extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-08-14 00:08:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into post

Browse Source
This commit is contained in:
Jonathan Turner
2019-08-31 15:12:03 +12:00
parent 9e167713b3 f274df6753
commit 1d77595576
28 changed files with 894 additions and 481 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/cli.rs
View File

@ -179,6 +179,8 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(ToBSON),
whole_stream_command(ToCSV),
whole_stream_command(ToJSON),
whole_stream_command(ToSQLite),
whole_stream_command(ToDB),
whole_stream_command(ToTOML),
whole_stream_command(ToTSV),
whole_stream_command(ToYAML),
@ -193,9 +195,12 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(FromINI),
whole_stream_command(FromBSON),
whole_stream_command(FromJSON),
whole_stream_command(FromDB),
whole_stream_command(FromSQLite),
whole_stream_command(FromTOML),
whole_stream_command(FromXML),
whole_stream_command(FromYAML),
whole_stream_command(FromYML),
whole_stream_command(Pick),
whole_stream_command(Get),
per_item_command(Remove),

8
src/commands.rs
View File

@ -19,6 +19,7 @@ pub(crate) mod from_bson;
pub(crate) mod from_csv;
pub(crate) mod from_ini;
pub(crate) mod from_json;
pub(crate) mod from_sqlite;
pub(crate) mod from_toml;
pub(crate) mod from_tsv;
pub(crate) mod from_xml;
@ -53,6 +54,7 @@ pub(crate) mod to_array;
pub(crate) mod to_bson;
pub(crate) mod to_csv;
pub(crate) mod to_json;
pub(crate) mod to_sqlite;
pub(crate) mod to_toml;
pub(crate) mod to_tsv;
pub(crate) mod to_yaml;
@ -68,6 +70,7 @@ pub(crate) use command::{
per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
UnevaluatedCallInfo, WholeStreamCommand,
};
pub(crate) use config::Config;
pub(crate) use cp::Cpy;
pub(crate) use date::Date;
@ -80,10 +83,13 @@ pub(crate) use from_bson::FromBSON;
pub(crate) use from_csv::FromCSV;
pub(crate) use from_ini::FromINI;
pub(crate) use from_json::FromJSON;
pub(crate) use from_sqlite::FromDB;
pub(crate) use from_sqlite::FromSQLite;
pub(crate) use from_toml::FromTOML;
pub(crate) use from_tsv::FromTSV;
pub(crate) use from_xml::FromXML;
pub(crate) use from_yaml::FromYAML;
pub(crate) use from_yaml::FromYML;
pub(crate) use get::Get;
pub(crate) use last::Last;
pub(crate) use lines::Lines;
@ -113,6 +119,8 @@ pub(crate) use to_array::ToArray;
pub(crate) use to_bson::ToBSON;
pub(crate) use to_csv::ToCSV;
pub(crate) use to_json::ToJSON;
pub(crate) use to_sqlite::ToDB;
pub(crate) use to_sqlite::ToSQLite;
pub(crate) use to_toml::ToTOML;
pub(crate) use to_tsv::ToTSV;
pub(crate) use to_yaml::ToYAML;

50
src/commands/classified.rs
View File

@ -146,53 +146,9 @@ impl InternalCommand {
.insert_at_current(Box::new(ValueShell::new(value)));
}
CommandAction::EnterShell(location) => {
let path = std::path::Path::new(&location);
if path.is_dir() {
// If it's a directory, add a new filesystem shell
context.shell_manager.insert_at_current(Box::new(
FilesystemShell::with_location(
location,
context.registry().clone(),
)?,
));
} else {
// If it's a file, attempt to open the file as a value and enter it
let cwd = context.shell_manager.path();
let full_path = std::path::PathBuf::from(cwd);
let (file_extension, contents, contents_tag, span_source) =
crate::commands::open::fetch(
&full_path,
&location,
Span::unknown(),
)
.await?;
if let Some(uuid) = contents_tag.origin {
// If we have loaded something, track its source
context.add_span_source(uuid, span_source);
}
match contents {
Value::Primitive(Primitive::String(string)) => {
let value = crate::commands::open::parse_string_as_value(
file_extension,
string,
contents_tag,
Span::unknown(),
)?;
context
.shell_manager
.insert_at_current(Box::new(ValueShell::new(value)));
}
value => context.shell_manager.insert_at_current(Box::new(
ValueShell::new(value.tagged(contents_tag)),
)),
}
}
context.shell_manager.insert_at_current(Box::new(
FilesystemShell::with_location(location, context.registry().clone())?,
));
}
CommandAction::PreviousShell => {
context.shell_manager.prev();

9
src/commands/command.rs
View File

@ -512,7 +512,7 @@ pub trait PerItemCommand: Send + Sync {
&self,
call_info: &CallInfo,
registry: &CommandRegistry,
shell_manager: &ShellManager,
raw_args: &RawCommandArgs,
input: Tagged<Value>,
) -> Result<OutputStream, ShellError>;
@ -579,7 +579,7 @@ impl Command {
.call_info
.evaluate(&registry, &Scope::it_value(x.clone()))
.unwrap();
match command.run(&call_info, &registry, &raw_args.shell_manager, x) {
match command.run(&call_info, &registry, &raw_args, x) {
Ok(o) => o,
Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(),
}
@ -596,7 +596,10 @@ impl Command {
.unwrap();
// We don't have an $it or block, so just execute what we have
match command.run(&call_info, &registry, &raw_args.shell_manager, nothing) {
match command
.run(&call_info, &registry, &raw_args, nothing)
.into()
{
Ok(o) => o,
Err(e) => OutputStream::one(Err(e)),
}

4
src/commands/cp.rs
View File

@ -19,10 +19,10 @@ impl PerItemCommand for Cpy {
&self,
call_info: &CallInfo,
_registry: &CommandRegistry,
shell_manager: &ShellManager,
raw_args: &RawCommandArgs,
_input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
call_info.process(shell_manager, cp)?.run()
call_info.process(&raw_args.shell_manager, cp)?.run()
}
fn name(&self) -> &str {

105
src/commands/enter.rs
View File

@ -1,8 +1,10 @@
use crate::commands::command::CommandAction;
use crate::commands::PerItemCommand;
use crate::commands::UnevaluatedCallInfo;
use crate::errors::ShellError;
use crate::parser::registry;
use crate::prelude::*;
use std::path::PathBuf;
pub struct Enter;
@ -18,18 +20,109 @@ impl PerItemCommand for Enter {
fn run(
&self,
call_info: &CallInfo,
_registry: &registry::CommandRegistry,
_shell_manager: &ShellManager,
registry: &registry::CommandRegistry,
raw_args: &RawCommandArgs,
_input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
let raw_args = raw_args.clone();
match call_info.args.expect_nth(0)? {
Tagged {
item: Value::Primitive(Primitive::String(location)),
..
} => Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterShell(
location.to_string(),
)))]
.into()),
} => {
let location = location.to_string();
let location_clone = location.to_string();
if PathBuf::from(location).is_dir() {
Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterShell(
location_clone,
)))]
.into())
} else {
let stream = async_stream_block! {
// If it's a file, attempt to open the file as a value and enter it
let cwd = raw_args.shell_manager.path();
let full_path = std::path::PathBuf::from(cwd);
let (file_extension, contents, contents_tag, span_source) =
crate::commands::open::fetch(
&full_path,
&location_clone,
Span::unknown(),
)
.await.unwrap();
if let Some(uuid) = contents_tag.origin {
// If we have loaded something, track its source
yield ReturnSuccess::action(CommandAction::AddSpanSource(
uuid,
span_source,
));
}
match contents {
Value::Primitive(Primitive::String(_)) => {
let tagged_contents = contents.tagged(contents_tag);
if let Some(extension) = file_extension {
let command_name = format!("from-{}", extension);
if let Some(converter) =
registry.get_command(&command_name)
{
let new_args = RawCommandArgs {
host: raw_args.host,
shell_manager: raw_args.shell_manager,
call_info: UnevaluatedCallInfo {
args: crate::parser::hir::Call {
head: raw_args.call_info.args.head,
positional: None,
named: None,
},
source: raw_args.call_info.source,
source_map: raw_args.call_info.source_map,
name_span: raw_args.call_info.name_span,
},
};
let mut result = converter.run(
new_args.with_input(vec![tagged_contents]),
&registry,
);
let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
result.drain_vec().await;
for res in result_vec {
match res {
Ok(ReturnSuccess::Value(Tagged {
item,
..
})) => {
yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(
Tagged {
item: item,
tag: contents_tag,
})));
}
x => yield x,
}
}
} else {
yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
}
} else {
yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
}
}
_ => {
let tagged_contents = contents.tagged(contents_tag);
yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
}
}
};
Ok(stream.to_output_stream())
}
}
x => Ok(
vec![Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(
x.clone(),

2
src/commands/from_bson.rs
View File

@ -45,7 +45,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value
collected.into_tagged_value()
}
Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
Bson::RegExp(r, opts) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(

4
src/commands/from_json.rs
View File

@ -32,9 +32,7 @@ fn convert_json_value_to_nu_value(v: &serde_hjson::Value, tag: impl Into<Tag>) -
let tag = tag.into();
match v {
serde_hjson::Value::Null => {
Value::Primitive(Primitive::String(String::from(""))).tagged(tag)
}
serde_hjson::Value::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
serde_hjson::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
serde_hjson::Value::F64(n) => {
Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag)

166
src/commands/from_sqlite.rs Normal file
View File

@ -0,0 +1,166 @@
use crate::commands::WholeStreamCommand;
use crate::errors::ShellError;
use crate::object::base::OF64;
use crate::object::{Primitive, TaggedDictBuilder, Value};
use crate::prelude::*;
use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
use std::io::Write;
use std::path::Path;
pub struct FromSQLite;
impl WholeStreamCommand for FromSQLite {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry)
}
fn name(&self) -> &str {
"from-sqlite"
}
fn signature(&self) -> Signature {
Signature::build("from-sqlite")
}
}
pub struct FromDB;
impl WholeStreamCommand for FromDB {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry)
}
fn name(&self) -> &str {
"from-db"
}
fn signature(&self) -> Signature {
Signature::build("from-db")
}
}
pub fn convert_sqlite_file_to_nu_value(
path: &Path,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, rusqlite::Error> {
let conn = Connection::open(path)?;
let mut meta_out = Vec::new();
let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
while let Some(meta_row) = meta_rows.next()? {
let table_name: String = meta_row.get(0)?;
let mut meta_dict = TaggedDictBuilder::new(tag.clone());
let mut out = Vec::new();
let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?;
let mut table_rows = table_stmt.query(NO_PARAMS)?;
while let Some(table_row) = table_rows.next()? {
out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?)
}
meta_dict.insert_tagged(
"table_name".to_string(),
Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()),
);
meta_dict.insert_tagged("table_values", Value::List(out).tagged(tag.clone()));
meta_out.push(meta_dict.into_tagged_value());
}
let tag = tag.into();
Ok(Value::List(meta_out).tagged(tag))
}
fn convert_sqlite_row_to_nu_value(
row: &Row,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, rusqlite::Error> {
let mut collected = TaggedDictBuilder::new(tag.clone());
for (i, c) in row.columns().iter().enumerate() {
collected.insert_tagged(
c.name().to_string(),
convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
);
}
return Ok(collected.into_tagged_value());
}
fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Tagged<Value> {
match value {
ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
ValueRef::Integer(i) => Value::Primitive(Primitive::Int(i)).tagged(tag),
ValueRef::Real(f) => Value::Primitive(Primitive::Float(OF64::from(f))).tagged(tag),
t @ ValueRef::Text(_) => {
// this unwrap is safe because we know the ValueRef is Text.
Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag)
}
ValueRef::Blob(u) => Value::Binary(u.to_owned()).tagged(tag),
}
}
pub fn from_sqlite_bytes_to_value(
mut bytes: Vec<u8>,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
tempfile.write_all(bytes.as_mut_slice())?;
match convert_sqlite_file_to_nu_value(tempfile.path(), tag) {
Ok(value) => Ok(value),
Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
}
}
fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let span = args.name_span();
let input = args.input;
let stream = async_stream_block! {
let values: Vec<Tagged<Value>> = input.values.collect().await;
for value in values {
let value_tag = value.tag();
match value.item {
Value::Binary(vb) =>
match from_sqlite_bytes_to_value(vb, span) {
Ok(x) => match x {
Tagged { item: Value::List(list), .. } => {
for l in list {
yield ReturnSuccess::value(l);
}
}
_ => yield ReturnSuccess::value(x),
}
Err(_) => {
yield Err(ShellError::labeled_error_with_secondary(
"Could not parse as SQLite",
"input cannot be parsed as SQLite",
span,
"value originates from here",
value_tag.span,
))
}
}
_ => yield Err(ShellError::labeled_error_with_secondary(
"Expected a string from pipeline",
"requires string input",
span,
"value originates from here",
value_tag.span,
)),
}
}
};
Ok(stream.to_output_stream())
}

20
src/commands/from_yaml.rs
View File

@ -23,6 +23,26 @@ impl WholeStreamCommand for FromYAML {
}
}
pub struct FromYML;
impl WholeStreamCommand for FromYML {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_yaml(args, registry)
}
fn name(&self) -> &str {
"from-yml"
}
fn signature(&self) -> Signature {
Signature::build("from-yml")
}
}
fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, tag: impl Into<Tag>) -> Tagged<Value> {
let tag = tag.into();

4
src/commands/mkdir.rs
View File

@ -16,10 +16,10 @@ impl PerItemCommand for Mkdir {
&self,
call_info: &CallInfo,
_registry: &CommandRegistry,
shell_manager: &ShellManager,
raw_args: &RawCommandArgs,
_input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
call_info.process(shell_manager, mkdir)?.run()
call_info.process(&raw_args.shell_manager, mkdir)?.run()
}
fn name(&self) -> &str {

4
src/commands/mv.rs
View File

@ -29,10 +29,10 @@ impl PerItemCommand for Move {
&self,
call_info: &CallInfo,
_registry: &CommandRegistry,
shell_manager: &ShellManager,
raw_args: &RawCommandArgs,
_input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
call_info.process(shell_manager, mv)?.run()
call_info.process(&raw_args.shell_manager, mv)?.run()
}
}

192
src/commands/open.rs
View File

@ -1,6 +1,7 @@
use crate::commands::UnevaluatedCallInfo;
use crate::context::SpanSource;
use crate::errors::ShellError;
use crate::object::{Primitive, Value};
use crate::object::Value;
use crate::parser::hir::SyntaxType;
use crate::parser::registry::Signature;
use crate::prelude::*;
@ -25,15 +26,20 @@ impl PerItemCommand for Open {
fn run(
&self,
call_info: &CallInfo,
_registry: &CommandRegistry,
shell_manager: &ShellManager,
registry: &CommandRegistry,
raw_args: &RawCommandArgs,
_input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
run(call_info, shell_manager)
run(call_info, registry, raw_args)
}
}
fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStream, ShellError> {
fn run(
call_info: &CallInfo,
registry: &CommandRegistry,
raw_args: &RawCommandArgs,
) -> Result<OutputStream, ShellError> {
let shell_manager = &raw_args.shell_manager;
let cwd = PathBuf::from(shell_manager.path());
let full_path = PathBuf::from(cwd);
@ -47,8 +53,9 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
let path_buf = path.as_path()?;
let path_str = path_buf.display().to_string();
let path_span = path.span();
let name_span = call_info.name_span;
let has_raw = call_info.args.has("raw");
let registry = registry.clone();
let raw_args = raw_args.clone();
let stream = async_stream_block! {
@ -65,7 +72,6 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
file_extension.or(path_str.split('.').last().map(String::from))
};
if let Some(uuid) = contents_tag.origin {
// If we have loaded something, track its source
yield ReturnSuccess::action(CommandAction::AddSpanSource(
@ -74,39 +80,46 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
));
}
match contents {
Value::Primitive(Primitive::String(string)) => {
let value = parse_string_as_value(file_extension, string, contents_tag, name_span).unwrap();
let tagged_contents = contents.tagged(contents_tag);
match value {
Tagged {
item: Value::List(list),
..
} => {
for elem in list {
yield ReturnSuccess::value(elem);
}
if let Some(extension) = file_extension {
let command_name = format!("from-{}", extension);
if let Some(converter) = registry.get_command(&command_name) {
let new_args = RawCommandArgs {
host: raw_args.host,
shell_manager: raw_args.shell_manager,
call_info: UnevaluatedCallInfo {
args: crate::parser::hir::Call {
head: raw_args.call_info.args.head,
positional: None,
named: None
},
source: raw_args.call_info.source,
source_map: raw_args.call_info.source_map,
name_span: raw_args.call_info.name_span,
}
x => yield ReturnSuccess::value(x),
}
}
Value::Binary(binary) => {
let value = parse_binary_as_value(file_extension, binary, contents_tag, name_span).unwrap();
match value {
Tagged {
item: Value::List(list),
..
} => {
for elem in list {
yield ReturnSuccess::value(elem);
};
let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry);
let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
for res in result_vec {
match res {
Ok(ReturnSuccess::Value(Tagged { item: Value::List(list), ..})) => {
for l in list {
yield Ok(ReturnSuccess::Value(l));
}
}
Ok(ReturnSuccess::Value(Tagged { item, .. })) => {
yield Ok(ReturnSuccess::Value(Tagged { item: item, tag: contents_tag }));
}
x => yield x,
}
x => yield ReturnSuccess::value(x),
}
} else {
yield ReturnSuccess::value(tagged_contents);
}
other => yield ReturnSuccess::value(other.tagged(contents_tag)),
};
} else {
yield ReturnSuccess::value(tagged_contents);
}
};
Ok(stream.to_output_stream())
@ -419,114 +432,3 @@ fn read_be_u16(input: &[u8]) -> Option<Vec<u16>> {
Some(result)
}
}
pub fn parse_string_as_value(
extension: Option<String>,
contents: String,
contents_tag: Tag,
name_span: Span,
) -> Result<Tagged<Value>, ShellError> {
match extension {
Some(ref x) if x == "csv" => {
crate::commands::from_csv::from_csv_string_to_value(contents, false, contents_tag)
.map_err(move |_| {
ShellError::labeled_error(
"Could not open as CSV",
"could not open as CSV",
name_span,
)
})
}
Some(ref x) if x == "tsv" => {
crate::commands::from_tsv::from_tsv_string_to_value(contents, false, contents_tag)
.map_err(move |_| {
ShellError::labeled_error(
"Could not open as TSV",
"could not open as TSV",
name_span,
)
})
}
Some(ref x) if x == "toml" => {
crate::commands::from_toml::from_toml_string_to_value(contents, contents_tag).map_err(
move |_| {
ShellError::labeled_error(
"Could not open as TOML",
"could not open as TOML",
name_span,
)
},
)
}
Some(ref x) if x == "json" => {
crate::commands::from_json::from_json_string_to_value(contents, contents_tag).map_err(
move |_| {
ShellError::labeled_error(
"Could not open as JSON",
"could not open as JSON",
name_span,
)
},
)
}
Some(ref x) if x == "ini" => crate::commands::from_ini::from_ini_string_to_value(
contents,
contents_tag,
)
.map_err(move |_| {
ShellError::labeled_error("Could not open as INI", "could not open as INI", name_span)
}),
Some(ref x) if x == "xml" => crate::commands::from_xml::from_xml_string_to_value(
contents,
contents_tag,
)
.map_err(move |_| {
ShellError::labeled_error("Could not open as XML", "could not open as XML", name_span)
}),
Some(ref x) if x == "yml" => {
crate::commands::from_yaml::from_yaml_string_to_value(contents, contents_tag).map_err(
move |_| {
ShellError::labeled_error(
"Could not open as YAML",
"could not open as YAML",
name_span,
)
},
)
}
Some(ref x) if x == "yaml" => {
crate::commands::from_yaml::from_yaml_string_to_value(contents, contents_tag).map_err(
move |_| {
ShellError::labeled_error(
"Could not open as YAML",
"could not open as YAML",
name_span,
)
},
)
}
_ => Ok(Value::string(contents).tagged(contents_tag)),
}
}
pub fn parse_binary_as_value(
extension: Option<String>,
contents: Vec<u8>,
contents_tag: Tag,
name_span: Span,
) -> Result<Tagged<Value>, ShellError> {
match extension {
Some(ref x) if x == "bson" => {
crate::commands::from_bson::from_bson_bytes_to_value(contents, contents_tag).map_err(
move |_| {
ShellError::labeled_error(
"Could not open as BSON",
"could not open as BSON",
name_span,
)
},
)
}
_ => Ok(Value::Binary(contents).tagged(contents_tag)),
}
}

71
src/commands/ps.rs
View File

@ -1,8 +1,13 @@
use crate::commands::WholeStreamCommand;
use crate::errors::ShellError;
use crate::object::process::process_dict;
use crate::object::TaggedDictBuilder;
use crate::prelude::*;
use sysinfo::SystemExt;
use std::time::Duration;
use std::usize;
use futures::stream::{StreamExt, TryStreamExt};
use heim::process::{self as process, Process, ProcessResult};
use heim::units::{ratio, Ratio};
pub struct PS;
@ -24,28 +29,44 @@ impl WholeStreamCommand for PS {
}
}
fn ps(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let system;
async fn usage(process: Process) -> ProcessResult<(process::Process, Ratio)> {
let usage_1 = process.cpu_usage().await?;
futures_timer::Delay::new(Duration::from_millis(100)).await?;
let usage_2 = process.cpu_usage().await?;
#[cfg(target_os = "linux")]
{
system = sysinfo::System::new();
}
#[cfg(not(target_os = "linux"))]
{
use sysinfo::RefreshKind;
let mut sy = sysinfo::System::new_with_specifics(RefreshKind::new().with_processes());
sy.refresh_processes();
system = sy;
}
let list = system.get_process_list();
let list = list
.into_iter()
.map(|(_, process)| process_dict(process, Tag::unknown_origin(args.call_info.name_span)))
.collect::<VecDeque<_>>();
Ok(list.from_input_stream())
Ok((process, usage_2 - usage_1))
}
fn ps(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let span = args.name_span();
let stream = async_stream_block! {
let processes = process::processes()
.map_ok(|process| {
// Note that there is no `.await` here,
// as we want to pass the returned future
// into the `.try_buffer_unordered`.
usage(process)
})
.try_buffer_unordered(usize::MAX);
pin_utils::pin_mut!(processes);
while let Some(res) = processes.next().await {
if let Ok((process, usage)) = res {
let mut dict = TaggedDictBuilder::new(Tag::unknown_origin(span));
dict.insert("pid", Value::int(process.pid()));
if let Ok(name) = process.name().await {
dict.insert("name", Value::string(name));
}
if let Ok(status) = process.status().await {
dict.insert("status", Value::string(format!("{:?}", status)));
}
dict.insert("cpu", Value::float(usage.get::<ratio::percent>() as f64));
yield ReturnSuccess::value(dict.into_tagged_value());
}
}
};
Ok(stream.to_output_stream())
}

4
src/commands/rm.rs
View File

@ -28,10 +28,10 @@ impl PerItemCommand for Remove {
&self,
call_info: &CallInfo,
_registry: &CommandRegistry,
shell_manager: &ShellManager,
raw_args: &RawCommandArgs,
_input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
call_info.process(shell_manager, rm)?.run()
call_info.process(&raw_args.shell_manager, rm)?.run()
}
}

205
src/commands/save.rs
View File

@ -1,8 +1,4 @@
use crate::commands::to_csv::{to_string as to_csv_to_string, value_to_csv_value};
use crate::commands::to_tsv::{to_string as to_tsv_to_string, value_to_tsv_value};
use crate::commands::to_json::value_to_json_value;
use crate::commands::to_toml::value_to_toml_value;
use crate::commands::to_yaml::value_to_yaml_value;
use crate::commands::UnevaluatedCallInfo;
use crate::commands::WholeStreamCommand;
use crate::errors::ShellError;
use crate::object::Value;
@ -33,7 +29,7 @@ impl WholeStreamCommand for Save {
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
args.process(registry, save)?.run()
Ok(args.process_raw(registry, save)?.run())
}
}
@ -47,16 +43,19 @@ fn save(
name,
shell_manager,
source_map,
host,
commands: registry,
..
}: RunnableContext,
raw_args: RawCommandArgs,
) -> Result<OutputStream, ShellError> {
let mut full_path = PathBuf::from(shell_manager.path());
let name_span = name;
if path.is_none() {
let source_map = source_map.clone();
let stream = async_stream_block! {
let input: Vec<Tagged<Value>> = input.values.collect().await;
let source_map = source_map.clone();
let stream = async_stream_block! {
let input: Vec<Tagged<Value>> = input.values.collect().await;
if path.is_none() {
// If there is no filename, check the metadata for the origin filename
if input.len() > 0 {
let origin = input[0].origin();
@ -88,50 +87,99 @@ fn save(
name_span,
));
}
let content = if !save_raw {
to_string_for(full_path.extension(), &input)
} else {
string_from(&input)
};
match content {
Ok(save_data) => match std::fs::write(full_path, save_data) {
Ok(o) => o,
Err(e) => yield Err(ShellError::string(e.to_string())),
},
Err(e) => yield Err(ShellError::string(e.to_string())),
} else {
if let Some(file) = path {
full_path.push(file.item());
}
};
Ok(OutputStream::new(stream))
} else {
if let Some(file) = path {
full_path.push(file.item());
}
let stream = async_stream_block! {
let input: Vec<Tagged<Value>> = input.values.collect().await;
let content = if !save_raw {
to_string_for(full_path.extension(), &input)
let content = if !save_raw {
if let Some(extension) = full_path.extension() {
let command_name = format!("to-{}", extension.to_str().unwrap());
if let Some(converter) = registry.get_command(&command_name) {
let new_args = RawCommandArgs {
host: host,
shell_manager: shell_manager,
call_info: UnevaluatedCallInfo {
args: crate::parser::hir::Call {
head: raw_args.call_info.args.head,
positional: None,
named: None
},
source: raw_args.call_info.source,
source_map: raw_args.call_info.source_map,
name_span: raw_args.call_info.name_span,
}
};
let mut result = converter.run(new_args.with_input(input), &registry);
let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
let mut result_string = String::new();
for res in result_vec {
match res {
Ok(ReturnSuccess::Value(Tagged { item: Value::Primitive(Primitive::String(s)), .. })) => {
result_string.push_str(&s);
}
_ => {
yield Err(ShellError::labeled_error(
"Save could not successfully save",
"unexpected data during saveS",
name_span,
));
},
}
}
Ok(result_string)
} else {
let mut result_string = String::new();
for res in input {
match res {
Tagged { item: Value::Primitive(Primitive::String(s)), .. } => {
result_string.push_str(&s);
}
_ => {
yield Err(ShellError::labeled_error(
"Save could not successfully save",
"unexpected data during saveS",
name_span,
));
},
}
}
Ok(result_string)
}
} else {
string_from(&input)
};
match content {
Ok(save_data) => match std::fs::write(full_path, save_data) {
Ok(o) => o,
Err(e) => yield Err(ShellError::string(e.to_string())),
},
Err(e) => yield Err(ShellError::string(e.to_string())),
let mut result_string = String::new();
for res in input {
match res {
Tagged { item: Value::Primitive(Primitive::String(s)), .. } => {
result_string.push_str(&s);
}
_ => {
yield Err(ShellError::labeled_error(
"Save could not successfully save",
"unexpected data during saveS",
name_span,
));
},
}
}
Ok(result_string)
}
} else {
string_from(&input)
};
Ok(OutputStream::new(stream))
}
match content {
Ok(save_data) => match std::fs::write(full_path, save_data) {
Ok(o) => o,
Err(e) => yield Err(ShellError::string(e.to_string())),
},
Err(e) => yield Err(ShellError::string(e.to_string())),
}
};
Ok(OutputStream::new(stream))
}
fn string_from(input: &Vec<Tagged<Value>>) -> Result<String, ShellError> {
@ -153,66 +201,3 @@ fn string_from(input: &Vec<Tagged<Value>>) -> Result<String, ShellError> {
Ok(save_data)
}
fn to_string_for(
ext: Option<&std::ffi::OsStr>,
input: &Vec<Tagged<Value>>,
) -> Result<String, ShellError> {
let contents = match ext {
Some(x) if x == "csv" => {
if input.len() != 1 {
return Err(ShellError::string(
"saving to csv requires a single object (or use --raw)",
));
}
to_csv_to_string(&value_to_csv_value(&input[0]))?
}
Some(x) if x == "tsv" => {
if input.len() != 1 {
return Err(ShellError::string(
"saving to tsv requires a single object (or use --raw)",
));
}
to_tsv_to_string(&value_to_tsv_value(&input[0]))?
}
Some(x) if x == "toml" => {
if input.len() != 1 {
return Err(ShellError::string(
"saving to toml requires a single object (or use --raw)",
));
}
toml::to_string(&value_to_toml_value(&input[0]))?
}
Some(x) if x == "json" => {
if input.len() != 1 {
return Err(ShellError::string(
"saving to json requires a single object (or use --raw)",
));
}
serde_json::to_string(&value_to_json_value(&input[0]))?
}
Some(x) if x == "yml" => {
if input.len() != 1 {
return Err(ShellError::string(
"saving to yml requires a single object (or use --raw)",
));
}
serde_yaml::to_string(&value_to_yaml_value(&input[0]))?
}
Some(x) if x == "yaml" => {
if input.len() != 1 {
return Err(ShellError::string(
"saving to yaml requires a single object (or use --raw)",
));
}
serde_yaml::to_string(&value_to_yaml_value(&input[0]))?
}
_ => {
return Err(ShellError::string(
"tried saving a single object with an unrecognized format.",
))
}
};
Ok(contents)
}

203
src/commands/to_sqlite.rs Normal file
View File

@ -0,0 +1,203 @@
use crate::commands::WholeStreamCommand;
use crate::object::{Dictionary, Primitive, Value};
use crate::prelude::*;
use hex::encode;
use rusqlite::{Connection, NO_PARAMS};
use std::io::Read;
pub struct ToSQLite;
impl WholeStreamCommand for ToSQLite {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry)
}
fn name(&self) -> &str {
"to-sqlite"
}
fn signature(&self) -> Signature {
Signature::build("to-sqlite")
}
}
pub struct ToDB;
impl WholeStreamCommand for ToDB {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry)
}
fn name(&self) -> &str {
"to-db"
}
fn signature(&self) -> Signature {
Signature::build("to-db")
}
}
fn comma_concat(acc: String, current: String) -> String {
if acc == "" {
current
} else {
format!("{}, {}", acc, current)
}
}
fn get_columns(rows: &Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
match &rows[0].item {
Value::Object(d) => Ok(d
.entries
.iter()
.map(|(k, _v)| k.clone())
.fold("".to_string(), comma_concat)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
}
}
fn nu_value_to_sqlite_string(v: Value) -> String {
match v {
Value::Binary(u) => format!("x'{}'", encode(u)),
Value::Primitive(p) => match p {
Primitive::Nothing => "NULL".into(),
Primitive::Int(i) => format!("{}", i),
Primitive::Float(f) => format!("{}", f.into_inner()),
Primitive::Bytes(u) => format!("{}", u),
Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
Primitive::Boolean(true) => "1".into(),
Primitive::Boolean(_) => "0".into(),
Primitive::Date(d) => format!("'{}'", d),
Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
Primitive::BeginningOfStream => "NULL".into(),
Primitive::EndOfStream => "NULL".into(),
},
_ => "NULL".into(),
}
}
fn get_insert_values(rows: Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
let values: Result<Vec<_>, _> = rows
.into_iter()
.map(|value| match value.item {
Value::Object(d) => Ok(format!(
"({})",
d.entries
.iter()
.map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone()))
.fold("".to_string(), comma_concat)
)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
})
.collect();
let values = values?;
Ok(values.into_iter().fold("".to_string(), comma_concat))
}
fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
let table_name = match table.entries.get("table_name") {
Some(Tagged {
item: Value::Primitive(Primitive::String(table_name)),
..
}) => table_name,
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table name",
))
}
};
let (columns, insert_values) = match table.entries.get("table_values") {
Some(Tagged {
item: Value::List(l),
..
}) => (get_columns(l), get_insert_values(l.to_vec())),
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table values",
))
}
};
let create = format!("create table {}({})", table_name, columns?);
let insert = format!("insert into {} values {}", table_name, insert_values?);
Ok((create, insert))
}
fn sqlite_input_stream_to_bytes(
values: Vec<Tagged<Value>>,
) -> Result<Tagged<Value>, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
let conn = match Connection::open(tempfile.path()) {
Ok(conn) => conn,
Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
};
let tag = values[0].tag.clone();
for value in values.into_iter() {
match value.item() {
Value::Object(d) => {
let (create, insert) = generate_statements(d.to_owned())?;
match conn
.execute(&create, NO_PARAMS)
.and_then(|_| conn.execute(&insert, NO_PARAMS))
{
Ok(_) => (),
Err(e) => {
println!("{}", create);
println!("{}", insert);
println!("{:?}", e);
return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
}
}
}
other => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Expected object, found {:?}", other),
))
}
}
}
let mut out = Vec::new();
tempfile.read_to_end(&mut out)?;
Ok(Value::Binary(out).tagged(tag))
}
fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let name_span = args.name_span();
let stream = async_stream_block! {
let values: Vec<_> = args.input.into_vec().await;
match sqlite_input_stream_to_bytes(values) {
Ok(out) => {
yield ReturnSuccess::value(out)
}
Err(_) => {
yield Err(ShellError::labeled_error(
"Expected an object with SQLite-compatible structure from pipeline",
"requires SQLite-compatible input",
name_span,
))
}
};
};
Ok(stream.to_output_stream())
}

2
src/commands/where_.rs
View File

@ -19,7 +19,7 @@ impl PerItemCommand for Where {
&self,
call_info: &CallInfo,
_registry: &registry::CommandRegistry,
_shell_manager: &ShellManager,
_raw_args: &RawCommandArgs,
input: Tagged<Value>,
) -> Result<OutputStream, ShellError> {
let input_clone = input.clone();

1
src/object.rs
View File

@ -4,7 +4,6 @@ pub(crate) mod dict;
pub(crate) mod files;
pub(crate) mod into;
pub(crate) mod meta;
pub(crate) mod process;
pub(crate) mod types;
#[allow(unused)]

29
src/object/process.rs
View File

@ -1,29 +0,0 @@
use crate::object::{TaggedDictBuilder, Value};
use crate::prelude::*;
use itertools::join;
use sysinfo::ProcessExt;
pub(crate) fn process_dict(proc: &sysinfo::Process, tag: impl Into<Tag>) -> Tagged<Value> {
let mut dict = TaggedDictBuilder::new(tag);
let cmd = proc.cmd();
let cmd_value = if cmd.len() == 0 {
Value::nothing()
} else {
Value::string(join(cmd, ""))
};
dict.insert("pid", Value::int(proc.pid() as i64));
dict.insert("status", Value::string(proc.status().to_string()));
dict.insert("cpu", Value::float(proc.cpu_usage() as f64));
//dict.insert("name", Value::string(proc.name()));
match cmd_value {
Value::Primitive(Primitive::Nothing) => {
dict.insert("name", Value::string(proc.name()));
}
_ => dict.insert("name", cmd_value),
}
dict.into_tagged_value()
}

6
src/parser/hir.rs
View File

@ -39,11 +39,11 @@ pub fn path(head: impl Into<Expression>, tail: Vec<Tagged<impl Into<String>>>) -
#[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)]
pub struct Call {
#[get = "crate"]
head: Box<Expression>,
pub head: Box<Expression>,
#[get = "crate"]
positional: Option<Vec<Expression>>,
pub positional: Option<Vec<Expression>>,
#[get = "crate"]
named: Option<NamedArguments>,
pub named: Option<NamedArguments>,
}
impl Call {

1
src/prelude.rs
View File

@ -52,6 +52,7 @@ pub(crate) use crate::commands::command::{
CallInfo, CommandAction, CommandArgs, ReturnSuccess, ReturnValue, RunnableContext,
};
pub(crate) use crate::commands::PerItemCommand;
pub(crate) use crate::commands::RawCommandArgs;
pub(crate) use crate::context::CommandRegistry;
pub(crate) use crate::context::{Context, SpanSource};
pub(crate) use crate::env::host::handle_unexpected;

4
src/utils.rs
View File

@ -266,6 +266,10 @@ mod tests {
loc: fixtures().join("sample.bson"),
at: 0
},
Res {
loc: fixtures().join("sample.db"),
at: 0
},
Res {
loc: fixtures().join("sample.ini"),
at: 0