From f730296e4564d87b4ddc62b17e54dc3601817adc Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Thu, 29 Aug 2019 15:53:45 +1200 Subject: [PATCH 1/9] WIP supporting from/to automatically --- src/cli.rs | 1 + src/commands.rs | 1 + src/commands/classified.rs | 24 ++--- src/commands/command.rs | 9 +- src/commands/cp.rs | 4 +- src/commands/enter.rs | 2 +- src/commands/from_yaml.rs | 20 +++++ src/commands/mkdir.rs | 4 +- src/commands/mv.rs | 4 +- src/commands/open.rs | 177 +++++++++---------------------------- src/commands/rm.rs | 4 +- src/commands/where_.rs | 2 +- src/parser/hir.rs | 6 +- src/prelude.rs | 2 +- 14 files changed, 90 insertions(+), 170 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index b650787d8d..46e4e338e8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -194,6 +194,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(FromTOML), whole_stream_command(FromXML), whole_stream_command(FromYAML), + whole_stream_command(FromYML), whole_stream_command(Pick), whole_stream_command(Get), per_item_command(Remove), diff --git a/src/commands.rs b/src/commands.rs index d1d9297fd4..529baf359a 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -80,6 +80,7 @@ crate use from_json::FromJSON; crate use from_toml::FromTOML; crate use from_xml::FromXML; crate use from_yaml::FromYAML; +crate use from_yaml::FromYML; crate use get::Get; crate use last::Last; crate use lines::Lines; diff --git a/src/commands/classified.rs b/src/commands/classified.rs index f74bab2f0e..b831a8007a 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -162,7 +162,7 @@ impl InternalCommand { let full_path = std::path::PathBuf::from(cwd); - let (file_extension, contents, contents_tag, span_source) = + let (_file_extension, contents, contents_tag, span_source) = crate::commands::open::fetch( &full_path, &location, @@ -175,23 +175,11 @@ impl InternalCommand { context.add_span_source(uuid, span_source); } - match contents { - Value::Primitive(Primitive::String(string)) => { - let value = crate::commands::open::parse_string_as_value( - file_extension, - string, - contents_tag, - Span::unknown(), - )?; - - context - .shell_manager - .insert_at_current(Box::new(ValueShell::new(value))); - } - value => context.shell_manager.insert_at_current(Box::new( - ValueShell::new(value.tagged(contents_tag)), - )), - } + context + .shell_manager + .insert_at_current(Box::new(ValueShell::new( + contents.tagged(contents_tag), + ))) } } CommandAction::PreviousShell => { diff --git a/src/commands/command.rs b/src/commands/command.rs index 9c8e786a1a..404d1d6b89 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -512,7 +512,7 @@ pub trait PerItemCommand: Send + Sync { &self, call_info: &CallInfo, registry: &CommandRegistry, - shell_manager: &ShellManager, + raw_args: &RawCommandArgs, input: Tagged, ) -> Result; @@ -579,7 +579,7 @@ impl Command { .call_info .evaluate(®istry, &Scope::it_value(x.clone())) .unwrap(); - match command.run(&call_info, ®istry, &raw_args.shell_manager, x) { + match command.run(&call_info, ®istry, &raw_args, x) { Ok(o) => o, Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(), } @@ -596,7 +596,10 @@ impl Command { .unwrap(); // We don't have an $it or block, so just execute what we have - match command.run(&call_info, ®istry, &raw_args.shell_manager, nothing) { + match command + .run(&call_info, ®istry, &raw_args, nothing) + .into() + { Ok(o) => o, Err(e) => OutputStream::one(Err(e)), } diff --git a/src/commands/cp.rs b/src/commands/cp.rs index 9ddbc8e4ae..ee1a289cd8 100644 --- a/src/commands/cp.rs +++ b/src/commands/cp.rs @@ -19,10 +19,10 @@ impl PerItemCommand for Cpy { &self, call_info: &CallInfo, _registry: &CommandRegistry, - shell_manager: &ShellManager, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { - call_info.process(shell_manager, cp)?.run() + call_info.process(&raw_args.shell_manager, cp)?.run() } fn name(&self) -> &str { diff --git a/src/commands/enter.rs b/src/commands/enter.rs index 2e9d0fd01a..19f6b9c8cb 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -19,7 +19,7 @@ impl PerItemCommand for Enter { &self, call_info: &CallInfo, _registry: ®istry::CommandRegistry, - _shell_manager: &ShellManager, + _raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { match call_info.args.expect_nth(0)? { diff --git a/src/commands/from_yaml.rs b/src/commands/from_yaml.rs index 349a6faeec..2294a39ee3 100644 --- a/src/commands/from_yaml.rs +++ b/src/commands/from_yaml.rs @@ -23,6 +23,26 @@ impl WholeStreamCommand for FromYAML { } } +pub struct FromYML; + +impl WholeStreamCommand for FromYML { + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + from_yaml(args, registry) + } + + fn name(&self) -> &str { + "from-yml" + } + + fn signature(&self) -> Signature { + Signature::build("from-yml") + } +} + fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, tag: impl Into) -> Tagged { let tag = tag.into(); diff --git a/src/commands/mkdir.rs b/src/commands/mkdir.rs index 5f92f1177b..96fd16afaf 100644 --- a/src/commands/mkdir.rs +++ b/src/commands/mkdir.rs @@ -16,10 +16,10 @@ impl PerItemCommand for Mkdir { &self, call_info: &CallInfo, _registry: &CommandRegistry, - shell_manager: &ShellManager, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { - call_info.process(shell_manager, mkdir)?.run() + call_info.process(&raw_args.shell_manager, mkdir)?.run() } fn name(&self) -> &str { diff --git a/src/commands/mv.rs b/src/commands/mv.rs index 37384e6b34..af93dde382 100644 --- a/src/commands/mv.rs +++ b/src/commands/mv.rs @@ -29,10 +29,10 @@ impl PerItemCommand for Move { &self, call_info: &CallInfo, _registry: &CommandRegistry, - shell_manager: &ShellManager, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { - call_info.process(shell_manager, mv)?.run() + call_info.process(&raw_args.shell_manager, mv)?.run() } } diff --git a/src/commands/open.rs b/src/commands/open.rs index cd3056e81d..012f398645 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -1,6 +1,7 @@ +use crate::commands::UnevaluatedCallInfo; use crate::context::SpanSource; use crate::errors::ShellError; -use crate::object::{Primitive, Value}; +use crate::object::Value; use crate::parser::hir::SyntaxType; use crate::parser::registry::Signature; use crate::prelude::*; @@ -25,15 +26,20 @@ impl PerItemCommand for Open { fn run( &self, call_info: &CallInfo, - _registry: &CommandRegistry, - shell_manager: &ShellManager, + registry: &CommandRegistry, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { - run(call_info, shell_manager) + run(call_info, registry, raw_args) } } -fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result { +fn run( + call_info: &CallInfo, + registry: &CommandRegistry, + raw_args: &RawCommandArgs, +) -> Result { + let shell_manager = &raw_args.shell_manager; let cwd = PathBuf::from(shell_manager.path()); let full_path = PathBuf::from(cwd); @@ -47,8 +53,9 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result Result Result { - let value = parse_string_as_value(file_extension, string, contents_tag, name_span).unwrap(); + let tagged_contents = contents.tagged(contents_tag); - match value { - Tagged { - item: Value::List(list), - .. - } => { - for elem in list { - yield ReturnSuccess::value(elem); - } + if let Some(extension) = file_extension { + let command_name = format!("from-{}", extension); + if let Some(converter) = registry.get_command(&command_name) { + let new_args = RawCommandArgs { + host: raw_args.host, + shell_manager: raw_args.shell_manager, + call_info: UnevaluatedCallInfo { + args: crate::parser::hir::Call { + head: raw_args.call_info.args.head, + positional: None, + named: None + }, + source: raw_args.call_info.source, + source_map: raw_args.call_info.source_map, + name_span: raw_args.call_info.name_span, } - x => yield ReturnSuccess::value(x), - } - } - Value::Binary(binary) => { - let value = parse_binary_as_value(file_extension, binary, contents_tag, name_span).unwrap(); - - match value { - Tagged { - item: Value::List(list), - .. - } => { - for elem in list { - yield ReturnSuccess::value(elem); + }; + let mut result = converter.run(new_args.with_input(vec![tagged_contents]), ®istry); + let result_vec: Vec> = result.drain_vec().await; + for res in result_vec { + match res { + Ok(ReturnSuccess::Value(Tagged { item, .. })) => { + yield Ok(ReturnSuccess::Value(Tagged { item: item, tag: contents_tag })); } + x => yield x, } - x => yield ReturnSuccess::value(x), } + } else { + yield ReturnSuccess::value(tagged_contents); } - other => yield ReturnSuccess::value(other.tagged(contents_tag)), - }; + } else { + yield ReturnSuccess::value(tagged_contents); + } }; Ok(stream.to_output_stream()) @@ -419,104 +427,3 @@ fn read_be_u16(input: &[u8]) -> Option> { Some(result) } } - -pub fn parse_string_as_value( - extension: Option, - contents: String, - contents_tag: Tag, - name_span: Span, -) -> Result, ShellError> { - match extension { - Some(ref x) if x == "csv" => { - crate::commands::from_csv::from_csv_string_to_value(contents, false, contents_tag) - .map_err(move |_| { - ShellError::labeled_error( - "Could not open as CSV", - "could not open as CSV", - name_span, - ) - }) - } - Some(ref x) if x == "toml" => { - crate::commands::from_toml::from_toml_string_to_value(contents, contents_tag).map_err( - move |_| { - ShellError::labeled_error( - "Could not open as TOML", - "could not open as TOML", - name_span, - ) - }, - ) - } - Some(ref x) if x == "json" => { - crate::commands::from_json::from_json_string_to_value(contents, contents_tag).map_err( - move |_| { - ShellError::labeled_error( - "Could not open as JSON", - "could not open as JSON", - name_span, - ) - }, - ) - } - Some(ref x) if x == "ini" => crate::commands::from_ini::from_ini_string_to_value( - contents, - contents_tag, - ) - .map_err(move |_| { - ShellError::labeled_error("Could not open as INI", "could not open as INI", name_span) - }), - Some(ref x) if x == "xml" => crate::commands::from_xml::from_xml_string_to_value( - contents, - contents_tag, - ) - .map_err(move |_| { - ShellError::labeled_error("Could not open as XML", "could not open as XML", name_span) - }), - Some(ref x) if x == "yml" => { - crate::commands::from_yaml::from_yaml_string_to_value(contents, contents_tag).map_err( - move |_| { - ShellError::labeled_error( - "Could not open as YAML", - "could not open as YAML", - name_span, - ) - }, - ) - } - Some(ref x) if x == "yaml" => { - crate::commands::from_yaml::from_yaml_string_to_value(contents, contents_tag).map_err( - move |_| { - ShellError::labeled_error( - "Could not open as YAML", - "could not open as YAML", - name_span, - ) - }, - ) - } - _ => Ok(Value::string(contents).tagged(contents_tag)), - } -} - -pub fn parse_binary_as_value( - extension: Option, - contents: Vec, - contents_tag: Tag, - name_span: Span, -) -> Result, ShellError> { - match extension { - Some(ref x) if x == "bson" => { - crate::commands::from_bson::from_bson_bytes_to_value(contents, contents_tag).map_err( - move |_| { - ShellError::labeled_error( - "Could not open as BSON", - "could not open as BSON", - name_span, - ) - }, - ) - } - _ => Ok(Value::Binary(contents).tagged(contents_tag)), - } -} diff --git a/src/commands/rm.rs b/src/commands/rm.rs index 9bbc352879..36f7aa6d8a 100644 --- a/src/commands/rm.rs +++ b/src/commands/rm.rs @@ -28,10 +28,10 @@ impl PerItemCommand for Remove { &self, call_info: &CallInfo, _registry: &CommandRegistry, - shell_manager: &ShellManager, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { - call_info.process(shell_manager, rm)?.run() + call_info.process(&raw_args.shell_manager, rm)?.run() } } diff --git a/src/commands/where_.rs b/src/commands/where_.rs index 4be427a4f8..f1d9b20023 100644 --- a/src/commands/where_.rs +++ b/src/commands/where_.rs @@ -19,7 +19,7 @@ impl PerItemCommand for Where { &self, call_info: &CallInfo, _registry: ®istry::CommandRegistry, - _shell_manager: &ShellManager, + _raw_args: &RawCommandArgs, input: Tagged, ) -> Result { let input_clone = input.clone(); diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 2148a179f4..cb25c3458d 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -39,11 +39,11 @@ pub fn path(head: impl Into, tail: Vec>>) - #[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)] pub struct Call { #[get = "crate"] - head: Box, + pub head: Box, #[get = "crate"] - positional: Option>, + pub positional: Option>, #[get = "crate"] - named: Option, + pub named: Option, } impl Call { diff --git a/src/prelude.rs b/src/prelude.rs index 38b61be38f..f696b86d1e 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -51,7 +51,7 @@ crate use crate::cli::MaybeOwned; crate use crate::commands::command::{ CallInfo, CommandAction, CommandArgs, ReturnSuccess, ReturnValue, RunnableContext, }; -crate use crate::commands::PerItemCommand; +crate use crate::commands::{PerItemCommand, RawCommandArgs}; crate use crate::context::CommandRegistry; crate use crate::context::{Context, SpanSource}; crate use crate::env::host::handle_unexpected; From b6db233c73a2688767d711690ace7768baf817a5 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Fri, 30 Aug 2019 03:39:16 +1200 Subject: [PATCH 2/9] Start working on save --- src/commands/save.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/save.rs b/src/commands/save.rs index da5c093502..52e1e97ef0 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -89,7 +89,7 @@ fn save( } let content = if !save_raw { - to_string_for(full_path.extension(), &input) + to_string_for(full_path.extension(), &input).await } else { string_from(&input) }; @@ -114,7 +114,7 @@ fn save( let input: Vec> = input.values.collect().await; let content = if !save_raw { - to_string_for(full_path.extension(), &input) + to_string_for(full_path.extension(), &input).await } else { string_from(&input) }; @@ -153,7 +153,7 @@ fn string_from(input: &Vec>) -> Result { Ok(save_data) } -fn to_string_for( +async fn to_string_for( ext: Option<&std::ffi::OsStr>, input: &Vec>, ) -> Result { From 60bfa277d0730c541c5a16859a0793196877f389 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 31 Aug 2019 07:07:07 +1200 Subject: [PATCH 3/9] Experiment with async/await-enabled ps --- Cargo.lock | 32 ++++++++----------- Cargo.toml | 3 +- src/commands/ps.rs | 71 ++++++++++++++++++++++++++++--------------- src/object.rs | 1 - src/object/process.rs | 29 ------------------ 5 files changed, 60 insertions(+), 76 deletions(-) delete mode 100644 src/object/process.rs diff --git a/Cargo.lock b/Cargo.lock index 02440e0c37..bf713a1326 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -641,11 +641,6 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "doc-comment" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "dtoa" version = "0.4.4" @@ -821,6 +816,15 @@ dependencies = [ "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "futures-timer" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "futures-util-preview" version = "0.3.0-alpha.18" @@ -1566,6 +1570,7 @@ dependencies = [ "enum-utils 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures-async-stream 0.1.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)", "futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures_codec 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "getset 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "git2 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1582,6 +1587,7 @@ dependencies = [ "nom5_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "onig_sys 69.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", "pretty-hex 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1602,7 +1608,6 @@ dependencies = [ "subprocess 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "surf 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "syntect 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sysinfo 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2619,18 +2624,6 @@ dependencies = [ "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "sysinfo" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "tempfile" version = "3.1.0" @@ -3181,7 +3174,6 @@ dependencies = [ "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" "checksum dirs-sys 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" -"checksum doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "923dea538cea0aa3025e8685b20d6ee21ef99c4f77e954a30febbaac5ec73a97" "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" "checksum dunce 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0ad6bf6a88548d1126045c413548df1453d9be094a8ab9fd59bf1fdd338da4f" "checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" @@ -3204,6 +3196,7 @@ dependencies = [ "checksum futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "ee7de0c1c9ed23f9457b0437fec7663ce64d9cc3c906597e714e529377b5ddd1" "checksum futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "efa8f90c4fb2328e381f8adfd4255b4a2b696f77d1c63a3dee6700b564c4e4b5" "checksum futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "e9b65a2481863d1b78e094a07e9c0eed458cc7dc6e72b22b7138b8a67d924859" +"checksum futures-timer 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f9eb554aa23143abc64ec4d0016f038caf53bb7cbc3d91490835c54edc96550" "checksum futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "7df53daff1e98cc024bf2720f3ceb0414d96fbb0a94f3cad3a5c3bf3be1d261c" "checksum futures_codec 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "36552cd31353fd135114510d53b8d120758120c36aa636a9341970f9efb1e4a0" "checksum getrandom 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "34f33de6f0ae7c9cb5e574502a562e2b512799e32abb801cd1e79ad952b62b49" @@ -3393,7 +3386,6 @@ dependencies = [ "checksum syn 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c65d951ab12d976b61a41cf9ed4531fc19735c6e6d84a4bb1453711e762ec731" "checksum synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02353edf96d6e4dc81aea2d8490a7e9db177bf8acb0e951c24940bf866cb313f" "checksum syntect 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e80b8831c5a543192ffc3727f01cf0e57579c6ac15558e3048bfb5708892167b" -"checksum sysinfo 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a8e841d2045e01c28343a09841a4c2323d0e936dbb813c95d6c76d13a88668d2" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum term 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" diff --git a/Cargo.toml b/Cargo.toml index 08a1272db1..ba12a45578 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ documentation = "https://book.nushell.sh" [dependencies] rustyline = "5.0.2" -sysinfo = "0.9.2" chrono = { version = "0.4.7", features = ["serde"] } derive-new = "0.5.7" prettytable-rs = "0.8.0" @@ -77,6 +76,8 @@ textwrap = {version = "0.11.0", features = ["term_size"]} rawkey = {version = "0.1.2", optional = true } clipboard = {version = "0.5", optional = true } shellexpand = "1.0.0" +futures-timer = "0.3.0" +pin-utils = "0.1.0-alpha.4" [features] raw-key = ["rawkey", "neso"] diff --git a/src/commands/ps.rs b/src/commands/ps.rs index 0dd9a1a6c9..d2a86487cc 100644 --- a/src/commands/ps.rs +++ b/src/commands/ps.rs @@ -1,8 +1,13 @@ use crate::commands::WholeStreamCommand; use crate::errors::ShellError; -use crate::object::process::process_dict; +use crate::object::TaggedDictBuilder; use crate::prelude::*; -use sysinfo::SystemExt; +use std::time::Duration; +use std::usize; + +use futures::stream::{StreamExt, TryStreamExt}; +use heim::process::{self as process, Process, ProcessResult}; +use heim::units::{ratio, Ratio}; pub struct PS; @@ -24,28 +29,44 @@ impl WholeStreamCommand for PS { } } -fn ps(args: CommandArgs, _registry: &CommandRegistry) -> Result { - let system; +async fn usage(process: Process) -> ProcessResult<(process::Process, Ratio)> { + let usage_1 = process.cpu_usage().await?; + futures_timer::Delay::new(Duration::from_millis(100)).await?; + let usage_2 = process.cpu_usage().await?; - #[cfg(target_os = "linux")] - { - system = sysinfo::System::new(); - } - - #[cfg(not(target_os = "linux"))] - { - use sysinfo::RefreshKind; - let mut sy = sysinfo::System::new_with_specifics(RefreshKind::new().with_processes()); - sy.refresh_processes(); - - system = sy; - } - let list = system.get_process_list(); - - let list = list - .into_iter() - .map(|(_, process)| process_dict(process, Tag::unknown_origin(args.call_info.name_span))) - .collect::>(); - - Ok(list.from_input_stream()) + Ok((process, usage_2 - usage_1)) +} + +fn ps(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let span = args.name_span(); + + let stream = async_stream_block! { + let processes = process::processes() + .map_ok(|process| { + // Note that there is no `.await` here, + // as we want to pass the returned future + // into the `.try_buffer_unordered`. + usage(process) + }) + .try_buffer_unordered(usize::MAX); + pin_utils::pin_mut!(processes); + + while let Some(res) = processes.next().await { + let (process, usage) = res.unwrap(); + + let mut dict = TaggedDictBuilder::new(Tag::unknown_origin(span)); + dict.insert("pid", Value::int(process.pid())); + if let Ok(name) = process.name().await { + dict.insert("name", Value::string(name)); + } + if let Ok(status) = process.status().await { + dict.insert("status", Value::string(format!("{:?}", status))); + } + dict.insert("cpu", Value::float(usage.get::() as f64)); + yield ReturnSuccess::value(dict.into_tagged_value()); + } + }; + + Ok(stream.to_output_stream()) } diff --git a/src/object.rs b/src/object.rs index 743a09cf68..dc5ecfc085 100644 --- a/src/object.rs +++ b/src/object.rs @@ -4,7 +4,6 @@ pub(crate) mod dict; pub(crate) mod files; pub(crate) mod into; pub(crate) mod meta; -pub(crate) mod process; pub(crate) mod types; #[allow(unused)] diff --git a/src/object/process.rs b/src/object/process.rs deleted file mode 100644 index a5b8e9ccd1..0000000000 --- a/src/object/process.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::object::{TaggedDictBuilder, Value}; -use crate::prelude::*; -use itertools::join; -use sysinfo::ProcessExt; - -pub(crate) fn process_dict(proc: &sysinfo::Process, tag: impl Into) -> Tagged { - let mut dict = TaggedDictBuilder::new(tag); - - let cmd = proc.cmd(); - - let cmd_value = if cmd.len() == 0 { - Value::nothing() - } else { - Value::string(join(cmd, "")) - }; - - dict.insert("pid", Value::int(proc.pid() as i64)); - dict.insert("status", Value::string(proc.status().to_string())); - dict.insert("cpu", Value::float(proc.cpu_usage() as f64)); - //dict.insert("name", Value::string(proc.name())); - match cmd_value { - Value::Primitive(Primitive::Nothing) => { - dict.insert("name", Value::string(proc.name())); - } - _ => dict.insert("name", cmd_value), - } - - dict.into_tagged_value() -} From c3abb3b6871f87d96c85d0015fb9f978e16aa920 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 31 Aug 2019 07:28:10 +1200 Subject: [PATCH 4/9] Fix unwrap --- src/commands/ps.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/commands/ps.rs b/src/commands/ps.rs index d2a86487cc..370b7b73d5 100644 --- a/src/commands/ps.rs +++ b/src/commands/ps.rs @@ -53,18 +53,18 @@ fn ps(args: CommandArgs, registry: &CommandRegistry) -> Result() as f64)); + yield ReturnSuccess::value(dict.into_tagged_value()); } - if let Ok(status) = process.status().await { - dict.insert("status", Value::string(format!("{:?}", status))); - } - dict.insert("cpu", Value::float(usage.get::() as f64)); - yield ReturnSuccess::value(dict.into_tagged_value()); } }; From 481722b80ac5131e75b91089abbae88443ac7d8a Mon Sep 17 00:00:00 2001 From: Patrick Meredith Date: Fri, 30 Aug 2019 17:34:35 -0400 Subject: [PATCH 5/9] Fix from_json to use Nothing --- src/commands/from_json.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/commands/from_json.rs b/src/commands/from_json.rs index ab4241f219..52ec333f36 100644 --- a/src/commands/from_json.rs +++ b/src/commands/from_json.rs @@ -32,9 +32,7 @@ fn convert_json_value_to_nu_value(v: &serde_hjson::Value, tag: impl Into) - let tag = tag.into(); match v { - serde_hjson::Value::Null => { - Value::Primitive(Primitive::String(String::from(""))).tagged(tag) - } + serde_hjson::Value::Null => Value::Primitive(Primitive::Nothing).tagged(tag), serde_hjson::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag), serde_hjson::Value::F64(n) => { Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag) From 3d147d11436c469eeb68cc11b82e6852abce9b69 Mon Sep 17 00:00:00 2001 From: Patrick Meredith Date: Tue, 27 Aug 2019 17:45:18 -0400 Subject: [PATCH 6/9] Add SQLite support --- Cargo.lock | 49 +++++++++ Cargo.toml | 5 + src/cli.rs | 2 + src/commands.rs | 5 + src/commands/from_bson.rs | 2 +- src/commands/from_sqlite.rs | 139 +++++++++++++++++++++++ src/commands/open.rs | 10 ++ src/commands/to_sqlite.rs | 183 +++++++++++++++++++++++++++++++ src/utils.rs | 4 + tests/command_open_tests.rs | 48 +++++--- tests/filters_test.rs | 139 ++++++++++++----------- tests/fixtures/formats/sample.db | Bin 0 -> 16384 bytes 12 files changed, 507 insertions(+), 79 deletions(-) create mode 100644 src/commands/from_sqlite.rs create mode 100644 src/commands/to_sqlite.rs create mode 100644 tests/fixtures/formats/sample.db diff --git a/Cargo.lock b/Cargo.lock index bf713a1326..b91214b5bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -715,6 +715,16 @@ dependencies = [ "synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fixedbitset" version = "0.1.9" @@ -1308,6 +1318,16 @@ dependencies = [ "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "libsqlite3-sys" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libz-sys" version = "1.0.25" @@ -1358,6 +1378,14 @@ dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "lzw" version = "0.10.0" @@ -1576,6 +1604,7 @@ dependencies = [ "git2 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "heim 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.22.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1596,6 +1625,7 @@ dependencies = [ "rawkey 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustyline 5.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2244,6 +2274,20 @@ dependencies = [ "xmlparser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusqlite" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rust-argon2" version = "0.5.0" @@ -3183,6 +3227,8 @@ dependencies = [ "checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" +"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" "checksum flate2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "550934ad4808d5d39365e5d61727309bf18b3b02c6c56b729cb92e7dd84bc3d8" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" @@ -3242,12 +3288,14 @@ dependencies = [ "checksum libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)" = "d44e80633f007889c7eff624b709ab43c92d708caad982295768a7b13ca3b5eb" "checksum libgit2-sys 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4c179ed6d19cd3a051e68c177fbbc214e79ac4724fac3a850ec9f3d3eb8a5578" "checksum libnghttp2-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02254d44f4435dd79e695f2c2b83cd06a47919adea30216ceaf0c57ca0a72463" +"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25" "checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" "checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" "checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" "checksum macaddr 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ff4752cb15cffb3e68f7dcb22e0818ac871f8c98fb07a634a81f41fb202a09f" "checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1" @@ -3342,6 +3390,7 @@ dependencies = [ "checksum render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68ed587df09cfb7ce1bc6fe8f77e24db219f222c049326ccbfb948ec67e31664" "checksum result 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560" "checksum roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "153c367ce9fb8ef7afe637ef92bd083ba0f88b03ef3fcf0287d40be05ae0a61c" +"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051" "checksum rust-argon2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81ed8d04228b44a740c8d46ff872a28e50fff3d659f307ab4da2cc502e019ff3" "checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" diff --git a/Cargo.toml b/Cargo.toml index ba12a45578..faa162f8cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ mime = "0.3.13" regex = "1.2.1" pretty-hex = "0.1.0" neso = { version = "0.5.0", optional = true } +hex = "0.3.2" crossterm = "0.10.2" tempfile = "3.1.0" image = "0.22.1" @@ -82,6 +83,10 @@ pin-utils = "0.1.0-alpha.4" [features] raw-key = ["rawkey", "neso"] +[dependencies.rusqlite] +version = "0.20.0" +features = ["bundled", "blob"] + [dev-dependencies] pretty_assertions = "0.6.1" diff --git a/src/cli.rs b/src/cli.rs index e01afe89ff..9ffab09c36 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -179,6 +179,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(ToBSON), whole_stream_command(ToCSV), whole_stream_command(ToJSON), + whole_stream_command(ToSQLite), whole_stream_command(ToTOML), whole_stream_command(ToTSV), whole_stream_command(ToYAML), @@ -193,6 +194,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(FromINI), whole_stream_command(FromBSON), whole_stream_command(FromJSON), + whole_stream_command(FromSQLite), whole_stream_command(FromTOML), whole_stream_command(FromXML), whole_stream_command(FromYAML), diff --git a/src/commands.rs b/src/commands.rs index 12cd6d56ee..2f5d174fdf 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -19,6 +19,7 @@ pub(crate) mod from_bson; pub(crate) mod from_csv; pub(crate) mod from_ini; pub(crate) mod from_json; +pub(crate) mod from_sqlite; pub(crate) mod from_toml; pub(crate) mod from_tsv; pub(crate) mod from_xml; @@ -52,6 +53,7 @@ pub(crate) mod to_array; pub(crate) mod to_bson; pub(crate) mod to_csv; pub(crate) mod to_json; +pub(crate) mod to_sqlite; pub(crate) mod to_toml; pub(crate) mod to_tsv; pub(crate) mod to_yaml; @@ -67,6 +69,7 @@ pub(crate) use command::{ per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs, UnevaluatedCallInfo, WholeStreamCommand, }; + pub(crate) use config::Config; pub(crate) use cp::Cpy; pub(crate) use date::Date; @@ -79,6 +82,7 @@ pub(crate) use from_bson::FromBSON; pub(crate) use from_csv::FromCSV; pub(crate) use from_ini::FromINI; pub(crate) use from_json::FromJSON; +pub(crate) use from_sqlite::FromSQLite; pub(crate) use from_toml::FromTOML; pub(crate) use from_tsv::FromTSV; pub(crate) use from_xml::FromXML; @@ -111,6 +115,7 @@ pub(crate) use to_array::ToArray; pub(crate) use to_bson::ToBSON; pub(crate) use to_csv::ToCSV; pub(crate) use to_json::ToJSON; +pub(crate) use to_sqlite::ToSQLite; pub(crate) use to_toml::ToTOML; pub(crate) use to_tsv::ToTSV; pub(crate) use to_yaml::ToYAML; diff --git a/src/commands/from_bson.rs b/src/commands/from_bson.rs index e244614ccf..ff1fc267c1 100644 --- a/src/commands/from_bson.rs +++ b/src/commands/from_bson.rs @@ -45,7 +45,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into) -> Tagged Value::Primitive(Primitive::Boolean(*b)).tagged(tag), - Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag), + Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag), Bson::RegExp(r, opts) => { let mut collected = TaggedDictBuilder::new(tag); collected.insert_tagged( diff --git a/src/commands/from_sqlite.rs b/src/commands/from_sqlite.rs new file mode 100644 index 0000000000..2ebf67690c --- /dev/null +++ b/src/commands/from_sqlite.rs @@ -0,0 +1,139 @@ +use crate::commands::WholeStreamCommand; +use crate::errors::ShellError; +use crate::object::base::OF64; +use crate::object::{Primitive, TaggedDictBuilder, Value}; +use crate::prelude::*; +use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS}; +use std::io::Write; +use std::path::Path; + +pub struct FromSQLite; + +impl WholeStreamCommand for FromSQLite { + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + from_sqlite(args, registry) + } + + fn name(&self) -> &str { + "from-sqlite" + } + + fn signature(&self) -> Signature { + Signature::build("from-sqlite") + } +} + +pub fn convert_sqlite_file_to_nu_value( + path: &Path, + tag: impl Into + Clone, +) -> Result, rusqlite::Error> { + let conn = Connection::open(path)?; + + let mut meta_out = Vec::new(); + let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?; + let mut meta_rows = meta_stmt.query(NO_PARAMS)?; + while let Some(meta_row) = meta_rows.next()? { + let table_name: String = meta_row.get(0)?; + let mut meta_dict = TaggedDictBuilder::new(tag.clone()); + let mut out = Vec::new(); + let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?; + let mut table_rows = table_stmt.query(NO_PARAMS)?; + while let Some(table_row) = table_rows.next()? { + out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?) + } + meta_dict.insert_tagged( + "table_name".to_string(), + Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()), + ); + meta_dict.insert_tagged("table_values", Value::List(out).tagged(tag.clone())); + meta_out.push(meta_dict.into_tagged_value()); + } + let tag = tag.into(); + Ok(Value::List(meta_out).tagged(tag)) +} + +fn convert_sqlite_row_to_nu_value( + row: &Row, + tag: impl Into + Clone, +) -> Result, rusqlite::Error> { + let mut collected = TaggedDictBuilder::new(tag.clone()); + for (i, c) in row.columns().iter().enumerate() { + collected.insert_tagged( + c.name().to_string(), + convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()), + ); + } + return Ok(collected.into_tagged_value()); +} + +fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into + Clone) -> Tagged { + match value { + ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag), + ValueRef::Integer(i) => Value::Primitive(Primitive::Int(i)).tagged(tag), + ValueRef::Real(f) => Value::Primitive(Primitive::Float(OF64::from(f))).tagged(tag), + t @ ValueRef::Text(_) => { + // this unwrap is safe because we know the ValueRef is Text. + Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag) + } + ValueRef::Blob(u) => Value::Binary(u.to_owned()).tagged(tag), + } +} + +pub fn from_sqlite_bytes_to_value( + mut bytes: Vec, + tag: impl Into + Clone, +) -> Result, std::io::Error> { + // FIXME: should probably write a sqlite virtual filesystem + // that will allow us to use bytes as a file to avoid this + // write out, but this will require C code. Might be + // best done as a PR to rusqlite. + let mut tempfile = tempfile::NamedTempFile::new()?; + tempfile.write_all(bytes.as_mut_slice())?; + match convert_sqlite_file_to_nu_value(tempfile.path(), tag) { + Ok(value) => Ok(value), + Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)), + } +} + +fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let span = args.name_span(); + let input = args.input; + + let stream = async_stream_block! { + let values: Vec> = input.values.collect().await; + + for value in values { + let value_tag = value.tag(); + match value.item { + Value::Binary(vb) => + match from_sqlite_bytes_to_value(vb, span) { + Ok(x) => yield ReturnSuccess::value(x), + Err(_) => { + yield Err(ShellError::labeled_error_with_secondary( + "Could not parse as SQLite", + "input cannot be parsed as SQLite", + span, + "value originates from here", + value_tag.span, + )) + } + } + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected a string from pipeline", + "requires string input", + span, + "value originates from here", + value_tag.span, + )), + + } + } + }; + + Ok(stream.to_output_stream()) +} diff --git a/src/commands/open.rs b/src/commands/open.rs index c4cd057147..af43e698ff 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -527,6 +527,16 @@ pub fn parse_binary_as_value( }, ) } + Some(ref x) if x == "db" => { + crate::commands::from_sqlite::from_sqlite_bytes_to_value(contents, contents_tag) + .map_err(move |_| { + ShellError::labeled_error( + "Could not open as SQLite", + "could not open as SQLite", + name_span, + ) + }) + } _ => Ok(Value::Binary(contents).tagged(contents_tag)), } } diff --git a/src/commands/to_sqlite.rs b/src/commands/to_sqlite.rs new file mode 100644 index 0000000000..55e001b0ba --- /dev/null +++ b/src/commands/to_sqlite.rs @@ -0,0 +1,183 @@ +use crate::commands::WholeStreamCommand; +use crate::object::{Dictionary, Primitive, Value}; +use crate::prelude::*; +use hex::encode; +use rusqlite::{Connection, NO_PARAMS}; +use std::io::Read; + +pub struct ToSQLite; + +impl WholeStreamCommand for ToSQLite { + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + to_sqlite(args, registry) + } + + fn name(&self) -> &str { + "to-sqlite" + } + + fn signature(&self) -> Signature { + Signature::build("to-sqlite") + } +} + +fn comma_concat(acc: String, current: String) -> String { + if acc == "" { + current + } else { + format!("{}, {}", acc, current) + } +} + +fn get_columns(rows: &Vec>) -> Result { + match &rows[0].item { + Value::Object(d) => Ok(d + .entries + .iter() + .map(|(k, _v)| k.clone()) + .fold("".to_string(), comma_concat)), + _ => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table column names", + )), + } +} + +fn nu_value_to_sqlite_string(v: Value) -> String { + match v { + Value::Binary(u) => format!("x'{}'", encode(u)), + Value::Primitive(p) => match p { + Primitive::Nothing => "NULL".into(), + Primitive::Int(i) => format!("{}", i), + Primitive::Float(f) => format!("{}", f.into_inner()), + Primitive::Bytes(u) => format!("{}", u), + Primitive::String(s) => format!("'{}'", s.replace("'", "''")), + Primitive::Boolean(true) => "1".into(), + Primitive::Boolean(_) => "0".into(), + Primitive::Date(d) => format!("'{}'", d), + Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")), + Primitive::BeginningOfStream => "NULL".into(), + Primitive::EndOfStream => "NULL".into(), + }, + _ => "NULL".into(), + } +} + +fn get_insert_values(rows: Vec>) -> Result { + let values: Result, _> = rows + .into_iter() + .map(|value| match value.item { + Value::Object(d) => Ok(format!( + "({})", + d.entries + .iter() + .map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone())) + .fold("".to_string(), comma_concat) + )), + _ => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table column names", + )), + }) + .collect(); + let values = values?; + Ok(values.into_iter().fold("".to_string(), comma_concat)) +} + +fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> { + let table_name = match table.entries.get("table_name") { + Some(Tagged { + item: Value::Primitive(Primitive::String(table_name)), + .. + }) => table_name, + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table name", + )) + } + }; + let (columns, insert_values) = match table.entries.get("table_values") { + Some(Tagged { + item: Value::List(l), + .. + }) => (get_columns(l), get_insert_values(l.to_vec())), + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table values", + )) + } + }; + let create = format!("create table {}({})", table_name, columns?); + let insert = format!("insert into {} values {}", table_name, insert_values?); + Ok((create, insert)) +} + +fn sqlite_input_stream_to_bytes( + values: Vec>, +) -> Result, std::io::Error> { + // FIXME: should probably write a sqlite virtual filesystem + // that will allow us to use bytes as a file to avoid this + // write out, but this will require C code. Might be + // best done as a PR to rusqlite. + let mut tempfile = tempfile::NamedTempFile::new()?; + let conn = match Connection::open(tempfile.path()) { + Ok(conn) => conn, + Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)), + }; + let tag = values[0].tag.clone(); + for value in values.into_iter() { + match value.item() { + Value::Object(d) => { + let (create, insert) = generate_statements(d.to_owned())?; + match conn + .execute(&create, NO_PARAMS) + .and_then(|_| conn.execute(&insert, NO_PARAMS)) + { + Ok(_) => (), + Err(e) => { + println!("{}", create); + println!("{}", insert); + println!("{:?}", e); + return Err(std::io::Error::new(std::io::ErrorKind::Other, e)); + } + } + } + other => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Expected object, found {:?}", other), + )) + } + } + } + let mut out = Vec::new(); + tempfile.read_to_end(&mut out)?; + Ok(Value::Binary(out).tagged(tag)) +} + +fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let name_span = args.name_span(); + let stream = async_stream_block! { + let values: Vec<_> = args.input.into_vec().await; + match sqlite_input_stream_to_bytes(values) { + Ok(out) => { + yield ReturnSuccess::value(out) + } + Err(_) => { + yield Err(ShellError::labeled_error( + "Expected an object with SQLite-compatible structure from pipeline", + "requires SQLite-compatible input", + name_span, + )) + } + }; + }; + Ok(stream.to_output_stream()) +} diff --git a/src/utils.rs b/src/utils.rs index 159907ea52..4ed9be2540 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -266,6 +266,10 @@ mod tests { loc: fixtures().join("sample.bson"), at: 0 }, + Res { + loc: fixtures().join("sample.db"), + at: 0 + }, Res { loc: fixtures().join("sample.ini"), at: 0 diff --git a/tests/command_open_tests.rs b/tests/command_open_tests.rs index 0b01cc6b8e..968bc7531b 100644 --- a/tests/command_open_tests.rs +++ b/tests/command_open_tests.rs @@ -6,23 +6,22 @@ use helpers::{Playground, Stub::*}; #[test] fn recognizes_csv() { Playground::setup("open_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "nu.zion.csv", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "nu.zion.csv", + r#" author,lang,source Jonathan Turner,Rust,New Zealand Andres N. Robalino,Rust,Ecuador Yehuda Katz,Rust,Estados Unidos - "# + "#, )]); let actual = nu!( cwd: dirs.test(), h::pipeline( r#" - open nu.zion.csv - | where author == "Andres N. Robalino" - | get source + open nu.zion.csv + | where author == "Andres N. Robalino" + | get source | echo $it "# )); @@ -46,11 +45,11 @@ fn open_can_parse_bson_2() { let actual = nu!( cwd: "tests/fixtures/formats", h::pipeline( r#" - open sample.bson - | get root - | nth 6 - | get b - | get '$binary_subtype' + open sample.bson + | get root + | nth 6 + | get b + | get '$binary_subtype' | echo $it "# )); @@ -58,6 +57,21 @@ fn open_can_parse_bson_2() { assert_eq!(actual, "function"); } +#[test] +fn open_can_parse_sqlite() { + let actual = nu!( + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open sample.db + | get table_values + | nth 2 + | get x + | echo $it"# + )); + + assert_eq!(actual, "hello"); +} + #[test] fn open_can_parse_toml() { let actual = nu!( @@ -74,8 +88,8 @@ fn open_can_parse_tsv() { cwd: "tests/fixtures/formats", h::pipeline( r#" open caco3_plastics.tsv - | first 1 - | get origin + | first 1 + | get origin | echo $it "# )); @@ -88,8 +102,8 @@ fn open_can_parse_json() { let actual = nu!( cwd: "tests/fixtures/formats", h::pipeline( r#" - open sgml_description.json - | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee + open sgml_description.json + | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee | echo $it "# )); diff --git a/tests/filters_test.rs b/tests/filters_test.rs index deabdaa257..33baf67fe3 100644 --- a/tests/filters_test.rs +++ b/tests/filters_test.rs @@ -16,14 +16,13 @@ fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() { #[test] fn converts_structured_table_to_csv_text() { Playground::setup("filter_to_csv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "csv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "csv_text_sample.txt", + r#" importer,shipper,tariff_item,name,origin Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia - "# + "#, )]); let actual = nu!( @@ -47,14 +46,13 @@ fn converts_structured_table_to_csv_text() { #[test] fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() { Playground::setup("filter_to_csv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "csv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "csv_text_sample.txt", + r#" importer,shipper,tariff_item,name,origin Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia - "# + "#, )]); let actual = nu!( @@ -76,10 +74,9 @@ fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() { #[test] fn converts_from_csv_text_to_structured_table() { Playground::setup("filter_from_csv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first_name,last_name,rusty_luck Andrés,Robalino,1 Jonathan,Turner,1 @@ -106,10 +103,9 @@ fn converts_from_csv_text_to_structured_table() { #[test] fn converts_from_csv_text_skipping_headers_to_structured_table() { Playground::setup("filter_from_csv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first_name,last_name,rusty_luck Andrés,Robalino,1 Jonathan,Turner,1 @@ -152,10 +148,9 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() { #[test] fn converts_from_json_text_to_structured_table() { Playground::setup("filter_from_json_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "katz.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "katz.txt", + r#" { "katz": [ {"name": "Yehuda", "rusty_luck": 1}, @@ -173,17 +168,15 @@ fn converts_from_json_text_to_structured_table() { ); assert_eq!(actual, "4"); - }) } #[test] fn converts_from_json_text_recognizing_objects_independendtly_to_structured_table() { Playground::setup("filter_from_json_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "katz.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "katz.txt", + r#" {"name": "Yehuda", "rusty_luck": 1} {"name": "Jonathan", "rusty_luck": 1} {"name": "Andres", "rusty_luck": 1} @@ -209,10 +202,9 @@ fn converts_from_json_text_recognizing_objects_independendtly_to_structured_tabl #[test] fn converts_structured_table_to_json_text() { Playground::setup("filter_to_json_test", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "sample.txt", + r#" JonAndrehudaTZ,3 GorbyPuff,100 "#, @@ -250,14 +242,13 @@ fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table() { #[test] fn converts_structured_table_to_tsv_text() { Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "tsv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "tsv_text_sample.txt", + r#" importer shipper tariff_item name origin Plasticos Rival Reverte 2509000000 Calcium carbonate Spain Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia - "# + "#, )]); let actual = nu!( @@ -281,14 +272,13 @@ fn converts_structured_table_to_tsv_text() { #[test] fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() { Playground::setup("filter_to_tsv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "tsv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "tsv_text_sample.txt", + r#" importer shipper tariff_item name origin Plasticos Rival Reverte 2509000000 Calcium carbonate Spain Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia - "# + "#, )]); let actual = nu!( @@ -310,10 +300,9 @@ fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() { #[test] fn converts_from_tsv_text_to_structured_table() { Playground::setup("filter_from_tsv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first Name Last Name rusty_luck Andrés Robalino 1 Jonathan Turner 1 @@ -340,10 +329,9 @@ fn converts_from_tsv_text_to_structured_table() { #[test] fn converts_from_tsv_text_skipping_headers_to_structured_table() { Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first Name Last Name rusty_luck Andrés Robalino 1 Jonathan Turner 1 @@ -354,11 +342,11 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() { let actual = nu!( cwd: dirs.test(), h::pipeline( r#" - open los_tres_amigos.txt - | from-tsv --headerless - | get Column3 - | str --to-int - | sum + open los_tres_amigos.txt + | from-tsv --headerless + | get Column3 + | str --to-int + | sum | echo $it "# )); @@ -368,21 +356,50 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() { } #[test] -fn can_convert_json_text_to_bson_and_back_into_table() { +fn can_convert_table_to_bson_and_back_into_table() { let actual = nu!( - cwd: "tests/fixtures/formats", - "open sample.bson | to-bson | from-bson | get root | nth 1 | get b | echo $it" - ); + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open sample.bson + | to-bson + | from-bson + | get root + | nth 1 + | get b + | echo $it"# + )); assert_eq!(actual, "whel"); } +#[test] +fn can_convert_table_to_sqlite_and_back_into_table() { + let actual = nu!( + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open sample.db + | to-sqlite + | from-sqlite + | get table_values + | nth 2 + | get x + | echo $it"# + )); + + assert_eq!(actual, "hello"); +} + #[test] fn can_convert_table_to_toml_text_and_from_toml_text_back_into_table() { let actual = nu!( - cwd: "tests/fixtures/formats", - "open cargo_sample.toml | to-toml | from-toml | get package.name | echo $it" - ); + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open cargo_sample.toml + | to-toml + | from-toml + | get package.name + | echo $it"# + )); assert_eq!(actual, "nu"); } diff --git a/tests/fixtures/formats/sample.db b/tests/fixtures/formats/sample.db new file mode 100644 index 0000000000000000000000000000000000000000..df67bfce3bcddb776545e14d4b6fb56c42990a1a GIT binary patch literal 16384 zcmeI(Jxd%x7zglq*<10#^4u2ZBBq#BxeJo!>VT&lHa1Z!Q|-k)cVTi%b|*2h^Ap$$ zegHd58*BXl;ui>_kV*(xB%m{UnrLIG82%4CFVE~V&957FXX*VyW)d~Xi%+pp9oAr+ zvl*orV`Q6W+igfYxtsIb7Y~fY-oD%mY=NW|MhCQI2LuEl009U<00Izz00bZa0SG|g zKN9FjKboB6b0+TVWT5ldl%w98TkLh0duq8myUWi+v#7kV`euUHt={-w;Oo=ww4 z>pXN3$h#>I#NU*MA+K{8HL7C9-TA*pXN>;PC0)=@J4HYM0uX=z1Rwwb2tWV=5P$## zAn-f|L`^Wy6QZ%>Ng?>I>)r@1#@+efqiy&7|D1l<2?7ETfB*y_009U<00Izz00bZa zfkzaW_vB{Nq* Date: Sat, 31 Aug 2019 12:59:21 +1200 Subject: [PATCH 7/9] Finish up enter and save --- src/commands/classified.rs | 38 +------ src/commands/enter.rs | 105 +++++++++++++++++-- src/commands/save.rs | 205 +++++++++++++++++-------------------- 3 files changed, 197 insertions(+), 151 deletions(-) diff --git a/src/commands/classified.rs b/src/commands/classified.rs index cf3689c1e6..1430707637 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -146,41 +146,9 @@ impl InternalCommand { .insert_at_current(Box::new(ValueShell::new(value))); } CommandAction::EnterShell(location) => { - let path = std::path::Path::new(&location); - - if path.is_dir() { - // If it's a directory, add a new filesystem shell - context.shell_manager.insert_at_current(Box::new( - FilesystemShell::with_location( - location, - context.registry().clone(), - )?, - )); - } else { - // If it's a file, attempt to open the file as a value and enter it - let cwd = context.shell_manager.path(); - - let full_path = std::path::PathBuf::from(cwd); - - let (_file_extension, contents, contents_tag, span_source) = - crate::commands::open::fetch( - &full_path, - &location, - Span::unknown(), - ) - .await?; - - if let Some(uuid) = contents_tag.origin { - // If we have loaded something, track its source - context.add_span_source(uuid, span_source); - } - - context - .shell_manager - .insert_at_current(Box::new(ValueShell::new( - contents.tagged(contents_tag), - ))) - } + context.shell_manager.insert_at_current(Box::new( + FilesystemShell::with_location(location, context.registry().clone())?, + )); } CommandAction::PreviousShell => { context.shell_manager.prev(); diff --git a/src/commands/enter.rs b/src/commands/enter.rs index 19f6b9c8cb..63316b7082 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -1,8 +1,10 @@ use crate::commands::command::CommandAction; use crate::commands::PerItemCommand; +use crate::commands::UnevaluatedCallInfo; use crate::errors::ShellError; use crate::parser::registry; use crate::prelude::*; +use std::path::PathBuf; pub struct Enter; @@ -18,18 +20,109 @@ impl PerItemCommand for Enter { fn run( &self, call_info: &CallInfo, - _registry: ®istry::CommandRegistry, - _raw_args: &RawCommandArgs, + registry: ®istry::CommandRegistry, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { + let registry = registry.clone(); + let raw_args = raw_args.clone(); match call_info.args.expect_nth(0)? { Tagged { item: Value::Primitive(Primitive::String(location)), .. - } => Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterShell( - location.to_string(), - )))] - .into()), + } => { + let location = location.to_string(); + let location_clone = location.to_string(); + if PathBuf::from(location).is_dir() { + Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterShell( + location_clone, + )))] + .into()) + } else { + let stream = async_stream_block! { + // If it's a file, attempt to open the file as a value and enter it + let cwd = raw_args.shell_manager.path(); + + let full_path = std::path::PathBuf::from(cwd); + + let (file_extension, contents, contents_tag, span_source) = + crate::commands::open::fetch( + &full_path, + &location_clone, + Span::unknown(), + ) + .await.unwrap(); + + if let Some(uuid) = contents_tag.origin { + // If we have loaded something, track its source + yield ReturnSuccess::action(CommandAction::AddSpanSource( + uuid, + span_source, + )); + } + + + match contents { + Value::Primitive(Primitive::String(_)) => { + let tagged_contents = contents.tagged(contents_tag); + + if let Some(extension) = file_extension { + let command_name = format!("from-{}", extension); + if let Some(converter) = + registry.get_command(&command_name) + { + let new_args = RawCommandArgs { + host: raw_args.host, + shell_manager: raw_args.shell_manager, + call_info: UnevaluatedCallInfo { + args: crate::parser::hir::Call { + head: raw_args.call_info.args.head, + positional: None, + named: None, + }, + source: raw_args.call_info.source, + source_map: raw_args.call_info.source_map, + name_span: raw_args.call_info.name_span, + }, + }; + let mut result = converter.run( + new_args.with_input(vec![tagged_contents]), + ®istry, + ); + let result_vec: Vec> = + result.drain_vec().await; + for res in result_vec { + match res { + Ok(ReturnSuccess::Value(Tagged { + item, + .. + })) => { + yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell( + Tagged { + item: item, + tag: contents_tag, + }))); + } + x => yield x, + } + } + } else { + yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents))); + } + } else { + yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents))); + } + } + _ => { + let tagged_contents = contents.tagged(contents_tag); + + yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents))); + } + } + }; + Ok(stream.to_output_stream()) + } + } x => Ok( vec![Ok(ReturnSuccess::Action(CommandAction::EnterValueShell( x.clone(), diff --git a/src/commands/save.rs b/src/commands/save.rs index 1ecd22e91f..11b245ef9c 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -1,8 +1,4 @@ -use crate::commands::to_csv::{to_string as to_csv_to_string, value_to_csv_value}; -use crate::commands::to_tsv::{to_string as to_tsv_to_string, value_to_tsv_value}; -use crate::commands::to_json::value_to_json_value; -use crate::commands::to_toml::value_to_toml_value; -use crate::commands::to_yaml::value_to_yaml_value; +use crate::commands::UnevaluatedCallInfo; use crate::commands::WholeStreamCommand; use crate::errors::ShellError; use crate::object::Value; @@ -33,7 +29,7 @@ impl WholeStreamCommand for Save { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - args.process(registry, save)?.run() + Ok(args.process_raw(registry, save)?.run()) } } @@ -47,16 +43,19 @@ fn save( name, shell_manager, source_map, + host, + commands: registry, .. }: RunnableContext, + raw_args: RawCommandArgs, ) -> Result { let mut full_path = PathBuf::from(shell_manager.path()); let name_span = name; - if path.is_none() { - let source_map = source_map.clone(); - let stream = async_stream_block! { - let input: Vec> = input.values.collect().await; + let source_map = source_map.clone(); + let stream = async_stream_block! { + let input: Vec> = input.values.collect().await; + if path.is_none() { // If there is no filename, check the metadata for the origin filename if input.len() > 0 { let origin = input[0].origin(); @@ -88,50 +87,99 @@ fn save( name_span, )); } - - let content = if !save_raw { - to_string_for(full_path.extension(), &input).await - } else { - string_from(&input) - }; - - match content { - Ok(save_data) => match std::fs::write(full_path, save_data) { - Ok(o) => o, - Err(e) => yield Err(ShellError::string(e.to_string())), - }, - Err(e) => yield Err(ShellError::string(e.to_string())), + } else { + if let Some(file) = path { + full_path.push(file.item()); } - - }; - - Ok(OutputStream::new(stream)) - } else { - if let Some(file) = path { - full_path.push(file.item()); } - let stream = async_stream_block! { - let input: Vec> = input.values.collect().await; - - let content = if !save_raw { - to_string_for(full_path.extension(), &input).await + let content = if !save_raw { + if let Some(extension) = full_path.extension() { + let command_name = format!("to-{}", extension.to_str().unwrap()); + if let Some(converter) = registry.get_command(&command_name) { + let new_args = RawCommandArgs { + host: host, + shell_manager: shell_manager, + call_info: UnevaluatedCallInfo { + args: crate::parser::hir::Call { + head: raw_args.call_info.args.head, + positional: None, + named: None + }, + source: raw_args.call_info.source, + source_map: raw_args.call_info.source_map, + name_span: raw_args.call_info.name_span, + } + }; + let mut result = converter.run(new_args.with_input(input), ®istry); + let result_vec: Vec> = result.drain_vec().await; + let mut result_string = String::new(); + for res in result_vec { + match res { + Ok(ReturnSuccess::Value(Tagged { item: Value::Primitive(Primitive::String(s)), .. })) => { + result_string.push_str(&s); + } + _ => { + yield Err(ShellError::labeled_error( + "Save could not successfully save", + "unexpected data during saveS", + name_span, + )); + }, + } + } + Ok(result_string) + } else { + let mut result_string = String::new(); + for res in input { + match res { + Tagged { item: Value::Primitive(Primitive::String(s)), .. } => { + result_string.push_str(&s); + } + _ => { + yield Err(ShellError::labeled_error( + "Save could not successfully save", + "unexpected data during saveS", + name_span, + )); + }, + } + } + Ok(result_string) + } } else { - string_from(&input) - }; - - match content { - Ok(save_data) => match std::fs::write(full_path, save_data) { - Ok(o) => o, - Err(e) => yield Err(ShellError::string(e.to_string())), - }, - Err(e) => yield Err(ShellError::string(e.to_string())), + let mut result_string = String::new(); + for res in input { + match res { + Tagged { item: Value::Primitive(Primitive::String(s)), .. } => { + result_string.push_str(&s); + } + _ => { + yield Err(ShellError::labeled_error( + "Save could not successfully save", + "unexpected data during saveS", + name_span, + )); + }, + } + } + Ok(result_string) } - + } else { + string_from(&input) }; - Ok(OutputStream::new(stream)) - } + match content { + Ok(save_data) => match std::fs::write(full_path, save_data) { + Ok(o) => o, + Err(e) => yield Err(ShellError::string(e.to_string())), + }, + Err(e) => yield Err(ShellError::string(e.to_string())), + } + + }; + + Ok(OutputStream::new(stream)) } fn string_from(input: &Vec>) -> Result { @@ -153,66 +201,3 @@ fn string_from(input: &Vec>) -> Result { Ok(save_data) } - -async fn to_string_for( - ext: Option<&std::ffi::OsStr>, - input: &Vec>, -) -> Result { - let contents = match ext { - Some(x) if x == "csv" => { - if input.len() != 1 { - return Err(ShellError::string( - "saving to csv requires a single object (or use --raw)", - )); - } - to_csv_to_string(&value_to_csv_value(&input[0]))? - } - Some(x) if x == "tsv" => { - if input.len() != 1 { - return Err(ShellError::string( - "saving to tsv requires a single object (or use --raw)", - )); - } - to_tsv_to_string(&value_to_tsv_value(&input[0]))? - } - Some(x) if x == "toml" => { - if input.len() != 1 { - return Err(ShellError::string( - "saving to toml requires a single object (or use --raw)", - )); - } - toml::to_string(&value_to_toml_value(&input[0]))? - } - Some(x) if x == "json" => { - if input.len() != 1 { - return Err(ShellError::string( - "saving to json requires a single object (or use --raw)", - )); - } - serde_json::to_string(&value_to_json_value(&input[0]))? - } - Some(x) if x == "yml" => { - if input.len() != 1 { - return Err(ShellError::string( - "saving to yml requires a single object (or use --raw)", - )); - } - serde_yaml::to_string(&value_to_yaml_value(&input[0]))? - } - Some(x) if x == "yaml" => { - if input.len() != 1 { - return Err(ShellError::string( - "saving to yaml requires a single object (or use --raw)", - )); - } - serde_yaml::to_string(&value_to_yaml_value(&input[0]))? - } - _ => { - return Err(ShellError::string( - "tried saving a single object with an unrecognized format.", - )) - } - }; - - Ok(contents) -} From 2cde4da43f08e0a91f77a14f9c8b733a36143c08 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 31 Aug 2019 13:35:53 +1200 Subject: [PATCH 8/9] Partially fix list support --- src/commands/open.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/commands/open.rs b/src/commands/open.rs index 012f398645..a085862cfe 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -103,6 +103,11 @@ fn run( let result_vec: Vec> = result.drain_vec().await; for res in result_vec { match res { + Ok(ReturnSuccess::Value(Tagged { item: Value::List(list), ..})) => { + for l in list { + yield Ok(ReturnSuccess::Value(l)); + } + } Ok(ReturnSuccess::Value(Tagged { item, .. })) => { yield Ok(ReturnSuccess::Value(Tagged { item: item, tag: contents_tag })); } From 2470e6dc24a19fe913def43fd35c31e7981e8020 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 31 Aug 2019 14:23:29 +1200 Subject: [PATCH 9/9] Expand lists loaded from sqlite --- src/commands/from_sqlite.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/commands/from_sqlite.rs b/src/commands/from_sqlite.rs index 2ebf67690c..3bcfa703cb 100644 --- a/src/commands/from_sqlite.rs +++ b/src/commands/from_sqlite.rs @@ -112,7 +112,14 @@ fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result match from_sqlite_bytes_to_value(vb, span) { - Ok(x) => yield ReturnSuccess::value(x), + Ok(x) => match x { + Tagged { item: Value::List(list), .. } => { + for l in list { + yield ReturnSuccess::value(l); + } + } + _ => yield ReturnSuccess::value(x), + } Err(_) => { yield Err(ShellError::labeled_error_with_secondary( "Could not parse as SQLite",