diff --git a/Cargo.lock b/Cargo.lock index 1c29147fe..3cc8903b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2247,6 +2247,7 @@ dependencies = [ "dirs 2.0.2", "dunce", "eml-parser", + "encoding_rs", "filesize", "futures 0.3.5", "futures-util", diff --git a/README.md b/README.md index db53f5cc0..df827e53b 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,10 @@ Nu is in heavy development, and will naturally change as it matures and people u | Completions | | X | | | | Completions are currently barebones, at best | Type-checking | | X | | | | Commands check basic types, but input/output isn't checked +# Current Roadmap + +We've added a `Roadmap Board` to help collaboratively capture the direction we're going for the current release as well as capture some important issues we'd like to see in NuShell. You can find the Roadmap [here](https://github.com/nushell/nushell/projects/2). + # Contributing See [Contributing](CONTRIBUTING.md) for details. diff --git a/TODO.md b/TODO.md index 2ac069f45..e69de29bb 100644 --- a/TODO.md +++ b/TODO.md @@ -1,60 +0,0 @@ -This pattern is extremely repetitive and can be abstracted: - -```rs - let args = args.evaluate_once(registry)?; - let tag = args.name_tag(); - let input = args.input; - - let stream = async_stream! { - let values: Vec = input.values.collect().await; - - let mut concat_string = String::new(); - let mut latest_tag: Option = None; - - for value in values { - latest_tag = Some(value_tag.clone()); - let value_span = value.tag.span; - - match &value.value { - UntaggedValue::Primitive(Primitive::String(s)) => { - concat_string.push_str(&s); - concat_string.push_str("\n"); - } - _ => yield Err(ShellError::labeled_error_with_secondary( - "Expected a string from pipeline", - "requires string input", - name_span, - "value originates from here", - value_span, - )), - - } - } - -``` - -Mandatory and Optional in parse_command - -trace_remaining? - -select_fields and select_fields take unnecessary Tag - -Value#value should be Value#untagged - -Unify dictionary building, probably around a macro - -sys plugin in own crate - -textview in own crate - -Combine atomic and atomic_parse in parser - -at_end_possible_ws needs to be comment and separator sensitive - -Eliminate unnecessary `nodes` parser - -#[derive(HasSpan)] - -Figure out a solution for the duplication in stuff like NumberShape vs. NumberExpressionShape - -use `struct Expander` from signature.rs \ No newline at end of file diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index 18652b40f..dcd062a7f 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -92,6 +92,7 @@ trash = { version = "1.0.1", optional = true } clipboard = { version = "0.5", optional = true } starship = { version = "0.41.3", optional = true } rayon = "1.3.0" +encoding_rs = "0.8.23" [target.'cfg(unix)'.dependencies] users = "0.10.0" diff --git a/crates/nu-cli/src/commands/alias.rs b/crates/nu-cli/src/commands/alias.rs index b6948572c..81ee024a3 100644 --- a/crates/nu-cli/src/commands/alias.rs +++ b/crates/nu-cli/src/commands/alias.rs @@ -45,8 +45,7 @@ impl WholeStreamCommand for Alias { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - // args.process(registry, alias)?.run() - alias(args, registry) + alias(args, registry).await } fn examples(&self) -> Vec { @@ -65,63 +64,78 @@ impl WholeStreamCommand for Alias { } } -// <<<<<<< HEAD -// pub fn alias(alias_args: AliasArgs, ctx: RunnableContext) -> Result { -// ======= -pub fn alias(args: CommandArgs, registry: &CommandRegistry) -> Result { +pub async fn alias( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let mut raw_input = args.raw_input.clone(); - let (AliasArgs { name, args: list, block, save}, ctx) = args.process(®istry).await?; - let mut processed_args: Vec = vec![]; + let mut raw_input = args.raw_input.clone(); + let ( + AliasArgs { + name, + args: list, + block, + save, + }, + _ctx, + ) = args.process(®istry).await?; + let mut processed_args: Vec = vec![]; - if let Some(true) = save { - let mut result = crate::data::config::read(name.clone().tag, &None)?; + if let Some(true) = save { + let mut result = crate::data::config::read(name.clone().tag, &None)?; - // process the alias to remove the --save flag - let left_brace = raw_input.find('{').unwrap_or(0); - let right_brace = raw_input.rfind('}').unwrap_or(raw_input.len()); - let mut left = raw_input[..left_brace].replace("--save", "").replace("-s", ""); - let mut right = raw_input[right_brace..].replace("--save", "").replace("-s", ""); - raw_input = format!("{}{}{}", left, &raw_input[left_brace..right_brace], right); + // process the alias to remove the --save flag + let left_brace = raw_input.find('{').unwrap_or(0); + let right_brace = raw_input.rfind('}').unwrap_or_else(|| raw_input.len()); + let left = raw_input[..left_brace] + .replace("--save", "") + .replace("-s", ""); + let right = raw_input[right_brace..] + .replace("--save", "") + .replace("-s", ""); + raw_input = format!("{}{}{}", left, &raw_input[left_brace..right_brace], right); - // create a value from raw_input alias - let alias: Value = raw_input.trim().to_string().into(); - let alias_start = raw_input.find("[").unwrap_or(0); // used to check if the same alias already exists + // create a value from raw_input alias + let alias: Value = raw_input.trim().to_string().into(); + let alias_start = raw_input.find('[').unwrap_or(0); // used to check if the same alias already exists - // add to startup if alias doesn't exist and replce if it does - match result.get_mut("startup") { - Some(startup) => { - if let UntaggedValue::Table(ref mut commands) = startup.value { - if let Some(command) = commands.iter_mut().find(|command| { - let cmd_str = command.as_string().unwrap_or_default(); - cmd_str.starts_with(&raw_input[..alias_start]) - }) { - *command = alias; - } else { - commands.push(alias); - } + // add to startup if alias doesn't exist and replce if it does + match result.get_mut("startup") { + Some(startup) => { + if let UntaggedValue::Table(ref mut commands) = startup.value { + if let Some(command) = commands.iter_mut().find(|command| { + let cmd_str = command.as_string().unwrap_or_default(); + cmd_str.starts_with(&raw_input[..alias_start]) + }) { + *command = alias; + } else { + commands.push(alias); } } - None => { - let mut table = UntaggedValue::table(&[alias]); - result.insert("startup".to_string(), table.into_value(Tag::default())); - } } - config::write(&result, &None)?; - } - - for item in list.iter() { - if let Ok(string) = item.as_string() { - processed_args.push(format!("${}", string)); - } else { - yield Err(ShellError::labeled_error("Expected a string", "expected a string", item.tag())); + None => { + let table = UntaggedValue::table(&[alias]); + result.insert("startup".to_string(), table.into_value(Tag::default())); } } - yield ReturnSuccess::action(CommandAction::AddAlias(name.to_string(), processed_args, block.clone())) - }; + config::write(&result, &None)?; + } - Ok(stream.to_output_stream()) + for item in list.iter() { + if let Ok(string) = item.as_string() { + processed_args.push(format!("${}", string)); + } else { + return Err(ShellError::labeled_error( + "Expected a string", + "expected a string", + item.tag(), + )); + } + } + + Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::AddAlias(name.to_string(), processed_args, block), + ))) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/autoview.rs b/crates/nu-cli/src/commands/autoview.rs index eee694efb..c587d603b 100644 --- a/crates/nu-cli/src/commands/autoview.rs +++ b/crates/nu-cli/src/commands/autoview.rs @@ -9,7 +9,6 @@ use parking_lot::Mutex; use prettytable::format::{FormatBuilder, LinePosition, LineSeparator}; use prettytable::{color, Attr, Cell, Row, Table}; use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; use textwrap::fill; pub struct Autoview; @@ -115,23 +114,12 @@ pub async fn autoview(context: RunnableContext) -> Result { let ctrl_c = context.ctrl_c.clone(); - let stream = async_stream! { - yield Ok(x); - yield Ok(y); + let xy = vec![x, y]; + let xy_stream = futures::stream::iter(xy) + .chain(input_stream) + .interruptible(ctrl_c); - loop { - match input_stream.next().await { - Some(z) => { - if ctrl_c.load(Ordering::SeqCst) { - break; - } - yield Ok(z); - } - _ => break, - } - } - }; - let stream = stream.to_input_stream(); + let stream = InputStream::from_stream(xy_stream); if let Some(table) = table { let command_args = create_default_command_args(&context).with_input(stream); diff --git a/crates/nu-cli/src/commands/average.rs b/crates/nu-cli/src/commands/average.rs index 80b786b6f..d7363f3c2 100644 --- a/crates/nu-cli/src/commands/average.rs +++ b/crates/nu-cli/src/commands/average.rs @@ -4,9 +4,7 @@ use crate::utils::data_processing::{reducer_for, Reduce}; use bigdecimal::FromPrimitive; use nu_errors::ShellError; use nu_protocol::hir::{convert_number_to_u64, Number, Operator}; -use nu_protocol::{ - Dictionary, Primitive, ReturnSuccess, ReturnValue, Signature, UntaggedValue, Value, -}; +use nu_protocol::{Dictionary, Primitive, ReturnSuccess, Signature, UntaggedValue, Value}; use num_traits::identities::Zero; use indexmap::map::IndexMap; @@ -42,6 +40,7 @@ impl WholeStreamCommand for Average { name: args.call_info.name_tag, raw_input: args.raw_input, }) + .await } fn examples(&self) -> Vec { @@ -53,53 +52,48 @@ impl WholeStreamCommand for Average { } } -fn average( +async fn average( RunnableContext { mut input, name, .. }: RunnableContext, ) -> Result { - let stream = async_stream! { - let mut values: Vec = input.drain_vec().await; - let action = reducer_for(Reduce::Sum); + let values: Vec = input.drain_vec().await; - if values.iter().all(|v| if let UntaggedValue::Primitive(_) = v.value {true} else {false}) { - match avg(&values, name) { - Ok(result) => yield ReturnSuccess::value(result), - Err(err) => yield Err(err), - } - } else { - let mut column_values = IndexMap::new(); - for value in values { - match value.value { - UntaggedValue::Row(row_dict) => { - for (key, value) in row_dict.entries.iter() { - column_values - .entry(key.clone()) - .and_modify(|v: &mut Vec| v.push(value.clone())) - .or_insert(vec![value.clone()]); - } - }, - table => {}, - }; - } - - let mut column_totals = IndexMap::new(); - for (col_name, col_vals) in column_values { - match avg(&col_vals, &name) { - Ok(result) => { - column_totals.insert(col_name, result); - } - Err(err) => yield Err(err), + if values.iter().all(|v| v.is_primitive()) { + match avg(&values, name) { + Ok(result) => Ok(OutputStream::one(ReturnSuccess::value(result))), + Err(err) => Err(err), + } + } else { + let mut column_values = IndexMap::new(); + for value in values { + if let UntaggedValue::Row(row_dict) = value.value { + for (key, value) in row_dict.entries.iter() { + column_values + .entry(key.clone()) + .and_modify(|v: &mut Vec| v.push(value.clone())) + .or_insert(vec![value.clone()]); } } - yield ReturnSuccess::value( - UntaggedValue::Row(Dictionary {entries: column_totals}).into_untagged_value()) } - }; - let stream: BoxStream<'static, ReturnValue> = stream.boxed(); + let mut column_totals = IndexMap::new(); + for (col_name, col_vals) in column_values { + match avg(&col_vals, &name) { + Ok(result) => { + column_totals.insert(col_name, result); + } + Err(err) => return Err(err), + } + } - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value( + UntaggedValue::Row(Dictionary { + entries: column_totals, + }) + .into_untagged_value(), + ))) + } } fn avg(values: &[Value], name: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/command.rs b/crates/nu-cli/src/commands/command.rs index 77b51a4d2..b5a138eda 100644 --- a/crates/nu-cli/src/commands/command.rs +++ b/crates/nu-cli/src/commands/command.rs @@ -389,42 +389,44 @@ impl WholeStreamCommand for FnFilterCommand { ctrl_c, shell_manager, call_info, - mut input, + input, .. }: CommandArgs, registry: &CommandRegistry, ) -> Result { - let host: Arc> = host.clone(); - let registry: CommandRegistry = registry.clone(); + let registry = Arc::new(registry.clone()); let func = self.func; - let stream = async_stream! { - while let Some(it) = input.next().await { + Ok(input + .then(move |it| { + let host = host.clone(); let registry = registry.clone(); - let call_info = match call_info.clone().evaluate_with_new_it(®istry, &it).await { - Err(err) => { yield Err(err); return; }, - Ok(args) => args, - }; - - let args = EvaluatedFilterCommandArgs::new( - host.clone(), - ctrl_c.clone(), - shell_manager.clone(), - call_info, - ); - - match func(args) { - Err(err) => yield Err(err), - Ok(mut stream) => { - while let Some(value) = stream.values.next().await { - yield value; + let ctrl_c = ctrl_c.clone(); + let shell_manager = shell_manager.clone(); + let call_info = call_info.clone(); + async move { + let call_info = match call_info.evaluate_with_new_it(&*registry, &it).await { + Err(err) => { + return OutputStream::one(Err(err)); } + Ok(args) => args, + }; + + let args = EvaluatedFilterCommandArgs::new( + host.clone(), + ctrl_c.clone(), + shell_manager.clone(), + call_info, + ); + + match func(args) { + Err(err) => return OutputStream::one(Err(err)), + Ok(stream) => stream, } } - } - }; - - Ok(stream.to_output_stream()) + }) + .flatten() + .to_output_stream()) } } diff --git a/crates/nu-cli/src/commands/echo.rs b/crates/nu-cli/src/commands/echo.rs index e510794c3..4d9b22f57 100644 --- a/crates/nu-cli/src/commands/echo.rs +++ b/crates/nu-cli/src/commands/echo.rs @@ -32,7 +32,7 @@ impl WholeStreamCommand for Echo { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - echo(args, registry) + echo(args, registry).await } fn examples(&self) -> Vec { @@ -51,67 +51,62 @@ impl WholeStreamCommand for Echo { } } -fn echo(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn echo(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (args, _): (EchoArgs, _) = args.process(®istry).await?; + let (args, _): (EchoArgs, _) = args.process(®istry).await?; - for i in args.rest { - match i.as_string() { - Ok(s) => { - yield Ok(ReturnSuccess::Value( - UntaggedValue::string(s).into_value(i.tag.clone()), - )); - } - _ => match i { - Value { - value: UntaggedValue::Table(table), - .. - } => { - for value in table { - yield Ok(ReturnSuccess::Value(value.clone())); - } - } - Value { - value: UntaggedValue::Primitive(Primitive::Range(range)), - tag - } => { - let mut current = range.from.0.item; - while current != range.to.0.item { - yield Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current.clone()).into_value(&tag))); - current = match crate::data::value::compute_values(Operator::Plus, &UntaggedValue::Primitive(current), &UntaggedValue::int(1)) { - Ok(result) => match result { - UntaggedValue::Primitive(p) => p, - _ => { - yield Err(ShellError::unimplemented("Internal error: expected a primitive result from increment")); - return; - } - }, - Err((left_type, right_type)) => { - yield Err(ShellError::coerce_error( - left_type.spanned(tag.span), - right_type.spanned(tag.span), - )); - return; - } - } - } - match range.to.1 { - RangeInclusion::Inclusive => { - yield Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current.clone()).into_value(&tag))); - } - _ => {} - } - } - _ => { - yield Ok(ReturnSuccess::Value(i.clone())); - } - }, + let stream = args.rest.into_iter().map(|i| { + match i.as_string() { + Ok(s) => { + OutputStream::one(Ok(ReturnSuccess::Value( + UntaggedValue::string(s).into_value(i.tag.clone()), + ))) } - } - }; + _ => match i { + Value { + value: UntaggedValue::Table(table), + .. + } => { + futures::stream::iter(table.into_iter().map(ReturnSuccess::value)).to_output_stream() + } + Value { + value: UntaggedValue::Primitive(Primitive::Range(range)), + tag + } => { + let mut output_vec = vec![]; - Ok(stream.to_output_stream()) + let mut current = range.from.0.item; + while current != range.to.0.item { + output_vec.push(Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current.clone()).into_value(&tag)))); + current = match crate::data::value::compute_values(Operator::Plus, &UntaggedValue::Primitive(current), &UntaggedValue::int(1)) { + Ok(result) => match result { + UntaggedValue::Primitive(p) => p, + _ => { + return OutputStream::one(Err(ShellError::unimplemented("Internal error: expected a primitive result from increment"))); + } + }, + Err((left_type, right_type)) => { + return OutputStream::one(Err(ShellError::coerce_error( + left_type.spanned(tag.span), + right_type.spanned(tag.span), + ))); + } + } + } + if let RangeInclusion::Inclusive = range.to.1 { + output_vec.push(Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current).into_value(&tag)))); + } + + futures::stream::iter(output_vec.into_iter()).to_output_stream() + } + _ => { + OutputStream::one(Ok(ReturnSuccess::Value(i.clone()))) + } + }, + } + }); + + Ok(futures::stream::iter(stream).flatten().to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/enter.rs b/crates/nu-cli/src/commands/enter.rs index deb17102f..4f6074142 100644 --- a/crates/nu-cli/src/commands/enter.rs +++ b/crates/nu-cli/src/commands/enter.rs @@ -14,6 +14,7 @@ pub struct Enter; #[derive(Deserialize)] pub struct EnterArgs { location: Tagged, + encoding: Option>, } #[async_trait] @@ -23,15 +24,29 @@ impl WholeStreamCommand for Enter { } fn signature(&self) -> Signature { - Signature::build("enter").required( - "location", - SyntaxShape::Path, - "the location to create a new shell from", - ) + Signature::build("enter") + .required( + "location", + SyntaxShape::Path, + "the location to create a new shell from", + ) + .named( + "encoding", + SyntaxShape::String, + "encoding to use to open file", + Some('e'), + ) } fn usage(&self) -> &str { - "Create a new shell and begin at this path." + r#"Create a new shell and begin at this path. + +Multiple encodings are supported for reading text files by using +the '--encoding ' parameter. Here is an example of a few: +big5, euc-jp, euc-kr, gbk, iso-8859-1, utf-16, cp1252, latin5 + +For a more complete list of encodings please refer to the encoding_rs +documentation link at https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics"# } async fn run( @@ -39,7 +54,7 @@ impl WholeStreamCommand for Enter { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - enter(args, registry) + enter(args, registry).await } fn examples(&self) -> Vec { @@ -54,121 +69,131 @@ impl WholeStreamCommand for Enter { example: "enter package.json", result: None, }, + Example { + description: "Enters file with iso-8859-1 encoding", + example: "enter file.csv --encoding iso-8859-1", + result: None, + }, ] } } -fn enter(raw_args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn enter( + raw_args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let scope = raw_args.call_info.scope.clone(); - let shell_manager = raw_args.shell_manager.clone(); - let head = raw_args.call_info.args.head.clone(); - let ctrl_c = raw_args.ctrl_c.clone(); - let current_errors = raw_args.current_errors.clone(); - let host = raw_args.host.clone(); - let tag = raw_args.call_info.name_tag.clone(); - let (EnterArgs { location }, _) = raw_args.process(®istry).await?; - let location_string = location.display().to_string(); - let location_clone = location_string.clone(); + let scope = raw_args.call_info.scope.clone(); + let shell_manager = raw_args.shell_manager.clone(); + let head = raw_args.call_info.args.head.clone(); + let ctrl_c = raw_args.ctrl_c.clone(); + let current_errors = raw_args.current_errors.clone(); + let host = raw_args.host.clone(); + let tag = raw_args.call_info.name_tag.clone(); + let (EnterArgs { location, encoding }, _) = raw_args.process(®istry).await?; + let location_string = location.display().to_string(); + let location_clone = location_string.clone(); - if location_string.starts_with("help") { - let spec = location_string.split(':').collect::>(); + if location_string.starts_with("help") { + let spec = location_string.split(':').collect::>(); - if spec.len() == 2 { - let (_, command) = (spec[0], spec[1]); + if spec.len() == 2 { + let (_, command) = (spec[0], spec[1]); - if registry.has(command) { - yield Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell( + if registry.has(command) { + return Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::EnterHelpShell( UntaggedValue::string(command).into_value(Tag::unknown()), - ))); - return; - } - } - yield Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell( - UntaggedValue::nothing().into_value(Tag::unknown()), - ))); - } else if location.is_dir() { - yield Ok(ReturnSuccess::Action(CommandAction::EnterShell( - location_clone, - ))); - } else { - // If it's a file, attempt to open the file as a value and enter it - let cwd = shell_manager.path(); - - let full_path = std::path::PathBuf::from(cwd); - - let (file_extension, contents, contents_tag) = - crate::commands::open::fetch( - &full_path, - &PathBuf::from(location_clone), - tag.span, - ).await?; - - match contents { - UntaggedValue::Primitive(Primitive::String(_)) => { - let tagged_contents = contents.into_value(&contents_tag); - - if let Some(extension) = file_extension { - let command_name = format!("from {}", extension); - if let Some(converter) = - registry.get_command(&command_name) - { - let new_args = RawCommandArgs { - host, - ctrl_c, - current_errors, - shell_manager, - call_info: UnevaluatedCallInfo { - args: nu_protocol::hir::Call { - head, - positional: None, - named: None, - span: Span::unknown(), - is_last: false, - }, - name_tag: tag.clone(), - scope: scope.clone() - }, - }; - let mut result = converter.run( - new_args.with_input(vec![tagged_contents]), - ®istry, - ).await; - let result_vec: Vec> = - result.drain_vec().await; - for res in result_vec { - match res { - Ok(ReturnSuccess::Value(Value { - value, - .. - })) => { - yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell( - Value { - value, - tag: contents_tag.clone(), - }))); - } - x => yield x, - } - } - } else { - yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents))); - } - } else { - yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents))); - } - } - _ => { - let tagged_contents = contents.into_value(contents_tag); - - yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents))); - } + ), + ))); } } - }; + Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::EnterHelpShell(UntaggedValue::nothing().into_value(Tag::unknown())), + ))) + } else if location.is_dir() { + Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::EnterShell(location_clone), + ))) + } else { + // If it's a file, attempt to open the file as a value and enter it + let cwd = shell_manager.path(); - Ok(stream.to_output_stream()) + let full_path = std::path::PathBuf::from(cwd); + + let (file_extension, contents, contents_tag) = crate::commands::open::fetch( + &full_path, + &PathBuf::from(location_clone), + tag.span, + match encoding { + Some(e) => e.to_string(), + _ => "".to_string(), + }, + ) + .await?; + + match contents { + UntaggedValue::Primitive(Primitive::String(_)) => { + let tagged_contents = contents.into_value(&contents_tag); + + if let Some(extension) = file_extension { + let command_name = format!("from {}", extension); + if let Some(converter) = registry.get_command(&command_name) { + let new_args = RawCommandArgs { + host, + ctrl_c, + current_errors, + shell_manager, + call_info: UnevaluatedCallInfo { + args: nu_protocol::hir::Call { + head, + positional: None, + named: None, + span: Span::unknown(), + is_last: false, + }, + name_tag: tag.clone(), + scope: scope.clone(), + }, + }; + let mut result = converter + .run(new_args.with_input(vec![tagged_contents]), ®istry) + .await; + let result_vec: Vec> = + result.drain_vec().await; + + Ok(futures::stream::iter(result_vec.into_iter().map( + move |res| match res { + Ok(ReturnSuccess::Value(Value { value, .. })) => Ok( + ReturnSuccess::Action(CommandAction::EnterValueShell(Value { + value, + tag: contents_tag.clone(), + })), + ), + x => x, + }, + )) + .to_output_stream()) + } else { + Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::EnterValueShell(tagged_contents), + ))) + } + } else { + Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::EnterValueShell(tagged_contents), + ))) + } + } + _ => { + let tagged_contents = contents.into_value(contents_tag); + + Ok(OutputStream::one(ReturnSuccess::action( + CommandAction::EnterValueShell(tagged_contents), + ))) + } + } + } } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/from.rs b/crates/nu-cli/src/commands/from.rs index 121451ade..e5ee31370 100644 --- a/crates/nu-cli/src/commands/from.rs +++ b/crates/nu-cli/src/commands/from.rs @@ -25,14 +25,10 @@ impl WholeStreamCommand for From { registry: &CommandRegistry, ) -> Result { let registry = registry.clone(); - let stream = async_stream! { - yield Ok(ReturnSuccess::Value( - UntaggedValue::string(crate::commands::help::get_help(&From, ®istry)) - .into_value(Tag::unknown()), - )); - }; - - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value( + UntaggedValue::string(crate::commands::help::get_help(&From, ®istry)) + .into_value(Tag::unknown()), + ))) } } diff --git a/crates/nu-cli/src/commands/from_json.rs b/crates/nu-cli/src/commands/from_json.rs index 7f0975c78..b94ca8fed 100644 --- a/crates/nu-cli/src/commands/from_json.rs +++ b/crates/nu-cli/src/commands/from_json.rs @@ -33,7 +33,7 @@ impl WholeStreamCommand for FromJSON { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - from_json(args, registry) + from_json(args, registry).await } } @@ -71,65 +71,73 @@ pub fn from_json_string_to_value(s: String, tag: impl Into) -> serde_hjson: Ok(convert_json_value_to_nu_value(&v, tag)) } -fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn from_json( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let name_tag = args.call_info.name_tag.clone(); let registry = registry.clone(); - let stream = async_stream! { - let (FromJSONArgs { objects }, mut input) = args.process(®istry).await?; - let concat_string = input.collect_string(name_tag.clone()).await?; + let (FromJSONArgs { objects }, input) = args.process(®istry).await?; + let concat_string = input.collect_string(name_tag.clone()).await?; - if objects { - for json_str in concat_string.item.lines() { + let string_clone: Vec<_> = concat_string.item.lines().map(|x| x.to_string()).collect(); + + if objects { + Ok( + futures::stream::iter(string_clone.into_iter().filter_map(move |json_str| { if json_str.is_empty() { - continue; + return None; } - match from_json_string_to_value(json_str.to_string(), &name_tag) { - Ok(x) => - yield ReturnSuccess::value(x), + match from_json_string_to_value(json_str, &name_tag) { + Ok(x) => Some(ReturnSuccess::value(x)), Err(e) => { let mut message = "Could not parse as JSON (".to_string(); message.push_str(&e.to_string()); message.push_str(")"); - yield Err(ShellError::labeled_error_with_secondary( + Some(Err(ShellError::labeled_error_with_secondary( message, "input cannot be parsed as JSON", - &name_tag, + name_tag.clone(), "value originates from here", - concat_string.tag.clone())) + concat_string.tag.clone(), + ))) } } - } - } else { - match from_json_string_to_value(concat_string.item, name_tag.clone()) { - Ok(x) => - match x { - Value { value: UntaggedValue::Table(list), .. } => { - for l in list { - yield ReturnSuccess::value(l); - } - } - x => yield ReturnSuccess::value(x), - } - Err(e) => { - let mut message = "Could not parse as JSON (".to_string(); - message.push_str(&e.to_string()); - message.push_str(")"); + })) + .to_output_stream(), + ) + } else { + match from_json_string_to_value(concat_string.item, name_tag.clone()) { + Ok(x) => match x { + Value { + value: UntaggedValue::Table(list), + .. + } => Ok( + futures::stream::iter(list.into_iter().map(ReturnSuccess::value)) + .to_output_stream(), + ), + x => Ok(OutputStream::one(ReturnSuccess::value(x))), + }, + Err(e) => { + let mut message = "Could not parse as JSON (".to_string(); + message.push_str(&e.to_string()); + message.push_str(")"); - yield Err(ShellError::labeled_error_with_secondary( + Ok(OutputStream::one(Err( + ShellError::labeled_error_with_secondary( message, "input cannot be parsed as JSON", name_tag, "value originates from here", - concat_string.tag)) - } + concat_string.tag, + ), + ))) } } - }; - - Ok(stream.to_output_stream()) + } } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/from_ods.rs b/crates/nu-cli/src/commands/from_ods.rs index 81f5cfc17..94c42f010 100644 --- a/crates/nu-cli/src/commands/from_ods.rs +++ b/crates/nu-cli/src/commands/from_ods.rs @@ -36,62 +36,66 @@ impl WholeStreamCommand for FromODS { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - from_ods(args, registry) + from_ods(args, registry).await } } -fn from_ods(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn from_ods( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let tag = args.call_info.name_tag.clone(); let registry = registry.clone(); - let stream = async_stream! { - let (FromODSArgs { headerless: _headerless }, mut input) = args.process(®istry).await?; - let bytes = input.collect_binary(tag.clone()).await?; - let mut buf: Cursor> = Cursor::new(bytes.item); - let mut ods = Ods::<_>::new(buf).map_err(|_| ShellError::labeled_error( - "Could not load ods file", - "could not load ods file", - &tag))?; + let ( + FromODSArgs { + headerless: _headerless, + }, + input, + ) = args.process(®istry).await?; + let bytes = input.collect_binary(tag.clone()).await?; + let buf: Cursor> = Cursor::new(bytes.item); + let mut ods = Ods::<_>::new(buf).map_err(|_| { + ShellError::labeled_error("Could not load ods file", "could not load ods file", &tag) + })?; - let mut dict = TaggedDictBuilder::new(&tag); + let mut dict = TaggedDictBuilder::new(&tag); - let sheet_names = ods.sheet_names().to_owned(); + let sheet_names = ods.sheet_names().to_owned(); - for sheet_name in &sheet_names { - let mut sheet_output = TaggedListBuilder::new(&tag); + for sheet_name in &sheet_names { + let mut sheet_output = TaggedListBuilder::new(&tag); - if let Some(Ok(current_sheet)) = ods.worksheet_range(sheet_name) { - for row in current_sheet.rows() { - let mut row_output = TaggedDictBuilder::new(&tag); - for (i, cell) in row.iter().enumerate() { - let value = match cell { - DataType::Empty => UntaggedValue::nothing(), - DataType::String(s) => UntaggedValue::string(s), - DataType::Float(f) => UntaggedValue::decimal(*f), - DataType::Int(i) => UntaggedValue::int(*i), - DataType::Bool(b) => UntaggedValue::boolean(*b), - _ => UntaggedValue::nothing(), - }; + if let Some(Ok(current_sheet)) = ods.worksheet_range(sheet_name) { + for row in current_sheet.rows() { + let mut row_output = TaggedDictBuilder::new(&tag); + for (i, cell) in row.iter().enumerate() { + let value = match cell { + DataType::Empty => UntaggedValue::nothing(), + DataType::String(s) => UntaggedValue::string(s), + DataType::Float(f) => UntaggedValue::decimal(*f), + DataType::Int(i) => UntaggedValue::int(*i), + DataType::Bool(b) => UntaggedValue::boolean(*b), + _ => UntaggedValue::nothing(), + }; - row_output.insert_untagged(&format!("Column{}", i), value); - } - - sheet_output.push_untagged(row_output.into_untagged_value()); + row_output.insert_untagged(&format!("Column{}", i), value); } - dict.insert_untagged(sheet_name, sheet_output.into_untagged_value()); - } else { - yield Err(ShellError::labeled_error( - "Could not load sheet", - "could not load sheet", - &tag)); + sheet_output.push_untagged(row_output.into_untagged_value()); } + + dict.insert_untagged(sheet_name, sheet_output.into_untagged_value()); + } else { + return Err(ShellError::labeled_error( + "Could not load sheet", + "could not load sheet", + &tag, + )); } + } - yield ReturnSuccess::value(dict.into_value()); - }; - - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value(dict.into_value()))) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/from_ssv.rs b/crates/nu-cli/src/commands/from_ssv.rs index f30a8c78f..c0b3a6151 100644 --- a/crates/nu-cli/src/commands/from_ssv.rs +++ b/crates/nu-cli/src/commands/from_ssv.rs @@ -51,7 +51,7 @@ impl WholeStreamCommand for FromSSV { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - from_ssv(args, registry) + from_ssv(args, registry).await } } @@ -251,37 +251,53 @@ fn from_ssv_string_to_value( Some(UntaggedValue::Table(rows).into_value(&tag)) } -fn from_ssv(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn from_ssv( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let name = args.call_info.name_tag.clone(); let registry = registry.clone(); - let stream = async_stream! { - let (FromSSVArgs { headerless, aligned_columns, minimum_spaces }, mut input) = args.process(®istry).await?; - let concat_string = input.collect_string(name.clone()).await?; - let split_at = match minimum_spaces { - Some(number) => number.item, - None => DEFAULT_MINIMUM_SPACES - }; + let ( + FromSSVArgs { + headerless, + aligned_columns, + minimum_spaces, + }, + input, + ) = args.process(®istry).await?; + let concat_string = input.collect_string(name.clone()).await?; + let split_at = match minimum_spaces { + Some(number) => number.item, + None => DEFAULT_MINIMUM_SPACES, + }; - match from_ssv_string_to_value(&concat_string.item, headerless, aligned_columns, split_at, name.clone()) { + Ok( + match from_ssv_string_to_value( + &concat_string.item, + headerless, + aligned_columns, + split_at, + name.clone(), + ) { Some(x) => match x { - Value { value: UntaggedValue::Table(list), ..} => { - for l in list { yield ReturnSuccess::value(l) } - } - x => yield ReturnSuccess::value(x) + Value { + value: UntaggedValue::Table(list), + .. + } => futures::stream::iter(list.into_iter().map(ReturnSuccess::value)) + .to_output_stream(), + x => OutputStream::one(ReturnSuccess::value(x)), }, None => { - yield Err(ShellError::labeled_error_with_secondary( + return Err(ShellError::labeled_error_with_secondary( "Could not parse as SSV", "input cannot be parsed ssv", &name, "value originates from here", &concat_string.tag, - )) - }, - } - }; - - Ok(stream.to_output_stream()) + )); + } + }, + ) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/from_toml.rs b/crates/nu-cli/src/commands/from_toml.rs index a0c58d61c..2dffe9e81 100644 --- a/crates/nu-cli/src/commands/from_toml.rs +++ b/crates/nu-cli/src/commands/from_toml.rs @@ -24,7 +24,7 @@ impl WholeStreamCommand for FromTOML { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - from_toml(args, registry) + from_toml(args, registry).await } } @@ -64,28 +64,28 @@ pub fn from_toml_string_to_value(s: String, tag: impl Into) -> Result Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let tag = args.name_tag(); - let input = args.input; + let args = args.evaluate_once(®istry).await?; + let tag = args.name_tag(); + let input = args.input; - let concat_string = input.collect_string(tag.clone()).await?; + let concat_string = input.collect_string(tag.clone()).await?; + Ok( match from_toml_string_to_value(concat_string.item, tag.clone()) { Ok(x) => match x { - Value { value: UntaggedValue::Table(list), .. } => { - for l in list { - yield ReturnSuccess::value(l); - } - } - x => yield ReturnSuccess::value(x), + Value { + value: UntaggedValue::Table(list), + .. + } => futures::stream::iter(list.into_iter().map(ReturnSuccess::value)) + .to_output_stream(), + x => OutputStream::one(ReturnSuccess::value(x)), }, Err(_) => { - yield Err(ShellError::labeled_error_with_secondary( + return Err(ShellError::labeled_error_with_secondary( "Could not parse as TOML", "input cannot be parsed as TOML", &tag, @@ -93,10 +93,8 @@ pub fn from_toml( concat_string.tag, )) } - } - }; - - Ok(stream.to_output_stream()) + }, + ) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/from_xlsx.rs b/crates/nu-cli/src/commands/from_xlsx.rs index 3bc6691ed..c518837ef 100644 --- a/crates/nu-cli/src/commands/from_xlsx.rs +++ b/crates/nu-cli/src/commands/from_xlsx.rs @@ -36,62 +36,66 @@ impl WholeStreamCommand for FromXLSX { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - from_xlsx(args, registry) + from_xlsx(args, registry).await } } -fn from_xlsx(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn from_xlsx( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let tag = args.call_info.name_tag.clone(); let registry = registry.clone(); - let stream = async_stream! { - let (FromXLSXArgs { headerless: _headerless }, mut input) = args.process(®istry).await?; - let value = input.collect_binary(tag.clone()).await?; + let ( + FromXLSXArgs { + headerless: _headerless, + }, + input, + ) = args.process(®istry).await?; + let value = input.collect_binary(tag.clone()).await?; - let mut buf: Cursor> = Cursor::new(value.item); - let mut xls = Xlsx::<_>::new(buf).map_err(|_| { - ShellError::labeled_error("Could not load xlsx file", "could not load xlsx file", &tag) - })?; + let buf: Cursor> = Cursor::new(value.item); + let mut xls = Xlsx::<_>::new(buf).map_err(|_| { + ShellError::labeled_error("Could not load xlsx file", "could not load xlsx file", &tag) + })?; - let mut dict = TaggedDictBuilder::new(&tag); + let mut dict = TaggedDictBuilder::new(&tag); - let sheet_names = xls.sheet_names().to_owned(); + let sheet_names = xls.sheet_names().to_owned(); - for sheet_name in &sheet_names { - let mut sheet_output = TaggedListBuilder::new(&tag); + for sheet_name in &sheet_names { + let mut sheet_output = TaggedListBuilder::new(&tag); - if let Some(Ok(current_sheet)) = xls.worksheet_range(sheet_name) { - for row in current_sheet.rows() { - let mut row_output = TaggedDictBuilder::new(&tag); - for (i, cell) in row.iter().enumerate() { - let value = match cell { - DataType::Empty => UntaggedValue::nothing(), - DataType::String(s) => UntaggedValue::string(s), - DataType::Float(f) => UntaggedValue::decimal(*f), - DataType::Int(i) => UntaggedValue::int(*i), - DataType::Bool(b) => UntaggedValue::boolean(*b), - _ => UntaggedValue::nothing(), - }; + if let Some(Ok(current_sheet)) = xls.worksheet_range(sheet_name) { + for row in current_sheet.rows() { + let mut row_output = TaggedDictBuilder::new(&tag); + for (i, cell) in row.iter().enumerate() { + let value = match cell { + DataType::Empty => UntaggedValue::nothing(), + DataType::String(s) => UntaggedValue::string(s), + DataType::Float(f) => UntaggedValue::decimal(*f), + DataType::Int(i) => UntaggedValue::int(*i), + DataType::Bool(b) => UntaggedValue::boolean(*b), + _ => UntaggedValue::nothing(), + }; - row_output.insert_untagged(&format!("Column{}", i), value); - } - - sheet_output.push_untagged(row_output.into_untagged_value()); + row_output.insert_untagged(&format!("Column{}", i), value); } - dict.insert_untagged(sheet_name, sheet_output.into_untagged_value()); - } else { - yield Err(ShellError::labeled_error( - "Could not load sheet", - "could not load sheet", - &tag, - )); + sheet_output.push_untagged(row_output.into_untagged_value()); } + + dict.insert_untagged(sheet_name, sheet_output.into_untagged_value()); + } else { + return Err(ShellError::labeled_error( + "Could not load sheet", + "could not load sheet", + &tag, + )); } + } - yield ReturnSuccess::value(dict.into_value()); - }; - - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value(dict.into_value()))) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/from_xml.rs b/crates/nu-cli/src/commands/from_xml.rs index 14c5cd514..6e13b7190 100644 --- a/crates/nu-cli/src/commands/from_xml.rs +++ b/crates/nu-cli/src/commands/from_xml.rs @@ -24,7 +24,7 @@ impl WholeStreamCommand for FromXML { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - from_xml(args, registry) + from_xml(args, registry).await } } @@ -99,37 +99,38 @@ pub fn from_xml_string_to_value(s: String, tag: impl Into) -> Result Result { +async fn from_xml( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let tag = args.name_tag(); - let input = args.input; + let args = args.evaluate_once(®istry).await?; + let tag = args.name_tag(); + let input = args.input; - let concat_string = input.collect_string(tag.clone()).await?; + let concat_string = input.collect_string(tag.clone()).await?; + Ok( match from_xml_string_to_value(concat_string.item, tag.clone()) { Ok(x) => match x { - Value { value: UntaggedValue::Table(list), .. } => { - for l in list { - yield ReturnSuccess::value(l); - } - } - x => yield ReturnSuccess::value(x), + Value { + value: UntaggedValue::Table(list), + .. + } => futures::stream::iter(list.into_iter().map(ReturnSuccess::value)) + .to_output_stream(), + x => OutputStream::one(ReturnSuccess::value(x)), }, Err(_) => { - yield Err(ShellError::labeled_error_with_secondary( + return Err(ShellError::labeled_error_with_secondary( "Could not parse as XML", "input cannot be parsed as XML", &tag, "value originates from here", &concat_string.tag, )) - } , - } - }; - - Ok(stream.to_output_stream()) + } + }, + ) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/headers.rs b/crates/nu-cli/src/commands/headers.rs index 5e734715b..4baf72842 100644 --- a/crates/nu-cli/src/commands/headers.rs +++ b/crates/nu-cli/src/commands/headers.rs @@ -28,7 +28,7 @@ impl WholeStreamCommand for Headers { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - headers(args, registry) + headers(args, registry).await } fn examples(&self) -> Vec { @@ -40,51 +40,65 @@ impl WholeStreamCommand for Headers { } } -pub fn headers(args: CommandArgs, _registry: &CommandRegistry) -> Result { - let stream = async_stream! { - let mut input = args.input; - let rows: Vec = input.collect().await; +pub async fn headers( + args: CommandArgs, + _registry: &CommandRegistry, +) -> Result { + let input = args.input; + let rows: Vec = input.collect().await; - if rows.len() < 1 { - yield Err(ShellError::untagged_runtime_error("Couldn't find headers, was the input a properly formatted, non-empty table?")); - } + if rows.is_empty() { + return Err(ShellError::untagged_runtime_error( + "Couldn't find headers, was the input a properly formatted, non-empty table?", + )); + } - //the headers are the first row in the table - let headers: Vec = match &rows[0].value { - UntaggedValue::Row(d) => { - Ok(d.entries.iter().map(|(k, v)| { + //the headers are the first row in the table + let headers: Vec = match &rows[0].value { + UntaggedValue::Row(d) => { + Ok(d.entries + .iter() + .map(|(k, v)| { match v.as_string() { Ok(s) => s, - Err(_) => { //If a cell that should contain a header name is empty, we name the column Column[index] + Err(_) => { + //If a cell that should contain a header name is empty, we name the column Column[index] match d.entries.get_full(k) { Some((index, _, _)) => format!("Column{}", index), - None => "unknownColumn".to_string() + None => "unknownColumn".to_string(), } } } - }).collect()) - } - _ => Err(ShellError::unexpected_eof("Could not get headers, is the table empty?", rows[0].tag.span)) - }?; + }) + .collect()) + } + _ => Err(ShellError::unexpected_eof( + "Could not get headers, is the table empty?", + rows[0].tag.span, + )), + }?; - //Each row is a dictionary with the headers as keys - for r in rows.iter().skip(1) { + Ok( + futures::stream::iter(rows.into_iter().skip(1).map(move |r| { + //Each row is a dictionary with the headers as keys match &r.value { UntaggedValue::Row(d) => { - let mut i = 0; let mut entries = IndexMap::new(); - for (_, v) in d.entries.iter() { + for (i, (_, v)) in d.entries.iter().enumerate() { entries.insert(headers[i].clone(), v.clone()); - i += 1; } - yield Ok(ReturnSuccess::Value(UntaggedValue::Row(Dictionary{entries}).into_value(r.tag.clone()))) + Ok(ReturnSuccess::Value( + UntaggedValue::Row(Dictionary { entries }).into_value(r.tag.clone()), + )) } - _ => yield Err(ShellError::unexpected_eof("Couldn't iterate through rows, was the input a properly formatted table?", r.tag.span)) + _ => Err(ShellError::unexpected_eof( + "Couldn't iterate through rows, was the input a properly formatted table?", + r.tag.span, + )), } - } - }; - - Ok(stream.to_output_stream()) + })) + .to_output_stream(), + ) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/histogram.rs b/crates/nu-cli/src/commands/histogram.rs index 003b1faf5..8a8c3b70a 100644 --- a/crates/nu-cli/src/commands/histogram.rs +++ b/crates/nu-cli/src/commands/histogram.rs @@ -45,7 +45,7 @@ impl WholeStreamCommand for Histogram { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - histogram(args, registry) + histogram(args, registry).await } fn examples(&self) -> Vec { @@ -70,95 +70,136 @@ impl WholeStreamCommand for Histogram { } } -pub fn histogram( +pub async fn histogram( args: CommandArgs, registry: &CommandRegistry, ) -> Result { let registry = registry.clone(); let name = args.call_info.name_tag.clone(); - let stream = async_stream! { - let (HistogramArgs { column_name, rest}, mut input) = args.process(®istry).await?; - let values: Vec = input.collect().await; + let (HistogramArgs { column_name, rest }, input) = args.process(®istry).await?; + let values: Vec = input.collect().await; - let Tagged { item: group_by, .. } = column_name.clone(); + let Tagged { item: group_by, .. } = column_name.clone(); - let groups = group(&column_name, values, &name)?; - let group_labels = columns_sorted(Some(group_by.clone()), &groups, &name); - let sorted = t_sort(Some(group_by.clone()), None, &groups, &name)?; - let evaled = evaluate(&sorted, None, &name)?; - let reduced = reduce(&evaled, None, &name)?; - let maxima = map_max(&reduced, None, &name)?; - let percents = percentages(&reduced, maxima, &name)?; + let groups = group(&column_name, values, &name)?; + let group_labels = columns_sorted(Some(group_by.clone()), &groups, &name); + let sorted = t_sort(Some(group_by), None, &groups, &name)?; + let evaled = evaluate(&sorted, None, &name)?; + let reduced = reduce(&evaled, None, &name)?; + let maxima = map_max(&reduced, None, &name)?; + let percents = percentages(&reduced, maxima, &name)?; - match percents { - Value { - value: UntaggedValue::Table(datasets), - .. - } => { + match percents { + Value { + value: UntaggedValue::Table(datasets), + .. + } => { + let mut idx = 0; - let mut idx = 0; + let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect(); - let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect(); + let frequency_column_name = if column_names_supplied.is_empty() { + "frequency".to_string() + } else { + column_names_supplied[0].clone() + }; - let frequency_column_name = if column_names_supplied.is_empty() { - "frequency".to_string() - } else { - column_names_supplied[0].clone() - }; + let column = (*column_name).clone(); - let column = (*column_name).clone(); + let count_column_name = "count".to_string(); + let count_shell_error = ShellError::labeled_error( + "Unable to load group count", + "unabled to load group count", + &name, + ); + let mut count_values: Vec = Vec::new(); - let count_column_name = "count".to_string(); - let count_shell_error = ShellError::labeled_error("Unable to load group count", "unabled to load group count", &name); - let mut count_values: Vec = Vec::new(); - - for table_entry in reduced.table_entries() { - match table_entry { - Value { - value: UntaggedValue::Table(list), - .. - } => { - for i in list { - if let Ok(count) = i.value.clone().into_value(&name).as_u64() { - count_values.push(count); - } else { - yield Err(count_shell_error); - return; - } + for table_entry in reduced.table_entries() { + match table_entry { + Value { + value: UntaggedValue::Table(list), + .. + } => { + for i in list { + if let Ok(count) = i.value.clone().into_value(&name).as_u64() { + count_values.push(count); + } else { + return Err(count_shell_error); } } - _ => { - yield Err(count_shell_error); - return; - } + } + _ => { + return Err(count_shell_error); } } + } - if let Value { value: UntaggedValue::Table(start), .. } = datasets.get(0).ok_or_else(|| ShellError::labeled_error("Unable to load dataset", "unabled to load dataset", &name))? { - for percentage in start.iter() { - + if let Value { + value: UntaggedValue::Table(start), + .. + } = datasets.get(0).ok_or_else(|| { + ShellError::labeled_error( + "Unable to load dataset", + "unabled to load dataset", + &name, + ) + })? { + let start = start.clone(); + Ok( + futures::stream::iter(start.into_iter().map(move |percentage| { let mut fact = TaggedDictBuilder::new(&name); - let value: Tagged = group_labels.get(idx).ok_or_else(|| ShellError::labeled_error("Unable to load group labels", "unabled to load group labels", &name))?.clone(); - fact.insert_value(&column, UntaggedValue::string(value.item).into_value(value.tag)); + let value: Tagged = group_labels + .get(idx) + .ok_or_else(|| { + ShellError::labeled_error( + "Unable to load group labels", + "unabled to load group labels", + &name, + ) + })? + .clone(); + fact.insert_value( + &column, + UntaggedValue::string(value.item).into_value(value.tag), + ); - fact.insert_untagged(&count_column_name, UntaggedValue::int(count_values[idx])); + fact.insert_untagged( + &count_column_name, + UntaggedValue::int(count_values[idx]), + ); - if let Value { value: UntaggedValue::Primitive(Primitive::Int(ref num)), ref tag } = percentage.clone() { - let string = std::iter::repeat("*").take(num.to_i32().ok_or_else(|| ShellError::labeled_error("Expected a number", "expected a number", tag))? as usize).collect::(); - fact.insert_untagged(&frequency_column_name, UntaggedValue::string(string)); + if let Value { + value: UntaggedValue::Primitive(Primitive::Int(ref num)), + ref tag, + } = percentage + { + let string = std::iter::repeat("*") + .take(num.to_i32().ok_or_else(|| { + ShellError::labeled_error( + "Expected a number", + "expected a number", + tag, + ) + })? as usize) + .collect::(); + fact.insert_untagged( + &frequency_column_name, + UntaggedValue::string(string), + ); } idx += 1; - yield ReturnSuccess::value(fact.into_value()); - } - } + ReturnSuccess::value(fact.into_value()) + })) + .to_output_stream(), + ) + } else { + Ok(OutputStream::empty()) } - _ => {} } - }; - - Ok(stream.to_output_stream()) + _ => Ok(OutputStream::empty()), + } } fn percentages(values: &Value, max: Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/history.rs b/crates/nu-cli/src/commands/history.rs index d3f3ed118..213583fb3 100644 --- a/crates/nu-cli/src/commands/history.rs +++ b/crates/nu-cli/src/commands/history.rs @@ -33,21 +33,25 @@ impl WholeStreamCommand for History { fn history(args: CommandArgs, _registry: &CommandRegistry) -> Result { let tag = args.call_info.name_tag; - let stream = async_stream! { - let history_path = HistoryFile::path(); - let file = File::open(history_path); - if let Ok(file) = file { - let reader = BufReader::new(file); - for line in reader.lines() { - if let Ok(line) = line { - yield ReturnSuccess::value(UntaggedValue::string(line).into_value(tag.clone())); - } - } - } else { - yield Err(ShellError::labeled_error("Could not open history", "history file could not be opened", tag.clone())); - } - }; - Ok(stream.to_output_stream()) + let history_path = HistoryFile::path(); + let file = File::open(history_path); + if let Ok(file) = file { + let reader = BufReader::new(file); + let output = reader.lines().filter_map(move |line| match line { + Ok(line) => Some(ReturnSuccess::value( + UntaggedValue::string(line).into_value(tag.clone()), + )), + Err(_) => None, + }); + + Ok(futures::stream::iter(output).to_output_stream()) + } else { + Err(ShellError::labeled_error( + "Could not open history", + "history file could not be opened", + tag, + )) + } } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/insert.rs b/crates/nu-cli/src/commands/insert.rs index 44a7f2431..51f895b4e 100644 --- a/crates/nu-cli/src/commands/insert.rs +++ b/crates/nu-cli/src/commands/insert.rs @@ -42,38 +42,32 @@ impl WholeStreamCommand for Insert { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - insert(args, registry) + insert(args, registry).await } } -fn insert(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn insert(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (InsertArgs { column, value }, mut input) = args.process(®istry).await?; - while let Some(row) = input.next().await { - match row { - Value { - value: UntaggedValue::Row(_), - .. - } => match row.insert_data_at_column_path(&column, value.clone()) { - Ok(v) => yield Ok(ReturnSuccess::Value(v)), - Err(err) => yield Err(err), - }, + let (InsertArgs { column, value }, input) = args.process(®istry).await?; - Value { tag, ..} => { - yield Err(ShellError::labeled_error( - "Unrecognized type in stream", - "original value", - tag, - )); - } + Ok(input + .map(move |row| match row { + Value { + value: UntaggedValue::Row(_), + .. + } => match row.insert_data_at_column_path(&column, value.clone()) { + Ok(v) => Ok(ReturnSuccess::Value(v)), + Err(err) => Err(err), + }, - } - }; - - }; - Ok(stream.to_output_stream()) + Value { tag, .. } => Err(ShellError::labeled_error( + "Unrecognized type in stream", + "original value", + tag, + )), + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/keep.rs b/crates/nu-cli/src/commands/keep.rs index bac83b5d9..5ba94cd60 100644 --- a/crates/nu-cli/src/commands/keep.rs +++ b/crates/nu-cli/src/commands/keep.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue}; +use nu_protocol::{Signature, SyntaxShape, UntaggedValue}; use nu_source::Tagged; pub struct Keep; @@ -35,7 +35,7 @@ impl WholeStreamCommand for Keep { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - keep(args, registry) + keep(args, registry).await } fn examples(&self) -> Vec { @@ -59,27 +59,16 @@ impl WholeStreamCommand for Keep { } } -fn keep(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn keep(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (KeepArgs { rows }, mut input) = args.process(®istry).await?; - let mut rows_desired = if let Some(quantity) = rows { - *quantity - } else { - 1 - }; - - while let Some(input) = input.next().await { - if rows_desired > 0 { - yield ReturnSuccess::value(input); - rows_desired -= 1; - } else { - break; - } - } + let (KeepArgs { rows }, input) = args.process(®istry).await?; + let rows_desired = if let Some(quantity) = rows { + *quantity + } else { + 1 }; - Ok(stream.to_output_stream()) + Ok(input.take(rows_desired).to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/kill.rs b/crates/nu-cli/src/commands/kill.rs index 4c39583df..8ea29665f 100644 --- a/crates/nu-cli/src/commands/kill.rs +++ b/crates/nu-cli/src/commands/kill.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue}; +use nu_protocol::{Signature, SyntaxShape}; use nu_source::Tagged; use std::process::{Command, Stdio}; @@ -43,7 +43,7 @@ impl WholeStreamCommand for Kill { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - kill(args, registry) + kill(args, registry).await } fn examples(&self) -> Vec { @@ -62,63 +62,60 @@ impl WholeStreamCommand for Kill { } } -fn kill(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn kill(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (KillArgs { + let ( + KillArgs { pid, rest, force, quiet, - }, mut input) = args.process(®istry).await?; - let mut cmd = if cfg!(windows) { - let mut cmd = Command::new("taskkill"); + }, + .., + ) = args.process(®istry).await?; + let mut cmd = if cfg!(windows) { + let mut cmd = Command::new("taskkill"); - if *force { - cmd.arg("/F"); - } + if *force { + cmd.arg("/F"); + } + cmd.arg("/PID"); + cmd.arg(pid.item().to_string()); + + // each pid must written as `/PID 0` otherwise + // taskkill will act as `killall` unix command + for id in &rest { cmd.arg("/PID"); - cmd.arg(pid.item().to_string()); - - // each pid must written as `/PID 0` otherwise - // taskkill will act as `killall` unix command - for id in &rest { - cmd.arg("/PID"); - cmd.arg(id.item().to_string()); - } - - cmd - } else { - let mut cmd = Command::new("kill"); - - if *force { - cmd.arg("-9"); - } - - cmd.arg(pid.item().to_string()); - - cmd.args(rest.iter().map(move |id| id.item().to_string())); - - cmd - }; - - // pipe everything to null - if *quiet { - cmd.stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); + cmd.arg(id.item().to_string()); } - cmd.status().expect("failed to execute shell command"); + cmd + } else { + let mut cmd = Command::new("kill"); - if false { - yield ReturnSuccess::value(UntaggedValue::nothing().into_value(Tag::unknown())); + if *force { + cmd.arg("-9"); } + + cmd.arg(pid.item().to_string()); + + cmd.args(rest.iter().map(move |id| id.item().to_string())); + + cmd }; - Ok(stream.to_output_stream()) + // pipe everything to null + if *quiet { + cmd.stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + } + + cmd.status().expect("failed to execute shell command"); + + Ok(OutputStream::empty()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/merge.rs b/crates/nu-cli/src/commands/merge.rs index 2948782d3..6cf6178d7 100644 --- a/crates/nu-cli/src/commands/merge.rs +++ b/crates/nu-cli/src/commands/merge.rs @@ -37,7 +37,7 @@ impl WholeStreamCommand for Merge { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - merge(args, registry) + merge(args, registry).await } fn examples(&self) -> Vec { @@ -49,57 +49,61 @@ impl WholeStreamCommand for Merge { } } -fn merge(raw_args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn merge( + raw_args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); let scope = raw_args.call_info.scope.clone(); - let stream = async_stream! { - let mut context = Context::from_raw(&raw_args, ®istry); - let name_tag = raw_args.call_info.name_tag.clone(); - let (merge_args, mut input): (MergeArgs, _) = raw_args.process(®istry).await?; - let block = merge_args.block; + let mut context = Context::from_raw(&raw_args, ®istry); + let name_tag = raw_args.call_info.name_tag.clone(); + let (merge_args, input): (MergeArgs, _) = raw_args.process(®istry).await?; + let block = merge_args.block; - let table: Option> = match run_block(&block, - &mut context, - InputStream::empty(), - &scope.it, - &scope.vars, - &scope.env).await { - Ok(mut stream) => Some(stream.drain_vec().await), - Err(err) => { - yield Err(err); - return; - } - }; - - - let table = table.unwrap_or_else(|| vec![Value { - value: UntaggedValue::row(IndexMap::default()), - tag: name_tag, - }]); - - let mut idx = 0; - - while let Some(value) = input.next().await { - let other = table.get(idx); - - match other { - Some(replacement) => { - match merge_values(&value.value, &replacement.value) { - Ok(merged_value) => yield ReturnSuccess::value(merged_value.into_value(&value.tag)), - Err(err) => { - let message = format!("The row at {:?} types mismatch", idx); - yield Err(ShellError::labeled_error("Could not merge", &message, &value.tag)); - } - } - } - None => yield ReturnSuccess::value(value), - } - - idx += 1; + let table: Option> = match run_block( + &block, + &mut context, + InputStream::empty(), + &scope.it, + &scope.vars, + &scope.env, + ) + .await + { + Ok(mut stream) => Some(stream.drain_vec().await), + Err(err) => { + return Err(err); } }; - Ok(stream.to_output_stream()) + let table = table.unwrap_or_else(|| { + vec![Value { + value: UntaggedValue::row(IndexMap::default()), + tag: name_tag, + }] + }); + + Ok(input + .enumerate() + .map(move |(idx, value)| { + let other = table.get(idx); + + match other { + Some(replacement) => match merge_values(&value.value, &replacement.value) { + Ok(merged_value) => ReturnSuccess::value(merged_value.into_value(&value.tag)), + Err(_) => { + let message = format!("The row at {:?} types mismatch", idx); + Err(ShellError::labeled_error( + "Could not merge", + &message, + &value.tag, + )) + } + }, + None => ReturnSuccess::value(value), + } + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/mv.rs b/crates/nu-cli/src/commands/mv.rs index 7386bc82b..03b3dbe70 100644 --- a/crates/nu-cli/src/commands/mv.rs +++ b/crates/nu-cli/src/commands/mv.rs @@ -43,7 +43,7 @@ impl WholeStreamCommand for Move { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - mv(args, registry) + mv(args, registry).await } fn examples(&self) -> Vec { @@ -67,20 +67,13 @@ impl WholeStreamCommand for Move { } } -fn mv(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn mv(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name = args.call_info.name_tag.clone(); - let shell_manager = args.shell_manager.clone(); - let (args, _) = args.process(®istry).await?; - let mut result = shell_manager.mv(args, name)?; + let name = args.call_info.name_tag.clone(); + let shell_manager = args.shell_manager.clone(); + let (args, _) = args.process(®istry).await?; - while let Some(item) = result.next().await { - yield item; - } - }; - - Ok(stream.to_output_stream()) + shell_manager.mv(args, name) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/nth.rs b/crates/nu-cli/src/commands/nth.rs index 61c32a949..9e10531e1 100644 --- a/crates/nu-cli/src/commands/nth.rs +++ b/crates/nu-cli/src/commands/nth.rs @@ -38,7 +38,7 @@ impl WholeStreamCommand for Nth { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - nth(args, registry) + nth(args, registry).await } fn examples(&self) -> Vec { @@ -57,30 +57,36 @@ impl WholeStreamCommand for Nth { } } -fn nth(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn nth(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (NthArgs { row_number, rest: and_rows}, input) = args.process(®istry).await?; + let ( + NthArgs { + row_number, + rest: and_rows, + }, + input, + ) = args.process(®istry).await?; - let mut inp = input.enumerate(); - while let Some((idx, item)) = inp.next().await { - let row_number = vec![row_number.clone()]; + let row_numbers = vec![vec![row_number], and_rows] + .into_iter() + .flatten() + .collect::>>(); - let row_numbers = vec![&row_number, &and_rows] - .into_iter() - .flatten() - .collect::>>(); - - if row_numbers - .iter() - .any(|requested| requested.item == idx as u64) - { - yield ReturnSuccess::value(item); - } - } - }; - - Ok(stream.to_output_stream()) + Ok(input + .enumerate() + .filter_map(move |(idx, item)| { + futures::future::ready( + if row_numbers + .iter() + .any(|requested| requested.item == idx as u64) + { + Some(ReturnSuccess::value(item)) + } else { + None + }, + ) + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/open.rs b/crates/nu-cli/src/commands/open.rs index ce7e52f2a..720ffa70d 100644 --- a/crates/nu-cli/src/commands/open.rs +++ b/crates/nu-cli/src/commands/open.rs @@ -4,6 +4,12 @@ use nu_errors::ShellError; use nu_protocol::{CommandAction, ReturnSuccess, Signature, SyntaxShape, UntaggedValue}; use nu_source::{AnchorLocation, Span, Tagged}; use std::path::{Path, PathBuf}; +extern crate encoding_rs; +use encoding_rs::*; +use std::fs::File; +use std::io::BufWriter; +use std::io::Read; +use std::io::Write; pub struct Open; @@ -11,6 +17,7 @@ pub struct Open; pub struct OpenArgs { path: Tagged, raw: Tagged, + encoding: Option>, } #[async_trait] @@ -31,10 +38,23 @@ impl WholeStreamCommand for Open { "load content as a string instead of a table", Some('r'), ) + .named( + "encoding", + SyntaxShape::String, + "encoding to use to open file", + Some('e'), + ) } fn usage(&self) -> &str { - "Load a file into a cell, convert to table if possible (avoid by appending '--raw')" + r#"Load a file into a cell, convert to table if possible (avoid by appending '--raw'). + +Multiple encodings are supported for reading text files by using +the '--encoding ' parameter. Here is an example of a few: +big5, euc-jp, euc-kr, gbk, iso-8859-1, utf-16, cp1252, latin5 + +For a more complete list of encodings please refer to the encoding_rs +documentation link at https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics"# } async fn run( @@ -46,11 +66,32 @@ impl WholeStreamCommand for Open { } fn examples(&self) -> Vec { - vec![Example { - description: "Opens \"users.csv\" and creates a table from the data", - example: "open users.csv", - result: None, - }] + vec![ + Example { + description: "Opens \"users.csv\" and creates a table from the data", + example: "open users.csv", + result: None, + }, + Example { + description: "Opens file with iso-8859-1 encoding", + example: "open file.csv --encoding iso-8859-1 | from csv", + result: None, + }, + ] + } +} + +pub fn get_encoding(opt: Option) -> &'static Encoding { + match opt { + None => UTF_8, + Some(label) => match Encoding::for_label((&label).as_bytes()) { + None => { + //print!("{} is not a known encoding label. Trying UTF-8.", label); + //std::process::exit(-2); + get_encoding(Some("utf-8".to_string())) + } + Some(encoding) => encoding, + }, } } @@ -59,8 +100,19 @@ async fn open(args: CommandArgs, registry: &CommandRegistry) -> Result e.to_string(), + _ => "".to_string(), + }; + let result = fetch(&full_path, &path.item, path.tag.span, enc).await; let (file_extension, contents, contents_tag) = result?; @@ -87,9 +139,173 @@ pub async fn fetch( cwd: &PathBuf, location: &PathBuf, span: Span, + encoding: String, ) -> Result<(Option, UntaggedValue, Tag), ShellError> { let mut cwd = cwd.clone(); + let output_encoding: &Encoding = get_encoding(Some("utf-8".to_string())); + let input_encoding: &Encoding = get_encoding(Some(encoding.clone())); + let mut decoder = input_encoding.new_decoder(); + let mut encoder = output_encoding.new_encoder(); + let mut _file: File; + let buf = Vec::new(); + let mut bufwriter = BufWriter::new(buf); + cwd.push(Path::new(location)); + if let Ok(cwd) = dunce::canonicalize(&cwd) { + if !encoding.is_empty() { + // use the encoding string + match File::open(&Path::new(&cwd)) { + Ok(mut _file) => { + convert_via_utf8( + &mut decoder, + &mut encoder, + &mut _file, + &mut bufwriter, + false, + ); + //bufwriter.flush()?; + Ok(( + cwd.extension() + .map(|name| name.to_string_lossy().to_string()), + UntaggedValue::string(String::from_utf8_lossy(&bufwriter.buffer())), + Tag { + span, + anchor: Some(AnchorLocation::File(cwd.to_string_lossy().to_string())), + }, + )) + } + Err(_) => Err(ShellError::labeled_error( + format!("Cannot open {:?} for reading.", &cwd), + "file not found", + span, + )), + } + } else { + // Do the old stuff + match std::fs::read(&cwd) { + Ok(bytes) => match std::str::from_utf8(&bytes) { + Ok(s) => Ok(( + cwd.extension() + .map(|name| name.to_string_lossy().to_string()), + UntaggedValue::string(s), + Tag { + span, + anchor: Some(AnchorLocation::File(cwd.to_string_lossy().to_string())), + }, + )), + Err(_) => { + //Non utf8 data. + match (bytes.get(0), bytes.get(1)) { + (Some(x), Some(y)) if *x == 0xff && *y == 0xfe => { + // Possibly UTF-16 little endian + let utf16 = read_le_u16(&bytes[2..]); + + if let Some(utf16) = utf16 { + match std::string::String::from_utf16(&utf16) { + Ok(s) => Ok(( + cwd.extension() + .map(|name| name.to_string_lossy().to_string()), + UntaggedValue::string(s), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )), + Err(_) => Ok(( + None, + UntaggedValue::binary(bytes), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )), + } + } else { + Ok(( + None, + UntaggedValue::binary(bytes), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )) + } + } + (Some(x), Some(y)) if *x == 0xfe && *y == 0xff => { + // Possibly UTF-16 big endian + let utf16 = read_be_u16(&bytes[2..]); + + if let Some(utf16) = utf16 { + match std::string::String::from_utf16(&utf16) { + Ok(s) => Ok(( + cwd.extension() + .map(|name| name.to_string_lossy().to_string()), + UntaggedValue::string(s), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )), + Err(_) => Ok(( + None, + UntaggedValue::binary(bytes), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )), + } + } else { + Ok(( + None, + UntaggedValue::binary(bytes), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )) + } + } + _ => Ok(( + None, + UntaggedValue::binary(bytes), + Tag { + span, + anchor: Some(AnchorLocation::File( + cwd.to_string_lossy().to_string(), + )), + }, + )), + } + } + }, + Err(_) => Err(ShellError::labeled_error( + format!("Cannot open {:?} for reading.", &cwd), + "file not found", + span, + )), + } + } + } else { + Err(ShellError::labeled_error( + format!("Cannot open {:?} for reading.", &cwd), + "file not found", + span, + )) + } + /* cwd.push(Path::new(location)); if let Ok(cwd) = dunce::canonicalize(cwd) { match std::fs::read(&cwd) { @@ -214,6 +430,103 @@ pub async fn fetch( span, )) } + */ +} + +fn convert_via_utf8( + decoder: &mut Decoder, + encoder: &mut Encoder, + read: &mut dyn Read, + write: &mut dyn Write, + last: bool, +) { + let mut input_buffer = [0u8; 2048]; + let mut intermediate_buffer_bytes = [0u8; 4096]; + // Is there a safe way to create a stack-allocated &mut str? + let mut intermediate_buffer: &mut str = + //unsafe { std::mem::transmute(&mut intermediate_buffer_bytes[..]) }; + std::str::from_utf8_mut(&mut intermediate_buffer_bytes[..]).expect("error with from_utf8_mut"); + let mut output_buffer = [0u8; 4096]; + let mut current_input_ended = false; + while !current_input_ended { + match read.read(&mut input_buffer) { + Err(_) => { + print!("Error reading input."); + //std::process::exit(-5); + } + Ok(decoder_input_end) => { + current_input_ended = decoder_input_end == 0; + let input_ended = last && current_input_ended; + let mut decoder_input_start = 0usize; + loop { + let (decoder_result, decoder_read, decoder_written, _) = decoder.decode_to_str( + &input_buffer[decoder_input_start..decoder_input_end], + &mut intermediate_buffer, + input_ended, + ); + decoder_input_start += decoder_read; + + let last_output = if input_ended { + match decoder_result { + CoderResult::InputEmpty => true, + CoderResult::OutputFull => false, + } + } else { + false + }; + + // Regardless of whether the intermediate buffer got full + // or the input buffer was exhausted, let's process what's + // in the intermediate buffer. + + if encoder.encoding() == UTF_8 { + // If the target is UTF-8, optimize out the encoder. + if write + .write_all(&intermediate_buffer.as_bytes()[..decoder_written]) + .is_err() + { + print!("Error writing output."); + //std::process::exit(-7); + } + } else { + let mut encoder_input_start = 0usize; + loop { + let (encoder_result, encoder_read, encoder_written, _) = encoder + .encode_from_utf8( + &intermediate_buffer[encoder_input_start..decoder_written], + &mut output_buffer, + last_output, + ); + encoder_input_start += encoder_read; + if write.write_all(&output_buffer[..encoder_written]).is_err() { + print!("Error writing output."); + //std::process::exit(-6); + } + match encoder_result { + CoderResult::InputEmpty => { + break; + } + CoderResult::OutputFull => { + continue; + } + } + } + } + + // Now let's see if we should read again or process the + // rest of the current input buffer. + match decoder_result { + CoderResult::InputEmpty => { + break; + } + CoderResult::OutputFull => { + continue; + } + } + } + } + } + } } fn read_le_u16(input: &[u8]) -> Option> { diff --git a/crates/nu-cli/src/commands/pivot.rs b/crates/nu-cli/src/commands/pivot.rs index 4f8748eef..30d56c5a7 100644 --- a/crates/nu-cli/src/commands/pivot.rs +++ b/crates/nu-cli/src/commands/pivot.rs @@ -51,92 +51,105 @@ impl WholeStreamCommand for Pivot { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - pivot(args, registry) + pivot(args, registry).await } } -pub fn pivot(args: CommandArgs, registry: &CommandRegistry) -> Result { +pub async fn pivot( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); let name = args.call_info.name_tag.clone(); - let stream = async_stream! { - let (args, mut input): (PivotArgs, _) = args.process(®istry).await?; - let input = input.into_vec().await; + let (args, input): (PivotArgs, _) = args.process(®istry).await?; + let input = input.into_vec().await; - let descs = merge_descriptors(&input); + let descs = merge_descriptors(&input); - let mut headers: Vec = vec![]; + let mut headers: Vec = vec![]; - if args.rest.len() > 0 && args.header_row { - yield Err(ShellError::labeled_error("Can not provide header names and use header row", "using header row", name)); - return; - } + if !args.rest.is_empty() && args.header_row { + return Err(ShellError::labeled_error( + "Can not provide header names and use header row", + "using header row", + name, + )); + } - if args.header_row { - for i in input.clone() { - if let Some(desc) = descs.get(0) { - match get_data_by_key(&i, desc[..].spanned_unknown()) { - Some(x) => { - if let Ok(s) = x.as_string() { - headers.push(s.to_string()); - } else { - yield Err(ShellError::labeled_error("Header row needs string headers", "used non-string headers", name)); - return; - } - } - _ => { - yield Err(ShellError::labeled_error("Header row is incomplete and can't be used", "using incomplete header row", name)); - return; - } - } - } else { - yield Err(ShellError::labeled_error("Header row is incomplete and can't be used", "using incomplete header row", name)); - return; - } - } - } else { - for i in 0..=input.len() { - if let Some(name) = args.rest.get(i) { - headers.push(name.to_string()) - } else { - headers.push(format!("Column{}", i)); - } - } - } - - let descs: Vec<_> = if args.header_row { - descs.iter().skip(1).collect() - } else { - descs.iter().collect() - }; - - for desc in descs { - let mut column_num: usize = 0; - let mut dict = TaggedDictBuilder::new(&name); - - if !args.ignore_titles && !args.header_row { - dict.insert_untagged(headers[column_num].clone(), UntaggedValue::string(desc.clone())); - column_num += 1 - } - - for i in input.clone() { + if args.header_row { + for i in input.clone() { + if let Some(desc) = descs.get(0) { match get_data_by_key(&i, desc[..].spanned_unknown()) { Some(x) => { - dict.insert_value(headers[column_num].clone(), x.clone()); + if let Ok(s) = x.as_string() { + headers.push(s.to_string()); + } else { + return Err(ShellError::labeled_error( + "Header row needs string headers", + "used non-string headers", + name, + )); + } } _ => { - dict.insert_untagged(headers[column_num].clone(), UntaggedValue::nothing()); + return Err(ShellError::labeled_error( + "Header row is incomplete and can't be used", + "using incomplete header row", + name, + )); } } - column_num += 1; + } else { + return Err(ShellError::labeled_error( + "Header row is incomplete and can't be used", + "using incomplete header row", + name, + )); } - - yield ReturnSuccess::value(dict.into_value()); } + } else { + for i in 0..=input.len() { + if let Some(name) = args.rest.get(i) { + headers.push(name.to_string()) + } else { + headers.push(format!("Column{}", i)); + } + } + } - + let descs: Vec<_> = if args.header_row { + descs.into_iter().skip(1).collect() + } else { + descs }; - Ok(OutputStream::new(stream)) + Ok(futures::stream::iter(descs.into_iter().map(move |desc| { + let mut column_num: usize = 0; + let mut dict = TaggedDictBuilder::new(&name); + + if !args.ignore_titles && !args.header_row { + dict.insert_untagged( + headers[column_num].clone(), + UntaggedValue::string(desc.clone()), + ); + column_num += 1 + } + + for i in input.clone() { + match get_data_by_key(&i, desc[..].spanned_unknown()) { + Some(x) => { + dict.insert_value(headers[column_num].clone(), x.clone()); + } + _ => { + dict.insert_untagged(headers[column_num].clone(), UntaggedValue::nothing()); + } + } + column_num += 1; + } + + ReturnSuccess::value(dict.into_value()) + })) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/prepend.rs b/crates/nu-cli/src/commands/prepend.rs index 8d9459805..6d7cdbe89 100644 --- a/crates/nu-cli/src/commands/prepend.rs +++ b/crates/nu-cli/src/commands/prepend.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value}; +use nu_protocol::{Signature, SyntaxShape, UntaggedValue, Value}; #[derive(Deserialize)] struct PrependArgs { @@ -34,7 +34,7 @@ impl WholeStreamCommand for Prepend { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - prepend(args, registry) + prepend(args, registry).await } fn examples(&self) -> Vec { @@ -51,19 +51,17 @@ impl WholeStreamCommand for Prepend { } } -fn prepend(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn prepend( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (PrependArgs { row }, mut input) = args.process(®istry).await?; + let (PrependArgs { row }, input) = args.process(®istry).await?; - yield ReturnSuccess::value(row); - while let Some(item) = input.next().await { - yield ReturnSuccess::value(item); - } - }; + let bos = futures::stream::iter(vec![row]); - Ok(stream.to_output_stream()) + Ok(bos.chain(input).to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/range.rs b/crates/nu-cli/src/commands/range.rs index 6b0057970..e180e4a02 100644 --- a/crates/nu-cli/src/commands/range.rs +++ b/crates/nu-cli/src/commands/range.rs @@ -36,28 +36,25 @@ impl WholeStreamCommand for Range { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - range(args, registry) + range(args, registry).await } } -fn range(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn range(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (RangeArgs { area }, mut input) = args.process(®istry).await?; - let range = area.item; - let (from, _) = range.from; - let (to, _) = range.to; + let (RangeArgs { area }, input) = args.process(®istry).await?; + let range = area.item; + let (from, _) = range.from; + let (to, _) = range.to; - let from = *from as usize; - let to = *to as usize; + let from = *from as usize; + let to = *to as usize; - let mut inp = input.skip(from).take(to - from + 1); - while let Some(item) = inp.next().await { - yield ReturnSuccess::value(item); - } - }; - - Ok(stream.to_output_stream()) + Ok(input + .skip(from) + .take(to - from + 1) + .map(ReturnSuccess::value) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/reject.rs b/crates/nu-cli/src/commands/reject.rs index e94a6f611..a3ff4899b 100644 --- a/crates/nu-cli/src/commands/reject.rs +++ b/crates/nu-cli/src/commands/reject.rs @@ -31,7 +31,7 @@ impl WholeStreamCommand for Reject { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - reject(args, registry) + reject(args, registry).await } fn examples(&self) -> Vec { @@ -43,28 +43,23 @@ impl WholeStreamCommand for Reject { } } -fn reject(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn reject(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name = args.call_info.name_tag.clone(); - let (RejectArgs { rest: fields }, mut input) = args.process(®istry).await?; - if fields.is_empty() { - yield Err(ShellError::labeled_error( - "Reject requires fields", - "needs parameter", - name, - )); - return; - } + let name = args.call_info.name_tag.clone(); + let (RejectArgs { rest: fields }, input) = args.process(®istry).await?; + if fields.is_empty() { + return Err(ShellError::labeled_error( + "Reject requires fields", + "needs parameter", + name, + )); + } - let fields: Vec<_> = fields.iter().map(|f| f.item.clone()).collect(); + let fields: Vec<_> = fields.iter().map(|f| f.item.clone()).collect(); - while let Some(item) = input.next().await { - yield ReturnSuccess::value(reject_fields(&item, &fields, &item.tag)); - } - }; - - Ok(stream.to_output_stream()) + Ok(input + .map(move |item| ReturnSuccess::value(reject_fields(&item, &fields, &item.tag))) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/rename.rs b/crates/nu-cli/src/commands/rename.rs index 751d99b49..9539870b3 100644 --- a/crates/nu-cli/src/commands/rename.rs +++ b/crates/nu-cli/src/commands/rename.rs @@ -38,7 +38,7 @@ impl WholeStreamCommand for Rename { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - rename(args, registry) + rename(args, registry).await } fn examples(&self) -> Vec { @@ -57,17 +57,20 @@ impl WholeStreamCommand for Rename { } } -pub fn rename(args: CommandArgs, registry: &CommandRegistry) -> Result { +pub async fn rename( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); let name = args.call_info.name_tag.clone(); - let stream = async_stream! { - let (Arguments { column_name, rest }, mut input) = args.process(®istry).await?; - let mut new_column_names = vec![vec![column_name]]; - new_column_names.push(rest); + let (Arguments { column_name, rest }, input) = args.process(®istry).await?; + let mut new_column_names = vec![vec![column_name]]; + new_column_names.push(rest); - let new_column_names = new_column_names.into_iter().flatten().collect::>(); + let new_column_names = new_column_names.into_iter().flatten().collect::>(); - while let Some(item) = input.next().await { + Ok(input + .map(move |item| { if let Value { value: UntaggedValue::Row(row), tag, @@ -87,21 +90,19 @@ pub fn rename(args: CommandArgs, registry: &CommandRegistry) -> Result Result { - reverse(args, registry) + reverse(args, registry).await } fn examples(&self) -> Vec { @@ -43,19 +43,16 @@ impl WholeStreamCommand for Reverse { } } -fn reverse(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn reverse( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let (input, _args) = args.parts(); + let args = args.evaluate_once(®istry).await?; + let (input, _args) = args.parts(); - let input = input.collect::>().await; - for output in input.into_iter().rev() { - yield ReturnSuccess::value(output); - } - }; - - Ok(stream.to_output_stream()) + let input = input.collect::>().await; + Ok(futures::stream::iter(input.into_iter().rev().map(ReturnSuccess::value)).to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/run_alias.rs b/crates/nu-cli/src/commands/run_alias.rs index 7ca73510b..8a23c58dd 100644 --- a/crates/nu-cli/src/commands/run_alias.rs +++ b/crates/nu-cli/src/commands/run_alias.rs @@ -4,7 +4,7 @@ use crate::prelude::*; use derive_new::new; use nu_errors::ShellError; -use nu_protocol::{hir::Block, ReturnSuccess, Signature, SyntaxShape}; +use nu_protocol::{hir::Block, Signature, SyntaxShape}; #[derive(new, Clone)] pub struct AliasCommand { @@ -38,7 +38,6 @@ impl WholeStreamCommand for AliasCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - let tag = args.call_info.name_tag.clone(); let call_info = args.call_info.clone(); let registry = registry.clone(); let block = self.block.clone(); @@ -46,61 +45,26 @@ impl WholeStreamCommand for AliasCommand { let mut context = Context::from_args(&args, ®istry); let input = args.input; - let stream = async_stream! { - let mut scope = call_info.scope.clone(); - let evaluated = call_info.evaluate(®istry).await?; - if let Some(positional) = &evaluated.args.positional { - for (pos, arg) in positional.iter().enumerate() { - scope.vars.insert(alias_command.args[pos].to_string(), arg.clone()); - } + let mut scope = call_info.scope.clone(); + let evaluated = call_info.evaluate(®istry).await?; + if let Some(positional) = &evaluated.args.positional { + for (pos, arg) in positional.iter().enumerate() { + scope + .vars + .insert(alias_command.args[pos].to_string(), arg.clone()); } + } - let result = run_block( - &block, - &mut context, - input, - &scope.it, - &scope.vars, - &scope.env, - ).await; - - match result { - Ok(stream) if stream.is_empty() => { - yield Err(ShellError::labeled_error( - "Expected a block", - "alias needs a block", - tag, - )); - } - Ok(mut stream) => { - // We collect first to ensure errors are put into the context - while let Some(result) = stream.next().await { - yield Ok(ReturnSuccess::Value(result)); - } - - let errors = context.get_errors(); - if let Some(x) = errors.first() { - yield Err(ShellError::labeled_error_with_secondary( - "Alias failed to run", - "alias failed to run", - tag.clone(), - x.to_string(), - tag - )); - } - } - Err(e) => { - yield Err(ShellError::labeled_error_with_secondary( - "Alias failed to run", - "alias failed to run", - tag.clone(), - e.to_string(), - tag - )); - } - } - }; - - Ok(stream.to_output_stream()) + // FIXME: we need to patch up the spans to point at the top-level error + Ok(run_block( + &block, + &mut context, + input, + &scope.it, + &scope.vars, + &scope.env, + ) + .await? + .to_output_stream()) } } diff --git a/crates/nu-cli/src/commands/run_external.rs b/crates/nu-cli/src/commands/run_external.rs index f6982cb20..e5326228b 100644 --- a/crates/nu-cli/src/commands/run_external.rs +++ b/crates/nu-cli/src/commands/run_external.rs @@ -9,7 +9,7 @@ use std::path::PathBuf; use nu_errors::ShellError; use nu_protocol::hir::{Expression, ExternalArgs, ExternalCommand, Literal, SpannedExpression}; -use nu_protocol::{ReturnSuccess, Signature, SyntaxShape}; +use nu_protocol::{Signature, SyntaxShape}; use nu_source::Tagged; #[derive(Deserialize)] @@ -99,69 +99,49 @@ impl WholeStreamCommand for RunExternalCommand { let is_interactive = self.interactive; - let stream = async_stream! { - let command = ExternalCommand { - name, - name_tag: args.call_info.name_tag.clone(), - args: ExternalArgs { - list: positionals.collect(), - span: args.call_info.args.span, - }, - }; - - // If we're in interactive mode, we will "auto cd". That is, instead of interpreting - // this as an external command, we will see it as a path and `cd` into it. - if is_interactive { - if let Some(path) = maybe_autocd_dir(&command, &mut external_context).await { - let cd_args = CdArgs { - path: Some(Tagged { - item: PathBuf::from(path), - tag: args.call_info.name_tag.clone(), - }) - }; - - let result = external_context.shell_manager.cd(cd_args, args.call_info.name_tag.clone()); - match result { - Ok(mut stream) => { - while let Some(value) = stream.next().await { - yield value; - } - }, - Err(e) => { - yield Err(e); - }, - _ => {} - } - - return; - } - } - - let scope = args.call_info.scope.clone(); - let is_last = args.call_info.args.is_last; - let input = args.input; - let result = external::run_external_command( - command, - &mut external_context, - input, - &scope, - is_last, - ).await; - - match result { - Ok(mut stream) => { - while let Some(value) = stream.next().await { - yield Ok(ReturnSuccess::Value(value)); - } - }, - Err(e) => { - yield Err(e); - }, - _ => {} - } + let command = ExternalCommand { + name, + name_tag: args.call_info.name_tag.clone(), + args: ExternalArgs { + list: positionals.collect(), + span: args.call_info.args.span, + }, }; - Ok(stream.to_output_stream()) + // If we're in interactive mode, we will "auto cd". That is, instead of interpreting + // this as an external command, we will see it as a path and `cd` into it. + if is_interactive { + if let Some(path) = maybe_autocd_dir(&command, &mut external_context).await { + let cd_args = CdArgs { + path: Some(Tagged { + item: PathBuf::from(path), + tag: args.call_info.name_tag.clone(), + }), + }; + + let result = external_context + .shell_manager + .cd(cd_args, args.call_info.name_tag.clone()); + match result { + Ok(stream) => return Ok(stream.to_output_stream()), + Err(e) => { + return Err(e); + } + } + } + } + + let scope = args.call_info.scope.clone(); + let is_last = args.call_info.args.is_last; + let input = args.input; + let result = + external::run_external_command(command, &mut external_context, input, &scope, is_last) + .await; + + match result { + Ok(stream) => Ok(stream.to_output_stream()), + Err(e) => Err(e), + } } } diff --git a/crates/nu-cli/src/commands/save.rs b/crates/nu-cli/src/commands/save.rs index a0ef140c3..0b520eef0 100644 --- a/crates/nu-cli/src/commands/save.rs +++ b/crates/nu-cli/src/commands/save.rs @@ -154,11 +154,14 @@ impl WholeStreamCommand for Save { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - save(args, registry) + save(args, registry).await } } -fn save(raw_args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn save( + raw_args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let mut full_path = PathBuf::from(raw_args.shell_manager.path()); let name_tag = raw_args.call_info.name_tag.clone(); let name = raw_args.call_info.name_tag.clone(); @@ -169,101 +172,96 @@ fn save(raw_args: CommandArgs, registry: &CommandRegistry) -> Result = input.collect().await; - if path.is_none() { - // If there is no filename, check the metadata for the anchor filename - if input.len() > 0 { - let anchor = input[0].tag.anchor(); - match anchor { - Some(path) => match path { - AnchorLocation::File(file) => { - full_path.push(Path::new(&file)); - } - _ => { - yield Err(ShellError::labeled_error( - "Save requires a filepath", - "needs path", - name_tag.clone(), - )); - } - }, - None => { - yield Err(ShellError::labeled_error( - "Save requires a filepath", - "needs path", - name_tag.clone(), - )); - } + let head = raw_args.call_info.args.head.clone(); + let ( + SaveArgs { + path, + raw: save_raw, + }, + input, + ) = raw_args.process(®istry).await?; + let input: Vec = input.collect().await; + if path.is_none() { + let mut should_return_file_path_error = true; + + // If there is no filename, check the metadata for the anchor filename + if !input.is_empty() { + let anchor = input[0].tag.anchor(); + + if let Some(path) = anchor { + if let AnchorLocation::File(file) = path { + should_return_file_path_error = false; + full_path.push(Path::new(&file)); } - } else { - yield Err(ShellError::labeled_error( - "Save requires a filepath", - "needs path", - name_tag.clone(), - )); - } - } else { - if let Some(file) = path { - full_path.push(file.item()); } } - // TODO use label_break_value once it is stable: - // https://github.com/rust-lang/rust/issues/48594 - let content : Result, ShellError> = 'scope: loop { - break if !save_raw { - if let Some(extension) = full_path.extension() { - let command_name = format!("to {}", extension.to_string_lossy()); - if let Some(converter) = registry.get_command(&command_name) { - let new_args = RawCommandArgs { - host, - ctrl_c, - current_errors, - shell_manager, - call_info: UnevaluatedCallInfo { - args: nu_protocol::hir::Call { - head, - positional: None, - named: None, - span: Span::unknown(), - is_last: false, - }, - name_tag: name_tag.clone(), - scope, - } - }; - let mut result = converter.run(new_args.with_input(input), ®istry).await; - let result_vec: Vec> = result.drain_vec().await; - if converter.is_binary() { - process_binary_return_success!('scope, result_vec, name_tag) - } else { - process_string_return_success!('scope, result_vec, name_tag) - } + if should_return_file_path_error { + return Err(ShellError::labeled_error( + "Save requires a filepath", + "needs path", + name_tag.clone(), + )); + } + } else if let Some(file) = path { + full_path.push(file.item()); + } + + // TODO use label_break_value once it is stable: + // https://github.com/rust-lang/rust/issues/48594 + #[allow(clippy::never_loop)] + let content: Result, ShellError> = 'scope: loop { + break if !save_raw { + if let Some(extension) = full_path.extension() { + let command_name = format!("to {}", extension.to_string_lossy()); + if let Some(converter) = registry.get_command(&command_name) { + let new_args = RawCommandArgs { + host, + ctrl_c, + current_errors, + shell_manager, + call_info: UnevaluatedCallInfo { + args: nu_protocol::hir::Call { + head, + positional: None, + named: None, + span: Span::unknown(), + is_last: false, + }, + name_tag: name_tag.clone(), + scope, + }, + }; + let mut result = converter.run(new_args.with_input(input), ®istry).await; + let result_vec: Vec> = + result.drain_vec().await; + if converter.is_binary() { + process_binary_return_success!('scope, result_vec, name_tag) } else { - process_unknown!('scope, input, name_tag) + process_string_return_success!('scope, result_vec, name_tag) } } else { process_unknown!('scope, input, name_tag) } } else { - Ok(string_from(&input).into_bytes()) - }; + process_unknown!('scope, input, name_tag) + } + } else { + Ok(string_from(&input).into_bytes()) }; - - match content { - Ok(save_data) => match std::fs::write(full_path, save_data) { - Ok(o) => o, - Err(e) => yield Err(ShellError::labeled_error(e.to_string(), "IO error while saving", name)), - }, - Err(e) => yield Err(e), - } - }; - Ok(OutputStream::new(stream)) + match content { + Ok(save_data) => match std::fs::write(full_path, save_data) { + Ok(_) => Ok(OutputStream::empty()), + Err(e) => Err(ShellError::labeled_error( + e.to_string(), + "IO error while saving", + name, + )), + }, + Err(e) => Err(e), + } } fn string_from(input: &[Value]) -> String { diff --git a/crates/nu-cli/src/commands/shuffle.rs b/crates/nu-cli/src/commands/shuffle.rs index 7b3297802..d1a00809f 100644 --- a/crates/nu-cli/src/commands/shuffle.rs +++ b/crates/nu-cli/src/commands/shuffle.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{ReturnSuccess, ReturnValue, Value}; +use nu_protocol::{ReturnSuccess, Value}; use rand::seq::SliceRandom; use rand::thread_rng; @@ -24,28 +24,20 @@ impl WholeStreamCommand for Shuffle { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - shuffle(args, registry) + shuffle(args, registry).await } } -fn shuffle(args: CommandArgs, _registry: &CommandRegistry) -> Result { - let stream = async_stream! { - let mut input = args.input; - let mut values: Vec = input.collect().await; +async fn shuffle( + args: CommandArgs, + _registry: &CommandRegistry, +) -> Result { + let input = args.input; + let mut values: Vec = input.collect().await; - let out = { - values.shuffle(&mut thread_rng()); - values.clone() - }; + values.shuffle(&mut thread_rng()); - for val in out.into_iter() { - yield ReturnSuccess::value(val); - } - }; - - let stream: BoxStream<'static, ReturnValue> = stream.boxed(); - - Ok(stream.to_output_stream()) + Ok(futures::stream::iter(values.into_iter().map(ReturnSuccess::value)).to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/split/column.rs b/crates/nu-cli/src/commands/split/column.rs index 5508d310c..6c419f3d3 100644 --- a/crates/nu-cli/src/commands/split/column.rs +++ b/crates/nu-cli/src/commands/split/column.rs @@ -43,16 +43,27 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - split_column(args, registry) + split_column(args, registry).await } } -fn split_column(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn split_column( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let name_span = args.call_info.name_tag.span; let registry = registry.clone(); - let stream = async_stream! { - let (SplitColumnArgs { separator, rest, collapse_empty }, mut input) = args.process(®istry).await?; - while let Some(v) = input.next().await { + let ( + SplitColumnArgs { + separator, + rest, + collapse_empty, + }, + input, + ) = args.process(®istry).await?; + + Ok(input + .map(move |v| { if let Ok(s) = v.as_string() { let splitter = separator.replace("\\n", "\n"); trace!("splitting with {:?}", splitter); @@ -79,7 +90,7 @@ fn split_column(args: CommandArgs, registry: &CommandRegistry) -> Result Result Result { let registry = registry.clone(); - let stream = async_stream! { - yield Ok(ReturnSuccess::Value( - UntaggedValue::string(crate::commands::help::get_help(&Command, ®istry)) - .into_value(Tag::unknown()), - )); - }; - - Ok(stream.to_output_stream()) + Ok(OutputStream::one(Ok(ReturnSuccess::Value( + UntaggedValue::string(crate::commands::help::get_help(&Command, ®istry)) + .into_value(Tag::unknown()), + )))) } } diff --git a/crates/nu-cli/src/commands/split/row.rs b/crates/nu-cli/src/commands/split/row.rs index 34179d046..df73bcdd3 100644 --- a/crates/nu-cli/src/commands/split/row.rs +++ b/crates/nu-cli/src/commands/split/row.rs @@ -35,41 +35,52 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - split_row(args, registry) + split_row(args, registry).await } } -fn split_row(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn split_row( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name = args.call_info.name_tag.clone(); - let (SplitRowArgs { separator }, mut input) = args.process(®istry).await?; - while let Some(v) = input.next().await { + let name = args.call_info.name_tag.clone(); + let (SplitRowArgs { separator }, input) = args.process(®istry).await?; + Ok(input + .flat_map(move |v| { if let Ok(s) = v.as_string() { let splitter = separator.item.replace("\\n", "\n"); trace!("splitting with {:?}", splitter); - let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); + let split_result: Vec = s + .split(&splitter) + .filter_map(|s| { + if s.trim() != "" { + Some(s.to_string()) + } else { + None + } + }) + .collect(); trace!("split result = {:?}", split_result); - for s in split_result { - yield ReturnSuccess::value( - UntaggedValue::Primitive(Primitive::String(s.into())).into_value(&v.tag), - ); - } + futures::stream::iter(split_result.into_iter().map(move |s| { + ReturnSuccess::value( + UntaggedValue::Primitive(Primitive::String(s)).into_value(&v.tag), + ) + })) + .to_output_stream() } else { - yield Err(ShellError::labeled_error_with_secondary( + OutputStream::one(Err(ShellError::labeled_error_with_secondary( "Expected a string from pipeline", "requires string input", name.span, "value originates from here", v.tag.span, - )); + ))) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/str_/capitalize.rs b/crates/nu-cli/src/commands/str_/capitalize.rs index 2fcb7c8ba..3b2c51548 100644 --- a/crates/nu-cli/src/commands/str_/capitalize.rs +++ b/crates/nu-cli/src/commands/str_/capitalize.rs @@ -37,7 +37,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -49,47 +49,44 @@ impl WholeStreamCommand for SubCommand { } } -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { rest }, mut input) = args.process(®istry).await?; + let (Arguments { rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } - Err(err) => { - yield Err(err); - return; - } + Err(err) => return Err(err), } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/command.rs b/crates/nu-cli/src/commands/str_/command.rs index c97a20b81..210ece0d6 100644 --- a/crates/nu-cli/src/commands/str_/command.rs +++ b/crates/nu-cli/src/commands/str_/command.rs @@ -28,14 +28,11 @@ impl WholeStreamCommand for Command { registry: &CommandRegistry, ) -> Result { let registry = registry.clone(); - let stream = async_stream! { - yield Ok(ReturnSuccess::Value( - UntaggedValue::string(crate::commands::help::get_help(&Command, ®istry)) - .into_value(Tag::unknown()), - )); - }; - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value( + UntaggedValue::string(crate::commands::help::get_help(&Command, ®istry)) + .into_value(Tag::unknown()), + ))) } } diff --git a/crates/nu-cli/src/commands/str_/downcase.rs b/crates/nu-cli/src/commands/str_/downcase.rs index b334003d3..0e8ca216e 100644 --- a/crates/nu-cli/src/commands/str_/downcase.rs +++ b/crates/nu-cli/src/commands/str_/downcase.rs @@ -37,7 +37,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -49,47 +49,46 @@ impl WholeStreamCommand for SubCommand { } } -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { rest }, mut input) = args.process(®istry).await?; + let (Arguments { rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/find_replace.rs b/crates/nu-cli/src/commands/str_/find_replace.rs index f22129eb1..c2ac63a0d 100644 --- a/crates/nu-cli/src/commands/str_/find_replace.rs +++ b/crates/nu-cli/src/commands/str_/find_replace.rs @@ -44,7 +44,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -59,52 +59,56 @@ impl WholeStreamCommand for SubCommand { #[derive(Clone)] struct FindReplace(String, String); -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { find, replace, rest }, mut input) = args.process(®istry).await?; - let options = FindReplace(find.item, replace.item); + let ( + Arguments { + find, + replace, + rest, + }, + input, + ) = args.process(®istry).await?; + let options = FindReplace(find.item, replace.item); - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, &options, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { let options = options.clone(); - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| { - action(old, &options, old.tag()) - })); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, &options, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, options: &FindReplace, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/set.rs b/crates/nu-cli/src/commands/str_/set.rs index 8c719f3ee..809fab0ae 100644 --- a/crates/nu-cli/src/commands/str_/set.rs +++ b/crates/nu-cli/src/commands/str_/set.rs @@ -37,7 +37,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -59,50 +59,49 @@ impl WholeStreamCommand for SubCommand { #[derive(Clone)] struct Replace(String); -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { replace, rest }, mut input) = args.process(®istry).await?; - let options = Replace(replace.item); + let (Arguments { replace, rest }, input) = args.process(®istry).await?; + let options = Replace(replace.item); - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, &options, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { let options = options.clone(); - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, &options, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, &options, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(_input: &Value, options: &Replace, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/substring.rs b/crates/nu-cli/src/commands/str_/substring.rs index ae9f3000a..5e4c649af 100644 --- a/crates/nu-cli/src/commands/str_/substring.rs +++ b/crates/nu-cli/src/commands/str_/substring.rs @@ -46,7 +46,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -73,91 +73,83 @@ impl WholeStreamCommand for SubCommand { #[derive(Clone)] struct Substring(usize, usize); -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let name = args.call_info.name_tag.clone(); let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { range, rest }, mut input) = args.process(®istry).await?; + let (Arguments { range, rest }, input) = args.process(®istry).await?; - let v: Vec<&str> = range.item.split(',').collect(); + let v: Vec<&str> = range.item.split(',').collect(); - let start = match v[0] { - "" => 0, - _ => v[0] - .trim() - .parse() - .map_err(|_| { - ShellError::labeled_error( - "could not perform substring", - "could not perform substring", - name.span, - ) - })? - }; - - let end = match v[1] { - "" => usize::max_value(), - _ => v[1] - .trim() - .parse() - .map_err(|_| { - ShellError::labeled_error( - "could not perform substring", - "could not perform substring", - name.span, - ) - })? - }; - - if start > end { - yield Err(ShellError::labeled_error( - "End must be greater than or equal to Start", - "End must be greater than or equal to Start", + let start = match v[0] { + "" => 0, + _ => v[0].trim().parse().map_err(|_| { + ShellError::labeled_error( + "could not perform substring", + "could not perform substring", name.span, - )); - return; - } + ) + })?, + }; - let options = Substring(start, end); + let end = match v[1] { + "" => usize::max_value(), + _ => v[1].trim().parse().map_err(|_| { + ShellError::labeled_error( + "could not perform substring", + "could not perform substring", + name.span, + ) + })?, + }; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + if start > end { + return Err(ShellError::labeled_error( + "End must be greater than or equal to Start", + "End must be greater than or equal to Start", + name.span, + )); + } - while let Some(v) = input.next().await { + let options = Substring(start, end); + + let column_paths: Vec<_> = rest; + + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, &options, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { let options = options.clone(); - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, &options, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, &options, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, options: &Substring, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/to_datetime.rs b/crates/nu-cli/src/commands/str_/to_datetime.rs index 33ccf6eaa..7814d83ed 100644 --- a/crates/nu-cli/src/commands/str_/to_datetime.rs +++ b/crates/nu-cli/src/commands/str_/to_datetime.rs @@ -47,7 +47,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -62,54 +62,53 @@ impl WholeStreamCommand for SubCommand { #[derive(Clone)] struct DatetimeFormat(String); -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { format, rest }, mut input) = args.process(®istry).await?; + let (Arguments { format, rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - let options = if let Some(Tagged { item: fmt, tag }) = format { - DatetimeFormat(fmt) - } else { - DatetimeFormat(String::from("%d.%m.%Y %H:%M %P %z")) - }; + let options = if let Some(Tagged { item: fmt, .. }) = format { + DatetimeFormat(fmt) + } else { + DatetimeFormat(String::from("%d.%m.%Y %H:%M %P %z")) + }; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, &options, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { let options = options.clone(); - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, &options, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, &options, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action( diff --git a/crates/nu-cli/src/commands/str_/to_decimal.rs b/crates/nu-cli/src/commands/str_/to_decimal.rs index a101b723a..3e908d22c 100644 --- a/crates/nu-cli/src/commands/str_/to_decimal.rs +++ b/crates/nu-cli/src/commands/str_/to_decimal.rs @@ -40,7 +40,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -52,47 +52,46 @@ impl WholeStreamCommand for SubCommand { } } -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { rest }, mut input) = args.process(®istry).await?; + let (Arguments { rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/to_integer.rs b/crates/nu-cli/src/commands/str_/to_integer.rs index ce4f20e7d..41277401a 100644 --- a/crates/nu-cli/src/commands/str_/to_integer.rs +++ b/crates/nu-cli/src/commands/str_/to_integer.rs @@ -40,7 +40,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -52,47 +52,46 @@ impl WholeStreamCommand for SubCommand { } } -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { rest }, mut input) = args.process(®istry).await?; + let (Arguments { rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/trim.rs b/crates/nu-cli/src/commands/str_/trim.rs index b8e91da70..3d5cf5b2d 100644 --- a/crates/nu-cli/src/commands/str_/trim.rs +++ b/crates/nu-cli/src/commands/str_/trim.rs @@ -37,7 +37,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -49,47 +49,46 @@ impl WholeStreamCommand for SubCommand { } } -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { rest }, mut input) = args.process(®istry).await?; + let (Arguments { rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/str_/upcase.rs b/crates/nu-cli/src/commands/str_/upcase.rs index 140b9534c..f13dad510 100644 --- a/crates/nu-cli/src/commands/str_/upcase.rs +++ b/crates/nu-cli/src/commands/str_/upcase.rs @@ -37,7 +37,7 @@ impl WholeStreamCommand for SubCommand { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - operate(args, registry) + operate(args, registry).await } fn examples(&self) -> Vec { @@ -49,47 +49,46 @@ impl WholeStreamCommand for SubCommand { } } -fn operate(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (Arguments { rest }, mut input) = args.process(®istry).await?; + let (Arguments { rest }, input) = args.process(®istry).await?; - let column_paths: Vec<_> = rest.iter().map(|x| x.clone()).collect(); + let column_paths: Vec<_> = rest; - while let Some(v) = input.next().await { + Ok(input + .map(move |v| { if column_paths.is_empty() { match action(&v, v.tag()) { - Ok(out) => yield ReturnSuccess::value(out), - Err(err) => { - yield Err(err); - return; - } + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), } } else { - - let mut ret = v.clone(); + let mut ret = v; for path in &column_paths { - let swapping = ret.swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag()))); + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag())), + ); match swapping { Ok(new_value) => { ret = new_value; } Err(err) => { - yield Err(err); - return; + return Err(err); } } } - yield ReturnSuccess::value(ret); + ReturnSuccess::value(ret) } - } - }; - - Ok(stream.to_output_stream()) + }) + .to_output_stream()) } fn action(input: &Value, tag: impl Into) -> Result { diff --git a/crates/nu-cli/src/commands/sum.rs b/crates/nu-cli/src/commands/sum.rs index 55b9aaeb8..b326b72e0 100644 --- a/crates/nu-cli/src/commands/sum.rs +++ b/crates/nu-cli/src/commands/sum.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::prelude::*; use crate::utils::data_processing::{reducer_for, Reduce}; use nu_errors::ShellError; -use nu_protocol::{Dictionary, ReturnSuccess, ReturnValue, Signature, UntaggedValue, Value}; +use nu_protocol::{Dictionary, ReturnSuccess, Signature, UntaggedValue, Value}; use num_traits::identities::Zero; use indexmap::map::IndexMap; @@ -38,6 +38,7 @@ impl WholeStreamCommand for Sum { name: args.call_info.name_tag, raw_input: args.raw_input, }) + .await } fn examples(&self) -> Vec { @@ -56,48 +57,45 @@ impl WholeStreamCommand for Sum { } } -fn sum(RunnableContext { mut input, .. }: RunnableContext) -> Result { - let stream = async_stream! { - let mut values: Vec = input.drain_vec().await; - let action = reducer_for(Reduce::Sum); +async fn sum( + RunnableContext { mut input, .. }: RunnableContext, +) -> Result { + let values: Vec = input.drain_vec().await; + let action = reducer_for(Reduce::Sum); - if values.iter().all(|v| if let UntaggedValue::Primitive(_) = v.value {true} else {false}) { - let total = action(Value::zero(), values)?; - yield ReturnSuccess::value(total) - } else { - let mut column_values = IndexMap::new(); - for value in values { - match value.value { - UntaggedValue::Row(row_dict) => { - for (key, value) in row_dict.entries.iter() { - column_values - .entry(key.clone()) - .and_modify(|v: &mut Vec| v.push(value.clone())) - .or_insert(vec![value.clone()]); - } - }, - table => {}, - }; - } - - let mut column_totals = IndexMap::new(); - for (col_name, col_vals) in column_values { - let sum = action(Value::zero(), col_vals); - match sum { - Ok(value) => { - column_totals.insert(col_name, value); - }, - Err(err) => yield Err(err), - }; - } - yield ReturnSuccess::value( - UntaggedValue::Row(Dictionary {entries: column_totals}).into_untagged_value()) + if values.iter().all(|v| v.is_primitive()) { + let total = action(Value::zero(), values)?; + Ok(OutputStream::one(ReturnSuccess::value(total))) + } else { + let mut column_values = IndexMap::new(); + for value in values { + if let UntaggedValue::Row(row_dict) = value.value { + for (key, value) in row_dict.entries.iter() { + column_values + .entry(key.clone()) + .and_modify(|v: &mut Vec| v.push(value.clone())) + .or_insert(vec![value.clone()]); + } + }; } - }; - let stream: BoxStream<'static, ReturnValue> = stream.boxed(); - - Ok(stream.to_output_stream()) + let mut column_totals = IndexMap::new(); + for (col_name, col_vals) in column_values { + let sum = action(Value::zero(), col_vals); + match sum { + Ok(value) => { + column_totals.insert(col_name, value); + } + Err(err) => return Err(err), + }; + } + Ok(OutputStream::one(ReturnSuccess::value( + UntaggedValue::Row(Dictionary { + entries: column_totals, + }) + .into_untagged_value(), + ))) + } } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/t_sort_by.rs b/crates/nu-cli/src/commands/t_sort_by.rs index 1c86da94d..98d76ec27 100644 --- a/crates/nu-cli/src/commands/t_sort_by.rs +++ b/crates/nu-cli/src/commands/t_sort_by.rs @@ -57,36 +57,47 @@ impl WholeStreamCommand for TSortBy { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - t_sort_by(args, registry) + t_sort_by(args, registry).await } } -fn t_sort_by(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn t_sort_by( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name = args.call_info.name_tag.clone(); - let (TSortByArgs { show_columns, group_by, ..}, mut input) = args.process(®istry).await?; - let values: Vec = input.collect().await; + let name = args.call_info.name_tag.clone(); + let ( + TSortByArgs { + show_columns, + group_by, + .. + }, + mut input, + ) = args.process(®istry).await?; + let values: Vec = input.collect().await; - let column_grouped_by_name = if let Some(grouped_by) = group_by { - Some(grouped_by.item().clone()) - } else { - None - }; - - if show_columns { - for label in columns_sorted(column_grouped_by_name, &values[0], &name).into_iter() { - yield ReturnSuccess::value(UntaggedValue::string(label.item).into_value(label.tag)); - } - } else { - match t_sort(column_grouped_by_name, None, &values[0], name) { - Ok(sorted) => yield ReturnSuccess::value(sorted), - Err(err) => yield Err(err) - } - } + let column_grouped_by_name = if let Some(grouped_by) = group_by { + Some(grouped_by.item().clone()) + } else { + None }; - Ok(stream.to_output_stream()) + if show_columns { + Ok(futures::stream::iter( + columns_sorted(column_grouped_by_name, &values[0], &name) + .into_iter() + .map(move |label| { + ReturnSuccess::value(UntaggedValue::string(label.item).into_value(label.tag)) + }), + ) + .to_output_stream()) + } else { + match t_sort(column_grouped_by_name, None, &values[0], name) { + Ok(sorted) => Ok(OutputStream::one(ReturnSuccess::value(sorted))), + Err(err) => Ok(OutputStream::one(Err(err))), + } + } } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/table.rs b/crates/nu-cli/src/commands/table.rs index 796aaa64b..ea8f3da02 100644 --- a/crates/nu-cli/src/commands/table.rs +++ b/crates/nu-cli/src/commands/table.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::format::TableView; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value}; +use nu_protocol::{Primitive, Signature, SyntaxShape, UntaggedValue, Value}; use std::time::Instant; const STREAM_PAGE_SIZE: usize = 1000; @@ -34,100 +34,97 @@ impl WholeStreamCommand for Table { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - table(args, registry) + table(args, registry).await } } -fn table(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn table(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let mut args = args.evaluate_once(®istry).await?; - let mut finished = false; + let mut args = args.evaluate_once(®istry).await?; + let mut finished = false; - let host = args.host.clone(); - let mut start_number = match args.get("start_number") { - Some(Value { value: UntaggedValue::Primitive(Primitive::Int(i)), .. }) => { - if let Some(num) = i.to_usize() { - num - } else { - yield Err(ShellError::labeled_error("Expected a row number", "expected a row number", &args.args.call_info.name_tag)); - 0 - } + let host = args.host.clone(); + let mut start_number = match args.get("start_number") { + Some(Value { + value: UntaggedValue::Primitive(Primitive::Int(i)), + .. + }) => { + if let Some(num) = i.to_usize() { + num + } else { + return Err(ShellError::labeled_error( + "Expected a row number", + "expected a row number", + &args.args.call_info.name_tag, + )); } - _ => { - 0 - } - }; + } + _ => 0, + }; - let mut delay_slot = None; + let mut delay_slot = None; - while !finished { - let mut new_input: VecDeque = VecDeque::new(); + while !finished { + let mut new_input: VecDeque = VecDeque::new(); - let start_time = Instant::now(); - for idx in 0..STREAM_PAGE_SIZE { - if let Some(val) = delay_slot { - new_input.push_back(val); - delay_slot = None; - } else { - match args.input.next().await { - Some(a) => { - if !new_input.is_empty() { - if let Some(descs) = new_input.get(0) { - let descs = descs.data_descriptors(); - let compare = a.data_descriptors(); - if descs != compare { - delay_slot = Some(a); - break; - } else { - new_input.push_back(a); - } + let start_time = Instant::now(); + for idx in 0..STREAM_PAGE_SIZE { + if let Some(val) = delay_slot { + new_input.push_back(val); + delay_slot = None; + } else { + match args.input.next().await { + Some(a) => { + if !new_input.is_empty() { + if let Some(descs) = new_input.get(0) { + let descs = descs.data_descriptors(); + let compare = a.data_descriptors(); + if descs != compare { + delay_slot = Some(a); + break; } else { new_input.push_back(a); } } else { new_input.push_back(a); } - } - _ => { - finished = true; - break; + } else { + new_input.push_back(a); } } + _ => { + finished = true; + break; + } + } - // Check if we've gone over our buffering threshold - if (idx + 1) % STREAM_TIMEOUT_CHECK_INTERVAL == 0 { - let end_time = Instant::now(); + // Check if we've gone over our buffering threshold + if (idx + 1) % STREAM_TIMEOUT_CHECK_INTERVAL == 0 { + let end_time = Instant::now(); - // If we've been buffering over a second, go ahead and send out what we have so far - if (end_time - start_time).as_secs() >= 1 { - break; - } + // If we've been buffering over a second, go ahead and send out what we have so far + if (end_time - start_time).as_secs() >= 1 { + break; } } } + } - let input: Vec = new_input.into(); + let input: Vec = new_input.into(); - if input.len() > 0 { - let mut host = host.lock(); - let view = TableView::from_list(&input, start_number); + if !input.is_empty() { + let mut host = host.lock(); + let view = TableView::from_list(&input, start_number); - if let Some(view) = view { - handle_unexpected(&mut *host, |host| crate::format::print_view(&view, host)); - } + if let Some(view) = view { + handle_unexpected(&mut *host, |host| crate::format::print_view(&view, host)); } - - start_number += input.len(); } - // Needed for async_stream to type check - if false { - yield ReturnSuccess::value(UntaggedValue::nothing().into_value(Tag::unknown())); - } - }; + start_number += input.len(); + } - Ok(OutputStream::new(stream)) + Ok(OutputStream::empty()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to.rs b/crates/nu-cli/src/commands/to.rs index 5db99546b..5c35d945e 100644 --- a/crates/nu-cli/src/commands/to.rs +++ b/crates/nu-cli/src/commands/to.rs @@ -26,14 +26,10 @@ impl WholeStreamCommand for To { registry: &CommandRegistry, ) -> Result { let registry = registry.clone(); - let stream = async_stream! { - yield Ok(ReturnSuccess::Value( - UntaggedValue::string(crate::commands::help::get_help(&To, ®istry)) - .into_value(Tag::unknown()), - )); - }; - - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value( + UntaggedValue::string(crate::commands::help::get_help(&To, ®istry)) + .into_value(Tag::unknown()), + ))) } } diff --git a/crates/nu-cli/src/commands/to_bson.rs b/crates/nu-cli/src/commands/to_bson.rs index 4ef5ba6a1..4f7541638 100644 --- a/crates/nu-cli/src/commands/to_bson.rs +++ b/crates/nu-cli/src/commands/to_bson.rs @@ -29,7 +29,7 @@ impl WholeStreamCommand for ToBSON { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_bson(args, registry) + to_bson(args, registry).await } fn is_binary(&self) -> bool { @@ -261,51 +261,53 @@ fn bson_value_to_bytes(bson: Bson, tag: Tag) -> Result, ShellError> { Ok(out) } -fn to_bson(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_bson( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let name_tag = args.name_tag(); - let name_span = name_tag.span; + let args = args.evaluate_once(®istry).await?; + let name_tag = args.name_tag(); + let name_span = name_tag.span; - let input: Vec = args.input.collect().await; + let input: Vec = args.input.collect().await; - let to_process_input = if input.len() > 1 { + let to_process_input = match input.len() { + x if x > 1 => { let tag = input[0].tag.clone(); - vec![Value { value: UntaggedValue::Table(input), tag } ] - } else if input.len() == 1 { - input - } else { - vec![] - }; - - for value in to_process_input { - match value_to_bson_value(&value) { - Ok(bson_value) => { - let value_span = value.tag.span; - - match bson_value_to_bytes(bson_value, name_tag.clone()) { - Ok(x) => yield ReturnSuccess::value( - UntaggedValue::binary(x).into_value(&name_tag), - ), - _ => yield Err(ShellError::labeled_error_with_secondary( - "Expected a table with BSON-compatible structure from pipeline", - "requires BSON-compatible input", - name_span, - "originates from here".to_string(), - value_span, - )), - } - } - _ => yield Err(ShellError::labeled_error( - "Expected a table with BSON-compatible structure from pipeline", - "requires BSON-compatible input", - &name_tag)) - } + vec![Value { + value: UntaggedValue::Table(input), + tag, + }] } + 1 => input, + _ => vec![], }; - Ok(stream.to_output_stream()) + Ok(futures::stream::iter(to_process_input.into_iter().map( + move |value| match value_to_bson_value(&value) { + Ok(bson_value) => { + let value_span = value.tag.span; + + match bson_value_to_bytes(bson_value, name_tag.clone()) { + Ok(x) => ReturnSuccess::value(UntaggedValue::binary(x).into_value(&name_tag)), + _ => Err(ShellError::labeled_error_with_secondary( + "Expected a table with BSON-compatible structure from pipeline", + "requires BSON-compatible input", + name_span, + "originates from here".to_string(), + value_span, + )), + } + } + _ => Err(ShellError::labeled_error( + "Expected a table with BSON-compatible structure from pipeline", + "requires BSON-compatible input", + &name_tag, + )), + }, + )) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to_csv.rs b/crates/nu-cli/src/commands/to_csv.rs index 9f8c641a2..b41aeb674 100644 --- a/crates/nu-cli/src/commands/to_csv.rs +++ b/crates/nu-cli/src/commands/to_csv.rs @@ -42,47 +42,44 @@ impl WholeStreamCommand for ToCSV { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_csv(args, registry) + to_csv(args, registry).await } } -fn to_csv(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_csv(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name = args.call_info.name_tag.clone(); - let (ToCSVArgs { separator, headerless }, mut input) = args.process(®istry).await?; - let sep = match separator { - Some(Value { - value: UntaggedValue::Primitive(Primitive::String(s)), - tag, - .. - }) => { - if s == r"\t" { - '\t' - } else { - let vec_s: Vec = s.chars().collect(); - if vec_s.len() != 1 { - yield Err(ShellError::labeled_error( - "Expected a single separator char from --separator", - "requires a single character string input", - tag, - )); - return; - }; - vec_s[0] - } + let name = args.call_info.name_tag.clone(); + let ( + ToCSVArgs { + separator, + headerless, + }, + input, + ) = args.process(®istry).await?; + let sep = match separator { + Some(Value { + value: UntaggedValue::Primitive(Primitive::String(s)), + tag, + .. + }) => { + if s == r"\t" { + '\t' + } else { + let vec_s: Vec = s.chars().collect(); + if vec_s.len() != 1 { + return Err(ShellError::labeled_error( + "Expected a single separator char from --separator", + "requires a single character string input", + tag, + )); + }; + vec_s[0] } - _ => ',', - }; - - let mut result = to_delimited_data(headerless, sep, "CSV", input, name)?; - - while let Some(item) = result.next().await { - yield item; } + _ => ',', }; - Ok(stream.to_output_stream()) + to_delimited_data(headerless, sep, "CSV", input, name).await } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to_delimited_data.rs b/crates/nu-cli/src/commands/to_delimited_data.rs index 995862031..010becdcb 100644 --- a/crates/nu-cli/src/commands/to_delimited_data.rs +++ b/crates/nu-cli/src/commands/to_delimited_data.rs @@ -165,7 +165,7 @@ fn merge_descriptors(values: &[Value]) -> Vec> { ret } -pub fn to_delimited_data( +pub async fn to_delimited_data( headerless: bool, sep: char, format_name: &'static str, @@ -175,33 +175,41 @@ pub fn to_delimited_data( let name_tag = name; let name_span = name_tag.span; - let stream = async_stream! { - let input: Vec = input.collect().await; + let input: Vec = input.collect().await; - let to_process_input = if input.len() > 1 { + let to_process_input = match input.len() { + x if x > 1 => { let tag = input[0].tag.clone(); - vec![Value { value: UntaggedValue::Table(input), tag } ] - } else if input.len() == 1 { - input - } else { - vec![] - }; + vec![Value { + value: UntaggedValue::Table(input), + tag, + }] + } + 1 => input, + _ => vec![], + }; - for value in to_process_input { + Ok( + futures::stream::iter(to_process_input.into_iter().map(move |value| { match from_value_to_delimited_string(&clone_tagged_value(&value), sep) { Ok(mut x) => { if headerless { - x.find('\n').map(|second_line|{ + if let Some(second_line) = x.find('\n') { let start = second_line + 1; x.replace_range(0..start, ""); - }); + } } - yield ReturnSuccess::value(UntaggedValue::Primitive(Primitive::String(x)).into_value(&name_tag)) + ReturnSuccess::value( + UntaggedValue::Primitive(Primitive::String(x)).into_value(&name_tag), + ) } - Err(x) => { - let expected = format!("Expected a table with {}-compatible structure from pipeline", format_name); + Err(_) => { + let expected = format!( + "Expected a table with {}-compatible structure from pipeline", + format_name + ); let requires = format!("requires {}-compatible input", format_name); - yield Err(ShellError::labeled_error_with_secondary( + Err(ShellError::labeled_error_with_secondary( expected, requires, name_span, @@ -210,8 +218,7 @@ pub fn to_delimited_data( )) } } - } - }; - - Ok(stream.to_output_stream()) + })) + .to_output_stream(), + ) } diff --git a/crates/nu-cli/src/commands/to_json.rs b/crates/nu-cli/src/commands/to_json.rs index f46c9c083..da2bced79 100644 --- a/crates/nu-cli/src/commands/to_json.rs +++ b/crates/nu-cli/src/commands/to_json.rs @@ -38,7 +38,7 @@ impl WholeStreamCommand for ToJSON { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_json(args, registry) + to_json(args, registry).await } fn examples(&self) -> Vec { @@ -163,78 +163,103 @@ fn json_list(input: &[Value]) -> Result, ShellError> { Ok(out) } -fn to_json(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_json( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name_tag = args.call_info.name_tag.clone(); - let (ToJSONArgs { pretty }, mut input) = args.process(®istry).await?; - let name_span = name_tag.span; - let input: Vec = input.collect().await; + let name_tag = args.call_info.name_tag.clone(); + let (ToJSONArgs { pretty }, input) = args.process(®istry).await?; + let name_span = name_tag.span; + let input: Vec = input.collect().await; - let to_process_input = if input.len() > 1 { + let to_process_input = match input.len() { + x if x > 1 => { let tag = input[0].tag.clone(); - vec![Value { value: UntaggedValue::Table(input), tag } ] - } else if input.len() == 1 { - input - } else { - vec![] - }; + vec![Value { + value: UntaggedValue::Table(input), + tag, + }] + } + 1 => input, + _ => vec![], + }; - for value in to_process_input { - match value_to_json_value(&value) { - Ok(json_value) => { - let value_span = value.tag.span; + Ok(futures::stream::iter(to_process_input.into_iter().map( + move |value| match value_to_json_value(&value) { + Ok(json_value) => { + let value_span = value.tag.span; - match serde_json::to_string(&json_value) { - Ok(mut serde_json_string) => { - if let Some(pretty_value) = &pretty { - let mut pretty_format_failed = true; + match serde_json::to_string(&json_value) { + Ok(mut serde_json_string) => { + if let Some(pretty_value) = &pretty { + let mut pretty_format_failed = true; - if let Ok(pretty_u64) = pretty_value.as_u64() { - if let Ok(serde_json_value) = serde_json::from_str::(serde_json_string.as_str()) { - let indentation_string = std::iter::repeat(" ").take(pretty_u64 as usize).collect::(); - let serde_formatter = serde_json::ser::PrettyFormatter::with_indent(indentation_string.as_bytes()); - let serde_buffer = Vec::new(); - let mut serde_serializer = serde_json::Serializer::with_formatter(serde_buffer, serde_formatter); - let serde_json_object = json!(serde_json_value); + if let Ok(pretty_u64) = pretty_value.as_u64() { + if let Ok(serde_json_value) = + serde_json::from_str::( + serde_json_string.as_str(), + ) + { + let indentation_string = std::iter::repeat(" ") + .take(pretty_u64 as usize) + .collect::(); + let serde_formatter = + serde_json::ser::PrettyFormatter::with_indent( + indentation_string.as_bytes(), + ); + let serde_buffer = Vec::new(); + let mut serde_serializer = + serde_json::Serializer::with_formatter( + serde_buffer, + serde_formatter, + ); + let serde_json_object = json!(serde_json_value); - if let Ok(()) = serde_json_object.serialize(&mut serde_serializer) { - if let Ok(ser_json_string) = String::from_utf8(serde_serializer.into_inner()) { - pretty_format_failed = false; - serde_json_string = ser_json_string - } + if let Ok(()) = + serde_json_object.serialize(&mut serde_serializer) + { + if let Ok(ser_json_string) = + String::from_utf8(serde_serializer.into_inner()) + { + pretty_format_failed = false; + serde_json_string = ser_json_string } } } - - if pretty_format_failed { - yield Err(ShellError::labeled_error("Pretty formatting failed", "failed", pretty_value.tag())); - return; - } } - yield ReturnSuccess::value( - UntaggedValue::Primitive(Primitive::String(serde_json_string)).into_value(&name_tag), - ) - }, - _ => yield Err(ShellError::labeled_error_with_secondary( - "Expected a table with JSON-compatible structure.tag() from pipeline", - "requires JSON-compatible input", - name_span, - "originates from here".to_string(), - value_span, - )), - } - } - _ => yield Err(ShellError::labeled_error( - "Expected a table with JSON-compatible structure from pipeline", - "requires JSON-compatible input", - &name_tag)) - } - } - }; + if pretty_format_failed { + return Err(ShellError::labeled_error( + "Pretty formatting failed", + "failed", + pretty_value.tag(), + )); + } + } - Ok(stream.to_output_stream()) + ReturnSuccess::value( + UntaggedValue::Primitive(Primitive::String(serde_json_string)) + .into_value(&name_tag), + ) + } + _ => Err(ShellError::labeled_error_with_secondary( + "Expected a table with JSON-compatible structure.tag() from pipeline", + "requires JSON-compatible input", + name_span, + "originates from here".to_string(), + value_span, + )), + } + } + _ => Err(ShellError::labeled_error( + "Expected a table with JSON-compatible structure from pipeline", + "requires JSON-compatible input", + &name_tag, + )), + }, + )) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to_toml.rs b/crates/nu-cli/src/commands/to_toml.rs index db61b51e5..f4c70b38c 100644 --- a/crates/nu-cli/src/commands/to_toml.rs +++ b/crates/nu-cli/src/commands/to_toml.rs @@ -24,7 +24,7 @@ impl WholeStreamCommand for ToTOML { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_toml(args, registry) + to_toml(args, registry).await } // TODO: add an example here. What commands to run to get a Row(Dictionary)? // fn examples(&self) -> Vec { @@ -135,49 +135,53 @@ fn collect_values(input: &[Value]) -> Result, ShellError> { Ok(out) } -fn to_toml(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_toml( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let name_tag = args.name_tag(); - let name_span = name_tag.span; - let input: Vec = args.input.collect().await; + let args = args.evaluate_once(®istry).await?; + let name_tag = args.name_tag(); + let name_span = name_tag.span; + let input: Vec = args.input.collect().await; - let to_process_input = if input.len() > 1 { + let to_process_input = match input.len() { + x if x > 1 => { let tag = input[0].tag.clone(); - vec![Value { value: UntaggedValue::Table(input), tag } ] - } else if input.len() == 1 { - input - } else { - vec![] - }; - - for value in to_process_input { - let value_span = value.tag.span; - match value_to_toml_value(&value) { - Ok(toml_value) => { - match toml::to_string(&toml_value) { - Ok(x) => yield ReturnSuccess::value( - UntaggedValue::Primitive(Primitive::String(x)).into_value(&name_tag), - ), - _ => yield Err(ShellError::labeled_error_with_secondary( - "Expected a table with TOML-compatible structure.tag() from pipeline", - "requires TOML-compatible input", - name_span, - "originates from here".to_string(), - value_span, - )), - } - } - _ => yield Err(ShellError::labeled_error( - "Expected a table with TOML-compatible structure from pipeline", - "requires TOML-compatible input", - &name_tag)) - } + vec![Value { + value: UntaggedValue::Table(input), + tag, + }] } + 1 => input, + _ => vec![], }; - Ok(stream.to_output_stream()) + Ok( + futures::stream::iter(to_process_input.into_iter().map(move |value| { + let value_span = value.tag.span; + match value_to_toml_value(&value) { + Ok(toml_value) => match toml::to_string(&toml_value) { + Ok(x) => ReturnSuccess::value( + UntaggedValue::Primitive(Primitive::String(x)).into_value(&name_tag), + ), + _ => Err(ShellError::labeled_error_with_secondary( + "Expected a table with TOML-compatible structure.tag() from pipeline", + "requires TOML-compatible input", + name_span, + "originates from here".to_string(), + value_span, + )), + }, + _ => Err(ShellError::labeled_error( + "Expected a table with TOML-compatible structure from pipeline", + "requires TOML-compatible input", + &name_tag, + )), + } + })) + .to_output_stream(), + ) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to_tsv.rs b/crates/nu-cli/src/commands/to_tsv.rs index 45b365736..974884986 100644 --- a/crates/nu-cli/src/commands/to_tsv.rs +++ b/crates/nu-cli/src/commands/to_tsv.rs @@ -34,29 +34,16 @@ impl WholeStreamCommand for ToTSV { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_tsv(args, registry) + to_tsv(args, registry).await } } -fn to_tsv(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_tsv(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let name = args.call_info.name_tag.clone(); - let (ToTSVArgs { headerless }, mut input) = args.process(®istry).await?; - let mut result = to_delimited_data( - headerless, - '\t', - "TSV", - input, - name, - )?; + let name = args.call_info.name_tag.clone(); + let (ToTSVArgs { headerless }, input) = args.process(®istry).await?; - while let Some(item) = result.next().await { - yield item; - } - }; - - Ok(stream.to_output_stream()) + to_delimited_data(headerless, '\t', "TSV", input, name).await } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to_url.rs b/crates/nu-cli/src/commands/to_url.rs index abc66a97c..d939ca271 100644 --- a/crates/nu-cli/src/commands/to_url.rs +++ b/crates/nu-cli/src/commands/to_url.rs @@ -24,67 +24,58 @@ impl WholeStreamCommand for ToURL { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_url(args, registry) + to_url(args, registry).await } } -fn to_url(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_url(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let tag = args.name_tag(); - let input = args.input; + let args = args.evaluate_once(®istry).await?; + let tag = args.name_tag(); + let input = args.input; - let input: Vec = input.collect().await; - - for value in input { - match value { - Value { value: UntaggedValue::Row(row), .. } => { - let mut row_vec = vec![]; - for (k,v) in row.entries { - match v.as_string() { - Ok(s) => { - row_vec.push((k.clone(), s.to_string())); - } - _ => { - yield Err(ShellError::labeled_error_with_secondary( - "Expected table with string values", - "requires table with strings", - &tag, - "value originates from here", - v.tag, - )) - } - } - } - - match serde_urlencoded::to_string(row_vec) { + Ok(input + .map(move |value| match value { + Value { + value: UntaggedValue::Row(row), + .. + } => { + let mut row_vec = vec![]; + for (k, v) in row.entries { + match v.as_string() { Ok(s) => { - yield ReturnSuccess::value(UntaggedValue::string(s).into_value(&tag)); + row_vec.push((k.clone(), s.to_string())); } _ => { - yield Err(ShellError::labeled_error( - "Failed to convert to url-encoded", - "cannot url-encode", + return Err(ShellError::labeled_error_with_secondary( + "Expected table with string values", + "requires table with strings", &tag, - )) + "value originates from here", + v.tag, + )); } } } - Value { tag: value_tag, .. } => { - yield Err(ShellError::labeled_error_with_secondary( - "Expected a table from pipeline", - "requires table input", + + match serde_urlencoded::to_string(row_vec) { + Ok(s) => ReturnSuccess::value(UntaggedValue::string(s).into_value(&tag)), + _ => Err(ShellError::labeled_error( + "Failed to convert to url-encoded", + "cannot url-encode", &tag, - "value originates from here", - value_tag.span, - )) + )), } } - } - }; - - Ok(stream.to_output_stream()) + Value { tag: value_tag, .. } => Err(ShellError::labeled_error_with_secondary( + "Expected a table from pipeline", + "requires table input", + &tag, + "value originates from here", + value_tag.span, + )), + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/to_yaml.rs b/crates/nu-cli/src/commands/to_yaml.rs index a30c9af01..3c0109f19 100644 --- a/crates/nu-cli/src/commands/to_yaml.rs +++ b/crates/nu-cli/src/commands/to_yaml.rs @@ -24,7 +24,7 @@ impl WholeStreamCommand for ToYAML { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - to_yaml(args, registry) + to_yaml(args, registry).await } } @@ -125,51 +125,55 @@ pub fn value_to_yaml_value(v: &Value) -> Result { }) } -fn to_yaml(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn to_yaml( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let args = args.evaluate_once(®istry).await?; - let name_tag = args.name_tag(); - let name_span = name_tag.span; + let args = args.evaluate_once(®istry).await?; + let name_tag = args.name_tag(); + let name_span = name_tag.span; - let input: Vec = args.input.collect().await; + let input: Vec = args.input.collect().await; - let to_process_input = if input.len() > 1 { + let to_process_input = match input.len() { + x if x > 1 => { let tag = input[0].tag.clone(); - vec![Value { value: UntaggedValue::Table(input), tag } ] - } else if input.len() == 1 { - input - } else { - vec![] - }; + vec![Value { + value: UntaggedValue::Table(input), + tag, + }] + } + 1 => input, + _ => vec![], + }; - for value in to_process_input { + Ok( + futures::stream::iter(to_process_input.into_iter().map(move |value| { let value_span = value.tag.span; match value_to_yaml_value(&value) { - Ok(yaml_value) => { - match serde_yaml::to_string(&yaml_value) { - Ok(x) => yield ReturnSuccess::value( - UntaggedValue::Primitive(Primitive::String(x)).into_value(&name_tag), - ), - _ => yield Err(ShellError::labeled_error_with_secondary( - "Expected a table with YAML-compatible structure from pipeline", - "requires YAML-compatible input", - name_span, - "originates from here".to_string(), - value_span, - )), - } - } - _ => yield Err(ShellError::labeled_error( + Ok(yaml_value) => match serde_yaml::to_string(&yaml_value) { + Ok(x) => ReturnSuccess::value( + UntaggedValue::Primitive(Primitive::String(x)).into_value(&name_tag), + ), + _ => Err(ShellError::labeled_error_with_secondary( + "Expected a table with YAML-compatible structure from pipeline", + "requires YAML-compatible input", + name_span, + "originates from here".to_string(), + value_span, + )), + }, + _ => Err(ShellError::labeled_error( "Expected a table with YAML-compatible structure from pipeline", "requires YAML-compatible input", - &name_tag)) + &name_tag, + )), } - } - }; - - Ok(stream.to_output_stream()) + })) + .to_output_stream(), + ) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/touch.rs b/crates/nu-cli/src/commands/touch.rs index aeab41560..0bd7fe549 100644 --- a/crates/nu-cli/src/commands/touch.rs +++ b/crates/nu-cli/src/commands/touch.rs @@ -33,7 +33,7 @@ impl WholeStreamCommand for Touch { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - touch(args, registry) + touch(args, registry).await } fn examples(&self) -> Vec { @@ -45,21 +45,18 @@ impl WholeStreamCommand for Touch { } } -fn touch(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn touch(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (TouchArgs { target }, _) = args.process(®istry).await?; - match OpenOptions::new().write(true).create(true).open(&target) { - Ok(_) => {}, - Err(err) => yield Err(ShellError::labeled_error( - "File Error", - err.to_string(), - &target.tag, - )), - } - }; + let (TouchArgs { target }, _) = args.process(®istry).await?; - Ok(stream.to_output_stream()) + match OpenOptions::new().write(true).create(true).open(&target) { + Ok(_) => Ok(OutputStream::empty()), + Err(err) => Err(ShellError::labeled_error( + "File Error", + err.to_string(), + &target.tag, + )), + } } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/update.rs b/crates/nu-cli/src/commands/update.rs index 239322d04..2590d42bb 100644 --- a/crates/nu-cli/src/commands/update.rs +++ b/crates/nu-cli/src/commands/update.rs @@ -3,7 +3,7 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{ColumnPath, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value}; +use nu_protocol::{ColumnPath, ReturnSuccess, Scope, Signature, SyntaxShape, UntaggedValue, Value}; use nu_value_ext::ValueExt; use futures::stream::once; @@ -44,105 +44,124 @@ impl WholeStreamCommand for Update { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - update(args, registry) + update(args, registry).await } } -fn update(raw_args: CommandArgs, registry: &CommandRegistry) -> Result { - let registry = registry.clone(); - let scope = raw_args.call_info.scope.clone(); +async fn process_row( + scope: Arc, + mut context: Arc, + input: Value, + mut replacement: Arc, + field: Arc, +) -> Result { + let replacement = Arc::make_mut(&mut replacement); - let stream = async_stream! { - let mut context = Context::from_raw(&raw_args, ®istry); - let (UpdateArgs { field, replacement }, mut input) = raw_args.process(®istry).await?; - while let Some(input) = input.next().await { - let replacement = replacement.clone(); - match replacement { - Value { - value: UntaggedValue::Block(block), - tag, - } => { - let for_block = input.clone(); - let input_stream = once(async { Ok(for_block) }).to_input_stream(); + Ok(match replacement { + Value { + value: UntaggedValue::Block(block), + .. + } => { + let for_block = input.clone(); + let input_stream = once(async { Ok(for_block) }).to_input_stream(); - let result = run_block( - &block, - &mut context, - input_stream, - &input, - &scope.vars, - &scope.env - ).await; + let result = run_block( + &block, + Arc::make_mut(&mut context), + input_stream, + &input, + &scope.vars, + &scope.env, + ) + .await; - match result { - Ok(mut stream) => { - let errors = context.get_errors(); - if let Some(error) = errors.first() { - yield Err(error.clone()); - } - - match input { - obj @ Value { - value: UntaggedValue::Row(_), - .. - } => { - if let Some(result) = stream.next().await { - match obj.replace_data_at_column_path(&field, result.clone()) { - Some(v) => yield Ok(ReturnSuccess::Value(v)), - None => { - yield Err(ShellError::labeled_error( - "update could not find place to insert column", - "column name", - obj.tag, - )) - } - } - } - } - Value { tag, ..} => { - yield Err(ShellError::labeled_error( - "Unrecognized type in stream", - "original value", - tag, - )) - } - } - } - Err(e) => { - yield Err(e); - } + match result { + Ok(mut stream) => { + let errors = context.get_errors(); + if let Some(error) = errors.first() { + return Err(error.clone()); } - } - _ => { + match input { - obj @ Value { + obj + @ + Value { value: UntaggedValue::Row(_), .. - } => match obj.replace_data_at_column_path(&field, replacement.clone()) { - Some(v) => yield Ok(ReturnSuccess::Value(v)), - None => { - yield Err(ShellError::labeled_error( - "update could not find place to insert column", - "column name", - obj.tag, - )) + } => { + if let Some(result) = stream.next().await { + match obj.replace_data_at_column_path(&field, result) { + Some(v) => OutputStream::one(ReturnSuccess::value(v)), + None => OutputStream::one(Err(ShellError::labeled_error( + "update could not find place to insert column", + "column name", + obj.tag, + ))), + } + } else { + OutputStream::empty() } - }, - Value { tag, ..} => { - yield Err(ShellError::labeled_error( - "Unrecognized type in stream", - "original value", - tag, - )) } - _ => {} + Value { tag, .. } => OutputStream::one(Err(ShellError::labeled_error( + "Unrecognized type in stream", + "original value", + tag, + ))), } } + Err(e) => OutputStream::one(Err(e)), } } - }; + _ => match input { + obj + @ + Value { + value: UntaggedValue::Row(_), + .. + } => match obj.replace_data_at_column_path(&field, replacement.clone()) { + Some(v) => OutputStream::one(ReturnSuccess::value(v)), + None => OutputStream::one(Err(ShellError::labeled_error( + "update could not find place to insert column", + "column name", + obj.tag, + ))), + }, + Value { tag, .. } => OutputStream::one(Err(ShellError::labeled_error( + "Unrecognized type in stream", + "original value", + tag, + ))), + }, + }) +} - Ok(stream.to_output_stream()) +async fn update( + raw_args: CommandArgs, + registry: &CommandRegistry, +) -> Result { + let registry = registry.clone(); + let scope = Arc::new(raw_args.call_info.scope.clone()); + let context = Arc::new(Context::from_raw(&raw_args, ®istry)); + let (UpdateArgs { field, replacement }, input) = raw_args.process(®istry).await?; + let replacement = Arc::new(replacement); + let field = Arc::new(field); + + Ok(input + .then(move |input| { + let replacement = replacement.clone(); + let scope = scope.clone(); + let context = context.clone(); + let field = field.clone(); + + async { + match process_row(scope, context, input, replacement, field).await { + Ok(s) => s, + Err(e) => OutputStream::one(Err(e)), + } + } + }) + .flatten() + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/what.rs b/crates/nu-cli/src/commands/what.rs index c8eabe027..3af287c0d 100644 --- a/crates/nu-cli/src/commands/what.rs +++ b/crates/nu-cli/src/commands/what.rs @@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand; use crate::prelude::*; use nu_errors::ShellError; -use nu_protocol::{ReturnSuccess, ReturnValue, Signature, UntaggedValue}; +use nu_protocol::{ReturnSuccess, Signature, UntaggedValue}; pub struct What; @@ -28,23 +28,23 @@ impl WholeStreamCommand for What { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - // args.process(registry, what)?.run() - what(args, registry) + what(args, registry).await } } -pub fn what(args: CommandArgs, _registry: &CommandRegistry) -> Result { - let stream = async_stream! { - let mut input = args.input; - while let Some(row) = input.next().await { +pub async fn what( + args: CommandArgs, + _registry: &CommandRegistry, +) -> Result { + Ok(args + .input + .map(|row| { let name = value::format_type(&row, 100); - yield ReturnSuccess::value(UntaggedValue::string(name).into_value(Tag::unknown_anchor(row.tag.span))); - } - }; - - let stream: BoxStream<'static, ReturnValue> = stream.boxed(); - - Ok(OutputStream::from(stream)) + ReturnSuccess::value( + UntaggedValue::string(name).into_value(Tag::unknown_anchor(row.tag.span)), + ) + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/where_.rs b/crates/nu-cli/src/commands/where_.rs index dfce3bcc3..eb9fd2ae7 100644 --- a/crates/nu-cli/src/commands/where_.rs +++ b/crates/nu-cli/src/commands/where_.rs @@ -35,7 +35,7 @@ impl WholeStreamCommand for Where { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - where_command(args, registry) + where_command(args, registry).await } fn examples(&self) -> Vec { @@ -63,65 +63,71 @@ impl WholeStreamCommand for Where { ] } } -fn where_command( +async fn where_command( raw_args: CommandArgs, registry: &CommandRegistry, ) -> Result { - let registry = registry.clone(); - let scope = raw_args.call_info.scope.clone(); + let registry = Arc::new(registry.clone()); + let scope = Arc::new(raw_args.call_info.scope.clone()); let tag = raw_args.call_info.name_tag.clone(); - let stream = async_stream! { - let (WhereArgs { block }, mut input) = raw_args.process(®istry).await?; - let condition = { - if block.block.len() != 1 { - yield Err(ShellError::labeled_error( - "Expected a condition", - "expected a condition", - tag, - )); - return; - } - match block.block[0].list.get(0) { - Some(item) => match item { - ClassifiedCommand::Expr(expr) => expr.clone(), - _ => { - yield Err(ShellError::labeled_error( - "Expected a condition", - "expected a condition", - tag, - )); - return; - } - }, - None => { - yield Err(ShellError::labeled_error( + let (WhereArgs { block }, input) = raw_args.process(®istry).await?; + let condition = { + if block.block.len() != 1 { + return Err(ShellError::labeled_error( + "Expected a condition", + "expected a condition", + tag, + )); + } + match block.block[0].list.get(0) { + Some(item) => match item { + ClassifiedCommand::Expr(expr) => expr.clone(), + _ => { + return Err(ShellError::labeled_error( "Expected a condition", "expected a condition", tag, )); - return; } + }, + None => { + return Err(ShellError::labeled_error( + "Expected a condition", + "expected a condition", + tag, + )); } - }; - - let mut input = input; - while let Some(input) = input.next().await { - - //FIXME: should we use the scope that's brought in as well? - let condition = evaluate_baseline_expr(&condition, ®istry, &input, &scope.vars, &scope.env).await?; - - match condition.as_bool() { - Ok(b) => { - if b { - yield Ok(ReturnSuccess::Value(input)); - } - } - Err(e) => yield Err(e), - }; } }; - Ok(stream.to_output_stream()) + Ok(input + .filter_map(move |input| { + let condition = condition.clone(); + let registry = registry.clone(); + let scope = scope.clone(); + + async move { + //FIXME: should we use the scope that's brought in as well? + let condition = + evaluate_baseline_expr(&condition, &*registry, &input, &scope.vars, &scope.env) + .await; + + match condition { + Ok(condition) => match condition.as_bool() { + Ok(b) => { + if b { + Some(Ok(ReturnSuccess::Value(input))) + } else { + None + } + } + Err(e) => Some(Err(e)), + }, + Err(e) => Some(Err(e)), + } + } + }) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/which_.rs b/crates/nu-cli/src/commands/which_.rs index e0a12fafe..2515cf885 100644 --- a/crates/nu-cli/src/commands/which_.rs +++ b/crates/nu-cli/src/commands/which_.rs @@ -28,7 +28,7 @@ impl WholeStreamCommand for Which { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - which(args, registry) + which(args, registry).await } } @@ -77,36 +77,42 @@ struct WhichArgs { all: bool, } -fn which(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn which(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let mut all = true; - let stream = async_stream! { - let (WhichArgs { application, all: all_items }, _) = args.process(®istry).await?; - all = all_items; - let external = application.starts_with('^'); - let item = if external { - application.item[1..].to_string() - } else { - application.item.clone() - }; - if !external { - let builtin = registry.has(&item); - if builtin { - yield ReturnSuccess::value(entry_builtin!(item, application.tag.clone())); - } - } - if let Ok(paths) = ichwh::which_all(&item).await { - for path in paths { - yield ReturnSuccess::value(entry_path!(item, path.into(), application.tag.clone())); - } - } + let mut output = vec![]; + + let (WhichArgs { application, all }, _) = args.process(®istry).await?; + let external = application.starts_with('^'); + let item = if external { + application.item[1..].to_string() + } else { + application.item.clone() }; + if !external { + let builtin = registry.has(&item); + if builtin { + output.push(ReturnSuccess::value(entry_builtin!( + item, + application.tag.clone() + ))); + } + } + + if let Ok(paths) = ichwh::which_all(&item).await { + for path in paths { + output.push(ReturnSuccess::value(entry_path!( + item, + path.into(), + application.tag.clone() + ))); + } + } if all { - Ok(stream.to_output_stream()) + Ok(futures::stream::iter(output.into_iter()).to_output_stream()) } else { - Ok(stream.take(1).to_output_stream()) + Ok(futures::stream::iter(output.into_iter().take(1)).to_output_stream()) } } diff --git a/crates/nu-cli/src/commands/wrap.rs b/crates/nu-cli/src/commands/wrap.rs index 8f6ae3e3c..27ed89ebb 100644 --- a/crates/nu-cli/src/commands/wrap.rs +++ b/crates/nu-cli/src/commands/wrap.rs @@ -38,8 +38,7 @@ impl WholeStreamCommand for Wrap { args: CommandArgs, registry: &CommandRegistry, ) -> Result { - // args.process(registry, wrap)?.run() - wrap(args, registry) + wrap(args, registry).await } fn examples(&self) -> Vec { @@ -84,64 +83,57 @@ impl WholeStreamCommand for Wrap { } } -fn wrap(args: CommandArgs, registry: &CommandRegistry) -> Result { +async fn wrap(args: CommandArgs, registry: &CommandRegistry) -> Result { let registry = registry.clone(); - let stream = async_stream! { - let (WrapArgs { column }, mut input) = args.process(®istry).await?; - let mut result_table = vec![]; - let mut are_all_rows = true; + let (WrapArgs { column }, mut input) = args.process(®istry).await?; + let mut result_table = vec![]; + let mut are_all_rows = true; - while let Some(value) = input.next().await { - match value { - Value { - value: UntaggedValue::Row(_), - .. - } => { - result_table.push(value); - } - _ => { - are_all_rows = false; + while let Some(value) = input.next().await { + match value { + Value { + value: UntaggedValue::Row(_), + .. + } => { + result_table.push(value); + } + _ => { + are_all_rows = false; - let mut index_map = IndexMap::new(); - index_map.insert( - match &column { - Some(key) => key.item.clone(), - None => DEFAULT_COLUMN_NAME.to_string(), - }, - value, - ); - - result_table.push(UntaggedValue::row(index_map).into_value(Tag::unknown())); - } + let mut index_map = IndexMap::new(); + index_map.insert( + match &column { + Some(key) => key.item.clone(), + None => DEFAULT_COLUMN_NAME.to_string(), + }, + value, + ); + result_table.push(UntaggedValue::row(index_map).into_value(Tag::unknown())); } } + } - if are_all_rows { - let mut index_map = IndexMap::new(); - index_map.insert( - match &column { - Some(key) => key.item.clone(), - None => DEFAULT_COLUMN_NAME.to_string(), - }, - UntaggedValue::table(&result_table).into_value(Tag::unknown()), - ); + if are_all_rows { + let mut index_map = IndexMap::new(); + index_map.insert( + match &column { + Some(key) => key.item.clone(), + None => DEFAULT_COLUMN_NAME.to_string(), + }, + UntaggedValue::table(&result_table).into_value(Tag::unknown()), + ); - let row = UntaggedValue::row(index_map).into_untagged_value(); + let row = UntaggedValue::row(index_map).into_untagged_value(); - yield ReturnSuccess::value(row); - } else { - for item in result_table - .iter() - .map(|row| ReturnSuccess::value(row.clone())) { - - yield item; - } - } - }; - - Ok(stream.to_output_stream()) + Ok(OutputStream::one(ReturnSuccess::value(row))) + } else { + Ok( + futures::stream::iter(result_table.into_iter().map(ReturnSuccess::value)) + .to_output_stream(), + ) + } } #[cfg(test)] diff --git a/crates/nu-cli/src/data/config.rs b/crates/nu-cli/src/data/config.rs index 5bda300d1..fbd296f27 100644 --- a/crates/nu-cli/src/data/config.rs +++ b/crates/nu-cli/src/data/config.rs @@ -65,7 +65,11 @@ pub fn read( Some(ref file) => file.clone(), }; - touch(&filename)?; + if !filename.exists() && touch(&filename).is_err() { + // If we can't create configs, let's just return an empty indexmap instead as we may be in + // a readonly environment + return Ok(IndexMap::new()); + } trace!("config file = {}", filename.display()); diff --git a/crates/nu-cli/src/shell/filesystem_shell.rs b/crates/nu-cli/src/shell/filesystem_shell.rs index 141fd00ce..4001e9b00 100644 --- a/crates/nu-cli/src/shell/filesystem_shell.rs +++ b/crates/nu-cli/src/shell/filesystem_shell.rs @@ -562,54 +562,54 @@ impl Shell for FilesystemShell { )); } - let stream = async_stream! { - for (f, tag) in all_targets.iter() { + Ok( + futures::stream::iter(all_targets.into_iter().map(move |(f, tag)| { let is_empty = || match f.read_dir() { Ok(mut p) => p.next().is_none(), - Err(_) => false + Err(_) => false, }; if let Ok(metadata) = f.symlink_metadata() { - if metadata.is_file() || metadata.file_type().is_symlink() || recursive.item || is_empty() { + if metadata.is_file() + || metadata.file_type().is_symlink() + || recursive.item + || is_empty() + { let result; #[cfg(feature = "trash-support")] { - let rm_always_trash = config::config(Tag::unknown())?.get("rm_always_trash").map(|val| val.is_true()).unwrap_or(false); + let rm_always_trash = config::config(Tag::unknown())? + .get("rm_always_trash") + .map(|val| val.is_true()) + .unwrap_or(false); result = if _trash.item || (rm_always_trash && !_permanent.item) { - trash::remove(f) - .map_err(|e| f.to_string_lossy()) + trash::remove(&f).map_err(|_| f.to_string_lossy()) } else if metadata.is_file() { - std::fs::remove_file(f) - .map_err(|e| f.to_string_lossy()) + std::fs::remove_file(&f).map_err(|_| f.to_string_lossy()) } else { - std::fs::remove_dir_all(f) - .map_err(|e| f.to_string_lossy()) + std::fs::remove_dir_all(&f).map_err(|_| f.to_string_lossy()) }; } #[cfg(not(feature = "trash-support"))] { result = if metadata.is_file() { - std::fs::remove_file(f) - .map_err(|e| f.to_string_lossy()) + std::fs::remove_file(&f).map_err(|_| f.to_string_lossy()) } else { - std::fs::remove_dir_all(f) - .map_err(|e| f.to_string_lossy()) + std::fs::remove_dir_all(&f).map_err(|_| f.to_string_lossy()) }; } if let Err(e) = result { let msg = format!("Could not delete {:}", e); - yield Err(ShellError::labeled_error(msg, e, tag)) + Err(ShellError::labeled_error(msg, e, tag)) } else { let val = format!("deleted {:}", f.to_string_lossy()).into(); - yield Ok(ReturnSuccess::Value(val)) + Ok(ReturnSuccess::Value(val)) } } else { - let msg = format!( - "Cannot remove {:}. try --recursive", - f.to_string_lossy() - ); - yield Err(ShellError::labeled_error( + let msg = + format!("Cannot remove {:}. try --recursive", f.to_string_lossy()); + Err(ShellError::labeled_error( msg, "cannot remove non-empty directory", tag, @@ -617,16 +617,15 @@ impl Shell for FilesystemShell { } } else { let msg = format!("no such file or directory: {:}", f.to_string_lossy()); - yield Err(ShellError::labeled_error( + Err(ShellError::labeled_error( msg, "no such file or directory", tag, )) } - } - }; - - Ok(stream.to_output_stream()) + })) + .to_output_stream(), + ) } fn path(&self) -> String { diff --git a/crates/nu-cli/tests/commands/enter.rs b/crates/nu-cli/tests/commands/enter.rs index 2bc87e74f..bbacd5a1f 100644 --- a/crates/nu-cli/tests/commands/enter.rs +++ b/crates/nu-cli/tests/commands/enter.rs @@ -80,7 +80,7 @@ fn errors_if_file_not_found() { "enter i_dont_exist.csv" ); - assert!(actual.err.contains("File could not be opened")); + //assert!(actual.err.contains("File could not be opened")); assert!(actual.err.contains("file not found")); }) } diff --git a/crates/nu-cli/tests/commands/math.rs b/crates/nu-cli/tests/commands/math.rs index 9822b48f9..680b1b641 100644 --- a/crates/nu-cli/tests/commands/math.rs +++ b/crates/nu-cli/tests/commands/math.rs @@ -84,6 +84,18 @@ fn division_of_ints2() { assert_eq!(actual.out, "0.25"); } +#[test] +fn proper_precedence_history() { + let actual = nu!( + cwd: "tests/fixtures/formats", pipeline( + r#" + = 2 / 2 / 2 + 1 + "# + )); + + assert_eq!(actual.out, "1.5"); +} + #[test] fn parens_precedence() { let actual = nu!( diff --git a/crates/nu-cli/tests/commands/open.rs b/crates/nu-cli/tests/commands/open.rs index b009f9821..b24eb1dca 100644 --- a/crates/nu-cli/tests/commands/open.rs +++ b/crates/nu-cli/tests/commands/open.rs @@ -225,6 +225,6 @@ fn errors_if_file_not_found() { "open i_dont_exist.txt" ); - assert!(actual.err.contains("File could not be opened")); - assert!(actual.err.contains("file not found")); + //assert!(actual.err.contains("File could not be opened")); + assert!(actual.err.contains("Cannot open")); } diff --git a/crates/nu-parser/src/parse.rs b/crates/nu-parser/src/parse.rs index 3b3ddcd79..39024f983 100644 --- a/crates/nu-parser/src/parse.rs +++ b/crates/nu-parser/src/parse.rs @@ -861,10 +861,8 @@ fn parse_math_expression( shorthand_mode: bool, ) -> (usize, SpannedExpression, Option) { // Precedence parsing is included - // Some notes: - // * short_hand mode means that the left-hand side of an expression can point to a column-path. To make this possible, - // we parse as normal, but then go back and when we detect a left-hand side, reparse that value if it's a string - // * parens are handled earlier, so they're not handled explicitly here + // Short_hand mode means that the left-hand side of an expression can point to a column-path. To make this possible, + // we parse as normal, but then go back and when we detect a left-hand side, reparse that value if it's a string let mut idx = 0; let mut error = None; @@ -948,6 +946,7 @@ fn parse_math_expression( } working_exprs.push((None, op)); working_exprs.push(rhs_working_expr); + prec.push(next_prec); } idx += 1; diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index 9852c8941..acef9d86c 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -339,6 +339,14 @@ impl Value { } } + /// View the Value as a Primitive value, if possible + pub fn is_primitive(&self) -> bool { + match &self.value { + UntaggedValue::Primitive(_) => true, + _ => false, + } + } + /// View the Value as unsigned 64-bit, if possible pub fn as_u64(&self) -> Result { match &self.value {