From 55edef5ddaf3d3d55290863446c2dd50c012e9bc Mon Sep 17 00:00:00 2001 From: Antoine Stevan <44101798+amtoine@users.noreply.github.com> Date: Fri, 19 Apr 2024 13:54:16 +0200 Subject: [PATCH] create `nuon` crate from `from nuon` and `to nuon` (#12553) # Description playing with the NUON format in Rust code in some plugins, we agreed with the team it was a great time to create a standalone NUON format to allow Rust devs to use this Nushell file format. > **Note** > this PR almost copy-pastes the code from `nu_commands/src/formats/from/nuon.rs` and `nu_commands/src/formats/to/nuon.rs` to `nuon/src/from.rs` and `nuon/src/to.rs`, with minor tweaks to make then standalone functions, e.g. remove the rest of the command implementations ### TODO - [x] add tests - [x] add documentation # User-Facing Changes devs will have access to a new crate, `nuon`, and two functions, `from_nuon` and `to_nuon` ```rust from_nuon( input: &str, span: Option, ) -> Result ``` ```rust to_nuon( input: &Value, raw: bool, tabs: Option, indent: Option, span: Option, ) -> Result ``` # Tests + Formatting i've basically taken all the tests from `crates/nu-command/tests/format_conversions/nuon.rs` and converted them to use `from_nuon` and `to_nuon` instead of Nushell commands - i've created a `nuon_end_to_end` to run both conversions with an optional middle value to check that all is fine > **Note** > the `nuon::tests::read_code_should_fail_rather_than_panic` test does give different results locally and in the CI... > i've left it ignored with comments to help future us :) # After Submitting mention that in the release notes for sure!! --- Cargo.lock | 13 + Cargo.toml | 1 + crates/nu-command/Cargo.toml | 1 + crates/nu-command/src/filters/uniq.rs | 3 +- crates/nu-command/src/formats/from/nuon.rs | 455 +---------------- crates/nu-command/src/formats/to/mod.rs | 1 - crates/nu-command/src/formats/to/nuon.rs | 259 +--------- .../tests/commands/database/into_sqlite.rs | 2 +- crates/nuon/Cargo.toml | 20 + crates/nuon/LICENSE | 21 + crates/nuon/src/from.rs | 465 ++++++++++++++++++ crates/nuon/src/lib.rs | 430 ++++++++++++++++ crates/nuon/src/to.rs | 283 +++++++++++ 13 files changed, 1240 insertions(+), 714 deletions(-) create mode 100644 crates/nuon/Cargo.toml create mode 100644 crates/nuon/LICENSE create mode 100644 crates/nuon/src/from.rs create mode 100644 crates/nuon/src/lib.rs create mode 100644 crates/nuon/src/to.rs diff --git a/Cargo.lock b/Cargo.lock index fee1981d32..63e890c002 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3068,6 +3068,7 @@ dependencies = [ "nu-utils", "num-format", "num-traits", + "nuon", "once_cell", "open", "os_pipe", @@ -3579,6 +3580,18 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "nuon" +version = "0.92.3" +dependencies = [ + "chrono", + "fancy-regex", + "nu-engine", + "nu-parser", + "nu-protocol", + "once_cell", +] + [[package]] name = "objc" version = "0.2.7" diff --git a/Cargo.toml b/Cargo.toml index bd616d8b3e..1e60a88ee6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ members = [ "crates/nu-term-grid", "crates/nu-test-support", "crates/nu-utils", + "crates/nuon", ] [workspace.dependencies] diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 7eea40b83c..83ec9ec838 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -27,6 +27,7 @@ nu-table = { path = "../nu-table", version = "0.92.3" } nu-term-grid = { path = "../nu-term-grid", version = "0.92.3" } nu-utils = { path = "../nu-utils", version = "0.92.3" } nu-ansi-term = { workspace = true } +nuon = { path = "../nuon", version = "0.92.3" } alphanumeric-sort = { workspace = true } base64 = { workspace = true } diff --git a/crates/nu-command/src/filters/uniq.rs b/crates/nu-command/src/filters/uniq.rs index e8ac53df52..1734b4dc10 100644 --- a/crates/nu-command/src/filters/uniq.rs +++ b/crates/nu-command/src/filters/uniq.rs @@ -1,4 +1,3 @@ -use crate::formats::value_to_string; use itertools::Itertools; use nu_engine::command_prelude::*; use nu_protocol::PipelineMetadata; @@ -216,7 +215,7 @@ fn sort_attributes(val: Value) -> Value { fn generate_key(item: &ValueCounter) -> Result { let value = sort_attributes(item.val_to_compare.clone()); //otherwise, keys could be different for Records - value_to_string(&value, Span::unknown(), 0, None) + nuon::to_nuon(&value, true, None, None, Some(Span::unknown())) } fn generate_results_with_count(head: Span, uniq_values: Vec) -> Vec { diff --git a/crates/nu-command/src/formats/from/nuon.rs b/crates/nu-command/src/formats/from/nuon.rs index 70919200db..bfffe7e5b4 100644 --- a/crates/nu-command/src/formats/from/nuon.rs +++ b/crates/nu-command/src/formats/from/nuon.rs @@ -1,10 +1,4 @@ use nu_engine::command_prelude::*; -use nu_protocol::{ - ast::{Expr, Expression, ListItem, RecordItem}, - engine::StateWorkingSet, - Range, Unit, -}; -use std::sync::Arc; #[derive(Clone)] pub struct FromNuon; @@ -46,7 +40,7 @@ impl Command for FromNuon { fn run( &self, - engine_state: &EngineState, + _engine_state: &EngineState, _stack: &mut Stack, call: &Call, input: PipelineData, @@ -54,98 +48,7 @@ impl Command for FromNuon { let head = call.head; let (string_input, _span, metadata) = input.collect_string_strict(head)?; - let engine_state = engine_state.clone(); - - let mut working_set = StateWorkingSet::new(&engine_state); - - let mut block = nu_parser::parse(&mut working_set, None, string_input.as_bytes(), false); - - if let Some(pipeline) = block.pipelines.get(1) { - if let Some(element) = pipeline.elements.first() { - return Err(ShellError::GenericError { - error: "error when loading nuon text".into(), - msg: "could not load nuon text".into(), - span: Some(head), - help: None, - inner: vec![ShellError::OutsideSpannedLabeledError { - src: string_input, - error: "error when loading".into(), - msg: "excess values when loading".into(), - span: element.expr.span, - }], - }); - } else { - return Err(ShellError::GenericError { - error: "error when loading nuon text".into(), - msg: "could not load nuon text".into(), - span: Some(head), - help: None, - inner: vec![ShellError::GenericError { - error: "error when loading".into(), - msg: "excess values when loading".into(), - span: Some(head), - help: None, - inner: vec![], - }], - }); - } - } - - let expr = if block.pipelines.is_empty() { - Expression { - expr: Expr::Nothing, - span: head, - custom_completion: None, - ty: Type::Nothing, - } - } else { - let mut pipeline = Arc::make_mut(&mut block).pipelines.remove(0); - - if let Some(expr) = pipeline.elements.get(1) { - return Err(ShellError::GenericError { - error: "error when loading nuon text".into(), - msg: "could not load nuon text".into(), - span: Some(head), - help: None, - inner: vec![ShellError::OutsideSpannedLabeledError { - src: string_input, - error: "error when loading".into(), - msg: "detected a pipeline in nuon file".into(), - span: expr.expr.span, - }], - }); - } - - if pipeline.elements.is_empty() { - Expression { - expr: Expr::Nothing, - span: head, - custom_completion: None, - ty: Type::Nothing, - } - } else { - pipeline.elements.remove(0).expr - } - }; - - if let Some(err) = working_set.parse_errors.first() { - return Err(ShellError::GenericError { - error: "error when parsing nuon text".into(), - msg: "could not parse nuon text".into(), - span: Some(head), - help: None, - inner: vec![ShellError::OutsideSpannedLabeledError { - src: string_input, - error: "error when parsing".into(), - msg: err.to_string(), - span: err.span(), - }], - }); - } - - let result = convert_to_value(expr, head, &string_input); - - match result { + match nuon::from_nuon(&string_input, Some(head)) { Ok(result) => Ok(result.into_pipeline_data_with_metadata(metadata)), Err(err) => Err(ShellError::GenericError { error: "error when loading nuon text".into(), @@ -158,360 +61,6 @@ impl Command for FromNuon { } } -fn convert_to_value( - expr: Expression, - span: Span, - original_text: &str, -) -> Result { - match expr.expr { - Expr::BinaryOp(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "binary operators not supported in nuon".into(), - span: expr.span, - }), - Expr::UnaryNot(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "unary operators not supported in nuon".into(), - span: expr.span, - }), - Expr::Block(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "blocks not supported in nuon".into(), - span: expr.span, - }), - Expr::Closure(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "closures not supported in nuon".into(), - span: expr.span, - }), - Expr::Binary(val) => Ok(Value::binary(val, span)), - Expr::Bool(val) => Ok(Value::bool(val, span)), - Expr::Call(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "calls not supported in nuon".into(), - span: expr.span, - }), - Expr::CellPath(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "subexpressions and cellpaths not supported in nuon".into(), - span: expr.span, - }), - Expr::DateTime(dt) => Ok(Value::date(dt, span)), - Expr::ExternalCall(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "calls not supported in nuon".into(), - span: expr.span, - }), - Expr::Filepath(val, _) => Ok(Value::string(val, span)), - Expr::Directory(val, _) => Ok(Value::string(val, span)), - Expr::Float(val) => Ok(Value::float(val, span)), - Expr::FullCellPath(full_cell_path) => { - if !full_cell_path.tail.is_empty() { - Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "subexpressions and cellpaths not supported in nuon".into(), - span: expr.span, - }) - } else { - convert_to_value(full_cell_path.head, span, original_text) - } - } - - Expr::Garbage => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "extra tokens in input file".into(), - span: expr.span, - }), - Expr::GlobPattern(val, _) => Ok(Value::string(val, span)), - Expr::ImportPattern(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "imports not supported in nuon".into(), - span: expr.span, - }), - Expr::Overlay(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "overlays not supported in nuon".into(), - span: expr.span, - }), - Expr::Int(val) => Ok(Value::int(val, span)), - Expr::Keyword(kw, ..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: format!("{} not supported in nuon", String::from_utf8_lossy(&kw)), - span: expr.span, - }), - Expr::List(vals) => { - let mut output = vec![]; - - for item in vals { - match item { - ListItem::Item(expr) => { - output.push(convert_to_value(expr, span, original_text)?); - } - ListItem::Spread(_, inner) => { - return Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "spread operator not supported in nuon".into(), - span: inner.span, - }); - } - } - } - - Ok(Value::list(output, span)) - } - Expr::MatchBlock(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "match blocks not supported in nuon".into(), - span: expr.span, - }), - Expr::Nothing => Ok(Value::nothing(span)), - Expr::Operator(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "operators not supported in nuon".into(), - span: expr.span, - }), - Expr::Range(from, next, to, operator) => { - let from = if let Some(f) = from { - convert_to_value(*f, span, original_text)? - } else { - Value::nothing(expr.span) - }; - - let next = if let Some(s) = next { - convert_to_value(*s, span, original_text)? - } else { - Value::nothing(expr.span) - }; - - let to = if let Some(t) = to { - convert_to_value(*t, span, original_text)? - } else { - Value::nothing(expr.span) - }; - - Ok(Value::range( - Range::new(from, next, to, operator.inclusion, expr.span)?, - expr.span, - )) - } - Expr::Record(key_vals) => { - let mut record = Record::with_capacity(key_vals.len()); - let mut key_spans = Vec::with_capacity(key_vals.len()); - - for key_val in key_vals { - match key_val { - RecordItem::Pair(key, val) => { - let key_str = match key.expr { - Expr::String(key_str) => key_str, - _ => { - return Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "only strings can be keys".into(), - span: key.span, - }) - } - }; - - if let Some(i) = record.index_of(&key_str) { - return Err(ShellError::ColumnDefinedTwice { - col_name: key_str, - second_use: key.span, - first_use: key_spans[i], - }); - } else { - key_spans.push(key.span); - record.push(key_str, convert_to_value(val, span, original_text)?); - } - } - RecordItem::Spread(_, inner) => { - return Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "spread operator not supported in nuon".into(), - span: inner.span, - }); - } - } - } - - Ok(Value::record(record, span)) - } - Expr::RowCondition(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "row conditions not supported in nuon".into(), - span: expr.span, - }), - Expr::Signature(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "signatures not supported in nuon".into(), - span: expr.span, - }), - Expr::String(s) => Ok(Value::string(s, span)), - Expr::StringInterpolation(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "string interpolation not supported in nuon".into(), - span: expr.span, - }), - Expr::Subexpression(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "subexpressions not supported in nuon".into(), - span: expr.span, - }), - Expr::Table(mut headers, cells) => { - let mut cols = vec![]; - - let mut output = vec![]; - - for key in headers.iter_mut() { - let key_str = match &mut key.expr { - Expr::String(key_str) => key_str, - _ => { - return Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "only strings can be keys".into(), - span: expr.span, - }) - } - }; - - if let Some(idx) = cols.iter().position(|existing| existing == key_str) { - return Err(ShellError::ColumnDefinedTwice { - col_name: key_str.clone(), - second_use: key.span, - first_use: headers[idx].span, - }); - } else { - cols.push(std::mem::take(key_str)); - } - } - - for row in cells { - if cols.len() != row.len() { - return Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "table has mismatched columns".into(), - span: expr.span, - }); - } - - let record = cols - .iter() - .zip(row) - .map(|(col, cell)| { - convert_to_value(cell, span, original_text).map(|val| (col.clone(), val)) - }) - .collect::>()?; - - output.push(Value::record(record, span)); - } - - Ok(Value::list(output, span)) - } - Expr::ValueWithUnit(val, unit) => { - let size = match val.expr { - Expr::Int(val) => val, - _ => { - return Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "non-integer unit value".into(), - span: expr.span, - }) - } - }; - - match unit.item { - Unit::Byte => Ok(Value::filesize(size, span)), - Unit::Kilobyte => Ok(Value::filesize(size * 1000, span)), - Unit::Megabyte => Ok(Value::filesize(size * 1000 * 1000, span)), - Unit::Gigabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000, span)), - Unit::Terabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000 * 1000, span)), - Unit::Petabyte => Ok(Value::filesize( - size * 1000 * 1000 * 1000 * 1000 * 1000, - span, - )), - Unit::Exabyte => Ok(Value::filesize( - size * 1000 * 1000 * 1000 * 1000 * 1000 * 1000, - span, - )), - - Unit::Kibibyte => Ok(Value::filesize(size * 1024, span)), - Unit::Mebibyte => Ok(Value::filesize(size * 1024 * 1024, span)), - Unit::Gibibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024, span)), - Unit::Tebibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024 * 1024, span)), - Unit::Pebibyte => Ok(Value::filesize( - size * 1024 * 1024 * 1024 * 1024 * 1024, - span, - )), - Unit::Exbibyte => Ok(Value::filesize( - size * 1024 * 1024 * 1024 * 1024 * 1024 * 1024, - span, - )), - - Unit::Nanosecond => Ok(Value::duration(size, span)), - Unit::Microsecond => Ok(Value::duration(size * 1000, span)), - Unit::Millisecond => Ok(Value::duration(size * 1000 * 1000, span)), - Unit::Second => Ok(Value::duration(size * 1000 * 1000 * 1000, span)), - Unit::Minute => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60, span)), - Unit::Hour => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60 * 60, span)), - Unit::Day => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24) { - Some(val) => Ok(Value::duration(val, span)), - None => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "day duration too large".into(), - msg: "day duration too large".into(), - span: expr.span, - }), - }, - - Unit::Week => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24 * 7) { - Some(val) => Ok(Value::duration(val, span)), - None => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "week duration too large".into(), - msg: "week duration too large".into(), - span: expr.span, - }), - }, - } - } - Expr::Var(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "variables not supported in nuon".into(), - span: expr.span, - }), - Expr::VarDecl(..) => Err(ShellError::OutsideSpannedLabeledError { - src: original_text.to_string(), - error: "Error when loading".into(), - msg: "variable declarations not supported in nuon".into(), - span: expr.span, - }), - } -} - #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/formats/to/mod.rs b/crates/nu-command/src/formats/to/mod.rs index c9e7ad0cd8..b49801c75f 100644 --- a/crates/nu-command/src/formats/to/mod.rs +++ b/crates/nu-command/src/formats/to/mod.rs @@ -15,7 +15,6 @@ pub use self::toml::ToToml; pub use command::To; pub use json::ToJson; pub use md::ToMd; -pub use nuon::value_to_string; pub use nuon::ToNuon; pub use text::ToText; pub use tsv::ToTsv; diff --git a/crates/nu-command/src/formats/to/nuon.rs b/crates/nu-command/src/formats/to/nuon.rs index f1b0c5ccf5..6352ade30a 100644 --- a/crates/nu-command/src/formats/to/nuon.rs +++ b/crates/nu-command/src/formats/to/nuon.rs @@ -1,10 +1,4 @@ -use core::fmt::Write; -use fancy_regex::Regex; -use nu_engine::{command_prelude::*, get_columns}; -use nu_parser::escape_quote_string; -use nu_protocol::Range; -use once_cell::sync::Lazy; -use std::ops::Bound; +use nu_engine::command_prelude::*; #[derive(Clone)] pub struct ToNuon; @@ -55,17 +49,7 @@ impl Command for ToNuon { let span = call.head; let value = input.into_value(span); - let nuon_result = if raw { - value_to_string(&value, span, 0, None) - } else if let Some(tab_count) = tabs { - value_to_string(&value, span, 0, Some(&"\t".repeat(tab_count))) - } else if let Some(indent) = indent { - value_to_string(&value, span, 0, Some(&" ".repeat(indent))) - } else { - value_to_string(&value, span, 0, None) - }; - - match nuon_result { + match nuon::to_nuon(&value, raw, tabs, indent, Some(span)) { Ok(serde_nuon_string) => { Ok(Value::string(serde_nuon_string, span).into_pipeline_data()) } @@ -108,245 +92,6 @@ impl Command for ToNuon { } } -pub fn value_to_string( - v: &Value, - span: Span, - depth: usize, - indent: Option<&str>, -) -> Result { - let (nl, sep) = get_true_separators(indent); - let idt = get_true_indentation(depth, indent); - let idt_po = get_true_indentation(depth + 1, indent); - let idt_pt = get_true_indentation(depth + 2, indent); - - match v { - Value::Binary { val, .. } => { - let mut s = String::with_capacity(2 * val.len()); - for byte in val { - if write!(s, "{byte:02X}").is_err() { - return Err(ShellError::UnsupportedInput { - msg: "could not convert binary to string".into(), - input: "value originates from here".into(), - msg_span: span, - input_span: v.span(), - }); - } - } - Ok(format!("0x[{s}]")) - } - Value::Block { .. } => Err(ShellError::UnsupportedInput { - msg: "blocks are currently not nuon-compatible".into(), - input: "value originates from here".into(), - msg_span: span, - input_span: v.span(), - }), - Value::Closure { .. } => Err(ShellError::UnsupportedInput { - msg: "closures are currently not nuon-compatible".into(), - input: "value originates from here".into(), - msg_span: span, - input_span: v.span(), - }), - Value::Bool { val, .. } => { - if *val { - Ok("true".to_string()) - } else { - Ok("false".to_string()) - } - } - Value::CellPath { .. } => Err(ShellError::UnsupportedInput { - msg: "cell-paths are currently not nuon-compatible".to_string(), - input: "value originates from here".into(), - msg_span: span, - input_span: v.span(), - }), - Value::Custom { .. } => Err(ShellError::UnsupportedInput { - msg: "custom values are currently not nuon-compatible".to_string(), - input: "value originates from here".into(), - msg_span: span, - input_span: v.span(), - }), - Value::Date { val, .. } => Ok(val.to_rfc3339()), - // FIXME: make durations use the shortest lossless representation. - Value::Duration { val, .. } => Ok(format!("{}ns", *val)), - // Propagate existing errors - Value::Error { error, .. } => Err(*error.clone()), - // FIXME: make filesizes use the shortest lossless representation. - Value::Filesize { val, .. } => Ok(format!("{}b", *val)), - Value::Float { val, .. } => { - // This serialises these as 'nan', 'inf' and '-inf', respectively. - if &val.round() == val && val.is_finite() { - Ok(format!("{}.0", *val)) - } else { - Ok(format!("{}", *val)) - } - } - Value::Int { val, .. } => Ok(format!("{}", *val)), - Value::List { vals, .. } => { - let headers = get_columns(vals); - if !headers.is_empty() && vals.iter().all(|x| x.columns().eq(headers.iter())) { - // Table output - let headers: Vec = headers - .iter() - .map(|string| { - if needs_quotes(string) { - format!("{idt}\"{string}\"") - } else { - format!("{idt}{string}") - } - }) - .collect(); - let headers_output = headers.join(&format!(",{sep}{nl}{idt_pt}")); - - let mut table_output = vec![]; - for val in vals { - let mut row = vec![]; - - if let Value::Record { val, .. } = val { - for val in val.values() { - row.push(value_to_string_without_quotes( - val, - span, - depth + 2, - indent, - )?); - } - } - - table_output.push(row.join(&format!(",{sep}{nl}{idt_pt}"))); - } - - Ok(format!( - "[{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}];{sep}{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}]{nl}{idt}]", - headers_output, - table_output.join(&format!("{nl}{idt_po}],{sep}{nl}{idt_po}[{nl}{idt_pt}")) - )) - } else { - let mut collection = vec![]; - for val in vals { - collection.push(format!( - "{idt_po}{}", - value_to_string_without_quotes(val, span, depth + 1, indent,)? - )); - } - Ok(format!( - "[{nl}{}{nl}{idt}]", - collection.join(&format!(",{sep}{nl}")) - )) - } - } - Value::Nothing { .. } => Ok("null".to_string()), - Value::Range { val, .. } => match val { - Range::IntRange(range) => Ok(range.to_string()), - Range::FloatRange(range) => { - let start = - value_to_string(&Value::float(range.start(), span), span, depth + 1, indent)?; - match range.end() { - Bound::Included(end) => Ok(format!( - "{}..{}", - start, - value_to_string(&Value::float(end, span), span, depth + 1, indent)? - )), - Bound::Excluded(end) => Ok(format!( - "{}..<{}", - start, - value_to_string(&Value::float(end, span), span, depth + 1, indent)? - )), - Bound::Unbounded => Ok(format!("{start}..",)), - } - } - }, - Value::Record { val, .. } => { - let mut collection = vec![]; - for (col, val) in &**val { - collection.push(if needs_quotes(col) { - format!( - "{idt_po}\"{}\": {}", - col, - value_to_string_without_quotes(val, span, depth + 1, indent)? - ) - } else { - format!( - "{idt_po}{}: {}", - col, - value_to_string_without_quotes(val, span, depth + 1, indent)? - ) - }); - } - Ok(format!( - "{{{nl}{}{nl}{idt}}}", - collection.join(&format!(",{sep}{nl}")) - )) - } - Value::LazyRecord { val, .. } => { - let collected = val.collect()?; - value_to_string(&collected, span, depth + 1, indent) - } - // All strings outside data structures are quoted because they are in 'command position' - // (could be mistaken for commands by the Nu parser) - Value::String { val, .. } => Ok(escape_quote_string(val)), - Value::Glob { val, .. } => Ok(escape_quote_string(val)), - } -} - -fn get_true_indentation(depth: usize, indent: Option<&str>) -> String { - match indent { - Some(i) => i.repeat(depth), - None => "".to_string(), - } -} - -fn get_true_separators(indent: Option<&str>) -> (String, String) { - match indent { - Some(_) => ("\n".to_string(), "".to_string()), - None => ("".to_string(), " ".to_string()), - } -} - -fn value_to_string_without_quotes( - v: &Value, - span: Span, - depth: usize, - indent: Option<&str>, -) -> Result { - match v { - Value::String { val, .. } => Ok({ - if needs_quotes(val) { - escape_quote_string(val) - } else { - val.clone() - } - }), - _ => value_to_string(v, span, depth, indent), - } -} - -// This hits, in order: -// • Any character of []:`{}#'";()|$, -// • Any digit (\d) -// • Any whitespace (\s) -// • Case-insensitive sign-insensitive float "keywords" inf, infinity and nan. -static NEEDS_QUOTES_REGEX: Lazy = Lazy::new(|| { - Regex::new(r#"[\[\]:`\{\}#'";\(\)\|\$,\d\s]|(?i)^[+\-]?(inf(inity)?|nan)$"#) - .expect("internal error: NEEDS_QUOTES_REGEX didn't compile") -}); - -fn needs_quotes(string: &str) -> bool { - if string.is_empty() { - return true; - } - // These are case-sensitive keywords - match string { - // `true`/`false`/`null` are active keywords in JSON and NUON - // `&&` is denied by the nu parser for diagnostics reasons - // (https://github.com/nushell/nushell/pull/7241) - // TODO: remove the extra check in the nuon codepath - "true" | "false" | "null" | "&&" => return true, - _ => (), - }; - // All other cases are handled here - NEEDS_QUOTES_REGEX.is_match(string).unwrap_or(false) -} - #[cfg(test)] mod test { #[test] diff --git a/crates/nu-command/tests/commands/database/into_sqlite.rs b/crates/nu-command/tests/commands/database/into_sqlite.rs index e70e5b5528..e30d7d2f8f 100644 --- a/crates/nu-command/tests/commands/database/into_sqlite.rs +++ b/crates/nu-command/tests/commands/database/into_sqlite.rs @@ -330,7 +330,7 @@ fn into_sqlite_big_insert() { ) .unwrap(); - let nuon = nu_command::value_to_string(&value, Span::unknown(), 0, None).unwrap() + let nuon = nuon::to_nuon(&value, true, None, None, Some(Span::unknown())).unwrap() + &line_ending(); nuon_file.write_all(nuon.as_bytes()).unwrap(); diff --git a/crates/nuon/Cargo.toml b/crates/nuon/Cargo.toml new file mode 100644 index 0000000000..54217e61b5 --- /dev/null +++ b/crates/nuon/Cargo.toml @@ -0,0 +1,20 @@ +[package] +authors = ["The Nushell Project Developers"] +description = "Support for the NUON format." +repository = "https://github.com/nushell/nushell/tree/main/crates/nuon" +edition = "2021" +license = "MIT" +name = "nuon" +version = "0.92.3" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nu-parser = { path = "../nu-parser", version = "0.92.3" } +nu-protocol = { path = "../nu-protocol", version = "0.92.3" } +nu-engine = { path = "../nu-engine", version = "0.92.3" } +once_cell = { workspace = true } +fancy-regex = { workspace = true } + +[dev-dependencies] +chrono = { workspace = true } diff --git a/crates/nuon/LICENSE b/crates/nuon/LICENSE new file mode 100644 index 0000000000..ae174e8595 --- /dev/null +++ b/crates/nuon/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 - 2023 The Nushell Project Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/nuon/src/from.rs b/crates/nuon/src/from.rs new file mode 100644 index 0000000000..b9b0427d0c --- /dev/null +++ b/crates/nuon/src/from.rs @@ -0,0 +1,465 @@ +use nu_protocol::{ + ast::{Expr, Expression, ListItem, RecordItem}, + engine::{EngineState, StateWorkingSet}, + Range, Record, ShellError, Span, Type, Unit, Value, +}; +use std::sync::Arc; + +/// convert a raw string representation of NUON data to an actual Nushell [`Value`] +/// +/// > **Note** +/// > [`Span`] can be passed to [`from_nuon`] if there is context available to the caller, e.g. when +/// > using this function in a command implementation such as +/// [`from nuon`](https://www.nushell.sh/commands/docs/from_nuon.html). +/// +/// also see [`super::to_nuon`] for the inverse operation +pub fn from_nuon(input: &str, span: Option) -> Result { + let mut engine_state = EngineState::default(); + // NOTE: the parser needs `$env.PWD` to be set, that's a know _API issue_ with the + // [`EngineState`] + engine_state.add_env_var("PWD".to_string(), Value::string("", Span::unknown())); + let mut working_set = StateWorkingSet::new(&engine_state); + + let mut block = nu_parser::parse(&mut working_set, None, input.as_bytes(), false); + + if let Some(pipeline) = block.pipelines.get(1) { + if let Some(element) = pipeline.elements.first() { + return Err(ShellError::GenericError { + error: "error when loading nuon text".into(), + msg: "could not load nuon text".into(), + span, + help: None, + inner: vec![ShellError::OutsideSpannedLabeledError { + src: input.to_string(), + error: "error when loading".into(), + msg: "excess values when loading".into(), + span: element.expr.span, + }], + }); + } else { + return Err(ShellError::GenericError { + error: "error when loading nuon text".into(), + msg: "could not load nuon text".into(), + span, + help: None, + inner: vec![ShellError::GenericError { + error: "error when loading".into(), + msg: "excess values when loading".into(), + span, + help: None, + inner: vec![], + }], + }); + } + } + + let expr = if block.pipelines.is_empty() { + Expression { + expr: Expr::Nothing, + span: span.unwrap_or(Span::unknown()), + custom_completion: None, + ty: Type::Nothing, + } + } else { + let mut pipeline = Arc::make_mut(&mut block).pipelines.remove(0); + + if let Some(expr) = pipeline.elements.get(1) { + return Err(ShellError::GenericError { + error: "error when loading nuon text".into(), + msg: "could not load nuon text".into(), + span, + help: None, + inner: vec![ShellError::OutsideSpannedLabeledError { + src: input.to_string(), + error: "error when loading".into(), + msg: "detected a pipeline in nuon file".into(), + span: expr.expr.span, + }], + }); + } + + if pipeline.elements.is_empty() { + Expression { + expr: Expr::Nothing, + span: span.unwrap_or(Span::unknown()), + custom_completion: None, + ty: Type::Nothing, + } + } else { + pipeline.elements.remove(0).expr + } + }; + + if let Some(err) = working_set.parse_errors.first() { + return Err(ShellError::GenericError { + error: "error when parsing nuon text".into(), + msg: "could not parse nuon text".into(), + span, + help: None, + inner: vec![ShellError::OutsideSpannedLabeledError { + src: input.to_string(), + error: "error when parsing".into(), + msg: err.to_string(), + span: err.span(), + }], + }); + } + + let value = convert_to_value(expr, span.unwrap_or(Span::unknown()), input)?; + + Ok(value) +} + +fn convert_to_value( + expr: Expression, + span: Span, + original_text: &str, +) -> Result { + match expr.expr { + Expr::BinaryOp(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "binary operators not supported in nuon".into(), + span: expr.span, + }), + Expr::UnaryNot(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "unary operators not supported in nuon".into(), + span: expr.span, + }), + Expr::Block(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "blocks not supported in nuon".into(), + span: expr.span, + }), + Expr::Closure(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "closures not supported in nuon".into(), + span: expr.span, + }), + Expr::Binary(val) => Ok(Value::binary(val, span)), + Expr::Bool(val) => Ok(Value::bool(val, span)), + Expr::Call(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "calls not supported in nuon".into(), + span: expr.span, + }), + Expr::CellPath(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "subexpressions and cellpaths not supported in nuon".into(), + span: expr.span, + }), + Expr::DateTime(dt) => Ok(Value::date(dt, span)), + Expr::ExternalCall(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "calls not supported in nuon".into(), + span: expr.span, + }), + Expr::Filepath(val, _) => Ok(Value::string(val, span)), + Expr::Directory(val, _) => Ok(Value::string(val, span)), + Expr::Float(val) => Ok(Value::float(val, span)), + Expr::FullCellPath(full_cell_path) => { + if !full_cell_path.tail.is_empty() { + Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "subexpressions and cellpaths not supported in nuon".into(), + span: expr.span, + }) + } else { + convert_to_value(full_cell_path.head, span, original_text) + } + } + + Expr::Garbage => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "extra tokens in input file".into(), + span: expr.span, + }), + Expr::GlobPattern(val, _) => Ok(Value::string(val, span)), + Expr::ImportPattern(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "imports not supported in nuon".into(), + span: expr.span, + }), + Expr::Overlay(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "overlays not supported in nuon".into(), + span: expr.span, + }), + Expr::Int(val) => Ok(Value::int(val, span)), + Expr::Keyword(kw, ..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: format!("{} not supported in nuon", String::from_utf8_lossy(&kw)), + span: expr.span, + }), + Expr::List(vals) => { + let mut output = vec![]; + + for item in vals { + match item { + ListItem::Item(expr) => { + output.push(convert_to_value(expr, span, original_text)?); + } + ListItem::Spread(_, inner) => { + return Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "spread operator not supported in nuon".into(), + span: inner.span, + }); + } + } + } + + Ok(Value::list(output, span)) + } + Expr::MatchBlock(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "match blocks not supported in nuon".into(), + span: expr.span, + }), + Expr::Nothing => Ok(Value::nothing(span)), + Expr::Operator(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "operators not supported in nuon".into(), + span: expr.span, + }), + Expr::Range(from, next, to, operator) => { + let from = if let Some(f) = from { + convert_to_value(*f, span, original_text)? + } else { + Value::nothing(expr.span) + }; + + let next = if let Some(s) = next { + convert_to_value(*s, span, original_text)? + } else { + Value::nothing(expr.span) + }; + + let to = if let Some(t) = to { + convert_to_value(*t, span, original_text)? + } else { + Value::nothing(expr.span) + }; + + Ok(Value::range( + Range::new(from, next, to, operator.inclusion, expr.span)?, + expr.span, + )) + } + Expr::Record(key_vals) => { + let mut record = Record::with_capacity(key_vals.len()); + let mut key_spans = Vec::with_capacity(key_vals.len()); + + for key_val in key_vals { + match key_val { + RecordItem::Pair(key, val) => { + let key_str = match key.expr { + Expr::String(key_str) => key_str, + _ => { + return Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "only strings can be keys".into(), + span: key.span, + }) + } + }; + + if let Some(i) = record.index_of(&key_str) { + return Err(ShellError::ColumnDefinedTwice { + col_name: key_str, + second_use: key.span, + first_use: key_spans[i], + }); + } else { + key_spans.push(key.span); + record.push(key_str, convert_to_value(val, span, original_text)?); + } + } + RecordItem::Spread(_, inner) => { + return Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "spread operator not supported in nuon".into(), + span: inner.span, + }); + } + } + } + + Ok(Value::record(record, span)) + } + Expr::RowCondition(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "row conditions not supported in nuon".into(), + span: expr.span, + }), + Expr::Signature(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "signatures not supported in nuon".into(), + span: expr.span, + }), + Expr::String(s) => Ok(Value::string(s, span)), + Expr::StringInterpolation(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "string interpolation not supported in nuon".into(), + span: expr.span, + }), + Expr::Subexpression(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "subexpressions not supported in nuon".into(), + span: expr.span, + }), + Expr::Table(mut headers, cells) => { + let mut cols = vec![]; + + let mut output = vec![]; + + for key in headers.iter_mut() { + let key_str = match &mut key.expr { + Expr::String(key_str) => key_str, + _ => { + return Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "only strings can be keys".into(), + span: expr.span, + }) + } + }; + + if let Some(idx) = cols.iter().position(|existing| existing == key_str) { + return Err(ShellError::ColumnDefinedTwice { + col_name: key_str.clone(), + second_use: key.span, + first_use: headers[idx].span, + }); + } else { + cols.push(std::mem::take(key_str)); + } + } + + for row in cells { + if cols.len() != row.len() { + return Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "table has mismatched columns".into(), + span: expr.span, + }); + } + + let record = cols + .iter() + .zip(row) + .map(|(col, cell)| { + convert_to_value(cell, span, original_text).map(|val| (col.clone(), val)) + }) + .collect::>()?; + + output.push(Value::record(record, span)); + } + + Ok(Value::list(output, span)) + } + Expr::ValueWithUnit(val, unit) => { + let size = match val.expr { + Expr::Int(val) => val, + _ => { + return Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "non-integer unit value".into(), + span: expr.span, + }) + } + }; + + match unit.item { + Unit::Byte => Ok(Value::filesize(size, span)), + Unit::Kilobyte => Ok(Value::filesize(size * 1000, span)), + Unit::Megabyte => Ok(Value::filesize(size * 1000 * 1000, span)), + Unit::Gigabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000, span)), + Unit::Terabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000 * 1000, span)), + Unit::Petabyte => Ok(Value::filesize( + size * 1000 * 1000 * 1000 * 1000 * 1000, + span, + )), + Unit::Exabyte => Ok(Value::filesize( + size * 1000 * 1000 * 1000 * 1000 * 1000 * 1000, + span, + )), + + Unit::Kibibyte => Ok(Value::filesize(size * 1024, span)), + Unit::Mebibyte => Ok(Value::filesize(size * 1024 * 1024, span)), + Unit::Gibibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024, span)), + Unit::Tebibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024 * 1024, span)), + Unit::Pebibyte => Ok(Value::filesize( + size * 1024 * 1024 * 1024 * 1024 * 1024, + span, + )), + Unit::Exbibyte => Ok(Value::filesize( + size * 1024 * 1024 * 1024 * 1024 * 1024 * 1024, + span, + )), + + Unit::Nanosecond => Ok(Value::duration(size, span)), + Unit::Microsecond => Ok(Value::duration(size * 1000, span)), + Unit::Millisecond => Ok(Value::duration(size * 1000 * 1000, span)), + Unit::Second => Ok(Value::duration(size * 1000 * 1000 * 1000, span)), + Unit::Minute => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60, span)), + Unit::Hour => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60 * 60, span)), + Unit::Day => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24) { + Some(val) => Ok(Value::duration(val, span)), + None => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "day duration too large".into(), + msg: "day duration too large".into(), + span: expr.span, + }), + }, + + Unit::Week => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24 * 7) { + Some(val) => Ok(Value::duration(val, span)), + None => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "week duration too large".into(), + msg: "week duration too large".into(), + span: expr.span, + }), + }, + } + } + Expr::Var(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "variables not supported in nuon".into(), + span: expr.span, + }), + Expr::VarDecl(..) => Err(ShellError::OutsideSpannedLabeledError { + src: original_text.to_string(), + error: "Error when loading".into(), + msg: "variable declarations not supported in nuon".into(), + span: expr.span, + }), + } +} diff --git a/crates/nuon/src/lib.rs b/crates/nuon/src/lib.rs new file mode 100644 index 0000000000..60817ca2d8 --- /dev/null +++ b/crates/nuon/src/lib.rs @@ -0,0 +1,430 @@ +//! Support for the NUON format. +//! +//! The NUON format is a superset of JSON designed to fit the feel of Nushell. +//! Some of its extra features are +//! - trailing commas are allowed +//! - quotes are not required around keys +mod from; +mod to; + +pub use from::from_nuon; +pub use to::to_nuon; + +#[cfg(test)] +mod tests { + use chrono::DateTime; + use nu_protocol::{ast::RangeInclusion, engine::Closure, record, IntRange, Range, Span, Value}; + + use crate::{from_nuon, to_nuon}; + + /// test something of the form + /// ```nushell + /// $v | from nuon | to nuon | $in == $v + /// ``` + /// + /// an optional "middle" value can be given to test what the value is between `from nuon` and + /// `to nuon`. + fn nuon_end_to_end(input: &str, middle: Option) { + let val = from_nuon(input, None).unwrap(); + if let Some(m) = middle { + assert_eq!(val, m); + } + assert_eq!(to_nuon(&val, true, None, None, None).unwrap(), input); + } + + #[test] + fn list_of_numbers() { + nuon_end_to_end( + "[1, 2, 3]", + Some(Value::test_list(vec![ + Value::test_int(1), + Value::test_int(2), + Value::test_int(3), + ])), + ); + } + + #[test] + fn list_of_strings() { + nuon_end_to_end( + "[abc, xyz, def]", + Some(Value::test_list(vec![ + Value::test_string("abc"), + Value::test_string("xyz"), + Value::test_string("def"), + ])), + ); + } + + #[test] + fn table() { + nuon_end_to_end( + "[[my, columns]; [abc, xyz], [def, ijk]]", + Some(Value::test_list(vec![ + Value::test_record(record!( + "my" => Value::test_string("abc"), + "columns" => Value::test_string("xyz") + )), + Value::test_record(record!( + "my" => Value::test_string("def"), + "columns" => Value::test_string("ijk") + )), + ])), + ); + } + + #[test] + fn from_nuon_illegal_table() { + assert!( + from_nuon("[[repeated repeated]; [abc, xyz], [def, ijk]]", None) + .unwrap_err() + .to_string() + .contains("Record field or table column used twice: repeated") + ); + } + + #[test] + fn bool() { + nuon_end_to_end("false", Some(Value::test_bool(false))); + } + + #[test] + fn escaping() { + nuon_end_to_end(r#""hello\"world""#, None); + } + + #[test] + fn escaping2() { + nuon_end_to_end(r#""hello\\world""#, None); + } + + #[test] + fn escaping3() { + nuon_end_to_end( + r#"[hello\\world]"#, + Some(Value::test_list(vec![Value::test_string( + r#"hello\\world"#, + )])), + ); + } + + #[test] + fn escaping4() { + nuon_end_to_end(r#"["hello\"world"]"#, None); + } + + #[test] + fn escaping5() { + nuon_end_to_end(r#"{s: "hello\"world"}"#, None); + } + + #[test] + fn negative_int() { + nuon_end_to_end("-1", Some(Value::test_int(-1))); + } + + #[test] + fn records() { + nuon_end_to_end( + r#"{name: "foo bar", age: 100, height: 10}"#, + Some(Value::test_record(record!( + "name" => Value::test_string("foo bar"), + "age" => Value::test_int(100), + "height" => Value::test_int(10), + ))), + ); + } + + #[test] + fn range() { + nuon_end_to_end( + "1..42", + Some(Value::test_range(Range::IntRange( + IntRange::new( + Value::test_int(1), + Value::test_int(2), + Value::test_int(42), + RangeInclusion::Inclusive, + Span::unknown(), + ) + .unwrap(), + ))), + ); + } + + #[test] + fn filesize() { + nuon_end_to_end("1024b", Some(Value::test_filesize(1024))); + assert_eq!(from_nuon("1kib", None).unwrap(), Value::test_filesize(1024),); + } + + #[test] + fn duration() { + nuon_end_to_end("60000000000ns", Some(Value::test_duration(60_000_000_000))); + } + + #[test] + fn to_nuon_datetime() { + nuon_end_to_end( + "1970-01-01T00:00:00+00:00", + Some(Value::test_date(DateTime::UNIX_EPOCH.into())), + ); + } + + #[test] + fn to_nuon_errs_on_closure() { + assert!(to_nuon( + &Value::test_closure(Closure { + block_id: 0, + captures: vec![] + }), + true, + None, + None, + None, + ) + .unwrap_err() + .to_string() + .contains("Unsupported input")); + } + + #[test] + fn binary() { + nuon_end_to_end( + "0x[ABCDEF]", + Some(Value::test_binary(vec![0xab, 0xcd, 0xef])), + ); + } + + #[test] + fn binary_roundtrip() { + assert_eq!( + to_nuon( + &from_nuon("0x[1f ff]", None).unwrap(), + true, + None, + None, + None + ) + .unwrap(), + "0x[1FFF]" + ); + } + + #[test] + fn read_sample_data() { + assert_eq!( + from_nuon( + include_str!("../../../tests/fixtures/formats/sample.nuon"), + None, + ) + .unwrap(), + Value::test_list(vec![ + Value::test_list(vec![ + Value::test_record(record!( + "a" => Value::test_int(1), + "nuon" => Value::test_int(2), + "table" => Value::test_int(3) + )), + Value::test_record(record!( + "a" => Value::test_int(4), + "nuon" => Value::test_int(5), + "table" => Value::test_int(6) + )), + ]), + Value::test_filesize(100 * 1024), + Value::test_duration(100 * 1_000_000_000), + Value::test_bool(true), + Value::test_record(record!( + "name" => Value::test_string("Bobby"), + "age" => Value::test_int(99) + ),), + Value::test_binary(vec![0x11, 0xff, 0xee, 0x1f]), + ]) + ); + } + + #[test] + fn float_doesnt_become_int() { + assert_eq!( + to_nuon(&Value::test_float(1.0), true, None, None, None).unwrap(), + "1.0" + ); + } + + #[test] + fn float_inf_parsed_properly() { + assert_eq!( + to_nuon(&Value::test_float(f64::INFINITY), true, None, None, None).unwrap(), + "inf" + ); + } + + #[test] + fn float_neg_inf_parsed_properly() { + assert_eq!( + to_nuon( + &Value::test_float(f64::NEG_INFINITY), + true, + None, + None, + None + ) + .unwrap(), + "-inf" + ); + } + + #[test] + fn float_nan_parsed_properly() { + assert_eq!( + to_nuon(&Value::test_float(-f64::NAN), true, None, None, None).unwrap(), + "NaN" + ); + } + + #[test] + fn to_nuon_converts_columns_with_spaces() { + assert!(from_nuon( + &to_nuon( + &Value::test_list(vec![ + Value::test_record(record!( + "a" => Value::test_int(1), + "b" => Value::test_int(2), + "c d" => Value::test_int(3) + )), + Value::test_record(record!( + "a" => Value::test_int(4), + "b" => Value::test_int(5), + "c d" => Value::test_int(6) + )) + ]), + true, + None, + None, + None + ) + .unwrap(), + None, + ) + .is_ok()); + } + + #[test] + fn to_nuon_quotes_empty_string() { + let res = to_nuon(&Value::test_string(""), true, None, None, None); + assert!(res.is_ok()); + assert_eq!(res.unwrap(), r#""""#); + } + + #[test] + fn to_nuon_quotes_empty_string_in_list() { + nuon_end_to_end( + r#"[""]"#, + Some(Value::test_list(vec![Value::test_string("")])), + ); + } + + #[test] + fn to_nuon_quotes_empty_string_in_table() { + nuon_end_to_end( + "[[a, b]; [\"\", la], [le, lu]]", + Some(Value::test_list(vec![ + Value::test_record(record!( + "a" => Value::test_string(""), + "b" => Value::test_string("la"), + )), + Value::test_record(record!( + "a" => Value::test_string("le"), + "b" => Value::test_string("lu"), + )), + ])), + ); + } + + #[test] + fn does_not_quote_strings_unnecessarily() { + assert_eq!( + to_nuon( + &Value::test_list(vec![ + Value::test_record(record!( + "a" => Value::test_int(1), + "b" => Value::test_int(2), + "c d" => Value::test_int(3) + )), + Value::test_record(record!( + "a" => Value::test_int(4), + "b" => Value::test_int(5), + "c d" => Value::test_int(6) + )) + ]), + true, + None, + None, + None + ) + .unwrap(), + "[[a, b, \"c d\"]; [1, 2, 3], [4, 5, 6]]" + ); + + assert_eq!( + to_nuon( + &Value::test_record(record!( + "ro name" => Value::test_string("sam"), + "rank" => Value::test_int(10) + )), + true, + None, + None, + None + ) + .unwrap(), + "{\"ro name\": sam, rank: 10}" + ); + } + + #[test] + fn quotes_some_strings_necessarily() { + nuon_end_to_end( + r#"["true", "false", "null", "NaN", "NAN", "nan", "+nan", "-nan", "inf", "+inf", "-inf", "INF", "Infinity", "+Infinity", "-Infinity", "INFINITY", "+19.99", "-19.99", "19.99b", "19.99kb", "19.99mb", "19.99gb", "19.99tb", "19.99pb", "19.99eb", "19.99zb", "19.99kib", "19.99mib", "19.99gib", "19.99tib", "19.99pib", "19.99eib", "19.99zib", "19ns", "19us", "19ms", "19sec", "19min", "19hr", "19day", "19wk", "-11.0..-15.0", "11.0..-15.0", "-11.0..15.0", "-11.0..<-15.0", "11.0..<-15.0", "-11.0..<15.0", "-11.0..", "11.0..", "..15.0", "..-15.0", "..<15.0", "..<-15.0", "2000-01-01", "2022-02-02T14:30:00", "2022-02-02T14:30:00+05:00", ", ", "", "&&"]"#, + None, + ); + } + + #[test] + // NOTE: this test could be stronger, but the output of [`from_nuon`] on the content of `../../../tests/fixtures/formats/code.nu` is + // not the same in the CI and locally... + // + // ## locally + // ``` + // OutsideSpannedLabeledError { + // src: "register", + // error: "Error when loading", + // msg: "calls not supported in nuon", + // span: Span { start: 0, end: 8 } + // } + // ``` + // + // ## in the CI + // ``` + // GenericError { + // error: "error when parsing nuon text", + // msg: "could not parse nuon text", + // span: None, + // help: None, + // inner: [OutsideSpannedLabeledError { + // src: "register", + // error: "error when parsing", + // msg: "Unknown state.", + // span: Span { start: 0, end: 8 } + // }] + // } + // ``` + fn read_code_should_fail_rather_than_panic() { + assert!(from_nuon( + include_str!("../../../tests/fixtures/formats/code.nu"), + None, + ) + .is_err()); + } +} diff --git a/crates/nuon/src/to.rs b/crates/nuon/src/to.rs new file mode 100644 index 0000000000..39c89fb0a5 --- /dev/null +++ b/crates/nuon/src/to.rs @@ -0,0 +1,283 @@ +use core::fmt::Write; +use fancy_regex::Regex; +use once_cell::sync::Lazy; + +use nu_engine::get_columns; +use nu_parser::escape_quote_string; +use nu_protocol::{Range, ShellError, Span, Value}; + +use std::ops::Bound; + +/// convert an actual Nushell [`Value`] to a raw string representation of the NUON data +/// +/// ## Arguments +/// - `tabs` and `indent` control the level of indentation, expressed in _tabulations_ and _spaces_ +/// respectively. `tabs` has higher precedence over `indent`. +/// - `raw` has the highest precedence and will for the output to be _raw_, i.e. the [`Value`] will +/// be _serialized_ on a single line, without extra whitespaces. +/// +/// > **Note** +/// > a [`Span`] can be passed to [`to_nuon`] if there is context available to the caller, e.g. when +/// > using this function in a command implementation such as [`to nuon`](https://www.nushell.sh/commands/docs/to_nuon.html). +/// +/// also see [`super::from_nuon`] for the inverse operation +pub fn to_nuon( + input: &Value, + raw: bool, + tabs: Option, + indent: Option, + span: Option, +) -> Result { + let span = span.unwrap_or(Span::unknown()); + + let nuon_result = if raw { + value_to_string(input, span, 0, None)? + } else if let Some(tab_count) = tabs { + value_to_string(input, span, 0, Some(&"\t".repeat(tab_count)))? + } else if let Some(indent) = indent { + value_to_string(input, span, 0, Some(&" ".repeat(indent)))? + } else { + value_to_string(input, span, 0, None)? + }; + + Ok(nuon_result) +} + +fn value_to_string( + v: &Value, + span: Span, + depth: usize, + indent: Option<&str>, +) -> Result { + let (nl, sep) = get_true_separators(indent); + let idt = get_true_indentation(depth, indent); + let idt_po = get_true_indentation(depth + 1, indent); + let idt_pt = get_true_indentation(depth + 2, indent); + + match v { + Value::Binary { val, .. } => { + let mut s = String::with_capacity(2 * val.len()); + for byte in val { + if write!(s, "{byte:02X}").is_err() { + return Err(ShellError::UnsupportedInput { + msg: "could not convert binary to string".into(), + input: "value originates from here".into(), + msg_span: span, + input_span: v.span(), + }); + } + } + Ok(format!("0x[{s}]")) + } + Value::Block { .. } => Err(ShellError::UnsupportedInput { + msg: "blocks are currently not nuon-compatible".into(), + input: "value originates from here".into(), + msg_span: span, + input_span: v.span(), + }), + Value::Closure { .. } => Err(ShellError::UnsupportedInput { + msg: "closures are currently not nuon-compatible".into(), + input: "value originates from here".into(), + msg_span: span, + input_span: v.span(), + }), + Value::Bool { val, .. } => { + if *val { + Ok("true".to_string()) + } else { + Ok("false".to_string()) + } + } + Value::CellPath { .. } => Err(ShellError::UnsupportedInput { + msg: "cell-paths are currently not nuon-compatible".to_string(), + input: "value originates from here".into(), + msg_span: span, + input_span: v.span(), + }), + Value::Custom { .. } => Err(ShellError::UnsupportedInput { + msg: "custom values are currently not nuon-compatible".to_string(), + input: "value originates from here".into(), + msg_span: span, + input_span: v.span(), + }), + Value::Date { val, .. } => Ok(val.to_rfc3339()), + // FIXME: make durations use the shortest lossless representation. + Value::Duration { val, .. } => Ok(format!("{}ns", *val)), + // Propagate existing errors + Value::Error { error, .. } => Err(*error.clone()), + // FIXME: make filesizes use the shortest lossless representation. + Value::Filesize { val, .. } => Ok(format!("{}b", *val)), + Value::Float { val, .. } => { + // This serialises these as 'nan', 'inf' and '-inf', respectively. + if &val.round() == val && val.is_finite() { + Ok(format!("{}.0", *val)) + } else { + Ok(format!("{}", *val)) + } + } + Value::Int { val, .. } => Ok(format!("{}", *val)), + Value::List { vals, .. } => { + let headers = get_columns(vals); + if !headers.is_empty() && vals.iter().all(|x| x.columns().eq(headers.iter())) { + // Table output + let headers: Vec = headers + .iter() + .map(|string| { + if needs_quotes(string) { + format!("{idt}\"{string}\"") + } else { + format!("{idt}{string}") + } + }) + .collect(); + let headers_output = headers.join(&format!(",{sep}{nl}{idt_pt}")); + + let mut table_output = vec![]; + for val in vals { + let mut row = vec![]; + + if let Value::Record { val, .. } = val { + for val in val.values() { + row.push(value_to_string_without_quotes( + val, + span, + depth + 2, + indent, + )?); + } + } + + table_output.push(row.join(&format!(",{sep}{nl}{idt_pt}"))); + } + + Ok(format!( + "[{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}];{sep}{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}]{nl}{idt}]", + headers_output, + table_output.join(&format!("{nl}{idt_po}],{sep}{nl}{idt_po}[{nl}{idt_pt}")) + )) + } else { + let mut collection = vec![]; + for val in vals { + collection.push(format!( + "{idt_po}{}", + value_to_string_without_quotes(val, span, depth + 1, indent,)? + )); + } + Ok(format!( + "[{nl}{}{nl}{idt}]", + collection.join(&format!(",{sep}{nl}")) + )) + } + } + Value::Nothing { .. } => Ok("null".to_string()), + Value::Range { val, .. } => match val { + Range::IntRange(range) => Ok(range.to_string()), + Range::FloatRange(range) => { + let start = + value_to_string(&Value::float(range.start(), span), span, depth + 1, indent)?; + match range.end() { + Bound::Included(end) => Ok(format!( + "{}..{}", + start, + value_to_string(&Value::float(end, span), span, depth + 1, indent)? + )), + Bound::Excluded(end) => Ok(format!( + "{}..<{}", + start, + value_to_string(&Value::float(end, span), span, depth + 1, indent)? + )), + Bound::Unbounded => Ok(format!("{start}..",)), + } + } + }, + Value::Record { val, .. } => { + let mut collection = vec![]; + for (col, val) in &**val { + collection.push(if needs_quotes(col) { + format!( + "{idt_po}\"{}\": {}", + col, + value_to_string_without_quotes(val, span, depth + 1, indent)? + ) + } else { + format!( + "{idt_po}{}: {}", + col, + value_to_string_without_quotes(val, span, depth + 1, indent)? + ) + }); + } + Ok(format!( + "{{{nl}{}{nl}{idt}}}", + collection.join(&format!(",{sep}{nl}")) + )) + } + Value::LazyRecord { val, .. } => { + let collected = val.collect()?; + value_to_string(&collected, span, depth + 1, indent) + } + // All strings outside data structures are quoted because they are in 'command position' + // (could be mistaken for commands by the Nu parser) + Value::String { val, .. } => Ok(escape_quote_string(val)), + Value::Glob { val, .. } => Ok(escape_quote_string(val)), + } +} + +fn get_true_indentation(depth: usize, indent: Option<&str>) -> String { + match indent { + Some(i) => i.repeat(depth), + None => "".to_string(), + } +} + +fn get_true_separators(indent: Option<&str>) -> (String, String) { + match indent { + Some(_) => ("\n".to_string(), "".to_string()), + None => ("".to_string(), " ".to_string()), + } +} + +fn value_to_string_without_quotes( + v: &Value, + span: Span, + depth: usize, + indent: Option<&str>, +) -> Result { + match v { + Value::String { val, .. } => Ok({ + if needs_quotes(val) { + escape_quote_string(val) + } else { + val.clone() + } + }), + _ => value_to_string(v, span, depth, indent), + } +} + +// This hits, in order: +// • Any character of []:`{}#'";()|$, +// • Any digit (\d) +// • Any whitespace (\s) +// • Case-insensitive sign-insensitive float "keywords" inf, infinity and nan. +static NEEDS_QUOTES_REGEX: Lazy = Lazy::new(|| { + Regex::new(r#"[\[\]:`\{\}#'";\(\)\|\$,\d\s]|(?i)^[+\-]?(inf(inity)?|nan)$"#) + .expect("internal error: NEEDS_QUOTES_REGEX didn't compile") +}); + +fn needs_quotes(string: &str) -> bool { + if string.is_empty() { + return true; + } + // These are case-sensitive keywords + match string { + // `true`/`false`/`null` are active keywords in JSON and NUON + // `&&` is denied by the nu parser for diagnostics reasons + // (https://github.com/nushell/nushell/pull/7241) + // TODO: remove the extra check in the nuon codepath + "true" | "false" | "null" | "&&" => return true, + _ => (), + }; + // All other cases are handled here + NEEDS_QUOTES_REGEX.is_match(string).unwrap_or(false) +}