extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-11-07 09:04:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

create nuon crate from from nuon and to nuon (#12553)

Browse Source 
# Description
playing with the NUON format in Rust code in some plugins, we agreed
with the team it was a great time to create a standalone NUON format to
allow Rust devs to use this Nushell file format.

> **Note**
> this PR almost copy-pastes the code from
`nu_commands/src/formats/from/nuon.rs` and
`nu_commands/src/formats/to/nuon.rs` to `nuon/src/from.rs` and
`nuon/src/to.rs`, with minor tweaks to make then standalone functions,
e.g. remove the rest of the command implementations

### TODO
- [x] add tests
- [x] add documentation

# User-Facing Changes
devs will have access to a new crate, `nuon`, and two functions,
`from_nuon` and `to_nuon`
```rust
from_nuon(
    input: &str,
    span: Option<Span>,
) -> Result<Value, ShellError>
```
```rust
to_nuon(
    input: &Value,
    raw: bool,
    tabs: Option<usize>,
    indent: Option<usize>,
    span: Option<Span>,
) -> Result<String, ShellError>
```

# Tests + Formatting
i've basically taken all the tests from
`crates/nu-command/tests/format_conversions/nuon.rs` and converted them
to use `from_nuon` and `to_nuon` instead of Nushell commands
- i've created a `nuon_end_to_end` to run both conversions with an
optional middle value to check that all is fine

> **Note** 
> the `nuon::tests::read_code_should_fail_rather_than_panic` test does
give different results locally and in the CI...
> i've left it ignored with comments to help future us :)

# After Submitting
mention that in the release notes for sure!!
This commit is contained in:
Antoine Stevan 2024-04-19 13:54:16 +02:00 committed by GitHub
parent fac2f43aa4
commit 55edef5dda
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 1240 additions and 714 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@ -3068,6 +3068,7 @@ dependencies = [
"nu-utils",
"num-format",
"num-traits",
"nuon",
"once_cell",
"open",
"os_pipe",
@ -3579,6 +3580,18 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "nuon"
version = "0.92.3"
dependencies = [
"chrono",
"fancy-regex",
"nu-engine",
"nu-parser",
"nu-protocol",
"once_cell",
]
[[package]]
name = "objc"
version = "0.2.7"

1
Cargo.toml
View File

@ -54,6 +54,7 @@ members = [
"crates/nu-term-grid",
"crates/nu-test-support",
"crates/nu-utils",
"crates/nuon",
]
[workspace.dependencies]

1
crates/nu-command/Cargo.toml
View File

@ -27,6 +27,7 @@ nu-table = { path = "../nu-table", version = "0.92.3" }
nu-term-grid = { path = "../nu-term-grid", version = "0.92.3" }
nu-utils = { path = "../nu-utils", version = "0.92.3" }
nu-ansi-term = { workspace = true }
nuon = { path = "../nuon", version = "0.92.3" }
alphanumeric-sort = { workspace = true }
base64 = { workspace = true }

3
crates/nu-command/src/filters/uniq.rs
View File

@ -1,4 +1,3 @@
use crate::formats::value_to_string;
use itertools::Itertools;
use nu_engine::command_prelude::*;
use nu_protocol::PipelineMetadata;
@ -216,7 +215,7 @@ fn sort_attributes(val: Value) -> Value {
fn generate_key(item: &ValueCounter) -> Result<String, ShellError> {
let value = sort_attributes(item.val_to_compare.clone()); //otherwise, keys could be different for Records
value_to_string(&value, Span::unknown(), 0, None)
nuon::to_nuon(&value, true, None, None, Some(Span::unknown()))
}
fn generate_results_with_count(head: Span, uniq_values: Vec<ValueCounter>) -> Vec<Value> {

455
crates/nu-command/src/formats/from/nuon.rs
View File

@ -1,10 +1,4 @@
use nu_engine::command_prelude::*;
use nu_protocol::{
ast::{Expr, Expression, ListItem, RecordItem},
engine::StateWorkingSet,
Range, Unit,
};
use std::sync::Arc;
#[derive(Clone)]
pub struct FromNuon;
@ -46,7 +40,7 @@ impl Command for FromNuon {
fn run(
&self,
engine_state: &EngineState,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
@ -54,98 +48,7 @@ impl Command for FromNuon {
let head = call.head;
let (string_input, _span, metadata) = input.collect_string_strict(head)?;
let engine_state = engine_state.clone();
let mut working_set = StateWorkingSet::new(&engine_state);
let mut block = nu_parser::parse(&mut working_set, None, string_input.as_bytes(), false);
if let Some(pipeline) = block.pipelines.get(1) {
if let Some(element) = pipeline.elements.first() {
return Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
msg: "could not load nuon text".into(),
span: Some(head),
help: None,
inner: vec![ShellError::OutsideSpannedLabeledError {
src: string_input,
error: "error when loading".into(),
msg: "excess values when loading".into(),
span: element.expr.span,
}],
});
} else {
return Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
msg: "could not load nuon text".into(),
span: Some(head),
help: None,
inner: vec![ShellError::GenericError {
error: "error when loading".into(),
msg: "excess values when loading".into(),
span: Some(head),
help: None,
inner: vec![],
}],
});
}
}
let expr = if block.pipelines.is_empty() {
Expression {
expr: Expr::Nothing,
span: head,
custom_completion: None,
ty: Type::Nothing,
}
} else {
let mut pipeline = Arc::make_mut(&mut block).pipelines.remove(0);
if let Some(expr) = pipeline.elements.get(1) {
return Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
msg: "could not load nuon text".into(),
span: Some(head),
help: None,
inner: vec![ShellError::OutsideSpannedLabeledError {
src: string_input,
error: "error when loading".into(),
msg: "detected a pipeline in nuon file".into(),
span: expr.expr.span,
}],
});
}
if pipeline.elements.is_empty() {
Expression {
expr: Expr::Nothing,
span: head,
custom_completion: None,
ty: Type::Nothing,
}
} else {
pipeline.elements.remove(0).expr
}
};
if let Some(err) = working_set.parse_errors.first() {
return Err(ShellError::GenericError {
error: "error when parsing nuon text".into(),
msg: "could not parse nuon text".into(),
span: Some(head),
help: None,
inner: vec![ShellError::OutsideSpannedLabeledError {
src: string_input,
error: "error when parsing".into(),
msg: err.to_string(),
span: err.span(),
}],
});
}
let result = convert_to_value(expr, head, &string_input);
match result {
match nuon::from_nuon(&string_input, Some(head)) {
Ok(result) => Ok(result.into_pipeline_data_with_metadata(metadata)),
Err(err) => Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
@ -158,360 +61,6 @@ impl Command for FromNuon {
}
}
fn convert_to_value(
expr: Expression,
span: Span,
original_text: &str,
) -> Result<Value, ShellError> {
match expr.expr {
Expr::BinaryOp(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "binary operators not supported in nuon".into(),
span: expr.span,
}),
Expr::UnaryNot(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "unary operators not supported in nuon".into(),
span: expr.span,
}),
Expr::Block(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "blocks not supported in nuon".into(),
span: expr.span,
}),
Expr::Closure(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "closures not supported in nuon".into(),
span: expr.span,
}),
Expr::Binary(val) => Ok(Value::binary(val, span)),
Expr::Bool(val) => Ok(Value::bool(val, span)),
Expr::Call(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "calls not supported in nuon".into(),
span: expr.span,
}),
Expr::CellPath(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "subexpressions and cellpaths not supported in nuon".into(),
span: expr.span,
}),
Expr::DateTime(dt) => Ok(Value::date(dt, span)),
Expr::ExternalCall(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "calls not supported in nuon".into(),
span: expr.span,
}),
Expr::Filepath(val, _) => Ok(Value::string(val, span)),
Expr::Directory(val, _) => Ok(Value::string(val, span)),
Expr::Float(val) => Ok(Value::float(val, span)),
Expr::FullCellPath(full_cell_path) => {
if !full_cell_path.tail.is_empty() {
Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "subexpressions and cellpaths not supported in nuon".into(),
span: expr.span,
})
} else {
convert_to_value(full_cell_path.head, span, original_text)
}
}
Expr::Garbage => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "extra tokens in input file".into(),
span: expr.span,
}),
Expr::GlobPattern(val, _) => Ok(Value::string(val, span)),
Expr::ImportPattern(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "imports not supported in nuon".into(),
span: expr.span,
}),
Expr::Overlay(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "overlays not supported in nuon".into(),
span: expr.span,
}),
Expr::Int(val) => Ok(Value::int(val, span)),
Expr::Keyword(kw, ..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: format!("{} not supported in nuon", String::from_utf8_lossy(&kw)),
span: expr.span,
}),
Expr::List(vals) => {
let mut output = vec![];
for item in vals {
match item {
ListItem::Item(expr) => {
output.push(convert_to_value(expr, span, original_text)?);
}
ListItem::Spread(_, inner) => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "spread operator not supported in nuon".into(),
span: inner.span,
});
}
}
}
Ok(Value::list(output, span))
}
Expr::MatchBlock(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "match blocks not supported in nuon".into(),
span: expr.span,
}),
Expr::Nothing => Ok(Value::nothing(span)),
Expr::Operator(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "operators not supported in nuon".into(),
span: expr.span,
}),
Expr::Range(from, next, to, operator) => {
let from = if let Some(f) = from {
convert_to_value(*f, span, original_text)?
} else {
Value::nothing(expr.span)
};
let next = if let Some(s) = next {
convert_to_value(*s, span, original_text)?
} else {
Value::nothing(expr.span)
};
let to = if let Some(t) = to {
convert_to_value(*t, span, original_text)?
} else {
Value::nothing(expr.span)
};
Ok(Value::range(
Range::new(from, next, to, operator.inclusion, expr.span)?,
expr.span,
))
}
Expr::Record(key_vals) => {
let mut record = Record::with_capacity(key_vals.len());
let mut key_spans = Vec::with_capacity(key_vals.len());
for key_val in key_vals {
match key_val {
RecordItem::Pair(key, val) => {
let key_str = match key.expr {
Expr::String(key_str) => key_str,
_ => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "only strings can be keys".into(),
span: key.span,
})
}
};
if let Some(i) = record.index_of(&key_str) {
return Err(ShellError::ColumnDefinedTwice {
col_name: key_str,
second_use: key.span,
first_use: key_spans[i],
});
} else {
key_spans.push(key.span);
record.push(key_str, convert_to_value(val, span, original_text)?);
}
}
RecordItem::Spread(_, inner) => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "spread operator not supported in nuon".into(),
span: inner.span,
});
}
}
}
Ok(Value::record(record, span))
}
Expr::RowCondition(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "row conditions not supported in nuon".into(),
span: expr.span,
}),
Expr::Signature(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "signatures not supported in nuon".into(),
span: expr.span,
}),
Expr::String(s) => Ok(Value::string(s, span)),
Expr::StringInterpolation(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "string interpolation not supported in nuon".into(),
span: expr.span,
}),
Expr::Subexpression(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "subexpressions not supported in nuon".into(),
span: expr.span,
}),
Expr::Table(mut headers, cells) => {
let mut cols = vec![];
let mut output = vec![];
for key in headers.iter_mut() {
let key_str = match &mut key.expr {
Expr::String(key_str) => key_str,
_ => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "only strings can be keys".into(),
span: expr.span,
})
}
};
if let Some(idx) = cols.iter().position(|existing| existing == key_str) {
return Err(ShellError::ColumnDefinedTwice {
col_name: key_str.clone(),
second_use: key.span,
first_use: headers[idx].span,
});
} else {
cols.push(std::mem::take(key_str));
}
}
for row in cells {
if cols.len() != row.len() {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "table has mismatched columns".into(),
span: expr.span,
});
}
let record = cols
.iter()
.zip(row)
.map(|(col, cell)| {
convert_to_value(cell, span, original_text).map(|val| (col.clone(), val))
})
.collect::<Result<_, _>>()?;
output.push(Value::record(record, span));
}
Ok(Value::list(output, span))
}
Expr::ValueWithUnit(val, unit) => {
let size = match val.expr {
Expr::Int(val) => val,
_ => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "non-integer unit value".into(),
span: expr.span,
})
}
};
match unit.item {
Unit::Byte => Ok(Value::filesize(size, span)),
Unit::Kilobyte => Ok(Value::filesize(size * 1000, span)),
Unit::Megabyte => Ok(Value::filesize(size * 1000 * 1000, span)),
Unit::Gigabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000, span)),
Unit::Terabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000 * 1000, span)),
Unit::Petabyte => Ok(Value::filesize(
size * 1000 * 1000 * 1000 * 1000 * 1000,
span,
)),
Unit::Exabyte => Ok(Value::filesize(
size * 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
span,
)),
Unit::Kibibyte => Ok(Value::filesize(size * 1024, span)),
Unit::Mebibyte => Ok(Value::filesize(size * 1024 * 1024, span)),
Unit::Gibibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024, span)),
Unit::Tebibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024 * 1024, span)),
Unit::Pebibyte => Ok(Value::filesize(
size * 1024 * 1024 * 1024 * 1024 * 1024,
span,
)),
Unit::Exbibyte => Ok(Value::filesize(
size * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
span,
)),
Unit::Nanosecond => Ok(Value::duration(size, span)),
Unit::Microsecond => Ok(Value::duration(size * 1000, span)),
Unit::Millisecond => Ok(Value::duration(size * 1000 * 1000, span)),
Unit::Second => Ok(Value::duration(size * 1000 * 1000 * 1000, span)),
Unit::Minute => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60, span)),
Unit::Hour => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60 * 60, span)),
Unit::Day => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24) {
Some(val) => Ok(Value::duration(val, span)),
None => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "day duration too large".into(),
msg: "day duration too large".into(),
span: expr.span,
}),
},
Unit::Week => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24 * 7) {
Some(val) => Ok(Value::duration(val, span)),
None => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "week duration too large".into(),
msg: "week duration too large".into(),
span: expr.span,
}),
},
}
}
Expr::Var(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "variables not supported in nuon".into(),
span: expr.span,
}),
Expr::VarDecl(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "variable declarations not supported in nuon".into(),
span: expr.span,
}),
}
}
#[cfg(test)]
mod test {
use super::*;

1
crates/nu-command/src/formats/to/mod.rs
View File

@ -15,7 +15,6 @@ pub use self::toml::ToToml;
pub use command::To;
pub use json::ToJson;
pub use md::ToMd;
pub use nuon::value_to_string;
pub use nuon::ToNuon;
pub use text::ToText;
pub use tsv::ToTsv;

259
crates/nu-command/src/formats/to/nuon.rs
View File

@ -1,10 +1,4 @@
use core::fmt::Write;
use fancy_regex::Regex;
use nu_engine::{command_prelude::*, get_columns};
use nu_parser::escape_quote_string;
use nu_protocol::Range;
use once_cell::sync::Lazy;
use std::ops::Bound;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ToNuon;
@ -55,17 +49,7 @@ impl Command for ToNuon {
let span = call.head;
let value = input.into_value(span);
let nuon_result = if raw {
value_to_string(&value, span, 0, None)
} else if let Some(tab_count) = tabs {
value_to_string(&value, span, 0, Some(&"\t".repeat(tab_count)))
} else if let Some(indent) = indent {
value_to_string(&value, span, 0, Some(&" ".repeat(indent)))
} else {
value_to_string(&value, span, 0, None)
};
match nuon_result {
match nuon::to_nuon(&value, raw, tabs, indent, Some(span)) {
Ok(serde_nuon_string) => {
Ok(Value::string(serde_nuon_string, span).into_pipeline_data())
}
@ -108,245 +92,6 @@ impl Command for ToNuon {
}
}
pub fn value_to_string(
v: &Value,
span: Span,
depth: usize,
indent: Option<&str>,
) -> Result<String, ShellError> {
let (nl, sep) = get_true_separators(indent);
let idt = get_true_indentation(depth, indent);
let idt_po = get_true_indentation(depth + 1, indent);
let idt_pt = get_true_indentation(depth + 2, indent);
match v {
Value::Binary { val, .. } => {
let mut s = String::with_capacity(2 * val.len());
for byte in val {
if write!(s, "{byte:02X}").is_err() {
return Err(ShellError::UnsupportedInput {
msg: "could not convert binary to string".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
});
}
}
Ok(format!("0x[{s}]"))
}
Value::Block { .. } => Err(ShellError::UnsupportedInput {
msg: "blocks are currently not nuon-compatible".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Closure { .. } => Err(ShellError::UnsupportedInput {
msg: "closures are currently not nuon-compatible".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Bool { val, .. } => {
if *val {
Ok("true".to_string())
} else {
Ok("false".to_string())
}
}
Value::CellPath { .. } => Err(ShellError::UnsupportedInput {
msg: "cell-paths are currently not nuon-compatible".to_string(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Custom { .. } => Err(ShellError::UnsupportedInput {
msg: "custom values are currently not nuon-compatible".to_string(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Date { val, .. } => Ok(val.to_rfc3339()),
// FIXME: make durations use the shortest lossless representation.
Value::Duration { val, .. } => Ok(format!("{}ns", *val)),
// Propagate existing errors
Value::Error { error, .. } => Err(*error.clone()),
// FIXME: make filesizes use the shortest lossless representation.
Value::Filesize { val, .. } => Ok(format!("{}b", *val)),
Value::Float { val, .. } => {
// This serialises these as 'nan', 'inf' and '-inf', respectively.
if &val.round() == val && val.is_finite() {
Ok(format!("{}.0", *val))
} else {
Ok(format!("{}", *val))
}
}
Value::Int { val, .. } => Ok(format!("{}", *val)),
Value::List { vals, .. } => {
let headers = get_columns(vals);
if !headers.is_empty() && vals.iter().all(|x| x.columns().eq(headers.iter())) {
// Table output
let headers: Vec<String> = headers
.iter()
.map(|string| {
if needs_quotes(string) {
format!("{idt}\"{string}\"")
} else {
format!("{idt}{string}")
}
})
.collect();
let headers_output = headers.join(&format!(",{sep}{nl}{idt_pt}"));
let mut table_output = vec![];
for val in vals {
let mut row = vec![];
if let Value::Record { val, .. } = val {
for val in val.values() {
row.push(value_to_string_without_quotes(
val,
span,
depth + 2,
indent,
)?);
}
}
table_output.push(row.join(&format!(",{sep}{nl}{idt_pt}")));
}
Ok(format!(
"[{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}];{sep}{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}]{nl}{idt}]",
headers_output,
table_output.join(&format!("{nl}{idt_po}],{sep}{nl}{idt_po}[{nl}{idt_pt}"))
))
} else {
let mut collection = vec![];
for val in vals {
collection.push(format!(
"{idt_po}{}",
value_to_string_without_quotes(val, span, depth + 1, indent,)?
));
}
Ok(format!(
"[{nl}{}{nl}{idt}]",
collection.join(&format!(",{sep}{nl}"))
))
}
}
Value::Nothing { .. } => Ok("null".to_string()),
Value::Range { val, .. } => match val {
Range::IntRange(range) => Ok(range.to_string()),
Range::FloatRange(range) => {
let start =
value_to_string(&Value::float(range.start(), span), span, depth + 1, indent)?;
match range.end() {
Bound::Included(end) => Ok(format!(
"{}..{}",
start,
value_to_string(&Value::float(end, span), span, depth + 1, indent)?
)),
Bound::Excluded(end) => Ok(format!(
"{}..<{}",
start,
value_to_string(&Value::float(end, span), span, depth + 1, indent)?
)),
Bound::Unbounded => Ok(format!("{start}..",)),
}
}
},
Value::Record { val, .. } => {
let mut collection = vec![];
for (col, val) in &**val {
collection.push(if needs_quotes(col) {
format!(
"{idt_po}\"{}\": {}",
col,
value_to_string_without_quotes(val, span, depth + 1, indent)?
)
} else {
format!(
"{idt_po}{}: {}",
col,
value_to_string_without_quotes(val, span, depth + 1, indent)?
)
});
}
Ok(format!(
"{{{nl}{}{nl}{idt}}}",
collection.join(&format!(",{sep}{nl}"))
))
}
Value::LazyRecord { val, .. } => {
let collected = val.collect()?;
value_to_string(&collected, span, depth + 1, indent)
}
// All strings outside data structures are quoted because they are in 'command position'
// (could be mistaken for commands by the Nu parser)
Value::String { val, .. } => Ok(escape_quote_string(val)),
Value::Glob { val, .. } => Ok(escape_quote_string(val)),
}
}
fn get_true_indentation(depth: usize, indent: Option<&str>) -> String {
match indent {
Some(i) => i.repeat(depth),
None => "".to_string(),
}
}
fn get_true_separators(indent: Option<&str>) -> (String, String) {
match indent {
Some(_) => ("\n".to_string(), "".to_string()),
None => ("".to_string(), " ".to_string()),
}
}
fn value_to_string_without_quotes(
v: &Value,
span: Span,
depth: usize,
indent: Option<&str>,
) -> Result<String, ShellError> {
match v {
Value::String { val, .. } => Ok({
if needs_quotes(val) {
escape_quote_string(val)
} else {
val.clone()
}
}),
_ => value_to_string(v, span, depth, indent),
}
}
// This hits, in order:
// • Any character of []:`{}#'";()|$,
// • Any digit (\d)
// • Any whitespace (\s)
// • Case-insensitive sign-insensitive float "keywords" inf, infinity and nan.
static NEEDS_QUOTES_REGEX: Lazy<Regex> = Lazy::new(|| {
Regex::new(r#"[\[\]:`\{\}#'";\(\)\|\$,\d\s]|(?i)^[+\-]?(inf(inity)?|nan)$"#)
.expect("internal error: NEEDS_QUOTES_REGEX didn't compile")
});
fn needs_quotes(string: &str) -> bool {
if string.is_empty() {
return true;
}
// These are case-sensitive keywords
match string {
// `true`/`false`/`null` are active keywords in JSON and NUON
// `&&` is denied by the nu parser for diagnostics reasons
// (https://github.com/nushell/nushell/pull/7241)
// TODO: remove the extra check in the nuon codepath
"true" | "false" | "null" | "&&" => return true,
_ => (),
};
// All other cases are handled here
NEEDS_QUOTES_REGEX.is_match(string).unwrap_or(false)
}
#[cfg(test)]
mod test {
#[test]

2
crates/nu-command/tests/commands/database/into_sqlite.rs
View File

@ -330,7 +330,7 @@ fn into_sqlite_big_insert() {
)
.unwrap();
let nuon = nu_command::value_to_string(&value, Span::unknown(), 0, None).unwrap()
let nuon = nuon::to_nuon(&value, true, None, None, Some(Span::unknown())).unwrap()
+ &line_ending();
nuon_file.write_all(nuon.as_bytes()).unwrap();

20
crates/nuon/Cargo.toml Normal file
View File

@ -0,0 +1,20 @@
[package]
authors = ["The Nushell Project Developers"]
description = "Support for the NUON format."
repository = "https://github.com/nushell/nushell/tree/main/crates/nuon"
edition = "2021"
license = "MIT"
name = "nuon"
version = "0.92.3"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
nu-parser = { path = "../nu-parser", version = "0.92.3" }
nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
nu-engine = { path = "../nu-engine", version = "0.92.3" }
once_cell = { workspace = true }
fancy-regex = { workspace = true }
[dev-dependencies]
chrono = { workspace = true }

21
crates/nuon/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 - 2023 The Nushell Project Developers
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

465
crates/nuon/src/from.rs Normal file
View File

@ -0,0 +1,465 @@
use nu_protocol::{
ast::{Expr, Expression, ListItem, RecordItem},
engine::{EngineState, StateWorkingSet},
Range, Record, ShellError, Span, Type, Unit, Value,
};
use std::sync::Arc;
/// convert a raw string representation of NUON data to an actual Nushell [`Value`]
///
/// > **Note**
/// > [`Span`] can be passed to [`from_nuon`] if there is context available to the caller, e.g. when
/// > using this function in a command implementation such as
/// [`from nuon`](https://www.nushell.sh/commands/docs/from_nuon.html).
///
/// also see [`super::to_nuon`] for the inverse operation
pub fn from_nuon(input: &str, span: Option<Span>) -> Result<Value, ShellError> {
let mut engine_state = EngineState::default();
// NOTE: the parser needs `$env.PWD` to be set, that's a know _API issue_ with the
// [`EngineState`]
engine_state.add_env_var("PWD".to_string(), Value::string("", Span::unknown()));
let mut working_set = StateWorkingSet::new(&engine_state);
let mut block = nu_parser::parse(&mut working_set, None, input.as_bytes(), false);
if let Some(pipeline) = block.pipelines.get(1) {
if let Some(element) = pipeline.elements.first() {
return Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
msg: "could not load nuon text".into(),
span,
help: None,
inner: vec![ShellError::OutsideSpannedLabeledError {
src: input.to_string(),
error: "error when loading".into(),
msg: "excess values when loading".into(),
span: element.expr.span,
}],
});
} else {
return Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
msg: "could not load nuon text".into(),
span,
help: None,
inner: vec![ShellError::GenericError {
error: "error when loading".into(),
msg: "excess values when loading".into(),
span,
help: None,
inner: vec![],
}],
});
}
}
let expr = if block.pipelines.is_empty() {
Expression {
expr: Expr::Nothing,
span: span.unwrap_or(Span::unknown()),
custom_completion: None,
ty: Type::Nothing,
}
} else {
let mut pipeline = Arc::make_mut(&mut block).pipelines.remove(0);
if let Some(expr) = pipeline.elements.get(1) {
return Err(ShellError::GenericError {
error: "error when loading nuon text".into(),
msg: "could not load nuon text".into(),
span,
help: None,
inner: vec![ShellError::OutsideSpannedLabeledError {
src: input.to_string(),
error: "error when loading".into(),
msg: "detected a pipeline in nuon file".into(),
span: expr.expr.span,
}],
});
}
if pipeline.elements.is_empty() {
Expression {
expr: Expr::Nothing,
span: span.unwrap_or(Span::unknown()),
custom_completion: None,
ty: Type::Nothing,
}
} else {
pipeline.elements.remove(0).expr
}
};
if let Some(err) = working_set.parse_errors.first() {
return Err(ShellError::GenericError {
error: "error when parsing nuon text".into(),
msg: "could not parse nuon text".into(),
span,
help: None,
inner: vec![ShellError::OutsideSpannedLabeledError {
src: input.to_string(),
error: "error when parsing".into(),
msg: err.to_string(),
span: err.span(),
}],
});
}
let value = convert_to_value(expr, span.unwrap_or(Span::unknown()), input)?;
Ok(value)
}
fn convert_to_value(
expr: Expression,
span: Span,
original_text: &str,
) -> Result<Value, ShellError> {
match expr.expr {
Expr::BinaryOp(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "binary operators not supported in nuon".into(),
span: expr.span,
}),
Expr::UnaryNot(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "unary operators not supported in nuon".into(),
span: expr.span,
}),
Expr::Block(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "blocks not supported in nuon".into(),
span: expr.span,
}),
Expr::Closure(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "closures not supported in nuon".into(),
span: expr.span,
}),
Expr::Binary(val) => Ok(Value::binary(val, span)),
Expr::Bool(val) => Ok(Value::bool(val, span)),
Expr::Call(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "calls not supported in nuon".into(),
span: expr.span,
}),
Expr::CellPath(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "subexpressions and cellpaths not supported in nuon".into(),
span: expr.span,
}),
Expr::DateTime(dt) => Ok(Value::date(dt, span)),
Expr::ExternalCall(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "calls not supported in nuon".into(),
span: expr.span,
}),
Expr::Filepath(val, _) => Ok(Value::string(val, span)),
Expr::Directory(val, _) => Ok(Value::string(val, span)),
Expr::Float(val) => Ok(Value::float(val, span)),
Expr::FullCellPath(full_cell_path) => {
if !full_cell_path.tail.is_empty() {
Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "subexpressions and cellpaths not supported in nuon".into(),
span: expr.span,
})
} else {
convert_to_value(full_cell_path.head, span, original_text)
}
}
Expr::Garbage => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "extra tokens in input file".into(),
span: expr.span,
}),
Expr::GlobPattern(val, _) => Ok(Value::string(val, span)),
Expr::ImportPattern(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "imports not supported in nuon".into(),
span: expr.span,
}),
Expr::Overlay(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "overlays not supported in nuon".into(),
span: expr.span,
}),
Expr::Int(val) => Ok(Value::int(val, span)),
Expr::Keyword(kw, ..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: format!("{} not supported in nuon", String::from_utf8_lossy(&kw)),
span: expr.span,
}),
Expr::List(vals) => {
let mut output = vec![];
for item in vals {
match item {
ListItem::Item(expr) => {
output.push(convert_to_value(expr, span, original_text)?);
}
ListItem::Spread(_, inner) => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "spread operator not supported in nuon".into(),
span: inner.span,
});
}
}
}
Ok(Value::list(output, span))
}
Expr::MatchBlock(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "match blocks not supported in nuon".into(),
span: expr.span,
}),
Expr::Nothing => Ok(Value::nothing(span)),
Expr::Operator(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "operators not supported in nuon".into(),
span: expr.span,
}),
Expr::Range(from, next, to, operator) => {
let from = if let Some(f) = from {
convert_to_value(*f, span, original_text)?
} else {
Value::nothing(expr.span)
};
let next = if let Some(s) = next {
convert_to_value(*s, span, original_text)?
} else {
Value::nothing(expr.span)
};
let to = if let Some(t) = to {
convert_to_value(*t, span, original_text)?
} else {
Value::nothing(expr.span)
};
Ok(Value::range(
Range::new(from, next, to, operator.inclusion, expr.span)?,
expr.span,
))
}
Expr::Record(key_vals) => {
let mut record = Record::with_capacity(key_vals.len());
let mut key_spans = Vec::with_capacity(key_vals.len());
for key_val in key_vals {
match key_val {
RecordItem::Pair(key, val) => {
let key_str = match key.expr {
Expr::String(key_str) => key_str,
_ => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "only strings can be keys".into(),
span: key.span,
})
}
};
if let Some(i) = record.index_of(&key_str) {
return Err(ShellError::ColumnDefinedTwice {
col_name: key_str,
second_use: key.span,
first_use: key_spans[i],
});
} else {
key_spans.push(key.span);
record.push(key_str, convert_to_value(val, span, original_text)?);
}
}
RecordItem::Spread(_, inner) => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "spread operator not supported in nuon".into(),
span: inner.span,
});
}
}
}
Ok(Value::record(record, span))
}
Expr::RowCondition(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "row conditions not supported in nuon".into(),
span: expr.span,
}),
Expr::Signature(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "signatures not supported in nuon".into(),
span: expr.span,
}),
Expr::String(s) => Ok(Value::string(s, span)),
Expr::StringInterpolation(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "string interpolation not supported in nuon".into(),
span: expr.span,
}),
Expr::Subexpression(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "subexpressions not supported in nuon".into(),
span: expr.span,
}),
Expr::Table(mut headers, cells) => {
let mut cols = vec![];
let mut output = vec![];
for key in headers.iter_mut() {
let key_str = match &mut key.expr {
Expr::String(key_str) => key_str,
_ => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "only strings can be keys".into(),
span: expr.span,
})
}
};
if let Some(idx) = cols.iter().position(|existing| existing == key_str) {
return Err(ShellError::ColumnDefinedTwice {
col_name: key_str.clone(),
second_use: key.span,
first_use: headers[idx].span,
});
} else {
cols.push(std::mem::take(key_str));
}
}
for row in cells {
if cols.len() != row.len() {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "table has mismatched columns".into(),
span: expr.span,
});
}
let record = cols
.iter()
.zip(row)
.map(|(col, cell)| {
convert_to_value(cell, span, original_text).map(|val| (col.clone(), val))
})
.collect::<Result<_, _>>()?;
output.push(Value::record(record, span));
}
Ok(Value::list(output, span))
}
Expr::ValueWithUnit(val, unit) => {
let size = match val.expr {
Expr::Int(val) => val,
_ => {
return Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "non-integer unit value".into(),
span: expr.span,
})
}
};
match unit.item {
Unit::Byte => Ok(Value::filesize(size, span)),
Unit::Kilobyte => Ok(Value::filesize(size * 1000, span)),
Unit::Megabyte => Ok(Value::filesize(size * 1000 * 1000, span)),
Unit::Gigabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000, span)),
Unit::Terabyte => Ok(Value::filesize(size * 1000 * 1000 * 1000 * 1000, span)),
Unit::Petabyte => Ok(Value::filesize(
size * 1000 * 1000 * 1000 * 1000 * 1000,
span,
)),
Unit::Exabyte => Ok(Value::filesize(
size * 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
span,
)),
Unit::Kibibyte => Ok(Value::filesize(size * 1024, span)),
Unit::Mebibyte => Ok(Value::filesize(size * 1024 * 1024, span)),
Unit::Gibibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024, span)),
Unit::Tebibyte => Ok(Value::filesize(size * 1024 * 1024 * 1024 * 1024, span)),
Unit::Pebibyte => Ok(Value::filesize(
size * 1024 * 1024 * 1024 * 1024 * 1024,
span,
)),
Unit::Exbibyte => Ok(Value::filesize(
size * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
span,
)),
Unit::Nanosecond => Ok(Value::duration(size, span)),
Unit::Microsecond => Ok(Value::duration(size * 1000, span)),
Unit::Millisecond => Ok(Value::duration(size * 1000 * 1000, span)),
Unit::Second => Ok(Value::duration(size * 1000 * 1000 * 1000, span)),
Unit::Minute => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60, span)),
Unit::Hour => Ok(Value::duration(size * 1000 * 1000 * 1000 * 60 * 60, span)),
Unit::Day => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24) {
Some(val) => Ok(Value::duration(val, span)),
None => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "day duration too large".into(),
msg: "day duration too large".into(),
span: expr.span,
}),
},
Unit::Week => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24 * 7) {
Some(val) => Ok(Value::duration(val, span)),
None => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "week duration too large".into(),
msg: "week duration too large".into(),
span: expr.span,
}),
},
}
}
Expr::Var(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "variables not supported in nuon".into(),
span: expr.span,
}),
Expr::VarDecl(..) => Err(ShellError::OutsideSpannedLabeledError {
src: original_text.to_string(),
error: "Error when loading".into(),
msg: "variable declarations not supported in nuon".into(),
span: expr.span,
}),
}
}

430
crates/nuon/src/lib.rs Normal file
View File

@ -0,0 +1,430 @@
//! Support for the NUON format.
//!
//! The NUON format is a superset of JSON designed to fit the feel of Nushell.
//! Some of its extra features are
//! - trailing commas are allowed
//! - quotes are not required around keys
mod from;
mod to;
pub use from::from_nuon;
pub use to::to_nuon;
#[cfg(test)]
mod tests {
use chrono::DateTime;
use nu_protocol::{ast::RangeInclusion, engine::Closure, record, IntRange, Range, Span, Value};
use crate::{from_nuon, to_nuon};
/// test something of the form
/// ```nushell
/// $v | from nuon | to nuon | $in == $v
/// ```
///
/// an optional "middle" value can be given to test what the value is between `from nuon` and
/// `to nuon`.
fn nuon_end_to_end(input: &str, middle: Option<Value>) {
let val = from_nuon(input, None).unwrap();
if let Some(m) = middle {
assert_eq!(val, m);
}
assert_eq!(to_nuon(&val, true, None, None, None).unwrap(), input);
}
#[test]
fn list_of_numbers() {
nuon_end_to_end(
"[1, 2, 3]",
Some(Value::test_list(vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
])),
);
}
#[test]
fn list_of_strings() {
nuon_end_to_end(
"[abc, xyz, def]",
Some(Value::test_list(vec![
Value::test_string("abc"),
Value::test_string("xyz"),
Value::test_string("def"),
])),
);
}
#[test]
fn table() {
nuon_end_to_end(
"[[my, columns]; [abc, xyz], [def, ijk]]",
Some(Value::test_list(vec![
Value::test_record(record!(
"my" => Value::test_string("abc"),
"columns" => Value::test_string("xyz")
)),
Value::test_record(record!(
"my" => Value::test_string("def"),
"columns" => Value::test_string("ijk")
)),
])),
);
}
#[test]
fn from_nuon_illegal_table() {
assert!(
from_nuon("[[repeated repeated]; [abc, xyz], [def, ijk]]", None)
.unwrap_err()
.to_string()
.contains("Record field or table column used twice: repeated")
);
}
#[test]
fn bool() {
nuon_end_to_end("false", Some(Value::test_bool(false)));
}
#[test]
fn escaping() {
nuon_end_to_end(r#""hello\"world""#, None);
}
#[test]
fn escaping2() {
nuon_end_to_end(r#""hello\\world""#, None);
}
#[test]
fn escaping3() {
nuon_end_to_end(
r#"[hello\\world]"#,
Some(Value::test_list(vec![Value::test_string(
r#"hello\\world"#,
)])),
);
}
#[test]
fn escaping4() {
nuon_end_to_end(r#"["hello\"world"]"#, None);
}
#[test]
fn escaping5() {
nuon_end_to_end(r#"{s: "hello\"world"}"#, None);
}
#[test]
fn negative_int() {
nuon_end_to_end("-1", Some(Value::test_int(-1)));
}
#[test]
fn records() {
nuon_end_to_end(
r#"{name: "foo bar", age: 100, height: 10}"#,
Some(Value::test_record(record!(
"name" => Value::test_string("foo bar"),
"age" => Value::test_int(100),
"height" => Value::test_int(10),
))),
);
}
#[test]
fn range() {
nuon_end_to_end(
"1..42",
Some(Value::test_range(Range::IntRange(
IntRange::new(
Value::test_int(1),
Value::test_int(2),
Value::test_int(42),
RangeInclusion::Inclusive,
Span::unknown(),
)
.unwrap(),
))),
);
}
#[test]
fn filesize() {
nuon_end_to_end("1024b", Some(Value::test_filesize(1024)));
assert_eq!(from_nuon("1kib", None).unwrap(), Value::test_filesize(1024),);
}
#[test]
fn duration() {
nuon_end_to_end("60000000000ns", Some(Value::test_duration(60_000_000_000)));
}
#[test]
fn to_nuon_datetime() {
nuon_end_to_end(
"1970-01-01T00:00:00+00:00",
Some(Value::test_date(DateTime::UNIX_EPOCH.into())),
);
}
#[test]
fn to_nuon_errs_on_closure() {
assert!(to_nuon(
&Value::test_closure(Closure {
block_id: 0,
captures: vec![]
}),
true,
None,
None,
None,
)
.unwrap_err()
.to_string()
.contains("Unsupported input"));
}
#[test]
fn binary() {
nuon_end_to_end(
"0x[ABCDEF]",
Some(Value::test_binary(vec![0xab, 0xcd, 0xef])),
);
}
#[test]
fn binary_roundtrip() {
assert_eq!(
to_nuon(
&from_nuon("0x[1f ff]", None).unwrap(),
true,
None,
None,
None
)
.unwrap(),
"0x[1FFF]"
);
}
#[test]
fn read_sample_data() {
assert_eq!(
from_nuon(
include_str!("../../../tests/fixtures/formats/sample.nuon"),
None,
)
.unwrap(),
Value::test_list(vec![
Value::test_list(vec![
Value::test_record(record!(
"a" => Value::test_int(1),
"nuon" => Value::test_int(2),
"table" => Value::test_int(3)
)),
Value::test_record(record!(
"a" => Value::test_int(4),
"nuon" => Value::test_int(5),
"table" => Value::test_int(6)
)),
]),
Value::test_filesize(100 * 1024),
Value::test_duration(100 * 1_000_000_000),
Value::test_bool(true),
Value::test_record(record!(
"name" => Value::test_string("Bobby"),
"age" => Value::test_int(99)
),),
Value::test_binary(vec![0x11, 0xff, 0xee, 0x1f]),
])
);
}
#[test]
fn float_doesnt_become_int() {
assert_eq!(
to_nuon(&Value::test_float(1.0), true, None, None, None).unwrap(),
"1.0"
);
}
#[test]
fn float_inf_parsed_properly() {
assert_eq!(
to_nuon(&Value::test_float(f64::INFINITY), true, None, None, None).unwrap(),
"inf"
);
}
#[test]
fn float_neg_inf_parsed_properly() {
assert_eq!(
to_nuon(
&Value::test_float(f64::NEG_INFINITY),
true,
None,
None,
None
)
.unwrap(),
"-inf"
);
}
#[test]
fn float_nan_parsed_properly() {
assert_eq!(
to_nuon(&Value::test_float(-f64::NAN), true, None, None, None).unwrap(),
"NaN"
);
}
#[test]
fn to_nuon_converts_columns_with_spaces() {
assert!(from_nuon(
&to_nuon(
&Value::test_list(vec![
Value::test_record(record!(
"a" => Value::test_int(1),
"b" => Value::test_int(2),
"c d" => Value::test_int(3)
)),
Value::test_record(record!(
"a" => Value::test_int(4),
"b" => Value::test_int(5),
"c d" => Value::test_int(6)
))
]),
true,
None,
None,
None
)
.unwrap(),
None,
)
.is_ok());
}
#[test]
fn to_nuon_quotes_empty_string() {
let res = to_nuon(&Value::test_string(""), true, None, None, None);
assert!(res.is_ok());
assert_eq!(res.unwrap(), r#""""#);
}
#[test]
fn to_nuon_quotes_empty_string_in_list() {
nuon_end_to_end(
r#"[""]"#,
Some(Value::test_list(vec![Value::test_string("")])),
);
}
#[test]
fn to_nuon_quotes_empty_string_in_table() {
nuon_end_to_end(
"[[a, b]; [\"\", la], [le, lu]]",
Some(Value::test_list(vec![
Value::test_record(record!(
"a" => Value::test_string(""),
"b" => Value::test_string("la"),
)),
Value::test_record(record!(
"a" => Value::test_string("le"),
"b" => Value::test_string("lu"),
)),
])),
);
}
#[test]
fn does_not_quote_strings_unnecessarily() {
assert_eq!(
to_nuon(
&Value::test_list(vec![
Value::test_record(record!(
"a" => Value::test_int(1),
"b" => Value::test_int(2),
"c d" => Value::test_int(3)
)),
Value::test_record(record!(
"a" => Value::test_int(4),
"b" => Value::test_int(5),
"c d" => Value::test_int(6)
))
]),
true,
None,
None,
None
)
.unwrap(),
"[[a, b, \"c d\"]; [1, 2, 3], [4, 5, 6]]"
);
assert_eq!(
to_nuon(
&Value::test_record(record!(
"ro name" => Value::test_string("sam"),
"rank" => Value::test_int(10)
)),
true,
None,
None,
None
)
.unwrap(),
"{\"ro name\": sam, rank: 10}"
);
}
#[test]
fn quotes_some_strings_necessarily() {
nuon_end_to_end(
r#"["true", "false", "null", "NaN", "NAN", "nan", "+nan", "-nan", "inf", "+inf", "-inf", "INF", "Infinity", "+Infinity", "-Infinity", "INFINITY", "+19.99", "-19.99", "19.99b", "19.99kb", "19.99mb", "19.99gb", "19.99tb", "19.99pb", "19.99eb", "19.99zb", "19.99kib", "19.99mib", "19.99gib", "19.99tib", "19.99pib", "19.99eib", "19.99zib", "19ns", "19us", "19ms", "19sec", "19min", "19hr", "19day", "19wk", "-11.0..-15.0", "11.0..-15.0", "-11.0..15.0", "-11.0..<-15.0", "11.0..<-15.0", "-11.0..<15.0", "-11.0..", "11.0..", "..15.0", "..-15.0", "..<15.0", "..<-15.0", "2000-01-01", "2022-02-02T14:30:00", "2022-02-02T14:30:00+05:00", ", ", "", "&&"]"#,
None,
);
}
#[test]
// NOTE: this test could be stronger, but the output of [`from_nuon`] on the content of `../../../tests/fixtures/formats/code.nu` is
// not the same in the CI and locally...
//
// ## locally
// ```
// OutsideSpannedLabeledError {
// src: "register",
// error: "Error when loading",
// msg: "calls not supported in nuon",
// span: Span { start: 0, end: 8 }
// }
// ```
//
// ## in the CI
// ```
// GenericError {
// error: "error when parsing nuon text",
// msg: "could not parse nuon text",
// span: None,
// help: None,
// inner: [OutsideSpannedLabeledError {
// src: "register",
// error: "error when parsing",
// msg: "Unknown state.",
// span: Span { start: 0, end: 8 }
// }]
// }
// ```
fn read_code_should_fail_rather_than_panic() {
assert!(from_nuon(
include_str!("../../../tests/fixtures/formats/code.nu"),
None,
)
.is_err());
}
}

283
crates/nuon/src/to.rs Normal file
View File

@ -0,0 +1,283 @@
use core::fmt::Write;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use nu_engine::get_columns;
use nu_parser::escape_quote_string;
use nu_protocol::{Range, ShellError, Span, Value};
use std::ops::Bound;
/// convert an actual Nushell [`Value`] to a raw string representation of the NUON data
///
/// ## Arguments
/// - `tabs` and `indent` control the level of indentation, expressed in _tabulations_ and _spaces_
/// respectively. `tabs` has higher precedence over `indent`.
/// - `raw` has the highest precedence and will for the output to be _raw_, i.e. the [`Value`] will
/// be _serialized_ on a single line, without extra whitespaces.
///
/// > **Note**
/// > a [`Span`] can be passed to [`to_nuon`] if there is context available to the caller, e.g. when
/// > using this function in a command implementation such as [`to nuon`](https://www.nushell.sh/commands/docs/to_nuon.html).
///
/// also see [`super::from_nuon`] for the inverse operation
pub fn to_nuon(
input: &Value,
raw: bool,
tabs: Option<usize>,
indent: Option<usize>,
span: Option<Span>,
) -> Result<String, ShellError> {
let span = span.unwrap_or(Span::unknown());
let nuon_result = if raw {
value_to_string(input, span, 0, None)?
} else if let Some(tab_count) = tabs {
value_to_string(input, span, 0, Some(&"\t".repeat(tab_count)))?
} else if let Some(indent) = indent {
value_to_string(input, span, 0, Some(&" ".repeat(indent)))?
} else {
value_to_string(input, span, 0, None)?
};
Ok(nuon_result)
}
fn value_to_string(
v: &Value,
span: Span,
depth: usize,
indent: Option<&str>,
) -> Result<String, ShellError> {
let (nl, sep) = get_true_separators(indent);
let idt = get_true_indentation(depth, indent);
let idt_po = get_true_indentation(depth + 1, indent);
let idt_pt = get_true_indentation(depth + 2, indent);
match v {
Value::Binary { val, .. } => {
let mut s = String::with_capacity(2 * val.len());
for byte in val {
if write!(s, "{byte:02X}").is_err() {
return Err(ShellError::UnsupportedInput {
msg: "could not convert binary to string".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
});
}
}
Ok(format!("0x[{s}]"))
}
Value::Block { .. } => Err(ShellError::UnsupportedInput {
msg: "blocks are currently not nuon-compatible".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Closure { .. } => Err(ShellError::UnsupportedInput {
msg: "closures are currently not nuon-compatible".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Bool { val, .. } => {
if *val {
Ok("true".to_string())
} else {
Ok("false".to_string())
}
}
Value::CellPath { .. } => Err(ShellError::UnsupportedInput {
msg: "cell-paths are currently not nuon-compatible".to_string(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Custom { .. } => Err(ShellError::UnsupportedInput {
msg: "custom values are currently not nuon-compatible".to_string(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Date { val, .. } => Ok(val.to_rfc3339()),
// FIXME: make durations use the shortest lossless representation.
Value::Duration { val, .. } => Ok(format!("{}ns", *val)),
// Propagate existing errors
Value::Error { error, .. } => Err(*error.clone()),
// FIXME: make filesizes use the shortest lossless representation.
Value::Filesize { val, .. } => Ok(format!("{}b", *val)),
Value::Float { val, .. } => {
// This serialises these as 'nan', 'inf' and '-inf', respectively.
if &val.round() == val && val.is_finite() {
Ok(format!("{}.0", *val))
} else {
Ok(format!("{}", *val))
}
}
Value::Int { val, .. } => Ok(format!("{}", *val)),
Value::List { vals, .. } => {
let headers = get_columns(vals);
if !headers.is_empty() && vals.iter().all(|x| x.columns().eq(headers.iter())) {
// Table output
let headers: Vec<String> = headers
.iter()
.map(|string| {
if needs_quotes(string) {
format!("{idt}\"{string}\"")
} else {
format!("{idt}{string}")
}
})
.collect();
let headers_output = headers.join(&format!(",{sep}{nl}{idt_pt}"));
let mut table_output = vec![];
for val in vals {
let mut row = vec![];
if let Value::Record { val, .. } = val {
for val in val.values() {
row.push(value_to_string_without_quotes(
val,
span,
depth + 2,
indent,
)?);
}
}
table_output.push(row.join(&format!(",{sep}{nl}{idt_pt}")));
}
Ok(format!(
"[{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}];{sep}{nl}{idt_po}[{nl}{idt_pt}{}{nl}{idt_po}]{nl}{idt}]",
headers_output,
table_output.join(&format!("{nl}{idt_po}],{sep}{nl}{idt_po}[{nl}{idt_pt}"))
))
} else {
let mut collection = vec![];
for val in vals {
collection.push(format!(
"{idt_po}{}",
value_to_string_without_quotes(val, span, depth + 1, indent,)?
));
}
Ok(format!(
"[{nl}{}{nl}{idt}]",
collection.join(&format!(",{sep}{nl}"))
))
}
}
Value::Nothing { .. } => Ok("null".to_string()),
Value::Range { val, .. } => match val {
Range::IntRange(range) => Ok(range.to_string()),
Range::FloatRange(range) => {
let start =
value_to_string(&Value::float(range.start(), span), span, depth + 1, indent)?;
match range.end() {
Bound::Included(end) => Ok(format!(
"{}..{}",
start,
value_to_string(&Value::float(end, span), span, depth + 1, indent)?
)),
Bound::Excluded(end) => Ok(format!(
"{}..<{}",
start,
value_to_string(&Value::float(end, span), span, depth + 1, indent)?
)),
Bound::Unbounded => Ok(format!("{start}..",)),
}
}
},
Value::Record { val, .. } => {
let mut collection = vec![];
for (col, val) in &**val {
collection.push(if needs_quotes(col) {
format!(
"{idt_po}\"{}\": {}",
col,
value_to_string_without_quotes(val, span, depth + 1, indent)?
)
} else {
format!(
"{idt_po}{}: {}",
col,
value_to_string_without_quotes(val, span, depth + 1, indent)?
)
});
}
Ok(format!(
"{{{nl}{}{nl}{idt}}}",
collection.join(&format!(",{sep}{nl}"))
))
}
Value::LazyRecord { val, .. } => {
let collected = val.collect()?;
value_to_string(&collected, span, depth + 1, indent)
}
// All strings outside data structures are quoted because they are in 'command position'
// (could be mistaken for commands by the Nu parser)
Value::String { val, .. } => Ok(escape_quote_string(val)),
Value::Glob { val, .. } => Ok(escape_quote_string(val)),
}
}
fn get_true_indentation(depth: usize, indent: Option<&str>) -> String {
match indent {
Some(i) => i.repeat(depth),
None => "".to_string(),
}
}
fn get_true_separators(indent: Option<&str>) -> (String, String) {
match indent {
Some(_) => ("\n".to_string(), "".to_string()),
None => ("".to_string(), " ".to_string()),
}
}
fn value_to_string_without_quotes(
v: &Value,
span: Span,
depth: usize,
indent: Option<&str>,
) -> Result<String, ShellError> {
match v {
Value::String { val, .. } => Ok({
if needs_quotes(val) {
escape_quote_string(val)
} else {
val.clone()
}
}),
_ => value_to_string(v, span, depth, indent),
}
}
// This hits, in order:
// • Any character of []:`{}#'";()|$,
// • Any digit (\d)
// • Any whitespace (\s)
// • Case-insensitive sign-insensitive float "keywords" inf, infinity and nan.
static NEEDS_QUOTES_REGEX: Lazy<Regex> = Lazy::new(|| {
Regex::new(r#"[\[\]:`\{\}#'";\(\)\|\$,\d\s]|(?i)^[+\-]?(inf(inity)?|nan)$"#)
.expect("internal error: NEEDS_QUOTES_REGEX didn't compile")
});
fn needs_quotes(string: &str) -> bool {
if string.is_empty() {
return true;
}
// These are case-sensitive keywords
match string {
// `true`/`false`/`null` are active keywords in JSON and NUON
// `&&` is denied by the nu parser for diagnostics reasons
// (https://github.com/nushell/nushell/pull/7241)
// TODO: remove the extra check in the nuon codepath
"true" | "false" | "null" | "&&" => return true,
_ => (),
};
// All other cases are handled here
NEEDS_QUOTES_REGEX.is_match(string).unwrap_or(false)
}