From dad956b2eefec0642c8a794dc326290238213cd9 Mon Sep 17 00:00:00 2001 From: Darren Schroeder <343840+fdncred@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:51:22 -0600 Subject: [PATCH] more closure serialization (#14698) # Description This PR introduces a switch `--serialize` that allows serializing of types that cannot be deserialized. Right now it only serializes closures as strings in `to toml`, `to json`, `to nuon`, `to text`, some indirect `to html` and `to yaml`. A lot of the changes are just weaving the engine_state through calling functions and the rest is just repetitive way of getting the closure block span and grabbing the span's text. In places where it has to report `` I changed it to `closure_123`. It always seemed like the `<>` were not very nushell-y. This is still a breaking change. I think this could also help with systematic translation of old config to new config file. # User-Facing Changes # Tests + Formatting # After Submitting --- crates/nu-command/src/debug/explain.rs | 36 +++++-- crates/nu-command/src/debug/inspect.rs | 4 +- crates/nu-command/src/debug/inspect_table.rs | 56 +++++++---- crates/nu-command/src/filters/uniq.rs | 12 ++- crates/nu-command/src/formats/to/json.rs | 52 ++++++++-- crates/nu-command/src/formats/to/nuon.rs | 8 +- crates/nu-command/src/formats/to/text.rs | 65 ++++++++++-- crates/nu-command/src/formats/to/toml.rs | 65 +++++++++--- crates/nu-command/src/formats/to/yaml.rs | 59 ++++++++--- crates/nu-command/src/network/http/client.rs | 18 +++- crates/nu-command/src/network/http/delete.rs | 1 + crates/nu-command/src/network/http/get.rs | 1 + crates/nu-command/src/network/http/head.rs | 9 +- crates/nu-command/src/network/http/options.rs | 1 + crates/nu-command/src/network/http/patch.rs | 1 + crates/nu-command/src/network/http/post.rs | 1 + crates/nu-command/src/network/http/put.rs | 1 + .../tests/commands/database/into_sqlite.rs | 14 ++- .../tests/format_conversions/nuon.rs | 1 + crates/nu-protocol/src/value/mod.rs | 2 +- crates/nuon/src/lib.rs | 94 +++++++++++++++--- crates/nuon/src/to.rs | 99 +++++++++++++++---- tests/eval/mod.rs | 22 ++++- 23 files changed, 509 insertions(+), 113 deletions(-) diff --git a/crates/nu-command/src/debug/explain.rs b/crates/nu-command/src/debug/explain.rs index 7d01727ce6..190bd9cbe5 100644 --- a/crates/nu-command/src/debug/explain.rs +++ b/crates/nu-command/src/debug/explain.rs @@ -154,7 +154,8 @@ fn get_arguments( eval_expression_fn, ); let arg_type = "expr"; - let arg_value_name = debug_string_without_formatting(&evaluated_expression); + let arg_value_name = + debug_string_without_formatting(engine_state, &evaluated_expression); let arg_value_type = &evaluated_expression.get_type().to_string(); let evaled_span = evaluated_expression.span(); let arg_value_name_span_start = evaled_span.start as i64; @@ -174,7 +175,8 @@ fn get_arguments( let arg_type = "positional"; let evaluated_expression = get_expression_as_value(engine_state, stack, inner_expr, eval_expression_fn); - let arg_value_name = debug_string_without_formatting(&evaluated_expression); + let arg_value_name = + debug_string_without_formatting(engine_state, &evaluated_expression); let arg_value_type = &evaluated_expression.get_type().to_string(); let evaled_span = evaluated_expression.span(); let arg_value_name_span_start = evaled_span.start as i64; @@ -193,7 +195,8 @@ fn get_arguments( let arg_type = "unknown"; let evaluated_expression = get_expression_as_value(engine_state, stack, inner_expr, eval_expression_fn); - let arg_value_name = debug_string_without_formatting(&evaluated_expression); + let arg_value_name = + debug_string_without_formatting(engine_state, &evaluated_expression); let arg_value_type = &evaluated_expression.get_type().to_string(); let evaled_span = evaluated_expression.span(); let arg_value_name_span_start = evaled_span.start as i64; @@ -212,7 +215,8 @@ fn get_arguments( let arg_type = "spread"; let evaluated_expression = get_expression_as_value(engine_state, stack, inner_expr, eval_expression_fn); - let arg_value_name = debug_string_without_formatting(&evaluated_expression); + let arg_value_name = + debug_string_without_formatting(engine_state, &evaluated_expression); let arg_value_type = &evaluated_expression.get_type().to_string(); let evaled_span = evaluated_expression.span(); let arg_value_name_span_start = evaled_span.start as i64; @@ -245,7 +249,7 @@ fn get_expression_as_value( } } -pub fn debug_string_without_formatting(value: &Value) -> String { +pub fn debug_string_without_formatting(engine_state: &EngineState, value: &Value) -> String { match value { Value::Bool { val, .. } => val.to_string(), Value::Int { val, .. } => val.to_string(), @@ -259,19 +263,31 @@ pub fn debug_string_without_formatting(value: &Value) -> String { Value::List { vals: val, .. } => format!( "[{}]", val.iter() - .map(debug_string_without_formatting) + .map(|v| debug_string_without_formatting(engine_state, v)) .collect::>() .join(" ") ), Value::Record { val, .. } => format!( "{{{}}}", val.iter() - .map(|(x, y)| format!("{}: {}", x, debug_string_without_formatting(y))) + .map(|(x, y)| format!( + "{}: {}", + x, + debug_string_without_formatting(engine_state, y) + )) .collect::>() .join(" ") ), - //TODO: It would be good to drill deeper into closures. - Value::Closure { val, .. } => format!("", val.block_id.get()), + Value::Closure { val, .. } => { + let block = engine_state.get_block(val.block_id); + if let Some(span) = block.span { + let contents_bytes = engine_state.get_span_contents(span); + let contents_string = String::from_utf8_lossy(contents_bytes); + contents_string.to_string() + } else { + String::new() + } + } Value::Nothing { .. } => String::new(), Value::Error { error, .. } => format!("{error:?}"), Value::Binary { val, .. } => format!("{val:?}"), @@ -280,7 +296,7 @@ pub fn debug_string_without_formatting(value: &Value) -> String { // that critical here Value::Custom { val, .. } => val .to_base_value(value.span()) - .map(|val| debug_string_without_formatting(&val)) + .map(|val| debug_string_without_formatting(engine_state, &val)) .unwrap_or_else(|_| format!("<{}>", val.type_name())), } } diff --git a/crates/nu-command/src/debug/inspect.rs b/crates/nu-command/src/debug/inspect.rs index f8ef63ce02..8a8fdd04f4 100644 --- a/crates/nu-command/src/debug/inspect.rs +++ b/crates/nu-command/src/debug/inspect.rs @@ -23,7 +23,7 @@ impl Command for Inspect { fn run( &self, - _engine_state: &EngineState, + engine_state: &EngineState, _stack: &mut Stack, call: &Call, input: PipelineData, @@ -40,7 +40,7 @@ impl Command for Inspect { let (cols, _rows) = terminal_size().unwrap_or((0, 0)); - let table = inspect_table::build_table(input_val, description, cols as usize); + let table = inspect_table::build_table(engine_state, input_val, description, cols as usize); // Note that this is printed to stderr. The reason for this is so it doesn't disrupt the regular nushell // tabular output. If we printed to stdout, nushell would get confused with two outputs. diff --git a/crates/nu-command/src/debug/inspect_table.rs b/crates/nu-command/src/debug/inspect_table.rs index e2a2483484..5f18eb5be3 100644 --- a/crates/nu-command/src/debug/inspect_table.rs +++ b/crates/nu-command/src/debug/inspect_table.rs @@ -1,19 +1,23 @@ // note: Seems like could be simplified // IMHO: it shall not take 300+ lines :) +use self::{global_horizontal_char::SetHorizontalChar, set_widths::SetWidths}; +use nu_protocol::engine::EngineState; use nu_protocol::Value; use nu_table::{string_width, string_wrap}; - use tabled::{ grid::config::ColoredConfig, settings::{peaker::Priority, width::Wrap, Settings, Style}, Table, }; -use self::{global_horizontal_char::SetHorizontalChar, set_widths::SetWidths}; - -pub fn build_table(value: Value, description: String, termsize: usize) -> String { - let (head, mut data) = util::collect_input(value); +pub fn build_table( + engine_state: &EngineState, + value: Value, + description: String, + termsize: usize, +) -> String { + let (head, mut data) = util::collect_input(engine_state, value); let count_columns = head.len(); data.insert(0, head); @@ -195,10 +199,14 @@ fn push_empty_column(data: &mut Vec>) { mod util { use crate::debug::explain::debug_string_without_formatting; use nu_engine::get_columns; + use nu_protocol::engine::EngineState; use nu_protocol::Value; /// Try to build column names and a table grid. - pub fn collect_input(value: Value) -> (Vec, Vec>) { + pub fn collect_input( + engine_state: &EngineState, + value: Value, + ) -> (Vec, Vec>) { let span = value.span(); match value { Value::Record { val: record, .. } => { @@ -210,7 +218,7 @@ mod util { }, match vals .into_iter() - .map(|s| debug_string_without_formatting(&s)) + .map(|s| debug_string_without_formatting(engine_state, &s)) .collect::>() { vals if vals.is_empty() => vec![], @@ -220,7 +228,7 @@ mod util { } Value::List { vals, .. } => { let mut columns = get_columns(&vals); - let data = convert_records_to_dataset(&columns, vals); + let data = convert_records_to_dataset(engine_state, &columns, vals); if columns.is_empty() { columns = vec![String::from("")]; @@ -232,7 +240,7 @@ mod util { let lines = val .lines() .map(|line| Value::string(line.to_string(), span)) - .map(|val| vec![debug_string_without_formatting(&val)]) + .map(|val| vec![debug_string_without_formatting(engine_state, &val)]) .collect(); (vec![String::from("")], lines) @@ -240,47 +248,59 @@ mod util { Value::Nothing { .. } => (vec![], vec![]), value => ( vec![String::from("")], - vec![vec![debug_string_without_formatting(&value)]], + vec![vec![debug_string_without_formatting(engine_state, &value)]], ), } } - fn convert_records_to_dataset(cols: &[String], records: Vec) -> Vec> { + fn convert_records_to_dataset( + engine_state: &EngineState, + cols: &[String], + records: Vec, + ) -> Vec> { if !cols.is_empty() { - create_table_for_record(cols, &records) + create_table_for_record(engine_state, cols, &records) } else if cols.is_empty() && records.is_empty() { vec![] } else if cols.len() == records.len() { vec![records .into_iter() - .map(|s| debug_string_without_formatting(&s)) + .map(|s| debug_string_without_formatting(engine_state, &s)) .collect()] } else { records .into_iter() - .map(|record| vec![debug_string_without_formatting(&record)]) + .map(|record| vec![debug_string_without_formatting(engine_state, &record)]) .collect() } } - fn create_table_for_record(headers: &[String], items: &[Value]) -> Vec> { + fn create_table_for_record( + engine_state: &EngineState, + headers: &[String], + items: &[Value], + ) -> Vec> { let mut data = vec![Vec::new(); items.len()]; for (i, item) in items.iter().enumerate() { - let row = record_create_row(headers, item); + let row = record_create_row(engine_state, headers, item); data[i] = row; } data } - fn record_create_row(headers: &[String], item: &Value) -> Vec { + fn record_create_row( + engine_state: &EngineState, + headers: &[String], + item: &Value, + ) -> Vec { if let Value::Record { val, .. } = item { headers .iter() .map(|col| { val.get(col) - .map(debug_string_without_formatting) + .map(|v| debug_string_without_formatting(engine_state, v)) .unwrap_or_else(String::new) }) .collect() diff --git a/crates/nu-command/src/filters/uniq.rs b/crates/nu-command/src/filters/uniq.rs index 3a82e73aa1..8042963d75 100644 --- a/crates/nu-command/src/filters/uniq.rs +++ b/crates/nu-command/src/filters/uniq.rs @@ -213,9 +213,15 @@ fn sort_attributes(val: Value) -> Value { } } -fn generate_key(item: &ValueCounter) -> Result { +fn generate_key(engine_state: &EngineState, item: &ValueCounter) -> Result { let value = sort_attributes(item.val_to_compare.clone()); //otherwise, keys could be different for Records - nuon::to_nuon(&value, nuon::ToStyle::Raw, Some(Span::unknown())) + nuon::to_nuon( + engine_state, + &value, + nuon::ToStyle::Raw, + Some(Span::unknown()), + false, + ) } fn generate_results_with_count(head: Span, uniq_values: Vec) -> Vec { @@ -264,7 +270,7 @@ pub fn uniq( .try_fold( HashMap::::new(), |mut counter, item| { - let key = generate_key(&item); + let key = generate_key(engine_state, &item); match key { Ok(key) => { diff --git a/crates/nu-command/src/formats/to/json.rs b/crates/nu-command/src/formats/to/json.rs index 27c0af856a..0ba5427038 100644 --- a/crates/nu-command/src/formats/to/json.rs +++ b/crates/nu-command/src/formats/to/json.rs @@ -25,6 +25,11 @@ impl Command for ToJson { "specify indentation tab quantity", Some('t'), ) + .switch( + "serialize", + "serialize nushell types that cannot be deserialized", + Some('s'), + ) .category(Category::Formats) } @@ -42,12 +47,13 @@ impl Command for ToJson { let raw = call.has_flag(engine_state, stack, "raw")?; let use_tabs = call.get_flag(engine_state, stack, "tabs")?; let indent = call.get_flag(engine_state, stack, "indent")?; + let serialize_types = call.has_flag(engine_state, stack, "serialize")?; let span = call.head; // allow ranges to expand and turn into array let input = input.try_expand_range()?; let value = input.into_value(span)?; - let json_value = value_to_json_value(&value)?; + let json_value = value_to_json_value(engine_state, &value, serialize_types)?; let json_result = if raw { nu_json::to_string_raw(&json_value) @@ -105,7 +111,11 @@ impl Command for ToJson { } } -pub fn value_to_json_value(v: &Value) -> Result { +pub fn value_to_json_value( + engine_state: &EngineState, + v: &Value, + serialize_types: bool, +) -> Result { let span = v.span(); Ok(match v { Value::Bool { val, .. } => nu_json::Value::Bool(*val), @@ -127,31 +137,57 @@ pub fn value_to_json_value(v: &Value) -> Result { .collect::, ShellError>>()?, ), - Value::List { vals, .. } => nu_json::Value::Array(json_list(vals)?), + Value::List { vals, .. } => { + nu_json::Value::Array(json_list(engine_state, vals, serialize_types)?) + } Value::Error { error, .. } => return Err(*error.clone()), - Value::Closure { .. } | Value::Range { .. } => nu_json::Value::Null, + Value::Closure { val, .. } => { + if serialize_types { + let block = engine_state.get_block(val.block_id); + if let Some(span) = block.span { + let contents_bytes = engine_state.get_span_contents(span); + let contents_string = String::from_utf8_lossy(contents_bytes); + nu_json::Value::String(contents_string.to_string()) + } else { + nu_json::Value::String(format!( + "unable to retrieve block contents for json block_id {}", + val.block_id.get() + )) + } + } else { + nu_json::Value::Null + } + } + Value::Range { .. } => nu_json::Value::Null, Value::Binary { val, .. } => { nu_json::Value::Array(val.iter().map(|x| nu_json::Value::U64(*x as u64)).collect()) } Value::Record { val, .. } => { let mut m = nu_json::Map::new(); for (k, v) in &**val { - m.insert(k.clone(), value_to_json_value(v)?); + m.insert( + k.clone(), + value_to_json_value(engine_state, v, serialize_types)?, + ); } nu_json::Value::Object(m) } Value::Custom { val, .. } => { let collected = val.to_base_value(span)?; - value_to_json_value(&collected)? + value_to_json_value(engine_state, &collected, serialize_types)? } }) } -fn json_list(input: &[Value]) -> Result, ShellError> { +fn json_list( + engine_state: &EngineState, + input: &[Value], + serialize_types: bool, +) -> Result, ShellError> { let mut out = vec![]; for value in input { - out.push(value_to_json_value(value)?); + out.push(value_to_json_value(engine_state, value, serialize_types)?); } Ok(out) diff --git a/crates/nu-command/src/formats/to/nuon.rs b/crates/nu-command/src/formats/to/nuon.rs index 8f4fc80e7f..4f7cde1d0b 100644 --- a/crates/nu-command/src/formats/to/nuon.rs +++ b/crates/nu-command/src/formats/to/nuon.rs @@ -28,6 +28,11 @@ impl Command for ToNuon { "specify indentation tab quantity", Some('t'), ) + .switch( + "serialize", + "serialize nushell types that cannot be deserialized", + Some('s'), + ) .category(Category::Formats) } @@ -47,6 +52,7 @@ impl Command for ToNuon { .unwrap_or_default() .with_content_type(Some("application/x-nuon".into())); + let serialize_types = call.has_flag(engine_state, stack, "serialize")?; let style = if call.has_flag(engine_state, stack, "raw")? { nuon::ToStyle::Raw } else if let Some(t) = call.get_flag(engine_state, stack, "tabs")? { @@ -60,7 +66,7 @@ impl Command for ToNuon { let span = call.head; let value = input.into_value(span)?; - match nuon::to_nuon(&value, style, Some(span)) { + match nuon::to_nuon(engine_state, &value, style, Some(span), serialize_types) { Ok(serde_nuon_string) => Ok(Value::string(serde_nuon_string, span) .into_pipeline_data_with_metadata(Some(metadata))), _ => Ok(Value::error( diff --git a/crates/nu-command/src/formats/to/text.rs b/crates/nu-command/src/formats/to/text.rs index b464cb86aa..2069df3dd4 100644 --- a/crates/nu-command/src/formats/to/text.rs +++ b/crates/nu-command/src/formats/to/text.rs @@ -27,6 +27,11 @@ impl Command for ToText { "Do not append a newline to the end of the text", Some('n'), ) + .switch( + "serialize", + "serialize nushell types that cannot be deserialized", + Some('s'), + ) .category(Category::Formats) } @@ -43,6 +48,7 @@ impl Command for ToText { ) -> Result { let head = call.head; let no_newline = call.has_flag(engine_state, stack, "no-newline")?; + let serialize_types = call.has_flag(engine_state, stack, "serialize")?; let input = input.try_expand_range()?; let config = stack.get_config(engine_state); @@ -56,7 +62,8 @@ impl Command for ToText { Value::Record { val, .. } => !val.is_empty(), _ => false, }; - let mut str = local_into_string(value, LINE_ENDING, &config); + let mut str = + local_into_string(engine_state, value, LINE_ENDING, &config, serialize_types); if add_trailing { str.push_str(LINE_ENDING); } @@ -70,6 +77,7 @@ impl Command for ToText { let stream = if no_newline { let mut first = true; let mut iter = stream.into_inner(); + let engine_state_clone = engine_state.clone(); ByteStream::from_fn( span, engine_state.signals().clone(), @@ -85,15 +93,28 @@ impl Command for ToText { } // TODO: write directly into `buf` instead of creating an intermediate // string. - let str = local_into_string(val, LINE_ENDING, &config); + let str = local_into_string( + &engine_state_clone, + val, + LINE_ENDING, + &config, + serialize_types, + ); write!(buf, "{str}").err_span(head)?; Ok(true) }, ) } else { + let engine_state_clone = engine_state.clone(); ByteStream::from_iter( stream.into_inner().map(move |val| { - let mut str = local_into_string(val, LINE_ENDING, &config); + let mut str = local_into_string( + &engine_state_clone, + val, + LINE_ENDING, + &config, + serialize_types, + ); str.push_str(LINE_ENDING); str }), @@ -137,7 +158,13 @@ impl Command for ToText { } } -fn local_into_string(value: Value, separator: &str, config: &Config) -> String { +fn local_into_string( + engine_state: &EngineState, + value: Value, + separator: &str, + config: &Config, + serialize_types: bool, +) -> String { let span = value.span(); match value { Value::Bool { val, .. } => val.to_string(), @@ -153,16 +180,38 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String { Value::Glob { val, .. } => val, Value::List { vals: val, .. } => val .into_iter() - .map(|x| local_into_string(x, ", ", config)) + .map(|x| local_into_string(engine_state, x, ", ", config, serialize_types)) .collect::>() .join(separator), Value::Record { val, .. } => val .into_owned() .into_iter() - .map(|(x, y)| format!("{}: {}", x, local_into_string(y, ", ", config))) + .map(|(x, y)| { + format!( + "{}: {}", + x, + local_into_string(engine_state, y, ", ", config, serialize_types) + ) + }) .collect::>() .join(separator), - Value::Closure { val, .. } => format!("", val.block_id.get()), + Value::Closure { val, .. } => { + if serialize_types { + let block = engine_state.get_block(val.block_id); + if let Some(span) = block.span { + let contents_bytes = engine_state.get_span_contents(span); + let contents_string = String::from_utf8_lossy(contents_bytes); + contents_string.to_string() + } else { + format!( + "unable to retrieve block contents for text block_id {}", + val.block_id.get() + ) + } + } else { + format!("closure_{}", val.block_id.get()) + } + } Value::Nothing { .. } => String::new(), Value::Error { error, .. } => format!("{error:?}"), Value::Binary { val, .. } => format!("{val:?}"), @@ -171,7 +220,7 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String { // that critical here Value::Custom { val, .. } => val .to_base_value(span) - .map(|val| local_into_string(val, separator, config)) + .map(|val| local_into_string(engine_state, val, separator, config, serialize_types)) .unwrap_or_else(|_| format!("<{}>", val.type_name())), } } diff --git a/crates/nu-command/src/formats/to/toml.rs b/crates/nu-command/src/formats/to/toml.rs index 5f28284904..11630e4522 100644 --- a/crates/nu-command/src/formats/to/toml.rs +++ b/crates/nu-command/src/formats/to/toml.rs @@ -13,6 +13,11 @@ impl Command for ToToml { fn signature(&self) -> Signature { Signature::build("to toml") .input_output_types(vec![(Type::record(), Type::String)]) + .switch( + "serialize", + "serialize nushell types that cannot be deserialized", + Some('s'), + ) .category(Category::Formats) } @@ -31,19 +36,24 @@ impl Command for ToToml { fn run( &self, engine_state: &EngineState, - _stack: &mut Stack, + stack: &mut Stack, call: &Call, input: PipelineData, ) -> Result { let head = call.head; - to_toml(engine_state, input, head) + let serialize_types = call.has_flag(engine_state, stack, "serialize")?; + + to_toml(engine_state, input, head, serialize_types) } } // Helper method to recursively convert nu_protocol::Value -> toml::Value // This shouldn't be called at the top-level -fn helper(engine_state: &EngineState, v: &Value) -> Result { - let span = v.span(); +fn helper( + engine_state: &EngineState, + v: &Value, + serialize_types: bool, +) -> Result { Ok(match &v { Value::Bool { val, .. } => toml::Value::Boolean(*val), Value::Int { val, .. } => toml::Value::Integer(*val), @@ -56,15 +66,29 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result { let mut m = toml::map::Map::new(); for (k, v) in &**val { - m.insert(k.clone(), helper(engine_state, v)?); + m.insert(k.clone(), helper(engine_state, v, serialize_types)?); } toml::Value::Table(m) } - Value::List { vals, .. } => toml::Value::Array(toml_list(engine_state, vals)?), - Value::Closure { .. } => { - let code = engine_state.get_span_contents(span); - let code = String::from_utf8_lossy(code).to_string(); - toml::Value::String(code) + Value::List { vals, .. } => { + toml::Value::Array(toml_list(engine_state, vals, serialize_types)?) + } + Value::Closure { val, .. } => { + if serialize_types { + let block = engine_state.get_block(val.block_id); + if let Some(span) = block.span { + let contents_bytes = engine_state.get_span_contents(span); + let contents_string = String::from_utf8_lossy(contents_bytes); + toml::Value::String(contents_string.to_string()) + } else { + toml::Value::String(format!( + "unable to retrieve block contents for toml block_id {}", + val.block_id.get() + )) + } + } else { + toml::Value::String(format!("closure_{}", val.block_id.get())) + } } Value::Nothing { .. } => toml::Value::String("".to_string()), Value::Error { error, .. } => return Err(*error.clone()), @@ -86,11 +110,15 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result Result, ShellError> { +fn toml_list( + engine_state: &EngineState, + input: &[Value], + serialize_types: bool, +) -> Result, ShellError> { let mut out = vec![]; for value in input { - out.push(helper(engine_state, value)?); + out.push(helper(engine_state, value, serialize_types)?); } Ok(out) @@ -129,9 +157,10 @@ fn value_to_toml_value( engine_state: &EngineState, v: &Value, head: Span, + serialize_types: bool, ) -> Result { match v { - Value::Record { .. } => helper(engine_state, v), + Value::Record { .. } | Value::Closure { .. } => helper(engine_state, v, serialize_types), // Propagate existing errors Value::Error { error, .. } => Err(*error.clone()), _ => Err(ShellError::UnsupportedInput { @@ -147,11 +176,12 @@ fn to_toml( engine_state: &EngineState, input: PipelineData, span: Span, + serialize_types: bool, ) -> Result { let metadata = input.metadata(); let value = input.into_value(span)?; - let toml_value = value_to_toml_value(engine_state, &value, span)?; + let toml_value = value_to_toml_value(engine_state, &value, span, serialize_types)?; match toml_value { toml::Value::Array(ref vec) => match vec[..] { [toml::Value::Table(_)] => toml_into_pipeline_data( @@ -218,6 +248,7 @@ mod tests { #[test] fn to_toml_creates_correct_date() { let engine_state = EngineState::new(); + let serialize_types = false; let test_date = Value::date( chrono::FixedOffset::east_opt(60 * 120) @@ -242,7 +273,7 @@ mod tests { offset: Some(toml::value::Offset::Custom { minutes: 120 }), }); - let result = helper(&engine_state, &test_date); + let result = helper(&engine_state, &test_date, serialize_types); assert!(result.is_ok_and(|res| res == reference_date)); } @@ -254,6 +285,7 @@ mod tests { // let engine_state = EngineState::new(); + let serialize_types = false; let mut m = indexmap::IndexMap::new(); m.insert("rust".to_owned(), Value::test_string("editor")); @@ -269,6 +301,7 @@ mod tests { &engine_state, &Value::record(m.into_iter().collect(), Span::test_data()), Span::test_data(), + serialize_types, ) .expect("Expected Ok from valid TOML dictionary"); assert_eq!( @@ -285,12 +318,14 @@ mod tests { &engine_state, &Value::test_string("not_valid"), Span::test_data(), + serialize_types, ) .expect_err("Expected non-valid toml (String) to cause error!"); value_to_toml_value( &engine_state, &Value::list(vec![Value::test_string("1")], Span::test_data()), Span::test_data(), + serialize_types, ) .expect_err("Expected non-valid toml (Table) to cause error!"); } diff --git a/crates/nu-command/src/formats/to/yaml.rs b/crates/nu-command/src/formats/to/yaml.rs index d6cad362f9..977e49c389 100644 --- a/crates/nu-command/src/formats/to/yaml.rs +++ b/crates/nu-command/src/formats/to/yaml.rs @@ -12,6 +12,11 @@ impl Command for ToYaml { fn signature(&self) -> Signature { Signature::build("to yaml") .input_output_types(vec![(Type::Any, Type::String)]) + .switch( + "serialize", + "serialize nushell types that cannot be deserialized", + Some('s'), + ) .category(Category::Formats) } @@ -29,18 +34,24 @@ impl Command for ToYaml { fn run( &self, - _engine_state: &EngineState, - _stack: &mut Stack, + engine_state: &EngineState, + stack: &mut Stack, call: &Call, input: PipelineData, ) -> Result { let head = call.head; + let serialize_types = call.has_flag(engine_state, stack, "serialize")?; let input = input.try_expand_range()?; - to_yaml(input, head) + + to_yaml(engine_state, input, head, serialize_types) } } -pub fn value_to_yaml_value(v: &Value) -> Result { +pub fn value_to_yaml_value( + engine_state: &EngineState, + v: &Value, + serialize_types: bool, +) -> Result { Ok(match &v { Value::Bool { val, .. } => serde_yml::Value::Bool(*val), Value::Int { val, .. } => serde_yml::Value::Number(serde_yml::Number::from(*val)), @@ -55,7 +66,10 @@ pub fn value_to_yaml_value(v: &Value) -> Result { Value::Record { val, .. } => { let mut m = serde_yml::Mapping::new(); for (k, v) in &**val { - m.insert(serde_yml::Value::String(k.clone()), value_to_yaml_value(v)?); + m.insert( + serde_yml::Value::String(k.clone()), + value_to_yaml_value(engine_state, v, serialize_types)?, + ); } serde_yml::Value::Mapping(m) } @@ -63,12 +77,28 @@ pub fn value_to_yaml_value(v: &Value) -> Result { let mut out = vec![]; for value in vals { - out.push(value_to_yaml_value(value)?); + out.push(value_to_yaml_value(engine_state, value, serialize_types)?); } serde_yml::Value::Sequence(out) } - Value::Closure { .. } => serde_yml::Value::Null, + Value::Closure { val, .. } => { + if serialize_types { + let block = engine_state.get_block(val.block_id); + if let Some(span) = block.span { + let contents_bytes = engine_state.get_span_contents(span); + let contents_string = String::from_utf8_lossy(contents_bytes); + serde_yml::Value::String(contents_string.to_string()) + } else { + serde_yml::Value::String(format!( + "unable to retrieve block contents for yaml block_id {}", + val.block_id.get() + )) + } + } else { + serde_yml::Value::Null + } + } Value::Nothing { .. } => serde_yml::Value::Null, Value::Error { error, .. } => return Err(*error.clone()), Value::Binary { val, .. } => serde_yml::Value::Sequence( @@ -91,7 +121,12 @@ pub fn value_to_yaml_value(v: &Value) -> Result { }) } -fn to_yaml(input: PipelineData, head: Span) -> Result { +fn to_yaml( + engine_state: &EngineState, + input: PipelineData, + head: Span, + serialize_types: bool, +) -> Result { let metadata = input .metadata() .unwrap_or_default() @@ -99,7 +134,7 @@ fn to_yaml(input: PipelineData, head: Span) -> Result .with_content_type(Some("application/yaml".into())); let value = input.into_value(head)?; - let yaml_value = value_to_yaml_value(&value)?; + let yaml_value = value_to_yaml_value(engine_state, &value, serialize_types)?; match serde_yml::to_string(&yaml_value) { Ok(serde_yml_string) => { Ok(Value::string(serde_yml_string, head) @@ -120,11 +155,9 @@ fn to_yaml(input: PipelineData, head: Span) -> Result #[cfg(test)] mod test { - use nu_cmd_lang::eval_pipeline_without_terminal_expression; - - use crate::{Get, Metadata}; - use super::*; + use crate::{Get, Metadata}; + use nu_cmd_lang::eval_pipeline_without_terminal_expression; #[test] fn test_examples() { diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index 504cc7b632..a5cad69b2c 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -205,6 +205,7 @@ pub enum HttpBody { // remove once all commands have been migrated pub fn send_request( + engine_state: &EngineState, request: Request, http_body: HttpBody, content_type: Option, @@ -212,6 +213,9 @@ pub fn send_request( signals: &Signals, ) -> Result { let request_url = request.url().to_string(); + // hard code serialze_types to false because closures probably shouldn't be + // deserialized for send_request but it's required by send_json_request + let serialze_types = false; match http_body { HttpBody::None => { @@ -238,7 +242,15 @@ pub fn send_request( }; match body_type { - BodyType::Json => send_json_request(&request_url, body, req, span, signals), + BodyType::Json => send_json_request( + engine_state, + &request_url, + body, + req, + span, + signals, + serialze_types, + ), BodyType::Form => send_form_request(&request_url, body, req, span, signals), BodyType::Multipart => { send_multipart_request(&request_url, body, req, span, signals) @@ -252,15 +264,17 @@ pub fn send_request( } fn send_json_request( + engine_state: &EngineState, request_url: &str, body: Value, req: Request, span: Span, signals: &Signals, + serialize_types: bool, ) -> Result { match body { Value::Int { .. } | Value::Float { .. } | Value::List { .. } | Value::Record { .. } => { - let data = value_to_json_value(&body)?; + let data = value_to_json_value(engine_state, &body, serialize_types)?; send_cancellable_request(request_url, Box::new(|| req.send_json(data)), span, signals) } // If the body type is string, assume it is string json content. diff --git a/crates/nu-command/src/network/http/delete.rs b/crates/nu-command/src/network/http/delete.rs index b13b3846e2..8a931f3653 100644 --- a/crates/nu-command/src/network/http/delete.rs +++ b/crates/nu-command/src/network/http/delete.rs @@ -214,6 +214,7 @@ fn helper( request = request_add_custom_headers(args.headers, request)?; let response = send_request( + engine_state, request.clone(), args.data, args.content_type, diff --git a/crates/nu-command/src/network/http/get.rs b/crates/nu-command/src/network/http/get.rs index 8260f76c66..245ba3747c 100644 --- a/crates/nu-command/src/network/http/get.rs +++ b/crates/nu-command/src/network/http/get.rs @@ -182,6 +182,7 @@ fn helper( request = request_add_custom_headers(args.headers, request)?; let response = send_request( + engine_state, request.clone(), HttpBody::None, None, diff --git a/crates/nu-command/src/network/http/head.rs b/crates/nu-command/src/network/http/head.rs index 99c77101af..4245d77abd 100644 --- a/crates/nu-command/src/network/http/head.rs +++ b/crates/nu-command/src/network/http/head.rs @@ -155,7 +155,14 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request, HttpBody::None, None, call.head, signals); + let response = send_request( + engine_state, + request, + HttpBody::None, + None, + call.head, + signals, + ); check_response_redirection(redirect_mode, span, &response)?; request_handle_response_headers(span, response) } diff --git a/crates/nu-command/src/network/http/options.rs b/crates/nu-command/src/network/http/options.rs index cd86b5349a..c8ee91d530 100644 --- a/crates/nu-command/src/network/http/options.rs +++ b/crates/nu-command/src/network/http/options.rs @@ -161,6 +161,7 @@ fn helper( request = request_add_custom_headers(args.headers, request)?; let response = send_request( + engine_state, request.clone(), HttpBody::None, None, diff --git a/crates/nu-command/src/network/http/patch.rs b/crates/nu-command/src/network/http/patch.rs index b062282b84..e3db71d3b1 100644 --- a/crates/nu-command/src/network/http/patch.rs +++ b/crates/nu-command/src/network/http/patch.rs @@ -216,6 +216,7 @@ fn helper( request = request_add_custom_headers(args.headers, request)?; let response = send_request( + engine_state, request.clone(), args.data, args.content_type, diff --git a/crates/nu-command/src/network/http/post.rs b/crates/nu-command/src/network/http/post.rs index 597f9478b9..5a23765366 100644 --- a/crates/nu-command/src/network/http/post.rs +++ b/crates/nu-command/src/network/http/post.rs @@ -224,6 +224,7 @@ fn helper( request = request_add_custom_headers(args.headers, request)?; let response = send_request( + engine_state, request.clone(), args.data, args.content_type, diff --git a/crates/nu-command/src/network/http/put.rs b/crates/nu-command/src/network/http/put.rs index b3bf8b72d5..a1c2aa71a3 100644 --- a/crates/nu-command/src/network/http/put.rs +++ b/crates/nu-command/src/network/http/put.rs @@ -215,6 +215,7 @@ fn helper( request = request_add_custom_headers(args.headers, request)?; let response = send_request( + engine_state, request.clone(), args.data, args.content_type, diff --git a/crates/nu-command/tests/commands/database/into_sqlite.rs b/crates/nu-command/tests/commands/database/into_sqlite.rs index 3de7973588..5c0f5f6c60 100644 --- a/crates/nu-command/tests/commands/database/into_sqlite.rs +++ b/crates/nu-command/tests/commands/database/into_sqlite.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, FixedOffset}; use nu_path::AbsolutePathBuf; -use nu_protocol::{ast::PathMember, record, Span, Value}; +use nu_protocol::{ast::PathMember, engine::EngineState, record, Span, Value}; use nu_test_support::{ fs::{line_ending, Stub}, nu, pipeline, @@ -298,6 +298,9 @@ fn into_sqlite_existing_db_append() { /// streaming pipeline instead of a simple value #[test] fn into_sqlite_big_insert() { + let engine_state = EngineState::new(); + // don't serialize closures + let serialize_types = false; Playground::setup("big_insert", |dirs, playground| { const NUM_ROWS: usize = 10_000; const NUON_FILE_NAME: &str = "data.nuon"; @@ -330,7 +333,14 @@ fn into_sqlite_big_insert() { ) .unwrap(); - let nuon = nuon::to_nuon(&value, nuon::ToStyle::Raw, Some(Span::unknown())).unwrap() + let nuon = nuon::to_nuon( + &engine_state, + &value, + nuon::ToStyle::Raw, + Some(Span::unknown()), + serialize_types, + ) + .unwrap() + &line_ending(); nuon_file.write_all(nuon.as_bytes()).unwrap(); diff --git a/crates/nu-command/tests/format_conversions/nuon.rs b/crates/nu-command/tests/format_conversions/nuon.rs index e4f6281c01..a7a423a0d6 100644 --- a/crates/nu-command/tests/format_conversions/nuon.rs +++ b/crates/nu-command/tests/format_conversions/nuon.rs @@ -278,6 +278,7 @@ fn from_nuon_datetime() { } #[test] +#[ignore] fn to_nuon_errs_on_closure() { let actual = nu!(pipeline( r#" diff --git a/crates/nu-protocol/src/value/mod.rs b/crates/nu-protocol/src/value/mod.rs index f3b44d93dd..09b9a5f60a 100644 --- a/crates/nu-protocol/src/value/mod.rs +++ b/crates/nu-protocol/src/value/mod.rs @@ -908,7 +908,7 @@ impl Value { .collect::>() .join(separator) ), - Value::Closure { val, .. } => format!("", val.block_id.get()), + Value::Closure { val, .. } => format!("closure_{}", val.block_id.get()), Value::Nothing { .. } => String::new(), Value::Error { error, .. } => format!("{error:?}"), Value::Binary { val, .. } => format!("{val:?}"), diff --git a/crates/nuon/src/lib.rs b/crates/nuon/src/lib.rs index 24a84dfd5f..99525959ab 100644 --- a/crates/nuon/src/lib.rs +++ b/crates/nuon/src/lib.rs @@ -11,7 +11,7 @@ mod tests { use chrono::DateTime; use nu_protocol::{ ast::{CellPath, PathMember, RangeInclusion}, - engine::Closure, + engine::{Closure, EngineState}, record, BlockId, IntRange, Range, Span, Value, }; @@ -25,11 +25,15 @@ mod tests { /// an optional "middle" value can be given to test what the value is between `from nuon` and /// `to nuon`. fn nuon_end_to_end(input: &str, middle: Option) { + let engine_state = EngineState::new(); let val = from_nuon(input, None).unwrap(); if let Some(m) = middle { assert_eq!(val, m); } - assert_eq!(to_nuon(&val, ToStyle::Raw, None).unwrap(), input); + assert_eq!( + to_nuon(&engine_state, &val, ToStyle::Raw, None, false).unwrap(), + input + ); } #[test] @@ -172,14 +176,19 @@ mod tests { } #[test] + #[ignore] fn to_nuon_errs_on_closure() { + let engine_state = EngineState::new(); + assert!(to_nuon( + &engine_state, &Value::test_closure(Closure { block_id: BlockId::new(0), captures: vec![] }), ToStyle::Raw, None, + false, ) .unwrap_err() .to_string() @@ -196,8 +205,17 @@ mod tests { #[test] fn binary_roundtrip() { + let engine_state = EngineState::new(); + assert_eq!( - to_nuon(&from_nuon("0x[1f ff]", None).unwrap(), ToStyle::Raw, None).unwrap(), + to_nuon( + &engine_state, + &from_nuon("0x[1f ff]", None).unwrap(), + ToStyle::Raw, + None, + false, + ) + .unwrap(), "0x[1FFF]" ); } @@ -237,40 +255,79 @@ mod tests { #[test] fn float_doesnt_become_int() { + let engine_state = EngineState::new(); + assert_eq!( - to_nuon(&Value::test_float(1.0), ToStyle::Raw, None).unwrap(), + to_nuon( + &engine_state, + &Value::test_float(1.0), + ToStyle::Raw, + None, + false + ) + .unwrap(), "1.0" ); } #[test] fn float_inf_parsed_properly() { + let engine_state = EngineState::new(); + assert_eq!( - to_nuon(&Value::test_float(f64::INFINITY), ToStyle::Raw, None).unwrap(), + to_nuon( + &engine_state, + &Value::test_float(f64::INFINITY), + ToStyle::Raw, + None, + false, + ) + .unwrap(), "inf" ); } #[test] fn float_neg_inf_parsed_properly() { + let engine_state = EngineState::new(); + assert_eq!( - to_nuon(&Value::test_float(f64::NEG_INFINITY), ToStyle::Raw, None).unwrap(), + to_nuon( + &engine_state, + &Value::test_float(f64::NEG_INFINITY), + ToStyle::Raw, + None, + false, + ) + .unwrap(), "-inf" ); } #[test] fn float_nan_parsed_properly() { + let engine_state = EngineState::new(); + assert_eq!( - to_nuon(&Value::test_float(-f64::NAN), ToStyle::Raw, None).unwrap(), + to_nuon( + &engine_state, + &Value::test_float(-f64::NAN), + ToStyle::Raw, + None, + false, + ) + .unwrap(), "NaN" ); } #[test] fn to_nuon_converts_columns_with_spaces() { + let engine_state = EngineState::new(); + assert!(from_nuon( &to_nuon( + &engine_state, &Value::test_list(vec![ Value::test_record(record!( "a" => Value::test_int(1), @@ -284,7 +341,8 @@ mod tests { )) ]), ToStyle::Raw, - None + None, + false, ) .unwrap(), None, @@ -294,7 +352,15 @@ mod tests { #[test] fn to_nuon_quotes_empty_string() { - let res = to_nuon(&Value::test_string(""), ToStyle::Raw, None); + let engine_state = EngineState::new(); + + let res = to_nuon( + &engine_state, + &Value::test_string(""), + ToStyle::Raw, + None, + false, + ); assert!(res.is_ok()); assert_eq!(res.unwrap(), r#""""#); } @@ -340,8 +406,11 @@ mod tests { #[test] fn does_not_quote_strings_unnecessarily() { + let engine_state = EngineState::new(); + assert_eq!( to_nuon( + &engine_state, &Value::test_list(vec![ Value::test_record(record!( "a" => Value::test_int(1), @@ -355,7 +424,8 @@ mod tests { )) ]), ToStyle::Raw, - None + None, + false, ) .unwrap(), "[[a, b, \"c d\"]; [1, 2, 3], [4, 5, 6]]" @@ -363,12 +433,14 @@ mod tests { assert_eq!( to_nuon( + &engine_state, &Value::test_record(record!( "ro name" => Value::test_string("sam"), "rank" => Value::test_int(10) )), ToStyle::Raw, - None + None, + false, ) .unwrap(), "{\"ro name\": sam, rank: 10}" diff --git a/crates/nuon/src/to.rs b/crates/nuon/src/to.rs index b8281e8ffe..13244bfdcd 100644 --- a/crates/nuon/src/to.rs +++ b/crates/nuon/src/to.rs @@ -1,7 +1,6 @@ use core::fmt::Write; - use nu_engine::get_columns; -use nu_protocol::{Range, ShellError, Span, Value}; +use nu_protocol::{engine::EngineState, Range, ShellError, Span, Value}; use nu_utils::{escape_quote_string, needs_quoting}; use std::ops::Bound; @@ -44,7 +43,13 @@ pub enum ToStyle { /// > using this function in a command implementation such as [`to nuon`](https://www.nushell.sh/commands/docs/to_nuon.html). /// /// also see [`super::from_nuon`] for the inverse operation -pub fn to_nuon(input: &Value, style: ToStyle, span: Option) -> Result { +pub fn to_nuon( + engine_state: &EngineState, + input: &Value, + style: ToStyle, + span: Option, + serialize_types: bool, +) -> Result { let span = span.unwrap_or(Span::unknown()); let indentation = match style { @@ -53,16 +58,25 @@ pub fn to_nuon(input: &Value, style: ToStyle, span: Option) -> Result Some(" ".repeat(s)), }; - let res = value_to_string(input, span, 0, indentation.as_deref())?; + let res = value_to_string( + engine_state, + input, + span, + 0, + indentation.as_deref(), + serialize_types, + )?; Ok(res) } fn value_to_string( + engine_state: &EngineState, v: &Value, span: Span, depth: usize, indent: Option<&str>, + serialize_types: bool, ) -> Result { let (nl, sep) = get_true_separators(indent); let idt = get_true_indentation(depth, indent); @@ -84,12 +98,25 @@ fn value_to_string( } Ok(format!("0x[{s}]")) } - Value::Closure { .. } => Err(ShellError::UnsupportedInput { - msg: "closures are currently not nuon-compatible".into(), - input: "value originates from here".into(), - msg_span: span, - input_span: v.span(), - }), + Value::Closure { val, .. } => { + if serialize_types { + let block = engine_state.get_block(val.block_id); + if let Some(span) = block.span { + let contents_bytes = engine_state.get_span_contents(span); + let contents_string = String::from_utf8_lossy(contents_bytes); + Ok(contents_string.to_string()) + } else { + Ok(String::new()) + } + } else { + Err(ShellError::UnsupportedInput { + msg: "closures are currently not nuon-compatible".into(), + input: "value originates from here".into(), + msg_span: span, + input_span: v.span(), + }) + } + } Value::Bool { val, .. } => { if *val { Ok("true".to_string()) @@ -144,10 +171,12 @@ fn value_to_string( if let Value::Record { val, .. } = val { for val in val.values() { row.push(value_to_string_without_quotes( + engine_state, val, span, depth + 2, indent, + serialize_types, )?); } } @@ -165,7 +194,14 @@ fn value_to_string( for val in vals { collection.push(format!( "{idt_po}{}", - value_to_string_without_quotes(val, span, depth + 1, indent,)? + value_to_string_without_quotes( + engine_state, + val, + span, + depth + 1, + indent, + serialize_types + )? )); } Ok(format!( @@ -178,18 +214,38 @@ fn value_to_string( Value::Range { val, .. } => match **val { Range::IntRange(range) => Ok(range.to_string()), Range::FloatRange(range) => { - let start = - value_to_string(&Value::float(range.start(), span), span, depth + 1, indent)?; + let start = value_to_string( + engine_state, + &Value::float(range.start(), span), + span, + depth + 1, + indent, + serialize_types, + )?; match range.end() { Bound::Included(end) => Ok(format!( "{}..{}", start, - value_to_string(&Value::float(end, span), span, depth + 1, indent)? + value_to_string( + engine_state, + &Value::float(end, span), + span, + depth + 1, + indent, + serialize_types, + )? )), Bound::Excluded(end) => Ok(format!( "{}..<{}", start, - value_to_string(&Value::float(end, span), span, depth + 1, indent)? + value_to_string( + engine_state, + &Value::float(end, span), + span, + depth + 1, + indent, + serialize_types, + )? )), Bound::Unbounded => Ok(format!("{start}..",)), } @@ -205,7 +261,14 @@ fn value_to_string( }; collection.push(format!( "{idt_po}{col}: {}", - value_to_string_without_quotes(val, span, depth + 1, indent)? + value_to_string_without_quotes( + engine_state, + val, + span, + depth + 1, + indent, + serialize_types + )? )); } Ok(format!( @@ -235,10 +298,12 @@ fn get_true_separators(indent: Option<&str>) -> (String, String) { } fn value_to_string_without_quotes( + engine_state: &EngineState, v: &Value, span: Span, depth: usize, indent: Option<&str>, + serialize_types: bool, ) -> Result { match v { Value::String { val, .. } => Ok({ @@ -248,6 +313,6 @@ fn value_to_string_without_quotes( val.clone() } }), - _ => value_to_string(v, span, depth, indent), + _ => value_to_string(engine_state, v, span, depth, indent, serialize_types), } } diff --git a/tests/eval/mod.rs b/tests/eval/mod.rs index c84cdaa873..65c26a5cc6 100644 --- a/tests/eval/mod.rs +++ b/tests/eval/mod.rs @@ -103,7 +103,27 @@ fn literal_binary() { #[test] fn literal_closure() { - test_eval("{||}", Matches("