more closure serialization (#14698)

# Description

This PR introduces a switch `--serialize` that allows serializing of
types that cannot be deserialized. Right now it only serializes closures
as strings in `to toml`, `to json`, `to nuon`, `to text`, some indirect
`to html` and `to yaml`.

A lot of the changes are just weaving the engine_state through calling
functions and the rest is just repetitive way of getting the closure
block span and grabbing the span's text.

In places where it has to report `<Closure 123>` I changed it to
`closure_123`. It always seemed like the `<>` were not very nushell-y.
This is still a breaking change.

I think this could also help with systematic translation of old config
to new config file.


# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the
tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

crates/nu-command/src/formats/to/json.rs

@@ -25,6 +25,11 @@ impl Command for ToJson {
                 "specify indentation tab quantity",
                 Some('t'),
             )
+            .switch(
+                "serialize",
+                "serialize nushell types that cannot be deserialized",
+                Some('s'),
+            )
             .category(Category::Formats)
     }
 
@@ -42,12 +47,13 @@ impl Command for ToJson {
         let raw = call.has_flag(engine_state, stack, "raw")?;
         let use_tabs = call.get_flag(engine_state, stack, "tabs")?;
         let indent = call.get_flag(engine_state, stack, "indent")?;
+        let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
 
         let span = call.head;
         // allow ranges to expand and turn into array
         let input = input.try_expand_range()?;
         let value = input.into_value(span)?;
-        let json_value = value_to_json_value(&value)?;
+        let json_value = value_to_json_value(engine_state, &value, serialize_types)?;
 
         let json_result = if raw {
             nu_json::to_string_raw(&json_value)
@@ -105,7 +111,11 @@ impl Command for ToJson {
     }
 }
 
-pub fn value_to_json_value(v: &Value) -> Result<nu_json::Value, ShellError> {
+pub fn value_to_json_value(
+    engine_state: &EngineState,
+    v: &Value,
+    serialize_types: bool,
+) -> Result<nu_json::Value, ShellError> {
     let span = v.span();
     Ok(match v {
         Value::Bool { val, .. } => nu_json::Value::Bool(*val),
@@ -127,31 +137,57 @@ pub fn value_to_json_value(v: &Value) -> Result<nu_json::Value, ShellError> {
                 .collect::<Result<Vec<nu_json::Value>, ShellError>>()?,
         ),
 
-        Value::List { vals, .. } => nu_json::Value::Array(json_list(vals)?),
+        Value::List { vals, .. } => {
+            nu_json::Value::Array(json_list(engine_state, vals, serialize_types)?)
+        }
         Value::Error { error, .. } => return Err(*error.clone()),
-        Value::Closure { .. } | Value::Range { .. } => nu_json::Value::Null,
+        Value::Closure { val, .. } => {
+            if serialize_types {
+                let block = engine_state.get_block(val.block_id);
+                if let Some(span) = block.span {
+                    let contents_bytes = engine_state.get_span_contents(span);
+                    let contents_string = String::from_utf8_lossy(contents_bytes);
+                    nu_json::Value::String(contents_string.to_string())
+                } else {
+                    nu_json::Value::String(format!(
+                        "unable to retrieve block contents for json block_id {}",
+                        val.block_id.get()
+                    ))
+                }
+            } else {
+                nu_json::Value::Null
+            }
+        }
+        Value::Range { .. } => nu_json::Value::Null,
         Value::Binary { val, .. } => {
             nu_json::Value::Array(val.iter().map(|x| nu_json::Value::U64(*x as u64)).collect())
         }
         Value::Record { val, .. } => {
             let mut m = nu_json::Map::new();
             for (k, v) in &**val {
-                m.insert(k.clone(), value_to_json_value(v)?);
+                m.insert(
+                    k.clone(),
+                    value_to_json_value(engine_state, v, serialize_types)?,
+                );
             }
             nu_json::Value::Object(m)
         }
         Value::Custom { val, .. } => {
             let collected = val.to_base_value(span)?;
-            value_to_json_value(&collected)?
+            value_to_json_value(engine_state, &collected, serialize_types)?
         }
     })
 }
 
-fn json_list(input: &[Value]) -> Result<Vec<nu_json::Value>, ShellError> {
+fn json_list(
+    engine_state: &EngineState,
+    input: &[Value],
+    serialize_types: bool,
+) -> Result<Vec<nu_json::Value>, ShellError> {
     let mut out = vec![];
 
     for value in input {
-        out.push(value_to_json_value(value)?);
+        out.push(value_to_json_value(engine_state, value, serialize_types)?);
     }
 
     Ok(out)

crates/nu-command/src/formats/to/nuon.rs

@@ -28,6 +28,11 @@ impl Command for ToNuon {
                 "specify indentation tab quantity",
                 Some('t'),
             )
+            .switch(
+                "serialize",
+                "serialize nushell types that cannot be deserialized",
+                Some('s'),
+            )
             .category(Category::Formats)
     }
 
@@ -47,6 +52,7 @@ impl Command for ToNuon {
             .unwrap_or_default()
             .with_content_type(Some("application/x-nuon".into()));
 
+        let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
         let style = if call.has_flag(engine_state, stack, "raw")? {
             nuon::ToStyle::Raw
         } else if let Some(t) = call.get_flag(engine_state, stack, "tabs")? {
@@ -60,7 +66,7 @@ impl Command for ToNuon {
         let span = call.head;
         let value = input.into_value(span)?;
 
-        match nuon::to_nuon(&value, style, Some(span)) {
+        match nuon::to_nuon(engine_state, &value, style, Some(span), serialize_types) {
             Ok(serde_nuon_string) => Ok(Value::string(serde_nuon_string, span)
                 .into_pipeline_data_with_metadata(Some(metadata))),
             _ => Ok(Value::error(

crates/nu-command/src/formats/to/text.rs

@@ -27,6 +27,11 @@ impl Command for ToText {
                 "Do not append a newline to the end of the text",
                 Some('n'),
             )
+            .switch(
+                "serialize",
+                "serialize nushell types that cannot be deserialized",
+                Some('s'),
+            )
             .category(Category::Formats)
     }
 
@@ -43,6 +48,7 @@ impl Command for ToText {
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
         let no_newline = call.has_flag(engine_state, stack, "no-newline")?;
+        let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
         let input = input.try_expand_range()?;
         let config = stack.get_config(engine_state);
 
@@ -56,7 +62,8 @@ impl Command for ToText {
                         Value::Record { val, .. } => !val.is_empty(),
                         _ => false,
                     };
-                let mut str = local_into_string(value, LINE_ENDING, &config);
+                let mut str =
+                    local_into_string(engine_state, value, LINE_ENDING, &config, serialize_types);
                 if add_trailing {
                     str.push_str(LINE_ENDING);
                 }
@@ -70,6 +77,7 @@ impl Command for ToText {
                 let stream = if no_newline {
                     let mut first = true;
                     let mut iter = stream.into_inner();
+                    let engine_state_clone = engine_state.clone();
                     ByteStream::from_fn(
                         span,
                         engine_state.signals().clone(),
@@ -85,15 +93,28 @@ impl Command for ToText {
                             }
                             // TODO: write directly into `buf` instead of creating an intermediate
                             // string.
-                            let str = local_into_string(val, LINE_ENDING, &config);
+                            let str = local_into_string(
+                                &engine_state_clone,
+                                val,
+                                LINE_ENDING,
+                                &config,
+                                serialize_types,
+                            );
                             write!(buf, "{str}").err_span(head)?;
                             Ok(true)
                         },
                     )
                 } else {
+                    let engine_state_clone = engine_state.clone();
                     ByteStream::from_iter(
                         stream.into_inner().map(move |val| {
-                            let mut str = local_into_string(val, LINE_ENDING, &config);
+                            let mut str = local_into_string(
+                                &engine_state_clone,
+                                val,
+                                LINE_ENDING,
+                                &config,
+                                serialize_types,
+                            );
                             str.push_str(LINE_ENDING);
                             str
                         }),
@@ -137,7 +158,13 @@ impl Command for ToText {
     }
 }
 
-fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
+fn local_into_string(
+    engine_state: &EngineState,
+    value: Value,
+    separator: &str,
+    config: &Config,
+    serialize_types: bool,
+) -> String {
     let span = value.span();
     match value {
         Value::Bool { val, .. } => val.to_string(),
@@ -153,16 +180,38 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
         Value::Glob { val, .. } => val,
         Value::List { vals: val, .. } => val
             .into_iter()
-            .map(|x| local_into_string(x, ", ", config))
+            .map(|x| local_into_string(engine_state, x, ", ", config, serialize_types))
             .collect::<Vec<_>>()
             .join(separator),
         Value::Record { val, .. } => val
             .into_owned()
             .into_iter()
-            .map(|(x, y)| format!("{}: {}", x, local_into_string(y, ", ", config)))
+            .map(|(x, y)| {
+                format!(
+                    "{}: {}",
+                    x,
+                    local_into_string(engine_state, y, ", ", config, serialize_types)
+                )
+            })
             .collect::<Vec<_>>()
             .join(separator),
-        Value::Closure { val, .. } => format!("<Closure {}>", val.block_id.get()),
+        Value::Closure { val, .. } => {
+            if serialize_types {
+                let block = engine_state.get_block(val.block_id);
+                if let Some(span) = block.span {
+                    let contents_bytes = engine_state.get_span_contents(span);
+                    let contents_string = String::from_utf8_lossy(contents_bytes);
+                    contents_string.to_string()
+                } else {
+                    format!(
+                        "unable to retrieve block contents for text block_id {}",
+                        val.block_id.get()
+                    )
+                }
+            } else {
+                format!("closure_{}", val.block_id.get())
+            }
+        }
         Value::Nothing { .. } => String::new(),
         Value::Error { error, .. } => format!("{error:?}"),
         Value::Binary { val, .. } => format!("{val:?}"),
@@ -171,7 +220,7 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
         // that critical here
         Value::Custom { val, .. } => val
             .to_base_value(span)
-            .map(|val| local_into_string(val, separator, config))
+            .map(|val| local_into_string(engine_state, val, separator, config, serialize_types))
             .unwrap_or_else(|_| format!("<{}>", val.type_name())),
     }
 }

crates/nu-command/src/formats/to/toml.rs

@@ -13,6 +13,11 @@ impl Command for ToToml {
     fn signature(&self) -> Signature {
         Signature::build("to toml")
             .input_output_types(vec![(Type::record(), Type::String)])
+            .switch(
+                "serialize",
+                "serialize nushell types that cannot be deserialized",
+                Some('s'),
+            )
             .category(Category::Formats)
     }
 
@@ -31,19 +36,24 @@ impl Command for ToToml {
     fn run(
         &self,
         engine_state: &EngineState,
-        _stack: &mut Stack,
+        stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
-        to_toml(engine_state, input, head)
+        let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
+
+        to_toml(engine_state, input, head, serialize_types)
     }
 }
 
 // Helper method to recursively convert nu_protocol::Value -> toml::Value
 // This shouldn't be called at the top-level
-fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellError> {
-    let span = v.span();
+fn helper(
+    engine_state: &EngineState,
+    v: &Value,
+    serialize_types: bool,
+) -> Result<toml::Value, ShellError> {
     Ok(match &v {
         Value::Bool { val, .. } => toml::Value::Boolean(*val),
         Value::Int { val, .. } => toml::Value::Integer(*val),
@@ -56,15 +66,29 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellErr
         Value::Record { val, .. } => {
             let mut m = toml::map::Map::new();
             for (k, v) in &**val {
-                m.insert(k.clone(), helper(engine_state, v)?);
+                m.insert(k.clone(), helper(engine_state, v, serialize_types)?);
             }
             toml::Value::Table(m)
         }
-        Value::List { vals, .. } => toml::Value::Array(toml_list(engine_state, vals)?),
-        Value::Closure { .. } => {
-            let code = engine_state.get_span_contents(span);
-            let code = String::from_utf8_lossy(code).to_string();
-            toml::Value::String(code)
+        Value::List { vals, .. } => {
+            toml::Value::Array(toml_list(engine_state, vals, serialize_types)?)
+        }
+        Value::Closure { val, .. } => {
+            if serialize_types {
+                let block = engine_state.get_block(val.block_id);
+                if let Some(span) = block.span {
+                    let contents_bytes = engine_state.get_span_contents(span);
+                    let contents_string = String::from_utf8_lossy(contents_bytes);
+                    toml::Value::String(contents_string.to_string())
+                } else {
+                    toml::Value::String(format!(
+                        "unable to retrieve block contents for toml block_id {}",
+                        val.block_id.get()
+                    ))
+                }
+            } else {
+                toml::Value::String(format!("closure_{}", val.block_id.get()))
+            }
         }
         Value::Nothing { .. } => toml::Value::String("<Nothing>".to_string()),
         Value::Error { error, .. } => return Err(*error.clone()),
@@ -86,11 +110,15 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellErr
     })
 }
 
-fn toml_list(engine_state: &EngineState, input: &[Value]) -> Result<Vec<toml::Value>, ShellError> {
+fn toml_list(
+    engine_state: &EngineState,
+    input: &[Value],
+    serialize_types: bool,
+) -> Result<Vec<toml::Value>, ShellError> {
     let mut out = vec![];
 
     for value in input {
-        out.push(helper(engine_state, value)?);
+        out.push(helper(engine_state, value, serialize_types)?);
     }
 
     Ok(out)
@@ -129,9 +157,10 @@ fn value_to_toml_value(
     engine_state: &EngineState,
     v: &Value,
     head: Span,
+    serialize_types: bool,
 ) -> Result<toml::Value, ShellError> {
     match v {
-        Value::Record { .. } => helper(engine_state, v),
+        Value::Record { .. } | Value::Closure { .. } => helper(engine_state, v, serialize_types),
         // Propagate existing errors
         Value::Error { error, .. } => Err(*error.clone()),
         _ => Err(ShellError::UnsupportedInput {
@@ -147,11 +176,12 @@ fn to_toml(
     engine_state: &EngineState,
     input: PipelineData,
     span: Span,
+    serialize_types: bool,
 ) -> Result<PipelineData, ShellError> {
     let metadata = input.metadata();
     let value = input.into_value(span)?;
 
-    let toml_value = value_to_toml_value(engine_state, &value, span)?;
+    let toml_value = value_to_toml_value(engine_state, &value, span, serialize_types)?;
     match toml_value {
         toml::Value::Array(ref vec) => match vec[..] {
             [toml::Value::Table(_)] => toml_into_pipeline_data(
@@ -218,6 +248,7 @@ mod tests {
     #[test]
     fn to_toml_creates_correct_date() {
         let engine_state = EngineState::new();
+        let serialize_types = false;
 
         let test_date = Value::date(
             chrono::FixedOffset::east_opt(60 * 120)
@@ -242,7 +273,7 @@ mod tests {
             offset: Some(toml::value::Offset::Custom { minutes: 120 }),
         });
 
-        let result = helper(&engine_state, &test_date);
+        let result = helper(&engine_state, &test_date, serialize_types);
 
         assert!(result.is_ok_and(|res| res == reference_date));
     }
@@ -254,6 +285,7 @@ mod tests {
         //
 
         let engine_state = EngineState::new();
+        let serialize_types = false;
 
         let mut m = indexmap::IndexMap::new();
         m.insert("rust".to_owned(), Value::test_string("editor"));
@@ -269,6 +301,7 @@ mod tests {
             &engine_state,
             &Value::record(m.into_iter().collect(), Span::test_data()),
             Span::test_data(),
+            serialize_types,
         )
         .expect("Expected Ok from valid TOML dictionary");
         assert_eq!(
@@ -285,12 +318,14 @@ mod tests {
             &engine_state,
             &Value::test_string("not_valid"),
             Span::test_data(),
+            serialize_types,
         )
         .expect_err("Expected non-valid toml (String) to cause error!");
         value_to_toml_value(
             &engine_state,
             &Value::list(vec![Value::test_string("1")], Span::test_data()),
             Span::test_data(),
+            serialize_types,
         )
         .expect_err("Expected non-valid toml (Table) to cause error!");
     }

crates/nu-command/src/formats/to/yaml.rs

@@ -12,6 +12,11 @@ impl Command for ToYaml {
     fn signature(&self) -> Signature {
         Signature::build("to yaml")
             .input_output_types(vec![(Type::Any, Type::String)])
+            .switch(
+                "serialize",
+                "serialize nushell types that cannot be deserialized",
+                Some('s'),
+            )
             .category(Category::Formats)
     }
 
@@ -29,18 +34,24 @@ impl Command for ToYaml {
 
     fn run(
         &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
+        engine_state: &EngineState,
+        stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
+        let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
         let input = input.try_expand_range()?;
-        to_yaml(input, head)
+
+        to_yaml(engine_state, input, head, serialize_types)
     }
 }
 
-pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
+pub fn value_to_yaml_value(
+    engine_state: &EngineState,
+    v: &Value,
+    serialize_types: bool,
+) -> Result<serde_yml::Value, ShellError> {
     Ok(match &v {
         Value::Bool { val, .. } => serde_yml::Value::Bool(*val),
         Value::Int { val, .. } => serde_yml::Value::Number(serde_yml::Number::from(*val)),
@@ -55,7 +66,10 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
         Value::Record { val, .. } => {
             let mut m = serde_yml::Mapping::new();
             for (k, v) in &**val {
-                m.insert(serde_yml::Value::String(k.clone()), value_to_yaml_value(v)?);
+                m.insert(
+                    serde_yml::Value::String(k.clone()),
+                    value_to_yaml_value(engine_state, v, serialize_types)?,
+                );
             }
             serde_yml::Value::Mapping(m)
         }
@@ -63,12 +77,28 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
             let mut out = vec![];
 
             for value in vals {
-                out.push(value_to_yaml_value(value)?);
+                out.push(value_to_yaml_value(engine_state, value, serialize_types)?);
             }
 
             serde_yml::Value::Sequence(out)
         }
-        Value::Closure { .. } => serde_yml::Value::Null,
+        Value::Closure { val, .. } => {
+            if serialize_types {
+                let block = engine_state.get_block(val.block_id);
+                if let Some(span) = block.span {
+                    let contents_bytes = engine_state.get_span_contents(span);
+                    let contents_string = String::from_utf8_lossy(contents_bytes);
+                    serde_yml::Value::String(contents_string.to_string())
+                } else {
+                    serde_yml::Value::String(format!(
+                        "unable to retrieve block contents for yaml block_id {}",
+                        val.block_id.get()
+                    ))
+                }
+            } else {
+                serde_yml::Value::Null
+            }
+        }
         Value::Nothing { .. } => serde_yml::Value::Null,
         Value::Error { error, .. } => return Err(*error.clone()),
         Value::Binary { val, .. } => serde_yml::Value::Sequence(
@@ -91,7 +121,12 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
     })
 }
 
-fn to_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
+fn to_yaml(
+    engine_state: &EngineState,
+    input: PipelineData,
+    head: Span,
+    serialize_types: bool,
+) -> Result<PipelineData, ShellError> {
     let metadata = input
         .metadata()
         .unwrap_or_default()
@@ -99,7 +134,7 @@ fn to_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError>
         .with_content_type(Some("application/yaml".into()));
     let value = input.into_value(head)?;
 
-    let yaml_value = value_to_yaml_value(&value)?;
+    let yaml_value = value_to_yaml_value(engine_state, &value, serialize_types)?;
     match serde_yml::to_string(&yaml_value) {
         Ok(serde_yml_string) => {
             Ok(Value::string(serde_yml_string, head)
@@ -120,11 +155,9 @@ fn to_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError>
 
 #[cfg(test)]
 mod test {
-    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
-
-    use crate::{Get, Metadata};
-
     use super::*;
+    use crate::{Get, Metadata};
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
 
     #[test]
     fn test_examples() {