remove content_type metadata from pipeline after from ... commands (#14602)

# Description

`from ...` conversions pass along all metadata except `content_type`,
which they set to `None`.

## Rationale

`open`ing a file results in no `content_type` metadata if it can be
parsed into a nu data structure, and using `open --raw` results in
`content_type` metadata.

`from ...` commands should preserve metadata ***except*** for
`content_type`, as after parsing it's no longer that `content_type` and
just structured nu data.

These commands should return identical data *and* identical metadata

```nushell
open foo.csv
```

```nushell
open foo.csv --raw | from csv
```

# User-Facing Changes

N/A

# Tests + Formatting
- 🟢 toolkit fmt
- 🟢 toolkit clippy
- 🟢 toolkit test
- 🟢 toolkit test stdlib

# After Submitting
N/A

crates/nu-command/src/formats/from/csv.rs

@@ -204,12 +204,45 @@ fn from_csv(
 
 #[cfg(test)]
 mod test {
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
+
     use super::*;
 
+    use crate::{Metadata, MetadataSet};
+
     #[test]
     fn test_examples() {
         use crate::test_examples;
 
         test_examples(FromCsv {})
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromCsv {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#""a,b\n1,2" | metadata set --content-type 'text/csv' --datasource-ls | from csv | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/delimited.rs

@@ -93,9 +93,10 @@ pub(super) fn from_delimited_data(
     input: PipelineData,
     name: Span,
 ) -> Result<PipelineData, ShellError> {
+    let metadata = input.metadata().map(|md| md.with_content_type(None));
     match input {
         PipelineData::Empty => Ok(PipelineData::Empty),
-        PipelineData::Value(value, metadata) => {
+        PipelineData::Value(value, ..) => {
             let string = value.into_string()?;
             let byte_stream = ByteStream::read_string(string, name, Signals::empty());
             Ok(PipelineData::ListStream(
@@ -109,7 +110,7 @@ pub(super) fn from_delimited_data(
             dst_span: name,
             src_span: list_stream.span(),
         }),
-        PipelineData::ByteStream(byte_stream, metadata) => Ok(PipelineData::ListStream(
+        PipelineData::ByteStream(byte_stream, ..) => Ok(PipelineData::ListStream(
             from_delimited_stream(config, byte_stream, name)?,
             metadata,
         )),

crates/nu-command/src/formats/from/json.rs

@@ -70,23 +70,22 @@ impl Command for FromJson {
         let span = call.head;
 
         let strict = call.has_flag(engine_state, stack, "strict")?;
+        let metadata = input.metadata().map(|md| md.with_content_type(None));
 
         // TODO: turn this into a structured underline of the nu_json error
         if call.has_flag(engine_state, stack, "objects")? {
             // Return a stream of JSON values, one for each non-empty line
             match input {
-                PipelineData::Value(Value::String { val, .. }, metadata) => {
+                PipelineData::Value(Value::String { val, .. }, ..) => Ok(PipelineData::ListStream(
-                    Ok(PipelineData::ListStream(
+                    read_json_lines(
-                        read_json_lines(
+                        Cursor::new(val),
-                            Cursor::new(val),
+                        span,
-                            span,
+                        strict,
-                            strict,
+                        engine_state.signals().clone(),
-                            engine_state.signals().clone(),
+                    ),
-                        ),
+                    metadata,
-                        metadata,
+                )),
-                    ))
+                PipelineData::ByteStream(stream, ..)
-                }
-                PipelineData::ByteStream(stream, metadata)
                     if stream.type_() != ByteStreamType::Binary =>
                 {
                     if let Some(reader) = stream.reader() {
@@ -107,7 +106,7 @@ impl Command for FromJson {
             }
         } else {
             // Return a single JSON value
-            let (string_input, span, metadata) = input.collect_string_strict(span)?;
+            let (string_input, span, ..) = input.collect_string_strict(span)?;
 
             if string_input.is_empty() {
                 return Ok(Value::nothing(span).into_pipeline_data());
@@ -267,6 +266,10 @@ fn convert_string_to_value_strict(string_input: &str, span: Span) -> Result<Valu
 
 #[cfg(test)]
 mod test {
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
+
+    use crate::{Metadata, MetadataSet};
+
     use super::*;
 
     #[test]
@@ -275,4 +278,33 @@ mod test {
 
         test_examples(FromJson {})
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromJson {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#"'{"a":1,"b":2}' | metadata set --content-type 'application/json' --datasource-ls | from json | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/msgpack.rs

@@ -113,7 +113,8 @@ MessagePack: https://msgpack.org/
             objects,
             signals: engine_state.signals().clone(),
         };
-        match input {
+        let metadata = input.metadata().map(|md| md.with_content_type(None));
+        let out = match input {
             // Deserialize from a byte buffer
             PipelineData::Value(Value::Binary { val: bytes, .. }, _) => {
                 read_msgpack(Cursor::new(bytes), opts)
@@ -136,7 +137,8 @@ MessagePack: https://msgpack.org/
                 dst_span: call.head,
                 src_span: input.span().unwrap_or(call.head),
             }),
-        }
+        };
+        out.map(|pd| pd.set_metadata(metadata))
     }
 }
 
@@ -510,6 +512,10 @@ fn assert_eof(input: &mut impl io::Read, span: Span) -> Result<(), ShellError> {
 
 #[cfg(test)]
 mod test {
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
+
+    use crate::{Metadata, MetadataSet, ToMsgpack};
+
     use super::*;
 
     #[test]
@@ -518,4 +524,34 @@ mod test {
 
         test_examples(FromMsgpack {})
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(ToMsgpack {}));
+            working_set.add_decl(Box::new(FromMsgpack {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#"{a: 1 b: 2} | to msgpack | metadata set --datasource-ls | from msgpack | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/msgpackz.rs

@@ -43,7 +43,8 @@ impl Command for FromMsgpackz {
             objects,
             signals: engine_state.signals().clone(),
         };
-        match input {
+        let metadata = input.metadata().map(|md| md.with_content_type(None));
+        let out = match input {
             // Deserialize from a byte buffer
             PipelineData::Value(Value::Binary { val: bytes, .. }, _) => {
                 let reader = brotli::Decompressor::new(Cursor::new(bytes), BUFFER_SIZE);
@@ -68,6 +69,7 @@ impl Command for FromMsgpackz {
                 dst_span: call.head,
                 src_span: span,
             }),
-        }
+        };
+        out.map(|pd| pd.set_metadata(metadata))
     }
 }

crates/nu-command/src/formats/from/nuon.rs

@@ -49,7 +49,8 @@ impl Command for FromNuon {
         let (string_input, _span, metadata) = input.collect_string_strict(head)?;
 
         match nuon::from_nuon(&string_input, Some(head)) {
-            Ok(result) => Ok(result.into_pipeline_data_with_metadata(metadata)),
+            Ok(result) => Ok(result
+                .into_pipeline_data_with_metadata(metadata.map(|md| md.with_content_type(None)))),
             Err(err) => Err(ShellError::GenericError {
                 error: "error when loading nuon text".into(),
                 msg: "could not load nuon text".into(),
@@ -63,6 +64,10 @@ impl Command for FromNuon {
 
 #[cfg(test)]
 mod test {
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
+
+    use crate::{Metadata, MetadataSet};
+
     use super::*;
 
     #[test]
@@ -71,4 +76,33 @@ mod test {
 
         test_examples(FromNuon {})
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromNuon {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#"'[[a, b]; [1, 2]]' | metadata set --content-type 'application/x-nuon' --datasource-ls | from nuon | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/ods.rs

@@ -46,7 +46,8 @@ impl Command for FromOds {
             vec![]
         };
 
-        from_ods(input, head, sel_sheets)
+        let metadata = input.metadata().map(|md| md.with_content_type(None));
+        from_ods(input, head, sel_sheets).map(|pd| pd.set_metadata(metadata))
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/formats/from/toml.rs

@@ -29,7 +29,8 @@ impl Command for FromToml {
         let span = call.head;
         let (mut string_input, span, metadata) = input.collect_string_strict(span)?;
         string_input.push('\n');
-        Ok(convert_string_to_value(string_input, span)?.into_pipeline_data_with_metadata(metadata))
+        Ok(convert_string_to_value(string_input, span)?
+            .into_pipeline_data_with_metadata(metadata.map(|md| md.with_content_type(None))))
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -144,8 +145,11 @@ pub fn convert_string_to_value(string_input: String, span: Span) -> Result<Value
 
 #[cfg(test)]
 mod tests {
+    use crate::{Metadata, MetadataSet};
+
     use super::*;
     use chrono::TimeZone;
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
     use toml::value::Datetime;
 
     #[test]
@@ -331,4 +335,33 @@ mod tests {
 
         assert_eq!(result, reference_date);
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromToml {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#""[a]\nb = 1\nc = 1" | metadata set --content-type 'text/x-toml' --datasource-ls | from toml | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/tsv.rs

@@ -165,6 +165,10 @@ fn from_tsv(
 
 #[cfg(test)]
 mod test {
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
+
+    use crate::{Metadata, MetadataSet};
+
     use super::*;
 
     #[test]
@@ -173,4 +177,33 @@ mod test {
 
         test_examples(FromTsv {})
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromTsv {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#""a\tb\n1\t2" | metadata set --content-type 'text/tab-separated-values' --datasource-ls | from tsv | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/xlsx.rs

@@ -47,7 +47,8 @@ impl Command for FromXlsx {
             vec![]
         };
 
-        from_xlsx(input, head, sel_sheets)
+        let metadata = input.metadata().map(|md| md.with_content_type(None));
+        from_xlsx(input, head, sel_sheets).map(|pd| pd.set_metadata(metadata))
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/formats/from/xml.rs

@@ -206,7 +206,9 @@ fn from_xml(input: PipelineData, info: &ParsingInfo) -> Result<PipelineData, She
     let (concat_string, span, metadata) = input.collect_string_strict(info.span)?;
 
     match from_xml_string_to_value(&concat_string, info) {
-        Ok(x) => Ok(x.into_pipeline_data_with_metadata(metadata)),
+        Ok(x) => {
+            Ok(x.into_pipeline_data_with_metadata(metadata.map(|md| md.with_content_type(None))))
+        }
         Err(err) => Err(process_xml_parse_error(err, span)),
     }
 }
@@ -322,10 +324,14 @@ fn make_cant_convert_error(help: impl Into<String>, span: Span) -> ShellError {
 
 #[cfg(test)]
 mod tests {
+    use crate::Metadata;
+    use crate::MetadataSet;
+
     use super::*;
 
     use indexmap::indexmap;
     use indexmap::IndexMap;
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
 
     fn string(input: impl Into<String>) -> Value {
         Value::test_string(input)
@@ -480,4 +486,36 @@ mod tests {
 
         test_examples(FromXml {})
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromXml {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#"'<?xml version="1.0" encoding="UTF-8"?>
+<note>
+  <remember>Event</remember>
+</note>' | metadata set --content-type 'application/xml' --datasource-ls | from xml | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }

crates/nu-command/src/formats/from/yaml.rs

@@ -235,14 +235,19 @@ fn from_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError
     let (concat_string, span, metadata) = input.collect_string_strict(head)?;
 
     match from_yaml_string_to_value(&concat_string, head, span) {
-        Ok(x) => Ok(x.into_pipeline_data_with_metadata(metadata)),
+        Ok(x) => {
+            Ok(x.into_pipeline_data_with_metadata(metadata.map(|md| md.with_content_type(None))))
+        }
         Err(other) => Err(other),
     }
 }
 
 #[cfg(test)]
 mod test {
+    use crate::{Metadata, MetadataSet};
+
     use super::*;
+    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
     use nu_protocol::Config;
 
     #[test]
@@ -395,4 +400,33 @@ mod test {
             assert!(result.ok().unwrap() == test_case.expected.ok().unwrap());
         }
     }
+
+    #[test]
+    fn test_content_type_metadata() {
+        let mut engine_state = Box::new(EngineState::new());
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+
+            working_set.add_decl(Box::new(FromYaml {}));
+            working_set.add_decl(Box::new(Metadata {}));
+            working_set.add_decl(Box::new(MetadataSet {}));
+
+            working_set.render()
+        };
+
+        engine_state
+            .merge_delta(delta)
+            .expect("Error merging delta");
+
+        let cmd = r#""a: 1\nb: 2" | metadata set --content-type 'application/yaml' --datasource-ls | from yaml | metadata | $in"#;
+        let result = eval_pipeline_without_terminal_expression(
+            cmd,
+            std::env::temp_dir().as_ref(),
+            &mut engine_state,
+        );
+        assert_eq!(
+            Value::test_record(record!("source" => Value::test_string("ls"))),
+            result.expect("There should be a result")
+        )
+    }
 }