Additional flags for commands from csv and from tsv (#8398)

# Description

Resolves issue #8370

Adds the following flags to commands `from csv` and `from tsv`:
- `--flexible`: allow the number of fields in records to be variable
- `-c --comment`: a comment character to ignore lines starting with it
- `-q --quote`: a quote character to ignore separators in strings,
defaults to '\"'
- `-e --escape`: an escape character for strings containing the quote
character

Internally, the `Value` struct has an additional helper function
`as_char` which converts it to a single `char`

# User-Facing Changes

The single quoted string `'\t'` can no longer be used as a parameter for
the flag `--separator '\t'` as it is interpreted as a two-character
string. One needs to use from now on the flag with a double quoted
string like so: `-s "\t"` which correctly interprets the string as a
single `char`.

crates/nu-command/tests/format_conversions/csv.rs

@@ -183,8 +183,92 @@ fn from_csv_text_with_tab_separator_to_table() {
 }
 
 #[test]
-fn from_csv_text_skipping_headers_to_table() {
+fn from_csv_text_with_comments_to_table() {
     Playground::setup("filter_from_csv_test_5", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                # This is a comment
+                first_name,last_name,rusty_luck
+                # This one too
+                Andrés,Robalino,1
+                Jonathan,Turner,1
+                Yehuda,Katz,1
+                # This one also
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r##"
+                open los_tres_caballeros.txt
+                | from csv --comment "#"
+                | get rusty_luck
+                | length
+            "##
+        ));
+
+        assert_eq!(actual.out, "3");
+    })
+}
+
+#[test]
+fn from_csv_text_with_custom_quotes_to_table() {
+    Playground::setup("filter_from_csv_test_6", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name,last_name,rusty_luck
+                'And''rés',Robalino,1
+                Jonathan,Turner,1
+                Yehuda,Katz,1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --quote "'"
+                | first
+                | get first_name
+            "#
+        ));
+
+        assert_eq!(actual.out, "And'rés");
+    })
+}
+
+#[test]
+fn from_csv_text_with_custom_escapes_to_table() {
+    Playground::setup("filter_from_csv_test_7", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name,last_name,rusty_luck
+                "And\"rés",Robalino,1
+                Jonathan,Turner,1
+                Yehuda,Katz,1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --escape '\'
+                | first
+                | get first_name
+            "#
+        ));
+
+        assert_eq!(actual.out, "And\"rés");
+    })
+}
+
+#[test]
+fn from_csv_text_skipping_headers_to_table() {
+    Playground::setup("filter_from_csv_test_8", |dirs, sandbox| {
         sandbox.with_files(vec![FileWithContentToBeTrimmed(
             "los_tres_amigos.txt",
             r#"
@@ -208,6 +292,84 @@ fn from_csv_text_skipping_headers_to_table() {
     })
 }
 
+#[test]
+fn from_csv_text_with_missing_columns_to_table() {
+    Playground::setup("filter_from_csv_test_9", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name,last_name,rusty_luck
+                Andrés,Robalino
+                Jonathan,Turner,1
+                Yehuda,Katz,1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --flexible
+                | get -i rusty_luck
+                | compact
+                | length
+            "#
+        ));
+
+        assert_eq!(actual.out, "2");
+    })
+}
+
+#[test]
+fn from_csv_text_with_multiple_char_separator() {
+    Playground::setup("filter_from_csv_test_10", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name,last_name,rusty_luck
+                Andrés,Robalino,1
+                Jonathan,Turner,1
+                Yehuda,Katz,1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --separator "li"
+            "#
+        ));
+
+        assert!(actual.err.contains("single character separator"));
+    })
+}
+
+#[test]
+fn from_csv_text_with_wrong_type_separator() {
+    Playground::setup("filter_from_csv_test_11", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name,last_name,rusty_luck
+                Andrés,Robalino,1
+                Jonathan,Turner,1
+                Yehuda,Katz,1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --separator ('123' | into int)
+            "#
+        ));
+
+        assert!(actual.err.contains("can't convert int to char"));
+    })
+}
+
 #[test]
 fn table_with_record_error() {
     let actual = nu!(

crates/nu-command/tests/format_conversions/tsv.rs

@@ -16,7 +16,7 @@ fn table_to_tsv_text_and_from_tsv_text_back_into_table() {
 fn table_to_tsv_text_and_from_tsv_text_back_into_table_using_csv_separator() {
     let actual = nu!(
         cwd: "tests/fixtures/formats",
-        r"open caco3_plastics.tsv | to tsv | from csv --separator '\t' | first | get origin"
+        r#"open caco3_plastics.tsv | to tsv | from csv --separator "\t" | first | get origin"#
     );
 
     assert_eq!(actual.out, "SPAIN");
@@ -106,8 +106,92 @@ fn from_tsv_text_to_table() {
 }
 
 #[test]
-fn from_tsv_text_skipping_headers_to_table() {
+fn from_tsv_text_with_comments_to_table() {
     Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                # This is a comment
+                first_name	last_name	rusty_luck
+                # This one too
+                Andrés	Robalino	1
+                Jonathan	Turner	1
+                Yehuda	Katz	1
+                # This one also
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r##"
+                open los_tres_caballeros.txt
+                | from tsv --comment "#"
+                | get rusty_luck
+                | length
+            "##
+        ));
+
+        assert_eq!(actual.out, "3");
+    })
+}
+
+#[test]
+fn from_tsv_text_with_custom_quotes_to_table() {
+    Playground::setup("filter_from_tsv_test_3", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name	last_name	rusty_luck
+                'And''rés'	Robalino	1
+                Jonathan	Turner	1
+                Yehuda	Katz	1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from tsv --quote "'"
+                | first
+                | get first_name
+            "#
+        ));
+
+        assert_eq!(actual.out, "And'rés");
+    })
+}
+
+#[test]
+fn from_tsv_text_with_custom_escapes_to_table() {
+    Playground::setup("filter_from_tsv_test_4", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name	last_name	rusty_luck
+                "And\"rés"	Robalino	1
+                Jonathan	Turner	1
+                Yehuda	Katz	1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from tsv --escape '\'
+                | first
+                | get first_name
+            "#
+        ));
+
+        assert_eq!(actual.out, "And\"rés");
+    })
+}
+
+#[test]
+fn from_tsv_text_skipping_headers_to_table() {
+    Playground::setup("filter_from_tsv_test_5", |dirs, sandbox| {
         sandbox.with_files(vec![FileWithContentToBeTrimmed(
             "los_tres_amigos.txt",
             r#"
@@ -130,3 +214,81 @@ fn from_tsv_text_skipping_headers_to_table() {
         assert_eq!(actual.out, "3");
     })
 }
+
+#[test]
+fn from_tsv_text_with_missing_columns_to_table() {
+    Playground::setup("filter_from_tsv_test_6", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name	last_name	rusty_luck
+                Andrés	Robalino
+                Jonathan	Turner	1
+                Yehuda	Katz	1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from tsv --flexible
+                | get -i rusty_luck
+                | compact
+                | length
+            "#
+        ));
+
+        assert_eq!(actual.out, "2");
+    })
+}
+
+#[test]
+fn from_tsv_text_with_multiple_char_comment() {
+    Playground::setup("filter_from_tsv_test_7", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name	last_name	rusty_luck
+                Andrés	Robalino	1
+                Jonathan	Turner	1
+                Yehuda	Katz	1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --comment "li"
+            "#
+        ));
+
+        assert!(actual.err.contains("single character separator"));
+    })
+}
+
+#[test]
+fn from_tsv_text_with_wrong_type_comment() {
+    Playground::setup("filter_from_csv_test_8", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_caballeros.txt",
+            r#"
+                first_name	last_name	rusty_luck
+                Andrés	Robalino	1
+                Jonathan	Turner	1
+                Yehuda	Katz	1
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open los_tres_caballeros.txt
+                | from csv --comment ('123' | into int)
+            "#
+        ));
+
+        assert!(actual.err.contains("can't convert int to char"));
+    })
+}