new command url parse (#6854) and url subcommands tests (#7124)

*code refactor from PR tips & clippy fixes

*added username, password, and fragment

*commands `url host`, `url scheme`, `url query`, and `url path` removed

*tests refactoring - avoid formatted output

crates/nu-command/src/default_context.rs

@@ -405,10 +405,7 @@ pub fn create_default_context() -> EngineState {
             Fetch,
             Post,
             Url,
-            UrlHost,
+            UrlParse,
-            UrlPath,
-            UrlQuery,
-            UrlScheme,
             Port,
         }
 

crates/nu-command/src/network/post.rs

@@ -186,7 +186,7 @@ fn helper(
         Ok(u) => u,
         Err(_e) => {
             return Err(ShellError::UnsupportedInput(
-                "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
+                "Incomplete or incorrect url. Expected a full url, e.g., https://www.example.com"
                     .to_string(),
                 span,
             ));

crates/nu-command/src/network/url/host.rs

@@ -1,70 +0,0 @@
-use super::{operator, url};
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Type, Value};
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "url host"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("url host")
-            .input_output_types(vec![(Type::String, Type::String)])
-            .rest(
-                "rest",
-                SyntaxShape::CellPath,
-                "optionally operate by cell path",
-            )
-            .category(Category::Network)
-    }
-
-    fn usage(&self) -> &str {
-        "Get the host of a URL"
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["hostname"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        operator(engine_state, stack, call, input, &host)
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        let span = Span::test_data();
-        vec![Example {
-            description: "Get host of a url",
-            example: "echo 'http://www.example.com/foo/bar' | url host",
-            result: Some(Value::String {
-                val: "www.example.com".to_string(),
-                span,
-            }),
-        }]
-    }
-}
-
-fn host(url: &url::Url) -> &str {
-    url.host_str().unwrap_or("")
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        use crate::test_examples;
-
-        test_examples(SubCommand {})
-    }
-}

crates/nu-command/src/network/url/mod.rs

@@ -1,92 +1,7 @@
-mod host;
+mod parse;
-mod path;
-mod query;
-mod scheme;
 mod url_;
 
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::{Call, CellPath},
-    engine::{EngineState, Stack},
-    PipelineData, ShellError, Span, Value,
-};
 use url::{self};
 
-pub use self::host::SubCommand as UrlHost;
+pub use self::parse::SubCommand as UrlParse;
-pub use self::path::SubCommand as UrlPath;
-pub use self::query::SubCommand as UrlQuery;
-pub use self::scheme::SubCommand as UrlScheme;
 pub use url_::Url;
-
-fn handle_value<F>(action: &F, v: &Value, span: Span) -> Value
-where
-    F: Fn(&url::Url) -> &str + Send + 'static,
-{
-    let a = |url| Value::String {
-        val: action(url).to_string(),
-        span,
-    };
-
-    match v {
-        Value::String { val: s, .. } => {
-            let s = s.trim();
-
-            match url::Url::parse(s) {
-                Ok(url) => a(&url),
-                Err(_) => Value::String {
-                    val: "".to_string(),
-                    span,
-                },
-            }
-        }
-        other => {
-            let span = other.span();
-            match span {
-                Ok(s) => {
-                    let got = format!("Expected a string, got {} instead", other.get_type());
-                    Value::Error {
-                        error: ShellError::UnsupportedInput(got, s),
-                    }
-                }
-                Err(e) => Value::Error { error: e },
-            }
-        }
-    }
-}
-
-fn operator<F>(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-    action: &'static F,
-) -> Result<PipelineData, ShellError>
-where
-    F: Fn(&url::Url) -> &str + Send + Sync + 'static,
-{
-    let span = call.head;
-    let column_paths: Vec<CellPath> = call.rest(engine_state, stack, 0)?;
-
-    input.map(
-        move |v| {
-            if column_paths.is_empty() {
-                handle_value(&action, &v, span)
-            } else {
-                let mut ret = v;
-
-                for path in &column_paths {
-                    let r = ret.update_cell_path(
-                        &path.members,
-                        Box::new(move |old| handle_value(&action, old, span)),
-                    );
-                    if let Err(error) = r {
-                        return Value::Error { error };
-                    }
-                }
-
-                ret
-            }
-        },
-        engine_state.ctrlc.clone(),
-    )
-}

crates/nu-command/src/network/url/parse.rs

@@ -0,0 +1,201 @@
+use super::url;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+};
+
+use url::Url;
+
+#[derive(Clone)]
+pub struct SubCommand;
+
+impl Command for SubCommand {
+    fn name(&self) -> &str {
+        "url parse"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("url parse")
+            .input_output_types(vec![(Type::String, Type::Record(vec![]))])
+            .rest(
+                "rest",
+                SyntaxShape::CellPath,
+                "optionally operate by cell path",
+            )
+            .category(Category::Network)
+    }
+
+    fn usage(&self) -> &str {
+        "Parses a url"
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec![
+            "scheme", "username", "password", "hostname", "port", "path", "query", "fragment",
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        _: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        parse(input.into_value(call.head), engine_state)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Parses a url",
+            example: "'http://user123:pass567@www.example.com:8081/foo/bar?param1=section&p2=&f[name]=vldc#hello' | url parse",
+            result: Some(Value::Record {
+                cols: vec![
+                    "scheme".to_string(),
+                    "username".to_string(),
+                    "password".to_string(),
+                    "host".to_string(),
+                    "port".to_string(),
+                    "path".to_string(),
+                    "query".to_string(),
+                    "fragment".to_string(),
+                    "params".to_string(),
+                ],
+                vals: vec![
+                    Value::test_string("http"),
+                    Value::test_string("user123"),
+                    Value::test_string("pass567"),
+                    Value::test_string("www.example.com"),
+                    Value::test_string("8081"),
+                    Value::test_string("/foo/bar"),
+                    Value::test_string("param1=section&p2=&f[name]=vldc"),
+                    Value::test_string("hello"),
+                    Value::Record {
+                        cols: vec!["param1".to_string(), "p2".to_string(), "f[name]".to_string()],
+                        vals: vec![
+                            Value::test_string("section"),
+                            Value::test_string(""),
+                            Value::test_string("vldc"),
+                        ],
+                        span: Span::test_data(),
+                    },
+                ],
+                span: Span::test_data(),
+            }),
+        }]
+    }
+}
+
+fn get_url_string(value: &Value, engine_state: &EngineState) -> String {
+    value.into_string("", engine_state.get_config())
+}
+
+fn parse(value: Value, engine_state: &EngineState) -> Result<PipelineData, ShellError> {
+    let url_string = get_url_string(&value, engine_state);
+
+    let result_url = Url::parse(url_string.as_str());
+
+    let head = value.span()?;
+
+    match result_url {
+        Ok(url) => {
+            let cols = vec![
+                String::from("scheme"),
+                String::from("username"),
+                String::from("password"),
+                String::from("host"),
+                String::from("port"),
+                String::from("path"),
+                String::from("query"),
+                String::from("fragment"),
+                String::from("params"),
+            ];
+            let mut vals: Vec<Value> = vec![
+                Value::String {
+                    val: String::from(url.scheme()),
+                    span: head,
+                },
+                Value::String {
+                    val: String::from(url.username()),
+                    span: head,
+                },
+                Value::String {
+                    val: String::from(url.password().unwrap_or("")),
+                    span: head,
+                },
+                Value::String {
+                    val: String::from(url.host_str().unwrap_or("")),
+                    span: head,
+                },
+                Value::String {
+                    val: url
+                        .port()
+                        .map(|p| p.to_string())
+                        .unwrap_or_else(|| "".into()),
+                    span: head,
+                },
+                Value::String {
+                    val: String::from(url.path()),
+                    span: head,
+                },
+                Value::String {
+                    val: String::from(url.query().unwrap_or("")),
+                    span: head,
+                },
+                Value::String {
+                    val: String::from(url.fragment().unwrap_or("")),
+                    span: head,
+                },
+            ];
+
+            let params =
+                serde_urlencoded::from_str::<Vec<(String, String)>>(url.query().unwrap_or(""));
+            match params {
+                Ok(result) => {
+                    let (param_cols, param_vals) = result
+                        .into_iter()
+                        .map(|(k, v)| (k, Value::String { val: v, span: head }))
+                        .unzip();
+
+                    vals.push(Value::Record {
+                        cols: param_cols,
+                        vals: param_vals,
+                        span: head,
+                    });
+
+                    Ok(PipelineData::Value(
+                        Value::Record {
+                            cols,
+                            vals,
+                            span: head,
+                        },
+                        None,
+                    ))
+                }
+
+                _ => Err(ShellError::UnsupportedInput(
+                    "String not compatible with url-encoding".to_string(),
+                    head,
+                )),
+            }
+        }
+        Err(_e) => Err(ShellError::UnsupportedInput(
+            "Incomplete or incorrect url. Expected a full url, e.g., https://www.example.com"
+                .to_string(),
+            head,
+        )),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(SubCommand {})
+    }
+}

crates/nu-command/src/network/url/path.rs

@@ -1,72 +0,0 @@
-use super::{operator, url};
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Type, Value};
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "url path"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("url path")
-            .input_output_types(vec![(Type::String, Type::String)])
-            .rest(
-                "rest",
-                SyntaxShape::CellPath,
-                "optionally operate by cell path",
-            )
-            .category(Category::Network)
-    }
-
-    fn usage(&self) -> &str {
-        "Get the path of a URL"
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        operator(engine_state, stack, call, input, &url::Url::path)
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        let span = Span::test_data();
-        vec![
-            Example {
-                description: "Get path of a url",
-                example: "echo 'http://www.example.com/foo/bar' | url path",
-                result: Some(Value::String {
-                    val: "/foo/bar".to_string(),
-                    span,
-                }),
-            },
-            Example {
-                description: "A trailing slash will be reflected in the path",
-                example: "echo 'http://www.example.com' | url path",
-                result: Some(Value::String {
-                    val: "/".to_string(),
-                    span,
-                }),
-            },
-        ]
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        use crate::test_examples;
-
-        test_examples(SubCommand {})
-    }
-}

crates/nu-command/src/network/url/query.rs

@@ -1,80 +0,0 @@
-use super::{operator, url};
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Type, Value};
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "url query"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("url query")
-            .input_output_types(vec![(Type::String, Type::String)])
-            .rest(
-                "rest",
-                SyntaxShape::CellPath,
-                "optionally operate by cell path",
-            )
-            .category(Category::Network)
-    }
-
-    fn usage(&self) -> &str {
-        "Get the query string of a URL"
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["parameter"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        operator(engine_state, stack, call, input, &query)
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        let span = Span::test_data();
-        vec![
-            Example {
-                description: "Get a query string",
-                example: "echo 'http://www.example.com/?foo=bar&baz=quux' | url query",
-                result: Some(Value::String {
-                    val: "foo=bar&baz=quux".to_string(),
-                    span,
-                }),
-            },
-            Example {
-                description: "Returns an empty string if there is no query string",
-                example: "echo 'http://www.example.com/' | url query",
-                result: Some(Value::String {
-                    val: "".to_string(),
-                    span,
-                }),
-            },
-        ]
-    }
-}
-
-fn query(url: &url::Url) -> &str {
-    url.query().unwrap_or("")
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        use crate::test_examples;
-
-        test_examples(SubCommand {})
-    }
-}

crates/nu-command/src/network/url/scheme.rs

@@ -1,76 +0,0 @@
-use super::{operator, url};
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Type, Value};
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "url scheme"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("url scheme")
-            .input_output_types(vec![(Type::String, Type::String)])
-            .rest(
-                "rest",
-                SyntaxShape::CellPath,
-                "optionally operate by cell path",
-            )
-            .category(Category::Network)
-    }
-
-    fn usage(&self) -> &str {
-        "Get the scheme (e.g. http, file) of a URL"
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["protocol"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        operator(engine_state, stack, call, input, &url::Url::scheme)
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        let span = Span::test_data();
-        vec![
-            Example {
-                description: "Get the scheme of a URL",
-                example: "echo 'http://www.example.com' | url scheme",
-                result: Some(Value::String {
-                    val: "http".to_string(),
-                    span,
-                }),
-            },
-            Example {
-                description: "You get an empty string if there is no scheme",
-                example: "echo 'test' | url scheme",
-                result: Some(Value::String {
-                    val: "".to_string(),
-                    span,
-                }),
-            },
-        ]
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        use crate::test_examples;
-
-        test_examples(SubCommand {})
-    }
-}

crates/nu-command/tests/commands/mod.rs

@@ -84,6 +84,7 @@ mod transpose;
 mod uniq;
 mod update;
 mod upsert;
+mod url;
 mod use_;
 mod where_;
 #[cfg(feature = "which-support")]

crates/nu-command/tests/commands/url/mod.rs

@@ -0,0 +1 @@
+mod parse;

crates/nu-command/tests/commands/url/parse.rs

@@ -0,0 +1,159 @@
+use nu_test_support::{nu, pipeline};
+
+#[test]
+fn url_parse_simple() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ("https://www.abc.com"
+                | url parse)
+                == {
+                    scheme: 'https',
+                    username: '',
+                    password: '',
+                    host: 'www.abc.com',
+                    port: '',
+                    path: '/',
+                    query: '',
+                    fragment: '',
+                    params: {}
+                }
+            "#
+    ));
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_with_port() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ("https://www.abc.com:8011"
+                | url parse)
+                == {
+                    scheme: 'https',
+                    username: '',
+                    password: '',
+                    host: 'www.abc.com',
+                    port: '8011',
+                    path: '/',
+                    query: '',
+                    fragment: '',
+                    params: {}
+                }
+            "#
+    ));
+
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_with_path() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ("http://www.abc.com:8811/def/ghj"
+                | url parse)
+                == {
+                    scheme: 'http',
+                    username: '',
+                    password: '',
+                    host: 'www.abc.com',
+                    port: '8811',
+                    path: '/def/ghj',
+                    query: '',
+                    fragment: '',
+                    params: {}
+                }
+            "#
+    ));
+
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_with_params() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ("http://www.abc.com:8811/def/ghj?param1=11&param2="
+                | url parse)
+                == {
+                    scheme: 'http',
+                    username: '',
+                    password: '',
+                    host: 'www.abc.com',
+                    port: '8811',
+                    path: '/def/ghj',
+                    query: 'param1=11&param2=',
+                    fragment: '',
+                    params: {param1: '11', param2: ''}
+                }
+            "#
+    ));
+
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_with_fragment() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ("http://www.abc.com:8811/def/ghj?param1=11&param2=#hello-fragment"
+                | url parse)
+                == {
+                    scheme: 'http',
+                    username: '',
+                    password: '',
+                    host: 'www.abc.com',
+                    port: '8811',
+                    path: '/def/ghj',
+                    query: 'param1=11&param2=',
+                    fragment: 'hello-fragment',
+                    params: {param1: '11', param2: ''}
+                }
+            "#
+    ));
+
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_with_username_and_password() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ("http://user123:password567@www.abc.com:8811/def/ghj?param1=11&param2=#hello-fragment"
+                | url parse)
+                == {
+                    scheme: 'http',
+                    username: 'user123',
+                    password: 'password567',
+                    host: 'www.abc.com',
+                    port: '8811',
+                    path: '/def/ghj',
+                    query: 'param1=11&param2=',
+                    fragment: 'hello-fragment',
+                    params: {param1: '11', param2: ''}
+                }
+            "#
+    ));
+
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_error_empty_url() {
+    let actual = nu!(
+        cwd: ".", pipeline(
+            r#"
+                ""
+                | url parse
+            "#
+    ));
+
+    assert!(actual.err.contains(
+        "Incomplete or incorrect url. Expected a full url, e.g., https://www.example.com"
+    ));
+}