From c3b6e07de631aec3b3f3506d140d961e4518ce36 Mon Sep 17 00:00:00 2001 From: Jae-Heon Ji <32578710+jaeheonji@users.noreply.github.com> Date: Fri, 10 Dec 2021 09:09:30 +0900 Subject: [PATCH] Port `network/url` command (#452) * feat: add url command * feat(network/url): add sub-command for url --- Cargo.lock | 1 + crates/nu-command/Cargo.toml | 1 + crates/nu-command/src/example_test.rs | 3 +- crates/nu-command/src/formats/from/mod.rs | 2 +- crates/nu-command/src/formats/to/mod.rs | 2 +- crates/nu-command/src/lib.rs | 2 + crates/nu-command/src/network/mod.rs | 3 + crates/nu-command/src/network/url/command.rs | 37 ++++++++ crates/nu-command/src/network/url/host.rs | 65 ++++++++++++++ crates/nu-command/src/network/url/mod.rs | 92 ++++++++++++++++++++ crates/nu-command/src/network/url/path.rs | 71 +++++++++++++++ crates/nu-command/src/network/url/query.rs | 75 ++++++++++++++++ crates/nu-command/src/network/url/scheme.rs | 71 +++++++++++++++ crates/nu-protocol/src/signature.rs | 2 + 14 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 crates/nu-command/src/network/mod.rs create mode 100644 crates/nu-command/src/network/url/command.rs create mode 100644 crates/nu-command/src/network/url/host.rs create mode 100644 crates/nu-command/src/network/url/mod.rs create mode 100644 crates/nu-command/src/network/url/path.rs create mode 100644 crates/nu-command/src/network/url/query.rs create mode 100644 crates/nu-command/src/network/url/scheme.rs diff --git a/Cargo.lock b/Cargo.lock index cfa186628b..2dff56a5da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1610,6 +1610,7 @@ dependencies = [ "toml", "trash", "unicode-segmentation", + "url", "uuid", ] diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 6c6c48eb61..68a76f04b7 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -16,6 +16,7 @@ nu-parser = { path = "../nu-parser" } nu-ansi-term = { path = "../nu-ansi-term" } # Potential dependencies for extras +url = "2.2.1" csv = "1.1.3" glob = "0.3.0" Inflector = "0.11" diff --git a/crates/nu-command/src/example_test.rs b/crates/nu-command/src/example_test.rs index da5789c917..ed7e9513e1 100644 --- a/crates/nu-command/src/example_test.rs +++ b/crates/nu-command/src/example_test.rs @@ -7,7 +7,7 @@ use nu_protocol::{ use crate::To; -use super::{Date, From, Into, Math, Random, Split, Str}; +use super::{Date, From, Into, Math, Random, Split, Str, Url}; pub fn test_examples(cmd: impl Command + 'static) { let examples = cmd.examples(); @@ -25,6 +25,7 @@ pub fn test_examples(cmd: impl Command + 'static) { working_set.add_decl(Box::new(Split)); working_set.add_decl(Box::new(Math)); working_set.add_decl(Box::new(Date)); + working_set.add_decl(Box::new(Url)); use super::Echo; working_set.add_decl(Box::new(Echo)); diff --git a/crates/nu-command/src/formats/from/mod.rs b/crates/nu-command/src/formats/from/mod.rs index 54f007d498..7f650d1641 100644 --- a/crates/nu-command/src/formats/from/mod.rs +++ b/crates/nu-command/src/formats/from/mod.rs @@ -17,6 +17,7 @@ mod yaml; pub use self::csv::FromCsv; pub use self::toml::FromToml; +pub use self::url::FromUrl; pub use command::From; pub use eml::FromEml; pub use ics::FromIcs; @@ -25,7 +26,6 @@ pub use json::FromJson; pub use ods::FromOds; pub use ssv::FromSsv; pub use tsv::FromTsv; -pub use url::FromUrl; pub use vcf::FromVcf; pub use xlsx::FromXlsx; pub use xml::FromXml; diff --git a/crates/nu-command/src/formats/to/mod.rs b/crates/nu-command/src/formats/to/mod.rs index 29f631915c..18209d3070 100644 --- a/crates/nu-command/src/formats/to/mod.rs +++ b/crates/nu-command/src/formats/to/mod.rs @@ -8,7 +8,7 @@ mod url; pub use self::csv::ToCsv; pub use self::toml::ToToml; +pub use self::url::ToUrl; pub use command::To; pub use json::ToJson; pub use tsv::ToTsv; -pub use url::ToUrl; diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index 593368639e..14a97fe741 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -10,6 +10,7 @@ mod filesystem; mod filters; mod formats; mod math; +mod network; mod platform; mod random; mod shells; @@ -29,6 +30,7 @@ pub use filesystem::*; pub use filters::*; pub use formats::*; pub use math::*; +pub use network::*; pub use platform::*; pub use random::*; pub use shells::*; diff --git a/crates/nu-command/src/network/mod.rs b/crates/nu-command/src/network/mod.rs new file mode 100644 index 0000000000..897fc3282d --- /dev/null +++ b/crates/nu-command/src/network/mod.rs @@ -0,0 +1,3 @@ +mod url; + +pub use self::url::*; diff --git a/crates/nu-command/src/network/url/command.rs b/crates/nu-command/src/network/url/command.rs new file mode 100644 index 0000000000..14cb545206 --- /dev/null +++ b/crates/nu-command/src/network/url/command.rs @@ -0,0 +1,37 @@ +use nu_engine::get_full_help; +use nu_protocol::{ + ast::Call, + engine::{Command, EngineState, Stack}, + Category, IntoPipelineData, PipelineData, Signature, Value, +}; + +#[derive(Clone)] +pub struct Url; + +impl Command for Url { + fn name(&self) -> &str { + "url" + } + + fn signature(&self) -> Signature { + Signature::build("url").category(Category::Network) + } + + fn usage(&self) -> &str { + "Apply url function." + } + + fn run( + &self, + engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + _input: PipelineData, + ) -> Result { + Ok(Value::String { + val: get_full_help(&Url.signature(), &Url.examples(), engine_state), + span: call.head, + } + .into_pipeline_data()) + } +} diff --git a/crates/nu-command/src/network/url/host.rs b/crates/nu-command/src/network/url/host.rs new file mode 100644 index 0000000000..215c268758 --- /dev/null +++ b/crates/nu-command/src/network/url/host.rs @@ -0,0 +1,65 @@ +use super::{operator, url}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "url host" + } + + fn signature(&self) -> Signature { + Signature::build("url host") + .rest( + "rest", + SyntaxShape::CellPath, + "optionally operate by cell path", + ) + .category(Category::Network) + } + + fn usage(&self) -> &str { + "gets the host of a url" + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + operator(engine_state, stack, call, input, &host) + } + + fn examples(&self) -> Vec { + let span = Span::unknown(); + vec![Example { + description: "Get host of a url", + example: "echo 'http://www.example.com/foo/bar' | url host", + result: Some(Value::String { + val: "www.example.com".to_string(), + span, + }), + }] + } +} + +fn host(url: &url::Url) -> &str { + url.host_str().unwrap_or("") +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-command/src/network/url/mod.rs b/crates/nu-command/src/network/url/mod.rs new file mode 100644 index 0000000000..5dcd97bdbf --- /dev/null +++ b/crates/nu-command/src/network/url/mod.rs @@ -0,0 +1,92 @@ +mod command; +mod host; +mod path; +mod query; +mod scheme; + +use nu_engine::CallExt; +use nu_protocol::{ + ast::{Call, CellPath}, + engine::{EngineState, Stack}, + PipelineData, ShellError, Span, Value, +}; +use url::{self}; + +pub use self::host::SubCommand as UrlHost; +pub use self::path::SubCommand as UrlPath; +pub use self::query::SubCommand as UrlQuery; +pub use self::scheme::SubCommand as UrlScheme; +pub use command::Url; + +fn handle_value(action: &F, v: &Value, span: Span) -> Value +where + F: Fn(&url::Url) -> &str + Send + 'static, +{ + let a = |url| Value::String { + val: action(url).to_string(), + span, + }; + + match v { + Value::String { val: s, .. } => { + let s = s.trim(); + + match url::Url::parse(s) { + Ok(url) => a(&url), + Err(_) => Value::String { + val: "".to_string(), + span, + }, + } + } + other => { + let span = other.span(); + match span { + Ok(s) => { + let got = format!("Expected a string, got {} instead", other.get_type()); + Value::Error { + error: ShellError::UnsupportedInput(got, s), + } + } + Err(e) => Value::Error { error: e }, + } + } + } +} + +fn operator( + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + action: &'static F, +) -> Result +where + F: Fn(&url::Url) -> &str + Send + Sync + 'static, +{ + let span = call.head; + let column_paths: Vec = call.rest(engine_state, stack, 0)?; + + input.map( + move |v| { + if column_paths.is_empty() { + handle_value(&action, &v, span) + } else { + let mut ret = v; + + for path in &column_paths { + let r = ret.update_cell_path( + &path.members, + Box::new(move |old| handle_value(&action, old, span)), + ); + if let Err(error) = r { + return Value::Error { error }; + } + } + + ret + } + }, + engine_state.ctrlc.clone(), + ) +} diff --git a/crates/nu-command/src/network/url/path.rs b/crates/nu-command/src/network/url/path.rs new file mode 100644 index 0000000000..c25b28557b --- /dev/null +++ b/crates/nu-command/src/network/url/path.rs @@ -0,0 +1,71 @@ +use super::{operator, url}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "url path" + } + + fn signature(&self) -> Signature { + Signature::build("url path") + .rest( + "rest", + SyntaxShape::CellPath, + "optionally operate by cell path", + ) + .category(Category::Network) + } + + fn usage(&self) -> &str { + "gets the path of a url" + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + operator(engine_state, stack, call, input, &url::Url::path) + } + + fn examples(&self) -> Vec { + let span = Span::unknown(); + vec![ + Example { + description: "Get path of a url", + example: "echo 'http://www.example.com/foo/bar' | url path", + result: Some(Value::String { + val: "/foo/bar".to_string(), + span, + }), + }, + Example { + description: "A trailing slash will be reflected in the path", + example: "echo 'http://www.example.com' | url path", + result: Some(Value::String { + val: "/".to_string(), + span, + }), + }, + ] + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-command/src/network/url/query.rs b/crates/nu-command/src/network/url/query.rs new file mode 100644 index 0000000000..b5b4c8ba38 --- /dev/null +++ b/crates/nu-command/src/network/url/query.rs @@ -0,0 +1,75 @@ +use super::{operator, url}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "url query" + } + + fn signature(&self) -> Signature { + Signature::build("url query") + .rest( + "rest", + SyntaxShape::CellPath, + "optionally operate by cell path", + ) + .category(Category::Network) + } + + fn usage(&self) -> &str { + "gets the query of a url" + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + operator(engine_state, stack, call, input, &query) + } + + fn examples(&self) -> Vec { + let span = Span::unknown(); + vec![ + Example { + description: "Get query of a url", + example: "echo 'http://www.example.com/?foo=bar&baz=quux' | url query", + result: Some(Value::String { + val: "foo=bar&baz=quux".to_string(), + span, + }), + }, + Example { + description: "No query gives the empty string", + example: "echo 'http://www.example.com/' | url query", + result: Some(Value::String { + val: "".to_string(), + span, + }), + }, + ] + } +} + +fn query(url: &url::Url) -> &str { + url.query().unwrap_or("") +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-command/src/network/url/scheme.rs b/crates/nu-command/src/network/url/scheme.rs new file mode 100644 index 0000000000..61c968b8dd --- /dev/null +++ b/crates/nu-command/src/network/url/scheme.rs @@ -0,0 +1,71 @@ +use super::{operator, url}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "url scheme" + } + + fn signature(&self) -> Signature { + Signature::build("url scheme") + .rest( + "rest", + SyntaxShape::CellPath, + "optionally operate by cell path", + ) + .category(Category::Network) + } + + fn usage(&self) -> &str { + "gets the scheme (eg http, file) of a url" + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + operator(engine_state, stack, call, input, &url::Url::scheme) + } + + fn examples(&self) -> Vec { + let span = Span::unknown(); + vec![ + Example { + description: "Get scheme of a url", + example: "echo 'http://www.example.com' | url scheme", + result: Some(Value::String { + val: "http".to_string(), + span, + }), + }, + Example { + description: "You get an empty string if there is no scheme", + example: "echo 'test' | url scheme", + result: Some(Value::String { + val: "".to_string(), + span, + }), + }, + ] + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-protocol/src/signature.rs b/crates/nu-protocol/src/signature.rs index 0b6b740158..4a33766b5f 100644 --- a/crates/nu-protocol/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -42,6 +42,7 @@ pub enum Category { Filters, Formats, Math, + Network, Random, Platform, Shells, @@ -64,6 +65,7 @@ impl std::fmt::Display for Category { Category::Filters => "filters", Category::Formats => "formats", Category::Math => "math", + Category::Network => "network", Category::Random => "random", Category::Platform => "platform", Category::Shells => "shells",