From 4e931fa73fd7fa088d633e2291fd9e25b849fd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Mon, 12 Oct 2020 18:18:39 -0500 Subject: [PATCH] Extract out xpath to a plugin. (#2661) --- Cargo.lock | 16 +++ Cargo.toml | 9 +- crates/nu-cli/src/cli.rs | 1 - crates/nu-cli/src/commands.rs | 2 - crates/nu-cli/src/commands/xpath.rs | 150 ----------------------- crates/nu-cli/tests/commands/xpath.rs | 41 ------- crates/nu_plugin_xpath/Cargo.toml | 24 ++++ crates/nu_plugin_xpath/src/lib.rs | 4 + crates/nu_plugin_xpath/src/main.rs | 6 + crates/nu_plugin_xpath/src/nu/mod.rs | 49 ++++++++ crates/nu_plugin_xpath/src/xpath.rs | 166 ++++++++++++++++++++++++++ src/plugins/nu_plugin_extra_xpath.rs | 6 + tests/shell/mod.rs | 13 +- wix/main.wxs | 8 ++ 14 files changed, 292 insertions(+), 203 deletions(-) delete mode 100644 crates/nu-cli/src/commands/xpath.rs delete mode 100644 crates/nu-cli/tests/commands/xpath.rs create mode 100644 crates/nu_plugin_xpath/Cargo.toml create mode 100644 crates/nu_plugin_xpath/src/lib.rs create mode 100644 crates/nu_plugin_xpath/src/main.rs create mode 100644 crates/nu_plugin_xpath/src/nu/mod.rs create mode 100644 crates/nu_plugin_xpath/src/xpath.rs create mode 100644 src/plugins/nu_plugin_extra_xpath.rs diff --git a/Cargo.lock b/Cargo.lock index b133732c62..a2c2fc18ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2890,6 +2890,7 @@ dependencies = [ "nu_plugin_to_bson", "nu_plugin_to_sqlite", "nu_plugin_tree", + "nu_plugin_xpath", "pretty_env_logger", "serde 1.0.115", "toml", @@ -3367,6 +3368,21 @@ dependencies = [ "ptree", ] +[[package]] +name = "nu_plugin_xpath" +version = "0.20.0" +dependencies = [ + "bigdecimal", + "indexmap", + "nu-errors", + "nu-plugin", + "nu-protocol", + "nu-source", + "nu-test-support", + "sxd-document", + "sxd-xpath", +] + [[package]] name = "num" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 68c9c6bf93..95f6a0d38d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ nu_plugin_textview = {version = "0.20.0", path = "./crates/nu_plugin_textview", nu_plugin_to_bson = {version = "0.20.0", path = "./crates/nu_plugin_to_bson", optional = true} nu_plugin_to_sqlite = {version = "0.20.0", path = "./crates/nu_plugin_to_sqlite", optional = true} nu_plugin_tree = {version = "0.20.0", path = "./crates/nu_plugin_tree", optional = true} +nu_plugin_xpath = {version = "0.20.0", path = "./crates/nu_plugin_xpath", optional = true} # Required to bootstrap the main binary clap = "2.33.3" @@ -88,7 +89,7 @@ default = [ "fetch", "rich-benchmark", ] -extra = ["default", "binaryview", "tree", "clipboard-cli", "trash-support", "start", "bson", "sqlite", "s3", "chart"] +extra = ["default", "binaryview", "tree", "clipboard-cli", "trash-support", "start", "bson", "sqlite", "s3", "chart", "xpath"] stable = ["default"] wasi = ["inc", "match", "directories-support", "ptree-support", "match", "tree", "rustyline-support"] @@ -114,6 +115,7 @@ sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"] start = ["nu_plugin_start"] trash-support = ["nu-cli/trash-support"] tree = ["nu_plugin_tree"] +xpath = ["nu_plugin_xpath"] # Core plugins that ship with `cargo install nu` by default # Currently, Cargo limits us to installing only one binary @@ -185,6 +187,11 @@ name = "nu_plugin_extra_chart_line" path = "src/plugins/nu_plugin_extra_chart_line.rs" required-features = ["chart"] +[[bin]] +name = "nu_plugin_extra_xpath" +path = "src/plugins/nu_plugin_extra_xpath.rs" +required-features = ["xpath"] + # Main nu binary [[bin]] name = "nu" diff --git a/crates/nu-cli/src/cli.rs b/crates/nu-cli/src/cli.rs index 90a90a2dcd..f38205b8e2 100644 --- a/crates/nu-cli/src/cli.rs +++ b/crates/nu-cli/src/cli.rs @@ -245,7 +245,6 @@ pub fn create_default_context(interactive: bool) -> Result, -} - -#[async_trait] -impl WholeStreamCommand for XPath { - fn name(&self) -> &str { - "xpath" - } - - fn signature(&self) -> Signature { - Signature::build("xpath").required("query", SyntaxShape::String, "xpath query") - } - - fn usage(&self) -> &str { - "execute xpath query on xml" - } - - fn examples(&self) -> Vec { - vec![Example { - description: "find items with name attribute", - example: r#"echo '
' | from xml | to xml | xpath '//nushell/@rocks'"#, - result: None, - }] - } - - async fn run( - &self, - args: CommandArgs, - registry: &CommandRegistry, - ) -> Result { - let tag = args.call_info.name_tag.clone(); - let (XPathArgs { query }, input) = args.process(®istry).await?; - - let query_string = query.as_str(); - let input_string = input.collect_string(tag.clone()).await?.item; - let result_string = execute_xpath_query(input_string, query_string.to_string()); - - match result_string { - Some(r) => Ok( - futures::stream::iter(r.into_iter().map(ReturnSuccess::value)).to_output_stream(), - ), - None => Err(ShellError::labeled_error( - "xpath query error", - "xpath query error", - query.tag(), - )), - } - } -} - -pub fn execute_xpath_query(input_string: String, query_string: String) -> Option> { - let xpath = build_xpath(&query_string); - let package = parser::parse(&input_string).expect("failed to parse xml"); - let document = package.as_document(); - let context = Context::new(); - - // leaving this here for augmentation at some point - // build_variables(&arguments, &mut context); - // build_namespaces(&arguments, &mut context); - - let res = xpath.evaluate(&context, document.root()); - - // Some xpath statements can be long, so let's truncate it with ellipsis - let mut key = query_string.clone(); - if query_string.len() >= 20 { - key.truncate(17); - key += "..."; - } else { - key = query_string; - }; - - match res { - Ok(r) => { - let rows: Vec = match r { - sxd_xpath::Value::Nodeset(ns) => ns - .into_iter() - .map(|a| { - let mut row = TaggedDictBuilder::new(Tag::unknown()); - row.insert_value(&key, UntaggedValue::string(a.string_value())); - row.into_value() - }) - .collect::>(), - sxd_xpath::Value::Boolean(b) => { - let mut row = TaggedDictBuilder::new(Tag::unknown()); - row.insert_value(&key, UntaggedValue::boolean(b)); - vec![row.into_value()] - } - sxd_xpath::Value::Number(n) => { - let mut row = TaggedDictBuilder::new(Tag::unknown()); - row.insert_value( - &key, - UntaggedValue::decimal(BigDecimal::from_f64(n).expect("error with f64")) - .into_untagged_value(), - ); - - vec![row.into_value()] - } - sxd_xpath::Value::String(s) => { - let mut row = TaggedDictBuilder::new(Tag::unknown()); - row.insert_value(&key, UntaggedValue::string(s)); - vec![row.into_value()] - } - }; - - if !rows.is_empty() { - Some(rows) - } else { - None - } - } - Err(_) => None, - } -} - -fn build_xpath(xpath_str: &str) -> sxd_xpath::XPath { - let factory = Factory::new(); - - factory - .build(xpath_str) - .unwrap_or_else(|e| panic!("Unable to compile XPath {}: {}", xpath_str, e)) - .expect("error with building the xpath factory") -} - -#[cfg(test)] -mod tests { - use super::ShellError; - use super::XPath; - - #[test] - fn examples_work_as_expected() -> Result<(), ShellError> { - use crate::examples::test as test_examples; - - Ok(test_examples(XPath {})?) - } -} diff --git a/crates/nu-cli/tests/commands/xpath.rs b/crates/nu-cli/tests/commands/xpath.rs deleted file mode 100644 index 27ebd54763..0000000000 --- a/crates/nu-cli/tests/commands/xpath.rs +++ /dev/null @@ -1,41 +0,0 @@ -use nu_test_support::fs::Stub::FileWithContentToBeTrimmed; -use nu_test_support::playground::Playground; -use nu_test_support::{nu, pipeline}; - -#[test] -fn position_function_in_predicate() { - let actual = nu!( - cwd: ".", pipeline( - r#" - echo "" | from xml | to xml | xpath "count(//a/*[position() = 2])" - "# - )); - - assert_eq!(actual.out, "1.0000"); -} - -#[test] -fn functions_implicitly_coerce_argument_types() { - let actual = nu!( - cwd: ".", pipeline( - r#" - echo "true" | from xml | to xml | xpath "count(//*[contains(., true)])" - "# - )); - - assert_eq!(actual.out, "1.0000"); -} - -#[test] -fn find_guid_permilink_is_true() { - let actual = nu!( - cwd: "tests/fixtures/formats", pipeline( - r#" - open jonathan.xml - | to xml - | xpath '//guid/@isPermaLink' - "# - )); - - assert_eq!(actual.out, "true"); -} diff --git a/crates/nu_plugin_xpath/Cargo.toml b/crates/nu_plugin_xpath/Cargo.toml new file mode 100644 index 0000000000..7f66468728 --- /dev/null +++ b/crates/nu_plugin_xpath/Cargo.toml @@ -0,0 +1,24 @@ +[package] +authors = ["The Nu Project Contributors"] +description = "Traverses xml" +edition = "2018" +license = "MIT" +name = "nu_plugin_xpath" +version = "0.20.0" + +[lib] +doctest = false + +[dependencies] +nu-plugin = {path = "../nu-plugin", version = "0.20.0"} +nu-errors = {version = "0.20.0", path = "../nu-errors"} +nu-protocol = {version = "0.20.0", path = "../nu-protocol"} +nu-source = {version = "0.20.0", path = "../nu-source"} + +sxd-xpath = "0.4.2" +sxd-document = "0.3.2" +bigdecimal = {version = "0.2.0", features = ["serde"]} +indexmap = {version = "1.6.0", features = ["serde-1"]} + +[dev-dependencies] +nu-test-support = {path = "../nu-test-support", version = "0.20.0"} \ No newline at end of file diff --git a/crates/nu_plugin_xpath/src/lib.rs b/crates/nu_plugin_xpath/src/lib.rs new file mode 100644 index 0000000000..7d6683f863 --- /dev/null +++ b/crates/nu_plugin_xpath/src/lib.rs @@ -0,0 +1,4 @@ +mod nu; +mod xpath; + +pub use xpath::Xpath; diff --git a/crates/nu_plugin_xpath/src/main.rs b/crates/nu_plugin_xpath/src/main.rs new file mode 100644 index 0000000000..66ae2ee151 --- /dev/null +++ b/crates/nu_plugin_xpath/src/main.rs @@ -0,0 +1,6 @@ +use nu_plugin::serve_plugin; +use nu_plugin_xpath::Xpath; + +fn main() { + serve_plugin(&mut Xpath::new()); +} diff --git a/crates/nu_plugin_xpath/src/nu/mod.rs b/crates/nu_plugin_xpath/src/nu/mod.rs new file mode 100644 index 0000000000..80619aa10a --- /dev/null +++ b/crates/nu_plugin_xpath/src/nu/mod.rs @@ -0,0 +1,49 @@ +use nu_errors::ShellError; +use nu_plugin::Plugin; +use nu_protocol::{ + CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value, +}; +use nu_source::TaggedItem; + +use crate::{xpath::string_to_value, Xpath}; + +impl Plugin for Xpath { + fn config(&mut self) -> Result { + Ok(Signature::build("xpath") + .desc("execute xpath query on xml") + .required("query", SyntaxShape::String, "xpath query") + .filter()) + } + + fn begin_filter(&mut self, call_info: CallInfo) -> Result, ShellError> { + let tag = call_info.name_tag; + + let query = call_info.args.nth(0).ok_or_else(|| { + ShellError::labeled_error("xpath query not passed", "xpath query not passed", &tag) + })?; + + self.query = query.as_string()?; + self.tag = tag; + + Ok(vec![]) + } + + fn filter(&mut self, input: Value) -> Result, ShellError> { + match input { + Value { + value: UntaggedValue::Primitive(Primitive::String(s)), + .. + } => Ok(string_to_value(s, (*self.query).tagged(&self.tag))? + .into_iter() + .map(ReturnSuccess::value) + .collect()), + Value { tag, .. } => Err(ShellError::labeled_error_with_secondary( + "Expected text from pipeline", + "requires text input", + &self.tag, + "value originates from here", + tag, + )), + } + } +} diff --git a/crates/nu_plugin_xpath/src/xpath.rs b/crates/nu_plugin_xpath/src/xpath.rs new file mode 100644 index 0000000000..857a8a770a --- /dev/null +++ b/crates/nu_plugin_xpath/src/xpath.rs @@ -0,0 +1,166 @@ +use nu_errors::ShellError; +use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value}; +use nu_source::{Tag, Tagged}; + +use bigdecimal::{BigDecimal, FromPrimitive}; + +use sxd_document::parser; +use sxd_xpath::{Context, Factory}; + +pub struct Xpath { + pub query: String, + pub tag: Tag, +} + +impl Xpath { + pub fn new() -> Xpath { + Xpath { + query: String::new(), + tag: Tag::unknown(), + } + } +} + +impl Default for Xpath { + fn default() -> Self { + Self::new() + } +} + +pub fn string_to_value(raw: String, query: Tagged<&str>) -> Result, ShellError> { + execute_xpath_query(raw, query.item.to_string(), query.tag()) +} + +fn execute_xpath_query( + input_string: String, + query_string: String, + tag: impl Into, +) -> Result, ShellError> { + let tag = tag.into(); + let xpath = build_xpath(&query_string)?; + + let package = parser::parse(&input_string); + + if package.is_err() { + return Err(ShellError::labeled_error( + "invalid xml document", + "invalid xml document", + tag.span, + )); + } + + let package = package.expect("invalid xml document"); + + let document = package.as_document(); + let context = Context::new(); + + // leaving this here for augmentation at some point + // build_variables(&arguments, &mut context); + // build_namespaces(&arguments, &mut context); + + let res = xpath.evaluate(&context, document.root()); + + // Some xpath statements can be long, so let's truncate it with ellipsis + let mut key = query_string.clone(); + if query_string.len() >= 20 { + key.truncate(17); + key += "..."; + } else { + key = query_string; + }; + + match res { + Ok(r) => { + let rows: Vec = match r { + sxd_xpath::Value::Nodeset(ns) => ns + .into_iter() + .map(|a| { + let mut row = TaggedDictBuilder::new(Tag::unknown()); + row.insert_value(&key, UntaggedValue::string(a.string_value())); + row.into_value() + }) + .collect::>(), + sxd_xpath::Value::Boolean(b) => { + let mut row = TaggedDictBuilder::new(Tag::unknown()); + row.insert_value(&key, UntaggedValue::boolean(b)); + vec![row.into_value()] + } + sxd_xpath::Value::Number(n) => { + let mut row = TaggedDictBuilder::new(Tag::unknown()); + row.insert_value( + &key, + UntaggedValue::decimal(BigDecimal::from_f64(n).expect("error with f64")) + .into_untagged_value(), + ); + + vec![row.into_value()] + } + sxd_xpath::Value::String(s) => { + let mut row = TaggedDictBuilder::new(Tag::unknown()); + row.insert_value(&key, UntaggedValue::string(s)); + vec![row.into_value()] + } + }; + + Ok(rows) + } + Err(_) => Err(ShellError::labeled_error( + "xpath query error", + "xpath query error", + tag, + )), + } +} + +fn build_xpath(xpath_str: &str) -> Result { + let factory = Factory::new(); + + match factory.build(xpath_str) { + Ok(xpath) => xpath.ok_or_else(|| ShellError::untagged_runtime_error("invalid xpath query")), + Err(_) => Err(ShellError::untagged_runtime_error( + "expected valid xpath query", + )), + } +} + +#[cfg(test)] +mod tests { + use super::string_to_value as query; + use nu_errors::ShellError; + use nu_source::{Span, TaggedItem}; + use nu_test_support::value::{decimal_from_float, row}; + + use indexmap::indexmap; + + #[test] + fn position_function_in_predicate() -> Result<(), ShellError> { + let text = String::from(r#""#); + + let actual = query(text, "count(//a/*[position() = 2])".tagged_unknown())?; + + assert_eq!( + actual[0], + row( + indexmap! { "count(//a/*[posit...".into() => decimal_from_float(1.0, Span::unknown()) } + ) + ); + + Ok(()) + } + + #[test] + fn functions_implicitly_coerce_argument_types() -> Result<(), ShellError> { + let text = String::from(r#"true"#); + + let actual = query(text, "count(//*[contains(., true)])".tagged_unknown())?; + + assert_eq!( + actual[0], + row( + indexmap! { "count(//*[contain...".into() => decimal_from_float(1.0, Span::unknown()) } + ) + ); + + Ok(()) + } +} diff --git a/src/plugins/nu_plugin_extra_xpath.rs b/src/plugins/nu_plugin_extra_xpath.rs new file mode 100644 index 0000000000..66ae2ee151 --- /dev/null +++ b/src/plugins/nu_plugin_extra_xpath.rs @@ -0,0 +1,6 @@ +use nu_plugin::serve_plugin; +use nu_plugin_xpath::Xpath; + +fn main() { + serve_plugin(&mut Xpath::new()); +} diff --git a/tests/shell/mod.rs b/tests/shell/mod.rs index ec78dee32f..967c4cfeb8 100644 --- a/tests/shell/mod.rs +++ b/tests/shell/mod.rs @@ -7,7 +7,8 @@ fn plugins_are_declared_with_wix() { let actual = nu!( cwd: ".", pipeline( r#" - echo $(open wix/main.wxs --raw | from xml + open Cargo.toml | get bin.name | drop | sort-by | wrap cargo | merge { + open wix/main.wxs --raw | from xml | get Wix.children.Product.children.0.Directory.children.0 | where Directory.attributes.Id == "$(var.PlatformProgramFilesFolder)" | get Directory.children.Directory.children.0 | last @@ -18,13 +19,9 @@ fn plugins_are_declared_with_wix() { | str substring [_, -4] File.attributes.Name | get File.attributes.Name | sort-by - | wrap wix) | merge { - open Cargo.toml | - get bin.name | - drop | - sort-by | - wrap cargo - } + | wrap wix + } + | default wix _ | if $it.wix != $it.cargo { = 1 } { = 0 } | math sum "# diff --git a/wix/main.wxs b/wix/main.wxs index 79a7c449da..00a20f0a9d 100644 --- a/wix/main.wxs +++ b/wix/main.wxs @@ -232,6 +232,14 @@ Source='target\$(var.Profile)\nu_plugin_extra_chart_line.exe' KeyPath='yes'/> + + +