add query json plugin for experimentation (#3327)

* add query json plugin for experimentation

* add some error handling

* closer but Kind::Array is still horked

* unravel the table so the output looks right

* clippy

* added the ability to use gjson modifiers

Cargo.lock

@@ -2051,6 +2051,12 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "gjson"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "864178e25a00c41404f1728997c9a21a7b746be1faefe6ce4dc41eb48bb4234f"
+
 [[package]]
 name = "glob"
 version = "0.3.0"
@@ -3110,6 +3116,7 @@ dependencies = [
  "nu_plugin_match",
  "nu_plugin_post",
  "nu_plugin_ps",
+ "nu_plugin_query_json",
  "nu_plugin_s3",
  "nu_plugin_selector",
  "nu_plugin_start",
@@ -3704,6 +3711,18 @@ dependencies = [
  "sysinfo",
 ]
 
+[[package]]
+name = "nu_plugin_query_json"
+version = "0.29.2"
+dependencies = [
+ "gjson",
+ "nu-errors",
+ "nu-plugin",
+ "nu-protocol",
+ "nu-source",
+ "nu-test-support",
+]
+
 [[package]]
 name = "nu_plugin_s3"
 version = "0.29.2"

Cargo.toml

@@ -38,6 +38,7 @@ nu_plugin_inc = { version = "0.29.2", path = "./crates/nu_plugin_inc", optional
 nu_plugin_match = { version = "0.29.2", path = "./crates/nu_plugin_match", optional = true }
 nu_plugin_post = { version = "0.29.2", path = "./crates/nu_plugin_post", optional = true }
 nu_plugin_ps = { version = "0.29.2", path = "./crates/nu_plugin_ps", optional = true }
+nu_plugin_query_json = { version = "0.29.2", path = "./crates/nu_plugin_query_json", optional = true }
 nu_plugin_s3 = { version = "0.29.2", path = "./crates/nu_plugin_s3", optional = true }
 nu_plugin_selector = { version = "0.29.2", path = "./crates/nu_plugin_selector", optional = true }
 nu_plugin_start = { version = "0.29.2", path = "./crates/nu_plugin_start", optional = true }
@@ -121,6 +122,7 @@ extra = [
     "chart",
     "xpath",
     "selector",
+    "query-json",
 ]
 
 wasi = ["inc", "match", "ptree-support", "match", "tree", "rustyline-support"]
@@ -141,6 +143,7 @@ binaryview = ["nu_plugin_binaryview"]
 bson = ["nu_plugin_from_bson", "nu_plugin_to_bson"]
 chart = ["nu_plugin_chart"]
 clipboard-cli = ["nu-cli/clipboard-cli", "nu-command/clipboard-cli"]
+query-json = ["nu_plugin_query_json"]
 s3 = ["nu_plugin_s3"]
 selector = ["nu_plugin_selector"]
 sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"]
@@ -214,6 +217,11 @@ name = "nu_plugin_extra_tree"
 path = "src/plugins/nu_plugin_extra_tree.rs"
 required-features = ["tree"]
 
+[[bin]]
+name = "nu_plugin_extra_query_json"
+path = "src/plugins/nu_plugin_extra_query_json.rs"
+required-features = ["query-json"]
+
 [[bin]]
 name = "nu_plugin_extra_start"
 path = "src/plugins/nu_plugin_extra_start.rs"

crates/nu_plugin_query_json/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+authors = ["The Nu Project Contributors"]
+description = "query json files with gjson"
+edition = "2018"
+license = "MIT"
+name = "nu_plugin_query_json"
+version = "0.29.2"
+
+[lib]
+doctest = false
+
+[dependencies]
+gjson = "0.7.1"
+nu-errors = { version = "0.29.2", path = "../nu-errors" }
+nu-plugin = { version = "0.29.2", path = "../nu-plugin" }
+nu-protocol = { version = "0.29.2", path = "../nu-protocol" }
+nu-source = { version = "0.29.2", path = "../nu-source" }
+
+[dev-dependencies]
+nu-test-support = { path = "../nu-test-support", version = "0.29.2" }

crates/nu_plugin_query_json/src/lib.rs

@@ -0,0 +1,4 @@
+mod nu;
+mod query_json;
+
+pub use query_json::QueryJson;

crates/nu_plugin_query_json/src/main.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_query_json::QueryJson;
+
+fn main() {
+    serve_plugin(&mut QueryJson::new());
+}

crates/nu_plugin_query_json/src/nu/mod.rs

@@ -0,0 +1,48 @@
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{
+    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value,
+};
+use nu_source::TaggedItem;
+
+use crate::{query_json::begin_json_query, QueryJson};
+
+impl Plugin for QueryJson {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("query json")
+            .desc("execute json query on json file (open --raw <file> | query json 'query string')\nsee https://gjson.dev/ for more info.")
+            .required("query", SyntaxShape::String, "json query")
+            .filter())
+    }
+
+    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        let tag = call_info.name_tag;
+        let query = call_info.args.nth(0).ok_or_else(|| {
+            ShellError::labeled_error("json query not passed", "json query not passed", &tag)
+        })?;
+
+        self.query = query.as_string()?;
+        self.tag = tag;
+
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        match input {
+            Value {
+                value: UntaggedValue::Primitive(Primitive::String(s)),
+                ..
+            } => match begin_json_query(s, (*self.query).tagged(&self.tag)) {
+                Ok(result) => Ok(result.into_iter().map(ReturnSuccess::value).collect()),
+                Err(err) => Err(err),
+            },
+            Value { tag, .. } => Err(ShellError::labeled_error_with_secondary(
+                "Expected text from pipeline",
+                "requires text input",
+                &self.tag,
+                "value originates from here",
+                tag,
+            )),
+        }
+    }
+}

crates/nu_plugin_query_json/src/query_json.rs

@@ -0,0 +1,162 @@
+use gjson::Value as gjValue;
+use nu_errors::ShellError;
+use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value};
+use nu_source::{Tag, Tagged};
+
+pub struct QueryJson {
+    pub query: String,
+    pub tag: Tag,
+}
+
+impl QueryJson {
+    pub fn new() -> QueryJson {
+        QueryJson {
+            query: String::new(),
+            tag: Tag::unknown(),
+        }
+    }
+}
+
+impl Default for QueryJson {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub fn begin_json_query(input: String, query: Tagged<&str>) -> Result<Vec<Value>, ShellError> {
+    execute_json_query(input, query.item.to_string(), query.tag())
+}
+
+fn execute_json_query(
+    input_string: String,
+    query_string: String,
+    tag: impl Into<Tag>,
+) -> Result<Vec<Value>, ShellError> {
+    let tag = tag.into();
+
+    // Validate the json before trying to query it
+    let is_valid_json = gjson::valid(input_string.as_str());
+    if !is_valid_json {
+        return Err(ShellError::labeled_error(
+            "invalid json",
+            "invalid json",
+            tag,
+        ));
+    }
+
+    let mut ret: Vec<Value> = vec![];
+    let val: gjValue = gjson::get(input_string.as_str(), &query_string);
+
+    if query_contains_modifiers(query_string.as_str()) {
+        let json_str = val.json();
+        let json_val = Value::from(json_str);
+        ret.push(json_val);
+    } else {
+        let gjv = convert_gjson_value_to_nu_value(&val, &tag);
+
+        match gjv.value {
+            UntaggedValue::Primitive(_) => ret.push(gjv),
+            UntaggedValue::Row(_) => ret.push(gjv),
+            UntaggedValue::Table(t) => {
+                // Unravel the table so it's not a table inside of a table in the output
+                for v in t.iter() {
+                    let c = v.clone();
+                    ret.push(c)
+                }
+            }
+            _ => (),
+        }
+    }
+
+    Ok(ret)
+}
+fn query_contains_modifiers(query: &str) -> bool {
+    // https://github.com/tidwall/gjson.rs documents 7 modifiers as of 4/19/21
+    // Some of these modifiers mean we really need to output the data as a string
+    // instead of tabular data. Others don't matter.
+
+    // Output as String
+    // @ugly: Remove all whitespace from a json document.
+    // @pretty: Make the json document more human readable.
+    query.contains("@ugly") || query.contains("@pretty")
+
+    // Output as Tablular
+    // Since it's output as tabular, which is our default, we can just ignore these
+    // @reverse: Reverse an array or the members of an object.
+    // @this: Returns the current element. It can be used to retrieve the root element.
+    // @valid: Ensure the json document is valid.
+    // @flatten: Flattens an array.
+    // @join: Joins multiple objects into a single object.
+}
+
+fn convert_gjson_value_to_nu_value(v: &gjValue, tag: impl Into<Tag>) -> Value {
+    let tag = tag.into();
+
+    match v.kind() {
+        gjson::Kind::Array => {
+            let mut values = vec![];
+            v.each(|_k, v| {
+                values.push(convert_gjson_value_to_nu_value(&v, &tag));
+                true
+            });
+
+            UntaggedValue::Table(values).into_value(&tag)
+        }
+        gjson::Kind::Null => UntaggedValue::nothing().into_value(&tag),
+        gjson::Kind::False => UntaggedValue::boolean(false).into_value(&tag),
+        gjson::Kind::Number => UntaggedValue::int(v.i64()).into_value(&tag),
+        gjson::Kind::String => UntaggedValue::string(v.str()).into_value(&tag),
+        gjson::Kind::True => UntaggedValue::boolean(true).into_value(&tag),
+        // I'm not sure how to test this, so it may not work
+        gjson::Kind::Object => {
+            // eprint!("Object: ");
+            let mut collected = TaggedDictBuilder::new(&tag);
+            v.each(|k, v| {
+                // eprintln!("k:{} v:{}", k.str(), v.str());
+                collected.insert_value(k.str(), convert_gjson_value_to_nu_value(&v, &tag));
+                true
+            });
+            collected.into_value()
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use gjson::{valid, Value as gjValue};
+
+    #[test]
+    fn validate_string() {
+        let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
+        let val = valid(json);
+        assert_eq!(val, true);
+    }
+
+    #[test]
+    fn answer_from_get_age() {
+        let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
+        let val: gjValue = gjson::get(json, "age");
+        assert_eq!(val.str(), "37");
+    }
+
+    #[test]
+    fn answer_from_get_children() {
+        let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
+        let val: gjValue = gjson::get(json, "children");
+        assert_eq!(val.str(), r#"["Sara", "Alex", "Jack"]"#);
+    }
+
+    #[test]
+    fn answer_from_get_children_count() {
+        let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
+        let val: gjValue = gjson::get(json, "children.#");
+        assert_eq!(val.str(), "3");
+    }
+
+    #[test]
+    fn answer_from_get_friends_first_name() {
+        let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
+        let val: gjValue = gjson::get(json, "friends.#.first");
+        assert_eq!(val.str(), r#"["James","Roger"]"#);
+    }
+}

src/plugins/nu_plugin_extra_query_json.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_query_json::QueryJson;
+
+fn main() {
+    serve_plugin(&mut QueryJson::new());
+}

wix/main.wxs

@@ -280,6 +280,14 @@
                                 Source='target\$(var.Profile)\nu_plugin_selector.exe'
                                 KeyPath='yes'/>
                         </Component>
+                        <Component Id='binary24' Guid='*' Win64='$(var.Win64)'>
+                            <File
+                                Id='exe24'
+                                Name='nu_plugin_query_json.exe'
+                                DiskId='1'
+                                Source='target\$(var.Profile)\nu_plugin_query_json.exe'
+                                KeyPath='yes'/>
+                        </Component>
                     </Directory>
                 </Directory>
             </Directory>
@@ -324,6 +332,7 @@
             <ComponentRef Id='binary21'/>
             <ComponentRef Id='binary22'/>
             <ComponentRef Id='binary23'/>
+            <ComponentRef Id='binary24'/>
 
             <Feature
                 Id='Environment'