add query json plugin for experimentation (#3327)

* add query json plugin for experimentation

* add some error handling

* closer but Kind::Array is still horked

* unravel the table so the output looks right

* clippy

* added the ability to use gjson modifiers
This commit is contained in:
Darren Schroeder 2021-04-19 11:19:06 -05:00 committed by GitHub
parent 9d24afcfe3
commit ea5bf9db36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 282 additions and 0 deletions

19
Cargo.lock generated
View File

@ -2051,6 +2051,12 @@ dependencies = [
"url",
]
[[package]]
name = "gjson"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "864178e25a00c41404f1728997c9a21a7b746be1faefe6ce4dc41eb48bb4234f"
[[package]]
name = "glob"
version = "0.3.0"
@ -3110,6 +3116,7 @@ dependencies = [
"nu_plugin_match",
"nu_plugin_post",
"nu_plugin_ps",
"nu_plugin_query_json",
"nu_plugin_s3",
"nu_plugin_selector",
"nu_plugin_start",
@ -3704,6 +3711,18 @@ dependencies = [
"sysinfo",
]
[[package]]
name = "nu_plugin_query_json"
version = "0.29.2"
dependencies = [
"gjson",
"nu-errors",
"nu-plugin",
"nu-protocol",
"nu-source",
"nu-test-support",
]
[[package]]
name = "nu_plugin_s3"
version = "0.29.2"

View File

@ -38,6 +38,7 @@ nu_plugin_inc = { version = "0.29.2", path = "./crates/nu_plugin_inc", optional
nu_plugin_match = { version = "0.29.2", path = "./crates/nu_plugin_match", optional = true }
nu_plugin_post = { version = "0.29.2", path = "./crates/nu_plugin_post", optional = true }
nu_plugin_ps = { version = "0.29.2", path = "./crates/nu_plugin_ps", optional = true }
nu_plugin_query_json = { version = "0.29.2", path = "./crates/nu_plugin_query_json", optional = true }
nu_plugin_s3 = { version = "0.29.2", path = "./crates/nu_plugin_s3", optional = true }
nu_plugin_selector = { version = "0.29.2", path = "./crates/nu_plugin_selector", optional = true }
nu_plugin_start = { version = "0.29.2", path = "./crates/nu_plugin_start", optional = true }
@ -121,6 +122,7 @@ extra = [
"chart",
"xpath",
"selector",
"query-json",
]
wasi = ["inc", "match", "ptree-support", "match", "tree", "rustyline-support"]
@ -141,6 +143,7 @@ binaryview = ["nu_plugin_binaryview"]
bson = ["nu_plugin_from_bson", "nu_plugin_to_bson"]
chart = ["nu_plugin_chart"]
clipboard-cli = ["nu-cli/clipboard-cli", "nu-command/clipboard-cli"]
query-json = ["nu_plugin_query_json"]
s3 = ["nu_plugin_s3"]
selector = ["nu_plugin_selector"]
sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"]
@ -214,6 +217,11 @@ name = "nu_plugin_extra_tree"
path = "src/plugins/nu_plugin_extra_tree.rs"
required-features = ["tree"]
[[bin]]
name = "nu_plugin_extra_query_json"
path = "src/plugins/nu_plugin_extra_query_json.rs"
required-features = ["query-json"]
[[bin]]
name = "nu_plugin_extra_start"
path = "src/plugins/nu_plugin_extra_start.rs"

View File

@ -0,0 +1,20 @@
[package]
authors = ["The Nu Project Contributors"]
description = "query json files with gjson"
edition = "2018"
license = "MIT"
name = "nu_plugin_query_json"
version = "0.29.2"
[lib]
doctest = false
[dependencies]
gjson = "0.7.1"
nu-errors = { version = "0.29.2", path = "../nu-errors" }
nu-plugin = { version = "0.29.2", path = "../nu-plugin" }
nu-protocol = { version = "0.29.2", path = "../nu-protocol" }
nu-source = { version = "0.29.2", path = "../nu-source" }
[dev-dependencies]
nu-test-support = { path = "../nu-test-support", version = "0.29.2" }

View File

@ -0,0 +1,4 @@
mod nu;
mod query_json;
pub use query_json::QueryJson;

View File

@ -0,0 +1,6 @@
use nu_plugin::serve_plugin;
use nu_plugin_query_json::QueryJson;
fn main() {
serve_plugin(&mut QueryJson::new());
}

View File

@ -0,0 +1,48 @@
use nu_errors::ShellError;
use nu_plugin::Plugin;
use nu_protocol::{
CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value,
};
use nu_source::TaggedItem;
use crate::{query_json::begin_json_query, QueryJson};
impl Plugin for QueryJson {
fn config(&mut self) -> Result<Signature, ShellError> {
Ok(Signature::build("query json")
.desc("execute json query on json file (open --raw <file> | query json 'query string')\nsee https://gjson.dev/ for more info.")
.required("query", SyntaxShape::String, "json query")
.filter())
}
fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
let tag = call_info.name_tag;
let query = call_info.args.nth(0).ok_or_else(|| {
ShellError::labeled_error("json query not passed", "json query not passed", &tag)
})?;
self.query = query.as_string()?;
self.tag = tag;
Ok(vec![])
}
fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
match input {
Value {
value: UntaggedValue::Primitive(Primitive::String(s)),
..
} => match begin_json_query(s, (*self.query).tagged(&self.tag)) {
Ok(result) => Ok(result.into_iter().map(ReturnSuccess::value).collect()),
Err(err) => Err(err),
},
Value { tag, .. } => Err(ShellError::labeled_error_with_secondary(
"Expected text from pipeline",
"requires text input",
&self.tag,
"value originates from here",
tag,
)),
}
}
}

View File

@ -0,0 +1,162 @@
use gjson::Value as gjValue;
use nu_errors::ShellError;
use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value};
use nu_source::{Tag, Tagged};
pub struct QueryJson {
pub query: String,
pub tag: Tag,
}
impl QueryJson {
pub fn new() -> QueryJson {
QueryJson {
query: String::new(),
tag: Tag::unknown(),
}
}
}
impl Default for QueryJson {
fn default() -> Self {
Self::new()
}
}
pub fn begin_json_query(input: String, query: Tagged<&str>) -> Result<Vec<Value>, ShellError> {
execute_json_query(input, query.item.to_string(), query.tag())
}
fn execute_json_query(
input_string: String,
query_string: String,
tag: impl Into<Tag>,
) -> Result<Vec<Value>, ShellError> {
let tag = tag.into();
// Validate the json before trying to query it
let is_valid_json = gjson::valid(input_string.as_str());
if !is_valid_json {
return Err(ShellError::labeled_error(
"invalid json",
"invalid json",
tag,
));
}
let mut ret: Vec<Value> = vec![];
let val: gjValue = gjson::get(input_string.as_str(), &query_string);
if query_contains_modifiers(query_string.as_str()) {
let json_str = val.json();
let json_val = Value::from(json_str);
ret.push(json_val);
} else {
let gjv = convert_gjson_value_to_nu_value(&val, &tag);
match gjv.value {
UntaggedValue::Primitive(_) => ret.push(gjv),
UntaggedValue::Row(_) => ret.push(gjv),
UntaggedValue::Table(t) => {
// Unravel the table so it's not a table inside of a table in the output
for v in t.iter() {
let c = v.clone();
ret.push(c)
}
}
_ => (),
}
}
Ok(ret)
}
fn query_contains_modifiers(query: &str) -> bool {
// https://github.com/tidwall/gjson.rs documents 7 modifiers as of 4/19/21
// Some of these modifiers mean we really need to output the data as a string
// instead of tabular data. Others don't matter.
// Output as String
// @ugly: Remove all whitespace from a json document.
// @pretty: Make the json document more human readable.
query.contains("@ugly") || query.contains("@pretty")
// Output as Tablular
// Since it's output as tabular, which is our default, we can just ignore these
// @reverse: Reverse an array or the members of an object.
// @this: Returns the current element. It can be used to retrieve the root element.
// @valid: Ensure the json document is valid.
// @flatten: Flattens an array.
// @join: Joins multiple objects into a single object.
}
fn convert_gjson_value_to_nu_value(v: &gjValue, tag: impl Into<Tag>) -> Value {
let tag = tag.into();
match v.kind() {
gjson::Kind::Array => {
let mut values = vec![];
v.each(|_k, v| {
values.push(convert_gjson_value_to_nu_value(&v, &tag));
true
});
UntaggedValue::Table(values).into_value(&tag)
}
gjson::Kind::Null => UntaggedValue::nothing().into_value(&tag),
gjson::Kind::False => UntaggedValue::boolean(false).into_value(&tag),
gjson::Kind::Number => UntaggedValue::int(v.i64()).into_value(&tag),
gjson::Kind::String => UntaggedValue::string(v.str()).into_value(&tag),
gjson::Kind::True => UntaggedValue::boolean(true).into_value(&tag),
// I'm not sure how to test this, so it may not work
gjson::Kind::Object => {
// eprint!("Object: ");
let mut collected = TaggedDictBuilder::new(&tag);
v.each(|k, v| {
// eprintln!("k:{} v:{}", k.str(), v.str());
collected.insert_value(k.str(), convert_gjson_value_to_nu_value(&v, &tag));
true
});
collected.into_value()
}
}
}
#[cfg(test)]
mod tests {
use gjson::{valid, Value as gjValue};
#[test]
fn validate_string() {
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
let val = valid(json);
assert_eq!(val, true);
}
#[test]
fn answer_from_get_age() {
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
let val: gjValue = gjson::get(json, "age");
assert_eq!(val.str(), "37");
}
#[test]
fn answer_from_get_children() {
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
let val: gjValue = gjson::get(json, "children");
assert_eq!(val.str(), r#"["Sara", "Alex", "Jack"]"#);
}
#[test]
fn answer_from_get_children_count() {
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
let val: gjValue = gjson::get(json, "children.#");
assert_eq!(val.str(), "3");
}
#[test]
fn answer_from_get_friends_first_name() {
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
let val: gjValue = gjson::get(json, "friends.#.first");
assert_eq!(val.str(), r#"["James","Roger"]"#);
}
}

View File

@ -0,0 +1,6 @@
use nu_plugin::serve_plugin;
use nu_plugin_query_json::QueryJson;
fn main() {
serve_plugin(&mut QueryJson::new());
}

View File

@ -280,6 +280,14 @@
Source='target\$(var.Profile)\nu_plugin_selector.exe'
KeyPath='yes'/>
</Component>
<Component Id='binary24' Guid='*' Win64='$(var.Win64)'>
<File
Id='exe24'
Name='nu_plugin_query_json.exe'
DiskId='1'
Source='target\$(var.Profile)\nu_plugin_query_json.exe'
KeyPath='yes'/>
</Component>
</Directory>
</Directory>
</Directory>
@ -324,6 +332,7 @@
<ComponentRef Id='binary21'/>
<ComponentRef Id='binary22'/>
<ComponentRef Id='binary23'/>
<ComponentRef Id='binary24'/>
<Feature
Id='Environment'