forked from extern/nushell
add query json plugin for experimentation (#3327)
* add query json plugin for experimentation * add some error handling * closer but Kind::Array is still horked * unravel the table so the output looks right * clippy * added the ability to use gjson modifiers
This commit is contained in:
parent
9d24afcfe3
commit
ea5bf9db36
19
Cargo.lock
generated
19
Cargo.lock
generated
@ -2051,6 +2051,12 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gjson"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "864178e25a00c41404f1728997c9a21a7b746be1faefe6ce4dc41eb48bb4234f"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.0"
|
||||
@ -3110,6 +3116,7 @@ dependencies = [
|
||||
"nu_plugin_match",
|
||||
"nu_plugin_post",
|
||||
"nu_plugin_ps",
|
||||
"nu_plugin_query_json",
|
||||
"nu_plugin_s3",
|
||||
"nu_plugin_selector",
|
||||
"nu_plugin_start",
|
||||
@ -3704,6 +3711,18 @@ dependencies = [
|
||||
"sysinfo",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu_plugin_query_json"
|
||||
version = "0.29.2"
|
||||
dependencies = [
|
||||
"gjson",
|
||||
"nu-errors",
|
||||
"nu-plugin",
|
||||
"nu-protocol",
|
||||
"nu-source",
|
||||
"nu-test-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu_plugin_s3"
|
||||
version = "0.29.2"
|
||||
|
@ -38,6 +38,7 @@ nu_plugin_inc = { version = "0.29.2", path = "./crates/nu_plugin_inc", optional
|
||||
nu_plugin_match = { version = "0.29.2", path = "./crates/nu_plugin_match", optional = true }
|
||||
nu_plugin_post = { version = "0.29.2", path = "./crates/nu_plugin_post", optional = true }
|
||||
nu_plugin_ps = { version = "0.29.2", path = "./crates/nu_plugin_ps", optional = true }
|
||||
nu_plugin_query_json = { version = "0.29.2", path = "./crates/nu_plugin_query_json", optional = true }
|
||||
nu_plugin_s3 = { version = "0.29.2", path = "./crates/nu_plugin_s3", optional = true }
|
||||
nu_plugin_selector = { version = "0.29.2", path = "./crates/nu_plugin_selector", optional = true }
|
||||
nu_plugin_start = { version = "0.29.2", path = "./crates/nu_plugin_start", optional = true }
|
||||
@ -121,6 +122,7 @@ extra = [
|
||||
"chart",
|
||||
"xpath",
|
||||
"selector",
|
||||
"query-json",
|
||||
]
|
||||
|
||||
wasi = ["inc", "match", "ptree-support", "match", "tree", "rustyline-support"]
|
||||
@ -141,6 +143,7 @@ binaryview = ["nu_plugin_binaryview"]
|
||||
bson = ["nu_plugin_from_bson", "nu_plugin_to_bson"]
|
||||
chart = ["nu_plugin_chart"]
|
||||
clipboard-cli = ["nu-cli/clipboard-cli", "nu-command/clipboard-cli"]
|
||||
query-json = ["nu_plugin_query_json"]
|
||||
s3 = ["nu_plugin_s3"]
|
||||
selector = ["nu_plugin_selector"]
|
||||
sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"]
|
||||
@ -214,6 +217,11 @@ name = "nu_plugin_extra_tree"
|
||||
path = "src/plugins/nu_plugin_extra_tree.rs"
|
||||
required-features = ["tree"]
|
||||
|
||||
[[bin]]
|
||||
name = "nu_plugin_extra_query_json"
|
||||
path = "src/plugins/nu_plugin_extra_query_json.rs"
|
||||
required-features = ["query-json"]
|
||||
|
||||
[[bin]]
|
||||
name = "nu_plugin_extra_start"
|
||||
path = "src/plugins/nu_plugin_extra_start.rs"
|
||||
|
20
crates/nu_plugin_query_json/Cargo.toml
Normal file
20
crates/nu_plugin_query_json/Cargo.toml
Normal file
@ -0,0 +1,20 @@
|
||||
[package]
|
||||
authors = ["The Nu Project Contributors"]
|
||||
description = "query json files with gjson"
|
||||
edition = "2018"
|
||||
license = "MIT"
|
||||
name = "nu_plugin_query_json"
|
||||
version = "0.29.2"
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
[dependencies]
|
||||
gjson = "0.7.1"
|
||||
nu-errors = { version = "0.29.2", path = "../nu-errors" }
|
||||
nu-plugin = { version = "0.29.2", path = "../nu-plugin" }
|
||||
nu-protocol = { version = "0.29.2", path = "../nu-protocol" }
|
||||
nu-source = { version = "0.29.2", path = "../nu-source" }
|
||||
|
||||
[dev-dependencies]
|
||||
nu-test-support = { path = "../nu-test-support", version = "0.29.2" }
|
4
crates/nu_plugin_query_json/src/lib.rs
Normal file
4
crates/nu_plugin_query_json/src/lib.rs
Normal file
@ -0,0 +1,4 @@
|
||||
mod nu;
|
||||
mod query_json;
|
||||
|
||||
pub use query_json::QueryJson;
|
6
crates/nu_plugin_query_json/src/main.rs
Normal file
6
crates/nu_plugin_query_json/src/main.rs
Normal file
@ -0,0 +1,6 @@
|
||||
use nu_plugin::serve_plugin;
|
||||
use nu_plugin_query_json::QueryJson;
|
||||
|
||||
fn main() {
|
||||
serve_plugin(&mut QueryJson::new());
|
||||
}
|
48
crates/nu_plugin_query_json/src/nu/mod.rs
Normal file
48
crates/nu_plugin_query_json/src/nu/mod.rs
Normal file
@ -0,0 +1,48 @@
|
||||
use nu_errors::ShellError;
|
||||
use nu_plugin::Plugin;
|
||||
use nu_protocol::{
|
||||
CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value,
|
||||
};
|
||||
use nu_source::TaggedItem;
|
||||
|
||||
use crate::{query_json::begin_json_query, QueryJson};
|
||||
|
||||
impl Plugin for QueryJson {
|
||||
fn config(&mut self) -> Result<Signature, ShellError> {
|
||||
Ok(Signature::build("query json")
|
||||
.desc("execute json query on json file (open --raw <file> | query json 'query string')\nsee https://gjson.dev/ for more info.")
|
||||
.required("query", SyntaxShape::String, "json query")
|
||||
.filter())
|
||||
}
|
||||
|
||||
fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
|
||||
let tag = call_info.name_tag;
|
||||
let query = call_info.args.nth(0).ok_or_else(|| {
|
||||
ShellError::labeled_error("json query not passed", "json query not passed", &tag)
|
||||
})?;
|
||||
|
||||
self.query = query.as_string()?;
|
||||
self.tag = tag;
|
||||
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
|
||||
match input {
|
||||
Value {
|
||||
value: UntaggedValue::Primitive(Primitive::String(s)),
|
||||
..
|
||||
} => match begin_json_query(s, (*self.query).tagged(&self.tag)) {
|
||||
Ok(result) => Ok(result.into_iter().map(ReturnSuccess::value).collect()),
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
Value { tag, .. } => Err(ShellError::labeled_error_with_secondary(
|
||||
"Expected text from pipeline",
|
||||
"requires text input",
|
||||
&self.tag,
|
||||
"value originates from here",
|
||||
tag,
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
162
crates/nu_plugin_query_json/src/query_json.rs
Normal file
162
crates/nu_plugin_query_json/src/query_json.rs
Normal file
@ -0,0 +1,162 @@
|
||||
use gjson::Value as gjValue;
|
||||
use nu_errors::ShellError;
|
||||
use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value};
|
||||
use nu_source::{Tag, Tagged};
|
||||
|
||||
pub struct QueryJson {
|
||||
pub query: String,
|
||||
pub tag: Tag,
|
||||
}
|
||||
|
||||
impl QueryJson {
|
||||
pub fn new() -> QueryJson {
|
||||
QueryJson {
|
||||
query: String::new(),
|
||||
tag: Tag::unknown(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for QueryJson {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn begin_json_query(input: String, query: Tagged<&str>) -> Result<Vec<Value>, ShellError> {
|
||||
execute_json_query(input, query.item.to_string(), query.tag())
|
||||
}
|
||||
|
||||
fn execute_json_query(
|
||||
input_string: String,
|
||||
query_string: String,
|
||||
tag: impl Into<Tag>,
|
||||
) -> Result<Vec<Value>, ShellError> {
|
||||
let tag = tag.into();
|
||||
|
||||
// Validate the json before trying to query it
|
||||
let is_valid_json = gjson::valid(input_string.as_str());
|
||||
if !is_valid_json {
|
||||
return Err(ShellError::labeled_error(
|
||||
"invalid json",
|
||||
"invalid json",
|
||||
tag,
|
||||
));
|
||||
}
|
||||
|
||||
let mut ret: Vec<Value> = vec![];
|
||||
let val: gjValue = gjson::get(input_string.as_str(), &query_string);
|
||||
|
||||
if query_contains_modifiers(query_string.as_str()) {
|
||||
let json_str = val.json();
|
||||
let json_val = Value::from(json_str);
|
||||
ret.push(json_val);
|
||||
} else {
|
||||
let gjv = convert_gjson_value_to_nu_value(&val, &tag);
|
||||
|
||||
match gjv.value {
|
||||
UntaggedValue::Primitive(_) => ret.push(gjv),
|
||||
UntaggedValue::Row(_) => ret.push(gjv),
|
||||
UntaggedValue::Table(t) => {
|
||||
// Unravel the table so it's not a table inside of a table in the output
|
||||
for v in t.iter() {
|
||||
let c = v.clone();
|
||||
ret.push(c)
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
fn query_contains_modifiers(query: &str) -> bool {
|
||||
// https://github.com/tidwall/gjson.rs documents 7 modifiers as of 4/19/21
|
||||
// Some of these modifiers mean we really need to output the data as a string
|
||||
// instead of tabular data. Others don't matter.
|
||||
|
||||
// Output as String
|
||||
// @ugly: Remove all whitespace from a json document.
|
||||
// @pretty: Make the json document more human readable.
|
||||
query.contains("@ugly") || query.contains("@pretty")
|
||||
|
||||
// Output as Tablular
|
||||
// Since it's output as tabular, which is our default, we can just ignore these
|
||||
// @reverse: Reverse an array or the members of an object.
|
||||
// @this: Returns the current element. It can be used to retrieve the root element.
|
||||
// @valid: Ensure the json document is valid.
|
||||
// @flatten: Flattens an array.
|
||||
// @join: Joins multiple objects into a single object.
|
||||
}
|
||||
|
||||
fn convert_gjson_value_to_nu_value(v: &gjValue, tag: impl Into<Tag>) -> Value {
|
||||
let tag = tag.into();
|
||||
|
||||
match v.kind() {
|
||||
gjson::Kind::Array => {
|
||||
let mut values = vec![];
|
||||
v.each(|_k, v| {
|
||||
values.push(convert_gjson_value_to_nu_value(&v, &tag));
|
||||
true
|
||||
});
|
||||
|
||||
UntaggedValue::Table(values).into_value(&tag)
|
||||
}
|
||||
gjson::Kind::Null => UntaggedValue::nothing().into_value(&tag),
|
||||
gjson::Kind::False => UntaggedValue::boolean(false).into_value(&tag),
|
||||
gjson::Kind::Number => UntaggedValue::int(v.i64()).into_value(&tag),
|
||||
gjson::Kind::String => UntaggedValue::string(v.str()).into_value(&tag),
|
||||
gjson::Kind::True => UntaggedValue::boolean(true).into_value(&tag),
|
||||
// I'm not sure how to test this, so it may not work
|
||||
gjson::Kind::Object => {
|
||||
// eprint!("Object: ");
|
||||
let mut collected = TaggedDictBuilder::new(&tag);
|
||||
v.each(|k, v| {
|
||||
// eprintln!("k:{} v:{}", k.str(), v.str());
|
||||
collected.insert_value(k.str(), convert_gjson_value_to_nu_value(&v, &tag));
|
||||
true
|
||||
});
|
||||
collected.into_value()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use gjson::{valid, Value as gjValue};
|
||||
|
||||
#[test]
|
||||
fn validate_string() {
|
||||
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
|
||||
let val = valid(json);
|
||||
assert_eq!(val, true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_from_get_age() {
|
||||
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
|
||||
let val: gjValue = gjson::get(json, "age");
|
||||
assert_eq!(val.str(), "37");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_from_get_children() {
|
||||
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
|
||||
let val: gjValue = gjson::get(json, "children");
|
||||
assert_eq!(val.str(), r#"["Sara", "Alex", "Jack"]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_from_get_children_count() {
|
||||
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
|
||||
let val: gjValue = gjson::get(json, "children.#");
|
||||
assert_eq!(val.str(), "3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_from_get_friends_first_name() {
|
||||
let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#;
|
||||
let val: gjValue = gjson::get(json, "friends.#.first");
|
||||
assert_eq!(val.str(), r#"["James","Roger"]"#);
|
||||
}
|
||||
}
|
6
src/plugins/nu_plugin_extra_query_json.rs
Normal file
6
src/plugins/nu_plugin_extra_query_json.rs
Normal file
@ -0,0 +1,6 @@
|
||||
use nu_plugin::serve_plugin;
|
||||
use nu_plugin_query_json::QueryJson;
|
||||
|
||||
fn main() {
|
||||
serve_plugin(&mut QueryJson::new());
|
||||
}
|
@ -280,6 +280,14 @@
|
||||
Source='target\$(var.Profile)\nu_plugin_selector.exe'
|
||||
KeyPath='yes'/>
|
||||
</Component>
|
||||
<Component Id='binary24' Guid='*' Win64='$(var.Win64)'>
|
||||
<File
|
||||
Id='exe24'
|
||||
Name='nu_plugin_query_json.exe'
|
||||
DiskId='1'
|
||||
Source='target\$(var.Profile)\nu_plugin_query_json.exe'
|
||||
KeyPath='yes'/>
|
||||
</Component>
|
||||
</Directory>
|
||||
</Directory>
|
||||
</Directory>
|
||||
@ -324,6 +332,7 @@
|
||||
<ComponentRef Id='binary21'/>
|
||||
<ComponentRef Id='binary22'/>
|
||||
<ComponentRef Id='binary23'/>
|
||||
<ComponentRef Id='binary24'/>
|
||||
|
||||
<Feature
|
||||
Id='Environment'
|
||||
|
Loading…
Reference in New Issue
Block a user