From 8f4feeb119bb7aee2df532be395621fa9fcfe6d8 Mon Sep 17 00:00:00 2001 From: Darren Schroeder <343840+fdncred@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:50:16 -0600 Subject: [PATCH] add `config flatten` command (#14621) # Description This is supposed to be a Quality-of-Life command that just makes some things easier when dealing with a nushell config. Really all it does is show you the current config in a flattened state. That's it. I was thinking this could be useful when comparing config settings between old and new config files. There are still room for improvements. For instance, closures are listed as an int. They can be updated with a `view source ` pipeline but that could all be built in too. ![image](https://github.com/user-attachments/assets/5d8981a3-8d03-4eb3-8361-2f3c3c560660) The command works by getting the current configuration, serializing it to json, then flattening that json. BTW, there's a new flatten_json.rs in nu-utils. Theoretically all this mess could be done in a custom command script, but it's proven to be exceedingly difficult based on the work from discord. Here's some more complex items to flatten. ![image](https://github.com/user-attachments/assets/b44e2ec8-cf17-41c4-bf8d-7f26317db071) # User-Facing Changes # Tests + Formatting # After Submitting --- Cargo.lock | 1 + crates/nu-command/src/default_context.rs | 1 + .../src/env/config/config_flatten.rs | 150 +++++++++++ crates/nu-command/src/env/config/config_nu.rs | 1 - crates/nu-command/src/env/config/mod.rs | 3 + crates/nu-command/src/env/mod.rs | 1 + crates/nu-utils/Cargo.toml | 3 +- crates/nu-utils/src/flatten_json.rs | 247 ++++++++++++++++++ crates/nu-utils/src/lib.rs | 2 + 9 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 crates/nu-command/src/env/config/config_flatten.rs create mode 100644 crates/nu-utils/src/flatten_json.rs diff --git a/Cargo.lock b/Cargo.lock index 9cc6b8c04c..c2170445a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3736,6 +3736,7 @@ dependencies = [ "nix 0.29.0", "num-format", "serde", + "serde_json", "strip-ansi-escapes", "sys-locale", "unicase", diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 3612c963c8..bcff06b5e8 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -351,6 +351,7 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { WithEnv, ConfigNu, ConfigEnv, + ConfigFlatten, ConfigMeta, ConfigReset, }; diff --git a/crates/nu-command/src/env/config/config_flatten.rs b/crates/nu-command/src/env/config/config_flatten.rs new file mode 100644 index 0000000000..3d27f02def --- /dev/null +++ b/crates/nu-command/src/env/config/config_flatten.rs @@ -0,0 +1,150 @@ +use nu_engine::command_prelude::*; +use nu_utils::JsonFlattener; // Ensure this import is present // Ensure this import is present + +#[derive(Clone)] +pub struct ConfigFlatten; + +impl Command for ConfigFlatten { + fn name(&self) -> &str { + "config flatten" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .category(Category::Debug) + .input_output_types(vec![(Type::Nothing, Type::record())]) + } + + fn description(&self) -> &str { + "Show the current configuration in a flattened form." + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Show the current configuration in a flattened form", + example: "config flatten", + result: None, + }] + } + + fn run( + &self, + engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + _input: PipelineData, + ) -> Result { + // Get the Config instance from the EngineState + let config = engine_state.get_config(); + // Serialize the Config instance to JSON + let serialized_config = + serde_json::to_value(&**config).map_err(|err| ShellError::GenericError { + error: format!("Failed to serialize config to json: {err}"), + msg: "".into(), + span: Some(call.head), + help: None, + inner: vec![], + })?; + // Create a JsonFlattener instance with appropriate arguments + let flattener = JsonFlattener { + separator: ".", + alt_array_flattening: false, + preserve_arrays: true, + }; + // Flatten the JSON value + let flattened_config_str = flattener.flatten(&serialized_config).to_string(); + let flattened_values = convert_string_to_value(&flattened_config_str, call.head)?; + + Ok(flattened_values.into_pipeline_data()) + } +} + +// From here below is taken from `from json`. Would be nice to have a nu-utils-value crate that could be shared +fn convert_string_to_value(string_input: &str, span: Span) -> Result { + match nu_json::from_str(string_input) { + Ok(value) => Ok(convert_nujson_to_value(value, span)), + + Err(x) => match x { + nu_json::Error::Syntax(_, row, col) => { + let label = x.to_string(); + let label_span = convert_row_column_to_span(row, col, string_input); + Err(ShellError::GenericError { + error: "Error while parsing JSON text".into(), + msg: "error parsing JSON text".into(), + span: Some(span), + help: None, + inner: vec![ShellError::OutsideSpannedLabeledError { + src: string_input.into(), + error: "Error while parsing JSON text".into(), + msg: label, + span: label_span, + }], + }) + } + x => Err(ShellError::CantConvert { + to_type: format!("structured json data ({x})"), + from_type: "string".into(), + span, + help: None, + }), + }, + } +} + +fn convert_nujson_to_value(value: nu_json::Value, span: Span) -> Value { + match value { + nu_json::Value::Array(array) => Value::list( + array + .into_iter() + .map(|x| convert_nujson_to_value(x, span)) + .collect(), + span, + ), + nu_json::Value::Bool(b) => Value::bool(b, span), + nu_json::Value::F64(f) => Value::float(f, span), + nu_json::Value::I64(i) => Value::int(i, span), + nu_json::Value::Null => Value::nothing(span), + nu_json::Value::Object(k) => Value::record( + k.into_iter() + .map(|(k, v)| (k, convert_nujson_to_value(v, span))) + .collect(), + span, + ), + nu_json::Value::U64(u) => { + if u > i64::MAX as u64 { + Value::error( + ShellError::CantConvert { + to_type: "i64 sized integer".into(), + from_type: "value larger than i64".into(), + span, + help: None, + }, + span, + ) + } else { + Value::int(u as i64, span) + } + } + nu_json::Value::String(s) => Value::string(s, span), + } +} + +// Converts row+column to a Span, assuming bytes (1-based rows) +fn convert_row_column_to_span(row: usize, col: usize, contents: &str) -> Span { + let mut cur_row = 1; + let mut cur_col = 1; + + for (offset, curr_byte) in contents.bytes().enumerate() { + if curr_byte == b'\n' { + cur_row += 1; + cur_col = 1; + } + if cur_row >= row && cur_col >= col { + return Span::new(offset, offset); + } else { + cur_col += 1; + } + } + + Span::new(contents.len(), contents.len()) +} diff --git a/crates/nu-command/src/env/config/config_nu.rs b/crates/nu-command/src/env/config/config_nu.rs index 66e24e2a94..5295fe5247 100644 --- a/crates/nu-command/src/env/config/config_nu.rs +++ b/crates/nu-command/src/env/config/config_nu.rs @@ -22,7 +22,6 @@ impl Command for ConfigNu { "Print a commented `config.nu` with documentation instead.", Some('s'), ) - // TODO: Signature narrower than what run actually supports theoretically } fn description(&self) -> &str { diff --git a/crates/nu-command/src/env/config/mod.rs b/crates/nu-command/src/env/config/mod.rs index fa6a3b4ca3..d596eb57b9 100644 --- a/crates/nu-command/src/env/config/mod.rs +++ b/crates/nu-command/src/env/config/mod.rs @@ -1,8 +1,11 @@ mod config_; mod config_env; +mod config_flatten; mod config_nu; mod config_reset; + pub use config_::ConfigMeta; pub use config_env::ConfigEnv; +pub use config_flatten::ConfigFlatten; pub use config_nu::ConfigNu; pub use config_reset::ConfigReset; diff --git a/crates/nu-command/src/env/mod.rs b/crates/nu-command/src/env/mod.rs index 2a72df5029..3f9312ed13 100644 --- a/crates/nu-command/src/env/mod.rs +++ b/crates/nu-command/src/env/mod.rs @@ -5,6 +5,7 @@ mod source_env; mod with_env; pub use config::ConfigEnv; +pub use config::ConfigFlatten; pub use config::ConfigMeta; pub use config::ConfigNu; pub use config::ConfigReset; diff --git a/crates/nu-utils/Cargo.toml b/crates/nu-utils/Cargo.toml index 309f5f3139..1e6194104b 100644 --- a/crates/nu-utils/Cargo.toml +++ b/crates/nu-utils/Cargo.toml @@ -24,6 +24,7 @@ log = { workspace = true } num-format = { workspace = true } strip-ansi-escapes = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } sys-locale = "0.3" unicase = "2.8.0" @@ -38,4 +39,4 @@ crossterm_winapi = "0.9" nix = { workspace = true, default-features = false, features = ["user", "fs"] } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/crates/nu-utils/src/flatten_json.rs b/crates/nu-utils/src/flatten_json.rs new file mode 100644 index 0000000000..31972d14eb --- /dev/null +++ b/crates/nu-utils/src/flatten_json.rs @@ -0,0 +1,247 @@ +use serde_json::{json, Map, Value as SerdeValue}; + +/// JsonFlattener is the main driver when flattening JSON +/// # Examples +/// ``` +/// use nu_utils; +/// +/// let flattener = nu_utils::JsonFlattener { ..Default::default() }; +/// ``` +pub struct JsonFlattener<'a> { + /// Alternate separator used between keys when flattening + /// # Examples + /// ``` + /// use nu_utils; + /// let flattener = nu_utils::JsonFlattener { separator: "_", ..Default::default()}; + /// ``` + pub separator: &'a str, + /// Opinionated flattening format that places values in an array if the object is nested inside an array + /// # Examples + /// ``` + /// use nu_utils; + /// let flattener = nu_utils::JsonFlattener { alt_array_flattening: true, ..Default::default()}; + /// ``` + pub alt_array_flattening: bool, + /// Completely flatten JSON and keep array structure in the key when flattening + /// # Examples + /// ``` + /// use nu_utils; + /// let flattener = nu_utils::JsonFlattener { preserve_arrays: true, ..Default::default()}; + /// ``` + pub preserve_arrays: bool, +} + +impl<'a> Default for JsonFlattener<'a> { + fn default() -> Self { + JsonFlattener { + separator: ".", + alt_array_flattening: false, + preserve_arrays: false, + } + } +} + +/// This implementation defines the core usage for the `JsonFlattener` structure. +/// # Examples +/// ``` +/// use nu_utils; +/// use serde_json::json; +/// +/// let flattener = nu_utils::JsonFlattener::new(); +/// let example = json!({ +/// "a": { +/// "b": "c" +/// } +/// }); +/// +/// let flattened_example = flattener.flatten(&example); +/// ``` +impl<'a> JsonFlattener<'a> { + /// Returns a flattener with the default arguments + /// # Examples + /// ``` + /// use nu_utils; + /// + /// let flattener = nu_utils::JsonFlattener::new(); + /// ``` + pub fn new() -> Self { + JsonFlattener { + ..Default::default() + } + } + + /// Flattens JSON variants into a JSON object + /// + /// # Arguments + /// + /// * `json` - A serde_json Value to flatten + /// + /// # Examples + /// ``` + /// use nu_utils; + /// use serde_json::json; + /// + /// let flattener = nu_utils::JsonFlattener::new(); + /// let example = json!({ + /// "name": "John Doe", + /// "age": 43, + /// "address": { + /// "street": "10 Downing Street", + /// "city": "London" + /// }, + /// "phones": [ + /// "+44 1234567", + /// "+44 2345678" + /// ] + /// }); + /// + /// let flattened_example = flattener.flatten(&example); + /// ``` + pub fn flatten(&self, json: &SerdeValue) -> SerdeValue { + let mut flattened_val = Map::::new(); + match json { + SerdeValue::Array(obj_arr) => { + self.flatten_array(&mut flattened_val, &"".to_string(), obj_arr) + } + SerdeValue::Object(obj_val) => { + self.flatten_object(&mut flattened_val, None, obj_val, false) + } + _ => self.flatten_value(&mut flattened_val, &"".to_string(), json, false), + } + SerdeValue::Object(flattened_val) + } + + fn flatten_object( + &self, + builder: &mut Map, + identifier: Option<&String>, + obj: &Map, + arr: bool, + ) { + for (k, v) in obj { + let expanded_identifier = identifier.map_or_else( + || k.clone(), + |identifier| format!("{identifier}{}{k}", self.separator), + ); + + if expanded_identifier.contains("span.start") + || expanded_identifier.contains("span.end") + { + continue; + } + + let expanded_identifier = self.filter_known_keys(&expanded_identifier); + + match v { + SerdeValue::Object(obj_val) => { + self.flatten_object(builder, Some(&expanded_identifier), obj_val, arr) + } + SerdeValue::Array(obj_arr) => { + self.flatten_array(builder, &expanded_identifier, obj_arr) + } + _ => self.flatten_value(builder, &expanded_identifier, v, arr), + } + } + } + + fn flatten_array( + &self, + builder: &mut Map, + identifier: &String, + obj: &[SerdeValue], + ) { + for (k, v) in obj.iter().enumerate() { + let with_key = format!("{identifier}{}{k}", self.separator); + if with_key.contains("span.start") || with_key.contains("span.end") { + continue; + } + + let with_key = self.filter_known_keys(&with_key); + + match v { + SerdeValue::Object(obj_val) => self.flatten_object( + builder, + Some(if self.preserve_arrays { + &with_key + } else { + identifier + }), + obj_val, + self.alt_array_flattening, + ), + SerdeValue::Array(obj_arr) => self.flatten_array( + builder, + if self.preserve_arrays { + &with_key + } else { + identifier + }, + obj_arr, + ), + _ => self.flatten_value( + builder, + if self.preserve_arrays { + &with_key + } else { + identifier + }, + v, + self.alt_array_flattening, + ), + } + } + } + + fn flatten_value( + &self, + builder: &mut Map, + identifier: &String, + obj: &SerdeValue, + arr: bool, + ) { + if let Some(v) = builder.get_mut(identifier) { + if let Some(arr) = v.as_array_mut() { + arr.push(obj.clone()); + } else { + let new_val = json!(vec![v, obj]); + builder.remove(identifier); + builder.insert(identifier.to_string(), new_val); + } + } else { + builder.insert( + identifier.to_string(), + if arr { + json!(vec![obj.clone()]) + } else { + obj.clone() + }, + ); + } + } + + fn filter_known_keys(&self, key: &str) -> String { + let mut filtered_key = key.to_string(); + if filtered_key.contains(".String.val") { + filtered_key = filtered_key.replace(".String.val", ""); + } + if filtered_key.contains(".Record.val") { + filtered_key = filtered_key.replace(".Record.val", ""); + } + if filtered_key.contains(".Closure.val") { + filtered_key = filtered_key.replace(".Closure.val", ""); + } + if filtered_key.contains(".List.vals") { + filtered_key = filtered_key.replace(".List.vals", ""); + } + if filtered_key.contains(".Int.val") { + filtered_key = filtered_key.replace(".Int.val", ""); + } + if filtered_key.contains(".Bool.val") { + filtered_key = filtered_key.replace(".Bool.val", ""); + } + if filtered_key.contains(".block_id") { + filtered_key = filtered_key.replace(".block_id", ""); + } + filtered_key + } +} diff --git a/crates/nu-utils/src/lib.rs b/crates/nu-utils/src/lib.rs index c43bda9a0e..117b6fc28b 100644 --- a/crates/nu-utils/src/lib.rs +++ b/crates/nu-utils/src/lib.rs @@ -3,6 +3,7 @@ mod casing; mod deansi; pub mod emoji; pub mod filesystem; +pub mod flatten_json; pub mod locale; mod quoting; mod shared_cow; @@ -20,6 +21,7 @@ pub use deansi::{ strip_ansi_likely, strip_ansi_string_likely, strip_ansi_string_unlikely, strip_ansi_unlikely, }; pub use emoji::contains_emoji; +pub use flatten_json::JsonFlattener; pub use quoting::{escape_quote_string, needs_quoting}; pub use shared_cow::SharedCow;