add a new inspect command for more debugging (#8028)

# Description

The purpose of this command is to help to debug pipelines. It works by
allowing you to inject the `inspect` command into a pipeline at any
point. Then it shows you what the input description is and what the
input values are that are passed into `inspect`. With each step it
prints this information out while also passing the value information on
to the next step in the pipeline.


![image](https://user-images.githubusercontent.com/343840/218154064-e107859b-d0da-41c6-8e34-2d717639b81c.png)

This command is kind of a "hack job" because it clones maybe too much
and I had to get creative in order to output two different tables. I'm
sure there are many ways this can be improved or combined into other
commands but I wanted to start here. Note that the `inspect` output is
written to stderr and the normal nushell output is written to stdout. If
we were to output both to stdout, nushell would get confused.

# User-Facing Changes



# Tests + Formatting

Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect` to check that you're using the standard code
style
- `cargo test --workspace` to check that all tests pass

# After Submitting

If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.

Cargo.lock

@@ -2801,6 +2801,7 @@ dependencies = [
  "shadow-rs",
  "sqlparser",
  "sysinfo",
+ "tabled",
  "terminal_size 0.2.1",
  "thiserror",
  "titlecase",

crates/nu-command/Cargo.toml

@@ -1,18 +1,20 @@
 [package]
 authors = ["The Nushell Project Developers"]
+build = "build.rs"
 description = "Nushell's built-in commands"
-repository = "https://github.com/nushell/nushell/tree/main/crates/nu-command"
 edition = "2021"
 license = "MIT"
 name = "nu-command"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-command"
 version = "0.75.1"
-build = "build.rs"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+nu-ansi-term = "0.46.0"
 nu-color-config = { path = "../nu-color-config", version = "0.75.1" }
 nu-engine = { path = "../nu-engine", version = "0.75.1" }
+nu-explore = { path = "../nu-explore", version = "0.75.1" }
 nu-glob = { path = "../nu-glob", version = "0.75.1" }
 nu-json = { path = "../nu-json", version = "0.75.1" }
 nu-parser = { path = "../nu-parser", version = "0.75.1" }
@@ -23,18 +25,17 @@ nu-system = { path = "../nu-system", version = "0.75.1" }
 nu-table = { path = "../nu-table", version = "0.75.1" }
 nu-term-grid = { path = "../nu-term-grid", version = "0.75.1" }
 nu-utils = { path = "../nu-utils", version = "0.75.1" }
-nu-explore = { path = "../nu-explore", version = "0.75.1" }
-nu-ansi-term = "0.46.0"
 num-format = { version = "0.4.3" }
 
 # Potential dependencies for extras
+Inflector = "0.11"
 alphanumeric-sort = "1.4.4"
 atty = "0.2.14"
 base64 = "0.21.0"
 byteorder = "1.4.3"
 bytesize = "1.1.0"
 calamine = "0.19.1"
-chrono = { version = "0.4.23", features = ["unstable-locales", "std"], default-features = false }
+chrono = { version = "0.4.23", features = ["std", "unstable-locales"], default-features = false }
 chrono-humanize = "0.2.1"
 chrono-tz = "0.8.1"
 crossterm = "0.24.0"
@@ -52,7 +53,6 @@ htmlescape = "0.3.1"
 ical = "0.8.0"
 indexmap = { version = "1.7", features = ["serde-1"] }
 indicatif = "0.17.2"
-Inflector = "0.11"
 is-root = "0.1.2"
 itertools = "0.10.0"
 log = "0.4.14"
@@ -74,45 +74,44 @@ regex = "1.7.1"
 reqwest = { version = "0.11", features = ["blocking", "json"] }
 roxmltree = "0.17.0"
 rust-embed = "6.3.0"
+rust-ini = "0.18.0"
 same-file = "1.0.6"
 serde = { version = "1.0.123", features = ["derive"] }
-rust-ini = "0.18.0"
 serde_urlencoded = "0.7.0"
 serde_yaml = "0.9.4"
 sha2 = "0.10.0"
 # Disable default features b/c the default features build Git (very slow to compile)
+percent-encoding = "2.2.0"
+reedline = { version = "0.15.0", features = ["bashisms", "sqlite"] }
+rusqlite = { version = "0.28.0", features = ["bundled"], optional = true }
 shadow-rs = { version = "0.20.0", default-features = false }
+sqlparser = { version = "0.30.0", features = ["serde"], optional = true }
 sysinfo = "0.27.7"
+tabled = "0.10.0"
 terminal_size = "0.2.1"
 thiserror = "1.0.31"
 titlecase = "2.0.0"
-unicode-segmentation = "1.10.0"
 toml = "0.7.1"
-url = "2.2.1"
+unicode-segmentation = "1.10.0"
-percent-encoding = "2.2.0"
-uuid = { version = "1.2.2", features = ["v4"] }
-which = { version = "4.4.0", optional = true }
-reedline = { version = "0.15.0", features = ["bashisms", "sqlite"] }
-wax = { version = "0.5.0" }
-rusqlite = { version = "0.28.0", features = ["bundled"], optional = true }
-sqlparser = { version = "0.30.0", features = ["serde"], optional = true }
 unicode-width = "0.1.10"
+url = "2.2.1"
+uuid = { version = "1.2.2", features = ["v4"] }
+wax = { version = "0.5.0" }
+which = { version = "4.4.0", optional = true }
 
 [target.'cfg(windows)'.dependencies]
 winreg = "0.10.1"
 
 [target.'cfg(unix)'.dependencies]
+libc = "0.2"
 umask = "2.0.0"
 users = "0.11.0"
-libc = "0.2"
 
 [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies.trash]
-version = "3.0.1"
 optional = true
+version = "3.0.1"
 
 [dependencies.polars]
-version = "0.26.1"
-optional = true
 features = [
 	"arg_where",
 	"checked_arithmetic",
@@ -121,9 +120,9 @@ features = [
 	"csv-file",
 	"cum_agg",
 	"default",
+	"dtype-categorical",
 	"dtype-datetime",
 	"dtype-struct",
-	"dtype-categorical",
 	"dynamic_groupby",
 	"ipc",
 	"is_in",
@@ -140,17 +139,19 @@ features = [
 	"strings",
 	"to_dummies",
 ]
+optional = true
+version = "0.26.1"
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.44.0"
 features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_SystemServices"]
+version = "0.44.0"
 
 [features]
+dataframe = ["num", "polars", "sqlparser"]
+plugin = ["nu-parser/plugin"]
+sqlite = ["rusqlite"]                      # TODO: given that rusqlite is included in reedline, should we just always include it?
 trash-support = ["trash"]
 which-support = ["which"]
-plugin = ["nu-parser/plugin"]
-dataframe = ["polars", "num", "sqlparser"]
-sqlite = ["rusqlite"]                      # TODO: given that rusqlite is included in reedline, should we just always include it?
 
 [build-dependencies]
 shadow-rs = { version = "0.20.0", default-features = false }
@@ -158,8 +159,8 @@ shadow-rs = { version = "0.20.0", default-features = false }
 [dev-dependencies]
 nu-test-support = { path = "../nu-test-support", version = "0.75.1" }
 
-hamcrest2 = "0.3.0"
 dirs-next = "2.0.0"
+hamcrest2 = "0.3.0"
 proptest = "1.0.0"
 quickcheck = "1.0.3"
 quickcheck_macros = "1.0.0"

crates/nu-command/src/default_context.rs

@@ -175,6 +175,7 @@ pub fn create_default_context() -> EngineState {
             Complete,
             Explain,
             External,
+            Inspect,
             NuCheck,
             Sys,
             TimeIt,

crates/nu-command/src/system/inspect.rs

@@ -0,0 +1,64 @@
+use super::inspect_table;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
+};
+use terminal_size::{terminal_size, Height, Width};
+
+#[derive(Clone)]
+pub struct Inspect;
+
+impl Command for Inspect {
+    fn name(&self) -> &str {
+        "inspect"
+    }
+
+    fn usage(&self) -> &str {
+        "Inspect pipeline results while running a pipeline"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("inspect")
+            .input_output_types(vec![(Type::Any, Type::Any)])
+            .allow_variants_without_examples(true)
+            .category(Category::Debug)
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let input_metadata = input.metadata();
+        let input_val = input.into_value(call.head);
+        let original_input = input_val.clone();
+        let description = match input_val {
+            Value::CustomValue { ref val, .. } => val.value_string(),
+            _ => input_val.get_type().to_string(),
+        };
+
+        let (cols, _rows) = match terminal_size() {
+            Some((w, h)) => (Width(w.0), Height(h.0)),
+            None => (Width(0), Height(0)),
+        };
+
+        let table = inspect_table::build_table(input_val, description, cols.0 as usize);
+
+        // Note that this is printed to stderr. The reason for this is so it doesn't disrupt the regular nushell
+        // tabular output. If we printed to stdout, nushell would get confused with two outputs.
+        eprintln!("{table}\n");
+
+        Ok(original_input.into_pipeline_data_with_metadata(input_metadata))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Inspect pipeline results",
+            example: "ls | inspect | get name | inspect",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/inspect_table.rs

@@ -0,0 +1,465 @@
+use nu_protocol::Value;
+use tabled::{
+    builder::Builder,
+    peaker::PriorityMax,
+    width::{MinWidth, Wrap},
+    Style,
+};
+
+use self::{
+    global_horizontal_char::SetHorizontalChar, peak2::Peak2, table_column_width::GetColumnWidths,
+    truncate_table::TruncateTable, width_increase::IncWidth,
+};
+
+pub fn build_table(value: Value, description: String, termsize: usize) -> String {
+    let (head, mut data) = util::collect_input(value);
+    data.insert(0, head);
+
+    let mut val_table = Builder::from(data).build();
+    let val_table_width = val_table.total_width();
+
+    let desc = vec![vec![String::from("description"), description]];
+
+    let mut desc_table = Builder::from(desc).build();
+    let desc_table_width = desc_table.total_width();
+
+    let width = val_table_width.clamp(desc_table_width, termsize);
+
+    desc_table
+        .with(Style::rounded().off_bottom())
+        .with(Wrap::new(width).priority::<PriorityMax>())
+        .with(MinWidth::new(width).priority::<Peak2>());
+
+    val_table
+        .with(Style::rounded().top_left_corner('├').top_right_corner('┤'))
+        .with(TruncateTable(width))
+        .with(Wrap::new(width).priority::<PriorityMax>())
+        .with(IncWidth(width));
+
+    let mut desc_widths = GetColumnWidths(Vec::new());
+    desc_table.with(&mut desc_widths);
+
+    val_table.with(SetHorizontalChar::new('┼', '┴', 0, desc_widths.0[0]));
+
+    format!("{desc_table}\n{val_table}")
+}
+
+mod truncate_table {
+    use tabled::{
+        papergrid::{
+            records::{Records, RecordsMut, Resizable},
+            width::{CfgWidthFunction, WidthEstimator},
+            Estimate,
+        },
+        TableOption,
+    };
+
+    pub struct TruncateTable(pub usize);
+
+    impl<R> TableOption<R> for TruncateTable
+    where
+        R: Records + RecordsMut<String> + Resizable,
+    {
+        fn change(&mut self, table: &mut tabled::Table<R>) {
+            let width = table.total_width();
+            if width <= self.0 {
+                return;
+            }
+
+            let count_columns = table.get_records().count_columns();
+            if count_columns < 1 {
+                return;
+            }
+
+            let mut evaluator = WidthEstimator::default();
+            evaluator.estimate(table.get_records(), table.get_config());
+            let columns_width: Vec<_> = evaluator.into();
+
+            const SPLIT_LINE_WIDTH: usize = 1;
+            let mut width = 0;
+            let mut i = 0;
+            for w in columns_width {
+                width += w + SPLIT_LINE_WIDTH;
+
+                if width >= self.0 {
+                    break;
+                }
+
+                i += 1;
+            }
+
+            if i == 0 && count_columns > 0 {
+                i = 1;
+            } else if i + 1 == count_columns {
+                // we want to left at least 1 column
+                i -= 1;
+            }
+
+            let count_columns = table.get_records().count_columns();
+            let y = count_columns - i;
+
+            let mut column = count_columns;
+            for _ in 0..y {
+                column -= 1;
+                table.get_records_mut().remove_column(column);
+            }
+
+            table.get_records_mut().push_column();
+
+            let width_ctrl = CfgWidthFunction::from_cfg(table.get_config());
+            let last_column = table.get_records().count_columns() - 1;
+            for row in 0..table.get_records().count_rows() {
+                table
+                    .get_records_mut()
+                    .set((row, last_column), String::from("‥"), &width_ctrl)
+            }
+        }
+    }
+}
+
+mod util {
+    use crate::system::explain::debug_string_without_formatting;
+    use nu_engine::get_columns;
+    use nu_protocol::{ast::PathMember, Span, Value};
+
+    /// Try to build column names and a table grid.
+    pub fn collect_input(value: Value) -> (Vec<String>, Vec<Vec<String>>) {
+        match value {
+            Value::Record { cols, vals, .. } => (
+                cols,
+                vec![vals
+                    .into_iter()
+                    .map(|s| debug_string_without_formatting(&s))
+                    .collect()],
+            ),
+            Value::List { vals, .. } => {
+                let mut columns = get_columns(&vals);
+                let data = convert_records_to_dataset(&columns, vals);
+
+                if columns.is_empty() && !data.is_empty() {
+                    columns = vec![String::from("")];
+                }
+
+                (columns, data)
+            }
+            Value::String { val, span } => {
+                let lines = val
+                    .lines()
+                    .map(|line| Value::String {
+                        val: line.to_string(),
+                        span,
+                    })
+                    .map(|val| vec![debug_string_without_formatting(&val)])
+                    .collect();
+
+                (vec![String::from("")], lines)
+            }
+            Value::Nothing { .. } => (vec![], vec![]),
+            value => (
+                vec![String::from("")],
+                vec![vec![debug_string_without_formatting(&value)]],
+            ),
+        }
+    }
+
+    fn convert_records_to_dataset(cols: &Vec<String>, records: Vec<Value>) -> Vec<Vec<String>> {
+        if !cols.is_empty() {
+            create_table_for_record(cols, &records)
+        } else if cols.is_empty() && records.is_empty() {
+            vec![]
+        } else if cols.len() == records.len() {
+            vec![records
+                .into_iter()
+                .map(|s| debug_string_without_formatting(&s))
+                .collect()]
+        } else {
+            records
+                .into_iter()
+                .map(|record| vec![debug_string_without_formatting(&record)])
+                .collect()
+        }
+    }
+
+    fn create_table_for_record(headers: &[String], items: &[Value]) -> Vec<Vec<String>> {
+        let mut data = vec![Vec::new(); items.len()];
+
+        for (i, item) in items.iter().enumerate() {
+            let row = record_create_row(headers, item);
+            data[i] = row;
+        }
+
+        data
+    }
+
+    fn record_create_row(headers: &[String], item: &Value) -> Vec<String> {
+        let mut rows = vec![String::default(); headers.len()];
+
+        for (i, header) in headers.iter().enumerate() {
+            let value = record_lookup_value(item, header);
+            rows[i] = debug_string_without_formatting(&value);
+        }
+
+        rows
+    }
+
+    fn record_lookup_value(item: &Value, header: &str) -> Value {
+        match item {
+            Value::Record { .. } => {
+                let path = PathMember::String {
+                    val: header.to_owned(),
+                    span: Span::unknown(),
+                };
+
+                item.clone()
+                    .follow_cell_path(&[path], false, false)
+                    .unwrap_or_else(|_| item.clone())
+            }
+            item => item.clone(),
+        }
+    }
+}
+
+mod style_no_left_right_1st {
+    use tabled::{papergrid::records::Records, Table, TableOption};
+
+    struct StyleOffLeftRightFirstLine;
+
+    impl<R> TableOption<R> for StyleOffLeftRightFirstLine
+    where
+        R: Records,
+    {
+        fn change(&mut self, table: &mut Table<R>) {
+            let shape = table.shape();
+            let cfg = table.get_config_mut();
+
+            let mut b = cfg.get_border((0, 0), shape);
+            b.left = Some(' ');
+            cfg.set_border((0, 0), b);
+
+            let mut b = cfg.get_border((0, shape.1 - 1), shape);
+            b.right = Some(' ');
+            cfg.set_border((0, 0), b);
+        }
+    }
+}
+
+mod peak2 {
+    use tabled::peaker::Peaker;
+
+    pub struct Peak2;
+
+    impl Peaker for Peak2 {
+        fn create() -> Self {
+            Self
+        }
+
+        fn peak(&mut self, _: &[usize], _: &[usize]) -> Option<usize> {
+            Some(1)
+        }
+    }
+}
+
+mod table_column_width {
+    use tabled::papergrid::{records::Records, Estimate};
+
+    pub struct GetColumnWidths(pub Vec<usize>);
+
+    impl<R> tabled::TableOption<R> for GetColumnWidths
+    where
+        R: Records,
+    {
+        fn change(&mut self, table: &mut tabled::Table<R>) {
+            let mut evaluator = tabled::papergrid::width::WidthEstimator::default();
+            evaluator.estimate(table.get_records(), table.get_config());
+            self.0 = evaluator.into();
+        }
+    }
+}
+
+mod global_horizontal_char {
+    use tabled::{
+        papergrid::{records::Records, width::WidthEstimator, Estimate, Offset::Begin},
+        Table, TableOption,
+    };
+
+    pub struct SetHorizontalChar {
+        c1: char,
+        c2: char,
+        line: usize,
+        position: usize,
+    }
+
+    impl SetHorizontalChar {
+        pub fn new(c1: char, c2: char, line: usize, position: usize) -> Self {
+            Self {
+                c1,
+                c2,
+                line,
+                position,
+            }
+        }
+    }
+
+    impl<R> TableOption<R> for SetHorizontalChar
+    where
+        R: Records,
+    {
+        fn change(&mut self, table: &mut Table<R>) {
+            let shape = table.shape();
+
+            let is_last_line = self.line == (shape.0 * 2);
+            let mut row = self.line;
+            if is_last_line {
+                row = self.line - 1;
+            }
+
+            let mut evaluator = WidthEstimator::default();
+            evaluator.estimate(table.get_records(), table.get_config());
+            let widths: Vec<_> = evaluator.into();
+
+            let mut i = 0;
+            #[allow(clippy::needless_range_loop)]
+            for column in 0..shape.1 {
+                let has_vertical = table.get_config().has_vertical(column, shape.1);
+
+                if has_vertical {
+                    if self.position == i {
+                        let mut border = table.get_config().get_border((row, column), shape);
+                        if is_last_line {
+                            border.left_bottom_corner = Some(self.c1);
+                        } else {
+                            border.left_top_corner = Some(self.c1);
+                        }
+
+                        table.get_config_mut().set_border((row, column), border);
+
+                        return;
+                    }
+
+                    i += 1;
+                }
+
+                let width = widths[column];
+
+                if self.position < i + width {
+                    let offset = self.position + 1 - i;
+                    // let offset = width - offset;
+
+                    table.get_config_mut().override_horizontal_border(
+                        (self.line, column),
+                        self.c2,
+                        Begin(offset),
+                    );
+
+                    return;
+                }
+
+                i += width;
+            }
+
+            let has_vertical = table.get_config().has_vertical(shape.1, shape.1);
+            if self.position == i && has_vertical {
+                let mut border = table.get_config().get_border((row, shape.1), shape);
+                if is_last_line {
+                    border.left_bottom_corner = Some(self.c1);
+                } else {
+                    border.left_top_corner = Some(self.c1);
+                }
+
+                table.get_config_mut().set_border((row, shape.1), border);
+            }
+        }
+    }
+}
+
+mod width_increase {
+    use tabled::{
+        object::Cell,
+        papergrid::{
+            records::{Records, RecordsMut},
+            width::WidthEstimator,
+            Entity, Estimate, GridConfig,
+        },
+        peaker::PriorityNone,
+        Modify, Width,
+    };
+
+    use tabled::{peaker::Peaker, Table, TableOption};
+
+    #[derive(Debug)]
+    pub struct IncWidth(pub usize);
+
+    impl<R> TableOption<R> for IncWidth
+    where
+        R: Records + RecordsMut<String>,
+    {
+        fn change(&mut self, table: &mut Table<R>) {
+            if table.is_empty() {
+                return;
+            }
+
+            let (widths, total_width) =
+                get_table_widths_with_total(table.get_records(), table.get_config());
+            if total_width >= self.0 {
+                return;
+            }
+
+            let increase_list =
+                get_increase_list(widths, self.0, total_width, PriorityNone::default());
+
+            for (col, width) in increase_list.into_iter().enumerate() {
+                for row in 0..table.get_records().count_rows() {
+                    let pad = table.get_config().get_padding(Entity::Cell(row, col));
+                    let width = width - pad.left.size - pad.right.size;
+
+                    table.with(Modify::new(Cell(row, col)).with(Width::increase(width)));
+                }
+            }
+        }
+    }
+
+    fn get_increase_list<F>(
+        mut widths: Vec<usize>,
+        total_width: usize,
+        mut width: usize,
+        mut peaker: F,
+    ) -> Vec<usize>
+    where
+        F: Peaker,
+    {
+        while width != total_width {
+            let col = match peaker.peak(&[], &widths) {
+                Some(col) => col,
+                None => break,
+            };
+
+            widths[col] += 1;
+            width += 1;
+        }
+
+        widths
+    }
+
+    fn get_table_widths_with_total<R>(records: R, cfg: &GridConfig) -> (Vec<usize>, usize)
+    where
+        R: Records,
+    {
+        let mut evaluator = WidthEstimator::default();
+        evaluator.estimate(&records, cfg);
+        let total_width = get_table_total_width(&records, cfg, &evaluator);
+        let widths = evaluator.into();
+
+        (widths, total_width)
+    }
+
+    pub(crate) fn get_table_total_width<W, R>(records: R, cfg: &GridConfig, ctrl: &W) -> usize
+    where
+        W: Estimate<R>,
+        R: Records,
+    {
+        ctrl.total()
+            + cfg.count_vertical(records.count_columns())
+            + cfg.get_margin().left.size
+            + cfg.get_margin().right.size
+    }
+}

crates/nu-command/src/system/mod.rs

@@ -2,6 +2,8 @@ mod complete;
 #[cfg(unix)]
 mod exec;
 mod explain;
+mod inspect;
+mod inspect_table;
 mod nu_check;
 #[cfg(any(
     target_os = "android",
@@ -21,6 +23,8 @@ pub use complete::Complete;
 #[cfg(unix)]
 pub use exec::Exec;
 pub use explain::Explain;
+pub use inspect::Inspect;
+pub use inspect_table::build_table;
 pub use nu_check::NuCheck;
 #[cfg(any(
     target_os = "android",