add --ide-ast for a simplistic ast for editors (#8995)

# Description
This is WIP. This generates a simplistic AST so that we can call it from
the vscode side for semantic token highlighting. The output is a
minified version of this screenshot.


![image](https://user-images.githubusercontent.com/343840/234354668-872d6267-9946-4b92-8a13-4fed45b4513a.png)

The script
```
def test [arg] {
  print $arg
  for i in (seq 1 10) {
    echo $i
  }
}
```

The simplistic AST
```json
[{"content":"def","index":0,"shape":"shape_internalcall","span":{"end":15,"start":12}},{"content":"test","index":1,"shape":"shape_string","span":{"end":20,"start":16}},{"content":"[arg]","index":2,"shape":"shape_signature","span":{"end":26,"start":21}},{"content":"{\r\n  ","index":3,"shape":"shape_closure","span":{"end":32,"start":27}},{"content":"print","index":4,"shape":"shape_internalcall","span":{"end":37,"start":32}},{"content":"$arg","index":5,"shape":"shape_variable","span":{"end":42,"start":38}},{"content":"for","index":6,"shape":"shape_internalcall","span":{"end":49,"start":46}},{"content":"i","index":7,"shape":"shape_vardecl","span":{"end":51,"start":50}},{"content":"in","index":8,"shape":"shape_keyword","span":{"end":54,"start":52}},{"content":"(","index":9,"shape":"shape_block","span":{"end":56,"start":55}},{"content":"seq","index":10,"shape":"shape_internalcall","span":{"end":59,"start":56}},{"content":"1","index":11,"shape":"shape_int","span":{"end":61,"start":60}},{"content":"10","index":12,"shape":"shape_int","span":{"end":64,"start":62}},{"content":")","index":13,"shape":"shape_block","span":{"end":65,"start":64}},{"content":"{\r\n    ","index":14,"shape":"shape_block","span":{"end":73,"start":66}},{"content":"echo","index":15,"shape":"shape_internalcall","span":{"end":77,"start":73}},{"content":"$i","index":16,"shape":"shape_variable","span":{"end":80,"start":78}},{"content":"\r\n  }","index":17,"shape":"shape_block","span":{"end":85,"start":80}},{"content":"\r\n}","index":18,"shape":"shape_closure","span":{"end":88,"start":85}}]
```

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect` to check that you're using the standard code
style
- `cargo test --workspace` to check that all tests pass
- `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the
standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

src/command.rs

@@ -33,7 +33,7 @@ pub(crate) fn gather_commandline_args() -> (Vec<String>, String, Vec<String>) {
 
         let flag_value = match arg.as_ref() {
             "--commands" | "-c" | "--table-mode" | "-m" | "-e" | "--execute" | "--config"
-            | "--env-config" | "-I" => args.next().map(|a| escape_quote_string(&a)),
+            | "--env-config" | "-I" | "ide-ast" => args.next().map(|a| escape_quote_string(&a)),
             #[cfg(feature = "plugin")]
             "--plugin-config" => args.next().map(|a| escape_quote_string(&a)),
             "--log-level" | "--log-target" | "--testbin" | "--threads" | "-t"
@@ -104,16 +104,18 @@ pub(crate) fn parse_commandline_args(
             let log_level: Option<Expression> = call.get_flag_expr("log-level");
             let log_target: Option<Expression> = call.get_flag_expr("log-target");
             let execute: Option<Expression> = call.get_flag_expr("execute");
-            let include_path: Option<Expression> = call.get_flag_expr("include-path");
             let table_mode: Option<Value> =
                 call.get_flag(engine_state, &mut stack, "table-mode")?;
 
+            // ide flags
+            let include_path: Option<Expression> = call.get_flag_expr("include-path");
             let ide_goto_def: Option<Value> =
                 call.get_flag(engine_state, &mut stack, "ide-goto-def")?;
             let ide_hover: Option<Value> = call.get_flag(engine_state, &mut stack, "ide-hover")?;
             let ide_complete: Option<Value> =
                 call.get_flag(engine_state, &mut stack, "ide-complete")?;
             let ide_check: Option<Value> = call.get_flag(engine_state, &mut stack, "ide-check")?;
+            let ide_ast: Option<Spanned<String>> = call.get_named_arg("ide-ast");
 
             fn extract_contents(
                 expression: Option<Expression>,
@@ -192,6 +194,7 @@ pub(crate) fn parse_commandline_args(
                 ide_hover,
                 ide_complete,
                 ide_check,
+                ide_ast,
                 table_mode,
             });
         }
@@ -230,6 +233,7 @@ pub(crate) struct NushellCliArgs {
     pub(crate) ide_hover: Option<Value>,
     pub(crate) ide_complete: Option<Value>,
     pub(crate) ide_check: Option<Value>,
+    pub(crate) ide_ast: Option<Spanned<String>>,
 }
 
 #[derive(Clone)]
@@ -317,7 +321,8 @@ impl Command for Nu {
                 SyntaxShape::Int,
                 "run a diagnostic check on the given source",
                 None,
-            );
+            )
+            .switch("ide-ast", "generate the ast on the given source", None);
 
         #[cfg(feature = "plugin")]
         {

src/ide.rs

@@ -7,7 +7,7 @@ use nu_protocol::{
     DeclId, ShellError, Span, Value, VarId,
 };
 use reedline::Completer;
-use serde_json::json;
+use serde_json::{json, Value as JsonValue};
 use std::sync::Arc;
 
 #[derive(Debug)]
@@ -602,3 +602,59 @@ pub fn complete(engine_reference: Arc<EngineState>, file_path: &String, location
         println!("]}}");
     }
 }
+
+pub fn ast(engine_state: &mut EngineState, file_path: &String) {
+    let cwd = std::env::current_dir().expect("Could not get current working directory.");
+    engine_state.add_env_var("PWD".into(), Value::test_string(cwd.to_string_lossy()));
+
+    let mut working_set = StateWorkingSet::new(engine_state);
+    let file = std::fs::read(file_path);
+
+    if let Ok(contents) = file {
+        let offset = working_set.next_span_start();
+        let parsed_block = parse(&mut working_set, Some(file_path), &contents, false);
+
+        let flat = flatten_block(&working_set, &parsed_block);
+        let mut json_val: JsonValue = json!([]);
+        for (span, shape) in flat {
+            let content = String::from_utf8_lossy(working_set.get_span_contents(span)).to_string();
+
+            let json = json!(
+                {
+                    "type": "ast",
+                    "span": {
+                        "start": span.start - offset,
+                        "end": span.end - offset,
+                    },
+                    "shape": shape.to_string(),
+                    "content": content // may not be necessary, but helpful for debugging
+                }
+            );
+            json_merge(&mut json_val, &json);
+        }
+        if let Ok(json_str) = serde_json::to_string(&json_val) {
+            println!("{json_str}");
+        } else {
+            println!("{{}}");
+        };
+    }
+}
+
+fn json_merge(a: &mut JsonValue, b: &JsonValue) {
+    match (a, b) {
+        (JsonValue::Object(ref mut a), JsonValue::Object(b)) => {
+            for (k, v) in b {
+                json_merge(a.entry(k).or_insert(JsonValue::Null), v);
+            }
+        }
+        (JsonValue::Array(ref mut a), JsonValue::Array(b)) => {
+            a.extend(b.clone());
+        }
+        (JsonValue::Array(ref mut a), JsonValue::Object(b)) => {
+            a.extend([JsonValue::Object(b.clone())]);
+        }
+        (a, b) => {
+            *a = b.clone();
+        }
+    }
+}

src/main.rs

@@ -170,6 +170,10 @@ fn main() -> Result<()> {
     } else if let Some(max_errors) = parsed_nu_cli_args.ide_check {
         ide::check(&mut engine_state, &script_name, &max_errors);
 
+        return Ok(());
+    } else if parsed_nu_cli_args.ide_ast.is_some() {
+        ide::ast(&mut engine_state, &script_name);
+
         return Ok(());
     }