Multiline scripts part 2 (#2795)

* Begin allowing comments and multiline scripts.

* clippy

* Finish moving to groups. Test pass

* Keep going

* WIP

* WIP

* BROKEN WIP

* WIP

* WIP

* Fix more tests

* WIP: alias starts working

* Broken WIP

* Broken WIP

* Variables begin to work

* captures start working

* A little better but needs fixed scope

* Shorthand env setting

* Update main merge

* Broken WIP

* WIP

* custom command parsing

* Custom commands start working

* Fix coloring and parsing of block

* Almost there

* Add some tests

* Add more param types

* Bump version

* Fix benchmark

* Fix stuff

crates/nu-protocol/src/hir.rs

@@ -5,7 +5,8 @@ use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
 
-use crate::{hir, Primitive, UntaggedValue};
+use crate::Signature;
+use crate::{hir, Dictionary, PositionalType, Primitive, SyntaxShape, UntaggedValue};
 use crate::{PathMember, ShellTypeName};
 use derive_new::new;
 
@@ -44,6 +45,10 @@ impl InternalCommand {
     pub fn has_it_usage(&self) -> bool {
         self.args.has_it_usage()
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        self.args.get_free_variables(known_variables)
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
@@ -62,11 +67,11 @@ impl ClassifiedBlock {
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
 pub struct ClassifiedPipeline {
-    pub commands: Commands,
+    pub commands: Pipeline,
 }
 
 impl ClassifiedPipeline {
-    pub fn new(commands: Commands) -> ClassifiedPipeline {
+    pub fn new(commands: Pipeline) -> ClassifiedPipeline {
         ClassifiedPipeline { commands }
     }
 }
@@ -74,7 +79,6 @@ impl ClassifiedPipeline {
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
 pub enum ClassifiedCommand {
     Expr(Box<SpannedExpression>),
-    #[allow(unused)]
     Dynamic(crate::hir::Call),
     Internal(InternalCommand),
     Error(ParseError),
@@ -89,17 +93,33 @@ impl ClassifiedCommand {
             ClassifiedCommand::Error(_) => false,
         }
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        match self {
+            ClassifiedCommand::Expr(expr) => expr.get_free_variables(known_variables),
+            ClassifiedCommand::Dynamic(call) => call.get_free_variables(known_variables),
+            ClassifiedCommand::Internal(internal) => internal.get_free_variables(known_variables),
+            _ => vec![],
+        }
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
-pub struct Commands {
+pub struct Pipeline {
     pub list: Vec<ClassifiedCommand>,
     pub span: Span,
 }
 
-impl Commands {
-    pub fn new(span: Span) -> Commands {
-        Commands { list: vec![], span }
+impl Pipeline {
+    pub fn new(span: Span) -> Pipeline {
+        Pipeline { list: vec![], span }
+    }
+
+    pub fn basic() -> Pipeline {
+        Pipeline {
+            list: vec![],
+            span: Span::unknown(),
+        }
     }
 
     pub fn push(&mut self, command: ClassifiedCommand) {
@@ -112,34 +132,87 @@ impl Commands {
 }
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+pub struct Group {
+    pub pipelines: Vec<Pipeline>,
+    pub span: Span,
+}
+impl Group {
+    pub fn new(pipelines: Vec<Pipeline>, span: Span) -> Group {
+        Group { pipelines, span }
+    }
+
+    pub fn basic() -> Group {
+        Group {
+            pipelines: vec![],
+            span: Span::unknown(),
+        }
+    }
+
+    pub fn push(&mut self, pipeline: Pipeline) {
+        self.pipelines.push(pipeline);
+    }
+
+    pub fn has_it_usage(&self) -> bool {
+        self.pipelines.iter().any(|cc| cc.has_it_usage())
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+pub struct CapturedBlock {
+    pub block: Block,
+    pub captured: Dictionary,
+}
+
+impl CapturedBlock {
+    pub fn new(block: Block, captured: Dictionary) -> Self {
+        Self { block, captured }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub struct Block {
-    pub params: Vec<String>,
-    pub block: Vec<Commands>,
+    pub params: Signature,
+    pub block: Vec<Group>,
+    pub definitions: IndexMap<String, Block>,
     pub span: Span,
 }
 
 impl Block {
-    pub fn new(params: Vec<String>, block: Vec<Commands>, span: Span) -> Block {
-        let mut output = Block {
+    pub fn new(
+        params: Signature,
+        block: Vec<Group>,
+        definitions: IndexMap<String, Block>,
+        span: Span,
+    ) -> Block {
+        Block {
             params,
             block,
+            definitions,
             span,
-        };
-
-        output.infer_params();
-        output
+        }
     }
 
-    pub fn push(&mut self, commands: Commands) {
-        self.block.push(commands);
+    pub fn basic() -> Block {
+        Block {
+            params: Signature::new("<basic>"),
+            block: vec![],
+            definitions: IndexMap::new(),
+            span: Span::unknown(),
+        }
+    }
+
+    pub fn push(&mut self, group: Group) {
+        self.block.push(group);
         self.infer_params();
     }
 
     pub fn set_redirect(&mut self, external_redirection: ExternalRedirection) {
-        if let Some(pipeline) = self.block.last_mut() {
-            if let Some(command) = pipeline.list.last_mut() {
-                if let ClassifiedCommand::Internal(internal) = command {
-                    internal.args.external_redirection = external_redirection;
+        if let Some(group) = self.block.last_mut() {
+            if let Some(pipeline) = group.pipelines.last_mut() {
+                if let Some(command) = pipeline.list.last_mut() {
+                    if let ClassifiedCommand::Internal(internal) = command {
+                        internal.args.external_redirection = external_redirection;
+                    }
                 }
             }
         }
@@ -150,10 +223,70 @@ impl Block {
     }
 
     pub fn infer_params(&mut self) {
-        if self.params.is_empty() && self.has_it_usage() {
-            self.params = vec!["$it".into()];
+        // FIXME: re-enable inference later
+        if self.params.positional.is_empty() && self.has_it_usage() {
+            self.params.positional = vec![(
+                PositionalType::Mandatory("$it".to_string(), SyntaxShape::Any),
+                "implied $it".to_string(),
+            )];
         }
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        let mut known_variables = known_variables.clone();
+        let positional_params: Vec<_> = self
+            .params
+            .positional
+            .iter()
+            .map(|(_, name)| name.clone())
+            .collect();
+        known_variables.extend_from_slice(&positional_params);
+
+        let mut free_variables = vec![];
+        for group in &self.block {
+            for pipeline in &group.pipelines {
+                for elem in &pipeline.list {
+                    free_variables
+                        .extend_from_slice(&elem.get_free_variables(&mut known_variables));
+                }
+            }
+        }
+
+        free_variables
+    }
+}
+
+#[allow(clippy::derive_hash_xor_eq)]
+impl Hash for Block {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        let mut entries = self.definitions.clone();
+        entries.sort_keys();
+
+        // FIXME: this is incomplete
+        entries.keys().collect::<Vec<&String>>().hash(state);
+    }
+}
+
+impl PartialOrd for Block {
+    /// Compare two dictionaries for sort ordering
+    fn partial_cmp(&self, other: &Block) -> Option<Ordering> {
+        let this: Vec<&String> = self.definitions.keys().collect();
+        let that: Vec<&String> = other.definitions.keys().collect();
+
+        // FIXME: this is incomplete
+        this.partial_cmp(&that)
+    }
+}
+
+impl Ord for Block {
+    /// Compare two dictionaries for ordering
+    fn cmp(&self, other: &Block) -> Ordering {
+        let this: Vec<&String> = self.definitions.keys().collect();
+        let that: Vec<&String> = other.definitions.keys().collect();
+
+        // FIXME: this is incomplete
+        this.cmp(&that)
+    }
 }
 
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, Hash, Deserialize, Serialize)]
@@ -546,6 +679,10 @@ impl SpannedExpression {
     pub fn has_it_usage(&self) -> bool {
         self.expr.has_it_usage()
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        self.expr.get_free_variables(known_variables)
+    }
 }
 
 impl std::ops::Deref for SpannedExpression {
@@ -1012,6 +1149,52 @@ impl Expression {
             _ => false,
         }
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        let mut output = vec![];
+        match self {
+            Expression::Variable(name, _) => {
+                if !known_variables.contains(name) {
+                    output.push(name.clone());
+                }
+            }
+            Expression::Table(headers, values) => {
+                for header in headers {
+                    output.extend(header.get_free_variables(known_variables));
+                }
+                for row in values {
+                    for value in row {
+                        output.extend(value.get_free_variables(known_variables));
+                    }
+                }
+            }
+            Expression::List(list) => {
+                for item in list {
+                    output.extend(item.get_free_variables(known_variables));
+                }
+            }
+            Expression::Invocation(block) => {
+                output.extend(block.get_free_variables(known_variables));
+            }
+            Expression::Binary(binary) => {
+                output.extend(binary.left.get_free_variables(known_variables));
+                output.extend(binary.right.get_free_variables(known_variables));
+            }
+            Expression::Path(path) => {
+                output.extend(path.head.get_free_variables(known_variables));
+            }
+            Expression::Range(range) => {
+                if let Some(left) = &range.left {
+                    output.extend(left.get_free_variables(known_variables));
+                }
+                if let Some(right) = &range.right {
+                    output.extend(right.get_free_variables(known_variables));
+                }
+            }
+            _ => {}
+        }
+        output
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
@@ -1019,7 +1202,7 @@ pub enum NamedValue {
     AbsentSwitch,
     PresentSwitch(Span),
     AbsentValue,
-    Value(Span, SpannedExpression),
+    Value(Span, Box<SpannedExpression>),
 }
 
 impl NamedValue {
@@ -1030,6 +1213,13 @@ impl NamedValue {
             false
         }
     }
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        if let NamedValue::Value(_, se) = self {
+            se.get_free_variables(known_variables)
+        } else {
+            vec![]
+        }
+    }
 }
 
 impl PrettyDebugWithSource for NamedValue {
@@ -1108,6 +1298,23 @@ impl Call {
                 false
             })
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        let mut free_variables = vec![];
+
+        free_variables.extend(self.head.get_free_variables(known_variables));
+        if let Some(pos) = &self.positional {
+            for pos in pos {
+                free_variables.extend(pos.get_free_variables(known_variables));
+            }
+        }
+
+        if let Some(named) = &self.named {
+            free_variables.extend(named.get_free_variables(known_variables));
+        }
+
+        free_variables
+    }
 }
 
 impl PrettyDebugWithSource for Call {
@@ -1229,7 +1436,7 @@ impl PartialOrd for NamedArguments {
         }
 
         let this: Vec<&NamedValue> = self.named.values().collect();
-        let that: Vec<&NamedValue> = self.named.values().collect();
+        let that: Vec<&NamedValue> = other.named.values().collect();
 
         this.partial_cmp(&that)
     }
@@ -1246,7 +1453,7 @@ impl Ord for NamedArguments {
         }
 
         let this: Vec<&NamedValue> = self.named.values().collect();
-        let that: Vec<&NamedValue> = self.named.values().collect();
+        let that: Vec<&NamedValue> = other.named.values().collect();
 
         this.cmp(&that)
     }
@@ -1272,6 +1479,14 @@ impl NamedArguments {
     pub fn has_it_usage(&self) -> bool {
         self.iter().any(|x| x.1.has_it_usage())
     }
+
+    pub fn get_free_variables(&self, known_variables: &mut Vec<String>) -> Vec<String> {
+        let mut free_variables = vec![];
+        for (_, val) in self.named.iter() {
+            free_variables.extend(val.get_free_variables(known_variables));
+        }
+        free_variables
+    }
 }
 
 impl NamedArguments {
@@ -1297,7 +1512,7 @@ impl NamedArguments {
             None => self.named.insert(name.into(), NamedValue::AbsentValue),
             Some(expr) => self
                 .named
-                .insert(name.into(), NamedValue::Value(flag_span, expr)),
+                .insert(name.into(), NamedValue::Value(flag_span, Box::new(expr))),
         };
     }
 
@@ -1308,7 +1523,7 @@ impl NamedArguments {
         expr: SpannedExpression,
     ) {
         self.named
-            .insert(name.into(), NamedValue::Value(flag_span, expr));
+            .insert(name.into(), NamedValue::Value(flag_span, Box::new(expr)));
     }
 
     pub fn switch_present(&self, switch: &str) -> bool {

crates/nu-protocol/src/lib.rs

@@ -21,7 +21,6 @@ pub use crate::type_shape::{Row as RowType, Type};
 pub use crate::value::column_path::{ColumnPath, PathMember, UnspannedPathMember};
 pub use crate::value::dict::{Dictionary, TaggedDictBuilder};
 pub use crate::value::did_you_mean::did_you_mean;
-pub use crate::value::evaluate::Scope;
 pub use crate::value::primitive::Primitive;
 pub use crate::value::primitive::{format_date, format_duration, format_primitive};
 pub use crate::value::range::{Range, RangeInclusion};

crates/nu-protocol/src/return_value.rs

@@ -1,5 +1,4 @@
-use crate::hir::Block;
-use crate::{value::Value, Signature};
+use crate::value::Value;
 use nu_errors::ShellError;
 use nu_source::{b, DebugDocBuilder, PrettyDebug};
 use serde::{Deserialize, Serialize};
@@ -21,9 +20,8 @@ pub enum CommandAction {
     EnterValueShell(Value),
     /// Enter the help shell, which allows exploring the help system
     EnterHelpShell(Value),
-    /// Add an alias command
-    /// Note: We are passing the Signature in a Box to decrease the memory size of AddAlias
-    AddAlias(Box<Signature>, Block),
+    /// Add a variable into scope
+    AddVariable(String, Value),
     /// Add plugins from path given
     AddPlugins(String),
     /// Go to the previous shell in the shell ring buffer
@@ -47,7 +45,7 @@ impl PrettyDebug for CommandAction {
             CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
             CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
             CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
-            CommandAction::AddAlias(..) => b::description("add alias"),
+            CommandAction::AddVariable(..) => b::description("add variable"),
             CommandAction::AddPlugins(..) => b::description("add plugins"),
             CommandAction::PreviousShell => b::description("previous shell"),
             CommandAction::NextShell => b::description("next shell"),

crates/nu-protocol/src/signature.rs

@@ -118,6 +118,18 @@ pub struct Signature {
     pub is_filter: bool,
 }
 
+impl PartialEq for Signature {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+            && self.usage == other.usage
+            && self.positional == other.positional
+            && self.rest_positional == other.rest_positional
+            && self.is_filter == other.is_filter
+    }
+}
+
+impl Eq for Signature {}
+
 impl Signature {
     pub fn shift_positional(&mut self) {
         self.positional = Vec::from(&self.positional[1..]);

crates/nu-protocol/src/syntax_shape.rs

@@ -30,8 +30,10 @@ pub enum SyntaxShape {
     Unit,
     /// An operator
     Operator,
-    /// A math expression, eg `foo > 1`
+    /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1`
     Math,
+    /// An initializer expression, eg the right hand side of `set x = 1 + 2`
+    Initializer,
 }
 
 impl PrettyDebug for SyntaxShape {
@@ -52,6 +54,7 @@ impl PrettyDebug for SyntaxShape {
             SyntaxShape::Unit => "unit",
             SyntaxShape::Operator => "operator",
             SyntaxShape::Math => "condition",
+            SyntaxShape::Initializer => "initializer",
         })
     }
 }

crates/nu-protocol/src/value.rs

@@ -3,7 +3,6 @@ mod convert;
 mod debug;
 pub mod dict;
 pub mod did_you_mean;
-pub mod evaluate;
 pub mod iter;
 pub mod primitive;
 pub mod range;
@@ -45,8 +44,8 @@ pub enum UntaggedValue {
     /// An error value that represents an error that occurred as the values in the pipeline were built
     Error(ShellError),
 
-    /// A block of Nu code, eg `{ ls | get name ; echo "done" }`
-    Block(hir::Block),
+    /// A block of Nu code, eg `{ ls | get name ; echo "done" }` with its captured values
+    Block(Box<hir::CapturedBlock>),
 }
 
 impl UntaggedValue {

crates/nu-protocol/src/value/dict.rs

@@ -122,6 +122,10 @@ impl Dictionary {
         }
     }
 
+    pub fn insert(&mut self, key: String, value: Value) -> Option<Value> {
+        self.entries.insert_full(key, value).1
+    }
+
     pub fn merge_from(&self, other: &Dictionary) -> Dictionary {
         let mut obj = self.clone();
 

crates/nu-protocol/src/value/evaluate.rs

@@ -1,106 +0,0 @@
-use crate::value::Value;
-use indexmap::IndexMap;
-use serde::{Deserialize, Serialize};
-use std::fmt::Debug;
-use std::sync::Arc;
-
-/// An evaluation scope. Scopes map variable names to Values and aid in evaluating blocks and expressions.
-#[derive(Deserialize, Serialize, Debug, Clone)]
-pub struct Scope {
-    vars: IndexMap<String, Value>,
-    env: IndexMap<String, String>,
-    parent: Option<Arc<Scope>>,
-}
-
-impl Scope {
-    pub fn vars(&self) -> IndexMap<String, Value> {
-        //FIXME: should this be an interator?
-
-        let mut output = IndexMap::new();
-
-        for v in &self.vars {
-            output.insert(v.0.clone(), v.1.clone());
-        }
-
-        if let Some(parent) = &self.parent {
-            for v in parent.vars() {
-                if !output.contains_key(&v.0) {
-                    output.insert(v.0.clone(), v.1.clone());
-                }
-            }
-        }
-
-        output
-    }
-
-    pub fn env(&self) -> IndexMap<String, String> {
-        //FIXME: should this be an interator?
-
-        let mut output = IndexMap::new();
-
-        for v in &self.env {
-            output.insert(v.0.clone(), v.1.clone());
-        }
-
-        if let Some(parent) = &self.parent {
-            for v in parent.env() {
-                if !output.contains_key(&v.0) {
-                    output.insert(v.0.clone(), v.1.clone());
-                }
-            }
-        }
-
-        output
-    }
-
-    pub fn var(&self, name: &str) -> Option<Value> {
-        if let Some(value) = self.vars().get(name) {
-            Some(value.clone())
-        } else {
-            None
-        }
-    }
-
-    pub fn from_env(env: IndexMap<String, String>) -> Arc<Scope> {
-        Arc::new(Scope {
-            vars: IndexMap::new(),
-            env,
-            parent: None,
-        })
-    }
-
-    pub fn append_var(this: Arc<Self>, name: impl Into<String>, value: Value) -> Arc<Scope> {
-        let mut vars = IndexMap::new();
-        vars.insert(name.into(), value);
-        Arc::new(Scope {
-            vars,
-            env: IndexMap::new(),
-            parent: Some(this),
-        })
-    }
-
-    pub fn append_vars(this: Arc<Self>, vars: IndexMap<String, Value>) -> Arc<Scope> {
-        Arc::new(Scope {
-            vars,
-            env: IndexMap::new(),
-            parent: Some(this),
-        })
-    }
-
-    pub fn append_env(this: Arc<Self>, env: IndexMap<String, String>) -> Arc<Scope> {
-        Arc::new(Scope {
-            vars: IndexMap::new(),
-            env,
-            parent: Some(this),
-        })
-    }
-
-    /// Create an empty scope
-    pub fn create() -> Arc<Scope> {
-        Arc::new(Scope {
-            vars: IndexMap::new(),
-            env: IndexMap::new(),
-            parent: None,
-        })
-    }
-}