Support evaluating most expressions

Blocks, paths, and others

Plus a bunch of other infra improvements

src/parser/hir.rs

@@ -9,11 +9,20 @@ use derive_new::new;
 use getset::Getters;
 
 crate use baseline_parse::baseline_parse_single_token;
-crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint};
+crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint, TokensIterator};
 crate use binary::Binary;
 crate use named::NamedArguments;
 crate use path::Path;
 
+pub fn path(head: impl Into<Expression>, tail: Vec<Spanned<impl Into<String>>>) -> Path {
+    Path::new(
+        head.into(),
+        tail.into_iter()
+            .map(|item| item.map(|string| string.into()))
+            .collect(),
+    )
+}
+
 #[derive(Debug, Clone, Eq, PartialEq, Getters, new)]
 pub struct Call {
     #[get = "crate"]
@@ -29,7 +38,7 @@ pub enum RawExpression {
     Literal(Literal),
     Variable(Variable),
     Binary(Box<Binary>),
-    Block(Box<Expression>),
+    Block(Vec<Expression>),
     Path(Box<Path>),
 
     #[allow(unused)]
@@ -76,6 +85,12 @@ impl Expression {
     }
 }
 
+impl From<Spanned<Path>> for Expression {
+    fn from(path: Spanned<Path>) -> Expression {
+        path.map(|p| RawExpression::Path(Box::new(p)))
+    }
+}
+
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub enum Literal {
     Integer(i64),

src/parser/hir/baseline_parse_tokens.rs

@@ -1,24 +1,27 @@
 use crate::errors::ShellError;
 use crate::parser::registry::CommandRegistry;
-use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode};
-use crate::Text;
+use crate::parser::{
+    hir, hir::baseline_parse_single_token, DelimitedNode, Delimiter, PathNode, RawToken, Span,
+    Spanned, TokenNode,
+};
+use crate::{SpannedItem, Text};
+use derive_new::new;
+use log::trace;
 
 pub fn baseline_parse_tokens(
-    token_nodes: &[TokenNode],
+    token_nodes: &mut TokensIterator<'_>,
     registry: &dyn CommandRegistry,
     source: &Text,
 ) -> Result<Vec<hir::Expression>, ShellError> {
     let mut exprs: Vec<hir::Expression> = vec![];
-    let mut rest = token_nodes;
 
     loop {
-        if rest.len() == 0 {
+        if token_nodes.at_end() {
             break;
         }
 
-        let (expr, remainder) = baseline_parse_next_expr(rest, registry, source, None)?;
+        let expr = baseline_parse_next_expr(token_nodes, registry, source, None)?;
         exprs.push(expr);
-        rest = remainder;
     }
 
     Ok(exprs)
@@ -35,25 +38,21 @@ pub enum ExpressionKindHint {
 }
 
 pub fn baseline_parse_next_expr(
-    token_nodes: &'nodes [TokenNode],
-    _registry: &dyn CommandRegistry,
+    tokens: &mut TokensIterator,
+    registry: &dyn CommandRegistry,
     source: &Text,
     coerce_hint: Option<ExpressionKindHint>,
-) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> {
-    let mut tokens = token_nodes.iter().peekable();
-
-    let first = next_token(&mut tokens);
-
-    let first = match first {
+) -> Result<hir::Expression, ShellError> {
+    let first = match tokens.next() {
         None => return Err(ShellError::string("Expected token, found none")),
-        Some(token) => baseline_parse_semantic_token(token, source)?,
+        Some(token) => baseline_parse_semantic_token(token, registry, source)?,
     };
 
     let possible_op = tokens.peek();
 
     let op = match possible_op {
-        Some(TokenNode::Operator(op)) => op,
-        _ => return Ok((first, &token_nodes[1..])),
+        Some(TokenNode::Operator(op)) => op.clone(),
+        _ => return Ok(first),
     };
 
     tokens.next();
@@ -64,7 +63,7 @@ pub fn baseline_parse_next_expr(
                 "Expected op followed by another expr, found nothing",
             ))
         }
-        Some(token) => baseline_parse_semantic_token(token, source)?,
+        Some(token) => baseline_parse_semantic_token(token, registry, source)?,
     };
 
     // We definitely have a binary expression here -- let's see if we should coerce it into a block
@@ -72,11 +71,11 @@ pub fn baseline_parse_next_expr(
     match coerce_hint {
         None => {
             let span = (first.span.start, second.span.end);
-            let binary = hir::Binary::new(first, *op, second);
+            let binary = hir::Binary::new(first, op, second);
             let binary = hir::RawExpression::Binary(Box::new(binary));
             let binary = Spanned::from_item(binary, span);
 
-            Ok((binary, &token_nodes[3..]))
+            Ok(binary)
         }
 
         Some(hint) => match hint {
@@ -133,14 +132,14 @@ pub fn baseline_parse_next_expr(
                     }
                 };
 
-                let binary = hir::Binary::new(path, *op, second);
+                let binary = hir::Binary::new(path, op, second);
                 let binary = hir::RawExpression::Binary(Box::new(binary));
                 let binary = Spanned::from_item(binary, span);
 
-                let block = hir::RawExpression::Block(Box::new(binary));
+                let block = hir::RawExpression::Block(vec![binary]);
                 let block = Spanned::from_item(block, span);
 
-                Ok((block, &token_nodes[3..]))
+                Ok(block)
             }
 
             other => unimplemented!("coerce hint {:?}", other),
@@ -150,27 +149,199 @@ pub fn baseline_parse_next_expr(
 
 pub fn baseline_parse_semantic_token(
     token: &TokenNode,
+    registry: &dyn CommandRegistry,
     source: &Text,
 ) -> Result<hir::Expression, ShellError> {
     match token {
         TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),
         TokenNode::Call(_call) => unimplemented!(),
-        TokenNode::Delimited(_delimited) => unimplemented!(),
+        TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, registry, source),
         TokenNode::Pipeline(_pipeline) => unimplemented!(),
         TokenNode::Operator(_op) => unreachable!(),
         TokenNode::Flag(_flag) => unimplemented!(),
         TokenNode::Identifier(_span) => unreachable!(),
         TokenNode::Whitespace(_span) => unreachable!(),
         TokenNode::Error(error) => Err(*error.item.clone()),
-        TokenNode::Path(_path) => unimplemented!(),
+        TokenNode::Path(path) => baseline_parse_path(path, registry, source),
     }
 }
 
-fn next_token(nodes: &mut impl Iterator<Item = &'a TokenNode>) -> Option<&'a TokenNode> {
-    loop {
-        match nodes.next() {
-            Some(TokenNode::Whitespace(_)) => continue,
-            other => return other,
+pub fn baseline_parse_delimited(
+    token: &Spanned<DelimitedNode>,
+    registry: &dyn CommandRegistry,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    match token.delimiter() {
+        Delimiter::Brace => {
+            let children = token.children();
+            let exprs =
+                baseline_parse_tokens(&mut TokensIterator::new(children), registry, source)?;
+
+            let expr = hir::RawExpression::Block(exprs);
+            Ok(Spanned::from_item(expr, token.span()))
+        }
+        Delimiter::Paren => unimplemented!(),
+        Delimiter::Square => unimplemented!(),
+    }
+}
+
+pub fn baseline_parse_path(
+    token: &Spanned<PathNode>,
+    registry: &dyn CommandRegistry,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    let head = baseline_parse_semantic_token(token.head(), registry, source)?;
+
+    let mut tail = vec![];
+
+    for part in token.tail() {
+        let string = match part {
+            TokenNode::Token(token) => match token.item() {
+                RawToken::Bare => token.span().slice(source),
+                RawToken::String(span) => span.slice(source),
+                RawToken::Integer(_) | RawToken::Size(..) | RawToken::Variable(_) => {
+                    return Err(ShellError::type_error(
+                        "String",
+                        token.type_name().spanned(part),
+                    ))
+                }
+            },
+
+            TokenNode::Identifier(span) => span.slice(source),
+
+            // TODO: Make this impossible
+            other => unreachable!("{:?}", other),
+        }
+        .to_string();
+
+        tail.push(string.spanned(part));
+    }
+
+    Ok(hir::path(head, tail).spanned(token).into())
+}
+
+#[derive(Debug, new)]
+pub struct TokensIterator<'a> {
+    tokens: &'a [TokenNode],
+    #[new(default)]
+    index: usize,
+    #[new(default)]
+    seen: indexmap::IndexSet<usize>,
+}
+
+impl TokensIterator<'a> {
+    pub fn remove(&mut self, position: usize) {
+        self.seen.insert(position);
+    }
+
+    pub fn len(&self) -> usize {
+        self.tokens.len()
+    }
+
+    pub fn at_end(&self) -> bool {
+        for index in self.index..self.tokens.len() {
+            if !self.seen.contains(&index) {
+                return false;
+            }
+        }
+
+        true
+    }
+
+    pub fn advance(&mut self) {
+        self.seen.insert(self.index);
+        self.index += 1;
+    }
+
+    pub fn extract<T>(&mut self, f: impl Fn(&TokenNode) -> Option<T>) -> Option<(usize, T)> {
+        for (i, item) in self.tokens.iter().enumerate() {
+            if self.seen.contains(&i) {
+                continue;
+            }
+
+            match f(item) {
+                None => {
+                    continue;
+                }
+                Some(value) => {
+                    self.seen.insert(i);
+                    return Some((i, value));
+                }
+            }
+        }
+
+        None
+    }
+
+    pub fn move_to(&mut self, pos: usize) {
+        self.index = pos;
+    }
+
+    pub fn restart(&mut self) {
+        self.index = 0;
+    }
+
+    pub fn clone(&self) -> TokensIterator {
+        TokensIterator {
+            tokens: self.tokens,
+            index: self.index,
+            seen: self.seen.clone(),
+        }
+    }
+
+    pub fn peek(&self) -> Option<&TokenNode> {
+        let mut tokens = self.clone();
+
+        tokens.next()
+    }
+
+    pub fn debug_remaining(&self) -> Vec<TokenNode> {
+        let mut tokens = self.clone();
+        tokens.restart();
+        tokens.cloned().collect()
+    }
+}
+
+impl Iterator for TokensIterator<'a> {
+    type Item = &'a TokenNode;
+
+    fn next(&mut self) -> Option<&'a TokenNode> {
+        loop {
+            if self.index >= self.tokens.len() {
+                return None;
+            }
+
+            if self.seen.contains(&self.index) {
+                self.advance();
+                continue;
+            }
+
+            if self.index >= self.tokens.len() {
+                return None;
+            }
+
+            match &self.tokens[self.index] {
+                TokenNode::Whitespace(_) => {
+                    self.advance();
+                }
+                other => {
+                    self.advance();
+                    return Some(other);
+                }
+            }
         }
     }
 }
+
+pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'a>, source: &Text) {
+    trace!(
+        "{} = {:?}",
+        desc,
+        itertools::join(
+            tail.debug_remaining()
+                .iter()
+                .map(|i| format!("%{:?}%", i.debug(source))),
+            " "
+        )
+    );
+}

src/parser/parse/span.rs

@@ -12,6 +12,14 @@ pub struct Spanned<T> {
     pub item: T,
 }
 
+pub trait SpannedItem: Sized {
+    fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
+        Spanned::from_item(self, span.into())
+    }
+}
+
+impl<T> SpannedItem for T {}
+
 impl<T> std::ops::Deref for Spanned<T> {
     type Target = T;
 
@@ -56,6 +64,12 @@ pub struct Span {
     // source: &'source str,
 }
 
+impl<T> From<&Spanned<T>> for Span {
+    fn from(input: &Spanned<T>) -> Span {
+        input.span
+    }
+}
+
 impl From<&Span> for Span {
     fn from(input: &Span) -> Span {
         *input

src/parser/parse/token_tree.rs

@@ -3,6 +3,8 @@ use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, span
 use crate::Text;
 use derive_new::new;
 use enum_utils::FromStr;
+use getset::Getters;
+use std::fmt;
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
 pub enum TokenNode {
@@ -20,6 +22,67 @@ pub enum TokenNode {
     Path(Spanned<PathNode>),
 }
 
+pub struct DebugTokenNode<'a> {
+    node: &'a TokenNode,
+    source: &'a Text,
+}
+
+impl fmt::Debug for DebugTokenNode<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.node {
+            TokenNode::Token(t) => write!(f, "{:?}", t.debug(self.source)),
+            TokenNode::Call(s) => {
+                write!(f, "(")?;
+
+                write!(f, "{:?}", s.head().debug(self.source))?;
+
+                if let Some(children) = s.children() {
+                    for child in children {
+                        write!(f, "{:?}", child.debug(self.source))?;
+                    }
+                }
+
+                write!(f, ")")
+            }
+
+            TokenNode::Delimited(d) => {
+                write!(
+                    f,
+                    "{}",
+                    match d.delimiter {
+                        Delimiter::Brace => "{",
+                        Delimiter::Paren => "(",
+                        Delimiter::Square => "[",
+                    }
+                )?;
+
+                for child in d.children() {
+                    write!(f, "{:?}", child.debug(self.source))?;
+                }
+
+                write!(
+                    f,
+                    "{}",
+                    match d.delimiter {
+                        Delimiter::Brace => "}",
+                        Delimiter::Paren => ")",
+                        Delimiter::Square => "]",
+                    }
+                )
+            }
+            TokenNode::Pipeline(s) => write!(f, "<todo:pipeline>"),
+            TokenNode::Error(s) => write!(f, "<error> for {:?}", s.span().slice(self.source)),
+            rest => write!(f, "{}", rest.span().slice(self.source)),
+        }
+    }
+}
+
+impl From<&TokenNode> for Span {
+    fn from(token: &TokenNode) -> Span {
+        token.span()
+    }
+}
+
 impl TokenNode {
     pub fn span(&self) -> Span {
         match self {
@@ -36,6 +99,10 @@ impl TokenNode {
         }
     }
 
+    pub fn debug(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
+        DebugTokenNode { node: self, source }
+    }
+
     pub fn as_external_arg(&self, source: &Text) -> String {
         self.span().slice(source).to_string()
     }
@@ -73,7 +140,8 @@ impl TokenNode {
     }
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+#[get = "crate"]
 pub struct DelimitedNode {
     delimiter: Delimiter,
     children: Vec<TokenNode>,
@@ -86,7 +154,8 @@ pub enum Delimiter {
     Square,
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+#[get = "crate"]
 pub struct PathNode {
     head: Box<TokenNode>,
     tail: Vec<TokenNode>,

src/parser/parse/tokens.rs

@@ -1,5 +1,7 @@
 use crate::parser::parse::span::*;
 use crate::parser::parse::unit::*;
+use crate::Text;
+use std::fmt;
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub enum RawToken {
@@ -10,4 +12,36 @@ pub enum RawToken {
     Bare,
 }
 
+impl RawToken {
+    pub fn type_name(&self) -> &'static str {
+        match self {
+            RawToken::Integer(_) => "Integer",
+            RawToken::Size(..) => "Size",
+            RawToken::String(_) => "String",
+            RawToken::Variable(_) => "Variable",
+            RawToken::Bare => "String",
+        }
+    }
+}
+
 pub type Token = Spanned<RawToken>;
+
+impl Token {
+    pub fn debug(&self, source: &'a Text) -> DebugToken<'a> {
+        DebugToken {
+            node: *self,
+            source,
+        }
+    }
+}
+
+pub struct DebugToken<'a> {
+    node: Token,
+    source: &'a Text,
+}
+
+impl fmt::Debug for DebugToken<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.node.span().slice(self.source))
+    }
+}

src/parser/parse_command.rs

@@ -67,77 +67,81 @@ fn parse_command_tail(
     tail: Option<Vec<TokenNode>>,
     source: &Text,
 ) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
-    let mut tail = match tail {
+    let tail = &mut match &tail {
         None => return Ok(None),
-        Some(tail) => tail,
+        Some(tail) => hir::TokensIterator::new(tail),
     };
 
     let mut named = NamedArguments::new();
 
+    trace_remaining("nodes", tail.clone(), source);
+
     for (name, kind) in config.named() {
         trace!("looking for {} : {:?}", name, kind);
 
         match kind {
             NamedType::Switch => {
-                let (rest, flag) = extract_switch(name, tail, source);
-
-                tail = rest;
+                let flag = extract_switch(name, tail, source);
 
                 named.insert_switch(name, flag);
             }
             NamedType::Mandatory(kind) => match extract_mandatory(name, tail, source) {
                 Err(err) => return Err(err), // produce a correct diagnostic
-                Ok((rest, pos, _flag)) => {
-                    let (expr, rest) = hir::baseline_parse_next_expr(
-                        &rest[pos..],
+                Ok((pos, _flag)) => {
+                    tail.move_to(pos);
+                    let expr = hir::baseline_parse_next_expr(
+                        tail,
                         registry,
                         source,
                         kind.to_coerce_hint(),
                     )?;
-                    tail = rest.to_vec();
 
+                    tail.restart();
                     named.insert_mandatory(name, expr);
                 }
             },
             NamedType::Optional(kind) => match extract_optional(name, tail, source) {
                 Err(err) => return Err(err), // produce a correct diagnostic
-                Ok((rest, Some((pos, _flag)))) => {
-                    let (expr, rest) = hir::baseline_parse_next_expr(
-                        &rest[pos..],
+                Ok(Some((pos, _flag))) => {
+                    tail.move_to(pos);
+                    let expr = hir::baseline_parse_next_expr(
+                        tail,
                         registry,
                         source,
                         kind.to_coerce_hint(),
                     )?;
-                    tail = rest.to_vec();
 
+                    tail.restart();
                     named.insert_optional(name, Some(expr));
                 }
 
-                Ok((rest, None)) => {
-                    tail = rest;
-
+                Ok(None) => {
+                    tail.restart();
                     named.insert_optional(name, None);
                 }
             },
         };
     }
 
+    trace_remaining("after named", tail.clone(), source);
+
     let mut positional = vec![];
     let mandatory = config.mandatory_positional();
 
     for arg in mandatory {
+        trace!("Processing mandatory {:?}", arg);
+
         if tail.len() == 0 {
             return Err(ShellError::unimplemented("Missing mandatory argument"));
         }
 
-        let (result, rest) =
-            hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?;
+        let result = hir::baseline_parse_next_expr(tail, registry, source, arg.to_coerce_hint())?;
 
         positional.push(result);
-
-        tail = rest.to_vec();
     }
 
+    trace_remaining("after mandatory", tail.clone(), source);
+
     let optional = config.optional_positional();
 
     for arg in optional {
@@ -145,18 +149,19 @@ fn parse_command_tail(
             break;
         }
 
-        let (result, rest) =
-            hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?;
+        let result = hir::baseline_parse_next_expr(tail, registry, source, arg.to_coerce_hint())?;
 
         positional.push(result);
-
-        tail = rest.to_vec();
     }
 
+    trace_remaining("after optional", tail.clone(), source);
+
     // TODO: Only do this if rest params are specified
-    let remainder = baseline_parse_tokens(&tail, registry, source)?;
+    let remainder = baseline_parse_tokens(tail, registry, source)?;
     positional.extend(remainder);
 
+    trace_remaining("after rest", tail.clone(), source);
+
     trace!("Constructed positional={:?} named={:?}", positional, named);
 
     let positional = match positional {
@@ -174,38 +179,20 @@ fn parse_command_tail(
     Ok(Some((positional, named)))
 }
 
-fn extract_switch(
-    name: &str,
-    mut tokens: Vec<TokenNode>,
-    source: &Text,
-) -> (Vec<TokenNode>, Option<Flag>) {
-    let pos = tokens
-        .iter()
-        .enumerate()
-        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
-        .nth(0);
-
-    match pos {
-        None => (tokens, None),
-        Some((pos, flag)) => {
-            tokens.remove(pos);
-            (tokens, Some(*flag))
-        }
-    }
+fn extract_switch(name: &str, tokens: &mut hir::TokensIterator<'_>, source: &Text) -> Option<Flag> {
+    tokens
+        .extract(|t| t.as_flag(name, source))
+        .map(|(_pos, flag)| flag.item)
 }
 
 fn extract_mandatory(
     name: &str,
-    mut tokens: Vec<TokenNode>,
+    tokens: &mut hir::TokensIterator<'a>,
     source: &Text,
-) -> Result<(Vec<TokenNode>, usize, Flag), ShellError> {
-    let pos = tokens
-        .iter()
-        .enumerate()
-        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
-        .nth(0);
+) -> Result<(usize, Flag), ShellError> {
+    let flag = tokens.extract(|t| t.as_flag(name, source));
 
-    match pos {
+    match flag {
         None => Err(ShellError::unimplemented(
             "Better error: mandatory flags must be present",
         )),
@@ -218,24 +205,20 @@ fn extract_mandatory(
 
             tokens.remove(pos);
 
-            Ok((tokens, pos, *flag))
+            Ok((pos, *flag))
         }
     }
 }
 
 fn extract_optional(
     name: &str,
-    mut tokens: Vec<TokenNode>,
+    tokens: &mut hir::TokensIterator<'a>,
     source: &Text,
-) -> Result<(Vec<TokenNode>, Option<(usize, Flag)>), ShellError> {
-    let pos = tokens
-        .iter()
-        .enumerate()
-        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
-        .nth(0);
+) -> Result<(Option<(usize, Flag)>), ShellError> {
+    let flag = tokens.extract(|t| t.as_flag(name, source));
 
-    match pos {
-        None => Ok((tokens, None)),
+    match flag {
+        None => Ok(None),
         Some((pos, flag)) => {
             if tokens.len() <= pos {
                 return Err(ShellError::unimplemented(
@@ -245,7 +228,20 @@ fn extract_optional(
 
             tokens.remove(pos);
 
-            Ok((tokens, Some((pos, *flag))))
+            Ok(Some((pos, *flag)))
         }
     }
 }
+
+pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'a>, source: &Text) {
+    trace!(
+        "{} = {:?}",
+        desc,
+        itertools::join(
+            tail.debug_remaining()
+                .iter()
+                .map(|i| format!("%{:?}%", i.debug(source))),
+            " "
+        )
+    );
+}