From 3379c23a49d125aa3825554f3d6ad88ef03678af Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sat, 29 Jun 2019 01:55:42 -0700 Subject: [PATCH] Support evaluating most expressions Blocks, paths, and others Plus a bunch of other infra improvements --- Cargo.toml | 2 - src/commands/plugin.rs | 31 ++-- src/errors.rs | 66 +++++-- src/evaluate/evaluator.rs | 4 +- src/lib.rs | 2 + src/object/base.rs | 45 +++-- src/parser.rs | 4 +- src/parser/hir.rs | 19 +- src/parser/hir/baseline_parse_tokens.rs | 233 ++++++++++++++++++++---- src/parser/parse/span.rs | 14 ++ src/parser/parse/token_tree.rs | 73 +++++++- src/parser/parse/tokens.rs | 34 ++++ src/parser/parse_command.rs | 120 ++++++------ 13 files changed, 497 insertions(+), 150 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 56f6eee90f..8b11d47987 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,3 @@ -cargo-features = ["default-run"] - [package] name = "nu" version = "0.1.2" diff --git a/src/commands/plugin.rs b/src/commands/plugin.rs index b3b85a2bfd..8d7b2c5d84 100644 --- a/src/commands/plugin.rs +++ b/src/commands/plugin.rs @@ -1,12 +1,9 @@ use crate::errors::ShellError; -use crate::parser::registry::{CommandConfig, PositionalType}; -use crate::parser::Spanned; use crate::prelude::*; use serde::{self, Deserialize, Serialize}; use std::io::prelude::*; use std::io::BufReader; -use std::io::{Read, Write}; -use subprocess::Exec; +use std::io::Write; #[derive(Debug, Serialize, Deserialize)] pub struct JsonRpc { @@ -14,6 +11,7 @@ pub struct JsonRpc { pub method: String, pub params: T, } + impl JsonRpc { pub fn new>(method: U, params: T) -> Self { JsonRpc { @@ -50,15 +48,16 @@ pub fn plugin(plugin_name: String, args: CommandArgs) -> Result Result { - let mut stdin = child.stdin.as_mut().expect("Failed to open stdin"); - let mut stdout = child.stdout.as_mut().expect("Failed to open stdout"); + let stdin = child.stdin.as_mut().expect("Failed to open stdin"); + let stdout = child.stdout.as_mut().expect("Failed to open stdout"); - let mut reader = BufReader::new(stdout); + let _ = BufReader::new(stdout); let request: JsonRpc> = JsonRpc::new("quit", vec![]); let request_raw = serde_json::to_string(&request).unwrap(); - stdin.write(format!("{}\n", request_raw).as_bytes()); + let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error VecDeque::new() } _ => { - let mut stdin = child.stdin.as_mut().expect("Failed to open stdin"); - let mut stdout = child.stdout.as_mut().expect("Failed to open stdout"); + let stdin = child.stdin.as_mut().expect("Failed to open stdin"); + let stdout = child.stdout.as_mut().expect("Failed to open stdout"); let mut reader = BufReader::new(stdout); let request = JsonRpc::new("filter", v); let request_raw = serde_json::to_string(&request).unwrap(); - stdin.write(format!("{}\n", request_raw).as_bytes()); + let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error let mut input = String::new(); match reader.read_line(&mut input) { @@ -101,7 +100,7 @@ pub fn plugin(plugin_name: String, args: CommandArgs) -> Result { + Err(_) => { let mut result = VecDeque::new(); result.push_back(ReturnValue::Value(Value::Error(Box::new( ShellError::string("Error while processing input"), diff --git a/src/errors.rs b/src/errors.rs index 8472e82dcc..72630b9f53 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -36,7 +36,10 @@ impl Description { #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] pub enum ShellError { String(StringError), - TypeError(Spanned), + TypeError { + expected: String, + actual: Spanned>, + }, MissingProperty { subpath: Description, expr: Description, @@ -49,6 +52,16 @@ pub enum ShellError { } impl ShellError { + crate fn type_error( + expected: impl Into, + actual: Spanned>, + ) -> ShellError { + ShellError::TypeError { + expected: expected.into(), + actual: actual.map(|i| Some(i.into())), + } + } + crate fn parse_error( error: nom::Err<(nom_locate::LocatedSpan<&str>, nom::error::ErrorKind)>, ) -> ShellError { @@ -57,9 +70,8 @@ impl ShellError { match error { nom::Err::Incomplete(_) => unreachable!(), nom::Err::Failure(span) | nom::Err::Error(span) => { - let diagnostic = - Diagnostic::new(Severity::Error, format!("Parse Error")) - .with_label(Label::new_primary(Span::from(span.0))); + let diagnostic = Diagnostic::new(Severity::Error, format!("Parse Error")) + .with_label(Label::new_primary(Span::from(span.0))); ShellError::diagnostic(diagnostic) // nom::Context::Code(span, kind) => { @@ -69,21 +81,20 @@ impl ShellError { // ShellError::diagnostic(diagnostic) // } - } - // ParseError::UnrecognizedToken { - // token: (start, SpannedToken { token, .. }, end), - // expected, - // } => { - // let diagnostic = Diagnostic::new( - // Severity::Error, - // format!("Unexpected {:?}, expected {:?}", token, expected), - // ) - // .with_label(Label::new_primary(Span::from((start, end)))); + } // ParseError::UnrecognizedToken { + // token: (start, SpannedToken { token, .. }, end), + // expected, + // } => { + // let diagnostic = Diagnostic::new( + // Severity::Error, + // format!("Unexpected {:?}, expected {:?}", token, expected), + // ) + // .with_label(Label::new_primary(Span::from((start, end)))); - // ShellError::diagnostic(diagnostic) - // } - // ParseError::User { error } => error, - // other => ShellError::string(format!("{:?}", other)), + // ShellError::diagnostic(diagnostic) + // } + // ParseError::User { error } => error, + // other => ShellError::string(format!("{:?}", other)), } } @@ -96,8 +107,23 @@ impl ShellError { ShellError::String(StringError { title, .. }) => { Diagnostic::new(Severity::Error, title) } - ShellError::TypeError(s) => Diagnostic::new(Severity::Error, "Type Error") - .with_label(Label::new_primary(s.span).with_message(s.item)), + ShellError::TypeError { + expected, + actual: + Spanned { + item: Some(actual), + span, + }, + } => Diagnostic::new(Severity::Error, "Type Error").with_label( + Label::new_primary(span) + .with_message(format!("Expected {}, found {}", expected, actual)), + ), + + ShellError::TypeError { + expected, + actual: Spanned { item: None, span }, + } => Diagnostic::new(Severity::Error, "Type Error") + .with_label(Label::new_primary(span).with_message(expected)), ShellError::MissingProperty { subpath, expr } => { let subpath = subpath.into_label(); diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index 4642ede45c..38f9012769 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -46,8 +46,8 @@ crate fn evaluate_baseline_expr( } } RawExpression::Block(block) => Ok(Spanned::from_item( - Value::Block(Block::new(*block.clone(), source.clone())), - block.span(), + Value::Block(Block::new(block.clone(), source.clone(), *expr.span())), + expr.span(), )), RawExpression::Path(path) => { let value = evaluate_baseline_expr(path.head(), registry, scope, source)?; diff --git a/src/lib.rs b/src/lib.rs index caf1dd8e59..1f801a4355 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ #![feature(async_await)] #![feature(try_trait)] #![feature(bind_by_move_pattern_guards)] +#![feature(box_syntax)] mod cli; mod commands; @@ -19,6 +20,7 @@ mod shell; mod stream; pub use crate::commands::command::ReturnValue; +pub use crate::parser::parse::span::SpannedItem; pub use crate::parser::Spanned; pub use cli::cli; pub use errors::ShellError; diff --git a/src/object/base.rs b/src/object/base.rs index b07c431611..acfc815067 100644 --- a/src/object/base.rs +++ b/src/object/base.rs @@ -1,7 +1,7 @@ use crate::errors::ShellError; use crate::evaluate::{evaluate_baseline_expr, Scope}; use crate::object::DataDescriptor; -use crate::parser::{hir, Operator, Spanned}; +use crate::parser::{hir, Operator, Span, Spanned}; use crate::prelude::*; use crate::Text; use ansi_term::Color; @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; use derive_new::new; use ordered_float::OrderedFloat; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer}; use std::fmt; use std::time::SystemTime; @@ -137,8 +137,9 @@ pub struct Operation { #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)] pub struct Block { - crate expression: hir::Expression, + crate expressions: Vec, crate source: Text, + crate span: Span, } impl Serialize for Block { @@ -146,7 +147,18 @@ impl Serialize for Block { where S: Serializer, { - serializer.serialize_str(&self.expression.source(&self.source.clone())) + let mut seq = serializer.serialize_seq(None)?; + + let list = self + .expressions + .iter() + .map(|e| e.source(&self.source.clone())); + + for item in list { + seq.serialize_element(item.as_ref())?; + } + + seq.end() } } @@ -167,7 +179,18 @@ impl Deserialize<'de> for Block { impl Block { pub fn invoke(&self, value: &Value) -> Result, ShellError> { let scope = Scope::new(value.copy()); - evaluate_baseline_expr(&self.expression, &(), &scope, &self.source) + + if self.expressions.len() == 0 { + return Ok(Spanned::from_item(Value::nothing(), self.span)); + } + + let mut last = None; + + for expr in self.expressions.iter() { + last = Some(evaluate_baseline_expr(&expr, &(), &scope, &self.source)?) + } + + Ok(last.unwrap()) } } @@ -305,7 +328,12 @@ impl Value { crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String { match self { Value::Primitive(p) => p.format(desc), - Value::Block(b) => b.expression.source(&b.source).to_string(), + Value::Block(b) => itertools::join( + b.expressions + .iter() + .map(|e| e.source(&b.source).to_string()), + "; ", + ), Value::Object(_) => format!("[object Object]"), Value::List(_) => format!("[list List]"), Value::Error(e) => format!("{}", e), @@ -403,11 +431,6 @@ impl Value { } } - #[allow(unused)] - pub fn block(e: hir::Expression, source: Text) -> Value { - Value::Block(Block::new(e, source)) - } - pub fn string(s: impl Into) -> Value { Value::Primitive(Primitive::String(s.into())) } diff --git a/src/parser.rs b/src/parser.rs index db584f4327..327d7cae9b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -5,7 +5,7 @@ crate mod registry; use crate::errors::ShellError; -crate use hir::baseline_parse_tokens::baseline_parse_tokens; +crate use hir::baseline_parse_tokens::{baseline_parse_tokens, trace_remaining}; crate use parse::call_node::CallNode; crate use parse::files::Files; crate use parse::flag::Flag; @@ -14,7 +14,7 @@ crate use parse::parser::{nom_input, pipeline}; crate use parse::pipeline::{Pipeline, PipelineElement}; pub use parse::span::{Span, Spanned}; crate use parse::text::Text; -crate use parse::token_tree::TokenNode; +crate use parse::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; crate use parse::tokens::{RawToken, Token}; crate use parse::unit::Unit; crate use parse_command::parse_command; diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 7ab79a8d93..16b79cf117 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -9,11 +9,20 @@ use derive_new::new; use getset::Getters; crate use baseline_parse::baseline_parse_single_token; -crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint}; +crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint, TokensIterator}; crate use binary::Binary; crate use named::NamedArguments; crate use path::Path; +pub fn path(head: impl Into, tail: Vec>>) -> Path { + Path::new( + head.into(), + tail.into_iter() + .map(|item| item.map(|string| string.into())) + .collect(), + ) +} + #[derive(Debug, Clone, Eq, PartialEq, Getters, new)] pub struct Call { #[get = "crate"] @@ -29,7 +38,7 @@ pub enum RawExpression { Literal(Literal), Variable(Variable), Binary(Box), - Block(Box), + Block(Vec), Path(Box), #[allow(unused)] @@ -76,6 +85,12 @@ impl Expression { } } +impl From> for Expression { + fn from(path: Spanned) -> Expression { + path.map(|p| RawExpression::Path(Box::new(p))) + } +} + #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum Literal { Integer(i64), diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index e9c094388b..89e2e32e95 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -1,24 +1,27 @@ use crate::errors::ShellError; use crate::parser::registry::CommandRegistry; -use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode}; -use crate::Text; +use crate::parser::{ + hir, hir::baseline_parse_single_token, DelimitedNode, Delimiter, PathNode, RawToken, Span, + Spanned, TokenNode, +}; +use crate::{SpannedItem, Text}; +use derive_new::new; +use log::trace; pub fn baseline_parse_tokens( - token_nodes: &[TokenNode], + token_nodes: &mut TokensIterator<'_>, registry: &dyn CommandRegistry, source: &Text, ) -> Result, ShellError> { let mut exprs: Vec = vec![]; - let mut rest = token_nodes; loop { - if rest.len() == 0 { + if token_nodes.at_end() { break; } - let (expr, remainder) = baseline_parse_next_expr(rest, registry, source, None)?; + let expr = baseline_parse_next_expr(token_nodes, registry, source, None)?; exprs.push(expr); - rest = remainder; } Ok(exprs) @@ -35,25 +38,21 @@ pub enum ExpressionKindHint { } pub fn baseline_parse_next_expr( - token_nodes: &'nodes [TokenNode], - _registry: &dyn CommandRegistry, + tokens: &mut TokensIterator, + registry: &dyn CommandRegistry, source: &Text, coerce_hint: Option, -) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> { - let mut tokens = token_nodes.iter().peekable(); - - let first = next_token(&mut tokens); - - let first = match first { +) -> Result { + let first = match tokens.next() { None => return Err(ShellError::string("Expected token, found none")), - Some(token) => baseline_parse_semantic_token(token, source)?, + Some(token) => baseline_parse_semantic_token(token, registry, source)?, }; let possible_op = tokens.peek(); let op = match possible_op { - Some(TokenNode::Operator(op)) => op, - _ => return Ok((first, &token_nodes[1..])), + Some(TokenNode::Operator(op)) => op.clone(), + _ => return Ok(first), }; tokens.next(); @@ -64,7 +63,7 @@ pub fn baseline_parse_next_expr( "Expected op followed by another expr, found nothing", )) } - Some(token) => baseline_parse_semantic_token(token, source)?, + Some(token) => baseline_parse_semantic_token(token, registry, source)?, }; // We definitely have a binary expression here -- let's see if we should coerce it into a block @@ -72,11 +71,11 @@ pub fn baseline_parse_next_expr( match coerce_hint { None => { let span = (first.span.start, second.span.end); - let binary = hir::Binary::new(first, *op, second); + let binary = hir::Binary::new(first, op, second); let binary = hir::RawExpression::Binary(Box::new(binary)); let binary = Spanned::from_item(binary, span); - Ok((binary, &token_nodes[3..])) + Ok(binary) } Some(hint) => match hint { @@ -133,14 +132,14 @@ pub fn baseline_parse_next_expr( } }; - let binary = hir::Binary::new(path, *op, second); + let binary = hir::Binary::new(path, op, second); let binary = hir::RawExpression::Binary(Box::new(binary)); let binary = Spanned::from_item(binary, span); - let block = hir::RawExpression::Block(Box::new(binary)); + let block = hir::RawExpression::Block(vec![binary]); let block = Spanned::from_item(block, span); - Ok((block, &token_nodes[3..])) + Ok(block) } other => unimplemented!("coerce hint {:?}", other), @@ -150,27 +149,199 @@ pub fn baseline_parse_next_expr( pub fn baseline_parse_semantic_token( token: &TokenNode, + registry: &dyn CommandRegistry, source: &Text, ) -> Result { match token { TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), TokenNode::Call(_call) => unimplemented!(), - TokenNode::Delimited(_delimited) => unimplemented!(), + TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, registry, source), TokenNode::Pipeline(_pipeline) => unimplemented!(), TokenNode::Operator(_op) => unreachable!(), TokenNode::Flag(_flag) => unimplemented!(), TokenNode::Identifier(_span) => unreachable!(), TokenNode::Whitespace(_span) => unreachable!(), TokenNode::Error(error) => Err(*error.item.clone()), - TokenNode::Path(_path) => unimplemented!(), + TokenNode::Path(path) => baseline_parse_path(path, registry, source), } } -fn next_token(nodes: &mut impl Iterator) -> Option<&'a TokenNode> { - loop { - match nodes.next() { - Some(TokenNode::Whitespace(_)) => continue, - other => return other, +pub fn baseline_parse_delimited( + token: &Spanned, + registry: &dyn CommandRegistry, + source: &Text, +) -> Result { + match token.delimiter() { + Delimiter::Brace => { + let children = token.children(); + let exprs = + baseline_parse_tokens(&mut TokensIterator::new(children), registry, source)?; + + let expr = hir::RawExpression::Block(exprs); + Ok(Spanned::from_item(expr, token.span())) + } + Delimiter::Paren => unimplemented!(), + Delimiter::Square => unimplemented!(), + } +} + +pub fn baseline_parse_path( + token: &Spanned, + registry: &dyn CommandRegistry, + source: &Text, +) -> Result { + let head = baseline_parse_semantic_token(token.head(), registry, source)?; + + let mut tail = vec![]; + + for part in token.tail() { + let string = match part { + TokenNode::Token(token) => match token.item() { + RawToken::Bare => token.span().slice(source), + RawToken::String(span) => span.slice(source), + RawToken::Integer(_) | RawToken::Size(..) | RawToken::Variable(_) => { + return Err(ShellError::type_error( + "String", + token.type_name().spanned(part), + )) + } + }, + + TokenNode::Identifier(span) => span.slice(source), + + // TODO: Make this impossible + other => unreachable!("{:?}", other), + } + .to_string(); + + tail.push(string.spanned(part)); + } + + Ok(hir::path(head, tail).spanned(token).into()) +} + +#[derive(Debug, new)] +pub struct TokensIterator<'a> { + tokens: &'a [TokenNode], + #[new(default)] + index: usize, + #[new(default)] + seen: indexmap::IndexSet, +} + +impl TokensIterator<'a> { + pub fn remove(&mut self, position: usize) { + self.seen.insert(position); + } + + pub fn len(&self) -> usize { + self.tokens.len() + } + + pub fn at_end(&self) -> bool { + for index in self.index..self.tokens.len() { + if !self.seen.contains(&index) { + return false; + } + } + + true + } + + pub fn advance(&mut self) { + self.seen.insert(self.index); + self.index += 1; + } + + pub fn extract(&mut self, f: impl Fn(&TokenNode) -> Option) -> Option<(usize, T)> { + for (i, item) in self.tokens.iter().enumerate() { + if self.seen.contains(&i) { + continue; + } + + match f(item) { + None => { + continue; + } + Some(value) => { + self.seen.insert(i); + return Some((i, value)); + } + } + } + + None + } + + pub fn move_to(&mut self, pos: usize) { + self.index = pos; + } + + pub fn restart(&mut self) { + self.index = 0; + } + + pub fn clone(&self) -> TokensIterator { + TokensIterator { + tokens: self.tokens, + index: self.index, + seen: self.seen.clone(), + } + } + + pub fn peek(&self) -> Option<&TokenNode> { + let mut tokens = self.clone(); + + tokens.next() + } + + pub fn debug_remaining(&self) -> Vec { + let mut tokens = self.clone(); + tokens.restart(); + tokens.cloned().collect() + } +} + +impl Iterator for TokensIterator<'a> { + type Item = &'a TokenNode; + + fn next(&mut self) -> Option<&'a TokenNode> { + loop { + if self.index >= self.tokens.len() { + return None; + } + + if self.seen.contains(&self.index) { + self.advance(); + continue; + } + + if self.index >= self.tokens.len() { + return None; + } + + match &self.tokens[self.index] { + TokenNode::Whitespace(_) => { + self.advance(); + } + other => { + self.advance(); + return Some(other); + } + } } } } + +pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'a>, source: &Text) { + trace!( + "{} = {:?}", + desc, + itertools::join( + tail.debug_remaining() + .iter() + .map(|i| format!("%{:?}%", i.debug(source))), + " " + ) + ); +} diff --git a/src/parser/parse/span.rs b/src/parser/parse/span.rs index 1777ff7424..4d3f38659a 100644 --- a/src/parser/parse/span.rs +++ b/src/parser/parse/span.rs @@ -12,6 +12,14 @@ pub struct Spanned { pub item: T, } +pub trait SpannedItem: Sized { + fn spanned(self, span: impl Into) -> Spanned { + Spanned::from_item(self, span.into()) + } +} + +impl SpannedItem for T {} + impl std::ops::Deref for Spanned { type Target = T; @@ -56,6 +64,12 @@ pub struct Span { // source: &'source str, } +impl From<&Spanned> for Span { + fn from(input: &Spanned) -> Span { + input.span + } +} + impl From<&Span> for Span { fn from(input: &Span) -> Span { *input diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index 4cfd73b9c9..8684880878 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -3,6 +3,8 @@ use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, span use crate::Text; use derive_new::new; use enum_utils::FromStr; +use getset::Getters; +use std::fmt; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { @@ -20,6 +22,67 @@ pub enum TokenNode { Path(Spanned), } +pub struct DebugTokenNode<'a> { + node: &'a TokenNode, + source: &'a Text, +} + +impl fmt::Debug for DebugTokenNode<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.node { + TokenNode::Token(t) => write!(f, "{:?}", t.debug(self.source)), + TokenNode::Call(s) => { + write!(f, "(")?; + + write!(f, "{:?}", s.head().debug(self.source))?; + + if let Some(children) = s.children() { + for child in children { + write!(f, "{:?}", child.debug(self.source))?; + } + } + + write!(f, ")") + } + + TokenNode::Delimited(d) => { + write!( + f, + "{}", + match d.delimiter { + Delimiter::Brace => "{", + Delimiter::Paren => "(", + Delimiter::Square => "[", + } + )?; + + for child in d.children() { + write!(f, "{:?}", child.debug(self.source))?; + } + + write!( + f, + "{}", + match d.delimiter { + Delimiter::Brace => "}", + Delimiter::Paren => ")", + Delimiter::Square => "]", + } + ) + } + TokenNode::Pipeline(s) => write!(f, ""), + TokenNode::Error(s) => write!(f, " for {:?}", s.span().slice(self.source)), + rest => write!(f, "{}", rest.span().slice(self.source)), + } + } +} + +impl From<&TokenNode> for Span { + fn from(token: &TokenNode) -> Span { + token.span() + } +} + impl TokenNode { pub fn span(&self) -> Span { match self { @@ -36,6 +99,10 @@ impl TokenNode { } } + pub fn debug(&'a self, source: &'a Text) -> DebugTokenNode<'a> { + DebugTokenNode { node: self, source } + } + pub fn as_external_arg(&self, source: &Text) -> String { self.span().slice(source).to_string() } @@ -73,7 +140,8 @@ impl TokenNode { } } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] +#[get = "crate"] pub struct DelimitedNode { delimiter: Delimiter, children: Vec, @@ -86,7 +154,8 @@ pub enum Delimiter { Square, } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] +#[get = "crate"] pub struct PathNode { head: Box, tail: Vec, diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index 5e3f942cbd..e9528c1506 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -1,5 +1,7 @@ use crate::parser::parse::span::*; use crate::parser::parse::unit::*; +use crate::Text; +use std::fmt; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum RawToken { @@ -10,4 +12,36 @@ pub enum RawToken { Bare, } +impl RawToken { + pub fn type_name(&self) -> &'static str { + match self { + RawToken::Integer(_) => "Integer", + RawToken::Size(..) => "Size", + RawToken::String(_) => "String", + RawToken::Variable(_) => "Variable", + RawToken::Bare => "String", + } + } +} + pub type Token = Spanned; + +impl Token { + pub fn debug(&self, source: &'a Text) -> DebugToken<'a> { + DebugToken { + node: *self, + source, + } + } +} + +pub struct DebugToken<'a> { + node: Token, + source: &'a Text, +} + +impl fmt::Debug for DebugToken<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.node.span().slice(self.source)) + } +} diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index c9c49b1b84..8e785e628e 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -67,77 +67,81 @@ fn parse_command_tail( tail: Option>, source: &Text, ) -> Result>, Option)>, ShellError> { - let mut tail = match tail { + let tail = &mut match &tail { None => return Ok(None), - Some(tail) => tail, + Some(tail) => hir::TokensIterator::new(tail), }; let mut named = NamedArguments::new(); + trace_remaining("nodes", tail.clone(), source); + for (name, kind) in config.named() { trace!("looking for {} : {:?}", name, kind); match kind { NamedType::Switch => { - let (rest, flag) = extract_switch(name, tail, source); - - tail = rest; + let flag = extract_switch(name, tail, source); named.insert_switch(name, flag); } NamedType::Mandatory(kind) => match extract_mandatory(name, tail, source) { Err(err) => return Err(err), // produce a correct diagnostic - Ok((rest, pos, _flag)) => { - let (expr, rest) = hir::baseline_parse_next_expr( - &rest[pos..], + Ok((pos, _flag)) => { + tail.move_to(pos); + let expr = hir::baseline_parse_next_expr( + tail, registry, source, kind.to_coerce_hint(), )?; - tail = rest.to_vec(); + tail.restart(); named.insert_mandatory(name, expr); } }, NamedType::Optional(kind) => match extract_optional(name, tail, source) { Err(err) => return Err(err), // produce a correct diagnostic - Ok((rest, Some((pos, _flag)))) => { - let (expr, rest) = hir::baseline_parse_next_expr( - &rest[pos..], + Ok(Some((pos, _flag))) => { + tail.move_to(pos); + let expr = hir::baseline_parse_next_expr( + tail, registry, source, kind.to_coerce_hint(), )?; - tail = rest.to_vec(); + tail.restart(); named.insert_optional(name, Some(expr)); } - Ok((rest, None)) => { - tail = rest; - + Ok(None) => { + tail.restart(); named.insert_optional(name, None); } }, }; } + trace_remaining("after named", tail.clone(), source); + let mut positional = vec![]; let mandatory = config.mandatory_positional(); for arg in mandatory { + trace!("Processing mandatory {:?}", arg); + if tail.len() == 0 { return Err(ShellError::unimplemented("Missing mandatory argument")); } - let (result, rest) = - hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?; + let result = hir::baseline_parse_next_expr(tail, registry, source, arg.to_coerce_hint())?; positional.push(result); - - tail = rest.to_vec(); } + trace_remaining("after mandatory", tail.clone(), source); + let optional = config.optional_positional(); for arg in optional { @@ -145,18 +149,19 @@ fn parse_command_tail( break; } - let (result, rest) = - hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?; + let result = hir::baseline_parse_next_expr(tail, registry, source, arg.to_coerce_hint())?; positional.push(result); - - tail = rest.to_vec(); } + trace_remaining("after optional", tail.clone(), source); + // TODO: Only do this if rest params are specified - let remainder = baseline_parse_tokens(&tail, registry, source)?; + let remainder = baseline_parse_tokens(tail, registry, source)?; positional.extend(remainder); + trace_remaining("after rest", tail.clone(), source); + trace!("Constructed positional={:?} named={:?}", positional, named); let positional = match positional { @@ -174,38 +179,20 @@ fn parse_command_tail( Ok(Some((positional, named))) } -fn extract_switch( - name: &str, - mut tokens: Vec, - source: &Text, -) -> (Vec, Option) { - let pos = tokens - .iter() - .enumerate() - .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) - .nth(0); - - match pos { - None => (tokens, None), - Some((pos, flag)) => { - tokens.remove(pos); - (tokens, Some(*flag)) - } - } +fn extract_switch(name: &str, tokens: &mut hir::TokensIterator<'_>, source: &Text) -> Option { + tokens + .extract(|t| t.as_flag(name, source)) + .map(|(_pos, flag)| flag.item) } fn extract_mandatory( name: &str, - mut tokens: Vec, + tokens: &mut hir::TokensIterator<'a>, source: &Text, -) -> Result<(Vec, usize, Flag), ShellError> { - let pos = tokens - .iter() - .enumerate() - .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) - .nth(0); +) -> Result<(usize, Flag), ShellError> { + let flag = tokens.extract(|t| t.as_flag(name, source)); - match pos { + match flag { None => Err(ShellError::unimplemented( "Better error: mandatory flags must be present", )), @@ -218,24 +205,20 @@ fn extract_mandatory( tokens.remove(pos); - Ok((tokens, pos, *flag)) + Ok((pos, *flag)) } } } fn extract_optional( name: &str, - mut tokens: Vec, + tokens: &mut hir::TokensIterator<'a>, source: &Text, -) -> Result<(Vec, Option<(usize, Flag)>), ShellError> { - let pos = tokens - .iter() - .enumerate() - .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) - .nth(0); +) -> Result<(Option<(usize, Flag)>), ShellError> { + let flag = tokens.extract(|t| t.as_flag(name, source)); - match pos { - None => Ok((tokens, None)), + match flag { + None => Ok(None), Some((pos, flag)) => { if tokens.len() <= pos { return Err(ShellError::unimplemented( @@ -245,7 +228,20 @@ fn extract_optional( tokens.remove(pos); - Ok((tokens, Some((pos, *flag)))) + Ok(Some((pos, *flag))) } } } + +pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'a>, source: &Text) { + trace!( + "{} = {:?}", + desc, + itertools::join( + tail.debug_remaining() + .iter() + .map(|i| format!("%{:?}%", i.debug(source))), + " " + ) + ); +}