diff --git a/.azure/azure-pipelines.yml b/.azure/azure-pipelines.yml index 70ff5e4af0..a963afc2ec 100644 --- a/.azure/azure-pipelines.yml +++ b/.azure/azure-pipelines.yml @@ -42,10 +42,10 @@ steps: echo "##vso[task.prependpath]$HOME/.cargo/bin" rustup component add rustfmt --toolchain "stable" displayName: Install Rust - - bash: RUSTFLAGS="-D warnings" cargo test --all --all-features + - bash: RUSTFLAGS="-D warnings" cargo test --all --features=user-visible condition: eq(variables['style'], 'unflagged') displayName: Run tests - - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --all-features + - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features=user-visible condition: eq(variables['style'], 'canary') displayName: Run tests - bash: cargo fmt --all -- --check diff --git a/.gitpod.yml b/.gitpod.yml index 8bbe78cf88..28a826a94f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,7 +1,7 @@ image: file: .gitpod.Dockerfile tasks: - - init: cargo install nu --all-features + - init: cargo install nu --features=user-visible command: nu github: prebuilds: diff --git a/Cargo.lock b/Cargo.lock index 1504f4612e..772bb51497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -814,6 +814,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enumflags2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33121c8782ba948ba332dab29311b026a8716dc65a1599e5b88f392d38496af8" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "env_logger" version = "0.6.2" @@ -1896,6 +1916,7 @@ dependencies = [ "nom_locate", "nu-build", "nu-errors", + "nu-macros", "nu-parser", "nu-protocol", "nu-source", @@ -1983,6 +2004,13 @@ dependencies = [ "toml 0.5.5", ] +[[package]] +name = "nu-macros" +version = "0.1.0" +dependencies = [ + "nu-protocol", +] + [[package]] name = "nu-parser" version = "0.1.0" @@ -1991,6 +2019,7 @@ dependencies = [ "bigdecimal", "cfg-if", "derive-new", + "enumflags2", "getset", "indexmap", "itertools 0.8.2", diff --git a/Cargo.toml b/Cargo.toml index d49f731735..bdeae804be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ documentation = "https://book.nushell.sh" [workspace] members = [ + "crates/nu-macros", "crates/nu-errors", "crates/nu-source", "crates/nu_plugin_average", @@ -54,6 +55,8 @@ nu_plugin_sum = {version = "0.1.0", path = "./crates/nu_plugin_sum", optional=tr nu_plugin_sys = {version = "0.1.0", path = "./crates/nu_plugin_sys", optional=true} nu_plugin_textview = {version = "0.1.0", path = "./crates/nu_plugin_textview", optional=true} nu_plugin_tree = {version = "0.1.0", path = "./crates/nu_plugin_tree", optional=true} +nu-macros = { version = "0.1.0", path = "./crates/nu-macros" } + query_interface = "0.3.5" typetag = "0.1.4" @@ -133,19 +136,21 @@ semver = {version = "0.9.0", optional = true} [features] default = ["sys", "ps", "textview", "inc", "str"] +user-visible = ["sys", "ps", "starship-prompt", "textview", "binaryview", "match", "tree", "average", "sum"] + sys = ["heim", "battery"] ps = ["heim", "futures-timer"] textview = ["crossterm", "syntect", "onig_sys", "url"] -inc = ["semver"] str = [] +inc = ["semver"] starship-prompt = ["starship"] binaryview = ["nu_plugin_binaryview"] match = ["nu_plugin_match"] tree = ["nu_plugin_tree"] average = ["nu_plugin_average"] sum = ["nu_plugin_sum"] -#trace = ["nu-parser/trace"] +trace = ["nu-parser/trace"] [dependencies.rusqlite] version = "0.20.0" diff --git a/README.md b/README.md index c1ae7a02f8..44264b843b 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ cargo install nu You can also install Nu with all the bells and whistles (be sure to have installed the [dependencies](https://book.nushell.sh/en/installation#dependencies) for your platform): ``` -cargo install nu --all-features +cargo install nu --features=user-visible ``` ## Docker diff --git a/TODO.md b/TODO.md index cc7c1e5ec5..db001731f2 100644 --- a/TODO.md +++ b/TODO.md @@ -46,3 +46,7 @@ Unify dictionary building, probably around a macro sys plugin in own crate textview in own crate + +Combine atomic and atomic_parse in parser + +at_end_possible_ws needs to be comment and separator sensitive diff --git a/crates/nu-errors/src/lib.rs b/crates/nu-errors/src/lib.rs index 1abeae9db4..231a93ba95 100644 --- a/crates/nu-errors/src/lib.rs +++ b/crates/nu-errors/src/lib.rs @@ -16,6 +16,9 @@ use std::ops::Range; pub enum ParseErrorReason { /// The parser encountered an EOF rather than what it was expecting Eof { expected: &'static str, span: Span }, + /// The parser expected to see the end of a token stream (possibly the token + /// stream from inside a delimited token node), but found something else. + ExtraTokens { actual: Spanned }, /// The parser encountered something other than what it was expecting Mismatch { expected: &'static str, @@ -43,6 +46,17 @@ impl ParseError { } } + /// Construct a [ParseErrorReason::ExtraTokens](ParseErrorReason::ExtraTokens) + pub fn extra_tokens(actual: Spanned>) -> ParseError { + let Spanned { span, item } = actual; + + ParseError { + reason: ParseErrorReason::ExtraTokens { + actual: item.into().spanned(span), + }, + } + } + /// Construct a [ParseErrorReason::Mismatch](ParseErrorReason::Mismatch) pub fn mismatch(expected: &'static str, actual: Spanned>) -> ParseError { let Spanned { span, item } = actual; @@ -71,6 +85,9 @@ impl From for ShellError { fn from(error: ParseError) -> ShellError { match error.reason { ParseErrorReason::Eof { expected, span } => ShellError::unexpected_eof(expected, span), + ParseErrorReason::ExtraTokens { actual } => { + ShellError::type_error("nothing", actual.clone()) + } ParseErrorReason::Mismatch { actual, expected } => { ShellError::type_error(expected, actual.clone()) } diff --git a/crates/nu-macros/Cargo.toml b/crates/nu-macros/Cargo.toml new file mode 100644 index 0000000000..cddf28a1c8 --- /dev/null +++ b/crates/nu-macros/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "nu-macros" +version = "0.1.0" +authors = ["Yehuda Katz "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nu-protocol = { path = "../nu-protocol", version = "0.1.0" } \ No newline at end of file diff --git a/crates/nu-macros/src/lib.rs b/crates/nu-macros/src/lib.rs new file mode 100644 index 0000000000..0c323f9aa1 --- /dev/null +++ b/crates/nu-macros/src/lib.rs @@ -0,0 +1,25 @@ +#[macro_export] +macro_rules! signature { + (def $name:tt { + $usage:tt + $( + $positional_name:tt $positional_ty:tt - $positional_desc:tt + )* + }) => {{ + let signature = Signature::new(stringify!($name)).desc($usage); + $( + $crate::positional! { signature, $positional_name $positional_ty - $positional_desc } + )* + signature + }}; +} + +#[macro_export] +macro_rules! positional { + ($ident:tt, $name:tt (optional $shape:tt) - $desc:tt) => { + let $ident = $ident.required(stringify!($name), SyntaxShape::$shape, $desc); + }; + ($ident:tt, $name:tt ($shape:tt)- $desc:tt) => { + let $ident = $ident.optional(stringify!($name), SyntaxShape::$shape, $desc); + }; +} diff --git a/crates/nu-parser/Cargo.toml b/crates/nu-parser/Cargo.toml index 804d3183fd..6a6038d517 100644 --- a/crates/nu-parser/Cargo.toml +++ b/crates/nu-parser/Cargo.toml @@ -32,6 +32,7 @@ ansi_term = "0.12.1" ptree = {version = "0.2" } language-reporting = "0.4.0" unicode-xid = "0.2.0" +enumflags2 = "0.6.2" [dev-dependencies] pretty_assertions = "0.6.1" @@ -40,4 +41,4 @@ pretty_assertions = "0.6.1" nu-build = { version = "0.1.0", path = "../nu-build" } [features] -#trace = ["nom-tracable/trace"] +trace = ["nom-tracable/trace"] diff --git a/crates/nu-parser/src/hir.rs b/crates/nu-parser/src/hir.rs index 42e01e2d6a..6c79d77ce6 100644 --- a/crates/nu-parser/src/hir.rs +++ b/crates/nu-parser/src/hir.rs @@ -4,11 +4,13 @@ pub(crate) mod expand_external_tokens; pub(crate) mod external_command; pub(crate) mod named; pub(crate) mod path; +pub(crate) mod range; +pub(crate) mod signature; pub mod syntax_shape; pub(crate) mod tokens_iterator; use crate::hir::syntax_shape::Member; -use crate::parse::operator::Operator; +use crate::parse::operator::CompareOperator; use crate::parse::parser::Number; use crate::parse::unit::Unit; use derive_new::new; @@ -24,12 +26,40 @@ use crate::parse::tokens::RawNumber; pub(crate) use self::binary::Binary; pub(crate) use self::path::Path; +pub(crate) use self::range::Range; pub(crate) use self::syntax_shape::ExpandContext; pub(crate) use self::tokens_iterator::TokensIterator; pub use self::external_command::ExternalCommand; pub use self::named::{NamedArguments, NamedValue}; +#[derive(Debug, Clone)] +pub struct Signature { + unspanned: nu_protocol::Signature, + span: Span, +} + +impl Signature { + pub fn new(unspanned: nu_protocol::Signature, span: impl Into) -> Signature { + Signature { + unspanned, + span: span.into(), + } + } +} + +impl HasSpan for Signature { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for Signature { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + self.unspanned.pretty_debug(source) + } +} + #[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)] pub struct Call { #[get = "pub(crate)"] @@ -68,6 +98,7 @@ pub enum RawExpression { Synthetic(Synthetic), Variable(Variable), Binary(Box), + Range(Box), Block(Vec), List(Vec), Path(Box), @@ -90,6 +121,7 @@ impl ShellTypeName for RawExpression { RawExpression::Variable(..) => "variable", RawExpression::List(..) => "list", RawExpression::Binary(..) => "binary", + RawExpression::Range(..) => "range", RawExpression::Block(..) => "block", RawExpression::Path(..) => "variable path", RawExpression::Boolean(..) => "boolean", @@ -159,6 +191,7 @@ impl PrettyDebugWithSource for Expression { }, RawExpression::Variable(_) => b::keyword(self.span.slice(source)), RawExpression::Binary(binary) => binary.pretty_debug(source), + RawExpression::Range(range) => range.pretty_debug(source), RawExpression::Block(_) => b::opaque("block"), RawExpression::List(list) => b::delimit( "[", @@ -245,7 +278,7 @@ impl Expression { pub fn infix( left: Expression, - op: Spanned>, + op: Spanned>, right: Expression, ) -> Expression { let new_span = left.span.until(right.span); @@ -254,6 +287,12 @@ impl Expression { .into_expr(new_span) } + pub fn range(left: Expression, op: Span, right: Expression) -> Expression { + let new_span = left.span.until(right.span); + + RawExpression::Range(Box::new(Range::new(left, op, right))).into_expr(new_span) + } + pub fn file_path(path: impl Into, outer: impl Into) -> Expression { RawExpression::FilePath(path.into()).into_expr(outer) } diff --git a/crates/nu-parser/src/hir/baseline_parse.rs b/crates/nu-parser/src/hir/baseline_parse.rs index 87c2771955..192cb697cc 100644 --- a/crates/nu-parser/src/hir/baseline_parse.rs +++ b/crates/nu-parser/src/hir/baseline_parse.rs @@ -1,2 +1,2 @@ #[cfg(test)] -mod tests; +pub mod tests; diff --git a/crates/nu-parser/src/hir/baseline_parse/tests.rs b/crates/nu-parser/src/hir/baseline_parse/tests.rs index 216be656b8..d6d3212563 100644 --- a/crates/nu-parser/src/hir/baseline_parse/tests.rs +++ b/crates/nu-parser/src/hir/baseline_parse/tests.rs @@ -23,7 +23,7 @@ fn test_parse_string() { fn test_parse_path() { parse_tokens( VariablePathShape, - vec![b::var("it"), b::op("."), b::bare("cpu")], + vec![b::var("it"), b::dot(), b::bare("cpu")], |tokens| { let (outer_var, inner_var) = tokens[0].expect_var(); let bare = tokens[2].expect_bare(); @@ -39,9 +39,9 @@ fn test_parse_path() { VariablePathShape, vec![ b::var("cpu"), - b::op("."), + b::dot(), b::bare("amount"), - b::op("."), + b::dot(), b::string("max ghz"), ], |tokens| { @@ -145,7 +145,7 @@ fn parse_tokens( let expr = match expr { Ok(expr) => expr, Err(err) => { - print_err(err.into(), context.source().clone()); + print_err(err.into(), &context.source().clone()); panic!("Parse failed"); } }; @@ -165,12 +165,10 @@ pub fn print_err(err: ShellError, source: &Text) { let mut source = source.to_string(); source.push_str(" "); let files = Files::new(source); - let _ = std::panic::catch_unwind(move || { - let _ = language_reporting::emit( - &mut writer.lock(), - &files, - &diag, - &language_reporting::DefaultConfig, - ); - }); + let _ = language_reporting::emit( + &mut writer.lock(), + &files, + &diag, + &language_reporting::DefaultConfig, + ); } diff --git a/crates/nu-parser/src/hir/binary.rs b/crates/nu-parser/src/hir/binary.rs index 47148fa6d8..cfc1e617ef 100644 --- a/crates/nu-parser/src/hir/binary.rs +++ b/crates/nu-parser/src/hir/binary.rs @@ -1,4 +1,4 @@ -use crate::{hir::Expression, Operator}; +use crate::{hir::Expression, CompareOperator}; use derive_new::new; use getset::Getters; @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; #[get = "pub"] pub struct Binary { left: Expression, - op: Spanned, + op: Spanned, right: Expression, } diff --git a/crates/nu-parser/src/hir/expand_external_tokens.rs b/crates/nu-parser/src/hir/expand_external_tokens.rs index 1ed67d44ce..5196a1ab47 100644 --- a/crates/nu-parser/src/hir/expand_external_tokens.rs +++ b/crates/nu-parser/src/hir/expand_external_tokens.rs @@ -10,6 +10,7 @@ use crate::{ TokensIterator, }; use nu_errors::ParseError; +use nu_protocol::SpannedTypeName; use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, SpannedItem}; #[derive(Debug, Clone)] @@ -195,11 +196,18 @@ impl ExpandExpression for ExternalHeadShape { UnspannedAtomicToken::Whitespace { .. } => { unreachable!("ExpansionRule doesn't allow Whitespace") } + UnspannedAtomicToken::Separator { .. } => { + unreachable!("ExpansionRule doesn't allow Separator") + } + UnspannedAtomicToken::Comment { .. } => { + unreachable!("ExpansionRule doesn't allow Comment") + } UnspannedAtomicToken::ShorthandFlag { .. } - | UnspannedAtomicToken::SquareDelimited { .. } => { + | UnspannedAtomicToken::SquareDelimited { .. } + | UnspannedAtomicToken::RoundDelimited { .. } => { return Err(ParseError::mismatch( "external command name", - "pipeline".spanned(atom.span), + atom.spanned_type_name(), )) } UnspannedAtomicToken::ExternalCommand { command } => { @@ -215,7 +223,10 @@ impl ExpandExpression for ExternalHeadShape { | UnspannedAtomicToken::GlobPattern { .. } | UnspannedAtomicToken::Word { .. } | UnspannedAtomicToken::Dot { .. } - | UnspannedAtomicToken::Operator { .. } => Expression::external_command(span, span), + | UnspannedAtomicToken::DotDot { .. } + | UnspannedAtomicToken::CompareOperator { .. } => { + Expression::external_command(span, span) + } }) } } @@ -257,6 +268,12 @@ impl ExpandExpression for ExternalContinuationShape { UnspannedAtomicToken::Whitespace { .. } => { unreachable!("ExpansionRule doesn't allow Whitespace") } + UnspannedAtomicToken::Separator { .. } => { + unreachable!("ExpansionRule doesn't allow Separator") + } + UnspannedAtomicToken::Comment { .. } => { + unreachable!("ExpansionRule doesn't allow Comment") + } UnspannedAtomicToken::String { body } => Expression::string(*body, span), UnspannedAtomicToken::ItVariable { name } => Expression::it_variable(*name, span), UnspannedAtomicToken::Variable { name } => Expression::variable(*name, span), @@ -265,11 +282,13 @@ impl ExpandExpression for ExternalContinuationShape { | UnspannedAtomicToken::Word { .. } | UnspannedAtomicToken::ShorthandFlag { .. } | UnspannedAtomicToken::Dot { .. } - | UnspannedAtomicToken::Operator { .. } => Expression::bare(span), - UnspannedAtomicToken::SquareDelimited { .. } => { + | UnspannedAtomicToken::DotDot { .. } + | UnspannedAtomicToken::CompareOperator { .. } => Expression::bare(span), + UnspannedAtomicToken::SquareDelimited { .. } + | UnspannedAtomicToken::RoundDelimited { .. } => { return Err(ParseError::mismatch( "external argument", - "pipeline".spanned(atom.span), + atom.spanned_type_name(), )) } }) diff --git a/crates/nu-parser/src/hir/range.rs b/crates/nu-parser/src/hir/range.rs new file mode 100644 index 0000000000..075dd3bd60 --- /dev/null +++ b/crates/nu-parser/src/hir/range.rs @@ -0,0 +1,33 @@ +use crate::hir::Expression; + +use derive_new::new; +use getset::Getters; +use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span}; +use serde::{Deserialize, Serialize}; + +#[derive( + Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new, +)] +pub struct Range { + #[get = "pub"] + left: Expression, + #[get = "pub"] + dotdot: Span, + #[get = "pub"] + right: Expression, +} + +impl PrettyDebugWithSource for Range { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + b::delimit( + "<", + self.left.pretty_debug(source) + + b::space() + + b::keyword(self.dotdot.slice(source)) + + b::space() + + self.right.pretty_debug(source), + ">", + ) + .group() + } +} diff --git a/crates/nu-parser/src/hir/signature.rs b/crates/nu-parser/src/hir/signature.rs new file mode 100644 index 0000000000..bf6278cb2b --- /dev/null +++ b/crates/nu-parser/src/hir/signature.rs @@ -0,0 +1,481 @@ +use crate::hir; +use crate::hir::syntax_shape::{ + expand_atom, expand_syntax, BareShape, ExpandContext, ExpandSyntax, ExpansionRule, + UnspannedAtomicToken, WhitespaceShape, +}; +use crate::hir::tokens_iterator::TokensIterator; +use crate::parse::comment::Comment; +use derive_new::new; +use nu_errors::ParseError; +use nu_protocol::{RowType, SpannedTypeName, Type}; +use nu_source::{ + b, DebugDocBuilder, HasFallibleSpan, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, +}; +use std::fmt::Debug; + +// A Signature is a command without implementation. +// +// In Nu, a command is a function combined with macro expansion rules. +// +// def cd +// # Change to a new path. +// optional directory(Path) # the directory to change to +// end + +#[derive(new)] +struct Expander<'a, 'b, 'c, 'd> { + iterator: &'b mut TokensIterator<'a>, + context: &'d ExpandContext<'c>, +} + +impl<'a, 'b, 'c, 'd> Expander<'a, 'b, 'c, 'd> { + fn expand(&mut self, syntax: impl ExpandSyntax) -> Result + where + O: HasFallibleSpan + Clone + std::fmt::Debug + 'static, + { + expand_syntax(&syntax, self.iterator, self.context) + } + + fn optional(&mut self, syntax: impl ExpandSyntax) -> Option + where + O: HasFallibleSpan + Clone + std::fmt::Debug + 'static, + { + match expand_syntax(&syntax, self.iterator, self.context) { + Err(_) => None, + Ok(value) => Some(value), + } + } + + fn pos(&mut self) -> Span { + self.iterator.span_at_cursor() + } + + fn slice_string(&mut self, span: impl Into) -> String { + span.into().slice(self.context.source()).to_string() + } +} + +#[derive(Debug, Copy, Clone)] +struct SignatureShape; + +impl ExpandSyntax for SignatureShape { + type Output = hir::Signature; + + fn name(&self) -> &'static str { + "signature" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let mut expander = Expander::new(token_nodes, context); + let start = expander.pos(); + expander.expand(keyword("def"))?; + expander.expand(WhitespaceShape)?; + let name = expander.expand(BareShape)?; + expander.expand(SeparatorShape)?; + let usage = expander.expand(CommentShape)?; + expander.expand(SeparatorShape)?; + let end = expander.pos(); + + Ok(hir::Signature::new( + nu_protocol::Signature::new(&name.word).desc(expander.slice_string(usage.text)), + start.until(end), + )) + }) + } +} + +fn keyword(kw: &'static str) -> KeywordShape { + KeywordShape { keyword: kw } +} + +#[derive(Debug, Copy, Clone)] +struct KeywordShape { + keyword: &'static str, +} + +impl ExpandSyntax for KeywordShape { + type Output = Span; + + fn name(&self) -> &'static str { + "keyword" + } + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "keyword", context, ExpansionRule::new())?; + + match &atom.unspanned { + UnspannedAtomicToken::Word { text } => { + let word = text.slice(context.source()); + + if word == self.keyword { + return Ok(atom.span); + } + } + _ => {} + } + + Err(ParseError::mismatch(self.keyword, atom.spanned_type_name())) + } +} + +#[derive(Debug, Copy, Clone)] +struct SeparatorShape; + +impl ExpandSyntax for SeparatorShape { + type Output = Span; + + fn name(&self) -> &'static str { + "separator" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "separator", context, ExpansionRule::new())?; + + match &atom.unspanned { + UnspannedAtomicToken::Separator { text } => Ok(*text), + _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())), + } + } +} + +#[derive(Debug, Copy, Clone)] +struct CommentShape; + +impl ExpandSyntax for CommentShape { + type Output = Comment; + + fn name(&self) -> &'static str { + "comment" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "comment", context, ExpansionRule::new())?; + + match &atom.unspanned { + UnspannedAtomicToken::Comment { body } => Ok(Comment::line(body, atom.span)), + _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())), + } + } +} + +#[derive(Debug, Copy, Clone, new)] +struct TupleShape { + first: A, + second: B, +} + +#[derive(Debug, Clone, new)] +struct TupleSyntax { + first: A, + second: B, +} + +impl PrettyDebugWithSource for TupleSyntax +where + A: PrettyDebugWithSource, + B: PrettyDebugWithSource, +{ + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + b::typed( + "pair", + self.first.pretty_debug(source) + b::space() + self.second.pretty_debug(source), + ) + } +} + +impl HasFallibleSpan for TupleSyntax +where + A: HasFallibleSpan + Debug + Clone, + B: HasFallibleSpan + Debug + Clone, +{ + fn maybe_span(&self) -> Option { + match (self.first.maybe_span(), self.second.maybe_span()) { + (Some(first), Some(second)) => Some(first.until(second)), + (Some(first), None) => Some(first), + (None, Some(second)) => Some(second), + (None, None) => None, + } + } +} + +impl ExpandSyntax for TupleShape +where + A: ExpandSyntax + Debug + Copy, + B: ExpandSyntax + Debug + Copy, + AOut: HasFallibleSpan + Debug + Clone + 'static, + BOut: HasFallibleSpan + Debug + Clone + 'static, +{ + type Output = TupleSyntax; + + fn name(&self) -> &'static str { + "pair" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let first = expand_syntax(&self.first, token_nodes, context)?; + let second = expand_syntax(&self.second, token_nodes, context)?; + + Ok(TupleSyntax { first, second }) + }) + } +} + +#[derive(Debug, Clone)] +pub struct PositionalParam { + optional: Option, + name: Identifier, + ty: Spanned, + desc: Spanned, + span: Span, +} + +impl HasSpan for PositionalParam { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for PositionalParam { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + (match self.optional { + Some(_) => b::description("optional") + b::space(), + None => b::blank(), + }) + self.ty.pretty_debug(source) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct PositionalParamShape; + +impl ExpandSyntax for PositionalParamShape { + type Output = PositionalParam; + + fn name(&self) -> &'static str { + "positional param" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let mut expander = Expander::new(token_nodes, context); + + let optional = expander + .optional(TupleShape::new(keyword("optional"), WhitespaceShape)) + .map(|s| s.first); + + let name = expander.expand(IdentifierShape)?; + + expander.optional(WhitespaceShape); + + let _ty = expander.expand(TypeShape)?; + + Ok(PositionalParam { + optional, + name, + ty: Type::Nothing.spanned(Span::unknown()), + desc: format!("").spanned(Span::unknown()), + span: Span::unknown(), + }) + }) + } +} + +#[derive(Debug, Clone)] +struct Identifier { + body: String, + span: Span, +} + +impl HasSpan for Identifier { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for Identifier { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + b::typed("id", b::description(self.span.slice(source))) + } +} + +#[derive(Debug, Copy, Clone)] +struct IdentifierShape; + +impl ExpandSyntax for IdentifierShape { + type Output = Identifier; + fn name(&self) -> &'static str { + "identifier" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "identifier", context, ExpansionRule::new())?; + + match atom.unspanned { + UnspannedAtomicToken::Word { text } => { + let body = text.slice(context.source()); + if is_id(body) { + return Ok(Identifier { + body: body.to_string(), + span: text, + }); + } + } + _ => {} + } + + Err(ParseError::mismatch("identifier", atom.spanned_type_name())) + } +} + +fn is_id(input: &str) -> bool { + let source = nu_source::nom_input(input); + match crate::parse::parser::ident(source) { + Err(_) => false, + Ok((input, _)) => input.fragment.len() == 0, + } +} + +#[derive(Debug, Clone, new)] +struct TypeSyntax { + ty: Type, + span: Span, +} + +impl HasSpan for TypeSyntax { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for TypeSyntax { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + self.ty.pretty_debug(source) + } +} + +#[derive(Debug, Copy, Clone)] +struct TypeShape; + +impl ExpandSyntax for TypeShape { + type Output = TypeSyntax; + + fn name(&self) -> &'static str { + "type" + } + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "type", context, ExpansionRule::new())?; + + match atom.unspanned { + UnspannedAtomicToken::Word { text } => { + let word = text.slice(context.source()); + + Ok(TypeSyntax::new( + match word { + "nothing" => Type::Nothing, + "integer" => Type::Int, + "decimal" => Type::Decimal, + "bytesize" => Type::Bytesize, + "string" => Type::String, + "column-path" => Type::ColumnPath, + "pattern" => Type::Pattern, + "boolean" => Type::Boolean, + "date" => Type::Date, + "duration" => Type::Duration, + "filename" => Type::Path, + "binary" => Type::Binary, + "row" => Type::Row(RowType::new()), + "table" => Type::Table(vec![]), + "block" => Type::Block, + _ => return Err(ParseError::mismatch("type", atom.spanned_type_name())), + }, + atom.span, + )) + } + _ => Err(ParseError::mismatch("type", atom.spanned_type_name())), + } + } +} + +#[derive(Debug, Copy, Clone)] +struct TypeAnnotation; + +impl ExpandSyntax for TypeAnnotation { + type Output = TypeSyntax; + + fn name(&self) -> &'static str { + "type annotation" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom( + token_nodes, + "type annotation", + context, + ExpansionRule::new(), + )?; + + match atom.unspanned { + UnspannedAtomicToken::RoundDelimited { nodes, .. } => { + token_nodes.atomic_parse(|token_nodes| { + token_nodes.child( + (&nodes[..]).spanned(atom.span), + context.source().clone(), + |token_nodes| { + let ty = expand_syntax(&TypeShape, token_nodes, context)?; + + let next = token_nodes.peek_non_ws(); + + match next.node { + None => Ok(ty), + Some(node) => { + Err(ParseError::extra_tokens(node.spanned_type_name())) + } + } + }, + ) + }) + } + + _ => Err(ParseError::mismatch( + "type annotation", + atom.spanned_type_name(), + )), + } + } +} diff --git a/crates/nu-parser/src/hir/syntax_shape.rs b/crates/nu-parser/src/hir/syntax_shape.rs index 04d89e93aa..5659116001 100644 --- a/crates/nu-parser/src/hir/syntax_shape.rs +++ b/crates/nu-parser/src/hir/syntax_shape.rs @@ -7,8 +7,9 @@ use crate::commands::external_command; use crate::hir; use crate::hir::expand_external_tokens::ExternalTokensShape; use crate::hir::syntax_shape::block::AnyBlockShape; +use crate::hir::syntax_shape::expression::range::RangeShape; use crate::hir::tokens_iterator::{Peeked, TokensIterator}; -use crate::parse::operator::Operator; +use crate::parse::operator::EvaluationOperator; use crate::parse::token_tree::TokenNode; use crate::parse::tokens::{Token, UnspannedToken}; use crate::parse_command::{parse_command_tail, CommandTailShape}; @@ -74,6 +75,7 @@ impl FallibleColorSyntax for SyntaxShape { context, shapes, ), + SyntaxShape::Range => color_fallible_syntax(&RangeShape, token_nodes, context, shapes), SyntaxShape::Member => { color_fallible_syntax(&MemberShape, token_nodes, context, shapes) } @@ -114,6 +116,7 @@ impl FallibleColorSyntax for SyntaxShape { match self { SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context), SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context), + SyntaxShape::Range => color_fallible_syntax(&RangeShape, token_nodes, context), SyntaxShape::String => { color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context) } @@ -134,6 +137,7 @@ impl ExpandExpression for SyntaxShape { match self { SyntaxShape::Any => "shape[any]", SyntaxShape::Int => "shape[integer]", + SyntaxShape::Range => "shape[range]", SyntaxShape::String => "shape[string]", SyntaxShape::Member => "shape[column name]", SyntaxShape::ColumnPath => "shape[column path]", @@ -152,6 +156,7 @@ impl ExpandExpression for SyntaxShape { match self { SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context), SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context), + SyntaxShape::Range => expand_expr(&RangeShape, token_nodes, context), SyntaxShape::String => expand_expr(&StringShape, token_nodes, context), SyntaxShape::Member => { let syntax = expand_syntax(&MemberShape, token_nodes, context)?; @@ -183,7 +188,6 @@ pub trait SignatureRegistry { pub struct ExpandContext<'context> { #[get = "pub(crate)"] pub registry: Box, - #[get = "pub(crate)"] pub source: &'context Text, pub homedir: Option, } @@ -192,6 +196,10 @@ impl<'context> ExpandContext<'context> { pub(crate) fn homedir(&self) -> Option<&Path> { self.homedir.as_ref().map(|h| h.as_path()) } + + pub(crate) fn source(&self) -> &'context Text { + self.source + } } pub trait TestSyntax: std::fmt::Debug + Copy { @@ -568,7 +576,7 @@ impl ExpandSyntax for BarePathShape { .. }) | TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), .. }) => true, @@ -604,7 +612,10 @@ impl FallibleColorSyntax for BareShape { } // otherwise, fail - other => Err(ParseError::mismatch("word", other.spanned_type_name())), + other => Err(ParseError::mismatch( + "word", + other.type_name().spanned(other.span()), + )), }) .map_err(|err| err.into()) } @@ -1600,7 +1611,7 @@ impl FallibleColorSyntax for SpaceShape { other => Err(ShellError::type_error( "whitespace", - other.spanned_type_name(), + other.type_name().spanned(other.span()), )), } } diff --git a/crates/nu-parser/src/hir/syntax_shape/block.rs b/crates/nu-parser/src/hir/syntax_shape/block.rs index 4a8f0aa9ac..c6ad13a0cd 100644 --- a/crates/nu-parser/src/hir/syntax_shape/block.rs +++ b/crates/nu-parser/src/hir/syntax_shape/block.rs @@ -390,6 +390,7 @@ impl FallibleColorSyntax for ShorthandHeadShape { ) -> Result<(), ShellError> { use crate::parse::token_tree::TokenNode; use crate::parse::tokens::{Token, UnspannedToken}; + use nu_protocol::SpannedTypeName; use nu_source::SpannedItem; // A shorthand path must not be at EOF diff --git a/crates/nu-parser/src/hir/syntax_shape/expression.rs b/crates/nu-parser/src/hir/syntax_shape/expression.rs index 4c7dfd3db6..02f68fe1dd 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression.rs @@ -4,6 +4,7 @@ pub(crate) mod file_path; pub(crate) mod list; pub(crate) mod number; pub(crate) mod pattern; +pub(crate) mod range; pub(crate) mod string; pub(crate) mod unit; pub(crate) mod variable_path; diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs b/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs index 6fe8a992ac..689c69bdd9 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs @@ -3,6 +3,7 @@ use crate::hir::syntax_shape::{ expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape, BarePatternShape, ExpandContext, UnitShape, UnitSyntax, }; +use crate::parse::operator::EvaluationOperator; use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; use crate::parse::tokens::UnspannedToken; use crate::parse::unit::Unit; @@ -12,7 +13,7 @@ use crate::{ parse::flag::{Flag, FlagKind}, }; use nu_errors::{ParseError, ShellError}; -use nu_protocol::ShellTypeName; +use nu_protocol::{ShellTypeName, SpannedTypeName}; use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem}; use std::ops::Deref; @@ -52,23 +53,36 @@ pub enum UnspannedAtomicToken<'tokens> { Word { text: Span, }, - #[allow(unused)] - Dot { - text: Span, - }, SquareDelimited { spans: (Span, Span), nodes: &'tokens Vec, }, + #[allow(unused)] + RoundDelimited { + spans: (Span, Span), + nodes: &'tokens Vec, + }, ShorthandFlag { name: Span, }, - Operator { + CompareOperator { + text: Span, + }, + Dot { + text: Span, + }, + DotDot { text: Span, }, Whitespace { text: Span, }, + Separator { + text: Span, + }, + Comment { + body: Span, + }, } impl<'tokens> UnspannedAtomicToken<'tokens> { @@ -80,15 +94,24 @@ impl<'tokens> UnspannedAtomicToken<'tokens> { } } +impl<'tokens> ShellTypeName for AtomicToken<'tokens> { + fn type_name(&self) -> &'static str { + self.unspanned.type_name() + } +} + impl<'tokens> ShellTypeName for UnspannedAtomicToken<'tokens> { fn type_name(&self) -> &'static str { match &self { UnspannedAtomicToken::Eof { .. } => "eof", UnspannedAtomicToken::Error { .. } => "error", - UnspannedAtomicToken::Operator { .. } => "operator", + UnspannedAtomicToken::CompareOperator { .. } => "compare operator", UnspannedAtomicToken::ShorthandFlag { .. } => "shorthand flag", UnspannedAtomicToken::Whitespace { .. } => "whitespace", + UnspannedAtomicToken::Separator { .. } => "separator", + UnspannedAtomicToken::Comment { .. } => "comment", UnspannedAtomicToken::Dot { .. } => "dot", + UnspannedAtomicToken::DotDot { .. } => "dotdot", UnspannedAtomicToken::Number { .. } => "number", UnspannedAtomicToken::Size { .. } => "size", UnspannedAtomicToken::String { .. } => "string", @@ -99,6 +122,7 @@ impl<'tokens> ShellTypeName for UnspannedAtomicToken<'tokens> { UnspannedAtomicToken::GlobPattern { .. } => "file pattern", UnspannedAtomicToken::Word { .. } => "word", UnspannedAtomicToken::SquareDelimited { .. } => "array literal", + UnspannedAtomicToken::RoundDelimited { .. } => "paren delimited", } } } @@ -109,6 +133,12 @@ pub struct AtomicToken<'tokens> { pub span: Span, } +impl<'tokens> HasSpan for AtomicToken<'tokens> { + fn span(&self) -> Span { + self.span + } +} + impl<'tokens> Deref for AtomicToken<'tokens> { type Target = UnspannedAtomicToken<'tokens>; @@ -131,31 +161,18 @@ impl<'tokens> AtomicToken<'tokens> { )) } UnspannedAtomicToken::Error { .. } => { - return Err(ParseError::mismatch( - expected, - "eof atomic token".spanned(self.span), - )) + return Err(ParseError::mismatch(expected, "error".spanned(self.span))) } - UnspannedAtomicToken::Operator { .. } => { - return Err(ParseError::mismatch( - expected, - "operator".spanned(self.span), - )) - } - UnspannedAtomicToken::ShorthandFlag { .. } => { - return Err(ParseError::mismatch( - expected, - "shorthand flag".spanned(self.span), - )) - } - UnspannedAtomicToken::Whitespace { .. } => { - return Err(ParseError::mismatch( - expected, - "whitespace".spanned(self.span), - )) - } - UnspannedAtomicToken::Dot { .. } => { - return Err(ParseError::mismatch(expected, "dot".spanned(self.span))) + UnspannedAtomicToken::RoundDelimited { .. } + | UnspannedAtomicToken::CompareOperator { .. } + | UnspannedAtomicToken::ShorthandFlag { .. } + | UnspannedAtomicToken::Whitespace { .. } + | UnspannedAtomicToken::Separator { .. } + | UnspannedAtomicToken::Comment { .. } + | UnspannedAtomicToken::Dot { .. } + | UnspannedAtomicToken::DotDot { .. } + | UnspannedAtomicToken::SquareDelimited { .. } => { + return Err(ParseError::mismatch(expected, self.spanned_type_name())); } UnspannedAtomicToken::Number { number } => { Expression::number(number.to_number(context.source), self.span) @@ -175,41 +192,17 @@ impl<'tokens> AtomicToken<'tokens> { self.span, ), UnspannedAtomicToken::Word { text } => Expression::string(*text, *text), - UnspannedAtomicToken::SquareDelimited { .. } => unimplemented!("into_hir"), }) } - #[cfg(not(coloring_in_tokens))] - pub fn spanned_type_name(&self) -> Spanned<&'static str> { - match &self.unspanned { - UnspannedAtomicToken::Eof { .. } => "eof", - UnspannedAtomicToken::Error { .. } => "error", - UnspannedAtomicToken::Operator { .. } => "operator", - UnspannedAtomicToken::ShorthandFlag { .. } => "shorthand flag", - UnspannedAtomicToken::Whitespace { .. } => "whitespace", - UnspannedAtomicToken::Dot { .. } => "dot", - UnspannedAtomicToken::Number { .. } => "number", - UnspannedAtomicToken::Size { .. } => "size", - UnspannedAtomicToken::String { .. } => "string", - UnspannedAtomicToken::ItVariable { .. } => "$it", - UnspannedAtomicToken::Variable { .. } => "variable", - UnspannedAtomicToken::ExternalCommand { .. } => "external command", - UnspannedAtomicToken::ExternalWord { .. } => "external word", - UnspannedAtomicToken::GlobPattern { .. } => "file pattern", - UnspannedAtomicToken::Word { .. } => "word", - UnspannedAtomicToken::SquareDelimited { .. } => "array literal", - } - .spanned(self.span) - } - pub(crate) fn color_tokens(&self, shapes: &mut Vec>) { match &self.unspanned { UnspannedAtomicToken::Eof { .. } => {} UnspannedAtomicToken::Error { .. } => { return shapes.push(FlatShape::Error.spanned(self.span)) } - UnspannedAtomicToken::Operator { .. } => { - return shapes.push(FlatShape::Operator.spanned(self.span)); + UnspannedAtomicToken::CompareOperator { .. } => { + return shapes.push(FlatShape::CompareOperator.spanned(self.span)); } UnspannedAtomicToken::ShorthandFlag { .. } => { return shapes.push(FlatShape::ShorthandFlag.spanned(self.span)); @@ -305,17 +298,30 @@ impl PrettyDebugWithSource for AtomicToken<'_> { b::intersperse_with_source(nodes.iter(), b::space(), source), "]", ), + UnspannedAtomicToken::RoundDelimited { nodes, .. } => b::delimit( + "(", + b::intersperse_with_source(nodes.iter(), b::space(), source), + ")", + ), UnspannedAtomicToken::ShorthandFlag { name } => { atom_kind("shorthand flag", b::key(name.slice(source))) } UnspannedAtomicToken::Dot { .. } => atom(b::kind("dot")), - UnspannedAtomicToken::Operator { text } => { + UnspannedAtomicToken::DotDot { .. } => atom(b::kind("dotdot")), + UnspannedAtomicToken::CompareOperator { text } => { atom_kind("operator", b::keyword(text.slice(source))) } UnspannedAtomicToken::Whitespace { text } => atom_kind( "whitespace", b::description(format!("{:?}", text.slice(source))), ), + UnspannedAtomicToken::Separator { text } => atom_kind( + "separator", + b::description(format!("{:?}", text.slice(source))), + ), + UnspannedAtomicToken::Comment { body } => { + atom_kind("comment", b::description(body.slice(source))) + } }) } } @@ -331,12 +337,15 @@ pub enum WhitespaceHandling { pub struct ExpansionRule { pub(crate) allow_external_command: bool, pub(crate) allow_external_word: bool, - pub(crate) allow_operator: bool, + pub(crate) allow_cmp_operator: bool, + pub(crate) allow_eval_operator: bool, pub(crate) allow_eof: bool, + pub(crate) allow_separator: bool, pub(crate) treat_size_as_word: bool, pub(crate) separate_members: bool, pub(crate) commit_errors: bool, pub(crate) whitespace: WhitespaceHandling, + pub(crate) allow_comments: bool, } impl ExpansionRule { @@ -344,12 +353,15 @@ impl ExpansionRule { ExpansionRule { allow_external_command: false, allow_external_word: false, - allow_operator: false, + allow_eval_operator: false, + allow_cmp_operator: false, allow_eof: false, treat_size_as_word: false, separate_members: false, commit_errors: false, + allow_separator: false, whitespace: WhitespaceHandling::RejectWhitespace, + allow_comments: false, } } @@ -360,11 +372,14 @@ impl ExpansionRule { ExpansionRule { allow_external_command: true, allow_external_word: true, - allow_operator: true, + allow_cmp_operator: true, + allow_eval_operator: true, allow_eof: true, separate_members: false, treat_size_as_word: false, commit_errors: true, + allow_separator: true, + allow_comments: true, whitespace: WhitespaceHandling::AllowWhitespace, } } @@ -376,14 +391,26 @@ impl ExpansionRule { } #[allow(unused)] - pub fn allow_operator(mut self) -> ExpansionRule { - self.allow_operator = true; + pub fn allow_cmp_operator(mut self) -> ExpansionRule { + self.allow_cmp_operator = true; + self + } + + #[allow(unused)] + pub fn no_cmp_operator(mut self) -> ExpansionRule { + self.allow_cmp_operator = false; + self + } + + #[allow(unused)] + pub fn allow_eval_operator(mut self) -> ExpansionRule { + self.allow_eval_operator = true; self } #[allow(unused)] pub fn no_operator(mut self) -> ExpansionRule { - self.allow_operator = false; + self.allow_eval_operator = false; self } @@ -440,6 +467,30 @@ impl ExpansionRule { self.whitespace = WhitespaceHandling::RejectWhitespace; self } + + #[allow(unused)] + pub fn allow_separator(mut self) -> ExpansionRule { + self.allow_separator = true; + self + } + + #[allow(unused)] + pub fn reject_separator(mut self) -> ExpansionRule { + self.allow_separator = false; + self + } + + #[allow(unused)] + pub fn allow_comments(mut self) -> ExpansionRule { + self.allow_comments = true; + self + } + + #[allow(unused)] + pub fn reject_comments(mut self) -> ExpansionRule { + self.allow_comments = false; + self + } } pub fn expand_atom<'me, 'content>( @@ -578,6 +629,17 @@ fn expand_atom_inner<'me, 'content>( .into_atomic_token(error.span)); } + TokenNode::Separator(span) if rule.allow_separator => { + peeked.commit(); + return Ok(UnspannedAtomicToken::Separator { text: *span }.into_atomic_token(span)); + } + + TokenNode::Comment(comment) if rule.allow_comments => { + peeked.commit(); + return Ok(UnspannedAtomicToken::Comment { body: comment.text } + .into_atomic_token(comment.span())); + } + // [ ... ] TokenNode::Delimited(Spanned { item: @@ -649,8 +711,16 @@ fn expand_atom_inner<'me, 'content>( // First, the error cases. Each error case corresponds to a expansion rule // flag that can be used to allow the case - // rule.allow_operator - UnspannedToken::Operator(_) if !rule.allow_operator => return Err(err.error()), + // rule.allow_cmp_operator + UnspannedToken::CompareOperator(_) if !rule.allow_cmp_operator => { + return Err(err.error()) + } + + // rule.allow_eval_operator + UnspannedToken::EvaluationOperator(_) if !rule.allow_eval_operator => { + return Err(err.error()) + } + // rule.allow_external_command UnspannedToken::ExternalCommand(_) if !rule.allow_external_command => { return Err(ParseError::mismatch( @@ -669,8 +739,15 @@ fn expand_atom_inner<'me, 'content>( UnspannedToken::Number(number) => { UnspannedAtomicToken::Number { number }.into_atomic_token(token_span) } - UnspannedToken::Operator(_) => { - UnspannedAtomicToken::Operator { text: token_span }.into_atomic_token(token_span) + UnspannedToken::CompareOperator(_) => { + UnspannedAtomicToken::CompareOperator { text: token_span } + .into_atomic_token(token_span) + } + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => { + UnspannedAtomicToken::Dot { text: token_span }.into_atomic_token(token_span) + } + UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => { + UnspannedAtomicToken::DotDot { text: token_span }.into_atomic_token(token_span) } UnspannedToken::String(body) => { UnspannedAtomicToken::String { body }.into_atomic_token(token_span) diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/number.rs b/crates/nu-parser/src/hir/syntax_shape/expression/number.rs index ed95439ce2..22494dd6a0 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/number.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/number.rs @@ -26,9 +26,9 @@ impl ExpandExpression for NumberShape { ) -> Result { parse_single_node(token_nodes, "Number", |token, token_span, err| { Ok(match token { - UnspannedToken::GlobPattern | UnspannedToken::Operator(..) => { - return Err(err.error()) - } + UnspannedToken::GlobPattern + | UnspannedToken::CompareOperator(..) + | UnspannedToken::EvaluationOperator(..) => return Err(err.error()), UnspannedToken::Variable(tag) if tag.slice(context.source) == "it" => { hir::Expression::it_variable(tag, token_span) } @@ -131,7 +131,8 @@ impl ExpandExpression for IntShape { parse_single_node(token_nodes, "Integer", |token, token_span, err| { Ok(match token { UnspannedToken::GlobPattern - | UnspannedToken::Operator(..) + | UnspannedToken::CompareOperator(..) + | UnspannedToken::EvaluationOperator(..) | UnspannedToken::ExternalWord => return Err(err.error()), UnspannedToken::Variable(span) if span.slice(context.source) == "it" => { hir::Expression::it_variable(span, token_span) diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs b/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs index 826d5d8286..96f7d9da9d 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs @@ -2,8 +2,9 @@ use crate::hir::syntax_shape::{ expand_atom, expand_bare, expression::expand_file_path, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, UnspannedAtomicToken, }; +use crate::parse::operator::EvaluationOperator; use crate::parse::tokens::{Token, UnspannedToken}; -use crate::{hir, hir::TokensIterator, Operator, TokenNode}; +use crate::{hir, hir::TokensIterator, TokenNode}; use nu_errors::{ParseError, ShellError}; #[cfg(coloring_in_tokens)] use nu_protocol::ShellTypeName; @@ -26,6 +27,8 @@ impl FallibleColorSyntax for PatternShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { + use nu_protocol::SpannedTypeName; + token_nodes.atomic(|token_nodes| { let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?; @@ -125,7 +128,7 @@ impl ExpandSyntax for BarePatternShape { .. }) | TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), .. }) | TokenNode::Token(Token { diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/range.rs b/crates/nu-parser/src/hir/syntax_shape/expression/range.rs new file mode 100644 index 0000000000..ca32db12f7 --- /dev/null +++ b/crates/nu-parser/src/hir/syntax_shape/expression/range.rs @@ -0,0 +1,154 @@ +use crate::hir::syntax_shape::expression::UnspannedAtomicToken; +use crate::hir::syntax_shape::{ + color_fallible_syntax, expand_atom, expand_expr, AnyExpressionShape, ExpandContext, + ExpandExpression, ExpansionRule, FallibleColorSyntax, FlatShape, +}; +use crate::parse::operator::EvaluationOperator; +use crate::parse::token_tree::TokenNode; +use crate::parse::tokens::{Token, UnspannedToken}; +use crate::{hir, hir::TokensIterator}; +use nu_errors::{ParseError, ShellError}; +use nu_protocol::SpannedTypeName; +use nu_source::SpannedItem; + +#[derive(Debug, Copy, Clone)] +pub struct RangeShape; + +impl ExpandExpression for RangeShape { + fn name(&self) -> &'static str { + "range" + } + + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let left = expand_expr(&AnyExpressionShape, token_nodes, context)?; + + let atom = expand_atom( + token_nodes, + "..", + context, + ExpansionRule::new().allow_eval_operator(), + )?; + + let span = match atom.unspanned { + UnspannedAtomicToken::DotDot { text } => text, + _ => return Err(ParseError::mismatch("..", atom.spanned_type_name())), + }; + + let right = expand_expr(&AnyExpressionShape, token_nodes, context)?; + + Ok(hir::Expression::range(left, span, right)) + }) + } +} + +#[cfg(coloring_in_tokens)] +impl FallibleColorSyntax for RangeShape { + type Info = (); + type Input = (); + + fn name(&self) -> &'static str { + "RangeShape" + } + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result<(), ShellError> { + token_nodes.atomic_parse(|token_nodes| { + color_fallible_syntax(&AnyExpressionShape, token_nodes, context)?; + color_fallible_syntax(&DotDotShape, token_nodes, context)?; + color_fallible_syntax(&AnyExpressionShape, token_nodes, context) + })?; + + Ok(()) + } +} + +#[cfg(not(coloring_in_tokens))] +impl FallibleColorSyntax for RangeShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.atomic_parse(|token_nodes| { + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)?; + color_fallible_syntax(&DotDotShape, token_nodes, context, shapes)?; + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) + })?; + + Ok(()) + } +} + +#[derive(Debug, Copy, Clone)] +struct DotDotShape; + +#[cfg(coloring_in_tokens)] +impl FallibleColorSyntax for DotDotShape { + type Info = (); + type Input = (); + + fn name(&self) -> &'static str { + ".." + } + + fn color_syntax<'a, 'b>( + &self, + _input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("..")?; + match &peeked.node { + TokenNode::Token(Token { + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot), + span, + }) => { + peeked.commit(); + token_nodes.color_shape(FlatShape::DotDot.spanned(span)); + Ok(()) + } + token => Err(ShellError::type_error("..", token.spanned_type_name())), + } + } +} + +#[cfg(not(coloring_in_tokens))] +impl FallibleColorSyntax for DotDotShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("..")?; + match &peeked.node { + TokenNode::Token(Token { + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot), + span, + }) => { + peeked.commit(); + shapes.push(FlatShape::DotDot.spanned(span)); + Ok(()) + } + token => Err(ShellError::type_error("..", token.spanned_type_name())), + } + } +} diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/string.rs b/crates/nu-parser/src/hir/syntax_shape/expression/string.rs index caf9b0abad..7844a9174a 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/string.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/string.rs @@ -91,7 +91,8 @@ impl ExpandExpression for StringShape { parse_single_node(token_nodes, "String", |token, token_span, err| { Ok(match token { UnspannedToken::GlobPattern - | UnspannedToken::Operator(..) + | UnspannedToken::CompareOperator(..) + | UnspannedToken::EvaluationOperator(..) | UnspannedToken::ExternalWord => return Err(err.error()), UnspannedToken::Variable(span) => { expand_variable(span, token_span, &context.source) diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs b/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs index 2f075a9d6f..e0173899b6 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs @@ -5,7 +5,7 @@ use crate::hir::syntax_shape::{ StringShape, TestSyntax, UnspannedAtomicToken, WhitespaceShape, }; use crate::parse::tokens::{RawNumber, UnspannedToken}; -use crate::{hir, hir::Expression, hir::TokensIterator, Operator}; +use crate::{hir, hir::Expression, hir::TokensIterator, CompareOperator, EvaluationOperator}; use nu_errors::ShellError; use nu_protocol::{PathMember, ShellTypeName}; use nu_source::{ @@ -271,7 +271,7 @@ impl ExpandSyntax for PathTailShape { #[derive(Debug, Clone)] pub enum ExpressionContinuation { DotSuffix(Span, PathMember), - InfixSuffix(Spanned, Expression), + InfixSuffix(Spanned, Expression), } impl PrettyDebugWithSource for ExpressionContinuation { @@ -484,6 +484,8 @@ impl FallibleColorSyntax for VariableShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { + use nu_protocol::SpannedTypeName; + let atom = expand_atom( token_nodes, "variable", @@ -1032,6 +1034,8 @@ impl FallibleColorSyntax for ColorableDotShape { _context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { + use nu_protocol::SpannedTypeName; + let peeked = token_nodes.peek_any().not_eof("dot")?; match peeked.node { @@ -1104,7 +1108,7 @@ impl ExpandSyntax for DotShape { ) -> Result { parse_single_node(token_nodes, "dot", |token, token_span, _| { Ok(match token { - UnspannedToken::Operator(Operator::Dot) => token_span, + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => token_span, _ => { return Err(ParseError::mismatch( "dot", @@ -1143,9 +1147,9 @@ impl FallibleColorSyntax for InfixShape { "infix operator", |token, token_span, err| { match token { - // If it's an operator (and not `.`), it's a match - UnspannedToken::Operator(operator) if operator != Operator::Dot => { - shapes.push(FlatShape::Operator.spanned(token_span)); + // If it's a comparison operator, it's a match + UnspannedToken::CompareOperator(_operator) => { + shapes.push(FlatShape::CompareOperator.spanned(token_span)); Ok(()) } @@ -1191,9 +1195,7 @@ impl FallibleColorSyntax for InfixShape { |token, token_span, _| { match token { // If it's an operator (and not `.`), it's a match - UnspannedToken::Operator(operator) if operator != Operator::Dot => { - Ok(token_span) - } + UnspannedToken::CompareOperator(_operator) => Ok(token_span), // Otherwise, it's not a match _ => Err(ParseError::mismatch( @@ -1206,7 +1208,7 @@ impl FallibleColorSyntax for InfixShape { checkpoint .iterator - .color_shape(FlatShape::Operator.spanned(operator_span)); + .color_shape(FlatShape::CompareOperator.spanned(operator_span)); // An infix operator must be followed by whitespace. If no whitespace was found, fail color_fallible_syntax(&WhitespaceShape, checkpoint.iterator, context)?; @@ -1266,7 +1268,7 @@ impl ExpandSyntax for InfixShape { #[derive(Debug, Clone)] pub struct InfixInnerSyntax { - pub operator: Spanned, + pub operator: Spanned, } impl HasSpan for InfixInnerSyntax { @@ -1298,12 +1300,10 @@ impl ExpandSyntax for InfixInnerShape { ) -> Result { parse_single_node(token_nodes, "infix operator", |token, token_span, err| { Ok(match token { - // If it's an operator (and not `.`), it's a match - UnspannedToken::Operator(operator) if operator != Operator::Dot => { - InfixInnerSyntax { - operator: operator.spanned(token_span), - } - } + // If it's a comparison operator, it's a match + UnspannedToken::CompareOperator(operator) => InfixInnerSyntax { + operator: operator.spanned(token_span), + }, // Otherwise, it's not a match _ => return Err(err.error()), diff --git a/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs b/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs index 2f2c865652..6c125ea547 100644 --- a/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs +++ b/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs @@ -1,5 +1,5 @@ use crate::parse::flag::{Flag, FlagKind}; -use crate::parse::operator::Operator; +use crate::parse::operator::EvaluationOperator; use crate::parse::token_tree::{Delimiter, TokenNode}; use crate::parse::tokens::{RawNumber, UnspannedToken}; use nu_source::{HasSpan, Span, Spanned, SpannedItem, Text}; @@ -10,8 +10,9 @@ pub enum FlatShape { CloseDelimiter(Delimiter), ItVariable, Variable, - Operator, + CompareOperator, Dot, + DotDot, InternalCommand, ExternalCommand, ExternalWord, @@ -27,7 +28,9 @@ pub enum FlatShape { Int, Decimal, Whitespace, + Separator, Error, + Comment, Size { number: Span, unit: Span }, } @@ -41,10 +44,15 @@ impl FlatShape { UnspannedToken::Number(RawNumber::Decimal(_)) => { shapes.push(FlatShape::Decimal.spanned(token.span)) } - UnspannedToken::Operator(Operator::Dot) => { + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => { shapes.push(FlatShape::Dot.spanned(token.span)) } - UnspannedToken::Operator(_) => shapes.push(FlatShape::Operator.spanned(token.span)), + UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => { + shapes.push(FlatShape::DotDot.spanned(token.span)) + } + UnspannedToken::CompareOperator(_) => { + shapes.push(FlatShape::CompareOperator.spanned(token.span)) + } UnspannedToken::String(_) => shapes.push(FlatShape::String.spanned(token.span)), UnspannedToken::Variable(v) if v.slice(source) == "it" => { shapes.push(FlatShape::ItVariable.spanned(token.span)) @@ -92,6 +100,8 @@ impl FlatShape { .. }) => shapes.push(FlatShape::ShorthandFlag.spanned(*span)), TokenNode::Whitespace(_) => shapes.push(FlatShape::Whitespace.spanned(token.span())), + TokenNode::Separator(_) => shapes.push(FlatShape::Separator.spanned(token.span())), + TokenNode::Comment(_) => shapes.push(FlatShape::Comment.spanned(token.span())), TokenNode::Error(v) => shapes.push(FlatShape::Error.spanned(v.span)), } } diff --git a/crates/nu-parser/src/hir/tokens_iterator.rs b/crates/nu-parser/src/hir/tokens_iterator.rs index 49ddd25a9b..a28b34d866 100644 --- a/crates/nu-parser/src/hir/tokens_iterator.rs +++ b/crates/nu-parser/src/hir/tokens_iterator.rs @@ -8,6 +8,7 @@ use crate::TokenNode; #[allow(unused)] use getset::{Getters, MutGetters}; use nu_errors::{ParseError, ShellError}; +use nu_protocol::SpannedTypeName; use nu_source::{HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Text}; cfg_if::cfg_if! { @@ -149,7 +150,7 @@ impl<'content, 'me> PeekedNode<'content, 'me> { pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError { match node { None => ParseError::unexpected_eof(expected, eof_span), - Some(node) => ParseError::mismatch(expected, node.type_name().spanned(node.span())), + Some(node) => ParseError::mismatch(expected, node.spanned_type_name()), } } @@ -498,10 +499,10 @@ impl<'content> TokensIterator<'content> { /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure /// that you'll succeed. - pub fn atomic_parse<'me, T>( + pub fn atomic_parse<'me, T, E>( &'me mut self, - block: impl FnOnce(&mut TokensIterator<'content>) -> Result, - ) -> Result { + block: impl FnOnce(&mut TokensIterator<'content>) -> Result, + ) -> Result { let state = &mut self.state; let index = state.index; diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 05194a67eb..f1e8baca90 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -12,9 +12,9 @@ pub use crate::hir::syntax_shape::{ pub use crate::hir::tokens_iterator::TokensIterator; pub use crate::parse::files::Files; pub use crate::parse::flag::Flag; -pub use crate::parse::operator::Operator; -pub use crate::parse::parser::pipeline; +pub use crate::parse::operator::{CompareOperator, EvaluationOperator}; pub use crate::parse::parser::Number; +pub use crate::parse::parser::{module, pipeline}; pub use crate::parse::token_tree::{Delimiter, TokenNode}; pub use crate::parse::token_tree_builder::TokenTreeBuilder; @@ -29,3 +29,12 @@ pub fn parse(input: &str) -> Result { Err(err) => Err(ShellError::parse_error(err)), } } + +pub fn parse_script(input: &str) -> Result { + let _ = pretty_env_logger::try_init(); + + match module(nom_input(input)) { + Ok((_rest, val)) => Ok(val), + Err(err) => Err(ShellError::parse_error(err)), + } +} diff --git a/crates/nu-parser/src/parse.rs b/crates/nu-parser/src/parse.rs index 6f9fac0b63..7528b99983 100644 --- a/crates/nu-parser/src/parse.rs +++ b/crates/nu-parser/src/parse.rs @@ -1,4 +1,5 @@ pub(crate) mod call_node; +pub(crate) mod comment; pub(crate) mod files; pub(crate) mod flag; pub(crate) mod operator; diff --git a/crates/nu-parser/src/parse/comment.rs b/crates/nu-parser/src/parse/comment.rs new file mode 100644 index 0000000000..57d26f338c --- /dev/null +++ b/crates/nu-parser/src/parse/comment.rs @@ -0,0 +1,42 @@ +use derive_new::new; +use getset::Getters; +use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum CommentKind { + Line, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)] +pub struct Comment { + pub(crate) kind: CommentKind, + pub(crate) text: Span, + pub(crate) span: Span, +} + +impl Comment { + pub fn line(text: impl Into, outer: impl Into) -> Comment { + Comment { + kind: CommentKind::Line, + text: text.into(), + span: outer.into(), + } + } +} + +impl PrettyDebugWithSource for Comment { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + let prefix = match self.kind { + CommentKind::Line => b::description("#"), + }; + + prefix + b::description(self.text.slice(source)) + } +} + +impl HasSpan for Comment { + fn span(&self) -> Span { + self.span + } +} diff --git a/crates/nu-parser/src/parse/files.rs b/crates/nu-parser/src/parse/files.rs index f472b59972..4df5d48fd4 100644 --- a/crates/nu-parser/src/parse/files.rs +++ b/crates/nu-parser/src/parse/files.rs @@ -34,13 +34,15 @@ impl language_reporting::ReportingFiles for Files { } fn location(&self, _file: Self::FileId, byte_index: usize) -> Option { + trace!("finding location for {}", byte_index); + let source = &self.snippet; let mut seen_lines = 0; let mut seen_bytes = 0; for (pos, slice) in source.match_indices('\n') { trace!( - "SEARCH={} SEEN={} POS={} SLICE={:?} LEN={} ALL={:?}", + "searching byte_index={} seen_bytes={} pos={} slice={:?} slice.len={} source={:?}", byte_index, seen_bytes, pos, @@ -50,9 +52,19 @@ impl language_reporting::ReportingFiles for Files { ); if pos >= byte_index { + trace!( + "returning {}:{} seen_lines={} byte_index={} pos={} seen_bytes={}", + seen_lines, + byte_index, + pos, + seen_lines, + byte_index, + seen_bytes + ); + return Some(language_reporting::Location::new( seen_lines, - byte_index - seen_bytes, + byte_index - pos, )); } else { seen_lines += 1; @@ -61,30 +73,70 @@ impl language_reporting::ReportingFiles for Files { } if seen_lines == 0 { - Some(language_reporting::Location::new(0, byte_index)) + trace!("seen_lines=0 end={}", source.len() - 1); + + // if we got here, there were no newlines in the source + Some(language_reporting::Location::new(0, source.len() - 1)) } else { - panic!("byte index {} wasn't valid", byte_index); + trace!( + "last line seen_lines={} end={}", + seen_lines, + source.len() - 1 - byte_index + ); + + // if we got here and we didn't return, it should mean that we're talking about + // the last line + Some(language_reporting::Location::new( + seen_lines, + source.len() - 1 - byte_index, + )) } } fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option { + trace!("finding line_span for {}", lineno); + let source = &self.snippet; let mut seen_lines = 0; let mut seen_bytes = 0; for (pos, _) in source.match_indices('\n') { + trace!( + "lineno={} seen_lines={} seen_bytes={} pos={}", + lineno, + seen_lines, + seen_bytes, + pos + ); + if seen_lines == lineno { - return Some(Span::new(seen_bytes, pos + 1)); + trace!("returning start={} end={}", seen_bytes, pos); + // If the number of seen lines is the lineno, seen_bytes is the start of the + // line and pos is the end of the line + return Some(Span::new(seen_bytes, pos)); } else { + // If it's not, increment seen_lines, and move seen_bytes to the beginning of + // the next line seen_lines += 1; seen_bytes = pos + 1; } } if seen_lines == 0 { + trace!("returning start={} end={}", 0, self.snippet.len() - 1); + + // if we got here, there were no newlines in the source Some(Span::new(0, self.snippet.len() - 1)) } else { - None + trace!( + "returning start={} end={}", + seen_bytes, + self.snippet.len() - 1 + ); + + // if we got here and we didn't return, it should mean that we're talking about + // the last line + Some(Span::new(seen_bytes, self.snippet.len() - 1)) } } diff --git a/crates/nu-parser/src/parse/operator.rs b/crates/nu-parser/src/parse/operator.rs index 99e5b5499e..a5504a6195 100644 --- a/crates/nu-parser/src/parse/operator.rs +++ b/crates/nu-parser/src/parse/operator.rs @@ -4,63 +4,102 @@ use serde::{Deserialize, Serialize}; use std::str::FromStr; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] -pub enum Operator { +pub enum CompareOperator { Equal, NotEqual, LessThan, GreaterThan, LessThanOrEqual, GreaterThanOrEqual, - Dot, Contains, NotContains, } -impl PrettyDebug for Operator { +impl PrettyDebug for CompareOperator { fn pretty(&self) -> DebugDocBuilder { b::operator(self.as_str()) } } -impl Operator { +impl CompareOperator { pub fn print(&self) -> String { self.as_str().to_string() } pub fn as_str(&self) -> &str { match *self { - Operator::Equal => "==", - Operator::NotEqual => "!=", - Operator::LessThan => "<", - Operator::GreaterThan => ">", - Operator::LessThanOrEqual => "<=", - Operator::GreaterThanOrEqual => ">=", - Operator::Dot => ".", - Operator::Contains => "=~", - Operator::NotContains => "!~", + CompareOperator::Equal => "==", + CompareOperator::NotEqual => "!=", + CompareOperator::LessThan => "<", + CompareOperator::GreaterThan => ">", + CompareOperator::LessThanOrEqual => "<=", + CompareOperator::GreaterThanOrEqual => ">=", + CompareOperator::Contains => "=~", + CompareOperator::NotContains => "!~", } } } -impl From<&str> for Operator { - fn from(input: &str) -> Operator { - Operator::from_str(input).unwrap() +impl From<&str> for CompareOperator { + fn from(input: &str) -> CompareOperator { + CompareOperator::from_str(input).unwrap() } } -impl FromStr for Operator { +impl FromStr for CompareOperator { type Err = (); fn from_str(input: &str) -> Result::Err> { match input { - "==" => Ok(Operator::Equal), - "!=" => Ok(Operator::NotEqual), - "<" => Ok(Operator::LessThan), - ">" => Ok(Operator::GreaterThan), - "<=" => Ok(Operator::LessThanOrEqual), - ">=" => Ok(Operator::GreaterThanOrEqual), - "." => Ok(Operator::Dot), - "=~" => Ok(Operator::Contains), - "!~" => Ok(Operator::NotContains), + "==" => Ok(CompareOperator::Equal), + "!=" => Ok(CompareOperator::NotEqual), + "<" => Ok(CompareOperator::LessThan), + ">" => Ok(CompareOperator::GreaterThan), + "<=" => Ok(CompareOperator::LessThanOrEqual), + ">=" => Ok(CompareOperator::GreaterThanOrEqual), + "=~" => Ok(CompareOperator::Contains), + "!~" => Ok(CompareOperator::NotContains), + _ => Err(()), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum EvaluationOperator { + Dot, + DotDot, +} + +impl PrettyDebug for EvaluationOperator { + fn pretty(&self) -> DebugDocBuilder { + b::operator(self.as_str()) + } +} + +impl EvaluationOperator { + pub fn print(&self) -> String { + self.as_str().to_string() + } + + pub fn as_str(&self) -> &str { + match *self { + EvaluationOperator::Dot => ".", + EvaluationOperator::DotDot => "..", + } + } +} + +impl From<&str> for EvaluationOperator { + fn from(input: &str) -> EvaluationOperator { + EvaluationOperator::from_str(input).unwrap() + } +} + +impl FromStr for EvaluationOperator { + type Err = (); + fn from_str(input: &str) -> Result::Err> { + match input { + "." => Ok(EvaluationOperator::Dot), + ".." => Ok(EvaluationOperator::DotDot), _ => Err(()), } } diff --git a/crates/nu-parser/src/parse/parser.rs b/crates/nu-parser/src/parse/parser.rs index f63d505908..78ae113d53 100644 --- a/crates/nu-parser/src/parse/parser.rs +++ b/crates/nu-parser/src/parse/parser.rs @@ -14,6 +14,7 @@ use nom::sequence::*; use bigdecimal::BigDecimal; use derive_new::new; +use enumflags2::BitFlags; use log::trace; use nom::dbg; use nom::*; @@ -32,7 +33,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::Debug; use std::str::FromStr; -macro_rules! operator { +macro_rules! cmp_operator { ($name:tt : $token:tt ) => { #[tracable_parser] pub fn $name(input: NomSpan) -> IResult { @@ -42,21 +43,38 @@ macro_rules! operator { Ok(( input, - TokenTreeBuilder::spanned_op(tag.fragment, Span::new(start, end)), + TokenTreeBuilder::spanned_cmp_op(tag.fragment, Span::new(start, end)), )) } }; } -operator! { gt: ">" } -operator! { lt: "<" } -operator! { gte: ">=" } -operator! { lte: "<=" } -operator! { eq: "==" } -operator! { neq: "!=" } -operator! { dot: "." } -operator! { cont: "=~" } -operator! { ncont: "!~" } +macro_rules! eval_operator { + ($name:tt : $token:tt ) => { + #[tracable_parser] + pub fn $name(input: NomSpan) -> IResult { + let start = input.offset; + let (input, tag) = tag($token)(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_eval_op(tag.fragment, Span::new(start, end)), + )) + } + }; +} + +cmp_operator! { gt: ">" } +cmp_operator! { lt: "<" } +cmp_operator! { gte: ">=" } +cmp_operator! { lte: "<=" } +cmp_operator! { eq: "==" } +cmp_operator! { neq: "!=" } +cmp_operator! { cont: "=~" } +cmp_operator! { ncont: "!~" } +eval_operator! { dot: "." } +eval_operator! { dotdot: ".." } #[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)] pub enum Number { @@ -213,6 +231,17 @@ pub fn raw_number(input: NomSpan) -> IResult { } } + let dotdot_result = dotdot(input); + + match dotdot_result { + // If we see a `..` immediately after an integer, it's a range, not a decimal + Ok((dotdot_input, _)) => { + return Ok((input, RawNumber::int(Span::new(start, input.offset)))) + } + + Err(_) => {} + } + let dot: IResult = tag(".")(input); let input = match dot { @@ -285,7 +314,7 @@ pub fn string(input: NomSpan) -> IResult { pub fn external(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = tag("^")(input)?; - let (input, bare) = take_while(is_bare_char)(input)?; + let (input, bare) = take_while(is_file_char)(input)?; let end = input.offset; Ok(( @@ -294,52 +323,186 @@ pub fn external(input: NomSpan) -> IResult { )) } -#[tracable_parser] -pub fn pattern(input: NomSpan) -> IResult { - let start = input.offset; - let (input, _) = take_while1(is_start_glob_char)(input)?; - let (input, _) = take_while(is_glob_char)(input)?; +fn word<'a, T, U, V>( + start_predicate: impl Fn(NomSpan<'a>) -> IResult, U>, + next_predicate: impl Fn(NomSpan<'a>) -> IResult, V> + Copy, + into: impl Fn(Span) -> T, +) -> impl Fn(NomSpan<'a>) -> IResult, T> { + move |input: NomSpan| { + let start = input.offset; - let next_char = &input.fragment.chars().nth(0); + let (input, _) = start_predicate(input)?; + let (input, _) = many0(next_predicate)(input)?; - if let Some(next_char) = next_char { - if is_external_word_char(*next_char) { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::TakeWhile1, - ))); + let next_char = &input.fragment.chars().nth(0); + + match next_char { + Some('.') => {} + Some(next_char) + if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) => + { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::TakeWhile1, + ))); + } + _ => {} } + + let end = input.offset; + + Ok((input, into(Span::new(start, end)))) } +} - let end = input.offset; - - Ok(( - input, - TokenTreeBuilder::spanned_pattern(Span::new(start, end)), - )) +pub fn matches(cond: fn(char) -> bool) -> impl Fn(NomSpan) -> IResult + Copy { + move |input: NomSpan| match input.iter_elements().next() { + Option::Some(c) if cond(c) => Ok((input.slice(1..), input.slice(0..1))), + _ => Err(nom::Err::Error(nom::error::ParseError::from_error_kind( + input, + nom::error::ErrorKind::Many0, + ))), + } } #[tracable_parser] -pub fn bare(input: NomSpan) -> IResult { - let start = input.offset; - let (input, _) = take_while1(is_start_bare_char)(input)?; - let (input, last) = take_while(is_bare_char)(input)?; +pub fn pattern(input: NomSpan) -> IResult { + word( + start_pattern, + matches(is_glob_char), + TokenTreeBuilder::spanned_pattern, + )(input) +} - let next_char = &input.fragment.chars().nth(0); - let prev_char = last.fragment.chars().nth(0); +#[tracable_parser] +pub fn start_pattern(input: NomSpan) -> IResult { + alt((take_while1(is_dot), matches(is_start_glob_char)))(input) +} - if let Some(next_char) = next_char { - if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::TakeWhile1, - ))); +#[tracable_parser] +pub fn filename(input: NomSpan) -> IResult { + let start_pos = input.offset; + + let (mut input, mut saw_special) = match start_file_char(input) { + Err(err) => return Err(err), + Ok((input, special)) => (input, special), + }; + + loop { + if saw_special.is_empty() { + match continue_file_char(input) { + Err(_) => { + return Ok(( + input, + TokenTreeBuilder::spanned_bare((start_pos, input.offset)), + )) + } + Ok((next_input, special)) => { + saw_special |= special; + input = next_input; + } + } + } else { + let rest = after_sep_file(input); + + let (input, span, updated_special) = match rest { + Err(_) => (input, (start_pos, input.offset), saw_special), + Ok((input, new_special)) => { + (input, (start_pos, input.offset), saw_special | new_special) + } + }; + + if updated_special.contains(SawSpecial::Glob) { + return Ok((input, TokenTreeBuilder::spanned_pattern(span))); + } else { + return Ok((input, TokenTreeBuilder::spanned_bare(span))); + } } } +} - let end = input.offset; +#[derive(BitFlags, Copy, Clone, Eq, PartialEq)] +enum SawSpecial { + PathSeparator = 0b01, + Glob = 0b10, +} - Ok((input, TokenTreeBuilder::spanned_bare(Span::new(start, end)))) +#[tracable_parser] +fn start_file_char(input: NomSpan) -> IResult> { + let path_sep_result = special_file_char(input); + + match path_sep_result { + Ok((input, special)) => return Ok((input, special)), + + Err(_) => {} + } + + start_filename(input).map(|(input, output)| (input, BitFlags::empty())) +} + +#[tracable_parser] +fn continue_file_char(input: NomSpan) -> IResult> { + let path_sep_result = special_file_char(input); + + match path_sep_result { + Ok((input, special)) => return Ok((input, special)), + Err(_) => {} + } + + matches(is_file_char)(input).map(|(input, _)| (input, BitFlags::empty())) +} + +#[tracable_parser] +fn special_file_char(input: NomSpan) -> IResult> { + match matches(is_path_separator)(input) { + Ok((input, _)) => return Ok((input, BitFlags::empty() | SawSpecial::PathSeparator)), + Err(_) => {} + } + + let (input, _) = matches(is_glob_specific_char)(input)?; + + Ok((input, BitFlags::empty() | SawSpecial::Glob)) +} + +#[tracable_parser] +fn after_sep_file(input: NomSpan) -> IResult> { + fn after_sep_char(c: char) -> bool { + is_external_word_char(c) || is_file_char(c) || c == '.' + } + + let start = input.offset; + let original_input = input; + let mut input = input; + + let (input, after_glob) = take_while1(after_sep_char)(input)?; + + let slice = original_input.slice(0..input.offset - start); + + let saw_special = if slice.fragment.chars().any(is_glob_specific_char) { + BitFlags::empty() | SawSpecial::Glob + } else { + BitFlags::empty() + }; + + Ok((input, saw_special)) +} + +pub fn start_filename(input: NomSpan) -> IResult { + alt((take_while1(is_dot), matches(is_start_file_char)))(input) +} + +#[tracable_parser] +pub fn member(input: NomSpan) -> IResult { + word( + matches(is_start_member_char), + matches(is_member_char), + TokenTreeBuilder::spanned_bare, + )(input) +} + +#[tracable_parser] +pub fn ident(input: NomSpan) -> IResult { + word(matches(is_id_start), matches(is_id_continue), Tag::from)(input) } #[tracable_parser] @@ -348,10 +511,7 @@ pub fn external_word(input: NomSpan) -> IResult { let (input, _) = take_while1(is_external_word_char)(input)?; let end = input.offset; - Ok(( - input, - TokenTreeBuilder::spanned_external_word(Span::new(start, end)), - )) + Ok((input, TokenTreeBuilder::spanned_external_word((start, end)))) } #[tracable_parser] @@ -367,21 +527,40 @@ pub fn var(input: NomSpan) -> IResult { )) } -#[tracable_parser] -pub fn ident(input: NomSpan) -> IResult { - let start = input.offset; - let (input, _) = take_while1(is_start_bare_char)(input)?; - let (input, _) = take_while(is_bare_char)(input)?; - let end = input.offset; +fn tight<'a>( + parser: impl Fn(NomSpan<'a>) -> IResult, Vec>, +) -> impl Fn(NomSpan<'a>) -> IResult, Vec> { + move |input: NomSpan| { + let mut result = vec![]; + let (input, head) = parser(input)?; + result.extend(head); - Ok((input, Tag::from((start, end, None)))) + let (input, tail) = opt(alt((many1(range_continuation), many1(dot_member))))(input)?; + + let next_char = &input.fragment.chars().nth(0); + + if is_boundary(*next_char) { + if let Some(tail) = tail { + for tokens in tail { + result.extend(tokens); + } + } + + Ok((input, result)) + } else { + Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Many0, + ))) + } + } } #[tracable_parser] pub fn flag(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = tag("--")(input)?; - let (input, bare) = bare(input)?; + let (input, bare) = filename(input)?; let end = input.offset; Ok(( @@ -394,7 +573,7 @@ pub fn flag(input: NomSpan) -> IResult { pub fn shorthand(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = tag("-")(input)?; - let (input, bare) = bare(input)?; + let (input, bare) = filename(input)?; let end = input.offset; Ok(( @@ -413,47 +592,97 @@ pub fn leaf(input: NomSpan) -> IResult { #[tracable_parser] pub fn token_list(input: NomSpan) -> IResult>> { let start = input.offset; - let (input, first) = node(input)?; + let mut node_list = vec![]; - let (input, mut list) = many0(pair(alt((whitespace, dot)), node))(input)?; + let mut next_input = input; + let mut before_space_input: Option = None; + let mut final_space_tokens = 0; - let end = input.offset; + loop { + let node_result = tight_node(next_input); - Ok(( - input, - make_token_list(first, list, None).spanned(Span::new(start, end)), - )) + let (after_node_input, next_nodes) = match node_result { + Err(_) => { + if let Some(before_space_input) = before_space_input { + next_input = before_space_input; + + for _ in 0..final_space_tokens { + node_list.pop(); + } + } + + break; + } + Ok((after_node_input, next_node)) => (after_node_input, next_node), + }; + + node_list.extend(next_nodes); + + // Special case that allows a parenthesized expression to immediate follow another + // token without a space, which could represent a type annotation. + let maybe_type = delimited_paren(after_node_input); + + let after_maybe_type_input = match maybe_type { + Err(_) => after_node_input, + Ok((after_maybe_type_input, parens)) => { + node_list.push(parens); + after_maybe_type_input + } + }; + + let maybe_space = any_space(after_maybe_type_input); + + let after_space_input = match maybe_space { + Err(_) => { + next_input = after_maybe_type_input; + + break; + } + Ok((after_space_input, space)) => { + final_space_tokens = space.len(); + node_list.extend(space); + before_space_input = Some(after_maybe_type_input); + after_space_input + } + }; + + next_input = after_space_input; + } + + let end = next_input.offset; + + Ok((next_input, node_list.spanned(Span::new(start, end)))) } #[tracable_parser] pub fn spaced_token_list(input: NomSpan) -> IResult>> { let start = input.offset; - let (input, pre_ws) = opt(whitespace)(input)?; + let (input, pre_ws) = opt(any_space)(input)?; let (input, items) = token_list(input)?; - let (input, post_ws) = opt(whitespace)(input)?; + let (input, post_ws) = opt(any_space)(input)?; let end = input.offset; let mut out = vec![]; - out.extend(pre_ws); + pre_ws.map(|pre_ws| out.extend(pre_ws)); out.extend(items.item); - out.extend(post_ws); + post_ws.map(|post_ws| out.extend(post_ws)); Ok((input, out.spanned(Span::new(start, end)))) } fn make_token_list( first: Vec, - list: Vec<(TokenNode, Vec)>, + list: Vec<(Vec, Vec)>, sp_right: Option, ) -> Vec { let mut nodes = vec![]; nodes.extend(first); - for (left, right) in list { - nodes.push(left); - nodes.extend(right); + for (sep, list) in list { + nodes.extend(sep); + nodes.extend(list); } if let Some(sp_right) = sp_right { @@ -463,6 +692,15 @@ fn make_token_list( nodes } +#[tracable_parser] +pub fn separator(input: NomSpan) -> IResult { + let left = input.offset; + let (input, ws1) = alt((tag(";"), tag("\n")))(input)?; + let right = input.offset; + + Ok((input, TokenTreeBuilder::spanned_sep(Span::new(left, right)))) +} + #[tracable_parser] pub fn whitespace(input: NomSpan) -> IResult { let left = input.offset; @@ -472,6 +710,30 @@ pub fn whitespace(input: NomSpan) -> IResult { Ok((input, TokenTreeBuilder::spanned_ws(Span::new(left, right)))) } +#[tracable_parser] +pub fn any_space(input: NomSpan) -> IResult> { + let left = input.offset; + let (input, tokens) = many1(alt((whitespace, separator, comment)))(input)?; + let right = input.offset; + + Ok((input, tokens)) +} + +#[tracable_parser] +pub fn comment(input: NomSpan) -> IResult { + let left = input.offset; + let (input, start) = tag("#")(input)?; + let (input, rest) = not_line_ending(input)?; + let right = input.offset; + + let span = (start.offset + 1, right); + + Ok(( + input, + TokenTreeBuilder::spanned_comment(span, Span::new(left, right)), + )) +} + pub fn delimited( input: NomSpan, delimiter: Delimiter, @@ -541,62 +803,43 @@ pub fn raw_call(input: NomSpan) -> IResult> { } #[tracable_parser] -pub fn bare_path(input: NomSpan) -> IResult> { - let (input, head) = alt((bare, dot))(input)?; +pub fn range_continuation(input: NomSpan) -> IResult> { + let original = input; - let (input, tail) = many0(alt((bare, dot, string)))(input)?; + let mut result = vec![]; - let next_char = &input.fragment.chars().nth(0); + let (input, dotdot_result) = dotdot(input)?; + result.push(dotdot_result); + let (input, node_result) = tight_node(input)?; + result.extend(node_result); - if is_boundary(*next_char) { - let mut result = vec![head]; - result.extend(tail); - - Ok((input, result)) - } else { - Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::Many0, - ))) - } + Ok((input, result)) } #[tracable_parser] -pub fn pattern_path(input: NomSpan) -> IResult> { - let (input, head) = alt((pattern, dot))(input)?; +pub fn dot_member(input: NomSpan) -> IResult> { + let (input, dot_result) = dot(input)?; + let (input, member_result) = any_member(input)?; - let (input, tail) = many0(alt((pattern, dot, string)))(input)?; - - let next_char = &input.fragment.chars().nth(0); - - if is_boundary(*next_char) { - let mut result = vec![head]; - result.extend(tail); - - Ok((input, result)) - } else { - Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::Many0, - ))) - } + Ok((input, vec![dot_result, member_result])) } #[tracable_parser] -pub fn node1(input: NomSpan) -> IResult { - alt((leaf, bare, pattern, external_word, delimited_paren))(input) +pub fn any_member(input: NomSpan) -> IResult { + alt((number, string, member))(input) } #[tracable_parser] -pub fn node(input: NomSpan) -> IResult> { +pub fn tight_node(input: NomSpan) -> IResult> { alt(( - to_list(leaf), - bare_path, - pattern_path, + tight(to_list(leaf)), + tight(to_list(filename)), + tight(to_list(pattern)), + to_list(comment), to_list(external_word), - to_list(delimited_paren), - to_list(delimited_brace), - to_list(delimited_square), + tight(to_list(delimited_paren)), + tight(to_list(delimited_brace)), + tight(to_list(delimited_square)), ))(input) } @@ -649,6 +892,23 @@ pub fn pipeline(input: NomSpan) -> IResult { )) } +#[tracable_parser] +pub fn module(input: NomSpan) -> IResult { + let (input, tokens) = spaced_token_list(input)?; + + if input.input_len() != 0 { + return Err(Err::Error(error_position!( + input, + nom::error::ErrorKind::Eof + ))); + } + + Ok(( + input, + TokenTreeBuilder::spanned_token_list(tokens.item, tokens.span), + )) +} + fn parse_int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -661,7 +921,7 @@ fn parse_int(frag: &str, neg: Option) -> i64 { fn is_boundary(c: Option) -> bool { match c { None => true, - Some(')') | Some(']') | Some('}') => true, + Some(')') | Some(']') | Some('}') | Some('(') => true, Some(c) if c.is_whitespace() => true, _ => false, } @@ -682,14 +942,25 @@ fn is_glob_specific_char(c: char) -> bool { } fn is_start_glob_char(c: char) -> bool { - is_start_bare_char(c) || is_glob_specific_char(c) || c == '.' + is_start_file_char(c) || is_glob_specific_char(c) || c == '.' } fn is_glob_char(c: char) -> bool { - is_bare_char(c) || is_glob_specific_char(c) + is_file_char(c) || is_glob_specific_char(c) } -fn is_start_bare_char(c: char) -> bool { +fn is_dot(c: char) -> bool { + c == '.' +} + +fn is_path_separator(c: char) -> bool { + match c { + '\\' | '/' | ':' => true, + _ => false, + } +} + +fn is_start_file_char(c: char) -> bool { match c { '+' => false, _ if c.is_alphanumeric() => true, @@ -698,11 +969,12 @@ fn is_start_bare_char(c: char) -> bool { '_' => true, '-' => true, '~' => true, + '.' => true, _ => false, } } -fn is_bare_char(c: char) -> bool { +fn is_file_char(c: char) -> bool { match c { '+' => true, _ if c.is_alphanumeric() => true, @@ -718,6 +990,24 @@ fn is_bare_char(c: char) -> bool { } } +fn is_start_member_char(c: char) -> bool { + match c { + _ if c.is_alphanumeric() => true, + '_' => true, + '-' => true, + _ => false, + } +} + +fn is_member_char(c: char) -> bool { + match c { + _ if c.is_alphanumeric() => true, + '_' => true, + '-' => true, + _ => false, + } +} + fn is_id_start(c: char) -> bool { unicode_xid::UnicodeXID::is_xid_start(c) } @@ -775,6 +1065,7 @@ mod tests { (<$parser:tt> $source:tt -> $tokens:expr) => { let result = apply($parser, stringify!($parser), $source); + let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens); if result != expected_tree { @@ -884,22 +1175,36 @@ mod tests { fn test_simple_path() { equal_tokens! { - "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")]) + "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::dot(), b::bare("exe")]) } equal_tokens! { - ".azure" -> b::token_list(vec![b::op(Operator::Dot), b::bare("azure")]) + ".azure" -> b::token_list(vec![b::bare(".azure")]) } equal_tokens! { - r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system"), b::op(Operator::Dot), b::bare("dll")]) + r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system.dll")]) } equal_tokens! { - r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests"), b::op(Operator::Dot), b::bare("js")]) + r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests.js")]) + } + + equal_tokens! { + + r"C:\Users\example\AppData\Local\Temp\.tmpZ4TVQ2\cd_test_8" -> b::token_list(vec![b::bare(r"C:\Users\example\AppData\Local\Temp\.tmpZ4TVQ2\cd_test_8")]) + } + + equal_tokens! { + + r"cd C:\Users\wycat\AppData\Local\Temp\.tmpaj5JKi\cd_test_11" -> b::pipeline(vec![vec![ + b::bare("cd"), + b::sp(), + b::bare(r"C:\Users\wycat\AppData\Local\Temp\.tmpaj5JKi\cd_test_11") + ]]) } } @@ -949,7 +1254,7 @@ mod tests { fn test_dot_prefixed_name() { equal_tokens! { - ".azure" -> b::token_list(vec![b::op("."), b::bare("azure")]) + ".azure" -> b::token_list(vec![b::bare(".azure")]) } } @@ -1003,33 +1308,43 @@ mod tests { } } + #[test] + fn test_range() { + let _ = pretty_env_logger::try_init(); + + equal_tokens! { + + "0..2" -> b::token_list(vec![b::int(0), b::dotdot(), b::int(2)]) + } + } + #[test] fn test_path() { let _ = pretty_env_logger::try_init(); equal_tokens! { - "$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")]) + "$it.print" -> b::token_list(vec![b::var("it"), b::dot(), b::bare("print")]) } equal_tokens! { - "$it.0" -> b::token_list(vec![b::var("it"), b::op("."), b::int(0)]) + "$it.0" -> b::token_list(vec![b::var("it"), b::dot(), b::int(0)]) } equal_tokens! { - "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")]) + "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::dot(), b::bare("part1"), b::dot(), b::bare("part2")]) } equal_tokens! { - "( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::bare("world")]) + "( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::dot(), b::bare("world")]) } equal_tokens! { - r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::string("world")]) + r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::dot(), b::string("world")]) } } @@ -1042,11 +1357,11 @@ mod tests { b::parens(vec![ b::sp(), b::var("it"), - b::op("."), + b::dot(), b::bare("is"), - b::op("."), + b::dot(), b::string("great news"), - b::op("."), + b::dot(), b::bare("right"), b::sp(), b::bare("yep"), @@ -1054,7 +1369,7 @@ mod tests { b::var("yep"), b::sp() ]), - b::op("."), b::string("world")] + b::dot(), b::string("world")] ) } @@ -1063,9 +1378,9 @@ mod tests { r#"$it."are PAS".0"# -> b::token_list( vec![ b::var("it"), - b::op("."), + b::dot(), b::string("are PAS"), - b::op("."), + b::dot(), b::int(0), ] ) @@ -1076,17 +1391,17 @@ mod tests { fn test_smoke_single_command() { equal_tokens! { - "git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::op(".")]) + "git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::bare(".")]) } equal_tokens! { - "open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml")]) + "open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml")]) } equal_tokens! { - "select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::op("."), b::bare("version")]) + "select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::dot(), b::bare("version")]) } equal_tokens! { @@ -1096,12 +1411,12 @@ mod tests { equal_tokens! { - "open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::flag("raw")]) + "open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml"), b::sp(), b::flag("raw")]) } equal_tokens! { - "open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::shorthand("r")]) + "open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml"), b::sp(), b::shorthand("r")]) } equal_tokens! { @@ -1117,7 +1432,7 @@ mod tests { b::sp(), b::flag("patch"), b::sp(), - b::bare("package"), b::op("."), b::bare("version") + b::bare("package"), b::dot(), b::bare("version") ] ) } @@ -1197,7 +1512,7 @@ mod tests { b::bare("where"), b::sp(), b::bare("cpu"), - b::op("."), + b::dot(), b::string("max ghz"), b::sp(), b::op(">"), @@ -1207,6 +1522,34 @@ mod tests { ); } + #[test] + fn test_signature() { + let _ = pretty_env_logger::try_init(); + + equal_tokens!( + + "def cd\n # Change to a new path.\n optional directory(Path) # the directory to change to\nend" -> + b::token_list(vec![ + b::bare("def"), + b::sp(), + b::bare("cd"), + b::sep("\n"), + b::ws(" "), + b::comment(" Change to a new path."), + b::sep("\n"), + b::ws(" "), + b::bare("optional"), + b::sp(), + b::bare("directory"), + b::parens(vec![b::bare("Path")]), + b::sp(), + b::comment(" the directory to change to"), + b::sep("\n"), + b::bare("end") + ]) + ); + } + // #[test] // fn test_smoke_pipeline() { // let _ = pretty_env_logger::try_init(); @@ -1279,7 +1622,18 @@ mod tests { desc: &str, string: &str, ) -> TokenNode { - f(nom_input(string)).unwrap().1 + let result = f(nom_input(string)); + + match result { + Ok(value) => value.1, + Err(err) => { + let err = nu_errors::ShellError::parse_error(err); + + println!("{:?}", string); + crate::hir::baseline_parse::tests::print_err(err, &nu_source::Text::from(string)); + panic!("test failed") + } + } } fn span((left, right): (usize, usize)) -> Span { diff --git a/crates/nu-parser/src/parse/token_tree.rs b/crates/nu-parser/src/parse/token_tree.rs index 2869a4b7ae..012c548b1b 100644 --- a/crates/nu-parser/src/parse/token_tree.rs +++ b/crates/nu-parser/src/parse/token_tree.rs @@ -1,4 +1,4 @@ -use crate::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; +use crate::parse::{call_node::*, comment::*, flag::*, operator::*, pipeline::*, tokens::*}; use derive_new::new; use getset::Getters; use nu_errors::{ParseError, ShellError}; @@ -18,7 +18,9 @@ pub enum TokenNode { Delimited(Spanned), Pipeline(Pipeline), Flag(Flag), + Comment(Comment), Whitespace(Span), + Separator(Span), Error(Spanned), } @@ -39,14 +41,32 @@ impl PrettyDebugWithSource for TokenNode { "whitespace", b::description(format!("{:?}", space.slice(source))), ), + TokenNode::Separator(span) => b::typed( + "separator", + b::description(format!("{:?}", span.slice(source))), + ), + TokenNode::Comment(comment) => { + b::typed("comment", b::description(comment.text.slice(source))) + } TokenNode::Error(_) => b::error("error"), } } } -impl HasSpan for TokenNode { - fn span(&self) -> Span { - self.get_span() +impl ShellTypeName for TokenNode { + fn type_name(&self) -> &'static str { + match self { + TokenNode::Token(t) => t.type_name(), + TokenNode::Nodes(_) => "nodes", + TokenNode::Call(_) => "command", + TokenNode::Delimited(d) => d.type_name(), + TokenNode::Pipeline(_) => "pipeline", + TokenNode::Flag(_) => "flag", + TokenNode::Whitespace(_) => "whitespace", + TokenNode::Separator(_) => "separator", + TokenNode::Comment(_) => "comment", + TokenNode::Error(_) => "error", + } } } @@ -107,12 +127,12 @@ impl fmt::Debug for DebugTokenNode<'_> { impl From<&TokenNode> for Span { fn from(token: &TokenNode) -> Span { - token.get_span() + token.span() } } -impl TokenNode { - pub fn get_span(&self) -> Span { +impl HasSpan for TokenNode { + fn span(&self) -> Span { match self { TokenNode::Token(t) => t.span, TokenNode::Nodes(t) => t.span, @@ -121,27 +141,14 @@ impl TokenNode { TokenNode::Pipeline(s) => s.span, TokenNode::Flag(s) => s.span, TokenNode::Whitespace(s) => *s, + TokenNode::Separator(s) => *s, + TokenNode::Comment(c) => c.span(), TokenNode::Error(s) => s.span, } } +} - pub fn type_name(&self) -> &'static str { - match self { - TokenNode::Token(t) => t.type_name(), - TokenNode::Nodes(_) => "nodes", - TokenNode::Call(_) => "command", - TokenNode::Delimited(d) => d.type_name(), - TokenNode::Pipeline(_) => "pipeline", - TokenNode::Flag(_) => "flag", - TokenNode::Whitespace(_) => "whitespace", - TokenNode::Error(_) => "error", - } - } - - pub fn spanned_type_name(&self) -> Spanned<&'static str> { - self.type_name().spanned(self.span()) - } - +impl TokenNode { pub fn tagged_type_name(&self) -> Tagged<&'static str> { self.type_name().tagged(self.span()) } @@ -244,7 +251,7 @@ impl TokenNode { pub fn is_dot(&self) -> bool { match self { TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), .. }) => true, _ => false, @@ -421,7 +428,7 @@ impl TokenNode { pub fn expect_dot(&self) -> Span { match self { TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), span, }) => *span, other => panic!("Expected dot, found {:?}", other), diff --git a/crates/nu-parser/src/parse/token_tree_builder.rs b/crates/nu-parser/src/parse/token_tree_builder.rs index 7b82cb4131..f88cb8b8e6 100644 --- a/crates/nu-parser/src/parse/token_tree_builder.rs +++ b/crates/nu-parser/src/parse/token_tree_builder.rs @@ -1,6 +1,7 @@ use crate::parse::call_node::CallNode; +use crate::parse::comment::Comment; use crate::parse::flag::{Flag, FlagKind}; -use crate::parse::operator::Operator; +use crate::parse::operator::{CompareOperator, EvaluationOperator}; use crate::parse::pipeline::{Pipeline, PipelineElement}; use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; use crate::parse::tokens::{RawNumber, UnspannedToken}; @@ -96,7 +97,7 @@ impl TokenTreeBuilder { TokenNode::Nodes(input.spanned(span.into())) } - pub fn op(input: impl Into) -> CurriedToken { + pub fn op(input: impl Into) -> CurriedToken { let input = input.into(); Box::new(move |b| { @@ -104,12 +105,39 @@ impl TokenTreeBuilder { b.pos = end; - TokenTreeBuilder::spanned_op(input, Span::new(start, end)) + TokenTreeBuilder::spanned_cmp_op(input, Span::new(start, end)) }) } - pub fn spanned_op(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(UnspannedToken::Operator(input.into()).into_token(span)) + pub fn spanned_cmp_op(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(UnspannedToken::CompareOperator(input.into()).into_token(span)) + } + + pub fn dot() -> CurriedToken { + Box::new(move |b| { + let (start, end) = b.consume("."); + + b.pos = end; + + TokenTreeBuilder::spanned_eval_op(".", Span::new(start, end)) + }) + } + + pub fn dotdot() -> CurriedToken { + Box::new(move |b| { + let (start, end) = b.consume(".."); + + b.pos = end; + + TokenTreeBuilder::spanned_eval_op("..", Span::new(start, end)) + }) + } + + pub fn spanned_eval_op( + input: impl Into, + span: impl Into, + ) -> TokenNode { + TokenNode::Token(UnspannedToken::EvaluationOperator(input.into()).into_token(span)) } pub fn string(input: impl Into) -> CurriedToken { @@ -398,6 +426,36 @@ impl TokenTreeBuilder { TokenNode::Whitespace(span.into()) } + pub fn sep(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&input); + TokenTreeBuilder::spanned_sep(Span::new(start, end)) + }) + } + + pub fn spanned_sep(span: impl Into) -> TokenNode { + TokenNode::Separator(span.into()) + } + + pub fn comment(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let outer_start = b.pos; + b.consume("#"); + let (start, end) = b.consume(&input); + let outer_end = b.pos; + + TokenTreeBuilder::spanned_comment((start, end), (outer_start, outer_end)) + }) + } + + pub fn spanned_comment(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Comment(Comment::line(input, span)) + } + fn consume(&mut self, input: &str) -> (usize, usize) { let start = self.pos; self.pos += input.len(); diff --git a/crates/nu-parser/src/parse/tokens.rs b/crates/nu-parser/src/parse/tokens.rs index 8430725313..4cb958c957 100644 --- a/crates/nu-parser/src/parse/tokens.rs +++ b/crates/nu-parser/src/parse/tokens.rs @@ -1,5 +1,5 @@ use crate::parse::parser::Number; -use crate::Operator; +use crate::{CompareOperator, EvaluationOperator}; use bigdecimal::BigDecimal; use nu_protocol::ShellTypeName; use nu_source::{ @@ -13,7 +13,8 @@ use std::str::FromStr; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum UnspannedToken { Number(RawNumber), - Operator(Operator), + CompareOperator(CompareOperator), + EvaluationOperator(EvaluationOperator), String(Span), Variable(Span), ExternalCommand(Span), @@ -35,7 +36,9 @@ impl ShellTypeName for UnspannedToken { fn type_name(&self) -> &'static str { match self { UnspannedToken::Number(_) => "number", - UnspannedToken::Operator(..) => "operator", + UnspannedToken::CompareOperator(..) => "comparison operator", + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => "dot", + UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => "dotdot", UnspannedToken::String(_) => "string", UnspannedToken::Variable(_) => "variable", UnspannedToken::ExternalCommand(_) => "syntax error", @@ -111,7 +114,8 @@ impl PrettyDebugWithSource for Token { fn pretty_debug(&self, source: &str) -> DebugDocBuilder { match self.unspanned { UnspannedToken::Number(number) => number.pretty_debug(source), - UnspannedToken::Operator(operator) => operator.pretty(), + UnspannedToken::CompareOperator(operator) => operator.pretty(), + UnspannedToken::EvaluationOperator(operator) => operator.pretty(), UnspannedToken::String(_) => b::primitive(self.span.slice(source)), UnspannedToken::Variable(_) => b::var(self.span.slice(source)), UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)), @@ -155,9 +159,9 @@ impl Token { } } - pub fn extract_operator(&self) -> Option> { + pub fn extract_operator(&self) -> Option> { match self.unspanned { - UnspannedToken::Operator(operator) => Some(operator.spanned(self.span)), + UnspannedToken::CompareOperator(operator) => Some(operator.spanned(self.span)), _ => None, } } diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs index f97fed9139..f77dc6f85a 100644 --- a/crates/nu-protocol/src/lib.rs +++ b/crates/nu-protocol/src/lib.rs @@ -8,6 +8,7 @@ mod return_value; mod signature; mod syntax_shape; mod type_name; +mod type_shape; mod value; pub use crate::call_info::{CallInfo, EvaluatedArgs}; @@ -17,9 +18,11 @@ pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue}; pub use crate::signature::{NamedType, PositionalType, Signature}; pub use crate::syntax_shape::SyntaxShape; pub use crate::type_name::{PrettyType, ShellTypeName, SpannedTypeName}; +pub use crate::type_shape::{Row as RowType, Type}; pub use crate::value::column_path::{did_you_mean, ColumnPath, PathMember, UnspannedPathMember}; pub use crate::value::dict::{Dictionary, TaggedDictBuilder}; pub use crate::value::evaluate::{Evaluate, EvaluateTrait, Scope}; pub use crate::value::primitive::format_primitive; pub use crate::value::primitive::Primitive; +pub use crate::value::range::{Range, RangeInclusion}; pub use crate::value::{UntaggedValue, Value}; diff --git a/crates/nu-protocol/src/signature.rs b/crates/nu-protocol/src/signature.rs index 7919506131..b75f6a9d4c 100644 --- a/crates/nu-protocol/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,4 +1,5 @@ use crate::syntax_shape::SyntaxShape; +use crate::type_shape::Type; use indexmap::IndexMap; use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource}; use serde::{Deserialize, Serialize}; @@ -76,6 +77,8 @@ pub struct Signature { pub positional: Vec<(PositionalType, Description)>, pub rest_positional: Option<(SyntaxShape, Description)>, pub named: IndexMap, + pub yields: Option, + pub input: Option, pub is_filter: bool, } @@ -98,14 +101,16 @@ impl PrettyDebugWithSource for Signature { } impl Signature { - pub fn new(name: String) -> Signature { + pub fn new(name: impl Into) -> Signature { Signature { - name, + name: name.into(), usage: String::new(), positional: vec![], rest_positional: None, named: IndexMap::new(), is_filter: false, + yields: None, + input: None, } } @@ -186,4 +191,14 @@ impl Signature { self.rest_positional = Some((ty, desc.into())); self } + + pub fn yields(mut self, ty: Type) -> Signature { + self.yields = Some(ty); + self + } + + pub fn input(mut self, ty: Type) -> Signature { + self.input = Some(ty); + self + } } diff --git a/crates/nu-protocol/src/syntax_shape.rs b/crates/nu-protocol/src/syntax_shape.rs index 7f6d486bfd..3f1d50f8b9 100644 --- a/crates/nu-protocol/src/syntax_shape.rs +++ b/crates/nu-protocol/src/syntax_shape.rs @@ -8,6 +8,7 @@ pub enum SyntaxShape { Member, ColumnPath, Number, + Range, Int, Path, Pattern, @@ -22,6 +23,7 @@ impl PrettyDebug for SyntaxShape { SyntaxShape::Member => "member shape", SyntaxShape::ColumnPath => "column path shape", SyntaxShape::Number => "number shape", + SyntaxShape::Range => "range shape", SyntaxShape::Int => "integer shape", SyntaxShape::Path => "file path shape", SyntaxShape::Pattern => "pattern shape", diff --git a/crates/nu-protocol/src/type_shape.rs b/crates/nu-protocol/src/type_shape.rs new file mode 100644 index 0000000000..be4aaaaabe --- /dev/null +++ b/crates/nu-protocol/src/type_shape.rs @@ -0,0 +1,382 @@ +use crate::value::dict::Dictionary; +use crate::value::primitive::Primitive; +use crate::value::range::RangeInclusion; +use crate::value::{UntaggedValue, Value}; +use derive_new::new; +use nu_source::{b, DebugDoc, DebugDocBuilder, PrettyDebug}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::hash::Hash; + +/** + This file describes the structural types of the nushell system. + + Its primary purpose today is to identify "equivalent" values for the purpose + of merging rows into a single table or identify rows in a table that have the + same shape for reflection. +*/ + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, new)] +pub struct RangeType { + from: (Type, RangeInclusion), + to: (Type, RangeInclusion), +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub enum Type { + Nothing, + Int, + Range(Box), + Decimal, + Bytesize, + String, + Line, + ColumnPath, + Pattern, + Boolean, + Date, + Duration, + Path, + Binary, + + Row(Row), + Table(Vec), + + // TODO: Block arguments + Block, + // TODO: Error type + Error, + + // Stream markers (used as bookend markers rather than actual values) + BeginningOfStream, + EndOfStream, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, new)] +pub struct Row { + #[new(default)] + map: BTreeMap, +} + +impl Serialize for Row { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_map(self.map.iter()) + } +} + +impl<'de> Deserialize<'de> for Row { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct RowVisitor; + + impl<'de> serde::de::Visitor<'de> for RowVisitor { + type Value = Row; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a row") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut new_map = BTreeMap::new(); + + loop { + let entry = map.next_entry()?; + + match entry { + None => return Ok(Row { map: new_map }), + Some((key, value)) => { + new_map.insert(key, value); + } + } + } + } + } + deserializer.deserialize_map(RowVisitor) + } +} + +impl Type { + pub fn from_primitive(primitive: &Primitive) -> Type { + match primitive { + Primitive::Nothing => Type::Nothing, + Primitive::Int(_) => Type::Int, + Primitive::Range(range) => { + let (left_value, left_inclusion) = &range.from; + let (right_value, right_inclusion) = &range.to; + + let left_type = (Type::from_primitive(left_value), *left_inclusion); + let right_type = (Type::from_primitive(right_value), *right_inclusion); + + let range = RangeType::new(left_type, right_type); + Type::Range(Box::new(range)) + } + Primitive::Decimal(_) => Type::Decimal, + Primitive::Bytes(_) => Type::Bytesize, + Primitive::String(_) => Type::String, + Primitive::Line(_) => Type::Line, + Primitive::ColumnPath(_) => Type::ColumnPath, + Primitive::Pattern(_) => Type::Pattern, + Primitive::Boolean(_) => Type::Boolean, + Primitive::Date(_) => Type::Date, + Primitive::Duration(_) => Type::Duration, + Primitive::Path(_) => Type::Path, + Primitive::Binary(_) => Type::Binary, + Primitive::BeginningOfStream => Type::BeginningOfStream, + Primitive::EndOfStream => Type::EndOfStream, + } + } + + pub fn from_dictionary(dictionary: &Dictionary) -> Type { + let mut map = BTreeMap::new(); + + for (key, value) in dictionary.entries.iter() { + let column = Column::String(key.clone()); + map.insert(column, Type::from_value(value)); + } + + Type::Row(Row { map }) + } + + pub fn from_table<'a>(table: impl IntoIterator) -> Type { + let mut vec = vec![]; + + for item in table.into_iter() { + vec.push(Type::from_value(item)) + } + + Type::Table(vec) + } + + pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> Type { + match value.into() { + UntaggedValue::Primitive(p) => Type::from_primitive(p), + UntaggedValue::Row(row) => Type::from_dictionary(row), + UntaggedValue::Table(table) => Type::from_table(table.iter()), + UntaggedValue::Error(_) => Type::Error, + UntaggedValue::Block(_) => Type::Block, + } + } +} + +impl PrettyDebug for Type { + fn pretty(&self) -> DebugDocBuilder { + match self { + Type::Nothing => ty("nothing"), + Type::Int => ty("integer"), + Type::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + let left_bracket = b::delimiter(match left_inclusion { + RangeInclusion::Exclusive => "(", + RangeInclusion::Inclusive => "[", + }); + + let right_bracket = b::delimiter(match right_inclusion { + RangeInclusion::Exclusive => ")", + RangeInclusion::Inclusive => "]", + }); + + b::typed( + "range", + (left_bracket + + left.pretty() + + b::operator(",") + + b::space() + + right.pretty() + + right_bracket) + .group(), + ) + } + Type::Decimal => ty("decimal"), + Type::Bytesize => ty("bytesize"), + Type::String => ty("string"), + Type::Line => ty("line"), + Type::ColumnPath => ty("column-path"), + Type::Pattern => ty("pattern"), + Type::Boolean => ty("boolean"), + Type::Date => ty("date"), + Type::Duration => ty("duration"), + Type::Path => ty("path"), + Type::Binary => ty("binary"), + Type::Error => b::error("error"), + Type::BeginningOfStream => b::keyword("beginning-of-stream"), + Type::EndOfStream => b::keyword("end-of-stream"), + Type::Row(row) => (b::kind("row") + + b::space() + + b::intersperse( + row.map.iter().map(|(key, ty)| { + (b::key(match key { + Column::String(string) => string.clone(), + Column::Value => "".to_string(), + }) + b::delimit("(", ty.pretty(), ")").into_kind()) + .nest() + }), + b::space(), + ) + .nest()) + .nest(), + + Type::Table(table) => { + let mut group: Group> = Group::new(); + + for (i, item) in table.iter().enumerate() { + group.add(item.to_doc(), i); + } + + (b::kind("table") + b::space() + b::keyword("of")).group() + + b::space() + + (if group.len() == 1 { + let (doc, _) = group.into_iter().nth(0).unwrap(); + DebugDocBuilder::from_doc(doc) + } else { + b::intersperse( + group.into_iter().map(|(doc, rows)| { + (b::intersperse( + rows.iter().map(|(from, to)| { + if from == to { + b::description(from) + } else { + (b::description(from) + + b::space() + + b::keyword("to") + + b::space() + + b::description(to)) + .group() + } + }), + b::description(", "), + ) + b::description(":") + + b::space() + + DebugDocBuilder::from_doc(doc)) + .nest() + }), + b::space(), + ) + }) + } + Type::Block => ty("block"), + } + } +} + +#[derive(Debug, new)] +struct DebugEntry<'a> { + key: &'a Column, + value: &'a Type, +} + +impl<'a> PrettyDebug for DebugEntry<'a> { + fn pretty(&self) -> DebugDocBuilder { + (b::key(match self.key { + Column::String(string) => string.clone(), + Column::Value => format!(""), + }) + b::delimit("(", self.value.pretty(), ")").into_kind()) + } +} + +fn ty(name: impl std::fmt::Display) -> DebugDocBuilder { + b::kind(format!("{}", name)) +} + +pub trait GroupedValue: Debug + Clone { + type Item; + + fn new() -> Self; + fn merge(&mut self, value: Self::Item); +} + +impl GroupedValue for Vec<(usize, usize)> { + type Item = usize; + + fn new() -> Vec<(usize, usize)> { + vec![] + } + + fn merge(&mut self, new_value: usize) { + match self.last_mut() { + Some(value) if value.1 == new_value - 1 => { + value.1 += 1; + } + + _ => self.push((new_value, new_value)), + } + } +} + +#[derive(Debug)] +pub struct Group { + values: indexmap::IndexMap, +} + +impl Group +where + K: Debug + Eq + Hash, + G: GroupedValue, +{ + pub fn new() -> Group { + Group { + values: indexmap::IndexMap::default(), + } + } + + pub fn len(&self) -> usize { + self.values.len() + } + + pub fn into_iter(self) -> impl Iterator { + self.values.into_iter() + } + + pub fn add(&mut self, key: impl Into, value: impl Into) { + let key = key.into(); + let value = value.into(); + + let group = self.values.get_mut(&key); + + match group { + None => { + self.values.insert(key, { + let mut group = G::new(); + group.merge(value.into()); + group + }); + } + Some(group) => { + group.merge(value.into()); + } + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] +pub enum Column { + String(String), + Value, +} + +impl Into for String { + fn into(self) -> Column { + Column::String(self) + } +} + +impl Into for &String { + fn into(self) -> Column { + Column::String(self.clone()) + } +} + +impl Into for &str { + fn into(self) -> Column { + Column::String(self.to_string()) + } +} diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index c5c6c47368..5082799996 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -4,6 +4,7 @@ mod debug; pub mod dict; pub mod evaluate; pub mod primitive; +pub mod range; mod serde_bigdecimal; mod serde_bigint; @@ -11,11 +12,12 @@ use crate::type_name::{ShellTypeName, SpannedTypeName}; use crate::value::dict::Dictionary; use crate::value::evaluate::Evaluate; use crate::value::primitive::Primitive; +use crate::value::range::{Range, RangeInclusion}; use crate::{ColumnPath, PathMember}; use bigdecimal::BigDecimal; use indexmap::IndexMap; use nu_errors::ShellError; -use nu_source::{AnchorLocation, HasSpan, Span, Tag}; +use nu_source::{AnchorLocation, HasSpan, Span, Spanned, Tag}; use num_bigint::BigInt; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -156,6 +158,13 @@ impl UntaggedValue { UntaggedValue::Primitive(Primitive::Binary(binary)) } + pub fn range( + left: (Spanned, RangeInclusion), + right: (Spanned, RangeInclusion), + ) -> UntaggedValue { + UntaggedValue::Primitive(Primitive::Range(Box::new(Range::new(left, right)))) + } + pub fn boolean(s: impl Into) -> UntaggedValue { UntaggedValue::Primitive(Primitive::Boolean(s.into())) } @@ -224,6 +233,23 @@ impl Value { _ => Err(ShellError::type_error("Path", self.spanned_type_name())), } } + + pub fn as_primitive(&self) -> Result { + match &self.value { + UntaggedValue::Primitive(primitive) => Ok(primitive.clone()), + _ => Err(ShellError::type_error( + "Primitive", + self.spanned_type_name(), + )), + } + } + + pub fn as_u64(&self) -> Result { + match &self.value { + UntaggedValue::Primitive(primitive) => primitive.as_u64(self.tag.span), + _ => Err(ShellError::type_error("integer", self.spanned_type_name())), + } + } } impl Into for &str { diff --git a/crates/nu-protocol/src/value/debug.rs b/crates/nu-protocol/src/value/debug.rs index f71be77124..f089b82b2c 100644 --- a/crates/nu-protocol/src/value/debug.rs +++ b/crates/nu-protocol/src/value/debug.rs @@ -28,6 +28,7 @@ impl PrettyType for Primitive { match self { Primitive::Nothing => ty("nothing"), Primitive::Int(_) => ty("integer"), + Primitive::Range(_) => ty("range"), Primitive::Decimal(_) => ty("decimal"), Primitive::Bytes(_) => ty("bytesize"), Primitive::String(_) => ty("string"), @@ -51,6 +52,21 @@ impl PrettyDebug for Primitive { Primitive::Nothing => b::primitive("nothing"), Primitive::Int(int) => prim(format_args!("{}", int)), Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)), + Primitive::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + b::typed( + "range", + (left_inclusion.debug_left_bracket() + + left.pretty() + + b::operator(",") + + b::space() + + right.pretty() + + right_inclusion.debug_right_bracket()) + .group(), + ) + } Primitive::Bytes(bytes) => primitive_doc(bytes, "bytesize"), Primitive::String(string) => prim(string), Primitive::Line(string) => prim(string), diff --git a/crates/nu-protocol/src/value/primitive.rs b/crates/nu-protocol/src/value/primitive.rs index 92bb3ee05d..30bf7c5222 100644 --- a/crates/nu-protocol/src/value/primitive.rs +++ b/crates/nu-protocol/src/value/primitive.rs @@ -1,12 +1,14 @@ use crate::type_name::ShellTypeName; use crate::value::column_path::ColumnPath; +use crate::value::range::Range; use crate::value::{serde_bigdecimal, serde_bigint}; use bigdecimal::BigDecimal; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; -use nu_source::PrettyDebug; +use nu_errors::{ExpectedRange, ShellError}; +use nu_source::{PrettyDebug, Span, SpannedItem}; use num_bigint::BigInt; -use num_traits::cast::FromPrimitive; +use num_traits::cast::{FromPrimitive, ToPrimitive}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -25,6 +27,7 @@ pub enum Primitive { Boolean(bool), Date(DateTime), Duration(u64), // Duration in seconds + Range(Box), Path(PathBuf), #[serde(with = "serde_bytes")] Binary(Vec), @@ -34,6 +37,25 @@ pub enum Primitive { EndOfStream, } +impl Primitive { + pub fn as_u64(&self, span: Span) -> Result { + match self { + Primitive::Int(int) => match int.to_u64() { + None => Err(ShellError::range_error( + ExpectedRange::U64, + &format!("{}", int).spanned(span), + "converting an integer into a 64-bit integer", + )), + Some(num) => Ok(num), + }, + other => Err(ShellError::type_error( + "integer", + other.type_name().spanned(span), + )), + } + } +} + impl From for Primitive { fn from(decimal: BigDecimal) -> Primitive { Primitive::Decimal(decimal) @@ -51,6 +73,7 @@ impl ShellTypeName for Primitive { match self { Primitive::Nothing => "nothing", Primitive::Int(_) => "integer", + Primitive::Range(_) => "range", Primitive::Decimal(_) => "decimal", Primitive::Bytes(_) => "bytes", Primitive::String(_) => "string", @@ -91,6 +114,11 @@ pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> S Primitive::Duration(sec) => format_duration(*sec), Primitive::Int(i) => i.to_string(), Primitive::Decimal(decimal) => decimal.to_string(), + Primitive::Range(range) => format!( + "{}..{}", + format_primitive(&range.from.0.item, None), + format_primitive(&range.to.0.item, None) + ), Primitive::Pattern(s) => s.to_string(), Primitive::String(s) => s.to_owned(), Primitive::Line(s) => s.to_owned(), @@ -125,7 +153,8 @@ pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> S Primitive::Date(d) => d.humanize().to_string(), } } -fn format_duration(sec: u64) -> String { + +pub fn format_duration(sec: u64) -> String { let (minutes, seconds) = (sec / 60, sec % 60); let (hours, minutes) = (minutes / 60, minutes % 60); let (days, hours) = (hours / 24, hours % 24); diff --git a/crates/nu-protocol/src/value/range.rs b/crates/nu-protocol/src/value/range.rs new file mode 100644 index 0000000000..f1f9a90fa8 --- /dev/null +++ b/crates/nu-protocol/src/value/range.rs @@ -0,0 +1,32 @@ +use crate::value::Primitive; +use derive_new::new; +use nu_source::{b, DebugDocBuilder, Spanned}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize, Hash)] +pub enum RangeInclusion { + Inclusive, + Exclusive, +} + +impl RangeInclusion { + pub fn debug_left_bracket(&self) -> DebugDocBuilder { + b::delimiter(match self { + RangeInclusion::Exclusive => "(", + RangeInclusion::Inclusive => "[", + }) + } + + pub fn debug_right_bracket(&self) -> DebugDocBuilder { + b::delimiter(match self { + RangeInclusion::Exclusive => ")", + RangeInclusion::Inclusive => "]", + }) + } +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize, new)] +pub struct Range { + pub from: (Spanned, RangeInclusion), + pub to: (Spanned, RangeInclusion), +} diff --git a/crates/nu-source/src/meta.rs b/crates/nu-source/src/meta.rs index 70012cc635..bca7135da9 100644 --- a/crates/nu-source/src/meta.rs +++ b/crates/nu-source/src/meta.rs @@ -220,10 +220,7 @@ impl nom_locate::LocatedSpanEx, ), ) -> Span { - Span { - start: input.0.offset, - end: input.1.offset, - } + Span::new(input.0.offset, input.1.offset) } } @@ -235,10 +232,7 @@ impl From<(usize, usize)> for Span { impl From<&std::ops::Range> for Span { fn from(input: &std::ops::Range) -> Span { - Span { - start: input.start, - end: input.end, - } + Span::new(input.start, input.end) } } @@ -321,10 +315,7 @@ impl Tag { pub fn for_char(pos: usize, anchor: AnchorLocation) -> Tag { Tag { anchor: Some(anchor), - span: Span { - start: pos, - end: pos + 1, - }, + span: Span::new(pos, pos + 1), } } @@ -528,11 +519,19 @@ impl Span { impl language_reporting::ReportingSpan for Span { fn with_start(&self, start: usize) -> Self { - Span::new(start, self.end) + if self.end < start { + Span::new(start, start) + } else { + Span::new(start, self.end) + } } fn with_end(&self, end: usize) -> Self { - Span::new(self.start, end) + if end < self.start { + Span::new(end, end) + } else { + Span::new(self.start, end) + } } fn start(&self) -> usize { diff --git a/crates/nu_plugin_post/src/main.rs b/crates/nu_plugin_post/src/main.rs index 8e802a2995..8a5bd606ad 100644 --- a/crates/nu_plugin_post/src/main.rs +++ b/crates/nu_plugin_post/src/main.rs @@ -447,7 +447,9 @@ pub fn value_to_json_value(v: &Value) -> Result { UntaggedValue::Table(l) => serde_json::Value::Array(json_list(l)?), UntaggedValue::Error(e) => return Err(e.clone()), - UntaggedValue::Block(_) => serde_json::Value::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => { + serde_json::Value::Null + } UntaggedValue::Primitive(Primitive::Binary(b)) => serde_json::Value::Array( b.iter() .map(|x| { diff --git a/src/commands/cd.rs b/src/commands/cd.rs index b96619caed..a57356d7ce 100644 --- a/src/commands/cd.rs +++ b/src/commands/cd.rs @@ -1,6 +1,7 @@ use crate::commands::WholeStreamCommand; use crate::prelude::*; use nu_errors::ShellError; +use nu_macros::signature; use nu_protocol::{Signature, SyntaxShape}; pub struct CD; @@ -11,11 +12,17 @@ impl WholeStreamCommand for CD { } fn signature(&self) -> Signature { - Signature::build("cd").optional( - "directory", - SyntaxShape::Path, - "the directory to change to", - ) + signature! { + def cd { + "the directory to change to" + directory(optional Path) - "the directory to change to" + } + } + // Signature::build("cd").optional( + // "directory", + // SyntaxShape::Path, + // "the directory to change to", + // ) } fn usage(&self) -> &str { diff --git a/src/commands/command.rs b/src/commands/command.rs index ef5443a3ed..5873d5ea20 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -378,14 +378,7 @@ pub trait WholeStreamCommand: Send + Sync { fn name(&self) -> &str; fn signature(&self) -> Signature { - Signature { - name: self.name().to_string(), - usage: self.usage().to_string(), - positional: vec![], - rest_positional: None, - named: indexmap::IndexMap::new(), - is_filter: true, - } + Signature::new(self.name()).desc(self.usage()).filter() } fn usage(&self) -> &str; @@ -405,14 +398,7 @@ pub trait PerItemCommand: Send + Sync { fn name(&self) -> &str; fn signature(&self) -> Signature { - Signature { - name: self.name().to_string(), - usage: self.usage().to_string(), - positional: vec![], - rest_positional: None, - named: indexmap::IndexMap::new(), - is_filter: true, - } + Signature::new(self.name()).desc(self.usage()).filter() } fn usage(&self) -> &str; diff --git a/src/commands/range.rs b/src/commands/range.rs index edb355451e..c2e9a14d83 100644 --- a/src/commands/range.rs +++ b/src/commands/range.rs @@ -1,5 +1,6 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; +use crate::deserializer::NumericRange; use crate::prelude::*; use nu_errors::ShellError; use nu_protocol::{Signature, SyntaxShape}; @@ -7,7 +8,7 @@ use nu_source::Tagged; #[derive(Deserialize)] struct RangeArgs { - area: Tagged, + area: Tagged, } pub struct Range; @@ -20,7 +21,7 @@ impl WholeStreamCommand for Range { fn signature(&self) -> Signature { Signature::build("range").required( "rows ", - SyntaxShape::Any, + SyntaxShape::Range, "range of rows to return: Eg) 4..7 (=> from 4 to 7)", ) } @@ -39,48 +40,14 @@ impl WholeStreamCommand for Range { } fn range( - RangeArgs { area: rows }: RangeArgs, - RunnableContext { input, name, .. }: RunnableContext, + RangeArgs { area }: RangeArgs, + RunnableContext { input, name: _, .. }: RunnableContext, ) -> Result { - match rows.item.find('.') { - Some(value) => { - let (first, last) = rows.item.split_at(value); - let first = match first.parse::() { - Ok(postion) => postion, - Err(_) => { - if first == "" { - 0 - } else { - return Err(ShellError::labeled_error( - "no correct start of range", - "'from' needs to be an Integer or empty", - name, - )); - } - } - }; - let last = match last.trim_start_matches('.').parse::() { - Ok(postion) => postion, - Err(_) => { - if last == ".." { - std::u64::MAX - 1 - } else { - return Err(ShellError::labeled_error( - "no correct end of range", - "'to' needs to be an Integer or empty", - name, - )); - } - } - }; - Ok(OutputStream::from_input( - input.values.skip(first).take(last - first + 1), - )) - } - None => Err(ShellError::labeled_error( - "No correct formatted range found", - "format: ..", - name, - )), - } + let range = area.item; + let (from, _) = range.from; + let (to, _) = range.to; + + return Ok(OutputStream::from_input( + input.values.skip(*from).take(*to - *from + 1), + )); } diff --git a/src/commands/to_bson.rs b/src/commands/to_bson.rs index 3fdc7e89b9..3b783b7af4 100644 --- a/src/commands/to_bson.rs +++ b/src/commands/to_bson.rs @@ -73,7 +73,7 @@ pub fn value_to_bson_value(v: &Value) -> Result { .map(|x| value_to_bson_value(x)) .collect::>()?, ), - UntaggedValue::Block(_) => Bson::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => Bson::Null, UntaggedValue::Error(e) => return Err(e.clone()), UntaggedValue::Primitive(Primitive::Binary(b)) => { Bson::Binary(BinarySubtype::Generic, b.clone()) diff --git a/src/commands/to_json.rs b/src/commands/to_json.rs index dfa4aef5c1..929fa8f195 100644 --- a/src/commands/to_json.rs +++ b/src/commands/to_json.rs @@ -76,7 +76,9 @@ pub fn value_to_json_value(v: &Value) -> Result { UntaggedValue::Table(l) => serde_json::Value::Array(json_list(l)?), UntaggedValue::Error(e) => return Err(e.clone()), - UntaggedValue::Block(_) => serde_json::Value::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => { + serde_json::Value::Null + } UntaggedValue::Primitive(Primitive::Binary(b)) => serde_json::Value::Array( b.iter() .map(|x| { diff --git a/src/commands/to_sqlite.rs b/src/commands/to_sqlite.rs index 6b76a7cb0f..9f6ef5546e 100644 --- a/src/commands/to_sqlite.rs +++ b/src/commands/to_sqlite.rs @@ -100,9 +100,10 @@ fn nu_value_to_sqlite_string(v: Value) -> String { Primitive::Date(d) => format!("'{}'", d), Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")), Primitive::Binary(u) => format!("x'{}'", encode(u)), - Primitive::BeginningOfStream | Primitive::EndOfStream | Primitive::ColumnPath(_) => { - "NULL".into() - } + Primitive::BeginningOfStream + | Primitive::EndOfStream + | Primitive::ColumnPath(_) + | Primitive::Range(_) => "NULL".into(), }, _ => "NULL".into(), } diff --git a/src/commands/to_toml.rs b/src/commands/to_toml.rs index 351602677a..48235723b3 100644 --- a/src/commands/to_toml.rs +++ b/src/commands/to_toml.rs @@ -69,6 +69,7 @@ pub fn value_to_toml_value(v: &Value) -> Result { UntaggedValue::Table(l) => toml::Value::Array(collect_values(l)?), UntaggedValue::Error(e) => return Err(e.clone()), UntaggedValue::Block(_) => toml::Value::String("".to_string()), + UntaggedValue::Primitive(Primitive::Range(_)) => toml::Value::String("".to_string()), UntaggedValue::Primitive(Primitive::Binary(b)) => { toml::Value::Array(b.iter().map(|x| toml::Value::Integer(*x as i64)).collect()) } diff --git a/src/commands/to_yaml.rs b/src/commands/to_yaml.rs index aa21108d99..f0c1c0da10 100644 --- a/src/commands/to_yaml.rs +++ b/src/commands/to_yaml.rs @@ -85,7 +85,9 @@ pub fn value_to_yaml_value(v: &Value) -> Result { serde_yaml::Value::Sequence(out) } UntaggedValue::Error(e) => return Err(e.clone()), - UntaggedValue::Block(_) => serde_yaml::Value::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => { + serde_yaml::Value::Null + } UntaggedValue::Primitive(Primitive::Binary(b)) => serde_yaml::Value::Sequence( b.iter() .map(|x| serde_yaml::Value::Number(serde_yaml::Number::from(*x))) diff --git a/src/data/base.rs b/src/data/base.rs index f30397a956..4fdee475dc 100644 --- a/src/data/base.rs +++ b/src/data/base.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use derive_new::new; use log::trace; use nu_errors::ShellError; -use nu_parser::{hir, Operator}; +use nu_parser::{hir, CompareOperator}; use nu_protocol::{ Evaluate, EvaluateTrait, Primitive, Scope, ShellTypeName, SpannedTypeName, TaggedDictBuilder, UntaggedValue, Value, @@ -23,7 +23,7 @@ use std::time::SystemTime; #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new, Serialize)] pub struct Operation { pub(crate) left: Value, - pub(crate) operator: Operator, + pub(crate) operator: CompareOperator, pub(crate) right: Value, } diff --git a/src/data/base/shape.rs b/src/data/base/shape.rs index 44521c772a..f6ffac0e7d 100644 --- a/src/data/base/shape.rs +++ b/src/data/base/shape.rs @@ -1,207 +1,24 @@ use crate::prelude::*; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; -use derive_new::new; use indexmap::IndexMap; use nu_errors::ShellError; +use nu_protocol::RangeInclusion; use nu_protocol::{ format_primitive, ColumnPath, Dictionary, Evaluate, Primitive, ShellTypeName, TaggedDictBuilder, UntaggedValue, Value, }; -use nu_source::{b, DebugDoc, PrettyDebug}; +use nu_source::{b, PrettyDebug}; use std::collections::BTreeMap; use std::fmt::Debug; use std::hash::Hash; use std::io::Write; use std::path::PathBuf; -/** - This file describes the structural types of the nushell system. - - Its primary purpose today is to identify "equivalent" values for the purpose - of merging rows into a single table or identify rows in a table that have the - same shape for reflection. - - It also serves as the primary vehicle for pretty-printing. -*/ - -#[allow(unused)] #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum TypeShape { - Nothing, - Int, - Decimal, - Bytesize, - String, - Line, - ColumnPath, - Pattern, - Boolean, - Date, - Duration, - Path, - Binary, - - Row(BTreeMap), - Table(Vec), - - // TODO: Block arguments - Block, - // TODO: Error type - Error, - - // Stream markers (used as bookend markers rather than actual values) - BeginningOfStream, - EndOfStream, -} - -impl TypeShape { - pub fn from_primitive(primitive: &Primitive) -> TypeShape { - match primitive { - Primitive::Nothing => TypeShape::Nothing, - Primitive::Int(_) => TypeShape::Int, - Primitive::Decimal(_) => TypeShape::Decimal, - Primitive::Bytes(_) => TypeShape::Bytesize, - Primitive::String(_) => TypeShape::String, - Primitive::Line(_) => TypeShape::Line, - Primitive::ColumnPath(_) => TypeShape::ColumnPath, - Primitive::Pattern(_) => TypeShape::Pattern, - Primitive::Boolean(_) => TypeShape::Boolean, - Primitive::Date(_) => TypeShape::Date, - Primitive::Duration(_) => TypeShape::Duration, - Primitive::Path(_) => TypeShape::Path, - Primitive::Binary(_) => TypeShape::Binary, - Primitive::BeginningOfStream => TypeShape::BeginningOfStream, - Primitive::EndOfStream => TypeShape::EndOfStream, - } - } - - pub fn from_dictionary(dictionary: &Dictionary) -> TypeShape { - let mut map = BTreeMap::new(); - - for (key, value) in dictionary.entries.iter() { - let column = Column::String(key.clone()); - map.insert(column, TypeShape::from_value(value)); - } - - TypeShape::Row(map) - } - - pub fn from_table<'a>(table: impl IntoIterator) -> TypeShape { - let mut vec = vec![]; - - for item in table.into_iter() { - vec.push(TypeShape::from_value(item)) - } - - TypeShape::Table(vec) - } - - pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> TypeShape { - match value.into() { - UntaggedValue::Primitive(p) => TypeShape::from_primitive(p), - UntaggedValue::Row(row) => TypeShape::from_dictionary(row), - UntaggedValue::Table(table) => TypeShape::from_table(table.iter()), - UntaggedValue::Error(_) => TypeShape::Error, - UntaggedValue::Block(_) => TypeShape::Block, - } - } -} - -impl PrettyDebug for TypeShape { - fn pretty(&self) -> DebugDocBuilder { - match self { - TypeShape::Nothing => ty("nothing"), - TypeShape::Int => ty("integer"), - TypeShape::Decimal => ty("decimal"), - TypeShape::Bytesize => ty("bytesize"), - TypeShape::String => ty("string"), - TypeShape::Line => ty("line"), - TypeShape::ColumnPath => ty("column-path"), - TypeShape::Pattern => ty("pattern"), - TypeShape::Boolean => ty("boolean"), - TypeShape::Date => ty("date"), - TypeShape::Duration => ty("duration"), - TypeShape::Path => ty("path"), - TypeShape::Binary => ty("binary"), - TypeShape::Error => b::error("error"), - TypeShape::BeginningOfStream => b::keyword("beginning-of-stream"), - TypeShape::EndOfStream => b::keyword("end-of-stream"), - TypeShape::Row(row) => (b::kind("row") - + b::space() - + b::intersperse( - row.iter().map(|(key, ty)| { - (b::key(match key { - Column::String(string) => string.clone(), - Column::Value => "".to_string(), - }) + b::delimit("(", ty.pretty(), ")").into_kind()) - .nest() - }), - b::space(), - ) - .nest()) - .nest(), - - TypeShape::Table(table) => { - let mut group: Group> = Group::new(); - - for (i, item) in table.iter().enumerate() { - group.add(item.to_doc(), i); - } - - (b::kind("table") + b::space() + b::keyword("of")).group() - + b::space() - + (if group.len() == 1 { - let (doc, _) = group.into_iter().nth(0).unwrap(); - DebugDocBuilder::from_doc(doc) - } else { - b::intersperse( - group.into_iter().map(|(doc, rows)| { - (b::intersperse( - rows.iter().map(|(from, to)| { - if from == to { - b::description(from) - } else { - (b::description(from) - + b::space() - + b::keyword("to") - + b::space() - + b::description(to)) - .group() - } - }), - b::description(", "), - ) + b::description(":") - + b::space() - + DebugDocBuilder::from_doc(doc)) - .nest() - }), - b::space(), - ) - }) - } - TypeShape::Block => ty("block"), - } - } -} - -#[derive(Debug, new)] -struct DebugEntry<'a> { - key: &'a Column, - value: &'a TypeShape, -} - -impl<'a> PrettyDebug for DebugEntry<'a> { - fn pretty(&self) -> DebugDocBuilder { - (b::key(match self.key { - Column::String(string) => string.clone(), - Column::Value => "".to_owned(), - }) + b::delimit("(", self.value.pretty(), ")").into_kind()) - } -} - -fn ty(name: impl std::fmt::Display) -> DebugDocBuilder { - b::kind(format!("{}", name)) +pub struct InlineRange { + from: (InlineShape, RangeInclusion), + to: (InlineShape, RangeInclusion), } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] @@ -209,6 +26,7 @@ pub enum InlineShape { Nothing, Int(BigInt), Decimal(BigDecimal), + Range(Box), Bytesize(u64), String(String), Line(String), @@ -243,6 +61,15 @@ impl InlineShape { match primitive { Primitive::Nothing => InlineShape::Nothing, Primitive::Int(int) => InlineShape::Int(int.clone()), + Primitive::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + InlineShape::Range(Box::new(InlineRange { + from: (InlineShape::from_primitive(left), *left_inclusion), + to: (InlineShape::from_primitive(right), *right_inclusion), + })) + } Primitive::Decimal(decimal) => InlineShape::Decimal(decimal.clone()), Primitive::Bytes(bytesize) => InlineShape::Bytesize(*bytesize), Primitive::String(string) => InlineShape::String(string.clone()), @@ -314,6 +141,17 @@ impl PrettyDebug for FormatInlineShape { InlineShape::Nothing => b::blank(), InlineShape::Int(int) => b::primitive(format!("{}", int)), InlineShape::Decimal(decimal) => b::primitive(format!("{}", decimal)), + InlineShape::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + let op = match (left_inclusion, right_inclusion) { + (RangeInclusion::Inclusive, RangeInclusion::Exclusive) => "..", + _ => unimplemented!("No syntax for ranges that aren't inclusive on the left and exclusive on the right") + }; + + left.clone().format().pretty() + b::operator(op) + right.clone().format().pretty() + } InlineShape::Bytesize(bytesize) => { let byte = byte_unit::Byte::from_bytes(*bytesize as u128); @@ -411,51 +249,6 @@ impl GroupedValue for Vec<(usize, usize)> { } } -#[derive(Debug)] -pub struct Group { - values: indexmap::IndexMap, -} - -impl Group -where - K: Debug + Eq + Hash, - G: GroupedValue, -{ - pub fn new() -> Group { - Group { - values: indexmap::IndexMap::default(), - } - } - - pub fn len(&self) -> usize { - self.values.len() - } - - pub fn into_iter(self) -> impl Iterator { - self.values.into_iter() - } - - pub fn add(&mut self, key: impl Into, value: impl Into) { - let key = key.into(); - let value = value.into(); - - let group = self.values.get_mut(&key); - - match group { - None => { - self.values.insert(key, { - let mut group = G::new(); - group.merge(value); - group - }); - } - Some(group) => { - group.merge(value); - } - } - } -} - #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum Column { String(String), diff --git a/src/data/value.rs b/src/data/value.rs index 8c8da375b8..aa1eb3d521 100644 --- a/src/data/value.rs +++ b/src/data/value.rs @@ -1,10 +1,10 @@ use crate::data::base::coerce_compare; -use crate::data::base::shape::{Column, InlineShape, TypeShape}; +use crate::data::base::shape::{Column, InlineShape}; use crate::data::primitive::style_primitive; use chrono::DateTime; use nu_errors::ShellError; -use nu_parser::Operator; -use nu_protocol::{Primitive, UntaggedValue}; +use nu_parser::CompareOperator; +use nu_protocol::{Primitive, Type, UntaggedValue}; use nu_source::{DebugDocBuilder, PrettyDebug, Tagged}; pub fn date_from_str(s: Tagged<&str>) -> Result { @@ -22,7 +22,7 @@ pub fn date_from_str(s: Tagged<&str>) -> Result { } pub fn compare_values( - operator: Operator, + operator: &CompareOperator, left: &UntaggedValue, right: &UntaggedValue, ) -> Result { @@ -34,16 +34,15 @@ pub fn compare_values( use std::cmp::Ordering; let result = match (operator, ordering) { - (Operator::Equal, Ordering::Equal) => true, - (Operator::NotEqual, Ordering::Less) | (Operator::NotEqual, Ordering::Greater) => { - true - } - (Operator::LessThan, Ordering::Less) => true, - (Operator::GreaterThan, Ordering::Greater) => true, - (Operator::GreaterThanOrEqual, Ordering::Greater) - | (Operator::GreaterThanOrEqual, Ordering::Equal) => true, - (Operator::LessThanOrEqual, Ordering::Less) - | (Operator::LessThanOrEqual, Ordering::Equal) => true, + (CompareOperator::Equal, Ordering::Equal) => true, + (CompareOperator::NotEqual, Ordering::Less) + | (CompareOperator::NotEqual, Ordering::Greater) => true, + (CompareOperator::LessThan, Ordering::Less) => true, + (CompareOperator::GreaterThan, Ordering::Greater) => true, + (CompareOperator::GreaterThanOrEqual, Ordering::Greater) + | (CompareOperator::GreaterThanOrEqual, Ordering::Equal) => true, + (CompareOperator::LessThanOrEqual, Ordering::Less) + | (CompareOperator::LessThanOrEqual, Ordering::Equal) => true, _ => false, }; @@ -53,7 +52,7 @@ pub fn compare_values( } pub fn format_type<'a>(value: impl Into<&'a UntaggedValue>, width: usize) -> String { - TypeShape::from_value(value.into()).colored_string(width) + Type::from_value(value.into()).colored_string(width) } pub fn format_leaf<'a>(value: impl Into<&'a UntaggedValue>) -> DebugDocBuilder { diff --git a/src/deserializer.rs b/src/deserializer.rs index 0b23b81657..b82ad64a1f 100644 --- a/src/deserializer.rs +++ b/src/deserializer.rs @@ -1,11 +1,20 @@ use log::trace; use nu_errors::{CoerceInto, ShellError}; -use nu_protocol::{CallInfo, ColumnPath, Evaluate, Primitive, ShellTypeName, UntaggedValue, Value}; -use nu_source::{HasSpan, SpannedItem, Tagged, TaggedItem}; +use nu_protocol::{ + CallInfo, ColumnPath, Evaluate, Primitive, RangeInclusion, ShellTypeName, UntaggedValue, Value, +}; +use nu_source::{HasSpan, Spanned, SpannedItem, Tagged, TaggedItem}; use nu_value_ext::ValueExt; use serde::de; +use serde::{Deserialize, Serialize}; use std::path::PathBuf; +#[derive(Copy, Clone, Deserialize, Serialize)] +pub struct NumericRange { + pub from: (Spanned, RangeInclusion), + pub to: (Spanned, RangeInclusion), +} + #[derive(Debug)] pub struct DeserializerItem<'de> { key_struct_field: Option<(String, &'de str)>, @@ -406,6 +415,25 @@ impl<'de, 'a> de::Deserializer<'de> for &'a mut ConfigDeserializer<'de> { value: UntaggedValue::Primitive(Primitive::String(string)), .. } => visit::, _>(string.tagged(tag), name, fields, visitor), + Value { + value: UntaggedValue::Primitive(Primitive::Range(range)), + .. + } => { + let (left, left_inclusion) = range.from; + let (right, right_inclusion) = range.to; + let left_span = left.span; + let right_span = right.span; + + let left = left.as_u64(left_span)?; + let right = right.as_u64(right_span)?; + + let numeric_range = NumericRange { + from: (left.spanned(left_span), left_inclusion), + to: (right.spanned(right_span), right_inclusion), + }; + + visit::, _>(numeric_range.tagged(tag), name, fields, visitor) + } other => Err(ShellError::type_error( name, diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index c3f89dfd38..03808f2a84 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -6,8 +6,8 @@ use log::trace; use nu_errors::{ArgumentError, ShellError}; use nu_parser::hir::{self, Expression, RawExpression}; use nu_protocol::{ - ColumnPath, Evaluate, Primitive, Scope, TaggedDictBuilder, UnspannedPathMember, UntaggedValue, - Value, + ColumnPath, Evaluate, Primitive, RangeInclusion, Scope, TaggedDictBuilder, UnspannedPathMember, + UntaggedValue, Value, }; use nu_source::Text; @@ -40,7 +40,7 @@ pub(crate) fn evaluate_baseline_expr( trace!("left={:?} right={:?}", left.value, right.value); - match apply_operator(**binary.op(), &left, &right) { + match apply_operator(&**binary.op(), &left, &right) { Ok(result) => Ok(result.into_value(tag)), Err((left_type, right_type)) => Err(ShellError::coerce_error( left_type.spanned(binary.left().span), @@ -48,6 +48,26 @@ pub(crate) fn evaluate_baseline_expr( )), } } + RawExpression::Range(range) => { + let left = range.left(); + let right = range.right(); + + let left = evaluate_baseline_expr(left, registry, scope, source)?; + let right = evaluate_baseline_expr(right, registry, scope, source)?; + let left_span = left.tag.span; + let right_span = right.tag.span; + + let left = ( + left.as_primitive()?.spanned(left_span), + RangeInclusion::Inclusive, + ); + let right = ( + right.as_primitive()?.spanned(right_span), + RangeInclusion::Exclusive, + ); + + Ok(UntaggedValue::range(left, right).into_value(tag)) + } RawExpression::List(list) => { let mut exprs = vec![]; diff --git a/src/evaluate/operator.rs b/src/evaluate/operator.rs index e96e6fd0fc..7eee06051b 100644 --- a/src/evaluate/operator.rs +++ b/src/evaluate/operator.rs @@ -1,25 +1,24 @@ use crate::data::value; -use nu_parser::Operator; +use nu_parser::CompareOperator; use nu_protocol::{Primitive, ShellTypeName, UntaggedValue, Value}; use std::ops::Not; pub fn apply_operator( - op: Operator, + op: &CompareOperator, left: &Value, right: &Value, ) -> Result { - match op { - Operator::Equal - | Operator::NotEqual - | Operator::LessThan - | Operator::GreaterThan - | Operator::LessThanOrEqual - | Operator::GreaterThanOrEqual => { + match *op { + CompareOperator::Equal + | CompareOperator::NotEqual + | CompareOperator::LessThan + | CompareOperator::GreaterThan + | CompareOperator::LessThanOrEqual + | CompareOperator::GreaterThanOrEqual => { value::compare_values(op, left, right).map(UntaggedValue::boolean) } - Operator::Dot => Ok(UntaggedValue::boolean(false)), - Operator::Contains => contains(left, right).map(UntaggedValue::boolean), - Operator::NotContains => contains(left, right) + CompareOperator::Contains => contains(left, right).map(UntaggedValue::boolean), + CompareOperator::NotContains => contains(left, right) .map(Not::not) .map(UntaggedValue::boolean), } diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 3008d5514d..8ad1b29771 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -144,7 +144,8 @@ fn paint_flat_shape(flat_shape: &Spanned, line: &str) -> String { FlatShape::CloseDelimiter(_) => Color::White.normal(), FlatShape::ItVariable => Color::Purple.bold(), FlatShape::Variable => Color::Purple.normal(), - FlatShape::Operator => Color::Yellow.normal(), + FlatShape::CompareOperator => Color::Yellow.normal(), + FlatShape::DotDot => Color::Yellow.bold(), FlatShape::Dot => Color::White.normal(), FlatShape::InternalCommand => Color::Cyan.bold(), FlatShape::ExternalCommand => Color::Cyan.normal(), @@ -160,7 +161,8 @@ fn paint_flat_shape(flat_shape: &Spanned, line: &str) -> String { FlatShape::ShorthandFlag => Color::Black.bold(), FlatShape::Int => Color::Purple.bold(), FlatShape::Decimal => Color::Purple.bold(), - FlatShape::Whitespace => Color::White.normal(), + FlatShape::Whitespace | FlatShape::Separator => Color::White.normal(), + FlatShape::Comment => Color::Black.bold(), FlatShape::Error => Color::Red.bold(), FlatShape::Size { number, unit } => { let number = number.slice(line); diff --git a/tests/command_cd_tests.rs b/tests/command_cd_tests.rs index 8b6592c940..1f67bf7fa9 100644 --- a/tests/command_cd_tests.rs +++ b/tests/command_cd_tests.rs @@ -130,8 +130,8 @@ fn filesystem_not_a_directory() { "cd ferris_did_it.txt" ); - assert!(actual.contains("ferris_did_it.txt")); - assert!(actual.contains("is not a directory")); + assert!(actual.contains("ferris_did_it.txt"), "actual={:?}", actual); + assert!(actual.contains("is not a directory"), "actual={:?}", actual); }) } @@ -142,8 +142,16 @@ fn filesystem_directory_not_found() { "cd dir_that_does_not_exist" ); - assert!(actual.contains("dir_that_does_not_exist")); - assert!(actual.contains("directory not found")); + assert!( + actual.contains("dir_that_does_not_exist"), + "actual={:?}", + actual + ); + assert!( + actual.contains("directory not found"), + "actual={:?}", + actual + ); } #[test] diff --git a/tests/commands_test.rs b/tests/commands_test.rs index 2d40cd7313..7b607830a1 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -248,30 +248,6 @@ fn range_selects_some_rows() { }); } -#[test] -fn range_selects_all_rows() { - Playground::setup("range_test_3", |dirs, sandbox| { - sandbox.with_files(vec![ - EmptyFile("notes.txt"), - EmptyFile("tests.txt"), - EmptyFile("persons.txt"), - ]); - - let actual = nu!( - cwd: dirs.test(), h::pipeline( - r#" - ls - | get name - | range .. - | count - | echo $it - "# - )); - - assert_eq!(actual, "3"); - }); -} - #[test] fn split_by() { Playground::setup("split_by_test_1", |dirs, sandbox| {