From 4291e31dc728d7f96757913eb2f09782f09c0bcd Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Mon, 10 Jun 2019 22:53:04 -0700 Subject: [PATCH 01/20] Start rebuilding lite parser using nom --- Cargo.lock | 55 +++--------- Cargo.toml | 3 +- src/errors.rs | 4 +- src/parser.rs | 1 + src/parser/parse2.rs | 6 ++ src/parser/parse2/operator.rs | 50 +++++++++++ src/parser/parse2/parser.rs | 146 ++++++++++++++++++++++++++++++++ src/parser/parse2/span.rs | 114 +++++++++++++++++++++++++ src/parser/parse2/token_tree.rs | 6 ++ src/parser/parse2/tokens.rs | 11 +++ src/parser/parse2/util.rs | 1 + 11 files changed, 352 insertions(+), 45 deletions(-) create mode 100644 src/parser/parse2.rs create mode 100644 src/parser/parse2/operator.rs create mode 100644 src/parser/parse2/parser.rs create mode 100644 src/parser/parse2/span.rs create mode 100644 src/parser/parse2/token_tree.rs create mode 100644 src/parser/parse2/tokens.rs create mode 100644 src/parser/parse2/util.rs diff --git a/Cargo.lock b/Cargo.lock index 4415d569d5..2ed37ff34e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,6 +166,11 @@ dependencies = [ "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bytecount" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "1.3.1" @@ -1238,17 +1243,6 @@ name = "lazycell" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "lexical-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "libc" version = "0.2.58" @@ -1530,13 +1524,13 @@ dependencies = [ ] [[package]] -name = "nom" -version = "5.0.0-beta1" +name = "nom_locate" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1573,7 +1567,8 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "logos 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "logos-derive 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 5.0.0-beta1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "pancurses 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2539,19 +2534,6 @@ name = "stable_deref_trait" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "stackvector" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "static_assertions" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "string" version = "0.2.0" @@ -2989,14 +2971,6 @@ name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "url" version = "1.7.2" @@ -3195,6 +3169,7 @@ dependencies = [ "checksum block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" "checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum byte-unit 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6754bb4703aa167bed5381f0c6842f1cc31a9ecde3b9443f726dde3ad3afb841" +"checksum bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "39f75544d7bbaf57560d2168f28fd649ff9c76153874db88bdbdfd839b1a7e7d" @@ -3312,7 +3287,6 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" -"checksum lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e82e023e062f1d25f807ad182008fba1b46538e999f908a08cc0c29e084462e" "checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" "checksum libgit2-sys 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "48441cb35dc255da8ae72825689a95368bf510659ae1ad55dc4aa88cb1789bf1" "checksum libssh2-sys 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "126a1f4078368b163bfdee65fbab072af08a1b374a5551b21e87ade27b1fbf9d" @@ -3343,7 +3317,7 @@ dependencies = [ "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -"checksum nom 5.0.0-beta1 (registry+https://github.com/rust-lang/crates.io-index)" = "6527f311b2baba609e980e008460ab5ebff6d6da15213bb8eb193b7746eefa24" +"checksum nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a47c112b3861d81f7fbf73892b9271af933af32bd5dee6889aa3c3fa9caed7e" "checksum num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cf4825417e1e1406b3782a8ce92f4d53f26ec055e3622e1881ca8e9f5f9e08db" "checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" "checksum num-integer 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8af8caa3184078cd419b430ff93684cb13937970fcb7639f728992f33ce674" @@ -3448,8 +3422,6 @@ dependencies = [ "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" -"checksum stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c049c77bf85fbc036484c97b008276d539d9ebff9dfbde37b632ebcd5b8746b6" -"checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum string 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0bbfb8937e38e34c3444ff00afb28b0811d9554f15c5ad64d12b0308d1d1995" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" @@ -3496,7 +3468,6 @@ dependencies = [ "checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" "checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" diff --git a/Cargo.toml b/Cargo.toml index 4fb8287c06..0628246995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ prettytable-rs = "0.8.0" itertools = "0.8.0" ansi_term = "0.11.0" conch-parser = "0.1.1" -nom = "5.0.0-beta1" +nom = "4.2.3" dunce = "1.0.0" indexmap = { version = "1.0.2", features = ["serde-1"] } chrono-humanize = "0.0.11" @@ -61,6 +61,7 @@ clipboard = "0.5" reqwest = "0.9" roxmltree = "0.6.0" pretty = "0.5.2" +nom_locate = "0.3.1" [dependencies.pancurses] version = "0.16" diff --git a/src/errors.rs b/src/errors.rs index 1023474d55..ddf45f4cd9 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -177,8 +177,8 @@ impl std::convert::From for ShellError { } } -impl std::convert::From> for ShellError { - fn from(input: nom::Err<(&str, nom::error::ErrorKind)>) -> ShellError { +impl std::convert::From> for ShellError { + fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError { ShellError::String(StringError { title: format!("{:?}", input), error: Value::nothing(), diff --git a/src/parser.rs b/src/parser.rs index 6e54183eb4..b2eb9f8468 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -4,6 +4,7 @@ crate mod lexer; crate mod parser; crate mod registry; crate mod span; +crate mod parse2; crate use ast::Pipeline; crate use registry::{Args, CommandConfig}; diff --git a/src/parser/parse2.rs b/src/parser/parse2.rs new file mode 100644 index 0000000000..ab4ef485be --- /dev/null +++ b/src/parser/parse2.rs @@ -0,0 +1,6 @@ +crate mod operator; +crate mod parser; +crate mod span; +crate mod token_tree; +crate mod tokens; +crate mod util; diff --git a/src/parser/parse2/operator.rs b/src/parser/parse2/operator.rs new file mode 100644 index 0000000000..89275afd74 --- /dev/null +++ b/src/parser/parse2/operator.rs @@ -0,0 +1,50 @@ +use serde_derive::{Deserialize, Serialize}; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum Operator { + Equal, + NotEqual, + LessThan, + GreaterThan, + LessThanOrEqual, + GreaterThanOrEqual, +} + +impl Operator { + pub fn print(&self) -> String { + self.as_str().to_string() + } + + pub fn as_str(&self) -> &str { + match *self { + Operator::Equal => "==", + Operator::NotEqual => "!=", + Operator::LessThan => "<", + Operator::GreaterThan => ">", + Operator::LessThanOrEqual => "<=", + Operator::GreaterThanOrEqual => ">=", + } + } +} + +impl From<&str> for Operator { + fn from(input: &str) -> Operator { + Operator::from_str(input).unwrap() + } +} + +impl FromStr for Operator { + type Err = (); + fn from_str(input: &str) -> Result::Err> { + match input { + "==" => Ok(Operator::Equal), + "!=" => Ok(Operator::NotEqual), + "<" => Ok(Operator::LessThan), + ">" => Ok(Operator::GreaterThan), + "<=" => Ok(Operator::LessThanOrEqual), + ">=" => Ok(Operator::GreaterThanOrEqual), + _ => Err(()), + } + } +} diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs new file mode 100644 index 0000000000..36c2fea082 --- /dev/null +++ b/src/parser/parse2/parser.rs @@ -0,0 +1,146 @@ +use crate::parser::parse2::{operator::*, span::*, tokens::*}; +use nom::types::CompleteStr; +use nom::*; +use nom_locate::{position, LocatedSpan}; +use std::str::FromStr; + +type NomSpan<'a> = LocatedSpan>; + +macro_rules! operator { + ($name:tt : $token:tt ) => { + named!($name( NomSpan ) -> Token, + do_parse!( + l: position!() + >> t: tag!(stringify!($token)) + >> r: position!() + >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) + ) + ); + }; +} + +operator! { gt: > } +operator! { lt: < } +operator! { gte: >= } +operator! { lte: <= } +operator! { eq: == } +operator! { neq: != } + +named!(integer( NomSpan ) -> Token, + do_parse!( + l: position!() + >> neg: opt!(tag!("-")) + >> num: digit1 + >> r: position!() + >> (Spanned::from_nom(RawToken::Integer(int(num.fragment.0, neg)), l, r)) + ) +); + +named!(operator( NomSpan ) -> Token, + alt!( + gte | lte | neq | gt | lt | eq + ) +); + +named!(dq_string( NomSpan ) -> Token, + do_parse!( + l: position!() + >> char!('"') + >> l1: position!() + >> many0!(none_of!("\"")) + >> r1: position!() + >> char!('"') + >> r: position!() + >> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r)) + ) +); + +named!(sq_string( NomSpan ) -> Token, + do_parse!( + l: position!() + >> char!('\'') + >> l1: position!() + >> many0!(none_of!("'")) + >> r1: position!() + >> char!('\'') + >> r: position!() + >> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r)) + ) +); + +fn int(frag: &str, neg: Option) -> i64 { + let int = FromStr::from_str(frag).unwrap(); + + match neg { + None => int, + Some(_) => int * -1, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_integer() { + assert_eq!( + integer(NomSpan::new(CompleteStr("123"))).unwrap().1, + Spanned::from_item(RawToken::Integer(123), (0, 3)) + ); + + assert_eq!( + integer(NomSpan::new(CompleteStr("-123"))).unwrap().1, + Spanned::from_item(RawToken::Integer(-123), (0, 4)) + ); + } + + #[test] + fn test_operator() { + assert_eq!( + operator(NomSpan::new(CompleteStr(">"))).unwrap().1, + Spanned::from_item(RawToken::Operator(Operator::GreaterThan), (0, 1)) + ); + + assert_eq!( + operator(NomSpan::new(CompleteStr(">="))).unwrap().1, + Spanned::from_item(RawToken::Operator(Operator::GreaterThanOrEqual), (0, 2)) + ); + + assert_eq!( + operator(NomSpan::new(CompleteStr("<"))).unwrap().1, + Spanned::from_item(RawToken::Operator(Operator::LessThan), (0, 1)) + ); + + assert_eq!( + operator(NomSpan::new(CompleteStr("<="))).unwrap().1, + Spanned::from_item(RawToken::Operator(Operator::LessThanOrEqual), (0, 2)) + ); + + assert_eq!( + operator(NomSpan::new(CompleteStr("=="))).unwrap().1, + Spanned::from_item(RawToken::Operator(Operator::Equal), (0, 2)) + ); + + assert_eq!( + operator(NomSpan::new(CompleteStr("!="))).unwrap().1, + Spanned::from_item(RawToken::Operator(Operator::NotEqual), (0, 2)) + ); + } + + #[test] + fn test_string() { + assert_eq!( + dq_string(NomSpan::new(CompleteStr(r#""hello world""#))) + .unwrap() + .1, + Spanned::from_item(RawToken::String(Span::from((1, 12))), (0, 13)) + ); + + assert_eq!( + sq_string(NomSpan::new(CompleteStr(r#"'hello world'"#))) + .unwrap() + .1, + Spanned::from_item(RawToken::String(Span::from((1, 12))), (0, 13)) + ); + } +} diff --git a/src/parser/parse2/span.rs b/src/parser/parse2/span.rs new file mode 100644 index 0000000000..2d3cbe7b5e --- /dev/null +++ b/src/parser/parse2/span.rs @@ -0,0 +1,114 @@ +use derive_new::new; +use std::ops::Range; + +#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct Spanned { + crate span: Span, + crate item: T, +} + +impl std::ops::Deref for Spanned { + type Target = T; + + fn deref(&self) -> &T { + &self.item + } +} + +impl Spanned { + crate fn from_nom( + item: T, + start: nom_locate::LocatedSpan, + end: nom_locate::LocatedSpan, + ) -> Spanned { + let start = start.offset; + let end = end.offset; + + Spanned { + span: Span::from((start, end)), + item, + } + } + + crate fn from_item(item: T, span: impl Into) -> Spanned { + Spanned { + span: span.into(), + item, + } + } + + crate fn map(self, input: impl FnOnce(T) -> U) -> Spanned { + let Spanned { span, item } = self; + + let mapped = input(item); + Spanned { span, item: mapped } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub struct Span { + crate start: usize, + crate end: usize, + // source: &'source str, +} + +impl From<(nom_locate::LocatedSpan, nom_locate::LocatedSpan)> for Span { + fn from(input: (nom_locate::LocatedSpan, nom_locate::LocatedSpan)) -> Span { + Span { + start: input.0.offset, + end: input.1.offset, + } + } +} + +impl From<(usize, usize)> for Span { + fn from(input: (usize, usize)) -> Span { + Span { + start: input.0, + end: input.1, + } + } +} + +impl From<&std::ops::Range> for Span { + fn from(input: &std::ops::Range) -> Span { + Span { + start: input.start, + end: input.end, + } + } +} + +impl Span { + fn new(range: &Range) -> Span { + Span { + start: range.start, + end: range.end, + // source, + } + } +} + +impl language_reporting::ReportingSpan for Span { + fn with_start(&self, start: usize) -> Self { + Span { + start, + end: self.end, + } + } + + fn with_end(&self, end: usize) -> Self { + Span { + start: self.start, + end, + } + } + + fn start(&self) -> usize { + self.start + } + + fn end(&self) -> usize { + self.end + } +} diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs new file mode 100644 index 0000000000..3f9f45ded7 --- /dev/null +++ b/src/parser/parse2/token_tree.rs @@ -0,0 +1,6 @@ +use crate::parser::parse2::{span::*, tokens::*}; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] +pub enum TokenNode { + Token(Token), +} diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs new file mode 100644 index 0000000000..b12c361b91 --- /dev/null +++ b/src/parser/parse2/tokens.rs @@ -0,0 +1,11 @@ +use crate::parser::parse2::operator::*; +use crate::parser::parse2::span::*; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RawToken { + Integer(i64), + Operator(Operator), + String(Span), +} + +pub type Token = Spanned; diff --git a/src/parser/parse2/util.rs b/src/parser/parse2/util.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/src/parser/parse2/util.rs @@ -0,0 +1 @@ + From c5c14e2d89fbfd74b9d208c64faf571f1ed71670 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Wed, 12 Jun 2019 10:44:45 -0700 Subject: [PATCH 02/20] Add a bunch more tokens and delimited () --- Cargo.lock | 51 ++++ Cargo.toml | 3 + src/parser/parse2.rs | 2 + src/parser/parse2/flag.rs | 7 + src/parser/parse2/parser.rs | 424 ++++++++++++++++++++++++++++---- src/parser/parse2/token_tree.rs | 18 +- src/parser/parse2/tokens.rs | 6 + src/parser/parse2/unit.rs | 50 ++++ 8 files changed, 514 insertions(+), 47 deletions(-) create mode 100644 src/parser/parse2/flag.rs create mode 100644 src/parser/parse2/unit.rs diff --git a/Cargo.lock b/Cargo.lock index 2ed37ff34e..3d72a03bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -643,6 +643,19 @@ dependencies = [ "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "derive_more" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "difference" version = "2.0.0" @@ -799,6 +812,28 @@ dependencies = [ "array-macro 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "enum-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "enum-utils-from-str 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "enum-utils-from-str" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "enum_derive" version = "0.1.7" @@ -1551,8 +1586,10 @@ dependencies = [ "ctrlc 3.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "cursive 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "derive-new 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "dirs 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "dunce 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "enum-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "enum_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "futures-preview 0.3.0-alpha.16 (registry+https://github.com/rust-lang/crates.io-index)", "futures-sink-preview 0.3.0-alpha.16 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1592,6 +1629,7 @@ dependencies = [ "tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "toml-query 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2442,6 +2480,15 @@ dependencies = [ "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_derive_internals" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_json" version = "1.0.39" @@ -3217,6 +3264,7 @@ dependencies = [ "checksum derive-new 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6ca414e896ae072546f4d789f452daaecf60ddee4c9df5dc6d5936d769e3d87c" "checksum derive_builder 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a0ca533e6abb78f9108585535ce2ae0b14c8b4504e138a9a28eaf8ba2b270c1d" "checksum derive_builder_core 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fb484fe06ba1dc5b82f88aff700191dfc127e02b06b35e302c169706168e2528" +"checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe" "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum directories 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "72d337a64190607d4fcca2cb78982c5dd57f4916e19696b48a575fa746b6cb0f" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" @@ -3237,6 +3285,8 @@ dependencies = [ "checksum enum-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ccd9b2d5e0eb5c2ff851791e2af90ab4531b1168cfc239d1c0bf467e60ba3c89" "checksum enum-map-derive 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "153f6e8a8b2868e2fedf921b165f30229edcccb74d6a9bb1ccf0480ef61cd07e" "checksum enum-map-internals 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "38b0bacf3ea7aba18ce84032efc3f0fa29f5c814048b742ab3e64d07d83ac3e8" +"checksum enum-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23c5d9343285feb320b4f2b0d0ba9501c11e1a19052026852d772a6e96e8de5d" +"checksum enum-utils-from-str 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6b5669381f76d7320e122abdd4a8307f986634f6d067fb69e31179422175801a" "checksum enum_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "406ac2a8c9eedf8af9ee1489bee9e50029278a6456c740f7454cf8a158abc816" "checksum enumset 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cac0a22e173f6570a7d69a2ab9e3fe79cf0dcdd0fdb162bfc932b97158f2b2a7" "checksum enumset_derive 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "01d93b926a992a4a526c2a14e2faf734fdef5bf9d0a52ba69a2ca7d4494c284b" @@ -3410,6 +3460,7 @@ dependencies = [ "checksum serde-hjson 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4640cf3168e40c00c874ff1ad436c0f18c37edec101d5d897a4396f617abce29" "checksum serde-value 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7a663f873dedc4eac1a559d4c6bc0d0b2c34dc5ac4702e105014b8281489e44f" "checksum serde_derive 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)" = "101b495b109a3e3ca8c4cbe44cf62391527cdfb6ba15821c5ce80bcd5ea23f9f" +"checksum serde_derive_internals 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8a80c6c0b1ebbcea4ec2c7e9e2e9fa197a425d17f1afec8ba79fcd1352b18ffb" "checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" "checksum serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" "checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" diff --git a/Cargo.toml b/Cargo.toml index 0628246995..95d677566a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,9 @@ reqwest = "0.9" roxmltree = "0.6.0" pretty = "0.5.2" nom_locate = "0.3.1" +derive_more = "0.15.0" +enum-utils = "0.1.0" +unicode-xid = "0.1.0" [dependencies.pancurses] version = "0.16" diff --git a/src/parser/parse2.rs b/src/parser/parse2.rs index ab4ef485be..510c3e8aaf 100644 --- a/src/parser/parse2.rs +++ b/src/parser/parse2.rs @@ -1,6 +1,8 @@ +crate mod flag; crate mod operator; crate mod parser; crate mod span; crate mod token_tree; crate mod tokens; +crate mod unit; crate mod util; diff --git a/src/parser/parse2/flag.rs b/src/parser/parse2/flag.rs new file mode 100644 index 0000000000..0d255e4515 --- /dev/null +++ b/src/parser/parse2/flag.rs @@ -0,0 +1,7 @@ +use serde_derive::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum Flag { + Shorthand, + Longhand, +} diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 36c2fea082..74bbc66394 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -1,4 +1,7 @@ -use crate::parser::parse2::{operator::*, span::*, tokens::*}; +#![allow(unused)] + +use crate::parser::parse2::{flag::*, operator::*, span::*, token_tree::*, tokens::*, unit::*}; +use nom; use nom::types::CompleteStr; use nom::*; use nom_locate::{position, LocatedSpan}; @@ -26,23 +29,30 @@ operator! { lte: <= } operator! { eq: == } operator! { neq: != } -named!(integer( NomSpan ) -> Token, +named!(pub raw_integer( NomSpan ) -> Spanned, do_parse!( l: position!() >> neg: opt!(tag!("-")) >> num: digit1 >> r: position!() - >> (Spanned::from_nom(RawToken::Integer(int(num.fragment.0, neg)), l, r)) + >> (Spanned::from_nom(int(num.fragment.0, neg), l, r)) ) ); -named!(operator( NomSpan ) -> Token, +named!(pub integer( NomSpan ) -> Token, + do_parse!( + int: raw_integer + >> (int.map(|i| RawToken::Integer(i))) + ) +); + +named!(pub operator( NomSpan ) -> Token, alt!( gte | lte | neq | gt | lt | eq ) ); -named!(dq_string( NomSpan ) -> Token, +named!(pub dq_string( NomSpan ) -> Token, do_parse!( l: position!() >> char!('"') @@ -55,7 +65,7 @@ named!(dq_string( NomSpan ) -> Token, ) ); -named!(sq_string( NomSpan ) -> Token, +named!(pub sq_string( NomSpan ) -> Token, do_parse!( l: position!() >> char!('\'') @@ -68,6 +78,114 @@ named!(sq_string( NomSpan ) -> Token, ) ); +named!(pub string( NomSpan ) -> Token, + alt!(sq_string | dq_string) +); + +named!(pub bare( NomSpan ) -> Token, + do_parse!( + l: position!() + >> take_while1!(is_start_bare_char) + >> take_while!(is_bare_char) + >> r: position!() + >> (Spanned::from_nom(RawToken::Bare, l, r)) + ) +); + +named!(pub var( NomSpan ) -> Token, + do_parse!( + l: position!() + >> tag!("$") + >> bare: identifier + >> r: position!() + >> (Spanned::from_nom(RawToken::Variable(bare.span), l, r)) + ) +); + +named!(pub identifier( NomSpan ) -> Spanned<()>, + do_parse!( + l: position!() + >> take_while1!(is_id_start) + >> take_while!(is_id_continue) + >> r: position!() + >> (Spanned::from_nom((), l, r)) + ) +); + +named!(pub flag( NomSpan ) -> Token, + do_parse!( + l: position!() + >> tag!("--") + >> bare: bare + >> r: position!() + >> (Spanned::from_nom(RawToken::Flag(Flag::Longhand, bare.span), l, r)) + ) +); + +named!(pub shorthand( NomSpan ) -> Token, + do_parse!( + l: position!() + >> tag!("-") + >> bare: bare + >> r: position!() + >> (Spanned::from_nom(RawToken::Flag(Flag::Shorthand, bare.span), l, r)) + ) +); + +named!(pub raw_unit( NomSpan ) -> Spanned, + do_parse!( + l: position!() + >> unit: alt!(tag!("B") | tag!("KB") | tag!("MB") | tag!("GB") | tag!("TB") | tag!("PB")) + >> r: position!() + >> (Spanned::from_nom(Unit::from(unit.fragment.0), l, r)) + ) +); + +named!(pub size( NomSpan ) -> Token, + do_parse!( + l: position!() + >> int: raw_integer + >> unit: raw_unit + >> r: position!() + >> (Spanned::from_nom(RawToken::Size(int.item, unit.item), l, r)) + ) +); + +// named!(pub unit_num( NomSpan ) -> Token, +// do_parse!( +// l: position!() +// >> +// ) +// ) + +named!(pub leaf( NomSpan ) -> Token, + alt!(size | integer | string | operator | flag | shorthand | bare) +); + +named!(pub leaf_node( NomSpan ) -> TokenNode, + do_parse!( + leaf: leaf + >> (TokenNode::Token(leaf)) + ) +); + +named!(pub delimited_paren( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> items: delimited!( + char!('('), + delimited!(space0, separated_list!(space1, node), space0), + char!(')') + ) + >> r: position!() + >> (TokenNode::Delimited(Spanned::from_nom(DelimitedNode::new(Delimiter::Paren, items), l, r))) + ) +); + +named!(pub node( NomSpan ) -> TokenNode, + alt!(leaf_node | delimited_paren) +); + fn int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -77,70 +195,284 @@ fn int(frag: &str, neg: Option) -> i64 { } } +fn is_start_bare_char(c: char) -> bool { + match c { + _ if c.is_alphabetic() => true, + '.' => true, + '\\' => true, + '/' => true, + '_' => true, + '-' => true, + _ => false, + } +} + +fn is_bare_char(c: char) -> bool { + match c { + _ if c.is_alphanumeric() => true, + ':' => true, + '.' => true, + '\\' => true, + '/' => true, + '_' => true, + '-' => true, + _ => false, + } +} + +fn is_id_start(c: char) -> bool { + unicode_xid::UnicodeXID::is_xid_start(c) +} + +fn is_id_continue(c: char) -> bool { + unicode_xid::UnicodeXID::is_xid_continue(c) + || match c { + '-' => true, + '?' => true, + '!' => true, + _ => false, + } +} + #[cfg(test)] mod tests { use super::*; + use pretty_assertions::assert_eq; + + macro_rules! assert_leaf { + (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt $parens:tt } ) => { + $( + assert_eq!( + apply($name, $input), + token(RawToken::$kind $parens, $left, $right) + ); + )* + + assert_eq!( + apply(leaf, $input), + token(RawToken::$kind $parens, $left, $right) + ); + + assert_eq!( + apply(leaf_node, $input), + TokenNode::Token(token(RawToken::$kind $parens, $left, $right)) + ); + + assert_eq!( + apply(node, $input), + TokenNode::Token(token(RawToken::$kind $parens, $left, $right)) + ); + }; + + (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt } ) => { + $( + assert_eq!( + apply($name, $input), + token(RawToken::$kind, $left, $right) + ); + )* + } + } #[test] fn test_integer() { - assert_eq!( - integer(NomSpan::new(CompleteStr("123"))).unwrap().1, - Spanned::from_item(RawToken::Integer(123), (0, 3)) - ); + assert_leaf! { + parsers [ integer ] + "123" -> 0..3 { Integer(123) } + } - assert_eq!( - integer(NomSpan::new(CompleteStr("-123"))).unwrap().1, - Spanned::from_item(RawToken::Integer(-123), (0, 4)) - ); + assert_leaf! { + parsers [ integer ] + "-123" -> 0..4 { Integer(-123) } + } + } + + #[test] + fn test_size() { + assert_leaf! { + parsers [ size ] + "123MB" -> 0..5 { Size(123, Unit::MB) } + } + + assert_leaf! { + parsers [ size ] + "10GB" -> 0..4 { Size(10, Unit::GB) } + } } #[test] fn test_operator() { - assert_eq!( - operator(NomSpan::new(CompleteStr(">"))).unwrap().1, - Spanned::from_item(RawToken::Operator(Operator::GreaterThan), (0, 1)) - ); + assert_leaf! { + parsers [ operator ] + ">" -> 0..1 { Operator(Operator::GreaterThan) } + } - assert_eq!( - operator(NomSpan::new(CompleteStr(">="))).unwrap().1, - Spanned::from_item(RawToken::Operator(Operator::GreaterThanOrEqual), (0, 2)) - ); + assert_leaf! { + parsers [ operator ] + ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } + } - assert_eq!( - operator(NomSpan::new(CompleteStr("<"))).unwrap().1, - Spanned::from_item(RawToken::Operator(Operator::LessThan), (0, 1)) - ); + assert_leaf! { + parsers [ operator ] + "<" -> 0..1 { Operator(Operator::LessThan) } + } - assert_eq!( - operator(NomSpan::new(CompleteStr("<="))).unwrap().1, - Spanned::from_item(RawToken::Operator(Operator::LessThanOrEqual), (0, 2)) - ); + assert_leaf! { + parsers [ operator ] + "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } + } - assert_eq!( - operator(NomSpan::new(CompleteStr("=="))).unwrap().1, - Spanned::from_item(RawToken::Operator(Operator::Equal), (0, 2)) - ); + assert_leaf! { + parsers [ operator ] + "==" -> 0..2 { Operator(Operator::Equal) } + } - assert_eq!( - operator(NomSpan::new(CompleteStr("!="))).unwrap().1, - Spanned::from_item(RawToken::Operator(Operator::NotEqual), (0, 2)) - ); + assert_leaf! { + parsers [ operator ] + "!=" -> 0..2 { Operator(Operator::NotEqual) } + } } #[test] fn test_string() { + assert_leaf! { + parsers [ string dq_string ] + r#""hello world""# -> 0..13 { String(span(1, 12)) } + } + + assert_leaf! { + parsers [ string sq_string ] + r"'hello world'" -> 0..13 { String(span(1, 12)) } + } + } + + #[test] + fn test_bare() { + assert_leaf! { + parsers [ bare ] + "hello" -> 0..5 { Bare } + } + + assert_leaf! { + parsers [ bare ] + "chrome.exe" -> 0..10 { Bare } + } + + assert_leaf! { + parsers [ bare ] + r"C:\windows\system.dll" -> 0..21 { Bare } + } + + assert_leaf! { + parsers [ bare ] + r"C:\Code\-testing\my_tests.js" -> 0..28 { Bare } + } + } + + #[test] + fn test_flag() { + assert_leaf! { + parsers [ flag ] + "--hello" -> 0..7 { Flag(Flag::Longhand, span(2, 7)) } + } + + assert_leaf! { + parsers [ flag ] + "--hello-world" -> 0..13 { Flag(Flag::Longhand, span(2, 13)) } + } + } + + #[test] + fn test_shorthand() { + assert_leaf! { + parsers [ shorthand ] + "-alt" -> 0..4 { Flag(Flag::Shorthand, span(1, 4)) } + } + } + + fn test_variable() { + assert_leaf! { + parsers [ var ] + "$it" -> 0..3 { Variable(span(1, 3)) } + } + + assert_leaf! { + parsers [ var ] + "$name" -> 0..5 { Variable(span(1, 5)) } + } + } + + #[test] + fn test_delimited() { assert_eq!( - dq_string(NomSpan::new(CompleteStr(r#""hello world""#))) - .unwrap() - .1, - Spanned::from_item(RawToken::String(Span::from((1, 12))), (0, 13)) + apply(node, "(abc)"), + delimited( + Delimiter::Paren, + vec![TokenNode::Token(token(RawToken::Bare, 1, 4))], + 0, + 5 + ) ); assert_eq!( - sq_string(NomSpan::new(CompleteStr(r#"'hello world'"#))) - .unwrap() - .1, - Spanned::from_item(RawToken::String(Span::from((1, 12))), (0, 13)) + apply(node, "( abc )"), + delimited( + Delimiter::Paren, + vec![TokenNode::Token(token(RawToken::Bare, 3, 6))], + 0, + 9 + ) + ); + + assert_eq!( + apply(node, "( abc def )"), + delimited( + Delimiter::Paren, + vec![ + TokenNode::Token(token(RawToken::Bare, 3, 6)), + TokenNode::Token(token(RawToken::Bare, 7, 10)), + ], + 0, + 12 + ) + ); + + assert_eq!( + apply(node, "( abc def 123 456GB )"), + delimited( + Delimiter::Paren, + vec![ + TokenNode::Token(token(RawToken::Bare, 3, 6)), + TokenNode::Token(token(RawToken::Bare, 7, 10)), + TokenNode::Token(token(RawToken::Integer(123), 11, 14)), + TokenNode::Token(token(RawToken::Size(456, Unit::GB), 15, 20)), + ], + 0, + 22 + ) ); } + + fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { + f(NomSpan::new(CompleteStr(string))).unwrap().1 + } + + fn span(left: usize, right: usize) -> Span { + Span::from((left, right)) + } + + fn delimited( + delimiter: Delimiter, + children: Vec, + left: usize, + right: usize, + ) -> TokenNode { + let node = DelimitedNode::new(delimiter, children); + let spanned = Spanned::from_item(node, (left, right)); + TokenNode::Delimited(spanned) + } + + fn token(token: RawToken, left: usize, right: usize) -> Token { + Spanned::from_item(token, (left, right)) + } } diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index 3f9f45ded7..ecb4094283 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,6 +1,22 @@ use crate::parser::parse2::{span::*, tokens::*}; +use derive_new::new; +use enum_utils::FromStr; -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { Token(Token), + Delimited(Spanned), +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct DelimitedNode { + delimiter: Delimiter, + children: Vec, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)] +pub enum Delimiter { + Paren, + Brace, + Square, } diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs index b12c361b91..eece2cd072 100644 --- a/src/parser/parse2/tokens.rs +++ b/src/parser/parse2/tokens.rs @@ -1,11 +1,17 @@ +use crate::parser::parse2::flag::*; use crate::parser::parse2::operator::*; use crate::parser::parse2::span::*; +use crate::parser::parse2::unit::*; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum RawToken { Integer(i64), + Size(i64, Unit), Operator(Operator), String(Span), + Variable(Span), + Bare, + Flag(Flag, Span), } pub type Token = Spanned; diff --git a/src/parser/parse2/unit.rs b/src/parser/parse2/unit.rs new file mode 100644 index 0000000000..883f4c5071 --- /dev/null +++ b/src/parser/parse2/unit.rs @@ -0,0 +1,50 @@ +use serde_derive::{Deserialize, Serialize}; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum Unit { + B, + KB, + MB, + GB, + TB, + PB, +} + +impl Unit { + pub fn print(&self) -> String { + self.as_str().to_string() + } + + pub fn as_str(&self) -> &str { + match *self { + Unit::B => "B", + Unit::KB => "KB", + Unit::MB => "MB", + Unit::GB => "GB", + Unit::TB => "TB", + Unit::PB => "PB", + } + } +} + +impl From<&str> for Unit { + fn from(input: &str) -> Unit { + Unit::from_str(input).unwrap() + } +} + +impl FromStr for Unit { + type Err = (); + fn from_str(input: &str) -> Result::Err> { + match input { + "B" => Ok(Unit::B), + "KB" => Ok(Unit::KB), + "MB" => Ok(Unit::MB), + "GB" => Ok(Unit::GB), + "TB" => Ok(Unit::TB), + "PB" => Ok(Unit::PB), + _ => Err(()), + } + } +} From 4f3a5f03005f77c30696a91ceae5b493736af3f8 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Wed, 12 Jun 2019 22:33:38 -0700 Subject: [PATCH 03/20] Paths --- Cargo.lock | 10 +++ Cargo.toml | 1 + src/parser/parse2/parser.rs | 131 ++++++++++++++++++++++++++++++-- src/parser/parse2/token_tree.rs | 7 ++ src/parser/parse2/tokens.rs | 1 + 5 files changed, 145 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d72a03bb9..a4de3e0f90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1558,6 +1558,14 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nom-trace" +version = "0.1.0" +source = "git+https://github.com/pythondude325/nom-trace.git#02f45edd3aa4ca9c82addb9c9a22e81044536e99" +dependencies = [ + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nom_locate" version = "0.3.1" @@ -1605,6 +1613,7 @@ dependencies = [ "logos 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "logos-derive 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)", "nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "pancurses 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3367,6 +3376,7 @@ dependencies = [ "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +"checksum nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)" = "" "checksum nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a47c112b3861d81f7fbf73892b9271af933af32bd5dee6889aa3c3fa9caed7e" "checksum num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cf4825417e1e1406b3782a8ce92f4d53f26ec055e3622e1881ca8e9f5f9e08db" "checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" diff --git a/Cargo.toml b/Cargo.toml index 95d677566a..ce9b612010 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ nom_locate = "0.3.1" derive_more = "0.15.0" enum-utils = "0.1.0" unicode-xid = "0.1.0" +nom-trace = { version = "0.1.0", git = "https://github.com/pythondude325/nom-trace.git" } [dependencies.pancurses] version = "0.16" diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 74bbc66394..dc4c7909c0 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -2,6 +2,7 @@ use crate::parser::parse2::{flag::*, operator::*, span::*, token_tree::*, tokens::*, unit::*}; use nom; +use nom::dbg; use nom::types::CompleteStr; use nom::*; use nom_locate::{position, LocatedSpan}; @@ -102,13 +103,13 @@ named!(pub var( NomSpan ) -> Token, ) ); -named!(pub identifier( NomSpan ) -> Spanned<()>, +named!(pub identifier( NomSpan ) -> Token, do_parse!( l: position!() >> take_while1!(is_id_start) >> take_while!(is_id_continue) >> r: position!() - >> (Spanned::from_nom((), l, r)) + >> (Spanned::from_nom(RawToken::Identifier, l, r)) ) ); @@ -159,7 +160,7 @@ named!(pub size( NomSpan ) -> Token, // ) named!(pub leaf( NomSpan ) -> Token, - alt!(size | integer | string | operator | flag | shorthand | bare) + alt!(size | integer | string | operator | flag | shorthand | var | bare) ); named!(pub leaf_node( NomSpan ) -> TokenNode, @@ -182,10 +183,25 @@ named!(pub delimited_paren( NomSpan ) -> TokenNode, ) ); -named!(pub node( NomSpan ) -> TokenNode, +named!(pub path( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> head: node1 + >> tag!(".") + >> tail: separated_list!(tag!("."), alt!(identifier | string)) + >> r: position!() + >> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail), l, r))) + ) +); + +named!(pub node1( NomSpan ) -> TokenNode, alt!(leaf_node | delimited_paren) ); +named!(pub node( NomSpan ) -> TokenNode, + alt!(path | leaf_node | delimited_paren) +); + fn int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -237,6 +253,7 @@ fn is_id_continue(c: char) -> bool { #[cfg(test)] mod tests { use super::*; + use nom_trace::{print_trace, reset_trace}; use pretty_assertions::assert_eq; macro_rules! assert_leaf { @@ -390,6 +407,7 @@ mod tests { } } + #[test] fn test_variable() { assert_leaf! { parsers [ var ] @@ -453,8 +471,101 @@ mod tests { ); } + #[test] + fn test_path() { + assert_eq!( + apply(node, "$it.print"), + path( + TokenNode::Token(token(RawToken::Variable(Span::from((1, 3))), 0, 3)), + vec![token(RawToken::Identifier, 4, 9)], + 0, + 9 + ) + ); + + assert_eq!( + apply(node, "$head.part1.part2"), + path( + TokenNode::Token(token(RawToken::Variable(Span::from((1, 5))), 0, 5)), + vec![ + token(RawToken::Identifier, 6, 11), + token(RawToken::Identifier, 12, 17) + ], + 0, + 17 + ) + ); + + assert_eq!( + apply(node, "( hello ).world"), + path( + delimited( + Delimiter::Paren, + vec![TokenNode::Token(token(RawToken::Bare, 2, 7))], + 0, + 9 + ), + vec![token(RawToken::Identifier, 10, 15)], + 0, + 15 + ) + ); + + assert_eq!( + apply(node, "( hello ).\"world\""), + path( + delimited( + Delimiter::Paren, + vec![TokenNode::Token(token(RawToken::Bare, 2, 7))], + 0, + 9 + ), + vec![token(RawToken::String(Span::from((11, 16))), 10, 17)], + 0, + 17 + ) + ); + } + + #[test] + fn test_nested_path() { + assert_eq!( + apply(node, "( $it.is.\"great news\".right yep $yep ).\"world\""), + path( + delimited( + Delimiter::Paren, + vec![ + path( + TokenNode::Token(token(RawToken::Variable(Span::from((3, 5))), 2, 5)), + vec![ + token(RawToken::Identifier, 6, 8), + token(RawToken::String(Span::from((10, 20))), 9, 21), + token(RawToken::Identifier, 22, 27) + ], + 2, + 27 + ), + leaf_token(RawToken::Bare, 28, 31), + leaf_token(RawToken::Variable(Span::from((33, 36))), 32, 36) + ], + 0, + 38 + ), + vec![token(RawToken::String(Span::from((40, 45))), 39, 46)], + 0, + 46 + ) + ); + } + fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { - f(NomSpan::new(CompleteStr(string))).unwrap().1 + match f(NomSpan::new(CompleteStr(string))) { + Ok(v) => v.1, + Err(other) => { + println!("{:?}", other); + panic!("No dice"); + } + } } fn span(left: usize, right: usize) -> Span { @@ -472,6 +583,16 @@ mod tests { TokenNode::Delimited(spanned) } + fn path(head: TokenNode, tail: Vec, left: usize, right: usize) -> TokenNode { + let node = PathNode::new(Box::new(head), tail); + let spanned = Spanned::from_item(node, (left, right)); + TokenNode::Path(spanned) + } + + fn leaf_token(token: RawToken, left: usize, right: usize) -> TokenNode { + TokenNode::Token(Spanned::from_item(token, (left, right))) + } + fn token(token: RawToken, left: usize, right: usize) -> Token { Spanned::from_item(token, (left, right)) } diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index ecb4094283..f79dd20554 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -6,6 +6,7 @@ use enum_utils::FromStr; pub enum TokenNode { Token(Token), Delimited(Spanned), + Path(Spanned), } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] @@ -20,3 +21,9 @@ pub enum Delimiter { Brace, Square, } + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct PathNode { + head: Box, + tail: Vec, +} diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs index eece2cd072..d1977498ce 100644 --- a/src/parser/parse2/tokens.rs +++ b/src/parser/parse2/tokens.rs @@ -10,6 +10,7 @@ pub enum RawToken { Operator(Operator), String(Span), Variable(Span), + Identifier, Bare, Flag(Flag, Span), } From 640484063b236860a8034d4abc2aeb75619a7048 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Fri, 14 Jun 2019 13:55:05 -0700 Subject: [PATCH 04/20] More tests --- src/parser/parse2.rs | 1 + src/parser/parse2/parser.rs | 160 +++++++---------- src/parser/parse2/token_tree.rs | 2 +- src/parser/parse2/token_tree_builder.rs | 221 ++++++++++++++++++++++++ 4 files changed, 284 insertions(+), 100 deletions(-) create mode 100644 src/parser/parse2/token_tree_builder.rs diff --git a/src/parser/parse2.rs b/src/parser/parse2.rs index 510c3e8aaf..9d2e522c98 100644 --- a/src/parser/parse2.rs +++ b/src/parser/parse2.rs @@ -3,6 +3,7 @@ crate mod operator; crate mod parser; crate mod span; crate mod token_tree; +crate mod token_tree_builder; crate mod tokens; crate mod unit; crate mod util; diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index dc4c7909c0..267cee78ed 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -190,7 +190,7 @@ named!(pub path( NomSpan ) -> TokenNode, >> tag!(".") >> tail: separated_list!(tag!("."), alt!(identifier | string)) >> r: position!() - >> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail), l, r))) + >> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail.into_iter().map(TokenNode::Token).collect()), l, r))) ) ); @@ -253,6 +253,8 @@ fn is_id_continue(c: char) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::parser::parse2::token_tree_builder::TokenTreeBuilder as b; + use crate::parser::parse2::token_tree_builder::{CurriedToken, TokenTreeBuilder}; use nom_trace::{print_trace, reset_trace}; use pretty_assertions::assert_eq; @@ -422,52 +424,37 @@ mod tests { #[test] fn test_delimited() { - assert_eq!( - apply(node, "(abc)"), - delimited( - Delimiter::Paren, - vec![TokenNode::Token(token(RawToken::Bare, 1, 4))], - 0, - 5 - ) - ); + assert_eq!(apply(node, "(abc)"), build(b::parens(vec![b::bare("abc")]))); assert_eq!( apply(node, "( abc )"), - delimited( - Delimiter::Paren, - vec![TokenNode::Token(token(RawToken::Bare, 3, 6))], - 0, - 9 - ) + build(b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) ); assert_eq!( apply(node, "( abc def )"), - delimited( - Delimiter::Paren, - vec![ - TokenNode::Token(token(RawToken::Bare, 3, 6)), - TokenNode::Token(token(RawToken::Bare, 7, 10)), - ], - 0, - 12 - ) + build(b::parens(vec![ + b::ws(" "), + b::bare("abc"), + b::sp(), + b::bare("def"), + b::sp() + ])) ); assert_eq!( apply(node, "( abc def 123 456GB )"), - delimited( - Delimiter::Paren, - vec![ - TokenNode::Token(token(RawToken::Bare, 3, 6)), - TokenNode::Token(token(RawToken::Bare, 7, 10)), - TokenNode::Token(token(RawToken::Integer(123), 11, 14)), - TokenNode::Token(token(RawToken::Size(456, Unit::GB), 15, 20)), - ], - 0, - 22 - ) + build(b::parens(vec![ + b::ws(" "), + b::bare("abc"), + b::sp(), + b::bare("def"), + b::sp(), + b::int(123), + b::sp(), + b::size(456, "GB"), + b::sp() + ])) ); } @@ -475,55 +462,31 @@ mod tests { fn test_path() { assert_eq!( apply(node, "$it.print"), - path( - TokenNode::Token(token(RawToken::Variable(Span::from((1, 3))), 0, 3)), - vec![token(RawToken::Identifier, 4, 9)], - 0, - 9 - ) + build(b::path(b::var("it"), vec![b::ident("print")])) ); assert_eq!( apply(node, "$head.part1.part2"), - path( - TokenNode::Token(token(RawToken::Variable(Span::from((1, 5))), 0, 5)), - vec![ - token(RawToken::Identifier, 6, 11), - token(RawToken::Identifier, 12, 17) - ], - 0, - 17 - ) + build(b::path( + b::var("head"), + vec![b::ident("part1"), b::ident("part2")] + )) ); assert_eq!( apply(node, "( hello ).world"), - path( - delimited( - Delimiter::Paren, - vec![TokenNode::Token(token(RawToken::Bare, 2, 7))], - 0, - 9 - ), - vec![token(RawToken::Identifier, 10, 15)], - 0, - 15 - ) + build(b::path( + b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), + vec![b::ident("world")] + )) ); assert_eq!( apply(node, "( hello ).\"world\""), - path( - delimited( - Delimiter::Paren, - vec![TokenNode::Token(token(RawToken::Bare, 2, 7))], - 0, - 9 - ), - vec![token(RawToken::String(Span::from((11, 16))), 10, 17)], - 0, - 17 - ) + build(b::path( + b::parens(vec![b::sp(), b::bare("hello"), b::sp()],), + vec![b::string("world")] + )) ); } @@ -531,31 +494,22 @@ mod tests { fn test_nested_path() { assert_eq!( apply(node, "( $it.is.\"great news\".right yep $yep ).\"world\""), - path( - delimited( - Delimiter::Paren, - vec![ - path( - TokenNode::Token(token(RawToken::Variable(Span::from((3, 5))), 2, 5)), - vec![ - token(RawToken::Identifier, 6, 8), - token(RawToken::String(Span::from((10, 20))), 9, 21), - token(RawToken::Identifier, 22, 27) - ], - 2, - 27 - ), - leaf_token(RawToken::Bare, 28, 31), - leaf_token(RawToken::Variable(Span::from((33, 36))), 32, 36) - ], - 0, - 38 - ), - vec![token(RawToken::String(Span::from((40, 45))), 39, 46)], - 0, - 46 - ) - ); + build(b::path( + b::parens(vec![ + b::sp(), + b::path( + b::var("it"), + vec![b::ident("is"), b::string("great news"), b::ident("right")] + ), + b::sp(), + b::bare("yep"), + b::sp(), + b::var("yep"), + b::sp() + ]), + vec![b::string("world")] + )) + ) } fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { @@ -584,7 +538,10 @@ mod tests { } fn path(head: TokenNode, tail: Vec, left: usize, right: usize) -> TokenNode { - let node = PathNode::new(Box::new(head), tail); + let node = PathNode::new( + Box::new(head), + tail.into_iter().map(TokenNode::Token).collect(), + ); let spanned = Spanned::from_item(node, (left, right)); TokenNode::Path(spanned) } @@ -596,4 +553,9 @@ mod tests { fn token(token: RawToken, left: usize, right: usize) -> Token { Spanned::from_item(token, (left, right)) } + + fn build(block: CurriedToken) -> TokenNode { + let mut builder = TokenTreeBuilder::new(); + block(&mut builder).expect("Expected to build into a token") + } } diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index f79dd20554..6ef1e78289 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -25,5 +25,5 @@ pub enum Delimiter { #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] pub struct PathNode { head: Box, - tail: Vec, + tail: Vec, } diff --git a/src/parser/parse2/token_tree_builder.rs b/src/parser/parse2/token_tree_builder.rs new file mode 100644 index 0000000000..5c7df7de77 --- /dev/null +++ b/src/parser/parse2/token_tree_builder.rs @@ -0,0 +1,221 @@ +use crate::parser::parse2::operator::Operator; +use crate::parser::parse2::span::{Span, Spanned}; +use crate::parser::parse2::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; +use crate::parser::parse2::tokens::{RawToken, Token}; +use crate::parser::parse2::unit::Unit; +use derive_new::new; + +#[derive(new)] +pub struct TokenTreeBuilder { + #[new(default)] + pos: usize, +} + +pub type CurriedToken = Box Option + 'static>; + +#[allow(unused)] +impl TokenTreeBuilder { + pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> TokenNode { + let mut builder = TokenTreeBuilder::new(); + block(&mut builder) + } + + pub fn op(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(input.as_str()); + + b.pos = end; + + Some(TokenTreeBuilder::spanned_op(input, (start, end))) + }) + } + + pub fn spanned_op(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::Operator(input.into()), + span.into(), + )) + } + + pub fn string(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, _) = b.consume("\""); + let (inner_start, inner_end) = b.consume(&input); + let (_, end) = b.consume("\""); + b.pos = end; + + Some(TokenTreeBuilder::spanned_string( + (inner_start, inner_end), + (start, end), + )) + }) + } + + pub fn spanned_string(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::String(input.into()), + span.into(), + )) + } + + pub fn bare(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&input); + b.pos = end; + + Some(TokenTreeBuilder::spanned_bare((start, end))) + }) + } + + pub fn spanned_bare(input: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item(RawToken::Bare, input.into())) + } + + pub fn int(input: impl Into) -> CurriedToken { + let int = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&int.to_string()); + b.pos = end; + + Some(TokenTreeBuilder::spanned_int(int, (start, end))) + }) + } + + pub fn spanned_int(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Token::from_item(RawToken::Integer(input.into()), span)) + } + + pub fn size(int: impl Into, unit: impl Into) -> CurriedToken { + let int = int.into(); + let unit = unit.into(); + + Box::new(move |b| { + let (start, _) = b.consume(&int.to_string()); + let (_, end) = b.consume(unit.as_str()); + b.pos = end; + + Some(TokenTreeBuilder::spanned_size((int, unit), (start, end))) + }) + } + + pub fn spanned_size( + input: (impl Into, impl Into), + span: impl Into, + ) -> TokenNode { + let (int, unit) = (input.0.into(), input.1.into()); + + TokenNode::Token(Spanned::from_item(RawToken::Size(int, unit), span)) + } + + pub fn path(head: CurriedToken, tail: Vec) -> CurriedToken { + Box::new(move |b| { + let start = b.pos; + let head = head(b).expect("The head of a path must not be whitespace"); + + let mut output = vec![]; + + for item in tail { + b.consume("."); + + match item(b) { + None => {} + Some(v) => output.push(v), + }; + } + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_path((head, output), (start, end))) + }) + } + + pub fn spanned_path(input: (TokenNode, Vec), span: impl Into) -> TokenNode { + TokenNode::Path(Spanned::from_item( + PathNode::new(Box::new(input.0), input.1), + span, + )) + } + + pub fn var(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, _) = b.consume("$"); + let (inner_start, end) = b.consume(&input); + + Some(TokenTreeBuilder::spanned_var( + (inner_start, end), + (start, end), + )) + }) + } + + pub fn spanned_var(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::Variable(input.into()), + span.into(), + )) + } + + pub fn ident(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&input); + Some(TokenTreeBuilder::spanned_ident((start, end))) + }) + } + + pub fn spanned_ident(span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into())) + } + + pub fn parens(input: Vec) -> CurriedToken { + Box::new(move |b| { + let (start, _) = b.consume("("); + let mut output = vec![]; + for item in input { + match item(b) { + None => {} + Some(v) => output.push(v), + }; + } + + let (_, end) = b.consume(")"); + + Some(TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Paren, output), + (start, end), + ))) + }) + } + + pub fn sp() -> CurriedToken { + Box::new(|b| { + b.consume(" "); + None + }) + } + + pub fn ws(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + b.consume(&input); + None + }) + } + + fn consume(&mut self, input: &str) -> (usize, usize) { + let start = self.pos; + self.pos += input.len(); + (start, self.pos) + } +} From 6e222eec2b0a0af4a0872f243d718dfe56206c67 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Fri, 14 Jun 2019 19:59:35 -0700 Subject: [PATCH 05/20] More delimiters --- src/parser/parse2/parser.rs | 226 +++++++++++++++++++----- src/parser/parse2/token_tree.rs | 31 +++- src/parser/parse2/token_tree_builder.rs | 188 +++++++++++++++++++- 3 files changed, 392 insertions(+), 53 deletions(-) diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 267cee78ed..5a2f6cc130 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -1,6 +1,8 @@ #![allow(unused)] -use crate::parser::parse2::{flag::*, operator::*, span::*, token_tree::*, tokens::*, unit::*}; +use crate::parser::parse2::{ + flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, unit::*, +}; use nom; use nom::dbg; use nom::types::CompleteStr; @@ -12,12 +14,13 @@ type NomSpan<'a> = LocatedSpan>; macro_rules! operator { ($name:tt : $token:tt ) => { - named!($name( NomSpan ) -> Token, + named!($name( NomSpan ) -> TokenNode, do_parse!( l: position!() >> t: tag!(stringify!($token)) >> r: position!() - >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) + >> (TokenTreeBuilder::spanned_op(t.fragment.0, (l, r))) + // >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) ) ); }; @@ -40,20 +43,20 @@ named!(pub raw_integer( NomSpan ) -> Spanned, ) ); -named!(pub integer( NomSpan ) -> Token, +named!(pub integer( NomSpan ) -> TokenNode, do_parse!( int: raw_integer - >> (int.map(|i| RawToken::Integer(i))) + >> (TokenTreeBuilder::spanned_int(*int, int.span)) ) ); -named!(pub operator( NomSpan ) -> Token, +named!(pub operator( NomSpan ) -> TokenNode, alt!( gte | lte | neq | gt | lt | eq ) ); -named!(pub dq_string( NomSpan ) -> Token, +named!(pub dq_string( NomSpan ) -> TokenNode, do_parse!( l: position!() >> char!('"') @@ -62,11 +65,11 @@ named!(pub dq_string( NomSpan ) -> Token, >> r1: position!() >> char!('"') >> r: position!() - >> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r)) + >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) ) ); -named!(pub sq_string( NomSpan ) -> Token, +named!(pub sq_string( NomSpan ) -> TokenNode, do_parse!( l: position!() >> char!('\'') @@ -75,61 +78,61 @@ named!(pub sq_string( NomSpan ) -> Token, >> r1: position!() >> char!('\'') >> r: position!() - >> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r)) + >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) ) ); -named!(pub string( NomSpan ) -> Token, +named!(pub string( NomSpan ) -> TokenNode, alt!(sq_string | dq_string) ); -named!(pub bare( NomSpan ) -> Token, +named!(pub bare( NomSpan ) -> TokenNode, do_parse!( l: position!() >> take_while1!(is_start_bare_char) >> take_while!(is_bare_char) >> r: position!() - >> (Spanned::from_nom(RawToken::Bare, l, r)) + >> (TokenTreeBuilder::spanned_bare((l, r))) ) ); -named!(pub var( NomSpan ) -> Token, +named!(pub var( NomSpan ) -> TokenNode, do_parse!( l: position!() >> tag!("$") >> bare: identifier >> r: position!() - >> (Spanned::from_nom(RawToken::Variable(bare.span), l, r)) + >> (TokenTreeBuilder::spanned_var(bare.span(), (l, r))) ) ); -named!(pub identifier( NomSpan ) -> Token, +named!(pub identifier( NomSpan ) -> TokenNode, do_parse!( l: position!() >> take_while1!(is_id_start) >> take_while!(is_id_continue) >> r: position!() - >> (Spanned::from_nom(RawToken::Identifier, l, r)) + >> (TokenTreeBuilder::spanned_ident((l, r))) ) ); -named!(pub flag( NomSpan ) -> Token, +named!(pub flag( NomSpan ) -> TokenNode, do_parse!( l: position!() >> tag!("--") >> bare: bare >> r: position!() - >> (Spanned::from_nom(RawToken::Flag(Flag::Longhand, bare.span), l, r)) + >> (TokenTreeBuilder::spanned_flag(bare.span(), (l, r))) ) ); -named!(pub shorthand( NomSpan ) -> Token, +named!(pub shorthand( NomSpan ) -> TokenNode, do_parse!( l: position!() >> tag!("-") >> bare: bare >> r: position!() - >> (Spanned::from_nom(RawToken::Flag(Flag::Shorthand, bare.span), l, r)) + >> (TokenTreeBuilder::spanned_shorthand(bare.span(), (l, r))) ) ); @@ -142,34 +145,20 @@ named!(pub raw_unit( NomSpan ) -> Spanned, ) ); -named!(pub size( NomSpan ) -> Token, +named!(pub size( NomSpan ) -> TokenNode, do_parse!( l: position!() >> int: raw_integer >> unit: raw_unit >> r: position!() - >> (Spanned::from_nom(RawToken::Size(int.item, unit.item), l, r)) + >> (TokenTreeBuilder::spanned_size((*int, *unit), (l, r))) ) ); -// named!(pub unit_num( NomSpan ) -> Token, -// do_parse!( -// l: position!() -// >> -// ) -// ) - -named!(pub leaf( NomSpan ) -> Token, +named!(pub leaf( NomSpan ) -> TokenNode, alt!(size | integer | string | operator | flag | shorthand | var | bare) ); -named!(pub leaf_node( NomSpan ) -> TokenNode, - do_parse!( - leaf: leaf - >> (TokenNode::Token(leaf)) - ) -); - named!(pub delimited_paren( NomSpan ) -> TokenNode, do_parse!( l: position!() @@ -179,7 +168,30 @@ named!(pub delimited_paren( NomSpan ) -> TokenNode, char!(')') ) >> r: position!() - >> (TokenNode::Delimited(Spanned::from_nom(DelimitedNode::new(Delimiter::Paren, items), l, r))) + >> (TokenTreeBuilder::spanned_parens(items, (l, r))) + ) +); + +named!(pub delimited_brace( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> items: delimited!( + char!('{'), + delimited!(space0, separated_list!(space1, node), space0), + char!('}') + ) + >> r: position!() + >> (TokenTreeBuilder::spanned_brace(items, (l, r))) + ) +); + +named!(pub raw_call( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> head: node + >> items: opt!(preceded!(space0, separated_nonempty_list!(space1, node))) + >> r: position!() + >> (TokenTreeBuilder::spanned_call((head, items), (l, r))) ) ); @@ -190,16 +202,25 @@ named!(pub path( NomSpan ) -> TokenNode, >> tag!(".") >> tail: separated_list!(tag!("."), alt!(identifier | string)) >> r: position!() - >> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail.into_iter().map(TokenNode::Token).collect()), l, r))) + >> (TokenTreeBuilder::spanned_path((head, tail), (l, r))) ) ); named!(pub node1( NomSpan ) -> TokenNode, - alt!(leaf_node | delimited_paren) + alt!(leaf | delimited_paren) ); named!(pub node( NomSpan ) -> TokenNode, - alt!(path | leaf_node | delimited_paren) + alt!(path | leaf | delimited_paren | delimited_brace) +); + +named!(pub pipeline( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> list: separated_list!(delimited!(space0, tag!("|"), space0), raw_call) + >> r: position!() + >> (TokenTreeBuilder::spanned_pipeline(list, (l, r))) + ) ); fn int(frag: &str, neg: Option) -> i64 { @@ -273,13 +294,13 @@ mod tests { ); assert_eq!( - apply(leaf_node, $input), - TokenNode::Token(token(RawToken::$kind $parens, $left, $right)) + apply(leaf, $input), + token(RawToken::$kind $parens, $left, $right) ); assert_eq!( apply(node, $input), - TokenNode::Token(token(RawToken::$kind $parens, $left, $right)) + token(RawToken::$kind $parens, $left, $right) ); }; @@ -512,6 +533,117 @@ mod tests { ) } + #[test] + fn test_smoke_single_command() { + assert_eq!( + apply(raw_call, "git add ."), + build(b::call( + b::bare("git"), + vec![b::sp(), b::bare("add"), b::sp(), b::bare(".")] + )) + ); + + assert_eq!( + apply(raw_call, "open Cargo.toml"), + build(b::call( + b::bare("open"), + vec![b::sp(), b::bare("Cargo.toml")] + )) + ); + + assert_eq!( + apply(raw_call, "select package.version"), + build(b::call( + b::bare("select"), + vec![b::sp(), b::bare("package.version")] + )) + ); + + assert_eq!( + apply(raw_call, "echo $it"), + build(b::call(b::bare("echo"), vec![b::sp(), b::var("it")])) + ); + + assert_eq!( + apply(raw_call, "open Cargo.toml --raw"), + build(b::call( + b::bare("open"), + vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::flag("raw")] + )) + ); + + assert_eq!( + apply(raw_call, "open Cargo.toml -r"), + build(b::call( + b::bare("open"), + vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::shorthand("r")] + )) + ); + + assert_eq!( + apply(raw_call, "config --set tabs 2"), + build(b::call( + b::bare("config"), + vec![ + b::sp(), + b::flag("set"), + b::sp(), + b::bare("tabs"), + b::sp(), + b::int(2) + ] + )) + ); + } + + #[test] + fn test_smoke_pipeline() { + assert_eq!( + apply( + pipeline, + r#"git branch --merged | split-row "`n" | where $it != "* master""# + ), + build(b::pipeline(vec![ + b::call( + b::bare("git"), + vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + ), + b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), + b::call( + b::bare("where"), + vec![ + b::sp(), + b::var("it"), + b::sp(), + b::op("!="), + b::sp(), + b::string("* master") + ] + ) + ])) + ); + + assert_eq!( + apply(pipeline, "ls | where { $it.size > 100 }"), + build(b::pipeline(vec![ + b::call(b::bare("ls"), vec![]), + b::call( + b::bare("where"), + vec![ + b::sp(), + b::braced(vec![ + b::path(b::var("it"), vec![b::ident("size")]), + b::sp(), + b::op(">"), + b::sp(), + b::int(100) + ]) + ] + ) + ])) + ) + } + fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { match f(NomSpan::new(CompleteStr(string))) { Ok(v) => v.1, @@ -550,8 +682,8 @@ mod tests { TokenNode::Token(Spanned::from_item(token, (left, right))) } - fn token(token: RawToken, left: usize, right: usize) -> Token { - Spanned::from_item(token, (left, right)) + fn token(token: RawToken, left: usize, right: usize) -> TokenNode { + TokenNode::Token(Spanned::from_item(token, (left, right))) } fn build(block: CurriedToken) -> TokenNode { diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index 6ef1e78289..b3cd722016 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,20 +1,42 @@ -use crate::parser::parse2::{span::*, tokens::*}; +use crate::parser::parse2::{operator::*, span::*, tokens::*}; use derive_new::new; use enum_utils::FromStr; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { Token(Token), + Call(Spanned), Delimited(Spanned), + Pipeline(Spanned>), + Binary(Spanned), Path(Spanned), } +impl TokenNode { + pub fn span(&self) -> Span { + match self { + TokenNode::Token(t) => t.span, + TokenNode::Call(s) => s.span, + TokenNode::Delimited(s) => s.span, + TokenNode::Pipeline(s) => s.span, + TokenNode::Binary(s) => s.span, + TokenNode::Path(s) => s.span, + } + } +} + #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] pub struct DelimitedNode { delimiter: Delimiter, children: Vec, } +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct CallNode { + head: Box, + children: Vec, +} + #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)] pub enum Delimiter { Paren, @@ -27,3 +49,10 @@ pub struct PathNode { head: Box, tail: Vec, } + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct BinaryNode { + left: Box, + op: Operator, + right: Box, +} diff --git a/src/parser/parse2/token_tree_builder.rs b/src/parser/parse2/token_tree_builder.rs index 5c7df7de77..a3fffaa8d7 100644 --- a/src/parser/parse2/token_tree_builder.rs +++ b/src/parser/parse2/token_tree_builder.rs @@ -1,6 +1,9 @@ +use crate::parser::parse2::flag::Flag; use crate::parser::parse2::operator::Operator; use crate::parser::parse2::span::{Span, Spanned}; -use crate::parser::parse2::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; +use crate::parser::parse2::token_tree::{ + BinaryNode, CallNode, DelimitedNode, Delimiter, PathNode, TokenNode, +}; use crate::parser::parse2::tokens::{RawToken, Token}; use crate::parser::parse2::unit::Unit; use derive_new::new; @@ -20,6 +23,38 @@ impl TokenTreeBuilder { block(&mut builder) } + pub fn pipeline(input: Vec) -> CurriedToken { + Box::new(move |b| { + let start = b.pos; + + let mut out = vec![]; + + let mut input = input.into_iter(); + let first = input + .next() + .expect("A pipeline must contain at least one element"); + + out.push(first(b).expect("The first element of a pipeline must not be whitespace")); + + for item in input { + b.consume(" | "); + + match item(b) { + None => {} + Some(v) => out.push(v), + } + } + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_pipeline(out, (start, end))) + }) + } + + pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { + TokenNode::Pipeline(Spanned::from_item(input, span)) + } + pub fn op(input: impl Into) -> CurriedToken { let input = input.into(); @@ -164,6 +199,48 @@ impl TokenTreeBuilder { )) } + pub fn flag(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, _) = b.consume("--"); + let (inner_start, end) = b.consume(&input); + + Some(TokenTreeBuilder::spanned_flag( + (inner_start, end), + (start, end), + )) + }) + } + + pub fn spanned_flag(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::Flag(Flag::Longhand, input.into()), + span.into(), + )) + } + + pub fn shorthand(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, _) = b.consume("-"); + let (inner_start, end) = b.consume(&input); + + Some(TokenTreeBuilder::spanned_shorthand( + (inner_start, end), + (start, end), + )) + }) + } + + pub fn spanned_shorthand(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::Flag(Flag::Shorthand, input.into()), + span.into(), + )) + } + pub fn ident(input: impl Into) -> CurriedToken { let input = input.into(); @@ -177,6 +254,39 @@ impl TokenTreeBuilder { TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into())) } + pub fn call(head: CurriedToken, input: Vec) -> CurriedToken { + Box::new(move |b| { + let start = b.pos; + + let head_node = head(b).expect("The head of a command must not be whitespace"); + + let mut tail_nodes = vec![]; + for item in input { + match item(b) { + None => {} + Some(v) => tail_nodes.push(v), + }; + } + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_call( + (head_node, Some(tail_nodes)), + (start, end), + )) + }) + } + + pub fn spanned_call( + input: (TokenNode, Option>), + span: impl Into, + ) -> TokenNode { + TokenNode::Call(Spanned::from_item( + CallNode::new(Box::new(input.0), input.1.unwrap_or_else(|| vec![])), + span, + )) + } + pub fn parens(input: Vec) -> CurriedToken { Box::new(move |b| { let (start, _) = b.consume("("); @@ -190,13 +300,81 @@ impl TokenTreeBuilder { let (_, end) = b.consume(")"); - Some(TokenNode::Delimited(Spanned::from_item( - DelimitedNode::new(Delimiter::Paren, output), - (start, end), - ))) + Some(TokenTreeBuilder::spanned_parens(output, (start, end))) }) } + pub fn spanned_parens(input: impl Into>, span: impl Into) -> TokenNode { + TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Paren, input.into()), + span, + )) + } + + pub fn braced(input: Vec) -> CurriedToken { + Box::new(move |b| { + let (start, _) = b.consume("{ "); + let mut output = vec![]; + for item in input { + match item(b) { + None => {} + Some(v) => output.push(v), + }; + } + + let (_, end) = b.consume(" }"); + + Some(TokenTreeBuilder::spanned_brace(output, (start, end))) + }) + } + + pub fn spanned_brace(input: impl Into>, span: impl Into) -> TokenNode { + TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Brace, input.into()), + span, + )) + } + + pub fn binary( + left: CurriedToken, + op: impl Into, + right: CurriedToken, + ) -> CurriedToken { + let op = op.into(); + + Box::new(move |b| { + let start = b.pos; + + let left = left(b).expect("The left side of a binary operation must not be whitespace"); + + b.consume(" "); + + b.consume(op.as_str()); + + b.consume(" "); + + let right = + right(b).expect("The right side of a binary operation must not be whitespace"); + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_binary( + (left, op, right), + (start, end), + )) + }) + } + + pub fn spanned_binary( + input: (impl Into, Operator, impl Into), + span: impl Into, + ) -> TokenNode { + TokenNode::Binary(Spanned::from_item( + BinaryNode::new(Box::new(input.0.into()), input.1, Box::new(input.2.into())), + span, + )) + } + pub fn sp() -> CurriedToken { Box::new(|b| { b.consume(" "); From e981129f1f2b562658c8ead274e1959d3178cd26 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Fri, 21 Jun 2019 21:36:57 -0400 Subject: [PATCH 06/20] Things work --- .cargo/config | 1 + Cargo.lock | 486 +++++++------ Cargo.toml | 5 +- src/cli.rs | 99 ++- src/commands.rs | 3 +- src/commands/cd.rs | 11 +- src/commands/classified.rs | 42 +- src/commands/command.rs | 50 +- src/commands/config.rs | 19 +- src/commands/enter.rs | 24 +- src/commands/exit.rs | 5 +- src/commands/first.rs | 6 +- src/commands/get.rs | 12 +- src/commands/lines.rs | 35 + src/commands/ls.rs | 6 +- src/commands/open.rs | 56 +- src/commands/pick.rs | 4 +- src/commands/reject.rs | 4 +- src/commands/save.rs | 13 +- src/commands/size.rs | 4 +- src/commands/skip.rs | 6 +- src/commands/sort_by.rs | 2 +- src/commands/split_column.rs | 25 +- src/commands/split_row.rs | 18 +- src/commands/view.rs | 4 +- src/commands/where_.rs | 7 +- src/context.rs | 21 +- src/errors.rs | 63 +- src/evaluate/evaluator.rs | 159 ++--- src/evaluate/mod.rs | 2 +- src/object/base.rs | 86 ++- src/object/desc.rs | 7 + src/object/dict.rs | 11 + src/parser.rs | 446 ++++++------ src/parser/hir.rs | 93 +++ src/parser/hir/baseline_parse.rs | 30 + src/parser/hir/baseline_parse_tokens.rs | 177 +++++ src/parser/hir/binary.rs | 11 + src/parser/hir/named.rs | 44 ++ src/parser/hir/path.rs | 10 + src/parser/parse2.rs | 5 + src/parser/parse2/call_node.rs | 26 + src/parser/parse2/files.rs | 77 ++ src/parser/parse2/flag.rs | 12 +- src/parser/parse2/operator.rs | 1 + src/parser/parse2/parser.rs | 904 ++++++++++++++++-------- src/parser/parse2/span.rs | 55 +- src/parser/parse2/text.rs | 204 ++++++ src/parser/parse2/token_tree.rs | 110 ++- src/parser/parse2/token_tree_builder.rs | 237 +++---- src/parser/parse2/tokens.rs | 28 +- src/parser/parse2/unit.rs | 4 - src/parser/parse_command.rs | 250 +++++++ src/parser/registry.rs | 412 +++++++---- src/prelude.rs | 1 - src/shell/helper.rs | 77 +- tests/sort_by.txt | 2 +- tests/split.txt | 2 +- 58 files changed, 3074 insertions(+), 1440 deletions(-) create mode 100644 src/commands/lines.rs create mode 100644 src/parser/hir.rs create mode 100644 src/parser/hir/baseline_parse.rs create mode 100644 src/parser/hir/baseline_parse_tokens.rs create mode 100644 src/parser/hir/binary.rs create mode 100644 src/parser/hir/named.rs create mode 100644 src/parser/hir/path.rs create mode 100644 src/parser/parse2/call_node.rs create mode 100644 src/parser/parse2/files.rs create mode 100644 src/parser/parse2/text.rs create mode 100644 src/parser/parse_command.rs diff --git a/.cargo/config b/.cargo/config index e69de29bb2..6727dab680 100644 --- a/.cargo/config +++ b/.cargo/config @@ -0,0 +1 @@ +paths = ["C:\\Users\\wycat\\Code\\nom_locate"] \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index a4de3e0f90..137ea71f93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,9 +7,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -85,25 +85,25 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "autocfg" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "backtrace" -version = "0.3.20" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -120,7 +120,7 @@ name = "base64" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -128,14 +128,14 @@ name = "bincode" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bitflags" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -163,7 +163,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -173,7 +173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "byteorder" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -181,7 +181,7 @@ name = "bytes" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -201,9 +201,9 @@ name = "chrono" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-integer 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -231,7 +231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -274,7 +274,7 @@ name = "cloudabi" version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -293,7 +293,7 @@ dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde-hjson 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "console" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -311,7 +311,7 @@ dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -349,7 +349,7 @@ dependencies = [ "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "publicsuffix 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -392,16 +392,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-deque" -version = "0.2.0" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -413,20 +413,6 @@ dependencies = [ "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "crossbeam-epoch" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam-epoch" version = "0.7.1" @@ -448,14 +434,6 @@ dependencies = [ "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "crossbeam-utils" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam-utils" version = "0.5.0" @@ -478,7 +456,7 @@ dependencies = [ "csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -495,7 +473,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -531,7 +509,7 @@ dependencies = [ "crossbeam-channel 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "enum-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "enumset 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -574,7 +552,7 @@ dependencies = [ "ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -587,7 +565,7 @@ dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -597,7 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "darling_core 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -607,7 +585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "darling_core 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -617,30 +595,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "derive_builder" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "darling 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_builder_core 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "darling 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_builder_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "derive_builder_core" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "darling 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", + "darling 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -651,9 +629,9 @@ dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -822,7 +800,7 @@ dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive_internals 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -845,7 +823,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "enumset_derive 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -856,7 +834,7 @@ dependencies = [ "darling 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -867,8 +845,8 @@ dependencies = [ "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -876,7 +854,7 @@ name = "error-chain" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.30 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -885,7 +863,7 @@ name = "failure" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.30 (registry+https://github.com/rust-lang/crates.io-index)", "failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -896,7 +874,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -944,7 +922,7 @@ name = "fuchsia-zircon" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -977,7 +955,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -988,7 +966,7 @@ dependencies = [ "futures-channel-preview 0.3.0-alpha.16 (registry+https://github.com/rust-lang/crates.io-index)", "futures-core-preview 0.3.0-alpha.16 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util-preview 0.3.0-alpha.16 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1054,7 +1032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1062,7 +1040,7 @@ name = "git2" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "libgit2-sys 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1073,10 +1051,10 @@ dependencies = [ [[package]] name = "h2" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1090,7 +1068,7 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1129,13 +1107,13 @@ dependencies = [ [[package]] name = "hyper" -version = "0.12.29" +version = "0.12.30" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "h2 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", + "h2 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1163,7 +1141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.12.29 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.30 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1188,7 +1166,7 @@ name = "indexmap" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1260,7 +1238,7 @@ dependencies = [ "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1278,6 +1256,17 @@ name = "lazycell" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lexical-core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libc" version = "0.2.58" @@ -1383,10 +1372,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1537,7 +1526,7 @@ name = "nix" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1559,21 +1548,23 @@ dependencies = [ ] [[package]] -name = "nom-trace" -version = "0.1.0" -source = "git+https://github.com/pythondude325/nom-trace.git#02f45edd3aa4ca9c82addb9c9a22e81044536e99" +name = "nom" +version = "5.0.0-beta2" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "nom_locate" version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/wycats/nom_locate.git?branch=nom5#9383cc779b23a746ff68cc7094d9ed7baaa661b2" dependencies = [ "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 5.0.0-beta2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1612,9 +1603,8 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "logos 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "logos-derive 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)", - "nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 5.0.0-beta2 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 0.3.1 (git+https://github.com/wycats/nom_locate.git?branch=nom5)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "pancurses 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1623,17 +1613,17 @@ dependencies = [ "prettyprint 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "prettytable-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ptree 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.18 (registry+https://github.com/rust-lang/crates.io-index)", "roxmltree 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustyline 4.1.0 (git+https://github.com/kkawakam/rustyline.git)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde-hjson 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)", "subprocess 0.1.19 (git+https://github.com/jonathandturner/rust-subprocess.git?branch=is_already_escaped)", - "sysinfo 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "sysinfo 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1646,47 +1636,49 @@ name = "num" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "num-iter 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", - "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-complex" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-integer" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-iter" -version = "0.1.38" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-rational" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-integer 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1694,20 +1686,20 @@ name = "num-traits" version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num_cpus" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1758,7 +1750,7 @@ name = "onig" version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "onig_sys 69.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1778,7 +1770,7 @@ name = "openssl" version = "0.10.23" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1796,7 +1788,7 @@ name = "openssl-sys" version = "0.9.47" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1808,7 +1800,7 @@ name = "ordered-float" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1871,7 +1863,7 @@ dependencies = [ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1886,7 +1878,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1895,7 +1887,7 @@ name = "parse-zoneinfo" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1968,14 +1960,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "plist" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "xml-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2017,9 +2009,9 @@ dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "console 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", + "console 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "content_inspector 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_builder 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_builder 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "directories 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2059,9 +2051,9 @@ dependencies = [ "directories 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "isatty 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde-value 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "tint 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2073,7 +2065,7 @@ dependencies = [ "error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2095,7 +2087,7 @@ name = "rand" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2113,7 +2105,7 @@ name = "rand_chacha" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2174,7 +2166,7 @@ name = "rand_pcg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2188,23 +2180,24 @@ dependencies = [ [[package]] name = "rayon" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rayon-core" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2241,19 +2234,19 @@ dependencies = [ [[package]] name = "regex" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2261,7 +2254,7 @@ dependencies = [ [[package]] name = "remove_dir_all" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2274,7 +2267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2290,13 +2283,13 @@ dependencies = [ "flate2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.12.29 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.30 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2324,7 +2317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rustc-demangle" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -2332,7 +2325,7 @@ name = "rustc-hash" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2440,10 +2433,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde_derive 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2454,7 +2447,7 @@ dependencies = [ "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2466,7 +2459,7 @@ dependencies = [ "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2476,17 +2469,17 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2495,7 +2488,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2505,7 +2498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2523,7 +2516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2534,7 +2527,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2582,7 +2575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "smallvec" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -2590,6 +2583,19 @@ name = "stable_deref_trait" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "stackvector" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "static_assertions" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "string" version = "0.2.0" @@ -2630,7 +2636,7 @@ dependencies = [ [[package]] name = "syn" -version = "0.15.34" +version = "0.15.36" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2645,7 +2651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2655,29 +2661,29 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "onig 4.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "plist 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "plist 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", - "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "sysinfo" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2690,7 +2696,7 @@ dependencies = [ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2699,7 +2705,7 @@ name = "term" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2716,7 +2722,7 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2724,7 +2730,7 @@ dependencies = [ [[package]] name = "termion" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2783,7 +2789,7 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2852,7 +2858,7 @@ dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2892,7 +2898,7 @@ dependencies = [ "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2922,7 +2928,7 @@ name = "toml" version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2930,7 +2936,7 @@ name = "toml" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2942,7 +2948,7 @@ dependencies = [ "failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "is-match 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "toml-query_derive 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2954,7 +2960,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "darling 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3009,7 +3015,7 @@ name = "unicode-normalization" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3027,6 +3033,14 @@ name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "url" version = "1.7.2" @@ -3039,7 +3053,7 @@ dependencies = [ [[package]] name = "utf8-ranges" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -3077,7 +3091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "walkdir" -version = "2.2.7" +version = "2.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3215,18 +3229,18 @@ dependencies = [ "checksum array-macro 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7c4ff37a25fb442a1fecfd399be0dde685558bca30fb998420532889a36852d2" "checksum arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" -"checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" -"checksum backtrace 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)" = "45934a579eff9fd0ff637ac376a4bd134f47f8fc603f0b211d696b54d61e35f1" +"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" +"checksum backtrace 0.3.30 (registry+https://github.com/rust-lang/crates.io-index)" = "ada4c783bb7e7443c14e0480f429ae2cc99da95065aeab7ee1b81ada0419404f" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" -"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" "checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum byte-unit 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6754bb4703aa167bed5381f0c6842f1cc31a9ecde3b9443f726dde3ad3afb841" "checksum bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8" -"checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "39f75544d7bbaf57560d2168f28fd649ff9c76153874db88bdbdfd839b1a7e7d" "checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" @@ -3240,7 +3254,7 @@ dependencies = [ "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum conch-parser 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "baf61cea7edff80a7d8a9a4c219d391664715559e9d4e7129494b45278705a41" "checksum config 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f9107d78ed62b3fa5a86e7d18e647abed48cfd8f8fab6c72f4cdb982d196f7e6" -"checksum console 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2bf3720d3f3fc30b721ef1ae54e13af3264af4af39dc476a8de56a6ee1e2184b" +"checksum console 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b11432ec626c0a221d0ff3914ea6887de61a12d89a814a4f1c49882fc3151a11" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum content_inspector 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38" "checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" @@ -3250,12 +3264,10 @@ dependencies = [ "checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum crossbeam-channel 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0f0ed1a4de2235cabda8558ff5840bffb97fcb64c97827f354a451307df5f72b" -"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" +"checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13" "checksum crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b18cd2e169ad86297e6bc0ad9aa679aee9daa4f19e8163860faf7c164e4f5a71" -"checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" "checksum crossbeam-epoch 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "04c9e3102cc2d69cd681412141b390abd55a362afc1540965dad0ad4d34280b4" "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" -"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" "checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c" "checksum csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9044e25afb0924b5a5fc5511689b0918629e85d68ea591e5e87fbf1e85ea1b3b" @@ -3271,8 +3283,8 @@ dependencies = [ "checksum darling_macro 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)" = "244e8987bd4e174385240cde20a3657f607fb0797563c28255c353b5819a07b1" "checksum darling_macro 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c6d8dac1c6f1d29a41c4712b4400f878cb4fcc4c7628f298dd75038e024998d1" "checksum derive-new 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6ca414e896ae072546f4d789f452daaecf60ddee4c9df5dc6d5936d769e3d87c" -"checksum derive_builder 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a0ca533e6abb78f9108585535ce2ae0b14c8b4504e138a9a28eaf8ba2b270c1d" -"checksum derive_builder_core 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fb484fe06ba1dc5b82f88aff700191dfc127e02b06b35e302c169706168e2528" +"checksum derive_builder 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ac53fa6a3cda160df823a9346442525dcaf1e171999a1cf23e67067e4fd64d4" +"checksum derive_builder_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0288a23da9333c246bb18c143426074a6ae96747995c5819d2947b64cd942b37" "checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe" "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum directories 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "72d337a64190607d4fcca2cb78982c5dd57f4916e19696b48a575fa746b6cb0f" @@ -3323,13 +3335,13 @@ dependencies = [ "checksum futures_codec 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b60f48aa03e365df015d2fbf0b79f17b440350c268a5e20305da17b394adcc1e" "checksum getset 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "19fbde0fad0c1c1f9474694b1f5c9ba22b09f2f74f74e6d2bd19c43f6656e2cb" "checksum git2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7339329bfa14a00223244311560d11f8f489b453fb90092af97f267a6090ab0" -"checksum h2 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "1e42e3daed5a7e17b12a0c23b5b2fbff23a925a570938ebee4baca1a9a1a2240" -"checksum hashbrown 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "570178d5e4952010d138b0f1d581271ff3a02406d990f887d1e87e3d6e43b0ac" +"checksum h2 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "69b2a5a3092cbebbc951fe55408402e696ee2ed09019137d1800fc2c411265d2" +"checksum hashbrown 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "29fba9abe4742d586dfd0c06ae4f7e73a1c2d86b856933509b269d82cdf06e18" "checksum http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "eed324f0f0daf6ec10c474f150505af2c143f251722bf9dbd1261bd1f2ee2c1a" "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" "checksum httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" -"checksum hyper 0.12.29 (registry+https://github.com/rust-lang/crates.io-index)" = "e2cd6adf83b3347d36e271f030621a8cf95fd1fd0760546b9fc5a24a0f1447c7" +"checksum hyper 0.12.30 (registry+https://github.com/rust-lang/crates.io-index)" = "40e7692b2009a70b1e9b362284add4d8b75880fefddb4acaa5e67194e843f219" "checksum hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" "checksum ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" @@ -3346,6 +3358,7 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +"checksum lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e82e023e062f1d25f807ad182008fba1b46538e999f908a08cc0c29e084462e" "checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" "checksum libgit2-sys 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "48441cb35dc255da8ae72825689a95368bf510659ae1ad55dc4aa88cb1789bf1" "checksum libssh2-sys 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "126a1f4078368b163bfdee65fbab072af08a1b374a5551b21e87ade27b1fbf9d" @@ -3376,16 +3389,16 @@ dependencies = [ "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -"checksum nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)" = "" -"checksum nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a47c112b3861d81f7fbf73892b9271af933af32bd5dee6889aa3c3fa9caed7e" +"checksum nom 5.0.0-beta2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfa215de9f747c2b3290ceee185976eb6ffc936087e19d810b57ba3ff85ae28c" +"checksum nom_locate 0.3.1 (git+https://github.com/wycats/nom_locate.git?branch=nom5)" = "" "checksum num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cf4825417e1e1406b3782a8ce92f4d53f26ec055e3622e1881ca8e9f5f9e08db" -"checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" -"checksum num-integer 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8af8caa3184078cd419b430ff93684cb13937970fcb7639f728992f33ce674" -"checksum num-iter 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "50b7cf973f09f27829571927831972c92bfd29de7a5a0148380afd12acf6d124" -"checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10" +"checksum num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fcb0cf31fb3ff77e6d2a6ebd6800df7fdcd106f2ad89113c9130bcd07f93dffc" +"checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" +"checksum num-iter 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "76bd5272412d173d6bf9afdf98db8612bbabc9a7a830b7bfc9c188911716132e" +"checksum num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2885278d5fe2adc2f75ced642d52d879bffaceb5a2e0b1d4309ffdfb239b454" "checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -"checksum num-traits 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d9c79c952a4a139f44a0fe205c4ee66ce239c0e6ce72cd935f5f7e2f717549dd" -"checksum num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" +"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" +"checksum num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcef43580c035376c0705c42792c294b66974abbfd2789b511784023f71f3273" "checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" "checksum objc 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "31d20fd2b37e07cf5125be68357b588672e8cefe9a96f8c17a9d46053b3e590d" "checksum objc-foundation 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" @@ -3415,7 +3428,7 @@ dependencies = [ "checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" "checksum pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c" -"checksum plist 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f4739851c08dd9a62a78beff2edf1a438517268b2c563c42fc6d9d3139e42d2a" +"checksum plist 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a9f075f6394100e7c105ed1af73fb1859d6fd14e49d4290d578120beb167f" "checksum pretty 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f60c0d9f6fc88ecdd245d90c1920ff76a430ab34303fc778d33b1d0a4c3bf6d3" "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" "checksum pretty_env_logger 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df8b3f4e0475def7d9c2e5de8e5a1306949849761e107b360d03e98eafaffd61" @@ -3436,20 +3449,20 @@ dependencies = [ "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" "checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -"checksum rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "373814f27745b2686b350dd261bfd24576a6fb0e2c5919b3a2b6005f820b0473" -"checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" +"checksum rayon 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4b0186e22767d5b9738a05eab7c6ac90b15db17e5b5f9bd87976dd7d89a10a4" +"checksum rayon-core 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbe0df8435ac0c397d467b6cad6d25543d06e8a019ef3f6af3c384597515bd2" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)" = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" -"checksum regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8f0a0bcab2fd7d1d7c54fa9eae6f43eddeb9ce2e7352f8518a814a4f65d60c58" -"checksum regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dcfd8681eebe297b81d98498869d4aae052137651ad7b96822f09ceb690d0a96" -"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0b2f0808e7d7e4fb1cb07feb6ff2f4bc827938f24f8c2e6a3beb7370af544bdd" +"checksum regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d76410686f9e3a17f06128962e0ecc5755870bb890c34820c7af7f1db2e1d48" +"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68ed587df09cfb7ce1bc6fe8f77e24db219f222c049326ccbfb948ec67e31664" "checksum reqwest 0.9.18 (registry+https://github.com/rust-lang/crates.io-index)" = "00eb63f212df0e358b427f0f40aa13aaea010b470be642ad422bcbca2feff2e4" "checksum roxmltree 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "53b0200cbfa8b3f6cfd6076592717d697a1ddc57cb2a8fbfd3d133c06011b579" "checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" -"checksum rustc-demangle 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "ccc78bfd5acd7bf3e89cffcf899e5cb1a52d6fafa8dec2739ad70c9577a57288" +"checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" "checksum rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7540fc8b0c49f096ee9c961cda096467dce8084bec6bdca2fc83895fd9b28cb8" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustyline 4.1.0 (git+https://github.com/kkawakam/rustyline.git)" = "" @@ -3465,11 +3478,11 @@ dependencies = [ "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" -"checksum serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)" = "a72e9b96fa45ce22a4bc23da3858dfccfd60acd28a25bcd328a98fdd6bea43fd" +"checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be" "checksum serde-hjson 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0b833c5ad67d52ced5f5938b2980f32a9c1c5ef047f0b4fb3127e7a423c76153" "checksum serde-hjson 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4640cf3168e40c00c874ff1ad436c0f18c37edec101d5d897a4396f617abce29" "checksum serde-value 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7a663f873dedc4eac1a559d4c6bc0d0b2c34dc5ac4702e105014b8281489e44f" -"checksum serde_derive 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)" = "101b495b109a3e3ca8c4cbe44cf62391527cdfb6ba15821c5ce80bcd5ea23f9f" +"checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e" "checksum serde_derive_internals 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8a80c6c0b1ebbcea4ec2c7e9e2e9fa197a425d17f1afec8ba79fcd1352b18ffb" "checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" "checksum serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" @@ -3481,22 +3494,24 @@ dependencies = [ "checksum signal-hook-registry 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cded4ffa32146722ec54ab1f16320568465aa922aa9ab4708129599740da85d7" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -"checksum smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" +"checksum smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ab606a9c5e214920bb66c458cd7be8ef094f813f20fe77a54cc7dbfff220d4b7" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" +"checksum stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c049c77bf85fbc036484c97b008276d539d9ebff9dfbde37b632ebcd5b8746b6" +"checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum string 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0bbfb8937e38e34c3444ff00afb28b0811d9554f15c5ad64d12b0308d1d1995" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" "checksum subprocess 0.1.19 (git+https://github.com/jonathandturner/rust-subprocess.git?branch=is_already_escaped)" = "" "checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" -"checksum syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)" = "a1393e4a97a19c01e900df2aec855a29f71cf02c402e2f443b8d2747c25c5dbe" +"checksum syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)" = "8b4f551a91e2e3848aeef8751d0d4eec9489b6474c720fd4c55958d8d31a430c" "checksum synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02353edf96d6e4dc81aea2d8490a7e9db177bf8acb0e951c24940bf866cb313f" "checksum syntect 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e80b8831c5a543192ffc3727f01cf0e57579c6ac15558e3048bfb5708892167b" -"checksum sysinfo 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "65f0e28a49b7bf142cee89befd7077b40627d7cc70aa8a8acfe03afc26016c33" +"checksum sysinfo 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cf62641ed7e88e20242b948d17b9fcc37e80b5599cf09cde190d6d4bb4bf289" "checksum tempfile 3.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7dc4738f2e68ed2855de5ac9cdbe05c9216773ecde4739b2f095002ab03a13ef" "checksum term 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" -"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" -"checksum termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" +"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" +"checksum termion 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6a8fb22f7cde82c8220e5aeacb3258ed7ce996142c77cba193f203515e26c330" "checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" @@ -3529,15 +3544,16 @@ dependencies = [ "checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -"checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" +"checksum utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9d50aa7650df78abf942826607c62468ce18d9019673d4a2ebe1865dbb96ffde" "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" "checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a" "checksum vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d9d7ed3431229a144296213105a390676cc49c9b6a72bd19f3176c98e129fa1" +"checksum walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c7904a7e2bb3cdf0cf5e783f44204a85a37a93151738fa349f06680f59a98b45" "checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" diff --git a/Cargo.toml b/Cargo.toml index ce9b612010..b8f9362100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ prettytable-rs = "0.8.0" itertools = "0.8.0" ansi_term = "0.11.0" conch-parser = "0.1.1" -nom = "4.2.3" +nom = "5.0.0-beta2" dunce = "1.0.0" indexmap = { version = "1.0.2", features = ["serde-1"] } chrono-humanize = "0.0.11" @@ -61,11 +61,10 @@ clipboard = "0.5" reqwest = "0.9" roxmltree = "0.6.0" pretty = "0.5.2" -nom_locate = "0.3.1" +nom_locate = { git = "https://github.com/wycats/nom_locate.git", branch = "nom5" } derive_more = "0.15.0" enum-utils = "0.1.0" unicode-xid = "0.1.0" -nom-trace = { version = "0.1.0", git = "https://github.com/pythondude325/nom-trace.git" } [dependencies.pancurses] version = "0.16" diff --git a/src/cli.rs b/src/cli.rs index 24e654da34..2263eb43fd 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -11,13 +11,13 @@ use crate::commands::classified::{ use crate::context::Context; crate use crate::errors::ShellError; use crate::evaluate::Scope; +use crate::parser::parse2::{PipelineElement, TokenNode}; +use crate::parser::registry; use crate::git::current_branch; use crate::object::Value; -use crate::parser::ast::{Expression, Leaf, RawExpression}; -use crate::parser::{Args, Pipeline}; -use log::debug; +use log::{debug, trace}; use rustyline::error::ReadlineError; use rustyline::{self, ColorMode, Config, Editor}; @@ -59,20 +59,21 @@ pub async fn cli() -> Result<(), Box> { command("from-xml", from_xml::from_xml), command("from-yaml", from_yaml::from_yaml), command("get", get::get), - command("open", open::open), command("enter", enter::enter), command("exit", exit::exit), command("pick", pick::pick), command("split-column", split_column::split_column), command("split-row", split_row::split_row), + command("lines", lines::lines), command("reject", reject::reject), command("trim", trim::trim), command("to-array", to_array::to_array), command("to-json", to_json::to_json), command("to-toml", to_toml::to_toml), + command("sort-by", sort_by::sort_by), + Arc::new(Open), Arc::new(Where), Arc::new(Config), - command("sort-by", sort_by::sort_by), ]); context.add_sinks(vec![ @@ -138,7 +139,7 @@ pub async fn cli() -> Result<(), Box> { let host = context.host.lock().unwrap(); let writer = host.err_termcolor(); line.push_str(" "); - let files = crate::parser::span::Files::new(line); + let files = crate::parser::Files::new(line); language_reporting::emit( &mut writer.lock(), @@ -231,7 +232,7 @@ async fn process_line(readline: Result, ctx: &mut Context debug!("=== Parsed ==="); debug!("{:#?}", result); - let mut pipeline = classify_pipeline(&result, ctx)?; + let mut pipeline = classify_pipeline(&result, ctx, &line)?; match pipeline.commands.last() { Some(ClassifiedCommand::Sink(_)) => {} @@ -239,9 +240,9 @@ async fn process_line(readline: Result, ctx: &mut Context _ => pipeline.commands.push(ClassifiedCommand::Sink(SinkCommand { command: sink("autoview", autoview::autoview), name_span: None, - args: Args { - positional: vec![], - named: indexmap::IndexMap::new(), + args: registry::Args { + positional: None, + named: None, }, })), } @@ -347,14 +348,15 @@ async fn process_line(readline: Result, ctx: &mut Context } fn classify_pipeline( - pipeline: &Pipeline, + pipeline: &TokenNode, context: &Context, + source: &str, ) -> Result { - let commands: Result, _> = pipeline - .commands + let pipeline = pipeline.as_pipeline()?; + + let commands: Result, ShellError> = pipeline .iter() - .cloned() - .map(|item| classify_command(&item, context)) + .map(|item| classify_command(&item, context, source)) .collect(); Ok(ClassifiedPipeline { @@ -363,61 +365,50 @@ fn classify_pipeline( } fn classify_command( - command: &Expression, + command: &PipelineElement, context: &Context, + source: &str, ) -> Result { - // let command_name = &command.name[..]; - // let args = &command.args; + let call = command.call(); - if let Expression { - expr: RawExpression::Call(call), - .. - } = command - { - match (&call.name, &call.args) { - ( - Expression { - expr: RawExpression::Leaf(Leaf::Bare(name)), - span, - }, - args, - ) => match context.has_command(&name.to_string()) { + match call { + call if call.head().is_bare() => { + let head = call.head(); + let name = head.source(source); + + match context.has_command(name) { true => { - let command = context.get_command(&name.to_string()); + let command = context.get_command(name); let config = command.config(); let scope = Scope::empty(); - let args = match args { - Some(args) => config.evaluate_args(args.iter(), &scope)?, - None => Args::default(), - }; + trace!("classifying {:?}", config); + + let args = config.evaluate_args(call, context, &scope, source)?; Ok(ClassifiedCommand::Internal(InternalCommand { command, - name_span: Some(span.clone()), + name_span: Some(head.span().clone()), args, })) } - false => match context.has_sink(&name.to_string()) { + false => match context.has_sink(name) { true => { - let command = context.get_sink(&name.to_string()); + let command = context.get_sink(name); let config = command.config(); let scope = Scope::empty(); - let args = match args { - Some(args) => config.evaluate_args(args.iter(), &scope)?, - None => Args::default(), - }; + let args = config.evaluate_args(call, context, &scope, source)?; Ok(ClassifiedCommand::Sink(SinkCommand { command, - name_span: Some(span.clone()), + name_span: Some(head.span().clone()), args, })) } false => { - let arg_list_strings: Vec = match args { - Some(args) => args.iter().map(|i| i.as_external_arg()).collect(), + let arg_list_strings: Vec = match call.children() { + Some(args) => args.iter().map(|i| i.as_external_arg(source)).collect(), None => vec![], }; @@ -427,17 +418,11 @@ fn classify_command( })) } }, - }, - - (_, None) => Err(ShellError::string( - "Unimplemented command that is just an expression (1)", - )), - (_, Some(_)) => Err(ShellError::string("Unimplemented dynamic command")), + } } - } else { - Err(ShellError::string(&format!( - "Unimplemented command that is just an expression (2) -- {:?}", - command - ))) + + _ => Err(ShellError::unimplemented( + "classify_command on command whose head is not bare", + )), } } diff --git a/src/commands.rs b/src/commands.rs index 5c21cfc20b..9178731248 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -13,6 +13,7 @@ crate mod from_toml; crate mod from_xml; crate mod from_yaml; crate mod get; +crate mod lines; crate mod ls; crate mod open; crate mod pick; @@ -34,5 +35,5 @@ crate mod where_; crate use command::command; crate use config::Config; - +crate use open::Open; crate use where_::Where; diff --git a/src/commands/cd.rs b/src/commands/cd.rs index d69f71e2bc..1ed46ab205 100644 --- a/src/commands/cd.rs +++ b/src/commands/cd.rs @@ -10,7 +10,8 @@ pub fn cd(args: CommandArgs) -> Result { match latest.obj { Value::Filesystem => { let cwd = latest.path().to_path_buf(); - let path = match args.positional.first() { + + let path = match args.nth(0) { None => match dirs::home_dir() { Some(o) => o, _ => return Err(ShellError::string("Can not change to home directory")), @@ -23,7 +24,7 @@ pub fn cd(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Can not change to directory", "directory not found", - args.positional[0].span.clone(), + v.span.clone(), )); } } @@ -34,11 +35,11 @@ pub fn cd(args: CommandArgs) -> Result { match env::set_current_dir(&path) { Ok(_) => {} Err(_) => { - if args.positional.len() > 0 { + if args.len() > 0 { return Err(ShellError::labeled_error( "Can not change to directory", "directory not found", - args.positional[0].span.clone(), + args.nth(0).unwrap().span.clone(), )); } else { return Err(ShellError::string("Can not change to directory")); @@ -50,7 +51,7 @@ pub fn cd(args: CommandArgs) -> Result { } _ => { let mut stream = VecDeque::new(); - match args.positional.first() { + match args.nth(0) { None => { stream.push_back(ReturnValue::change_cwd(PathBuf::from("/"))); } diff --git a/src/commands/classified.rs b/src/commands/classified.rs index fcd9586f2d..c412bcd50d 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -1,14 +1,14 @@ use crate::commands::command::Sink; -use crate::parser::ast::Expression; -use crate::parser::lexer::Span; -use crate::parser::registry::Args; +use crate::parser::{registry::Args, Span, TokenNode}; use crate::prelude::*; use bytes::{BufMut, BytesMut}; use futures_codec::{Decoder, Encoder, Framed}; +use log::{log_enabled, trace}; use std::io::{Error, ErrorKind}; use std::path::PathBuf; use std::sync::Arc; use subprocess::Exec; + /// A simple `Codec` implementation that splits up data into lines. pub struct LinesCodec {} @@ -79,7 +79,7 @@ crate struct ClassifiedPipeline { crate enum ClassifiedCommand { #[allow(unused)] - Expr(Expression), + Expr(TokenNode), Internal(InternalCommand), Sink(SinkCommand), External(ExternalCommand), @@ -109,12 +109,22 @@ impl InternalCommand { context: &mut Context, input: ClassifiedInputStream, ) -> Result { - let result = context.run_command( - self.command, - self.name_span.clone(), - self.args, - input.objects, - )?; + let objects = if log_enabled!(log::Level::Trace) { + trace!("->"); + trace!("{}", self.command.name()); + trace!("{:?}", self.args.debug()); + let objects: Vec<_> = input.objects.collect().await; + trace!( + "input = {:#?}", + objects.iter().map(|o| o.debug()).collect::>(), + ); + VecDeque::from(objects).boxed() + } else { + input.objects + }; + + let result = + context.run_command(self.command, self.name_span.clone(), self.args, objects)?; let env = context.env.clone(); let stream = result.filter_map(move |v| match v { @@ -171,9 +181,10 @@ impl ExternalCommand { ) -> Result { let inputs: Vec = input.objects.collect().await; + trace!("{:?} -> {}", inputs, self.name); + let mut arg_string = format!("{}", self.name); for arg in &self.args { - arg_string.push_str(" "); arg_string.push_str(&arg); } @@ -184,6 +195,7 @@ impl ExternalCommand { if arg_string.contains("$it") { let mut first = true; + for i in &inputs { if !first { process = process.arg("&&"); @@ -193,6 +205,10 @@ impl ExternalCommand { } for arg in &self.args { + if arg.chars().all(|c| c.is_whitespace()) { + continue; + } + process = process.arg(&arg.replace("$it", &i.as_string().unwrap())); } } @@ -217,6 +233,10 @@ impl ExternalCommand { } for arg in &self.args { + if arg.chars().all(|c| c.is_whitespace()) { + continue; + } + new_arg_string.push_str(" "); new_arg_string.push_str(&arg.replace("$it", &i.as_string().unwrap())); } diff --git a/src/commands/command.rs b/src/commands/command.rs index 2d9faff09e..335ab2ae3e 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -1,25 +1,53 @@ use crate::errors::ShellError; use crate::object::Value; -use crate::parser::lexer::Span; -use crate::parser::lexer::Spanned; -use crate::parser::CommandConfig; +use crate::parser::{ + registry::{self, Args}, + Span, Spanned, +}; use crate::prelude::*; +use getset::Getters; use std::path::PathBuf; +#[derive(Getters)] +#[get = "crate"] pub struct CommandArgs { pub host: Arc>, pub env: Arc>>, pub name_span: Option, - pub positional: Vec>, - pub named: indexmap::IndexMap, + pub args: Args, pub input: InputStream, } +impl CommandArgs { + pub fn nth(&self, pos: usize) -> Option<&Spanned> { + self.args.nth(pos) + } + + pub fn positional_iter(&self) -> impl Iterator> { + self.args.positional_iter() + } + + pub fn expect_nth(&self, pos: usize) -> Result<&Spanned, ShellError> { + self.args.expect_nth(pos) + } + + pub fn len(&self) -> usize { + self.args.len() + } + + pub fn get(&self, name: &str) -> Option<&Spanned> { + self.args.get(name) + } + + pub fn has(&self, name: &str) -> bool { + self.args.has(name) + } +} + pub struct SinkCommandArgs { pub ctx: Context, pub name_span: Option, - pub positional: Vec>, - pub named: indexmap::IndexMap, + pub args: Args, pub input: Vec, } @@ -46,8 +74,8 @@ pub trait Command { fn run(&self, args: CommandArgs) -> Result; fn name(&self) -> &str; - fn config(&self) -> CommandConfig { - CommandConfig { + fn config(&self) -> registry::CommandConfig { + registry::CommandConfig { name: self.name().to_string(), mandatory_positional: vec![], optional_positional: vec![], @@ -61,8 +89,8 @@ pub trait Sink { fn run(&self, args: SinkCommandArgs) -> Result<(), ShellError>; fn name(&self) -> &str; - fn config(&self) -> CommandConfig { - CommandConfig { + fn config(&self) -> registry::CommandConfig { + registry::CommandConfig { name: self.name().to_string(), mandatory_positional: vec![], optional_positional: vec![], diff --git a/src/commands/config.rs b/src/commands/config.rs index 11b35a84e2..d18bc2f17f 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1,8 +1,7 @@ use crate::errors::ShellError; use crate::object::config; use crate::object::Value; -use crate::parser::registry::{NamedType, NamedValue}; -use crate::parser::CommandConfig; +use crate::parser::registry::{CommandConfig, NamedType, NamedValue}; use crate::prelude::*; use indexmap::IndexMap; use log::trace; @@ -19,7 +18,7 @@ impl Command for Config { fn config(&self) -> CommandConfig { let mut named: IndexMap = IndexMap::new(); - named.insert("set".to_string(), NamedType::Optional(NamedValue::Tuple)); + named.insert("set".to_string(), NamedType::Optional(NamedValue::Single)); named.insert("get".to_string(), NamedType::Optional(NamedValue::Single)); named.insert("clear".to_string(), NamedType::Switch); @@ -41,10 +40,10 @@ impl Command for Config { pub fn config(args: CommandArgs) -> Result { let mut result = crate::object::config::config()?; - trace!("{:#?}", args.positional); - trace!("{:#?}", args.named); + trace!("{:#?}", args.args.positional); + trace!("{:#?}", args.args.named); - if let Some(v) = args.named.get("get") { + if let Some(v) = args.get("get") { let key = v.as_string()?; let value = result .get(&key) @@ -56,7 +55,7 @@ pub fn config(args: CommandArgs) -> Result { ); } - if let Some(v) = args.named.get("set") { + if let Some(v) = args.get("set") { if let Ok((key, value)) = v.as_pair() { result.insert(key.as_string()?, value.clone()); @@ -71,7 +70,7 @@ pub fn config(args: CommandArgs) -> Result { } } - if let Some(_) = args.named.get("clear") { + if let Some(_) = args.get("clear") { result.clear(); config::write_config(&result)?; @@ -84,7 +83,7 @@ pub fn config(args: CommandArgs) -> Result { ); } - if let Some(v) = args.named.get("remove") { + if let Some(v) = args.get("remove") { let key = v.as_string()?; if result.contains_key(&key) { @@ -104,7 +103,7 @@ pub fn config(args: CommandArgs) -> Result { ); } - if args.positional.len() == 0 { + if args.len() == 0 { return Ok( futures::stream::once(futures::future::ready(ReturnValue::Value(Value::Object( result.into(), diff --git a/src/commands/enter.rs b/src/commands/enter.rs index d1bbef5da2..bd00ab2d91 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -1,26 +1,28 @@ use crate::commands::command::CommandAction; use crate::errors::ShellError; use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::Spanned; use crate::prelude::*; use std::path::{Path, PathBuf}; pub fn enter(args: CommandArgs) -> Result { - if args.positional.len() == 0 { - return Err(ShellError::string("open requires a filepath or url")); - } + let path = match args.nth(0) { + None => return Err(ShellError::string("open requires a filepath or url")), + Some(p) => p, + }; let cwd = args - .env + .env() .lock() .unwrap() .first() .unwrap() .path() .to_path_buf(); + let mut full_path = PathBuf::from(cwd); - let (file_extension, contents) = match &args.positional[0].item { + let (file_extension, contents) = match path.item() { Value::Primitive(Primitive::String(s)) => { if s.starts_with("http:") || s.starts_with("https:") { let response = reqwest::get(s); @@ -43,7 +45,7 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Web page contents corrupt", "received garbled data", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }, @@ -51,7 +53,7 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "URL could not be opened", "url not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -68,7 +70,7 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "File cound not be opened", "file not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -78,14 +80,14 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Expected string value for filename", "expected filename", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }; let mut stream = VecDeque::new(); - let open_raw = match args.positional.get(1) { + let open_raw = match args.nth(1) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), .. diff --git a/src/commands/exit.rs b/src/commands/exit.rs index 0669adee76..27438f6654 100644 --- a/src/commands/exit.rs +++ b/src/commands/exit.rs @@ -1,11 +1,8 @@ use crate::commands::command::CommandAction; use crate::errors::ShellError; -use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; use crate::prelude::*; -use std::path::{Path, PathBuf}; -pub fn exit(args: CommandArgs) -> Result { +pub fn exit(_args: CommandArgs) -> Result { let mut stream = VecDeque::new(); stream.push_back(ReturnValue::Action(CommandAction::Exit)); Ok(stream.boxed()) diff --git a/src/commands/first.rs b/src/commands/first.rs index afc1ff8f52..48d6091488 100644 --- a/src/commands/first.rs +++ b/src/commands/first.rs @@ -4,7 +4,7 @@ use crate::prelude::*; // TODO: "Amount remaining" wrapper pub fn first(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { if let Some(span) = args.name_span { return Err(ShellError::labeled_error( "First requires an amount", @@ -16,7 +16,7 @@ pub fn first(args: CommandArgs) -> Result { } } - let amount = args.positional[0].as_i64(); + let amount = args.expect_nth(0)?.as_i64(); let amount = match amount { Ok(o) => o, @@ -24,7 +24,7 @@ pub fn first(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Value is not a number", "expected integer", - args.positional[0].span, + args.expect_nth(0)?.span, )) } }; diff --git a/src/commands/get.rs b/src/commands/get.rs index b3869d97a3..37f83a695e 100644 --- a/src/commands/get.rs +++ b/src/commands/get.rs @@ -20,7 +20,7 @@ fn get_member(path: &str, obj: &Value) -> Option { } pub fn get(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { if let Some(span) = args.name_span { return Err(ShellError::labeled_error( "Get requires a field or field path", @@ -32,7 +32,7 @@ pub fn get(args: CommandArgs) -> Result { } } - let amount = args.positional[0].as_i64(); + let amount = args.expect_nth(0)?.as_i64(); // If it's a number, get the row instead of the column if let Ok(amount) = amount { @@ -44,7 +44,13 @@ pub fn get(args: CommandArgs) -> Result { .boxed()); } - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args + .args + .positional + .unwrap() + .iter() + .map(|a| a.as_string()) + .collect(); let fields = fields?; let stream = args diff --git a/src/commands/lines.rs b/src/commands/lines.rs new file mode 100644 index 0000000000..420a1c0177 --- /dev/null +++ b/src/commands/lines.rs @@ -0,0 +1,35 @@ +use crate::errors::ShellError; +use crate::object::{Primitive, Value}; +use crate::prelude::*; +use log::trace; + +// TODO: "Amount remaining" wrapper + +pub fn lines(args: CommandArgs) -> Result { + let input = args.input; + + let stream = input + .map(move |v| match v { + Value::Primitive(Primitive::String(s)) => { + let splitter = "\n"; + let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); + + trace!("split result = {:?}", split_result); + + let mut result = VecDeque::new(); + for s in split_result { + result.push_back(ReturnValue::Value(Value::Primitive(Primitive::String( + s.to_string(), + )))); + } + result + } + _ => { + let result = VecDeque::new(); + result + } + }) + .flatten(); + + Ok(stream.boxed()) +} diff --git a/src/commands/ls.rs b/src/commands/ls.rs index 70cbf9d53e..405fa0fc37 100644 --- a/src/commands/ls.rs +++ b/src/commands/ls.rs @@ -1,6 +1,6 @@ use crate::errors::ShellError; use crate::object::{dir_entry_dict, Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::Spanned; use crate::prelude::*; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -10,7 +10,7 @@ pub fn ls(args: CommandArgs) -> Result { let path = env.last().unwrap().path.to_path_buf(); let obj = &env.last().unwrap().obj; let mut full_path = PathBuf::from(path); - match &args.positional.get(0) { + match &args.nth(0) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), .. @@ -24,7 +24,7 @@ pub fn ls(args: CommandArgs) -> Result { let entries = match entries { Err(e) => { - if let Some(s) = args.positional.get(0) { + if let Some(s) = args.nth(0) { return Err(ShellError::labeled_error( e.to_string(), e.to_string(), diff --git a/src/commands/open.rs b/src/commands/open.rs index 15392e48d3..414d0b5863 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -1,11 +1,36 @@ use crate::errors::ShellError; use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::registry::{CommandConfig, NamedType}; use crate::prelude::*; +use indexmap::IndexMap; use std::path::{Path, PathBuf}; -pub fn open(args: CommandArgs) -> Result { - if args.positional.len() == 0 { +pub struct Open; + +impl Command for Open { + fn run(&self, args: CommandArgs) -> Result { + open(args) + } + fn name(&self) -> &str { + "open" + } + + fn config(&self) -> CommandConfig { + let mut named: IndexMap = IndexMap::new(); + named.insert("raw".to_string(), NamedType::Switch); + + CommandConfig { + name: self.name().to_string(), + mandatory_positional: vec![], + optional_positional: vec![], + rest_positional: false, + named, + } + } +} + +fn open(args: CommandArgs) -> Result { + if args.len() == 0 { return Err(ShellError::string("open requires a filepath or url")); } @@ -19,7 +44,7 @@ pub fn open(args: CommandArgs) -> Result { .to_path_buf(); let mut full_path = PathBuf::from(cwd); - let (file_extension, contents) = match &args.positional[0].item { + let (file_extension, contents) = match &args.expect_nth(0)?.item { Value::Primitive(Primitive::String(s)) => { if s.starts_with("http:") || s.starts_with("https:") { let response = reqwest::get(s); @@ -42,7 +67,7 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Web page contents corrupt", "received garbled data", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }, @@ -50,7 +75,7 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "URL could not be opened", "url not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -67,7 +92,7 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "File cound not be opened", "file not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -77,27 +102,14 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Expected string value for filename", "expected filename", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }; let mut stream = VecDeque::new(); - let open_raw = match args.positional.get(1) { - Some(Spanned { - item: Value::Primitive(Primitive::String(s)), - .. - }) if s == "--raw" => true, - Some(v) => { - return Err(ShellError::labeled_error( - "Unknown flag for open", - "unknown flag", - v.span, - )) - } - _ => false, - }; + let open_raw = args.has("raw"); match file_extension { Some(x) if x == "toml" && !open_raw => { diff --git a/src/commands/pick.rs b/src/commands/pick.rs index 73e953ad0c..b5a17e0372 100644 --- a/src/commands/pick.rs +++ b/src/commands/pick.rs @@ -4,7 +4,7 @@ use crate::object::Value; use crate::prelude::*; pub fn pick(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { if let Some(span) = args.name_span { return Err(ShellError::labeled_error( "Pick requires fields", @@ -16,7 +16,7 @@ pub fn pick(args: CommandArgs) -> Result { } } - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let objects = args diff --git a/src/commands/reject.rs b/src/commands/reject.rs index 372adcf7c5..4a9f28a7ed 100644 --- a/src/commands/reject.rs +++ b/src/commands/reject.rs @@ -4,7 +4,7 @@ use crate::object::Value; use crate::prelude::*; pub fn reject(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { if let Some(span) = args.name_span { return Err(ShellError::labeled_error( "Reject requires fields", @@ -16,7 +16,7 @@ pub fn reject(args: CommandArgs) -> Result { } } - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let stream = args diff --git a/src/commands/save.rs b/src/commands/save.rs index 5d94496cf3..7ba3ebf3ad 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -1,14 +1,19 @@ use crate::commands::command::SinkCommandArgs; use crate::errors::ShellError; use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::Spanned; use std::path::{Path, PathBuf}; pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> { - if args.positional.len() == 0 { + if args.args.positional.is_none() { return Err(ShellError::string("save requires a filepath")); } + let positional = match args.args.positional { + None => return Err(ShellError::string("save requires a filepath")), + Some(p) => p, + }; + let cwd = args .ctx .env @@ -19,12 +24,12 @@ pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> { .path() .to_path_buf(); let mut full_path = PathBuf::from(cwd); - match &(args.positional[0].item) { + match &(positional[0].item) { Value::Primitive(Primitive::String(s)) => full_path.push(Path::new(s)), _ => {} } - let save_raw = match args.positional.get(1) { + let save_raw = match positional.get(1) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), .. diff --git a/src/commands/size.rs b/src/commands/size.rs index 4fe90e25ff..59df8165c3 100644 --- a/src/commands/size.rs +++ b/src/commands/size.rs @@ -6,7 +6,7 @@ use std::fs::File; use std::io::prelude::*; pub fn size(args: CommandArgs) -> Result { - if args.positional.is_empty() { + if args.len() == 0 { return Err(ShellError::string("size requires at least one file")); } let cwd = args @@ -21,7 +21,7 @@ pub fn size(args: CommandArgs) -> Result { let mut contents = String::new(); let mut list = VecDeque::new(); - for name in args.positional { + for name in args.positional_iter() { let name = name.as_string()?; let path = cwd.join(&name); let mut file = File::open(path)?; diff --git a/src/commands/skip.rs b/src/commands/skip.rs index 38af0c1a79..61e71cc2de 100644 --- a/src/commands/skip.rs +++ b/src/commands/skip.rs @@ -2,7 +2,7 @@ use crate::errors::ShellError; use crate::prelude::*; pub fn skip(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { if let Some(span) = args.name_span { return Err(ShellError::labeled_error( "Skip requires an amount", @@ -14,7 +14,7 @@ pub fn skip(args: CommandArgs) -> Result { } } - let amount = args.positional[0].as_i64(); + let amount = args.expect_nth(0)?.as_i64(); let amount = match amount { Ok(o) => o, @@ -22,7 +22,7 @@ pub fn skip(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Value is not a number", "expected integer", - args.positional[0].span, + args.expect_nth(0)?.span, )) } }; diff --git a/src/commands/sort_by.rs b/src/commands/sort_by.rs index d488d5cbd4..9793e9b1b9 100644 --- a/src/commands/sort_by.rs +++ b/src/commands/sort_by.rs @@ -2,7 +2,7 @@ use crate::errors::ShellError; use crate::prelude::*; pub fn sort_by(args: CommandArgs) -> Result { - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let output = args.input.collect::>(); diff --git a/src/commands/split_column.rs b/src/commands/split_column.rs index c768ec6b83..b546955793 100644 --- a/src/commands/split_column.rs +++ b/src/commands/split_column.rs @@ -6,20 +6,33 @@ use log::trace; // TODO: "Amount remaining" wrapper pub fn split_column(args: CommandArgs) -> Result { + let positional: Vec<_> = args.positional_iter().cloned().collect(); + + if positional.len() == 0 { + if let Some(span) = args.name_span { + return Err(ShellError::labeled_error( + "split-column requires arguments", + "needs parameter", + span, + )); + } else { + return Err(ShellError::string("split-column requires arguments.")); + } + } + let input = args.input; - let args = args.positional; Ok(input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - let splitter = args[0].as_string().unwrap().replace("\\n", "\n"); + let splitter = positional[0].as_string().unwrap().replace("\\n", "\n"); trace!("splitting with {:?}", splitter); let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); trace!("split result = {:?}", split_result); // If they didn't provide column names, make up our own - if (args.len() - 1) == 0 { + if (positional.len() - 1) == 0 { let mut gen_columns = vec![]; for i in 0..split_result.len() { gen_columns.push(format!("Column{}", i + 1)); @@ -33,9 +46,9 @@ pub fn split_column(args: CommandArgs) -> Result { ); } ReturnValue::Value(Value::Object(dict)) - } else if split_result.len() == (args.len() - 1) { + } else if split_result.len() == (positional.len() - 1) { let mut dict = crate::object::Dictionary::default(); - for (k, v) in split_result.iter().zip(args.iter().skip(1)) { + for (k, v) in split_result.iter().zip(positional.iter().skip(1)) { dict.add( v.as_string().unwrap(), Value::Primitive(Primitive::String(k.to_string())), @@ -44,7 +57,7 @@ pub fn split_column(args: CommandArgs) -> Result { ReturnValue::Value(Value::Object(dict)) } else { let mut dict = crate::object::Dictionary::default(); - for k in args.iter().skip(1) { + for k in positional.iter().skip(1) { dict.add( k.as_string().unwrap().trim(), Value::Primitive(Primitive::String("".to_string())), diff --git a/src/commands/split_row.rs b/src/commands/split_row.rs index fead9f8e1f..80dbf29adc 100644 --- a/src/commands/split_row.rs +++ b/src/commands/split_row.rs @@ -1,18 +1,32 @@ use crate::errors::ShellError; use crate::object::{Primitive, Value}; +use crate::parser::Spanned; use crate::prelude::*; use log::trace; // TODO: "Amount remaining" wrapper pub fn split_row(args: CommandArgs) -> Result { + let positional: Vec> = args.positional_iter().cloned().collect(); + + if positional.len() == 0 { + if let Some(span) = args.name_span { + return Err(ShellError::labeled_error( + "split-row requires arguments", + "needs parameter", + span, + )); + } else { + return Err(ShellError::string("split-row requires arguments.")); + } + } + let input = args.input; - let args = args.positional; let stream = input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - let splitter = args[0].as_string().unwrap().replace("\\n", "\n"); + let splitter = positional[0].as_string().unwrap().replace("\\n", "\n"); trace!("splitting with {:?}", splitter); let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); diff --git a/src/commands/view.rs b/src/commands/view.rs index 30dce91d88..a24b28ff74 100644 --- a/src/commands/view.rs +++ b/src/commands/view.rs @@ -3,7 +3,7 @@ use crate::prelude::*; use prettyprint::PrettyPrinter; pub fn view(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { if let Some(span) = args.name_span { return Err(ShellError::labeled_error( "View requires a filename", @@ -15,7 +15,7 @@ pub fn view(args: CommandArgs) -> Result { } } - let target = match args.positional[0].as_string() { + let target = match args.expect_nth(0)?.as_string() { Ok(s) => s.clone(), Err(e) => { if let Some(span) = args.name_span { diff --git a/src/commands/where_.rs b/src/commands/where_.rs index 45ac0c7fef..265dbf8caf 100644 --- a/src/commands/where_.rs +++ b/src/commands/where_.rs @@ -1,6 +1,5 @@ use crate::errors::ShellError; -use crate::parser::registry::PositionalType; -use crate::parser::CommandConfig; +use crate::parser::registry::{CommandConfig, PositionalType}; use crate::prelude::*; pub struct Where; @@ -25,11 +24,11 @@ impl Command for Where { } pub fn r#where(args: CommandArgs) -> Result { - if args.positional.is_empty() { + if args.len() == 0 { return Err(ShellError::string("select requires a field")); } - let block = args.positional[0].as_block()?; + let block = args.expect_nth(0)?.as_block()?; let input = args.input; let objects = input.filter_map(move |item| { diff --git a/src/context.rs b/src/context.rs index ff571369d4..c0bc2bb303 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,7 +1,8 @@ -use crate::commands::command::Sink; -use crate::commands::command::SinkCommandArgs; -use crate::parser::lexer::Span; -use crate::parser::Args; +use crate::commands::command::{Sink, SinkCommandArgs}; +use crate::parser::{ + registry::{Args, CommandConfig, CommandRegistry}, + Span, +}; use crate::prelude::*; use indexmap::IndexMap; @@ -56,8 +57,7 @@ impl Context { let command_args = SinkCommandArgs { ctx: self.clone(), name_span, - positional: args.positional, - named: args.named, + args, input, }; @@ -87,11 +87,16 @@ impl Context { host: self.host.clone(), env: self.env.clone(), name_span, - positional: args.positional, - named: args.named, + args, input, }; command.run(command_args) } } + +impl CommandRegistry for Context { + fn get(&self, name: &str) -> Option { + self.commands.get(name).map(|c| c.config()) + } +} diff --git a/src/errors.rs b/src/errors.rs index ddf45f4cd9..fee2f0e798 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,6 +1,7 @@ -use crate::parser::lexer::{Span, SpannedToken}; #[allow(unused)] use crate::prelude::*; + +use crate::parser::Span; use derive_new::new; use language_reporting::{Diagnostic, Label, Severity}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -15,26 +16,40 @@ pub enum ShellError { impl ShellError { crate fn parse_error( - error: lalrpop_util::ParseError, + error: nom::Err<(nom_locate::LocatedSpan<&str>, nom::error::ErrorKind)>, ) -> ShellError { - use lalrpop_util::ParseError; use language_reporting::*; match error { - ParseError::UnrecognizedToken { - token: (start, SpannedToken { token, .. }, end), - expected, - } => { - let diagnostic = Diagnostic::new( - Severity::Error, - format!("Unexpected {:?}, expected {:?}", token, expected), - ) - .with_label(Label::new_primary(Span::from((start, end)))); + nom::Err::Incomplete(_) => unreachable!(), + nom::Err::Failure(span) | nom::Err::Error(span) => { + let diagnostic = + Diagnostic::new(Severity::Error, format!("{:?}", span)) + .with_label(Label::new_primary(Span::from(span.0))); ShellError::diagnostic(diagnostic) + // nom::Context::Code(span, kind) => { + // let diagnostic = + // Diagnostic::new(Severity::Error, format!("{}", kind.description())) + // .with_label(Label::new_primary(Span::from(span))); + + // ShellError::diagnostic(diagnostic) + // } } - ParseError::User { error } => error, - other => ShellError::string(format!("{:?}", other)), + // ParseError::UnrecognizedToken { + // token: (start, SpannedToken { token, .. }, end), + // expected, + // } => { + // let diagnostic = Diagnostic::new( + // Severity::Error, + // format!("Unexpected {:?}, expected {:?}", token, expected), + // ) + // .with_label(Label::new_primary(Span::from((start, end)))); + + // ShellError::diagnostic(diagnostic) + // } + // ParseError::User { error } => error, + // other => ShellError::string(format!("{:?}", other)), } } @@ -61,6 +76,10 @@ impl ShellError { ShellError::string(&format!("Unimplemented: {}", title.into())) } + crate fn unexpected(title: impl Into) -> ShellError { + ShellError::string(&format!("Unexpected: {}", title.into())) + } + crate fn copy_error(&self) -> ShellError { self.clone() } @@ -177,14 +196,14 @@ impl std::convert::From for ShellError { } } -impl std::convert::From> for ShellError { - fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError { - ShellError::String(StringError { - title: format!("{:?}", input), - error: Value::nothing(), - }) - } -} +// impl std::convert::From> for ShellError { +// fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError { +// ShellError::String(StringError { +// title: format!("{:?}", input), +// error: Value::nothing(), +// }) +// } +// } impl std::convert::From for ShellError { fn from(input: toml::ser::Error) -> ShellError { diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index c60b7fe08d..70cd486e71 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -1,6 +1,8 @@ -use crate::object::Primitive; -use crate::parser::ast; -use crate::parser::lexer::Spanned; +use crate::object::base::Block; +use crate::parser::{ + hir::{self, Expression, RawExpression}, + CommandRegistry, Span, Spanned, Text, +}; use crate::prelude::*; use derive_new::new; use indexmap::IndexMap; @@ -21,96 +23,75 @@ impl Scope { } } -crate fn evaluate_expr( - expr: &ast::Expression, +crate fn evaluate_baseline_expr( + expr: &Expression, + registry: &dyn CommandRegistry, scope: &Scope, + source: &str, ) -> Result, ShellError> { - use ast::*; - match &expr.expr { - RawExpression::Call(_) => Err(ShellError::unimplemented("Evaluating call expression")), - RawExpression::Leaf(l) => Ok(Spanned::from_item(evaluate_leaf(l), expr.span.clone())), - RawExpression::Parenthesized(p) => evaluate_expr(&p.expr, scope), - RawExpression::Flag(f) => Ok(Spanned::from_item( - Value::Primitive(Primitive::String(f.print())), - expr.span.clone(), - )), - RawExpression::Block(b) => evaluate_block(&b, scope), - RawExpression::Path(p) => evaluate_path(&p, scope), - RawExpression::Binary(b) => evaluate_binary(b, scope), - RawExpression::VariableReference(r) => { - evaluate_reference(r, scope).map(|x| Spanned::from_item(x, expr.span.clone())) - } - } -} + match &expr.item { + RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(*literal), source)), + RawExpression::Variable(var) => evaluate_reference(var, scope, source), + RawExpression::Binary(binary) => { + let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?; + let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?; -fn evaluate_leaf(leaf: &ast::Leaf) -> Value { - use ast::*; - - match leaf { - Leaf::String(s) => Value::string(s), - Leaf::Bare(path) => Value::string(path.to_string()), - Leaf::Boolean(b) => Value::boolean(*b), - Leaf::Int(i) => Value::int(*i), - Leaf::Unit(i, unit) => unit.compute(*i), - } -} - -fn evaluate_reference(r: &ast::Variable, scope: &Scope) -> Result { - use ast::Variable::*; - - match r { - It => Ok(scope.it.copy()), - Other(s) => Ok(scope - .vars - .get(s) - .map(|v| v.copy()) - .unwrap_or_else(|| Value::nothing())), - } -} - -fn evaluate_binary(binary: &ast::Binary, scope: &Scope) -> Result, ShellError> { - let left = evaluate_expr(&binary.left, scope)?; - let right = evaluate_expr(&binary.right, scope)?; - - match left.compare(&binary.operator, &right) { - Some(v) => Ok(Spanned::from_item( - Value::boolean(v), - binary.operator.span.clone(), - )), - None => Err(ShellError::TypeError(format!( - "Can't compare {} and {}", - left.type_name(), - right.type_name() - ))), - } -} - -fn evaluate_block(block: &ast::Block, _scope: &Scope) -> Result, ShellError> { - Ok(Spanned::from_item( - Value::block(block.expr.clone()), - block.expr.span.clone(), - )) -} - -fn evaluate_path(path: &ast::Path, scope: &Scope) -> Result, ShellError> { - let head = path.head(); - let mut value = evaluate_expr(head, scope)?; - let mut seen = vec![]; - - for name in path.tail() { - let next = value.get_data_by_key(&name.item); - seen.push(name.item.clone()); - - match next { - None => { - return Err(ShellError::MissingProperty { - expr: path.print(), - subpath: itertools::join(seen, "."), - }); + match left.compare(binary.op(), &*right) { + Some(result) => Ok(Spanned::from_item(Value::boolean(result), *expr.span())), + None => Err(ShellError::unimplemented(&format!( + "Comparison failure {:?}", + binary + ))), } - Some(v) => value = Spanned::from_item(v.copy(), name.span.clone()), } - } + RawExpression::Block(block) => Ok(Spanned::from_item( + Value::Block(Block::new(*block.clone(), Text::from(source))), // TODO: Pass Text around + block.span(), + )), + RawExpression::Path(path) => { + let value = evaluate_baseline_expr(path.head(), registry, scope, source)?; + let mut value = value.item(); - Ok(value) + for name in path.tail() { + let next = value.get_data_by_key(name); + + match next { + None => return Err(ShellError::unimplemented("Invalid property from path")), + Some(next) => value = next, + }; + } + + Ok(Spanned::from_item(value.clone(), expr.span())) + } + RawExpression::Boolean(_boolean) => unimplemented!(), + } +} + +fn evaluate_literal(literal: Spanned, source: &str) -> Spanned { + let result = match literal.item { + hir::Literal::Integer(int) => Value::int(int), + hir::Literal::Size(_int, _unit) => unimplemented!(), + hir::Literal::String(span) => Value::string(span.slice(source)), + hir::Literal::Bare => Value::string(literal.span().slice(source)), + }; + + literal.map(|_| result) +} + +fn evaluate_reference( + name: &hir::Variable, + scope: &Scope, + source: &str, +) -> Result, ShellError> { + match name { + hir::Variable::It(span) => Ok(Spanned::from_item(scope.it.copy(), span)), + hir::Variable::Other(span) => Ok(Spanned::from_item( + scope + .vars + .get(span.slice(source)) + .map(|v| v.copy()) + .unwrap_or_else(|| Value::nothing()), + span, + )), + } } diff --git a/src/evaluate/mod.rs b/src/evaluate/mod.rs index febfb1bd70..09451fbb49 100644 --- a/src/evaluate/mod.rs +++ b/src/evaluate/mod.rs @@ -1,3 +1,3 @@ crate mod evaluator; -crate use evaluator::{evaluate_expr, Scope}; +crate use evaluator::{evaluate_baseline_expr, Scope}; diff --git a/src/object/base.rs b/src/object/base.rs index 506de9e8d4..e5c9bdfd24 100644 --- a/src/object/base.rs +++ b/src/object/base.rs @@ -1,16 +1,17 @@ use crate::errors::ShellError; -use crate::evaluate::{evaluate_expr, Scope}; +use crate::evaluate::{evaluate_baseline_expr, Scope}; use crate::object::DataDescriptor; -use crate::parser::ast::{self, Operator}; -use crate::parser::lexer::Spanned; +use crate::parser::{hir, Operator, Spanned}; use crate::prelude::*; use ansi_term::Color; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; use derive_new::new; use ordered_float::OrderedFloat; +use std::fmt; use std::time::SystemTime; +use crate::parser::Text; use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq, new)] @@ -75,6 +76,20 @@ impl Primitive { .to_string() } + crate fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result { + use Primitive::*; + + match self { + Nothing => write!(f, "Nothing"), + Int(int) => write!(f, "{}", int), + Float(float) => write!(f, "{:?}", float), + Bytes(bytes) => write!(f, "{}", bytes), + String(string) => write!(f, "{:?}", string), + Boolean(boolean) => write!(f, "{}", boolean), + Date(date) => write!(f, "{}", date), + } + } + crate fn format(&self, field_name: Option<&DataDescriptor>) -> String { match self { Primitive::Nothing => format!("{}", Color::Black.bold().paint("-")), @@ -117,7 +132,8 @@ pub struct Operation { #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)] pub struct Block { - crate expression: ast::Expression, + crate expression: hir::Expression, + crate source: Text, } impl Serialize for Block { @@ -125,7 +141,7 @@ impl Serialize for Block { where S: Serializer, { - serializer.serialize_str(&self.expression.print()) + serializer.serialize_str(&self.expression.source(self.source.as_ref())) } } @@ -134,17 +150,19 @@ impl Deserialize<'de> for Block { where D: Deserializer<'de>, { - let mut builder = ast::ExpressionBuilder::new(); - let expr: ast::Expression = builder.string("Unserializable block"); - - Ok(Block::new(expr)) + unimplemented!("deserialize block") + // let s = "\"unimplemented deserialize block\""; + // Ok(Block::new( + // TokenTreeBuilder::spanned_string((1, s.len() - 1), (0, s.len())), + // Text::from(s), + // )) } } impl Block { pub fn invoke(&self, value: &Value) -> Result, ShellError> { let scope = Scope::new(value.copy()); - evaluate_expr(&self.expression, &scope) + evaluate_baseline_expr(&self.expression, &(), &scope, self.source.as_ref()) } } @@ -153,6 +171,7 @@ pub enum Value { Primitive(Primitive), Object(crate::object::Dictionary), List(Vec), + #[allow(unused)] Block(Block), Filesystem, @@ -160,6 +179,39 @@ pub enum Value { Error(Box), } +pub fn debug_list(values: &'a Vec) -> ValuesDebug<'a> { + ValuesDebug { values } +} + +pub struct ValuesDebug<'a> { + values: &'a Vec, +} + +impl fmt::Debug for ValuesDebug<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_list() + .entries(self.values.iter().map(|i| i.debug())) + .finish() + } +} + +pub struct ValueDebug<'a> { + value: &'a Value, +} + +impl fmt::Debug for ValueDebug<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.value { + Value::Primitive(p) => p.debug(f), + Value::Object(o) => o.debug(f), + Value::List(l) => debug_list(l).fmt(f), + Value::Block(_) => write!(f, "[[block]]"), + Value::Error(err) => write!(f, "[[error :: {} ]]", err), + Value::Filesystem => write!(f, "[[filesystem]]"), + } + } +} + impl Value { crate fn type_name(&self) -> String { match self { @@ -172,6 +224,10 @@ impl Value { } } + crate fn debug(&'a self) -> ValueDebug<'a> { + ValueDebug { value: self } + } + crate fn data_descriptors(&self) -> Vec { match self { Value::Primitive(_) => vec![DataDescriptor::value_of()], @@ -237,7 +293,7 @@ impl Value { crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String { match self { Value::Primitive(p) => p.format(desc), - Value::Block(b) => b.expression.print(), + Value::Block(b) => b.expression.source(b.source.as_ref()).to_string(), Value::Object(_) => format!("[object Object]"), Value::List(_) => format!("[list List]"), Value::Error(e) => format!("{}", e), @@ -245,7 +301,8 @@ impl Value { } } - crate fn compare(&self, operator: &ast::Operator, other: &Value) -> Option { + #[allow(unused)] + crate fn compare(&self, operator: &Operator, other: &Value) -> Option { match operator { _ => { let coerced = coerce_compare(self, other)?; @@ -342,8 +399,9 @@ impl Value { } } - crate fn block(e: ast::Expression) -> Value { - Value::Block(Block::new(e)) + #[allow(unused)] + crate fn block(e: hir::Expression, source: Text) -> Value { + Value::Block(Block::new(e, source)) } crate fn string(s: impl Into) -> Value { diff --git a/src/object/desc.rs b/src/object/desc.rs index 8693cba1cc..d69705b466 100644 --- a/src/object/desc.rs +++ b/src/object/desc.rs @@ -16,6 +16,13 @@ impl DescriptorName { } } + crate fn debug(&self) -> &str { + match self { + DescriptorName::String(s) => s, + DescriptorName::ValueOf => "[[value]]", + } + } + crate fn as_string(&self) -> Option<&str> { match self { DescriptorName::String(s) => Some(s), diff --git a/src/object/dict.rs b/src/object/dict.rs index c9ff168a0c..3cd13261b4 100644 --- a/src/object/dict.rs +++ b/src/object/dict.rs @@ -7,6 +7,7 @@ use indexmap::IndexMap; use serde::ser::{Serialize, SerializeMap, Serializer}; use serde_derive::Deserialize; use std::cmp::{Ordering, PartialOrd}; +use std::fmt; #[derive(Debug, Default, Eq, PartialEq, Deserialize, Clone, new)] pub struct Dictionary { @@ -113,4 +114,14 @@ impl Dictionary { None => None, } } + + crate fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_struct("Dictionary"); + + for (desc, value) in self.entries.iter() { + debug.field(desc.name.debug(), &value.debug()); + } + + debug.finish() + } } diff --git a/src/parser.rs b/src/parser.rs index b2eb9f8468..3af5481c76 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,279 +1,275 @@ -crate mod ast; -crate mod completer; -crate mod lexer; -crate mod parser; -crate mod registry; -crate mod span; +crate mod hir; crate mod parse2; - -crate use ast::Pipeline; -crate use registry::{Args, CommandConfig}; +crate mod parse_command; +crate mod registry; use crate::errors::ShellError; -use ast::Module; -use lexer::Lexer; -use log::trace; -use parser::{ModuleParser, ReplLineParser}; -pub fn parse(input: &str) -> Result { +crate use hir::baseline_parse_tokens::baseline_parse_tokens; +crate use parse2::call_node::CallNode; +crate use parse2::files::Files; +crate use parse2::flag::Flag; +crate use parse2::operator::Operator; +crate use parse2::parser::{nom_input, pipeline}; +crate use parse2::span::{Span, Spanned}; +crate use parse2::text::Text; +crate use parse2::token_tree::TokenNode; +crate use parse2::tokens::{RawToken, Token}; +crate use parse2::unit::Unit; +crate use parse_command::parse_command; +crate use registry::CommandRegistry; + +pub fn parse(input: &str) -> Result { let _ = pretty_env_logger::try_init(); - let parser = ReplLineParser::new(); - let tokens = Lexer::new(input, false); - - trace!( - "Tokens: {:?}", - tokens.clone().collect::, _>>() - ); - - match parser.parse(tokens) { - Ok(val) => Ok(val), + match pipeline(nom_input(input)) { + Ok((_rest, val)) => Ok(val), Err(err) => Err(ShellError::parse_error(err)), } } -#[allow(unused)] -pub fn parse_module(input: &str) -> Result { - let _ = pretty_env_logger::try_init(); +// #[allow(unused)] +// pub fn parse_module(input: &str) -> Result { +// let _ = pretty_env_logger::try_init(); - let parser = ModuleParser::new(); - let tokens = Lexer::new(input, false); +// let parser = ModuleParser::new(); +// let tokens = Lexer::new(input, false); - trace!( - "Tokens: {:?}", - tokens.clone().collect::, _>>() - ); +// trace!( +// "Tokens: {:?}", +// tokens.clone().collect::, _>>() +// ); - match parser.parse(tokens) { - Ok(val) => Ok(val), - Err(err) => Err(ShellError::parse_error(err)), - } -} +// match parser.parse(tokens) { +// Ok(val) => Ok(val), +// Err(err) => Err(ShellError::parse_error(err)), +// } +// } -#[cfg(test)] -mod tests { - use super::*; - use crate::parser::ast::Pipeline; - use pretty_assertions::assert_eq; +// #[cfg(test)] +// mod tests { +// use super::*; +// use crate::parser::ast::Pipeline; +// use pretty_assertions::assert_eq; - fn assert_parse(source: &str, expected: Pipeline) { - let parsed = match parse(source) { - Ok(p) => p, - Err(ShellError::Diagnostic(diag)) => { - use language_reporting::termcolor; +// fn assert_parse(source: &str, expected: Pipeline) { +// let parsed = match parse(source) { +// Ok(p) => p, +// Err(ShellError::Diagnostic(diag)) => { +// use language_reporting::termcolor; - let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); - let files = crate::parser::span::Files::new(source.to_string()); +// let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); +// let files = crate::parser::span::Files::new(source.to_string()); - language_reporting::emit( - &mut writer.lock(), - &files, - &diag.diagnostic, - &language_reporting::DefaultConfig, - ) - .unwrap(); +// language_reporting::emit( +// &mut writer.lock(), +// &files, +// &diag.diagnostic, +// &language_reporting::DefaultConfig, +// ) +// .unwrap(); - panic!("Test failed") - } - Err(err) => panic!("Something went wrong during parse: {:#?}", err), - }; +// panic!("Test failed") +// } +// Err(err) => panic!("Something went wrong during parse: {:#?}", err), +// }; - let printed = parsed.print(); +// let printed = parsed.print(); - assert_eq!(parsed, expected); - assert_eq!(printed, source); +// assert_eq!(parsed, expected); +// assert_eq!(printed, source); - let span = expected.span; +// let span = expected.span; - let expected_module = ModuleBuilder::spanned_items( - vec![Spanned::from_item(RawItem::Expression(expected), span)], - span.start, - span.end, - ); +// let expected_module = ModuleBuilder::spanned_items( +// vec![Spanned::from_item(RawItem::Expression(expected), span)], +// span.start, +// span.end, +// ); - assert_parse_module(source, expected_module); - } +// assert_parse_module(source, expected_module); +// } - fn assert_parse_module(source: &str, expected: Module) { - let parsed = match parse_module(source) { - Ok(p) => p, - Err(ShellError::Diagnostic(diag)) => { - use language_reporting::termcolor; +// fn assert_parse_module(source: &str, expected: Module) { +// let parsed = match parse_module(source) { +// Ok(p) => p, +// Err(ShellError::Diagnostic(diag)) => { +// use language_reporting::termcolor; - let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); - let files = crate::parser::span::Files::new(source.to_string()); +// let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); +// let files = crate::parser::span::Files::new(source.to_string()); - language_reporting::emit( - &mut writer.lock(), - &files, - &diag.diagnostic, - &language_reporting::DefaultConfig, - ) - .unwrap(); +// language_reporting::emit( +// &mut writer.lock(), +// &files, +// &diag.diagnostic, +// &language_reporting::DefaultConfig, +// ) +// .unwrap(); - panic!("Test failed") - } - Err(err) => panic!("Something went wrong during parse: {:#?}", err), - }; +// panic!("Test failed") +// } +// Err(err) => panic!("Something went wrong during parse: {:#?}", err), +// }; - let printed = parsed.print(); +// let printed = parsed.print(); - assert_eq!(parsed, expected); - assert_eq!(printed, source); - } +// assert_eq!(parsed, expected); +// assert_eq!(printed, source); +// } - macro_rules! commands { - ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{ - use $crate::parser::ast::{Expression, ExpressionBuilder}; - let mut builder = crate::parser::ast::ExpressionBuilder::new(); +// macro_rules! commands { +// ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{ +// use $crate::parser::ast::{Expression, ExpressionBuilder}; +// let mut builder = crate::parser::ast::ExpressionBuilder::new(); - builder.pipeline(vec![ - $( - (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression)) - ),* - ]) - }} - } +// builder.pipeline(vec![ +// $( +// (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression)) +// ),* +// ]) +// }} +// } - macro_rules! command { - ($name:ident) => { - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(( - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)), - vec![] - )) - }; +// macro_rules! command { +// ($name:ident) => { +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(( +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)), +// vec![] +// )) +// }; - ($name:ident $( $command:ident ( $body:expr ) )*) => {{ - use $crate::parser::ast::{Expression, ExpressionBuilder}; - &|b: &mut ExpressionBuilder| b.call(( - (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression), - vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ])) +// ($name:ident $( $command:ident ( $body:expr ) )*) => {{ +// use $crate::parser::ast::{Expression, ExpressionBuilder}; +// &|b: &mut ExpressionBuilder| b.call(( +// (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression), +// vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ])) - }}; +// }}; - ($name:ident $( $command:ident ( $body:expr ) )*) => { - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ]) - }; +// ($name:ident $( $command:ident ( $body:expr ) )*) => { +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ]) +// }; - ($name:tt $( $command:ident ( $body:expr ) )*) => { - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ])) - }; - } +// ($name:tt $( $command:ident ( $body:expr ) )*) => { +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ])) +// }; +// } - #[test] - fn parse_simple_command() { - assert_parse("ls", commands![(ls)]); - } +// #[test] +// fn parse_simple_command() { +// assert_parse("ls", commands![(ls)]); +// } - #[test] - fn parse_command_with_args() { - assert_parse( - r#"open Cargo.toml | select package.authors | split-row " ""#, - commands![ - (open bare("Cargo.toml")) - | (select bare("package.authors")) - | ("split-row" string(" ")) - ], - ); +// #[test] +// fn parse_command_with_args() { +// assert_parse( +// r#"open Cargo.toml | select package.authors | split-row " ""#, +// commands![ +// (open bare("Cargo.toml")) +// | (select bare("package.authors")) +// | ("split-row" string(" ")) +// ], +// ); - assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]); +// assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]); - assert_parse( - "open Cargo.toml | select package.version | echo $it", - commands![ - (open bare("Cargo.toml")) - | (select bare("package.version")) - | (echo var("it")) - ], - ); +// assert_parse( +// "open Cargo.toml | select package.version | echo $it", +// commands![ +// (open bare("Cargo.toml")) +// | (select bare("package.version")) +// | (echo var("it")) +// ], +// ); - assert_parse( - "open Cargo.toml --raw", - commands![(open bare("Cargo.toml") flag("raw"))], - ); +// assert_parse( +// "open Cargo.toml --raw", +// commands![(open bare("Cargo.toml") flag("raw"))], +// ); - assert_parse( - "open Cargo.toml -r", - commands![(open bare("Cargo.toml") shorthand("r"))], - ); +// assert_parse( +// "open Cargo.toml -r", +// commands![(open bare("Cargo.toml") shorthand("r"))], +// ); - assert_parse( - "open Cargo.toml | from-toml | to-toml", - commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")], - ); +// assert_parse( +// "open Cargo.toml | from-toml | to-toml", +// commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")], +// ); - assert_parse( - r#"config --get "ignore dups" | format-list"#, - commands![(config flag("get") string("ignore dups")) | ("format-list")], - ); +// assert_parse( +// r#"config --get "ignore dups" | format-list"#, +// commands![(config flag("get") string("ignore dups")) | ("format-list")], +// ); - assert_parse( - "open Cargo.toml | from-toml | select dependencies | column serde", - commands![ - (open bare("Cargo.toml")) - | ("from-toml") - | (select bare("dependencies")) - | (column bare("serde")) - ], - ); +// assert_parse( +// "open Cargo.toml | from-toml | select dependencies | column serde", +// commands![ +// (open bare("Cargo.toml")) +// | ("from-toml") +// | (select bare("dependencies")) +// | (column bare("serde")) +// ], +// ); - assert_parse( - "config --set tabs 2", - commands![(config flag("set") bare("tabs") int(2))], - ); +// assert_parse( +// "config --set tabs 2", +// commands![(config flag("set") bare("tabs") int(2))], +// ); - assert_parse( - r#"ls | skip 1 | first 2 | select "file name" | rm $it"#, - commands![ - (ls) - | (skip int(1)) - | (first int(2)) - | (select string("file name")) - | (rm var("it")) - ], - ); +// assert_parse( +// r#"ls | skip 1 | first 2 | select "file name" | rm $it"#, +// commands![ +// (ls) +// | (skip int(1)) +// | (first int(2)) +// | (select string("file name")) +// | (rm var("it")) +// ], +// ); - assert_parse( - r#"git branch --merged | split-row "`n" | where $it != "* master""#, - commands![ - // TODO: Handle escapes correctly. Should we do ` escape because of paths? - (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master")))) - ], - ); +// assert_parse( +// r#"git branch --merged | split-row "`n" | where $it != "* master""#, +// commands![ +// // TODO: Handle escapes correctly. Should we do ` escape because of paths? +// (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master")))) +// ], +// ); - assert_parse( - r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#, - commands![ - (open bare("input2.json")) - | ("from-json") - | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso")) - | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML")))) - ] - ); +// assert_parse( +// r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#, +// commands![ +// (open bare("input2.json")) +// | ("from-json") +// | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso")) +// | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML")))) +// ] +// ); - assert_parse( - r"cd ..\.cargo\", - commands![ - (cd bare(r"..\.cargo\")) - ], - ); +// assert_parse( +// r"cd ..\.cargo\", +// commands![ +// (cd bare(r"..\.cargo\")) +// ], +// ); - assert_parse( - "ls | where size < 1KB", - commands![ - (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB"))))) - ], - ); +// assert_parse( +// "ls | where size < 1KB", +// commands![ +// (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB"))))) +// ], +// ); - assert_parse( - "ls | where { $it.size > 100 }", - commands![ - (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100))))) - ], - ) - } +// assert_parse( +// "ls | where { $it.size > 100 }", +// commands![ +// (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100))))) +// ], +// ) +// } - use crate::parser::ast::{ModuleBuilder, RawItem}; - use crate::parser::lexer::Spanned; +// use crate::parser::ast::{ModuleBuilder, RawItem}; +// use crate::parser::lexer::Spanned; -} +// } diff --git a/src/parser/hir.rs b/src/parser/hir.rs new file mode 100644 index 0000000000..70b5f5c3be --- /dev/null +++ b/src/parser/hir.rs @@ -0,0 +1,93 @@ +crate mod baseline_parse; +crate mod baseline_parse_tokens; +crate mod binary; +crate mod named; +crate mod path; + +use crate::parser::{Span, Spanned, Unit}; +use derive_new::new; +use getset::Getters; + +crate use baseline_parse::baseline_parse_single_token; +crate use baseline_parse_tokens::{ + baseline_parse_next_expr, baseline_parse_tokens, ExpressionKindHint, +}; +crate use binary::Binary; +crate use named::NamedArguments; +crate use path::Path; + +#[derive(Debug, Clone, Eq, PartialEq, Getters, new)] +pub struct Call { + #[get = "crate"] + head: Box, + #[get = "crate"] + positional: Option>, + #[get = "crate"] + named: Option, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RawExpression { + Literal(Literal), + Variable(Variable), + Binary(Box), + Block(Box), + Path(Box), + + #[allow(unused)] + Boolean(bool), +} + +pub type Expression = Spanned; + +impl Expression { + fn int(i: impl Into, span: impl Into) -> Expression { + Spanned::from_item(RawExpression::Literal(Literal::Integer(i.into())), span) + } + + fn size(i: impl Into, unit: impl Into, span: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Literal(Literal::Size(i.into(), unit.into())), + span, + ) + } + + fn string(inner: impl Into, outer: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Literal(Literal::String(inner.into())), + outer.into(), + ) + } + + fn bare(span: impl Into) -> Expression { + Spanned::from_item(RawExpression::Literal(Literal::Bare), span.into()) + } + + fn variable(inner: impl Into, outer: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Variable(Variable::Other(inner.into())), + outer.into(), + ) + } + + fn it_variable(inner: impl Into, outer: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Variable(Variable::It(inner.into())), + outer.into(), + ) + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum Literal { + Integer(i64), + Size(i64, Unit), + String(Span), + Bare, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum Variable { + It(Span), + Other(Span), +} diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs new file mode 100644 index 0000000000..1150434fcb --- /dev/null +++ b/src/parser/hir/baseline_parse.rs @@ -0,0 +1,30 @@ +use crate::errors::ShellError; +use crate::parser::{hir, CommandRegistry, RawToken, Token, TokenNode}; + +// pub fn baseline_parse_token( +// token_node: TokenNode, +// _registry: &dyn CommandRegistry, +// ) -> Result { +// match token_node { +// TokenNode::Token(token) => Ok(baseline_parse_single_token(token)), +// TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")), +// TokenNode::Delimited(_delimited) => { +// Err(ShellError::unimplemented("baseline_parse Delimited")) +// } +// TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")), +// TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")), +// } +// } + +pub fn baseline_parse_single_token(token: &Token, source: &str) -> hir::Expression { + match *token.item() { + RawToken::Integer(int) => hir::Expression::int(int, token.span), + RawToken::Size(int, unit) => hir::Expression::size(int, unit, token.span), + RawToken::String(span) => hir::Expression::string(span, token.span), + RawToken::Variable(span) if span.slice(source) == "it" => { + hir::Expression::it_variable(span, token.span) + } + RawToken::Variable(span) => hir::Expression::variable(span, token.span), + RawToken::Bare => hir::Expression::bare(token.span), + } +} diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs new file mode 100644 index 0000000000..4cf8ae86bb --- /dev/null +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -0,0 +1,177 @@ +use crate::errors::ShellError; +use crate::parser::registry::CommandRegistry; +use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode}; + +pub fn baseline_parse_tokens( + token_nodes: &[TokenNode], + registry: &dyn CommandRegistry, + source: &str, +) -> Result, ShellError> { + let mut exprs: Vec = vec![]; + let mut rest = token_nodes; + + loop { + if rest.len() == 0 { + break; + } + + let (expr, remainder) = baseline_parse_next_expr(rest, registry, source, None)?; + exprs.push(expr); + rest = remainder; + } + + Ok(exprs) +} + +#[derive(Debug)] +pub enum ExpressionKindHint { + Literal, + Variable, + Binary, + Block, + Boolean, +} + +pub fn baseline_parse_next_expr( + token_nodes: &'nodes [TokenNode], + _registry: &dyn CommandRegistry, + source: &str, + coerce_hint: Option, +) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> { + println!( + "baseline_parse_next_expr {:?} - {:?}", + token_nodes, coerce_hint + ); + + let mut tokens = token_nodes.iter().peekable(); + + let first = next_token(&mut tokens); + + let first = match first { + None => return Err(ShellError::unimplemented("Expected token, found none")), + Some(token) => baseline_parse_semantic_token(token, source)?, + }; + + let possible_op = tokens.peek(); + + let op = match possible_op { + Some(TokenNode::Operator(op)) => op, + _ => return Ok((first, &token_nodes[1..])), + }; + + tokens.next(); + + let second = match tokens.next() { + None => { + return Err(ShellError::unimplemented( + "Expected op followed by another expr, found nothing", + )) + } + Some(token) => baseline_parse_semantic_token(token, source)?, + }; + + // We definitely have a binary expression here -- let's see if we should coerce it into a block + + match coerce_hint { + None => { + let span = (first.span.start, second.span.end); + let binary = hir::Binary::new(first, *op, second); + let binary = hir::RawExpression::Binary(Box::new(binary)); + let binary = Spanned::from_item(binary, span); + + Ok((binary, &token_nodes[3..])) + } + + Some(hint) => match hint { + ExpressionKindHint::Block => { + let span = (first.span.start, second.span.end); + + let string: Spanned = match first { + Spanned { + item: hir::RawExpression::Literal(hir::Literal::Bare), + span, + } => Spanned::from_item(span.slice(source).to_string(), span), + Spanned { + item: hir::RawExpression::Literal(hir::Literal::String(inner)), + span, + } => Spanned::from_item(inner.slice(source).to_string(), span), + _ => { + return Err(ShellError::unimplemented( + "The first part of a block must be a string", + )) + } + }; + + let path = hir::Path::new( + Spanned::from_item( + // TODO: Deal with synthetic nodes that have no representation at all in source + hir::RawExpression::Variable(hir::Variable::It(Span::from((0, 0)))), + (0, 0), + ), + vec![string], + ); + let path = hir::RawExpression::Path(Box::new(path)); + let path = Spanned::from_item(path, first.span); + + let binary = hir::Binary::new(path, *op, second); + let binary = hir::RawExpression::Binary(Box::new(binary)); + let binary = Spanned::from_item(binary, span); + + let block = hir::RawExpression::Block(Box::new(binary)); + let block = Spanned::from_item(block, span); + + Ok((block, &token_nodes[3..])) + } + + other => unimplemented!("coerce hint {:?}", other), + }, + } +} + +pub fn baseline_parse_semantic_token( + token: &TokenNode, + source: &str, +) -> Result { + match token { + TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), + TokenNode::Call(call) => unimplemented!(), + TokenNode::Delimited(delimited) => unimplemented!(), + TokenNode::Pipeline(pipeline) => unimplemented!(), + TokenNode::Operator(_op) => unreachable!(), + TokenNode::Flag(flag) => unimplemented!(), + TokenNode::Identifier(_span) => unreachable!(), + TokenNode::Whitespace(_span) => unreachable!(), + TokenNode::Error(error) => Err(*error.item.clone()), + TokenNode::Path(path) => unimplemented!(), + } +} + +fn next_token(nodes: &mut impl Iterator) -> Option<&'a TokenNode> { + loop { + match nodes.next() { + Some(TokenNode::Whitespace(_)) => continue, + other => return other, + } + } +} + +fn baseline_parse_token( + token_node: &TokenNode, + _registry: &dyn CommandRegistry, + source: &str, +) -> Result { + match token_node { + TokenNode::Token(token) => Ok(hir::baseline_parse_single_token(token, source)), + TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")), + TokenNode::Delimited(_delimited) => { + Err(ShellError::unimplemented("baseline_parse Delimited")) + } + TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")), + TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")), + TokenNode::Operator(_op) => Err(ShellError::unimplemented("baseline_parse Operator")), + TokenNode::Flag(_op) => Err(ShellError::unimplemented("baseline_parse Flag")), + TokenNode::Identifier(_op) => Err(ShellError::unimplemented("baseline_parse Identifier")), + TokenNode::Whitespace(_op) => Err(ShellError::unimplemented("baseline_parse Whitespace")), + TokenNode::Error(err) => Err(*err.item.clone()), + } +} diff --git a/src/parser/hir/binary.rs b/src/parser/hir/binary.rs new file mode 100644 index 0000000000..315964ed5f --- /dev/null +++ b/src/parser/hir/binary.rs @@ -0,0 +1,11 @@ +use crate::parser::{hir::Expression, Operator, Spanned}; +use derive_new::new; +use getset::Getters; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, new)] +#[get = "crate"] +pub struct Binary { + left: Expression, + op: Spanned, + right: Expression, +} diff --git a/src/parser/hir/named.rs b/src/parser/hir/named.rs new file mode 100644 index 0000000000..07c6db655e --- /dev/null +++ b/src/parser/hir/named.rs @@ -0,0 +1,44 @@ +use crate::parser::hir::Expression; +use crate::parser::{Flag, Span}; +use derive_new::new; +use indexmap::IndexMap; +use log::trace; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum NamedValue { + AbsentSwitch, + PresentSwitch(Span), + AbsentValue, + Value(Expression), +} + +#[derive(Debug, Clone, Eq, PartialEq, new)] +pub struct NamedArguments { + #[new(default)] + crate named: IndexMap, +} + +impl NamedArguments { + pub fn insert_switch(&mut self, name: impl Into, switch: Option) { + let name = name.into(); + trace!("Inserting switch -- {} = {:?}", name, switch); + + match switch { + None => self.named.insert(name.into(), NamedValue::AbsentSwitch), + Some(flag) => self + .named + .insert(name, NamedValue::PresentSwitch(*flag.name())), + }; + } + + pub fn insert_optional(&mut self, name: impl Into, expr: Option) { + match expr { + None => self.named.insert(name.into(), NamedValue::AbsentValue), + Some(expr) => self.named.insert(name.into(), NamedValue::Value(expr)), + }; + } + + pub fn insert_mandatory(&mut self, name: impl Into, expr: Expression) { + self.named.insert(name.into(), NamedValue::Value(expr)); + } +} diff --git a/src/parser/hir/path.rs b/src/parser/hir/path.rs new file mode 100644 index 0000000000..132f3112b8 --- /dev/null +++ b/src/parser/hir/path.rs @@ -0,0 +1,10 @@ +use crate::parser::{hir::Expression, Operator, Spanned}; +use derive_new::new; +use getset::Getters; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, new)] +#[get = "crate"] +pub struct Path { + head: Expression, + tail: Vec>, +} diff --git a/src/parser/parse2.rs b/src/parser/parse2.rs index 9d2e522c98..b841c4e5fa 100644 --- a/src/parser/parse2.rs +++ b/src/parser/parse2.rs @@ -1,9 +1,14 @@ +crate mod call_node; +crate mod files; crate mod flag; crate mod operator; crate mod parser; crate mod span; +crate mod text; crate mod token_tree; crate mod token_tree_builder; crate mod tokens; crate mod unit; crate mod util; + +crate use token_tree::{PipelineElement, TokenNode}; diff --git a/src/parser/parse2/call_node.rs b/src/parser/parse2/call_node.rs new file mode 100644 index 0000000000..eab5c8f328 --- /dev/null +++ b/src/parser/parse2/call_node.rs @@ -0,0 +1,26 @@ +use crate::parser::TokenNode; +use getset::Getters; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)] +pub struct CallNode { + #[get = "crate"] + head: Box, + #[get = "crate"] + children: Option>, +} + +impl CallNode { + pub fn new(head: Box, children: Vec) -> CallNode { + if children.len() == 0 { + CallNode { + head, + children: None, + } + } else { + CallNode { + head, + children: Some(children), + } + } + } +} diff --git a/src/parser/parse2/files.rs b/src/parser/parse2/files.rs new file mode 100644 index 0000000000..c606055f6c --- /dev/null +++ b/src/parser/parse2/files.rs @@ -0,0 +1,77 @@ +use crate::parser::parse2::span::Span; +use derive_new::new; +use language_reporting::{FileName, Location}; + +#[derive(new, Debug, Clone)] +pub struct Files { + snippet: String, +} + +impl language_reporting::ReportingFiles for Files { + type Span = Span; + type FileId = usize; + + fn byte_span( + &self, + _file: Self::FileId, + from_index: usize, + to_index: usize, + ) -> Option { + Some(Span::from((from_index, to_index))) + } + fn file_id(&self, _span: Self::Span) -> Self::FileId { + 0 + } + fn file_name(&self, _file: Self::FileId) -> FileName { + FileName::Verbatim(format!("")) + } + fn byte_index(&self, _file: Self::FileId, _line: usize, _column: usize) -> Option { + unimplemented!("byte_index") + } + fn location(&self, _file: Self::FileId, byte_index: usize) -> Option { + let source = &self.snippet; + let mut seen_lines = 0; + let mut seen_bytes = 0; + + for (pos, _) in source.match_indices('\n') { + if pos > byte_index { + return Some(language_reporting::Location::new( + seen_lines, + byte_index - seen_bytes, + )); + } else { + seen_lines += 1; + seen_bytes = pos; + } + } + + if seen_lines == 0 { + Some(language_reporting::Location::new(0, byte_index)) + } else { + None + } + } + fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option { + let source = &self.snippet; + let mut seen_lines = 0; + let mut seen_bytes = 0; + + for (pos, _) in source.match_indices('\n') { + if seen_lines == lineno { + return Some(Span::from((seen_bytes, pos))); + } else { + seen_lines += 1; + seen_bytes = pos + 1; + } + } + + if seen_lines == 0 { + Some(Span::from((0, self.snippet.len() - 1))) + } else { + None + } + } + fn source(&self, span: Self::Span) -> Option { + Some(self.snippet[span.start..span.end].to_string()) + } +} diff --git a/src/parser/parse2/flag.rs b/src/parser/parse2/flag.rs index 0d255e4515..3b587c3354 100644 --- a/src/parser/parse2/flag.rs +++ b/src/parser/parse2/flag.rs @@ -1,7 +1,17 @@ +use crate::parser::Span; +use derive_new::new; +use getset::Getters; use serde_derive::{Deserialize, Serialize}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] -pub enum Flag { +pub enum FlagKind { Shorthand, Longhand, } + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)] +#[get = "crate"] +pub struct Flag { + kind: FlagKind, + name: Span, +} diff --git a/src/parser/parse2/operator.rs b/src/parser/parse2/operator.rs index 89275afd74..c5ce66bfd8 100644 --- a/src/parser/parse2/operator.rs +++ b/src/parser/parse2/operator.rs @@ -12,6 +12,7 @@ pub enum Operator { } impl Operator { + #[allow(unused)] pub fn print(&self) -> String { self.as_str().to_string() } diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 5a2f6cc130..81d5293642 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -1,28 +1,43 @@ #![allow(unused)] use crate::parser::parse2::{ - flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, unit::*, + call_node::*, flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, + unit::*, }; use nom; +use nom::branch::*; +use nom::bytes::complete::*; +use nom::character::complete::*; +use nom::combinator::*; +use nom::multi::*; +use nom::sequence::*; + +use log::trace; use nom::dbg; -use nom::types::CompleteStr; use nom::*; +use nom::{AsBytes, FindSubstring, IResult, InputLength, InputTake, Slice}; use nom_locate::{position, LocatedSpan}; +use std::fmt::Debug; use std::str::FromStr; -type NomSpan<'a> = LocatedSpan>; +pub type NomSpan<'a> = LocatedSpan<&'a str>; + +pub fn nom_input(s: &'a str) -> NomSpan<'a> { + LocatedSpan::new(s) +} macro_rules! operator { ($name:tt : $token:tt ) => { - named!($name( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> t: tag!(stringify!($token)) - >> r: position!() - >> (TokenTreeBuilder::spanned_op(t.fragment.0, (l, r))) - // >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) - ) - ); + pub fn $name(input: NomSpan) -> IResult { + let start = input.offset; + let (input, tag) = tag(stringify!($token))(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_op(tag.fragment, (start, end)), + )) + } }; } @@ -33,195 +48,420 @@ operator! { lte: <= } operator! { eq: == } operator! { neq: != } -named!(pub raw_integer( NomSpan ) -> Spanned, - do_parse!( - l: position!() - >> neg: opt!(tag!("-")) - >> num: digit1 - >> r: position!() - >> (Spanned::from_nom(int(num.fragment.0, neg), l, r)) +fn trace_step<'a, T: Debug>( + input: NomSpan<'a>, + name: &str, + block: impl FnOnce(NomSpan<'a>) -> IResult, T>, +) -> IResult, T> { + trace!("+ before {} @ {:?}", name, input); + match block(input) { + Ok((input, result)) => { + trace!("after {} @ {:?} -> {:?}", name, input, result); + Ok((input, result)) + } + + Err(e) => { + trace!("- failed {} :: {:?}", name, e); + Err(e) + } + } +} + +pub fn raw_integer(input: NomSpan) -> IResult> { + let start = input.offset; + trace_step(input, "raw_integer", move |input| { + let (input, neg) = opt(tag("-"))(input)?; + let (input, num) = digit1(input)?; + let end = input.offset; + + Ok(( + input, + Spanned::from_item(int(num.fragment, neg), (start, end)), + )) + }) +} + +pub fn integer(input: NomSpan) -> IResult { + trace_step(input, "integer", move |input| { + let (input, int) = raw_integer(input)?; + + Ok((input, TokenTreeBuilder::spanned_int(*int, int.span))) + }) +} + +pub fn operator(input: NomSpan) -> IResult { + trace_step(input, "operator", |input| { + let (input, operator) = alt((gte, lte, neq, gt, lt, eq))(input)?; + + Ok((input, operator)) + }) +} + +pub fn dq_string(input: NomSpan) -> IResult { + trace_step(input, "dq_string", |input| { + let start = input.offset; + let (input, _) = char('"')(input)?; + let start1 = input.offset; + let (input, _) = many0(none_of("\""))(input)?; + let end1 = input.offset; + let (input, _) = char('"')(input)?; + let end = input.offset; + Ok(( + input, + TokenTreeBuilder::spanned_string((start1, end1), (start, end)), + )) + }) +} + +pub fn sq_string(input: NomSpan) -> IResult { + trace_step(input, "sq_string", move |input| { + let start = input.offset; + let (input, _) = char('\'')(input)?; + let start1 = input.offset; + let (input, _) = many0(none_of("\'"))(input)?; + let end1 = input.offset; + let (input, _) = char('\'')(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_string((start1, end1), (start, end)), + )) + }) +} + +pub fn string(input: NomSpan) -> IResult { + trace_step(input, "string", move |input| { + alt((sq_string, dq_string))(input) + }) +} + +pub fn bare(input: NomSpan) -> IResult { + trace_step(input, "bare", move |input| { + let start = input.offset; + let (input, _) = take_while1(is_start_bare_char)(input)?; + let (input, _) = take_while(is_bare_char)(input)?; + let end = input.offset; + + Ok((input, TokenTreeBuilder::spanned_bare((start, end)))) + }) +} + +pub fn var(input: NomSpan) -> IResult { + trace_step(input, "var", move |input| { + let start = input.offset; + let (input, _) = tag("$")(input)?; + let (input, bare) = identifier(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_var(bare.span(), (start, end)), + )) + }) +} + +// let start = input.offset; +// let (input, _) = take_while1(is_start_bare_char)(input)?; +// let (input, _) = take_while(is_bare_char)(input)?; +// let end = input.offset; + +// Ok((input, TokenTreeBuilder::spanned_bare((start, end)))) + +pub fn identifier(input: NomSpan) -> IResult { + trace_step(input, "identifier", move |input| { + let start = input.offset; + let (input, _) = take_while1(is_id_start)(input)?; + let (input, _) = take_while(is_id_continue)(input)?; + + let end = input.offset; + + Ok((input, TokenTreeBuilder::spanned_ident((start, end)))) + }) +} + +pub fn flag(input: NomSpan) -> IResult { + trace_step(input, "flag", move |input| { + let start = input.offset; + let (input, _) = tag("--")(input)?; + let (input, bare) = bare(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_flag(bare.span(), (start, end)), + )) + }) +} + +pub fn shorthand(input: NomSpan) -> IResult { + trace_step(input, "shorthand", move |input| { + let start = input.offset; + let (input, _) = tag("-")(input)?; + let (input, bare) = bare(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_shorthand(bare.span(), (start, end)), + )) + }) +} + +pub fn raw_unit(input: NomSpan) -> IResult> { + trace_step(input, "raw_unit", move |input| { + let start = input.offset; + let (input, unit) = alt(( + tag("B"), + tag("KB"), + tag("MB"), + tag("GB"), + tag("TB"), + tag("PB"), + ))(input)?; + let end = input.offset; + + Ok(( + input, + Spanned::from_item(Unit::from(unit.fragment), (start, end)), + )) + }) +} + +pub fn size(input: NomSpan) -> IResult { + trace_step(input, "size", move |input| { + let start = input.offset; + let (input, int) = raw_integer(input)?; + let (input, unit) = raw_unit(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_size((*int, *unit), (start, end)), + )) + }) +} + +pub fn leaf(input: NomSpan) -> IResult { + trace_step(input, "leaf", move |input| { + let (input, node) = + alt((size, integer, string, operator, flag, shorthand, var, bare))(input)?; + + Ok((input, node)) + }) +} + +pub fn token_list(input: NomSpan) -> IResult> { + trace_step(input, "token_list", move |input| { + let (input, first) = node(input)?; + let (input, list) = many0(pair(space1, node))(input)?; + + Ok((input, make_token_list(None, first, list, None))) + }) +} + +pub fn spaced_token_list(input: NomSpan) -> IResult> { + trace_step(input, "spaced_token_list", move |input| { + let (input, sp_left) = opt(space1)(input)?; + let (input, first) = node(input)?; + let (input, list) = many0(pair(space1, node))(input)?; + let (input, sp_right) = opt(space1)(input)?; + + Ok((input, make_token_list(sp_left, first, list, sp_right))) + }) +} + +fn make_token_list( + sp_left: Option, + first: TokenNode, + list: Vec<(NomSpan, TokenNode)>, + sp_right: Option, +) -> Vec { + let mut nodes = vec![]; + + if let Some(sp_left) = sp_left { + nodes.push(TokenNode::Whitespace(Span::from(sp_left))); + } + + nodes.push(first); + + for (ws, token) in list { + nodes.push(TokenNode::Whitespace(Span::from(ws))); + nodes.push(token); + } + + if let Some(sp_right) = sp_right { + nodes.push(TokenNode::Whitespace(Span::from(sp_right))); + } + + nodes +} + +pub fn whitespace(input: NomSpan) -> IResult { + trace_step(input, "whitespace", move |input| { + let left = input.offset; + let (input, ws1) = space1(input)?; + let right = input.offset; + + Ok((input, TokenTreeBuilder::spanned_ws((left, right)))) + }) +} + +pub fn delimited_paren(input: NomSpan) -> IResult { + trace_step(input, "delimited_paren", move |input| { + let left = input.offset; + let (input, _) = char('(')(input)?; + let (input, ws1) = opt(whitespace)(input)?; + let (input, inner_items) = opt(token_list)(input)?; + let (input, ws2) = opt(whitespace)(input)?; + let (input, _) = char(')')(input)?; + let right = input.offset; + + let mut items = vec![]; + + if let Some(space) = ws1 { + items.push(space); + } + + if let Some(inner_items) = inner_items { + items.extend(inner_items); + } + + if let Some(space) = ws2 { + items.push(space); + } + + Ok(( + input, + TokenTreeBuilder::spanned_parens(items, (left, right)), + )) + }) +} + +pub fn delimited_square(input: NomSpan) -> IResult { + trace_step(input, "delimited_paren", move |input| { + let left = input.offset; + let (input, _) = char('[')(input)?; + let (input, ws1) = opt(whitespace)(input)?; + let (input, inner_items) = opt(token_list)(input)?; + let (input, ws2) = opt(whitespace)(input)?; + let (input, _) = char(']')(input)?; + let right = input.offset; + + let mut items = vec![]; + + if let Some(space) = ws1 { + items.push(space); + } + + if let Some(inner_items) = inner_items { + items.extend(inner_items); + } + + if let Some(space) = ws2 { + items.push(space); + } + + Ok(( + input, + TokenTreeBuilder::spanned_square(items, (left, right)), + )) + }) +} + +pub fn delimited_brace(input: NomSpan) -> IResult { + trace_step(input, "delimited_brace", move |input| { + let left = input.offset; + let (input, _) = char('{')(input)?; + let (input, _) = opt(space1)(input)?; + let (input, items) = opt(token_list)(input)?; + let (input, _) = opt(space1)(input)?; + let (input, _) = char('}')(input)?; + let right = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_brace(items.unwrap_or_else(|| vec![]), (left, right)), + )) + }) +} + +pub fn raw_call(input: NomSpan) -> IResult> { + trace_step(input, "raw_call", move |input| { + let left = input.offset; + let (input, items) = token_list(input)?; + let right = input.offset; + + Ok((input, TokenTreeBuilder::spanned_call(items, (left, right)))) + }) +} + +pub fn path(input: NomSpan) -> IResult { + trace_step(input, "path", move |input| { + let left = input.offset; + let (input, head) = node1(input)?; + let (input, _) = tag(".")(input)?; + let (input, tail) = separated_list(tag("."), alt((identifier, string)))(input)?; + let right = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_path((head, tail), (left, right)), + )) + }) +} + +pub fn node1(input: NomSpan) -> IResult { + trace_step(input, "node1", alt((leaf, delimited_paren))) +} + +pub fn node(input: NomSpan) -> IResult { + trace_step( + input, + "node", + alt(( + path, + leaf, + delimited_paren, + delimited_brace, + delimited_square, + )), ) -); +} -named!(pub integer( NomSpan ) -> TokenNode, - do_parse!( - int: raw_integer - >> (TokenTreeBuilder::spanned_int(*int, int.span)) - ) -); +pub fn pipeline(input: NomSpan) -> IResult { + trace_step(input, "pipeline", |input| { + let start = input.offset; + let (input, head) = tuple((raw_call, opt(space1)))(input)?; + let (input, items) = trace_step( + input, + "many0", + many0(tuple((tag("|"), opt(space1), raw_call, opt(space1)))), + )?; + let end = input.offset; -named!(pub operator( NomSpan ) -> TokenNode, - alt!( - gte | lte | neq | gt | lt | eq - ) -); + Ok(( + input, + TokenTreeBuilder::spanned_pipeline(make_call_list(head, items), (start, end)), + )) + }) +} -named!(pub dq_string( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> char!('"') - >> l1: position!() - >> many0!(none_of!("\"")) - >> r1: position!() - >> char!('"') - >> r: position!() - >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) - ) -); +fn make_call_list( + head: (Spanned, Option), + tail: Vec<(NomSpan, Option, Spanned, Option)>, +) -> Vec { + let mut out = vec![]; + let el = PipelineElement::new(None, head.0, head.1.map(Span::from)); + out.push(el); -named!(pub sq_string( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> char!('\'') - >> l1: position!() - >> many0!(none_of!("'")) - >> r1: position!() - >> char!('\'') - >> r: position!() - >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) - ) -); + for (pipe, ws1, call, ws2) in tail { + let el = PipelineElement::new(ws1.map(Span::from), call, ws2.map(Span::from)); + out.push(el); + } -named!(pub string( NomSpan ) -> TokenNode, - alt!(sq_string | dq_string) -); - -named!(pub bare( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> take_while1!(is_start_bare_char) - >> take_while!(is_bare_char) - >> r: position!() - >> (TokenTreeBuilder::spanned_bare((l, r))) - ) -); - -named!(pub var( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> tag!("$") - >> bare: identifier - >> r: position!() - >> (TokenTreeBuilder::spanned_var(bare.span(), (l, r))) - ) -); - -named!(pub identifier( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> take_while1!(is_id_start) - >> take_while!(is_id_continue) - >> r: position!() - >> (TokenTreeBuilder::spanned_ident((l, r))) - ) -); - -named!(pub flag( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> tag!("--") - >> bare: bare - >> r: position!() - >> (TokenTreeBuilder::spanned_flag(bare.span(), (l, r))) - ) -); - -named!(pub shorthand( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> tag!("-") - >> bare: bare - >> r: position!() - >> (TokenTreeBuilder::spanned_shorthand(bare.span(), (l, r))) - ) -); - -named!(pub raw_unit( NomSpan ) -> Spanned, - do_parse!( - l: position!() - >> unit: alt!(tag!("B") | tag!("KB") | tag!("MB") | tag!("GB") | tag!("TB") | tag!("PB")) - >> r: position!() - >> (Spanned::from_nom(Unit::from(unit.fragment.0), l, r)) - ) -); - -named!(pub size( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> int: raw_integer - >> unit: raw_unit - >> r: position!() - >> (TokenTreeBuilder::spanned_size((*int, *unit), (l, r))) - ) -); - -named!(pub leaf( NomSpan ) -> TokenNode, - alt!(size | integer | string | operator | flag | shorthand | var | bare) -); - -named!(pub delimited_paren( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> items: delimited!( - char!('('), - delimited!(space0, separated_list!(space1, node), space0), - char!(')') - ) - >> r: position!() - >> (TokenTreeBuilder::spanned_parens(items, (l, r))) - ) -); - -named!(pub delimited_brace( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> items: delimited!( - char!('{'), - delimited!(space0, separated_list!(space1, node), space0), - char!('}') - ) - >> r: position!() - >> (TokenTreeBuilder::spanned_brace(items, (l, r))) - ) -); - -named!(pub raw_call( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> head: node - >> items: opt!(preceded!(space0, separated_nonempty_list!(space1, node))) - >> r: position!() - >> (TokenTreeBuilder::spanned_call((head, items), (l, r))) - ) -); - -named!(pub path( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> head: node1 - >> tag!(".") - >> tail: separated_list!(tag!("."), alt!(identifier | string)) - >> r: position!() - >> (TokenTreeBuilder::spanned_path((head, tail), (l, r))) - ) -); - -named!(pub node1( NomSpan ) -> TokenNode, - alt!(leaf | delimited_paren) -); - -named!(pub node( NomSpan ) -> TokenNode, - alt!(path | leaf | delimited_paren | delimited_brace) -); - -named!(pub pipeline( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> list: separated_list!(delimited!(space0, tag!("|"), space0), raw_call) - >> r: position!() - >> (TokenTreeBuilder::spanned_pipeline(list, (l, r))) - ) -); + out +} fn int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -276,30 +516,29 @@ mod tests { use super::*; use crate::parser::parse2::token_tree_builder::TokenTreeBuilder as b; use crate::parser::parse2::token_tree_builder::{CurriedToken, TokenTreeBuilder}; - use nom_trace::{print_trace, reset_trace}; use pretty_assertions::assert_eq; macro_rules! assert_leaf { (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt $parens:tt } ) => { $( assert_eq!( - apply($name, $input), + apply($name, stringify!($name), $input), token(RawToken::$kind $parens, $left, $right) ); )* assert_eq!( - apply(leaf, $input), + apply(leaf, "leaf", $input), token(RawToken::$kind $parens, $left, $right) ); assert_eq!( - apply(leaf, $input), + apply(leaf, "leaf", $input), token(RawToken::$kind $parens, $left, $right) ); assert_eq!( - apply(node, $input), + apply(node, "node", $input), token(RawToken::$kind $parens, $left, $right) ); }; @@ -307,7 +546,7 @@ mod tests { (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt } ) => { $( assert_eq!( - apply($name, $input), + apply($name, stringify!($name), $input), token(RawToken::$kind, $left, $right) ); )* @@ -342,35 +581,32 @@ mod tests { #[test] fn test_operator() { - assert_leaf! { - parsers [ operator ] - ">" -> 0..1 { Operator(Operator::GreaterThan) } - } + assert_eq!(apply(node, "node", ">"), build_token(b::op(">"))); - assert_leaf! { - parsers [ operator ] - ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } - } + // assert_leaf! { + // parsers [ operator ] + // ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } + // } - assert_leaf! { - parsers [ operator ] - "<" -> 0..1 { Operator(Operator::LessThan) } - } + // assert_leaf! { + // parsers [ operator ] + // "<" -> 0..1 { Operator(Operator::LessThan) } + // } - assert_leaf! { - parsers [ operator ] - "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } - } + // assert_leaf! { + // parsers [ operator ] + // "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } + // } - assert_leaf! { - parsers [ operator ] - "==" -> 0..2 { Operator(Operator::Equal) } - } + // assert_leaf! { + // parsers [ operator ] + // "==" -> 0..2 { Operator(Operator::Equal) } + // } - assert_leaf! { - parsers [ operator ] - "!=" -> 0..2 { Operator(Operator::NotEqual) } - } + // assert_leaf! { + // parsers [ operator ] + // "!=" -> 0..2 { Operator(Operator::NotEqual) } + // } } #[test] @@ -411,23 +647,23 @@ mod tests { #[test] fn test_flag() { - assert_leaf! { - parsers [ flag ] - "--hello" -> 0..7 { Flag(Flag::Longhand, span(2, 7)) } - } + // assert_leaf! { + // parsers [ flag ] + // "--hello" -> 0..7 { Flag(Spanned::from_item(FlagKind::Longhand, span(2, 7))) } + // } - assert_leaf! { - parsers [ flag ] - "--hello-world" -> 0..13 { Flag(Flag::Longhand, span(2, 13)) } - } + // assert_leaf! { + // parsers [ flag ] + // "--hello-world" -> 0..13 { Flag(Spanned::from_item(FlagKind::Longhand, span(2, 13))) } + // } } #[test] fn test_shorthand() { - assert_leaf! { - parsers [ shorthand ] - "-alt" -> 0..4 { Flag(Flag::Shorthand, span(1, 4)) } - } + // assert_leaf! { + // parsers [ shorthand ] + // "-alt" -> 0..4 { Flag(Spanned::from_item(FlagKind::Shorthand, span(1, 4))) } + // } } #[test] @@ -444,17 +680,20 @@ mod tests { } #[test] - fn test_delimited() { - assert_eq!(apply(node, "(abc)"), build(b::parens(vec![b::bare("abc")]))); - + fn test_delimited_paren() { assert_eq!( - apply(node, "( abc )"), - build(b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) + apply(node, "node", "(abc)"), + build_token(b::parens(vec![b::bare("abc")])) ); assert_eq!( - apply(node, "( abc def )"), - build(b::parens(vec![ + apply(node, "node", "( abc )"), + build_token(b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) + ); + + assert_eq!( + apply(node, "node", "( abc def )"), + build_token(b::parens(vec![ b::ws(" "), b::bare("abc"), b::sp(), @@ -464,8 +703,47 @@ mod tests { ); assert_eq!( - apply(node, "( abc def 123 456GB )"), - build(b::parens(vec![ + apply(node, "node", "( abc def 123 456GB )"), + build_token(b::parens(vec![ + b::ws(" "), + b::bare("abc"), + b::sp(), + b::bare("def"), + b::sp(), + b::int(123), + b::sp(), + b::size(456, "GB"), + b::sp() + ])) + ); + } + + #[test] + fn test_delimited_square() { + assert_eq!( + apply(node, "node", "[abc]"), + build_token(b::square(vec![b::bare("abc")])) + ); + + assert_eq!( + apply(node, "node", "[ abc ]"), + build_token(b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) + ); + + assert_eq!( + apply(node, "node", "[ abc def ]"), + build_token(b::square(vec![ + b::ws(" "), + b::bare("abc"), + b::sp(), + b::bare("def"), + b::sp() + ])) + ); + + assert_eq!( + apply(node, "node", "[ abc def 123 456GB ]"), + build_token(b::square(vec![ b::ws(" "), b::bare("abc"), b::sp(), @@ -481,30 +759,31 @@ mod tests { #[test] fn test_path() { + let _ = pretty_env_logger::try_init(); assert_eq!( - apply(node, "$it.print"), - build(b::path(b::var("it"), vec![b::ident("print")])) + apply(node, "node", "$it.print"), + build_token(b::path(b::var("it"), vec![b::ident("print")])) ); assert_eq!( - apply(node, "$head.part1.part2"), - build(b::path( + apply(node, "node", "$head.part1.part2"), + build_token(b::path( b::var("head"), vec![b::ident("part1"), b::ident("part2")] )) ); assert_eq!( - apply(node, "( hello ).world"), - build(b::path( + apply(node, "node", "( hello ).world"), + build_token(b::path( b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), vec![b::ident("world")] )) ); assert_eq!( - apply(node, "( hello ).\"world\""), - build(b::path( + apply(node, "node", "( hello ).\"world\""), + build_token(b::path( b::parens(vec![b::sp(), b::bare("hello"), b::sp()],), vec![b::string("world")] )) @@ -514,8 +793,12 @@ mod tests { #[test] fn test_nested_path() { assert_eq!( - apply(node, "( $it.is.\"great news\".right yep $yep ).\"world\""), - build(b::path( + apply( + node, + "node", + "( $it.is.\"great news\".right yep $yep ).\"world\"" + ), + build_token(b::path( b::parens(vec![ b::sp(), b::path( @@ -536,7 +819,7 @@ mod tests { #[test] fn test_smoke_single_command() { assert_eq!( - apply(raw_call, "git add ."), + apply(raw_call, "raw_call", "git add ."), build(b::call( b::bare("git"), vec![b::sp(), b::bare("add"), b::sp(), b::bare(".")] @@ -544,7 +827,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "open Cargo.toml"), + apply(raw_call, "raw_call", "open Cargo.toml"), build(b::call( b::bare("open"), vec![b::sp(), b::bare("Cargo.toml")] @@ -552,7 +835,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "select package.version"), + apply(raw_call, "raw_call", "select package.version"), build(b::call( b::bare("select"), vec![b::sp(), b::bare("package.version")] @@ -560,12 +843,12 @@ mod tests { ); assert_eq!( - apply(raw_call, "echo $it"), + apply(raw_call, "raw_call", "echo $it"), build(b::call(b::bare("echo"), vec![b::sp(), b::var("it")])) ); assert_eq!( - apply(raw_call, "open Cargo.toml --raw"), + apply(raw_call, "raw_call", "open Cargo.toml --raw"), build(b::call( b::bare("open"), vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::flag("raw")] @@ -573,7 +856,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "open Cargo.toml -r"), + apply(raw_call, "raw_call", "open Cargo.toml -r"), build(b::call( b::bare("open"), vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::shorthand("r")] @@ -581,7 +864,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "config --set tabs 2"), + apply(raw_call, "raw_call", "config --set tabs 2"), build(b::call( b::bare("config"), vec![ @@ -598,57 +881,81 @@ mod tests { #[test] fn test_smoke_pipeline() { + let _ = pretty_env_logger::try_init(); + assert_eq!( apply( pipeline, + "pipeline", r#"git branch --merged | split-row "`n" | where $it != "* master""# ), - build(b::pipeline(vec![ - b::call( - b::bare("git"), - vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + build_token(b::pipeline(vec![ + ( + None, + b::call( + b::bare("git"), + vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + ), + Some(" ") ), - b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), - b::call( - b::bare("where"), - vec![ - b::sp(), - b::var("it"), - b::sp(), - b::op("!="), - b::sp(), - b::string("* master") - ] + ( + Some(" "), + b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), + Some(" ") + ), + ( + Some(" "), + b::call( + b::bare("where"), + vec![ + b::sp(), + b::var("it"), + b::sp(), + b::op("!="), + b::sp(), + b::string("* master") + ] + ), + None ) ])) ); assert_eq!( - apply(pipeline, "ls | where { $it.size > 100 }"), - build(b::pipeline(vec![ - b::call(b::bare("ls"), vec![]), - b::call( - b::bare("where"), - vec![ - b::sp(), - b::braced(vec![ - b::path(b::var("it"), vec![b::ident("size")]), + apply(pipeline, "pipeline", "ls | where { $it.size > 100 }"), + build_token(b::pipeline(vec![ + (None, b::call(b::bare("ls"), vec![]), Some(" ")), + ( + Some(" "), + b::call( + b::bare("where"), + vec![ b::sp(), - b::op(">"), - b::sp(), - b::int(100) - ]) - ] + b::braced(vec![ + b::path(b::var("it"), vec![b::ident("size")]), + b::sp(), + b::op(">"), + b::sp(), + b::int(100) + ]) + ] + ), + None ) ])) ) } - fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { - match f(NomSpan::new(CompleteStr(string))) { + fn apply( + f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err<(NomSpan, nom::error::ErrorKind)>>, + desc: &str, + string: &str, + ) -> T { + match f(NomSpan::new(string)) { Ok(v) => v.1, Err(other) => { println!("{:?}", other); + println!("for {} @ {}", string, desc); panic!("No dice"); } } @@ -686,8 +993,13 @@ mod tests { TokenNode::Token(Spanned::from_item(token, (left, right))) } - fn build(block: CurriedToken) -> TokenNode { + fn build(block: CurriedNode) -> T { let mut builder = TokenTreeBuilder::new(); - block(&mut builder).expect("Expected to build into a token") + block(&mut builder) + } + + fn build_token(block: CurriedToken) -> TokenNode { + let mut builder = TokenTreeBuilder::new(); + block(&mut builder) } } diff --git a/src/parser/parse2/span.rs b/src/parser/parse2/span.rs index 2d3cbe7b5e..4214d29aaa 100644 --- a/src/parser/parse2/span.rs +++ b/src/parser/parse2/span.rs @@ -1,7 +1,8 @@ use derive_new::new; -use std::ops::Range; +use getset::Getters; -#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Getters)] +#[get = "crate"] pub struct Spanned { crate span: Span, crate item: T, @@ -16,20 +17,6 @@ impl std::ops::Deref for Spanned { } impl Spanned { - crate fn from_nom( - item: T, - start: nom_locate::LocatedSpan, - end: nom_locate::LocatedSpan, - ) -> Spanned { - let start = start.offset; - let end = end.offset; - - Spanned { - span: Span::from((start, end)), - item, - } - } - crate fn from_item(item: T, span: impl Into) -> Spanned { Spanned { span: span.into(), @@ -43,6 +30,19 @@ impl Spanned { let mapped = input(item); Spanned { span, item: mapped } } + + crate fn copy_span(&self, output: U) -> Spanned { + let Spanned { span, item } = self; + + Spanned { + span: *span, + item: output, + } + } + + pub fn source(&self, source: &'source str) -> &'source str { + self.span().slice(source) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] @@ -52,6 +52,21 @@ pub struct Span { // source: &'source str, } +impl From<&Span> for Span { + fn from(input: &Span) -> Span { + *input + } +} + +impl From> for Span { + fn from(input: nom_locate::LocatedSpan<&str>) -> Span { + Span { + start: input.offset, + end: input.offset + input.fragment.len(), + } + } +} + impl From<(nom_locate::LocatedSpan, nom_locate::LocatedSpan)> for Span { fn from(input: (nom_locate::LocatedSpan, nom_locate::LocatedSpan)) -> Span { Span { @@ -80,12 +95,8 @@ impl From<&std::ops::Range> for Span { } impl Span { - fn new(range: &Range) -> Span { - Span { - start: range.start, - end: range.end, - // source, - } + pub fn slice(&self, source: &'source str) -> &'source str { + &source[self.start..self.end] } } diff --git a/src/parser/parse2/text.rs b/src/parser/parse2/text.rs new file mode 100644 index 0000000000..7246392283 --- /dev/null +++ b/src/parser/parse2/text.rs @@ -0,0 +1,204 @@ +use std::cmp::Ordering; +use std::hash::Hash; +use std::hash::Hasher; +use std::ops::Range; +use std::sync::Arc; + +/// A "Text" is like a string except that it can be cheaply cloned. +/// You can also "extract" subtexts quite cheaply. You can also deref +/// an `&Text` into a `&str` for interoperability. +/// +/// Used to represent the value of an input file. +#[derive(Clone)] +pub struct Text { + text: Arc, + start: usize, + end: usize, +} + +impl Text { + /// Modifies this restrict to a subset of its current range. + pub fn select(&mut self, range: Range) { + let len = range.end - range.start; + let new_start = self.start + range.start; + let new_end = new_start + len; + assert!(new_end <= self.end); + + self.start = new_start; + self.end = new_end; + } + + /// Extract a new `Text` that is a subset of an old `Text` + /// -- `text.extract(1..3)` is similar to `&foo[1..3]` except that + /// it gives back an owned value instead of a borrowed value. + pub fn extract(&self, range: Range) -> Self { + let mut result = self.clone(); + result.select(range); + result + } +} + +impl From> for Text { + fn from(text: Arc) -> Self { + let end = text.len(); + Self { + text, + start: 0, + end, + } + } +} + +impl AsRef for Text { + fn as_ref(&self) -> &str { + &*self + } +} + +impl From for Text { + fn from(text: String) -> Self { + Text::from(Arc::new(text)) + } +} + +impl From<&String> for Text { + fn from(text: &String) -> Self { + Text::from(text.to_string()) + } +} + +impl From<&str> for Text { + fn from(text: &str) -> Self { + Text::from(text.to_string()) + } +} + +impl std::borrow::Borrow for Text { + fn borrow(&self) -> &str { + &*self + } +} + +impl std::ops::Deref for Text { + type Target = str; + + fn deref(&self) -> &str { + &self.text[self.start..self.end] + } +} + +impl std::fmt::Display for Text { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + ::fmt(self, fmt) + } +} + +impl std::fmt::Debug for Text { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + ::fmt(self, fmt) + } +} + +impl PartialEq for Text { + fn eq(&self, other: &Text) -> bool { + let this: &str = self; + let other: &str = other; + this == other + } +} + +impl Eq for Text {} + +impl PartialEq for Text { + fn eq(&self, other: &str) -> bool { + let this: &str = self; + this == other + } +} + +impl PartialEq for Text { + fn eq(&self, other: &String) -> bool { + let this: &str = self; + let other: &str = other; + this == other + } +} + +impl PartialEq for str { + fn eq(&self, other: &Text) -> bool { + other == self + } +} + +impl PartialEq for String { + fn eq(&self, other: &Text) -> bool { + other == self + } +} + +impl PartialEq<&T> for Text +where + Text: PartialEq, +{ + fn eq(&self, other: &&T) -> bool { + self == *other + } +} + +impl Hash for Text { + fn hash(&self, state: &mut H) { + ::hash(self, state) + } +} + +impl PartialOrd for Text { + fn partial_cmp(&self, other: &Text) -> Option { + let this: &str = self; + let other: &str = other; + this.partial_cmp(other) + } +} + +impl Ord for Text { + fn cmp(&self, other: &Text) -> Ordering { + let this: &str = self; + let other: &str = other; + this.cmp(other) + } +} + +impl PartialOrd for Text { + fn partial_cmp(&self, other: &str) -> Option { + let this: &str = self; + this.partial_cmp(other) + } +} + +impl PartialOrd for Text { + fn partial_cmp(&self, other: &String) -> Option { + let this: &str = self; + let other: &str = other; + this.partial_cmp(other) + } +} + +impl PartialOrd for str { + fn partial_cmp(&self, other: &Text) -> Option { + other.partial_cmp(self) + } +} + +impl PartialOrd for String { + fn partial_cmp(&self, other: &Text) -> Option { + other.partial_cmp(self) + } +} + +impl PartialOrd<&T> for Text +where + Text: PartialOrd, +{ + fn partial_cmp(&self, other: &&T) -> Option { + self.partial_cmp(*other) + } +} diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index b3cd722016..9718e9def5 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,14 +1,22 @@ -use crate::parser::parse2::{operator::*, span::*, tokens::*}; +use crate::errors::ShellError; +use crate::parser::parse2::{call_node::*, flag::*, operator::*, span::*, tokens::*}; use derive_new::new; use enum_utils::FromStr; +use getset::Getters; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { Token(Token), + #[allow(unused)] Call(Spanned), Delimited(Spanned), - Pipeline(Spanned>), - Binary(Spanned), + Pipeline(Spanned>), + Operator(Spanned), + Flag(Spanned), + Identifier(Span), + Whitespace(Span), + #[allow(unused)] + Error(Spanned>), Path(Spanned), } @@ -19,10 +27,71 @@ impl TokenNode { TokenNode::Call(s) => s.span, TokenNode::Delimited(s) => s.span, TokenNode::Pipeline(s) => s.span, - TokenNode::Binary(s) => s.span, + TokenNode::Operator(s) => s.span, + TokenNode::Flag(s) => s.span, + TokenNode::Identifier(s) => *s, + TokenNode::Whitespace(s) => *s, + TokenNode::Error(s) => s.span, TokenNode::Path(s) => s.span, } } + + pub fn as_external_arg(&self, source: &str) -> String { + self.span().slice(source).to_string() + } + + pub fn source(&self, source: &'source str) -> &'source str { + self.span().slice(source) + } + + pub fn is_ws(&self) -> bool { + match self { + TokenNode::Whitespace(_) => true, + _ => false, + } + } + + pub fn is_bare(&self) -> bool { + match self { + TokenNode::Token(Spanned { + item: RawToken::Bare, + .. + }) => true, + _ => false, + } + } + + crate fn as_string(&self, source: &str) -> Option> { + match self { + TokenNode::Token(Spanned { + item: RawToken::Bare, + span, + }) => Some(Spanned::from_item(span.slice(source).to_string(), span)), + TokenNode::Token(Spanned { + item: RawToken::String(inner), + span, + }) => Some(Spanned::from_item(inner.slice(source).to_string(), span)), + _ => None, + } + } + + crate fn as_flag(&self, value: &str, source: &str) -> Option> { + match self { + TokenNode::Flag( + flag @ Spanned { + item: Flag { .. }, .. + }, + ) if value == flag.name().slice(source) => Some(*flag), + _ => None, + } + } + + pub fn as_pipeline(&self) -> Result, ShellError> { + match self { + TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()), + _ => Err(ShellError::string("unimplemented")), + } + } } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] @@ -31,12 +100,6 @@ pub struct DelimitedNode { children: Vec, } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] -pub struct CallNode { - head: Box, - children: Vec, -} - #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)] pub enum Delimiter { Paren, @@ -50,9 +113,26 @@ pub struct PathNode { tail: Vec, } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] -pub struct BinaryNode { - left: Box, - op: Operator, - right: Box, +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] +pub struct PipelineElement { + pre_ws: Option, + #[get = "crate"] + call: Spanned, + post_ws: Option, +} + +impl PipelineElement { + crate fn span(&self) -> Span { + let start = match self.pre_ws { + None => self.call.span.start, + Some(span) => span.start, + }; + + let end = match self.post_ws { + None => self.call.span.end, + Some(span) => span.end, + }; + + Span::from((start, end)) + } } diff --git a/src/parser/parse2/token_tree_builder.rs b/src/parser/parse2/token_tree_builder.rs index a3fffaa8d7..08740d7b32 100644 --- a/src/parser/parse2/token_tree_builder.rs +++ b/src/parser/parse2/token_tree_builder.rs @@ -1,11 +1,15 @@ -use crate::parser::parse2::flag::Flag; +#[allow(unused)] +use crate::prelude::*; + +use crate::parser::parse2::flag::{Flag, FlagKind}; use crate::parser::parse2::operator::Operator; use crate::parser::parse2::span::{Span, Spanned}; use crate::parser::parse2::token_tree::{ - BinaryNode, CallNode, DelimitedNode, Delimiter, PathNode, TokenNode, + DelimitedNode, Delimiter, PathNode, PipelineElement, TokenNode, }; use crate::parser::parse2::tokens::{RawToken, Token}; use crate::parser::parse2::unit::Unit; +use crate::parser::CallNode; use derive_new::new; #[derive(new)] @@ -14,7 +18,10 @@ pub struct TokenTreeBuilder { pos: usize, } -pub type CurriedToken = Box Option + 'static>; +#[allow(unused)] +pub type CurriedNode = Box T + 'static>; +pub type CurriedToken = Box TokenNode + 'static>; +pub type CurriedCall = Box Spanned + 'static>; #[allow(unused)] impl TokenTreeBuilder { @@ -23,35 +30,57 @@ impl TokenTreeBuilder { block(&mut builder) } - pub fn pipeline(input: Vec) -> CurriedToken { + pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken { + let input: Vec<(Option, CurriedCall, Option)> = input + .into_iter() + .map(|(pre, call, post)| { + ( + pre.map(|s| s.to_string()), + call, + post.map(|s| s.to_string()), + ) + }) + .collect(); + Box::new(move |b| { let start = b.pos; - let mut out = vec![]; + let mut out: Vec = vec![]; let mut input = input.into_iter(); - let first = input + let (pre, call, post) = input .next() .expect("A pipeline must contain at least one element"); - out.push(first(b).expect("The first element of a pipeline must not be whitespace")); + let pre_span = pre.map(|pre| b.consume(&pre)); + let call = call(b); + let post_span = post.map(|post| b.consume(&post)); + out.push(PipelineElement::new( + pre_span.map(Span::from), + call, + post_span.map(Span::from), + )); - for item in input { - b.consume(" | "); + for (pre, call, post) in input { + b.consume("|"); + let pre_span = pre.map(|pre| b.consume(&pre)); + let call = call(b); + let post_span = post.map(|post| b.consume(&post)); - match item(b) { - None => {} - Some(v) => out.push(v), - } + out.push(PipelineElement::new( + pre_span.map(Span::from), + call, + post_span.map(Span::from), + )); } let end = b.pos; - Some(TokenTreeBuilder::spanned_pipeline(out, (start, end))) + TokenTreeBuilder::spanned_pipeline(out, (start, end)) }) } - pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { + pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { TokenNode::Pipeline(Spanned::from_item(input, span)) } @@ -63,15 +92,12 @@ impl TokenTreeBuilder { b.pos = end; - Some(TokenTreeBuilder::spanned_op(input, (start, end))) + TokenTreeBuilder::spanned_op(input, (start, end)) }) } pub fn spanned_op(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item( - RawToken::Operator(input.into()), - span.into(), - )) + TokenNode::Operator(Spanned::from_item(input.into(), span.into())) } pub fn string(input: impl Into) -> CurriedToken { @@ -83,10 +109,7 @@ impl TokenTreeBuilder { let (_, end) = b.consume("\""); b.pos = end; - Some(TokenTreeBuilder::spanned_string( - (inner_start, inner_end), - (start, end), - )) + TokenTreeBuilder::spanned_string((inner_start, inner_end), (start, end)) }) } @@ -104,7 +127,7 @@ impl TokenTreeBuilder { let (start, end) = b.consume(&input); b.pos = end; - Some(TokenTreeBuilder::spanned_bare((start, end))) + TokenTreeBuilder::spanned_bare((start, end)) }) } @@ -119,7 +142,7 @@ impl TokenTreeBuilder { let (start, end) = b.consume(&int.to_string()); b.pos = end; - Some(TokenTreeBuilder::spanned_int(int, (start, end))) + TokenTreeBuilder::spanned_int(int, (start, end)) }) } @@ -136,7 +159,7 @@ impl TokenTreeBuilder { let (_, end) = b.consume(unit.as_str()); b.pos = end; - Some(TokenTreeBuilder::spanned_size((int, unit), (start, end))) + TokenTreeBuilder::spanned_size((int, unit), (start, end)) }) } @@ -152,22 +175,19 @@ impl TokenTreeBuilder { pub fn path(head: CurriedToken, tail: Vec) -> CurriedToken { Box::new(move |b| { let start = b.pos; - let head = head(b).expect("The head of a path must not be whitespace"); + let head = head(b); let mut output = vec![]; for item in tail { b.consume("."); - match item(b) { - None => {} - Some(v) => output.push(v), - }; + output.push(item(b)); } let end = b.pos; - Some(TokenTreeBuilder::spanned_path((head, output), (start, end))) + TokenTreeBuilder::spanned_path((head, output), (start, end)) }) } @@ -185,10 +205,7 @@ impl TokenTreeBuilder { let (start, _) = b.consume("$"); let (inner_start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_var( - (inner_start, end), - (start, end), - )) + TokenTreeBuilder::spanned_var((inner_start, end), (start, end)) }) } @@ -206,16 +223,13 @@ impl TokenTreeBuilder { let (start, _) = b.consume("--"); let (inner_start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_flag( - (inner_start, end), - (start, end), - )) + TokenTreeBuilder::spanned_flag((inner_start, end), (start, end)) }) } pub fn spanned_flag(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item( - RawToken::Flag(Flag::Longhand, input.into()), + TokenNode::Flag(Spanned::from_item( + Flag::new(FlagKind::Longhand, input.into()), span.into(), )) } @@ -227,16 +241,13 @@ impl TokenTreeBuilder { let (start, _) = b.consume("-"); let (inner_start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_shorthand( - (inner_start, end), - (start, end), - )) + TokenTreeBuilder::spanned_shorthand((inner_start, end), (start, end)) }) } pub fn spanned_shorthand(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item( - RawToken::Flag(Flag::Shorthand, input.into()), + TokenNode::Flag(Spanned::from_item( + Flag::new(FlagKind::Shorthand, input.into()), span.into(), )) } @@ -246,45 +257,42 @@ impl TokenTreeBuilder { Box::new(move |b| { let (start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_ident((start, end))) + TokenTreeBuilder::spanned_ident((start, end)) }) } pub fn spanned_ident(span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into())) + TokenNode::Identifier(span.into()) } - pub fn call(head: CurriedToken, input: Vec) -> CurriedToken { + pub fn call(head: CurriedToken, input: Vec) -> CurriedCall { Box::new(move |b| { let start = b.pos; - let head_node = head(b).expect("The head of a command must not be whitespace"); + let head_node = head(b); - let mut tail_nodes = vec![]; + let mut nodes = vec![head_node]; for item in input { - match item(b) { - None => {} - Some(v) => tail_nodes.push(v), - }; + nodes.push(item(b)); } let end = b.pos; - Some(TokenTreeBuilder::spanned_call( - (head_node, Some(tail_nodes)), - (start, end), - )) + TokenTreeBuilder::spanned_call(nodes, (start, end)) }) } - pub fn spanned_call( - input: (TokenNode, Option>), - span: impl Into, - ) -> TokenNode { - TokenNode::Call(Spanned::from_item( - CallNode::new(Box::new(input.0), input.1.unwrap_or_else(|| vec![])), - span, - )) + pub fn spanned_call(input: Vec, span: impl Into) -> Spanned { + if input.len() == 0 { + panic!("BUG: spanned call (TODO)") + } + + let mut input = input.into_iter(); + + let head = input.next().unwrap(); + let tail = input.collect(); + + Spanned::from_item(CallNode::new(Box::new(head), tail), span) } pub fn parens(input: Vec) -> CurriedToken { @@ -292,15 +300,12 @@ impl TokenTreeBuilder { let (start, _) = b.consume("("); let mut output = vec![]; for item in input { - match item(b) { - None => {} - Some(v) => output.push(v), - }; + output.push(item(b)); } let (_, end) = b.consume(")"); - Some(TokenTreeBuilder::spanned_parens(output, (start, end))) + TokenTreeBuilder::spanned_parens(output, (start, end)) }) } @@ -311,20 +316,38 @@ impl TokenTreeBuilder { )) } + pub fn square(input: Vec) -> CurriedToken { + Box::new(move |b| { + let (start, _) = b.consume("["); + let mut output = vec![]; + for item in input { + output.push(item(b)); + } + + let (_, end) = b.consume("]"); + + TokenTreeBuilder::spanned_square(output, (start, end)) + }) + } + + pub fn spanned_square(input: impl Into>, span: impl Into) -> TokenNode { + TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Square, input.into()), + span, + )) + } + pub fn braced(input: Vec) -> CurriedToken { Box::new(move |b| { let (start, _) = b.consume("{ "); let mut output = vec![]; for item in input { - match item(b) { - None => {} - Some(v) => output.push(v), - }; + output.push(item(b)); } let (_, end) = b.consume(" }"); - Some(TokenTreeBuilder::spanned_brace(output, (start, end))) + TokenTreeBuilder::spanned_brace(output, (start, end)) }) } @@ -335,50 +358,10 @@ impl TokenTreeBuilder { )) } - pub fn binary( - left: CurriedToken, - op: impl Into, - right: CurriedToken, - ) -> CurriedToken { - let op = op.into(); - - Box::new(move |b| { - let start = b.pos; - - let left = left(b).expect("The left side of a binary operation must not be whitespace"); - - b.consume(" "); - - b.consume(op.as_str()); - - b.consume(" "); - - let right = - right(b).expect("The right side of a binary operation must not be whitespace"); - - let end = b.pos; - - Some(TokenTreeBuilder::spanned_binary( - (left, op, right), - (start, end), - )) - }) - } - - pub fn spanned_binary( - input: (impl Into, Operator, impl Into), - span: impl Into, - ) -> TokenNode { - TokenNode::Binary(Spanned::from_item( - BinaryNode::new(Box::new(input.0.into()), input.1, Box::new(input.2.into())), - span, - )) - } - pub fn sp() -> CurriedToken { Box::new(|b| { - b.consume(" "); - None + let (start, end) = b.consume(" "); + TokenNode::Whitespace(Span::from((start, end))) }) } @@ -386,11 +369,17 @@ impl TokenTreeBuilder { let input = input.into(); Box::new(move |b| { - b.consume(&input); - None + let (start, end) = b.consume(&input); + TokenTreeBuilder::spanned_ws((start, end)) }) } + pub fn spanned_ws(span: impl Into) -> TokenNode { + let span = span.into(); + + TokenNode::Whitespace(span.into()) + } + fn consume(&mut self, input: &str) -> (usize, usize) { let start = self.pos; self.pos += input.len(); diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs index d1977498ce..3e415c5261 100644 --- a/src/parser/parse2/tokens.rs +++ b/src/parser/parse2/tokens.rs @@ -7,12 +7,34 @@ use crate::parser::parse2::unit::*; pub enum RawToken { Integer(i64), Size(i64, Unit), - Operator(Operator), String(Span), Variable(Span), - Identifier, Bare, - Flag(Flag, Span), } pub type Token = Spanned; + +impl Token { + pub fn to_semantic_token(&self) -> Option { + let semantic_token = match self.item { + RawToken::Integer(int) => RawSemanticToken::Integer(int), + RawToken::Size(int, unit) => RawSemanticToken::Size(int, unit), + RawToken::String(span) => RawSemanticToken::String(span), + RawToken::Variable(span) => RawSemanticToken::Variable(span), + RawToken::Bare => RawSemanticToken::Bare, + }; + + Some(Spanned::from_item(semantic_token, self.span)) + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RawSemanticToken { + Integer(i64), + Size(i64, Unit), + String(Span), + Variable(Span), + Bare, +} + +pub type SemanticToken = Spanned; diff --git a/src/parser/parse2/unit.rs b/src/parser/parse2/unit.rs index 883f4c5071..7e51103cfc 100644 --- a/src/parser/parse2/unit.rs +++ b/src/parser/parse2/unit.rs @@ -12,10 +12,6 @@ pub enum Unit { } impl Unit { - pub fn print(&self) -> String { - self.as_str().to_string() - } - pub fn as_str(&self) -> &str { match *self { Unit::B => "B", diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs new file mode 100644 index 0000000000..d3064cbae0 --- /dev/null +++ b/src/parser/parse_command.rs @@ -0,0 +1,250 @@ +use crate::errors::ShellError; +use crate::parser::registry::{CommandConfig, CommandRegistry, NamedType}; +use crate::parser::{baseline_parse_tokens, CallNode, Spanned}; +use crate::parser::{ + hir::{self, NamedArguments}, + Flag, RawToken, TokenNode, +}; +use log::trace; + +pub fn parse_command( + config: &CommandConfig, + registry: &dyn CommandRegistry, + call: &Spanned, + source: &str, +) -> Result { + let Spanned { item: call, .. } = call; + + trace!("Processing {:?}", config); + + let head = parse_command_head(call.head())?; + + let children: Option> = call.children().as_ref().map(|nodes| { + nodes + .iter() + .cloned() + .filter(|node| match node { + TokenNode::Whitespace(_) => false, + _ => true, + }) + .collect() + }); + + match parse_command_tail(&config, registry, children, source)? { + None => Ok(hir::Call::new(Box::new(head), None, None)), + Some((positional, named)) => Ok(hir::Call::new(Box::new(head), positional, named)), + } +} + +fn parse_command_head(head: &TokenNode) -> Result { + match head { + TokenNode::Token( + spanned @ Spanned { + item: RawToken::Bare, + .. + }, + ) => Ok(spanned.map(|_| hir::RawExpression::Literal(hir::Literal::Bare))), + + TokenNode::Token(Spanned { + item: RawToken::String(inner_span), + span, + }) => Ok(Spanned::from_item( + hir::RawExpression::Literal(hir::Literal::String(*inner_span)), + *span, + )), + + other => Err(ShellError::unexpected(&format!( + "command head -> {:?}", + other + ))), + } +} + +fn parse_command_tail( + config: &CommandConfig, + registry: &dyn CommandRegistry, + tail: Option>, + source: &str, +) -> Result>, Option)>, ShellError> { + let mut tail = match tail { + None => return Ok(None), + Some(tail) => tail, + }; + + let mut named = NamedArguments::new(); + + for (name, kind) in config.named() { + trace!("looking for {} : {:?}", name, kind); + + match kind { + NamedType::Switch => { + let (rest, flag) = extract_switch(name, tail, source); + + tail = rest; + + named.insert_switch(name, flag); + } + NamedType::Mandatory(kind) => match extract_mandatory(name, tail, source) { + Err(err) => return Err(err), // produce a correct diagnostic + Ok((rest, pos, _flag)) => { + let (expr, rest) = hir::baseline_parse_next_expr( + &rest[pos..], + registry, + source, + kind.to_coerce_hint(), + )?; + tail = rest.to_vec(); + + named.insert_mandatory(name, expr); + } + }, + NamedType::Optional(kind) => match extract_optional(name, tail, source) { + Err(err) => return Err(err), // produce a correct diagnostic + Ok((rest, Some((pos, _flag)))) => { + let (expr, rest) = hir::baseline_parse_next_expr( + &rest[pos..], + registry, + source, + kind.to_coerce_hint(), + )?; + tail = rest.to_vec(); + + named.insert_optional(name, Some(expr)); + } + + Ok((rest, None)) => { + tail = rest; + + named.insert_optional(name, None); + } + }, + }; + } + + let mut positional = vec![]; + let mandatory = config.mandatory_positional(); + + for arg in mandatory { + if tail.len() == 0 { + return Err(ShellError::unimplemented("Missing mandatory argument")); + } + + let (result, rest) = + hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?; + + positional.push(result); + + tail = rest.to_vec(); + } + + let optional = config.optional_positional(); + + for arg in optional { + if tail.len() == 0 { + break; + } + + let (result, rest) = + hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?; + + positional.push(result); + + tail = rest.to_vec(); + } + + // TODO: Only do this if rest params are specified + let remainder = baseline_parse_tokens(&tail, registry, source)?; + positional.extend(remainder); + + trace!("Constructed positional={:?} named={:?}", positional, named); + + let positional = match positional { + positional if positional.len() == 0 => None, + positional => Some(positional), + }; + + let named = match named { + named if named.named.is_empty() => None, + named => Some(named), + }; + + trace!("Normalized positional={:?} named={:?}", positional, named); + + Ok(Some((positional, named))) +} + +fn extract_switch( + name: &str, + mut tokens: Vec, + source: &str, +) -> (Vec, Option) { + let pos = tokens + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) + .nth(0); + + match pos { + None => (tokens, None), + Some((pos, flag)) => { + tokens.remove(pos); + (tokens, Some(*flag)) + } + } +} + +fn extract_mandatory( + name: &str, + mut tokens: Vec, + source: &str, +) -> Result<(Vec, usize, Flag), ShellError> { + let pos = tokens + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) + .nth(0); + + match pos { + None => Err(ShellError::unimplemented( + "Better error: mandatory flags must be present", + )), + Some((pos, flag)) => { + if tokens.len() <= pos { + return Err(ShellError::unimplemented( + "Better errors: mandatory flags must be followed by values", + )); + } + + tokens.remove(pos); + + Ok((tokens, pos, *flag)) + } + } +} + +fn extract_optional( + name: &str, + mut tokens: Vec, + source: &str, +) -> Result<(Vec, Option<(usize, Flag)>), ShellError> { + let pos = tokens + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) + .nth(0); + + match pos { + None => Ok((tokens, None)), + Some((pos, flag)) => { + if tokens.len() <= pos { + return Err(ShellError::unimplemented( + "Better errors: optional flags must be followed by values", + )); + } + + tokens.remove(pos); + + Ok((tokens, Some((pos, *flag)))) + } + } +} diff --git a/src/parser/registry.rs b/src/parser/registry.rs index a1adc63b9c..d50c67aa43 100644 --- a/src/parser/registry.rs +++ b/src/parser/registry.rs @@ -1,7 +1,11 @@ -use crate::evaluate::{evaluate_expr, Scope}; -use crate::parser::lexer::Spanned; +use crate::evaluate::{evaluate_baseline_expr, Scope}; +use crate::parser::{hir, hir::ExpressionKindHint, parse_command, CallNode, Spanned}; use crate::prelude::*; +use derive_new::new; +use getset::Getters; use indexmap::IndexMap; +use log::trace; +use std::fmt; #[allow(unused)] #[derive(Debug)] @@ -14,13 +18,18 @@ pub enum NamedType { #[derive(Debug)] pub enum NamedValue { Single, - Tuple, #[allow(unused)] Block, +} - #[allow(unused)] - Array, +impl NamedValue { + crate fn to_coerce_hint(&self) -> Option { + match self { + NamedValue::Single => None, + NamedValue::Block => Some(ExpressionKindHint::Block), + } + } } #[allow(unused)] @@ -31,61 +40,16 @@ pub enum PositionalType { } impl PositionalType { - crate fn name(&self) -> String { + crate fn to_coerce_hint(&self) -> Option { match self { - PositionalType::Value(s) => s.clone(), - PositionalType::Block(s) => s.clone(), - } - } - - crate fn evaluate( - &self, - arg: ast::Expression, - scope: &Scope, - ) -> Result, ShellError> { - match self { - PositionalType::Value(_) => evaluate_expr(&arg, scope), - PositionalType::Block(_) => match arg { - ast::Expression { - expr: ast::RawExpression::Block(b), - .. - } => Ok(Spanned::from_item(Value::block(b.expr), arg.span.clone())), - ast::Expression { - expr: ast::RawExpression::Binary(binary), - .. - } => { - // TODO: Use original spans - let mut b = ast::ExpressionBuilder::new(); - if let Some(s) = binary.left.as_string() { - Ok(Spanned::from_item( - Value::block(b.binary(( - &|b| b.path((&|b| b.var("it"), vec![s.clone()])), - &|_| binary.operator.clone(), - &|_| binary.right.clone(), - ))), - arg.span.clone(), - )) - } else { - let mut b = ast::ExpressionBuilder::new(); - let expr = b.binary(( - &|_| binary.left.clone(), - &|_| binary.operator.clone(), - &|_| binary.right.clone(), - )); - - Ok(Spanned::from_item(Value::block(expr), arg.span.clone())) - } - } - other => { - let span = other.span.clone(); - Ok(Spanned::from_item(Value::block(other), span)) - } - }, + PositionalType::Value(_) => None, + PositionalType::Block(_) => Some(ExpressionKindHint::Block), } } } -#[derive(Debug)] +#[derive(Debug, Getters)] +#[get = "crate"] pub struct CommandConfig { crate name: String, crate mandatory_positional: Vec, @@ -94,90 +58,218 @@ pub struct CommandConfig { crate named: IndexMap, } -#[derive(Debug, Default)] +#[derive(Debug, Default, new)] pub struct Args { - pub positional: Vec>, - pub named: IndexMap, + pub positional: Option>>, + pub named: Option>>, +} + +#[derive(new)] +pub struct DebugPositional<'a> { + positional: &'a Option>>, +} + +impl fmt::Debug for DebugPositional<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.positional { + None => write!(f, "None"), + Some(positional) => f + .debug_list() + .entries(positional.iter().map(|p| p.item().debug())) + .finish(), + } + } +} + +#[derive(new)] +pub struct DebugNamed<'a> { + named: &'a Option>>, +} + +impl fmt::Debug for DebugNamed<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.named { + None => write!(f, "None"), + Some(named) => f + .debug_map() + .entries(named.iter().map(|(k, v)| (k, v.item().debug()))) + .finish(), + } + } +} + +pub struct DebugArgs<'a> { + args: &'a Args, +} + +impl fmt::Debug for DebugArgs<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut s = f.debug_struct("Args"); + + s.field("positional", &DebugPositional::new(&self.args.positional)); + s.field("named", &DebugNamed::new(&self.args.named)); + + s.finish() + } +} + +impl Args { + pub fn debug(&'a self) -> DebugArgs<'a> { + DebugArgs { args: self } + } + + pub fn nth(&self, pos: usize) -> Option<&Spanned> { + match &self.positional { + None => None, + Some(array) => array.iter().nth(pos), + } + } + + pub fn expect_nth(&self, pos: usize) -> Result<&Spanned, ShellError> { + match &self.positional { + None => Err(ShellError::unimplemented("Better error: expect_nth")), + Some(array) => match array.iter().nth(pos) { + None => Err(ShellError::unimplemented("Better error: expect_nth")), + Some(item) => Ok(item), + }, + } + } + + pub fn len(&self) -> usize { + match &self.positional { + None => 0, + Some(array) => array.len(), + } + } + + pub fn has(&self, name: &str) -> bool { + match &self.named { + None => false, + Some(named) => named.contains_key(name), + } + } + + pub fn get(&self, name: &str) -> Option<&Spanned> { + match &self.named { + None => None, + Some(named) => named.get(name), + } + } + + pub fn positional_iter(&'a self) -> PositionalIter<'a> { + match &self.positional { + None => PositionalIter::Empty, + Some(v) => { + let iter = v.iter(); + PositionalIter::Array(iter) + } + } + } +} + +pub enum PositionalIter<'a> { + Empty, + Array(std::slice::Iter<'a, Spanned>), +} + +impl Iterator for PositionalIter<'a> { + type Item = &'a Spanned; + + fn next(&mut self) -> Option { + match self { + PositionalIter::Empty => None, + PositionalIter::Array(iter) => iter.next(), + } + } } impl CommandConfig { crate fn evaluate_args( &self, - args: impl Iterator, + call: &Spanned, + registry: &dyn CommandRegistry, scope: &Scope, + source: &str, ) -> Result { - let mut positional: Vec> = vec![]; - let mut named: IndexMap = IndexMap::default(); + let args = parse_command(self, registry, call, source)?; - let mut args: Vec = args.cloned().collect(); + trace!("parsed args: {:?}", args); - for (key, ty) in self.named.iter() { - let index = args.iter().position(|a| a.is_flag(&key)); + evaluate_args(args, registry, scope, source) - match (index, ty) { - (Some(i), NamedType::Switch) => { - args.remove(i); - named.insert(key.clone(), Value::boolean(true)); - } + // let mut positional: Vec> = vec![]; + // let mut named: IndexMap = IndexMap::default(); - (None, NamedType::Switch) => {} + // let mut args: Vec = args.cloned().collect(); - (Some(i), NamedType::Optional(v)) => { - args.remove(i); - named.insert(key.clone(), extract_named(&mut args, i, v)?); - } + // for (key, ty) in self.named.iter() { + // let index = args.iter().position(|a| a.is_flag(&key, source)); - (None, NamedType::Optional(_)) => {} + // match (index, ty) { + // (Some(i), NamedType::Switch) => { + // args.remove(i); + // named.insert(key.clone(), Value::boolean(true)); + // } - (Some(i), NamedType::Mandatory(v)) => { - args.remove(i); - named.insert(key.clone(), extract_named(&mut args, i, v)?); - } + // (None, NamedType::Switch) => {} - (None, NamedType::Mandatory(_)) => { - return Err(ShellError::string(&format!( - "Expected mandatory argument {}, but it was missing", - key - ))) - } - } - } + // (Some(i), NamedType::Optional(v)) => { + // args.remove(i); + // named.insert(key.clone(), extract_named(&mut args, i, v)?); + // } - let mut args = args.into_iter(); + // (None, NamedType::Optional(_)) => {} - for param in &self.mandatory_positional { - let arg = args.next(); + // (Some(i), NamedType::Mandatory(v)) => { + // args.remove(i); + // named.insert(key.clone(), extract_named(&mut args, i, v)?); + // } - let value = match arg { - None => { - return Err(ShellError::string(format!( - "expected mandatory positional argument {}", - param.name() - ))) - } + // (None, NamedType::Mandatory(_)) => { + // return Err(ShellError::string(&format!( + // "Expected mandatory argument {}, but it was missing", + // key + // ))) + // } + // } + // } - Some(arg) => param.evaluate(arg.clone(), scope)?, - }; + // let mut args = args.into_iter(); - positional.push(value); - } + // for param in &self.mandatory_positional { + // let arg = args.next(); - if self.rest_positional { - let rest: Result>, _> = - args.map(|i| evaluate_expr(&i, &Scope::empty())).collect(); - positional.extend(rest?); - } else { - let rest: Vec = args.collect(); + // let value = match arg { + // None => { + // return Err(ShellError::string(format!( + // "expected mandatory positional argument {}", + // param.name() + // ))) + // } - if rest.len() > 0 { - return Err(ShellError::string(&format!( - "Too many arguments, extras: {:?}", - rest - ))); - } - } + // Some(arg) => param.evaluate(arg.clone(), scope, source)?, + // }; - Ok(Args { positional, named }) + // positional.push(value); + // } + + // if self.rest_positional { + // let rest: Result>, _> = args + // .map(|i| evaluate_baseline_expr(&i, &Scope::empty(), source)) + // .collect(); + // positional.extend(rest?); + // } else { + // let rest: Vec = args.collect(); + + // if rest.len() > 0 { + // return Err(ShellError::string(&format!( + // "Too many arguments, extras: {:?}", + // rest + // ))); + // } + // } + + // Ok(Args { positional, named }) } #[allow(unused)] @@ -186,50 +278,64 @@ impl CommandConfig { } } -fn extract_named( - v: &mut Vec, - position: usize, - ty: &NamedValue, -) -> Result { - match ty { - NamedValue::Single => { - let expr = v.remove(position); - expect_simple_expr(expr) - } +fn evaluate_args( + args: hir::Call, + registry: &dyn CommandRegistry, + scope: &Scope, + source: &str, +) -> Result { + let positional: Result>, _> = args + .positional() + .as_ref() + .map(|p| { + p.iter() + .map(|e| evaluate_baseline_expr(e, &(), scope, source)) + .collect() + }) + .transpose(); - NamedValue::Tuple => { - let expr = v.remove(position); - let next = v.remove(position); + let positional = positional?; - let list = vec![expect_simple_expr(expr)?, expect_simple_expr(next)?]; - Ok(Value::List(list)) - } + let named: Result>>, ShellError> = args + .named() + .as_ref() + .map(|n| { + let mut results = IndexMap::new(); - other => Err(ShellError::string(&format!( - "Unimplemented named argument {:?}", - other - ))), - } -} + for (name, value) in n.named.iter() { + match value { + hir::named::NamedValue::PresentSwitch(span) => { + results.insert( + name.clone(), + Spanned::from_item(Value::boolean(true), *span), + ); + } + hir::named::NamedValue::Value(expr) => { + results.insert( + name.clone(), + evaluate_baseline_expr(expr, registry, scope, source)?, + ); + } -fn expect_simple_expr(expr: ast::Expression) -> Result { - match &*expr { - ast::RawExpression::Leaf(l) => Ok(match l { - ast::Leaf::Bare(s) => Value::string(s.to_string()), - ast::Leaf::String(s) => Value::string(s), - ast::Leaf::Boolean(b) => Value::boolean(*b), - ast::Leaf::Int(i) => Value::int(*i), - ast::Leaf::Unit(i, unit) => unit.compute(*i), - }), + _ => {} + }; + } - // TODO: Diagnostic - other => Err(ShellError::string(&format!( - "Expected a value, found {}", - other.print() - ))), - } + Ok(results) + }) + .transpose(); + + let named = named?; + + Ok(Args::new(positional, named)) } pub trait CommandRegistry { - fn get(&self, name: &str) -> CommandConfig; + fn get(&self, name: &str) -> Option; +} + +impl CommandRegistry for () { + fn get(&self, _name: &str) -> Option { + None + } } diff --git a/src/prelude.rs b/src/prelude.rs index d943d90827..b9fec2540c 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -5,7 +5,6 @@ crate use crate::env::host::handle_unexpected; crate use crate::env::{Environment, Host}; crate use crate::errors::ShellError; crate use crate::object::Value; -crate use crate::parser::ast; crate use crate::stream::{single_output, InputStream, OutputStream}; crate use futures::{FutureExt, StreamExt}; crate use std::collections::VecDeque; diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 039a0b719e..85ad50de6a 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,9 +1,8 @@ use crate::shell::completer::NuCompleter; -use crate::parser::lexer::SpannedToken; +use crate::parser::nom_input; use crate::prelude::*; use ansi_term::Color; -use log::trace; use rustyline::completion::{self, Completer, FilenameCompleter}; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -47,7 +46,7 @@ impl Hinter for Helper { } impl Highlighter for Helper { - fn highlight_prompt<'b, 's: 'b, 'p:'b>(&'s self, prompt: &'p str, _: bool) -> Cow<'b, str> { + fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&'s self, prompt: &'p str, _: bool) -> Cow<'b, str> { Owned("\x1b[32m".to_owned() + &prompt[0..prompt.len() - 2] + "\x1b[m> ") } @@ -56,30 +55,39 @@ impl Highlighter for Helper { } fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { - let tokens = crate::parser::lexer::Lexer::new(line, true); - let tokens: Result, _> = tokens.collect(); + let tokens = crate::parser::pipeline(nom_input(line)); match tokens { Err(_) => Cow::Borrowed(line), - Ok(v) => { + Ok((_rest, v)) => { let mut out = String::new(); - let mut iter = v.iter(); + let tokens = match v.as_pipeline() { + Err(_) => return Cow::Borrowed(line), + Ok(v) => v, + }; - let mut state = State::Command; + let mut iter = tokens.into_iter(); + + match iter.next() { + None => return Cow::Owned(out), + Some(v) => out.push_str(v.span().slice(line)), + }; loop { match iter.next() { None => return Cow::Owned(out), - Some((start, token, end)) => { - let (style, new_state) = token_style(&token, state); + Some(token) => { + // let styled = token_style(&token, state); - trace!("token={:?}", token); - trace!("style={:?}", style); - trace!("new_state={:?}", new_state); + // trace!("token={:?}", token); + // trace!("style={:?}", style); + // trace!("new_state={:?}", new_state); - state = new_state; - let slice = &line[*start..*end]; - let styled = style.paint(slice); + // state = new_state; + // let slice = &line[*start..*end]; + // let styled = style.paint(slice); + out.push_str("|"); + let styled = Color::Black.bold().paint(token.span().slice(line)); out.push_str(&styled.to_string()); } } @@ -93,41 +101,4 @@ impl Highlighter for Helper { } } -#[derive(Debug)] -enum State { - Command, - Flag, - Var, - Bare, - None, -} - -fn token_style( - token: &crate::parser::lexer::SpannedToken, - state: State, -) -> (ansi_term::Style, State) { - use crate::parser::lexer::Token::*; - - match (state, &token.token) { - (State::Command, Bare) => (Color::Cyan.bold(), State::None), - (State::Command, Whitespace) => (Color::White.normal(), State::Command), - - (State::Flag, Bare) => (Color::Black.bold(), State::None), - - (State::Var, Variable) => (Color::Yellow.bold(), State::None), - - (State::Bare, PathDot) => (Color::Green.normal(), State::Bare), - (State::Bare, Member) => (Color::Green.normal(), State::Bare), - - (_, Dash) | (_, DashDash) => (Color::Black.bold(), State::Flag), - (_, Dollar) => (Color::Yellow.bold(), State::Var), - (_, Bare) => (Color::Green.normal(), State::Bare), - (_, Member) => (Color::Cyan.normal(), State::None), - (_, Num) => (Color::Purple.bold(), State::None), - (_, DQString) | (_, SQString) => (Color::Green.normal(), State::None), - (_, Pipe) => (Color::White.normal(), State::Command), - _ => (Color::White.normal(), State::None), - } -} - impl rustyline::Helper for Helper {} diff --git a/tests/sort_by.txt b/tests/sort_by.txt index b84c5e7ea2..aa933f8a2d 100644 --- a/tests/sort_by.txt +++ b/tests/sort_by.txt @@ -1,3 +1,3 @@ cd tests -open test.toml --raw | split-row "\n" | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it +open test.toml --raw | lines | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it exit diff --git a/tests/split.txt b/tests/split.txt index 5bc3d841f6..68713c675f 100644 --- a/tests/split.txt +++ b/tests/split.txt @@ -1,3 +1,3 @@ cd tests -open test.toml --raw | split-row "\n" | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it +open test.toml --raw | lines | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it exit From 9ae9beb94a8d656770154ad3e7698bd9d18e7405 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 22 Jun 2019 15:43:37 +1200 Subject: [PATCH 07/20] WIP --- Cargo.lock | 67 +- Cargo.toml | 7 +- src/cli.rs | 103 ++- src/commands.rs | 2 +- src/commands/cd.rs | 15 +- src/commands/classified.rs | 49 +- src/commands/command.rs | 50 +- src/commands/config.rs | 19 +- src/commands/enter.rs | 26 +- src/commands/first.rs | 6 +- src/commands/get.rs | 9 +- src/commands/lines.rs | 8 +- src/commands/ls.rs | 8 +- src/commands/open.rs | 48 +- src/commands/pick.rs | 4 +- src/commands/reject.rs | 4 +- src/commands/save.rs | 13 +- src/commands/size.rs | 4 +- src/commands/skip.rs | 6 +- src/commands/skip_while.rs | 8 +- src/commands/sort_by.rs | 2 +- src/commands/split_column.rs | 18 +- src/commands/split_row.rs | 10 +- src/commands/view.rs | 4 +- src/commands/where_.rs | 7 +- src/context.rs | 21 +- src/errors.rs | 63 +- src/evaluate/evaluator.rs | 159 ++--- src/evaluate/mod.rs | 2 +- src/object/base.rs | 86 ++- src/object/desc.rs | 7 + src/object/dict.rs | 11 + src/parser.rs | 446 ++++++------ src/parser/hir.rs | 93 +++ src/parser/hir/baseline_parse.rs | 30 + src/parser/hir/baseline_parse_tokens.rs | 177 +++++ src/parser/hir/binary.rs | 11 + src/parser/hir/named.rs | 44 ++ src/parser/hir/path.rs | 10 + src/parser/parse2.rs | 5 + src/parser/parse2/call_node.rs | 26 + src/parser/parse2/files.rs | 77 ++ src/parser/parse2/flag.rs | 12 +- src/parser/parse2/operator.rs | 1 + src/parser/parse2/parser.rs | 907 ++++++++++++++++-------- src/parser/parse2/span.rs | 55 +- src/parser/parse2/text.rs | 204 ++++++ src/parser/parse2/token_tree.rs | 110 ++- src/parser/parse2/token_tree_builder.rs | 237 +++---- src/parser/parse2/tokens.rs | 30 +- src/parser/parse2/unit.rs | 4 - src/parser/parse_command.rs | 250 +++++++ src/parser/registry.rs | 412 +++++++---- src/prelude.rs | 1 - src/shell/helper.rs | 80 +-- tests/sort_by.txt | 2 +- tests/split.txt | 2 +- 57 files changed, 2848 insertions(+), 1224 deletions(-) create mode 100644 src/parser/hir.rs create mode 100644 src/parser/hir/baseline_parse.rs create mode 100644 src/parser/hir/baseline_parse_tokens.rs create mode 100644 src/parser/hir/binary.rs create mode 100644 src/parser/hir/named.rs create mode 100644 src/parser/hir/path.rs create mode 100644 src/parser/parse2/call_node.rs create mode 100644 src/parser/parse2/files.rs create mode 100644 src/parser/parse2/text.rs create mode 100644 src/parser/parse_command.rs diff --git a/Cargo.lock b/Cargo.lock index 77339f6fd7..0db81d7831 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1242,6 +1242,18 @@ name = "lazycell" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lexical-core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "stackvector 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libc" version = "0.2.58" @@ -1522,21 +1534,23 @@ dependencies = [ ] [[package]] -name = "nom-trace" -version = "0.1.0" -source = "git+https://github.com/pythondude325/nom-trace.git#02f45edd3aa4ca9c82addb9c9a22e81044536e99" +name = "nom" +version = "5.0.0-beta2" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical-core 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "nom_locate" version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/wycats/nom_locate.git?branch=nom5#9383cc779b23a746ff68cc7094d9ed7baaa661b2" dependencies = [ "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 5.0.0-beta2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1575,9 +1589,8 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "logos 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "logos-derive 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)", - "nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 5.0.0-beta2 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 0.3.1 (git+https://github.com/wycats/nom_locate.git?branch=nom5)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "pancurses 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1589,7 +1602,7 @@ dependencies = [ "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.18 (registry+https://github.com/rust-lang/crates.io-index)", "roxmltree 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustyline 5.0.0 (git+https://github.com/kkawakam/rustyline.git)", + "rustyline 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde-hjson 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2319,7 +2332,7 @@ dependencies = [ [[package]] name = "rustyline" version = "5.0.0" -source = "git+https://github.com/kkawakam/rustyline.git#70e45caf47b699e43bec0d2fb722aae2cdad602f" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dirs 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2573,6 +2586,20 @@ name = "stable_deref_trait" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "stackvector" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "static_assertions" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "string" version = "0.2.0" @@ -3019,6 +3046,14 @@ name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "url" version = "1.7.2" @@ -3335,6 +3370,7 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +"checksum lexical-core 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d224b31f370c8dfc0dea1932d9e6bd451e65aef6f5f2318846664c04b42a796" "checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" "checksum libgit2-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "941a41e23f77323b8c9d2ee118aec9ee39dfc176078c18b4757d3bad049d9ff7" "checksum libssh2-sys 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "126a1f4078368b163bfdee65fbab072af08a1b374a5551b21e87ade27b1fbf9d" @@ -3365,8 +3401,8 @@ dependencies = [ "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -"checksum nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)" = "" -"checksum nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a47c112b3861d81f7fbf73892b9271af933af32bd5dee6889aa3c3fa9caed7e" +"checksum nom 5.0.0-beta2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfa215de9f747c2b3290ceee185976eb6ffc936087e19d810b57ba3ff85ae28c" +"checksum nom_locate 0.3.1 (git+https://github.com/wycats/nom_locate.git?branch=nom5)" = "" "checksum num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cf4825417e1e1406b3782a8ce92f4d53f26ec055e3622e1881ca8e9f5f9e08db" "checksum num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fcb0cf31fb3ff77e6d2a6ebd6800df7fdcd106f2ad89113c9130bcd07f93dffc" "checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" @@ -3442,7 +3478,7 @@ dependencies = [ "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" "checksum rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7540fc8b0c49f096ee9c961cda096467dce8084bec6bdca2fc83895fd9b28cb8" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum rustyline 5.0.0 (git+https://github.com/kkawakam/rustyline.git)" = "" +"checksum rustyline 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67e12e40e0240de07f0dab4f4dd01bdb15d74dc977026d4ba91666c41c679ade" "checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f" "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" "checksum same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8f20c4be53a8a1ff4c1f1b2bd14570d2f634628709752f0702ecdd2b3f9a5267" @@ -3474,6 +3510,8 @@ dependencies = [ "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ab606a9c5e214920bb66c458cd7be8ef094f813f20fe77a54cc7dbfff220d4b7" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" +"checksum stackvector 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1c4725650978235083241fab0fdc8e694c3de37821524e7534a1a9061d1068af" +"checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum string 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0bbfb8937e38e34c3444ff00afb28b0811d9554f15c5ad64d12b0308d1d1995" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" @@ -3521,6 +3559,7 @@ dependencies = [ "checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" "checksum utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9d50aa7650df78abf942826607c62468ce18d9019673d4a2ebe1865dbb96ffde" "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" diff --git a/Cargo.toml b/Cargo.toml index ba0b3b98f9..1d48b667c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" [dependencies] #rustyline = "4.1.0" -rustyline = { git ="https://github.com/kkawakam/rustyline.git" } +rustyline = "5.0.0" sysinfo = "0.8.5" chrono = { version = "0.4.6", features = ["serde"] } chrono-tz = "0.5.1" @@ -19,7 +19,7 @@ prettytable-rs = "0.8.0" itertools = "0.8.0" ansi_term = "0.11.0" conch-parser = "0.1.1" -nom = "4.2.3" +nom = "5.0.0-beta2" dunce = "1.0.0" indexmap = { version = "1.0.2", features = ["serde-1"] } chrono-humanize = "0.0.11" @@ -61,11 +61,10 @@ clipboard = "0.5" reqwest = "0.9" roxmltree = "0.6.0" pretty = "0.5.2" -nom_locate = "0.3.1" +nom_locate = { git = "https://github.com/wycats/nom_locate.git", branch = "nom5" } derive_more = "0.15.0" enum-utils = "0.1.0" unicode-xid = "0.1.0" -nom-trace = { version = "0.1.0", git = "https://github.com/pythondude325/nom-trace.git" } serde_ini = "0.2.0" subprocess = "0.1.18" sys-info = "0.5.7" diff --git a/src/cli.rs b/src/cli.rs index 1dbba22c02..82dc18f7c9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -11,14 +11,14 @@ use crate::commands::classified::{ use crate::context::Context; crate use crate::errors::ShellError; use crate::evaluate::Scope; +use crate::parser::parse2::span::Spanned; +use crate::parser::parse2::{PipelineElement, TokenNode}; +use crate::parser::registry; use crate::git::current_branch; use crate::object::Value; -use crate::parser::ast::{Expression, Leaf, RawExpression}; -use crate::parser::lexer::Spanned; -use crate::parser::{Args, Pipeline}; -use log::debug; +use log::{debug, trace}; use rustyline::error::ReadlineError; use rustyline::{self, ColorMode, Config, Editor}; @@ -62,19 +62,21 @@ pub async fn cli() -> Result<(), Box> { command("from-xml", from_xml::from_xml), command("from-yaml", from_yaml::from_yaml), command("get", get::get), - command("open", open::open), command("enter", enter::enter), command("exit", exit::exit), command("lines", lines::lines), command("pick", pick::pick), command("split-column", split_column::split_column), command("split-row", split_row::split_row), + command("lines", lines::lines), command("reject", reject::reject), command("trim", trim::trim), command("to-array", to_array::to_array), command("to-ini", to_ini::to_ini), command("to-json", to_json::to_json), command("to-toml", to_toml::to_toml), + command("sort-by", sort_by::sort_by), + Arc::new(Open), Arc::new(Where), Arc::new(Config), Arc::new(SkipWhile), @@ -159,7 +161,7 @@ pub async fn cli() -> Result<(), Box> { let host = context.host.lock().unwrap(); let writer = host.err_termcolor(); line.push_str(" "); - let files = crate::parser::span::Files::new(line); + let files = crate::parser::Files::new(line); language_reporting::emit( &mut writer.lock(), @@ -258,7 +260,7 @@ async fn process_line(readline: Result, ctx: &mut Context debug!("=== Parsed ==="); debug!("{:#?}", result); - let mut pipeline = classify_pipeline(&result, ctx)?; + let mut pipeline = classify_pipeline(&result, ctx, &line)?; match pipeline.commands.last() { Some(ClassifiedCommand::Sink(_)) => {} @@ -266,9 +268,9 @@ async fn process_line(readline: Result, ctx: &mut Context _ => pipeline.commands.push(ClassifiedCommand::Sink(SinkCommand { command: sink("autoview", autoview::autoview), name_span: None, - args: Args { - positional: vec![], - named: indexmap::IndexMap::new(), + args: registry::Args { + positional: None, + named: None, }, })), } @@ -371,14 +373,15 @@ async fn process_line(readline: Result, ctx: &mut Context } fn classify_pipeline( - pipeline: &Pipeline, + pipeline: &TokenNode, context: &Context, + source: &str, ) -> Result { - let commands: Result, _> = pipeline - .commands + let pipeline = pipeline.as_pipeline()?; + + let commands: Result, ShellError> = pipeline .iter() - .cloned() - .map(|item| classify_command(&item, context)) + .map(|item| classify_command(&item, context, source)) .collect(); Ok(ClassifiedPipeline { @@ -387,85 +390,69 @@ fn classify_pipeline( } fn classify_command( - command: &Expression, + command: &PipelineElement, context: &Context, + source: &str, ) -> Result { - // let command_name = &command.name[..]; - // let args = &command.args; + let call = command.call(); - if let Expression { - expr: RawExpression::Call(call), - .. - } = command - { - match (&call.name, &call.args) { - ( - Expression { - expr: RawExpression::Leaf(Leaf::Bare(name)), - span, - }, - args, - ) => match context.has_command(&name.to_string()) { + match call { + call if call.head().is_bare() => { + let head = call.head(); + let name = head.source(source); + + match context.has_command(name) { true => { - let command = context.get_command(&name.to_string()); + let command = context.get_command(name); let config = command.config(); let scope = Scope::empty(); - let args = match args { - Some(args) => config.evaluate_args(args.iter(), &scope)?, - None => Args::default(), - }; + trace!("classifying {:?}", config); + + let args = config.evaluate_args(call, context, &scope, source)?; Ok(ClassifiedCommand::Internal(InternalCommand { command, - name_span: Some(span.clone()), + name_span: Some(head.span().clone()), args, })) } - false => match context.has_sink(&name.to_string()) { + false => match context.has_sink(name) { true => { - let command = context.get_sink(&name.to_string()); + let command = context.get_sink(name); let config = command.config(); let scope = Scope::empty(); - let args = match args { - Some(args) => config.evaluate_args(args.iter(), &scope)?, - None => Args::default(), - }; + let args = config.evaluate_args(call, context, &scope, source)?; Ok(ClassifiedCommand::Sink(SinkCommand { command, - name_span: Some(span.clone()), + name_span: Some(head.span().clone()), args, })) } false => { - let arg_list_strings: Vec> = match args { + let arg_list_strings: Vec> = match call.children() { + //Some(args) => args.iter().map(|i| i.as_external_arg(source)).collect(), Some(args) => args .iter() - .map(|i| Spanned::from_item(i.as_external_arg(), i.span)) + .map(|i| Spanned::from_item(i.as_external_arg(source), i.span())) .collect(), None => vec![], }; Ok(ClassifiedCommand::External(ExternalCommand { name: name.to_string(), - name_span: Some(span.clone()), + name_span: Some(head.span().clone()), args: arg_list_strings, })) } }, - }, - - (_, None) => Err(ShellError::string( - "Unimplemented command that is just an expression (1)", - )), - (_, Some(_)) => Err(ShellError::string("Unimplemented dynamic command")), + } } - } else { - Err(ShellError::string(&format!( - "Unimplemented command that is just an expression (2) -- {:?}", - command - ))) + + _ => Err(ShellError::unimplemented( + "classify_command on command whose head is not bare", + )), } } diff --git a/src/commands.rs b/src/commands.rs index 2923d449ff..3454a0c01d 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -41,6 +41,6 @@ crate mod where_; crate use command::command; crate use config::Config; - +crate use open::Open; crate use where_::Where; crate use skip_while::SkipWhile; diff --git a/src/commands/cd.rs b/src/commands/cd.rs index 380c7d5832..bf6d581e9b 100644 --- a/src/commands/cd.rs +++ b/src/commands/cd.rs @@ -10,7 +10,8 @@ pub fn cd(args: CommandArgs) -> Result { match latest.obj { Value::Filesystem => { let cwd = latest.path().to_path_buf(); - let path = match args.positional.first() { + + let path = match args.nth(0) { None => match dirs::home_dir() { Some(o) => o, _ => { @@ -26,10 +27,10 @@ pub fn cd(args: CommandArgs) -> Result { match dunce::canonicalize(cwd.join(&target).as_path()) { Ok(p) => p, Err(_) => { - return Err(ShellError::maybe_labeled_error( + return Err(ShellError::labeled_error( "Can not change to directory", "directory not found", - Some(args.positional[0].span.clone()), + v.span.clone(), )); } } @@ -40,11 +41,11 @@ pub fn cd(args: CommandArgs) -> Result { match env::set_current_dir(&path) { Ok(_) => {} Err(_) => { - if args.positional.len() > 0 { - return Err(ShellError::maybe_labeled_error( + if args.len() > 0 { + return Err(ShellError::labeled_error( "Can not change to directory", "directory not found", - Some(args.positional[0].span.clone()), + args.nth(0).unwrap().span.clone(), )); } else { return Err(ShellError::string("Can not change to directory")); @@ -56,7 +57,7 @@ pub fn cd(args: CommandArgs) -> Result { } _ => { let mut stream = VecDeque::new(); - match args.positional.first() { + match args.nth(0) { None => { stream.push_back(ReturnValue::change_cwd(PathBuf::from("/"))); } diff --git a/src/commands/classified.rs b/src/commands/classified.rs index bb476187ee..65c5ed8753 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -1,15 +1,16 @@ use crate::commands::command::Sink; -use crate::parser::ast::Expression; -use crate::parser::lexer::{Span, Spanned}; -use crate::parser::registry::Args; +use crate::parser::parse2::span::Spanned; +use crate::parser::{registry::Args, Span, TokenNode}; use crate::prelude::*; use bytes::{BufMut, BytesMut}; use futures::stream::StreamExt; use futures_codec::{Decoder, Encoder, Framed}; +use log::{log_enabled, trace}; use std::io::{Error, ErrorKind}; use std::path::PathBuf; use std::sync::Arc; use subprocess::Exec; + /// A simple `Codec` implementation that splits up data into lines. pub struct LinesCodec {} @@ -80,7 +81,7 @@ crate struct ClassifiedPipeline { crate enum ClassifiedCommand { #[allow(unused)] - Expr(Expression), + Expr(TokenNode), Internal(InternalCommand), Sink(SinkCommand), External(ExternalCommand), @@ -110,18 +111,31 @@ impl InternalCommand { context: &mut Context, input: ClassifiedInputStream, ) -> Result { - let mut result = context.run_command( - self.command, - self.name_span.clone(), - self.args, - input.objects, - )?; + let objects = if log_enabled!(log::Level::Trace) { + trace!("->"); + trace!("{}", self.command.name()); + trace!("{:?}", self.args.debug()); + let objects: Vec<_> = input.objects.collect().await; + trace!( + "input = {:#?}", + objects.iter().map(|o| o.debug()).collect::>(), + ); + VecDeque::from(objects).boxed() + } else { + input.objects + }; + + let mut result = + context.run_command(self.command, self.name_span.clone(), self.args, objects)?; + let env = context.env.clone(); + let mut stream = VecDeque::new(); while let Some(item) = result.next().await { match item { ReturnValue::Value(Value::Error(err)) => { return Err(*err); } + ReturnValue::Action(action) => match action { CommandAction::ChangePath(path) => { context.env.lock().unwrap().back_mut().map(|x| { @@ -177,10 +191,12 @@ impl ExternalCommand { ) -> Result { let inputs: Vec = input.objects.collect().await; + trace!("{:?} -> {}", inputs, self.name); + let mut arg_string = format!("{}", self.name); for arg in &self.args { - arg_string.push_str(" "); - arg_string.push_str(&arg.item); + //arg_string.push_str(" "); + arg_string.push_str(&arg); } let mut process; @@ -191,6 +207,7 @@ impl ExternalCommand { if arg_string.contains("$it") { let mut first = true; + for i in &inputs { if i.as_string().is_err() { let mut span = None; @@ -217,6 +234,10 @@ impl ExternalCommand { } for arg in &self.args { + if arg.chars().all(|c| c.is_whitespace()) { + continue; + } + process = process.arg(&arg.replace("$it", &i.as_string().unwrap())); } } @@ -254,6 +275,10 @@ impl ExternalCommand { } for arg in &self.args { + if arg.chars().all(|c| c.is_whitespace()) { + continue; + } + new_arg_string.push_str(" "); new_arg_string.push_str(&arg.replace("$it", &i.as_string().unwrap())); } diff --git a/src/commands/command.rs b/src/commands/command.rs index ff2046206e..d25cf52b67 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -1,25 +1,53 @@ use crate::errors::ShellError; use crate::object::Value; -use crate::parser::lexer::Span; -use crate::parser::lexer::Spanned; -use crate::parser::CommandConfig; +use crate::parser::{ + registry::{self, Args}, + Span, Spanned, +}; use crate::prelude::*; +use getset::Getters; use std::path::PathBuf; +#[derive(Getters)] +#[get = "crate"] pub struct CommandArgs { pub host: Arc>, pub env: Arc>>, pub name_span: Option, - pub positional: Vec>, - pub named: indexmap::IndexMap, + pub args: Args, pub input: InputStream, } +impl CommandArgs { + pub fn nth(&self, pos: usize) -> Option<&Spanned> { + self.args.nth(pos) + } + + pub fn positional_iter(&self) -> impl Iterator> { + self.args.positional_iter() + } + + pub fn expect_nth(&self, pos: usize) -> Result<&Spanned, ShellError> { + self.args.expect_nth(pos) + } + + pub fn len(&self) -> usize { + self.args.len() + } + + pub fn get(&self, name: &str) -> Option<&Spanned> { + self.args.get(name) + } + + pub fn has(&self, name: &str) -> bool { + self.args.has(name) + } +} + pub struct SinkCommandArgs { pub ctx: Context, pub name_span: Option, - pub positional: Vec>, - pub named: indexmap::IndexMap, + pub args: Args, pub input: Vec, } @@ -46,8 +74,8 @@ pub trait Command { fn run(&self, args: CommandArgs) -> Result; fn name(&self) -> &str; - fn config(&self) -> CommandConfig { - CommandConfig { + fn config(&self) -> registry::CommandConfig { + registry::CommandConfig { name: self.name().to_string(), mandatory_positional: vec![], optional_positional: vec![], @@ -61,8 +89,8 @@ pub trait Sink { fn run(&self, args: SinkCommandArgs) -> Result<(), ShellError>; fn name(&self) -> &str; - fn config(&self) -> CommandConfig { - CommandConfig { + fn config(&self) -> registry::CommandConfig { + registry::CommandConfig { name: self.name().to_string(), mandatory_positional: vec![], optional_positional: vec![], diff --git a/src/commands/config.rs b/src/commands/config.rs index 11b35a84e2..d18bc2f17f 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1,8 +1,7 @@ use crate::errors::ShellError; use crate::object::config; use crate::object::Value; -use crate::parser::registry::{NamedType, NamedValue}; -use crate::parser::CommandConfig; +use crate::parser::registry::{CommandConfig, NamedType, NamedValue}; use crate::prelude::*; use indexmap::IndexMap; use log::trace; @@ -19,7 +18,7 @@ impl Command for Config { fn config(&self) -> CommandConfig { let mut named: IndexMap = IndexMap::new(); - named.insert("set".to_string(), NamedType::Optional(NamedValue::Tuple)); + named.insert("set".to_string(), NamedType::Optional(NamedValue::Single)); named.insert("get".to_string(), NamedType::Optional(NamedValue::Single)); named.insert("clear".to_string(), NamedType::Switch); @@ -41,10 +40,10 @@ impl Command for Config { pub fn config(args: CommandArgs) -> Result { let mut result = crate::object::config::config()?; - trace!("{:#?}", args.positional); - trace!("{:#?}", args.named); + trace!("{:#?}", args.args.positional); + trace!("{:#?}", args.args.named); - if let Some(v) = args.named.get("get") { + if let Some(v) = args.get("get") { let key = v.as_string()?; let value = result .get(&key) @@ -56,7 +55,7 @@ pub fn config(args: CommandArgs) -> Result { ); } - if let Some(v) = args.named.get("set") { + if let Some(v) = args.get("set") { if let Ok((key, value)) = v.as_pair() { result.insert(key.as_string()?, value.clone()); @@ -71,7 +70,7 @@ pub fn config(args: CommandArgs) -> Result { } } - if let Some(_) = args.named.get("clear") { + if let Some(_) = args.get("clear") { result.clear(); config::write_config(&result)?; @@ -84,7 +83,7 @@ pub fn config(args: CommandArgs) -> Result { ); } - if let Some(v) = args.named.get("remove") { + if let Some(v) = args.get("remove") { let key = v.as_string()?; if result.contains_key(&key) { @@ -104,7 +103,7 @@ pub fn config(args: CommandArgs) -> Result { ); } - if args.positional.len() == 0 { + if args.len() == 0 { return Ok( futures::stream::once(futures::future::ready(ReturnValue::Value(Value::Object( result.into(), diff --git a/src/commands/enter.rs b/src/commands/enter.rs index 2ca3c2ccaf..df0315a9ba 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -1,32 +1,34 @@ use crate::commands::command::CommandAction; use crate::errors::ShellError; use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::Spanned; use crate::prelude::*; use std::path::{Path, PathBuf}; pub fn enter(args: CommandArgs) -> Result { - if args.positional.len() == 0 { - return Err(ShellError::maybe_labeled_error( + let path = match args.nth(0) { + None => return Err(ShellError::maybe_labeled_error( "open requires a path or url", "missing path", args.name_span, - )); - } + )), + Some(p) => p, + }; let span = args.name_span; let cwd = args - .env + .env() .lock() .unwrap() .front() .unwrap() .path() .to_path_buf(); + let mut full_path = PathBuf::from(cwd); - let (file_extension, contents) = match &args.positional[0].item { + let (file_extension, contents) = match path.item() { Value::Primitive(Primitive::String(s)) => { if s.starts_with("http:") || s.starts_with("https:") { let response = reqwest::get(s); @@ -49,7 +51,7 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Web page contents corrupt", "received garbled data", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }, @@ -57,7 +59,7 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "URL could not be opened", "url not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -74,7 +76,7 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "File cound not be opened", "file not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -84,14 +86,14 @@ pub fn enter(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Expected string value for filename", "expected filename", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }; let mut stream = VecDeque::new(); - let file_extension = match args.positional.get(1) { + let file_extension = match args.nth(1) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), span, diff --git a/src/commands/first.rs b/src/commands/first.rs index 89bb4ff9c9..68831eeaea 100644 --- a/src/commands/first.rs +++ b/src/commands/first.rs @@ -4,7 +4,7 @@ use crate::prelude::*; // TODO: "Amount remaining" wrapper pub fn first(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "First requires an amount", "needs parameter", @@ -12,7 +12,7 @@ pub fn first(args: CommandArgs) -> Result { )); } - let amount = args.positional[0].as_i64(); + let amount = args.expect_nth(0)?.as_i64(); let amount = match amount { Ok(o) => o, @@ -20,7 +20,7 @@ pub fn first(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Value is not a number", "expected integer", - args.positional[0].span, + args.expect_nth(0)?.span, )) } }; diff --git a/src/commands/get.rs b/src/commands/get.rs index 60b835d62f..81a000a010 100644 --- a/src/commands/get.rs +++ b/src/commands/get.rs @@ -1,6 +1,6 @@ use crate::errors::ShellError; use crate::object::Value; -use crate::parser::lexer::Span; +use crate::parser::Span; use crate::prelude::*; fn get_member(path: &str, span: Span, obj: &Value) -> Option { @@ -22,7 +22,7 @@ fn get_member(path: &str, span: Span, obj: &Value) -> Option { } pub fn get(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Get requires a field or field path", "needs parameter", @@ -30,7 +30,7 @@ pub fn get(args: CommandArgs) -> Result { )); } - let amount = args.positional[0].as_i64(); + let amount = args.expect_nth(0)?.as_i64(); // If it's a number, get the row instead of the column if let Ok(amount) = amount { @@ -43,8 +43,7 @@ pub fn get(args: CommandArgs) -> Result { } let fields: Result, _> = args - .positional - .iter() + .positional_iter() .map(|a| (a.as_string().map(|x| (x, a.span)))) .collect(); let fields = fields?; diff --git a/src/commands/lines.rs b/src/commands/lines.rs index b1db3feb2f..86ac1ac09b 100644 --- a/src/commands/lines.rs +++ b/src/commands/lines.rs @@ -1,6 +1,9 @@ use crate::errors::ShellError; use crate::object::{Primitive, Value}; use crate::prelude::*; +use log::trace; + +// TODO: "Amount remaining" wrapper pub fn lines(args: CommandArgs) -> Result { let input = args.input; @@ -9,7 +12,10 @@ pub fn lines(args: CommandArgs) -> Result { let stream = input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - let split_result: Vec<_> = s.lines().filter(|s| s.trim() != "").collect(); + let splitter = "\n"; + let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); + + trace!("split result = {:?}", split_result); let mut result = VecDeque::new(); for s in split_result { diff --git a/src/commands/ls.rs b/src/commands/ls.rs index f8205e3a1a..2f0b09156c 100644 --- a/src/commands/ls.rs +++ b/src/commands/ls.rs @@ -1,6 +1,6 @@ use crate::errors::ShellError; use crate::object::{dir_entry_dict, Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::Spanned; use crate::prelude::*; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -10,7 +10,7 @@ pub fn ls(args: CommandArgs) -> Result { let path = env.back().unwrap().path.to_path_buf(); let obj = &env.back().unwrap().obj; let mut full_path = PathBuf::from(path); - match &args.positional.get(0) { + match &args.nth(0) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), .. @@ -24,7 +24,7 @@ pub fn ls(args: CommandArgs) -> Result { let entries = match entries { Err(e) => { - if let Some(s) = args.positional.get(0) { + if let Some(s) = args.nth(0) { return Err(ShellError::labeled_error( e.to_string(), e.to_string(), @@ -97,7 +97,7 @@ pub fn ls(args: CommandArgs) -> Result { return Err(ShellError::maybe_labeled_error( "Index not closed", format!("path missing closing ']'"), - if args.positional.len() > 0 { Some(args.positional[0].span) } else { args.name_span }, + if args.len() > 0 { Some(args.nth(0).unwrap().span) } else { args.name_span }, )) } } diff --git a/src/commands/open.rs b/src/commands/open.rs index 0edce69880..17be0b7063 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -1,11 +1,37 @@ use crate::errors::ShellError; use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::registry::{CommandConfig, NamedType}; use crate::prelude::*; +use indexmap::IndexMap; use std::path::{Path, PathBuf}; +use crate::parser::parse2::span::Spanned; -pub fn open(args: CommandArgs) -> Result { - if args.positional.len() == 0 { +pub struct Open; + +impl Command for Open { + fn run(&self, args: CommandArgs) -> Result { + open(args) + } + fn name(&self) -> &str { + "open" + } + + fn config(&self) -> CommandConfig { + let mut named: IndexMap = IndexMap::new(); + named.insert("raw".to_string(), NamedType::Switch); + + CommandConfig { + name: self.name().to_string(), + mandatory_positional: vec![], + optional_positional: vec![], + rest_positional: false, + named, + } + } +} + +fn open(args: CommandArgs) -> Result { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Open requires a path or url", "needs path or url", @@ -25,7 +51,7 @@ pub fn open(args: CommandArgs) -> Result { .to_path_buf(); let mut full_path = PathBuf::from(cwd); - let (file_extension, contents) = match &args.positional[0].item { + let (file_extension, contents) = match &args.expect_nth(0)?.item { Value::Primitive(Primitive::String(s)) => { if s.starts_with("http:") || s.starts_with("https:") { let response = reqwest::get(s); @@ -48,7 +74,7 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Web page contents corrupt", "received garbled data", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }, @@ -56,7 +82,7 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "URL could not be opened", "url not found", - args.positional[0].span, + args.expect_nth(0)?.span, )); } } @@ -71,9 +97,9 @@ pub fn open(args: CommandArgs) -> Result { ), Err(_) => { return Err(ShellError::labeled_error( - "File could not be opened", - "could not be opened", - args.positional[0].span, + "File cound not be opened", + "file not found", + args.expect_nth(0)?.span, )); } } @@ -83,14 +109,14 @@ pub fn open(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Expected string value for filename", "expected filename", - args.positional[0].span, + args.expect_nth(0)?.span, )); } }; let mut stream = VecDeque::new(); - let file_extension = match args.positional.get(1) { + let file_extension = match args.nth(1) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), span, diff --git a/src/commands/pick.rs b/src/commands/pick.rs index 9de5e80683..12d2a271a4 100644 --- a/src/commands/pick.rs +++ b/src/commands/pick.rs @@ -4,7 +4,7 @@ use crate::object::Value; use crate::prelude::*; pub fn pick(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Pick requires fields", "needs parameter", @@ -12,7 +12,7 @@ pub fn pick(args: CommandArgs) -> Result { )); } - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let objects = args diff --git a/src/commands/reject.rs b/src/commands/reject.rs index ec5703738b..50bce3d724 100644 --- a/src/commands/reject.rs +++ b/src/commands/reject.rs @@ -4,7 +4,7 @@ use crate::object::Value; use crate::prelude::*; pub fn reject(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Reject requires fields", "needs parameter", @@ -12,7 +12,7 @@ pub fn reject(args: CommandArgs) -> Result { )); } - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let stream = args diff --git a/src/commands/save.rs b/src/commands/save.rs index 3aeb751033..6c61df3f94 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -1,11 +1,11 @@ use crate::commands::command::SinkCommandArgs; use crate::errors::ShellError; use crate::object::{Primitive, Value}; -use crate::parser::lexer::Spanned; +use crate::parser::Spanned; use std::path::{Path, PathBuf}; pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> { - if args.positional.len() == 0 { + if args.args.positional.is_none() { return Err(ShellError::maybe_labeled_error( "Save requires a filepath", "needs path", @@ -13,6 +13,11 @@ pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> { )); } + let positional = match args.args.positional { + None => return Err(ShellError::string("save requires a filepath")), + Some(p) => p, + }; + let cwd = args .ctx .env @@ -23,12 +28,12 @@ pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> { .path() .to_path_buf(); let mut full_path = PathBuf::from(cwd); - match &(args.positional[0].item) { + match &(positional[0].item) { Value::Primitive(Primitive::String(s)) => full_path.push(Path::new(s)), _ => {} } - let save_raw = match args.positional.get(1) { + let save_raw = match positional.get(1) { Some(Spanned { item: Value::Primitive(Primitive::String(s)), .. diff --git a/src/commands/size.rs b/src/commands/size.rs index 6fdf78cfae..5d99cb5e55 100644 --- a/src/commands/size.rs +++ b/src/commands/size.rs @@ -6,7 +6,7 @@ use std::fs::File; use std::io::prelude::*; pub fn size(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Size requires a filepath", "needs path", @@ -25,7 +25,7 @@ pub fn size(args: CommandArgs) -> Result { let mut contents = String::new(); let mut list = VecDeque::new(); - for name in args.positional { + for name in args.positional_iter() { let name = name.as_string()?; let path = cwd.join(&name); let mut file = File::open(path)?; diff --git a/src/commands/skip.rs b/src/commands/skip.rs index d556151053..19b53ae582 100644 --- a/src/commands/skip.rs +++ b/src/commands/skip.rs @@ -2,7 +2,7 @@ use crate::errors::ShellError; use crate::prelude::*; pub fn skip(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Skip requires an amount", "needs parameter", @@ -10,7 +10,7 @@ pub fn skip(args: CommandArgs) -> Result { )); } - let amount = args.positional[0].as_i64(); + let amount = args.expect_nth(0)?.as_i64(); let amount = match amount { Ok(o) => o, @@ -18,7 +18,7 @@ pub fn skip(args: CommandArgs) -> Result { return Err(ShellError::labeled_error( "Value is not a number", "expected integer", - args.positional[0].span, + args.expect_nth(0)?.span, )) } }; diff --git a/src/commands/skip_while.rs b/src/commands/skip_while.rs index 15cb439812..eb6a948823 100644 --- a/src/commands/skip_while.rs +++ b/src/commands/skip_while.rs @@ -1,6 +1,6 @@ use crate::errors::ShellError; +use crate::parser::registry::CommandConfig; use crate::parser::registry::PositionalType; -use crate::parser::CommandConfig; use crate::prelude::*; pub struct SkipWhile; @@ -25,7 +25,7 @@ impl Command for SkipWhile { } pub fn skip_while(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Where requires a condition", "needs condition", @@ -33,7 +33,7 @@ pub fn skip_while(args: CommandArgs) -> Result { )); } - let block = args.positional[0].as_block()?; + let block = args.nth(0).unwrap().as_block()?; let input = args.input; let objects = input.skip_while(move |item| { @@ -48,4 +48,4 @@ pub fn skip_while(args: CommandArgs) -> Result { }); Ok(objects.map(|x| ReturnValue::Value(x)).boxed()) -} \ No newline at end of file +} diff --git a/src/commands/sort_by.rs b/src/commands/sort_by.rs index d488d5cbd4..9793e9b1b9 100644 --- a/src/commands/sort_by.rs +++ b/src/commands/sort_by.rs @@ -2,7 +2,7 @@ use crate::errors::ShellError; use crate::prelude::*; pub fn sort_by(args: CommandArgs) -> Result { - let fields: Result, _> = args.positional.iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let output = args.input.collect::>(); diff --git a/src/commands/split_column.rs b/src/commands/split_column.rs index 653b9e6180..e8890a1b35 100644 --- a/src/commands/split_column.rs +++ b/src/commands/split_column.rs @@ -6,28 +6,30 @@ use log::trace; // TODO: "Amount remaining" wrapper pub fn split_column(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + let positional: Vec<_> = args.positional_iter().cloned().collect(); + let span = args.name_span; + + if positional.len() == 0 { return Err(ShellError::maybe_labeled_error( "Split-column needs more information", "needs parameter (eg split-column \",\")", args.name_span, )); } + let input = args.input; - let span = args.name_span; - let args = args.positional; Ok(input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - let splitter = args[0].as_string().unwrap().replace("\\n", "\n"); + let splitter = positional[0].as_string().unwrap().replace("\\n", "\n"); trace!("splitting with {:?}", splitter); let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); trace!("split result = {:?}", split_result); // If they didn't provide column names, make up our own - if (args.len() - 1) == 0 { + if (positional.len() - 1) == 0 { let mut gen_columns = vec![]; for i in 0..split_result.len() { gen_columns.push(format!("Column{}", i + 1)); @@ -41,9 +43,9 @@ pub fn split_column(args: CommandArgs) -> Result { ); } ReturnValue::Value(Value::Object(dict)) - } else if split_result.len() == (args.len() - 1) { + } else if split_result.len() == (positional.len() - 1) { let mut dict = crate::object::Dictionary::default(); - for (k, v) in split_result.iter().zip(args.iter().skip(1)) { + for (k, v) in split_result.iter().zip(positional.iter().skip(1)) { dict.add( v.as_string().unwrap(), Value::Primitive(Primitive::String(k.to_string())), @@ -52,7 +54,7 @@ pub fn split_column(args: CommandArgs) -> Result { ReturnValue::Value(Value::Object(dict)) } else { let mut dict = crate::object::Dictionary::default(); - for k in args.iter().skip(1) { + for k in positional.iter().skip(1) { dict.add( k.as_string().unwrap().trim(), Value::Primitive(Primitive::String("".to_string())), diff --git a/src/commands/split_row.rs b/src/commands/split_row.rs index ed382bb761..f29f4bc261 100644 --- a/src/commands/split_row.rs +++ b/src/commands/split_row.rs @@ -1,12 +1,16 @@ use crate::errors::ShellError; use crate::object::{Primitive, Value}; +use crate::parser::Spanned; use crate::prelude::*; use log::trace; // TODO: "Amount remaining" wrapper pub fn split_row(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + let positional: Vec> = args.positional_iter().cloned().collect(); + let span = args.name_span; + + if positional.len() == 0 { return Err(ShellError::maybe_labeled_error( "Split-row needs more information", "needs parameter (eg split-row \"\\n\")", @@ -15,13 +19,11 @@ pub fn split_row(args: CommandArgs) -> Result { } let input = args.input; - let span = args.name_span; - let args = args.positional; let stream = input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - let splitter = args[0].as_string().unwrap().replace("\\n", "\n"); + let splitter = positional[0].as_string().unwrap().replace("\\n", "\n"); trace!("splitting with {:?}", splitter); let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); diff --git a/src/commands/view.rs b/src/commands/view.rs index 51e29cdbaa..2aba83d0bb 100644 --- a/src/commands/view.rs +++ b/src/commands/view.rs @@ -3,7 +3,7 @@ use crate::prelude::*; use prettyprint::PrettyPrinter; pub fn view(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "View requires a filename", "needs parameter", @@ -11,7 +11,7 @@ pub fn view(args: CommandArgs) -> Result { )); } - let target = match args.positional[0].as_string() { + let target = match args.expect_nth(0)?.as_string() { Ok(s) => s.clone(), Err(e) => { if let Some(span) = args.name_span { diff --git a/src/commands/where_.rs b/src/commands/where_.rs index 045c6f7df6..e5c2279ad3 100644 --- a/src/commands/where_.rs +++ b/src/commands/where_.rs @@ -1,6 +1,5 @@ use crate::errors::ShellError; -use crate::parser::registry::PositionalType; -use crate::parser::CommandConfig; +use crate::parser::registry::{CommandConfig, PositionalType}; use crate::prelude::*; pub struct Where; @@ -25,7 +24,7 @@ impl Command for Where { } pub fn r#where(args: CommandArgs) -> Result { - if args.positional.len() == 0 { + if args.len() == 0 { return Err(ShellError::maybe_labeled_error( "Where requires a condition", "needs condition", @@ -33,7 +32,7 @@ pub fn r#where(args: CommandArgs) -> Result { )); } - let block = args.positional[0].as_block()?; + let block = args.expect_nth(0)?.as_block()?; let input = args.input; let objects = input.filter_map(move |item| { diff --git a/src/context.rs b/src/context.rs index 931190acad..aee27d17e9 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,7 +1,8 @@ -use crate::commands::command::Sink; -use crate::commands::command::SinkCommandArgs; -use crate::parser::lexer::Span; -use crate::parser::Args; +use crate::commands::command::{Sink, SinkCommandArgs}; +use crate::parser::{ + registry::{Args, CommandConfig, CommandRegistry}, + Span, +}; use crate::prelude::*; use indexmap::IndexMap; @@ -58,8 +59,7 @@ impl Context { let command_args = SinkCommandArgs { ctx: self.clone(), name_span, - positional: args.positional, - named: args.named, + args, input, }; @@ -89,11 +89,16 @@ impl Context { host: self.host.clone(), env: self.env.clone(), name_span, - positional: args.positional, - named: args.named, + args, input, }; command.run(command_args) } } + +impl CommandRegistry for Context { + fn get(&self, name: &str) -> Option { + self.commands.get(name).map(|c| c.config()) + } +} diff --git a/src/errors.rs b/src/errors.rs index 520a2633d7..88d5e78fd3 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,6 +1,7 @@ -use crate::parser::lexer::{Span, SpannedToken}; #[allow(unused)] use crate::prelude::*; + +use crate::parser::Span; use derive_new::new; use language_reporting::{Diagnostic, Label, Severity}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -15,26 +16,40 @@ pub enum ShellError { impl ShellError { crate fn parse_error( - error: lalrpop_util::ParseError, + error: nom::Err<(nom_locate::LocatedSpan<&str>, nom::error::ErrorKind)>, ) -> ShellError { - use lalrpop_util::ParseError; use language_reporting::*; match error { - ParseError::UnrecognizedToken { - token: (start, SpannedToken { token, .. }, end), - expected, - } => { - let diagnostic = Diagnostic::new( - Severity::Error, - format!("Unexpected {:?}, expected {:?}", token, expected), - ) - .with_label(Label::new_primary(Span::from((start, end)))); + nom::Err::Incomplete(_) => unreachable!(), + nom::Err::Failure(span) | nom::Err::Error(span) => { + let diagnostic = + Diagnostic::new(Severity::Error, format!("{:?}", span)) + .with_label(Label::new_primary(Span::from(span.0))); ShellError::diagnostic(diagnostic) + // nom::Context::Code(span, kind) => { + // let diagnostic = + // Diagnostic::new(Severity::Error, format!("{}", kind.description())) + // .with_label(Label::new_primary(Span::from(span))); + + // ShellError::diagnostic(diagnostic) + // } } - ParseError::User { error } => error, - other => ShellError::string(format!("{:?}", other)), + // ParseError::UnrecognizedToken { + // token: (start, SpannedToken { token, .. }, end), + // expected, + // } => { + // let diagnostic = Diagnostic::new( + // Severity::Error, + // format!("Unexpected {:?}, expected {:?}", token, expected), + // ) + // .with_label(Label::new_primary(Span::from((start, end)))); + + // ShellError::diagnostic(diagnostic) + // } + // ParseError::User { error } => error, + // other => ShellError::string(format!("{:?}", other)), } } @@ -75,6 +90,10 @@ impl ShellError { ShellError::string(&format!("Unimplemented: {}", title.into())) } + crate fn unexpected(title: impl Into) -> ShellError { + ShellError::string(&format!("Unexpected: {}", title.into())) + } + crate fn copy_error(&self) -> ShellError { self.clone() } @@ -191,14 +210,14 @@ impl std::convert::From for ShellError { } } -impl std::convert::From> for ShellError { - fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError { - ShellError::String(StringError { - title: format!("{:?}", input), - error: Value::nothing(), - }) - } -} +// impl std::convert::From> for ShellError { +// fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError { +// ShellError::String(StringError { +// title: format!("{:?}", input), +// error: Value::nothing(), +// }) +// } +// } impl std::convert::From for ShellError { fn from(input: toml::ser::Error) -> ShellError { diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index c60b7fe08d..70cd486e71 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -1,6 +1,8 @@ -use crate::object::Primitive; -use crate::parser::ast; -use crate::parser::lexer::Spanned; +use crate::object::base::Block; +use crate::parser::{ + hir::{self, Expression, RawExpression}, + CommandRegistry, Span, Spanned, Text, +}; use crate::prelude::*; use derive_new::new; use indexmap::IndexMap; @@ -21,96 +23,75 @@ impl Scope { } } -crate fn evaluate_expr( - expr: &ast::Expression, +crate fn evaluate_baseline_expr( + expr: &Expression, + registry: &dyn CommandRegistry, scope: &Scope, + source: &str, ) -> Result, ShellError> { - use ast::*; - match &expr.expr { - RawExpression::Call(_) => Err(ShellError::unimplemented("Evaluating call expression")), - RawExpression::Leaf(l) => Ok(Spanned::from_item(evaluate_leaf(l), expr.span.clone())), - RawExpression::Parenthesized(p) => evaluate_expr(&p.expr, scope), - RawExpression::Flag(f) => Ok(Spanned::from_item( - Value::Primitive(Primitive::String(f.print())), - expr.span.clone(), - )), - RawExpression::Block(b) => evaluate_block(&b, scope), - RawExpression::Path(p) => evaluate_path(&p, scope), - RawExpression::Binary(b) => evaluate_binary(b, scope), - RawExpression::VariableReference(r) => { - evaluate_reference(r, scope).map(|x| Spanned::from_item(x, expr.span.clone())) - } - } -} + match &expr.item { + RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(*literal), source)), + RawExpression::Variable(var) => evaluate_reference(var, scope, source), + RawExpression::Binary(binary) => { + let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?; + let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?; -fn evaluate_leaf(leaf: &ast::Leaf) -> Value { - use ast::*; - - match leaf { - Leaf::String(s) => Value::string(s), - Leaf::Bare(path) => Value::string(path.to_string()), - Leaf::Boolean(b) => Value::boolean(*b), - Leaf::Int(i) => Value::int(*i), - Leaf::Unit(i, unit) => unit.compute(*i), - } -} - -fn evaluate_reference(r: &ast::Variable, scope: &Scope) -> Result { - use ast::Variable::*; - - match r { - It => Ok(scope.it.copy()), - Other(s) => Ok(scope - .vars - .get(s) - .map(|v| v.copy()) - .unwrap_or_else(|| Value::nothing())), - } -} - -fn evaluate_binary(binary: &ast::Binary, scope: &Scope) -> Result, ShellError> { - let left = evaluate_expr(&binary.left, scope)?; - let right = evaluate_expr(&binary.right, scope)?; - - match left.compare(&binary.operator, &right) { - Some(v) => Ok(Spanned::from_item( - Value::boolean(v), - binary.operator.span.clone(), - )), - None => Err(ShellError::TypeError(format!( - "Can't compare {} and {}", - left.type_name(), - right.type_name() - ))), - } -} - -fn evaluate_block(block: &ast::Block, _scope: &Scope) -> Result, ShellError> { - Ok(Spanned::from_item( - Value::block(block.expr.clone()), - block.expr.span.clone(), - )) -} - -fn evaluate_path(path: &ast::Path, scope: &Scope) -> Result, ShellError> { - let head = path.head(); - let mut value = evaluate_expr(head, scope)?; - let mut seen = vec![]; - - for name in path.tail() { - let next = value.get_data_by_key(&name.item); - seen.push(name.item.clone()); - - match next { - None => { - return Err(ShellError::MissingProperty { - expr: path.print(), - subpath: itertools::join(seen, "."), - }); + match left.compare(binary.op(), &*right) { + Some(result) => Ok(Spanned::from_item(Value::boolean(result), *expr.span())), + None => Err(ShellError::unimplemented(&format!( + "Comparison failure {:?}", + binary + ))), } - Some(v) => value = Spanned::from_item(v.copy(), name.span.clone()), } - } + RawExpression::Block(block) => Ok(Spanned::from_item( + Value::Block(Block::new(*block.clone(), Text::from(source))), // TODO: Pass Text around + block.span(), + )), + RawExpression::Path(path) => { + let value = evaluate_baseline_expr(path.head(), registry, scope, source)?; + let mut value = value.item(); - Ok(value) + for name in path.tail() { + let next = value.get_data_by_key(name); + + match next { + None => return Err(ShellError::unimplemented("Invalid property from path")), + Some(next) => value = next, + }; + } + + Ok(Spanned::from_item(value.clone(), expr.span())) + } + RawExpression::Boolean(_boolean) => unimplemented!(), + } +} + +fn evaluate_literal(literal: Spanned, source: &str) -> Spanned { + let result = match literal.item { + hir::Literal::Integer(int) => Value::int(int), + hir::Literal::Size(_int, _unit) => unimplemented!(), + hir::Literal::String(span) => Value::string(span.slice(source)), + hir::Literal::Bare => Value::string(literal.span().slice(source)), + }; + + literal.map(|_| result) +} + +fn evaluate_reference( + name: &hir::Variable, + scope: &Scope, + source: &str, +) -> Result, ShellError> { + match name { + hir::Variable::It(span) => Ok(Spanned::from_item(scope.it.copy(), span)), + hir::Variable::Other(span) => Ok(Spanned::from_item( + scope + .vars + .get(span.slice(source)) + .map(|v| v.copy()) + .unwrap_or_else(|| Value::nothing()), + span, + )), + } } diff --git a/src/evaluate/mod.rs b/src/evaluate/mod.rs index febfb1bd70..09451fbb49 100644 --- a/src/evaluate/mod.rs +++ b/src/evaluate/mod.rs @@ -1,3 +1,3 @@ crate mod evaluator; -crate use evaluator::{evaluate_expr, Scope}; +crate use evaluator::{evaluate_baseline_expr, Scope}; diff --git a/src/object/base.rs b/src/object/base.rs index d9a531387c..7b4e108ab8 100644 --- a/src/object/base.rs +++ b/src/object/base.rs @@ -1,16 +1,17 @@ use crate::errors::ShellError; -use crate::evaluate::{evaluate_expr, Scope}; +use crate::evaluate::{evaluate_baseline_expr, Scope}; use crate::object::DataDescriptor; -use crate::parser::ast::{self, Operator}; -use crate::parser::lexer::Spanned; +use crate::parser::{hir, Operator, Spanned}; use crate::prelude::*; use ansi_term::Color; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; use derive_new::new; use ordered_float::OrderedFloat; +use std::fmt; use std::time::SystemTime; +use crate::parser::Text; use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq, new)] @@ -75,6 +76,20 @@ impl Primitive { .to_string() } + crate fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result { + use Primitive::*; + + match self { + Nothing => write!(f, "Nothing"), + Int(int) => write!(f, "{}", int), + Float(float) => write!(f, "{:?}", float), + Bytes(bytes) => write!(f, "{}", bytes), + String(string) => write!(f, "{:?}", string), + Boolean(boolean) => write!(f, "{}", boolean), + Date(date) => write!(f, "{}", date), + } + } + crate fn format(&self, field_name: Option<&DataDescriptor>) -> String { match self { Primitive::Nothing => format!("{}", Color::Black.bold().paint("-")), @@ -117,7 +132,8 @@ pub struct Operation { #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)] pub struct Block { - crate expression: ast::Expression, + crate expression: hir::Expression, + crate source: Text, } impl Serialize for Block { @@ -125,7 +141,7 @@ impl Serialize for Block { where S: Serializer, { - serializer.serialize_str(&self.expression.print()) + serializer.serialize_str(&self.expression.source(self.source.as_ref())) } } @@ -134,17 +150,19 @@ impl Deserialize<'de> for Block { where D: Deserializer<'de>, { - let mut builder = ast::ExpressionBuilder::new(); - let expr: ast::Expression = builder.string("Unserializable block"); - - Ok(Block::new(expr)) + unimplemented!("deserialize block") + // let s = "\"unimplemented deserialize block\""; + // Ok(Block::new( + // TokenTreeBuilder::spanned_string((1, s.len() - 1), (0, s.len())), + // Text::from(s), + // )) } } impl Block { pub fn invoke(&self, value: &Value) -> Result, ShellError> { let scope = Scope::new(value.copy()); - evaluate_expr(&self.expression, &scope) + evaluate_baseline_expr(&self.expression, &(), &scope, self.source.as_ref()) } } @@ -153,6 +171,7 @@ pub enum Value { Primitive(Primitive), Object(crate::object::Dictionary), List(Vec), + #[allow(unused)] Block(Block), Filesystem, @@ -160,6 +179,39 @@ pub enum Value { Error(Box), } +pub fn debug_list(values: &'a Vec) -> ValuesDebug<'a> { + ValuesDebug { values } +} + +pub struct ValuesDebug<'a> { + values: &'a Vec, +} + +impl fmt::Debug for ValuesDebug<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_list() + .entries(self.values.iter().map(|i| i.debug())) + .finish() + } +} + +pub struct ValueDebug<'a> { + value: &'a Value, +} + +impl fmt::Debug for ValueDebug<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.value { + Value::Primitive(p) => p.debug(f), + Value::Object(o) => o.debug(f), + Value::List(l) => debug_list(l).fmt(f), + Value::Block(_) => write!(f, "[[block]]"), + Value::Error(err) => write!(f, "[[error :: {} ]]", err), + Value::Filesystem => write!(f, "[[filesystem]]"), + } + } +} + impl Value { crate fn type_name(&self) -> String { match self { @@ -172,6 +224,10 @@ impl Value { } } + crate fn debug(&'a self) -> ValueDebug<'a> { + ValueDebug { value: self } + } + crate fn data_descriptors(&self) -> Vec { match self { Value::Primitive(_) => vec![DataDescriptor::value_of()], @@ -237,7 +293,7 @@ impl Value { crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String { match self { Value::Primitive(p) => p.format(desc), - Value::Block(b) => b.expression.print(), + Value::Block(b) => b.expression.source(b.source.as_ref()).to_string(), Value::Object(_) => format!("[object Object]"), Value::List(_) => format!("[list List]"), Value::Error(e) => format!("{}", e), @@ -245,7 +301,8 @@ impl Value { } } - crate fn compare(&self, operator: &ast::Operator, other: &Value) -> Option { + #[allow(unused)] + crate fn compare(&self, operator: &Operator, other: &Value) -> Option { match operator { _ => { let coerced = coerce_compare(self, other)?; @@ -347,8 +404,9 @@ impl Value { } } - crate fn block(e: ast::Expression) -> Value { - Value::Block(Block::new(e)) + #[allow(unused)] + crate fn block(e: hir::Expression, source: Text) -> Value { + Value::Block(Block::new(e, source)) } crate fn string(s: impl Into) -> Value { diff --git a/src/object/desc.rs b/src/object/desc.rs index 8693cba1cc..d69705b466 100644 --- a/src/object/desc.rs +++ b/src/object/desc.rs @@ -16,6 +16,13 @@ impl DescriptorName { } } + crate fn debug(&self) -> &str { + match self { + DescriptorName::String(s) => s, + DescriptorName::ValueOf => "[[value]]", + } + } + crate fn as_string(&self) -> Option<&str> { match self { DescriptorName::String(s) => Some(s), diff --git a/src/object/dict.rs b/src/object/dict.rs index c9ff168a0c..3cd13261b4 100644 --- a/src/object/dict.rs +++ b/src/object/dict.rs @@ -7,6 +7,7 @@ use indexmap::IndexMap; use serde::ser::{Serialize, SerializeMap, Serializer}; use serde_derive::Deserialize; use std::cmp::{Ordering, PartialOrd}; +use std::fmt; #[derive(Debug, Default, Eq, PartialEq, Deserialize, Clone, new)] pub struct Dictionary { @@ -113,4 +114,14 @@ impl Dictionary { None => None, } } + + crate fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_struct("Dictionary"); + + for (desc, value) in self.entries.iter() { + debug.field(desc.name.debug(), &value.debug()); + } + + debug.finish() + } } diff --git a/src/parser.rs b/src/parser.rs index b2eb9f8468..3af5481c76 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,279 +1,275 @@ -crate mod ast; -crate mod completer; -crate mod lexer; -crate mod parser; -crate mod registry; -crate mod span; +crate mod hir; crate mod parse2; - -crate use ast::Pipeline; -crate use registry::{Args, CommandConfig}; +crate mod parse_command; +crate mod registry; use crate::errors::ShellError; -use ast::Module; -use lexer::Lexer; -use log::trace; -use parser::{ModuleParser, ReplLineParser}; -pub fn parse(input: &str) -> Result { +crate use hir::baseline_parse_tokens::baseline_parse_tokens; +crate use parse2::call_node::CallNode; +crate use parse2::files::Files; +crate use parse2::flag::Flag; +crate use parse2::operator::Operator; +crate use parse2::parser::{nom_input, pipeline}; +crate use parse2::span::{Span, Spanned}; +crate use parse2::text::Text; +crate use parse2::token_tree::TokenNode; +crate use parse2::tokens::{RawToken, Token}; +crate use parse2::unit::Unit; +crate use parse_command::parse_command; +crate use registry::CommandRegistry; + +pub fn parse(input: &str) -> Result { let _ = pretty_env_logger::try_init(); - let parser = ReplLineParser::new(); - let tokens = Lexer::new(input, false); - - trace!( - "Tokens: {:?}", - tokens.clone().collect::, _>>() - ); - - match parser.parse(tokens) { - Ok(val) => Ok(val), + match pipeline(nom_input(input)) { + Ok((_rest, val)) => Ok(val), Err(err) => Err(ShellError::parse_error(err)), } } -#[allow(unused)] -pub fn parse_module(input: &str) -> Result { - let _ = pretty_env_logger::try_init(); +// #[allow(unused)] +// pub fn parse_module(input: &str) -> Result { +// let _ = pretty_env_logger::try_init(); - let parser = ModuleParser::new(); - let tokens = Lexer::new(input, false); +// let parser = ModuleParser::new(); +// let tokens = Lexer::new(input, false); - trace!( - "Tokens: {:?}", - tokens.clone().collect::, _>>() - ); +// trace!( +// "Tokens: {:?}", +// tokens.clone().collect::, _>>() +// ); - match parser.parse(tokens) { - Ok(val) => Ok(val), - Err(err) => Err(ShellError::parse_error(err)), - } -} +// match parser.parse(tokens) { +// Ok(val) => Ok(val), +// Err(err) => Err(ShellError::parse_error(err)), +// } +// } -#[cfg(test)] -mod tests { - use super::*; - use crate::parser::ast::Pipeline; - use pretty_assertions::assert_eq; +// #[cfg(test)] +// mod tests { +// use super::*; +// use crate::parser::ast::Pipeline; +// use pretty_assertions::assert_eq; - fn assert_parse(source: &str, expected: Pipeline) { - let parsed = match parse(source) { - Ok(p) => p, - Err(ShellError::Diagnostic(diag)) => { - use language_reporting::termcolor; +// fn assert_parse(source: &str, expected: Pipeline) { +// let parsed = match parse(source) { +// Ok(p) => p, +// Err(ShellError::Diagnostic(diag)) => { +// use language_reporting::termcolor; - let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); - let files = crate::parser::span::Files::new(source.to_string()); +// let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); +// let files = crate::parser::span::Files::new(source.to_string()); - language_reporting::emit( - &mut writer.lock(), - &files, - &diag.diagnostic, - &language_reporting::DefaultConfig, - ) - .unwrap(); +// language_reporting::emit( +// &mut writer.lock(), +// &files, +// &diag.diagnostic, +// &language_reporting::DefaultConfig, +// ) +// .unwrap(); - panic!("Test failed") - } - Err(err) => panic!("Something went wrong during parse: {:#?}", err), - }; +// panic!("Test failed") +// } +// Err(err) => panic!("Something went wrong during parse: {:#?}", err), +// }; - let printed = parsed.print(); +// let printed = parsed.print(); - assert_eq!(parsed, expected); - assert_eq!(printed, source); +// assert_eq!(parsed, expected); +// assert_eq!(printed, source); - let span = expected.span; +// let span = expected.span; - let expected_module = ModuleBuilder::spanned_items( - vec![Spanned::from_item(RawItem::Expression(expected), span)], - span.start, - span.end, - ); +// let expected_module = ModuleBuilder::spanned_items( +// vec![Spanned::from_item(RawItem::Expression(expected), span)], +// span.start, +// span.end, +// ); - assert_parse_module(source, expected_module); - } +// assert_parse_module(source, expected_module); +// } - fn assert_parse_module(source: &str, expected: Module) { - let parsed = match parse_module(source) { - Ok(p) => p, - Err(ShellError::Diagnostic(diag)) => { - use language_reporting::termcolor; +// fn assert_parse_module(source: &str, expected: Module) { +// let parsed = match parse_module(source) { +// Ok(p) => p, +// Err(ShellError::Diagnostic(diag)) => { +// use language_reporting::termcolor; - let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); - let files = crate::parser::span::Files::new(source.to_string()); +// let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); +// let files = crate::parser::span::Files::new(source.to_string()); - language_reporting::emit( - &mut writer.lock(), - &files, - &diag.diagnostic, - &language_reporting::DefaultConfig, - ) - .unwrap(); +// language_reporting::emit( +// &mut writer.lock(), +// &files, +// &diag.diagnostic, +// &language_reporting::DefaultConfig, +// ) +// .unwrap(); - panic!("Test failed") - } - Err(err) => panic!("Something went wrong during parse: {:#?}", err), - }; +// panic!("Test failed") +// } +// Err(err) => panic!("Something went wrong during parse: {:#?}", err), +// }; - let printed = parsed.print(); +// let printed = parsed.print(); - assert_eq!(parsed, expected); - assert_eq!(printed, source); - } +// assert_eq!(parsed, expected); +// assert_eq!(printed, source); +// } - macro_rules! commands { - ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{ - use $crate::parser::ast::{Expression, ExpressionBuilder}; - let mut builder = crate::parser::ast::ExpressionBuilder::new(); +// macro_rules! commands { +// ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{ +// use $crate::parser::ast::{Expression, ExpressionBuilder}; +// let mut builder = crate::parser::ast::ExpressionBuilder::new(); - builder.pipeline(vec![ - $( - (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression)) - ),* - ]) - }} - } +// builder.pipeline(vec![ +// $( +// (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression)) +// ),* +// ]) +// }} +// } - macro_rules! command { - ($name:ident) => { - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(( - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)), - vec![] - )) - }; +// macro_rules! command { +// ($name:ident) => { +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(( +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)), +// vec![] +// )) +// }; - ($name:ident $( $command:ident ( $body:expr ) )*) => {{ - use $crate::parser::ast::{Expression, ExpressionBuilder}; - &|b: &mut ExpressionBuilder| b.call(( - (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression), - vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ])) +// ($name:ident $( $command:ident ( $body:expr ) )*) => {{ +// use $crate::parser::ast::{Expression, ExpressionBuilder}; +// &|b: &mut ExpressionBuilder| b.call(( +// (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression), +// vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ])) - }}; +// }}; - ($name:ident $( $command:ident ( $body:expr ) )*) => { - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ]) - }; +// ($name:ident $( $command:ident ( $body:expr ) )*) => { +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ]) +// }; - ($name:tt $( $command:ident ( $body:expr ) )*) => { - &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ])) - }; - } +// ($name:tt $( $command:ident ( $body:expr ) )*) => { +// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ])) +// }; +// } - #[test] - fn parse_simple_command() { - assert_parse("ls", commands![(ls)]); - } +// #[test] +// fn parse_simple_command() { +// assert_parse("ls", commands![(ls)]); +// } - #[test] - fn parse_command_with_args() { - assert_parse( - r#"open Cargo.toml | select package.authors | split-row " ""#, - commands![ - (open bare("Cargo.toml")) - | (select bare("package.authors")) - | ("split-row" string(" ")) - ], - ); +// #[test] +// fn parse_command_with_args() { +// assert_parse( +// r#"open Cargo.toml | select package.authors | split-row " ""#, +// commands![ +// (open bare("Cargo.toml")) +// | (select bare("package.authors")) +// | ("split-row" string(" ")) +// ], +// ); - assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]); +// assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]); - assert_parse( - "open Cargo.toml | select package.version | echo $it", - commands![ - (open bare("Cargo.toml")) - | (select bare("package.version")) - | (echo var("it")) - ], - ); +// assert_parse( +// "open Cargo.toml | select package.version | echo $it", +// commands![ +// (open bare("Cargo.toml")) +// | (select bare("package.version")) +// | (echo var("it")) +// ], +// ); - assert_parse( - "open Cargo.toml --raw", - commands![(open bare("Cargo.toml") flag("raw"))], - ); +// assert_parse( +// "open Cargo.toml --raw", +// commands![(open bare("Cargo.toml") flag("raw"))], +// ); - assert_parse( - "open Cargo.toml -r", - commands![(open bare("Cargo.toml") shorthand("r"))], - ); +// assert_parse( +// "open Cargo.toml -r", +// commands![(open bare("Cargo.toml") shorthand("r"))], +// ); - assert_parse( - "open Cargo.toml | from-toml | to-toml", - commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")], - ); +// assert_parse( +// "open Cargo.toml | from-toml | to-toml", +// commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")], +// ); - assert_parse( - r#"config --get "ignore dups" | format-list"#, - commands![(config flag("get") string("ignore dups")) | ("format-list")], - ); +// assert_parse( +// r#"config --get "ignore dups" | format-list"#, +// commands![(config flag("get") string("ignore dups")) | ("format-list")], +// ); - assert_parse( - "open Cargo.toml | from-toml | select dependencies | column serde", - commands![ - (open bare("Cargo.toml")) - | ("from-toml") - | (select bare("dependencies")) - | (column bare("serde")) - ], - ); +// assert_parse( +// "open Cargo.toml | from-toml | select dependencies | column serde", +// commands![ +// (open bare("Cargo.toml")) +// | ("from-toml") +// | (select bare("dependencies")) +// | (column bare("serde")) +// ], +// ); - assert_parse( - "config --set tabs 2", - commands![(config flag("set") bare("tabs") int(2))], - ); +// assert_parse( +// "config --set tabs 2", +// commands![(config flag("set") bare("tabs") int(2))], +// ); - assert_parse( - r#"ls | skip 1 | first 2 | select "file name" | rm $it"#, - commands![ - (ls) - | (skip int(1)) - | (first int(2)) - | (select string("file name")) - | (rm var("it")) - ], - ); +// assert_parse( +// r#"ls | skip 1 | first 2 | select "file name" | rm $it"#, +// commands![ +// (ls) +// | (skip int(1)) +// | (first int(2)) +// | (select string("file name")) +// | (rm var("it")) +// ], +// ); - assert_parse( - r#"git branch --merged | split-row "`n" | where $it != "* master""#, - commands![ - // TODO: Handle escapes correctly. Should we do ` escape because of paths? - (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master")))) - ], - ); +// assert_parse( +// r#"git branch --merged | split-row "`n" | where $it != "* master""#, +// commands![ +// // TODO: Handle escapes correctly. Should we do ` escape because of paths? +// (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master")))) +// ], +// ); - assert_parse( - r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#, - commands![ - (open bare("input2.json")) - | ("from-json") - | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso")) - | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML")))) - ] - ); +// assert_parse( +// r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#, +// commands![ +// (open bare("input2.json")) +// | ("from-json") +// | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso")) +// | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML")))) +// ] +// ); - assert_parse( - r"cd ..\.cargo\", - commands![ - (cd bare(r"..\.cargo\")) - ], - ); +// assert_parse( +// r"cd ..\.cargo\", +// commands![ +// (cd bare(r"..\.cargo\")) +// ], +// ); - assert_parse( - "ls | where size < 1KB", - commands![ - (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB"))))) - ], - ); +// assert_parse( +// "ls | where size < 1KB", +// commands![ +// (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB"))))) +// ], +// ); - assert_parse( - "ls | where { $it.size > 100 }", - commands![ - (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100))))) - ], - ) - } +// assert_parse( +// "ls | where { $it.size > 100 }", +// commands![ +// (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100))))) +// ], +// ) +// } - use crate::parser::ast::{ModuleBuilder, RawItem}; - use crate::parser::lexer::Spanned; +// use crate::parser::ast::{ModuleBuilder, RawItem}; +// use crate::parser::lexer::Spanned; -} +// } diff --git a/src/parser/hir.rs b/src/parser/hir.rs new file mode 100644 index 0000000000..0b7e90cdc5 --- /dev/null +++ b/src/parser/hir.rs @@ -0,0 +1,93 @@ +crate mod baseline_parse; +crate mod baseline_parse_tokens; +crate mod binary; +crate mod named; +crate mod path; + +use crate::parser::{Span, Spanned, Unit}; +use derive_new::new; +use getset::Getters; + +crate use baseline_parse::baseline_parse_single_token; +crate use baseline_parse_tokens::{ + baseline_parse_next_expr, ExpressionKindHint, +}; +crate use binary::Binary; +crate use named::NamedArguments; +crate use path::Path; + +#[derive(Debug, Clone, Eq, PartialEq, Getters, new)] +pub struct Call { + #[get = "crate"] + head: Box, + #[get = "crate"] + positional: Option>, + #[get = "crate"] + named: Option, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RawExpression { + Literal(Literal), + Variable(Variable), + Binary(Box), + Block(Box), + Path(Box), + + #[allow(unused)] + Boolean(bool), +} + +pub type Expression = Spanned; + +impl Expression { + fn int(i: impl Into, span: impl Into) -> Expression { + Spanned::from_item(RawExpression::Literal(Literal::Integer(i.into())), span) + } + + fn size(i: impl Into, unit: impl Into, span: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Literal(Literal::Size(i.into(), unit.into())), + span, + ) + } + + fn string(inner: impl Into, outer: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Literal(Literal::String(inner.into())), + outer.into(), + ) + } + + fn bare(span: impl Into) -> Expression { + Spanned::from_item(RawExpression::Literal(Literal::Bare), span.into()) + } + + fn variable(inner: impl Into, outer: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Variable(Variable::Other(inner.into())), + outer.into(), + ) + } + + fn it_variable(inner: impl Into, outer: impl Into) -> Expression { + Spanned::from_item( + RawExpression::Variable(Variable::It(inner.into())), + outer.into(), + ) + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum Literal { + Integer(i64), + Size(i64, Unit), + String(Span), + Bare, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum Variable { + It(Span), + Other(Span), +} diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs new file mode 100644 index 0000000000..1150434fcb --- /dev/null +++ b/src/parser/hir/baseline_parse.rs @@ -0,0 +1,30 @@ +use crate::errors::ShellError; +use crate::parser::{hir, CommandRegistry, RawToken, Token, TokenNode}; + +// pub fn baseline_parse_token( +// token_node: TokenNode, +// _registry: &dyn CommandRegistry, +// ) -> Result { +// match token_node { +// TokenNode::Token(token) => Ok(baseline_parse_single_token(token)), +// TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")), +// TokenNode::Delimited(_delimited) => { +// Err(ShellError::unimplemented("baseline_parse Delimited")) +// } +// TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")), +// TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")), +// } +// } + +pub fn baseline_parse_single_token(token: &Token, source: &str) -> hir::Expression { + match *token.item() { + RawToken::Integer(int) => hir::Expression::int(int, token.span), + RawToken::Size(int, unit) => hir::Expression::size(int, unit, token.span), + RawToken::String(span) => hir::Expression::string(span, token.span), + RawToken::Variable(span) if span.slice(source) == "it" => { + hir::Expression::it_variable(span, token.span) + } + RawToken::Variable(span) => hir::Expression::variable(span, token.span), + RawToken::Bare => hir::Expression::bare(token.span), + } +} diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs new file mode 100644 index 0000000000..4cf8ae86bb --- /dev/null +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -0,0 +1,177 @@ +use crate::errors::ShellError; +use crate::parser::registry::CommandRegistry; +use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode}; + +pub fn baseline_parse_tokens( + token_nodes: &[TokenNode], + registry: &dyn CommandRegistry, + source: &str, +) -> Result, ShellError> { + let mut exprs: Vec = vec![]; + let mut rest = token_nodes; + + loop { + if rest.len() == 0 { + break; + } + + let (expr, remainder) = baseline_parse_next_expr(rest, registry, source, None)?; + exprs.push(expr); + rest = remainder; + } + + Ok(exprs) +} + +#[derive(Debug)] +pub enum ExpressionKindHint { + Literal, + Variable, + Binary, + Block, + Boolean, +} + +pub fn baseline_parse_next_expr( + token_nodes: &'nodes [TokenNode], + _registry: &dyn CommandRegistry, + source: &str, + coerce_hint: Option, +) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> { + println!( + "baseline_parse_next_expr {:?} - {:?}", + token_nodes, coerce_hint + ); + + let mut tokens = token_nodes.iter().peekable(); + + let first = next_token(&mut tokens); + + let first = match first { + None => return Err(ShellError::unimplemented("Expected token, found none")), + Some(token) => baseline_parse_semantic_token(token, source)?, + }; + + let possible_op = tokens.peek(); + + let op = match possible_op { + Some(TokenNode::Operator(op)) => op, + _ => return Ok((first, &token_nodes[1..])), + }; + + tokens.next(); + + let second = match tokens.next() { + None => { + return Err(ShellError::unimplemented( + "Expected op followed by another expr, found nothing", + )) + } + Some(token) => baseline_parse_semantic_token(token, source)?, + }; + + // We definitely have a binary expression here -- let's see if we should coerce it into a block + + match coerce_hint { + None => { + let span = (first.span.start, second.span.end); + let binary = hir::Binary::new(first, *op, second); + let binary = hir::RawExpression::Binary(Box::new(binary)); + let binary = Spanned::from_item(binary, span); + + Ok((binary, &token_nodes[3..])) + } + + Some(hint) => match hint { + ExpressionKindHint::Block => { + let span = (first.span.start, second.span.end); + + let string: Spanned = match first { + Spanned { + item: hir::RawExpression::Literal(hir::Literal::Bare), + span, + } => Spanned::from_item(span.slice(source).to_string(), span), + Spanned { + item: hir::RawExpression::Literal(hir::Literal::String(inner)), + span, + } => Spanned::from_item(inner.slice(source).to_string(), span), + _ => { + return Err(ShellError::unimplemented( + "The first part of a block must be a string", + )) + } + }; + + let path = hir::Path::new( + Spanned::from_item( + // TODO: Deal with synthetic nodes that have no representation at all in source + hir::RawExpression::Variable(hir::Variable::It(Span::from((0, 0)))), + (0, 0), + ), + vec![string], + ); + let path = hir::RawExpression::Path(Box::new(path)); + let path = Spanned::from_item(path, first.span); + + let binary = hir::Binary::new(path, *op, second); + let binary = hir::RawExpression::Binary(Box::new(binary)); + let binary = Spanned::from_item(binary, span); + + let block = hir::RawExpression::Block(Box::new(binary)); + let block = Spanned::from_item(block, span); + + Ok((block, &token_nodes[3..])) + } + + other => unimplemented!("coerce hint {:?}", other), + }, + } +} + +pub fn baseline_parse_semantic_token( + token: &TokenNode, + source: &str, +) -> Result { + match token { + TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), + TokenNode::Call(call) => unimplemented!(), + TokenNode::Delimited(delimited) => unimplemented!(), + TokenNode::Pipeline(pipeline) => unimplemented!(), + TokenNode::Operator(_op) => unreachable!(), + TokenNode::Flag(flag) => unimplemented!(), + TokenNode::Identifier(_span) => unreachable!(), + TokenNode::Whitespace(_span) => unreachable!(), + TokenNode::Error(error) => Err(*error.item.clone()), + TokenNode::Path(path) => unimplemented!(), + } +} + +fn next_token(nodes: &mut impl Iterator) -> Option<&'a TokenNode> { + loop { + match nodes.next() { + Some(TokenNode::Whitespace(_)) => continue, + other => return other, + } + } +} + +fn baseline_parse_token( + token_node: &TokenNode, + _registry: &dyn CommandRegistry, + source: &str, +) -> Result { + match token_node { + TokenNode::Token(token) => Ok(hir::baseline_parse_single_token(token, source)), + TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")), + TokenNode::Delimited(_delimited) => { + Err(ShellError::unimplemented("baseline_parse Delimited")) + } + TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")), + TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")), + TokenNode::Operator(_op) => Err(ShellError::unimplemented("baseline_parse Operator")), + TokenNode::Flag(_op) => Err(ShellError::unimplemented("baseline_parse Flag")), + TokenNode::Identifier(_op) => Err(ShellError::unimplemented("baseline_parse Identifier")), + TokenNode::Whitespace(_op) => Err(ShellError::unimplemented("baseline_parse Whitespace")), + TokenNode::Error(err) => Err(*err.item.clone()), + } +} diff --git a/src/parser/hir/binary.rs b/src/parser/hir/binary.rs new file mode 100644 index 0000000000..315964ed5f --- /dev/null +++ b/src/parser/hir/binary.rs @@ -0,0 +1,11 @@ +use crate::parser::{hir::Expression, Operator, Spanned}; +use derive_new::new; +use getset::Getters; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, new)] +#[get = "crate"] +pub struct Binary { + left: Expression, + op: Spanned, + right: Expression, +} diff --git a/src/parser/hir/named.rs b/src/parser/hir/named.rs new file mode 100644 index 0000000000..07c6db655e --- /dev/null +++ b/src/parser/hir/named.rs @@ -0,0 +1,44 @@ +use crate::parser::hir::Expression; +use crate::parser::{Flag, Span}; +use derive_new::new; +use indexmap::IndexMap; +use log::trace; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum NamedValue { + AbsentSwitch, + PresentSwitch(Span), + AbsentValue, + Value(Expression), +} + +#[derive(Debug, Clone, Eq, PartialEq, new)] +pub struct NamedArguments { + #[new(default)] + crate named: IndexMap, +} + +impl NamedArguments { + pub fn insert_switch(&mut self, name: impl Into, switch: Option) { + let name = name.into(); + trace!("Inserting switch -- {} = {:?}", name, switch); + + match switch { + None => self.named.insert(name.into(), NamedValue::AbsentSwitch), + Some(flag) => self + .named + .insert(name, NamedValue::PresentSwitch(*flag.name())), + }; + } + + pub fn insert_optional(&mut self, name: impl Into, expr: Option) { + match expr { + None => self.named.insert(name.into(), NamedValue::AbsentValue), + Some(expr) => self.named.insert(name.into(), NamedValue::Value(expr)), + }; + } + + pub fn insert_mandatory(&mut self, name: impl Into, expr: Expression) { + self.named.insert(name.into(), NamedValue::Value(expr)); + } +} diff --git a/src/parser/hir/path.rs b/src/parser/hir/path.rs new file mode 100644 index 0000000000..132f3112b8 --- /dev/null +++ b/src/parser/hir/path.rs @@ -0,0 +1,10 @@ +use crate::parser::{hir::Expression, Operator, Spanned}; +use derive_new::new; +use getset::Getters; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, new)] +#[get = "crate"] +pub struct Path { + head: Expression, + tail: Vec>, +} diff --git a/src/parser/parse2.rs b/src/parser/parse2.rs index 9d2e522c98..b841c4e5fa 100644 --- a/src/parser/parse2.rs +++ b/src/parser/parse2.rs @@ -1,9 +1,14 @@ +crate mod call_node; +crate mod files; crate mod flag; crate mod operator; crate mod parser; crate mod span; +crate mod text; crate mod token_tree; crate mod token_tree_builder; crate mod tokens; crate mod unit; crate mod util; + +crate use token_tree::{PipelineElement, TokenNode}; diff --git a/src/parser/parse2/call_node.rs b/src/parser/parse2/call_node.rs new file mode 100644 index 0000000000..eab5c8f328 --- /dev/null +++ b/src/parser/parse2/call_node.rs @@ -0,0 +1,26 @@ +use crate::parser::TokenNode; +use getset::Getters; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)] +pub struct CallNode { + #[get = "crate"] + head: Box, + #[get = "crate"] + children: Option>, +} + +impl CallNode { + pub fn new(head: Box, children: Vec) -> CallNode { + if children.len() == 0 { + CallNode { + head, + children: None, + } + } else { + CallNode { + head, + children: Some(children), + } + } + } +} diff --git a/src/parser/parse2/files.rs b/src/parser/parse2/files.rs new file mode 100644 index 0000000000..c606055f6c --- /dev/null +++ b/src/parser/parse2/files.rs @@ -0,0 +1,77 @@ +use crate::parser::parse2::span::Span; +use derive_new::new; +use language_reporting::{FileName, Location}; + +#[derive(new, Debug, Clone)] +pub struct Files { + snippet: String, +} + +impl language_reporting::ReportingFiles for Files { + type Span = Span; + type FileId = usize; + + fn byte_span( + &self, + _file: Self::FileId, + from_index: usize, + to_index: usize, + ) -> Option { + Some(Span::from((from_index, to_index))) + } + fn file_id(&self, _span: Self::Span) -> Self::FileId { + 0 + } + fn file_name(&self, _file: Self::FileId) -> FileName { + FileName::Verbatim(format!("")) + } + fn byte_index(&self, _file: Self::FileId, _line: usize, _column: usize) -> Option { + unimplemented!("byte_index") + } + fn location(&self, _file: Self::FileId, byte_index: usize) -> Option { + let source = &self.snippet; + let mut seen_lines = 0; + let mut seen_bytes = 0; + + for (pos, _) in source.match_indices('\n') { + if pos > byte_index { + return Some(language_reporting::Location::new( + seen_lines, + byte_index - seen_bytes, + )); + } else { + seen_lines += 1; + seen_bytes = pos; + } + } + + if seen_lines == 0 { + Some(language_reporting::Location::new(0, byte_index)) + } else { + None + } + } + fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option { + let source = &self.snippet; + let mut seen_lines = 0; + let mut seen_bytes = 0; + + for (pos, _) in source.match_indices('\n') { + if seen_lines == lineno { + return Some(Span::from((seen_bytes, pos))); + } else { + seen_lines += 1; + seen_bytes = pos + 1; + } + } + + if seen_lines == 0 { + Some(Span::from((0, self.snippet.len() - 1))) + } else { + None + } + } + fn source(&self, span: Self::Span) -> Option { + Some(self.snippet[span.start..span.end].to_string()) + } +} diff --git a/src/parser/parse2/flag.rs b/src/parser/parse2/flag.rs index 0d255e4515..3b587c3354 100644 --- a/src/parser/parse2/flag.rs +++ b/src/parser/parse2/flag.rs @@ -1,7 +1,17 @@ +use crate::parser::Span; +use derive_new::new; +use getset::Getters; use serde_derive::{Deserialize, Serialize}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] -pub enum Flag { +pub enum FlagKind { Shorthand, Longhand, } + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)] +#[get = "crate"] +pub struct Flag { + kind: FlagKind, + name: Span, +} diff --git a/src/parser/parse2/operator.rs b/src/parser/parse2/operator.rs index 89275afd74..c5ce66bfd8 100644 --- a/src/parser/parse2/operator.rs +++ b/src/parser/parse2/operator.rs @@ -12,6 +12,7 @@ pub enum Operator { } impl Operator { + #[allow(unused)] pub fn print(&self) -> String { self.as_str().to_string() } diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 5a2f6cc130..4a9a134338 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -1,28 +1,43 @@ #![allow(unused)] use crate::parser::parse2::{ - flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, unit::*, + call_node::*, flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, + unit::*, }; use nom; +use nom::branch::*; +use nom::bytes::complete::*; +use nom::character::complete::*; +use nom::combinator::*; +use nom::multi::*; +use nom::sequence::*; + +use log::trace; use nom::dbg; -use nom::types::CompleteStr; use nom::*; +use nom::{AsBytes, FindSubstring, IResult, InputLength, InputTake, Slice}; use nom_locate::{position, LocatedSpan}; +use std::fmt::Debug; use std::str::FromStr; -type NomSpan<'a> = LocatedSpan>; +pub type NomSpan<'a> = LocatedSpan<&'a str>; + +pub fn nom_input(s: &'a str) -> NomSpan<'a> { + LocatedSpan::new(s) +} macro_rules! operator { ($name:tt : $token:tt ) => { - named!($name( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> t: tag!(stringify!($token)) - >> r: position!() - >> (TokenTreeBuilder::spanned_op(t.fragment.0, (l, r))) - // >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) - ) - ); + pub fn $name(input: NomSpan) -> IResult { + let start = input.offset; + let (input, tag) = tag(stringify!($token))(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_op(tag.fragment, (start, end)), + )) + } }; } @@ -33,195 +48,423 @@ operator! { lte: <= } operator! { eq: == } operator! { neq: != } -named!(pub raw_integer( NomSpan ) -> Spanned, - do_parse!( - l: position!() - >> neg: opt!(tag!("-")) - >> num: digit1 - >> r: position!() - >> (Spanned::from_nom(int(num.fragment.0, neg), l, r)) +fn trace_step<'a, T: Debug>( + input: NomSpan<'a>, + name: &str, + block: impl FnOnce(NomSpan<'a>) -> IResult, T>, +) -> IResult, T> { + trace!("+ before {} @ {:?}", name, input); + match block(input) { + Ok((input, result)) => { + trace!("after {} @ {:?} -> {:?}", name, input, result); + Ok((input, result)) + } + + Err(e) => { + trace!("- failed {} :: {:?}", name, e); + Err(e) + } + } +} + +pub fn raw_integer(input: NomSpan) -> IResult> { + let start = input.offset; + trace_step(input, "raw_integer", move |input| { + let (input, neg) = opt(tag("-"))(input)?; + println!("(input,neg) = {:?} {:?}", input, neg); + let (input, num) = digit1(input)?; + println!("(input,num) = {:?} {:?}", input, num); + let end = input.offset; + + Ok(( + input, + Spanned::from_item(int(num.fragment, neg), (start, end)), + )) + }) +} + +pub fn integer(input: NomSpan) -> IResult { + trace_step(input, "integer", move |input| { + let (input, int) = raw_integer(input)?; + + Ok((input, TokenTreeBuilder::spanned_int(*int, int.span))) + }) +} + +pub fn operator(input: NomSpan) -> IResult { + trace_step(input, "operator", |input| { + let (input, operator) = alt((gte, lte, neq, gt, lt, eq))(input)?; + + Ok((input, operator)) + }) +} + +pub fn dq_string(input: NomSpan) -> IResult { + trace_step(input, "dq_string", |input| { + let start = input.offset; + let (input, _) = char('"')(input)?; + let start1 = input.offset; + let (input, _) = many0(none_of("\""))(input)?; + let end1 = input.offset; + let (input, _) = char('"')(input)?; + let end = input.offset; + Ok(( + input, + TokenTreeBuilder::spanned_string((start1, end1), (start, end)), + )) + }) +} + +pub fn sq_string(input: NomSpan) -> IResult { + trace_step(input, "sq_string", move |input| { + let start = input.offset; + let (input, _) = char('\'')(input)?; + let start1 = input.offset; + let (input, _) = many0(none_of("\'"))(input)?; + let end1 = input.offset; + let (input, _) = char('\'')(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_string((start1, end1), (start, end)), + )) + }) +} + +pub fn string(input: NomSpan) -> IResult { + trace_step(input, "string", move |input| { + alt((sq_string, dq_string))(input) + }) +} + +pub fn bare(input: NomSpan) -> IResult { + trace_step(input, "bare", move |input| { + let start = input.offset; + let (input, _) = take_while1(is_start_bare_char)(input)?; + let (input, _) = take_while(is_bare_char)(input)?; + let end = input.offset; + + Ok((input, TokenTreeBuilder::spanned_bare((start, end)))) + }) +} + +pub fn var(input: NomSpan) -> IResult { + trace_step(input, "var", move |input| { + let start = input.offset; + let (input, _) = tag("$")(input)?; + let (input, bare) = identifier(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_var(bare.span(), (start, end)), + )) + }) +} + +// let start = input.offset; +// let (input, _) = take_while1(is_start_bare_char)(input)?; +// let (input, _) = take_while(is_bare_char)(input)?; +// let end = input.offset; + +// Ok((input, TokenTreeBuilder::spanned_bare((start, end)))) + +pub fn identifier(input: NomSpan) -> IResult { + trace_step(input, "identifier", move |input| { + let start = input.offset; + let (input, _) = take_while1(is_id_start)(input)?; + let (input, _) = take_while(is_id_continue)(input)?; + + let end = input.offset; + + Ok((input, TokenTreeBuilder::spanned_ident((start, end)))) + }) +} + +pub fn flag(input: NomSpan) -> IResult { + trace_step(input, "flag", move |input| { + let start = input.offset; + let (input, _) = tag("--")(input)?; + let (input, bare) = bare(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_flag(bare.span(), (start, end)), + )) + }) +} + +pub fn shorthand(input: NomSpan) -> IResult { + trace_step(input, "shorthand", move |input| { + let start = input.offset; + let (input, _) = tag("-")(input)?; + let (input, bare) = bare(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_shorthand(bare.span(), (start, end)), + )) + }) +} + +pub fn raw_unit(input: NomSpan) -> IResult> { + trace_step(input, "raw_unit", move |input| { + let start = input.offset; + let (input, unit) = alt(( + tag("B"), + tag("KB"), + tag("MB"), + tag("GB"), + tag("TB"), + tag("PB"), + ))(input)?; + let end = input.offset; + + Ok(( + input, + Spanned::from_item(Unit::from(unit.fragment), (start, end)), + )) + }) +} + +pub fn size(input: NomSpan) -> IResult { + trace_step(input, "size", move |input| { + let start = input.offset; + let (input, int) = raw_integer(input)?; + let (input, unit) = raw_unit(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_size((*int, *unit), (start, end)), + )) + }) +} + +pub fn leaf(input: NomSpan) -> IResult { + trace_step(input, "leaf", move |input| { + println!("alt: {:?}", input); + let (input, node) = + alt((size, integer, string, operator, flag, shorthand, var, bare))(input)?; + + Ok((input, node)) + }) +} + +pub fn token_list(input: NomSpan) -> IResult> { + trace_step(input, "token_list", move |input| { + let (input, first) = node(input)?; + let (input, list) = many0(pair(space1, node))(input)?; + + Ok((input, make_token_list(None, first, list, None))) + }) +} + +pub fn spaced_token_list(input: NomSpan) -> IResult> { + trace_step(input, "spaced_token_list", move |input| { + let (input, sp_left) = opt(space1)(input)?; + let (input, first) = node(input)?; + let (input, list) = many0(pair(space1, node))(input)?; + let (input, sp_right) = opt(space1)(input)?; + + Ok((input, make_token_list(sp_left, first, list, sp_right))) + }) +} + +fn make_token_list( + sp_left: Option, + first: TokenNode, + list: Vec<(NomSpan, TokenNode)>, + sp_right: Option, +) -> Vec { + let mut nodes = vec![]; + + if let Some(sp_left) = sp_left { + nodes.push(TokenNode::Whitespace(Span::from(sp_left))); + } + + nodes.push(first); + + for (ws, token) in list { + nodes.push(TokenNode::Whitespace(Span::from(ws))); + nodes.push(token); + } + + if let Some(sp_right) = sp_right { + nodes.push(TokenNode::Whitespace(Span::from(sp_right))); + } + + nodes +} + +pub fn whitespace(input: NomSpan) -> IResult { + trace_step(input, "whitespace", move |input| { + let left = input.offset; + let (input, ws1) = space1(input)?; + let right = input.offset; + + Ok((input, TokenTreeBuilder::spanned_ws((left, right)))) + }) +} + +pub fn delimited_paren(input: NomSpan) -> IResult { + trace_step(input, "delimited_paren", move |input| { + let left = input.offset; + let (input, _) = char('(')(input)?; + let (input, ws1) = opt(whitespace)(input)?; + let (input, inner_items) = opt(token_list)(input)?; + let (input, ws2) = opt(whitespace)(input)?; + let (input, _) = char(')')(input)?; + let right = input.offset; + + let mut items = vec![]; + + if let Some(space) = ws1 { + items.push(space); + } + + if let Some(inner_items) = inner_items { + items.extend(inner_items); + } + + if let Some(space) = ws2 { + items.push(space); + } + + Ok(( + input, + TokenTreeBuilder::spanned_parens(items, (left, right)), + )) + }) +} + +pub fn delimited_square(input: NomSpan) -> IResult { + trace_step(input, "delimited_paren", move |input| { + let left = input.offset; + let (input, _) = char('[')(input)?; + let (input, ws1) = opt(whitespace)(input)?; + let (input, inner_items) = opt(token_list)(input)?; + let (input, ws2) = opt(whitespace)(input)?; + let (input, _) = char(']')(input)?; + let right = input.offset; + + let mut items = vec![]; + + if let Some(space) = ws1 { + items.push(space); + } + + if let Some(inner_items) = inner_items { + items.extend(inner_items); + } + + if let Some(space) = ws2 { + items.push(space); + } + + Ok(( + input, + TokenTreeBuilder::spanned_square(items, (left, right)), + )) + }) +} + +pub fn delimited_brace(input: NomSpan) -> IResult { + trace_step(input, "delimited_brace", move |input| { + let left = input.offset; + let (input, _) = char('{')(input)?; + let (input, _) = opt(space1)(input)?; + let (input, items) = opt(token_list)(input)?; + let (input, _) = opt(space1)(input)?; + let (input, _) = char('}')(input)?; + let right = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_brace(items.unwrap_or_else(|| vec![]), (left, right)), + )) + }) +} + +pub fn raw_call(input: NomSpan) -> IResult> { + trace_step(input, "raw_call", move |input| { + let left = input.offset; + let (input, items) = token_list(input)?; + let right = input.offset; + + Ok((input, TokenTreeBuilder::spanned_call(items, (left, right)))) + }) +} + +pub fn path(input: NomSpan) -> IResult { + trace_step(input, "path", move |input| { + let left = input.offset; + let (input, head) = node1(input)?; + let (input, _) = tag(".")(input)?; + let (input, tail) = separated_list(tag("."), alt((identifier, string)))(input)?; + let right = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_path((head, tail), (left, right)), + )) + }) +} + +pub fn node1(input: NomSpan) -> IResult { + trace_step(input, "node1", alt((leaf, delimited_paren))) +} + +pub fn node(input: NomSpan) -> IResult { + trace_step( + input, + "node", + alt(( + path, + leaf, + delimited_paren, + delimited_brace, + delimited_square, + )), ) -); +} -named!(pub integer( NomSpan ) -> TokenNode, - do_parse!( - int: raw_integer - >> (TokenTreeBuilder::spanned_int(*int, int.span)) - ) -); +pub fn pipeline(input: NomSpan) -> IResult { + trace_step(input, "pipeline", |input| { + let start = input.offset; + let (input, head) = tuple((raw_call, opt(space1)))(input)?; + let (input, items) = trace_step( + input, + "many0", + many0(tuple((tag("|"), opt(space1), raw_call, opt(space1)))), + )?; + let end = input.offset; -named!(pub operator( NomSpan ) -> TokenNode, - alt!( - gte | lte | neq | gt | lt | eq - ) -); + Ok(( + input, + TokenTreeBuilder::spanned_pipeline(make_call_list(head, items), (start, end)), + )) + }) +} -named!(pub dq_string( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> char!('"') - >> l1: position!() - >> many0!(none_of!("\"")) - >> r1: position!() - >> char!('"') - >> r: position!() - >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) - ) -); +fn make_call_list( + head: (Spanned, Option), + tail: Vec<(NomSpan, Option, Spanned, Option)>, +) -> Vec { + let mut out = vec![]; + let el = PipelineElement::new(None, head.0, head.1.map(Span::from)); + out.push(el); -named!(pub sq_string( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> char!('\'') - >> l1: position!() - >> many0!(none_of!("'")) - >> r1: position!() - >> char!('\'') - >> r: position!() - >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) - ) -); + for (pipe, ws1, call, ws2) in tail { + let el = PipelineElement::new(ws1.map(Span::from), call, ws2.map(Span::from)); + out.push(el); + } -named!(pub string( NomSpan ) -> TokenNode, - alt!(sq_string | dq_string) -); - -named!(pub bare( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> take_while1!(is_start_bare_char) - >> take_while!(is_bare_char) - >> r: position!() - >> (TokenTreeBuilder::spanned_bare((l, r))) - ) -); - -named!(pub var( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> tag!("$") - >> bare: identifier - >> r: position!() - >> (TokenTreeBuilder::spanned_var(bare.span(), (l, r))) - ) -); - -named!(pub identifier( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> take_while1!(is_id_start) - >> take_while!(is_id_continue) - >> r: position!() - >> (TokenTreeBuilder::spanned_ident((l, r))) - ) -); - -named!(pub flag( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> tag!("--") - >> bare: bare - >> r: position!() - >> (TokenTreeBuilder::spanned_flag(bare.span(), (l, r))) - ) -); - -named!(pub shorthand( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> tag!("-") - >> bare: bare - >> r: position!() - >> (TokenTreeBuilder::spanned_shorthand(bare.span(), (l, r))) - ) -); - -named!(pub raw_unit( NomSpan ) -> Spanned, - do_parse!( - l: position!() - >> unit: alt!(tag!("B") | tag!("KB") | tag!("MB") | tag!("GB") | tag!("TB") | tag!("PB")) - >> r: position!() - >> (Spanned::from_nom(Unit::from(unit.fragment.0), l, r)) - ) -); - -named!(pub size( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> int: raw_integer - >> unit: raw_unit - >> r: position!() - >> (TokenTreeBuilder::spanned_size((*int, *unit), (l, r))) - ) -); - -named!(pub leaf( NomSpan ) -> TokenNode, - alt!(size | integer | string | operator | flag | shorthand | var | bare) -); - -named!(pub delimited_paren( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> items: delimited!( - char!('('), - delimited!(space0, separated_list!(space1, node), space0), - char!(')') - ) - >> r: position!() - >> (TokenTreeBuilder::spanned_parens(items, (l, r))) - ) -); - -named!(pub delimited_brace( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> items: delimited!( - char!('{'), - delimited!(space0, separated_list!(space1, node), space0), - char!('}') - ) - >> r: position!() - >> (TokenTreeBuilder::spanned_brace(items, (l, r))) - ) -); - -named!(pub raw_call( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> head: node - >> items: opt!(preceded!(space0, separated_nonempty_list!(space1, node))) - >> r: position!() - >> (TokenTreeBuilder::spanned_call((head, items), (l, r))) - ) -); - -named!(pub path( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> head: node1 - >> tag!(".") - >> tail: separated_list!(tag!("."), alt!(identifier | string)) - >> r: position!() - >> (TokenTreeBuilder::spanned_path((head, tail), (l, r))) - ) -); - -named!(pub node1( NomSpan ) -> TokenNode, - alt!(leaf | delimited_paren) -); - -named!(pub node( NomSpan ) -> TokenNode, - alt!(path | leaf | delimited_paren | delimited_brace) -); - -named!(pub pipeline( NomSpan ) -> TokenNode, - do_parse!( - l: position!() - >> list: separated_list!(delimited!(space0, tag!("|"), space0), raw_call) - >> r: position!() - >> (TokenTreeBuilder::spanned_pipeline(list, (l, r))) - ) -); + out +} fn int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -276,30 +519,29 @@ mod tests { use super::*; use crate::parser::parse2::token_tree_builder::TokenTreeBuilder as b; use crate::parser::parse2::token_tree_builder::{CurriedToken, TokenTreeBuilder}; - use nom_trace::{print_trace, reset_trace}; use pretty_assertions::assert_eq; macro_rules! assert_leaf { (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt $parens:tt } ) => { $( assert_eq!( - apply($name, $input), + apply($name, stringify!($name), $input), token(RawToken::$kind $parens, $left, $right) ); )* assert_eq!( - apply(leaf, $input), + apply(leaf, "leaf", $input), token(RawToken::$kind $parens, $left, $right) ); assert_eq!( - apply(leaf, $input), + apply(leaf, "leaf", $input), token(RawToken::$kind $parens, $left, $right) ); assert_eq!( - apply(node, $input), + apply(node, "node", $input), token(RawToken::$kind $parens, $left, $right) ); }; @@ -307,7 +549,7 @@ mod tests { (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt } ) => { $( assert_eq!( - apply($name, $input), + apply($name, stringify!($name), $input), token(RawToken::$kind, $left, $right) ); )* @@ -342,35 +584,32 @@ mod tests { #[test] fn test_operator() { - assert_leaf! { - parsers [ operator ] - ">" -> 0..1 { Operator(Operator::GreaterThan) } - } + assert_eq!(apply(node, "node", ">"), build_token(b::op(">"))); - assert_leaf! { - parsers [ operator ] - ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } - } + // assert_leaf! { + // parsers [ operator ] + // ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } + // } - assert_leaf! { - parsers [ operator ] - "<" -> 0..1 { Operator(Operator::LessThan) } - } + // assert_leaf! { + // parsers [ operator ] + // "<" -> 0..1 { Operator(Operator::LessThan) } + // } - assert_leaf! { - parsers [ operator ] - "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } - } + // assert_leaf! { + // parsers [ operator ] + // "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } + // } - assert_leaf! { - parsers [ operator ] - "==" -> 0..2 { Operator(Operator::Equal) } - } + // assert_leaf! { + // parsers [ operator ] + // "==" -> 0..2 { Operator(Operator::Equal) } + // } - assert_leaf! { - parsers [ operator ] - "!=" -> 0..2 { Operator(Operator::NotEqual) } - } + // assert_leaf! { + // parsers [ operator ] + // "!=" -> 0..2 { Operator(Operator::NotEqual) } + // } } #[test] @@ -411,23 +650,23 @@ mod tests { #[test] fn test_flag() { - assert_leaf! { - parsers [ flag ] - "--hello" -> 0..7 { Flag(Flag::Longhand, span(2, 7)) } - } + // assert_leaf! { + // parsers [ flag ] + // "--hello" -> 0..7 { Flag(Spanned::from_item(FlagKind::Longhand, span(2, 7))) } + // } - assert_leaf! { - parsers [ flag ] - "--hello-world" -> 0..13 { Flag(Flag::Longhand, span(2, 13)) } - } + // assert_leaf! { + // parsers [ flag ] + // "--hello-world" -> 0..13 { Flag(Spanned::from_item(FlagKind::Longhand, span(2, 13))) } + // } } #[test] fn test_shorthand() { - assert_leaf! { - parsers [ shorthand ] - "-alt" -> 0..4 { Flag(Flag::Shorthand, span(1, 4)) } - } + // assert_leaf! { + // parsers [ shorthand ] + // "-alt" -> 0..4 { Flag(Spanned::from_item(FlagKind::Shorthand, span(1, 4))) } + // } } #[test] @@ -444,17 +683,20 @@ mod tests { } #[test] - fn test_delimited() { - assert_eq!(apply(node, "(abc)"), build(b::parens(vec![b::bare("abc")]))); - + fn test_delimited_paren() { assert_eq!( - apply(node, "( abc )"), - build(b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) + apply(node, "node", "(abc)"), + build_token(b::parens(vec![b::bare("abc")])) ); assert_eq!( - apply(node, "( abc def )"), - build(b::parens(vec![ + apply(node, "node", "( abc )"), + build_token(b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) + ); + + assert_eq!( + apply(node, "node", "( abc def )"), + build_token(b::parens(vec![ b::ws(" "), b::bare("abc"), b::sp(), @@ -464,8 +706,47 @@ mod tests { ); assert_eq!( - apply(node, "( abc def 123 456GB )"), - build(b::parens(vec![ + apply(node, "node", "( abc def 123 456GB )"), + build_token(b::parens(vec![ + b::ws(" "), + b::bare("abc"), + b::sp(), + b::bare("def"), + b::sp(), + b::int(123), + b::sp(), + b::size(456, "GB"), + b::sp() + ])) + ); + } + + #[test] + fn test_delimited_square() { + assert_eq!( + apply(node, "node", "[abc]"), + build_token(b::square(vec![b::bare("abc")])) + ); + + assert_eq!( + apply(node, "node", "[ abc ]"), + build_token(b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) + ); + + assert_eq!( + apply(node, "node", "[ abc def ]"), + build_token(b::square(vec![ + b::ws(" "), + b::bare("abc"), + b::sp(), + b::bare("def"), + b::sp() + ])) + ); + + assert_eq!( + apply(node, "node", "[ abc def 123 456GB ]"), + build_token(b::square(vec![ b::ws(" "), b::bare("abc"), b::sp(), @@ -481,30 +762,31 @@ mod tests { #[test] fn test_path() { + let _ = pretty_env_logger::try_init(); assert_eq!( - apply(node, "$it.print"), - build(b::path(b::var("it"), vec![b::ident("print")])) + apply(node, "node", "$it.print"), + build_token(b::path(b::var("it"), vec![b::ident("print")])) ); assert_eq!( - apply(node, "$head.part1.part2"), - build(b::path( + apply(node, "node", "$head.part1.part2"), + build_token(b::path( b::var("head"), vec![b::ident("part1"), b::ident("part2")] )) ); assert_eq!( - apply(node, "( hello ).world"), - build(b::path( + apply(node, "node", "( hello ).world"), + build_token(b::path( b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), vec![b::ident("world")] )) ); assert_eq!( - apply(node, "( hello ).\"world\""), - build(b::path( + apply(node, "node", "( hello ).\"world\""), + build_token(b::path( b::parens(vec![b::sp(), b::bare("hello"), b::sp()],), vec![b::string("world")] )) @@ -514,8 +796,12 @@ mod tests { #[test] fn test_nested_path() { assert_eq!( - apply(node, "( $it.is.\"great news\".right yep $yep ).\"world\""), - build(b::path( + apply( + node, + "node", + "( $it.is.\"great news\".right yep $yep ).\"world\"" + ), + build_token(b::path( b::parens(vec![ b::sp(), b::path( @@ -536,7 +822,7 @@ mod tests { #[test] fn test_smoke_single_command() { assert_eq!( - apply(raw_call, "git add ."), + apply(raw_call, "raw_call", "git add ."), build(b::call( b::bare("git"), vec![b::sp(), b::bare("add"), b::sp(), b::bare(".")] @@ -544,7 +830,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "open Cargo.toml"), + apply(raw_call, "raw_call", "open Cargo.toml"), build(b::call( b::bare("open"), vec![b::sp(), b::bare("Cargo.toml")] @@ -552,7 +838,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "select package.version"), + apply(raw_call, "raw_call", "select package.version"), build(b::call( b::bare("select"), vec![b::sp(), b::bare("package.version")] @@ -560,12 +846,12 @@ mod tests { ); assert_eq!( - apply(raw_call, "echo $it"), + apply(raw_call, "raw_call", "echo $it"), build(b::call(b::bare("echo"), vec![b::sp(), b::var("it")])) ); assert_eq!( - apply(raw_call, "open Cargo.toml --raw"), + apply(raw_call, "raw_call", "open Cargo.toml --raw"), build(b::call( b::bare("open"), vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::flag("raw")] @@ -573,7 +859,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "open Cargo.toml -r"), + apply(raw_call, "raw_call", "open Cargo.toml -r"), build(b::call( b::bare("open"), vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::shorthand("r")] @@ -581,7 +867,7 @@ mod tests { ); assert_eq!( - apply(raw_call, "config --set tabs 2"), + apply(raw_call, "raw_call", "config --set tabs 2"), build(b::call( b::bare("config"), vec![ @@ -598,57 +884,81 @@ mod tests { #[test] fn test_smoke_pipeline() { + let _ = pretty_env_logger::try_init(); + assert_eq!( apply( pipeline, + "pipeline", r#"git branch --merged | split-row "`n" | where $it != "* master""# ), - build(b::pipeline(vec![ - b::call( - b::bare("git"), - vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + build_token(b::pipeline(vec![ + ( + None, + b::call( + b::bare("git"), + vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + ), + Some(" ") ), - b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), - b::call( - b::bare("where"), - vec![ - b::sp(), - b::var("it"), - b::sp(), - b::op("!="), - b::sp(), - b::string("* master") - ] + ( + Some(" "), + b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), + Some(" ") + ), + ( + Some(" "), + b::call( + b::bare("where"), + vec![ + b::sp(), + b::var("it"), + b::sp(), + b::op("!="), + b::sp(), + b::string("* master") + ] + ), + None ) ])) ); assert_eq!( - apply(pipeline, "ls | where { $it.size > 100 }"), - build(b::pipeline(vec![ - b::call(b::bare("ls"), vec![]), - b::call( - b::bare("where"), - vec![ - b::sp(), - b::braced(vec![ - b::path(b::var("it"), vec![b::ident("size")]), + apply(pipeline, "pipeline", "ls | where { $it.size > 100 }"), + build_token(b::pipeline(vec![ + (None, b::call(b::bare("ls"), vec![]), Some(" ")), + ( + Some(" "), + b::call( + b::bare("where"), + vec![ b::sp(), - b::op(">"), - b::sp(), - b::int(100) - ]) - ] + b::braced(vec![ + b::path(b::var("it"), vec![b::ident("size")]), + b::sp(), + b::op(">"), + b::sp(), + b::int(100) + ]) + ] + ), + None ) ])) ) } - fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { - match f(NomSpan::new(CompleteStr(string))) { + fn apply( + f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err<(NomSpan, nom::error::ErrorKind)>>, + desc: &str, + string: &str, + ) -> T { + match f(NomSpan::new(string)) { Ok(v) => v.1, Err(other) => { println!("{:?}", other); + println!("for {} @ {}", string, desc); panic!("No dice"); } } @@ -686,8 +996,13 @@ mod tests { TokenNode::Token(Spanned::from_item(token, (left, right))) } - fn build(block: CurriedToken) -> TokenNode { + fn build(block: CurriedNode) -> T { let mut builder = TokenTreeBuilder::new(); - block(&mut builder).expect("Expected to build into a token") + block(&mut builder) + } + + fn build_token(block: CurriedToken) -> TokenNode { + let mut builder = TokenTreeBuilder::new(); + block(&mut builder) } } diff --git a/src/parser/parse2/span.rs b/src/parser/parse2/span.rs index 2d3cbe7b5e..4214d29aaa 100644 --- a/src/parser/parse2/span.rs +++ b/src/parser/parse2/span.rs @@ -1,7 +1,8 @@ use derive_new::new; -use std::ops::Range; +use getset::Getters; -#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Getters)] +#[get = "crate"] pub struct Spanned { crate span: Span, crate item: T, @@ -16,20 +17,6 @@ impl std::ops::Deref for Spanned { } impl Spanned { - crate fn from_nom( - item: T, - start: nom_locate::LocatedSpan, - end: nom_locate::LocatedSpan, - ) -> Spanned { - let start = start.offset; - let end = end.offset; - - Spanned { - span: Span::from((start, end)), - item, - } - } - crate fn from_item(item: T, span: impl Into) -> Spanned { Spanned { span: span.into(), @@ -43,6 +30,19 @@ impl Spanned { let mapped = input(item); Spanned { span, item: mapped } } + + crate fn copy_span(&self, output: U) -> Spanned { + let Spanned { span, item } = self; + + Spanned { + span: *span, + item: output, + } + } + + pub fn source(&self, source: &'source str) -> &'source str { + self.span().slice(source) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] @@ -52,6 +52,21 @@ pub struct Span { // source: &'source str, } +impl From<&Span> for Span { + fn from(input: &Span) -> Span { + *input + } +} + +impl From> for Span { + fn from(input: nom_locate::LocatedSpan<&str>) -> Span { + Span { + start: input.offset, + end: input.offset + input.fragment.len(), + } + } +} + impl From<(nom_locate::LocatedSpan, nom_locate::LocatedSpan)> for Span { fn from(input: (nom_locate::LocatedSpan, nom_locate::LocatedSpan)) -> Span { Span { @@ -80,12 +95,8 @@ impl From<&std::ops::Range> for Span { } impl Span { - fn new(range: &Range) -> Span { - Span { - start: range.start, - end: range.end, - // source, - } + pub fn slice(&self, source: &'source str) -> &'source str { + &source[self.start..self.end] } } diff --git a/src/parser/parse2/text.rs b/src/parser/parse2/text.rs new file mode 100644 index 0000000000..7246392283 --- /dev/null +++ b/src/parser/parse2/text.rs @@ -0,0 +1,204 @@ +use std::cmp::Ordering; +use std::hash::Hash; +use std::hash::Hasher; +use std::ops::Range; +use std::sync::Arc; + +/// A "Text" is like a string except that it can be cheaply cloned. +/// You can also "extract" subtexts quite cheaply. You can also deref +/// an `&Text` into a `&str` for interoperability. +/// +/// Used to represent the value of an input file. +#[derive(Clone)] +pub struct Text { + text: Arc, + start: usize, + end: usize, +} + +impl Text { + /// Modifies this restrict to a subset of its current range. + pub fn select(&mut self, range: Range) { + let len = range.end - range.start; + let new_start = self.start + range.start; + let new_end = new_start + len; + assert!(new_end <= self.end); + + self.start = new_start; + self.end = new_end; + } + + /// Extract a new `Text` that is a subset of an old `Text` + /// -- `text.extract(1..3)` is similar to `&foo[1..3]` except that + /// it gives back an owned value instead of a borrowed value. + pub fn extract(&self, range: Range) -> Self { + let mut result = self.clone(); + result.select(range); + result + } +} + +impl From> for Text { + fn from(text: Arc) -> Self { + let end = text.len(); + Self { + text, + start: 0, + end, + } + } +} + +impl AsRef for Text { + fn as_ref(&self) -> &str { + &*self + } +} + +impl From for Text { + fn from(text: String) -> Self { + Text::from(Arc::new(text)) + } +} + +impl From<&String> for Text { + fn from(text: &String) -> Self { + Text::from(text.to_string()) + } +} + +impl From<&str> for Text { + fn from(text: &str) -> Self { + Text::from(text.to_string()) + } +} + +impl std::borrow::Borrow for Text { + fn borrow(&self) -> &str { + &*self + } +} + +impl std::ops::Deref for Text { + type Target = str; + + fn deref(&self) -> &str { + &self.text[self.start..self.end] + } +} + +impl std::fmt::Display for Text { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + ::fmt(self, fmt) + } +} + +impl std::fmt::Debug for Text { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + ::fmt(self, fmt) + } +} + +impl PartialEq for Text { + fn eq(&self, other: &Text) -> bool { + let this: &str = self; + let other: &str = other; + this == other + } +} + +impl Eq for Text {} + +impl PartialEq for Text { + fn eq(&self, other: &str) -> bool { + let this: &str = self; + this == other + } +} + +impl PartialEq for Text { + fn eq(&self, other: &String) -> bool { + let this: &str = self; + let other: &str = other; + this == other + } +} + +impl PartialEq for str { + fn eq(&self, other: &Text) -> bool { + other == self + } +} + +impl PartialEq for String { + fn eq(&self, other: &Text) -> bool { + other == self + } +} + +impl PartialEq<&T> for Text +where + Text: PartialEq, +{ + fn eq(&self, other: &&T) -> bool { + self == *other + } +} + +impl Hash for Text { + fn hash(&self, state: &mut H) { + ::hash(self, state) + } +} + +impl PartialOrd for Text { + fn partial_cmp(&self, other: &Text) -> Option { + let this: &str = self; + let other: &str = other; + this.partial_cmp(other) + } +} + +impl Ord for Text { + fn cmp(&self, other: &Text) -> Ordering { + let this: &str = self; + let other: &str = other; + this.cmp(other) + } +} + +impl PartialOrd for Text { + fn partial_cmp(&self, other: &str) -> Option { + let this: &str = self; + this.partial_cmp(other) + } +} + +impl PartialOrd for Text { + fn partial_cmp(&self, other: &String) -> Option { + let this: &str = self; + let other: &str = other; + this.partial_cmp(other) + } +} + +impl PartialOrd for str { + fn partial_cmp(&self, other: &Text) -> Option { + other.partial_cmp(self) + } +} + +impl PartialOrd for String { + fn partial_cmp(&self, other: &Text) -> Option { + other.partial_cmp(self) + } +} + +impl PartialOrd<&T> for Text +where + Text: PartialOrd, +{ + fn partial_cmp(&self, other: &&T) -> Option { + self.partial_cmp(*other) + } +} diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index b3cd722016..9718e9def5 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,14 +1,22 @@ -use crate::parser::parse2::{operator::*, span::*, tokens::*}; +use crate::errors::ShellError; +use crate::parser::parse2::{call_node::*, flag::*, operator::*, span::*, tokens::*}; use derive_new::new; use enum_utils::FromStr; +use getset::Getters; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { Token(Token), + #[allow(unused)] Call(Spanned), Delimited(Spanned), - Pipeline(Spanned>), - Binary(Spanned), + Pipeline(Spanned>), + Operator(Spanned), + Flag(Spanned), + Identifier(Span), + Whitespace(Span), + #[allow(unused)] + Error(Spanned>), Path(Spanned), } @@ -19,10 +27,71 @@ impl TokenNode { TokenNode::Call(s) => s.span, TokenNode::Delimited(s) => s.span, TokenNode::Pipeline(s) => s.span, - TokenNode::Binary(s) => s.span, + TokenNode::Operator(s) => s.span, + TokenNode::Flag(s) => s.span, + TokenNode::Identifier(s) => *s, + TokenNode::Whitespace(s) => *s, + TokenNode::Error(s) => s.span, TokenNode::Path(s) => s.span, } } + + pub fn as_external_arg(&self, source: &str) -> String { + self.span().slice(source).to_string() + } + + pub fn source(&self, source: &'source str) -> &'source str { + self.span().slice(source) + } + + pub fn is_ws(&self) -> bool { + match self { + TokenNode::Whitespace(_) => true, + _ => false, + } + } + + pub fn is_bare(&self) -> bool { + match self { + TokenNode::Token(Spanned { + item: RawToken::Bare, + .. + }) => true, + _ => false, + } + } + + crate fn as_string(&self, source: &str) -> Option> { + match self { + TokenNode::Token(Spanned { + item: RawToken::Bare, + span, + }) => Some(Spanned::from_item(span.slice(source).to_string(), span)), + TokenNode::Token(Spanned { + item: RawToken::String(inner), + span, + }) => Some(Spanned::from_item(inner.slice(source).to_string(), span)), + _ => None, + } + } + + crate fn as_flag(&self, value: &str, source: &str) -> Option> { + match self { + TokenNode::Flag( + flag @ Spanned { + item: Flag { .. }, .. + }, + ) if value == flag.name().slice(source) => Some(*flag), + _ => None, + } + } + + pub fn as_pipeline(&self) -> Result, ShellError> { + match self { + TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()), + _ => Err(ShellError::string("unimplemented")), + } + } } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] @@ -31,12 +100,6 @@ pub struct DelimitedNode { children: Vec, } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] -pub struct CallNode { - head: Box, - children: Vec, -} - #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)] pub enum Delimiter { Paren, @@ -50,9 +113,26 @@ pub struct PathNode { tail: Vec, } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] -pub struct BinaryNode { - left: Box, - op: Operator, - right: Box, +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] +pub struct PipelineElement { + pre_ws: Option, + #[get = "crate"] + call: Spanned, + post_ws: Option, +} + +impl PipelineElement { + crate fn span(&self) -> Span { + let start = match self.pre_ws { + None => self.call.span.start, + Some(span) => span.start, + }; + + let end = match self.post_ws { + None => self.call.span.end, + Some(span) => span.end, + }; + + Span::from((start, end)) + } } diff --git a/src/parser/parse2/token_tree_builder.rs b/src/parser/parse2/token_tree_builder.rs index a3fffaa8d7..08740d7b32 100644 --- a/src/parser/parse2/token_tree_builder.rs +++ b/src/parser/parse2/token_tree_builder.rs @@ -1,11 +1,15 @@ -use crate::parser::parse2::flag::Flag; +#[allow(unused)] +use crate::prelude::*; + +use crate::parser::parse2::flag::{Flag, FlagKind}; use crate::parser::parse2::operator::Operator; use crate::parser::parse2::span::{Span, Spanned}; use crate::parser::parse2::token_tree::{ - BinaryNode, CallNode, DelimitedNode, Delimiter, PathNode, TokenNode, + DelimitedNode, Delimiter, PathNode, PipelineElement, TokenNode, }; use crate::parser::parse2::tokens::{RawToken, Token}; use crate::parser::parse2::unit::Unit; +use crate::parser::CallNode; use derive_new::new; #[derive(new)] @@ -14,7 +18,10 @@ pub struct TokenTreeBuilder { pos: usize, } -pub type CurriedToken = Box Option + 'static>; +#[allow(unused)] +pub type CurriedNode = Box T + 'static>; +pub type CurriedToken = Box TokenNode + 'static>; +pub type CurriedCall = Box Spanned + 'static>; #[allow(unused)] impl TokenTreeBuilder { @@ -23,35 +30,57 @@ impl TokenTreeBuilder { block(&mut builder) } - pub fn pipeline(input: Vec) -> CurriedToken { + pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken { + let input: Vec<(Option, CurriedCall, Option)> = input + .into_iter() + .map(|(pre, call, post)| { + ( + pre.map(|s| s.to_string()), + call, + post.map(|s| s.to_string()), + ) + }) + .collect(); + Box::new(move |b| { let start = b.pos; - let mut out = vec![]; + let mut out: Vec = vec![]; let mut input = input.into_iter(); - let first = input + let (pre, call, post) = input .next() .expect("A pipeline must contain at least one element"); - out.push(first(b).expect("The first element of a pipeline must not be whitespace")); + let pre_span = pre.map(|pre| b.consume(&pre)); + let call = call(b); + let post_span = post.map(|post| b.consume(&post)); + out.push(PipelineElement::new( + pre_span.map(Span::from), + call, + post_span.map(Span::from), + )); - for item in input { - b.consume(" | "); + for (pre, call, post) in input { + b.consume("|"); + let pre_span = pre.map(|pre| b.consume(&pre)); + let call = call(b); + let post_span = post.map(|post| b.consume(&post)); - match item(b) { - None => {} - Some(v) => out.push(v), - } + out.push(PipelineElement::new( + pre_span.map(Span::from), + call, + post_span.map(Span::from), + )); } let end = b.pos; - Some(TokenTreeBuilder::spanned_pipeline(out, (start, end))) + TokenTreeBuilder::spanned_pipeline(out, (start, end)) }) } - pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { + pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { TokenNode::Pipeline(Spanned::from_item(input, span)) } @@ -63,15 +92,12 @@ impl TokenTreeBuilder { b.pos = end; - Some(TokenTreeBuilder::spanned_op(input, (start, end))) + TokenTreeBuilder::spanned_op(input, (start, end)) }) } pub fn spanned_op(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item( - RawToken::Operator(input.into()), - span.into(), - )) + TokenNode::Operator(Spanned::from_item(input.into(), span.into())) } pub fn string(input: impl Into) -> CurriedToken { @@ -83,10 +109,7 @@ impl TokenTreeBuilder { let (_, end) = b.consume("\""); b.pos = end; - Some(TokenTreeBuilder::spanned_string( - (inner_start, inner_end), - (start, end), - )) + TokenTreeBuilder::spanned_string((inner_start, inner_end), (start, end)) }) } @@ -104,7 +127,7 @@ impl TokenTreeBuilder { let (start, end) = b.consume(&input); b.pos = end; - Some(TokenTreeBuilder::spanned_bare((start, end))) + TokenTreeBuilder::spanned_bare((start, end)) }) } @@ -119,7 +142,7 @@ impl TokenTreeBuilder { let (start, end) = b.consume(&int.to_string()); b.pos = end; - Some(TokenTreeBuilder::spanned_int(int, (start, end))) + TokenTreeBuilder::spanned_int(int, (start, end)) }) } @@ -136,7 +159,7 @@ impl TokenTreeBuilder { let (_, end) = b.consume(unit.as_str()); b.pos = end; - Some(TokenTreeBuilder::spanned_size((int, unit), (start, end))) + TokenTreeBuilder::spanned_size((int, unit), (start, end)) }) } @@ -152,22 +175,19 @@ impl TokenTreeBuilder { pub fn path(head: CurriedToken, tail: Vec) -> CurriedToken { Box::new(move |b| { let start = b.pos; - let head = head(b).expect("The head of a path must not be whitespace"); + let head = head(b); let mut output = vec![]; for item in tail { b.consume("."); - match item(b) { - None => {} - Some(v) => output.push(v), - }; + output.push(item(b)); } let end = b.pos; - Some(TokenTreeBuilder::spanned_path((head, output), (start, end))) + TokenTreeBuilder::spanned_path((head, output), (start, end)) }) } @@ -185,10 +205,7 @@ impl TokenTreeBuilder { let (start, _) = b.consume("$"); let (inner_start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_var( - (inner_start, end), - (start, end), - )) + TokenTreeBuilder::spanned_var((inner_start, end), (start, end)) }) } @@ -206,16 +223,13 @@ impl TokenTreeBuilder { let (start, _) = b.consume("--"); let (inner_start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_flag( - (inner_start, end), - (start, end), - )) + TokenTreeBuilder::spanned_flag((inner_start, end), (start, end)) }) } pub fn spanned_flag(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item( - RawToken::Flag(Flag::Longhand, input.into()), + TokenNode::Flag(Spanned::from_item( + Flag::new(FlagKind::Longhand, input.into()), span.into(), )) } @@ -227,16 +241,13 @@ impl TokenTreeBuilder { let (start, _) = b.consume("-"); let (inner_start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_shorthand( - (inner_start, end), - (start, end), - )) + TokenTreeBuilder::spanned_shorthand((inner_start, end), (start, end)) }) } pub fn spanned_shorthand(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item( - RawToken::Flag(Flag::Shorthand, input.into()), + TokenNode::Flag(Spanned::from_item( + Flag::new(FlagKind::Shorthand, input.into()), span.into(), )) } @@ -246,45 +257,42 @@ impl TokenTreeBuilder { Box::new(move |b| { let (start, end) = b.consume(&input); - Some(TokenTreeBuilder::spanned_ident((start, end))) + TokenTreeBuilder::spanned_ident((start, end)) }) } pub fn spanned_ident(span: impl Into) -> TokenNode { - TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into())) + TokenNode::Identifier(span.into()) } - pub fn call(head: CurriedToken, input: Vec) -> CurriedToken { + pub fn call(head: CurriedToken, input: Vec) -> CurriedCall { Box::new(move |b| { let start = b.pos; - let head_node = head(b).expect("The head of a command must not be whitespace"); + let head_node = head(b); - let mut tail_nodes = vec![]; + let mut nodes = vec![head_node]; for item in input { - match item(b) { - None => {} - Some(v) => tail_nodes.push(v), - }; + nodes.push(item(b)); } let end = b.pos; - Some(TokenTreeBuilder::spanned_call( - (head_node, Some(tail_nodes)), - (start, end), - )) + TokenTreeBuilder::spanned_call(nodes, (start, end)) }) } - pub fn spanned_call( - input: (TokenNode, Option>), - span: impl Into, - ) -> TokenNode { - TokenNode::Call(Spanned::from_item( - CallNode::new(Box::new(input.0), input.1.unwrap_or_else(|| vec![])), - span, - )) + pub fn spanned_call(input: Vec, span: impl Into) -> Spanned { + if input.len() == 0 { + panic!("BUG: spanned call (TODO)") + } + + let mut input = input.into_iter(); + + let head = input.next().unwrap(); + let tail = input.collect(); + + Spanned::from_item(CallNode::new(Box::new(head), tail), span) } pub fn parens(input: Vec) -> CurriedToken { @@ -292,15 +300,12 @@ impl TokenTreeBuilder { let (start, _) = b.consume("("); let mut output = vec![]; for item in input { - match item(b) { - None => {} - Some(v) => output.push(v), - }; + output.push(item(b)); } let (_, end) = b.consume(")"); - Some(TokenTreeBuilder::spanned_parens(output, (start, end))) + TokenTreeBuilder::spanned_parens(output, (start, end)) }) } @@ -311,20 +316,38 @@ impl TokenTreeBuilder { )) } + pub fn square(input: Vec) -> CurriedToken { + Box::new(move |b| { + let (start, _) = b.consume("["); + let mut output = vec![]; + for item in input { + output.push(item(b)); + } + + let (_, end) = b.consume("]"); + + TokenTreeBuilder::spanned_square(output, (start, end)) + }) + } + + pub fn spanned_square(input: impl Into>, span: impl Into) -> TokenNode { + TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Square, input.into()), + span, + )) + } + pub fn braced(input: Vec) -> CurriedToken { Box::new(move |b| { let (start, _) = b.consume("{ "); let mut output = vec![]; for item in input { - match item(b) { - None => {} - Some(v) => output.push(v), - }; + output.push(item(b)); } let (_, end) = b.consume(" }"); - Some(TokenTreeBuilder::spanned_brace(output, (start, end))) + TokenTreeBuilder::spanned_brace(output, (start, end)) }) } @@ -335,50 +358,10 @@ impl TokenTreeBuilder { )) } - pub fn binary( - left: CurriedToken, - op: impl Into, - right: CurriedToken, - ) -> CurriedToken { - let op = op.into(); - - Box::new(move |b| { - let start = b.pos; - - let left = left(b).expect("The left side of a binary operation must not be whitespace"); - - b.consume(" "); - - b.consume(op.as_str()); - - b.consume(" "); - - let right = - right(b).expect("The right side of a binary operation must not be whitespace"); - - let end = b.pos; - - Some(TokenTreeBuilder::spanned_binary( - (left, op, right), - (start, end), - )) - }) - } - - pub fn spanned_binary( - input: (impl Into, Operator, impl Into), - span: impl Into, - ) -> TokenNode { - TokenNode::Binary(Spanned::from_item( - BinaryNode::new(Box::new(input.0.into()), input.1, Box::new(input.2.into())), - span, - )) - } - pub fn sp() -> CurriedToken { Box::new(|b| { - b.consume(" "); - None + let (start, end) = b.consume(" "); + TokenNode::Whitespace(Span::from((start, end))) }) } @@ -386,11 +369,17 @@ impl TokenTreeBuilder { let input = input.into(); Box::new(move |b| { - b.consume(&input); - None + let (start, end) = b.consume(&input); + TokenTreeBuilder::spanned_ws((start, end)) }) } + pub fn spanned_ws(span: impl Into) -> TokenNode { + let span = span.into(); + + TokenNode::Whitespace(span.into()) + } + fn consume(&mut self, input: &str) -> (usize, usize) { let start = self.pos; self.pos += input.len(); diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs index d1977498ce..4e8fdcf5b1 100644 --- a/src/parser/parse2/tokens.rs +++ b/src/parser/parse2/tokens.rs @@ -1,5 +1,3 @@ -use crate::parser::parse2::flag::*; -use crate::parser::parse2::operator::*; use crate::parser::parse2::span::*; use crate::parser::parse2::unit::*; @@ -7,12 +5,34 @@ use crate::parser::parse2::unit::*; pub enum RawToken { Integer(i64), Size(i64, Unit), - Operator(Operator), String(Span), Variable(Span), - Identifier, Bare, - Flag(Flag, Span), } pub type Token = Spanned; + +impl Token { + pub fn to_semantic_token(&self) -> Option { + let semantic_token = match self.item { + RawToken::Integer(int) => RawSemanticToken::Integer(int), + RawToken::Size(int, unit) => RawSemanticToken::Size(int, unit), + RawToken::String(span) => RawSemanticToken::String(span), + RawToken::Variable(span) => RawSemanticToken::Variable(span), + RawToken::Bare => RawSemanticToken::Bare, + }; + + Some(Spanned::from_item(semantic_token, self.span)) + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RawSemanticToken { + Integer(i64), + Size(i64, Unit), + String(Span), + Variable(Span), + Bare, +} + +pub type SemanticToken = Spanned; diff --git a/src/parser/parse2/unit.rs b/src/parser/parse2/unit.rs index 883f4c5071..7e51103cfc 100644 --- a/src/parser/parse2/unit.rs +++ b/src/parser/parse2/unit.rs @@ -12,10 +12,6 @@ pub enum Unit { } impl Unit { - pub fn print(&self) -> String { - self.as_str().to_string() - } - pub fn as_str(&self) -> &str { match *self { Unit::B => "B", diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs new file mode 100644 index 0000000000..d3064cbae0 --- /dev/null +++ b/src/parser/parse_command.rs @@ -0,0 +1,250 @@ +use crate::errors::ShellError; +use crate::parser::registry::{CommandConfig, CommandRegistry, NamedType}; +use crate::parser::{baseline_parse_tokens, CallNode, Spanned}; +use crate::parser::{ + hir::{self, NamedArguments}, + Flag, RawToken, TokenNode, +}; +use log::trace; + +pub fn parse_command( + config: &CommandConfig, + registry: &dyn CommandRegistry, + call: &Spanned, + source: &str, +) -> Result { + let Spanned { item: call, .. } = call; + + trace!("Processing {:?}", config); + + let head = parse_command_head(call.head())?; + + let children: Option> = call.children().as_ref().map(|nodes| { + nodes + .iter() + .cloned() + .filter(|node| match node { + TokenNode::Whitespace(_) => false, + _ => true, + }) + .collect() + }); + + match parse_command_tail(&config, registry, children, source)? { + None => Ok(hir::Call::new(Box::new(head), None, None)), + Some((positional, named)) => Ok(hir::Call::new(Box::new(head), positional, named)), + } +} + +fn parse_command_head(head: &TokenNode) -> Result { + match head { + TokenNode::Token( + spanned @ Spanned { + item: RawToken::Bare, + .. + }, + ) => Ok(spanned.map(|_| hir::RawExpression::Literal(hir::Literal::Bare))), + + TokenNode::Token(Spanned { + item: RawToken::String(inner_span), + span, + }) => Ok(Spanned::from_item( + hir::RawExpression::Literal(hir::Literal::String(*inner_span)), + *span, + )), + + other => Err(ShellError::unexpected(&format!( + "command head -> {:?}", + other + ))), + } +} + +fn parse_command_tail( + config: &CommandConfig, + registry: &dyn CommandRegistry, + tail: Option>, + source: &str, +) -> Result>, Option)>, ShellError> { + let mut tail = match tail { + None => return Ok(None), + Some(tail) => tail, + }; + + let mut named = NamedArguments::new(); + + for (name, kind) in config.named() { + trace!("looking for {} : {:?}", name, kind); + + match kind { + NamedType::Switch => { + let (rest, flag) = extract_switch(name, tail, source); + + tail = rest; + + named.insert_switch(name, flag); + } + NamedType::Mandatory(kind) => match extract_mandatory(name, tail, source) { + Err(err) => return Err(err), // produce a correct diagnostic + Ok((rest, pos, _flag)) => { + let (expr, rest) = hir::baseline_parse_next_expr( + &rest[pos..], + registry, + source, + kind.to_coerce_hint(), + )?; + tail = rest.to_vec(); + + named.insert_mandatory(name, expr); + } + }, + NamedType::Optional(kind) => match extract_optional(name, tail, source) { + Err(err) => return Err(err), // produce a correct diagnostic + Ok((rest, Some((pos, _flag)))) => { + let (expr, rest) = hir::baseline_parse_next_expr( + &rest[pos..], + registry, + source, + kind.to_coerce_hint(), + )?; + tail = rest.to_vec(); + + named.insert_optional(name, Some(expr)); + } + + Ok((rest, None)) => { + tail = rest; + + named.insert_optional(name, None); + } + }, + }; + } + + let mut positional = vec![]; + let mandatory = config.mandatory_positional(); + + for arg in mandatory { + if tail.len() == 0 { + return Err(ShellError::unimplemented("Missing mandatory argument")); + } + + let (result, rest) = + hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?; + + positional.push(result); + + tail = rest.to_vec(); + } + + let optional = config.optional_positional(); + + for arg in optional { + if tail.len() == 0 { + break; + } + + let (result, rest) = + hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?; + + positional.push(result); + + tail = rest.to_vec(); + } + + // TODO: Only do this if rest params are specified + let remainder = baseline_parse_tokens(&tail, registry, source)?; + positional.extend(remainder); + + trace!("Constructed positional={:?} named={:?}", positional, named); + + let positional = match positional { + positional if positional.len() == 0 => None, + positional => Some(positional), + }; + + let named = match named { + named if named.named.is_empty() => None, + named => Some(named), + }; + + trace!("Normalized positional={:?} named={:?}", positional, named); + + Ok(Some((positional, named))) +} + +fn extract_switch( + name: &str, + mut tokens: Vec, + source: &str, +) -> (Vec, Option) { + let pos = tokens + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) + .nth(0); + + match pos { + None => (tokens, None), + Some((pos, flag)) => { + tokens.remove(pos); + (tokens, Some(*flag)) + } + } +} + +fn extract_mandatory( + name: &str, + mut tokens: Vec, + source: &str, +) -> Result<(Vec, usize, Flag), ShellError> { + let pos = tokens + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) + .nth(0); + + match pos { + None => Err(ShellError::unimplemented( + "Better error: mandatory flags must be present", + )), + Some((pos, flag)) => { + if tokens.len() <= pos { + return Err(ShellError::unimplemented( + "Better errors: mandatory flags must be followed by values", + )); + } + + tokens.remove(pos); + + Ok((tokens, pos, *flag)) + } + } +} + +fn extract_optional( + name: &str, + mut tokens: Vec, + source: &str, +) -> Result<(Vec, Option<(usize, Flag)>), ShellError> { + let pos = tokens + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f))) + .nth(0); + + match pos { + None => Ok((tokens, None)), + Some((pos, flag)) => { + if tokens.len() <= pos { + return Err(ShellError::unimplemented( + "Better errors: optional flags must be followed by values", + )); + } + + tokens.remove(pos); + + Ok((tokens, Some((pos, *flag)))) + } + } +} diff --git a/src/parser/registry.rs b/src/parser/registry.rs index a1adc63b9c..d50c67aa43 100644 --- a/src/parser/registry.rs +++ b/src/parser/registry.rs @@ -1,7 +1,11 @@ -use crate::evaluate::{evaluate_expr, Scope}; -use crate::parser::lexer::Spanned; +use crate::evaluate::{evaluate_baseline_expr, Scope}; +use crate::parser::{hir, hir::ExpressionKindHint, parse_command, CallNode, Spanned}; use crate::prelude::*; +use derive_new::new; +use getset::Getters; use indexmap::IndexMap; +use log::trace; +use std::fmt; #[allow(unused)] #[derive(Debug)] @@ -14,13 +18,18 @@ pub enum NamedType { #[derive(Debug)] pub enum NamedValue { Single, - Tuple, #[allow(unused)] Block, +} - #[allow(unused)] - Array, +impl NamedValue { + crate fn to_coerce_hint(&self) -> Option { + match self { + NamedValue::Single => None, + NamedValue::Block => Some(ExpressionKindHint::Block), + } + } } #[allow(unused)] @@ -31,61 +40,16 @@ pub enum PositionalType { } impl PositionalType { - crate fn name(&self) -> String { + crate fn to_coerce_hint(&self) -> Option { match self { - PositionalType::Value(s) => s.clone(), - PositionalType::Block(s) => s.clone(), - } - } - - crate fn evaluate( - &self, - arg: ast::Expression, - scope: &Scope, - ) -> Result, ShellError> { - match self { - PositionalType::Value(_) => evaluate_expr(&arg, scope), - PositionalType::Block(_) => match arg { - ast::Expression { - expr: ast::RawExpression::Block(b), - .. - } => Ok(Spanned::from_item(Value::block(b.expr), arg.span.clone())), - ast::Expression { - expr: ast::RawExpression::Binary(binary), - .. - } => { - // TODO: Use original spans - let mut b = ast::ExpressionBuilder::new(); - if let Some(s) = binary.left.as_string() { - Ok(Spanned::from_item( - Value::block(b.binary(( - &|b| b.path((&|b| b.var("it"), vec![s.clone()])), - &|_| binary.operator.clone(), - &|_| binary.right.clone(), - ))), - arg.span.clone(), - )) - } else { - let mut b = ast::ExpressionBuilder::new(); - let expr = b.binary(( - &|_| binary.left.clone(), - &|_| binary.operator.clone(), - &|_| binary.right.clone(), - )); - - Ok(Spanned::from_item(Value::block(expr), arg.span.clone())) - } - } - other => { - let span = other.span.clone(); - Ok(Spanned::from_item(Value::block(other), span)) - } - }, + PositionalType::Value(_) => None, + PositionalType::Block(_) => Some(ExpressionKindHint::Block), } } } -#[derive(Debug)] +#[derive(Debug, Getters)] +#[get = "crate"] pub struct CommandConfig { crate name: String, crate mandatory_positional: Vec, @@ -94,90 +58,218 @@ pub struct CommandConfig { crate named: IndexMap, } -#[derive(Debug, Default)] +#[derive(Debug, Default, new)] pub struct Args { - pub positional: Vec>, - pub named: IndexMap, + pub positional: Option>>, + pub named: Option>>, +} + +#[derive(new)] +pub struct DebugPositional<'a> { + positional: &'a Option>>, +} + +impl fmt::Debug for DebugPositional<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.positional { + None => write!(f, "None"), + Some(positional) => f + .debug_list() + .entries(positional.iter().map(|p| p.item().debug())) + .finish(), + } + } +} + +#[derive(new)] +pub struct DebugNamed<'a> { + named: &'a Option>>, +} + +impl fmt::Debug for DebugNamed<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.named { + None => write!(f, "None"), + Some(named) => f + .debug_map() + .entries(named.iter().map(|(k, v)| (k, v.item().debug()))) + .finish(), + } + } +} + +pub struct DebugArgs<'a> { + args: &'a Args, +} + +impl fmt::Debug for DebugArgs<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut s = f.debug_struct("Args"); + + s.field("positional", &DebugPositional::new(&self.args.positional)); + s.field("named", &DebugNamed::new(&self.args.named)); + + s.finish() + } +} + +impl Args { + pub fn debug(&'a self) -> DebugArgs<'a> { + DebugArgs { args: self } + } + + pub fn nth(&self, pos: usize) -> Option<&Spanned> { + match &self.positional { + None => None, + Some(array) => array.iter().nth(pos), + } + } + + pub fn expect_nth(&self, pos: usize) -> Result<&Spanned, ShellError> { + match &self.positional { + None => Err(ShellError::unimplemented("Better error: expect_nth")), + Some(array) => match array.iter().nth(pos) { + None => Err(ShellError::unimplemented("Better error: expect_nth")), + Some(item) => Ok(item), + }, + } + } + + pub fn len(&self) -> usize { + match &self.positional { + None => 0, + Some(array) => array.len(), + } + } + + pub fn has(&self, name: &str) -> bool { + match &self.named { + None => false, + Some(named) => named.contains_key(name), + } + } + + pub fn get(&self, name: &str) -> Option<&Spanned> { + match &self.named { + None => None, + Some(named) => named.get(name), + } + } + + pub fn positional_iter(&'a self) -> PositionalIter<'a> { + match &self.positional { + None => PositionalIter::Empty, + Some(v) => { + let iter = v.iter(); + PositionalIter::Array(iter) + } + } + } +} + +pub enum PositionalIter<'a> { + Empty, + Array(std::slice::Iter<'a, Spanned>), +} + +impl Iterator for PositionalIter<'a> { + type Item = &'a Spanned; + + fn next(&mut self) -> Option { + match self { + PositionalIter::Empty => None, + PositionalIter::Array(iter) => iter.next(), + } + } } impl CommandConfig { crate fn evaluate_args( &self, - args: impl Iterator, + call: &Spanned, + registry: &dyn CommandRegistry, scope: &Scope, + source: &str, ) -> Result { - let mut positional: Vec> = vec![]; - let mut named: IndexMap = IndexMap::default(); + let args = parse_command(self, registry, call, source)?; - let mut args: Vec = args.cloned().collect(); + trace!("parsed args: {:?}", args); - for (key, ty) in self.named.iter() { - let index = args.iter().position(|a| a.is_flag(&key)); + evaluate_args(args, registry, scope, source) - match (index, ty) { - (Some(i), NamedType::Switch) => { - args.remove(i); - named.insert(key.clone(), Value::boolean(true)); - } + // let mut positional: Vec> = vec![]; + // let mut named: IndexMap = IndexMap::default(); - (None, NamedType::Switch) => {} + // let mut args: Vec = args.cloned().collect(); - (Some(i), NamedType::Optional(v)) => { - args.remove(i); - named.insert(key.clone(), extract_named(&mut args, i, v)?); - } + // for (key, ty) in self.named.iter() { + // let index = args.iter().position(|a| a.is_flag(&key, source)); - (None, NamedType::Optional(_)) => {} + // match (index, ty) { + // (Some(i), NamedType::Switch) => { + // args.remove(i); + // named.insert(key.clone(), Value::boolean(true)); + // } - (Some(i), NamedType::Mandatory(v)) => { - args.remove(i); - named.insert(key.clone(), extract_named(&mut args, i, v)?); - } + // (None, NamedType::Switch) => {} - (None, NamedType::Mandatory(_)) => { - return Err(ShellError::string(&format!( - "Expected mandatory argument {}, but it was missing", - key - ))) - } - } - } + // (Some(i), NamedType::Optional(v)) => { + // args.remove(i); + // named.insert(key.clone(), extract_named(&mut args, i, v)?); + // } - let mut args = args.into_iter(); + // (None, NamedType::Optional(_)) => {} - for param in &self.mandatory_positional { - let arg = args.next(); + // (Some(i), NamedType::Mandatory(v)) => { + // args.remove(i); + // named.insert(key.clone(), extract_named(&mut args, i, v)?); + // } - let value = match arg { - None => { - return Err(ShellError::string(format!( - "expected mandatory positional argument {}", - param.name() - ))) - } + // (None, NamedType::Mandatory(_)) => { + // return Err(ShellError::string(&format!( + // "Expected mandatory argument {}, but it was missing", + // key + // ))) + // } + // } + // } - Some(arg) => param.evaluate(arg.clone(), scope)?, - }; + // let mut args = args.into_iter(); - positional.push(value); - } + // for param in &self.mandatory_positional { + // let arg = args.next(); - if self.rest_positional { - let rest: Result>, _> = - args.map(|i| evaluate_expr(&i, &Scope::empty())).collect(); - positional.extend(rest?); - } else { - let rest: Vec = args.collect(); + // let value = match arg { + // None => { + // return Err(ShellError::string(format!( + // "expected mandatory positional argument {}", + // param.name() + // ))) + // } - if rest.len() > 0 { - return Err(ShellError::string(&format!( - "Too many arguments, extras: {:?}", - rest - ))); - } - } + // Some(arg) => param.evaluate(arg.clone(), scope, source)?, + // }; - Ok(Args { positional, named }) + // positional.push(value); + // } + + // if self.rest_positional { + // let rest: Result>, _> = args + // .map(|i| evaluate_baseline_expr(&i, &Scope::empty(), source)) + // .collect(); + // positional.extend(rest?); + // } else { + // let rest: Vec = args.collect(); + + // if rest.len() > 0 { + // return Err(ShellError::string(&format!( + // "Too many arguments, extras: {:?}", + // rest + // ))); + // } + // } + + // Ok(Args { positional, named }) } #[allow(unused)] @@ -186,50 +278,64 @@ impl CommandConfig { } } -fn extract_named( - v: &mut Vec, - position: usize, - ty: &NamedValue, -) -> Result { - match ty { - NamedValue::Single => { - let expr = v.remove(position); - expect_simple_expr(expr) - } +fn evaluate_args( + args: hir::Call, + registry: &dyn CommandRegistry, + scope: &Scope, + source: &str, +) -> Result { + let positional: Result>, _> = args + .positional() + .as_ref() + .map(|p| { + p.iter() + .map(|e| evaluate_baseline_expr(e, &(), scope, source)) + .collect() + }) + .transpose(); - NamedValue::Tuple => { - let expr = v.remove(position); - let next = v.remove(position); + let positional = positional?; - let list = vec![expect_simple_expr(expr)?, expect_simple_expr(next)?]; - Ok(Value::List(list)) - } + let named: Result>>, ShellError> = args + .named() + .as_ref() + .map(|n| { + let mut results = IndexMap::new(); - other => Err(ShellError::string(&format!( - "Unimplemented named argument {:?}", - other - ))), - } -} + for (name, value) in n.named.iter() { + match value { + hir::named::NamedValue::PresentSwitch(span) => { + results.insert( + name.clone(), + Spanned::from_item(Value::boolean(true), *span), + ); + } + hir::named::NamedValue::Value(expr) => { + results.insert( + name.clone(), + evaluate_baseline_expr(expr, registry, scope, source)?, + ); + } -fn expect_simple_expr(expr: ast::Expression) -> Result { - match &*expr { - ast::RawExpression::Leaf(l) => Ok(match l { - ast::Leaf::Bare(s) => Value::string(s.to_string()), - ast::Leaf::String(s) => Value::string(s), - ast::Leaf::Boolean(b) => Value::boolean(*b), - ast::Leaf::Int(i) => Value::int(*i), - ast::Leaf::Unit(i, unit) => unit.compute(*i), - }), + _ => {} + }; + } - // TODO: Diagnostic - other => Err(ShellError::string(&format!( - "Expected a value, found {}", - other.print() - ))), - } + Ok(results) + }) + .transpose(); + + let named = named?; + + Ok(Args::new(positional, named)) } pub trait CommandRegistry { - fn get(&self, name: &str) -> CommandConfig; + fn get(&self, name: &str) -> Option; +} + +impl CommandRegistry for () { + fn get(&self, _name: &str) -> Option { + None + } } diff --git a/src/prelude.rs b/src/prelude.rs index d943d90827..b9fec2540c 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -5,7 +5,6 @@ crate use crate::env::host::handle_unexpected; crate use crate::env::{Environment, Host}; crate use crate::errors::ShellError; crate use crate::object::Value; -crate use crate::parser::ast; crate use crate::stream::{single_output, InputStream, OutputStream}; crate use futures::{FutureExt, StreamExt}; crate use std::collections::VecDeque; diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 039a0b719e..e234aca42c 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,9 +1,8 @@ use crate::shell::completer::NuCompleter; -use crate::parser::lexer::SpannedToken; +use crate::parser::nom_input; use crate::prelude::*; use ansi_term::Color; -use log::trace; use rustyline::completion::{self, Completer, FilenameCompleter}; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -47,7 +46,7 @@ impl Hinter for Helper { } impl Highlighter for Helper { - fn highlight_prompt<'b, 's: 'b, 'p:'b>(&'s self, prompt: &'p str, _: bool) -> Cow<'b, str> { + fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&'s self, prompt: &'p str, _: bool) -> Cow<'b, str> { Owned("\x1b[32m".to_owned() + &prompt[0..prompt.len() - 2] + "\x1b[m> ") } @@ -56,30 +55,42 @@ impl Highlighter for Helper { } fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { - let tokens = crate::parser::lexer::Lexer::new(line, true); - let tokens: Result, _> = tokens.collect(); + + return Cow::Borrowed(line); + + let tokens = crate::parser::pipeline(nom_input(line)); match tokens { Err(_) => Cow::Borrowed(line), - Ok(v) => { + Ok((_rest, v)) => { let mut out = String::new(); - let mut iter = v.iter(); + let tokens = match v.as_pipeline() { + Err(_) => return Cow::Borrowed(line), + Ok(v) => v, + }; - let mut state = State::Command; + let mut iter = tokens.into_iter(); + + match iter.next() { + None => return Cow::Owned(out), + Some(v) => out.push_str(v.span().slice(line)), + }; loop { match iter.next() { None => return Cow::Owned(out), - Some((start, token, end)) => { - let (style, new_state) = token_style(&token, state); + Some(token) => { + // let styled = token_style(&token, state); - trace!("token={:?}", token); - trace!("style={:?}", style); - trace!("new_state={:?}", new_state); + // trace!("token={:?}", token); + // trace!("style={:?}", style); + // trace!("new_state={:?}", new_state); - state = new_state; - let slice = &line[*start..*end]; - let styled = style.paint(slice); + // state = new_state; + // let slice = &line[*start..*end]; + // let styled = style.paint(slice); + out.push_str("|"); + let styled = Color::Black.bold().paint(token.span().slice(line)); out.push_str(&styled.to_string()); } } @@ -93,41 +104,4 @@ impl Highlighter for Helper { } } -#[derive(Debug)] -enum State { - Command, - Flag, - Var, - Bare, - None, -} - -fn token_style( - token: &crate::parser::lexer::SpannedToken, - state: State, -) -> (ansi_term::Style, State) { - use crate::parser::lexer::Token::*; - - match (state, &token.token) { - (State::Command, Bare) => (Color::Cyan.bold(), State::None), - (State::Command, Whitespace) => (Color::White.normal(), State::Command), - - (State::Flag, Bare) => (Color::Black.bold(), State::None), - - (State::Var, Variable) => (Color::Yellow.bold(), State::None), - - (State::Bare, PathDot) => (Color::Green.normal(), State::Bare), - (State::Bare, Member) => (Color::Green.normal(), State::Bare), - - (_, Dash) | (_, DashDash) => (Color::Black.bold(), State::Flag), - (_, Dollar) => (Color::Yellow.bold(), State::Var), - (_, Bare) => (Color::Green.normal(), State::Bare), - (_, Member) => (Color::Cyan.normal(), State::None), - (_, Num) => (Color::Purple.bold(), State::None), - (_, DQString) | (_, SQString) => (Color::Green.normal(), State::None), - (_, Pipe) => (Color::White.normal(), State::Command), - _ => (Color::White.normal(), State::None), - } -} - impl rustyline::Helper for Helper {} diff --git a/tests/sort_by.txt b/tests/sort_by.txt index b84c5e7ea2..aa933f8a2d 100644 --- a/tests/sort_by.txt +++ b/tests/sort_by.txt @@ -1,3 +1,3 @@ cd tests -open test.toml --raw | split-row "\n" | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it +open test.toml --raw | lines | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it exit diff --git a/tests/split.txt b/tests/split.txt index 5bc3d841f6..68713c675f 100644 --- a/tests/split.txt +++ b/tests/split.txt @@ -1,3 +1,3 @@ cd tests -open test.toml --raw | split-row "\n" | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it +open test.toml --raw | lines | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it exit From 1c0096b2fb94ad7b350d0ddaa216bb150f028f67 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 22 Jun 2019 15:49:49 +1200 Subject: [PATCH 08/20] WIP --- .cargo/config | 1 + src/parser/parse2/parser.rs | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.cargo/config b/.cargo/config index e69de29bb2..6727dab680 100644 --- a/.cargo/config +++ b/.cargo/config @@ -0,0 +1 @@ +paths = ["C:\\Users\\wycat\\Code\\nom_locate"] \ No newline at end of file diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 4a9a134338..e7cac1bc2d 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -71,9 +71,7 @@ pub fn raw_integer(input: NomSpan) -> IResult> { let start = input.offset; trace_step(input, "raw_integer", move |input| { let (input, neg) = opt(tag("-"))(input)?; - println!("(input,neg) = {:?} {:?}", input, neg); let (input, num) = digit1(input)?; - println!("(input,num) = {:?} {:?}", input, num); let end = input.offset; Ok(( From 8cc91e05c4a8500ca3ddae5767ad1f9d20372277 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 22 Jun 2019 15:52:19 +1200 Subject: [PATCH 09/20] WIP --- src/shell/helper.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/shell/helper.rs b/src/shell/helper.rs index e234aca42c..b09a11a7ad 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -56,8 +56,6 @@ impl Highlighter for Helper { fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { - return Cow::Borrowed(line); - let tokens = crate::parser::pipeline(nom_input(line)); match tokens { From 3b35dcb61928fa22cf174dff77e89460cbc9f382 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sat, 22 Jun 2019 10:08:53 -0400 Subject: [PATCH 10/20] Finish nom upgrade --- .cargo/config | 1 - src/evaluate/evaluator.rs | 2 +- src/parser/hir.rs | 4 +--- src/parser/hir/baseline_parse.rs | 18 +------------- src/parser/hir/baseline_parse_tokens.rs | 32 +++++-------------------- src/parser/hir/path.rs | 2 +- src/parser/parse2/span.rs | 2 +- src/parser/parse2/token_tree.rs | 21 ---------------- src/parser/parse2/tokens.rs | 27 --------------------- 9 files changed, 11 insertions(+), 98 deletions(-) diff --git a/.cargo/config b/.cargo/config index 6727dab680..e69de29bb2 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1 +0,0 @@ -paths = ["C:\\Users\\wycat\\Code\\nom_locate"] \ No newline at end of file diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index 70cd486e71..7163bf6973 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -1,7 +1,7 @@ use crate::object::base::Block; use crate::parser::{ hir::{self, Expression, RawExpression}, - CommandRegistry, Span, Spanned, Text, + CommandRegistry, Spanned, Text, }; use crate::prelude::*; use derive_new::new; diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 70b5f5c3be..7ab79a8d93 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -9,9 +9,7 @@ use derive_new::new; use getset::Getters; crate use baseline_parse::baseline_parse_single_token; -crate use baseline_parse_tokens::{ - baseline_parse_next_expr, baseline_parse_tokens, ExpressionKindHint, -}; +crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint}; crate use binary::Binary; crate use named::NamedArguments; crate use path::Path; diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs index 1150434fcb..ae9240cd4c 100644 --- a/src/parser/hir/baseline_parse.rs +++ b/src/parser/hir/baseline_parse.rs @@ -1,20 +1,4 @@ -use crate::errors::ShellError; -use crate::parser::{hir, CommandRegistry, RawToken, Token, TokenNode}; - -// pub fn baseline_parse_token( -// token_node: TokenNode, -// _registry: &dyn CommandRegistry, -// ) -> Result { -// match token_node { -// TokenNode::Token(token) => Ok(baseline_parse_single_token(token)), -// TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")), -// TokenNode::Delimited(_delimited) => { -// Err(ShellError::unimplemented("baseline_parse Delimited")) -// } -// TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")), -// TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")), -// } -// } +use crate::parser::{hir, RawToken, Token}; pub fn baseline_parse_single_token(token: &Token, source: &str) -> hir::Expression { match *token.item() { diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index 4cf8ae86bb..39c1fcaee8 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -23,6 +23,7 @@ pub fn baseline_parse_tokens( Ok(exprs) } +#[allow(unused)] #[derive(Debug)] pub enum ExpressionKindHint { Literal, @@ -134,15 +135,15 @@ pub fn baseline_parse_semantic_token( ) -> Result { match token { TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), - TokenNode::Call(call) => unimplemented!(), - TokenNode::Delimited(delimited) => unimplemented!(), - TokenNode::Pipeline(pipeline) => unimplemented!(), + TokenNode::Call(_call) => unimplemented!(), + TokenNode::Delimited(_delimited) => unimplemented!(), + TokenNode::Pipeline(_pipeline) => unimplemented!(), TokenNode::Operator(_op) => unreachable!(), - TokenNode::Flag(flag) => unimplemented!(), + TokenNode::Flag(_flag) => unimplemented!(), TokenNode::Identifier(_span) => unreachable!(), TokenNode::Whitespace(_span) => unreachable!(), TokenNode::Error(error) => Err(*error.item.clone()), - TokenNode::Path(path) => unimplemented!(), + TokenNode::Path(_path) => unimplemented!(), } } @@ -154,24 +155,3 @@ fn next_token(nodes: &mut impl Iterator) -> Option<&'a Tok } } } - -fn baseline_parse_token( - token_node: &TokenNode, - _registry: &dyn CommandRegistry, - source: &str, -) -> Result { - match token_node { - TokenNode::Token(token) => Ok(hir::baseline_parse_single_token(token, source)), - TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")), - TokenNode::Delimited(_delimited) => { - Err(ShellError::unimplemented("baseline_parse Delimited")) - } - TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")), - TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")), - TokenNode::Operator(_op) => Err(ShellError::unimplemented("baseline_parse Operator")), - TokenNode::Flag(_op) => Err(ShellError::unimplemented("baseline_parse Flag")), - TokenNode::Identifier(_op) => Err(ShellError::unimplemented("baseline_parse Identifier")), - TokenNode::Whitespace(_op) => Err(ShellError::unimplemented("baseline_parse Whitespace")), - TokenNode::Error(err) => Err(*err.item.clone()), - } -} diff --git a/src/parser/hir/path.rs b/src/parser/hir/path.rs index 132f3112b8..02bf8e5ce3 100644 --- a/src/parser/hir/path.rs +++ b/src/parser/hir/path.rs @@ -1,4 +1,4 @@ -use crate::parser::{hir::Expression, Operator, Spanned}; +use crate::parser::{hir::Expression, Spanned}; use derive_new::new; use getset::Getters; diff --git a/src/parser/parse2/span.rs b/src/parser/parse2/span.rs index 4214d29aaa..928352d69b 100644 --- a/src/parser/parse2/span.rs +++ b/src/parser/parse2/span.rs @@ -32,7 +32,7 @@ impl Spanned { } crate fn copy_span(&self, output: U) -> Spanned { - let Spanned { span, item } = self; + let Spanned { span, .. } = self; Spanned { span: *span, diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index 9718e9def5..e618feaa93 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -44,13 +44,6 @@ impl TokenNode { self.span().slice(source) } - pub fn is_ws(&self) -> bool { - match self { - TokenNode::Whitespace(_) => true, - _ => false, - } - } - pub fn is_bare(&self) -> bool { match self { TokenNode::Token(Spanned { @@ -61,20 +54,6 @@ impl TokenNode { } } - crate fn as_string(&self, source: &str) -> Option> { - match self { - TokenNode::Token(Spanned { - item: RawToken::Bare, - span, - }) => Some(Spanned::from_item(span.slice(source).to_string(), span)), - TokenNode::Token(Spanned { - item: RawToken::String(inner), - span, - }) => Some(Spanned::from_item(inner.slice(source).to_string(), span)), - _ => None, - } - } - crate fn as_flag(&self, value: &str, source: &str) -> Option> { match self { TokenNode::Flag( diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs index 3e415c5261..e2d36b3d76 100644 --- a/src/parser/parse2/tokens.rs +++ b/src/parser/parse2/tokens.rs @@ -1,5 +1,3 @@ -use crate::parser::parse2::flag::*; -use crate::parser::parse2::operator::*; use crate::parser::parse2::span::*; use crate::parser::parse2::unit::*; @@ -13,28 +11,3 @@ pub enum RawToken { } pub type Token = Spanned; - -impl Token { - pub fn to_semantic_token(&self) -> Option { - let semantic_token = match self.item { - RawToken::Integer(int) => RawSemanticToken::Integer(int), - RawToken::Size(int, unit) => RawSemanticToken::Size(int, unit), - RawToken::String(span) => RawSemanticToken::String(span), - RawToken::Variable(span) => RawSemanticToken::Variable(span), - RawToken::Bare => RawSemanticToken::Bare, - }; - - Some(Spanned::from_item(semantic_token, self.span)) - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum RawSemanticToken { - Integer(i64), - Size(i64, Unit), - String(Span), - Variable(Span), - Bare, -} - -pub type SemanticToken = Spanned; From 72a61f557b42b5c28e65cfa46b9668411ac18582 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sun, 23 Jun 2019 04:59:37 +1200 Subject: [PATCH 11/20] Remove debug println --- src/parser/hir/baseline_parse_tokens.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index 39c1fcaee8..3f4b7868f1 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -39,11 +39,6 @@ pub fn baseline_parse_next_expr( source: &str, coerce_hint: Option, ) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> { - println!( - "baseline_parse_next_expr {:?} - {:?}", - token_nodes, coerce_hint - ); - let mut tokens = token_nodes.iter().peekable(); let first = next_token(&mut tokens); From 9af08eb91131ce502144ec1fe8d3922cd3521dea Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sun, 23 Jun 2019 06:02:23 +1200 Subject: [PATCH 12/20] Fix flag oversight --- src/commands/enter.rs | 41 ++++++++++++++++++++--------------------- src/commands/open.rs | 42 ++++++++++++++++++++---------------------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/src/commands/enter.rs b/src/commands/enter.rs index df0315a9ba..48a51a4151 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -93,32 +93,31 @@ pub fn enter(args: CommandArgs) -> Result { let mut stream = VecDeque::new(); - let file_extension = match args.nth(1) { - Some(Spanned { - item: Value::Primitive(Primitive::String(s)), - span, - }) => { - if s == "--raw" { - None - } else if s == "--json" { - Some("json".to_string()) - } else if s == "--xml" { - Some("xml".to_string()) - } else if s == "--ini" { - Some("ini".to_string()) - } else if s == "--yaml" { - Some("yaml".to_string()) - } else if s == "--toml" { - Some("toml".to_string()) - } else { + let file_extension = if args.has("raw") { + None + } else if args.has("json") { + Some("json".to_string()) + } else if args.has("xml") { + Some("xml".to_string()) + } else if args.has("ini") { + Some("ini".to_string()) + } else if args.has("yaml") { + Some("yaml".to_string()) + } else if args.has("toml") { + Some("toml".to_string()) + } else { + if let Some(ref named_args) = args.args.named { + for named in named_args.iter() { return Err(ShellError::labeled_error( - "Unknown flag for open", + "Unknown flag for enter", "unknown flag", - span.clone(), + named.1.span.clone(), )); } + file_extension + } else { + file_extension } - _ => file_extension, }; match file_extension { diff --git a/src/commands/open.rs b/src/commands/open.rs index 17be0b7063..d98b404de2 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -1,10 +1,10 @@ use crate::errors::ShellError; use crate::object::{Primitive, Value}; +use crate::parser::parse2::span::Spanned; use crate::parser::registry::{CommandConfig, NamedType}; use crate::prelude::*; use indexmap::IndexMap; use std::path::{Path, PathBuf}; -use crate::parser::parse2::span::Spanned; pub struct Open; @@ -116,34 +116,32 @@ fn open(args: CommandArgs) -> Result { let mut stream = VecDeque::new(); - let file_extension = match args.nth(1) { - Some(Spanned { - item: Value::Primitive(Primitive::String(s)), - span, - }) => { - if s == "--raw" { - None - } else if s == "--json" { - Some("json".to_string()) - } else if s == "--xml" { - Some("xml".to_string()) - } else if s == "--ini" { - Some("ini".to_string()) - } else if s == "--yaml" { - Some("yaml".to_string()) - } else if s == "--toml" { - Some("toml".to_string()) - } else { + let file_extension = if args.has("raw") { + None + } else if args.has("json") { + Some("json".to_string()) + } else if args.has("xml") { + Some("xml".to_string()) + } else if args.has("ini") { + Some("ini".to_string()) + } else if args.has("yaml") { + Some("yaml".to_string()) + } else if args.has("toml") { + Some("toml".to_string()) + } else { + if let Some(ref named_args) = args.args.named { + for named in named_args.iter() { return Err(ShellError::labeled_error( "Unknown flag for open", "unknown flag", - span.clone(), + named.1.span.clone(), )); } + file_extension + } else { + file_extension } - _ => file_extension, }; - match file_extension { Some(x) if x == "toml" => { stream.push_back(ReturnValue::Value( From 37c4fb92f893058de0cde90ae1481d39790dd030 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sun, 23 Jun 2019 06:32:58 +1200 Subject: [PATCH 13/20] Fix raw as a variable --- src/parser/hir/baseline_parse_tokens.rs | 45 +++++++++++++++++-------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index 3f4b7868f1..a04a06c5a7 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -82,15 +82,43 @@ pub fn baseline_parse_next_expr( ExpressionKindHint::Block => { let span = (first.span.start, second.span.end); - let string: Spanned = match first { + let path: Spanned = match first { Spanned { item: hir::RawExpression::Literal(hir::Literal::Bare), span, - } => Spanned::from_item(span.slice(source).to_string(), span), + } => { + let string = Spanned::from_item(span.slice(source).to_string(), span); + let path = hir::Path::new( + Spanned::from_item( + // TODO: Deal with synthetic nodes that have no representation at all in source + hir::RawExpression::Variable(hir::Variable::It(Span::from((0, 0)))), + (0, 0), + ), + vec![string], + ); + let path = hir::RawExpression::Path(Box::new(path)); + Spanned { item: path, span: first.span } + } Spanned { item: hir::RawExpression::Literal(hir::Literal::String(inner)), span, - } => Spanned::from_item(inner.slice(source).to_string(), span), + } => { + let string = Spanned::from_item(inner.slice(source).to_string(), span); + let path = hir::Path::new( + Spanned::from_item( + // TODO: Deal with synthetic nodes that have no representation at all in source + hir::RawExpression::Variable(hir::Variable::It(Span::from((0, 0)))), + (0, 0), + ), + vec![string], + ); + let path = hir::RawExpression::Path(Box::new(path)); + Spanned { item: path, span: first.span } + } + Spanned { + item: hir::RawExpression::Variable(..), + .. + } => first, _ => { return Err(ShellError::unimplemented( "The first part of a block must be a string", @@ -98,17 +126,6 @@ pub fn baseline_parse_next_expr( } }; - let path = hir::Path::new( - Spanned::from_item( - // TODO: Deal with synthetic nodes that have no representation at all in source - hir::RawExpression::Variable(hir::Variable::It(Span::from((0, 0)))), - (0, 0), - ), - vec![string], - ); - let path = hir::RawExpression::Path(Box::new(path)); - let path = Spanned::from_item(path, first.span); - let binary = hir::Binary::new(path, *op, second); let binary = hir::RawExpression::Binary(Box::new(binary)); let binary = Spanned::from_item(binary, span); From dc081151bcd029d49aaf00a629653627fdc99aca Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sun, 23 Jun 2019 07:47:29 +1200 Subject: [PATCH 14/20] add back some shell coloring --- src/parser/parse2/token_tree.rs | 4 +- src/shell/helper.rs | 70 ++++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index e618feaa93..d2c6e7ffb8 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -94,10 +94,10 @@ pub struct PathNode { #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] pub struct PipelineElement { - pre_ws: Option, + pub pre_ws: Option, #[get = "crate"] call: Spanned, - post_ws: Option, + pub post_ws: Option, } impl PipelineElement { diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 85ad50de6a..d91f55f0c0 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,5 +1,8 @@ use crate::shell::completer::NuCompleter; - +use crate::parser::parse2::PipelineElement; +use crate::parser::parse2::token_tree::TokenNode; +use crate::parser::parse2::tokens::RawToken; +use crate::parser::parse2::span::Spanned; use crate::parser::nom_input; use crate::prelude::*; use ansi_term::Color; @@ -68,26 +71,24 @@ impl Highlighter for Helper { let mut iter = tokens.into_iter(); - match iter.next() { - None => return Cow::Owned(out), - Some(v) => out.push_str(v.span().slice(line)), - }; + // match iter.next() { + // None => return Cow::Owned(out), + // Some(v) => out.push_str(v.span().slice(line)), + // }; + let mut first = true; loop { match iter.next() { None => return Cow::Owned(out), Some(token) => { - // let styled = token_style(&token, state); + let styled = paint_pipeline_element(&token, line); - // trace!("token={:?}", token); - // trace!("style={:?}", style); - // trace!("new_state={:?}", new_state); + if !first { + out.push_str("|"); + } else { + first = false; + } - // state = new_state; - // let slice = &line[*start..*end]; - // let styled = style.paint(slice); - out.push_str("|"); - let styled = Color::Black.bold().paint(token.span().slice(line)); out.push_str(&styled.to_string()); } } @@ -101,4 +102,45 @@ impl Highlighter for Helper { } } +fn paint_token_node(token_node: &TokenNode, line: &str) -> String { + let styled = match token_node { + TokenNode::Call(..) => Color::Cyan.bold().paint(token_node.span().slice(line)), + TokenNode::Whitespace(..) => Color::White.normal().paint(token_node.span().slice(line)), + TokenNode::Flag(..) => Color::Black.bold().paint(token_node.span().slice(line)), + TokenNode::Identifier(..) => Color::Yellow.bold().paint(token_node.span().slice(line)), + TokenNode::Path(..) => Color::Green.bold().paint(token_node.span().slice(line)), + TokenNode::Error(..) => Color::Red.bold().paint(token_node.span().slice(line)), + TokenNode::Delimited(..) => Color::White.paint(token_node.span().slice(line)), + TokenNode::Operator(..) => Color::Purple.bold().paint(token_node.span().slice(line)), + TokenNode::Pipeline(..) => Color::Blue.normal().paint(token_node.span().slice(line)), + TokenNode::Token(Spanned { item: RawToken::Integer(..), ..}) => Color::Purple.bold().paint(token_node.span().slice(line)), + TokenNode::Token(Spanned { item: RawToken::Size(..), ..}) => Color::Purple.bold().paint(token_node.span().slice(line)), + TokenNode::Token(Spanned { item: RawToken::String(..), ..}) => Color::Green.normal().paint(token_node.span().slice(line)), + TokenNode::Token(Spanned { item: RawToken::Variable(..), ..}) => Color::Yellow.bold().paint(token_node.span().slice(line)), + TokenNode::Token(Spanned { item: RawToken::Bare, ..}) => Color::Green.normal().paint(token_node.span().slice(line)), + }; + + styled.to_string() +} + +fn paint_pipeline_element(pipeline_element: &PipelineElement, line: &str) -> String { + let mut styled = String::new(); + + if let Some(ws) = pipeline_element.pre_ws { + styled.push_str(&Color::White.normal().paint(ws.slice(line))); + } + styled.push_str(&paint_token_node(pipeline_element.call().head(), line)); + + if let Some(children) = pipeline_element.call().children() { + for child in children { + styled.push_str(&paint_token_node(child, line)); + } + } + if let Some(ws) = pipeline_element.post_ws { + styled.push_str(&Color::White.normal().paint(ws.slice(line))); + } + + styled.to_string() +} + impl rustyline::Helper for Helper {} From 4036bf1ffd55cb7f2ea4cee3184f30da19bac6bc Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sat, 22 Jun 2019 16:46:16 -0400 Subject: [PATCH 15/20] &str -> Text --- Cargo.lock | 2 +- src/cli.rs | 8 ++++---- src/commands/cd.rs | 8 ++++---- src/commands/config.rs | 8 ++++---- src/commands/enter.rs | 2 +- src/commands/from_json.rs | 8 ++++---- src/commands/from_toml.rs | 21 ++++++++++++++------- src/commands/from_xml.rs | 18 ++++++++++-------- src/commands/from_yaml.rs | 8 ++++---- src/commands/get.rs | 4 +++- src/commands/lines.rs | 2 +- src/commands/ls.rs | 2 +- src/commands/open.rs | 6 ++---- src/commands/pick.rs | 2 +- src/commands/reject.rs | 2 +- src/commands/size.rs | 4 ++-- src/commands/split_column.rs | 13 +++++-------- src/commands/split_row.rs | 2 +- src/commands/to_json.rs | 6 +++++- src/commands/to_toml.rs | 6 +++++- src/commands/trim.rs | 2 +- src/commands/view.rs | 2 +- src/evaluate/evaluator.rs | 8 ++++---- src/format/generic.rs | 2 +- src/main.rs | 2 ++ src/object/base.rs | 19 +++++++++---------- src/object/desc.rs | 17 ++++++++++++++--- src/parser/hir/baseline_parse.rs | 3 ++- src/parser/hir/baseline_parse_tokens.rs | 12 ++++-------- src/parser/parse2/span.rs | 7 ++++--- src/parser/parse2/text.rs | 2 +- src/parser/parse2/token_tree.rs | 7 ++++--- src/parser/parse_command.rs | 11 ++++++----- src/parser/registry.rs | 4 ++-- src/prelude.rs | 1 + src/shell/helper.rs | 2 +- 36 files changed, 130 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 137ea71f93..8d3466a975 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1560,7 +1560,7 @@ dependencies = [ [[package]] name = "nom_locate" version = "0.3.1" -source = "git+https://github.com/wycats/nom_locate.git?branch=nom5#9383cc779b23a746ff68cc7094d9ed7baaa661b2" +source = "git+https://github.com/wycats/nom_locate.git?branch=nom5#5baeede19605e2049994eca8f7b366ddc85bb9cd" dependencies = [ "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/cli.rs b/src/cli.rs index 2263eb43fd..f438f53b98 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -232,7 +232,7 @@ async fn process_line(readline: Result, ctx: &mut Context debug!("=== Parsed ==="); debug!("{:#?}", result); - let mut pipeline = classify_pipeline(&result, ctx, &line)?; + let mut pipeline = classify_pipeline(&result, ctx, &Text::from(line))?; match pipeline.commands.last() { Some(ClassifiedCommand::Sink(_)) => {} @@ -350,13 +350,13 @@ async fn process_line(readline: Result, ctx: &mut Context fn classify_pipeline( pipeline: &TokenNode, context: &Context, - source: &str, + source: &Text, ) -> Result { let pipeline = pipeline.as_pipeline()?; let commands: Result, ShellError> = pipeline .iter() - .map(|item| classify_command(&item, context, source)) + .map(|item| classify_command(&item, context, &source)) .collect(); Ok(ClassifiedPipeline { @@ -367,7 +367,7 @@ fn classify_pipeline( fn classify_command( command: &PipelineElement, context: &Context, - source: &str, + source: &Text, ) -> Result { let call = command.call(); diff --git a/src/commands/cd.rs b/src/commands/cd.rs index 1ed46ab205..2cb2fc1b67 100644 --- a/src/commands/cd.rs +++ b/src/commands/cd.rs @@ -17,8 +17,8 @@ pub fn cd(args: CommandArgs) -> Result { _ => return Err(ShellError::string("Can not change to home directory")), }, Some(v) => { - let target = v.as_string()?.clone(); - match dunce::canonicalize(cwd.join(&target).as_path()) { + let target = v.as_string()?; + match dunce::canonicalize(cwd.join(target.as_ref()).as_path()) { Ok(p) => p, Err(_) => { return Err(ShellError::labeled_error( @@ -63,9 +63,9 @@ pub fn cd(args: CommandArgs) -> Result { cwd.pop(); } _ => match target.chars().nth(0) { - Some(x) if x == '/' => cwd = PathBuf::from(target), + Some(x) if x == '/' => cwd = PathBuf::from(target.as_ref()), _ => { - cwd.push(target); + cwd.push(target.as_ref()); } }, } diff --git a/src/commands/config.rs b/src/commands/config.rs index d18bc2f17f..d2161dd62b 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -46,7 +46,7 @@ pub fn config(args: CommandArgs) -> Result { if let Some(v) = args.get("get") { let key = v.as_string()?; let value = result - .get(&key) + .get(key.as_ref()) .ok_or_else(|| ShellError::string(&format!("Missing key {} in config", key)))?; return Ok( @@ -57,7 +57,7 @@ pub fn config(args: CommandArgs) -> Result { if let Some(v) = args.get("set") { if let Ok((key, value)) = v.as_pair() { - result.insert(key.as_string()?, value.clone()); + result.insert(key.as_string()?.as_ref().to_string(), value.clone()); config::write_config(&result)?; @@ -86,8 +86,8 @@ pub fn config(args: CommandArgs) -> Result { if let Some(v) = args.get("remove") { let key = v.as_string()?; - if result.contains_key(&key) { - result.remove(&key); + if result.contains_key(key.as_ref()) { + result.remove(key.as_ref()); } else { return Err(ShellError::string(&format!( "{} does not exist in config", diff --git a/src/commands/enter.rs b/src/commands/enter.rs index bd00ab2d91..d3f97512a7 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -58,7 +58,7 @@ pub fn enter(args: CommandArgs) -> Result { } } } else { - full_path.push(Path::new(&s)); + full_path.push(Path::new(s)); match std::fs::read_to_string(&full_path) { Ok(s) => ( full_path diff --git a/src/commands/from_json.rs b/src/commands/from_json.rs index 1624d9c7e1..5162632dd3 100644 --- a/src/commands/from_json.rs +++ b/src/commands/from_json.rs @@ -4,12 +4,12 @@ use crate::prelude::*; fn convert_json_value_to_nu_value(v: &serde_hjson::Value) -> Value { match v { - serde_hjson::Value::Null => Value::Primitive(Primitive::String("".to_string())), + serde_hjson::Value::Null => Value::Primitive(Primitive::String(String::from(""))), serde_hjson::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)), serde_hjson::Value::F64(n) => Value::Primitive(Primitive::Float(OF64::from(*n))), serde_hjson::Value::U64(n) => Value::Primitive(Primitive::Int(*n as i64)), serde_hjson::Value::I64(n) => Value::Primitive(Primitive::Int(*n as i64)), - serde_hjson::Value::String(s) => Value::Primitive(Primitive::String(s.clone())), + serde_hjson::Value::String(s) => Value::Primitive(Primitive::String(String::from(s))), serde_hjson::Value::Array(a) => Value::List( a.iter() .map(|x| convert_json_value_to_nu_value(x)) @@ -38,9 +38,9 @@ pub fn from_json(args: CommandArgs) -> Result { Ok(out .map(|a| match a { Value::Primitive(Primitive::String(s)) => { - ReturnValue::Value(from_json_string_to_value(s)) + ReturnValue::Value(from_json_string_to_value(s.to_string())) } - _ => ReturnValue::Value(Value::Primitive(Primitive::String("".to_string()))), + _ => ReturnValue::Value(Value::Primitive(Primitive::String(String::from("")))), }) .boxed()) } diff --git a/src/commands/from_toml.rs b/src/commands/from_toml.rs index 59610a8ffd..59c7f39bf6 100644 --- a/src/commands/from_toml.rs +++ b/src/commands/from_toml.rs @@ -1,5 +1,5 @@ -use crate::object::{Primitive, Value, Dictionary, DataDescriptor}; use crate::object::base::OF64; +use crate::object::{DataDescriptor, Dictionary, Primitive, Value}; use crate::prelude::*; fn convert_toml_value_to_nu_value(v: &toml::Value) -> Value { @@ -7,21 +7,28 @@ fn convert_toml_value_to_nu_value(v: &toml::Value) -> Value { toml::Value::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)), toml::Value::Integer(n) => Value::Primitive(Primitive::Int(*n)), toml::Value::Float(n) => Value::Primitive(Primitive::Float(OF64::from(*n))), - toml::Value::String(s) => Value::Primitive(Primitive::String(s.clone())), - toml::Value::Array(a) => Value::List(a.iter().map(|x| convert_toml_value_to_nu_value(x)).collect()), + toml::Value::String(s) => Value::Primitive(Primitive::String(String::from(s))), + toml::Value::Array(a) => Value::List( + a.iter() + .map(|x| convert_toml_value_to_nu_value(x)) + .collect(), + ), toml::Value::Datetime(dt) => Value::Primitive(Primitive::String(dt.to_string())), toml::Value::Table(t) => { let mut collected = Dictionary::default(); for (k, v) in t.iter() { - collected.add(DataDescriptor::from(k.clone()), convert_toml_value_to_nu_value(v)); + collected.add( + DataDescriptor::from(k.clone()), + convert_toml_value_to_nu_value(v), + ); } Value::Object(collected) } } } -pub fn from_toml_string_to_value(s: String) -> Value { - let v: toml::Value = s.parse::().unwrap(); +pub fn from_toml_string_to_value(s: impl AsRef) -> Value { + let v: toml::Value = s.as_ref().parse::().unwrap(); convert_toml_value_to_nu_value(&v) } @@ -32,7 +39,7 @@ pub fn from_toml(args: CommandArgs) -> Result { Value::Primitive(Primitive::String(s)) => { ReturnValue::Value(from_toml_string_to_value(s)) } - _ => ReturnValue::Value(Value::Primitive(Primitive::String("".to_string()))), + _ => ReturnValue::Value(Value::Primitive(Primitive::String(String::from("")))), }) .boxed()) } diff --git a/src/commands/from_xml.rs b/src/commands/from_xml.rs index 83e964e09d..8a28e9bee8 100644 --- a/src/commands/from_xml.rs +++ b/src/commands/from_xml.rs @@ -15,7 +15,7 @@ fn from_node_to_value<'a, 'd>(n: &roxmltree::Node<'a, 'd>) -> Value { .filter(|x| match x { Value::Primitive(Primitive::String(f)) => { if f.trim() == "" { - false + false } else { true } @@ -32,13 +32,13 @@ fn from_node_to_value<'a, 'd>(n: &roxmltree::Node<'a, 'd>) -> Value { Value::Object(collected) } else if n.is_comment() { - Value::Primitive(Primitive::String("".to_string())) + Value::string("") } else if n.is_pi() { - Value::Primitive(Primitive::String("".to_string())) + Value::string("") } else if n.is_text() { - Value::Primitive(Primitive::String(n.text().unwrap().to_string())) + Value::string(n.text().unwrap()) } else { - Value::Primitive(Primitive::String("".to_string())) + Value::string("") } } @@ -46,8 +46,8 @@ fn from_document_to_value(d: &roxmltree::Document) -> Value { from_node_to_value(&d.root_element()) } -pub fn from_xml_string_to_value(s: String) -> Value { - match roxmltree::Document::parse(&s) { +pub fn from_xml_string_to_value(s: impl AsRef) -> Value { + match roxmltree::Document::parse(s.as_ref()) { Ok(doc) => from_document_to_value(&doc), Err(_) => Value::Error(Box::new(ShellError::string( "Can't convert string from xml".to_string(), @@ -59,7 +59,9 @@ pub fn from_xml(args: CommandArgs) -> Result { let out = args.input; Ok(out .map(|a| match a { - Value::Primitive(Primitive::String(s)) => ReturnValue::Value(from_xml_string_to_value(s)), + Value::Primitive(Primitive::String(s)) => { + ReturnValue::Value(from_xml_string_to_value(s)) + } _ => ReturnValue::Value(Value::Error(Box::new(ShellError::string( "Trying to convert XML from non-string".to_string(), )))), diff --git a/src/commands/from_yaml.rs b/src/commands/from_yaml.rs index 171ab3ae0d..12340cd6dc 100644 --- a/src/commands/from_yaml.rs +++ b/src/commands/from_yaml.rs @@ -11,7 +11,7 @@ fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value) -> Value { serde_yaml::Value::Number(n) if n.is_f64() => { Value::Primitive(Primitive::Float(OF64::from(n.as_f64().unwrap()))) } - serde_yaml::Value::String(s) => Value::Primitive(Primitive::String(s.clone())), + serde_yaml::Value::String(s) => Value::string(s), serde_yaml::Value::Sequence(a) => Value::List( a.iter() .map(|x| convert_yaml_value_to_nu_value(x)) @@ -36,8 +36,8 @@ fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value) -> Value { } } -pub fn from_yaml_string_to_value(s: String) -> Value { - let v: serde_yaml::Value = serde_yaml::from_str(&s).unwrap(); +pub fn from_yaml_string_to_value(s: impl AsRef) -> Value { + let v: serde_yaml::Value = serde_yaml::from_str(s.as_ref()).unwrap(); convert_yaml_value_to_nu_value(&v) } @@ -48,7 +48,7 @@ pub fn from_yaml(args: CommandArgs) -> Result { Value::Primitive(Primitive::String(s)) => { ReturnValue::Value(from_yaml_string_to_value(s)) } - _ => ReturnValue::Value(Value::Primitive(Primitive::String("".to_string()))), + _ => ReturnValue::Value(Value::Primitive(Primitive::String("".into()))), }) .boxed()) } diff --git a/src/commands/get.rs b/src/commands/get.rs index 37f83a695e..5811d4156b 100644 --- a/src/commands/get.rs +++ b/src/commands/get.rs @@ -1,6 +1,7 @@ use crate::errors::ShellError; use crate::object::Value; use crate::prelude::*; +use crate::Text; fn get_member(path: &str, obj: &Value) -> Option { let mut current = obj; @@ -44,13 +45,14 @@ pub fn get(args: CommandArgs) -> Result { .boxed()); } - let fields: Result, _> = args + let fields: Result, _> = args .args .positional .unwrap() .iter() .map(|a| a.as_string()) .collect(); + let fields = fields?; let stream = args diff --git a/src/commands/lines.rs b/src/commands/lines.rs index 420a1c0177..7764927eb9 100644 --- a/src/commands/lines.rs +++ b/src/commands/lines.rs @@ -19,7 +19,7 @@ pub fn lines(args: CommandArgs) -> Result { let mut result = VecDeque::new(); for s in split_result { result.push_back(ReturnValue::Value(Value::Primitive(Primitive::String( - s.to_string(), + s.into(), )))); } result diff --git a/src/commands/ls.rs b/src/commands/ls.rs index 405fa0fc37..9dc3c3354d 100644 --- a/src/commands/ls.rs +++ b/src/commands/ls.rs @@ -14,7 +14,7 @@ pub fn ls(args: CommandArgs) -> Result { Some(Spanned { item: Value::Primitive(Primitive::String(s)), .. - }) => full_path.push(Path::new(s)), + }) => full_path.push(Path::new(&s)), _ => {} } diff --git a/src/commands/open.rs b/src/commands/open.rs index 414d0b5863..527bdee04c 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -80,7 +80,7 @@ fn open(args: CommandArgs) -> Result { } } } else { - full_path.push(Path::new(&s)); + full_path.push(Path::new(s)); match std::fs::read_to_string(&full_path) { Ok(s) => ( full_path @@ -138,9 +138,7 @@ fn open(args: CommandArgs) -> Result { )); } _ => { - stream.push_back(ReturnValue::Value(Value::Primitive(Primitive::String( - contents, - )))); + stream.push_back(ReturnValue::Value(Value::string(contents))); } } diff --git a/src/commands/pick.rs b/src/commands/pick.rs index b5a17e0372..3d86fd72e6 100644 --- a/src/commands/pick.rs +++ b/src/commands/pick.rs @@ -16,7 +16,7 @@ pub fn pick(args: CommandArgs) -> Result { } } - let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let objects = args diff --git a/src/commands/reject.rs b/src/commands/reject.rs index 4a9f28a7ed..b30036fcf7 100644 --- a/src/commands/reject.rs +++ b/src/commands/reject.rs @@ -16,7 +16,7 @@ pub fn reject(args: CommandArgs) -> Result { } } - let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); + let fields: Result, _> = args.positional_iter().map(|a| a.as_string()).collect(); let fields = fields?; let stream = args diff --git a/src/commands/size.rs b/src/commands/size.rs index 59df8165c3..c99276375b 100644 --- a/src/commands/size.rs +++ b/src/commands/size.rs @@ -23,7 +23,7 @@ pub fn size(args: CommandArgs) -> Result { let mut list = VecDeque::new(); for name in args.positional_iter() { let name = name.as_string()?; - let path = cwd.join(&name); + let path = cwd.join(name.as_ref()); let mut file = File::open(path)?; file.read_to_string(&mut contents)?; list.push_back(count(&name, &contents)); @@ -59,7 +59,7 @@ fn count(name: &str, contents: &str) -> ReturnValue { } let mut dict = Dictionary::default(); - dict.add("name", Value::string(name.to_owned())); + dict.add("name", Value::string(name)); dict.add("lines", Value::int(lines)); dict.add("words", Value::int(words)); dict.add("chars", Value::int(chars)); diff --git a/src/commands/split_column.rs b/src/commands/split_column.rs index b546955793..32f1f75946 100644 --- a/src/commands/split_column.rs +++ b/src/commands/split_column.rs @@ -39,19 +39,16 @@ pub fn split_column(args: CommandArgs) -> Result { } let mut dict = crate::object::Dictionary::default(); - for (k, v) in split_result.iter().zip(gen_columns.iter()) { - dict.add( - v.clone(), - Value::Primitive(Primitive::String(k.to_string())), - ); + for (&k, v) in split_result.iter().zip(gen_columns.iter()) { + dict.add(v.clone(), Value::Primitive(Primitive::String(k.into()))); } ReturnValue::Value(Value::Object(dict)) } else if split_result.len() == (positional.len() - 1) { let mut dict = crate::object::Dictionary::default(); - for (k, v) in split_result.iter().zip(positional.iter().skip(1)) { + for (&k, v) in split_result.iter().zip(positional.iter().skip(1)) { dict.add( v.as_string().unwrap(), - Value::Primitive(Primitive::String(k.to_string())), + Value::Primitive(Primitive::String(k.into())), ); } ReturnValue::Value(Value::Object(dict)) @@ -60,7 +57,7 @@ pub fn split_column(args: CommandArgs) -> Result { for k in positional.iter().skip(1) { dict.add( k.as_string().unwrap().trim(), - Value::Primitive(Primitive::String("".to_string())), + Value::Primitive(Primitive::String("".into())), ); } ReturnValue::Value(Value::Object(dict)) diff --git a/src/commands/split_row.rs b/src/commands/split_row.rs index 80dbf29adc..1fb10e353b 100644 --- a/src/commands/split_row.rs +++ b/src/commands/split_row.rs @@ -35,7 +35,7 @@ pub fn split_row(args: CommandArgs) -> Result { let mut result = VecDeque::new(); for s in split_result { result.push_back(ReturnValue::Value(Value::Primitive(Primitive::String( - s.to_string(), + s.into(), )))); } result diff --git a/src/commands/to_json.rs b/src/commands/to_json.rs index fa8aadd0a6..eaae203084 100644 --- a/src/commands/to_json.rs +++ b/src/commands/to_json.rs @@ -4,6 +4,10 @@ use crate::prelude::*; pub fn to_json(args: CommandArgs) -> Result { let out = args.input; Ok(out - .map(|a| ReturnValue::Value(Value::Primitive(Primitive::String(serde_json::to_string(&a).unwrap())))) + .map(|a| { + ReturnValue::Value(Value::Primitive(Primitive::String( + serde_json::to_string(&a).unwrap().into(), + ))) + }) .boxed()) } diff --git a/src/commands/to_toml.rs b/src/commands/to_toml.rs index ac0fe05a6d..a7727e7279 100644 --- a/src/commands/to_toml.rs +++ b/src/commands/to_toml.rs @@ -4,6 +4,10 @@ use crate::prelude::*; pub fn to_toml(args: CommandArgs) -> Result { let out = args.input; Ok(out - .map(|a| ReturnValue::Value(Value::Primitive(Primitive::String(toml::to_string(&a).unwrap())))) + .map(|a| { + ReturnValue::Value(Value::Primitive(Primitive::String( + toml::to_string(&a).unwrap().into(), + ))) + }) .boxed()) } diff --git a/src/commands/trim.rs b/src/commands/trim.rs index 2974b4e65b..0ef840fea1 100644 --- a/src/commands/trim.rs +++ b/src/commands/trim.rs @@ -10,7 +10,7 @@ pub fn trim(args: CommandArgs) -> Result { Ok(input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - ReturnValue::Value(Value::Primitive(Primitive::String(s.trim().to_string()))) + ReturnValue::Value(Value::Primitive(Primitive::String(s.trim().into()))) } x => ReturnValue::Value(x), }) diff --git a/src/commands/view.rs b/src/commands/view.rs index a24b28ff74..71c4cdec09 100644 --- a/src/commands/view.rs +++ b/src/commands/view.rs @@ -46,7 +46,7 @@ pub fn view(args: CommandArgs) -> Result { .build() .map_err(|e| ShellError::string(e))?; - let file = cwd.join(target); + let file = cwd.join(target.as_ref()); let _ = printer.file(file.display().to_string()); diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index 7163bf6973..31958af787 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -27,7 +27,7 @@ crate fn evaluate_baseline_expr( expr: &Expression, registry: &dyn CommandRegistry, scope: &Scope, - source: &str, + source: &Text, ) -> Result, ShellError> { match &expr.item { RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(*literal), source)), @@ -45,7 +45,7 @@ crate fn evaluate_baseline_expr( } } RawExpression::Block(block) => Ok(Spanned::from_item( - Value::Block(Block::new(*block.clone(), Text::from(source))), // TODO: Pass Text around + Value::Block(Block::new(*block.clone(), source.clone())), block.span(), )), RawExpression::Path(path) => { @@ -67,7 +67,7 @@ crate fn evaluate_baseline_expr( } } -fn evaluate_literal(literal: Spanned, source: &str) -> Spanned { +fn evaluate_literal(literal: Spanned, source: &Text) -> Spanned { let result = match literal.item { hir::Literal::Integer(int) => Value::int(int), hir::Literal::Size(_int, _unit) => unimplemented!(), @@ -81,7 +81,7 @@ fn evaluate_literal(literal: Spanned, source: &str) -> Spanned Result, ShellError> { match name { hir::Variable::It(span) => Ok(Spanned::from_item(scope.it.copy(), span)), diff --git a/src/format/generic.rs b/src/format/generic.rs index 06032cf713..ba99123e8c 100644 --- a/src/format/generic.rs +++ b/src/format/generic.rs @@ -31,7 +31,7 @@ impl RenderView for GenericView<'value> { b @ Value::Block(_) => { let printed = b.format_leaf(None); - let view = EntriesView::from_value(&Value::string(&printed)); + let view = EntriesView::from_value(&Value::string(printed)); view.render_view(host)?; Ok(()) } diff --git a/src/main.rs b/src/main.rs index 5d5aaba469..ac4f251c32 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,8 @@ use clap::{App, Arg}; use log::LevelFilter; use std::error::Error; +crate use parser::parse2::text::Text; + fn main() -> Result<(), Box> { let matches = App::new("nu shell") .version("0.5") diff --git a/src/object/base.rs b/src/object/base.rs index e5c9bdfd24..18229c68b8 100644 --- a/src/object/base.rs +++ b/src/object/base.rs @@ -3,17 +3,16 @@ use crate::evaluate::{evaluate_baseline_expr, Scope}; use crate::object::DataDescriptor; use crate::parser::{hir, Operator, Spanned}; use crate::prelude::*; +use crate::Text; use ansi_term::Color; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; use derive_new::new; use ordered_float::OrderedFloat; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::fmt; use std::time::SystemTime; -use crate::parser::Text; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - #[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq, new)] pub struct OF64 { crate inner: OrderedFloat, @@ -141,7 +140,7 @@ impl Serialize for Block { where S: Serializer, { - serializer.serialize_str(&self.expression.source(self.source.as_ref())) + serializer.serialize_str(&self.expression.source(&self.source.clone())) } } @@ -162,7 +161,7 @@ impl Deserialize<'de> for Block { impl Block { pub fn invoke(&self, value: &Value) -> Result, ShellError> { let scope = Scope::new(value.copy()); - evaluate_baseline_expr(&self.expression, &(), &scope, self.source.as_ref()) + evaluate_baseline_expr(&self.expression, &(), &scope, &self.source) } } @@ -293,7 +292,7 @@ impl Value { crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String { match self { Value::Primitive(p) => p.format(desc), - Value::Block(b) => b.expression.source(b.source.as_ref()).to_string(), + Value::Block(b) => b.expression.source(&b.source).to_string(), Value::Object(_) => format!("[object Object]"), Value::List(_) => format!("[list List]"), Value::Error(e) => format!("{}", e), @@ -346,9 +345,9 @@ impl Value { } } - crate fn as_string(&self) -> Result { + crate fn as_string(&self) -> Result { match self { - Value::Primitive(Primitive::String(s)) => Ok(s.clone()), + Value::Primitive(Primitive::String(s)) => Ok(Text::from(s.clone())), // TODO: this should definitely be more general with better errors other => Err(ShellError::string(format!( "Expected string, got {:?}", @@ -456,7 +455,7 @@ impl Value { } } -crate fn select_fields(obj: &Value, fields: &[String]) -> crate::object::Dictionary { +crate fn select_fields(obj: &Value, fields: &[Text]) -> crate::object::Dictionary { let mut out = crate::object::Dictionary::default(); let descs = obj.data_descriptors(); @@ -471,7 +470,7 @@ crate fn select_fields(obj: &Value, fields: &[String]) -> crate::object::Diction out } -crate fn reject_fields(obj: &Value, fields: &[String]) -> crate::object::Dictionary { +crate fn reject_fields(obj: &Value, fields: &[Text]) -> crate::object::Dictionary { let mut out = crate::object::Dictionary::default(); let descs = obj.data_descriptors(); diff --git a/src/object/desc.rs b/src/object/desc.rs index d69705b466..7a7e24c765 100644 --- a/src/object/desc.rs +++ b/src/object/desc.rs @@ -1,4 +1,5 @@ use crate::object::types::Type; +use crate::Text; use derive_new::new; use serde::{Deserialize, Serialize, Serializer}; @@ -90,9 +91,19 @@ impl From for DataDescriptor { } } +impl From for DataDescriptor { + fn from(input: Text) -> DataDescriptor { + DataDescriptor { + name: DescriptorName::String(input.to_string()), + readonly: true, + ty: Type::Any, + } + } +} + impl DescriptorName { - crate fn for_string_name(name: impl Into) -> DescriptorName { - DescriptorName::String(name.into()) + crate fn for_string_name(name: impl AsRef) -> DescriptorName { + DescriptorName::String(name.as_ref().into()) } } @@ -113,7 +124,7 @@ impl DataDescriptor { } } - crate fn for_string_name(name: impl Into) -> DataDescriptor { + crate fn for_string_name(name: impl AsRef) -> DataDescriptor { DataDescriptor::for_name(DescriptorName::for_string_name(name)) } diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs index ae9240cd4c..2661fb7726 100644 --- a/src/parser/hir/baseline_parse.rs +++ b/src/parser/hir/baseline_parse.rs @@ -1,6 +1,7 @@ use crate::parser::{hir, RawToken, Token}; +use crate::Text; -pub fn baseline_parse_single_token(token: &Token, source: &str) -> hir::Expression { +pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression { match *token.item() { RawToken::Integer(int) => hir::Expression::int(int, token.span), RawToken::Size(int, unit) => hir::Expression::size(int, unit, token.span), diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index 39c1fcaee8..95c26904d7 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -1,11 +1,12 @@ use crate::errors::ShellError; use crate::parser::registry::CommandRegistry; use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode}; +use crate::Text; pub fn baseline_parse_tokens( token_nodes: &[TokenNode], registry: &dyn CommandRegistry, - source: &str, + source: &Text, ) -> Result, ShellError> { let mut exprs: Vec = vec![]; let mut rest = token_nodes; @@ -36,14 +37,9 @@ pub enum ExpressionKindHint { pub fn baseline_parse_next_expr( token_nodes: &'nodes [TokenNode], _registry: &dyn CommandRegistry, - source: &str, + source: &Text, coerce_hint: Option, ) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> { - println!( - "baseline_parse_next_expr {:?} - {:?}", - token_nodes, coerce_hint - ); - let mut tokens = token_nodes.iter().peekable(); let first = next_token(&mut tokens); @@ -131,7 +127,7 @@ pub fn baseline_parse_next_expr( pub fn baseline_parse_semantic_token( token: &TokenNode, - source: &str, + source: &Text, ) -> Result { match token { TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), diff --git a/src/parser/parse2/span.rs b/src/parser/parse2/span.rs index 928352d69b..e5905ea4df 100644 --- a/src/parser/parse2/span.rs +++ b/src/parser/parse2/span.rs @@ -1,3 +1,4 @@ +use crate::Text; use derive_new::new; use getset::Getters; @@ -40,8 +41,8 @@ impl Spanned { } } - pub fn source(&self, source: &'source str) -> &'source str { - self.span().slice(source) + pub fn source(&self, source: &Text) -> Text { + Text::from(self.span().slice(source)) } } @@ -95,7 +96,7 @@ impl From<&std::ops::Range> for Span { } impl Span { - pub fn slice(&self, source: &'source str) -> &'source str { + pub fn slice(&self, source: &'a str) -> &'a str { &source[self.start..self.end] } } diff --git a/src/parser/parse2/text.rs b/src/parser/parse2/text.rs index 7246392283..60f014d9eb 100644 --- a/src/parser/parse2/text.rs +++ b/src/parser/parse2/text.rs @@ -31,7 +31,7 @@ impl Text { /// Extract a new `Text` that is a subset of an old `Text` /// -- `text.extract(1..3)` is similar to `&foo[1..3]` except that /// it gives back an owned value instead of a borrowed value. - pub fn extract(&self, range: Range) -> Self { + pub fn slice(&self, range: Range) -> Self { let mut result = self.clone(); result.select(range); result diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index e618feaa93..098e024b60 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,5 +1,6 @@ use crate::errors::ShellError; use crate::parser::parse2::{call_node::*, flag::*, operator::*, span::*, tokens::*}; +use crate::Text; use derive_new::new; use enum_utils::FromStr; use getset::Getters; @@ -36,11 +37,11 @@ impl TokenNode { } } - pub fn as_external_arg(&self, source: &str) -> String { + pub fn as_external_arg(&self, source: &Text) -> String { self.span().slice(source).to_string() } - pub fn source(&self, source: &'source str) -> &'source str { + pub fn source(&self, source: &'a Text) -> &'a str { self.span().slice(source) } @@ -54,7 +55,7 @@ impl TokenNode { } } - crate fn as_flag(&self, value: &str, source: &str) -> Option> { + crate fn as_flag(&self, value: &str, source: &Text) -> Option> { match self { TokenNode::Flag( flag @ Spanned { diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index d3064cbae0..c9c49b1b84 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -5,13 +5,14 @@ use crate::parser::{ hir::{self, NamedArguments}, Flag, RawToken, TokenNode, }; +use crate::Text; use log::trace; pub fn parse_command( config: &CommandConfig, registry: &dyn CommandRegistry, call: &Spanned, - source: &str, + source: &Text, ) -> Result { let Spanned { item: call, .. } = call; @@ -64,7 +65,7 @@ fn parse_command_tail( config: &CommandConfig, registry: &dyn CommandRegistry, tail: Option>, - source: &str, + source: &Text, ) -> Result>, Option)>, ShellError> { let mut tail = match tail { None => return Ok(None), @@ -176,7 +177,7 @@ fn parse_command_tail( fn extract_switch( name: &str, mut tokens: Vec, - source: &str, + source: &Text, ) -> (Vec, Option) { let pos = tokens .iter() @@ -196,7 +197,7 @@ fn extract_switch( fn extract_mandatory( name: &str, mut tokens: Vec, - source: &str, + source: &Text, ) -> Result<(Vec, usize, Flag), ShellError> { let pos = tokens .iter() @@ -225,7 +226,7 @@ fn extract_mandatory( fn extract_optional( name: &str, mut tokens: Vec, - source: &str, + source: &Text, ) -> Result<(Vec, Option<(usize, Flag)>), ShellError> { let pos = tokens .iter() diff --git a/src/parser/registry.rs b/src/parser/registry.rs index d50c67aa43..6f786616bf 100644 --- a/src/parser/registry.rs +++ b/src/parser/registry.rs @@ -189,7 +189,7 @@ impl CommandConfig { call: &Spanned, registry: &dyn CommandRegistry, scope: &Scope, - source: &str, + source: &Text, ) -> Result { let args = parse_command(self, registry, call, source)?; @@ -282,7 +282,7 @@ fn evaluate_args( args: hir::Call, registry: &dyn CommandRegistry, scope: &Scope, - source: &str, + source: &Text, ) -> Result { let positional: Result>, _> = args .positional() diff --git a/src/prelude.rs b/src/prelude.rs index b9fec2540c..241b8aad4f 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,6 +6,7 @@ crate use crate::env::{Environment, Host}; crate use crate::errors::ShellError; crate use crate::object::Value; crate use crate::stream::{single_output, InputStream, OutputStream}; +crate use crate::Text; crate use futures::{FutureExt, StreamExt}; crate use std::collections::VecDeque; crate use std::pin::Pin; diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 85ad50de6a..c79b0cc89e 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -70,7 +70,7 @@ impl Highlighter for Helper { match iter.next() { None => return Cow::Owned(out), - Some(v) => out.push_str(v.span().slice(line)), + Some(v) => out.push_str(v.span().slice(&Text::from(line))), }; loop { From 6af4dafd871c8747beacd8f10eb55e558982a7cb Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sat, 22 Jun 2019 18:31:05 -0400 Subject: [PATCH 16/20] Update lockfile --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4ccd32ff63..1a17417ed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "nom_locate" version = "0.3.1" -source = "git+https://github.com/wycats/nom_locate.git?branch=nom5#b981e045fdaf29dbbf6e83803a01896cf280d862" +source = "git+https://github.com/wycats/nom_locate.git?branch=nom5#5baeede19605e2049994eca8f7b366ddc85bb9cd" dependencies = [ "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", From cbab97174e704e849a08e7fee7a412c097a76715 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sun, 23 Jun 2019 13:35:43 -0400 Subject: [PATCH 17/20] Fix painting --- Cargo.toml | 3 +- src/cli.rs | 6 +- src/parser.rs | 247 +----------------------- src/parser/ast/expression.rs | 1 + src/parser/hir/baseline_parse_tokens.rs | 11 +- src/parser/parse2.rs | 3 +- src/parser/parse2/parser.rs | 51 +++-- src/parser/parse2/pipeline.rs | 18 ++ src/parser/parse2/token_tree.rs | 17 +- src/parser/parse2/token_tree_builder.rs | 48 +++-- src/shell/helper.rs | 40 ++-- 11 files changed, 133 insertions(+), 312 deletions(-) create mode 100644 src/parser/parse2/pipeline.rs diff --git a/Cargo.toml b/Cargo.toml index 1d48b667c5..41049d3535 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -#rustyline = "4.1.0" rustyline = "5.0.0" sysinfo = "0.8.5" chrono = { version = "0.4.6", features = ["serde"] } @@ -59,7 +58,7 @@ ctrlc = "3.1.3" ptree = "0.2" clipboard = "0.5" reqwest = "0.9" -roxmltree = "0.6.0" +roxmltree = "0.6.1" pretty = "0.5.2" nom_locate = { git = "https://github.com/wycats/nom_locate.git", branch = "nom5" } derive_more = "0.15.0" diff --git a/src/cli.rs b/src/cli.rs index 8d2cca722b..52101ff6a3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -12,8 +12,8 @@ use crate::context::Context; crate use crate::errors::ShellError; use crate::evaluate::Scope; use crate::parser::parse2::span::Spanned; -use crate::parser::parse2::{PipelineElement, TokenNode}; use crate::parser::registry; +use crate::parser::{Pipeline, PipelineElement, TokenNode}; use crate::git::current_branch; use crate::object::Value; @@ -379,7 +379,9 @@ fn classify_pipeline( ) -> Result { let pipeline = pipeline.as_pipeline()?; - let commands: Result, ShellError> = pipeline + let Pipeline { parts, .. } = pipeline; + + let commands: Result, ShellError> = parts .iter() .map(|item| classify_command(&item, context, &source)) .collect(); diff --git a/src/parser.rs b/src/parser.rs index 3af5481c76..fcc299d4ec 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -11,6 +11,7 @@ crate use parse2::files::Files; crate use parse2::flag::Flag; crate use parse2::operator::Operator; crate use parse2::parser::{nom_input, pipeline}; +crate use parse2::pipeline::{Pipeline, PipelineElement}; crate use parse2::span::{Span, Spanned}; crate use parse2::text::Text; crate use parse2::token_tree::TokenNode; @@ -27,249 +28,3 @@ pub fn parse(input: &str) -> Result { Err(err) => Err(ShellError::parse_error(err)), } } - -// #[allow(unused)] -// pub fn parse_module(input: &str) -> Result { -// let _ = pretty_env_logger::try_init(); - -// let parser = ModuleParser::new(); -// let tokens = Lexer::new(input, false); - -// trace!( -// "Tokens: {:?}", -// tokens.clone().collect::, _>>() -// ); - -// match parser.parse(tokens) { -// Ok(val) => Ok(val), -// Err(err) => Err(ShellError::parse_error(err)), -// } -// } - -// #[cfg(test)] -// mod tests { -// use super::*; -// use crate::parser::ast::Pipeline; -// use pretty_assertions::assert_eq; - -// fn assert_parse(source: &str, expected: Pipeline) { -// let parsed = match parse(source) { -// Ok(p) => p, -// Err(ShellError::Diagnostic(diag)) => { -// use language_reporting::termcolor; - -// let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); -// let files = crate::parser::span::Files::new(source.to_string()); - -// language_reporting::emit( -// &mut writer.lock(), -// &files, -// &diag.diagnostic, -// &language_reporting::DefaultConfig, -// ) -// .unwrap(); - -// panic!("Test failed") -// } -// Err(err) => panic!("Something went wrong during parse: {:#?}", err), -// }; - -// let printed = parsed.print(); - -// assert_eq!(parsed, expected); -// assert_eq!(printed, source); - -// let span = expected.span; - -// let expected_module = ModuleBuilder::spanned_items( -// vec![Spanned::from_item(RawItem::Expression(expected), span)], -// span.start, -// span.end, -// ); - -// assert_parse_module(source, expected_module); -// } - -// fn assert_parse_module(source: &str, expected: Module) { -// let parsed = match parse_module(source) { -// Ok(p) => p, -// Err(ShellError::Diagnostic(diag)) => { -// use language_reporting::termcolor; - -// let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto); -// let files = crate::parser::span::Files::new(source.to_string()); - -// language_reporting::emit( -// &mut writer.lock(), -// &files, -// &diag.diagnostic, -// &language_reporting::DefaultConfig, -// ) -// .unwrap(); - -// panic!("Test failed") -// } -// Err(err) => panic!("Something went wrong during parse: {:#?}", err), -// }; - -// let printed = parsed.print(); - -// assert_eq!(parsed, expected); -// assert_eq!(printed, source); -// } - -// macro_rules! commands { -// ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{ -// use $crate::parser::ast::{Expression, ExpressionBuilder}; -// let mut builder = crate::parser::ast::ExpressionBuilder::new(); - -// builder.pipeline(vec![ -// $( -// (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression)) -// ),* -// ]) -// }} -// } - -// macro_rules! command { -// ($name:ident) => { -// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(( -// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)), -// vec![] -// )) -// }; - -// ($name:ident $( $command:ident ( $body:expr ) )*) => {{ -// use $crate::parser::ast::{Expression, ExpressionBuilder}; -// &|b: &mut ExpressionBuilder| b.call(( -// (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression), -// vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ])) - -// }}; - -// ($name:ident $( $command:ident ( $body:expr ) )*) => { -// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ]) -// }; - -// ($name:tt $( $command:ident ( $body:expr ) )*) => { -// &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ])) -// }; -// } - -// #[test] -// fn parse_simple_command() { -// assert_parse("ls", commands![(ls)]); -// } - -// #[test] -// fn parse_command_with_args() { -// assert_parse( -// r#"open Cargo.toml | select package.authors | split-row " ""#, -// commands![ -// (open bare("Cargo.toml")) -// | (select bare("package.authors")) -// | ("split-row" string(" ")) -// ], -// ); - -// assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]); - -// assert_parse( -// "open Cargo.toml | select package.version | echo $it", -// commands![ -// (open bare("Cargo.toml")) -// | (select bare("package.version")) -// | (echo var("it")) -// ], -// ); - -// assert_parse( -// "open Cargo.toml --raw", -// commands![(open bare("Cargo.toml") flag("raw"))], -// ); - -// assert_parse( -// "open Cargo.toml -r", -// commands![(open bare("Cargo.toml") shorthand("r"))], -// ); - -// assert_parse( -// "open Cargo.toml | from-toml | to-toml", -// commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")], -// ); - -// assert_parse( -// r#"config --get "ignore dups" | format-list"#, -// commands![(config flag("get") string("ignore dups")) | ("format-list")], -// ); - -// assert_parse( -// "open Cargo.toml | from-toml | select dependencies | column serde", -// commands![ -// (open bare("Cargo.toml")) -// | ("from-toml") -// | (select bare("dependencies")) -// | (column bare("serde")) -// ], -// ); - -// assert_parse( -// "config --set tabs 2", -// commands![(config flag("set") bare("tabs") int(2))], -// ); - -// assert_parse( -// r#"ls | skip 1 | first 2 | select "file name" | rm $it"#, -// commands![ -// (ls) -// | (skip int(1)) -// | (first int(2)) -// | (select string("file name")) -// | (rm var("it")) -// ], -// ); - -// assert_parse( -// r#"git branch --merged | split-row "`n" | where $it != "* master""#, -// commands![ -// // TODO: Handle escapes correctly. Should we do ` escape because of paths? -// (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master")))) -// ], -// ); - -// assert_parse( -// r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#, -// commands![ -// (open bare("input2.json")) -// | ("from-json") -// | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso")) -// | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML")))) -// ] -// ); - -// assert_parse( -// r"cd ..\.cargo\", -// commands![ -// (cd bare(r"..\.cargo\")) -// ], -// ); - -// assert_parse( -// "ls | where size < 1KB", -// commands![ -// (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB"))))) -// ], -// ); - -// assert_parse( -// "ls | where { $it.size > 100 }", -// commands![ -// (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100))))) -// ], -// ) -// } - -// use crate::parser::ast::{ModuleBuilder, RawItem}; -// use crate::parser::lexer::Spanned; - -// } diff --git a/src/parser/ast/expression.rs b/src/parser/ast/expression.rs index 8eb35be6fe..9d2c9833ca 100644 --- a/src/parser/ast/expression.rs +++ b/src/parser/ast/expression.rs @@ -524,6 +524,7 @@ impl Call { #[derive(new, Debug, Eq, PartialEq, Clone)] pub struct Pipeline { crate commands: Vec, + crate trailing: Span, crate span: Span, } diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index ef88f0d460..45826f2076 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -98,7 +98,10 @@ pub fn baseline_parse_next_expr( vec![string], ); let path = hir::RawExpression::Path(Box::new(path)); - Spanned { item: path, span: first.span } + Spanned { + item: path, + span: first.span, + } } Spanned { item: hir::RawExpression::Literal(hir::Literal::String(inner)), @@ -114,7 +117,10 @@ pub fn baseline_parse_next_expr( vec![string], ); let path = hir::RawExpression::Path(Box::new(path)); - Spanned { item: path, span: first.span } + Spanned { + item: path, + span: first.span, + } } Spanned { item: hir::RawExpression::Variable(..), @@ -157,6 +163,7 @@ pub fn baseline_parse_semantic_token( TokenNode::Whitespace(_span) => unreachable!(), TokenNode::Error(error) => Err(*error.item.clone()), TokenNode::Path(_path) => unimplemented!(), + TokenNode::EOF(_span) => unimplemented!(), } } diff --git a/src/parser/parse2.rs b/src/parser/parse2.rs index b841c4e5fa..ed2ffd8928 100644 --- a/src/parser/parse2.rs +++ b/src/parser/parse2.rs @@ -3,6 +3,7 @@ crate mod files; crate mod flag; crate mod operator; crate mod parser; +crate mod pipeline; crate mod span; crate mod text; crate mod token_tree; @@ -10,5 +11,3 @@ crate mod token_tree_builder; crate mod tokens; crate mod unit; crate mod util; - -crate use token_tree::{PipelineElement, TokenNode}; diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 81d5293642..5260a6fccc 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -1,8 +1,8 @@ #![allow(unused)] use crate::parser::parse2::{ - call_node::*, flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, - unit::*, + call_node::*, flag::*, operator::*, pipeline::*, span::*, token_tree::*, token_tree_builder::*, + tokens::*, unit::*, }; use nom; use nom::branch::*; @@ -429,34 +429,63 @@ pub fn node(input: NomSpan) -> IResult { ) } +pub fn eof(input: NomSpan) -> IResult { + if input.input_len() == 0 { + Ok((input, TokenNode::EOF(Span::from(input)))) + } else { + Err(Err::Error(error_position!( + input, + nom::error::ErrorKind::Eof + ))) + } +} + pub fn pipeline(input: NomSpan) -> IResult { trace_step(input, "pipeline", |input| { let start = input.offset; - let (input, head) = tuple((raw_call, opt(space1)))(input)?; + let (input, head) = opt(tuple((raw_call, opt(space1), opt(tag("|")))))(input)?; let (input, items) = trace_step( input, "many0", - many0(tuple((tag("|"), opt(space1), raw_call, opt(space1)))), + many0(tuple((opt(space1), raw_call, opt(space1), opt(tag("|"))))), )?; + + let (input, tail) = tuple((opt(space1), eof))(input)?; let end = input.offset; Ok(( input, - TokenTreeBuilder::spanned_pipeline(make_call_list(head, items), (start, end)), + TokenTreeBuilder::spanned_pipeline( + (make_call_list(head, items), tail.0.map(Span::from)), + (start, end), + ), )) }) } fn make_call_list( - head: (Spanned, Option), - tail: Vec<(NomSpan, Option, Spanned, Option)>, + head: Option<(Spanned, Option, Option)>, + items: Vec<( + Option, + Spanned, + Option, + Option, + )>, ) -> Vec { let mut out = vec![]; - let el = PipelineElement::new(None, head.0, head.1.map(Span::from)); - out.push(el); - for (pipe, ws1, call, ws2) in tail { - let el = PipelineElement::new(ws1.map(Span::from), call, ws2.map(Span::from)); + if let Some(head) = head { + let el = PipelineElement::new(None, head.0, head.1.map(Span::from), head.2.map(Span::from)); + out.push(el); + } + + for (ws1, call, ws2, pipe) in items { + let el = PipelineElement::new( + ws1.map(Span::from), + call, + ws2.map(Span::from), + pipe.map(Span::from), + ); out.push(el); } diff --git a/src/parser/parse2/pipeline.rs b/src/parser/parse2/pipeline.rs new file mode 100644 index 0000000000..8fe8938841 --- /dev/null +++ b/src/parser/parse2/pipeline.rs @@ -0,0 +1,18 @@ +use crate::parser::{CallNode, Span, Spanned}; +use derive_new::new; +use getset::Getters; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +pub struct Pipeline { + crate parts: Vec, + crate post_ws: Option, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] +pub struct PipelineElement { + pub pre_ws: Option, + #[get = "crate"] + call: Spanned, + pub post_ws: Option, + pub post_pipe: Option, +} diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index a45418dfdb..c0a7778f42 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,9 +1,8 @@ use crate::errors::ShellError; -use crate::parser::parse2::{call_node::*, flag::*, operator::*, span::*, tokens::*}; +use crate::parser::parse2::{call_node::*, flag::*, operator::*, pipeline::*, span::*, tokens::*}; use crate::Text; use derive_new::new; use enum_utils::FromStr; -use getset::Getters; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { @@ -11,7 +10,7 @@ pub enum TokenNode { #[allow(unused)] Call(Spanned), Delimited(Spanned), - Pipeline(Spanned>), + Pipeline(Spanned), Operator(Spanned), Flag(Spanned), Identifier(Span), @@ -19,6 +18,7 @@ pub enum TokenNode { #[allow(unused)] Error(Spanned>), Path(Spanned), + EOF(Span), } impl TokenNode { @@ -34,6 +34,7 @@ impl TokenNode { TokenNode::Whitespace(s) => *s, TokenNode::Error(s) => s.span, TokenNode::Path(s) => s.span, + TokenNode::EOF(s) => *s, } } @@ -66,7 +67,7 @@ impl TokenNode { } } - pub fn as_pipeline(&self) -> Result, ShellError> { + pub fn as_pipeline(&self) -> Result { match self { TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()), _ => Err(ShellError::string("unimplemented")), @@ -92,11 +93,3 @@ pub struct PathNode { head: Box, tail: Vec, } - -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] -pub struct PipelineElement { - pub pre_ws: Option, - #[get = "crate"] - call: Spanned, - pub post_ws: Option, -} diff --git a/src/parser/parse2/token_tree_builder.rs b/src/parser/parse2/token_tree_builder.rs index 08740d7b32..f962691c41 100644 --- a/src/parser/parse2/token_tree_builder.rs +++ b/src/parser/parse2/token_tree_builder.rs @@ -3,10 +3,9 @@ use crate::prelude::*; use crate::parser::parse2::flag::{Flag, FlagKind}; use crate::parser::parse2::operator::Operator; +use crate::parser::parse2::pipeline::{Pipeline, PipelineElement}; use crate::parser::parse2::span::{Span, Spanned}; -use crate::parser::parse2::token_tree::{ - DelimitedNode, Delimiter, PathNode, PipelineElement, TokenNode, -}; +use crate::parser::parse2::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; use crate::parser::parse2::tokens::{RawToken, Token}; use crate::parser::parse2::unit::Unit; use crate::parser::CallNode; @@ -47,7 +46,7 @@ impl TokenTreeBuilder { let mut out: Vec = vec![]; - let mut input = input.into_iter(); + let mut input = input.into_iter().peekable(); let (pre, call, post) = input .next() .expect("A pipeline must contain at least one element"); @@ -55,33 +54,48 @@ impl TokenTreeBuilder { let pre_span = pre.map(|pre| b.consume(&pre)); let call = call(b); let post_span = post.map(|post| b.consume(&post)); + let pipe = input.peek().map(|_| Span::from(b.consume("|"))); out.push(PipelineElement::new( pre_span.map(Span::from), call, post_span.map(Span::from), + pipe, )); - for (pre, call, post) in input { - b.consume("|"); - let pre_span = pre.map(|pre| b.consume(&pre)); - let call = call(b); - let post_span = post.map(|post| b.consume(&post)); + loop { + match input.next() { + None => break, + Some((pre, call, post)) => { + let pre_span = pre.map(|pre| b.consume(&pre)); + let call = call(b); + let post_span = post.map(|post| b.consume(&post)); - out.push(PipelineElement::new( - pre_span.map(Span::from), - call, - post_span.map(Span::from), - )); + let pipe = input.peek().map(|_| Span::from(b.consume("|"))); + + out.push(PipelineElement::new( + pre_span.map(Span::from), + call, + post_span.map(Span::from), + pipe, + )); + } + } } let end = b.pos; - TokenTreeBuilder::spanned_pipeline(out, (start, end)) + TokenTreeBuilder::spanned_pipeline((out, None), (start, end)) }) } - pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { - TokenNode::Pipeline(Spanned::from_item(input, span)) + pub fn spanned_pipeline( + input: (Vec, Option), + span: impl Into, + ) -> TokenNode { + TokenNode::Pipeline(Spanned::from_item( + Pipeline::new(input.0, input.1.into()), + span, + )) } pub fn op(input: impl Into) -> CurriedToken { diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 08341e6f64..78444cf37d 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -2,7 +2,7 @@ use crate::parser::nom_input; use crate::parser::parse2::span::Spanned; use crate::parser::parse2::token_tree::TokenNode; use crate::parser::parse2::tokens::RawToken; -use crate::parser::parse2::PipelineElement; +use crate::parser::{Pipeline, PipelineElement}; use crate::prelude::*; use crate::shell::completer::NuCompleter; use ansi_term::Color; @@ -61,34 +61,31 @@ impl Highlighter for Helper { let tokens = crate::parser::pipeline(nom_input(line)); match tokens { - Err(_) => Cow::Borrowed(line), + Err(e) => { + println!("error: {:?}", e); + Cow::Borrowed(line) + } Ok((_rest, v)) => { let mut out = String::new(); - let tokens = match v.as_pipeline() { + let pipeline = match v.as_pipeline() { Err(_) => return Cow::Borrowed(line), Ok(v) => v, }; - let mut iter = tokens.into_iter(); - - // match iter.next() { - // None => return Cow::Owned(out), - // Some(v) => out.push_str(v.span().slice(line)), - // }; - let mut first = true; + let Pipeline { parts, post_ws } = pipeline; + let mut iter = parts.into_iter(); loop { match iter.next() { - None => return Cow::Owned(out), - Some(token) => { - let styled = paint_pipeline_element(&token, line); - - if !first { - out.push_str("|"); - } else { - first = false; + None => { + if let Some(ws) = post_ws { + out.push_str(ws.slice(line)); } + return Cow::Owned(out); + } + Some(token) => { + let styled = paint_pipeline_element(&token, line); out.push_str(&styled.to_string()); } } @@ -133,6 +130,7 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String { item: RawToken::Bare, .. }) => Color::Green.normal().paint(token_node.span().slice(line)), + TokenNode::EOF(_) => return format!(""), }; styled.to_string() @@ -144,6 +142,7 @@ fn paint_pipeline_element(pipeline_element: &PipelineElement, line: &str) -> Str if let Some(ws) = pipeline_element.pre_ws { styled.push_str(&Color::White.normal().paint(ws.slice(line))); } + styled.push_str(&paint_token_node(pipeline_element.call().head(), line)); if let Some(children) = pipeline_element.call().children() { @@ -151,10 +150,15 @@ fn paint_pipeline_element(pipeline_element: &PipelineElement, line: &str) -> Str styled.push_str(&paint_token_node(child, line)); } } + if let Some(ws) = pipeline_element.post_ws { styled.push_str(&Color::White.normal().paint(ws.slice(line))); } + if let Some(_) = pipeline_element.post_pipe { + styled.push_str(&Color::Purple.paint("|")); + } + styled.to_string() } From bed5ba52d39141781d37e2d905a744006b1a43c6 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sun, 23 Jun 2019 15:32:26 -0400 Subject: [PATCH 18/20] Fixed trailing issues --- src/parser/hir/baseline_parse_tokens.rs | 1 - src/parser/parse2/parser.rs | 24 +++++++++++------------- src/parser/parse2/token_tree.rs | 2 -- src/shell/helper.rs | 1 - 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index 45826f2076..81222f66f9 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -163,7 +163,6 @@ pub fn baseline_parse_semantic_token( TokenNode::Whitespace(_span) => unreachable!(), TokenNode::Error(error) => Err(*error.item.clone()), TokenNode::Path(_path) => unimplemented!(), - TokenNode::EOF(_span) => unimplemented!(), } } diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 5260a6fccc..2a069bd552 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -429,17 +429,6 @@ pub fn node(input: NomSpan) -> IResult { ) } -pub fn eof(input: NomSpan) -> IResult { - if input.input_len() == 0 { - Ok((input, TokenNode::EOF(Span::from(input)))) - } else { - Err(Err::Error(error_position!( - input, - nom::error::ErrorKind::Eof - ))) - } -} - pub fn pipeline(input: NomSpan) -> IResult { trace_step(input, "pipeline", |input| { let start = input.offset; @@ -450,13 +439,22 @@ pub fn pipeline(input: NomSpan) -> IResult { many0(tuple((opt(space1), raw_call, opt(space1), opt(tag("|"))))), )?; - let (input, tail) = tuple((opt(space1), eof))(input)?; + let (input, tail) = opt(space1)(input)?; + let (input, newline) = opt(multispace1)(input)?; + + if input.input_len() != 0 { + return Err(Err::Error(error_position!( + input, + nom::error::ErrorKind::Eof + ))); + } + let end = input.offset; Ok(( input, TokenTreeBuilder::spanned_pipeline( - (make_call_list(head, items), tail.0.map(Span::from)), + (make_call_list(head, items), tail.map(Span::from)), (start, end), ), )) diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index c0a7778f42..e11295bba4 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -18,7 +18,6 @@ pub enum TokenNode { #[allow(unused)] Error(Spanned>), Path(Spanned), - EOF(Span), } impl TokenNode { @@ -34,7 +33,6 @@ impl TokenNode { TokenNode::Whitespace(s) => *s, TokenNode::Error(s) => s.span, TokenNode::Path(s) => s.span, - TokenNode::EOF(s) => *s, } } diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 78444cf37d..114fd3a1f5 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -130,7 +130,6 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String { item: RawToken::Bare, .. }) => Color::Green.normal().paint(token_node.span().slice(line)), - TokenNode::EOF(_) => return format!(""), }; styled.to_string() From 7957fc502f45920b88972f7761e96a8cba289805 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sun, 23 Jun 2019 18:55:31 -0600 Subject: [PATCH 19/20] Fix a bunch of bugs --- src/cli.rs | 106 +++++++++++++++--------- src/errors.rs | 78 ++++++++++++++++- src/evaluate/evaluator.rs | 29 ++++--- src/object/base.rs | 52 ++++++------ src/object/desc.rs | 2 +- src/object/dict.rs | 31 +++++-- src/object/types.rs | 2 +- src/parser/hir/baseline_parse_tokens.rs | 2 +- src/parser/parse2/span.rs | 7 +- src/shell/helper.rs | 5 +- 10 files changed, 217 insertions(+), 97 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 52101ff6a3..a4f7c2de9b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -156,45 +156,60 @@ pub async fn cli() -> Result<(), Box> { LineResult::Error(mut line, err) => { rl.add_history_entry(line.clone()); - match err { - ShellError::Diagnostic(diag) => { - let host = context.host.lock().unwrap(); - let writer = host.err_termcolor(); - line.push_str(" "); - let files = crate::parser::Files::new(line); - language_reporting::emit( - &mut writer.lock(), - &files, - &diag.diagnostic, - &language_reporting::DefaultConfig, - ) - .unwrap(); - } + let diag = err.to_diagnostic(); + let host = context.host.lock().unwrap(); + let writer = host.err_termcolor(); + line.push_str(" "); + let files = crate::parser::Files::new(line); - ShellError::TypeError(desc) => context - .host - .lock() - .unwrap() - .stdout(&format!("TypeError: {}", desc)), + language_reporting::emit( + &mut writer.lock(), + &files, + &diag, + &language_reporting::DefaultConfig, + ) + .unwrap(); - ShellError::MissingProperty { subpath, .. } => context - .host - .lock() - .unwrap() - .stdout(&format!("Missing property {}", subpath)), + // match err { + // ShellError::Diagnostic(diag) => { + // let host = context.host.lock().unwrap(); + // let writer = host.err_termcolor(); + // line.push_str(" "); + // let files = crate::parser::Files::new(line); - ShellError::String(_) => { - context.host.lock().unwrap().stdout(&format!("{}", err)) - } - } + // language_reporting::emit( + // &mut writer.lock(), + // &files, + // &diag.diagnostic, + // &language_reporting::DefaultConfig, + // ) + // .unwrap(); + // } + + // ShellError::TypeError(desc) => context + // .host + // .lock() + // .unwrap() + // .stdout(&format!("TypeError: {}", desc)), + + // ShellError::MissingProperty { subpath, .. } => context + // .host + // .lock() + // .unwrap() + // .stdout(&format!("Missing property {}", subpath)), + + // ShellError::String(_) => { + // context.host.lock().unwrap().stdout(&format!("{}", err)) + // } + // } } LineResult::Break => { break; } - LineResult::FatalError(err) => { + LineResult::FatalError(_, err) => { context .host .lock() @@ -216,24 +231,24 @@ enum LineResult { Break, #[allow(unused)] - FatalError(ShellError), + FatalError(String, ShellError), } impl std::ops::Try for LineResult { type Ok = Option; - type Error = ShellError; + type Error = (String, ShellError); - fn into_result(self) -> Result, ShellError> { + fn into_result(self) -> Result, (String, ShellError)> { match self { LineResult::Success(s) => Ok(Some(s)), - LineResult::Error(_, s) => Err(s), + LineResult::Error(string, err) => Err((string, err)), LineResult::Break => Ok(None), LineResult::CtrlC => Ok(None), - LineResult::FatalError(err) => Err(err), + LineResult::FatalError(string, err) => Err((string, err)), } } - fn from_error(v: ShellError) -> Self { - LineResult::Error(String::new(), v) + fn from_error(v: (String, ShellError)) -> Self { + LineResult::Error(v.0, v.1) } fn from_ok(v: Option) -> Self { @@ -260,7 +275,8 @@ async fn process_line(readline: Result, ctx: &mut Context debug!("=== Parsed ==="); debug!("{:#?}", result); - let mut pipeline = classify_pipeline(&result, ctx, &Text::from(line))?; + let mut pipeline = classify_pipeline(&result, ctx, &Text::from(line)) + .map_err(|err| (line.clone(), err))?; match pipeline.commands.last() { Some(ClassifiedCommand::Sink(_)) => {} @@ -438,7 +454,13 @@ fn classify_command( //Some(args) => args.iter().map(|i| i.as_external_arg(source)).collect(), Some(args) => args .iter() - .map(|i| Spanned::from_item(i.as_external_arg(source), i.span())) + .filter_map(|i| match i { + TokenNode::Whitespace(_) => None, + other => Some(Spanned::from_item( + other.as_external_arg(source), + other.span(), + )), + }) .collect(), None => vec![], }; @@ -453,8 +475,12 @@ fn classify_command( } } - _ => Err(ShellError::unimplemented( - "classify_command on command whose head is not bare", + call => Err(ShellError::diagnostic( + language_reporting::Diagnostic::new( + language_reporting::Severity::Error, + "Invalid command", + ) + .with_label(language_reporting::Label::new_primary(call.head().span())), )), } } diff --git a/src/errors.rs b/src/errors.rs index 88d5e78fd3..506aaaa0c6 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,17 +1,51 @@ #[allow(unused)] use crate::prelude::*; -use crate::parser::Span; +use crate::parser::{Span, Spanned}; use derive_new::new; use language_reporting::{Diagnostic, Label, Severity}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] +pub enum Description { + Source(Spanned), + Synthetic(String), +} + +impl Description { + pub fn from(item: Spanned>) -> Description { + match item { + Spanned { + span: Span { start: 0, end: 0 }, + item, + } => Description::Synthetic(item.into()), + Spanned { span, item } => Description::Source(Spanned::from_item(item.into(), span)), + } + } +} + +impl Description { + fn into_label(self) -> Result, String> { + match self { + Description::Source(s) => Ok(Label::new_primary(s.span).with_message(s.item)), + Description::Synthetic(s) => Err(s), + } + } +} + #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] pub enum ShellError { String(StringError), - TypeError(String), - MissingProperty { subpath: String, expr: String }, + TypeError(Spanned), + MissingProperty { + subpath: Description, + expr: Description, + }, Diagnostic(ShellDiagnostic), + CoerceError { + left: Spanned, + right: Spanned, + }, } impl ShellError { @@ -24,7 +58,7 @@ impl ShellError { nom::Err::Incomplete(_) => unreachable!(), nom::Err::Failure(span) | nom::Err::Error(span) => { let diagnostic = - Diagnostic::new(Severity::Error, format!("{:?}", span)) + Diagnostic::new(Severity::Error, format!("Parse Error")) .with_label(Label::new_primary(Span::from(span.0))); ShellError::diagnostic(diagnostic) @@ -57,6 +91,41 @@ impl ShellError { ShellError::Diagnostic(ShellDiagnostic { diagnostic }) } + crate fn to_diagnostic(self) -> Diagnostic { + match self { + ShellError::String(StringError { title, .. }) => { + Diagnostic::new(Severity::Error, title) + } + ShellError::TypeError(s) => Diagnostic::new(Severity::Error, "Type Error") + .with_label(Label::new_primary(s.span).with_message(s.item)), + + ShellError::MissingProperty { subpath, expr } => { + let subpath = subpath.into_label(); + let expr = expr.into_label(); + + let mut diag = Diagnostic::new(Severity::Error, "Missing property"); + + match subpath { + Ok(label) => diag = diag.with_label(label), + Err(ty) => diag.message = format!("Missing property (for {})", ty), + } + + if let Ok(label) = expr { + diag = diag.with_label(label); + } + + diag + } + + ShellError::Diagnostic(diag) => diag.diagnostic, + ShellError::CoerceError { left, right } => { + Diagnostic::new(Severity::Error, "Coercion error") + .with_label(Label::new_primary(left.span).with_message(left.item)) + .with_label(Label::new_secondary(right.span).with_message(right.item)) + } + } + } + crate fn labeled_error( msg: impl Into, label: impl Into, @@ -177,6 +246,7 @@ impl std::fmt::Display for ShellError { ShellError::TypeError { .. } => write!(f, "TypeError"), ShellError::MissingProperty { .. } => write!(f, "MissingProperty"), ShellError::Diagnostic(_) => write!(f, ""), + ShellError::CoerceError { .. } => write!(f, "CoerceError"), } } } diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index 31958af787..16da0ad78c 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -1,3 +1,4 @@ +use crate::errors::Description; use crate::object::base::Block; use crate::parser::{ hir::{self, Expression, RawExpression}, @@ -37,11 +38,11 @@ crate fn evaluate_baseline_expr( let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?; match left.compare(binary.op(), &*right) { - Some(result) => Ok(Spanned::from_item(Value::boolean(result), *expr.span())), - None => Err(ShellError::unimplemented(&format!( - "Comparison failure {:?}", - binary - ))), + Ok(result) => Ok(Spanned::from_item(Value::boolean(result), *expr.span())), + Err((left_type, right_type)) => Err(ShellError::CoerceError { + left: binary.left().copy_span(left_type), + right: binary.right().copy_span(right_type), + }), } } RawExpression::Block(block) => Ok(Spanned::from_item( @@ -50,18 +51,26 @@ crate fn evaluate_baseline_expr( )), RawExpression::Path(path) => { let value = evaluate_baseline_expr(path.head(), registry, scope, source)?; - let mut value = value.item(); + let mut item = value; for name in path.tail() { - let next = value.get_data_by_key(name); + let next = item.get_data_by_key(name); match next { - None => return Err(ShellError::unimplemented("Invalid property from path")), - Some(next) => value = next, + None => { + return Err(ShellError::MissingProperty { + subpath: Description::from(item.spanned_type_name()), + expr: Description::from(name.clone()), + }) + } + Some(next) => { + item = + Spanned::from_item(next.clone(), (expr.span().start, name.span().end)) + } }; } - Ok(Spanned::from_item(value.clone(), expr.span())) + Ok(Spanned::from_item(item.item().clone(), expr.span())) } RawExpression::Boolean(_boolean) => unimplemented!(), } diff --git a/src/object/base.rs b/src/object/base.rs index 81ecf00eae..2a2a5b3c8d 100644 --- a/src/object/base.rs +++ b/src/object/base.rs @@ -165,7 +165,7 @@ impl Block { } } -#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)] +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Clone)] pub enum Value { Primitive(Primitive), Object(crate::object::Dictionary), @@ -211,6 +211,13 @@ impl fmt::Debug for ValueDebug<'a> { } } +impl Spanned { + crate fn spanned_type_name(&self) -> Spanned { + let name = self.type_name(); + Spanned::from_item(name, self.span) + } +} + impl Value { crate fn type_name(&self) -> String { match self { @@ -301,7 +308,7 @@ impl Value { } #[allow(unused)] - crate fn compare(&self, operator: &Operator, other: &Value) -> Option { + crate fn compare(&self, operator: &Operator, other: &Value) -> Result { match operator { _ => { let coerced = coerce_compare(self, other)?; @@ -322,7 +329,7 @@ impl Value { _ => false, }; - Some(result) + Ok(result) } } } @@ -383,18 +390,6 @@ impl Value { } } - #[allow(unused)] - crate fn as_bool(&self) -> Result { - match self { - Value::Primitive(Primitive::Boolean(b)) => Ok(*b), - // TODO: this should definitely be more general with better errors - other => Err(ShellError::string(format!( - "Expected integer, got {:?}", - other - ))), - } - } - crate fn is_true(&self) -> bool { match self { Value::Primitive(Primitive::Boolean(true)) => true, @@ -591,24 +586,27 @@ impl CompareValues { } } -fn coerce_compare(left: &Value, right: &Value) -> Option { +fn coerce_compare(left: &Value, right: &Value) -> Result { match (left, right) { (Value::Primitive(left), Value::Primitive(right)) => coerce_compare_primitive(left, right), - _ => None, + _ => Err((left.type_name(), right.type_name())), } } -fn coerce_compare_primitive(left: &Primitive, right: &Primitive) -> Option { +fn coerce_compare_primitive( + left: &Primitive, + right: &Primitive, +) -> Result { use Primitive::*; - match (left, right) { - (Int(left), Int(right)) => Some(CompareValues::Ints(*left, *right)), - (Float(left), Int(right)) => Some(CompareValues::Floats(*left, (*right as f64).into())), - (Int(left), Float(right)) => Some(CompareValues::Floats((*left as f64).into(), *right)), - (Int(left), Bytes(right)) => Some(CompareValues::Bytes(*left as i128, *right as i128)), - (Bytes(left), Int(right)) => Some(CompareValues::Bytes(*left as i128, *right as i128)), - (String(left), String(right)) => Some(CompareValues::String(left.clone(), right.clone())), - _ => None, - } + Ok(match (left, right) { + (Int(left), Int(right)) => CompareValues::Ints(*left, *right), + (Float(left), Int(right)) => CompareValues::Floats(*left, (*right as f64).into()), + (Int(left), Float(right)) => CompareValues::Floats((*left as f64).into(), *right), + (Int(left), Bytes(right)) => CompareValues::Bytes(*left as i128, *right as i128), + (Bytes(left), Int(right)) => CompareValues::Bytes(*left as i128, *right as i128), + (String(left), String(right)) => CompareValues::String(left.clone(), right.clone()), + _ => return Err((left.type_name(), right.type_name())), + }) } diff --git a/src/object/desc.rs b/src/object/desc.rs index 7a7e24c765..a3d7784a63 100644 --- a/src/object/desc.rs +++ b/src/object/desc.rs @@ -39,7 +39,7 @@ impl DescriptorName { } } -#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Hash, new)] +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, new)] pub struct DataDescriptor { crate name: DescriptorName, crate readonly: bool, diff --git a/src/object/dict.rs b/src/object/dict.rs index 3cd13261b4..146967c81c 100644 --- a/src/object/dict.rs +++ b/src/object/dict.rs @@ -15,9 +15,18 @@ pub struct Dictionary { } impl PartialOrd for Dictionary { - // TODO: FIXME - fn partial_cmp(&self, _other: &Dictionary) -> Option { - Some(Ordering::Less) + fn partial_cmp(&self, other: &Dictionary) -> Option { + let this: Vec<&DataDescriptor> = self.entries.keys().collect(); + let that: Vec<&DataDescriptor> = other.entries.keys().collect(); + + if this != that { + return this.partial_cmp(&that); + } + + let this: Vec<&Value> = self.entries.values().collect(); + let that: Vec<&Value> = self.entries.values().collect(); + + this.partial_cmp(&that) } } @@ -56,9 +65,18 @@ impl From> for Dictionary { } impl Ord for Dictionary { - // TODO: FIXME - fn cmp(&self, _other: &Dictionary) -> Ordering { - Ordering::Less + fn cmp(&self, other: &Dictionary) -> Ordering { + let this: Vec<&DataDescriptor> = self.entries.keys().collect(); + let that: Vec<&DataDescriptor> = other.entries.keys().collect(); + + if this != that { + return this.cmp(&that); + } + + let this: Vec<&Value> = self.entries.values().collect(); + let that: Vec<&Value> = self.entries.values().collect(); + + this.cmp(&that) } } @@ -69,7 +87,6 @@ impl PartialOrd for Dictionary { } impl PartialEq for Dictionary { - // TODO: FIXME fn eq(&self, other: &Value) -> bool { match other { Value::Object(d) => self == d, diff --git a/src/object/types.rs b/src/object/types.rs index 2f1bc2e4e6..34055e0b0d 100644 --- a/src/object/types.rs +++ b/src/object/types.rs @@ -1,7 +1,7 @@ use derive_new::new; use serde_derive::{Deserialize, Serialize}; -#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash, new)] +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, new)] pub enum Type { Any, } diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index 81222f66f9..e9c094388b 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -45,7 +45,7 @@ pub fn baseline_parse_next_expr( let first = next_token(&mut tokens); let first = match first { - None => return Err(ShellError::unimplemented("Expected token, found none")), + None => return Err(ShellError::string("Expected token, found none")), Some(token) => baseline_parse_semantic_token(token, source)?, }; diff --git a/src/parser/parse2/span.rs b/src/parser/parse2/span.rs index e5905ea4df..ca279eeaa2 100644 --- a/src/parser/parse2/span.rs +++ b/src/parser/parse2/span.rs @@ -1,8 +1,11 @@ use crate::Text; use derive_new::new; use getset::Getters; +use serde_derive::{Deserialize, Serialize}; -#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Getters)] +#[derive( + new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, +)] #[get = "crate"] pub struct Spanned { crate span: Span, @@ -46,7 +49,7 @@ impl Spanned { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)] pub struct Span { crate start: usize, crate end: usize, diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 114fd3a1f5..101b45a971 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -61,10 +61,7 @@ impl Highlighter for Helper { let tokens = crate::parser::pipeline(nom_input(line)); match tokens { - Err(e) => { - println!("error: {:?}", e); - Cow::Borrowed(line) - } + Err(_) => Cow::Borrowed(line), Ok((_rest, v)) => { let mut out = String::new(); let pipeline = match v.as_pipeline() { From 62368644fac13043f51f4dc4c242bb2e6c59bcce Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Mon, 24 Jun 2019 14:00:53 +1200 Subject: [PATCH 20/20] Fix lines for windows --- src/commands/lines.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/commands/lines.rs b/src/commands/lines.rs index e170ca00be..933268758a 100644 --- a/src/commands/lines.rs +++ b/src/commands/lines.rs @@ -12,8 +12,7 @@ pub fn lines(args: CommandArgs) -> Result { let stream = input .map(move |v| match v { Value::Primitive(Primitive::String(s)) => { - let splitter = "\n"; - let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect(); + let split_result: Vec<_> = s.lines().filter(|s| s.trim() != "").collect(); trace!("split result = {:?}", split_result);