diff --git a/crates/nu-parser/src/hir/syntax_shape.rs b/crates/nu-parser/src/hir/syntax_shape.rs index 1468178b56..ace2f43ff4 100644 --- a/crates/nu-parser/src/hir/syntax_shape.rs +++ b/crates/nu-parser/src/hir/syntax_shape.rs @@ -240,12 +240,12 @@ pub fn expand_bare( match node { Some(token) if predicate(token) => { peeked.commit(); - state = state.clone().seen(token.span()); + state = state.seen(token.span()); let shapes = FlatShape::shapes(token, &source); token_nodes.color_shapes(shapes); } token => { - state = state.clone().end(token, "word"); + state = state.end(token, "word"); break; } } diff --git a/crates/nu-parser/src/hir/tokens_iterator.rs b/crates/nu-parser/src/hir/tokens_iterator.rs index 56661ba182..b2d104d4bc 100644 --- a/crates/nu-parser/src/hir/tokens_iterator.rs +++ b/crates/nu-parser/src/hir/tokens_iterator.rs @@ -3,6 +3,9 @@ pub(crate) mod into_shapes; pub(crate) mod pattern; pub(crate) mod state; +#[cfg(test)] +mod tests; + use self::debug::ExpandTracer; use self::into_shapes::IntoShapes; use self::state::{Peeked, TokensIteratorState}; diff --git a/crates/nu-parser/src/hir/tokens_iterator/tests.rs b/crates/nu-parser/src/hir/tokens_iterator/tests.rs index 7d6d2a762f..8c22aec77d 100644 --- a/crates/nu-parser/src/hir/tokens_iterator/tests.rs +++ b/crates/nu-parser/src/hir/tokens_iterator/tests.rs @@ -1,21 +1,46 @@ -use crate::hir::TokensIterator; +use crate::hir::{syntax_shape::ExpandContext, syntax_shape::SignatureRegistry, TokensIterator}; use crate::parse::token_tree_builder::TokenTreeBuilder as b; -use crate::Span; +use nu_protocol::Signature; +use nu_source::{Span, Text}; + +use derive_new::new; + +#[derive(Debug, Clone, new)] +struct TestRegistry { + #[new(default)] + signatures: indexmap::IndexMap, +} + +impl TestRegistry {} + +impl SignatureRegistry for TestRegistry { + fn has(&self, name: &str) -> bool { + self.signatures.contains_key(name) + } + fn get(&self, name: &str) -> Option { + self.signatures.get(name).cloned() + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} #[test] -<<<<<<< HEAD fn supplies_tokens() { - let tokens = b::token_list(vec![b::it_var(), b::op("."), b::bare("cpu")]); -======= -fn supplies_tokens() -> Result<(), Box> { - let tokens = b::token_list(vec![b::var("it"), b::op("."), b::bare("cpu")]); ->>>>>>> master - let (tokens, _) = b::build(tokens); + let token = b::it_var(); - let tokens = tokens.expect_list(); - let mut iterator = TokensIterator::new(tokens, Span::unknown()); + let (tokens, source) = b::build(token); - iterator.next()?.expect_var(); - iterator.next()?.expect_dot(); - iterator.next()?.expect_bare(); + let tokens = vec![tokens]; + let source = Text::from(&source); + + let mut iterator = TokensIterator::new( + &tokens, + ExpandContext::new(Box::new(TestRegistry::new()), &source, None), + Span::unknown(), + ); + + let token = iterator.next().expect("Token expected."); + + token.expect_var(); } diff --git a/crates/nu-parser/src/parse/tokens.rs b/crates/nu-parser/src/parse/tokens.rs deleted file mode 100644 index 0b15694699..0000000000 --- a/crates/nu-parser/src/parse/tokens.rs +++ /dev/null @@ -1,231 +0,0 @@ -use crate::parse::parser::Number; -use crate::{CompareOperator, EvaluationOperator}; -use bigdecimal::BigDecimal; -use nu_protocol::ShellTypeName; -use nu_source::{ - b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem, - Text, -}; -use num_bigint::BigInt; -use std::fmt; -use std::str::FromStr; - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum UnspannedToken { - Number(RawNumber), - CompareOperator(CompareOperator), - EvaluationOperator(EvaluationOperator), - String(Span), - Variable(Span), - ExternalCommand(Span), - ExternalWord, - GlobPattern, - Bare, -} - -impl UnspannedToken { - pub fn into_token(self, span: impl Into) -> Token { - Token { - unspanned: self, - span: span.into(), - } - } -} - -impl ShellTypeName for UnspannedToken { - fn type_name(&self) -> &'static str { - match self { - UnspannedToken::Number(_) => "number", - UnspannedToken::CompareOperator(..) => "comparison operator", - UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => "dot", - UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => "dotdot", - UnspannedToken::String(_) => "string", - UnspannedToken::Variable(_) => "variable", - UnspannedToken::ExternalCommand(_) => "syntax error", - UnspannedToken::ExternalWord => "syntax error", - UnspannedToken::GlobPattern => "glob pattern", - UnspannedToken::Bare => "string", - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum RawNumber { - Int(Span), - Decimal(Span), -} - -impl HasSpan for RawNumber { - fn span(&self) -> Span { - match self { - RawNumber::Int(span) => *span, - RawNumber::Decimal(span) => *span, - } - } -} - -impl PrettyDebugWithSource for RawNumber { - fn pretty_debug(&self, source: &str) -> DebugDocBuilder { - match self { - RawNumber::Int(span) => b::primitive(span.slice(source)), - RawNumber::Decimal(span) => b::primitive(span.slice(source)), - } - } -} - -impl RawNumber { - pub fn int(span: impl Into) -> RawNumber { - let span = span.into(); - - RawNumber::Int(span) - } - - pub fn decimal(span: impl Into) -> RawNumber { - let span = span.into(); - - RawNumber::Decimal(span) - } - - pub(crate) fn to_number(self, source: &Text) -> Number { - match self { - RawNumber::Int(tag) => { - if let Ok(int) = BigInt::from_str(tag.slice(source)) { - Number::Int(int) - } else { - unreachable!("Internal error: to_number failed") - } - } - RawNumber::Decimal(tag) => { - if let Ok(decimal) = BigDecimal::from_str(tag.slice(source)) { - Number::Decimal(decimal) - } else { - unreachable!("Internal error: to_number failed") - } - } - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct Token { - pub unspanned: UnspannedToken, - pub span: Span, -} - -impl std::ops::Deref for Token { - type Target = UnspannedToken; - - fn deref(&self) -> &UnspannedToken { - &self.unspanned - } -} - -impl PrettyDebugWithSource for Token { - fn pretty_debug(&self, source: &str) -> DebugDocBuilder { - match self.unspanned { - UnspannedToken::Number(number) => number.pretty_debug(source), - UnspannedToken::CompareOperator(operator) => operator.pretty(), - UnspannedToken::EvaluationOperator(operator) => operator.pretty(), - UnspannedToken::String(_) => b::primitive(self.span.slice(source)), - UnspannedToken::Variable(_) => b::var(self.span.slice(source)), - UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)), - UnspannedToken::ExternalWord => { - b::typed("external", b::description(self.span.slice(source))) - } - UnspannedToken::GlobPattern => { - b::typed("pattern", b::description(self.span.slice(source))) - } - UnspannedToken::Bare => b::primitive(self.span.slice(source)), - } - } -} - -impl Token { - pub fn debug<'a>(&self, source: &'a Text) -> DebugToken<'a> { - DebugToken { - node: *self, - source, - } - } - - pub fn extract_number(&self) -> Option { - match self.unspanned { - UnspannedToken::Number(number) => Some(number), - _ => None, - } - } - - pub fn extract_int(&self) -> Option<(Span, Span)> { - match self.unspanned { - UnspannedToken::Number(RawNumber::Int(int)) => Some((int, self.span)), - _ => None, - } - } - - pub fn extract_decimal(&self) -> Option<(Span, Span)> { - match self.unspanned { - UnspannedToken::Number(RawNumber::Decimal(decimal)) => Some((decimal, self.span)), - _ => None, - } - } - - pub fn extract_operator(&self) -> Option> { - match self.unspanned { - UnspannedToken::CompareOperator(operator) => Some(operator.spanned(self.span)), - _ => None, - } - } - - pub fn extract_string(&self) -> Option<(Span, Span)> { - match self.unspanned { - UnspannedToken::String(span) => Some((span, self.span)), - _ => None, - } - } - - pub fn extract_variable(&self) -> Option<(Span, Span)> { - match self.unspanned { - UnspannedToken::Variable(span) => Some((span, self.span)), - _ => None, - } - } - - pub fn extract_external_command(&self) -> Option<(Span, Span)> { - match self.unspanned { - UnspannedToken::ExternalCommand(span) => Some((span, self.span)), - _ => None, - } - } - - pub fn extract_external_word(&self) -> Option { - match self.unspanned { - UnspannedToken::ExternalWord => Some(self.span), - _ => None, - } - } - - pub fn extract_glob_pattern(&self) -> Option { - match self.unspanned { - UnspannedToken::GlobPattern => Some(self.span), - _ => None, - } - } - - pub fn extract_bare(&self) -> Option { - match self.unspanned { - UnspannedToken::Bare => Some(self.span), - _ => None, - } - } -} - -pub struct DebugToken<'a> { - node: Token, - source: &'a Text, -} - -impl fmt::Debug for DebugToken<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.node.span.slice(self.source)) - } -}