forked from extern/nushell
Add Range and start Signature support
This commit contains two improvements: - Support for a Range syntax (and a corresponding Range value) - Work towards a signature syntax Implementing the Range syntax resulted in cleaning up how operators in the core syntax works. There are now two kinds of infix operators - tight operators (`.` and `..`) - loose operators Tight operators may not be interspersed (`$it.left..$it.right` is a syntax error). Loose operators require whitespace on both sides of the operator, and can be arbitrarily interspersed. Precedence is left to right in the core syntax. Note that delimited syntax (like `( ... )` or `[ ... ]`) is a single token node in the core syntax. A single token node can be parsed from beginning to end in a context-free manner. The rule for `.` is `<token node>.<member>`. The rule for `..` is `<token node>..<token node>`. Loose operators all have the same syntactic rule: `<token node><space><loose op><space><token node>`. The second aspect of this pull request is the beginning of support for a signature syntax. Before implementing signatures, a necessary prerequisite is for the core syntax to support multi-line programs. That work establishes a few things: - `;` and newlines are handled in the core grammar, and both count as "separators" - line comments begin with `#` and continue until the end of the line In this commit, multi-token productions in the core grammar can use separators interchangably with spaces. However, I think we will ultimately want a different rule preventing separators from occurring before an infix operator, so that the end of a line is always unambiguous. This would avoid gratuitous differences between modules and repl usage. We already effectively have this rule, because otherwise `x<newline> | y` would be a single pipeline, but of course that wouldn't work.
This commit is contained in:
Yehuda Katz
42
crates/nu-parser/src/parse/comment.rs
Normal file
42
crates/nu-parser/src/parse/comment.rs
Normal file
@ -0,0 +1,42 @@
|
||||
use derive_new::new;
|
||||
use getset::Getters;
|
||||
use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
|
||||
pub enum CommentKind {
|
||||
Line,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)]
|
||||
pub struct Comment {
|
||||
pub(crate) kind: CommentKind,
|
||||
pub(crate) text: Span,
|
||||
pub(crate) span: Span,
|
||||
}
|
||||
|
||||
impl Comment {
|
||||
pub fn line(text: impl Into<Span>, outer: impl Into<Span>) -> Comment {
|
||||
Comment {
|
||||
kind: CommentKind::Line,
|
||||
text: text.into(),
|
||||
span: outer.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PrettyDebugWithSource for Comment {
|
||||
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
|
||||
let prefix = match self.kind {
|
||||
CommentKind::Line => b::description("#"),
|
||||
};
|
||||
|
||||
prefix + b::description(self.text.slice(source))
|
||||
}
|
||||
}
|
||||
|
||||
impl HasSpan for Comment {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
@ -34,13 +34,15 @@ impl language_reporting::ReportingFiles for Files {
|
||||
}
|
||||
|
||||
fn location(&self, _file: Self::FileId, byte_index: usize) -> Option<Location> {
|
||||
trace!("finding location for {}", byte_index);
|
||||
|
||||
let source = &self.snippet;
|
||||
let mut seen_lines = 0;
|
||||
let mut seen_bytes = 0;
|
||||
|
||||
for (pos, slice) in source.match_indices('\n') {
|
||||
trace!(
|
||||
"SEARCH={} SEEN={} POS={} SLICE={:?} LEN={} ALL={:?}",
|
||||
"searching byte_index={} seen_bytes={} pos={} slice={:?} slice.len={} source={:?}",
|
||||
byte_index,
|
||||
seen_bytes,
|
||||
pos,
|
||||
@ -50,9 +52,19 @@ impl language_reporting::ReportingFiles for Files {
|
||||
);
|
||||
|
||||
if pos >= byte_index {
|
||||
trace!(
|
||||
"returning {}:{} seen_lines={} byte_index={} pos={} seen_bytes={}",
|
||||
seen_lines,
|
||||
byte_index,
|
||||
pos,
|
||||
seen_lines,
|
||||
byte_index,
|
||||
seen_bytes
|
||||
);
|
||||
|
||||
return Some(language_reporting::Location::new(
|
||||
seen_lines,
|
||||
byte_index - seen_bytes,
|
||||
byte_index - pos,
|
||||
));
|
||||
} else {
|
||||
seen_lines += 1;
|
||||
@ -61,30 +73,70 @@ impl language_reporting::ReportingFiles for Files {
|
||||
}
|
||||
|
||||
if seen_lines == 0 {
|
||||
Some(language_reporting::Location::new(0, byte_index))
|
||||
trace!("seen_lines=0 end={}", source.len() - 1);
|
||||
|
||||
// if we got here, there were no newlines in the source
|
||||
Some(language_reporting::Location::new(0, source.len() - 1))
|
||||
} else {
|
||||
panic!("byte index {} wasn't valid", byte_index);
|
||||
trace!(
|
||||
"last line seen_lines={} end={}",
|
||||
seen_lines,
|
||||
source.len() - 1 - byte_index
|
||||
);
|
||||
|
||||
// if we got here and we didn't return, it should mean that we're talking about
|
||||
// the last line
|
||||
Some(language_reporting::Location::new(
|
||||
seen_lines,
|
||||
source.len() - 1 - byte_index,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option<Self::Span> {
|
||||
trace!("finding line_span for {}", lineno);
|
||||
|
||||
let source = &self.snippet;
|
||||
let mut seen_lines = 0;
|
||||
let mut seen_bytes = 0;
|
||||
|
||||
for (pos, _) in source.match_indices('\n') {
|
||||
trace!(
|
||||
"lineno={} seen_lines={} seen_bytes={} pos={}",
|
||||
lineno,
|
||||
seen_lines,
|
||||
seen_bytes,
|
||||
pos
|
||||
);
|
||||
|
||||
if seen_lines == lineno {
|
||||
return Some(Span::new(seen_bytes, pos + 1));
|
||||
trace!("returning start={} end={}", seen_bytes, pos);
|
||||
// If the number of seen lines is the lineno, seen_bytes is the start of the
|
||||
// line and pos is the end of the line
|
||||
return Some(Span::new(seen_bytes, pos));
|
||||
} else {
|
||||
// If it's not, increment seen_lines, and move seen_bytes to the beginning of
|
||||
// the next line
|
||||
seen_lines += 1;
|
||||
seen_bytes = pos + 1;
|
||||
}
|
||||
}
|
||||
|
||||
if seen_lines == 0 {
|
||||
trace!("returning start={} end={}", 0, self.snippet.len() - 1);
|
||||
|
||||
// if we got here, there were no newlines in the source
|
||||
Some(Span::new(0, self.snippet.len() - 1))
|
||||
} else {
|
||||
None
|
||||
trace!(
|
||||
"returning start={} end={}",
|
||||
seen_bytes,
|
||||
self.snippet.len() - 1
|
||||
);
|
||||
|
||||
// if we got here and we didn't return, it should mean that we're talking about
|
||||
// the last line
|
||||
Some(Span::new(seen_bytes, self.snippet.len() - 1))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -4,63 +4,102 @@ use serde::{Deserialize, Serialize};
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
|
||||
pub enum Operator {
|
||||
pub enum CompareOperator {
|
||||
Equal,
|
||||
NotEqual,
|
||||
LessThan,
|
||||
GreaterThan,
|
||||
LessThanOrEqual,
|
||||
GreaterThanOrEqual,
|
||||
Dot,
|
||||
Contains,
|
||||
NotContains,
|
||||
}
|
||||
|
||||
impl PrettyDebug for Operator {
|
||||
impl PrettyDebug for CompareOperator {
|
||||
fn pretty(&self) -> DebugDocBuilder {
|
||||
b::operator(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl Operator {
|
||||
impl CompareOperator {
|
||||
pub fn print(&self) -> String {
|
||||
self.as_str().to_string()
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
match *self {
|
||||
Operator::Equal => "==",
|
||||
Operator::NotEqual => "!=",
|
||||
Operator::LessThan => "<",
|
||||
Operator::GreaterThan => ">",
|
||||
Operator::LessThanOrEqual => "<=",
|
||||
Operator::GreaterThanOrEqual => ">=",
|
||||
Operator::Dot => ".",
|
||||
Operator::Contains => "=~",
|
||||
Operator::NotContains => "!~",
|
||||
CompareOperator::Equal => "==",
|
||||
CompareOperator::NotEqual => "!=",
|
||||
CompareOperator::LessThan => "<",
|
||||
CompareOperator::GreaterThan => ">",
|
||||
CompareOperator::LessThanOrEqual => "<=",
|
||||
CompareOperator::GreaterThanOrEqual => ">=",
|
||||
CompareOperator::Contains => "=~",
|
||||
CompareOperator::NotContains => "!~",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Operator {
|
||||
fn from(input: &str) -> Operator {
|
||||
Operator::from_str(input).unwrap()
|
||||
impl From<&str> for CompareOperator {
|
||||
fn from(input: &str) -> CompareOperator {
|
||||
CompareOperator::from_str(input).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Operator {
|
||||
impl FromStr for CompareOperator {
|
||||
type Err = ();
|
||||
fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
|
||||
match input {
|
||||
"==" => Ok(Operator::Equal),
|
||||
"!=" => Ok(Operator::NotEqual),
|
||||
"<" => Ok(Operator::LessThan),
|
||||
">" => Ok(Operator::GreaterThan),
|
||||
"<=" => Ok(Operator::LessThanOrEqual),
|
||||
">=" => Ok(Operator::GreaterThanOrEqual),
|
||||
"." => Ok(Operator::Dot),
|
||||
"=~" => Ok(Operator::Contains),
|
||||
"!~" => Ok(Operator::NotContains),
|
||||
"==" => Ok(CompareOperator::Equal),
|
||||
"!=" => Ok(CompareOperator::NotEqual),
|
||||
"<" => Ok(CompareOperator::LessThan),
|
||||
">" => Ok(CompareOperator::GreaterThan),
|
||||
"<=" => Ok(CompareOperator::LessThanOrEqual),
|
||||
">=" => Ok(CompareOperator::GreaterThanOrEqual),
|
||||
"=~" => Ok(CompareOperator::Contains),
|
||||
"!~" => Ok(CompareOperator::NotContains),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
|
||||
pub enum EvaluationOperator {
|
||||
Dot,
|
||||
DotDot,
|
||||
}
|
||||
|
||||
impl PrettyDebug for EvaluationOperator {
|
||||
fn pretty(&self) -> DebugDocBuilder {
|
||||
b::operator(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl EvaluationOperator {
|
||||
pub fn print(&self) -> String {
|
||||
self.as_str().to_string()
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
match *self {
|
||||
EvaluationOperator::Dot => ".",
|
||||
EvaluationOperator::DotDot => "..",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for EvaluationOperator {
|
||||
fn from(input: &str) -> EvaluationOperator {
|
||||
EvaluationOperator::from_str(input).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for EvaluationOperator {
|
||||
type Err = ();
|
||||
fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
|
||||
match input {
|
||||
"." => Ok(EvaluationOperator::Dot),
|
||||
".." => Ok(EvaluationOperator::DotDot),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@ use nom::sequence::*;
|
||||
|
||||
use bigdecimal::BigDecimal;
|
||||
use derive_new::new;
|
||||
use enumflags2::BitFlags;
|
||||
use log::trace;
|
||||
use nom::dbg;
|
||||
use nom::*;
|
||||
@ -32,7 +33,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::fmt::Debug;
|
||||
use std::str::FromStr;
|
||||
|
||||
macro_rules! operator {
|
||||
macro_rules! cmp_operator {
|
||||
($name:tt : $token:tt ) => {
|
||||
#[tracable_parser]
|
||||
pub fn $name(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
@ -42,21 +43,38 @@ macro_rules! operator {
|
||||
|
||||
Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_op(tag.fragment, Span::new(start, end)),
|
||||
TokenTreeBuilder::spanned_cmp_op(tag.fragment, Span::new(start, end)),
|
||||
))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
operator! { gt: ">" }
|
||||
operator! { lt: "<" }
|
||||
operator! { gte: ">=" }
|
||||
operator! { lte: "<=" }
|
||||
operator! { eq: "==" }
|
||||
operator! { neq: "!=" }
|
||||
operator! { dot: "." }
|
||||
operator! { cont: "=~" }
|
||||
operator! { ncont: "!~" }
|
||||
macro_rules! eval_operator {
|
||||
($name:tt : $token:tt ) => {
|
||||
#[tracable_parser]
|
||||
pub fn $name(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start = input.offset;
|
||||
let (input, tag) = tag($token)(input)?;
|
||||
let end = input.offset;
|
||||
|
||||
Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_eval_op(tag.fragment, Span::new(start, end)),
|
||||
))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
cmp_operator! { gt: ">" }
|
||||
cmp_operator! { lt: "<" }
|
||||
cmp_operator! { gte: ">=" }
|
||||
cmp_operator! { lte: "<=" }
|
||||
cmp_operator! { eq: "==" }
|
||||
cmp_operator! { neq: "!=" }
|
||||
cmp_operator! { cont: "=~" }
|
||||
cmp_operator! { ncont: "!~" }
|
||||
eval_operator! { dot: "." }
|
||||
eval_operator! { dotdot: ".." }
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
pub enum Number {
|
||||
@ -213,6 +231,17 @@ pub fn raw_number(input: NomSpan) -> IResult<NomSpan, RawNumber> {
|
||||
}
|
||||
}
|
||||
|
||||
let dotdot_result = dotdot(input);
|
||||
|
||||
match dotdot_result {
|
||||
// If we see a `..` immediately after an integer, it's a range, not a decimal
|
||||
Ok((dotdot_input, _)) => {
|
||||
return Ok((input, RawNumber::int(Span::new(start, input.offset))))
|
||||
}
|
||||
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
let dot: IResult<NomSpan, NomSpan, (NomSpan, nom::error::ErrorKind)> = tag(".")(input);
|
||||
|
||||
let input = match dot {
|
||||
@ -285,7 +314,7 @@ pub fn string(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
pub fn external(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start = input.offset;
|
||||
let (input, _) = tag("^")(input)?;
|
||||
let (input, bare) = take_while(is_bare_char)(input)?;
|
||||
let (input, bare) = take_while(is_file_char)(input)?;
|
||||
let end = input.offset;
|
||||
|
||||
Ok((
|
||||
@ -294,52 +323,186 @@ pub fn external(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn pattern(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start = input.offset;
|
||||
let (input, _) = take_while1(is_start_glob_char)(input)?;
|
||||
let (input, _) = take_while(is_glob_char)(input)?;
|
||||
fn word<'a, T, U, V>(
|
||||
start_predicate: impl Fn(NomSpan<'a>) -> IResult<NomSpan<'a>, U>,
|
||||
next_predicate: impl Fn(NomSpan<'a>) -> IResult<NomSpan<'a>, V> + Copy,
|
||||
into: impl Fn(Span) -> T,
|
||||
) -> impl Fn(NomSpan<'a>) -> IResult<NomSpan<'a>, T> {
|
||||
move |input: NomSpan| {
|
||||
let start = input.offset;
|
||||
|
||||
let next_char = &input.fragment.chars().nth(0);
|
||||
let (input, _) = start_predicate(input)?;
|
||||
let (input, _) = many0(next_predicate)(input)?;
|
||||
|
||||
if let Some(next_char) = next_char {
|
||||
if is_external_word_char(*next_char) {
|
||||
return Err(nom::Err::Error(nom::error::make_error(
|
||||
input,
|
||||
nom::error::ErrorKind::TakeWhile1,
|
||||
)));
|
||||
let next_char = &input.fragment.chars().nth(0);
|
||||
|
||||
match next_char {
|
||||
Some('.') => {}
|
||||
Some(next_char)
|
||||
if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) =>
|
||||
{
|
||||
return Err(nom::Err::Error(nom::error::make_error(
|
||||
input,
|
||||
nom::error::ErrorKind::TakeWhile1,
|
||||
)));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let end = input.offset;
|
||||
|
||||
Ok((input, into(Span::new(start, end))))
|
||||
}
|
||||
}
|
||||
|
||||
let end = input.offset;
|
||||
|
||||
Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_pattern(Span::new(start, end)),
|
||||
))
|
||||
pub fn matches(cond: fn(char) -> bool) -> impl Fn(NomSpan) -> IResult<NomSpan, NomSpan> + Copy {
|
||||
move |input: NomSpan| match input.iter_elements().next() {
|
||||
Option::Some(c) if cond(c) => Ok((input.slice(1..), input.slice(0..1))),
|
||||
_ => Err(nom::Err::Error(nom::error::ParseError::from_error_kind(
|
||||
input,
|
||||
nom::error::ErrorKind::Many0,
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn bare(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start = input.offset;
|
||||
let (input, _) = take_while1(is_start_bare_char)(input)?;
|
||||
let (input, last) = take_while(is_bare_char)(input)?;
|
||||
pub fn pattern(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
word(
|
||||
start_pattern,
|
||||
matches(is_glob_char),
|
||||
TokenTreeBuilder::spanned_pattern,
|
||||
)(input)
|
||||
}
|
||||
|
||||
let next_char = &input.fragment.chars().nth(0);
|
||||
let prev_char = last.fragment.chars().nth(0);
|
||||
#[tracable_parser]
|
||||
pub fn start_pattern(input: NomSpan) -> IResult<NomSpan, NomSpan> {
|
||||
alt((take_while1(is_dot), matches(is_start_glob_char)))(input)
|
||||
}
|
||||
|
||||
if let Some(next_char) = next_char {
|
||||
if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) {
|
||||
return Err(nom::Err::Error(nom::error::make_error(
|
||||
input,
|
||||
nom::error::ErrorKind::TakeWhile1,
|
||||
)));
|
||||
#[tracable_parser]
|
||||
pub fn filename(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start_pos = input.offset;
|
||||
|
||||
let (mut input, mut saw_special) = match start_file_char(input) {
|
||||
Err(err) => return Err(err),
|
||||
Ok((input, special)) => (input, special),
|
||||
};
|
||||
|
||||
loop {
|
||||
if saw_special.is_empty() {
|
||||
match continue_file_char(input) {
|
||||
Err(_) => {
|
||||
return Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_bare((start_pos, input.offset)),
|
||||
))
|
||||
}
|
||||
Ok((next_input, special)) => {
|
||||
saw_special |= special;
|
||||
input = next_input;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let rest = after_sep_file(input);
|
||||
|
||||
let (input, span, updated_special) = match rest {
|
||||
Err(_) => (input, (start_pos, input.offset), saw_special),
|
||||
Ok((input, new_special)) => {
|
||||
(input, (start_pos, input.offset), saw_special | new_special)
|
||||
}
|
||||
};
|
||||
|
||||
if updated_special.contains(SawSpecial::Glob) {
|
||||
return Ok((input, TokenTreeBuilder::spanned_pattern(span)));
|
||||
} else {
|
||||
return Ok((input, TokenTreeBuilder::spanned_bare(span)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let end = input.offset;
|
||||
#[derive(BitFlags, Copy, Clone, Eq, PartialEq)]
|
||||
enum SawSpecial {
|
||||
PathSeparator = 0b01,
|
||||
Glob = 0b10,
|
||||
}
|
||||
|
||||
Ok((input, TokenTreeBuilder::spanned_bare(Span::new(start, end))))
|
||||
#[tracable_parser]
|
||||
fn start_file_char(input: NomSpan) -> IResult<NomSpan, BitFlags<SawSpecial>> {
|
||||
let path_sep_result = special_file_char(input);
|
||||
|
||||
match path_sep_result {
|
||||
Ok((input, special)) => return Ok((input, special)),
|
||||
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
start_filename(input).map(|(input, output)| (input, BitFlags::empty()))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
fn continue_file_char(input: NomSpan) -> IResult<NomSpan, BitFlags<SawSpecial>> {
|
||||
let path_sep_result = special_file_char(input);
|
||||
|
||||
match path_sep_result {
|
||||
Ok((input, special)) => return Ok((input, special)),
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
matches(is_file_char)(input).map(|(input, _)| (input, BitFlags::empty()))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
fn special_file_char(input: NomSpan) -> IResult<NomSpan, BitFlags<SawSpecial>> {
|
||||
match matches(is_path_separator)(input) {
|
||||
Ok((input, _)) => return Ok((input, BitFlags::empty() | SawSpecial::PathSeparator)),
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
let (input, _) = matches(is_glob_specific_char)(input)?;
|
||||
|
||||
Ok((input, BitFlags::empty() | SawSpecial::Glob))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
fn after_sep_file(input: NomSpan) -> IResult<NomSpan, BitFlags<SawSpecial>> {
|
||||
fn after_sep_char(c: char) -> bool {
|
||||
is_external_word_char(c) || is_file_char(c) || c == '.'
|
||||
}
|
||||
|
||||
let start = input.offset;
|
||||
let original_input = input;
|
||||
let mut input = input;
|
||||
|
||||
let (input, after_glob) = take_while1(after_sep_char)(input)?;
|
||||
|
||||
let slice = original_input.slice(0..input.offset - start);
|
||||
|
||||
let saw_special = if slice.fragment.chars().any(is_glob_specific_char) {
|
||||
BitFlags::empty() | SawSpecial::Glob
|
||||
} else {
|
||||
BitFlags::empty()
|
||||
};
|
||||
|
||||
Ok((input, saw_special))
|
||||
}
|
||||
|
||||
pub fn start_filename(input: NomSpan) -> IResult<NomSpan, NomSpan> {
|
||||
alt((take_while1(is_dot), matches(is_start_file_char)))(input)
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn member(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
word(
|
||||
matches(is_start_member_char),
|
||||
matches(is_member_char),
|
||||
TokenTreeBuilder::spanned_bare,
|
||||
)(input)
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn ident(input: NomSpan) -> IResult<NomSpan, Tag> {
|
||||
word(matches(is_id_start), matches(is_id_continue), Tag::from)(input)
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
@ -348,10 +511,7 @@ pub fn external_word(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let (input, _) = take_while1(is_external_word_char)(input)?;
|
||||
let end = input.offset;
|
||||
|
||||
Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_external_word(Span::new(start, end)),
|
||||
))
|
||||
Ok((input, TokenTreeBuilder::spanned_external_word((start, end))))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
@ -367,21 +527,40 @@ pub fn var(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn ident(input: NomSpan) -> IResult<NomSpan, Tag> {
|
||||
let start = input.offset;
|
||||
let (input, _) = take_while1(is_start_bare_char)(input)?;
|
||||
let (input, _) = take_while(is_bare_char)(input)?;
|
||||
let end = input.offset;
|
||||
fn tight<'a>(
|
||||
parser: impl Fn(NomSpan<'a>) -> IResult<NomSpan<'a>, Vec<TokenNode>>,
|
||||
) -> impl Fn(NomSpan<'a>) -> IResult<NomSpan<'a>, Vec<TokenNode>> {
|
||||
move |input: NomSpan| {
|
||||
let mut result = vec![];
|
||||
let (input, head) = parser(input)?;
|
||||
result.extend(head);
|
||||
|
||||
Ok((input, Tag::from((start, end, None))))
|
||||
let (input, tail) = opt(alt((many1(range_continuation), many1(dot_member))))(input)?;
|
||||
|
||||
let next_char = &input.fragment.chars().nth(0);
|
||||
|
||||
if is_boundary(*next_char) {
|
||||
if let Some(tail) = tail {
|
||||
for tokens in tail {
|
||||
result.extend(tokens);
|
||||
}
|
||||
}
|
||||
|
||||
Ok((input, result))
|
||||
} else {
|
||||
Err(nom::Err::Error(nom::error::make_error(
|
||||
input,
|
||||
nom::error::ErrorKind::Many0,
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn flag(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start = input.offset;
|
||||
let (input, _) = tag("--")(input)?;
|
||||
let (input, bare) = bare(input)?;
|
||||
let (input, bare) = filename(input)?;
|
||||
let end = input.offset;
|
||||
|
||||
Ok((
|
||||
@ -394,7 +573,7 @@ pub fn flag(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
pub fn shorthand(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let start = input.offset;
|
||||
let (input, _) = tag("-")(input)?;
|
||||
let (input, bare) = bare(input)?;
|
||||
let (input, bare) = filename(input)?;
|
||||
let end = input.offset;
|
||||
|
||||
Ok((
|
||||
@ -413,47 +592,97 @@ pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
#[tracable_parser]
|
||||
pub fn token_list(input: NomSpan) -> IResult<NomSpan, Spanned<Vec<TokenNode>>> {
|
||||
let start = input.offset;
|
||||
let (input, first) = node(input)?;
|
||||
let mut node_list = vec![];
|
||||
|
||||
let (input, mut list) = many0(pair(alt((whitespace, dot)), node))(input)?;
|
||||
let mut next_input = input;
|
||||
let mut before_space_input: Option<NomSpan> = None;
|
||||
let mut final_space_tokens = 0;
|
||||
|
||||
let end = input.offset;
|
||||
loop {
|
||||
let node_result = tight_node(next_input);
|
||||
|
||||
Ok((
|
||||
input,
|
||||
make_token_list(first, list, None).spanned(Span::new(start, end)),
|
||||
))
|
||||
let (after_node_input, next_nodes) = match node_result {
|
||||
Err(_) => {
|
||||
if let Some(before_space_input) = before_space_input {
|
||||
next_input = before_space_input;
|
||||
|
||||
for _ in 0..final_space_tokens {
|
||||
node_list.pop();
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
Ok((after_node_input, next_node)) => (after_node_input, next_node),
|
||||
};
|
||||
|
||||
node_list.extend(next_nodes);
|
||||
|
||||
// Special case that allows a parenthesized expression to immediate follow another
|
||||
// token without a space, which could represent a type annotation.
|
||||
let maybe_type = delimited_paren(after_node_input);
|
||||
|
||||
let after_maybe_type_input = match maybe_type {
|
||||
Err(_) => after_node_input,
|
||||
Ok((after_maybe_type_input, parens)) => {
|
||||
node_list.push(parens);
|
||||
after_maybe_type_input
|
||||
}
|
||||
};
|
||||
|
||||
let maybe_space = any_space(after_maybe_type_input);
|
||||
|
||||
let after_space_input = match maybe_space {
|
||||
Err(_) => {
|
||||
next_input = after_maybe_type_input;
|
||||
|
||||
break;
|
||||
}
|
||||
Ok((after_space_input, space)) => {
|
||||
final_space_tokens = space.len();
|
||||
node_list.extend(space);
|
||||
before_space_input = Some(after_maybe_type_input);
|
||||
after_space_input
|
||||
}
|
||||
};
|
||||
|
||||
next_input = after_space_input;
|
||||
}
|
||||
|
||||
let end = next_input.offset;
|
||||
|
||||
Ok((next_input, node_list.spanned(Span::new(start, end))))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn spaced_token_list(input: NomSpan) -> IResult<NomSpan, Spanned<Vec<TokenNode>>> {
|
||||
let start = input.offset;
|
||||
let (input, pre_ws) = opt(whitespace)(input)?;
|
||||
let (input, pre_ws) = opt(any_space)(input)?;
|
||||
let (input, items) = token_list(input)?;
|
||||
let (input, post_ws) = opt(whitespace)(input)?;
|
||||
let (input, post_ws) = opt(any_space)(input)?;
|
||||
let end = input.offset;
|
||||
|
||||
let mut out = vec![];
|
||||
|
||||
out.extend(pre_ws);
|
||||
pre_ws.map(|pre_ws| out.extend(pre_ws));
|
||||
out.extend(items.item);
|
||||
out.extend(post_ws);
|
||||
post_ws.map(|post_ws| out.extend(post_ws));
|
||||
|
||||
Ok((input, out.spanned(Span::new(start, end))))
|
||||
}
|
||||
|
||||
fn make_token_list(
|
||||
first: Vec<TokenNode>,
|
||||
list: Vec<(TokenNode, Vec<TokenNode>)>,
|
||||
list: Vec<(Vec<TokenNode>, Vec<TokenNode>)>,
|
||||
sp_right: Option<TokenNode>,
|
||||
) -> Vec<TokenNode> {
|
||||
let mut nodes = vec![];
|
||||
|
||||
nodes.extend(first);
|
||||
|
||||
for (left, right) in list {
|
||||
nodes.push(left);
|
||||
nodes.extend(right);
|
||||
for (sep, list) in list {
|
||||
nodes.extend(sep);
|
||||
nodes.extend(list);
|
||||
}
|
||||
|
||||
if let Some(sp_right) = sp_right {
|
||||
@ -463,6 +692,15 @@ fn make_token_list(
|
||||
nodes
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn separator(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let left = input.offset;
|
||||
let (input, ws1) = alt((tag(";"), tag("\n")))(input)?;
|
||||
let right = input.offset;
|
||||
|
||||
Ok((input, TokenTreeBuilder::spanned_sep(Span::new(left, right))))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn whitespace(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let left = input.offset;
|
||||
@ -472,6 +710,30 @@ pub fn whitespace(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
Ok((input, TokenTreeBuilder::spanned_ws(Span::new(left, right))))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn any_space(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
let left = input.offset;
|
||||
let (input, tokens) = many1(alt((whitespace, separator, comment)))(input)?;
|
||||
let right = input.offset;
|
||||
|
||||
Ok((input, tokens))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn comment(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let left = input.offset;
|
||||
let (input, start) = tag("#")(input)?;
|
||||
let (input, rest) = not_line_ending(input)?;
|
||||
let right = input.offset;
|
||||
|
||||
let span = (start.offset + 1, right);
|
||||
|
||||
Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_comment(span, Span::new(left, right)),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn delimited(
|
||||
input: NomSpan,
|
||||
delimiter: Delimiter,
|
||||
@ -541,62 +803,43 @@ pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Spanned<CallNode>> {
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn bare_path(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
let (input, head) = alt((bare, dot))(input)?;
|
||||
pub fn range_continuation(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
let original = input;
|
||||
|
||||
let (input, tail) = many0(alt((bare, dot, string)))(input)?;
|
||||
let mut result = vec![];
|
||||
|
||||
let next_char = &input.fragment.chars().nth(0);
|
||||
let (input, dotdot_result) = dotdot(input)?;
|
||||
result.push(dotdot_result);
|
||||
let (input, node_result) = tight_node(input)?;
|
||||
result.extend(node_result);
|
||||
|
||||
if is_boundary(*next_char) {
|
||||
let mut result = vec![head];
|
||||
result.extend(tail);
|
||||
|
||||
Ok((input, result))
|
||||
} else {
|
||||
Err(nom::Err::Error(nom::error::make_error(
|
||||
input,
|
||||
nom::error::ErrorKind::Many0,
|
||||
)))
|
||||
}
|
||||
Ok((input, result))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn pattern_path(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
let (input, head) = alt((pattern, dot))(input)?;
|
||||
pub fn dot_member(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
let (input, dot_result) = dot(input)?;
|
||||
let (input, member_result) = any_member(input)?;
|
||||
|
||||
let (input, tail) = many0(alt((pattern, dot, string)))(input)?;
|
||||
|
||||
let next_char = &input.fragment.chars().nth(0);
|
||||
|
||||
if is_boundary(*next_char) {
|
||||
let mut result = vec![head];
|
||||
result.extend(tail);
|
||||
|
||||
Ok((input, result))
|
||||
} else {
|
||||
Err(nom::Err::Error(nom::error::make_error(
|
||||
input,
|
||||
nom::error::ErrorKind::Many0,
|
||||
)))
|
||||
}
|
||||
Ok((input, vec![dot_result, member_result]))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn node1(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
alt((leaf, bare, pattern, external_word, delimited_paren))(input)
|
||||
pub fn any_member(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
alt((number, string, member))(input)
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn node(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
pub fn tight_node(input: NomSpan) -> IResult<NomSpan, Vec<TokenNode>> {
|
||||
alt((
|
||||
to_list(leaf),
|
||||
bare_path,
|
||||
pattern_path,
|
||||
tight(to_list(leaf)),
|
||||
tight(to_list(filename)),
|
||||
tight(to_list(pattern)),
|
||||
to_list(comment),
|
||||
to_list(external_word),
|
||||
to_list(delimited_paren),
|
||||
to_list(delimited_brace),
|
||||
to_list(delimited_square),
|
||||
tight(to_list(delimited_paren)),
|
||||
tight(to_list(delimited_brace)),
|
||||
tight(to_list(delimited_square)),
|
||||
))(input)
|
||||
}
|
||||
|
||||
@ -649,6 +892,23 @@ pub fn pipeline(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
))
|
||||
}
|
||||
|
||||
#[tracable_parser]
|
||||
pub fn module(input: NomSpan) -> IResult<NomSpan, TokenNode> {
|
||||
let (input, tokens) = spaced_token_list(input)?;
|
||||
|
||||
if input.input_len() != 0 {
|
||||
return Err(Err::Error(error_position!(
|
||||
input,
|
||||
nom::error::ErrorKind::Eof
|
||||
)));
|
||||
}
|
||||
|
||||
Ok((
|
||||
input,
|
||||
TokenTreeBuilder::spanned_token_list(tokens.item, tokens.span),
|
||||
))
|
||||
}
|
||||
|
||||
fn parse_int<T>(frag: &str, neg: Option<T>) -> i64 {
|
||||
let int = FromStr::from_str(frag).unwrap();
|
||||
|
||||
@ -661,7 +921,7 @@ fn parse_int<T>(frag: &str, neg: Option<T>) -> i64 {
|
||||
fn is_boundary(c: Option<char>) -> bool {
|
||||
match c {
|
||||
None => true,
|
||||
Some(')') | Some(']') | Some('}') => true,
|
||||
Some(')') | Some(']') | Some('}') | Some('(') => true,
|
||||
Some(c) if c.is_whitespace() => true,
|
||||
_ => false,
|
||||
}
|
||||
@ -682,14 +942,25 @@ fn is_glob_specific_char(c: char) -> bool {
|
||||
}
|
||||
|
||||
fn is_start_glob_char(c: char) -> bool {
|
||||
is_start_bare_char(c) || is_glob_specific_char(c) || c == '.'
|
||||
is_start_file_char(c) || is_glob_specific_char(c) || c == '.'
|
||||
}
|
||||
|
||||
fn is_glob_char(c: char) -> bool {
|
||||
is_bare_char(c) || is_glob_specific_char(c)
|
||||
is_file_char(c) || is_glob_specific_char(c)
|
||||
}
|
||||
|
||||
fn is_start_bare_char(c: char) -> bool {
|
||||
fn is_dot(c: char) -> bool {
|
||||
c == '.'
|
||||
}
|
||||
|
||||
fn is_path_separator(c: char) -> bool {
|
||||
match c {
|
||||
'\\' | '/' | ':' => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_start_file_char(c: char) -> bool {
|
||||
match c {
|
||||
'+' => false,
|
||||
_ if c.is_alphanumeric() => true,
|
||||
@ -698,11 +969,12 @@ fn is_start_bare_char(c: char) -> bool {
|
||||
'_' => true,
|
||||
'-' => true,
|
||||
'~' => true,
|
||||
'.' => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_bare_char(c: char) -> bool {
|
||||
fn is_file_char(c: char) -> bool {
|
||||
match c {
|
||||
'+' => true,
|
||||
_ if c.is_alphanumeric() => true,
|
||||
@ -718,6 +990,24 @@ fn is_bare_char(c: char) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_start_member_char(c: char) -> bool {
|
||||
match c {
|
||||
_ if c.is_alphanumeric() => true,
|
||||
'_' => true,
|
||||
'-' => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_member_char(c: char) -> bool {
|
||||
match c {
|
||||
_ if c.is_alphanumeric() => true,
|
||||
'_' => true,
|
||||
'-' => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_id_start(c: char) -> bool {
|
||||
unicode_xid::UnicodeXID::is_xid_start(c)
|
||||
}
|
||||
@ -775,6 +1065,7 @@ mod tests {
|
||||
|
||||
(<$parser:tt> $source:tt -> $tokens:expr) => {
|
||||
let result = apply($parser, stringify!($parser), $source);
|
||||
|
||||
let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens);
|
||||
|
||||
if result != expected_tree {
|
||||
@ -884,22 +1175,36 @@ mod tests {
|
||||
fn test_simple_path() {
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")])
|
||||
"chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::dot(), b::bare("exe")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
".azure" -> b::token_list(vec![b::op(Operator::Dot), b::bare("azure")])
|
||||
".azure" -> b::token_list(vec![b::bare(".azure")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system"), b::op(Operator::Dot), b::bare("dll")])
|
||||
r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system.dll")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests"), b::op(Operator::Dot), b::bare("js")])
|
||||
r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests.js")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
r"C:\Users\example\AppData\Local\Temp\.tmpZ4TVQ2\cd_test_8" -> b::token_list(vec![b::bare(r"C:\Users\example\AppData\Local\Temp\.tmpZ4TVQ2\cd_test_8")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<pipeline>
|
||||
r"cd C:\Users\wycat\AppData\Local\Temp\.tmpaj5JKi\cd_test_11" -> b::pipeline(vec![vec![
|
||||
b::bare("cd"),
|
||||
b::sp(),
|
||||
b::bare(r"C:\Users\wycat\AppData\Local\Temp\.tmpaj5JKi\cd_test_11")
|
||||
]])
|
||||
}
|
||||
}
|
||||
|
||||
@ -949,7 +1254,7 @@ mod tests {
|
||||
fn test_dot_prefixed_name() {
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
".azure" -> b::token_list(vec![b::op("."), b::bare("azure")])
|
||||
".azure" -> b::token_list(vec![b::bare(".azure")])
|
||||
}
|
||||
}
|
||||
|
||||
@ -1003,33 +1308,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_range() {
|
||||
let _ = pretty_env_logger::try_init();
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"0..2" -> b::token_list(vec![b::int(0), b::dotdot(), b::int(2)])
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_path() {
|
||||
let _ = pretty_env_logger::try_init();
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")])
|
||||
"$it.print" -> b::token_list(vec![b::var("it"), b::dot(), b::bare("print")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"$it.0" -> b::token_list(vec![b::var("it"), b::op("."), b::int(0)])
|
||||
"$it.0" -> b::token_list(vec![b::var("it"), b::dot(), b::int(0)])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")])
|
||||
"$head.part1.part2" -> b::token_list(vec![b::var("head"), b::dot(), b::bare("part1"), b::dot(), b::bare("part2")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::bare("world")])
|
||||
"( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::dot(), b::bare("world")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::string("world")])
|
||||
r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::dot(), b::string("world")])
|
||||
}
|
||||
}
|
||||
|
||||
@ -1042,11 +1357,11 @@ mod tests {
|
||||
b::parens(vec![
|
||||
b::sp(),
|
||||
b::var("it"),
|
||||
b::op("."),
|
||||
b::dot(),
|
||||
b::bare("is"),
|
||||
b::op("."),
|
||||
b::dot(),
|
||||
b::string("great news"),
|
||||
b::op("."),
|
||||
b::dot(),
|
||||
b::bare("right"),
|
||||
b::sp(),
|
||||
b::bare("yep"),
|
||||
@ -1054,7 +1369,7 @@ mod tests {
|
||||
b::var("yep"),
|
||||
b::sp()
|
||||
]),
|
||||
b::op("."), b::string("world")]
|
||||
b::dot(), b::string("world")]
|
||||
)
|
||||
}
|
||||
|
||||
@ -1063,9 +1378,9 @@ mod tests {
|
||||
r#"$it."are PAS".0"# -> b::token_list(
|
||||
vec![
|
||||
b::var("it"),
|
||||
b::op("."),
|
||||
b::dot(),
|
||||
b::string("are PAS"),
|
||||
b::op("."),
|
||||
b::dot(),
|
||||
b::int(0),
|
||||
]
|
||||
)
|
||||
@ -1076,17 +1391,17 @@ mod tests {
|
||||
fn test_smoke_single_command() {
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::op(".")])
|
||||
"git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::bare(".")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml")])
|
||||
"open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::op("."), b::bare("version")])
|
||||
"select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::dot(), b::bare("version")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
@ -1096,12 +1411,12 @@ mod tests {
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::flag("raw")])
|
||||
"open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml"), b::sp(), b::flag("raw")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
<nodes>
|
||||
"open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::shorthand("r")])
|
||||
"open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml"), b::sp(), b::shorthand("r")])
|
||||
}
|
||||
|
||||
equal_tokens! {
|
||||
@ -1117,7 +1432,7 @@ mod tests {
|
||||
b::sp(),
|
||||
b::flag("patch"),
|
||||
b::sp(),
|
||||
b::bare("package"), b::op("."), b::bare("version")
|
||||
b::bare("package"), b::dot(), b::bare("version")
|
||||
]
|
||||
)
|
||||
}
|
||||
@ -1197,7 +1512,7 @@ mod tests {
|
||||
b::bare("where"),
|
||||
b::sp(),
|
||||
b::bare("cpu"),
|
||||
b::op("."),
|
||||
b::dot(),
|
||||
b::string("max ghz"),
|
||||
b::sp(),
|
||||
b::op(">"),
|
||||
@ -1207,6 +1522,34 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature() {
|
||||
let _ = pretty_env_logger::try_init();
|
||||
|
||||
equal_tokens!(
|
||||
<module>
|
||||
"def cd\n # Change to a new path.\n optional directory(Path) # the directory to change to\nend" ->
|
||||
b::token_list(vec![
|
||||
b::bare("def"),
|
||||
b::sp(),
|
||||
b::bare("cd"),
|
||||
b::sep("\n"),
|
||||
b::ws(" "),
|
||||
b::comment(" Change to a new path."),
|
||||
b::sep("\n"),
|
||||
b::ws(" "),
|
||||
b::bare("optional"),
|
||||
b::sp(),
|
||||
b::bare("directory"),
|
||||
b::parens(vec![b::bare("Path")]),
|
||||
b::sp(),
|
||||
b::comment(" the directory to change to"),
|
||||
b::sep("\n"),
|
||||
b::bare("end")
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
// #[test]
|
||||
// fn test_smoke_pipeline() {
|
||||
// let _ = pretty_env_logger::try_init();
|
||||
@ -1279,7 +1622,18 @@ mod tests {
|
||||
desc: &str,
|
||||
string: &str,
|
||||
) -> TokenNode {
|
||||
f(nom_input(string)).unwrap().1
|
||||
let result = f(nom_input(string));
|
||||
|
||||
match result {
|
||||
Ok(value) => value.1,
|
||||
Err(err) => {
|
||||
let err = nu_errors::ShellError::parse_error(err);
|
||||
|
||||
println!("{:?}", string);
|
||||
crate::hir::baseline_parse::tests::print_err(err, &nu_source::Text::from(string));
|
||||
panic!("test failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn span((left, right): (usize, usize)) -> Span {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
use crate::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
|
||||
use crate::parse::{call_node::*, comment::*, flag::*, operator::*, pipeline::*, tokens::*};
|
||||
use derive_new::new;
|
||||
use getset::Getters;
|
||||
use nu_errors::{ParseError, ShellError};
|
||||
@ -18,7 +18,9 @@ pub enum TokenNode {
|
||||
Delimited(Spanned<DelimitedNode>),
|
||||
Pipeline(Pipeline),
|
||||
Flag(Flag),
|
||||
Comment(Comment),
|
||||
Whitespace(Span),
|
||||
Separator(Span),
|
||||
|
||||
Error(Spanned<ShellError>),
|
||||
}
|
||||
@ -39,14 +41,32 @@ impl PrettyDebugWithSource for TokenNode {
|
||||
"whitespace",
|
||||
b::description(format!("{:?}", space.slice(source))),
|
||||
),
|
||||
TokenNode::Separator(span) => b::typed(
|
||||
"separator",
|
||||
b::description(format!("{:?}", span.slice(source))),
|
||||
),
|
||||
TokenNode::Comment(comment) => {
|
||||
b::typed("comment", b::description(comment.text.slice(source)))
|
||||
}
|
||||
TokenNode::Error(_) => b::error("error"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HasSpan for TokenNode {
|
||||
fn span(&self) -> Span {
|
||||
self.get_span()
|
||||
impl ShellTypeName for TokenNode {
|
||||
fn type_name(&self) -> &'static str {
|
||||
match self {
|
||||
TokenNode::Token(t) => t.type_name(),
|
||||
TokenNode::Nodes(_) => "nodes",
|
||||
TokenNode::Call(_) => "command",
|
||||
TokenNode::Delimited(d) => d.type_name(),
|
||||
TokenNode::Pipeline(_) => "pipeline",
|
||||
TokenNode::Flag(_) => "flag",
|
||||
TokenNode::Whitespace(_) => "whitespace",
|
||||
TokenNode::Separator(_) => "separator",
|
||||
TokenNode::Comment(_) => "comment",
|
||||
TokenNode::Error(_) => "error",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -107,12 +127,12 @@ impl fmt::Debug for DebugTokenNode<'_> {
|
||||
|
||||
impl From<&TokenNode> for Span {
|
||||
fn from(token: &TokenNode) -> Span {
|
||||
token.get_span()
|
||||
token.span()
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenNode {
|
||||
pub fn get_span(&self) -> Span {
|
||||
impl HasSpan for TokenNode {
|
||||
fn span(&self) -> Span {
|
||||
match self {
|
||||
TokenNode::Token(t) => t.span,
|
||||
TokenNode::Nodes(t) => t.span,
|
||||
@ -121,27 +141,14 @@ impl TokenNode {
|
||||
TokenNode::Pipeline(s) => s.span,
|
||||
TokenNode::Flag(s) => s.span,
|
||||
TokenNode::Whitespace(s) => *s,
|
||||
TokenNode::Separator(s) => *s,
|
||||
TokenNode::Comment(c) => c.span(),
|
||||
TokenNode::Error(s) => s.span,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn type_name(&self) -> &'static str {
|
||||
match self {
|
||||
TokenNode::Token(t) => t.type_name(),
|
||||
TokenNode::Nodes(_) => "nodes",
|
||||
TokenNode::Call(_) => "command",
|
||||
TokenNode::Delimited(d) => d.type_name(),
|
||||
TokenNode::Pipeline(_) => "pipeline",
|
||||
TokenNode::Flag(_) => "flag",
|
||||
TokenNode::Whitespace(_) => "whitespace",
|
||||
TokenNode::Error(_) => "error",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spanned_type_name(&self) -> Spanned<&'static str> {
|
||||
self.type_name().spanned(self.span())
|
||||
}
|
||||
|
||||
impl TokenNode {
|
||||
pub fn tagged_type_name(&self) -> Tagged<&'static str> {
|
||||
self.type_name().tagged(self.span())
|
||||
}
|
||||
@ -244,7 +251,7 @@ impl TokenNode {
|
||||
pub fn is_dot(&self) -> bool {
|
||||
match self {
|
||||
TokenNode::Token(Token {
|
||||
unspanned: UnspannedToken::Operator(Operator::Dot),
|
||||
unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot),
|
||||
..
|
||||
}) => true,
|
||||
_ => false,
|
||||
@ -421,7 +428,7 @@ impl TokenNode {
|
||||
pub fn expect_dot(&self) -> Span {
|
||||
match self {
|
||||
TokenNode::Token(Token {
|
||||
unspanned: UnspannedToken::Operator(Operator::Dot),
|
||||
unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot),
|
||||
span,
|
||||
}) => *span,
|
||||
other => panic!("Expected dot, found {:?}", other),
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
use crate::parse::call_node::CallNode;
|
||||
use crate::parse::comment::Comment;
|
||||
use crate::parse::flag::{Flag, FlagKind};
|
||||
use crate::parse::operator::Operator;
|
||||
use crate::parse::operator::{CompareOperator, EvaluationOperator};
|
||||
use crate::parse::pipeline::{Pipeline, PipelineElement};
|
||||
use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
|
||||
use crate::parse::tokens::{RawNumber, UnspannedToken};
|
||||
@ -96,7 +97,7 @@ impl TokenTreeBuilder {
|
||||
TokenNode::Nodes(input.spanned(span.into()))
|
||||
}
|
||||
|
||||
pub fn op(input: impl Into<Operator>) -> CurriedToken {
|
||||
pub fn op(input: impl Into<CompareOperator>) -> CurriedToken {
|
||||
let input = input.into();
|
||||
|
||||
Box::new(move |b| {
|
||||
@ -104,12 +105,39 @@ impl TokenTreeBuilder {
|
||||
|
||||
b.pos = end;
|
||||
|
||||
TokenTreeBuilder::spanned_op(input, Span::new(start, end))
|
||||
TokenTreeBuilder::spanned_cmp_op(input, Span::new(start, end))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn spanned_op(input: impl Into<Operator>, span: impl Into<Span>) -> TokenNode {
|
||||
TokenNode::Token(UnspannedToken::Operator(input.into()).into_token(span))
|
||||
pub fn spanned_cmp_op(input: impl Into<CompareOperator>, span: impl Into<Span>) -> TokenNode {
|
||||
TokenNode::Token(UnspannedToken::CompareOperator(input.into()).into_token(span))
|
||||
}
|
||||
|
||||
pub fn dot() -> CurriedToken {
|
||||
Box::new(move |b| {
|
||||
let (start, end) = b.consume(".");
|
||||
|
||||
b.pos = end;
|
||||
|
||||
TokenTreeBuilder::spanned_eval_op(".", Span::new(start, end))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn dotdot() -> CurriedToken {
|
||||
Box::new(move |b| {
|
||||
let (start, end) = b.consume("..");
|
||||
|
||||
b.pos = end;
|
||||
|
||||
TokenTreeBuilder::spanned_eval_op("..", Span::new(start, end))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn spanned_eval_op(
|
||||
input: impl Into<EvaluationOperator>,
|
||||
span: impl Into<Span>,
|
||||
) -> TokenNode {
|
||||
TokenNode::Token(UnspannedToken::EvaluationOperator(input.into()).into_token(span))
|
||||
}
|
||||
|
||||
pub fn string(input: impl Into<String>) -> CurriedToken {
|
||||
@ -398,6 +426,36 @@ impl TokenTreeBuilder {
|
||||
TokenNode::Whitespace(span.into())
|
||||
}
|
||||
|
||||
pub fn sep(input: impl Into<String>) -> CurriedToken {
|
||||
let input = input.into();
|
||||
|
||||
Box::new(move |b| {
|
||||
let (start, end) = b.consume(&input);
|
||||
TokenTreeBuilder::spanned_sep(Span::new(start, end))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn spanned_sep(span: impl Into<Span>) -> TokenNode {
|
||||
TokenNode::Separator(span.into())
|
||||
}
|
||||
|
||||
pub fn comment(input: impl Into<String>) -> CurriedToken {
|
||||
let input = input.into();
|
||||
|
||||
Box::new(move |b| {
|
||||
let outer_start = b.pos;
|
||||
b.consume("#");
|
||||
let (start, end) = b.consume(&input);
|
||||
let outer_end = b.pos;
|
||||
|
||||
TokenTreeBuilder::spanned_comment((start, end), (outer_start, outer_end))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn spanned_comment(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
|
||||
TokenNode::Comment(Comment::line(input, span))
|
||||
}
|
||||
|
||||
fn consume(&mut self, input: &str) -> (usize, usize) {
|
||||
let start = self.pos;
|
||||
self.pos += input.len();
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
use crate::parse::parser::Number;
|
||||
use crate::Operator;
|
||||
use crate::{CompareOperator, EvaluationOperator};
|
||||
use bigdecimal::BigDecimal;
|
||||
use nu_protocol::ShellTypeName;
|
||||
use nu_source::{
|
||||
@ -13,7 +13,8 @@ use std::str::FromStr;
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
|
||||
pub enum UnspannedToken {
|
||||
Number(RawNumber),
|
||||
Operator(Operator),
|
||||
CompareOperator(CompareOperator),
|
||||
EvaluationOperator(EvaluationOperator),
|
||||
String(Span),
|
||||
Variable(Span),
|
||||
ExternalCommand(Span),
|
||||
@ -35,7 +36,9 @@ impl ShellTypeName for UnspannedToken {
|
||||
fn type_name(&self) -> &'static str {
|
||||
match self {
|
||||
UnspannedToken::Number(_) => "number",
|
||||
UnspannedToken::Operator(..) => "operator",
|
||||
UnspannedToken::CompareOperator(..) => "comparison operator",
|
||||
UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => "dot",
|
||||
UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => "dotdot",
|
||||
UnspannedToken::String(_) => "string",
|
||||
UnspannedToken::Variable(_) => "variable",
|
||||
UnspannedToken::ExternalCommand(_) => "syntax error",
|
||||
@ -111,7 +114,8 @@ impl PrettyDebugWithSource for Token {
|
||||
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
|
||||
match self.unspanned {
|
||||
UnspannedToken::Number(number) => number.pretty_debug(source),
|
||||
UnspannedToken::Operator(operator) => operator.pretty(),
|
||||
UnspannedToken::CompareOperator(operator) => operator.pretty(),
|
||||
UnspannedToken::EvaluationOperator(operator) => operator.pretty(),
|
||||
UnspannedToken::String(_) => b::primitive(self.span.slice(source)),
|
||||
UnspannedToken::Variable(_) => b::var(self.span.slice(source)),
|
||||
UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)),
|
||||
@ -155,9 +159,9 @@ impl Token {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_operator(&self) -> Option<Spanned<Operator>> {
|
||||
pub fn extract_operator(&self) -> Option<Spanned<CompareOperator>> {
|
||||
match self.unspanned {
|
||||
UnspannedToken::Operator(operator) => Some(operator.spanned(self.span)),
|
||||
UnspannedToken::CompareOperator(operator) => Some(operator.spanned(self.span)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user