Extract core stuff into own crates

This commit extracts five new crates:

- nu-source, which contains the core source-code handling logic in Nu,
  including Text, Span, and also the pretty.rs-based debug logic
- nu-parser, which is the parser and expander logic
- nu-protocol, which is the bulk of the types and basic conveniences
  used by plugins
- nu-errors, which contains ShellError, ParseError and error handling
  conveniences
- nu-textview, which is the textview plugin extracted into a crate

One of the major consequences of this refactor is that it's no longer
possible to `impl X for Spanned<Y>` outside of the `nu-source` crate, so
a lot of types became more concrete (Value became a concrete type
instead of Spanned<Value>, for example).

This also turned a number of inherent methods in the main nu crate into
plain functions (impl Value {} became a bunch of functions in the
`value` namespace in `crate::data::value`).
This commit is contained in:
Yehuda Katz
2019-11-25 18:30:48 -08:00
parent 2eae5a2a89
commit e4226def16
205 changed files with 3491 additions and 2605 deletions

View File

@ -0,0 +1,45 @@
use crate::TokenNode;
use getset::Getters;
use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource};
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
pub struct CallNode {
#[get = "pub(crate)"]
head: Box<TokenNode>,
#[get = "pub(crate)"]
children: Option<Vec<TokenNode>>,
}
impl PrettyDebugWithSource for CallNode {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
b::typed(
"call",
self.head.pretty_debug(source)
+ b::preceded(
b::space(),
b::intersperse(
self.children.iter().flat_map(|children| {
children.iter().map(|child| child.pretty_debug(source))
}),
b::space(),
),
),
)
}
}
impl CallNode {
pub fn new(head: Box<TokenNode>, children: Vec<TokenNode>) -> CallNode {
if children.len() == 0 {
CallNode {
head,
children: None,
}
} else {
CallNode {
head,
children: Some(children),
}
}
}
}

View File

@ -0,0 +1,101 @@
use derive_new::new;
use language_reporting::{FileName, Location};
use log::trace;
use nu_source::Span;
#[derive(new, Debug, Clone)]
pub struct Files {
snippet: String,
}
impl language_reporting::ReportingFiles for Files {
type Span = Span;
type FileId = usize;
fn byte_span(
&self,
_file: Self::FileId,
from_index: usize,
to_index: usize,
) -> Option<Self::Span> {
Some(Span::new(from_index, to_index))
}
fn file_id(&self, _tag: Self::Span) -> Self::FileId {
0
}
fn file_name(&self, _file: Self::FileId) -> FileName {
FileName::Verbatim(format!("shell"))
}
fn byte_index(&self, _file: Self::FileId, _line: usize, _column: usize) -> Option<usize> {
unimplemented!("byte_index")
}
fn location(&self, _file: Self::FileId, byte_index: usize) -> Option<Location> {
let source = &self.snippet;
let mut seen_lines = 0;
let mut seen_bytes = 0;
for (pos, slice) in source.match_indices('\n') {
trace!(
"SEARCH={} SEEN={} POS={} SLICE={:?} LEN={} ALL={:?}",
byte_index,
seen_bytes,
pos,
slice,
source.len(),
source
);
if pos >= byte_index {
return Some(language_reporting::Location::new(
seen_lines,
byte_index - seen_bytes,
));
} else {
seen_lines += 1;
seen_bytes = pos;
}
}
if seen_lines == 0 {
Some(language_reporting::Location::new(0, byte_index))
} else {
panic!("byte index {} wasn't valid", byte_index);
}
}
fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option<Self::Span> {
let source = &self.snippet;
let mut seen_lines = 0;
let mut seen_bytes = 0;
for (pos, _) in source.match_indices('\n') {
if seen_lines == lineno {
return Some(Span::new(seen_bytes, pos + 1));
} else {
seen_lines += 1;
seen_bytes = pos + 1;
}
}
if seen_lines == 0 {
Some(Span::new(0, self.snippet.len() - 1))
} else {
None
}
}
fn source(&self, span: Self::Span) -> Option<String> {
trace!("source(tag={:?}) snippet={:?}", span, self.snippet);
if span.start() > span.end() {
return None;
} else if span.end() > self.snippet.len() {
return None;
}
Some(span.slice(&self.snippet).to_string())
}
}

View File

@ -0,0 +1,39 @@
use crate::hir::syntax_shape::flat_shape::FlatShape;
use derive_new::new;
use getset::Getters;
use nu_source::{Span, b, Spanned, SpannedItem, PrettyDebugWithSource, DebugDocBuilder};
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub enum FlagKind {
Shorthand,
Longhand,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)]
#[get = "pub(crate)"]
pub struct Flag {
pub(crate) kind: FlagKind,
pub(crate) name: Span,
pub(crate) span: Span,
}
impl PrettyDebugWithSource for Flag {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
let prefix = match self.kind {
FlagKind::Longhand => b::description("--"),
FlagKind::Shorthand => b::description("-"),
};
prefix + b::description(self.name.slice(source))
}
}
impl Flag {
pub fn color(&self) -> Spanned<FlatShape> {
match self.kind {
FlagKind::Longhand => FlatShape::Flag.spanned(self.span),
FlagKind::Shorthand => FlatShape::ShorthandFlag.spanned(self.span),
}
}
}

View File

@ -0,0 +1,67 @@
use serde::{Deserialize, Serialize};
use nu_source::{b, PrettyDebug, DebugDocBuilder};
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub enum Operator {
Equal,
NotEqual,
LessThan,
GreaterThan,
LessThanOrEqual,
GreaterThanOrEqual,
Dot,
Contains,
NotContains,
}
impl PrettyDebug for Operator {
fn pretty(&self) -> DebugDocBuilder {
b::operator(self.as_str())
}
}
impl Operator {
pub fn print(&self) -> String {
self.as_str().to_string()
}
pub fn as_str(&self) -> &str {
match *self {
Operator::Equal => "==",
Operator::NotEqual => "!=",
Operator::LessThan => "<",
Operator::GreaterThan => ">",
Operator::LessThanOrEqual => "<=",
Operator::GreaterThanOrEqual => ">=",
Operator::Dot => ".",
Operator::Contains => "=~",
Operator::NotContains => "!~",
}
}
}
impl From<&str> for Operator {
fn from(input: &str) -> Operator {
Operator::from_str(input).unwrap()
}
}
impl FromStr for Operator {
type Err = ();
fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
match input {
"==" => Ok(Operator::Equal),
"!=" => Ok(Operator::NotEqual),
"<" => Ok(Operator::LessThan),
">" => Ok(Operator::GreaterThan),
"<=" => Ok(Operator::LessThanOrEqual),
">=" => Ok(Operator::GreaterThanOrEqual),
"." => Ok(Operator::Dot),
"=~" => Ok(Operator::Contains),
"!~" => Ok(Operator::NotContains),
_ => Err(()),
}
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,75 @@
use crate::TokenNode;
use derive_new::new;
use getset::Getters;
use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span, Spanned, HasSpan};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
pub struct Pipeline {
#[get = "pub"]
pub(crate) parts: Vec<PipelineElement>,
pub(crate) span: Span,
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
pub struct Tokens {
pub(crate) tokens: Vec<TokenNode>,
pub(crate) span: Span,
}
impl Tokens {
pub fn iter(&self) -> impl Iterator<Item = &TokenNode> {
self.tokens.iter()
}
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
pub struct PipelineElement {
pub pipe: Option<Span>,
pub tokens: Tokens,
}
impl HasSpan for PipelineElement {
fn span(&self) -> Span {
match self.pipe {
Option::None => self.tokens.span,
Option::Some(pipe) => pipe.until(self.tokens.span),
}
}
}
impl PipelineElement {
pub fn new(pipe: Option<Span>, tokens: Spanned<Vec<TokenNode>>) -> PipelineElement {
PipelineElement {
pipe,
tokens: Tokens {
tokens: tokens.item,
span: tokens.span,
},
}
}
pub fn tokens(&self) -> &[TokenNode] {
&self.tokens.tokens
}
}
impl PrettyDebugWithSource for Pipeline {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
b::intersperse(
self.parts.iter().map(|token| token.pretty_debug(source)),
b::operator(" | "),
)
}
}
impl PrettyDebugWithSource for PipelineElement {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
b::intersperse(
self.tokens.iter().map(|token| match token {
TokenNode::Whitespace(_) => b::blank(),
token => token.pretty_debug(source),
}),
b::space(),
)
}
}

View File

@ -0,0 +1,440 @@
use crate::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
use derive_new::new;
use getset::Getters;
use nu_errors::{ParseError, ShellError};
use nu_protocol::ShellTypeName;
use nu_source::{
b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, Tagged,
TaggedItem, Text,
};
use std::fmt;
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub enum TokenNode {
Token(Token),
Call(Spanned<CallNode>),
Nodes(Spanned<Vec<TokenNode>>),
Delimited(Spanned<DelimitedNode>),
Pipeline(Pipeline),
Flag(Flag),
Whitespace(Span),
Error(Spanned<ShellError>),
}
impl PrettyDebugWithSource for TokenNode {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
match self {
TokenNode::Token(token) => token.pretty_debug(source),
TokenNode::Call(call) => call.pretty_debug(source),
TokenNode::Nodes(nodes) => b::intersperse(
nodes.iter().map(|node| node.pretty_debug(source)),
b::space(),
),
TokenNode::Delimited(delimited) => delimited.pretty_debug(source),
TokenNode::Pipeline(pipeline) => pipeline.pretty_debug(source),
TokenNode::Flag(flag) => flag.pretty_debug(source),
TokenNode::Whitespace(space) => b::typed(
"whitespace",
b::description(format!("{:?}", space.slice(source))),
),
TokenNode::Error(_) => b::error("error"),
}
}
}
impl HasSpan for TokenNode {
fn span(&self) -> Span {
self.get_span()
}
}
pub struct DebugTokenNode<'a> {
node: &'a TokenNode,
source: &'a Text,
}
impl fmt::Debug for DebugTokenNode<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self.node {
TokenNode::Token(t) => write!(f, "{:?}", t.debug(self.source)),
TokenNode::Call(s) => {
write!(f, "(")?;
write!(f, "{}", s.head().debug(self.source))?;
if let Some(children) = s.children() {
for child in children {
write!(f, "{}", child.debug(self.source))?;
}
}
write!(f, ")")
}
TokenNode::Delimited(d) => {
write!(
f,
"{}",
match d.delimiter {
Delimiter::Brace => "{",
Delimiter::Paren => "(",
Delimiter::Square => "[",
}
)?;
for child in d.children() {
write!(f, "{:?}", child.old_debug(self.source))?;
}
write!(
f,
"{}",
match d.delimiter {
Delimiter::Brace => "}",
Delimiter::Paren => ")",
Delimiter::Square => "]",
}
)
}
TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)),
TokenNode::Error(_) => write!(f, "<error>"),
rest => write!(f, "{}", rest.span().slice(self.source)),
}
}
}
impl From<&TokenNode> for Span {
fn from(token: &TokenNode) -> Span {
token.get_span()
}
}
impl TokenNode {
pub fn get_span(&self) -> Span {
match self {
TokenNode::Token(t) => t.span,
TokenNode::Nodes(t) => t.span,
TokenNode::Call(s) => s.span,
TokenNode::Delimited(s) => s.span,
TokenNode::Pipeline(s) => s.span,
TokenNode::Flag(s) => s.span,
TokenNode::Whitespace(s) => *s,
TokenNode::Error(s) => s.span,
}
}
pub fn type_name(&self) -> &'static str {
match self {
TokenNode::Token(t) => t.type_name(),
TokenNode::Nodes(_) => "nodes",
TokenNode::Call(_) => "command",
TokenNode::Delimited(d) => d.type_name(),
TokenNode::Pipeline(_) => "pipeline",
TokenNode::Flag(_) => "flag",
TokenNode::Whitespace(_) => "whitespace",
TokenNode::Error(_) => "error",
}
}
pub fn spanned_type_name(&self) -> Spanned<&'static str> {
self.type_name().spanned(self.span())
}
pub fn tagged_type_name(&self) -> Tagged<&'static str> {
self.type_name().tagged(self.span())
}
pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
DebugTokenNode { node: self, source }
}
pub fn as_external_arg(&self, source: &Text) -> String {
self.span().slice(source).to_string()
}
pub fn source<'a>(&self, source: &'a Text) -> &'a str {
self.span().slice(source)
}
pub fn get_variable(&self) -> Result<(Span, Span), ShellError> {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Variable(inner_span),
span: outer_span,
}) => Ok((*outer_span, *inner_span)),
_ => Err(ShellError::type_error(
"variable",
self.type_name().spanned(self.span()),
)),
}
}
pub fn is_bare(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Bare,
..
}) => true,
_ => false,
}
}
pub fn is_string(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::String(_),
..
}) => true,
_ => false,
}
}
pub fn is_number(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Number(_),
..
}) => true,
_ => false,
}
}
pub fn as_string(&self) -> Option<(Span, Span)> {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::String(inner_span),
span: outer_span,
}) => Some((*outer_span, *inner_span)),
_ => None,
}
}
pub fn is_pattern(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::GlobPattern,
..
}) => true,
_ => false,
}
}
pub fn is_word(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Bare,
..
}) => true,
_ => false,
}
}
pub fn is_int(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Number(RawNumber::Int(_)),
..
}) => true,
_ => false,
}
}
pub fn is_dot(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Operator(Operator::Dot),
..
}) => true,
_ => false,
}
}
pub fn as_block(&self) -> Option<(Spanned<&[TokenNode]>, (Span, Span))> {
match self {
TokenNode::Delimited(Spanned {
item:
DelimitedNode {
delimiter,
children,
spans,
},
span,
}) if *delimiter == Delimiter::Brace => Some(((&children[..]).spanned(*span), *spans)),
_ => None,
}
}
pub fn is_external(&self) -> bool {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::ExternalCommand(..),
..
}) => true,
_ => false,
}
}
pub(crate) fn as_flag(&self, value: &str, source: &Text) -> Option<Flag> {
match self {
TokenNode::Flag(flag @ Flag { .. }) if value == flag.name().slice(source) => {
Some(*flag)
}
_ => None,
}
}
pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
match self {
TokenNode::Pipeline(pipeline) => Ok(pipeline.clone()),
other => Err(ParseError::mismatch(
"pipeline",
other.type_name().spanned(other.span()),
)),
}
}
pub fn is_whitespace(&self) -> bool {
match self {
TokenNode::Whitespace(_) => true,
_ => false,
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
#[get = "pub(crate)"]
pub struct DelimitedNode {
pub(crate) delimiter: Delimiter,
pub(crate) spans: (Span, Span),
pub(crate) children: Vec<TokenNode>,
}
impl PrettyDebugWithSource for DelimitedNode {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
b::delimit(
self.delimiter.open(),
b::intersperse(
self.children.iter().map(|child| child.pretty_debug(source)),
b::space(),
),
self.delimiter.close(),
)
}
}
impl DelimitedNode {
pub fn type_name(&self) -> &'static str {
match self.delimiter {
Delimiter::Brace => "braced expression",
Delimiter::Paren => "parenthesized expression",
Delimiter::Square => "array literal or index operator",
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub enum Delimiter {
Paren,
Brace,
Square,
}
impl Delimiter {
pub(crate) fn open(&self) -> &'static str {
match self {
Delimiter::Paren => "(",
Delimiter::Brace => "{",
Delimiter::Square => "[",
}
}
pub(crate) fn close(&self) -> &'static str {
match self {
Delimiter::Paren => ")",
Delimiter::Brace => "}",
Delimiter::Square => "]",
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
#[get = "pub(crate)"]
pub struct PathNode {
head: Box<TokenNode>,
tail: Vec<TokenNode>,
}
#[cfg(test)]
impl TokenNode {
pub fn expect_external(&self) -> Span {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::ExternalCommand(span),
..
}) => *span,
other => panic!(
"Only call expect_external if you checked is_external first, found {:?}",
other
),
}
}
pub fn expect_string(&self) -> (Span, Span) {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::String(inner_span),
span: outer_span,
}) => (*outer_span, *inner_span),
other => panic!("Expected string, found {:?}", other),
}
}
pub fn expect_list(&self) -> Spanned<&[TokenNode]> {
match self {
TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span),
other => panic!("Expected list, found {:?}", other),
}
}
pub fn expect_pattern(&self) -> Span {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::GlobPattern,
span: outer_span,
}) => *outer_span,
other => panic!("Expected pattern, found {:?}", other),
}
}
pub fn expect_var(&self) -> (Span, Span) {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Variable(inner_span),
span: outer_span,
}) => (*outer_span, *inner_span),
other => panic!("Expected var, found {:?}", other),
}
}
pub fn expect_dot(&self) -> Span {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Operator(Operator::Dot),
span,
}) => *span,
other => panic!("Expected dot, found {:?}", other),
}
}
pub fn expect_bare(&self) -> Span {
match self {
TokenNode::Token(Token {
unspanned: UnspannedToken::Bare,
span,
}) => *span,
other => panic!("Expected bare, found {:?}", other),
}
}
}

View File

@ -0,0 +1,414 @@
use crate::parse::call_node::CallNode;
use crate::parse::flag::{Flag, FlagKind};
use crate::parse::operator::Operator;
use crate::parse::pipeline::{Pipeline, PipelineElement};
use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
use crate::parse::tokens::{RawNumber, UnspannedToken};
use bigdecimal::BigDecimal;
use nu_source::{Span, Spanned, SpannedItem};
use num_bigint::BigInt;
pub struct TokenTreeBuilder {
pos: usize,
output: String,
}
impl TokenTreeBuilder {
pub fn new() -> TokenTreeBuilder {
TokenTreeBuilder {
pos: 0,
output: String::new(),
}
}
}
pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;
impl TokenTreeBuilder {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
let mut builder = TokenTreeBuilder::new();
let node = block(&mut builder);
(node, builder.output)
}
fn build_spanned<T>(
&mut self,
callback: impl FnOnce(&mut TokenTreeBuilder) -> T,
) -> Spanned<T> {
let start = self.pos;
let ret = callback(self);
let end = self.pos;
ret.spanned(Span::new(start, end))
}
pub fn pipeline(input: Vec<Vec<CurriedToken>>) -> CurriedToken {
Box::new(move |b| {
let start = b.pos;
let mut out: Vec<PipelineElement> = vec![];
let mut input = input.into_iter().peekable();
let head = input
.next()
.expect("A pipeline must contain at least one element");
let pipe = None;
let head = b.build_spanned(|b| head.into_iter().map(|node| node(b)).collect());
out.push(PipelineElement::new(pipe, head));
loop {
match input.next() {
None => break,
Some(node) => {
let pipe = Some(b.consume_span("|"));
let node =
b.build_spanned(|b| node.into_iter().map(|node| node(b)).collect());
out.push(PipelineElement::new(pipe, node));
}
}
}
let end = b.pos;
TokenTreeBuilder::spanned_pipeline(out, Span::new(start, end))
})
}
pub fn spanned_pipeline(input: Vec<PipelineElement>, span: impl Into<Span>) -> TokenNode {
TokenNode::Pipeline(Pipeline::new(input, span.into()))
}
pub fn token_list(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let start = b.pos;
let tokens = input.into_iter().map(|i| i(b)).collect();
let end = b.pos;
TokenTreeBuilder::spanned_token_list(tokens, Span::new(start, end))
})
}
pub fn spanned_token_list(input: Vec<TokenNode>, span: impl Into<Span>) -> TokenNode {
TokenNode::Nodes(input.spanned(span.into()))
}
pub fn op(input: impl Into<Operator>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(input.as_str());
b.pos = end;
TokenTreeBuilder::spanned_op(input, Span::new(start, end))
})
}
pub fn spanned_op(input: impl Into<Operator>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::Operator(input.into()).into_token(span))
}
pub fn string(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, _) = b.consume("\"");
let (inner_start, inner_end) = b.consume(&input);
let (_, end) = b.consume("\"");
b.pos = end;
TokenTreeBuilder::spanned_string(
Span::new(inner_start, inner_end),
Span::new(start, end),
)
})
}
pub fn spanned_string(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::String(input.into()).into_token(span))
}
pub fn bare(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_bare(Span::new(start, end))
})
}
pub fn spanned_bare(span: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::Bare.into_token(span))
}
pub fn pattern(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_pattern(Span::new(start, end))
})
}
pub fn spanned_pattern(input: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::GlobPattern.into_token(input))
}
pub fn external_word(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_external_word(Span::new(start, end))
})
}
pub fn spanned_external_word(input: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::ExternalWord.into_token(input))
}
pub fn external_command(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (outer_start, _) = b.consume("^");
let (inner_start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_external_command(
Span::new(inner_start, end),
Span::new(outer_start, end),
)
})
}
pub fn spanned_external_command(inner: impl Into<Span>, outer: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::ExternalCommand(inner.into()).into_token(outer))
}
pub fn int(input: impl Into<BigInt>) -> CurriedToken {
let int = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&int.to_string());
b.pos = end;
TokenTreeBuilder::spanned_number(
RawNumber::Int(Span::new(start, end)),
Span::new(start, end),
)
})
}
pub fn decimal(input: impl Into<BigDecimal>) -> CurriedToken {
let decimal = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&decimal.to_string());
b.pos = end;
TokenTreeBuilder::spanned_number(
RawNumber::Decimal(Span::new(start, end)),
Span::new(start, end),
)
})
}
pub fn spanned_number(input: impl Into<RawNumber>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::Number(input.into()).into_token(span))
}
pub fn var(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, _) = b.consume("$");
let (inner_start, end) = b.consume(&input);
TokenTreeBuilder::spanned_var(Span::new(inner_start, end), Span::new(start, end))
})
}
pub fn spanned_var(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(UnspannedToken::Variable(input.into()).into_token(span))
}
pub fn flag(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, _) = b.consume("--");
let (inner_start, end) = b.consume(&input);
TokenTreeBuilder::spanned_flag(Span::new(inner_start, end), Span::new(start, end))
})
}
pub fn spanned_flag(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Flag(Flag::new(FlagKind::Longhand, input.into(), span.into()))
}
pub fn shorthand(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, _) = b.consume("-");
let (inner_start, end) = b.consume(&input);
TokenTreeBuilder::spanned_shorthand((inner_start, end), (start, end))
})
}
pub fn spanned_shorthand(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Flag(Flag::new(FlagKind::Shorthand, input.into(), span.into()))
}
pub fn call(head: CurriedToken, input: Vec<CurriedToken>) -> CurriedCall {
Box::new(move |b| {
let start = b.pos;
let head_node = head(b);
let mut nodes = vec![head_node];
for item in input {
nodes.push(item(b));
}
let end = b.pos;
TokenTreeBuilder::spanned_call(nodes, Span::new(start, end))
})
}
pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Spanned<CallNode> {
if input.len() == 0 {
panic!("BUG: spanned call (TODO)")
}
let mut input = input.into_iter();
let head = input.next().unwrap();
let tail = input.collect();
CallNode::new(Box::new(head), tail).spanned(span.into())
}
fn consume_delimiter(
&mut self,
input: Vec<CurriedToken>,
_open: &str,
_close: &str,
) -> (Span, Span, Span, Vec<TokenNode>) {
let (start_open_paren, end_open_paren) = self.consume("(");
let mut output = vec![];
for item in input {
output.push(item(self));
}
let (start_close_paren, end_close_paren) = self.consume(")");
let open = Span::new(start_open_paren, end_open_paren);
let close = Span::new(start_close_paren, end_close_paren);
let whole = Span::new(start_open_paren, end_close_paren);
(open, close, whole, output)
}
pub fn parens(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let (open, close, whole, output) = b.consume_delimiter(input, "(", ")");
TokenTreeBuilder::spanned_parens(output, (open, close), whole)
})
}
pub fn spanned_parens(
input: impl Into<Vec<TokenNode>>,
spans: (Span, Span),
span: impl Into<Span>,
) -> TokenNode {
TokenNode::Delimited(
DelimitedNode::new(Delimiter::Paren, spans, input.into()).spanned(span.into()),
)
}
pub fn square(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let (open, close, whole, tokens) = b.consume_delimiter(input, "[", "]");
TokenTreeBuilder::spanned_square(tokens, (open, close), whole)
})
}
pub fn spanned_square(
input: impl Into<Vec<TokenNode>>,
spans: (Span, Span),
span: impl Into<Span>,
) -> TokenNode {
TokenNode::Delimited(
DelimitedNode::new(Delimiter::Square, spans, input.into()).spanned(span.into()),
)
}
pub fn braced(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let (open, close, whole, tokens) = b.consume_delimiter(input, "{", "}");
TokenTreeBuilder::spanned_brace(tokens, (open, close), whole)
})
}
pub fn spanned_brace(
input: impl Into<Vec<TokenNode>>,
spans: (Span, Span),
span: impl Into<Span>,
) -> TokenNode {
TokenNode::Delimited(
DelimitedNode::new(Delimiter::Brace, spans, input.into()).spanned(span.into()),
)
}
pub fn sp() -> CurriedToken {
Box::new(|b| {
let (start, end) = b.consume(" ");
TokenNode::Whitespace(Span::new(start, end))
})
}
pub fn ws(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
TokenTreeBuilder::spanned_ws(Span::new(start, end))
})
}
pub fn spanned_ws(span: impl Into<Span>) -> TokenNode {
TokenNode::Whitespace(span.into())
}
fn consume(&mut self, input: &str) -> (usize, usize) {
let start = self.pos;
self.pos += input.len();
self.output.push_str(input);
(start, self.pos)
}
fn consume_span(&mut self, input: &str) -> Span {
let start = self.pos;
self.pos += input.len();
self.output.push_str(input);
Span::new(start, self.pos)
}
}

View File

@ -0,0 +1,217 @@
use crate::parse::parser::Number;
use crate::Operator;
use bigdecimal::BigDecimal;
use nu_protocol::ShellTypeName;
use nu_source::{
b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
Text,
};
use num_bigint::BigInt;
use std::fmt;
use std::str::FromStr;
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum UnspannedToken {
Number(RawNumber),
Operator(Operator),
String(Span),
Variable(Span),
ExternalCommand(Span),
ExternalWord,
GlobPattern,
Bare,
}
impl UnspannedToken {
pub fn into_token(self, span: impl Into<Span>) -> Token {
Token {
unspanned: self,
span: span.into(),
}
}
}
impl ShellTypeName for UnspannedToken {
fn type_name(&self) -> &'static str {
match self {
UnspannedToken::Number(_) => "number",
UnspannedToken::Operator(..) => "operator",
UnspannedToken::String(_) => "string",
UnspannedToken::Variable(_) => "variable",
UnspannedToken::ExternalCommand(_) => "syntax error",
UnspannedToken::ExternalWord => "syntax error",
UnspannedToken::GlobPattern => "glob pattern",
UnspannedToken::Bare => "string",
}
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum RawNumber {
Int(Span),
Decimal(Span),
}
impl HasSpan for RawNumber {
fn span(&self) -> Span {
match self {
RawNumber::Int(span) => *span,
RawNumber::Decimal(span) => *span,
}
}
}
impl PrettyDebugWithSource for RawNumber {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
match self {
RawNumber::Int(span) => b::primitive(span.slice(source)),
RawNumber::Decimal(span) => b::primitive(span.slice(source)),
}
}
}
impl RawNumber {
pub fn int(span: impl Into<Span>) -> RawNumber {
let span = span.into();
RawNumber::Int(span)
}
pub fn decimal(span: impl Into<Span>) -> RawNumber {
let span = span.into();
RawNumber::Decimal(span)
}
pub(crate) fn to_number(self, source: &Text) -> Number {
match self {
RawNumber::Int(tag) => Number::Int(BigInt::from_str(tag.slice(source)).unwrap()),
RawNumber::Decimal(tag) => {
Number::Decimal(BigDecimal::from_str(tag.slice(source)).unwrap())
}
}
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Token {
pub unspanned: UnspannedToken,
pub span: Span,
}
impl std::ops::Deref for Token {
type Target = UnspannedToken;
fn deref(&self) -> &UnspannedToken {
&self.unspanned
}
}
impl PrettyDebugWithSource for Token {
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
match self.unspanned {
UnspannedToken::Number(number) => number.pretty_debug(source),
UnspannedToken::Operator(operator) => operator.pretty(),
UnspannedToken::String(_) => b::primitive(self.span.slice(source)),
UnspannedToken::Variable(_) => b::var(self.span.slice(source)),
UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)),
UnspannedToken::ExternalWord => {
b::typed("external", b::description(self.span.slice(source)))
}
UnspannedToken::GlobPattern => {
b::typed("pattern", b::description(self.span.slice(source)))
}
UnspannedToken::Bare => b::primitive(self.span.slice(source)),
}
}
}
impl Token {
pub fn debug<'a>(&self, source: &'a Text) -> DebugToken<'a> {
DebugToken {
node: *self,
source,
}
}
pub fn extract_number(&self) -> Option<RawNumber> {
match self.unspanned {
UnspannedToken::Number(number) => Some(number),
_ => None,
}
}
pub fn extract_int(&self) -> Option<(Span, Span)> {
match self.unspanned {
UnspannedToken::Number(RawNumber::Int(int)) => Some((int, self.span)),
_ => None,
}
}
pub fn extract_decimal(&self) -> Option<(Span, Span)> {
match self.unspanned {
UnspannedToken::Number(RawNumber::Decimal(decimal)) => Some((decimal, self.span)),
_ => None,
}
}
pub fn extract_operator(&self) -> Option<Spanned<Operator>> {
match self.unspanned {
UnspannedToken::Operator(operator) => Some(operator.spanned(self.span)),
_ => None,
}
}
pub fn extract_string(&self) -> Option<(Span, Span)> {
match self.unspanned {
UnspannedToken::String(span) => Some((span, self.span)),
_ => None,
}
}
pub fn extract_variable(&self) -> Option<(Span, Span)> {
match self.unspanned {
UnspannedToken::Variable(span) => Some((span, self.span)),
_ => None,
}
}
pub fn extract_external_command(&self) -> Option<(Span, Span)> {
match self.unspanned {
UnspannedToken::ExternalCommand(span) => Some((span, self.span)),
_ => None,
}
}
pub fn extract_external_word(&self) -> Option<Span> {
match self.unspanned {
UnspannedToken::ExternalWord => Some(self.span),
_ => None,
}
}
pub fn extract_glob_pattern(&self) -> Option<Span> {
match self.unspanned {
UnspannedToken::GlobPattern => Some(self.span),
_ => None,
}
}
pub fn extract_bare(&self) -> Option<Span> {
match self.unspanned {
UnspannedToken::Bare => Some(self.span),
_ => None,
}
}
}
pub struct DebugToken<'a> {
node: Token,
source: &'a Text,
}
impl fmt::Debug for DebugToken<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self.node.span.slice(self.source))
}
}

View File

@ -0,0 +1,115 @@
use crate::parse::parser::Number;
use nu_protocol::{Primitive, UntaggedValue};
use nu_source::{b, DebugDocBuilder, PrettyDebug};
use num_traits::ToPrimitive;
use serde::{Deserialize, Serialize};
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub enum Unit {
// Filesize units
Byte,
Kilobyte,
Megabyte,
Gigabyte,
Terabyte,
Petabyte,
// Duration units
Second,
Minute,
Hour,
Day,
Week,
Month,
Year,
}
impl PrettyDebug for Unit {
fn pretty(&self) -> DebugDocBuilder {
b::keyword(self.as_str())
}
}
fn convert_number_to_u64(number: &Number) -> u64 {
match number {
Number::Int(big_int) => big_int.to_u64().unwrap(),
Number::Decimal(big_decimal) => big_decimal.to_u64().unwrap(),
}
}
impl Unit {
pub fn as_str(&self) -> &str {
match *self {
Unit::Byte => "B",
Unit::Kilobyte => "KB",
Unit::Megabyte => "MB",
Unit::Gigabyte => "GB",
Unit::Terabyte => "TB",
Unit::Petabyte => "PB",
Unit::Second => "s",
Unit::Minute => "m",
Unit::Hour => "h",
Unit::Day => "d",
Unit::Week => "w",
Unit::Month => "M",
Unit::Year => "y",
}
}
pub fn compute(&self, size: &Number) -> UntaggedValue {
let size = size.clone();
match &self {
Unit::Byte => number(size),
Unit::Kilobyte => number(size * 1024),
Unit::Megabyte => number(size * 1024 * 1024),
Unit::Gigabyte => number(size * 1024 * 1024 * 1024),
Unit::Terabyte => number(size * 1024 * 1024 * 1024 * 1024),
Unit::Petabyte => number(size * 1024 * 1024 * 1024 * 1024 * 1024),
Unit::Second => duration(convert_number_to_u64(&size)),
Unit::Minute => duration(60 * convert_number_to_u64(&size)),
Unit::Hour => duration(60 * 60 * convert_number_to_u64(&size)),
Unit::Day => duration(24 * 60 * 60 * convert_number_to_u64(&size)),
Unit::Week => duration(7 * 24 * 60 * 60 * convert_number_to_u64(&size)),
Unit::Month => duration(30 * 24 * 60 * 60 * convert_number_to_u64(&size)),
Unit::Year => duration(365 * 24 * 60 * 60 * convert_number_to_u64(&size)),
}
}
}
fn number(number: impl Into<Number>) -> UntaggedValue {
let number = number.into();
match number {
Number::Int(int) => UntaggedValue::Primitive(Primitive::Int(int)),
Number::Decimal(decimal) => UntaggedValue::Primitive(Primitive::Decimal(decimal)),
}
}
pub fn duration(secs: u64) -> UntaggedValue {
UntaggedValue::Primitive(Primitive::Duration(secs))
}
impl FromStr for Unit {
type Err = ();
fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
match input {
"B" | "b" => Ok(Unit::Byte),
"KB" | "kb" | "Kb" | "K" | "k" => Ok(Unit::Kilobyte),
"MB" | "mb" | "Mb" => Ok(Unit::Megabyte),
"GB" | "gb" | "Gb" => Ok(Unit::Gigabyte),
"TB" | "tb" | "Tb" => Ok(Unit::Terabyte),
"PB" | "pb" | "Pb" => Ok(Unit::Petabyte),
"s" => Ok(Unit::Second),
"m" => Ok(Unit::Minute),
"h" => Ok(Unit::Hour),
"d" => Ok(Unit::Day),
"w" => Ok(Unit::Week),
"M" => Ok(Unit::Month),
"y" => Ok(Unit::Year),
_ => Err(()),
}
}
}

View File

@ -0,0 +1 @@