extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-08-10 20:15:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

initial change to Tagged<Value>

Browse Source
This commit is contained in:
Jonathan Turner
2019-08-01 13:58:42 +12:00
parent c61a1108ff
commit 462f783fac
78 changed files with 713 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/parser/parse/files.rs
View File

@ -1,4 +1,4 @@
use crate::parser::parse::span::Span;
use crate::Span;
use derive_new::new;
use language_reporting::{FileName, Location};

2
src/parser/parse/flag.rs
View File

@ -1,4 +1,4 @@
use crate::parser::Span;
use crate::Span;
use derive_new::new;
use getset::Getters;
use serde_derive::{Deserialize, Serialize};

33
src/parser/parse/parser.rs
View File

@ -1,9 +1,10 @@
#![allow(unused)]
use crate::parser::parse::{
call_node::*, flag::*, operator::*, pipeline::*, span::*, token_tree::*, token_tree_builder::*,
call_node::*, flag::*, operator::*, pipeline::*, token_tree::*, token_tree_builder::*,
tokens::*, unit::*,
};
use crate::{Span, Tagged};
use nom;
use nom::branch::*;
use nom::bytes::complete::*;
@ -67,7 +68,7 @@ fn trace_step<'a, T: Debug>(
}
}
pub fn raw_integer(input: NomSpan) -> IResult<NomSpan, Spanned<i64>> {
pub fn raw_integer(input: NomSpan) -> IResult<NomSpan, Tagged<i64>> {
let start = input.offset;
trace_step(input, "raw_integer", move |input| {
let (input, neg) = opt(tag("-"))(input)?;
@ -76,7 +77,7 @@ pub fn raw_integer(input: NomSpan) -> IResult<NomSpan, Spanned<i64>> {
Ok((
input,
Spanned::from_item(int(num.fragment, neg), (start, end)),
Tagged::from_item(int(num.fragment, neg), (start, end)),
))
})
}
@ -85,7 +86,7 @@ pub fn integer(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "integer", move |input| {
let (input, int) = raw_integer(input)?;
Ok((input, TokenTreeBuilder::spanned_int(*int, int.span)))
Ok((input, TokenTreeBuilder::spanned_int(*int, int.span())))
})
}
*/
@ -202,7 +203,7 @@ pub fn shorthand(input: NomSpan) -> IResult<NomSpan, TokenNode> {
})
}
pub fn raw_unit(input: NomSpan) -> IResult<NomSpan, Spanned<Unit>> {
pub fn raw_unit(input: NomSpan) -> IResult<NomSpan, Tagged<Unit>> {
trace_step(input, "raw_unit", move |input| {
let start = input.offset;
let (input, unit) = alt((
@ -230,7 +231,7 @@ pub fn raw_unit(input: NomSpan) -> IResult<NomSpan, Spanned<Unit>> {
Ok((
input,
Spanned::from_item(Unit::from(unit.fragment), (start, end)),
Tagged::from_item(Unit::from(unit.fragment), (start, end)),
))
})
}
@ -408,7 +409,7 @@ pub fn delimited_brace(input: NomSpan) -> IResult<NomSpan, TokenNode> {
})
}
pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Spanned<CallNode>> {
pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Tagged<CallNode>> {
trace_step(input, "raw_call", move |input| {
let left = input.offset;
let (input, items) = token_list(input)?;
@ -484,10 +485,10 @@ pub fn pipeline(input: NomSpan) -> IResult<NomSpan, TokenNode> {
}
fn make_call_list(
head: Option<(Spanned<CallNode>, Option<NomSpan>, Option<NomSpan>)>,
head: Option<(Tagged<CallNode>, Option<NomSpan>, Option<NomSpan>)>,
items: Vec<(
Option<NomSpan>,
Spanned<CallNode>,
Tagged<CallNode>,
Option<NomSpan>,
Option<NomSpan>,
)>,
@ -701,12 +702,12 @@ mod tests {
fn test_flag() {
// assert_leaf! {
// parsers [ flag ]
// "--hello" -> 0..7 { Flag(Spanned::from_item(FlagKind::Longhand, span(2, 7))) }
// "--hello" -> 0..7 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 7))) }
// }
// assert_leaf! {
// parsers [ flag ]
// "--hello-world" -> 0..13 { Flag(Spanned::from_item(FlagKind::Longhand, span(2, 13))) }
// "--hello-world" -> 0..13 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 13))) }
// }
}
@ -714,7 +715,7 @@ mod tests {
fn test_shorthand() {
// assert_leaf! {
// parsers [ shorthand ]
// "-alt" -> 0..4 { Flag(Spanned::from_item(FlagKind::Shorthand, span(1, 4))) }
// "-alt" -> 0..4 { Flag(Tagged::from_item(FlagKind::Shorthand, span(1, 4))) }
// }
}
@ -1024,7 +1025,7 @@ mod tests {
right: usize,
) -> TokenNode {
let node = DelimitedNode::new(delimiter, children);
let spanned = Spanned::from_item(node, (left, right));
let spanned = Tagged::from_item(node, (left, right));
TokenNode::Delimited(spanned)
}
@ -1033,16 +1034,16 @@ mod tests {
Box::new(head),
tail.into_iter().map(TokenNode::Token).collect(),
);
let spanned = Spanned::from_item(node, (left, right));
let spanned = Tagged::from_item(node, (left, right));
TokenNode::Path(spanned)
}
fn leaf_token(token: RawToken, left: usize, right: usize) -> TokenNode {
TokenNode::Token(Spanned::from_item(token, (left, right)))
TokenNode::Token(Tagged::from_item(token, (left, right)))
}
fn token(token: RawToken, left: usize, right: usize) -> TokenNode {
TokenNode::Token(Spanned::from_item(token, (left, right)))
TokenNode::Token(Tagged::from_item(token, (left, right)))
}
fn build<T>(block: CurriedNode<T>) -> T {

5
src/parser/parse/pipeline.rs
View File

@ -1,4 +1,5 @@
use crate::parser::{CallNode, Span, Spanned};
use crate::parser::CallNode;
use crate::{Span, Tagged};
use derive_new::new;
use getset::Getters;
@ -12,7 +13,7 @@ pub struct Pipeline {
pub struct PipelineElement {
pub pre_ws: Option<Span>,
#[get = "crate"]
call: Spanned<CallNode>,
call: Tagged<CallNode>,
pub post_ws: Option<Span>,
pub post_pipe: Option<Span>,
}

197
src/parser/parse/span.rs
View File

@ -1,197 +0,0 @@
use crate::Text;
use derive_new::new;
use getset::Getters;
use serde::Serialize;
use serde_derive::Deserialize;
use uuid::Uuid;
#[derive(
new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters,
)]
#[get = "crate"]
pub struct Spanned<T> {
pub span: Span,
pub item: T,
}
impl<T> Spanned<T> {
pub fn spanned(self, span: impl Into<Span>) -> Spanned<T> {
Spanned::from_item(self.item, span.into())
}
}
pub trait SpannedItem: Sized {
fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
Spanned::from_item(self, span.into())
}
// For now, this is a temporary facility. In many cases, there are other useful spans that we
// could be using, such as the original source spans of JSON or Toml files, but we don't yet
// have the infrastructure to make that work.
fn spanned_unknown(self) -> Spanned<Self> {
Spanned::from_item(self, (0, 0))
}
}
impl<T> SpannedItem for T {}
impl<T> std::ops::Deref for Spanned<T> {
type Target = T;
fn deref(&self) -> &T {
&self.item
}
}
impl<T> Spanned<T> {
crate fn from_item(item: T, span: impl Into<Span>) -> Spanned<T> {
Spanned {
span: span.into(),
item,
}
}
pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Spanned<U> {
let Spanned { span, item } = self;
let mapped = input(item);
Spanned { span, item: mapped }
}
crate fn copy_span<U>(&self, output: U) -> Spanned<U> {
let Spanned { span, .. } = self;
Spanned {
span: *span,
item: output,
}
}
pub fn source(&self, source: &Text) -> Text {
Text::from(self.span().slice(source))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
pub struct Span {
crate start: usize,
crate end: usize,
pub source: Option<Uuid>,
}
impl From<Option<Span>> for Span {
fn from(input: Option<Span>) -> Span {
match input {
None => Span {
start: 0,
end: 0,
source: None,
},
Some(span) => span,
}
}
}
impl<T> From<&Spanned<T>> for Span {
fn from(input: &Spanned<T>) -> Span {
input.span
}
}
impl From<&Span> for Span {
fn from(input: &Span) -> Span {
*input
}
}
impl From<nom5_locate::LocatedSpan<&str>> for Span {
fn from(input: nom5_locate::LocatedSpan<&str>) -> Span {
Span {
start: input.offset,
end: input.offset + input.fragment.len(),
source: None,
}
}
}
impl<T> From<(nom5_locate::LocatedSpan<T>, nom5_locate::LocatedSpan<T>)> for Span {
fn from(input: (nom5_locate::LocatedSpan<T>, nom5_locate::LocatedSpan<T>)) -> Span {
Span {
start: input.0.offset,
end: input.1.offset,
source: None,
}
}
}
impl From<(usize, usize)> for Span {
fn from(input: (usize, usize)) -> Span {
Span {
start: input.0,
end: input.1,
source: None,
}
}
}
impl From<&std::ops::Range<usize>> for Span {
fn from(input: &std::ops::Range<usize>) -> Span {
Span {
start: input.start,
end: input.end,
source: None,
}
}
}
impl Span {
pub fn unknown() -> Span {
Span {
start: 0,
end: 0,
source: None,
}
}
pub fn unknown_with_uuid(uuid: Uuid) -> Span {
Span {
start: 0,
end: 0,
source: Some(uuid),
}
}
pub fn is_unknown(&self) -> bool {
self.start == 0 && self.end == 0
}
pub fn slice(&self, source: &'a str) -> &'a str {
&source[self.start..self.end]
}
}
impl language_reporting::ReportingSpan for Span {
fn with_start(&self, start: usize) -> Self {
Span {
start,
end: self.end,
source: None,
}
}
fn with_end(&self, end: usize) -> Self {
Span {
start: self.start,
end,
source: None,
}
}
fn start(&self) -> usize {
self.start
}
fn end(&self) -> usize {
self.end
}
}

42
src/parser/parse/token_tree.rs
View File

@ -1,6 +1,6 @@
use crate::errors::ShellError;
use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, span::*, tokens::*};
use crate::Text;
use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
use crate::{Span, Tagged, Text};
use derive_new::new;
use enum_utils::FromStr;
use getset::Getters;
@ -10,16 +10,16 @@ use std::fmt;
pub enum TokenNode {
Token(Token),
#[allow(unused)]
Call(Spanned<CallNode>),
Delimited(Spanned<DelimitedNode>),
Pipeline(Spanned<Pipeline>),
Operator(Spanned<Operator>),
Flag(Spanned<Flag>),
Call(Tagged<CallNode>),
Delimited(Tagged<DelimitedNode>),
Pipeline(Tagged<Pipeline>),
Operator(Tagged<Operator>),
Flag(Tagged<Flag>),
Member(Span),
Whitespace(Span),
#[allow(unused)]
Error(Spanned<Box<ShellError>>),
Path(Spanned<PathNode>),
Error(Tagged<Box<ShellError>>),
Path(Tagged<PathNode>),
}
pub struct DebugTokenNode<'a> {
@ -86,16 +86,16 @@ impl From<&TokenNode> for Span {
impl TokenNode {
pub fn span(&self) -> Span {
match self {
TokenNode::Token(t) => t.span,
TokenNode::Call(s) => s.span,
TokenNode::Delimited(s) => s.span,
TokenNode::Pipeline(s) => s.span,
TokenNode::Operator(s) => s.span,
TokenNode::Flag(s) => s.span,
TokenNode::Token(t) => t.span(),
TokenNode::Call(s) => s.span(),
TokenNode::Delimited(s) => s.span(),
TokenNode::Pipeline(s) => s.span(),
TokenNode::Operator(s) => s.span(),
TokenNode::Flag(s) => s.span(),
TokenNode::Member(s) => *s,
TokenNode::Whitespace(s) => *s,
TokenNode::Error(s) => s.span,
TokenNode::Path(s) => s.span,
TokenNode::Error(s) => s.span(),
TokenNode::Path(s) => s.span(),
}
}
@ -129,7 +129,7 @@ impl TokenNode {
pub fn is_bare(&self) -> bool {
match self {
TokenNode::Token(Spanned {
TokenNode::Token(Tagged {
item: RawToken::Bare,
..
}) => true,
@ -137,10 +137,10 @@ impl TokenNode {
}
}
crate fn as_flag(&self, value: &str, source: &Text) -> Option<Spanned<Flag>> {
crate fn as_flag(&self, value: &str, source: &Text) -> Option<Tagged<Flag>> {
match self {
TokenNode::Flag(
flag @ Spanned {
flag @ Tagged {
item: Flag { .. }, ..
},
) if value == flag.name().slice(source) => Some(*flag),
@ -150,7 +150,7 @@ impl TokenNode {
pub fn as_pipeline(&self) -> Result<Pipeline, ShellError> {
match self {
TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
TokenNode::Pipeline(Tagged { item, .. }) => Ok(item.clone()),
_ => Err(ShellError::string("unimplemented")),
}
}

32
src/parser/parse/token_tree_builder.rs
View File

@ -4,11 +4,11 @@ use crate::prelude::*;
use crate::parser::parse::flag::{Flag, FlagKind};
use crate::parser::parse::operator::Operator;
use crate::parser::parse::pipeline::{Pipeline, PipelineElement};
use crate::parser::parse::span::{Span, Spanned};
use crate::parser::parse::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode};
use crate::parser::parse::tokens::{RawToken, Token};
use crate::parser::parse::unit::Unit;
use crate::parser::CallNode;
use crate::Span;
use derive_new::new;
#[derive(new)]
@ -20,7 +20,7 @@ pub struct TokenTreeBuilder {
#[allow(unused)]
pub type CurriedNode<T> = Box<dyn FnOnce(&mut TokenTreeBuilder) -> T + 'static>;
pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>;
#[allow(unused)]
impl TokenTreeBuilder {
@ -92,7 +92,7 @@ impl TokenTreeBuilder {
input: (Vec<PipelineElement>, Option<Span>),
span: impl Into<Span>,
) -> TokenNode {
TokenNode::Pipeline(Spanned::from_item(
TokenNode::Pipeline(Tagged::from_item(
Pipeline::new(input.0, input.1.into()),
span,
))
@ -111,7 +111,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_op(input: impl Into<Operator>, span: impl Into<Span>) -> TokenNode {
TokenNode::Operator(Spanned::from_item(input.into(), span.into()))
TokenNode::Operator(Tagged::from_item(input.into(), span.into()))
}
pub fn string(input: impl Into<String>) -> CurriedToken {
@ -128,7 +128,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_string(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(Spanned::from_item(
TokenNode::Token(Tagged::from_item(
RawToken::String(input.into()),
span.into(),
))
@ -146,7 +146,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_bare(input: impl Into<Span>) -> TokenNode {
TokenNode::Token(Spanned::from_item(RawToken::Bare, input.into()))
TokenNode::Token(Tagged::from_item(RawToken::Bare, input.into()))
}
pub fn int(input: impl Into<i64>) -> CurriedToken {
@ -183,7 +183,7 @@ impl TokenTreeBuilder {
) -> TokenNode {
let (int, unit) = (input.0.into(), input.1.into());
TokenNode::Token(Spanned::from_item(RawToken::Size(int, unit), span))
TokenNode::Token(Tagged::from_item(RawToken::Size(int, unit), span))
}
pub fn path(head: CurriedToken, tail: Vec<CurriedToken>) -> CurriedToken {
@ -206,7 +206,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_path(input: (TokenNode, Vec<TokenNode>), span: impl Into<Span>) -> TokenNode {
TokenNode::Path(Spanned::from_item(
TokenNode::Path(Tagged::from_item(
PathNode::new(Box::new(input.0), input.1),
span,
))
@ -224,7 +224,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_var(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(Spanned::from_item(
TokenNode::Token(Tagged::from_item(
RawToken::Variable(input.into()),
span.into(),
))
@ -242,7 +242,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_flag(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Flag(Spanned::from_item(
TokenNode::Flag(Tagged::from_item(
Flag::new(FlagKind::Longhand, input.into()),
span.into(),
))
@ -260,7 +260,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_shorthand(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Flag(Spanned::from_item(
TokenNode::Flag(Tagged::from_item(
Flag::new(FlagKind::Shorthand, input.into()),
span.into(),
))
@ -296,7 +296,7 @@ impl TokenTreeBuilder {
})
}
pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Spanned<CallNode> {
pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Tagged<CallNode> {
if input.len() == 0 {
panic!("BUG: spanned call (TODO)")
}
@ -306,7 +306,7 @@ impl TokenTreeBuilder {
let head = input.next().unwrap();
let tail = input.collect();
Spanned::from_item(CallNode::new(Box::new(head), tail), span)
Tagged::from_item(CallNode::new(Box::new(head), tail), span)
}
pub fn parens(input: Vec<CurriedToken>) -> CurriedToken {
@ -324,7 +324,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_parens(input: impl Into<Vec<TokenNode>>, span: impl Into<Span>) -> TokenNode {
TokenNode::Delimited(Spanned::from_item(
TokenNode::Delimited(Tagged::from_item(
DelimitedNode::new(Delimiter::Paren, input.into()),
span,
))
@ -345,7 +345,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_square(input: impl Into<Vec<TokenNode>>, span: impl Into<Span>) -> TokenNode {
TokenNode::Delimited(Spanned::from_item(
TokenNode::Delimited(Tagged::from_item(
DelimitedNode::new(Delimiter::Square, input.into()),
span,
))
@ -366,7 +366,7 @@ impl TokenTreeBuilder {
}
pub fn spanned_brace(input: impl Into<Vec<TokenNode>>, span: impl Into<Span>) -> TokenNode {
TokenNode::Delimited(Spanned::from_item(
TokenNode::Delimited(Tagged::from_item(
DelimitedNode::new(Delimiter::Brace, input.into()),
span,
))

5
src/parser/parse/tokens.rs
View File

@ -1,6 +1,5 @@
use crate::parser::parse::span::*;
use crate::parser::parse::unit::*;
use crate::Text;
use crate::{Span, Tagged, Text};
use std::fmt;
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
@ -24,7 +23,7 @@ impl RawToken {
}
}
pub type Token = Spanned<RawToken>;
pub type Token = Tagged<RawToken>;
impl Token {
pub fn debug(&self, source: &'a Text) -> DebugToken<'a> {