extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-12-27 01:19:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

basic signature parse

Browse Source
This commit is contained in:
JT 2021-07-17 09:55:12 +12:00
parent 949c6a5932
commit 6aef00ecff
5 changed files with 255 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/eval.rs
View File

@ -64,6 +64,7 @@ impl Engine {
Expr::Table(_, _) => Err(ShellError::Unsupported(expr.span)),
Expr::Literal(_) => Err(ShellError::Unsupported(expr.span)),
Expr::String(_) => Err(ShellError::Unsupported(expr.span)),
Expr::Signature(_) => Err(ShellError::Unsupported(expr.span)),
Expr::Garbage => Err(ShellError::Unsupported(expr.span)),
}
}

66
src/lex.rs
View File

@ -38,18 +38,28 @@ impl BlockKind {
}
}
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
#[derive(PartialEq, Eq, Debug, Clone)]
pub enum LexMode {
Normal,
CommaIsSpace,
NewlineIsSpace,
CommaAndNewlineIsSpace,
Custom {
whitespace: Vec<u8>,
special: Vec<u8>,
},
}
impl LexMode {
pub fn whitespace_contains(&self, b: u8) -> bool {
match self {
LexMode::Custom { ref whitespace, .. } => whitespace.contains(&b),
_ => false,
}
}
}
// A baseline token is terminated if it's not nested inside of a paired
// delimiter and the next character is one of: `|`, `;`, `#` or any
// whitespace.
fn is_item_terminator(block_level: &[BlockKind], c: u8, lex_mode: LexMode) -> bool {
fn is_item_terminator(block_level: &[BlockKind], c: u8, lex_mode: &LexMode) -> bool {
block_level.is_empty()
&& (c == b' '
|| c == b'\t'
@ -57,14 +67,25 @@ fn is_item_terminator(block_level: &[BlockKind], c: u8, lex_mode: LexMode) -> bo
|| c == b'|'
|| c == b';'
|| c == b'#'
|| (c == b',' && lex_mode == LexMode::CommaIsSpace)
|| (c == b',' && lex_mode == LexMode::CommaAndNewlineIsSpace))
|| lex_mode.whitespace_contains(c))
}
// A special token is one that is a byte that stands alone as its own token. For example
// when parsing a signature you may want to have `:` be able to separate tokens and also
// to be handled as its own token to notify you you're about to parse a type in the example
// `foo:bar`
fn is_special_item(block_level: &[BlockKind], c: u8, lex_mode: &LexMode) -> bool {
block_level.is_empty()
&& (match lex_mode {
LexMode::Custom { special, .. } => special.contains(&c),
_ => false,
})
}
pub fn lex_item(
input: &[u8],
curr_offset: &mut usize,
lex_mode: LexMode,
lex_mode: &LexMode,
) -> (Span, Option<ParseError>) {
// This variable tracks the starting character of a string literal, so that
// we remain inside the string literal lexer mode until we encounter the
@ -99,19 +120,22 @@ pub fn lex_item(
quote_start = None;
}
} else if c == b'#' {
if is_item_terminator(&block_level, c, lex_mode) {
if is_item_terminator(&block_level, c, &lex_mode) {
break;
}
in_comment = true;
} else if c == b'\n' {
in_comment = false;
if is_item_terminator(&block_level, c, lex_mode) {
if is_item_terminator(&block_level, c, &lex_mode) {
break;
}
} else if in_comment {
if is_item_terminator(&block_level, c, lex_mode) {
if is_item_terminator(&block_level, c, &lex_mode) {
break;
}
} else if is_special_item(&block_level, c, &lex_mode) && token_start == *curr_offset {
*curr_offset += 1;
break;
} else if c == b'\'' || c == b'"' {
// We encountered the opening quote of a string literal.
quote_start = Some(c);
@ -140,7 +164,7 @@ pub fn lex_item(
if let Some(BlockKind::Paren) = block_level.last() {
let _ = block_level.pop();
}
} else if is_item_terminator(&block_level, c, lex_mode) {
} else if is_item_terminator(&block_level, c, &lex_mode) {
break;
}
@ -182,7 +206,7 @@ pub fn lex_item(
pub fn lex(
input: &[u8],
span_offset: usize,
lex_mode: LexMode,
lex_mode: &LexMode,
) -> (Vec<Token>, Option<ParseError>) {
let mut error = None;
@ -239,7 +263,7 @@ pub fn lex(
let idx = curr_offset;
curr_offset += 1;
if lex_mode != LexMode::NewlineIsSpace && lex_mode != LexMode::CommaAndNewlineIsSpace {
if !lex_mode.whitespace_contains(c) {
output.push(Token::new(TokenContents::Eol, Span::new(idx, idx + 1)));
}
} else if c == b'#' {
@ -265,17 +289,13 @@ pub fn lex(
Span::new(start, curr_offset),
));
}
} else if c == b' '
|| c == b'\t'
|| (c == b',' && lex_mode == LexMode::CommaIsSpace)
|| (c == b',' && lex_mode == LexMode::CommaAndNewlineIsSpace)
{
} else if c == b' ' || c == b'\t' || lex_mode.whitespace_contains(c) {
// If the next character is non-newline whitespace, skip it.
curr_offset += 1;
} else {
// Otherwise, try to consume an unclassified token.
let (span, err) = lex_item(input, &mut curr_offset, lex_mode);
let (span, err) = lex_item(input, &mut curr_offset, &lex_mode);
if error.is_none() {
error = err;
}
@ -294,7 +314,7 @@ mod lex_tests {
fn lex_basic() {
let file = b"let x = 4";
let output = lex(file, 0, LexMode::Normal);
let output = lex(file, 0, &LexMode::Normal);
assert!(output.1.is_none());
}
@ -303,7 +323,7 @@ mod lex_tests {
fn lex_newline() {
let file = b"let x = 300\nlet y = 500;";
let output = lex(file, 0, LexMode::Normal);
let output = lex(file, 0, &LexMode::Normal);
println!("{:#?}", output.0);
assert!(output.0.contains(&Token {
@ -316,7 +336,7 @@ mod lex_tests {
fn lex_empty() {
let file = b"";
let output = lex(file, 0, LexMode::Normal);
let output = lex(file, 0, &LexMode::Normal);
assert!(output.0.is_empty());
assert!(output.1.is_none());

2
src/lite_parse.rs
View File

@ -128,7 +128,7 @@ mod tests {
use crate::{lex, lite_parse, LiteBlock, ParseError, Span};
fn lite_parse_helper(input: &[u8]) -> Result<LiteBlock, ParseError> {
let (output, err) = lex(input, 0, crate::LexMode::Normal);
let (output, err) = lex(input, 0, &crate::LexMode::Normal);
if let Some(err) = err {
return Err(err);
}

6
src/main.rs
View File

@ -56,11 +56,7 @@ fn main() -> std::io::Result<()> {
let sig = Signature::build("def")
.required("def_name", SyntaxShape::String, "definition name")
.required(
"params",
SyntaxShape::List(Box::new(SyntaxShape::VarWithOptType)),
"parameters",
)
.required("params", SyntaxShape::Signature, "parameters")
.required("block", SyntaxShape::Block, "body of the definition");
working_set.add_decl(sig.into());

226
src/parser.rs
View File

@ -3,8 +3,8 @@ use std::ops::{Index, IndexMut};
use crate::{
lex, lite_parse,
parser_state::{Type, VarId},
signature::Flag,
BlockId, DeclId, Declaration, LiteBlock, ParseError, ParserWorkingSet, Signature, Span,
signature::{Flag, PositionalArg},
BlockId, DeclId, Declaration, LiteBlock, ParseError, ParserWorkingSet, Signature, Span, Token,
};
/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function.
@ -71,6 +71,9 @@ pub enum SyntaxShape {
/// A variable with optional type, `x` or `x: int`
VarWithOptType,
/// A signature for a definition, `[x:int, --foo]`
Signature,
/// A general expression, eg `1 + 2` or `foo --bar`
Expression,
}
@ -135,6 +138,7 @@ pub enum Expr {
Table(Vec<Expression>, Vec<Vec<Expression>>),
Literal(Vec<u8>),
String(String), // FIXME: improve this in the future?
Signature(Signature),
Garbage,
}
@ -185,6 +189,13 @@ impl Expression {
}
}
pub fn as_signature(self) -> Option<Signature> {
match self.expr {
Expr::Signature(sig) => Some(sig),
_ => None,
}
}
pub fn as_list(self) -> Option<Vec<Expression>> {
match self.expr {
Expr::List(list) => Some(list),
@ -787,7 +798,7 @@ impl ParserWorkingSet {
let source = self.get_span_contents(span);
let (output, err) = lex(&source, start, crate::LexMode::Normal);
let (output, err) = lex(&source, start, &crate::LexMode::Normal);
error = error.or(err);
let (output, err) = lite_parse(&output);
@ -826,6 +837,28 @@ impl ParserWorkingSet {
}
}
//TODO: Handle error case
pub fn parse_shape_name(&self, bytes: &[u8]) -> SyntaxShape {
match bytes {
b"any" => SyntaxShape::Any,
b"string" => SyntaxShape::String,
b"column-path" => SyntaxShape::ColumnPath,
b"number" => SyntaxShape::Number,
b"range" => SyntaxShape::Range,
b"int" => SyntaxShape::Int,
b"path" => SyntaxShape::FilePath,
b"glob" => SyntaxShape::GlobPattern,
b"block" => SyntaxShape::Block,
b"cond" => SyntaxShape::RowCondition,
b"operator" => SyntaxShape::Operator,
b"math" => SyntaxShape::MathExpression,
b"variable" => SyntaxShape::Variable,
b"signature" => SyntaxShape::Signature,
b"expr" => SyntaxShape::Expression,
_ => SyntaxShape::Any,
}
}
pub fn parse_type(&self, bytes: &[u8]) -> Type {
if bytes == b"int" {
Type::Int
@ -887,6 +920,140 @@ impl ParserWorkingSet {
self.parse_math_expression(spans)
}
pub fn parse_signature(&mut self, span: Span) -> (Expression, Option<ParseError>) {
enum ParseMode {
ArgMode,
TypeMode,
}
enum Arg {
Positional(PositionalArg),
Flag(Flag),
}
println!("parse signature");
let bytes = self.get_span_contents(span);
let mut error = None;
let mut start = span.start;
let mut end = span.end;
if bytes.starts_with(b"[") {
start += 1;
}
if bytes.ends_with(b"]") {
end -= 1;
} else {
error = error.or_else(|| {
Some(ParseError::Unclosed(
"]".into(),
Span {
start: end,
end: end + 1,
},
))
});
}
let span = Span { start, end };
let source = &self.file_contents[..span.end];
let (output, err) = lex(
&source,
span.start,
&crate::LexMode::Custom {
whitespace: vec![b'\n', b','],
special: vec![b':', b'?'],
},
);
error = error.or(err);
let mut args: Vec<Arg> = vec![];
let mut parse_mode = ParseMode::ArgMode;
for token in &output {
match token {
Token {
contents: crate::TokenContents::Item,
span,
} => {
let contents = &self.file_contents[span.start..span.end];
if contents == b":" {
match parse_mode {
ParseMode::ArgMode => {
parse_mode = ParseMode::TypeMode;
}
ParseMode::TypeMode => {
// We're seeing two types for the same thing for some reason, error
error = error.or(Some(ParseError::Mismatch("type".into(), *span)));
}
}
} else {
match parse_mode {
ParseMode::ArgMode => {
if contents.starts_with(b"--") {
// Long flag
args.push(Arg::Flag(Flag {
arg: None,
desc: String::new(),
long: String::from_utf8_lossy(contents).to_string(),
short: None,
required: true,
}));
} else {
// Positional arg
args.push(Arg::Positional(PositionalArg {
desc: String::new(),
name: String::from_utf8_lossy(contents).to_string(),
shape: SyntaxShape::Any,
}))
}
}
ParseMode::TypeMode => {
if let Some(last) = args.last_mut() {
let syntax_shape = self.parse_shape_name(contents);
//TODO check if we're replacing one already
match last {
Arg::Positional(PositionalArg { name, desc, shape }) => {
*shape = syntax_shape;
}
Arg::Flag(Flag {
long,
short,
arg,
required,
desc,
}) => *arg = Some(syntax_shape),
}
}
parse_mode = ParseMode::ArgMode;
}
}
}
}
_ => {}
}
}
let mut sig = Signature::new(String::new());
for arg in args {
match arg {
Arg::Positional(positional) => sig.required_positional.push(positional),
Arg::Flag(flag) => sig.named.push(flag),
}
}
(
Expression {
expr: Expr::Signature(sig),
span,
},
error,
)
}
pub fn parse_list_expression(
&mut self,
span: Span,
@ -919,7 +1086,14 @@ impl ParserWorkingSet {
let span = Span { start, end };
let source = &self.file_contents[..span.end];
let (output, err) = lex(&source, span.start, crate::LexMode::CommaAndNewlineIsSpace);
let (output, err) = lex(
&source,
span.start,
&crate::LexMode::Custom {
whitespace: vec![b'\n', b','],
special: vec![],
},
);
error = error.or(err);
let (output, err) = lite_parse(&output);
@ -983,7 +1157,14 @@ impl ParserWorkingSet {
let source = &self.file_contents[..end];
let (output, err) = lex(&source, start, crate::LexMode::CommaAndNewlineIsSpace);
let (output, err) = lex(
&source,
start,
&crate::LexMode::Custom {
whitespace: vec![b'\n', b','],
special: vec![],
},
);
error = error.or(err);
let (output, err) = lite_parse(&output);
@ -1073,7 +1254,7 @@ impl ParserWorkingSet {
let source = &self.file_contents[..end];
let (output, err) = lex(&source, start, crate::LexMode::Normal);
let (output, err) = lex(&source, start, &crate::LexMode::Normal);
error = error.or(err);
let (output, err) = lite_parse(&output);
@ -1116,11 +1297,14 @@ impl ParserWorkingSet {
return self.parse_full_column_path(span);
} else if bytes.starts_with(b"[") {
match shape {
SyntaxShape::Any | SyntaxShape::List(_) | SyntaxShape::Table => {}
SyntaxShape::Any
| SyntaxShape::List(_)
| SyntaxShape::Table
| SyntaxShape::Signature => {}
_ => {
return (
Expression::garbage(span),
Some(ParseError::Mismatch("non-table/non-list".into(), span)),
Some(ParseError::Mismatch("non-[] value".into(), span)),
);
}
}
@ -1179,6 +1363,16 @@ impl ParserWorkingSet {
)
}
}
SyntaxShape::Signature => {
if bytes.starts_with(b"[") {
self.parse_signature(span)
} else {
(
Expression::garbage(span),
Some(ParseError::Mismatch("signature".into(), span)),
)
}
}
SyntaxShape::List(elem) => {
if bytes.starts_with(b"[") {
self.parse_list_expression(span, &elem)
@ -1419,22 +1613,20 @@ impl ParserWorkingSet {
.remove(0)
.as_string()
.expect("internal error: expected def name");
let args = call
let mut signature = call
.positional
.remove(0)
.as_list()
.expect("internal error: expected param list")
.into_iter()
.map(|x| x.as_var().expect("internal error: expected parameter"))
.collect::<Vec<_>>();
.as_signature()
.expect("internal error: expected param list");
let block_id = call
.positional
.remove(0)
.as_block()
.expect("internal error: expected block");
signature.name = name;
let decl = Declaration {
signature: Signature::new(name),
signature,
body: Some(block_id),
};
@ -1526,7 +1718,7 @@ impl ParserWorkingSet {
pub fn parse_file(&mut self, fname: &str, contents: Vec<u8>) -> (Block, Option<ParseError>) {
let mut error = None;
let (output, err) = lex(&contents, 0, crate::LexMode::Normal);
let (output, err) = lex(&contents, 0, &crate::LexMode::Normal);
error = error.or(err);
self.add_file(fname.into(), contents);
@ -1545,7 +1737,7 @@ impl ParserWorkingSet {
self.add_file("source".into(), source.into());
let (output, err) = lex(source, 0, crate::LexMode::Normal);
let (output, err) = lex(source, 0, &crate::LexMode::Normal);
error = error.or(err);
let (output, err) = lite_parse(&output);