holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix running multiple times, add reedline

Browse Source
This commit is contained in:
JT 2021-07-22 18:04:50 +12:00
parent 4deed7c836
commit c25209eb34
5 changed files with 148 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@ -6,3 +6,4 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
reedline = {git = "https://github.com/jntrnr/reedline"}

15
src/lex.rs
View File

@ -69,6 +69,7 @@ fn is_special_item(block_level: &[BlockKind], c: u8, special_tokens: &[u8]) -> b
pub fn lex_item(
input: &[u8],
curr_offset: &mut usize,
span_offset: usize,
additional_whitespace: &[u8],
special_tokens: &[u8],
) -> (Span, Option<ParseError>) {
@ -156,7 +157,7 @@ pub fn lex_item(
*curr_offset += 1;
}
let span = Span::new(token_start, *curr_offset);
let span = Span::new(span_offset + token_start, span_offset + *curr_offset);
// If there is still unclosed opening delimiters, close them and add
// synthetic closing characters to the accumulated token.
@ -196,7 +197,7 @@ pub fn lex(
) -> (Vec<Token>, Option<ParseError>) {
let mut error = None;
let mut curr_offset = span_offset;
let mut curr_offset = 0;
let mut output = vec![];
let mut is_complete = true;
@ -242,7 +243,7 @@ pub fn lex(
curr_offset += 1;
output.push(Token::new(
TokenContents::Semicolon,
Span::new(idx, idx + 1),
Span::new(span_offset + idx, span_offset + idx + 1),
));
} else if c == b'\n' || c == b'\r' {
// If the next character is a newline, we're looking at an EOL (end of line) token.
@ -250,7 +251,10 @@ pub fn lex(
let idx = curr_offset;
curr_offset += 1;
if !additional_whitespace.contains(&c) {
output.push(Token::new(TokenContents::Eol, Span::new(idx, idx + 1)));
output.push(Token::new(
TokenContents::Eol,
Span::new(span_offset + idx, span_offset + idx + 1),
));
}
} else if c == b'#' {
// If the next character is `#`, we're at the beginning of a line
@ -272,7 +276,7 @@ pub fn lex(
if start != curr_offset {
output.push(Token::new(
TokenContents::Comment,
Span::new(start, curr_offset),
Span::new(span_offset + start, span_offset + curr_offset),
));
}
} else if c == b' ' || c == b'\t' || additional_whitespace.contains(&c) {
@ -284,6 +288,7 @@ pub fn lex(
let (span, err) = lex_item(
input,
&mut curr_offset,
span_offset,
additional_whitespace,
special_tokens,
);

167
src/main.rs
View File

@ -3,78 +3,76 @@ use std::sync::Arc;
use engine_q::{ParserState, ParserWorkingSet, Signature, SyntaxShape};
fn main() -> std::io::Result<()> {
if let Some(path) = std::env::args().nth(1) {
let mut parser_state = Arc::new(ParserState::new());
let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
let mut parser_state = Arc::new(ParserState::new());
let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
working_set.add_decl(sig.into());
let sig = Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition");
working_set.add_decl(sig.into());
// let sig = Signature::build("bar")
// .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
// .switch("--rock", "rock!!", Some('r'));
// working_set.add_decl(sig.into());
let sig = Signature::build("if")
.required("cond", SyntaxShape::RowCondition, "condition")
.required("then_block", SyntaxShape::Block, "then block")
.required(
"else",
SyntaxShape::Literal(b"else".to_vec()),
"else keyword",
)
.required("else_block", SyntaxShape::Block, "else block");
working_set.add_decl(sig.into());
let sig =
Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition");
working_set.add_decl(sig.into());
let sig = Signature::build("if")
.required("cond", SyntaxShape::RowCondition, "condition")
.required("then_block", SyntaxShape::Block, "then block")
.required(
"else",
SyntaxShape::Literal(b"else".to_vec()),
"else keyword",
)
.required("else_block", SyntaxShape::Block, "else block");
working_set.add_decl(sig.into());
let sig = Signature::build("let")
.required("var_name", SyntaxShape::Variable, "variable name")
.required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign")
.required(
"value",
SyntaxShape::Expression,
"the value to set the variable to",
);
working_set.add_decl(sig.into());
let sig = Signature::build("alias")
.required("var_name", SyntaxShape::Variable, "variable name")
.required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign")
.required(
"value",
SyntaxShape::Expression,
"the value to set the variable to",
);
working_set.add_decl(sig.into());
let sig = Signature::build("sum").required(
"arg",
SyntaxShape::List(Box::new(SyntaxShape::Number)),
"list of numbers",
let sig = Signature::build("let")
.required("var_name", SyntaxShape::Variable, "variable name")
.required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign")
.required(
"value",
SyntaxShape::Expression,
"the value to set the variable to",
);
working_set.add_decl(sig.into());
working_set.add_decl(sig.into());
let sig = Signature::build("def")
.required("def_name", SyntaxShape::String, "definition name")
.required("params", SyntaxShape::Signature, "parameters")
.required("block", SyntaxShape::Block, "body of the definition");
working_set.add_decl(sig.into());
let sig = Signature::build("alias")
.required("var_name", SyntaxShape::Variable, "variable name")
.required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign")
.required(
"value",
SyntaxShape::Expression,
"the value to set the variable to",
);
working_set.add_decl(sig.into());
let sig = Signature::build("add");
working_set.add_decl(sig.into());
let sig = Signature::build("add it");
working_set.add_decl(sig.into());
let sig = Signature::build("sum").required(
"arg",
SyntaxShape::List(Box::new(SyntaxShape::Number)),
"list of numbers",
);
working_set.add_decl(sig.into());
let sig = Signature::build("add it together")
.required("x", SyntaxShape::Int, "x value")
.required("y", SyntaxShape::Int, "y value");
working_set.add_decl(sig.into());
let sig = Signature::build("def")
.required("def_name", SyntaxShape::String, "definition name")
.required("params", SyntaxShape::Signature, "parameters")
.required("block", SyntaxShape::Block, "body of the definition");
working_set.add_decl(sig.into());
ParserState::merge_working_set(&mut parser_state, working_set);
// let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
// working_set.add_decl(sig.into());
// let sig = Signature::build("bar")
// .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
// .switch("--rock", "rock!!", Some('r'));
// working_set.add_decl(sig.into());
let sig = Signature::build("add");
working_set.add_decl(sig.into());
let sig = Signature::build("add it");
working_set.add_decl(sig.into());
let sig = Signature::build("add it together")
.required("x", SyntaxShape::Int, "x value")
.required("y", SyntaxShape::Int, "y value");
working_set.add_decl(sig.into());
ParserState::merge_working_set(&mut parser_state, working_set);
if let Some(path) = std::env::args().nth(1) {
// let file = std::fs::read(&path)?;
// let (output, err) = working_set.parse_file(&path, file);
@ -99,7 +97,48 @@ fn main() -> std::io::Result<()> {
Ok(())
} else {
println!("specify file to lex");
use reedline::{DefaultPrompt, FileBackedHistory, Reedline, Signal};
let mut line_editor =
Reedline::new().with_history(Box::new(FileBackedHistory::new(1000)))?;
let prompt = DefaultPrompt::new(1);
let mut current_line = 1;
loop {
let input = line_editor.read_line(&prompt)?;
match input {
Signal::Success(s) => {
if s.trim() == "exit" {
break;
}
println!("input: '{}'", s);
let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
let (output, err) = working_set.parse_file(
&format!("line_{}", current_line),
s.as_bytes(),
false,
);
ParserState::merge_working_set(&mut parser_state, working_set);
println!("{:#?}", parser_state);
println!("{:#?}", output);
println!("Error: {:?}", err);
}
Signal::CtrlC => {
println!("Ctrl-c");
}
Signal::CtrlD => {
break;
}
Signal::CtrlL => {
line_editor.clear_screen()?;
}
}
current_line += 1;
}
Ok(())
}
}

40
src/parser.rs
View File

@ -549,6 +549,7 @@ impl ParserWorkingSet {
pub fn parse_internal_call(
&mut self,
command_span: Span,
spans: &[Span],
decl_id: usize,
) -> (Box<Call>, Span, Option<ParseError>) {
@ -567,7 +568,7 @@ impl ParserWorkingSet {
// The index into the spans of argument data given to parse
// Starting at the first argument
let mut spans_idx = 1;
let mut spans_idx = 0;
while spans_idx < spans.len() {
let arg_span = spans[spans_idx];
@ -615,8 +616,17 @@ impl ParserWorkingSet {
let end = if decl.signature.rest_positional.is_some() {
spans.len()
} else {
println!("num_positionals: {}", decl.signature.num_positionals());
println!("positional_idx: {}", positional_idx);
println!("spans.len(): {}", spans.len());
println!("spans_idx: {}", spans_idx);
let remainder = decl.signature.num_positionals() - positional_idx;
spans.len() - remainder + 1
if remainder > spans.len() {
spans.len()
} else {
spans.len() - remainder + 1
}
};
let (arg, err) =
@ -632,7 +642,7 @@ impl ParserWorkingSet {
spans_idx += 1;
}
let err = check_call(spans[0], &decl.signature, &call);
let err = check_call(command_span, &decl.signature, &call);
error = error.or(err);
// FIXME: type unknown
@ -651,6 +661,7 @@ impl ParserWorkingSet {
let mut new_name = name.to_vec();
new_name.push(b' ');
new_name.extend(self.get_span_contents(spans[pos]));
if let Some(did) = self.find_decl(&new_name) {
decl_id = did;
} else {
@ -660,11 +671,12 @@ impl ParserWorkingSet {
pos += 1;
}
// parse internal command
let (call, span, err) = self.parse_internal_call(&spans[(pos - 1)..], decl_id);
let (call, _, err) =
self.parse_internal_call(span(&spans[0..pos]), &spans[pos..], decl_id);
(
Expression {
expr: Expr::Call(call),
span,
span: span(spans),
},
err,
)
@ -1757,7 +1769,8 @@ impl ParserWorkingSet {
if name == b"def" {
if let Some(decl_id) = self.find_decl(b"def") {
let (call, call_span, err) = self.parse_internal_call(spans, decl_id);
let (call, call_span, err) =
self.parse_internal_call(spans[0], &spans[1..], decl_id);
if err.is_some() {
return (
@ -1813,7 +1826,8 @@ impl ParserWorkingSet {
if name == b"let" {
if let Some(decl_id) = self.find_decl(b"let") {
let (call, call_span, err) = self.parse_internal_call(spans, decl_id);
let (call, call_span, err) =
self.parse_internal_call(spans[0], &spans[1..], decl_id);
return (
Statement::Expression(Expression {
@ -1890,16 +1904,18 @@ impl ParserWorkingSet {
pub fn parse_file(
&mut self,
fname: &str,
contents: Vec<u8>,
contents: &[u8],
scoped: bool,
) -> (Block, Option<ParseError>) {
let mut error = None;
let (output, err) = lex(&contents, 0, &[], &[]);
error = error.or(err);
let span_offset = self.next_span_start();
self.add_file(fname.into(), contents);
let (output, err) = lex(&contents, span_offset, &[], &[]);
error = error.or(err);
let (output, err) = lite_parse(&output);
error = error.or(err);
@ -1912,9 +1928,11 @@ impl ParserWorkingSet {
pub fn parse_source(&mut self, source: &[u8], scoped: bool) -> (Block, Option<ParseError>) {
let mut error = None;
let span_offset = self.next_span_start();
self.add_file("source".into(), source.into());
let (output, err) = lex(source, 0, &[], &[]);
let (output, err) = lex(source, span_offset, &[], &[]);
error = error.or(err);
let (output, err) = lite_parse(&output);

10
src/parser_state.rs
View File

@ -205,10 +205,10 @@ impl ParserWorkingSet {
}
}
pub fn add_file(&mut self, filename: String, contents: Vec<u8>) -> usize {
pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize {
let next_span_start = self.next_span_start();
self.file_contents.extend(&contents);
self.file_contents.extend(contents);
let next_span_end = self.next_span_start();
@ -354,7 +354,7 @@ mod parser_state_tests {
#[test]
fn add_file_gives_id() {
let mut parser_state = ParserWorkingSet::new(Some(Arc::new(ParserState::new())));
let id = parser_state.add_file("test.nu".into(), vec![]);
let id = parser_state.add_file("test.nu".into(), &[]);
assert_eq!(id, 0);
}
@ -365,7 +365,7 @@ mod parser_state_tests {
let parent_id = parser_state.add_file("test.nu".into(), vec![]);
let mut working_set = ParserWorkingSet::new(Some(Arc::new(parser_state)));
let working_set_id = working_set.add_file("child.nu".into(), vec![]);
let working_set_id = working_set.add_file("child.nu".into(), &[]);
assert_eq!(parent_id, 0);
assert_eq!(working_set_id, 1);
@ -378,7 +378,7 @@ mod parser_state_tests {
let mut parser_state = Arc::new(parser_state);
let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
working_set.add_file("child.nu".into(), vec![]);
working_set.add_file("child.nu".into(), &[]);
ParserState::merge_working_set(&mut parser_state, working_set);