parser fixes for windows and pretty errors

This commit is contained in:
Jonathan Turner 2021-08-10 17:08:10 +12:00
parent 1a3e1e0959
commit ef4af443a5
8 changed files with 410 additions and 18 deletions

View File

@ -8,6 +8,7 @@ edition = "2018"
[dependencies]
reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" }
nu-ansi-term = "0.32.0"
codespan-reporting = "0.11.1"
# mimalloc = { version = "*", default-features = false }
[dev-dependencies]

322
src/errors.rs Normal file
View File

@ -0,0 +1,322 @@
use core::ops::Range;
use crate::{ParseError, ParserWorkingSet, ShellError, Span};
use codespan_reporting::diagnostic::{Diagnostic, Label};
use codespan_reporting::term::termcolor::{ColorChoice, StandardStream};
impl<'a> codespan_reporting::files::Files<'a> for ParserWorkingSet<'a> {
type FileId = usize;
type Name = String;
type Source = String;
fn name(&'a self, id: Self::FileId) -> Result<Self::Name, codespan_reporting::files::Error> {
Ok(self.get_filename(id))
}
fn source(
&'a self,
id: Self::FileId,
) -> Result<Self::Source, codespan_reporting::files::Error> {
Ok(self.get_file_source(id))
}
fn line_index(
&'a self,
id: Self::FileId,
byte_index: usize,
) -> Result<usize, codespan_reporting::files::Error> {
let source = self.get_file_source(id);
let mut count = 0;
for byte in source.bytes().enumerate() {
if byte.0 == byte_index {
// println!("count: {} for file: {} index: {}", count, id, byte_index);
return Ok(count);
}
if byte.1 == b'\n' {
count += 1;
}
}
// println!("count: {} for file: {} index: {}", count, id, byte_index);
Ok(count)
}
fn line_range(
&'a self,
id: Self::FileId,
line_index: usize,
) -> Result<Range<usize>, codespan_reporting::files::Error> {
let source = self.get_file_source(id);
let mut count = 0;
let mut start = Some(0);
let mut end = None;
for byte in source.bytes().enumerate() {
#[allow(clippy::comparison_chain)]
if count > line_index {
let start = start.expect("internal error: couldn't find line");
let end = end.expect("internal error: couldn't find line");
// println!(
// "Span: {}..{} for fileid: {} index: {}",
// start, end, id, line_index
// );
return Ok(start..end);
} else if count == line_index {
end = Some(byte.0 + 1);
}
#[allow(clippy::comparison_chain)]
if byte.1 == b'\n' {
count += 1;
if count > line_index {
break;
} else if count == line_index {
start = Some(byte.0);
}
}
}
match (start, end) {
(Some(start), Some(end)) => {
// println!(
// "Span: {}..{} for fileid: {} index: {}",
// start, end, id, line_index
// );
Ok(start..end)
}
_ => Err(codespan_reporting::files::Error::FileMissing),
}
}
}
fn convert_span_to_diag(
working_set: &ParserWorkingSet,
span: &Span,
) -> Result<(usize, Range<usize>), Box<dyn std::error::Error>> {
for (file_id, (_, start, end)) in working_set.files().enumerate() {
if span.start >= *start && span.end <= *end {
let new_start = span.start - start;
let new_end = span.end - start;
return Ok((file_id, new_start..new_end));
}
}
panic!("internal error: can't find span in parser state")
}
pub fn report_parsing_error(
working_set: &ParserWorkingSet,
error: &ParseError,
) -> Result<(), Box<dyn std::error::Error>> {
let writer = StandardStream::stderr(ColorChoice::Always);
let config = codespan_reporting::term::Config::default();
let diagnostic =
match error {
ParseError::Mismatch(missing, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Type mismatch during operation")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("expected {}", missing))])
}
ParseError::ExtraTokens(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Extra tokens in code")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("extra tokens")
])
}
ParseError::ExtraPositional(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Extra positional argument")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message("extra positional argument")])
}
ParseError::UnexpectedEof(s, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unexpected end of code")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("expected {}", s))])
}
ParseError::Unclosed(delim, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unclosed delimiter")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("unclosed {}", delim))])
}
ParseError::UnknownStatement(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unknown statement")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("unknown statement")
])
}
ParseError::MultipleRestParams(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Multiple rest params")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message("multiple rest params")])
}
ParseError::VariableNotFound(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Variable not found")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("variable not found")
])
}
ParseError::UnknownCommand(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unknown command")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("unknown command")
])
}
ParseError::UnknownFlag(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unknown flag")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("unknown flag")
])
}
ParseError::UnknownType(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unknown type")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("unknown type")
])
}
ParseError::MissingFlagParam(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Missing flag param")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message("flag missing parameter")])
}
ParseError::ShortFlagBatchCantTakeArg(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Batches of short flags can't take arguments")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message("short flag batches can't take args")])
}
ParseError::MissingPositional(name, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Missing required positional arg")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("missing {}", name))])
}
ParseError::MissingType(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Missing type")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("expected type")
])
}
ParseError::TypeMismatch(ty, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Type mismatch")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("expected {:?}", ty))])
}
ParseError::MissingRequiredFlag(name, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Missing required flag")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("missing required flag {}", name))])
}
ParseError::IncompleteMathExpression(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Incomplete math expresssion")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message("incomplete math expression")])
}
ParseError::UnknownState(name, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unknown state")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("unknown state {}", name))])
}
ParseError::NonUtf8(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Non-UTF8 code")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("non-UTF8 code")
])
}
};
// println!("DIAG");
// println!("{:?}", diagnostic);
codespan_reporting::term::emit(&mut writer.lock(), &config, working_set, &diagnostic)?;
Ok(())
}
pub fn report_shell_error(
working_set: &ParserWorkingSet,
error: &ShellError,
) -> Result<(), Box<dyn std::error::Error>> {
let writer = StandardStream::stderr(ColorChoice::Always);
let config = codespan_reporting::term::Config::default();
let diagnostic = match error {
ShellError::Mismatch(missing, span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Type mismatch during operation")
.with_labels(vec![Label::primary(diag_file_id, diag_range)
.with_message(format!("expected {}", missing))])
}
ShellError::Unsupported(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Unsupported operation")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("unsupported operation")
])
}
ShellError::InternalError(s) => {
Diagnostic::error().with_message(format!("Internal error: {}", s))
}
ShellError::VariableNotFound(span) => {
let (diag_file_id, diag_range) = convert_span_to_diag(working_set, span)?;
Diagnostic::error()
.with_message("Variable not found")
.with_labels(vec![
Label::primary(diag_file_id, diag_range).with_message("variable not found")
])
}
};
// println!("DIAG");
// println!("{:?}", diagnostic);
codespan_reporting::term::emit(&mut writer.lock(), &config, working_set, &diagnostic)?;
Ok(())
}

View File

@ -358,9 +358,7 @@ fn eval_call(state: &State, stack: Stack, call: &Call) -> Result<Value, ShellErr
} else if decl.signature.name == "stack" {
stack.print_stack();
Ok(Value::Nothing { span: call.head })
} else if decl.signature.name == "def" {
Ok(Value::Nothing { span: call.head })
} else if decl.signature.name == "alias" {
} else if decl.signature.name == "def" || decl.signature.name == "alias" {
Ok(Value::Nothing { span: call.head })
} else {
Err(ShellError::Unsupported(call.head))

View File

@ -51,6 +51,7 @@ fn is_item_terminator(
&& (c == b' '
|| c == b'\t'
|| c == b'\n'
|| c == b'\r'
|| c == b'|'
|| c == b';'
|| c == b'#'
@ -269,7 +270,7 @@ pub fn lex(
while let Some(input) = input.get(curr_offset) {
curr_offset += 1;
if *input == b'\n' {
if *input == b'\n' || *input == b'\r' {
output.push(Token::new(
TokenContents::Comment,
Span::new(start, curr_offset),

View File

@ -1,4 +1,5 @@
mod declaration;
mod errors;
mod eval;
mod flatten;
mod lex;
@ -14,7 +15,8 @@ mod tests;
mod type_check;
pub use declaration::Declaration;
pub use eval::{eval_block, eval_expression, Stack, StackFrame, State};
pub use errors::{report_parsing_error, report_shell_error};
pub use eval::{eval_block, eval_expression, ShellError, Stack, StackFrame, State};
pub use lex::{lex, Token, TokenContents};
pub use lite_parse::{lite_parse, LiteBlock, LiteCommand, LiteStatement};
pub use parse_error::ParseError;

View File

@ -1,7 +1,8 @@
use std::{cell::RefCell, rc::Rc};
use engine_q::{
eval_block, NuHighlighter, ParserState, ParserWorkingSet, Signature, Stack, State, SyntaxShape,
eval_block, report_parsing_error, report_shell_error, NuHighlighter, ParserState,
ParserWorkingSet, Signature, Stack, State, SyntaxShape,
};
fn main() -> std::io::Result<()> {
@ -130,7 +131,8 @@ fn main() -> std::io::Result<()> {
let mut working_set = ParserWorkingSet::new(&*parser_state);
let (output, err) = working_set.parse_file(&path, &file, false);
if let Some(err) = err {
eprintln!("Parse Error: {:?}", err);
let _ = report_parsing_error(&working_set, &err);
std::process::exit(1);
}
(output, working_set.render())
@ -149,7 +151,11 @@ fn main() -> std::io::Result<()> {
println!("{}", value);
}
Err(err) => {
eprintln!("Eval Error: {:?}", err);
let parser_state = parser_state.borrow();
let working_set = ParserWorkingSet::new(&*parser_state);
let _ = engine_q::report_shell_error(&working_set, &err);
std::process::exit(1);
}
}
@ -189,7 +195,7 @@ fn main() -> std::io::Result<()> {
false,
);
if let Some(err) = err {
eprintln!("Parse Error: {:?}", err);
let _ = report_parsing_error(&working_set, &err);
continue;
}
(output, working_set.render())
@ -206,7 +212,10 @@ fn main() -> std::io::Result<()> {
println!("{}", value);
}
Err(err) => {
eprintln!("Eval Error: {:?}", err);
let parser_state = parser_state.borrow();
let working_set = ParserWorkingSet::new(&*parser_state);
let _ = report_shell_error(&working_set, &err);
}
}
}

View File

@ -1,6 +1,6 @@
use crate::{parser::Block, Declaration, Span};
use core::panic;
use std::collections::HashMap;
use std::{collections::HashMap, slice::Iter};
#[derive(Debug)]
pub struct ParserState {
@ -155,6 +155,33 @@ impl ParserState {
self.file_contents.len()
}
pub fn files(&self) -> Iter<(String, usize, usize)> {
self.files.iter()
}
pub fn get_filename(&self, file_id: usize) -> String {
for file in self.files.iter().enumerate() {
if file.0 == file_id {
return file.1 .0.clone();
}
}
"<unknown>".into()
}
pub fn get_file_source(&self, file_id: usize) -> String {
for file in self.files.iter().enumerate() {
if file.0 == file_id {
let output =
String::from_utf8_lossy(&self.file_contents[file.1 .1..file.1 .2]).to_string();
return output;
}
}
"<unknown>".into()
}
#[allow(unused)]
pub(crate) fn add_file(&mut self, filename: String, contents: Vec<u8>) -> usize {
let next_span_start = self.next_span_start();
@ -264,6 +291,36 @@ impl<'a> ParserWorkingSet<'a> {
self.permanent_state.next_span_start()
}
pub fn files(&'a self) -> impl Iterator<Item = &(String, usize, usize)> {
self.permanent_state.files().chain(self.delta.files.iter())
}
pub fn get_filename(&self, file_id: usize) -> String {
for file in self.files().enumerate() {
if file.0 == file_id {
return file.1 .0.clone();
}
}
"<unknown>".into()
}
pub fn get_file_source(&self, file_id: usize) -> String {
for file in self.files().enumerate() {
if file.0 == file_id {
let output = String::from_utf8_lossy(self.get_span_contents(Span {
start: file.1 .1,
end: file.1 .2,
}))
.to_string();
return output;
}
}
"<unknown>".into()
}
pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize {
let next_span_start = self.next_span_start();

View File

@ -43,11 +43,13 @@ fn fail_test(input: &str, expected: &str) -> TestResult {
let output = cmd.output()?;
let output = String::from_utf8_lossy(&output.stderr).to_string();
println!("{}", output);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output.contains("Error:"));
assert!(output.contains(expected));
println!("stdout: {}", stdout);
println!("stderr: {}", stderr);
assert!(stderr.contains(expected));
Ok(())
}
@ -64,7 +66,7 @@ fn add_simple2() -> TestResult {
#[test]
fn broken_math() -> TestResult {
fail_test("3 + ", "Incomplete")
fail_test("3 + ", "incomplete")
}
#[test]
@ -81,7 +83,7 @@ fn if_test2() -> TestResult {
fn no_scope_leak1() -> TestResult {
fail_test(
"if $false { let $x = 10 } else { let $x = 20 }; $x",
"VariableNotFound",
"variable not found",
)
}
@ -89,7 +91,7 @@ fn no_scope_leak1() -> TestResult {
fn no_scope_leak2() -> TestResult {
fail_test(
"def foo [] { $x }; def bar [] { let $x = 10; foo }; bar",
"VariableNotFound",
"Variable not found",
)
}