extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-10-05 01:43:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

Modernize external parse and improve trace

Browse Source 
The original purpose of this PR was to modernize the external parser to
use the new Shape system.

This commit does include some of that change, but a more important
aspect of this change is an improvement to the expansion trace.

Previous commit 6a7c00ea adding trace infrastructure to the syntax coloring
feature. This commit adds tracing to the expander.

The bulk of that work, in addition to the tree builder logic, was an
overhaul of the formatter traits to make them more general purpose, and
more structured.

Some highlights:

- `ToDebug` was split into two traits (`ToDebug` and `DebugFormat`)
  because implementations needed to become objects, but a convenience
  method on `ToDebug` didn't qualify
- `DebugFormat`'s `fmt_debug` method now takes a `DebugFormatter` rather
  than a standard formatter, and `DebugFormatter` has a new (but still
  limited) facility for structured formatting.
- Implementations of `ExpandSyntax` need to produce output that
  implements `DebugFormat`.

Unlike the highlighter changes, these changes are fairly focused in the
trace output, so these changes aren't behind a flag.
This commit is contained in:
Yehuda Katz 2019-10-28 07:46:50 -07:00
parent 7f18ff10b2
commit 4be88ff572
45 changed files with 2301 additions and 1029 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.cargo/config
View File

@ -0,0 +1,3 @@
[build]
rustflags = "--cfg coloring_in_tokens"

9
Cargo.lock generated
View File

@ -1498,6 +1498,7 @@ dependencies = [
"bson 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono-humanize 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1540,7 +1541,7 @@ dependencies = [
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"roxmltree 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)",
"rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
"serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2077,8 +2078,8 @@ dependencies = [
[[package]]
name = "rustyline"
version = "5.0.3"
source = "git+https://github.com/kkawakam/rustyline.git#449c811998f630102bb2d9fb0b59b890d9eabac5"
version = "5.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3056,7 +3057,7 @@ dependencies = [
"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)" = "<none>"
"checksum rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e9d8eb9912bc492db051324d36f5cea56984fc2afeaa5c6fa84e0b0e3cde550f"
"checksum ryu 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19d2271fa48eaf61e53cc88b4ad9adcbafa2d512c531e7fadb6dc11a4d3656c5"
"checksum safemem 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0"
"checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421"

1
Cargo.toml
View File

@ -75,6 +75,7 @@ serde_urlencoded = "0.6.1"
sublime_fuzzy = "0.5"
trash = "1.0.0"
regex = "1"
cfg-if = "0.1"
neso = { version = "0.5.0", optional = true }
crossterm = { version = "0.10.2", optional = true }

36
src/cli.rs
View File

@ -14,13 +14,13 @@ use crate::git::current_branch;
use crate::parser::registry::Signature;
use crate::parser::{
hir,
hir::syntax_shape::{expand_syntax, PipelineShape},
hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator},
hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape},
hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator},
TokenNode,
};
use crate::prelude::*;
use log::{debug, trace};
use log::{debug, log_enabled, trace};
use rustyline::error::ReadlineError;
use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
use std::env;
@ -506,6 +506,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
Some(ClassifiedCommand::External(_)) => {}
_ => pipeline
.commands
.item
.push(ClassifiedCommand::Internal(InternalCommand {
name: "autoview".to_string(),
name_tag: Tag::unknown(),
@ -513,13 +514,14 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
Box::new(hir::Expression::synthetic_string("autoview")),
None,
None,
),
)
.spanned_unknown(),
})),
}
let mut input = ClassifiedInputStream::new();
let mut iter = pipeline.commands.into_iter().peekable();
let mut iter = pipeline.commands.item.into_iter().peekable();
let mut is_first_command = true;
// Check the config to see if we need to update the path
@ -679,11 +681,20 @@ fn classify_pipeline(
let mut pipeline_list = vec![pipeline.clone()];
let mut iterator = TokensIterator::all(&mut pipeline_list, pipeline.span());
expand_syntax(
let result = expand_syntax(
&PipelineShape,
&mut iterator,
&context.expand_context(source, pipeline.span()),
&context.expand_context(source),
)
.map_err(|err| err.into());
if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
println!("");
ptree::print_tree(&iterator.expand_tracer().print(source.clone())).unwrap();
println!("");
}
result
}
// Classify this command as an external command, which doesn't give special meaning
@ -691,21 +702,22 @@ fn classify_pipeline(
// strings.
pub(crate) fn external_command(
tokens: &mut TokensIterator,
source: &Text,
context: &ExpandContext,
name: Tagged<&str>,
) -> Result<ClassifiedCommand, ShellError> {
let arg_list_strings = expand_external_tokens(tokens, source)?;
) -> Result<ClassifiedCommand, ParseError> {
let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?;
Ok(ClassifiedCommand::External(ExternalCommand {
name: name.to_string(),
name_tag: name.tag(),
args: arg_list_strings
args: item
.iter()
.map(|x| Tagged {
tag: x.span.into(),
item: x.item.clone(),
})
.collect(),
.collect::<Vec<_>>()
.spanned(span),
}))
}

94
src/commands/classified.rs
View File

@ -4,7 +4,9 @@ use bytes::{BufMut, BytesMut};
use derive_new::new;
use futures::stream::StreamExt;
use futures_codec::{Decoder, Encoder, Framed};
use itertools::Itertools;
use log::{log_enabled, trace};
use std::fmt;
use std::io::{Error, ErrorKind};
use subprocess::Exec;
@ -72,26 +74,77 @@ impl ClassifiedInputStream {
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub(crate) struct ClassifiedPipeline {
pub(crate) commands: Vec<ClassifiedCommand>,
pub(crate) commands: Spanned<Vec<ClassifiedCommand>>,
}
#[derive(Debug, Eq, PartialEq)]
impl FormatDebug for ClassifiedPipeline {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str(
"classified pipeline",
self.commands.iter().map(|c| c.debug(source)).join(" | "),
)
}
}
impl HasSpan for ClassifiedPipeline {
fn span(&self) -> Span {
self.commands.span
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) enum ClassifiedCommand {
#[allow(unused)]
Expr(TokenNode),
Internal(InternalCommand),
#[allow(unused)]
Dynamic(hir::Call),
Dynamic(Spanned<hir::Call>),
External(ExternalCommand),
}
#[derive(new, Debug, Eq, PartialEq)]
impl FormatDebug for ClassifiedCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
ClassifiedCommand::Expr(expr) => expr.fmt_debug(f, source),
ClassifiedCommand::Internal(internal) => internal.fmt_debug(f, source),
ClassifiedCommand::Dynamic(dynamic) => dynamic.fmt_debug(f, source),
ClassifiedCommand::External(external) => external.fmt_debug(f, source),
}
}
}
impl HasSpan for ClassifiedCommand {
fn span(&self) -> Span {
match self {
ClassifiedCommand::Expr(node) => node.span(),
ClassifiedCommand::Internal(command) => command.span(),
ClassifiedCommand::Dynamic(call) => call.span,
ClassifiedCommand::External(command) => command.span(),
}
}
}
#[derive(new, Debug, Clone, Eq, PartialEq)]
pub(crate) struct InternalCommand {
pub(crate) name: String,
pub(crate) name_tag: Tag,
pub(crate) args: hir::Call,
pub(crate) args: Spanned<hir::Call>,
}
impl HasSpan for InternalCommand {
fn span(&self) -> Span {
let start = self.name_tag.span;
start.until(self.args.span)
}
}
impl FormatDebug for InternalCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say("internal", self.args.debug(source))
}
}
#[derive(new, Debug, Eq, PartialEq)]
@ -122,7 +175,7 @@ impl InternalCommand {
context.run_command(
command,
self.name_tag.clone(),
self.args,
self.args.item,
&source,
objects,
is_first_command,
@ -201,12 +254,31 @@ impl InternalCommand {
}
}
#[derive(Debug, Eq, PartialEq)]
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) struct ExternalCommand {
pub(crate) name: String,
pub(crate) name_tag: Tag,
pub(crate) args: Vec<Tagged<String>>,
pub(crate) args: Spanned<Vec<Tagged<String>>>,
}
impl FormatDebug for ExternalCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.name)?;
if self.args.item.len() > 0 {
write!(f, " ")?;
write!(f, "{}", self.args.iter().map(|i| i.debug(source)).join(" "))?;
}
Ok(())
}
}
impl HasSpan for ExternalCommand {
fn span(&self) -> Span {
self.name_tag.span.until(self.args.span)
}
}
#[derive(Debug)]
@ -230,7 +302,7 @@ impl ExternalCommand {
trace!(target: "nu::run::external", "inputs = {:?}", inputs);
let mut arg_string = format!("{}", self.name);
for arg in &self.args {
for arg in &self.args.item {
arg_string.push_str(&arg);
}
@ -275,7 +347,7 @@ impl ExternalCommand {
process = Exec::shell(itertools::join(commands, " && "))
} else {
process = Exec::cmd(&self.name);
for arg in &self.args {
for arg in &self.args.item {
let arg_chars: Vec<_> = arg.chars().collect();
if arg_chars.len() > 1
&& arg_chars[0] == '"'

26
src/commands/command.rs
View File

@ -19,8 +19,8 @@ pub struct UnevaluatedCallInfo {
pub name_tag: Tag,
}
impl ToDebug for UnevaluatedCallInfo {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for UnevaluatedCallInfo {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.args.fmt_debug(f, source)
}
}
@ -96,8 +96,14 @@ impl RawCommandArgs {
}
}
impl ToDebug for CommandArgs {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl std::fmt::Debug for CommandArgs {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.call_info.fmt(f)
}
}
impl FormatDebug for CommandArgs {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.call_info.fmt_debug(f, source)
}
}
@ -377,7 +383,7 @@ impl EvaluatedCommandArgs {
}
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CommandAction {
ChangePath(String),
Exit,
@ -389,8 +395,8 @@ pub enum CommandAction {
LeaveShell,
}
impl ToDebug for CommandAction {
fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
impl FormatDebug for CommandAction {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
match self {
CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
CommandAction::Exit => write!(f, "action:exit"),
@ -408,7 +414,7 @@ impl ToDebug for CommandAction {
}
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ReturnSuccess {
Value(Tagged<Value>),
Action(CommandAction),
@ -416,8 +422,8 @@ pub enum ReturnSuccess {
pub type ReturnValue = Result<ReturnSuccess, ShellError>;
impl ToDebug for ReturnValue {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for ReturnValue {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Err(err) => write!(f, "{}", err.debug(source)),
Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),

3
src/context.rs
View File

@ -71,9 +71,8 @@ impl Context {
pub(crate) fn expand_context<'context>(
&'context self,
source: &'context Text,
span: Span,
) -> ExpandContext<'context> {
ExpandContext::new(&self.registry, span, source, self.shell_manager.homedir())
ExpandContext::new(&self.registry, source, self.shell_manager.homedir())
}
pub(crate) fn basic() -> Result<Context, Box<dyn Error>> {

132
src/data/meta.rs
View File

@ -5,6 +5,7 @@ use derive_new::new;
use getset::Getters;
use serde::Deserialize;
use serde::Serialize;
use std::fmt;
use std::path::{Path, PathBuf};
#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
@ -461,3 +462,134 @@ impl language_reporting::ReportingSpan for Span {
self.end
}
}
pub trait HasSpan: ToDebug {
fn span(&self) -> Span;
}
pub trait HasFallibleSpan: ToDebug {
fn maybe_span(&self) -> Option<Span>;
}
impl<T: HasSpan> HasFallibleSpan for T {
fn maybe_span(&self) -> Option<Span> {
Some(HasSpan::span(self))
}
}
impl<T> HasSpan for Spanned<T>
where
Spanned<T>: ToDebug,
{
fn span(&self) -> Span {
self.span
}
}
impl HasFallibleSpan for Option<Span> {
fn maybe_span(&self) -> Option<Span> {
*self
}
}
impl FormatDebug for Option<Span> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Option::None => write!(f, "no span"),
Option::Some(span) => FormatDebug::fmt_debug(span, f, source),
}
}
}
impl FormatDebug for Span {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{:?}", self.slice(source))
}
}
impl HasSpan for Span {
fn span(&self) -> Span {
*self
}
}
impl<T> FormatDebug for Option<Spanned<T>>
where
Spanned<T>: ToDebug,
{
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Option::None => write!(f, "nothing"),
Option::Some(spanned) => FormatDebug::fmt_debug(spanned, f, source),
}
}
}
impl<T> HasFallibleSpan for Option<Spanned<T>>
where
Spanned<T>: ToDebug,
{
fn maybe_span(&self) -> Option<Span> {
match self {
None => None,
Some(value) => Some(value.span),
}
}
}
impl<T> FormatDebug for Option<Tagged<T>>
where
Tagged<T>: ToDebug,
{
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Option::None => write!(f, "nothing"),
Option::Some(item) => FormatDebug::fmt_debug(item, f, source),
}
}
}
impl<T> HasFallibleSpan for Option<Tagged<T>>
where
Tagged<T>: ToDebug,
{
fn maybe_span(&self) -> Option<Span> {
match self {
None => None,
Some(value) => Some(value.tag.span),
}
}
}
impl<T> HasSpan for Tagged<T>
where
Tagged<T>: ToDebug,
{
fn span(&self) -> Span {
self.tag.span
}
}
impl<T: ToDebug> FormatDebug for Vec<T> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "[ ")?;
write!(
f,
"{}",
self.iter().map(|item| item.debug(source)).join(" ")
)?;
write!(f, " ]")
}
}
impl FormatDebug for String {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self)
}
}
impl FormatDebug for Spanned<String> {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self.item)
}
}

94
src/errors.rs
View File

@ -30,6 +30,82 @@ impl Description {
}
}
#[derive(Debug, Clone)]
pub enum ParseErrorReason {
Eof {
expected: &'static str,
},
Mismatch {
expected: &'static str,
actual: Tagged<String>,
},
ArgumentError {
command: String,
error: ArgumentError,
tag: Tag,
},
}
#[derive(Debug, Clone)]
pub struct ParseError {
reason: ParseErrorReason,
tag: Tag,
}
impl ParseError {
pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError {
ParseError {
reason: ParseErrorReason::Eof { expected },
tag: span.into(),
}
}
pub fn mismatch(expected: &'static str, actual: Tagged<impl Into<String>>) -> ParseError {
let Tagged { tag, item } = actual;
ParseError {
reason: ParseErrorReason::Mismatch {
expected,
actual: item.into().tagged(tag.clone()),
},
tag,
}
}
pub fn argument_error(
command: impl Into<String>,
kind: ArgumentError,
tag: impl Into<Tag>,
) -> ParseError {
let tag = tag.into();
ParseError {
reason: ParseErrorReason::ArgumentError {
command: command.into(),
error: kind,
tag: tag.clone(),
},
tag: tag.clone(),
}
}
}
impl From<ParseError> for ShellError {
fn from(error: ParseError) -> ShellError {
match error.reason {
ParseErrorReason::Eof { expected } => ShellError::unexpected_eof(expected, error.tag),
ParseErrorReason::Mismatch { actual, expected } => {
ShellError::type_error(expected, actual.clone())
}
ParseErrorReason::ArgumentError {
command,
error,
tag,
} => ShellError::argument_error(command, error, tag),
}
}
}
#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
pub enum ArgumentError {
MissingMandatoryFlag(String),
@ -51,8 +127,8 @@ impl ShellError {
}
}
impl ToDebug for ShellError {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for ShellError {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.error.fmt_debug(f, source)
}
}
@ -153,16 +229,6 @@ impl ShellError {
.start()
}
pub(crate) fn invalid_external_word(tag: impl Into<Tag>) -> ShellError {
ProximateShellError::ArgumentError {
command: "Invalid argument to Nu command (did you mean to call an external command?)"
.into(),
error: ArgumentError::InvalidExternalWord,
tag: tag.into(),
}
.start()
}
pub(crate) fn parse_error(
error: nom::Err<(
nom_locate::LocatedSpanEx<&str, TracableContext>,
@ -490,8 +556,8 @@ impl ProximateShellError {
}
}
impl ToDebug for ProximateShellError {
fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
impl FormatDebug for ProximateShellError {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
// TODO: Custom debug for inner spans
write!(f, "{:?}", self)
}

6
src/lib.rs
View File

@ -30,12 +30,16 @@ pub use crate::env::host::BasicHost;
pub use crate::parser::hir::SyntaxShape;
pub use crate::parser::parse::token_tree_builder::TokenTreeBuilder;
pub use crate::plugin::{serve_plugin, Plugin};
pub use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
pub use crate::utils::{did_you_mean, AbsoluteFile, AbsolutePath, RelativePath};
pub use cli::cli;
pub use data::base::{Primitive, Value};
pub use data::config::{config_path, APP_INFO};
pub use data::dict::{Dictionary, TaggedDictBuilder};
pub use data::meta::{tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem};
pub use data::meta::{
tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
TaggedItem,
};
pub use errors::{CoerceInto, ShellError};
pub use num_traits::cast::ToPrimitive;
pub use parser::parse::text::Text;

15
src/main.rs
View File

@ -19,6 +19,12 @@ fn main() -> Result<(), Box<dyn Error>> {
.multiple(true)
.takes_value(true),
)
.arg(
Arg::with_name("debug")
.long("debug")
.multiple(true)
.takes_value(true),
)
.get_matches();
let loglevel = match matches.value_of("loglevel") {
@ -48,6 +54,15 @@ fn main() -> Result<(), Box<dyn Error>> {
}
}
match matches.values_of("debug") {
None => {}
Some(values) => {
for item in values {
builder.filter_module(&format!("nu::{}", item), LevelFilter::Debug);
}
}
}
builder.try_init()?;
futures::executor::block_on(nu::cli())?;

41
src/parser/hir.rs
View File

@ -24,7 +24,6 @@ pub(crate) use self::external_command::ExternalCommand;
pub(crate) use self::named::NamedArguments;
pub(crate) use self::path::Path;
pub(crate) use self::syntax_shape::ExpandContext;
pub(crate) use self::tokens_iterator::debug::debug_tokens;
pub(crate) use self::tokens_iterator::TokensIterator;
pub use self::syntax_shape::SyntaxShape;
@ -50,8 +49,8 @@ impl Call {
}
}
impl ToDebug for Call {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Call {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "({}", self.head.debug(source))?;
if let Some(positional) = &self.positional {
@ -242,10 +241,14 @@ impl Expression {
pub(crate) fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
RawExpression::Variable(Variable::It(inner.into())).spanned(outer)
}
pub(crate) fn tagged_type_name(&self) -> Tagged<&'static str> {
self.item.type_name().tagged(self.span)
}
}
impl ToDebug for Expression {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Spanned<RawExpression> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match &self.item {
RawExpression::Literal(l) => l.spanned(self.span).fmt_debug(f, source),
RawExpression::FilePath(p) => write!(f, "{}", p.display()),
@ -256,7 +259,7 @@ impl ToDebug for Expression {
RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
RawExpression::Binary(b) => write!(f, "{}", b.debug(source)),
RawExpression::ExternalCommand(c) => write!(f, "^{}", c.name().slice(source)),
RawExpression::Block(exprs) => {
RawExpression::Block(exprs) => f.say_block("block", |f| {
write!(f, "{{ ")?;
for expr in exprs {
@ -264,8 +267,8 @@ impl ToDebug for Expression {
}
write!(f, "}}")
}
RawExpression::List(exprs) => {
}),
RawExpression::List(exprs) => f.say_block("list", |f| {
write!(f, "[ ")?;
for expr in exprs {
@ -273,7 +276,7 @@ impl ToDebug for Expression {
}
write!(f, "]")
}
}),
RawExpression::Path(p) => write!(f, "{}", p.debug(source)),
RawExpression::Boolean(true) => write!(f, "$yes"),
RawExpression::Boolean(false) => write!(f, "$no"),
@ -321,14 +324,14 @@ impl std::fmt::Display for Tagged<&Literal> {
}
}
impl ToDebug for Spanned<&Literal> {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Spanned<&Literal> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self.item {
Literal::Number(number) => write!(f, "{:?}", number),
Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit),
Literal::String(tag) => write!(f, "{}", tag.slice(source)),
Literal::GlobPattern(_) => write!(f, "{}", self.span.slice(source)),
Literal::Bare => write!(f, "{}", self.span.slice(source)),
Literal::Number(..) => f.say_str("number", self.span.slice(source)),
Literal::Size(..) => f.say_str("size", self.span.slice(source)),
Literal::String(..) => f.say_str("string", self.span.slice(source)),
Literal::GlobPattern(..) => f.say_str("glob", self.span.slice(source)),
Literal::Bare => f.say_str("word", self.span.slice(source)),
}
}
}
@ -359,3 +362,9 @@ impl std::fmt::Display for Variable {
}
}
}
impl FormatDebug for Spanned<Variable> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.span.slice(source))
}
}

19
src/parser/hir/baseline_parse/tests.rs
View File

@ -6,7 +6,7 @@ use crate::parser::hir::syntax_shape::*;
use crate::parser::hir::TokensIterator;
use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
use crate::parser::TokenNode;
use crate::{Span, SpannedItem, Tag, Text};
use crate::{HasSpan, Span, SpannedItem, Tag, Text};
use pretty_assertions::assert_eq;
use std::fmt::Debug;
@ -63,7 +63,9 @@ fn test_parse_command() {
vec![b::bare("ls"), b::sp(), b::pattern("*.txt")],
|tokens| {
let bare = tokens[0].expect_bare();
let pattern = tokens[2].expect_pattern();
let pat = tokens[2].expect_pattern();
eprintln!("{:?} {:?} {:?}", bare, pat, bare.until(pat));
ClassifiedCommand::Internal(InternalCommand::new(
"ls".to_string(),
@ -73,9 +75,10 @@ fn test_parse_command() {
},
hir::Call {
head: Box::new(hir::RawExpression::Command(bare).spanned(bare)),
positional: Some(vec![hir::Expression::pattern("*.txt", pattern)]),
positional: Some(vec![hir::Expression::pattern("*.txt", pat)]),
named: None,
},
}
.spanned(bare.until(pat)),
))
// hir::Expression::path(
// hir::Expression::variable(inner_var, outer_var),
@ -86,7 +89,7 @@ fn test_parse_command() {
);
}
fn parse_tokens<T: Eq + Debug>(
fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
shape: impl ExpandSyntax<Output = T>,
tokens: Vec<CurriedToken>,
expected: impl FnOnce(&[TokenNode]) -> T,
@ -96,19 +99,19 @@ fn parse_tokens<T: Eq + Debug>(
ExpandContext::with_empty(&Text::from(source), |context| {
let tokens = tokens.expect_list();
let mut iterator = TokensIterator::all(tokens, *context.span());
let mut iterator = TokensIterator::all(tokens.item, tokens.span);
let expr = expand_syntax(&shape, &mut iterator, &context);
let expr = match expr {
Ok(expr) => expr,
Err(err) => {
crate::cli::print_err(err, &BasicHost, context.source().clone());
crate::cli::print_err(err.into(), &BasicHost, context.source().clone());
panic!("Parse failed");
}
};
assert_eq!(expr, expected(tokens));
assert_eq!(expr, expected(tokens.item));
})
}

4
src/parser/hir/binary.rs
View File

@ -22,8 +22,8 @@ impl fmt::Display for Binary {
}
}
impl ToDebug for Binary {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Binary {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.left.debug(source))?;
write!(f, " {} ", self.op.debug(source))?;
write!(f, "{}", self.right.debug(source))?;

362
src/parser/hir/expand_external_tokens.rs
View File

@ -1,35 +1,55 @@
use crate::errors::ShellError;
use crate::errors::ParseError;
#[cfg(not(coloring_in_tokens))]
use crate::parser::hir::syntax_shape::FlatShape;
use crate::parser::{
hir::syntax_shape::{
color_syntax, expand_atom, AtomicToken, ColorSyntax, ExpandContext, ExpansionRule,
MaybeSpaceShape,
color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax,
ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape,
},
TokenNode, TokensIterator,
hir::Expression,
TokensIterator,
};
use crate::{Span, Spanned, Text};
pub fn expand_external_tokens(
token_nodes: &mut TokensIterator<'_>,
source: &Text,
) -> Result<Vec<Spanned<String>>, ShellError> {
let mut out: Vec<Spanned<String>> = vec![];
loop {
if let Some(span) = expand_next_expression(token_nodes)? {
out.push(span.spanned_string(source));
} else {
break;
}
}
Ok(out)
}
use crate::{DebugFormatter, FormatDebug, Span, Spanned, SpannedItem};
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct ExternalTokensShape;
impl FormatDebug for Spanned<Vec<Spanned<String>>> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
FormatDebug::fmt_debug(&self.item, f, source)
}
}
impl ExpandSyntax for ExternalTokensShape {
type Output = Spanned<Vec<Spanned<String>>>;
fn name(&self) -> &'static str {
"external command"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
let mut out: Vec<Spanned<String>> = vec![];
let start = token_nodes.span_at_cursor();
loop {
match expand_syntax(&ExternalExpressionShape, token_nodes, context) {
Err(_) | Ok(None) => break,
Ok(Some(span)) => out.push(span.spanned_string(context.source())),
}
}
let end = token_nodes.span_at_cursor();
Ok(out.spanned(start.until(end)))
}
}
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for ExternalTokensShape {
type Info = ();
@ -85,109 +105,200 @@ impl ColorSyntax for ExternalTokensShape {
}
}
pub fn expand_next_expression(
token_nodes: &mut TokensIterator<'_>,
) -> Result<Option<Span>, ShellError> {
let first = token_nodes.next_non_ws();
#[derive(Debug, Copy, Clone)]
pub struct ExternalExpressionShape;
let first = match first {
None => return Ok(None),
Some(v) => v,
};
impl ExpandSyntax for ExternalExpressionShape {
type Output = Option<Span>;
let first = triage_external_head(first)?;
let mut last = first;
loop {
let continuation = triage_continuation(token_nodes)?;
if let Some(continuation) = continuation {
last = continuation;
} else {
break;
}
fn name(&self) -> &'static str {
"external expression"
}
Ok(Some(first.until(last)))
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
fn triage_external_head(node: &TokenNode) -> Result<Span, ShellError> {
Ok(match node {
TokenNode::Token(token) => token.span,
TokenNode::Call(_call) => unimplemented!("TODO: OMG"),
TokenNode::Nodes(_nodes) => unimplemented!("TODO: OMG"),
TokenNode::Delimited(_delimited) => unimplemented!("TODO: OMG"),
TokenNode::Pipeline(_pipeline) => unimplemented!("TODO: OMG"),
TokenNode::Flag(flag) => flag.span,
TokenNode::Whitespace(_whitespace) => {
unreachable!("This function should be called after next_non_ws()")
let first = expand_atom(
token_nodes,
"external command",
context,
ExpansionRule::new().allow_external_command(),
)?
.span;
let mut last = first;
loop {
let continuation = expand_expr(&ExternalContinuationShape, token_nodes, context);
if let Ok(continuation) = continuation {
last = continuation.span;
} else {
break;
}
}
TokenNode::Error(_error) => unimplemented!("TODO: OMG"),
})
}
fn triage_continuation<'a, 'b>(
nodes: &'a mut TokensIterator<'b>,
) -> Result<Option<Span>, ShellError> {
let mut peeked = nodes.peek_any();
let node = match peeked.node {
None => return Ok(None),
Some(node) => node,
};
match &node {
node if node.is_whitespace() => return Ok(None),
TokenNode::Token(..) | TokenNode::Flag(..) => {}
TokenNode::Call(..) => unimplemented!("call"),
TokenNode::Nodes(..) => unimplemented!("nodes"),
TokenNode::Delimited(..) => unimplemented!("delimited"),
TokenNode::Pipeline(..) => unimplemented!("pipeline"),
TokenNode::Whitespace(..) => unimplemented!("whitespace"),
TokenNode::Error(..) => unimplemented!("error"),
Ok(Some(first.until(last)))
}
peeked.commit();
Ok(Some(node.span()))
}
#[must_use]
enum ExternalExpressionResult {
Eof,
Processed,
}
#[derive(Debug, Copy, Clone)]
struct ExternalExpression;
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for ExternalExpression {
type Info = ExternalExpressionResult;
type Input = ();
impl ExpandSyntax for ExternalExpression {
type Output = Option<Span>;
fn color_syntax<'a, 'b>(
fn name(&self) -> &'static str {
"external expression"
}
fn expand_syntax<'a, 'b>(
&self,
_input: &(),
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
shapes: &mut Vec<Spanned<FlatShape>>,
) -> ExternalExpressionResult {
let atom = match expand_atom(
token_nodes,
"external word",
context,
ExpansionRule::permissive(),
) {
Err(_) => unreachable!("TODO: separate infallible expand_atom"),
Ok(Spanned {
item: AtomicToken::Eof { .. },
..
}) => return ExternalExpressionResult::Eof,
Ok(atom) => atom,
};
) -> Result<Self::Output, ParseError> {
expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
atom.color_tokens(shapes);
return ExternalExpressionResult::Processed;
let first = expand_syntax(&ExternalHeadShape, token_nodes, context)?.span;
let mut last = first;
loop {
let continuation = expand_syntax(&ExternalContinuationShape, token_nodes, context);
if let Ok(continuation) = continuation {
last = continuation.span;
} else {
break;
}
}
Ok(Some(first.until(last)))
}
}
#[derive(Debug, Copy, Clone)]
struct ExternalHeadShape;
impl ExpandExpression for ExternalHeadShape {
fn name(&self) -> &'static str {
"external argument"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Expression, ParseError> {
match expand_atom(
token_nodes,
"external argument",
context,
ExpansionRule::new()
.allow_external_word()
.treat_size_as_word(),
)? {
atom => match &atom {
Spanned { item, span } => Ok(match item {
AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
AtomicToken::Whitespace { .. } => {
unreachable!("ExpansionRule doesn't allow Whitespace")
}
AtomicToken::ShorthandFlag { .. }
| AtomicToken::LonghandFlag { .. }
| AtomicToken::SquareDelimited { .. }
| AtomicToken::ParenDelimited { .. }
| AtomicToken::BraceDelimited { .. }
| AtomicToken::Pipeline { .. } => {
return Err(ParseError::mismatch(
"external command name",
atom.tagged_type_name(),
))
}
AtomicToken::ExternalCommand { command } => {
Expression::external_command(*command, *span)
}
AtomicToken::Number { number } => {
Expression::number(number.to_number(context.source()), *span)
}
AtomicToken::String { body } => Expression::string(*body, *span),
AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
AtomicToken::Variable { name } => Expression::variable(*name, *span),
AtomicToken::ExternalWord { .. }
| AtomicToken::GlobPattern { .. }
| AtomicToken::FilePath { .. }
| AtomicToken::Word { .. }
| AtomicToken::Dot { .. }
| AtomicToken::Operator { .. } => Expression::external_command(*span, *span),
}),
},
}
}
}
#[derive(Debug, Copy, Clone)]
struct ExternalContinuationShape;
impl ExpandExpression for ExternalContinuationShape {
fn name(&self) -> &'static str {
"external argument"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Expression, ParseError> {
match expand_atom(
token_nodes,
"external argument",
context,
ExpansionRule::new()
.allow_external_word()
.treat_size_as_word(),
)? {
atom => match &atom {
Spanned { item, span } => Ok(match item {
AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
AtomicToken::Number { number } => {
Expression::number(number.to_number(context.source()), *span)
}
AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
AtomicToken::ExternalCommand { .. } => {
unreachable!("ExpansionRule doesn't allow ExternalCommand")
}
AtomicToken::Whitespace { .. } => {
unreachable!("ExpansionRule doesn't allow Whitespace")
}
AtomicToken::String { body } => Expression::string(*body, *span),
AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
AtomicToken::Variable { name } => Expression::variable(*name, *span),
AtomicToken::ExternalWord { .. }
| AtomicToken::GlobPattern { .. }
| AtomicToken::FilePath { .. }
| AtomicToken::Word { .. }
| AtomicToken::ShorthandFlag { .. }
| AtomicToken::LonghandFlag { .. }
| AtomicToken::Dot { .. }
| AtomicToken::Operator { .. } => Expression::bare(*span),
AtomicToken::SquareDelimited { .. }
| AtomicToken::ParenDelimited { .. }
| AtomicToken::BraceDelimited { .. }
| AtomicToken::Pipeline { .. } => {
return Err(ParseError::mismatch(
"external argument",
atom.tagged_type_name(),
))
}
}),
},
}
}
}
@ -224,3 +335,40 @@ impl ColorSyntax for ExternalExpression {
return ExternalExpressionResult::Processed;
}
}
#[must_use]
enum ExternalExpressionResult {
Eof,
Processed,
}
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for ExternalExpression {
type Info = ExternalExpressionResult;
type Input = ();
fn color_syntax<'a, 'b>(
&self,
_input: &(),
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
shapes: &mut Vec<Spanned<FlatShape>>,
) -> ExternalExpressionResult {
let atom = match expand_atom(
token_nodes,
"external word",
context,
ExpansionRule::permissive(),
) {
Err(_) => unreachable!("TODO: separate infallible expand_atom"),
Ok(Spanned {
item: AtomicToken::Eof { .. },
..
}) => return ExternalExpressionResult::Eof,
Ok(atom) => atom,
};
atom.color_tokens(shapes);
return ExternalExpressionResult::Processed;
}
}

4
src/parser/hir/external_command.rs
View File

@ -12,8 +12,8 @@ pub struct ExternalCommand {
pub(crate) name: Span,
}
impl ToDebug for ExternalCommand {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for ExternalCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.name.slice(source))?;
Ok(())

4
src/parser/hir/named.rs
View File

@ -21,8 +21,8 @@ pub struct NamedArguments {
pub(crate) named: IndexMap<String, NamedValue>,
}
impl ToDebug for NamedArguments {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for NamedArguments {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
for (name, value) in &self.named {
match value {
NamedValue::AbsentSwitch => continue,

4
src/parser/hir/path.rs
View File

@ -44,8 +44,8 @@ impl Path {
}
}
impl ToDebug for Path {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Path {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.head.debug(source))?;
for part in &self.tail {

415
src/parser/hir/syntax_shape.rs
View File

@ -11,16 +11,12 @@ use crate::parser::hir::expand_external_tokens::ExternalTokensShape;
use crate::parser::hir::syntax_shape::block::AnyBlockShape;
use crate::parser::hir::tokens_iterator::Peeked;
use crate::parser::parse_command::{parse_command_tail, CommandTailShape};
use crate::parser::{
hir,
hir::{debug_tokens, TokensIterator},
Operator, RawToken, TokenNode,
};
use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
use crate::prelude::*;
use derive_new::new;
use getset::Getters;
use log::{self, trace};
use serde::{Deserialize, Serialize};
use std::fmt;
use std::path::{Path, PathBuf};
pub(crate) use self::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
@ -40,15 +36,16 @@ pub(crate) use self::expression::variable_path::{
pub(crate) use self::expression::{continue_expression, AnyExpressionShape};
pub(crate) use self::flat_shape::FlatShape;
#[cfg(not(coloring_in_tokens))]
use crate::parser::hir::tokens_iterator::debug::debug_tokens;
#[cfg(not(coloring_in_tokens))]
use crate::parser::parse::pipeline::Pipeline;
#[cfg(not(coloring_in_tokens))]
use log::log_enabled;
use log::{log_enabled, trace};
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum SyntaxShape {
Any,
List,
String,
Member,
ColumnPath,
@ -75,10 +72,6 @@ impl FallibleColorSyntax for SyntaxShape {
SyntaxShape::Any => {
color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)
}
SyntaxShape::List => {
color_syntax(&ExpressionListShape, token_nodes, context, shapes);
Ok(())
}
SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes),
SyntaxShape::String => color_fallible_syntax_with(
&StringShape,
@ -126,10 +119,6 @@ impl FallibleColorSyntax for SyntaxShape {
) -> Result<(), ShellError> {
match self {
SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context),
SyntaxShape::List => {
color_syntax(&ExpressionListShape, token_nodes, context);
Ok(())
}
SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context),
SyntaxShape::String => {
color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context)
@ -147,14 +136,27 @@ impl FallibleColorSyntax for SyntaxShape {
}
impl ExpandExpression for SyntaxShape {
fn name(&self) -> &'static str {
match self {
SyntaxShape::Any => "any",
SyntaxShape::Int => "integer",
SyntaxShape::String => "string",
SyntaxShape::Member => "column name",
SyntaxShape::ColumnPath => "column path",
SyntaxShape::Number => "number",
SyntaxShape::Path => "file path",
SyntaxShape::Pattern => "glob pattern",
SyntaxShape::Block => "block",
}
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
match self {
SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context),
SyntaxShape::List => Err(ShellError::unimplemented("SyntaxShape:List")),
SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context),
SyntaxShape::String => expand_expr(&StringShape, token_nodes, context),
SyntaxShape::Member => {
@ -162,8 +164,9 @@ impl ExpandExpression for SyntaxShape {
Ok(syntax.to_expr())
}
SyntaxShape::ColumnPath => {
let Tagged { item: members, tag } =
expand_syntax(&ColumnPathShape, token_nodes, context)?;
let column_path = expand_syntax(&ColumnPathShape, token_nodes, context)?;
let Tagged { item: members, tag } = column_path.path();
Ok(hir::Expression::list(
members.into_iter().map(|s| s.to_expr()).collect(),
@ -182,7 +185,6 @@ impl std::fmt::Display for SyntaxShape {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
SyntaxShape::Any => write!(f, "Any"),
SyntaxShape::List => write!(f, "List"),
SyntaxShape::String => write!(f, "String"),
SyntaxShape::Int => write!(f, "Integer"),
SyntaxShape::Member => write!(f, "Member"),
@ -200,8 +202,6 @@ pub struct ExpandContext<'context> {
#[get = "pub(crate)"]
registry: &'context CommandRegistry,
#[get = "pub(crate)"]
span: Span,
#[get = "pub(crate)"]
source: &'context Text,
homedir: Option<PathBuf>,
}
@ -221,7 +221,6 @@ impl<'context> ExpandContext<'context> {
callback(ExpandContext {
registry: &registry,
span: Span::unknown(),
source,
homedir: None,
})
@ -237,11 +236,13 @@ pub trait TestSyntax: std::fmt::Debug + Copy {
}
pub trait ExpandExpression: std::fmt::Debug + Copy {
fn name(&self) -> &'static str;
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError>;
) -> Result<hir::Expression, ParseError>;
}
#[cfg(coloring_in_tokens)]
@ -303,35 +304,49 @@ pub trait ColorSyntax: std::fmt::Debug + Copy {
}
pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy {
type Output: std::fmt::Debug;
type Output: HasFallibleSpan + Clone + std::fmt::Debug + 'static;
fn name(&self) -> &'static str;
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError>;
) -> Result<Self::Output, ParseError>;
}
pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<T::Output, ShellError> {
trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
) -> Result<T::Output, ParseError> {
token_nodes.expand_frame(shape.name(), |token_nodes| {
shape.expand_syntax(token_nodes, context)
})
}
let result = shape.expand_syntax(token_nodes, context);
pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ParseError> {
token_nodes.expand_expr_frame(shape.name(), |token_nodes| {
shape.expand_expr(token_nodes, context)
})
}
match result {
Err(err) => {
trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
Err(err)
}
Ok(result) => {
trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
Ok(result)
}
}
#[cfg(coloring_in_tokens)]
pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> ((), U) {
(
(),
token_nodes.color_frame(shape.name(), |token_nodes| {
shape.color_syntax(&(), token_nodes, context)
}),
)
}
#[cfg(not(coloring_in_tokens))]
@ -363,20 +378,6 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
((), result)
}
#[cfg(coloring_in_tokens)]
pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> ((), U) {
(
(),
token_nodes.color_frame(shape.name(), |token_nodes| {
shape.color_syntax(&(), token_nodes, context)
}),
)
}
#[cfg(not(coloring_in_tokens))]
pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax<Info = U, Input = ()>, U>(
shape: &T,
@ -492,36 +493,18 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax<Info = U, Input
})
}
pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
trace!(target: "nu::expand_expression", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
let result = shape.expand_expr(token_nodes, context);
match result {
Err(err) => {
trace!(target: "nu::expand_expression", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
Err(err)
}
Ok(result) => {
trace!(target: "nu::expand_expression", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
Ok(result)
}
}
}
impl<T: ExpandExpression> ExpandSyntax for T {
type Output = hir::Expression;
fn name(&self) -> &'static str {
ExpandExpression::name(self)
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
ExpandExpression::expand_expr(self, token_nodes, context)
}
}
@ -537,7 +520,7 @@ pub trait SkipSyntax: std::fmt::Debug + Copy {
enum BarePathState {
Initial,
Seen(Span, Span),
Error(ShellError),
Error(ParseError),
}
impl BarePathState {
@ -549,7 +532,7 @@ impl BarePathState {
}
}
pub fn end(self, peeked: Peeked, reason: impl Into<String>) -> BarePathState {
pub fn end(self, peeked: Peeked, reason: &'static str) -> BarePathState {
match self {
BarePathState::Initial => BarePathState::Error(peeked.type_error(reason)),
BarePathState::Seen(start, end) => BarePathState::Seen(start, end),
@ -557,7 +540,7 @@ impl BarePathState {
}
}
pub fn into_bare(self) -> Result<Span, ShellError> {
pub fn into_bare(self) -> Result<Span, ParseError> {
match self {
BarePathState::Initial => unreachable!("into_bare in initial state"),
BarePathState::Seen(start, end) => Ok(start.until(end)),
@ -570,7 +553,7 @@ pub fn expand_bare<'a, 'b>(
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
predicate: impl Fn(&TokenNode) -> bool,
) -> Result<Span, ShellError> {
) -> Result<Span, ParseError> {
let mut state = BarePathState::Initial;
loop {
@ -603,11 +586,15 @@ pub struct BarePathShape;
impl ExpandSyntax for BarePathShape {
type Output = Span;
fn name(&self) -> &'static str {
"shorthand path"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Span, ShellError> {
) -> Result<Span, ParseError> {
expand_bare(token_nodes, context, |token| match token {
TokenNode::Token(Spanned {
item: RawToken::Bare,
@ -638,19 +625,21 @@ impl FallibleColorSyntax for BareShape {
_context: &ExpandContext,
shapes: &mut Vec<Spanned<FlatShape>>,
) -> Result<(), ShellError> {
token_nodes.peek_any_token("word", |token| match token {
// If it's a bare token, color it
TokenNode::Token(Spanned {
item: RawToken::Bare,
span,
}) => {
shapes.push((*input).spanned(*span));
Ok(())
}
token_nodes
.peek_any_token("word", |token| match token {
// If it's a bare token, color it
TokenNode::Token(Spanned {
item: RawToken::Bare,
span,
}) => {
shapes.push((*input).spanned(*span));
Ok(())
}
// otherwise, fail
other => Err(ShellError::type_error("word", other.tagged_type_name())),
})
// otherwise, fail
other => Err(ParseError::mismatch("word", other.tagged_type_name())),
})
.map_err(|err| err.into())
}
}
@ -677,7 +666,7 @@ impl FallibleColorSyntax for BareShape {
}) => Ok(span),
// otherwise, fail
other => Err(ShellError::type_error("word", other.tagged_type_name())),
other => Err(ParseError::mismatch("word", other.tagged_type_name())),
})?;
token_nodes.color_shape((*input).spanned(*span));
@ -689,11 +678,15 @@ impl FallibleColorSyntax for BareShape {
impl ExpandSyntax for BareShape {
type Output = Spanned<String>;
fn name(&self) -> &'static str {
"word"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let peeked = token_nodes.peek_any().not_eof("word")?;
match peeked.node {
@ -705,7 +698,7 @@ impl ExpandSyntax for BareShape {
Ok(span.spanned_string(context.source))
}
other => Err(ShellError::type_error("word", other.tagged_type_name())),
other => Err(ParseError::mismatch("word", other.tagged_type_name())),
}
}
}
@ -725,7 +718,7 @@ impl TestSyntax for BareShape {
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum CommandSignature {
Internal(Spanned<Arc<Command>>),
LiteralExternal { outer: Span, inner: Span },
@ -733,6 +726,34 @@ pub enum CommandSignature {
Expression(hir::Expression),
}
impl FormatDebug for CommandSignature {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
CommandSignature::Internal(internal) => {
f.say_str("internal", internal.span.slice(source))
}
CommandSignature::LiteralExternal { outer, .. } => {
f.say_str("external", outer.slice(source))
}
CommandSignature::External(external) => {
write!(f, "external:{}", external.slice(source))
}
CommandSignature::Expression(expr) => expr.fmt_debug(f, source),
}
}
}
impl HasSpan for CommandSignature {
fn span(&self) -> Span {
match self {
CommandSignature::Internal(spanned) => spanned.span,
CommandSignature::LiteralExternal { outer, .. } => *outer,
CommandSignature::External(span) => *span,
CommandSignature::Expression(expr) => expr.span,
}
}
}
impl CommandSignature {
pub fn to_expression(&self) -> hir::Expression {
match self {
@ -833,12 +854,17 @@ impl FallibleColorSyntax for PipelineShape {
#[cfg(coloring_in_tokens)]
impl ExpandSyntax for PipelineShape {
type Output = ClassifiedPipeline;
fn name(&self) -> &'static str {
"pipeline"
}
fn expand_syntax<'content, 'me>(
&self,
iterator: &'me mut TokensIterator<'content>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
let source = context.source;
) -> Result<Self::Output, ParseError> {
let start = iterator.span_at_cursor();
let peeked = iterator.peek_any().not_eof("pipeline")?;
let pipeline = peeked.commit().as_pipeline()?;
@ -851,25 +877,34 @@ impl ExpandSyntax for PipelineShape {
let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
let classified = iterator.child(tokens, move |token_nodes| {
classify_command(token_nodes, context, &source)
expand_syntax(&ClassifiedCommandShape, token_nodes, context)
})?;
out.push(classified);
}
Ok(ClassifiedPipeline { commands: out })
let end = iterator.span_at_cursor();
Ok(ClassifiedPipeline {
commands: out.spanned(start.until(end)),
})
}
}
#[cfg(not(coloring_in_tokens))]
impl ExpandSyntax for PipelineShape {
type Output = ClassifiedPipeline;
fn name(&self) -> &'static str {
"pipeline"
}
fn expand_syntax<'content, 'me>(
&self,
iterator: &'me mut TokensIterator<'content>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
let source = context.source;
) -> Result<Self::Output, ParseError> {
let start = iterator.span_at_cursor();
let peeked = iterator.peek_any().not_eof("pipeline")?;
let pipeline = peeked.commit().as_pipeline()?;
@ -882,13 +917,17 @@ impl ExpandSyntax for PipelineShape {
let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
let classified = iterator.child(tokens, move |token_nodes| {
classify_command(token_nodes, context, &source)
expand_syntax(&ClassifiedCommandShape, token_nodes, context)
})?;
out.push(classified);
}
Ok(ClassifiedPipeline { commands: out })
let end = iterator.span_at_cursor();
Ok(ClassifiedPipeline {
commands: out.spanned(start.until(end)),
})
}
}
@ -1014,11 +1053,15 @@ impl FallibleColorSyntax for CommandHeadShape {
impl ExpandSyntax for CommandHeadShape {
type Output = CommandSignature;
fn name(&self) -> &'static str {
"command head"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<CommandSignature, ShellError> {
) -> Result<CommandSignature, ParseError> {
let node =
parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_span, _| {
Ok(match token {
@ -1060,29 +1103,34 @@ pub struct ClassifiedCommandShape;
impl ExpandSyntax for ClassifiedCommandShape {
type Output = ClassifiedCommand;
fn name(&self) -> &'static str {
"classified command"
}
fn expand_syntax<'a, 'b>(
&self,
iterator: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let start = iterator.span_at_cursor();
let head = expand_syntax(&CommandHeadShape, iterator, context)?;
match &head {
CommandSignature::Expression(expr) => Err(ShellError::syntax_error(
"Unexpected expression in command position".tagged(expr.span),
)),
CommandSignature::Expression(expr) => {
Err(ParseError::mismatch("command", expr.tagged_type_name()))
}
// If the command starts with `^`, treat it as an external command no matter what
CommandSignature::External(name) => {
let name_str = name.slice(&context.source);
external_command(iterator, &context.source, name_str.tagged(name))
external_command(iterator, context, name_str.tagged(name))
}
CommandSignature::LiteralExternal { outer, inner } => {
let name_str = inner.slice(&context.source);
external_command(iterator, &context.source, name_str.tagged(outer))
external_command(iterator, context, name_str.tagged(outer))
}
CommandSignature::Internal(command) => {
@ -1094,11 +1142,14 @@ impl ExpandSyntax for ClassifiedCommandShape {
Some((positional, named)) => (positional, named),
};
let end = iterator.span_at_cursor();
let call = hir::Call {
head: Box::new(head.to_expression()),
positional,
named,
};
}
.spanned(start.until(end));
Ok(ClassifiedCommand::Internal(InternalCommand::new(
command.item.name().to_string(),
@ -1198,12 +1249,16 @@ impl FallibleColorSyntax for InternalCommandHeadShape {
}
impl ExpandExpression for InternalCommandHeadShape {
fn name(&self) -> &'static str {
"internal command head"
}
fn expand_expr(
&self,
token_nodes: &mut TokensIterator<'_>,
_context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
let peeked_head = token_nodes.peek_non_ws().not_eof("command head4")?;
) -> Result<hir::Expression, ParseError> {
let peeked_head = token_nodes.peek_non_ws().not_eof("command head")?;
let expr = match peeked_head.node {
TokenNode::Token(
@ -1219,8 +1274,8 @@ impl ExpandExpression for InternalCommandHeadShape {
}) => hir::RawExpression::Literal(hir::Literal::String(*inner_span)).spanned(*span),
node => {
return Err(ShellError::type_error(
"command head5",
return Err(ParseError::mismatch(
"command head",
node.tagged_type_name(),
))
}
@ -1238,16 +1293,16 @@ pub(crate) struct SingleError<'token> {
}
impl<'token> SingleError<'token> {
pub(crate) fn error(&self) -> ShellError {
ShellError::type_error(self.expected, self.node.type_name().tagged(self.node.span))
pub(crate) fn error(&self) -> ParseError {
ParseError::mismatch(self.expected, self.node.type_name().tagged(self.node.span))
}
}
fn parse_single_node<'a, 'b, T>(
token_nodes: &'b mut TokensIterator<'a>,
expected: &'static str,
callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ShellError>,
) -> Result<T, ShellError> {
callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ParseError>,
) -> Result<T, ParseError> {
token_nodes.peek_any_token(expected, |node| match node {
TokenNode::Token(token) => callback(
token.item,
@ -1258,7 +1313,7 @@ fn parse_single_node<'a, 'b, T>(
},
),
other => Err(ShellError::type_error(expected, other.tagged_type_name())),
other => Err(ParseError::mismatch(expected, other.tagged_type_name())),
})
}
@ -1360,22 +1415,21 @@ impl FallibleColorSyntax for WhitespaceShape {
impl ExpandSyntax for WhitespaceShape {
type Output = Span;
fn name(&self) -> &'static str {
"whitespace"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let peeked = token_nodes.peek_any().not_eof("whitespace")?;
let span = match peeked.node {
TokenNode::Whitespace(tag) => *tag,
other => {
return Err(ShellError::type_error(
"whitespace",
other.tagged_type_name(),
))
}
other => return Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
};
peeked.commit();
@ -1390,11 +1444,15 @@ pub struct SpacedExpression<T: ExpandExpression> {
}
impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
fn name(&self) -> &'static str {
"spaced expression"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// TODO: Make the name part of the trait
let peeked = token_nodes.peek_any().not_eof("whitespace")?;
@ -1404,10 +1462,7 @@ impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
expand_expr(&self.inner, token_nodes, context)
}
other => Err(ShellError::type_error(
"whitespace",
other.tagged_type_name(),
)),
other => Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
}
}
}
@ -1424,6 +1479,36 @@ pub struct MaybeSpacedExpression<T: ExpandExpression> {
#[derive(Debug, Copy, Clone)]
pub struct MaybeSpaceShape;
impl ExpandSyntax for MaybeSpaceShape {
type Output = Option<Span>;
fn name(&self) -> &'static str {
"maybe space"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
let peeked = token_nodes.peek_any().not_eof("whitespace");
let span = match peeked {
Err(_) => None,
Ok(peeked) => {
if let TokenNode::Whitespace(..) = peeked.node {
let node = peeked.commit();
Some(node.span())
} else {
None
}
}
};
Ok(span)
}
}
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for MaybeSpaceShape {
type Info = ();
@ -1544,11 +1629,15 @@ impl FallibleColorSyntax for SpaceShape {
}
impl<T: ExpandExpression> ExpandExpression for MaybeSpacedExpression<T> {
fn name(&self) -> &'static str {
"maybe space"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// TODO: Make the name part of the trait
let peeked = token_nodes.peek_any().not_eof("whitespace")?;
@ -1578,58 +1667,6 @@ fn expand_variable(span: Span, token_span: Span, source: &Text) -> hir::Expressi
}
}
fn classify_command(
mut iterator: &mut TokensIterator,
context: &ExpandContext,
source: &Text,
) -> Result<ClassifiedCommand, ShellError> {
let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?;
match &head {
CommandSignature::Expression(_) => Err(ShellError::syntax_error(
"Unexpected expression in command position".tagged(iterator.whole_span()),
)),
// If the command starts with `^`, treat it as an external command no matter what
CommandSignature::External(name) => {
let name_str = name.slice(source);
external_command(&mut iterator, source, name_str.tagged(name))
}
CommandSignature::LiteralExternal { outer, inner } => {
let name_str = inner.slice(source);
external_command(&mut iterator, source, name_str.tagged(outer))
}
CommandSignature::Internal(command) => {
let tail =
parse_command_tail(&command.signature(), &context, &mut iterator, command.span)?;
let (positional, named) = match tail {
None => (None, None),
Some((positional, named)) => (positional, named),
};
let call = hir::Call {
head: Box::new(head.to_expression()),
positional,
named,
};
Ok(ClassifiedCommand::Internal(InternalCommand::new(
command.name().to_string(),
Tag {
span: command.span,
anchor: None,
},
call,
)))
}
}
}
#[derive(Debug, Copy, Clone)]
pub struct CommandShape;

39
src/parser/hir/syntax_shape/block.rs
View File

@ -6,7 +6,8 @@ use crate::parser::{
hir::syntax_shape::{
color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax,
DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape,
ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, VariablePathShape,
ExpressionListShape, FallibleColorSyntax, MemberShape, ParseError, PathTailShape,
VariablePathShape,
},
hir::tokens_iterator::TokensIterator,
parse::token_tree::Delimiter,
@ -42,7 +43,7 @@ impl FallibleColorSyntax for AnyBlockShape {
match block {
// If so, color it as a block
Some((children, spans)) => {
let mut token_nodes = TokensIterator::new(children.item, context.span, false);
let mut token_nodes = TokensIterator::new(children.item, children.span, false);
color_syntax_with(
&DelimitedShape,
&(Delimiter::Brace, spans.0, spans.1),
@ -109,11 +110,15 @@ impl FallibleColorSyntax for AnyBlockShape {
}
impl ExpandExpression for AnyBlockShape {
fn name(&self) -> &'static str {
"any block"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let block = token_nodes.peek_non_ws().not_eof("block")?;
// is it just a block?
@ -121,11 +126,11 @@ impl ExpandExpression for AnyBlockShape {
match block {
Some((block, _tags)) => {
let mut iterator = TokensIterator::new(&block.item, context.span, false);
let mut iterator = TokensIterator::new(&block.item, block.span, false);
let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?;
return Ok(hir::RawExpression::Block(exprs).spanned(block.span));
return Ok(hir::RawExpression::Block(exprs.item).spanned(block.span));
}
_ => {}
}
@ -204,14 +209,18 @@ impl FallibleColorSyntax for ShorthandBlock {
}
impl ExpandExpression for ShorthandBlock {
fn name(&self) -> &'static str {
"shorthand block"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let path = expand_expr(&ShorthandPath, token_nodes, context)?;
let start = path.span;
let expr = continue_expression(path, token_nodes, context)?;
let expr = continue_expression(path, token_nodes, context);
let end = expr.span;
let block = hir::RawExpression::Block(vec![expr]).spanned(start.until(end));
@ -317,11 +326,15 @@ impl FallibleColorSyntax for ShorthandPath {
}
impl ExpandExpression for ShorthandPath {
fn name(&self) -> &'static str {
"shorthand path"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// if it's a variable path, that's the head part
let path = expand_expr(&VariablePathShape, token_nodes, context);
@ -339,7 +352,7 @@ impl ExpandExpression for ShorthandPath {
match tail {
Err(_) => return Ok(head),
Ok((tail, _)) => {
Ok(Spanned { item: tail, .. }) => {
// For each member that `PathTailShape` expanded, join it onto the existing expression
// to form a new path
for member in tail {
@ -446,11 +459,15 @@ impl FallibleColorSyntax for ShorthandHeadShape {
}
impl ExpandExpression for ShorthandHeadShape {
fn name(&self) -> &'static str {
"shorthand head"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// A shorthand path must not be at EOF
let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?;
@ -495,7 +512,7 @@ impl ExpandExpression for ShorthandHeadShape {
// Any other token is not a valid bare head
other => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
"shorthand path",
other.tagged_type_name(),
))

26
src/parser/hir/syntax_shape/expression.rs
View File

@ -12,7 +12,7 @@ use crate::parser::hir::syntax_shape::{
color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom,
expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape,
DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation,
ExpressionContinuationShape, FallibleColorSyntax, FlatShape,
ExpressionContinuationShape, FallibleColorSyntax, FlatShape, ParseError,
};
use crate::parser::{
hir,
@ -25,15 +25,19 @@ use std::path::PathBuf;
pub struct AnyExpressionShape;
impl ExpandExpression for AnyExpressionShape {
fn name(&self) -> &'static str {
"any expression"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// Look for an expression at the cursor
let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?;
continue_expression(head, token_nodes, context)
Ok(continue_expression(head, token_nodes, context))
}
}
@ -98,14 +102,14 @@ pub(crate) fn continue_expression(
mut head: hir::Expression,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> hir::Expression {
loop {
// Check to see whether there's any continuation after the head expression
let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context);
match continuation {
// If there's no continuation, return the head
Err(_) => return Ok(head),
Err(_) => return head,
// Otherwise, form a new expression by combining the head with the continuation
Ok(continuation) => match continuation {
// If the continuation is a `.member`, form a path with the new member
@ -174,11 +178,15 @@ pub(crate) fn continue_coloring_expression(
pub struct AnyExpressionStartShape;
impl ExpandExpression for AnyExpressionStartShape {
fn name(&self) -> &'static str {
"any expression start"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?;
match atom.item {
@ -445,13 +453,17 @@ impl FallibleColorSyntax for BareTailShape {
}
impl ExpandSyntax for BareTailShape {
fn name(&self) -> &'static str {
"word continuation"
}
type Output = Option<Span>;
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Option<Span>, ShellError> {
) -> Result<Option<Span>, ParseError> {
let mut end: Option<Span> = None;
loop {

48
src/parser/hir/syntax_shape/expression/atom.rs
View File

@ -90,40 +90,40 @@ impl<'tokens> SpannedAtomicToken<'tokens> {
&self,
context: &ExpandContext,
expected: &'static str,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
Ok(match &self.item {
AtomicToken::Eof { .. } => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
"eof atomic token".tagged(self.span),
))
}
AtomicToken::Error { .. } => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
"eof atomic token".tagged(self.span),
))
}
AtomicToken::Operator { .. } => {
return Err(ShellError::type_error(
expected,
"operator".tagged(self.span),
))
return Err(ParseError::mismatch(expected, "operator".tagged(self.span)))
}
AtomicToken::ShorthandFlag { .. } => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
"shorthand flag".tagged(self.span),
))
}
AtomicToken::LonghandFlag { .. } => {
return Err(ShellError::type_error(expected, "flag".tagged(self.span)))
return Err(ParseError::mismatch(expected, "flag".tagged(self.span)))
}
AtomicToken::Whitespace { .. } => {
return Err(ShellError::unimplemented("whitespace in AtomicToken"))
return Err(ParseError::mismatch(
expected,
"whitespace".tagged(self.span),
))
}
AtomicToken::Dot { .. } => {
return Err(ShellError::type_error(expected, "dot".tagged(self.span)))
return Err(ParseError::mismatch(expected, "dot".tagged(self.span)))
}
AtomicToken::Number { number } => {
Expression::number(number.to_number(context.source), self.span)
@ -381,7 +381,7 @@ pub fn expand_atom<'me, 'content>(
expected: &'static str,
context: &ExpandContext,
rule: ExpansionRule,
) -> Result<SpannedAtomicToken<'content>, ShellError> {
) -> Result<SpannedAtomicToken<'content>, ParseError> {
if token_nodes.at_end() {
match rule.allow_eof {
true => {
@ -390,7 +390,7 @@ pub fn expand_atom<'me, 'content>(
}
.spanned(Span::unknown()))
}
false => return Err(ShellError::unexpected_eof("anything", Tag::unknown())),
false => return Err(ParseError::unexpected_eof("anything", Span::unknown())),
}
}
@ -515,12 +515,13 @@ pub fn expand_atom<'me, 'content>(
// if whitespace is disallowed, return an error
WhitespaceHandling::RejectWhitespace => {
return Err(ShellError::syntax_error("Unexpected whitespace".tagged(
Tag {
return Err(ParseError::mismatch(
expected,
"whitespace".tagged(Tag {
span: *span,
anchor: None,
},
)))
}),
))
}
},
@ -544,7 +545,7 @@ pub fn expand_atom<'me, 'content>(
RawToken::Operator(_) if !rule.allow_operator => return Err(err.error()),
// rule.allow_external_command
RawToken::ExternalCommand(_) if !rule.allow_external_command => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
token.type_name().tagged(Tag {
span: token_span,
@ -554,10 +555,13 @@ pub fn expand_atom<'me, 'content>(
}
// rule.allow_external_word
RawToken::ExternalWord if !rule.allow_external_word => {
return Err(ShellError::invalid_external_word(Tag {
span: token_span,
anchor: None,
}))
return Err(ParseError::mismatch(
expected,
"external word".tagged(Tag {
span: token_span,
anchor: None,
}),
))
}
RawToken::Number(number) => AtomicToken::Number { number }.spanned(token_span),

7
src/parser/hir/syntax_shape/expression/delimited.rs
View File

@ -8,12 +8,15 @@ pub fn expand_delimited_square(
children: &Vec<TokenNode>,
span: Span,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let mut tokens = TokensIterator::new(&children, span, false);
let list = expand_syntax(&ExpressionListShape, &mut tokens, context);
Ok(hir::Expression::list(list?, Tag { span, anchor: None }))
Ok(hir::Expression::list(
list?.item,
Tag { span, anchor: None },
))
}
#[cfg(not(coloring_in_tokens))]

7
src/parser/hir/syntax_shape/expression/file_path.rs
View File

@ -1,6 +1,7 @@
use crate::parser::hir::syntax_shape::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
use crate::parser::hir::syntax_shape::{
expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape,
ParseError,
};
use crate::parser::{hir, hir::TokensIterator};
use crate::prelude::*;
@ -90,11 +91,15 @@ impl FallibleColorSyntax for FilePathShape {
}
impl ExpandExpression for FilePathShape {
fn name(&self) -> &'static str {
"file path"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let atom = expand_atom(token_nodes, "file path", context, ExpansionRule::new())?;
match atom.item {

27
src/parser/hir/syntax_shape/expression/list.rs
View File

@ -1,4 +1,4 @@
use crate::errors::ShellError;
use crate::errors::ParseError;
#[cfg(not(coloring_in_tokens))]
use crate::parser::hir::syntax_shape::FlatShape;
use crate::parser::{
@ -10,24 +10,36 @@ use crate::parser::{
},
hir::TokensIterator,
};
#[cfg(not(coloring_in_tokens))]
use crate::Spanned;
use crate::{DebugFormatter, FormatDebug, Spanned, SpannedItem};
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct ExpressionListShape;
impl FormatDebug for Spanned<Vec<hir::Expression>> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
FormatDebug::fmt_debug(&self.item, f, source)
}
}
impl ExpandSyntax for ExpressionListShape {
type Output = Vec<hir::Expression>;
type Output = Spanned<Vec<hir::Expression>>;
fn name(&self) -> &'static str {
"expression list"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<Vec<hir::Expression>, ShellError> {
) -> Result<Spanned<Vec<hir::Expression>>, ParseError> {
let mut exprs = vec![];
let start = token_nodes.span_at_cursor();
if token_nodes.at_end_possible_ws() {
return Ok(exprs);
return Ok(exprs.spanned(start));
}
let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?;
@ -36,7 +48,8 @@ impl ExpandSyntax for ExpressionListShape {
loop {
if token_nodes.at_end_possible_ws() {
return Ok(exprs);
let end = token_nodes.span_at_cursor();
return Ok(exprs.spanned(start.until(end)));
}
let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?;

30
src/parser/hir/syntax_shape/expression/number.rs
View File

@ -1,6 +1,6 @@
use crate::parser::hir::syntax_shape::{
expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule,
FallibleColorSyntax, FlatShape,
FallibleColorSyntax, FlatShape, ParseError,
};
use crate::parser::{
hir,
@ -13,11 +13,15 @@ use crate::prelude::*;
pub struct NumberShape;
impl ExpandExpression for NumberShape {
fn name(&self) -> &'static str {
"number"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "Number", |token, token_span, err| {
Ok(match token {
RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
@ -28,10 +32,13 @@ impl ExpandExpression for NumberShape {
hir::Expression::external_command(tag, token_span)
}
RawToken::ExternalWord => {
return Err(ShellError::invalid_external_word(Tag {
span: token_span,
anchor: None,
}))
return Err(ParseError::mismatch(
"number",
"syntax error".tagged(Tag {
span: token_span,
anchor: None,
}),
))
}
RawToken::Variable(tag) => hir::Expression::variable(tag, token_span),
RawToken::Number(number) => {
@ -111,16 +118,19 @@ impl FallibleColorSyntax for NumberShape {
pub struct IntShape;
impl ExpandExpression for IntShape {
fn name(&self) -> &'static str {
"integer"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "Integer", |token, token_span, err| {
Ok(match token {
RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
RawToken::ExternalWord => {
return Err(ShellError::invalid_external_word(token_span))
RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
return Err(err.error())
}
RawToken::Variable(span) if span.slice(context.source) == "it" => {
hir::Expression::it_variable(span, token_span)

14
src/parser/hir/syntax_shape/expression/pattern.rs
View File

@ -1,6 +1,6 @@
use crate::parser::hir::syntax_shape::{
expand_atom, expand_bare, expression::expand_file_path, AtomicToken, ExpandContext,
ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape,
ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError,
};
use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
use crate::prelude::*;
@ -66,11 +66,15 @@ impl FallibleColorSyntax for PatternShape {
}
impl ExpandExpression for PatternShape {
fn name(&self) -> &'static str {
"glob pattern"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::new())?;
match atom.item {
@ -91,11 +95,15 @@ pub struct BarePatternShape;
impl ExpandSyntax for BarePatternShape {
type Output = Span;
fn name(&self) -> &'static str {
"bare pattern"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Span, ShellError> {
) -> Result<Span, ParseError> {
expand_bare(token_nodes, context, |token| match token {
TokenNode::Token(Spanned {
item: RawToken::Bare,

26
src/parser/hir/syntax_shape/expression/string.rs
View File

@ -1,6 +1,6 @@
use crate::parser::hir::syntax_shape::{
expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression,
ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax,
ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, TestSyntax,
};
use crate::parser::hir::tokens_iterator::Peeked;
use crate::parser::{hir, hir::TokensIterator, RawToken};
@ -75,32 +75,24 @@ impl FallibleColorSyntax for StringShape {
}
impl ExpandExpression for StringShape {
fn name(&self) -> &'static str {
"string"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
parse_single_node(token_nodes, "String", |token, token_span, _| {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "String", |token, token_span, err| {
Ok(match token {
RawToken::GlobPattern => {
return Err(ShellError::type_error(
"String",
"glob pattern".tagged(token_span),
))
}
RawToken::Operator(..) => {
return Err(ShellError::type_error(
"String",
"operator".tagged(token_span),
))
RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
return Err(err.error())
}
RawToken::Variable(span) => expand_variable(span, token_span, &context.source),
RawToken::ExternalCommand(span) => {
hir::Expression::external_command(span, token_span)
}
RawToken::ExternalWord => {
return Err(ShellError::invalid_external_word(token_span))
}
RawToken::Number(_) => hir::Expression::bare(token_span),
RawToken::Bare => hir::Expression::bare(token_span),
RawToken::String(span) => hir::Expression::string(span, token_span),

27
src/parser/hir/syntax_shape/expression/unit.rs
View File

@ -1,5 +1,5 @@
use crate::data::meta::Span;
use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax};
use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax, ParseError};
use crate::parser::parse::tokens::RawNumber;
use crate::parser::parse::unit::Unit;
use crate::parser::{hir::TokensIterator, RawToken, TokenNode};
@ -9,18 +9,34 @@ use nom::bytes::complete::tag;
use nom::character::complete::digit1;
use nom::combinator::{all_consuming, opt, value};
use nom::IResult;
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct UnitShape;
impl FormatDebug for Spanned<(Spanned<RawNumber>, Spanned<Unit>)> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
let dict = indexmap::indexmap! {
"number" => format!("{}", self.item.0.item.debug(source)),
"unit" => format!("{}", self.item.1.debug(source)),
};
f.say_dict("unit", dict)
}
}
impl ExpandSyntax for UnitShape {
type Output = Spanned<(Spanned<RawNumber>, Spanned<Unit>)>;
fn name(&self) -> &'static str {
"unit"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ShellError> {
) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ParseError> {
let peeked = token_nodes.peek_any().not_eof("unit")?;
let span = match peeked.node {
@ -34,12 +50,7 @@ impl ExpandSyntax for UnitShape {
let unit = unit_size(span.slice(context.source), *span);
let (_, (number, unit)) = match unit {
Err(_) => {
return Err(ShellError::type_error(
"unit",
"word".tagged(Tag::unknown()),
))
}
Err(_) => return Err(ParseError::mismatch("unit", "word".tagged(Tag::unknown()))),
Ok((number, unit)) => (number, unit),
};

246
src/parser/hir/syntax_shape/expression/variable_path.rs
View File

@ -1,21 +1,28 @@
use crate::parser::hir::syntax_shape::{
color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax,
parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression,
ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape,
TestSyntax, WhitespaceShape,
ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, Peeked, SkipSyntax,
StringShape, TestSyntax, WhitespaceShape,
};
use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken};
use crate::prelude::*;
use derive_new::new;
use getset::Getters;
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct VariablePathShape;
impl ExpandExpression for VariablePathShape {
fn name(&self) -> &'static str {
"variable path"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// 1. let the head be the first token, expecting a variable
// 2. let the tail be an empty list of members
// 2. while the next token (excluding ws) is a dot:
@ -200,12 +207,17 @@ impl FallibleColorSyntax for PathTailShape {
}
impl ExpandSyntax for PathTailShape {
type Output = (Vec<Spanned<String>>, Span);
type Output = Spanned<Vec<Spanned<String>>>;
fn name(&self) -> &'static str {
"path continuation"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let mut end: Option<Span> = None;
let mut tail = vec![];
@ -223,7 +235,7 @@ impl ExpandSyntax for PathTailShape {
match end {
None => {
return Err(ShellError::type_error("path tail", {
return Err(ParseError::mismatch("path tail", {
let typed_span = token_nodes.typed_span_at_cursor();
Tagged {
@ -233,17 +245,41 @@ impl ExpandSyntax for PathTailShape {
}))
}
Some(end) => Ok((tail, end)),
Some(end) => Ok(tail.spanned(end)),
}
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum ExpressionContinuation {
DotSuffix(Span, Spanned<String>),
InfixSuffix(Spanned<Operator>, Expression),
}
impl FormatDebug for ExpressionContinuation {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
ExpressionContinuation::DotSuffix(dot, rest) => {
f.say_str("dot suffix", dot.until(rest.span).slice(source))
}
ExpressionContinuation::InfixSuffix(operator, expr) => {
f.say_str("infix suffix", operator.span.until(expr.span).slice(source))
}
}
}
}
impl HasSpan for ExpressionContinuation {
fn span(&self) -> Span {
match self {
ExpressionContinuation::DotSuffix(dot, column) => dot.until(column.span),
ExpressionContinuation::InfixSuffix(operator, expression) => {
operator.span.until(expression.span)
}
}
}
}
/// An expression continuation
#[derive(Debug, Copy, Clone)]
pub struct ExpressionContinuationShape;
@ -251,11 +287,15 @@ pub struct ExpressionContinuationShape;
impl ExpandSyntax for ExpressionContinuationShape {
type Output = ExpressionContinuation;
fn name(&self) -> &'static str {
"expression continuation"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<ExpressionContinuation, ShellError> {
) -> Result<ExpressionContinuation, ParseError> {
// Try to expand a `.`
let dot = expand_syntax(&DotShape, token_nodes, context);
@ -270,7 +310,7 @@ impl ExpandSyntax for ExpressionContinuationShape {
// Otherwise, we expect an infix operator and an expression next
Err(_) => {
let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?;
let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?.item;
let next = expand_expr(&AnyExpressionShape, token_nodes, context)?;
Ok(ExpressionContinuation::InfixSuffix(op, next))
@ -390,12 +430,16 @@ impl FallibleColorSyntax for ExpressionContinuationShape {
pub struct VariableShape;
impl ExpandExpression for VariableShape {
fn name(&self) -> &'static str {
"variable"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
parse_single_node(token_nodes, "variable", |token, token_tag, _| {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "variable", |token, token_tag, err| {
Ok(match token {
RawToken::Variable(tag) => {
if tag.slice(context.source) == "it" {
@ -404,12 +448,7 @@ impl ExpandExpression for VariableShape {
hir::Expression::variable(tag, token_tag)
}
}
_ => {
return Err(ShellError::type_error(
"variable",
token.type_name().tagged(token_tag),
))
}
_ => return Err(err.error()),
})
})
}
@ -435,7 +474,7 @@ impl FallibleColorSyntax for VariableShape {
);
let atom = match atom {
Err(err) => return Err(err),
Err(err) => return Err(err.into()),
Ok(atom) => atom,
};
@ -476,7 +515,7 @@ impl FallibleColorSyntax for VariableShape {
);
let atom = match atom {
Err(err) => return Err(err),
Err(err) => return Err(err.into()),
Ok(atom) => atom,
};
@ -489,7 +528,7 @@ impl FallibleColorSyntax for VariableShape {
token_nodes.color_shape(FlatShape::ItVariable.spanned(atom.span));
Ok(())
}
_ => Err(ShellError::type_error("variable", atom.tagged_type_name())),
_ => Err(ParseError::mismatch("variable", atom.tagged_type_name()).into()),
}
}
}
@ -500,6 +539,24 @@ pub enum Member {
Bare(Span),
}
impl FormatDebug for Member {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Member::String(outer, _) => write!(f, "member ({})", outer.slice(source)),
Member::Bare(bare) => write!(f, "member ({})", bare.slice(source)),
}
}
}
impl HasSpan for Member {
fn span(&self) -> Span {
match self {
Member::String(outer, ..) => *outer,
Member::Bare(name) => *name,
}
}
}
impl Member {
pub(crate) fn to_expr(&self) -> hir::Expression {
match self {
@ -538,7 +595,7 @@ enum ColumnPathState {
LeadingDot(Span),
Dot(Span, Vec<Member>, Span),
Member(Span, Vec<Member>),
Error(ShellError),
Error(ParseError),
}
impl ColumnPathState {
@ -546,10 +603,10 @@ impl ColumnPathState {
match self {
ColumnPathState::Initial => ColumnPathState::LeadingDot(dot),
ColumnPathState::LeadingDot(_) => {
ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Dot(..) => {
ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot),
ColumnPathState::Error(err) => ColumnPathState::Error(err),
@ -570,20 +627,20 @@ impl ColumnPathState {
})
}
ColumnPathState::Member(..) => {
ColumnPathState::Error(ShellError::type_error("column", member.tagged_type_name()))
ColumnPathState::Error(ParseError::mismatch("column", member.tagged_type_name()))
}
ColumnPathState::Error(err) => ColumnPathState::Error(err),
}
}
pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ShellError> {
pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ParseError> {
match self {
ColumnPathState::Initial => Err(next.type_error("column path")),
ColumnPathState::LeadingDot(dot) => {
Err(ShellError::type_error("column", "dot".tagged(dot)))
Err(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Dot(_tag, _members, dot) => {
Err(ShellError::type_error("column", "dot".tagged(dot)))
Err(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)),
ColumnPathState::Error(err) => Err(err),
@ -594,7 +651,7 @@ impl ColumnPathState {
pub fn expand_column_path<'a, 'b>(
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Tagged<Vec<Member>>, ShellError> {
) -> Result<Tagged<Vec<Member>>, ParseError> {
let mut state = ColumnPathState::Initial;
loop {
@ -720,15 +777,43 @@ impl FallibleColorSyntax for ColumnPathShape {
}
}
impl FormatDebug for Tagged<Vec<Member>> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.item.fmt_debug(f, source)
}
}
#[derive(Debug, Clone, Getters, new)]
pub struct ColumnPath {
#[get = "pub"]
path: Tagged<Vec<Member>>,
}
impl HasSpan for ColumnPath {
fn span(&self) -> Span {
self.path.tag.span
}
}
impl FormatDebug for ColumnPath {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say("column path", self.path.item.debug(source))
}
}
impl ExpandSyntax for ColumnPathShape {
type Output = Tagged<Vec<Member>>;
type Output = ColumnPath;
fn name(&self) -> &'static str {
"column path"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
expand_column_path(token_nodes, context)
) -> Result<Self::Output, ParseError> {
Ok(ColumnPath::new(expand_column_path(token_nodes, context)?))
}
}
@ -806,11 +891,15 @@ impl FallibleColorSyntax for MemberShape {
impl ExpandSyntax for MemberShape {
type Output = Member;
fn name(&self) -> &'static str {
"column"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<Member, ShellError> {
) -> Result<Member, ParseError> {
let bare = BareShape.test(token_nodes, context);
if let Some(peeked) = bare {
let node = peeked.not_eof("column")?.commit();
@ -906,16 +995,20 @@ impl SkipSyntax for DotShape {
impl ExpandSyntax for DotShape {
type Output = Span;
fn name(&self) -> &'static str {
"dot"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
parse_single_node(token_nodes, "dot", |token, token_span, _| {
Ok(match token {
RawToken::Operator(Operator::Dot) => token_span,
_ => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
"dot",
token.type_name().tagged(token_span),
))
@ -950,7 +1043,7 @@ impl FallibleColorSyntax for InfixShape {
parse_single_node(
checkpoint.iterator,
"infix operator",
|token, token_span, _| {
|token, token_span, err| {
match token {
// If it's an operator (and not `.`), it's a match
RawToken::Operator(operator) if operator != Operator::Dot => {
@ -959,10 +1052,7 @@ impl FallibleColorSyntax for InfixShape {
}
// Otherwise, it's not a match
_ => Err(ShellError::type_error(
"infix operator",
token.type_name().tagged(token_span),
)),
_ => Err(err.error()),
}
},
)?;
@ -1006,7 +1096,7 @@ impl FallibleColorSyntax for InfixShape {
RawToken::Operator(operator) if operator != Operator::Dot => Ok(token_span),
// Otherwise, it's not a match
_ => Err(ShellError::type_error(
_ => Err(ParseError::mismatch(
"infix operator",
token.type_name().tagged(token_span),
)),
@ -1026,46 +1116,72 @@ impl FallibleColorSyntax for InfixShape {
}
}
impl FormatDebug for Spanned<(Span, Spanned<Operator>, Span)> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str("operator", self.item.1.span.slice(source))
}
}
impl ExpandSyntax for InfixShape {
type Output = (Span, Spanned<Operator>, Span);
type Output = Spanned<(Span, Spanned<Operator>, Span)>;
fn name(&self) -> &'static str {
"infix operator"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
let checkpoint = token_nodes.checkpoint();
) -> Result<Self::Output, ParseError> {
let mut checkpoint = token_nodes.checkpoint();
// An infix operator must be prefixed by whitespace
let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
// Parse the next TokenNode after the whitespace
let operator = parse_single_node(
checkpoint.iterator,
"infix operator",
|token, token_span, _| {
Ok(match token {
// If it's an operator (and not `.`), it's a match
RawToken::Operator(operator) if operator != Operator::Dot => {
operator.spanned(token_span)
}
// Otherwise, it's not a match
_ => {
return Err(ShellError::type_error(
"infix operator",
token.type_name().tagged(token_span),
))
}
})
},
)?;
let operator = expand_syntax(&InfixInnerShape, &mut checkpoint.iterator, context)?;
// An infix operator must be followed by whitespace
let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
checkpoint.commit();
Ok((start, operator, end))
Ok((start, operator, end).spanned(start.until(end)))
}
}
#[derive(Debug, Copy, Clone)]
pub struct InfixInnerShape;
impl FormatDebug for Spanned<Operator> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str("operator", self.span.slice(source))
}
}
impl ExpandSyntax for InfixInnerShape {
type Output = Spanned<Operator>;
fn name(&self) -> &'static str {
"infix inner"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
parse_single_node(token_nodes, "infix operator", |token, token_span, err| {
Ok(match token {
// If it's an operator (and not `.`), it's a match
RawToken::Operator(operator) if operator != Operator::Dot => {
operator.spanned(token_span)
}
// Otherwise, it's not a match
_ => return Err(err.error()),
})
})
}
}

2
src/parser/hir/syntax_shape/flat_shape.rs
View File

@ -1,5 +1,5 @@
use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode};
use crate::{Span, Spanned, SpannedItem, Text};
use crate::{HasSpan, Span, Spanned, SpannedItem, Text};
#[derive(Debug, Copy, Clone)]
pub enum FlatShape {

219
src/parser/hir/tokens_iterator.rs
View File

@ -1,25 +1,38 @@
pub(crate) mod debug;
use self::debug::Tracer;
use self::debug::{ColorTracer, ExpandTracer};
use crate::errors::ShellError;
#[cfg(coloring_in_tokens)]
use crate::parser::hir::syntax_shape::FlatShape;
use crate::parser::hir::Expression;
use crate::parser::TokenNode;
use crate::prelude::*;
use crate::{Span, Spanned, SpannedItem};
#[allow(unused)]
use getset::{Getters, MutGetters};
#[derive(Getters, Debug)]
pub struct TokensIteratorState<'content> {
tokens: &'content [TokenNode],
span: Span,
skip_ws: bool,
index: usize,
seen: indexmap::IndexSet<usize>,
#[cfg(coloring_in_tokens)]
#[cfg_attr(coloring_in_tokens, get = "pub")]
shapes: Vec<Spanned<FlatShape>>,
cfg_if::cfg_if! {
if #[cfg(coloring_in_tokens)] {
#[derive(Getters, Debug)]
pub struct TokensIteratorState<'content> {
tokens: &'content [TokenNode],
span: Span,
skip_ws: bool,
index: usize,
seen: indexmap::IndexSet<usize>,
#[get = "pub"]
shapes: Vec<Spanned<FlatShape>>,
}
} else {
#[derive(Getters, Debug)]
pub struct TokensIteratorState<'content> {
tokens: &'content [TokenNode],
span: Span,
skip_ws: bool,
index: usize,
seen: indexmap::IndexSet<usize>,
}
}
}
#[derive(Getters, MutGetters, Debug)]
@ -29,7 +42,10 @@ pub struct TokensIterator<'content> {
state: TokensIteratorState<'content>,
#[get = "pub"]
#[get_mut = "pub"]
tracer: Tracer,
color_tracer: ColorTracer,
#[get = "pub"]
#[get_mut = "pub"]
expand_tracer: ExpandTracer,
}
#[derive(Debug)]
@ -83,12 +99,9 @@ impl<'content, 'me> Peeked<'content, 'me> {
Some(node)
}
pub fn not_eof(
self,
expected: impl Into<String>,
) -> Result<PeekedNode<'content, 'me>, ShellError> {
pub fn not_eof(self, expected: &'static str) -> Result<PeekedNode<'content, 'me>, ParseError> {
match self.node {
None => Err(ShellError::unexpected_eof(
None => Err(ParseError::unexpected_eof(
expected,
self.iterator.eof_span(),
)),
@ -101,7 +114,7 @@ impl<'content, 'me> Peeked<'content, 'me> {
}
}
pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
pub fn type_error(&self, expected: &'static str) -> ParseError {
peek_error(&self.node, self.iterator.eof_span(), expected)
}
}
@ -129,19 +142,15 @@ impl<'content, 'me> PeekedNode<'content, 'me> {
pub fn rollback(self) {}
pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
pub fn type_error(&self, expected: &'static str) -> ParseError {
peek_error(&Some(self.node), self.iterator.eof_span(), expected)
}
}
pub fn peek_error(
node: &Option<&TokenNode>,
eof_span: Span,
expected: impl Into<String>,
) -> ShellError {
pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError {
match node {
None => ShellError::unexpected_eof(expected, eof_span),
Some(node) => ShellError::type_error(expected, node.tagged_type_name()),
None => ParseError::unexpected_eof(expected, eof_span),
Some(node) => ParseError::mismatch(expected, node.tagged_type_name()),
}
}
@ -161,7 +170,8 @@ impl<'content> TokensIterator<'content> {
#[cfg(coloring_in_tokens)]
shapes: vec![],
},
tracer: Tracer::new(),
color_tracer: ColorTracer::new(),
expand_tracer: ExpandTracer::new(),
}
}
@ -188,7 +198,7 @@ impl<'content> TokensIterator<'content> {
#[cfg(coloring_in_tokens)]
pub fn color_shape(&mut self, shape: Spanned<FlatShape>) {
self.with_tracer(|_, tracer| tracer.add_shape(shape));
self.with_color_tracer(|_, tracer| tracer.add_shape(shape));
self.state.shapes.push(shape);
}
@ -201,7 +211,7 @@ impl<'content> TokensIterator<'content> {
(len..(shapes.len())).map(|i| shapes[i]).collect()
};
self.with_tracer(|_, tracer| {
self.with_color_tracer(|_, tracer| {
for shape in new_shapes {
tracer.add_shape(shape)
}
@ -233,8 +243,11 @@ impl<'content> TokensIterator<'content> {
let mut shapes = vec![];
std::mem::swap(&mut shapes, &mut self.state.shapes);
let mut tracer = Tracer::new();
std::mem::swap(&mut tracer, &mut self.tracer);
let mut color_tracer = ColorTracer::new();
std::mem::swap(&mut color_tracer, &mut self.color_tracer);
let mut expand_tracer = ExpandTracer::new();
std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
let mut iterator = TokensIterator {
state: TokensIteratorState {
@ -245,13 +258,15 @@ impl<'content> TokensIterator<'content> {
seen: indexmap::IndexSet::new(),
shapes,
},
tracer,
color_tracer,
expand_tracer,
};
let result = block(&mut iterator);
std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes);
std::mem::swap(&mut iterator.tracer, &mut self.tracer);
std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
result
}
@ -262,8 +277,11 @@ impl<'content> TokensIterator<'content> {
tokens: Spanned<&'me [TokenNode]>,
block: impl FnOnce(&mut TokensIterator<'me>) -> T,
) -> T {
let mut tracer = Tracer::new();
std::mem::swap(&mut tracer, &mut self.tracer);
let mut color_tracer = ColorTracer::new();
std::mem::swap(&mut color_tracer, &mut self.color_tracer);
let mut expand_tracer = ExpandTracer::new();
std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
let mut iterator = TokensIterator {
state: TokensIteratorState {
@ -273,19 +291,34 @@ impl<'content> TokensIterator<'content> {
index: 0,
seen: indexmap::IndexSet::new(),
},
tracer,
color_tracer,
expand_tracer,
};
let result = block(&mut iterator);
std::mem::swap(&mut iterator.tracer, &mut self.tracer);
std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
result
}
pub fn with_tracer(&mut self, block: impl FnOnce(&mut TokensIteratorState, &mut Tracer)) {
pub fn with_color_tracer(
&mut self,
block: impl FnOnce(&mut TokensIteratorState, &mut ColorTracer),
) {
let state = &mut self.state;
let tracer = &mut self.tracer;
let color_tracer = &mut self.color_tracer;
block(state, color_tracer)
}
pub fn with_expand_tracer(
&mut self,
block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer),
) {
let state = &mut self.state;
let tracer = &mut self.expand_tracer;
block(state, tracer)
}
@ -296,32 +329,77 @@ impl<'content> TokensIterator<'content> {
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> T,
) -> T {
self.with_tracer(|_, tracer| tracer.start(desc));
self.with_color_tracer(|_, tracer| tracer.start(desc));
let result = block(self);
self.with_tracer(|_, tracer| {
self.with_color_tracer(|_, tracer| {
tracer.success();
});
result
}
pub fn expand_frame<T>(
&mut self,
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> Result<T, ParseError>,
) -> Result<T, ParseError>
where
T: std::fmt::Debug + FormatDebug + Clone + HasFallibleSpan + 'static,
{
self.with_expand_tracer(|_, tracer| tracer.start(desc));
let result = block(self);
self.with_expand_tracer(|_, tracer| match &result {
Ok(result) => {
tracer.add_result(Box::new(result.clone()));
tracer.success();
}
Err(err) => tracer.failed(err),
});
result
}
pub fn expand_expr_frame(
&mut self,
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> Result<Expression, ParseError>,
) -> Result<Expression, ParseError> {
self.with_expand_tracer(|_, tracer| tracer.start(desc));
let result = block(self);
self.with_expand_tracer(|_, tracer| match &result {
Ok(expr) => {
tracer.add_expr(expr.clone());
tracer.success()
}
Err(err) => tracer.failed(err),
});
result
}
pub fn color_fallible_frame<T>(
&mut self,
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> Result<T, ShellError>,
) -> Result<T, ShellError> {
self.with_tracer(|_, tracer| tracer.start(desc));
self.with_color_tracer(|_, tracer| tracer.start(desc));
if self.at_end() {
self.with_tracer(|_, tracer| tracer.eof_frame());
self.with_color_tracer(|_, tracer| tracer.eof_frame());
return Err(ShellError::unexpected_eof("coloring", Tag::unknown()));
}
let result = block(self);
self.with_tracer(|_, tracer| match &result {
self.with_color_tracer(|_, tracer| match &result {
Ok(_) => {
tracer.success();
}
@ -431,10 +509,6 @@ impl<'content> TokensIterator<'content> {
}
}
pub fn whole_span(&self) -> Span {
self.state.span
}
pub fn span_at_cursor(&mut self) -> Span {
let next = self.peek_any();
@ -491,27 +565,22 @@ impl<'content> TokensIterator<'content> {
self.state.index = 0;
}
pub fn clone(&self) -> TokensIterator<'content> {
let state = &self.state;
TokensIterator {
state: TokensIteratorState {
tokens: state.tokens,
span: state.span,
index: state.index,
seen: state.seen.clone(),
skip_ws: state.skip_ws,
#[cfg(coloring_in_tokens)]
shapes: state.shapes.clone(),
},
tracer: self.tracer.clone(),
}
}
// Get the next token, not including whitespace
pub fn next_non_ws(&mut self) -> Option<&TokenNode> {
let mut peeked = start_next(self, true);
peeked.commit()
}
// pub fn clone(&self) -> TokensIterator<'content> {
// let state = &self.state;
// TokensIterator {
// state: TokensIteratorState {
// tokens: state.tokens,
// span: state.span,
// index: state.index,
// seen: state.seen.clone(),
// skip_ws: state.skip_ws,
// #[cfg(coloring_in_tokens)]
// shapes: state.shapes.clone(),
// },
// color_tracer: self.color_tracer.clone(),
// expand_tracer: self.expand_tracer.clone(),
// }
// }
// Peek the next token, not including whitespace
pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> {
@ -527,8 +596,8 @@ impl<'content> TokensIterator<'content> {
pub fn peek_any_token<'me, T>(
&'me mut self,
expected: &'static str,
block: impl FnOnce(&'content TokenNode) -> Result<T, ShellError>,
) -> Result<T, ShellError> {
block: impl FnOnce(&'content TokenNode) -> Result<T, ParseError>,
) -> Result<T, ParseError> {
let peeked = start_next(self, false);
let peeked = peeked.not_eof(expected);
@ -557,9 +626,11 @@ impl<'content> TokensIterator<'content> {
}
pub fn debug_remaining(&self) -> Vec<TokenNode> {
let mut tokens = self.clone();
tokens.restart();
tokens.cloned().collect()
// TODO: TODO: TODO: Clean up
vec![]
// let mut tokens = self.clone();
// tokens.restart();
// tokens.cloned().collect()
}
}

357
src/parser/hir/tokens_iterator/debug.rs
View File

@ -1,13 +1,13 @@
use crate::errors::ShellError;
use crate::parser::hir::syntax_shape::FlatShape;
#![allow(unused)]
pub(crate) mod color_trace;
pub(crate) mod expand_trace;
pub(crate) use self::color_trace::*;
pub(crate) use self::expand_trace::*;
use crate::parser::hir::tokens_iterator::TokensIteratorState;
use crate::prelude::*;
use crate::traits::ToDebug;
use ansi_term::Color;
use log::trace;
use ptree::*;
use std::borrow::Cow;
use std::io;
#[derive(Debug)]
pub(crate) enum DebugIteratorToken {
@ -36,344 +36,3 @@ pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec<Deb
out
}
#[derive(Debug, Clone)]
pub enum FrameChild {
#[allow(unused)]
Shape(Spanned<FlatShape>),
Frame(ColorFrame),
}
impl FrameChild {
fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
match self {
FrameChild::Shape(shape) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
FrameChild::Frame(frame) => frame.colored_leaf_description(f),
}
}
fn into_tree_child(self, text: &Text) -> TreeChild {
match self {
FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
}
}
}
#[derive(Debug, Clone)]
pub struct ColorFrame {
description: &'static str,
children: Vec<FrameChild>,
error: Option<ShellError>,
}
impl ColorFrame {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
if self.has_only_error_descendents() {
if self.children.len() == 0 {
write!(
f,
"{}",
Color::White.bold().on(Color::Red).paint(self.description)
)
} else {
write!(f, "{}", Color::Red.normal().paint(self.description))
}
} else if self.has_descendent_shapes() {
write!(f, "{}", Color::Green.normal().paint(self.description))
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))
}
}
fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
if self.children.len() == 1 {
let child = &self.children[0];
self.colored_leaf_description(f)?;
write!(f, " -> ")?;
child.colored_leaf_description(text, f)
} else {
self.colored_leaf_description(f)
}
}
fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
if self.children.len() == 1 {
let child = &self.children[0];
match child {
FrameChild::Shape(_) => vec![],
FrameChild::Frame(frame) => frame.tree_children(text),
}
} else {
self.tree_children(text)
}
}
fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
self.children
.clone()
.into_iter()
.map(|c| c.into_tree_child(text))
.collect()
}
#[allow(unused)]
fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.children.push(FrameChild::Shape(shape))
}
fn has_child_shapes(&self) -> bool {
self.any_child_shape(|_| true)
}
fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Shape(shape) => {
if predicate(*shape) {
return true;
}
}
_ => {}
}
}
false
}
fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Frame(frame) => {
if predicate(frame) {
return true;
}
}
_ => {}
}
}
false
}
fn has_descendent_shapes(&self) -> bool {
if self.has_child_shapes() {
true
} else {
self.any_child_frame(|frame| frame.has_descendent_shapes())
}
}
fn has_only_error_descendents(&self) -> bool {
if self.children.len() == 0 {
// if this frame has no children at all, it has only error descendents if this frame
// is an error
self.error.is_some()
} else {
// otherwise, it has only error descendents if all of its children terminate in an
// error (transitively)
let mut seen_error = false;
for child in &self.children {
match child {
// if this frame has at least one child shape, this frame has non-error descendents
FrameChild::Shape(_) => return false,
FrameChild::Frame(frame) => {
// if the chi
if frame.has_only_error_descendents() {
seen_error = true;
} else {
return false;
}
}
}
}
seen_error
}
}
}
#[derive(Debug, Clone)]
pub enum TreeChild {
Shape(Spanned<FlatShape>, Text),
Frame(ColorFrame, Text),
}
impl TreeChild {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
match self {
TreeChild::Shape(shape, text) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
}
}
}
impl TreeItem for TreeChild {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
match self {
shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
TreeChild::Frame(frame, text) => frame.colored_description(text, f),
}
}
fn children(&self) -> Cow<[Self::Child]> {
match self {
TreeChild::Shape(..) => Cow::Borrowed(&[]),
TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
}
}
}
#[derive(Debug, Clone)]
pub struct Tracer {
frame_stack: Vec<ColorFrame>,
}
impl Tracer {
pub fn print(self, source: Text) -> PrintTracer {
PrintTracer {
tracer: self,
source,
}
}
pub fn new() -> Tracer {
let root = ColorFrame {
description: "Trace",
children: vec![],
error: None,
};
Tracer {
frame_stack: vec![root],
}
}
fn current_frame(&mut self) -> &mut ColorFrame {
let frames = &mut self.frame_stack;
let last = frames.len() - 1;
&mut frames[last]
}
fn pop_frame(&mut self) -> ColorFrame {
let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
if self.frame_stack.len() == 0 {
panic!("Can't pop root tracer frame");
}
self.debug();
result
}
pub fn start(&mut self, description: &'static str) {
let frame = ColorFrame {
description,
children: vec![],
error: None,
};
self.frame_stack.push(frame);
self.debug();
}
pub fn eof_frame(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
#[allow(unused)]
pub fn finish(&mut self) {
loop {
if self.frame_stack.len() == 1 {
break;
}
let frame = self.pop_frame();
self.current_frame().children.push(FrameChild::Frame(frame));
}
}
#[allow(unused)]
pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.current_frame().add_shape(shape);
}
pub fn success(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
pub fn failed(&mut self, error: &ShellError) {
let mut current = self.pop_frame();
current.error = Some(error.clone());
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
fn debug(&self) {
trace!(target: "nu::color_syntax",
"frames = {:?}",
self.frame_stack
.iter()
.map(|f| f.description)
.collect::<Vec<_>>()
);
trace!(target: "nu::color_syntax", "{:#?}", self);
}
}
#[derive(Debug, Clone)]
pub struct PrintTracer {
tracer: Tracer,
source: Text,
}
impl TreeItem for PrintTracer {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
write!(f, "{}", style.paint("Color Trace"))
}
fn children(&self) -> Cow<[Self::Child]> {
Cow::Owned(vec![TreeChild::Frame(
self.tracer.frame_stack[0].clone(),
self.source.clone(),
)])
}
}

351
src/parser/hir/tokens_iterator/debug/color_trace.rs Normal file
View File

@ -0,0 +1,351 @@
use crate::errors::ShellError;
use crate::parser::hir::syntax_shape::FlatShape;
use crate::prelude::*;
use ansi_term::Color;
use log::trace;
use ptree::*;
use std::borrow::Cow;
use std::io;
#[derive(Debug, Clone)]
pub enum FrameChild {
#[allow(unused)]
Shape(Spanned<FlatShape>),
Frame(ColorFrame),
}
impl FrameChild {
fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
match self {
FrameChild::Shape(shape) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
FrameChild::Frame(frame) => frame.colored_leaf_description(f),
}
}
fn into_tree_child(self, text: &Text) -> TreeChild {
match self {
FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
}
}
}
#[derive(Debug, Clone)]
pub struct ColorFrame {
description: &'static str,
children: Vec<FrameChild>,
error: Option<ShellError>,
}
impl ColorFrame {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
if self.has_only_error_descendents() {
if self.children.len() == 0 {
write!(
f,
"{}",
Color::White.bold().on(Color::Red).paint(self.description)
)
} else {
write!(f, "{}", Color::Red.normal().paint(self.description))
}
} else if self.has_descendent_shapes() {
write!(f, "{}", Color::Green.normal().paint(self.description))
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))
}
}
fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
if self.children.len() == 1 {
let child = &self.children[0];
self.colored_leaf_description(f)?;
write!(f, " -> ")?;
child.colored_leaf_description(text, f)
} else {
self.colored_leaf_description(f)
}
}
fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
if self.children.len() == 1 {
let child = &self.children[0];
match child {
FrameChild::Shape(_) => vec![],
FrameChild::Frame(frame) => frame.tree_children(text),
}
} else {
self.tree_children(text)
}
}
fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
self.children
.clone()
.into_iter()
.map(|c| c.into_tree_child(text))
.collect()
}
#[allow(unused)]
fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.children.push(FrameChild::Shape(shape))
}
fn has_child_shapes(&self) -> bool {
self.any_child_shape(|_| true)
}
fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Shape(shape) => {
if predicate(*shape) {
return true;
}
}
_ => {}
}
}
false
}
fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Frame(frame) => {
if predicate(frame) {
return true;
}
}
_ => {}
}
}
false
}
fn has_descendent_shapes(&self) -> bool {
if self.has_child_shapes() {
true
} else {
self.any_child_frame(|frame| frame.has_descendent_shapes())
}
}
fn has_only_error_descendents(&self) -> bool {
if self.children.len() == 0 {
// if this frame has no children at all, it has only error descendents if this frame
// is an error
self.error.is_some()
} else {
// otherwise, it has only error descendents if all of its children terminate in an
// error (transitively)
let mut seen_error = false;
for child in &self.children {
match child {
// if this frame has at least one child shape, this frame has non-error descendents
FrameChild::Shape(_) => return false,
FrameChild::Frame(frame) => {
// if the chi
if frame.has_only_error_descendents() {
seen_error = true;
} else {
return false;
}
}
}
}
seen_error
}
}
}
#[derive(Debug, Clone)]
pub enum TreeChild {
Shape(Spanned<FlatShape>, Text),
Frame(ColorFrame, Text),
}
impl TreeChild {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
match self {
TreeChild::Shape(shape, text) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
}
}
}
impl TreeItem for TreeChild {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
match self {
shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
TreeChild::Frame(frame, text) => frame.colored_description(text, f),
}
}
fn children(&self) -> Cow<[Self::Child]> {
match self {
TreeChild::Shape(..) => Cow::Borrowed(&[]),
TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
}
}
}
#[derive(Debug, Clone)]
pub struct ColorTracer {
frame_stack: Vec<ColorFrame>,
}
impl ColorTracer {
pub fn print(self, source: Text) -> PrintTracer {
PrintTracer {
tracer: self,
source,
}
}
pub fn new() -> ColorTracer {
let root = ColorFrame {
description: "Trace",
children: vec![],
error: None,
};
ColorTracer {
frame_stack: vec![root],
}
}
fn current_frame(&mut self) -> &mut ColorFrame {
let frames = &mut self.frame_stack;
let last = frames.len() - 1;
&mut frames[last]
}
fn pop_frame(&mut self) -> ColorFrame {
trace!(target: "nu::color_syntax", "Popping {:#?}", self);
let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
if self.frame_stack.len() == 0 {
panic!("Can't pop root tracer frame {:#?}", self);
}
self.debug();
result
}
pub fn start(&mut self, description: &'static str) {
let frame = ColorFrame {
description,
children: vec![],
error: None,
};
self.frame_stack.push(frame);
self.debug();
}
pub fn eof_frame(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
#[allow(unused)]
pub fn finish(&mut self) {
loop {
if self.frame_stack.len() == 1 {
break;
}
let frame = self.pop_frame();
self.current_frame().children.push(FrameChild::Frame(frame));
}
}
#[allow(unused)]
pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.current_frame().add_shape(shape);
}
pub fn success(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
pub fn failed(&mut self, error: &ShellError) {
let mut current = self.pop_frame();
current.error = Some(error.clone());
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
fn debug(&self) {
trace!(target: "nu::color_syntax",
"frames = {:?}",
self.frame_stack
.iter()
.map(|f| f.description)
.collect::<Vec<_>>()
);
trace!(target: "nu::color_syntax", "{:#?}", self);
}
}
#[derive(Debug, Clone)]
pub struct PrintTracer {
tracer: ColorTracer,
source: Text,
}
impl TreeItem for PrintTracer {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
write!(f, "{}", style.paint("Color Trace"))
}
fn children(&self) -> Cow<[Self::Child]> {
Cow::Owned(vec![TreeChild::Frame(
self.tracer.frame_stack[0].clone(),
self.source.clone(),
)])
}
}

365
src/parser/hir/tokens_iterator/debug/expand_trace.rs Normal file
View File

@ -0,0 +1,365 @@
use crate::parser::hir::Expression;
use crate::prelude::*;
use ansi_term::Color;
use log::trace;
use ptree::*;
use std::borrow::Cow;
use std::io;
#[derive(Debug)]
pub enum FrameChild {
Expr(Expression),
Frame(ExprFrame),
Result(Box<dyn FormatDebug>),
}
impl FrameChild {
fn get_error_leaf(&self) -> Option<&'static str> {
match self {
FrameChild::Frame(frame) if frame.error.is_some() => {
if frame.children.len() == 0 {
Some(frame.description)
} else {
None
}
}
_ => None,
}
}
fn to_tree_child(&self, text: &Text) -> TreeChild {
match self {
FrameChild::Expr(expr) => TreeChild::OkExpr(expr.clone(), text.clone()),
FrameChild::Result(result) => {
let result = format!("{}", result.debug(text));
TreeChild::OkNonExpr(result)
}
FrameChild::Frame(frame) => {
if frame.error.is_some() {
if frame.children.len() == 0 {
TreeChild::ErrorLeaf(vec![frame.description])
} else {
TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone())
}
} else {
TreeChild::OkFrame(frame.to_tree_frame(text), text.clone())
}
}
}
}
}
#[derive(Debug)]
pub struct ExprFrame {
description: &'static str,
children: Vec<FrameChild>,
error: Option<ParseError>,
}
impl ExprFrame {
fn to_tree_frame(&self, text: &Text) -> TreeFrame {
let mut children = vec![];
let mut errors = vec![];
for child in &self.children {
if let Some(error_leaf) = child.get_error_leaf() {
errors.push(error_leaf);
continue;
} else if errors.len() > 0 {
children.push(TreeChild::ErrorLeaf(errors));
errors = vec![];
}
children.push(child.to_tree_child(text));
}
if errors.len() > 0 {
children.push(TreeChild::ErrorLeaf(errors));
}
TreeFrame {
description: self.description,
children,
error: self.error.clone(),
}
}
fn add_expr(&mut self, expr: Expression) {
self.children.push(FrameChild::Expr(expr))
}
fn add_result(&mut self, result: Box<dyn FormatDebug>) {
self.children.push(FrameChild::Result(result))
}
}
#[derive(Debug, Clone)]
pub struct TreeFrame {
description: &'static str,
children: Vec<TreeChild>,
error: Option<ParseError>,
}
impl TreeFrame {
fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
if self.children.len() == 1 {
if self.error.is_some() {
write!(f, "{}", Color::Red.normal().paint(self.description))?;
} else if self.has_descendent_green() {
write!(f, "{}", Color::Green.normal().paint(self.description))?;
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))?;
}
write!(f, " -> ")?;
self.children[0].leaf_description(f)
} else {
if self.error.is_some() {
if self.children.len() == 0 {
write!(
f,
"{}",
Color::White.bold().on(Color::Red).paint(self.description)
)
} else {
write!(f, "{}", Color::Red.normal().paint(self.description))
}
} else if self.has_descendent_green() {
write!(f, "{}", Color::Green.normal().paint(self.description))
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))
}
}
}
fn has_child_green(&self) -> bool {
self.children.iter().any(|item| match item {
TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false,
TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) => true,
})
}
fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool {
for item in &self.children {
match item {
TreeChild::OkFrame(frame, ..) => {
if predicate(frame) {
return true;
}
}
_ => {}
}
}
false
}
fn has_descendent_green(&self) -> bool {
if self.has_child_green() {
true
} else {
self.any_child_frame(|frame| frame.has_child_green())
}
}
fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
if self.children.len() == 1 {
let child: &TreeChild = &self.children[0];
match child {
TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
vec![]
}
TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => {
frame.children_for_formatting(text)
}
}
} else {
self.children.clone()
}
}
}
#[derive(Debug, Clone)]
pub enum TreeChild {
OkNonExpr(String),
OkExpr(Expression, Text),
OkFrame(TreeFrame, Text),
ErrorFrame(TreeFrame, Text),
ErrorLeaf(Vec<&'static str>),
}
impl TreeChild {
fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
match self {
TreeChild::OkExpr(expr, text) => write!(
f,
"{} {} {}",
Color::Cyan.normal().paint("returns"),
Color::White.bold().on(Color::Green).paint(expr.type_name()),
expr.span.slice(text)
),
TreeChild::OkNonExpr(result) => write!(
f,
"{} {}",
Color::Cyan.normal().paint("returns"),
Color::White
.bold()
.on(Color::Green)
.paint(format!("{}", result))
),
TreeChild::ErrorLeaf(desc) => {
let last = desc.len() - 1;
for (i, item) in desc.iter().enumerate() {
write!(f, "{}", Color::White.bold().on(Color::Red).paint(*item))?;
if i != last {
write!(f, "{}", Color::White.normal().paint(", "))?;
}
}
Ok(())
}
TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => {
frame.leaf_description(f)
}
}
}
}
impl TreeItem for TreeChild {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
self.leaf_description(f)
}
fn children(&self) -> Cow<[Self::Child]> {
match self {
TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
Cow::Borrowed(&[])
}
TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => {
Cow::Owned(frame.children_for_formatting(text))
}
}
}
}
#[derive(Debug)]
pub struct ExpandTracer {
frame_stack: Vec<ExprFrame>,
}
impl ExpandTracer {
pub fn print(&self, source: Text) -> PrintTracer {
let root = self
.frame_stack
.iter()
.nth(0)
.unwrap()
.to_tree_frame(&source);
PrintTracer { root, source }
}
pub fn new() -> ExpandTracer {
let root = ExprFrame {
description: "Trace",
children: vec![],
error: None,
};
ExpandTracer {
frame_stack: vec![root],
}
}
fn current_frame(&mut self) -> &mut ExprFrame {
let frames = &mut self.frame_stack;
let last = frames.len() - 1;
&mut frames[last]
}
fn pop_frame(&mut self) -> ExprFrame {
let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
if self.frame_stack.len() == 0 {
panic!("Can't pop root tracer frame");
}
self.debug();
result
}
pub fn start(&mut self, description: &'static str) {
let frame = ExprFrame {
description,
children: vec![],
error: None,
};
self.frame_stack.push(frame);
self.debug();
}
pub fn add_expr(&mut self, shape: Expression) {
self.current_frame().add_expr(shape);
}
pub fn add_result(&mut self, result: Box<dyn FormatDebug>) {
self.current_frame().add_result(result);
}
pub fn success(&mut self) {
trace!(target: "parser::expand_syntax", "success {:#?}", self);
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
pub fn failed(&mut self, error: &ParseError) {
let mut current = self.pop_frame();
current.error = Some(error.clone());
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
fn debug(&self) {
trace!(target: "nu::parser::expand",
"frames = {:?}",
self.frame_stack
.iter()
.map(|f| f.description)
.collect::<Vec<_>>()
);
trace!(target: "nu::parser::expand", "{:#?}", self);
}
}
#[derive(Debug, Clone)]
pub struct PrintTracer {
root: TreeFrame,
source: Text,
}
impl TreeItem for PrintTracer {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
write!(f, "{}", style.paint("Expansion Trace"))
}
fn children(&self) -> Cow<[Self::Child]> {
Cow::Borrowed(&self.root.children)
}
}

8
src/parser/parse/call_node.rs
View File

@ -1,7 +1,7 @@
use crate::parser::TokenNode;
use crate::traits::ToDebug;
use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
use getset::Getters;
use std::fmt;
use std::fmt::{self, Write};
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
pub struct CallNode {
@ -27,8 +27,8 @@ impl CallNode {
}
}
impl ToDebug for CallNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for CallNode {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.head.debug(source))?;
if let Some(children) = &self.children {

4
src/parser/parse/operator.rs
View File

@ -14,8 +14,8 @@ pub enum Operator {
Dot,
}
impl ToDebug for Operator {
fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
impl FormatDebug for Operator {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self.as_str())
}
}

24
src/parser/parse/pipeline.rs
View File

@ -1,24 +1,22 @@
use crate::parser::TokenNode;
use crate::traits::ToDebug;
use crate::{Span, Spanned};
use crate::{DebugFormatter, FormatDebug, Span, Spanned, ToDebug};
use derive_new::new;
use getset::Getters;
use std::fmt;
use itertools::Itertools;
use std::fmt::{self, Write};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
pub struct Pipeline {
#[get = "pub"]
pub(crate) parts: Vec<Spanned<PipelineElement>>,
// pub(crate) post_ws: Option<Tag>,
}
impl ToDebug for Pipeline {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
for part in self.parts.iter() {
write!(f, "{}", part.debug(source))?;
}
Ok(())
impl FormatDebug for Pipeline {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str(
"pipeline",
self.parts.iter().map(|p| p.debug(source)).join(" "),
)
}
}
@ -29,8 +27,8 @@ pub struct PipelineElement {
pub tokens: Spanned<Vec<TokenNode>>,
}
impl ToDebug for PipelineElement {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for PipelineElement {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
if let Some(pipe) = self.pipe {
write!(f, "{}", pipe.slice(source))?;
}

24
src/parser/parse/token_tree.rs
View File

@ -1,4 +1,4 @@
use crate::errors::ShellError;
use crate::errors::{ParseError, ShellError};
use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
use crate::prelude::*;
use crate::traits::ToDebug;
@ -21,8 +21,14 @@ pub enum TokenNode {
Error(Spanned<ShellError>),
}
impl ToDebug for TokenNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl HasSpan for TokenNode {
fn span(&self) -> Span {
self.get_span()
}
}
impl FormatDebug for TokenNode {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{:?}", self.old_debug(&Text::from(source)))
}
}
@ -84,12 +90,12 @@ impl fmt::Debug for DebugTokenNode<'_> {
impl From<&TokenNode> for Span {
fn from(token: &TokenNode) -> Span {
token.span()
token.get_span()
}
}
impl TokenNode {
pub fn span(&self) -> Span {
pub fn get_span(&self) -> Span {
match self {
TokenNode::Token(t) => t.span,
TokenNode::Nodes(t) => t.span,
@ -231,10 +237,10 @@ impl TokenNode {
}
}
pub fn as_pipeline(&self) -> Result<Pipeline, ShellError> {
pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
match self {
TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
_ => Err(ShellError::type_error("pipeline", self.tagged_type_name())),
other => Err(ParseError::mismatch("pipeline", other.tagged_type_name())),
}
}
@ -321,9 +327,9 @@ impl TokenNode {
}
}
pub fn expect_list(&self) -> &[TokenNode] {
pub fn expect_list(&self) -> Spanned<&[TokenNode]> {
match self {
TokenNode::Nodes(token_nodes) => &token_nodes[..],
TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span),
other => panic!("Expected list, found {:?}", other),
}
}

13
src/parser/parse/tokens.rs
View File

@ -23,8 +23,8 @@ impl RawToken {
RawToken::Operator(..) => "operator",
RawToken::String(_) => "string",
RawToken::Variable(_) => "variable",
RawToken::ExternalCommand(_) => "external command",
RawToken::ExternalWord => "external word",
RawToken::ExternalCommand(_) => "syntax error",
RawToken::ExternalWord => "syntax error",
RawToken::GlobPattern => "glob pattern",
RawToken::Bare => "string",
}
@ -37,6 +37,15 @@ pub enum RawNumber {
Decimal(Span),
}
impl FormatDebug for RawNumber {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
RawNumber::Int(span) => f.say_str("int", span.slice(source)),
RawNumber::Decimal(span) => f.say_str("decimal", span.slice(source)),
}
}
}
impl RawNumber {
pub fn int(span: impl Into<Span>) -> Spanned<RawNumber> {
let span = span.into();

7
src/parser/parse/unit.rs
View File

@ -1,6 +1,7 @@
use crate::data::base::Value;
use crate::prelude::*;
use serde::{Deserialize, Serialize};
use std::fmt;
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
@ -13,6 +14,12 @@ pub enum Unit {
PB,
}
impl FormatDebug for Spanned<Unit> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.span.slice(source))
}
}
impl Unit {
pub fn as_str(&self) -> &str {
match *self {

38
src/parser/parse_command.rs
View File

@ -1,4 +1,4 @@
use crate::errors::{ArgumentError, ShellError};
use crate::errors::{ArgumentError, ParseError};
use crate::parser::hir::syntax_shape::{
color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced,
BackoffColoringMode, ColorSyntax, MaybeSpaceShape,
@ -18,9 +18,9 @@ pub fn parse_command_tail(
context: &ExpandContext,
tail: &mut TokensIterator,
command_span: Span,
) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ParseError> {
let mut named = NamedArguments::new();
trace_remaining("nodes", tail.clone(), context.source());
trace_remaining("nodes", &tail, context.source());
for (name, kind) in &config.named {
trace!(target: "nu::parse", "looking for {} : {:?}", name, kind);
@ -38,7 +38,7 @@ pub fn parse_command_tail(
tail.move_to(pos);
if tail.at_end() {
return Err(ShellError::argument_error(
return Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingValueForName(name.to_string()),
flag.span,
@ -59,7 +59,7 @@ pub fn parse_command_tail(
tail.move_to(pos);
if tail.at_end() {
return Err(ShellError::argument_error(
return Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingValueForName(name.to_string()),
flag.span,
@ -85,7 +85,7 @@ pub fn parse_command_tail(
};
}
trace_remaining("after named", tail.clone(), context.source());
trace_remaining("after named", &tail, context.source());
let mut positional = vec![];
@ -95,7 +95,7 @@ pub fn parse_command_tail(
match &arg.0 {
PositionalType::Mandatory(..) => {
if tail.at_end_possible_ws() {
return Err(ShellError::argument_error(
return Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()),
Tag {
@ -118,7 +118,7 @@ pub fn parse_command_tail(
positional.push(result);
}
trace_remaining("after positional", tail.clone(), context.source());
trace_remaining("after positional", &tail, context.source());
if let Some((syntax_type, _)) = config.rest_positional {
let mut out = vec![];
@ -136,7 +136,7 @@ pub fn parse_command_tail(
positional.extend(out);
}
trace_remaining("after rest", tail.clone(), context.source());
trace_remaining("after rest", &tail, context.source());
trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named);
@ -202,8 +202,6 @@ impl ColorSyntax for CommandTailShape {
shapes: &mut Vec<Spanned<FlatShape>>,
) -> Self::Info {
let mut args = ColoringArgs::new(token_nodes.len());
trace_remaining("nodes", token_nodes.clone(), context.source());
for (name, kind) in &signature.named {
trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
@ -295,8 +293,6 @@ impl ColorSyntax for CommandTailShape {
};
}
trace_remaining("after named", token_nodes.clone(), context.source());
for arg in &signature.positional {
trace!("Processing positional {:?}", arg);
@ -341,8 +337,6 @@ impl ColorSyntax for CommandTailShape {
}
}
trace_remaining("after positional", token_nodes.clone(), context.source());
if let Some((syntax_type, _)) = signature.rest_positional {
loop {
if token_nodes.at_end_possible_ws() {
@ -402,7 +396,7 @@ impl ColorSyntax for CommandTailShape {
context: &ExpandContext,
) -> Self::Info {
let mut args = ColoringArgs::new(token_nodes.len());
trace_remaining("nodes", token_nodes.clone(), context.source());
trace_remaining("nodes", &token_nodes, context.source());
for (name, kind) in &signature.named {
trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
@ -497,7 +491,7 @@ impl ColorSyntax for CommandTailShape {
};
}
trace_remaining("after named", token_nodes.clone(), context.source());
trace_remaining("after named", &token_nodes, context.source());
for arg in &signature.positional {
trace!("Processing positional {:?}", arg);
@ -537,7 +531,7 @@ impl ColorSyntax for CommandTailShape {
}
}
trace_remaining("after positional", token_nodes.clone(), context.source());
trace_remaining("after positional", &token_nodes, context.source());
if let Some((syntax_type, _)) = signature.rest_positional {
loop {
@ -594,11 +588,11 @@ fn extract_mandatory(
tokens: &mut hir::TokensIterator<'_>,
source: &Text,
span: Span,
) -> Result<(usize, Spanned<Flag>), ShellError> {
) -> Result<(usize, Spanned<Flag>), ParseError> {
let flag = tokens.extract(|t| t.as_flag(name, source));
match flag {
None => Err(ShellError::argument_error(
None => Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingMandatoryFlag(name.to_string()),
span,
@ -615,7 +609,7 @@ fn extract_optional(
name: &str,
tokens: &mut hir::TokensIterator<'_>,
source: &Text,
) -> Result<(Option<(usize, Spanned<Flag>)>), ShellError> {
) -> Result<(Option<(usize, Spanned<Flag>)>), ParseError> {
let flag = tokens.extract(|t| t.as_flag(name, source));
match flag {
@ -627,7 +621,7 @@ fn extract_optional(
}
}
pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) {
pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>, source: &Text) {
trace!(
target: "nu::parse",
"{} = {:?}",

10
src/prelude.rs
View File

@ -67,13 +67,14 @@ pub(crate) use crate::context::CommandRegistry;
pub(crate) use crate::context::{AnchorLocation, Context};
pub(crate) use crate::data::base as value;
pub(crate) use crate::data::meta::{
tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
TaggedItem,
};
pub(crate) use crate::data::types::ExtractType;
pub(crate) use crate::data::{Primitive, Value};
pub(crate) use crate::env::host::handle_unexpected;
pub(crate) use crate::env::Host;
pub(crate) use crate::errors::{CoerceInto, ShellError};
pub(crate) use crate::errors::{CoerceInto, ParseError, ShellError};
pub(crate) use crate::parser::hir::SyntaxShape;
pub(crate) use crate::parser::parse::parser::Number;
pub(crate) use crate::parser::registry::Signature;
@ -82,7 +83,7 @@ pub(crate) use crate::shell::help_shell::HelpShell;
pub(crate) use crate::shell::shell_manager::ShellManager;
pub(crate) use crate::shell::value_shell::ValueShell;
pub(crate) use crate::stream::{InputStream, OutputStream};
pub(crate) use crate::traits::{HasTag, ToDebug};
pub(crate) use crate::traits::{DebugFormatter, FormatDebug, HasTag, ToDebug};
pub(crate) use crate::Text;
pub(crate) use async_stream::stream as async_stream;
pub(crate) use bigdecimal::BigDecimal;
@ -93,9 +94,12 @@ pub(crate) use num_traits::cast::{FromPrimitive, ToPrimitive};
pub(crate) use num_traits::identities::Zero;
pub(crate) use serde::Deserialize;
pub(crate) use std::collections::VecDeque;
pub(crate) use std::fmt::Write;
pub(crate) use std::future::Future;
pub(crate) use std::sync::{Arc, Mutex};
pub(crate) use itertools::Itertools;
pub trait FromInputStream {
fn from_input_stream(self) -> OutputStream;
}

15
src/shell/helper.rs
View File

@ -3,7 +3,7 @@ use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, Pipelin
use crate::parser::hir::TokensIterator;
use crate::parser::nom_input;
use crate::parser::parse::token_tree::TokenNode;
use crate::{Span, Spanned, SpannedItem, Tag, Tagged, Text};
use crate::{HasSpan, Spanned, SpannedItem, Tag, Tagged, Text};
use ansi_term::Color;
use log::{log_enabled, trace};
use rustyline::completion::Completer;
@ -65,9 +65,7 @@ impl Highlighter for Helper {
let mut tokens = TokensIterator::all(&tokens[..], v.span());
let text = Text::from(line);
let expand_context = self
.context
.expand_context(&text, Span::new(0, line.len() - 1));
let expand_context = self.context.expand_context(&text);
#[cfg(not(coloring_in_tokens))]
let shapes = {
@ -86,16 +84,17 @@ impl Highlighter for Helper {
let shapes = {
// We just constructed a token list that only contains a pipeline, so it can't fail
color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context).unwrap();
tokens.with_tracer(|_, tracer| tracer.finish());
tokens.with_color_tracer(|_, tracer| tracer.finish());
tokens.state().shapes()
};
trace!(target: "nu::color_syntax", "{:#?}", tokens.tracer());
trace!(target: "nu::color_syntax", "{:#?}", tokens.color_tracer());
if log_enabled!(target: "nu::color_syntax", log::Level::Trace) {
if log_enabled!(target: "nu::color_syntax", log::Level::Debug) {
println!("");
ptree::print_tree(&tokens.tracer().clone().print(Text::from(line))).unwrap();
ptree::print_tree(&tokens.color_tracer().clone().print(Text::from(line)))
.unwrap();
println!("");
}

124
src/traits.rs
View File

@ -1,14 +1,28 @@
use crate::prelude::*;
use std::fmt;
use derive_new::new;
use std::fmt::{self, Write};
pub struct Debuggable<'a, T: ToDebug> {
pub struct Debuggable<'a, T: FormatDebug> {
inner: &'a T,
source: &'a str,
}
impl FormatDebug for str {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self)
}
}
impl<T: ToDebug> fmt::Display for Debuggable<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt_debug(f, self.source)
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.inner.fmt_debug(
&mut DebugFormatter::new(
f,
ansi_term::Color::White.bold(),
ansi_term::Color::Black.bold(),
),
self.source,
)
}
}
@ -16,13 +30,109 @@ pub trait HasTag {
fn tag(&self) -> Tag;
}
pub trait ToDebug: Sized {
#[derive(new)]
pub struct DebugFormatter<'me, 'args> {
formatter: &'me mut std::fmt::Formatter<'args>,
style: ansi_term::Style,
default_style: ansi_term::Style,
}
impl<'me, 'args> DebugFormatter<'me, 'args> {
pub fn say<'debuggable>(
&mut self,
kind: &str,
debuggable: Debuggable<'debuggable, impl FormatDebug>,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
write!(
self,
"{}",
self.default_style.paint(format!("{}", debuggable))
)
}
pub fn say_str<'debuggable>(
&mut self,
kind: &str,
string: impl AsRef<str>,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
write!(self, "{}", self.default_style.paint(string.as_ref()))
}
pub fn say_block(
&mut self,
kind: &str,
block: impl FnOnce(&mut Self) -> std::fmt::Result,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
block(self)
}
pub fn say_dict<'debuggable>(
&mut self,
kind: &str,
dict: indexmap::IndexMap<&str, String>,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
let last = dict.len() - 1;
for (i, (key, value)) in dict.into_iter().enumerate() {
write!(self, "{}", self.default_style.paint(key))?;
write!(self, "{}", self.default_style.paint("=["))?;
write!(self, "{}", self.style.paint(value))?;
write!(self, "{}", self.default_style.paint("]"))?;
if i != last {
write!(self, "{}", self.default_style.paint(" "))?;
}
}
Ok(())
}
}
impl<'a, 'b> std::fmt::Write for DebugFormatter<'a, 'b> {
fn write_str(&mut self, s: &str) -> std::fmt::Result {
self.formatter.write_str(s)
}
fn write_char(&mut self, c: char) -> std::fmt::Result {
self.formatter.write_char(c)
}
fn write_fmt(self: &mut Self, args: std::fmt::Arguments<'_>) -> std::fmt::Result {
self.formatter.write_fmt(args)
}
}
pub trait FormatDebug: std::fmt::Debug {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result;
}
pub trait ToDebug: Sized + FormatDebug {
fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self>;
}
impl FormatDebug for Box<dyn FormatDebug> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
(&**self).fmt_debug(f, source)
}
}
impl<T> ToDebug for T
where
T: FormatDebug + Sized,
{
fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self> {
Debuggable {
inner: self,
source,
}
}
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result;
}