holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #897 from nushell/modernize_external_tokens

Browse Source 
Modernize external tokens
This commit is contained in:
Jonathan Turner 2019-11-02 06:18:38 +13:00 committed by GitHub
commit e75fdc2865
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 2301 additions and 1029 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.cargo/config
View File

@ -0,0 +1,3 @@
[build]
rustflags = "--cfg coloring_in_tokens"

9
Cargo.lock generated
View File

@ -1498,6 +1498,7 @@ dependencies = [
"bson 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono-humanize 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1540,7 +1541,7 @@ dependencies = [
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"roxmltree 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)",
"rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
"serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2077,8 +2078,8 @@ dependencies = [
[[package]]
name = "rustyline"
version = "5.0.3"
source = "git+https://github.com/kkawakam/rustyline.git#449c811998f630102bb2d9fb0b59b890d9eabac5"
version = "5.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3056,7 +3057,7 @@ dependencies = [
"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)" = "<none>"
"checksum rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e9d8eb9912bc492db051324d36f5cea56984fc2afeaa5c6fa84e0b0e3cde550f"
"checksum ryu 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19d2271fa48eaf61e53cc88b4ad9adcbafa2d512c531e7fadb6dc11a4d3656c5"
"checksum safemem 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0"
"checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421"

1
Cargo.toml
View File

@ -75,6 +75,7 @@ serde_urlencoded = "0.6.1"
sublime_fuzzy = "0.5"
trash = "1.0.0"
regex = "1"
cfg-if = "0.1"
neso = { version = "0.5.0", optional = true }
crossterm = { version = "0.10.2", optional = true }

36
src/cli.rs
View File

@ -14,13 +14,13 @@ use crate::git::current_branch;
use crate::parser::registry::Signature;
use crate::parser::{
hir,
hir::syntax_shape::{expand_syntax, PipelineShape},
hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator},
hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape},
hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator},
TokenNode,
};
use crate::prelude::*;
use log::{debug, trace};
use log::{debug, log_enabled, trace};
use rustyline::error::ReadlineError;
use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
use std::env;
@ -506,6 +506,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
Some(ClassifiedCommand::External(_)) => {}
_ => pipeline
.commands
.item
.push(ClassifiedCommand::Internal(InternalCommand {
name: "autoview".to_string(),
name_tag: Tag::unknown(),
@ -513,13 +514,14 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
Box::new(hir::Expression::synthetic_string("autoview")),
None,
None,
),
)
.spanned_unknown(),
})),
}
let mut input = ClassifiedInputStream::new();
let mut iter = pipeline.commands.into_iter().peekable();
let mut iter = pipeline.commands.item.into_iter().peekable();
let mut is_first_command = true;
// Check the config to see if we need to update the path
@ -679,11 +681,20 @@ fn classify_pipeline(
let mut pipeline_list = vec![pipeline.clone()];
let mut iterator = TokensIterator::all(&mut pipeline_list, pipeline.span());
expand_syntax(
let result = expand_syntax(
&PipelineShape,
&mut iterator,
&context.expand_context(source, pipeline.span()),
&context.expand_context(source),
)
.map_err(|err| err.into());
if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
println!("");
ptree::print_tree(&iterator.expand_tracer().print(source.clone())).unwrap();
println!("");
}
result
}
// Classify this command as an external command, which doesn't give special meaning
@ -691,21 +702,22 @@ fn classify_pipeline(
// strings.
pub(crate) fn external_command(
tokens: &mut TokensIterator,
source: &Text,
context: &ExpandContext,
name: Tagged<&str>,
) -> Result<ClassifiedCommand, ShellError> {
let arg_list_strings = expand_external_tokens(tokens, source)?;
) -> Result<ClassifiedCommand, ParseError> {
let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?;
Ok(ClassifiedCommand::External(ExternalCommand {
name: name.to_string(),
name_tag: name.tag(),
args: arg_list_strings
args: item
.iter()
.map(|x| Tagged {
tag: x.span.into(),
item: x.item.clone(),
})
.collect(),
.collect::<Vec<_>>()
.spanned(span),
}))
}

94
src/commands/classified.rs
View File

@ -4,7 +4,9 @@ use bytes::{BufMut, BytesMut};
use derive_new::new;
use futures::stream::StreamExt;
use futures_codec::{Decoder, Encoder, Framed};
use itertools::Itertools;
use log::{log_enabled, trace};
use std::fmt;
use std::io::{Error, ErrorKind};
use subprocess::Exec;
@ -72,26 +74,77 @@ impl ClassifiedInputStream {
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub(crate) struct ClassifiedPipeline {
pub(crate) commands: Vec<ClassifiedCommand>,
pub(crate) commands: Spanned<Vec<ClassifiedCommand>>,
}
#[derive(Debug, Eq, PartialEq)]
impl FormatDebug for ClassifiedPipeline {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str(
"classified pipeline",
self.commands.iter().map(|c| c.debug(source)).join(" | "),
)
}
}
impl HasSpan for ClassifiedPipeline {
fn span(&self) -> Span {
self.commands.span
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) enum ClassifiedCommand {
#[allow(unused)]
Expr(TokenNode),
Internal(InternalCommand),
#[allow(unused)]
Dynamic(hir::Call),
Dynamic(Spanned<hir::Call>),
External(ExternalCommand),
}
#[derive(new, Debug, Eq, PartialEq)]
impl FormatDebug for ClassifiedCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
ClassifiedCommand::Expr(expr) => expr.fmt_debug(f, source),
ClassifiedCommand::Internal(internal) => internal.fmt_debug(f, source),
ClassifiedCommand::Dynamic(dynamic) => dynamic.fmt_debug(f, source),
ClassifiedCommand::External(external) => external.fmt_debug(f, source),
}
}
}
impl HasSpan for ClassifiedCommand {
fn span(&self) -> Span {
match self {
ClassifiedCommand::Expr(node) => node.span(),
ClassifiedCommand::Internal(command) => command.span(),
ClassifiedCommand::Dynamic(call) => call.span,
ClassifiedCommand::External(command) => command.span(),
}
}
}
#[derive(new, Debug, Clone, Eq, PartialEq)]
pub(crate) struct InternalCommand {
pub(crate) name: String,
pub(crate) name_tag: Tag,
pub(crate) args: hir::Call,
pub(crate) args: Spanned<hir::Call>,
}
impl HasSpan for InternalCommand {
fn span(&self) -> Span {
let start = self.name_tag.span;
start.until(self.args.span)
}
}
impl FormatDebug for InternalCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say("internal", self.args.debug(source))
}
}
#[derive(new, Debug, Eq, PartialEq)]
@ -122,7 +175,7 @@ impl InternalCommand {
context.run_command(
command,
self.name_tag.clone(),
self.args,
self.args.item,
&source,
objects,
is_first_command,
@ -201,12 +254,31 @@ impl InternalCommand {
}
}
#[derive(Debug, Eq, PartialEq)]
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) struct ExternalCommand {
pub(crate) name: String,
pub(crate) name_tag: Tag,
pub(crate) args: Vec<Tagged<String>>,
pub(crate) args: Spanned<Vec<Tagged<String>>>,
}
impl FormatDebug for ExternalCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.name)?;
if self.args.item.len() > 0 {
write!(f, " ")?;
write!(f, "{}", self.args.iter().map(|i| i.debug(source)).join(" "))?;
}
Ok(())
}
}
impl HasSpan for ExternalCommand {
fn span(&self) -> Span {
self.name_tag.span.until(self.args.span)
}
}
#[derive(Debug)]
@ -230,7 +302,7 @@ impl ExternalCommand {
trace!(target: "nu::run::external", "inputs = {:?}", inputs);
let mut arg_string = format!("{}", self.name);
for arg in &self.args {
for arg in &self.args.item {
arg_string.push_str(&arg);
}
@ -275,7 +347,7 @@ impl ExternalCommand {
process = Exec::shell(itertools::join(commands, " && "))
} else {
process = Exec::cmd(&self.name);
for arg in &self.args {
for arg in &self.args.item {
let arg_chars: Vec<_> = arg.chars().collect();
if arg_chars.len() > 1
&& arg_chars[0] == '"'

26
src/commands/command.rs
View File

@ -19,8 +19,8 @@ pub struct UnevaluatedCallInfo {
pub name_tag: Tag,
}
impl ToDebug for UnevaluatedCallInfo {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for UnevaluatedCallInfo {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.args.fmt_debug(f, source)
}
}
@ -96,8 +96,14 @@ impl RawCommandArgs {
}
}
impl ToDebug for CommandArgs {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl std::fmt::Debug for CommandArgs {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.call_info.fmt(f)
}
}
impl FormatDebug for CommandArgs {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.call_info.fmt_debug(f, source)
}
}
@ -377,7 +383,7 @@ impl EvaluatedCommandArgs {
}
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CommandAction {
ChangePath(String),
Exit,
@ -389,8 +395,8 @@ pub enum CommandAction {
LeaveShell,
}
impl ToDebug for CommandAction {
fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
impl FormatDebug for CommandAction {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
match self {
CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
CommandAction::Exit => write!(f, "action:exit"),
@ -408,7 +414,7 @@ impl ToDebug for CommandAction {
}
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ReturnSuccess {
Value(Tagged<Value>),
Action(CommandAction),
@ -416,8 +422,8 @@ pub enum ReturnSuccess {
pub type ReturnValue = Result<ReturnSuccess, ShellError>;
impl ToDebug for ReturnValue {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for ReturnValue {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Err(err) => write!(f, "{}", err.debug(source)),
Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),

3
src/context.rs
View File

@ -71,9 +71,8 @@ impl Context {
pub(crate) fn expand_context<'context>(
&'context self,
source: &'context Text,
span: Span,
) -> ExpandContext<'context> {
ExpandContext::new(&self.registry, span, source, self.shell_manager.homedir())
ExpandContext::new(&self.registry, source, self.shell_manager.homedir())
}
pub(crate) fn basic() -> Result<Context, Box<dyn Error>> {

132
src/data/meta.rs
View File

@ -5,6 +5,7 @@ use derive_new::new;
use getset::Getters;
use serde::Deserialize;
use serde::Serialize;
use std::fmt;
use std::path::{Path, PathBuf};
#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
@ -461,3 +462,134 @@ impl language_reporting::ReportingSpan for Span {
self.end
}
}
pub trait HasSpan: ToDebug {
fn span(&self) -> Span;
}
pub trait HasFallibleSpan: ToDebug {
fn maybe_span(&self) -> Option<Span>;
}
impl<T: HasSpan> HasFallibleSpan for T {
fn maybe_span(&self) -> Option<Span> {
Some(HasSpan::span(self))
}
}
impl<T> HasSpan for Spanned<T>
where
Spanned<T>: ToDebug,
{
fn span(&self) -> Span {
self.span
}
}
impl HasFallibleSpan for Option<Span> {
fn maybe_span(&self) -> Option<Span> {
*self
}
}
impl FormatDebug for Option<Span> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Option::None => write!(f, "no span"),
Option::Some(span) => FormatDebug::fmt_debug(span, f, source),
}
}
}
impl FormatDebug for Span {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{:?}", self.slice(source))
}
}
impl HasSpan for Span {
fn span(&self) -> Span {
*self
}
}
impl<T> FormatDebug for Option<Spanned<T>>
where
Spanned<T>: ToDebug,
{
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Option::None => write!(f, "nothing"),
Option::Some(spanned) => FormatDebug::fmt_debug(spanned, f, source),
}
}
}
impl<T> HasFallibleSpan for Option<Spanned<T>>
where
Spanned<T>: ToDebug,
{
fn maybe_span(&self) -> Option<Span> {
match self {
None => None,
Some(value) => Some(value.span),
}
}
}
impl<T> FormatDebug for Option<Tagged<T>>
where
Tagged<T>: ToDebug,
{
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Option::None => write!(f, "nothing"),
Option::Some(item) => FormatDebug::fmt_debug(item, f, source),
}
}
}
impl<T> HasFallibleSpan for Option<Tagged<T>>
where
Tagged<T>: ToDebug,
{
fn maybe_span(&self) -> Option<Span> {
match self {
None => None,
Some(value) => Some(value.tag.span),
}
}
}
impl<T> HasSpan for Tagged<T>
where
Tagged<T>: ToDebug,
{
fn span(&self) -> Span {
self.tag.span
}
}
impl<T: ToDebug> FormatDebug for Vec<T> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "[ ")?;
write!(
f,
"{}",
self.iter().map(|item| item.debug(source)).join(" ")
)?;
write!(f, " ]")
}
}
impl FormatDebug for String {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self)
}
}
impl FormatDebug for Spanned<String> {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self.item)
}
}

94
src/errors.rs
View File

@ -30,6 +30,82 @@ impl Description {
}
}
#[derive(Debug, Clone)]
pub enum ParseErrorReason {
Eof {
expected: &'static str,
},
Mismatch {
expected: &'static str,
actual: Tagged<String>,
},
ArgumentError {
command: String,
error: ArgumentError,
tag: Tag,
},
}
#[derive(Debug, Clone)]
pub struct ParseError {
reason: ParseErrorReason,
tag: Tag,
}
impl ParseError {
pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError {
ParseError {
reason: ParseErrorReason::Eof { expected },
tag: span.into(),
}
}
pub fn mismatch(expected: &'static str, actual: Tagged<impl Into<String>>) -> ParseError {
let Tagged { tag, item } = actual;
ParseError {
reason: ParseErrorReason::Mismatch {
expected,
actual: item.into().tagged(tag.clone()),
},
tag,
}
}
pub fn argument_error(
command: impl Into<String>,
kind: ArgumentError,
tag: impl Into<Tag>,
) -> ParseError {
let tag = tag.into();
ParseError {
reason: ParseErrorReason::ArgumentError {
command: command.into(),
error: kind,
tag: tag.clone(),
},
tag: tag.clone(),
}
}
}
impl From<ParseError> for ShellError {
fn from(error: ParseError) -> ShellError {
match error.reason {
ParseErrorReason::Eof { expected } => ShellError::unexpected_eof(expected, error.tag),
ParseErrorReason::Mismatch { actual, expected } => {
ShellError::type_error(expected, actual.clone())
}
ParseErrorReason::ArgumentError {
command,
error,
tag,
} => ShellError::argument_error(command, error, tag),
}
}
}
#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
pub enum ArgumentError {
MissingMandatoryFlag(String),
@ -51,8 +127,8 @@ impl ShellError {
}
}
impl ToDebug for ShellError {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for ShellError {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.error.fmt_debug(f, source)
}
}
@ -153,16 +229,6 @@ impl ShellError {
.start()
}
pub(crate) fn invalid_external_word(tag: impl Into<Tag>) -> ShellError {
ProximateShellError::ArgumentError {
command: "Invalid argument to Nu command (did you mean to call an external command?)"
.into(),
error: ArgumentError::InvalidExternalWord,
tag: tag.into(),
}
.start()
}
pub(crate) fn parse_error(
error: nom::Err<(
nom_locate::LocatedSpanEx<&str, TracableContext>,
@ -490,8 +556,8 @@ impl ProximateShellError {
}
}
impl ToDebug for ProximateShellError {
fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
impl FormatDebug for ProximateShellError {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
// TODO: Custom debug for inner spans
write!(f, "{:?}", self)
}

6
src/lib.rs
View File

@ -30,12 +30,16 @@ pub use crate::env::host::BasicHost;
pub use crate::parser::hir::SyntaxShape;
pub use crate::parser::parse::token_tree_builder::TokenTreeBuilder;
pub use crate::plugin::{serve_plugin, Plugin};
pub use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
pub use crate::utils::{did_you_mean, AbsoluteFile, AbsolutePath, RelativePath};
pub use cli::cli;
pub use data::base::{Primitive, Value};
pub use data::config::{config_path, APP_INFO};
pub use data::dict::{Dictionary, TaggedDictBuilder};
pub use data::meta::{tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem};
pub use data::meta::{
tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
TaggedItem,
};
pub use errors::{CoerceInto, ShellError};
pub use num_traits::cast::ToPrimitive;
pub use parser::parse::text::Text;

15
src/main.rs
View File

@ -19,6 +19,12 @@ fn main() -> Result<(), Box<dyn Error>> {
.multiple(true)
.takes_value(true),
)
.arg(
Arg::with_name("debug")
.long("debug")
.multiple(true)
.takes_value(true),
)
.get_matches();
let loglevel = match matches.value_of("loglevel") {
@ -48,6 +54,15 @@ fn main() -> Result<(), Box<dyn Error>> {
}
}
match matches.values_of("debug") {
None => {}
Some(values) => {
for item in values {
builder.filter_module(&format!("nu::{}", item), LevelFilter::Debug);
}
}
}
builder.try_init()?;
futures::executor::block_on(nu::cli())?;

41
src/parser/hir.rs
View File

@ -24,7 +24,6 @@ pub(crate) use self::external_command::ExternalCommand;
pub(crate) use self::named::NamedArguments;
pub(crate) use self::path::Path;
pub(crate) use self::syntax_shape::ExpandContext;
pub(crate) use self::tokens_iterator::debug::debug_tokens;
pub(crate) use self::tokens_iterator::TokensIterator;
pub use self::syntax_shape::SyntaxShape;
@ -50,8 +49,8 @@ impl Call {
}
}
impl ToDebug for Call {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Call {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "({}", self.head.debug(source))?;
if let Some(positional) = &self.positional {
@ -242,10 +241,14 @@ impl Expression {
pub(crate) fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
RawExpression::Variable(Variable::It(inner.into())).spanned(outer)
}
pub(crate) fn tagged_type_name(&self) -> Tagged<&'static str> {
self.item.type_name().tagged(self.span)
}
}
impl ToDebug for Expression {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Spanned<RawExpression> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match &self.item {
RawExpression::Literal(l) => l.spanned(self.span).fmt_debug(f, source),
RawExpression::FilePath(p) => write!(f, "{}", p.display()),
@ -256,7 +259,7 @@ impl ToDebug for Expression {
RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
RawExpression::Binary(b) => write!(f, "{}", b.debug(source)),
RawExpression::ExternalCommand(c) => write!(f, "^{}", c.name().slice(source)),
RawExpression::Block(exprs) => {
RawExpression::Block(exprs) => f.say_block("block", |f| {
write!(f, "{{ ")?;
for expr in exprs {
@ -264,8 +267,8 @@ impl ToDebug for Expression {
}
write!(f, "}}")
}
RawExpression::List(exprs) => {
}),
RawExpression::List(exprs) => f.say_block("list", |f| {
write!(f, "[ ")?;
for expr in exprs {
@ -273,7 +276,7 @@ impl ToDebug for Expression {
}
write!(f, "]")
}
}),
RawExpression::Path(p) => write!(f, "{}", p.debug(source)),
RawExpression::Boolean(true) => write!(f, "$yes"),
RawExpression::Boolean(false) => write!(f, "$no"),
@ -321,14 +324,14 @@ impl std::fmt::Display for Tagged<&Literal> {
}
}
impl ToDebug for Spanned<&Literal> {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Spanned<&Literal> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self.item {
Literal::Number(number) => write!(f, "{:?}", number),
Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit),
Literal::String(tag) => write!(f, "{}", tag.slice(source)),
Literal::GlobPattern(_) => write!(f, "{}", self.span.slice(source)),
Literal::Bare => write!(f, "{}", self.span.slice(source)),
Literal::Number(..) => f.say_str("number", self.span.slice(source)),
Literal::Size(..) => f.say_str("size", self.span.slice(source)),
Literal::String(..) => f.say_str("string", self.span.slice(source)),
Literal::GlobPattern(..) => f.say_str("glob", self.span.slice(source)),
Literal::Bare => f.say_str("word", self.span.slice(source)),
}
}
}
@ -359,3 +362,9 @@ impl std::fmt::Display for Variable {
}
}
}
impl FormatDebug for Spanned<Variable> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.span.slice(source))
}
}

19
src/parser/hir/baseline_parse/tests.rs
View File

@ -6,7 +6,7 @@ use crate::parser::hir::syntax_shape::*;
use crate::parser::hir::TokensIterator;
use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
use crate::parser::TokenNode;
use crate::{Span, SpannedItem, Tag, Text};
use crate::{HasSpan, Span, SpannedItem, Tag, Text};
use pretty_assertions::assert_eq;
use std::fmt::Debug;
@ -63,7 +63,9 @@ fn test_parse_command() {
vec![b::bare("ls"), b::sp(), b::pattern("*.txt")],
|tokens| {
let bare = tokens[0].expect_bare();
let pattern = tokens[2].expect_pattern();
let pat = tokens[2].expect_pattern();
eprintln!("{:?} {:?} {:?}", bare, pat, bare.until(pat));
ClassifiedCommand::Internal(InternalCommand::new(
"ls".to_string(),
@ -73,9 +75,10 @@ fn test_parse_command() {
},
hir::Call {
head: Box::new(hir::RawExpression::Command(bare).spanned(bare)),
positional: Some(vec![hir::Expression::pattern("*.txt", pattern)]),
positional: Some(vec![hir::Expression::pattern("*.txt", pat)]),
named: None,
},
}
.spanned(bare.until(pat)),
))
// hir::Expression::path(
// hir::Expression::variable(inner_var, outer_var),
@ -86,7 +89,7 @@ fn test_parse_command() {
);
}
fn parse_tokens<T: Eq + Debug>(
fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
shape: impl ExpandSyntax<Output = T>,
tokens: Vec<CurriedToken>,
expected: impl FnOnce(&[TokenNode]) -> T,
@ -96,19 +99,19 @@ fn parse_tokens<T: Eq + Debug>(
ExpandContext::with_empty(&Text::from(source), |context| {
let tokens = tokens.expect_list();
let mut iterator = TokensIterator::all(tokens, *context.span());
let mut iterator = TokensIterator::all(tokens.item, tokens.span);
let expr = expand_syntax(&shape, &mut iterator, &context);
let expr = match expr {
Ok(expr) => expr,
Err(err) => {
crate::cli::print_err(err, &BasicHost, context.source().clone());
crate::cli::print_err(err.into(), &BasicHost, context.source().clone());
panic!("Parse failed");
}
};
assert_eq!(expr, expected(tokens));
assert_eq!(expr, expected(tokens.item));
})
}

4
src/parser/hir/binary.rs
View File

@ -22,8 +22,8 @@ impl fmt::Display for Binary {
}
}
impl ToDebug for Binary {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Binary {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.left.debug(source))?;
write!(f, " {} ", self.op.debug(source))?;
write!(f, "{}", self.right.debug(source))?;

362
src/parser/hir/expand_external_tokens.rs
View File

@ -1,35 +1,55 @@
use crate::errors::ShellError;
use crate::errors::ParseError;
#[cfg(not(coloring_in_tokens))]
use crate::parser::hir::syntax_shape::FlatShape;
use crate::parser::{
hir::syntax_shape::{
color_syntax, expand_atom, AtomicToken, ColorSyntax, ExpandContext, ExpansionRule,
MaybeSpaceShape,
color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax,
ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape,
},
TokenNode, TokensIterator,
hir::Expression,
TokensIterator,
};
use crate::{Span, Spanned, Text};
pub fn expand_external_tokens(
token_nodes: &mut TokensIterator<'_>,
source: &Text,
) -> Result<Vec<Spanned<String>>, ShellError> {
let mut out: Vec<Spanned<String>> = vec![];
loop {
if let Some(span) = expand_next_expression(token_nodes)? {
out.push(span.spanned_string(source));
} else {
break;
}
}
Ok(out)
}
use crate::{DebugFormatter, FormatDebug, Span, Spanned, SpannedItem};
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct ExternalTokensShape;
impl FormatDebug for Spanned<Vec<Spanned<String>>> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
FormatDebug::fmt_debug(&self.item, f, source)
}
}
impl ExpandSyntax for ExternalTokensShape {
type Output = Spanned<Vec<Spanned<String>>>;
fn name(&self) -> &'static str {
"external command"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
let mut out: Vec<Spanned<String>> = vec![];
let start = token_nodes.span_at_cursor();
loop {
match expand_syntax(&ExternalExpressionShape, token_nodes, context) {
Err(_) | Ok(None) => break,
Ok(Some(span)) => out.push(span.spanned_string(context.source())),
}
}
let end = token_nodes.span_at_cursor();
Ok(out.spanned(start.until(end)))
}
}
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for ExternalTokensShape {
type Info = ();
@ -85,109 +105,200 @@ impl ColorSyntax for ExternalTokensShape {
}
}
pub fn expand_next_expression(
token_nodes: &mut TokensIterator<'_>,
) -> Result<Option<Span>, ShellError> {
let first = token_nodes.next_non_ws();
#[derive(Debug, Copy, Clone)]
pub struct ExternalExpressionShape;
let first = match first {
None => return Ok(None),
Some(v) => v,
};
impl ExpandSyntax for ExternalExpressionShape {
type Output = Option<Span>;
let first = triage_external_head(first)?;
let mut last = first;
loop {
let continuation = triage_continuation(token_nodes)?;
if let Some(continuation) = continuation {
last = continuation;
} else {
break;
}
fn name(&self) -> &'static str {
"external expression"
}
Ok(Some(first.until(last)))
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
fn triage_external_head(node: &TokenNode) -> Result<Span, ShellError> {
Ok(match node {
TokenNode::Token(token) => token.span,
TokenNode::Call(_call) => unimplemented!("TODO: OMG"),
TokenNode::Nodes(_nodes) => unimplemented!("TODO: OMG"),
TokenNode::Delimited(_delimited) => unimplemented!("TODO: OMG"),
TokenNode::Pipeline(_pipeline) => unimplemented!("TODO: OMG"),
TokenNode::Flag(flag) => flag.span,
TokenNode::Whitespace(_whitespace) => {
unreachable!("This function should be called after next_non_ws()")
let first = expand_atom(
token_nodes,
"external command",
context,
ExpansionRule::new().allow_external_command(),
)?
.span;
let mut last = first;
loop {
let continuation = expand_expr(&ExternalContinuationShape, token_nodes, context);
if let Ok(continuation) = continuation {
last = continuation.span;
} else {
break;
}
}
TokenNode::Error(_error) => unimplemented!("TODO: OMG"),
})
}
fn triage_continuation<'a, 'b>(
nodes: &'a mut TokensIterator<'b>,
) -> Result<Option<Span>, ShellError> {
let mut peeked = nodes.peek_any();
let node = match peeked.node {
None => return Ok(None),
Some(node) => node,
};
match &node {
node if node.is_whitespace() => return Ok(None),
TokenNode::Token(..) | TokenNode::Flag(..) => {}
TokenNode::Call(..) => unimplemented!("call"),
TokenNode::Nodes(..) => unimplemented!("nodes"),
TokenNode::Delimited(..) => unimplemented!("delimited"),
TokenNode::Pipeline(..) => unimplemented!("pipeline"),
TokenNode::Whitespace(..) => unimplemented!("whitespace"),
TokenNode::Error(..) => unimplemented!("error"),
Ok(Some(first.until(last)))
}
peeked.commit();
Ok(Some(node.span()))
}
#[must_use]
enum ExternalExpressionResult {
Eof,
Processed,
}
#[derive(Debug, Copy, Clone)]
struct ExternalExpression;
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for ExternalExpression {
type Info = ExternalExpressionResult;
type Input = ();
impl ExpandSyntax for ExternalExpression {
type Output = Option<Span>;
fn color_syntax<'a, 'b>(
fn name(&self) -> &'static str {
"external expression"
}
fn expand_syntax<'a, 'b>(
&self,
_input: &(),
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
shapes: &mut Vec<Spanned<FlatShape>>,
) -> ExternalExpressionResult {
let atom = match expand_atom(
token_nodes,
"external word",
context,
ExpansionRule::permissive(),
) {
Err(_) => unreachable!("TODO: separate infallible expand_atom"),
Ok(Spanned {
item: AtomicToken::Eof { .. },
..
}) => return ExternalExpressionResult::Eof,
Ok(atom) => atom,
};
) -> Result<Self::Output, ParseError> {
expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
atom.color_tokens(shapes);
return ExternalExpressionResult::Processed;
let first = expand_syntax(&ExternalHeadShape, token_nodes, context)?.span;
let mut last = first;
loop {
let continuation = expand_syntax(&ExternalContinuationShape, token_nodes, context);
if let Ok(continuation) = continuation {
last = continuation.span;
} else {
break;
}
}
Ok(Some(first.until(last)))
}
}
#[derive(Debug, Copy, Clone)]
struct ExternalHeadShape;
impl ExpandExpression for ExternalHeadShape {
fn name(&self) -> &'static str {
"external argument"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Expression, ParseError> {
match expand_atom(
token_nodes,
"external argument",
context,
ExpansionRule::new()
.allow_external_word()
.treat_size_as_word(),
)? {
atom => match &atom {
Spanned { item, span } => Ok(match item {
AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
AtomicToken::Whitespace { .. } => {
unreachable!("ExpansionRule doesn't allow Whitespace")
}
AtomicToken::ShorthandFlag { .. }
| AtomicToken::LonghandFlag { .. }
| AtomicToken::SquareDelimited { .. }
| AtomicToken::ParenDelimited { .. }
| AtomicToken::BraceDelimited { .. }
| AtomicToken::Pipeline { .. } => {
return Err(ParseError::mismatch(
"external command name",
atom.tagged_type_name(),
))
}
AtomicToken::ExternalCommand { command } => {
Expression::external_command(*command, *span)
}
AtomicToken::Number { number } => {
Expression::number(number.to_number(context.source()), *span)
}
AtomicToken::String { body } => Expression::string(*body, *span),
AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
AtomicToken::Variable { name } => Expression::variable(*name, *span),
AtomicToken::ExternalWord { .. }
| AtomicToken::GlobPattern { .. }
| AtomicToken::FilePath { .. }
| AtomicToken::Word { .. }
| AtomicToken::Dot { .. }
| AtomicToken::Operator { .. } => Expression::external_command(*span, *span),
}),
},
}
}
}
#[derive(Debug, Copy, Clone)]
struct ExternalContinuationShape;
impl ExpandExpression for ExternalContinuationShape {
fn name(&self) -> &'static str {
"external argument"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Expression, ParseError> {
match expand_atom(
token_nodes,
"external argument",
context,
ExpansionRule::new()
.allow_external_word()
.treat_size_as_word(),
)? {
atom => match &atom {
Spanned { item, span } => Ok(match item {
AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
AtomicToken::Number { number } => {
Expression::number(number.to_number(context.source()), *span)
}
AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
AtomicToken::ExternalCommand { .. } => {
unreachable!("ExpansionRule doesn't allow ExternalCommand")
}
AtomicToken::Whitespace { .. } => {
unreachable!("ExpansionRule doesn't allow Whitespace")
}
AtomicToken::String { body } => Expression::string(*body, *span),
AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
AtomicToken::Variable { name } => Expression::variable(*name, *span),
AtomicToken::ExternalWord { .. }
| AtomicToken::GlobPattern { .. }
| AtomicToken::FilePath { .. }
| AtomicToken::Word { .. }
| AtomicToken::ShorthandFlag { .. }
| AtomicToken::LonghandFlag { .. }
| AtomicToken::Dot { .. }
| AtomicToken::Operator { .. } => Expression::bare(*span),
AtomicToken::SquareDelimited { .. }
| AtomicToken::ParenDelimited { .. }
| AtomicToken::BraceDelimited { .. }
| AtomicToken::Pipeline { .. } => {
return Err(ParseError::mismatch(
"external argument",
atom.tagged_type_name(),
))
}
}),
},
}
}
}
@ -224,3 +335,40 @@ impl ColorSyntax for ExternalExpression {
return ExternalExpressionResult::Processed;
}
}
#[must_use]
enum ExternalExpressionResult {
Eof,
Processed,
}
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for ExternalExpression {
type Info = ExternalExpressionResult;
type Input = ();
fn color_syntax<'a, 'b>(
&self,
_input: &(),
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
shapes: &mut Vec<Spanned<FlatShape>>,
) -> ExternalExpressionResult {
let atom = match expand_atom(
token_nodes,
"external word",
context,
ExpansionRule::permissive(),
) {
Err(_) => unreachable!("TODO: separate infallible expand_atom"),
Ok(Spanned {
item: AtomicToken::Eof { .. },
..
}) => return ExternalExpressionResult::Eof,
Ok(atom) => atom,
};
atom.color_tokens(shapes);
return ExternalExpressionResult::Processed;
}
}

4
src/parser/hir/external_command.rs
View File

@ -12,8 +12,8 @@ pub struct ExternalCommand {
pub(crate) name: Span,
}
impl ToDebug for ExternalCommand {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for ExternalCommand {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.name.slice(source))?;
Ok(())

4
src/parser/hir/named.rs
View File

@ -21,8 +21,8 @@ pub struct NamedArguments {
pub(crate) named: IndexMap<String, NamedValue>,
}
impl ToDebug for NamedArguments {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for NamedArguments {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
for (name, value) in &self.named {
match value {
NamedValue::AbsentSwitch => continue,

4
src/parser/hir/path.rs
View File

@ -44,8 +44,8 @@ impl Path {
}
}
impl ToDebug for Path {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for Path {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.head.debug(source))?;
for part in &self.tail {

415
src/parser/hir/syntax_shape.rs
View File

@ -11,16 +11,12 @@ use crate::parser::hir::expand_external_tokens::ExternalTokensShape;
use crate::parser::hir::syntax_shape::block::AnyBlockShape;
use crate::parser::hir::tokens_iterator::Peeked;
use crate::parser::parse_command::{parse_command_tail, CommandTailShape};
use crate::parser::{
hir,
hir::{debug_tokens, TokensIterator},
Operator, RawToken, TokenNode,
};
use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
use crate::prelude::*;
use derive_new::new;
use getset::Getters;
use log::{self, trace};
use serde::{Deserialize, Serialize};
use std::fmt;
use std::path::{Path, PathBuf};
pub(crate) use self::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
@ -40,15 +36,16 @@ pub(crate) use self::expression::variable_path::{
pub(crate) use self::expression::{continue_expression, AnyExpressionShape};
pub(crate) use self::flat_shape::FlatShape;
#[cfg(not(coloring_in_tokens))]
use crate::parser::hir::tokens_iterator::debug::debug_tokens;
#[cfg(not(coloring_in_tokens))]
use crate::parser::parse::pipeline::Pipeline;
#[cfg(not(coloring_in_tokens))]
use log::log_enabled;
use log::{log_enabled, trace};
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum SyntaxShape {
Any,
List,
String,
Member,
ColumnPath,
@ -75,10 +72,6 @@ impl FallibleColorSyntax for SyntaxShape {
SyntaxShape::Any => {
color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)
}
SyntaxShape::List => {
color_syntax(&ExpressionListShape, token_nodes, context, shapes);
Ok(())
}
SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes),
SyntaxShape::String => color_fallible_syntax_with(
&StringShape,
@ -126,10 +119,6 @@ impl FallibleColorSyntax for SyntaxShape {
) -> Result<(), ShellError> {
match self {
SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context),
SyntaxShape::List => {
color_syntax(&ExpressionListShape, token_nodes, context);
Ok(())
}
SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context),
SyntaxShape::String => {
color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context)
@ -147,14 +136,27 @@ impl FallibleColorSyntax for SyntaxShape {
}
impl ExpandExpression for SyntaxShape {
fn name(&self) -> &'static str {
match self {
SyntaxShape::Any => "any",
SyntaxShape::Int => "integer",
SyntaxShape::String => "string",
SyntaxShape::Member => "column name",
SyntaxShape::ColumnPath => "column path",
SyntaxShape::Number => "number",
SyntaxShape::Path => "file path",
SyntaxShape::Pattern => "glob pattern",
SyntaxShape::Block => "block",
}
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
match self {
SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context),
SyntaxShape::List => Err(ShellError::unimplemented("SyntaxShape:List")),
SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context),
SyntaxShape::String => expand_expr(&StringShape, token_nodes, context),
SyntaxShape::Member => {
@ -162,8 +164,9 @@ impl ExpandExpression for SyntaxShape {
Ok(syntax.to_expr())
}
SyntaxShape::ColumnPath => {
let Tagged { item: members, tag } =
expand_syntax(&ColumnPathShape, token_nodes, context)?;
let column_path = expand_syntax(&ColumnPathShape, token_nodes, context)?;
let Tagged { item: members, tag } = column_path.path();
Ok(hir::Expression::list(
members.into_iter().map(|s| s.to_expr()).collect(),
@ -182,7 +185,6 @@ impl std::fmt::Display for SyntaxShape {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
SyntaxShape::Any => write!(f, "Any"),
SyntaxShape::List => write!(f, "List"),
SyntaxShape::String => write!(f, "String"),
SyntaxShape::Int => write!(f, "Integer"),
SyntaxShape::Member => write!(f, "Member"),
@ -200,8 +202,6 @@ pub struct ExpandContext<'context> {
#[get = "pub(crate)"]
registry: &'context CommandRegistry,
#[get = "pub(crate)"]
span: Span,
#[get = "pub(crate)"]
source: &'context Text,
homedir: Option<PathBuf>,
}
@ -221,7 +221,6 @@ impl<'context> ExpandContext<'context> {
callback(ExpandContext {
registry: &registry,
span: Span::unknown(),
source,
homedir: None,
})
@ -237,11 +236,13 @@ pub trait TestSyntax: std::fmt::Debug + Copy {
}
pub trait ExpandExpression: std::fmt::Debug + Copy {
fn name(&self) -> &'static str;
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError>;
) -> Result<hir::Expression, ParseError>;
}
#[cfg(coloring_in_tokens)]
@ -303,35 +304,49 @@ pub trait ColorSyntax: std::fmt::Debug + Copy {
}
pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy {
type Output: std::fmt::Debug;
type Output: HasFallibleSpan + Clone + std::fmt::Debug + 'static;
fn name(&self) -> &'static str;
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError>;
) -> Result<Self::Output, ParseError>;
}
pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<T::Output, ShellError> {
trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
) -> Result<T::Output, ParseError> {
token_nodes.expand_frame(shape.name(), |token_nodes| {
shape.expand_syntax(token_nodes, context)
})
}
let result = shape.expand_syntax(token_nodes, context);
pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ParseError> {
token_nodes.expand_expr_frame(shape.name(), |token_nodes| {
shape.expand_expr(token_nodes, context)
})
}
match result {
Err(err) => {
trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
Err(err)
}
Ok(result) => {
trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
Ok(result)
}
}
#[cfg(coloring_in_tokens)]
pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> ((), U) {
(
(),
token_nodes.color_frame(shape.name(), |token_nodes| {
shape.color_syntax(&(), token_nodes, context)
}),
)
}
#[cfg(not(coloring_in_tokens))]
@ -363,20 +378,6 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
((), result)
}
#[cfg(coloring_in_tokens)]
pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> ((), U) {
(
(),
token_nodes.color_frame(shape.name(), |token_nodes| {
shape.color_syntax(&(), token_nodes, context)
}),
)
}
#[cfg(not(coloring_in_tokens))]
pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax<Info = U, Input = ()>, U>(
shape: &T,
@ -492,36 +493,18 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax<Info = U, Input
})
}
pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
shape: &T,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
trace!(target: "nu::expand_expression", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
let result = shape.expand_expr(token_nodes, context);
match result {
Err(err) => {
trace!(target: "nu::expand_expression", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
Err(err)
}
Ok(result) => {
trace!(target: "nu::expand_expression", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
Ok(result)
}
}
}
impl<T: ExpandExpression> ExpandSyntax for T {
type Output = hir::Expression;
fn name(&self) -> &'static str {
ExpandExpression::name(self)
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
ExpandExpression::expand_expr(self, token_nodes, context)
}
}
@ -537,7 +520,7 @@ pub trait SkipSyntax: std::fmt::Debug + Copy {
enum BarePathState {
Initial,
Seen(Span, Span),
Error(ShellError),
Error(ParseError),
}
impl BarePathState {
@ -549,7 +532,7 @@ impl BarePathState {
}
}
pub fn end(self, peeked: Peeked, reason: impl Into<String>) -> BarePathState {
pub fn end(self, peeked: Peeked, reason: &'static str) -> BarePathState {
match self {
BarePathState::Initial => BarePathState::Error(peeked.type_error(reason)),
BarePathState::Seen(start, end) => BarePathState::Seen(start, end),
@ -557,7 +540,7 @@ impl BarePathState {
}
}
pub fn into_bare(self) -> Result<Span, ShellError> {
pub fn into_bare(self) -> Result<Span, ParseError> {
match self {
BarePathState::Initial => unreachable!("into_bare in initial state"),
BarePathState::Seen(start, end) => Ok(start.until(end)),
@ -570,7 +553,7 @@ pub fn expand_bare<'a, 'b>(
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
predicate: impl Fn(&TokenNode) -> bool,
) -> Result<Span, ShellError> {
) -> Result<Span, ParseError> {
let mut state = BarePathState::Initial;
loop {
@ -603,11 +586,15 @@ pub struct BarePathShape;
impl ExpandSyntax for BarePathShape {
type Output = Span;
fn name(&self) -> &'static str {
"shorthand path"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Span, ShellError> {
) -> Result<Span, ParseError> {
expand_bare(token_nodes, context, |token| match token {
TokenNode::Token(Spanned {
item: RawToken::Bare,
@ -638,19 +625,21 @@ impl FallibleColorSyntax for BareShape {
_context: &ExpandContext,
shapes: &mut Vec<Spanned<FlatShape>>,
) -> Result<(), ShellError> {
token_nodes.peek_any_token("word", |token| match token {
// If it's a bare token, color it
TokenNode::Token(Spanned {
item: RawToken::Bare,
span,
}) => {
shapes.push((*input).spanned(*span));
Ok(())
}
token_nodes
.peek_any_token("word", |token| match token {
// If it's a bare token, color it
TokenNode::Token(Spanned {
item: RawToken::Bare,
span,
}) => {
shapes.push((*input).spanned(*span));
Ok(())
}
// otherwise, fail
other => Err(ShellError::type_error("word", other.tagged_type_name())),
})
// otherwise, fail
other => Err(ParseError::mismatch("word", other.tagged_type_name())),
})
.map_err(|err| err.into())
}
}
@ -677,7 +666,7 @@ impl FallibleColorSyntax for BareShape {
}) => Ok(span),
// otherwise, fail
other => Err(ShellError::type_error("word", other.tagged_type_name())),
other => Err(ParseError::mismatch("word", other.tagged_type_name())),
})?;
token_nodes.color_shape((*input).spanned(*span));
@ -689,11 +678,15 @@ impl FallibleColorSyntax for BareShape {
impl ExpandSyntax for BareShape {
type Output = Spanned<String>;
fn name(&self) -> &'static str {
"word"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let peeked = token_nodes.peek_any().not_eof("word")?;
match peeked.node {
@ -705,7 +698,7 @@ impl ExpandSyntax for BareShape {
Ok(span.spanned_string(context.source))
}
other => Err(ShellError::type_error("word", other.tagged_type_name())),
other => Err(ParseError::mismatch("word", other.tagged_type_name())),
}
}
}
@ -725,7 +718,7 @@ impl TestSyntax for BareShape {
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum CommandSignature {
Internal(Spanned<Arc<Command>>),
LiteralExternal { outer: Span, inner: Span },
@ -733,6 +726,34 @@ pub enum CommandSignature {
Expression(hir::Expression),
}
impl FormatDebug for CommandSignature {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
CommandSignature::Internal(internal) => {
f.say_str("internal", internal.span.slice(source))
}
CommandSignature::LiteralExternal { outer, .. } => {
f.say_str("external", outer.slice(source))
}
CommandSignature::External(external) => {
write!(f, "external:{}", external.slice(source))
}
CommandSignature::Expression(expr) => expr.fmt_debug(f, source),
}
}
}
impl HasSpan for CommandSignature {
fn span(&self) -> Span {
match self {
CommandSignature::Internal(spanned) => spanned.span,
CommandSignature::LiteralExternal { outer, .. } => *outer,
CommandSignature::External(span) => *span,
CommandSignature::Expression(expr) => expr.span,
}
}
}
impl CommandSignature {
pub fn to_expression(&self) -> hir::Expression {
match self {
@ -833,12 +854,17 @@ impl FallibleColorSyntax for PipelineShape {
#[cfg(coloring_in_tokens)]
impl ExpandSyntax for PipelineShape {
type Output = ClassifiedPipeline;
fn name(&self) -> &'static str {
"pipeline"
}
fn expand_syntax<'content, 'me>(
&self,
iterator: &'me mut TokensIterator<'content>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
let source = context.source;
) -> Result<Self::Output, ParseError> {
let start = iterator.span_at_cursor();
let peeked = iterator.peek_any().not_eof("pipeline")?;
let pipeline = peeked.commit().as_pipeline()?;
@ -851,25 +877,34 @@ impl ExpandSyntax for PipelineShape {
let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
let classified = iterator.child(tokens, move |token_nodes| {
classify_command(token_nodes, context, &source)
expand_syntax(&ClassifiedCommandShape, token_nodes, context)
})?;
out.push(classified);
}
Ok(ClassifiedPipeline { commands: out })
let end = iterator.span_at_cursor();
Ok(ClassifiedPipeline {
commands: out.spanned(start.until(end)),
})
}
}
#[cfg(not(coloring_in_tokens))]
impl ExpandSyntax for PipelineShape {
type Output = ClassifiedPipeline;
fn name(&self) -> &'static str {
"pipeline"
}
fn expand_syntax<'content, 'me>(
&self,
iterator: &'me mut TokensIterator<'content>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
let source = context.source;
) -> Result<Self::Output, ParseError> {
let start = iterator.span_at_cursor();
let peeked = iterator.peek_any().not_eof("pipeline")?;
let pipeline = peeked.commit().as_pipeline()?;
@ -882,13 +917,17 @@ impl ExpandSyntax for PipelineShape {
let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
let classified = iterator.child(tokens, move |token_nodes| {
classify_command(token_nodes, context, &source)
expand_syntax(&ClassifiedCommandShape, token_nodes, context)
})?;
out.push(classified);
}
Ok(ClassifiedPipeline { commands: out })
let end = iterator.span_at_cursor();
Ok(ClassifiedPipeline {
commands: out.spanned(start.until(end)),
})
}
}
@ -1014,11 +1053,15 @@ impl FallibleColorSyntax for CommandHeadShape {
impl ExpandSyntax for CommandHeadShape {
type Output = CommandSignature;
fn name(&self) -> &'static str {
"command head"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<CommandSignature, ShellError> {
) -> Result<CommandSignature, ParseError> {
let node =
parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_span, _| {
Ok(match token {
@ -1060,29 +1103,34 @@ pub struct ClassifiedCommandShape;
impl ExpandSyntax for ClassifiedCommandShape {
type Output = ClassifiedCommand;
fn name(&self) -> &'static str {
"classified command"
}
fn expand_syntax<'a, 'b>(
&self,
iterator: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let start = iterator.span_at_cursor();
let head = expand_syntax(&CommandHeadShape, iterator, context)?;
match &head {
CommandSignature::Expression(expr) => Err(ShellError::syntax_error(
"Unexpected expression in command position".tagged(expr.span),
)),
CommandSignature::Expression(expr) => {
Err(ParseError::mismatch("command", expr.tagged_type_name()))
}
// If the command starts with `^`, treat it as an external command no matter what
CommandSignature::External(name) => {
let name_str = name.slice(&context.source);
external_command(iterator, &context.source, name_str.tagged(name))
external_command(iterator, context, name_str.tagged(name))
}
CommandSignature::LiteralExternal { outer, inner } => {
let name_str = inner.slice(&context.source);
external_command(iterator, &context.source, name_str.tagged(outer))
external_command(iterator, context, name_str.tagged(outer))
}
CommandSignature::Internal(command) => {
@ -1094,11 +1142,14 @@ impl ExpandSyntax for ClassifiedCommandShape {
Some((positional, named)) => (positional, named),
};
let end = iterator.span_at_cursor();
let call = hir::Call {
head: Box::new(head.to_expression()),
positional,
named,
};
}
.spanned(start.until(end));
Ok(ClassifiedCommand::Internal(InternalCommand::new(
command.item.name().to_string(),
@ -1198,12 +1249,16 @@ impl FallibleColorSyntax for InternalCommandHeadShape {
}
impl ExpandExpression for InternalCommandHeadShape {
fn name(&self) -> &'static str {
"internal command head"
}
fn expand_expr(
&self,
token_nodes: &mut TokensIterator<'_>,
_context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
let peeked_head = token_nodes.peek_non_ws().not_eof("command head4")?;
) -> Result<hir::Expression, ParseError> {
let peeked_head = token_nodes.peek_non_ws().not_eof("command head")?;
let expr = match peeked_head.node {
TokenNode::Token(
@ -1219,8 +1274,8 @@ impl ExpandExpression for InternalCommandHeadShape {
}) => hir::RawExpression::Literal(hir::Literal::String(*inner_span)).spanned(*span),
node => {
return Err(ShellError::type_error(
"command head5",
return Err(ParseError::mismatch(
"command head",
node.tagged_type_name(),
))
}
@ -1238,16 +1293,16 @@ pub(crate) struct SingleError<'token> {
}
impl<'token> SingleError<'token> {
pub(crate) fn error(&self) -> ShellError {
ShellError::type_error(self.expected, self.node.type_name().tagged(self.node.span))
pub(crate) fn error(&self) -> ParseError {
ParseError::mismatch(self.expected, self.node.type_name().tagged(self.node.span))
}
}
fn parse_single_node<'a, 'b, T>(
token_nodes: &'b mut TokensIterator<'a>,
expected: &'static str,
callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ShellError>,
) -> Result<T, ShellError> {
callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ParseError>,
) -> Result<T, ParseError> {
token_nodes.peek_any_token(expected, |node| match node {
TokenNode::Token(token) => callback(
token.item,
@ -1258,7 +1313,7 @@ fn parse_single_node<'a, 'b, T>(
},
),
other => Err(ShellError::type_error(expected, other.tagged_type_name())),
other => Err(ParseError::mismatch(expected, other.tagged_type_name())),
})
}
@ -1360,22 +1415,21 @@ impl FallibleColorSyntax for WhitespaceShape {
impl ExpandSyntax for WhitespaceShape {
type Output = Span;
fn name(&self) -> &'static str {
"whitespace"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let peeked = token_nodes.peek_any().not_eof("whitespace")?;
let span = match peeked.node {
TokenNode::Whitespace(tag) => *tag,
other => {
return Err(ShellError::type_error(
"whitespace",
other.tagged_type_name(),
))
}
other => return Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
};
peeked.commit();
@ -1390,11 +1444,15 @@ pub struct SpacedExpression<T: ExpandExpression> {
}
impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
fn name(&self) -> &'static str {
"spaced expression"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// TODO: Make the name part of the trait
let peeked = token_nodes.peek_any().not_eof("whitespace")?;
@ -1404,10 +1462,7 @@ impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
expand_expr(&self.inner, token_nodes, context)
}
other => Err(ShellError::type_error(
"whitespace",
other.tagged_type_name(),
)),
other => Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
}
}
}
@ -1424,6 +1479,36 @@ pub struct MaybeSpacedExpression<T: ExpandExpression> {
#[derive(Debug, Copy, Clone)]
pub struct MaybeSpaceShape;
impl ExpandSyntax for MaybeSpaceShape {
type Output = Option<Span>;
fn name(&self) -> &'static str {
"maybe space"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
let peeked = token_nodes.peek_any().not_eof("whitespace");
let span = match peeked {
Err(_) => None,
Ok(peeked) => {
if let TokenNode::Whitespace(..) = peeked.node {
let node = peeked.commit();
Some(node.span())
} else {
None
}
}
};
Ok(span)
}
}
#[cfg(not(coloring_in_tokens))]
impl ColorSyntax for MaybeSpaceShape {
type Info = ();
@ -1544,11 +1629,15 @@ impl FallibleColorSyntax for SpaceShape {
}
impl<T: ExpandExpression> ExpandExpression for MaybeSpacedExpression<T> {
fn name(&self) -> &'static str {
"maybe space"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// TODO: Make the name part of the trait
let peeked = token_nodes.peek_any().not_eof("whitespace")?;
@ -1578,58 +1667,6 @@ fn expand_variable(span: Span, token_span: Span, source: &Text) -> hir::Expressi
}
}
fn classify_command(
mut iterator: &mut TokensIterator,
context: &ExpandContext,
source: &Text,
) -> Result<ClassifiedCommand, ShellError> {
let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?;
match &head {
CommandSignature::Expression(_) => Err(ShellError::syntax_error(
"Unexpected expression in command position".tagged(iterator.whole_span()),
)),
// If the command starts with `^`, treat it as an external command no matter what
CommandSignature::External(name) => {
let name_str = name.slice(source);
external_command(&mut iterator, source, name_str.tagged(name))
}
CommandSignature::LiteralExternal { outer, inner } => {
let name_str = inner.slice(source);
external_command(&mut iterator, source, name_str.tagged(outer))
}
CommandSignature::Internal(command) => {
let tail =
parse_command_tail(&command.signature(), &context, &mut iterator, command.span)?;
let (positional, named) = match tail {
None => (None, None),
Some((positional, named)) => (positional, named),
};
let call = hir::Call {
head: Box::new(head.to_expression()),
positional,
named,
};
Ok(ClassifiedCommand::Internal(InternalCommand::new(
command.name().to_string(),
Tag {
span: command.span,
anchor: None,
},
call,
)))
}
}
}
#[derive(Debug, Copy, Clone)]
pub struct CommandShape;

39
src/parser/hir/syntax_shape/block.rs
View File

@ -6,7 +6,8 @@ use crate::parser::{
hir::syntax_shape::{
color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax,
DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape,
ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, VariablePathShape,
ExpressionListShape, FallibleColorSyntax, MemberShape, ParseError, PathTailShape,
VariablePathShape,
},
hir::tokens_iterator::TokensIterator,
parse::token_tree::Delimiter,
@ -42,7 +43,7 @@ impl FallibleColorSyntax for AnyBlockShape {
match block {
// If so, color it as a block
Some((children, spans)) => {
let mut token_nodes = TokensIterator::new(children.item, context.span, false);
let mut token_nodes = TokensIterator::new(children.item, children.span, false);
color_syntax_with(
&DelimitedShape,
&(Delimiter::Brace, spans.0, spans.1),
@ -109,11 +110,15 @@ impl FallibleColorSyntax for AnyBlockShape {
}
impl ExpandExpression for AnyBlockShape {
fn name(&self) -> &'static str {
"any block"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let block = token_nodes.peek_non_ws().not_eof("block")?;
// is it just a block?
@ -121,11 +126,11 @@ impl ExpandExpression for AnyBlockShape {
match block {
Some((block, _tags)) => {
let mut iterator = TokensIterator::new(&block.item, context.span, false);
let mut iterator = TokensIterator::new(&block.item, block.span, false);
let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?;
return Ok(hir::RawExpression::Block(exprs).spanned(block.span));
return Ok(hir::RawExpression::Block(exprs.item).spanned(block.span));
}
_ => {}
}
@ -204,14 +209,18 @@ impl FallibleColorSyntax for ShorthandBlock {
}
impl ExpandExpression for ShorthandBlock {
fn name(&self) -> &'static str {
"shorthand block"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let path = expand_expr(&ShorthandPath, token_nodes, context)?;
let start = path.span;
let expr = continue_expression(path, token_nodes, context)?;
let expr = continue_expression(path, token_nodes, context);
let end = expr.span;
let block = hir::RawExpression::Block(vec![expr]).spanned(start.until(end));
@ -317,11 +326,15 @@ impl FallibleColorSyntax for ShorthandPath {
}
impl ExpandExpression for ShorthandPath {
fn name(&self) -> &'static str {
"shorthand path"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// if it's a variable path, that's the head part
let path = expand_expr(&VariablePathShape, token_nodes, context);
@ -339,7 +352,7 @@ impl ExpandExpression for ShorthandPath {
match tail {
Err(_) => return Ok(head),
Ok((tail, _)) => {
Ok(Spanned { item: tail, .. }) => {
// For each member that `PathTailShape` expanded, join it onto the existing expression
// to form a new path
for member in tail {
@ -446,11 +459,15 @@ impl FallibleColorSyntax for ShorthandHeadShape {
}
impl ExpandExpression for ShorthandHeadShape {
fn name(&self) -> &'static str {
"shorthand head"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// A shorthand path must not be at EOF
let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?;
@ -495,7 +512,7 @@ impl ExpandExpression for ShorthandHeadShape {
// Any other token is not a valid bare head
other => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
"shorthand path",
other.tagged_type_name(),
))

26
src/parser/hir/syntax_shape/expression.rs
View File

@ -12,7 +12,7 @@ use crate::parser::hir::syntax_shape::{
color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom,
expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape,
DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation,
ExpressionContinuationShape, FallibleColorSyntax, FlatShape,
ExpressionContinuationShape, FallibleColorSyntax, FlatShape, ParseError,
};
use crate::parser::{
hir,
@ -25,15 +25,19 @@ use std::path::PathBuf;
pub struct AnyExpressionShape;
impl ExpandExpression for AnyExpressionShape {
fn name(&self) -> &'static str {
"any expression"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// Look for an expression at the cursor
let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?;
continue_expression(head, token_nodes, context)
Ok(continue_expression(head, token_nodes, context))
}
}
@ -98,14 +102,14 @@ pub(crate) fn continue_expression(
mut head: hir::Expression,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> hir::Expression {
loop {
// Check to see whether there's any continuation after the head expression
let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context);
match continuation {
// If there's no continuation, return the head
Err(_) => return Ok(head),
Err(_) => return head,
// Otherwise, form a new expression by combining the head with the continuation
Ok(continuation) => match continuation {
// If the continuation is a `.member`, form a path with the new member
@ -174,11 +178,15 @@ pub(crate) fn continue_coloring_expression(
pub struct AnyExpressionStartShape;
impl ExpandExpression for AnyExpressionStartShape {
fn name(&self) -> &'static str {
"any expression start"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?;
match atom.item {
@ -445,13 +453,17 @@ impl FallibleColorSyntax for BareTailShape {
}
impl ExpandSyntax for BareTailShape {
fn name(&self) -> &'static str {
"word continuation"
}
type Output = Option<Span>;
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Option<Span>, ShellError> {
) -> Result<Option<Span>, ParseError> {
let mut end: Option<Span> = None;
loop {

48
src/parser/hir/syntax_shape/expression/atom.rs
View File

@ -90,40 +90,40 @@ impl<'tokens> SpannedAtomicToken<'tokens> {
&self,
context: &ExpandContext,
expected: &'static str,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
Ok(match &self.item {
AtomicToken::Eof { .. } => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
"eof atomic token".tagged(self.span),
))
}
AtomicToken::Error { .. } => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
"eof atomic token".tagged(self.span),
))
}
AtomicToken::Operator { .. } => {
return Err(ShellError::type_error(
expected,
"operator".tagged(self.span),
))
return Err(ParseError::mismatch(expected, "operator".tagged(self.span)))
}
AtomicToken::ShorthandFlag { .. } => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
"shorthand flag".tagged(self.span),
))
}
AtomicToken::LonghandFlag { .. } => {
return Err(ShellError::type_error(expected, "flag".tagged(self.span)))
return Err(ParseError::mismatch(expected, "flag".tagged(self.span)))
}
AtomicToken::Whitespace { .. } => {
return Err(ShellError::unimplemented("whitespace in AtomicToken"))
return Err(ParseError::mismatch(
expected,
"whitespace".tagged(self.span),
))
}
AtomicToken::Dot { .. } => {
return Err(ShellError::type_error(expected, "dot".tagged(self.span)))
return Err(ParseError::mismatch(expected, "dot".tagged(self.span)))
}
AtomicToken::Number { number } => {
Expression::number(number.to_number(context.source), self.span)
@ -381,7 +381,7 @@ pub fn expand_atom<'me, 'content>(
expected: &'static str,
context: &ExpandContext,
rule: ExpansionRule,
) -> Result<SpannedAtomicToken<'content>, ShellError> {
) -> Result<SpannedAtomicToken<'content>, ParseError> {
if token_nodes.at_end() {
match rule.allow_eof {
true => {
@ -390,7 +390,7 @@ pub fn expand_atom<'me, 'content>(
}
.spanned(Span::unknown()))
}
false => return Err(ShellError::unexpected_eof("anything", Tag::unknown())),
false => return Err(ParseError::unexpected_eof("anything", Span::unknown())),
}
}
@ -515,12 +515,13 @@ pub fn expand_atom<'me, 'content>(
// if whitespace is disallowed, return an error
WhitespaceHandling::RejectWhitespace => {
return Err(ShellError::syntax_error("Unexpected whitespace".tagged(
Tag {
return Err(ParseError::mismatch(
expected,
"whitespace".tagged(Tag {
span: *span,
anchor: None,
},
)))
}),
))
}
},
@ -544,7 +545,7 @@ pub fn expand_atom<'me, 'content>(
RawToken::Operator(_) if !rule.allow_operator => return Err(err.error()),
// rule.allow_external_command
RawToken::ExternalCommand(_) if !rule.allow_external_command => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
expected,
token.type_name().tagged(Tag {
span: token_span,
@ -554,10 +555,13 @@ pub fn expand_atom<'me, 'content>(
}
// rule.allow_external_word
RawToken::ExternalWord if !rule.allow_external_word => {
return Err(ShellError::invalid_external_word(Tag {
span: token_span,
anchor: None,
}))
return Err(ParseError::mismatch(
expected,
"external word".tagged(Tag {
span: token_span,
anchor: None,
}),
))
}
RawToken::Number(number) => AtomicToken::Number { number }.spanned(token_span),

7
src/parser/hir/syntax_shape/expression/delimited.rs
View File

@ -8,12 +8,15 @@ pub fn expand_delimited_square(
children: &Vec<TokenNode>,
span: Span,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let mut tokens = TokensIterator::new(&children, span, false);
let list = expand_syntax(&ExpressionListShape, &mut tokens, context);
Ok(hir::Expression::list(list?, Tag { span, anchor: None }))
Ok(hir::Expression::list(
list?.item,
Tag { span, anchor: None },
))
}
#[cfg(not(coloring_in_tokens))]

7
src/parser/hir/syntax_shape/expression/file_path.rs
View File

@ -1,6 +1,7 @@
use crate::parser::hir::syntax_shape::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
use crate::parser::hir::syntax_shape::{
expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape,
ParseError,
};
use crate::parser::{hir, hir::TokensIterator};
use crate::prelude::*;
@ -90,11 +91,15 @@ impl FallibleColorSyntax for FilePathShape {
}
impl ExpandExpression for FilePathShape {
fn name(&self) -> &'static str {
"file path"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let atom = expand_atom(token_nodes, "file path", context, ExpansionRule::new())?;
match atom.item {

27
src/parser/hir/syntax_shape/expression/list.rs
View File

@ -1,4 +1,4 @@
use crate::errors::ShellError;
use crate::errors::ParseError;
#[cfg(not(coloring_in_tokens))]
use crate::parser::hir::syntax_shape::FlatShape;
use crate::parser::{
@ -10,24 +10,36 @@ use crate::parser::{
},
hir::TokensIterator,
};
#[cfg(not(coloring_in_tokens))]
use crate::Spanned;
use crate::{DebugFormatter, FormatDebug, Spanned, SpannedItem};
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct ExpressionListShape;
impl FormatDebug for Spanned<Vec<hir::Expression>> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
FormatDebug::fmt_debug(&self.item, f, source)
}
}
impl ExpandSyntax for ExpressionListShape {
type Output = Vec<hir::Expression>;
type Output = Spanned<Vec<hir::Expression>>;
fn name(&self) -> &'static str {
"expression list"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<Vec<hir::Expression>, ShellError> {
) -> Result<Spanned<Vec<hir::Expression>>, ParseError> {
let mut exprs = vec![];
let start = token_nodes.span_at_cursor();
if token_nodes.at_end_possible_ws() {
return Ok(exprs);
return Ok(exprs.spanned(start));
}
let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?;
@ -36,7 +48,8 @@ impl ExpandSyntax for ExpressionListShape {
loop {
if token_nodes.at_end_possible_ws() {
return Ok(exprs);
let end = token_nodes.span_at_cursor();
return Ok(exprs.spanned(start.until(end)));
}
let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?;

30
src/parser/hir/syntax_shape/expression/number.rs
View File

@ -1,6 +1,6 @@
use crate::parser::hir::syntax_shape::{
expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule,
FallibleColorSyntax, FlatShape,
FallibleColorSyntax, FlatShape, ParseError,
};
use crate::parser::{
hir,
@ -13,11 +13,15 @@ use crate::prelude::*;
pub struct NumberShape;
impl ExpandExpression for NumberShape {
fn name(&self) -> &'static str {
"number"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "Number", |token, token_span, err| {
Ok(match token {
RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
@ -28,10 +32,13 @@ impl ExpandExpression for NumberShape {
hir::Expression::external_command(tag, token_span)
}
RawToken::ExternalWord => {
return Err(ShellError::invalid_external_word(Tag {
span: token_span,
anchor: None,
}))
return Err(ParseError::mismatch(
"number",
"syntax error".tagged(Tag {
span: token_span,
anchor: None,
}),
))
}
RawToken::Variable(tag) => hir::Expression::variable(tag, token_span),
RawToken::Number(number) => {
@ -111,16 +118,19 @@ impl FallibleColorSyntax for NumberShape {
pub struct IntShape;
impl ExpandExpression for IntShape {
fn name(&self) -> &'static str {
"integer"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "Integer", |token, token_span, err| {
Ok(match token {
RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
RawToken::ExternalWord => {
return Err(ShellError::invalid_external_word(token_span))
RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
return Err(err.error())
}
RawToken::Variable(span) if span.slice(context.source) == "it" => {
hir::Expression::it_variable(span, token_span)

14
src/parser/hir/syntax_shape/expression/pattern.rs
View File

@ -1,6 +1,6 @@
use crate::parser::hir::syntax_shape::{
expand_atom, expand_bare, expression::expand_file_path, AtomicToken, ExpandContext,
ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape,
ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError,
};
use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
use crate::prelude::*;
@ -66,11 +66,15 @@ impl FallibleColorSyntax for PatternShape {
}
impl ExpandExpression for PatternShape {
fn name(&self) -> &'static str {
"glob pattern"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::new())?;
match atom.item {
@ -91,11 +95,15 @@ pub struct BarePatternShape;
impl ExpandSyntax for BarePatternShape {
type Output = Span;
fn name(&self) -> &'static str {
"bare pattern"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Span, ShellError> {
) -> Result<Span, ParseError> {
expand_bare(token_nodes, context, |token| match token {
TokenNode::Token(Spanned {
item: RawToken::Bare,

26
src/parser/hir/syntax_shape/expression/string.rs
View File

@ -1,6 +1,6 @@
use crate::parser::hir::syntax_shape::{
expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression,
ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax,
ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, TestSyntax,
};
use crate::parser::hir::tokens_iterator::Peeked;
use crate::parser::{hir, hir::TokensIterator, RawToken};
@ -75,32 +75,24 @@ impl FallibleColorSyntax for StringShape {
}
impl ExpandExpression for StringShape {
fn name(&self) -> &'static str {
"string"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
parse_single_node(token_nodes, "String", |token, token_span, _| {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "String", |token, token_span, err| {
Ok(match token {
RawToken::GlobPattern => {
return Err(ShellError::type_error(
"String",
"glob pattern".tagged(token_span),
))
}
RawToken::Operator(..) => {
return Err(ShellError::type_error(
"String",
"operator".tagged(token_span),
))
RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
return Err(err.error())
}
RawToken::Variable(span) => expand_variable(span, token_span, &context.source),
RawToken::ExternalCommand(span) => {
hir::Expression::external_command(span, token_span)
}
RawToken::ExternalWord => {
return Err(ShellError::invalid_external_word(token_span))
}
RawToken::Number(_) => hir::Expression::bare(token_span),
RawToken::Bare => hir::Expression::bare(token_span),
RawToken::String(span) => hir::Expression::string(span, token_span),

27
src/parser/hir/syntax_shape/expression/unit.rs
View File

@ -1,5 +1,5 @@
use crate::data::meta::Span;
use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax};
use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax, ParseError};
use crate::parser::parse::tokens::RawNumber;
use crate::parser::parse::unit::Unit;
use crate::parser::{hir::TokensIterator, RawToken, TokenNode};
@ -9,18 +9,34 @@ use nom::bytes::complete::tag;
use nom::character::complete::digit1;
use nom::combinator::{all_consuming, opt, value};
use nom::IResult;
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct UnitShape;
impl FormatDebug for Spanned<(Spanned<RawNumber>, Spanned<Unit>)> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
let dict = indexmap::indexmap! {
"number" => format!("{}", self.item.0.item.debug(source)),
"unit" => format!("{}", self.item.1.debug(source)),
};
f.say_dict("unit", dict)
}
}
impl ExpandSyntax for UnitShape {
type Output = Spanned<(Spanned<RawNumber>, Spanned<Unit>)>;
fn name(&self) -> &'static str {
"unit"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ShellError> {
) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ParseError> {
let peeked = token_nodes.peek_any().not_eof("unit")?;
let span = match peeked.node {
@ -34,12 +50,7 @@ impl ExpandSyntax for UnitShape {
let unit = unit_size(span.slice(context.source), *span);
let (_, (number, unit)) = match unit {
Err(_) => {
return Err(ShellError::type_error(
"unit",
"word".tagged(Tag::unknown()),
))
}
Err(_) => return Err(ParseError::mismatch("unit", "word".tagged(Tag::unknown()))),
Ok((number, unit)) => (number, unit),
};

246
src/parser/hir/syntax_shape/expression/variable_path.rs
View File

@ -1,21 +1,28 @@
use crate::parser::hir::syntax_shape::{
color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax,
parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression,
ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape,
TestSyntax, WhitespaceShape,
ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, Peeked, SkipSyntax,
StringShape, TestSyntax, WhitespaceShape,
};
use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken};
use crate::prelude::*;
use derive_new::new;
use getset::Getters;
use std::fmt;
#[derive(Debug, Copy, Clone)]
pub struct VariablePathShape;
impl ExpandExpression for VariablePathShape {
fn name(&self) -> &'static str {
"variable path"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
) -> Result<hir::Expression, ParseError> {
// 1. let the head be the first token, expecting a variable
// 2. let the tail be an empty list of members
// 2. while the next token (excluding ws) is a dot:
@ -200,12 +207,17 @@ impl FallibleColorSyntax for PathTailShape {
}
impl ExpandSyntax for PathTailShape {
type Output = (Vec<Spanned<String>>, Span);
type Output = Spanned<Vec<Spanned<String>>>;
fn name(&self) -> &'static str {
"path continuation"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
let mut end: Option<Span> = None;
let mut tail = vec![];
@ -223,7 +235,7 @@ impl ExpandSyntax for PathTailShape {
match end {
None => {
return Err(ShellError::type_error("path tail", {
return Err(ParseError::mismatch("path tail", {
let typed_span = token_nodes.typed_span_at_cursor();
Tagged {
@ -233,17 +245,41 @@ impl ExpandSyntax for PathTailShape {
}))
}
Some(end) => Ok((tail, end)),
Some(end) => Ok(tail.spanned(end)),
}
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum ExpressionContinuation {
DotSuffix(Span, Spanned<String>),
InfixSuffix(Spanned<Operator>, Expression),
}
impl FormatDebug for ExpressionContinuation {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
ExpressionContinuation::DotSuffix(dot, rest) => {
f.say_str("dot suffix", dot.until(rest.span).slice(source))
}
ExpressionContinuation::InfixSuffix(operator, expr) => {
f.say_str("infix suffix", operator.span.until(expr.span).slice(source))
}
}
}
}
impl HasSpan for ExpressionContinuation {
fn span(&self) -> Span {
match self {
ExpressionContinuation::DotSuffix(dot, column) => dot.until(column.span),
ExpressionContinuation::InfixSuffix(operator, expression) => {
operator.span.until(expression.span)
}
}
}
}
/// An expression continuation
#[derive(Debug, Copy, Clone)]
pub struct ExpressionContinuationShape;
@ -251,11 +287,15 @@ pub struct ExpressionContinuationShape;
impl ExpandSyntax for ExpressionContinuationShape {
type Output = ExpressionContinuation;
fn name(&self) -> &'static str {
"expression continuation"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<ExpressionContinuation, ShellError> {
) -> Result<ExpressionContinuation, ParseError> {
// Try to expand a `.`
let dot = expand_syntax(&DotShape, token_nodes, context);
@ -270,7 +310,7 @@ impl ExpandSyntax for ExpressionContinuationShape {
// Otherwise, we expect an infix operator and an expression next
Err(_) => {
let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?;
let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?.item;
let next = expand_expr(&AnyExpressionShape, token_nodes, context)?;
Ok(ExpressionContinuation::InfixSuffix(op, next))
@ -390,12 +430,16 @@ impl FallibleColorSyntax for ExpressionContinuationShape {
pub struct VariableShape;
impl ExpandExpression for VariableShape {
fn name(&self) -> &'static str {
"variable"
}
fn expand_expr<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<hir::Expression, ShellError> {
parse_single_node(token_nodes, "variable", |token, token_tag, _| {
) -> Result<hir::Expression, ParseError> {
parse_single_node(token_nodes, "variable", |token, token_tag, err| {
Ok(match token {
RawToken::Variable(tag) => {
if tag.slice(context.source) == "it" {
@ -404,12 +448,7 @@ impl ExpandExpression for VariableShape {
hir::Expression::variable(tag, token_tag)
}
}
_ => {
return Err(ShellError::type_error(
"variable",
token.type_name().tagged(token_tag),
))
}
_ => return Err(err.error()),
})
})
}
@ -435,7 +474,7 @@ impl FallibleColorSyntax for VariableShape {
);
let atom = match atom {
Err(err) => return Err(err),
Err(err) => return Err(err.into()),
Ok(atom) => atom,
};
@ -476,7 +515,7 @@ impl FallibleColorSyntax for VariableShape {
);
let atom = match atom {
Err(err) => return Err(err),
Err(err) => return Err(err.into()),
Ok(atom) => atom,
};
@ -489,7 +528,7 @@ impl FallibleColorSyntax for VariableShape {
token_nodes.color_shape(FlatShape::ItVariable.spanned(atom.span));
Ok(())
}
_ => Err(ShellError::type_error("variable", atom.tagged_type_name())),
_ => Err(ParseError::mismatch("variable", atom.tagged_type_name()).into()),
}
}
}
@ -500,6 +539,24 @@ pub enum Member {
Bare(Span),
}
impl FormatDebug for Member {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
Member::String(outer, _) => write!(f, "member ({})", outer.slice(source)),
Member::Bare(bare) => write!(f, "member ({})", bare.slice(source)),
}
}
}
impl HasSpan for Member {
fn span(&self) -> Span {
match self {
Member::String(outer, ..) => *outer,
Member::Bare(name) => *name,
}
}
}
impl Member {
pub(crate) fn to_expr(&self) -> hir::Expression {
match self {
@ -538,7 +595,7 @@ enum ColumnPathState {
LeadingDot(Span),
Dot(Span, Vec<Member>, Span),
Member(Span, Vec<Member>),
Error(ShellError),
Error(ParseError),
}
impl ColumnPathState {
@ -546,10 +603,10 @@ impl ColumnPathState {
match self {
ColumnPathState::Initial => ColumnPathState::LeadingDot(dot),
ColumnPathState::LeadingDot(_) => {
ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Dot(..) => {
ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot),
ColumnPathState::Error(err) => ColumnPathState::Error(err),
@ -570,20 +627,20 @@ impl ColumnPathState {
})
}
ColumnPathState::Member(..) => {
ColumnPathState::Error(ShellError::type_error("column", member.tagged_type_name()))
ColumnPathState::Error(ParseError::mismatch("column", member.tagged_type_name()))
}
ColumnPathState::Error(err) => ColumnPathState::Error(err),
}
}
pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ShellError> {
pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ParseError> {
match self {
ColumnPathState::Initial => Err(next.type_error("column path")),
ColumnPathState::LeadingDot(dot) => {
Err(ShellError::type_error("column", "dot".tagged(dot)))
Err(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Dot(_tag, _members, dot) => {
Err(ShellError::type_error("column", "dot".tagged(dot)))
Err(ParseError::mismatch("column", "dot".tagged(dot)))
}
ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)),
ColumnPathState::Error(err) => Err(err),
@ -594,7 +651,7 @@ impl ColumnPathState {
pub fn expand_column_path<'a, 'b>(
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Tagged<Vec<Member>>, ShellError> {
) -> Result<Tagged<Vec<Member>>, ParseError> {
let mut state = ColumnPathState::Initial;
loop {
@ -720,15 +777,43 @@ impl FallibleColorSyntax for ColumnPathShape {
}
}
impl FormatDebug for Tagged<Vec<Member>> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
self.item.fmt_debug(f, source)
}
}
#[derive(Debug, Clone, Getters, new)]
pub struct ColumnPath {
#[get = "pub"]
path: Tagged<Vec<Member>>,
}
impl HasSpan for ColumnPath {
fn span(&self) -> Span {
self.path.tag.span
}
}
impl FormatDebug for ColumnPath {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say("column path", self.path.item.debug(source))
}
}
impl ExpandSyntax for ColumnPathShape {
type Output = Tagged<Vec<Member>>;
type Output = ColumnPath;
fn name(&self) -> &'static str {
"column path"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
expand_column_path(token_nodes, context)
) -> Result<Self::Output, ParseError> {
Ok(ColumnPath::new(expand_column_path(token_nodes, context)?))
}
}
@ -806,11 +891,15 @@ impl FallibleColorSyntax for MemberShape {
impl ExpandSyntax for MemberShape {
type Output = Member;
fn name(&self) -> &'static str {
"column"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &mut TokensIterator<'_>,
context: &ExpandContext,
) -> Result<Member, ShellError> {
) -> Result<Member, ParseError> {
let bare = BareShape.test(token_nodes, context);
if let Some(peeked) = bare {
let node = peeked.not_eof("column")?.commit();
@ -906,16 +995,20 @@ impl SkipSyntax for DotShape {
impl ExpandSyntax for DotShape {
type Output = Span;
fn name(&self) -> &'static str {
"dot"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
) -> Result<Self::Output, ParseError> {
parse_single_node(token_nodes, "dot", |token, token_span, _| {
Ok(match token {
RawToken::Operator(Operator::Dot) => token_span,
_ => {
return Err(ShellError::type_error(
return Err(ParseError::mismatch(
"dot",
token.type_name().tagged(token_span),
))
@ -950,7 +1043,7 @@ impl FallibleColorSyntax for InfixShape {
parse_single_node(
checkpoint.iterator,
"infix operator",
|token, token_span, _| {
|token, token_span, err| {
match token {
// If it's an operator (and not `.`), it's a match
RawToken::Operator(operator) if operator != Operator::Dot => {
@ -959,10 +1052,7 @@ impl FallibleColorSyntax for InfixShape {
}
// Otherwise, it's not a match
_ => Err(ShellError::type_error(
"infix operator",
token.type_name().tagged(token_span),
)),
_ => Err(err.error()),
}
},
)?;
@ -1006,7 +1096,7 @@ impl FallibleColorSyntax for InfixShape {
RawToken::Operator(operator) if operator != Operator::Dot => Ok(token_span),
// Otherwise, it's not a match
_ => Err(ShellError::type_error(
_ => Err(ParseError::mismatch(
"infix operator",
token.type_name().tagged(token_span),
)),
@ -1026,46 +1116,72 @@ impl FallibleColorSyntax for InfixShape {
}
}
impl FormatDebug for Spanned<(Span, Spanned<Operator>, Span)> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str("operator", self.item.1.span.slice(source))
}
}
impl ExpandSyntax for InfixShape {
type Output = (Span, Spanned<Operator>, Span);
type Output = Spanned<(Span, Spanned<Operator>, Span)>;
fn name(&self) -> &'static str {
"infix operator"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
context: &ExpandContext,
) -> Result<Self::Output, ShellError> {
let checkpoint = token_nodes.checkpoint();
) -> Result<Self::Output, ParseError> {
let mut checkpoint = token_nodes.checkpoint();
// An infix operator must be prefixed by whitespace
let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
// Parse the next TokenNode after the whitespace
let operator = parse_single_node(
checkpoint.iterator,
"infix operator",
|token, token_span, _| {
Ok(match token {
// If it's an operator (and not `.`), it's a match
RawToken::Operator(operator) if operator != Operator::Dot => {
operator.spanned(token_span)
}
// Otherwise, it's not a match
_ => {
return Err(ShellError::type_error(
"infix operator",
token.type_name().tagged(token_span),
))
}
})
},
)?;
let operator = expand_syntax(&InfixInnerShape, &mut checkpoint.iterator, context)?;
// An infix operator must be followed by whitespace
let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
checkpoint.commit();
Ok((start, operator, end))
Ok((start, operator, end).spanned(start.until(end)))
}
}
#[derive(Debug, Copy, Clone)]
pub struct InfixInnerShape;
impl FormatDebug for Spanned<Operator> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str("operator", self.span.slice(source))
}
}
impl ExpandSyntax for InfixInnerShape {
type Output = Spanned<Operator>;
fn name(&self) -> &'static str {
"infix inner"
}
fn expand_syntax<'a, 'b>(
&self,
token_nodes: &'b mut TokensIterator<'a>,
_context: &ExpandContext,
) -> Result<Self::Output, ParseError> {
parse_single_node(token_nodes, "infix operator", |token, token_span, err| {
Ok(match token {
// If it's an operator (and not `.`), it's a match
RawToken::Operator(operator) if operator != Operator::Dot => {
operator.spanned(token_span)
}
// Otherwise, it's not a match
_ => return Err(err.error()),
})
})
}
}

2
src/parser/hir/syntax_shape/flat_shape.rs
View File

@ -1,5 +1,5 @@
use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode};
use crate::{Span, Spanned, SpannedItem, Text};
use crate::{HasSpan, Span, Spanned, SpannedItem, Text};
#[derive(Debug, Copy, Clone)]
pub enum FlatShape {

219
src/parser/hir/tokens_iterator.rs
View File

@ -1,25 +1,38 @@
pub(crate) mod debug;
use self::debug::Tracer;
use self::debug::{ColorTracer, ExpandTracer};
use crate::errors::ShellError;
#[cfg(coloring_in_tokens)]
use crate::parser::hir::syntax_shape::FlatShape;
use crate::parser::hir::Expression;
use crate::parser::TokenNode;
use crate::prelude::*;
use crate::{Span, Spanned, SpannedItem};
#[allow(unused)]
use getset::{Getters, MutGetters};
#[derive(Getters, Debug)]
pub struct TokensIteratorState<'content> {
tokens: &'content [TokenNode],
span: Span,
skip_ws: bool,
index: usize,
seen: indexmap::IndexSet<usize>,
#[cfg(coloring_in_tokens)]
#[cfg_attr(coloring_in_tokens, get = "pub")]
shapes: Vec<Spanned<FlatShape>>,
cfg_if::cfg_if! {
if #[cfg(coloring_in_tokens)] {
#[derive(Getters, Debug)]
pub struct TokensIteratorState<'content> {
tokens: &'content [TokenNode],
span: Span,
skip_ws: bool,
index: usize,
seen: indexmap::IndexSet<usize>,
#[get = "pub"]
shapes: Vec<Spanned<FlatShape>>,
}
} else {
#[derive(Getters, Debug)]
pub struct TokensIteratorState<'content> {
tokens: &'content [TokenNode],
span: Span,
skip_ws: bool,
index: usize,
seen: indexmap::IndexSet<usize>,
}
}
}
#[derive(Getters, MutGetters, Debug)]
@ -29,7 +42,10 @@ pub struct TokensIterator<'content> {
state: TokensIteratorState<'content>,
#[get = "pub"]
#[get_mut = "pub"]
tracer: Tracer,
color_tracer: ColorTracer,
#[get = "pub"]
#[get_mut = "pub"]
expand_tracer: ExpandTracer,
}
#[derive(Debug)]
@ -83,12 +99,9 @@ impl<'content, 'me> Peeked<'content, 'me> {
Some(node)
}
pub fn not_eof(
self,
expected: impl Into<String>,
) -> Result<PeekedNode<'content, 'me>, ShellError> {
pub fn not_eof(self, expected: &'static str) -> Result<PeekedNode<'content, 'me>, ParseError> {
match self.node {
None => Err(ShellError::unexpected_eof(
None => Err(ParseError::unexpected_eof(
expected,
self.iterator.eof_span(),
)),
@ -101,7 +114,7 @@ impl<'content, 'me> Peeked<'content, 'me> {
}
}
pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
pub fn type_error(&self, expected: &'static str) -> ParseError {
peek_error(&self.node, self.iterator.eof_span(), expected)
}
}
@ -129,19 +142,15 @@ impl<'content, 'me> PeekedNode<'content, 'me> {
pub fn rollback(self) {}
pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
pub fn type_error(&self, expected: &'static str) -> ParseError {
peek_error(&Some(self.node), self.iterator.eof_span(), expected)
}
}
pub fn peek_error(
node: &Option<&TokenNode>,
eof_span: Span,
expected: impl Into<String>,
) -> ShellError {
pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError {
match node {
None => ShellError::unexpected_eof(expected, eof_span),
Some(node) => ShellError::type_error(expected, node.tagged_type_name()),
None => ParseError::unexpected_eof(expected, eof_span),
Some(node) => ParseError::mismatch(expected, node.tagged_type_name()),
}
}
@ -161,7 +170,8 @@ impl<'content> TokensIterator<'content> {
#[cfg(coloring_in_tokens)]
shapes: vec![],
},
tracer: Tracer::new(),
color_tracer: ColorTracer::new(),
expand_tracer: ExpandTracer::new(),
}
}
@ -188,7 +198,7 @@ impl<'content> TokensIterator<'content> {
#[cfg(coloring_in_tokens)]
pub fn color_shape(&mut self, shape: Spanned<FlatShape>) {
self.with_tracer(|_, tracer| tracer.add_shape(shape));
self.with_color_tracer(|_, tracer| tracer.add_shape(shape));
self.state.shapes.push(shape);
}
@ -201,7 +211,7 @@ impl<'content> TokensIterator<'content> {
(len..(shapes.len())).map(|i| shapes[i]).collect()
};
self.with_tracer(|_, tracer| {
self.with_color_tracer(|_, tracer| {
for shape in new_shapes {
tracer.add_shape(shape)
}
@ -233,8 +243,11 @@ impl<'content> TokensIterator<'content> {
let mut shapes = vec![];
std::mem::swap(&mut shapes, &mut self.state.shapes);
let mut tracer = Tracer::new();
std::mem::swap(&mut tracer, &mut self.tracer);
let mut color_tracer = ColorTracer::new();
std::mem::swap(&mut color_tracer, &mut self.color_tracer);
let mut expand_tracer = ExpandTracer::new();
std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
let mut iterator = TokensIterator {
state: TokensIteratorState {
@ -245,13 +258,15 @@ impl<'content> TokensIterator<'content> {
seen: indexmap::IndexSet::new(),
shapes,
},
tracer,
color_tracer,
expand_tracer,
};
let result = block(&mut iterator);
std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes);
std::mem::swap(&mut iterator.tracer, &mut self.tracer);
std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
result
}
@ -262,8 +277,11 @@ impl<'content> TokensIterator<'content> {
tokens: Spanned<&'me [TokenNode]>,
block: impl FnOnce(&mut TokensIterator<'me>) -> T,
) -> T {
let mut tracer = Tracer::new();
std::mem::swap(&mut tracer, &mut self.tracer);
let mut color_tracer = ColorTracer::new();
std::mem::swap(&mut color_tracer, &mut self.color_tracer);
let mut expand_tracer = ExpandTracer::new();
std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
let mut iterator = TokensIterator {
state: TokensIteratorState {
@ -273,19 +291,34 @@ impl<'content> TokensIterator<'content> {
index: 0,
seen: indexmap::IndexSet::new(),
},
tracer,
color_tracer,
expand_tracer,
};
let result = block(&mut iterator);
std::mem::swap(&mut iterator.tracer, &mut self.tracer);
std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
result
}
pub fn with_tracer(&mut self, block: impl FnOnce(&mut TokensIteratorState, &mut Tracer)) {
pub fn with_color_tracer(
&mut self,
block: impl FnOnce(&mut TokensIteratorState, &mut ColorTracer),
) {
let state = &mut self.state;
let tracer = &mut self.tracer;
let color_tracer = &mut self.color_tracer;
block(state, color_tracer)
}
pub fn with_expand_tracer(
&mut self,
block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer),
) {
let state = &mut self.state;
let tracer = &mut self.expand_tracer;
block(state, tracer)
}
@ -296,32 +329,77 @@ impl<'content> TokensIterator<'content> {
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> T,
) -> T {
self.with_tracer(|_, tracer| tracer.start(desc));
self.with_color_tracer(|_, tracer| tracer.start(desc));
let result = block(self);
self.with_tracer(|_, tracer| {
self.with_color_tracer(|_, tracer| {
tracer.success();
});
result
}
pub fn expand_frame<T>(
&mut self,
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> Result<T, ParseError>,
) -> Result<T, ParseError>
where
T: std::fmt::Debug + FormatDebug + Clone + HasFallibleSpan + 'static,
{
self.with_expand_tracer(|_, tracer| tracer.start(desc));
let result = block(self);
self.with_expand_tracer(|_, tracer| match &result {
Ok(result) => {
tracer.add_result(Box::new(result.clone()));
tracer.success();
}
Err(err) => tracer.failed(err),
});
result
}
pub fn expand_expr_frame(
&mut self,
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> Result<Expression, ParseError>,
) -> Result<Expression, ParseError> {
self.with_expand_tracer(|_, tracer| tracer.start(desc));
let result = block(self);
self.with_expand_tracer(|_, tracer| match &result {
Ok(expr) => {
tracer.add_expr(expr.clone());
tracer.success()
}
Err(err) => tracer.failed(err),
});
result
}
pub fn color_fallible_frame<T>(
&mut self,
desc: &'static str,
block: impl FnOnce(&mut TokensIterator) -> Result<T, ShellError>,
) -> Result<T, ShellError> {
self.with_tracer(|_, tracer| tracer.start(desc));
self.with_color_tracer(|_, tracer| tracer.start(desc));
if self.at_end() {
self.with_tracer(|_, tracer| tracer.eof_frame());
self.with_color_tracer(|_, tracer| tracer.eof_frame());
return Err(ShellError::unexpected_eof("coloring", Tag::unknown()));
}
let result = block(self);
self.with_tracer(|_, tracer| match &result {
self.with_color_tracer(|_, tracer| match &result {
Ok(_) => {
tracer.success();
}
@ -431,10 +509,6 @@ impl<'content> TokensIterator<'content> {
}
}
pub fn whole_span(&self) -> Span {
self.state.span
}
pub fn span_at_cursor(&mut self) -> Span {
let next = self.peek_any();
@ -491,27 +565,22 @@ impl<'content> TokensIterator<'content> {
self.state.index = 0;
}
pub fn clone(&self) -> TokensIterator<'content> {
let state = &self.state;
TokensIterator {
state: TokensIteratorState {
tokens: state.tokens,
span: state.span,
index: state.index,
seen: state.seen.clone(),
skip_ws: state.skip_ws,
#[cfg(coloring_in_tokens)]
shapes: state.shapes.clone(),
},
tracer: self.tracer.clone(),
}
}
// Get the next token, not including whitespace
pub fn next_non_ws(&mut self) -> Option<&TokenNode> {
let mut peeked = start_next(self, true);
peeked.commit()
}
// pub fn clone(&self) -> TokensIterator<'content> {
// let state = &self.state;
// TokensIterator {
// state: TokensIteratorState {
// tokens: state.tokens,
// span: state.span,
// index: state.index,
// seen: state.seen.clone(),
// skip_ws: state.skip_ws,
// #[cfg(coloring_in_tokens)]
// shapes: state.shapes.clone(),
// },
// color_tracer: self.color_tracer.clone(),
// expand_tracer: self.expand_tracer.clone(),
// }
// }
// Peek the next token, not including whitespace
pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> {
@ -527,8 +596,8 @@ impl<'content> TokensIterator<'content> {
pub fn peek_any_token<'me, T>(
&'me mut self,
expected: &'static str,
block: impl FnOnce(&'content TokenNode) -> Result<T, ShellError>,
) -> Result<T, ShellError> {
block: impl FnOnce(&'content TokenNode) -> Result<T, ParseError>,
) -> Result<T, ParseError> {
let peeked = start_next(self, false);
let peeked = peeked.not_eof(expected);
@ -557,9 +626,11 @@ impl<'content> TokensIterator<'content> {
}
pub fn debug_remaining(&self) -> Vec<TokenNode> {
let mut tokens = self.clone();
tokens.restart();
tokens.cloned().collect()
// TODO: TODO: TODO: Clean up
vec![]
// let mut tokens = self.clone();
// tokens.restart();
// tokens.cloned().collect()
}
}

357
src/parser/hir/tokens_iterator/debug.rs
View File

@ -1,13 +1,13 @@
use crate::errors::ShellError;
use crate::parser::hir::syntax_shape::FlatShape;
#![allow(unused)]
pub(crate) mod color_trace;
pub(crate) mod expand_trace;
pub(crate) use self::color_trace::*;
pub(crate) use self::expand_trace::*;
use crate::parser::hir::tokens_iterator::TokensIteratorState;
use crate::prelude::*;
use crate::traits::ToDebug;
use ansi_term::Color;
use log::trace;
use ptree::*;
use std::borrow::Cow;
use std::io;
#[derive(Debug)]
pub(crate) enum DebugIteratorToken {
@ -36,344 +36,3 @@ pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec<Deb
out
}
#[derive(Debug, Clone)]
pub enum FrameChild {
#[allow(unused)]
Shape(Spanned<FlatShape>),
Frame(ColorFrame),
}
impl FrameChild {
fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
match self {
FrameChild::Shape(shape) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
FrameChild::Frame(frame) => frame.colored_leaf_description(f),
}
}
fn into_tree_child(self, text: &Text) -> TreeChild {
match self {
FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
}
}
}
#[derive(Debug, Clone)]
pub struct ColorFrame {
description: &'static str,
children: Vec<FrameChild>,
error: Option<ShellError>,
}
impl ColorFrame {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
if self.has_only_error_descendents() {
if self.children.len() == 0 {
write!(
f,
"{}",
Color::White.bold().on(Color::Red).paint(self.description)
)
} else {
write!(f, "{}", Color::Red.normal().paint(self.description))
}
} else if self.has_descendent_shapes() {
write!(f, "{}", Color::Green.normal().paint(self.description))
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))
}
}
fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
if self.children.len() == 1 {
let child = &self.children[0];
self.colored_leaf_description(f)?;
write!(f, " -> ")?;
child.colored_leaf_description(text, f)
} else {
self.colored_leaf_description(f)
}
}
fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
if self.children.len() == 1 {
let child = &self.children[0];
match child {
FrameChild::Shape(_) => vec![],
FrameChild::Frame(frame) => frame.tree_children(text),
}
} else {
self.tree_children(text)
}
}
fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
self.children
.clone()
.into_iter()
.map(|c| c.into_tree_child(text))
.collect()
}
#[allow(unused)]
fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.children.push(FrameChild::Shape(shape))
}
fn has_child_shapes(&self) -> bool {
self.any_child_shape(|_| true)
}
fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Shape(shape) => {
if predicate(*shape) {
return true;
}
}
_ => {}
}
}
false
}
fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Frame(frame) => {
if predicate(frame) {
return true;
}
}
_ => {}
}
}
false
}
fn has_descendent_shapes(&self) -> bool {
if self.has_child_shapes() {
true
} else {
self.any_child_frame(|frame| frame.has_descendent_shapes())
}
}
fn has_only_error_descendents(&self) -> bool {
if self.children.len() == 0 {
// if this frame has no children at all, it has only error descendents if this frame
// is an error
self.error.is_some()
} else {
// otherwise, it has only error descendents if all of its children terminate in an
// error (transitively)
let mut seen_error = false;
for child in &self.children {
match child {
// if this frame has at least one child shape, this frame has non-error descendents
FrameChild::Shape(_) => return false,
FrameChild::Frame(frame) => {
// if the chi
if frame.has_only_error_descendents() {
seen_error = true;
} else {
return false;
}
}
}
}
seen_error
}
}
}
#[derive(Debug, Clone)]
pub enum TreeChild {
Shape(Spanned<FlatShape>, Text),
Frame(ColorFrame, Text),
}
impl TreeChild {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
match self {
TreeChild::Shape(shape, text) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
}
}
}
impl TreeItem for TreeChild {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
match self {
shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
TreeChild::Frame(frame, text) => frame.colored_description(text, f),
}
}
fn children(&self) -> Cow<[Self::Child]> {
match self {
TreeChild::Shape(..) => Cow::Borrowed(&[]),
TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
}
}
}
#[derive(Debug, Clone)]
pub struct Tracer {
frame_stack: Vec<ColorFrame>,
}
impl Tracer {
pub fn print(self, source: Text) -> PrintTracer {
PrintTracer {
tracer: self,
source,
}
}
pub fn new() -> Tracer {
let root = ColorFrame {
description: "Trace",
children: vec![],
error: None,
};
Tracer {
frame_stack: vec![root],
}
}
fn current_frame(&mut self) -> &mut ColorFrame {
let frames = &mut self.frame_stack;
let last = frames.len() - 1;
&mut frames[last]
}
fn pop_frame(&mut self) -> ColorFrame {
let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
if self.frame_stack.len() == 0 {
panic!("Can't pop root tracer frame");
}
self.debug();
result
}
pub fn start(&mut self, description: &'static str) {
let frame = ColorFrame {
description,
children: vec![],
error: None,
};
self.frame_stack.push(frame);
self.debug();
}
pub fn eof_frame(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
#[allow(unused)]
pub fn finish(&mut self) {
loop {
if self.frame_stack.len() == 1 {
break;
}
let frame = self.pop_frame();
self.current_frame().children.push(FrameChild::Frame(frame));
}
}
#[allow(unused)]
pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.current_frame().add_shape(shape);
}
pub fn success(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
pub fn failed(&mut self, error: &ShellError) {
let mut current = self.pop_frame();
current.error = Some(error.clone());
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
fn debug(&self) {
trace!(target: "nu::color_syntax",
"frames = {:?}",
self.frame_stack
.iter()
.map(|f| f.description)
.collect::<Vec<_>>()
);
trace!(target: "nu::color_syntax", "{:#?}", self);
}
}
#[derive(Debug, Clone)]
pub struct PrintTracer {
tracer: Tracer,
source: Text,
}
impl TreeItem for PrintTracer {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
write!(f, "{}", style.paint("Color Trace"))
}
fn children(&self) -> Cow<[Self::Child]> {
Cow::Owned(vec![TreeChild::Frame(
self.tracer.frame_stack[0].clone(),
self.source.clone(),
)])
}
}

351
src/parser/hir/tokens_iterator/debug/color_trace.rs Normal file
View File

@ -0,0 +1,351 @@
use crate::errors::ShellError;
use crate::parser::hir::syntax_shape::FlatShape;
use crate::prelude::*;
use ansi_term::Color;
use log::trace;
use ptree::*;
use std::borrow::Cow;
use std::io;
#[derive(Debug, Clone)]
pub enum FrameChild {
#[allow(unused)]
Shape(Spanned<FlatShape>),
Frame(ColorFrame),
}
impl FrameChild {
fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
match self {
FrameChild::Shape(shape) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
FrameChild::Frame(frame) => frame.colored_leaf_description(f),
}
}
fn into_tree_child(self, text: &Text) -> TreeChild {
match self {
FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
}
}
}
#[derive(Debug, Clone)]
pub struct ColorFrame {
description: &'static str,
children: Vec<FrameChild>,
error: Option<ShellError>,
}
impl ColorFrame {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
if self.has_only_error_descendents() {
if self.children.len() == 0 {
write!(
f,
"{}",
Color::White.bold().on(Color::Red).paint(self.description)
)
} else {
write!(f, "{}", Color::Red.normal().paint(self.description))
}
} else if self.has_descendent_shapes() {
write!(f, "{}", Color::Green.normal().paint(self.description))
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))
}
}
fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
if self.children.len() == 1 {
let child = &self.children[0];
self.colored_leaf_description(f)?;
write!(f, " -> ")?;
child.colored_leaf_description(text, f)
} else {
self.colored_leaf_description(f)
}
}
fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
if self.children.len() == 1 {
let child = &self.children[0];
match child {
FrameChild::Shape(_) => vec![],
FrameChild::Frame(frame) => frame.tree_children(text),
}
} else {
self.tree_children(text)
}
}
fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
self.children
.clone()
.into_iter()
.map(|c| c.into_tree_child(text))
.collect()
}
#[allow(unused)]
fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.children.push(FrameChild::Shape(shape))
}
fn has_child_shapes(&self) -> bool {
self.any_child_shape(|_| true)
}
fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Shape(shape) => {
if predicate(*shape) {
return true;
}
}
_ => {}
}
}
false
}
fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
for item in &self.children {
match item {
FrameChild::Frame(frame) => {
if predicate(frame) {
return true;
}
}
_ => {}
}
}
false
}
fn has_descendent_shapes(&self) -> bool {
if self.has_child_shapes() {
true
} else {
self.any_child_frame(|frame| frame.has_descendent_shapes())
}
}
fn has_only_error_descendents(&self) -> bool {
if self.children.len() == 0 {
// if this frame has no children at all, it has only error descendents if this frame
// is an error
self.error.is_some()
} else {
// otherwise, it has only error descendents if all of its children terminate in an
// error (transitively)
let mut seen_error = false;
for child in &self.children {
match child {
// if this frame has at least one child shape, this frame has non-error descendents
FrameChild::Shape(_) => return false,
FrameChild::Frame(frame) => {
// if the chi
if frame.has_only_error_descendents() {
seen_error = true;
} else {
return false;
}
}
}
}
seen_error
}
}
}
#[derive(Debug, Clone)]
pub enum TreeChild {
Shape(Spanned<FlatShape>, Text),
Frame(ColorFrame, Text),
}
impl TreeChild {
fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
match self {
TreeChild::Shape(shape, text) => write!(
f,
"{} {:?}",
Color::White
.bold()
.on(Color::Green)
.paint(format!("{:?}", shape.item)),
shape.span.slice(text)
),
TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
}
}
}
impl TreeItem for TreeChild {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
match self {
shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
TreeChild::Frame(frame, text) => frame.colored_description(text, f),
}
}
fn children(&self) -> Cow<[Self::Child]> {
match self {
TreeChild::Shape(..) => Cow::Borrowed(&[]),
TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
}
}
}
#[derive(Debug, Clone)]
pub struct ColorTracer {
frame_stack: Vec<ColorFrame>,
}
impl ColorTracer {
pub fn print(self, source: Text) -> PrintTracer {
PrintTracer {
tracer: self,
source,
}
}
pub fn new() -> ColorTracer {
let root = ColorFrame {
description: "Trace",
children: vec![],
error: None,
};
ColorTracer {
frame_stack: vec![root],
}
}
fn current_frame(&mut self) -> &mut ColorFrame {
let frames = &mut self.frame_stack;
let last = frames.len() - 1;
&mut frames[last]
}
fn pop_frame(&mut self) -> ColorFrame {
trace!(target: "nu::color_syntax", "Popping {:#?}", self);
let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
if self.frame_stack.len() == 0 {
panic!("Can't pop root tracer frame {:#?}", self);
}
self.debug();
result
}
pub fn start(&mut self, description: &'static str) {
let frame = ColorFrame {
description,
children: vec![],
error: None,
};
self.frame_stack.push(frame);
self.debug();
}
pub fn eof_frame(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
#[allow(unused)]
pub fn finish(&mut self) {
loop {
if self.frame_stack.len() == 1 {
break;
}
let frame = self.pop_frame();
self.current_frame().children.push(FrameChild::Frame(frame));
}
}
#[allow(unused)]
pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
self.current_frame().add_shape(shape);
}
pub fn success(&mut self) {
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
pub fn failed(&mut self, error: &ShellError) {
let mut current = self.pop_frame();
current.error = Some(error.clone());
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
fn debug(&self) {
trace!(target: "nu::color_syntax",
"frames = {:?}",
self.frame_stack
.iter()
.map(|f| f.description)
.collect::<Vec<_>>()
);
trace!(target: "nu::color_syntax", "{:#?}", self);
}
}
#[derive(Debug, Clone)]
pub struct PrintTracer {
tracer: ColorTracer,
source: Text,
}
impl TreeItem for PrintTracer {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
write!(f, "{}", style.paint("Color Trace"))
}
fn children(&self) -> Cow<[Self::Child]> {
Cow::Owned(vec![TreeChild::Frame(
self.tracer.frame_stack[0].clone(),
self.source.clone(),
)])
}
}

365
src/parser/hir/tokens_iterator/debug/expand_trace.rs Normal file
View File

@ -0,0 +1,365 @@
use crate::parser::hir::Expression;
use crate::prelude::*;
use ansi_term::Color;
use log::trace;
use ptree::*;
use std::borrow::Cow;
use std::io;
#[derive(Debug)]
pub enum FrameChild {
Expr(Expression),
Frame(ExprFrame),
Result(Box<dyn FormatDebug>),
}
impl FrameChild {
fn get_error_leaf(&self) -> Option<&'static str> {
match self {
FrameChild::Frame(frame) if frame.error.is_some() => {
if frame.children.len() == 0 {
Some(frame.description)
} else {
None
}
}
_ => None,
}
}
fn to_tree_child(&self, text: &Text) -> TreeChild {
match self {
FrameChild::Expr(expr) => TreeChild::OkExpr(expr.clone(), text.clone()),
FrameChild::Result(result) => {
let result = format!("{}", result.debug(text));
TreeChild::OkNonExpr(result)
}
FrameChild::Frame(frame) => {
if frame.error.is_some() {
if frame.children.len() == 0 {
TreeChild::ErrorLeaf(vec![frame.description])
} else {
TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone())
}
} else {
TreeChild::OkFrame(frame.to_tree_frame(text), text.clone())
}
}
}
}
}
#[derive(Debug)]
pub struct ExprFrame {
description: &'static str,
children: Vec<FrameChild>,
error: Option<ParseError>,
}
impl ExprFrame {
fn to_tree_frame(&self, text: &Text) -> TreeFrame {
let mut children = vec![];
let mut errors = vec![];
for child in &self.children {
if let Some(error_leaf) = child.get_error_leaf() {
errors.push(error_leaf);
continue;
} else if errors.len() > 0 {
children.push(TreeChild::ErrorLeaf(errors));
errors = vec![];
}
children.push(child.to_tree_child(text));
}
if errors.len() > 0 {
children.push(TreeChild::ErrorLeaf(errors));
}
TreeFrame {
description: self.description,
children,
error: self.error.clone(),
}
}
fn add_expr(&mut self, expr: Expression) {
self.children.push(FrameChild::Expr(expr))
}
fn add_result(&mut self, result: Box<dyn FormatDebug>) {
self.children.push(FrameChild::Result(result))
}
}
#[derive(Debug, Clone)]
pub struct TreeFrame {
description: &'static str,
children: Vec<TreeChild>,
error: Option<ParseError>,
}
impl TreeFrame {
fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
if self.children.len() == 1 {
if self.error.is_some() {
write!(f, "{}", Color::Red.normal().paint(self.description))?;
} else if self.has_descendent_green() {
write!(f, "{}", Color::Green.normal().paint(self.description))?;
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))?;
}
write!(f, " -> ")?;
self.children[0].leaf_description(f)
} else {
if self.error.is_some() {
if self.children.len() == 0 {
write!(
f,
"{}",
Color::White.bold().on(Color::Red).paint(self.description)
)
} else {
write!(f, "{}", Color::Red.normal().paint(self.description))
}
} else if self.has_descendent_green() {
write!(f, "{}", Color::Green.normal().paint(self.description))
} else {
write!(f, "{}", Color::Yellow.bold().paint(self.description))
}
}
}
fn has_child_green(&self) -> bool {
self.children.iter().any(|item| match item {
TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false,
TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) => true,
})
}
fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool {
for item in &self.children {
match item {
TreeChild::OkFrame(frame, ..) => {
if predicate(frame) {
return true;
}
}
_ => {}
}
}
false
}
fn has_descendent_green(&self) -> bool {
if self.has_child_green() {
true
} else {
self.any_child_frame(|frame| frame.has_child_green())
}
}
fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
if self.children.len() == 1 {
let child: &TreeChild = &self.children[0];
match child {
TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
vec![]
}
TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => {
frame.children_for_formatting(text)
}
}
} else {
self.children.clone()
}
}
}
#[derive(Debug, Clone)]
pub enum TreeChild {
OkNonExpr(String),
OkExpr(Expression, Text),
OkFrame(TreeFrame, Text),
ErrorFrame(TreeFrame, Text),
ErrorLeaf(Vec<&'static str>),
}
impl TreeChild {
fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
match self {
TreeChild::OkExpr(expr, text) => write!(
f,
"{} {} {}",
Color::Cyan.normal().paint("returns"),
Color::White.bold().on(Color::Green).paint(expr.type_name()),
expr.span.slice(text)
),
TreeChild::OkNonExpr(result) => write!(
f,
"{} {}",
Color::Cyan.normal().paint("returns"),
Color::White
.bold()
.on(Color::Green)
.paint(format!("{}", result))
),
TreeChild::ErrorLeaf(desc) => {
let last = desc.len() - 1;
for (i, item) in desc.iter().enumerate() {
write!(f, "{}", Color::White.bold().on(Color::Red).paint(*item))?;
if i != last {
write!(f, "{}", Color::White.normal().paint(", "))?;
}
}
Ok(())
}
TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => {
frame.leaf_description(f)
}
}
}
}
impl TreeItem for TreeChild {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
self.leaf_description(f)
}
fn children(&self) -> Cow<[Self::Child]> {
match self {
TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
Cow::Borrowed(&[])
}
TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => {
Cow::Owned(frame.children_for_formatting(text))
}
}
}
}
#[derive(Debug)]
pub struct ExpandTracer {
frame_stack: Vec<ExprFrame>,
}
impl ExpandTracer {
pub fn print(&self, source: Text) -> PrintTracer {
let root = self
.frame_stack
.iter()
.nth(0)
.unwrap()
.to_tree_frame(&source);
PrintTracer { root, source }
}
pub fn new() -> ExpandTracer {
let root = ExprFrame {
description: "Trace",
children: vec![],
error: None,
};
ExpandTracer {
frame_stack: vec![root],
}
}
fn current_frame(&mut self) -> &mut ExprFrame {
let frames = &mut self.frame_stack;
let last = frames.len() - 1;
&mut frames[last]
}
fn pop_frame(&mut self) -> ExprFrame {
let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
if self.frame_stack.len() == 0 {
panic!("Can't pop root tracer frame");
}
self.debug();
result
}
pub fn start(&mut self, description: &'static str) {
let frame = ExprFrame {
description,
children: vec![],
error: None,
};
self.frame_stack.push(frame);
self.debug();
}
pub fn add_expr(&mut self, shape: Expression) {
self.current_frame().add_expr(shape);
}
pub fn add_result(&mut self, result: Box<dyn FormatDebug>) {
self.current_frame().add_result(result);
}
pub fn success(&mut self) {
trace!(target: "parser::expand_syntax", "success {:#?}", self);
let current = self.pop_frame();
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
pub fn failed(&mut self, error: &ParseError) {
let mut current = self.pop_frame();
current.error = Some(error.clone());
self.current_frame()
.children
.push(FrameChild::Frame(current));
}
fn debug(&self) {
trace!(target: "nu::parser::expand",
"frames = {:?}",
self.frame_stack
.iter()
.map(|f| f.description)
.collect::<Vec<_>>()
);
trace!(target: "nu::parser::expand", "{:#?}", self);
}
}
#[derive(Debug, Clone)]
pub struct PrintTracer {
root: TreeFrame,
source: Text,
}
impl TreeItem for PrintTracer {
type Child = TreeChild;
fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
write!(f, "{}", style.paint("Expansion Trace"))
}
fn children(&self) -> Cow<[Self::Child]> {
Cow::Borrowed(&self.root.children)
}
}

8
src/parser/parse/call_node.rs
View File

@ -1,7 +1,7 @@
use crate::parser::TokenNode;
use crate::traits::ToDebug;
use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
use getset::Getters;
use std::fmt;
use std::fmt::{self, Write};
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
pub struct CallNode {
@ -27,8 +27,8 @@ impl CallNode {
}
}
impl ToDebug for CallNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for CallNode {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.head.debug(source))?;
if let Some(children) = &self.children {

4
src/parser/parse/operator.rs
View File

@ -14,8 +14,8 @@ pub enum Operator {
Dot,
}
impl ToDebug for Operator {
fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
impl FormatDebug for Operator {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self.as_str())
}
}

24
src/parser/parse/pipeline.rs
View File

@ -1,24 +1,22 @@
use crate::parser::TokenNode;
use crate::traits::ToDebug;
use crate::{Span, Spanned};
use crate::{DebugFormatter, FormatDebug, Span, Spanned, ToDebug};
use derive_new::new;
use getset::Getters;
use std::fmt;
use itertools::Itertools;
use std::fmt::{self, Write};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
pub struct Pipeline {
#[get = "pub"]
pub(crate) parts: Vec<Spanned<PipelineElement>>,
// pub(crate) post_ws: Option<Tag>,
}
impl ToDebug for Pipeline {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
for part in self.parts.iter() {
write!(f, "{}", part.debug(source))?;
}
Ok(())
impl FormatDebug for Pipeline {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
f.say_str(
"pipeline",
self.parts.iter().map(|p| p.debug(source)).join(" "),
)
}
}
@ -29,8 +27,8 @@ pub struct PipelineElement {
pub tokens: Spanned<Vec<TokenNode>>,
}
impl ToDebug for PipelineElement {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl FormatDebug for PipelineElement {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
if let Some(pipe) = self.pipe {
write!(f, "{}", pipe.slice(source))?;
}

24
src/parser/parse/token_tree.rs
View File

@ -1,4 +1,4 @@
use crate::errors::ShellError;
use crate::errors::{ParseError, ShellError};
use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
use crate::prelude::*;
use crate::traits::ToDebug;
@ -21,8 +21,14 @@ pub enum TokenNode {
Error(Spanned<ShellError>),
}
impl ToDebug for TokenNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
impl HasSpan for TokenNode {
fn span(&self) -> Span {
self.get_span()
}
}
impl FormatDebug for TokenNode {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{:?}", self.old_debug(&Text::from(source)))
}
}
@ -84,12 +90,12 @@ impl fmt::Debug for DebugTokenNode<'_> {
impl From<&TokenNode> for Span {
fn from(token: &TokenNode) -> Span {
token.span()
token.get_span()
}
}
impl TokenNode {
pub fn span(&self) -> Span {
pub fn get_span(&self) -> Span {
match self {
TokenNode::Token(t) => t.span,
TokenNode::Nodes(t) => t.span,
@ -231,10 +237,10 @@ impl TokenNode {
}
}
pub fn as_pipeline(&self) -> Result<Pipeline, ShellError> {
pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
match self {
TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
_ => Err(ShellError::type_error("pipeline", self.tagged_type_name())),
other => Err(ParseError::mismatch("pipeline", other.tagged_type_name())),
}
}
@ -321,9 +327,9 @@ impl TokenNode {
}
}
pub fn expect_list(&self) -> &[TokenNode] {
pub fn expect_list(&self) -> Spanned<&[TokenNode]> {
match self {
TokenNode::Nodes(token_nodes) => &token_nodes[..],
TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span),
other => panic!("Expected list, found {:?}", other),
}
}

13
src/parser/parse/tokens.rs
View File

@ -23,8 +23,8 @@ impl RawToken {
RawToken::Operator(..) => "operator",
RawToken::String(_) => "string",
RawToken::Variable(_) => "variable",
RawToken::ExternalCommand(_) => "external command",
RawToken::ExternalWord => "external word",
RawToken::ExternalCommand(_) => "syntax error",
RawToken::ExternalWord => "syntax error",
RawToken::GlobPattern => "glob pattern",
RawToken::Bare => "string",
}
@ -37,6 +37,15 @@ pub enum RawNumber {
Decimal(Span),
}
impl FormatDebug for RawNumber {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
match self {
RawNumber::Int(span) => f.say_str("int", span.slice(source)),
RawNumber::Decimal(span) => f.say_str("decimal", span.slice(source)),
}
}
}
impl RawNumber {
pub fn int(span: impl Into<Span>) -> Spanned<RawNumber> {
let span = span.into();

7
src/parser/parse/unit.rs
View File

@ -1,6 +1,7 @@
use crate::data::base::Value;
use crate::prelude::*;
use serde::{Deserialize, Serialize};
use std::fmt;
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
@ -13,6 +14,12 @@ pub enum Unit {
PB,
}
impl FormatDebug for Spanned<Unit> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
write!(f, "{}", self.span.slice(source))
}
}
impl Unit {
pub fn as_str(&self) -> &str {
match *self {

38
src/parser/parse_command.rs
View File

@ -1,4 +1,4 @@
use crate::errors::{ArgumentError, ShellError};
use crate::errors::{ArgumentError, ParseError};
use crate::parser::hir::syntax_shape::{
color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced,
BackoffColoringMode, ColorSyntax, MaybeSpaceShape,
@ -18,9 +18,9 @@ pub fn parse_command_tail(
context: &ExpandContext,
tail: &mut TokensIterator,
command_span: Span,
) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ParseError> {
let mut named = NamedArguments::new();
trace_remaining("nodes", tail.clone(), context.source());
trace_remaining("nodes", &tail, context.source());
for (name, kind) in &config.named {
trace!(target: "nu::parse", "looking for {} : {:?}", name, kind);
@ -38,7 +38,7 @@ pub fn parse_command_tail(
tail.move_to(pos);
if tail.at_end() {
return Err(ShellError::argument_error(
return Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingValueForName(name.to_string()),
flag.span,
@ -59,7 +59,7 @@ pub fn parse_command_tail(
tail.move_to(pos);
if tail.at_end() {
return Err(ShellError::argument_error(
return Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingValueForName(name.to_string()),
flag.span,
@ -85,7 +85,7 @@ pub fn parse_command_tail(
};
}
trace_remaining("after named", tail.clone(), context.source());
trace_remaining("after named", &tail, context.source());
let mut positional = vec![];
@ -95,7 +95,7 @@ pub fn parse_command_tail(
match &arg.0 {
PositionalType::Mandatory(..) => {
if tail.at_end_possible_ws() {
return Err(ShellError::argument_error(
return Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()),
Tag {
@ -118,7 +118,7 @@ pub fn parse_command_tail(
positional.push(result);
}
trace_remaining("after positional", tail.clone(), context.source());
trace_remaining("after positional", &tail, context.source());
if let Some((syntax_type, _)) = config.rest_positional {
let mut out = vec![];
@ -136,7 +136,7 @@ pub fn parse_command_tail(
positional.extend(out);
}
trace_remaining("after rest", tail.clone(), context.source());
trace_remaining("after rest", &tail, context.source());
trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named);
@ -202,8 +202,6 @@ impl ColorSyntax for CommandTailShape {
shapes: &mut Vec<Spanned<FlatShape>>,
) -> Self::Info {
let mut args = ColoringArgs::new(token_nodes.len());
trace_remaining("nodes", token_nodes.clone(), context.source());
for (name, kind) in &signature.named {
trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
@ -295,8 +293,6 @@ impl ColorSyntax for CommandTailShape {
};
}
trace_remaining("after named", token_nodes.clone(), context.source());
for arg in &signature.positional {
trace!("Processing positional {:?}", arg);
@ -341,8 +337,6 @@ impl ColorSyntax for CommandTailShape {
}
}
trace_remaining("after positional", token_nodes.clone(), context.source());
if let Some((syntax_type, _)) = signature.rest_positional {
loop {
if token_nodes.at_end_possible_ws() {
@ -402,7 +396,7 @@ impl ColorSyntax for CommandTailShape {
context: &ExpandContext,
) -> Self::Info {
let mut args = ColoringArgs::new(token_nodes.len());
trace_remaining("nodes", token_nodes.clone(), context.source());
trace_remaining("nodes", &token_nodes, context.source());
for (name, kind) in &signature.named {
trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
@ -497,7 +491,7 @@ impl ColorSyntax for CommandTailShape {
};
}
trace_remaining("after named", token_nodes.clone(), context.source());
trace_remaining("after named", &token_nodes, context.source());
for arg in &signature.positional {
trace!("Processing positional {:?}", arg);
@ -537,7 +531,7 @@ impl ColorSyntax for CommandTailShape {
}
}
trace_remaining("after positional", token_nodes.clone(), context.source());
trace_remaining("after positional", &token_nodes, context.source());
if let Some((syntax_type, _)) = signature.rest_positional {
loop {
@ -594,11 +588,11 @@ fn extract_mandatory(
tokens: &mut hir::TokensIterator<'_>,
source: &Text,
span: Span,
) -> Result<(usize, Spanned<Flag>), ShellError> {
) -> Result<(usize, Spanned<Flag>), ParseError> {
let flag = tokens.extract(|t| t.as_flag(name, source));
match flag {
None => Err(ShellError::argument_error(
None => Err(ParseError::argument_error(
config.name.clone(),
ArgumentError::MissingMandatoryFlag(name.to_string()),
span,
@ -615,7 +609,7 @@ fn extract_optional(
name: &str,
tokens: &mut hir::TokensIterator<'_>,
source: &Text,
) -> Result<(Option<(usize, Spanned<Flag>)>), ShellError> {
) -> Result<(Option<(usize, Spanned<Flag>)>), ParseError> {
let flag = tokens.extract(|t| t.as_flag(name, source));
match flag {
@ -627,7 +621,7 @@ fn extract_optional(
}
}
pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) {
pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>, source: &Text) {
trace!(
target: "nu::parse",
"{} = {:?}",

10
src/prelude.rs
View File

@ -67,13 +67,14 @@ pub(crate) use crate::context::CommandRegistry;
pub(crate) use crate::context::{AnchorLocation, Context};
pub(crate) use crate::data::base as value;
pub(crate) use crate::data::meta::{
tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
TaggedItem,
};
pub(crate) use crate::data::types::ExtractType;
pub(crate) use crate::data::{Primitive, Value};
pub(crate) use crate::env::host::handle_unexpected;
pub(crate) use crate::env::Host;
pub(crate) use crate::errors::{CoerceInto, ShellError};
pub(crate) use crate::errors::{CoerceInto, ParseError, ShellError};
pub(crate) use crate::parser::hir::SyntaxShape;
pub(crate) use crate::parser::parse::parser::Number;
pub(crate) use crate::parser::registry::Signature;
@ -82,7 +83,7 @@ pub(crate) use crate::shell::help_shell::HelpShell;
pub(crate) use crate::shell::shell_manager::ShellManager;
pub(crate) use crate::shell::value_shell::ValueShell;
pub(crate) use crate::stream::{InputStream, OutputStream};
pub(crate) use crate::traits::{HasTag, ToDebug};
pub(crate) use crate::traits::{DebugFormatter, FormatDebug, HasTag, ToDebug};
pub(crate) use crate::Text;
pub(crate) use async_stream::stream as async_stream;
pub(crate) use bigdecimal::BigDecimal;
@ -93,9 +94,12 @@ pub(crate) use num_traits::cast::{FromPrimitive, ToPrimitive};
pub(crate) use num_traits::identities::Zero;
pub(crate) use serde::Deserialize;
pub(crate) use std::collections::VecDeque;
pub(crate) use std::fmt::Write;
pub(crate) use std::future::Future;
pub(crate) use std::sync::{Arc, Mutex};
pub(crate) use itertools::Itertools;
pub trait FromInputStream {
fn from_input_stream(self) -> OutputStream;
}

15
src/shell/helper.rs
View File

@ -3,7 +3,7 @@ use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, Pipelin
use crate::parser::hir::TokensIterator;
use crate::parser::nom_input;
use crate::parser::parse::token_tree::TokenNode;
use crate::{Span, Spanned, SpannedItem, Tag, Tagged, Text};
use crate::{HasSpan, Spanned, SpannedItem, Tag, Tagged, Text};
use ansi_term::Color;
use log::{log_enabled, trace};
use rustyline::completion::Completer;
@ -65,9 +65,7 @@ impl Highlighter for Helper {
let mut tokens = TokensIterator::all(&tokens[..], v.span());
let text = Text::from(line);
let expand_context = self
.context
.expand_context(&text, Span::new(0, line.len() - 1));
let expand_context = self.context.expand_context(&text);
#[cfg(not(coloring_in_tokens))]
let shapes = {
@ -86,16 +84,17 @@ impl Highlighter for Helper {
let shapes = {
// We just constructed a token list that only contains a pipeline, so it can't fail
color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context).unwrap();
tokens.with_tracer(|_, tracer| tracer.finish());
tokens.with_color_tracer(|_, tracer| tracer.finish());
tokens.state().shapes()
};
trace!(target: "nu::color_syntax", "{:#?}", tokens.tracer());
trace!(target: "nu::color_syntax", "{:#?}", tokens.color_tracer());
if log_enabled!(target: "nu::color_syntax", log::Level::Trace) {
if log_enabled!(target: "nu::color_syntax", log::Level::Debug) {
println!("");
ptree::print_tree(&tokens.tracer().clone().print(Text::from(line))).unwrap();
ptree::print_tree(&tokens.color_tracer().clone().print(Text::from(line)))
.unwrap();
println!("");
}

124
src/traits.rs
View File

@ -1,14 +1,28 @@
use crate::prelude::*;
use std::fmt;
use derive_new::new;
use std::fmt::{self, Write};
pub struct Debuggable<'a, T: ToDebug> {
pub struct Debuggable<'a, T: FormatDebug> {
inner: &'a T,
source: &'a str,
}
impl FormatDebug for str {
fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
write!(f, "{}", self)
}
}
impl<T: ToDebug> fmt::Display for Debuggable<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt_debug(f, self.source)
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.inner.fmt_debug(
&mut DebugFormatter::new(
f,
ansi_term::Color::White.bold(),
ansi_term::Color::Black.bold(),
),
self.source,
)
}
}
@ -16,13 +30,109 @@ pub trait HasTag {
fn tag(&self) -> Tag;
}
pub trait ToDebug: Sized {
#[derive(new)]
pub struct DebugFormatter<'me, 'args> {
formatter: &'me mut std::fmt::Formatter<'args>,
style: ansi_term::Style,
default_style: ansi_term::Style,
}
impl<'me, 'args> DebugFormatter<'me, 'args> {
pub fn say<'debuggable>(
&mut self,
kind: &str,
debuggable: Debuggable<'debuggable, impl FormatDebug>,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
write!(
self,
"{}",
self.default_style.paint(format!("{}", debuggable))
)
}
pub fn say_str<'debuggable>(
&mut self,
kind: &str,
string: impl AsRef<str>,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
write!(self, "{}", self.default_style.paint(string.as_ref()))
}
pub fn say_block(
&mut self,
kind: &str,
block: impl FnOnce(&mut Self) -> std::fmt::Result,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
block(self)
}
pub fn say_dict<'debuggable>(
&mut self,
kind: &str,
dict: indexmap::IndexMap<&str, String>,
) -> std::fmt::Result {
write!(self, "{}", self.style.paint(kind))?;
write!(self, "{}", self.default_style.paint(" "))?;
let last = dict.len() - 1;
for (i, (key, value)) in dict.into_iter().enumerate() {
write!(self, "{}", self.default_style.paint(key))?;
write!(self, "{}", self.default_style.paint("=["))?;
write!(self, "{}", self.style.paint(value))?;
write!(self, "{}", self.default_style.paint("]"))?;
if i != last {
write!(self, "{}", self.default_style.paint(" "))?;
}
}
Ok(())
}
}
impl<'a, 'b> std::fmt::Write for DebugFormatter<'a, 'b> {
fn write_str(&mut self, s: &str) -> std::fmt::Result {
self.formatter.write_str(s)
}
fn write_char(&mut self, c: char) -> std::fmt::Result {
self.formatter.write_char(c)
}
fn write_fmt(self: &mut Self, args: std::fmt::Arguments<'_>) -> std::fmt::Result {
self.formatter.write_fmt(args)
}
}
pub trait FormatDebug: std::fmt::Debug {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result;
}
pub trait ToDebug: Sized + FormatDebug {
fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self>;
}
impl FormatDebug for Box<dyn FormatDebug> {
fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
(&**self).fmt_debug(f, source)
}
}
impl<T> ToDebug for T
where
T: FormatDebug + Sized,
{
fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self> {
Debuggable {
inner: self,
source,
}
}
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result;
}