mirror of
https://github.com/nushell/nushell.git
synced 2025-05-02 00:54:25 +02:00
Restructure and streamline token expansion The purpose of this commit is to streamline the token expansion code, by removing aspects of the code that are no longer relevant, removing pointless duplication, and eliminating the need to pass the same arguments to `expand_syntax`. The first big-picture change in this commit is that instead of a handful of `expand_` functions, which take a TokensIterator and ExpandContext, a smaller number of methods on the `TokensIterator` do the same job. The second big-picture change in this commit is fully eliminating the coloring traits, making coloring a responsibility of the base expansion implementations. This also means that the coloring tracer is merged into the expansion tracer, so you can follow a single expansion and see how the expansion process produced colored tokens. One side effect of this change is that the expander itself is marginally more error-correcting. The error correction works by switching from structured expansion to `BackoffColoringMode` when an unexpected token is found, which guarantees that all spans of the source are colored, but may not be the most optimal error recovery strategy. That said, because `BackoffColoringMode` only extends as far as a closing delimiter (`)`, `]`, `}`) or pipe (`|`), it does result in fairly granular correction strategy. The current code still produces an `Err` (plus a complete list of colored shapes) from the parsing process if any errors are encountered, but this could easily be addressed now that the underlying expansion is error-correcting. This commit also colors any spans that are syntax errors in red, and causes the parser to include some additional information about what tokens were expected at any given point where an error was encountered, so that completions and hinting could be more robust in the future. Co-authored-by: Jonathan Turner <jonathandturner@users.noreply.github.com> Co-authored-by: Andrés N. Robalino <andres@androbtech.com>
712 lines
16 KiB
Rust
712 lines
16 KiB
Rust
use crate::pretty::{b, DebugDocBuilder, PrettyDebugWithSource};
|
|
use crate::text::Text;
|
|
use crate::tracable::TracableContext;
|
|
|
|
use derive_new::new;
|
|
use getset::Getters;
|
|
use serde::Deserialize;
|
|
use serde::Serialize;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// Anchors represent a location that a value originated from. The value may have been loaded from a file, fetched from a website, or parsed from some text
|
|
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
|
pub enum AnchorLocation {
|
|
/// The originating site where the value was first found
|
|
Url(String),
|
|
/// The original file where the value was loaded from
|
|
File(String),
|
|
/// The text where the value was parsed from
|
|
Source(Text),
|
|
}
|
|
|
|
pub trait HasTag {
|
|
/// Get the associated metadata
|
|
fn tag(&self) -> Tag;
|
|
}
|
|
|
|
/// A wrapper type that attaches a Span to a value
|
|
#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
|
|
pub struct Spanned<T> {
|
|
pub span: Span,
|
|
pub item: T,
|
|
}
|
|
|
|
impl<T> Spanned<T> {
|
|
/// Allows mapping over a Spanned value
|
|
pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Spanned<U> {
|
|
let span = self.span;
|
|
|
|
let mapped = input(self.item);
|
|
mapped.spanned(span)
|
|
}
|
|
}
|
|
|
|
impl Spanned<String> {
|
|
/// Iterates over the contained String
|
|
pub fn items<'a, U>(
|
|
items: impl Iterator<Item = &'a Spanned<String>>,
|
|
) -> impl Iterator<Item = &'a str> {
|
|
items.map(|item| &item.item[..])
|
|
}
|
|
}
|
|
|
|
impl Spanned<String> {
|
|
/// Borrows the contained String
|
|
pub fn borrow_spanned(&self) -> Spanned<&str> {
|
|
let span = self.span;
|
|
self.item[..].spanned(span)
|
|
}
|
|
}
|
|
|
|
pub trait SpannedItem: Sized {
|
|
/// Converts a value into a Spanned value
|
|
fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
|
|
Spanned {
|
|
item: self,
|
|
span: span.into(),
|
|
}
|
|
}
|
|
|
|
/// Converts a value into a Spanned value, using an unknown Span
|
|
fn spanned_unknown(self) -> Spanned<Self> {
|
|
Spanned {
|
|
item: self,
|
|
span: Span::unknown(),
|
|
}
|
|
}
|
|
}
|
|
impl<T> SpannedItem for T {}
|
|
|
|
impl<T> std::ops::Deref for Spanned<T> {
|
|
type Target = T;
|
|
|
|
/// Shorthand to deref to the contained value
|
|
fn deref(&self) -> &T {
|
|
&self.item
|
|
}
|
|
}
|
|
|
|
/// A wrapper type that attaches a Tag to a value
|
|
#[derive(new, Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
|
|
pub struct Tagged<T> {
|
|
pub tag: Tag,
|
|
pub item: T,
|
|
}
|
|
|
|
impl Tagged<String> {
|
|
/// Allows borrowing the contained string slice as a spanned value
|
|
pub fn borrow_spanned(&self) -> Spanned<&str> {
|
|
let span = self.tag.span;
|
|
self.item[..].spanned(span)
|
|
}
|
|
|
|
/// Allows borrowing the contained string slice as a tagged value
|
|
pub fn borrow_tagged(&self) -> Tagged<&str> {
|
|
self.item[..].tagged(self.tag.clone())
|
|
}
|
|
}
|
|
|
|
impl<T> Tagged<Vec<T>> {
|
|
/// Iterates over the contained value(s)
|
|
pub fn items(&self) -> impl Iterator<Item = &T> {
|
|
self.item.iter()
|
|
}
|
|
}
|
|
|
|
impl<T> HasTag for Tagged<T> {
|
|
/// Helper for getting the Tag from the Tagged value
|
|
fn tag(&self) -> Tag {
|
|
self.tag.clone()
|
|
}
|
|
}
|
|
|
|
impl AsRef<Path> for Tagged<PathBuf> {
|
|
/// Gets the reference to the contained Path
|
|
fn as_ref(&self) -> &Path {
|
|
self.item.as_ref()
|
|
}
|
|
}
|
|
|
|
pub trait TaggedItem: Sized {
|
|
fn tagged(self, tag: impl Into<Tag>) -> Tagged<Self> {
|
|
Tagged {
|
|
item: self,
|
|
tag: tag.into(),
|
|
}
|
|
}
|
|
|
|
// For now, this is a temporary facility. In many cases, there are other useful spans that we
|
|
// could be using, such as the original source spans of JSON or Toml files, but we don't yet
|
|
// have the infrastructure to make that work.
|
|
fn tagged_unknown(self) -> Tagged<Self> {
|
|
Tagged {
|
|
item: self,
|
|
tag: Tag {
|
|
span: Span::unknown(),
|
|
anchor: None,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T> TaggedItem for T {}
|
|
|
|
impl<T> std::ops::Deref for Tagged<T> {
|
|
type Target = T;
|
|
|
|
fn deref(&self) -> &T {
|
|
&self.item
|
|
}
|
|
}
|
|
|
|
impl<T> Tagged<T> {
|
|
pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Tagged<U> {
|
|
let tag = self.tag();
|
|
|
|
let mapped = input(self.item);
|
|
mapped.tagged(tag)
|
|
}
|
|
|
|
pub fn map_anchored(self, anchor: &Option<AnchorLocation>) -> Tagged<T> {
|
|
let mut tag = self.tag;
|
|
|
|
tag.anchor = anchor.clone();
|
|
|
|
Tagged {
|
|
item: self.item,
|
|
tag,
|
|
}
|
|
}
|
|
|
|
pub fn transpose(&self) -> Tagged<&T> {
|
|
Tagged {
|
|
item: &self.item,
|
|
tag: self.tag.clone(),
|
|
}
|
|
}
|
|
|
|
pub fn tag(&self) -> Tag {
|
|
self.tag.clone()
|
|
}
|
|
|
|
pub fn span(&self) -> Span {
|
|
self.tag.span
|
|
}
|
|
|
|
pub fn anchor(&self) -> Option<AnchorLocation> {
|
|
self.tag.anchor.clone()
|
|
}
|
|
|
|
pub fn anchor_name(&self) -> Option<String> {
|
|
match self.tag.anchor {
|
|
Some(AnchorLocation::File(ref file)) => Some(file.clone()),
|
|
Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
pub fn item(&self) -> &T {
|
|
&self.item
|
|
}
|
|
|
|
pub fn into_parts(self) -> (T, Tag) {
|
|
(self.item, self.tag)
|
|
}
|
|
}
|
|
|
|
impl From<&Tag> for Tag {
|
|
fn from(input: &Tag) -> Tag {
|
|
input.clone()
|
|
}
|
|
}
|
|
|
|
impl<T> From<nom_locate::LocatedSpanEx<&str, T>> for Span {
|
|
fn from(input: nom_locate::LocatedSpanEx<&str, T>) -> Span {
|
|
Span::new(input.offset, input.offset + input.fragment.len())
|
|
}
|
|
}
|
|
|
|
impl<T>
|
|
From<(
|
|
nom_locate::LocatedSpanEx<T, u64>,
|
|
nom_locate::LocatedSpanEx<T, u64>,
|
|
)> for Span
|
|
{
|
|
fn from(
|
|
input: (
|
|
nom_locate::LocatedSpanEx<T, u64>,
|
|
nom_locate::LocatedSpanEx<T, u64>,
|
|
),
|
|
) -> Span {
|
|
Span::new(input.0.offset, input.1.offset)
|
|
}
|
|
}
|
|
|
|
impl From<(usize, usize)> for Span {
|
|
fn from(input: (usize, usize)) -> Span {
|
|
Span::new(input.0, input.1)
|
|
}
|
|
}
|
|
|
|
impl From<&std::ops::Range<usize>> for Span {
|
|
fn from(input: &std::ops::Range<usize>) -> Span {
|
|
Span::new(input.start, input.end)
|
|
}
|
|
}
|
|
|
|
/// The set of metadata that can be associated with a value
|
|
#[derive(
|
|
Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, new,
|
|
)]
|
|
pub struct Tag {
|
|
/// The original source for this value
|
|
pub anchor: Option<AnchorLocation>,
|
|
/// The span in the source text for the command that created this value
|
|
pub span: Span,
|
|
}
|
|
|
|
impl From<Span> for Tag {
|
|
fn from(span: Span) -> Self {
|
|
Tag { anchor: None, span }
|
|
}
|
|
}
|
|
|
|
impl From<&Span> for Tag {
|
|
fn from(span: &Span) -> Self {
|
|
Tag {
|
|
anchor: None,
|
|
span: *span,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<(usize, usize, TracableContext)> for Tag {
|
|
fn from((start, end, _context): (usize, usize, TracableContext)) -> Self {
|
|
Tag {
|
|
anchor: None,
|
|
span: Span::new(start, end),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<(usize, usize, AnchorLocation)> for Tag {
|
|
fn from((start, end, anchor): (usize, usize, AnchorLocation)) -> Self {
|
|
Tag {
|
|
anchor: Some(anchor),
|
|
span: Span::new(start, end),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<(usize, usize, Option<AnchorLocation>)> for Tag {
|
|
fn from((start, end, anchor): (usize, usize, Option<AnchorLocation>)) -> Self {
|
|
Tag {
|
|
anchor,
|
|
span: Span::new(start, end),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<nom_locate::LocatedSpanEx<&str, TracableContext>> for Tag {
|
|
fn from(input: nom_locate::LocatedSpanEx<&str, TracableContext>) -> Tag {
|
|
Tag {
|
|
anchor: None,
|
|
span: Span::new(input.offset, input.offset + input.fragment.len()),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<Tag> for Span {
|
|
fn from(tag: Tag) -> Self {
|
|
tag.span
|
|
}
|
|
}
|
|
|
|
impl From<&Tag> for Span {
|
|
fn from(tag: &Tag) -> Self {
|
|
tag.span
|
|
}
|
|
}
|
|
|
|
impl Tag {
|
|
pub fn unknown_anchor(span: Span) -> Tag {
|
|
Tag { anchor: None, span }
|
|
}
|
|
|
|
pub fn for_char(pos: usize, anchor: AnchorLocation) -> Tag {
|
|
Tag {
|
|
anchor: Some(anchor),
|
|
span: Span::new(pos, pos + 1),
|
|
}
|
|
}
|
|
|
|
pub fn unknown_span(anchor: AnchorLocation) -> Tag {
|
|
Tag {
|
|
anchor: Some(anchor),
|
|
span: Span::unknown(),
|
|
}
|
|
}
|
|
|
|
pub fn unknown() -> Tag {
|
|
Tag {
|
|
anchor: None,
|
|
span: Span::unknown(),
|
|
}
|
|
}
|
|
|
|
pub fn anchor(&self) -> Option<AnchorLocation> {
|
|
self.anchor.clone()
|
|
}
|
|
|
|
pub fn until(&self, other: impl Into<Tag>) -> Tag {
|
|
let other = other.into();
|
|
debug_assert!(
|
|
self.anchor == other.anchor,
|
|
"Can only merge two tags with the same anchor"
|
|
);
|
|
|
|
Tag {
|
|
span: Span::new(self.span.start, other.span.end),
|
|
anchor: self.anchor.clone(),
|
|
}
|
|
}
|
|
|
|
pub fn until_option(&self, other: Option<impl Into<Tag>>) -> Tag {
|
|
match other {
|
|
Some(other) => {
|
|
let other = other.into();
|
|
debug_assert!(
|
|
self.anchor == other.anchor,
|
|
"Can only merge two tags with the same anchor"
|
|
);
|
|
|
|
Tag {
|
|
span: Span::new(self.span.start, other.span.end),
|
|
anchor: self.anchor.clone(),
|
|
}
|
|
}
|
|
None => self.clone(),
|
|
}
|
|
}
|
|
|
|
pub fn slice<'a>(&self, source: &'a str) -> &'a str {
|
|
self.span.slice(source)
|
|
}
|
|
|
|
pub fn string<'a>(&self, source: &'a str) -> String {
|
|
self.span.slice(source).to_string()
|
|
}
|
|
|
|
pub fn tagged_slice<'a>(&self, source: &'a str) -> Tagged<&'a str> {
|
|
self.span.slice(source).tagged(self)
|
|
}
|
|
|
|
pub fn tagged_string<'a>(&self, source: &'a str) -> Tagged<String> {
|
|
self.span.slice(source).to_string().tagged(self)
|
|
}
|
|
|
|
pub fn anchor_name(&self) -> Option<String> {
|
|
match self.anchor {
|
|
Some(AnchorLocation::File(ref file)) => Some(file.clone()),
|
|
Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn tag_for_tagged_list(mut iter: impl Iterator<Item = Tag>) -> Tag {
|
|
let first = iter.next();
|
|
|
|
let first = match first {
|
|
None => return Tag::unknown(),
|
|
Some(first) => first,
|
|
};
|
|
|
|
let last = iter.last();
|
|
|
|
match last {
|
|
None => first,
|
|
Some(last) => first.until(last),
|
|
}
|
|
}
|
|
|
|
pub fn span_for_spanned_list(mut iter: impl Iterator<Item = Span>) -> Span {
|
|
let first = iter.next();
|
|
|
|
let first = match first {
|
|
None => return Span::unknown(),
|
|
Some(first) => first,
|
|
};
|
|
|
|
let last = iter.last();
|
|
|
|
match last {
|
|
None => first,
|
|
Some(last) => first.until(last),
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
|
|
pub struct Span {
|
|
start: usize,
|
|
end: usize,
|
|
}
|
|
|
|
impl From<&Span> for Span {
|
|
fn from(span: &Span) -> Span {
|
|
*span
|
|
}
|
|
}
|
|
|
|
impl From<Option<Span>> for Span {
|
|
fn from(input: Option<Span>) -> Span {
|
|
match input {
|
|
None => Span::new(0, 0),
|
|
Some(span) => span,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Span {
|
|
pub fn unknown() -> Span {
|
|
Span::new(0, 0)
|
|
}
|
|
|
|
pub fn new(start: usize, end: usize) -> Span {
|
|
assert!(
|
|
end >= start,
|
|
"Can't create a Span whose end < start, start={}, end={}",
|
|
start,
|
|
end
|
|
);
|
|
|
|
Span { start, end }
|
|
}
|
|
|
|
pub fn for_char(pos: usize) -> Span {
|
|
Span {
|
|
start: pos,
|
|
end: pos + 1,
|
|
}
|
|
}
|
|
|
|
pub fn contains(&self, pos: usize) -> bool {
|
|
self.start <= pos && self.end >= pos
|
|
}
|
|
|
|
pub fn since(&self, other: impl Into<Span>) -> Span {
|
|
let other = other.into();
|
|
|
|
Span::new(other.start, self.end)
|
|
}
|
|
|
|
pub fn until(&self, other: impl Into<Span>) -> Span {
|
|
let other = other.into();
|
|
|
|
Span::new(self.start, other.end)
|
|
}
|
|
|
|
pub fn until_option(&self, other: Option<impl Into<Span>>) -> Span {
|
|
match other {
|
|
Some(other) => {
|
|
let other = other.into();
|
|
|
|
Span::new(self.start, other.end)
|
|
}
|
|
None => *self,
|
|
}
|
|
}
|
|
|
|
pub fn string<'a>(&self, source: &'a str) -> String {
|
|
self.slice(source).to_string()
|
|
}
|
|
|
|
pub fn spanned_slice<'a>(&self, source: &'a str) -> Spanned<&'a str> {
|
|
self.slice(source).spanned(*self)
|
|
}
|
|
|
|
pub fn spanned_string<'a>(&self, source: &'a str) -> Spanned<String> {
|
|
self.slice(source).to_string().spanned(*self)
|
|
}
|
|
|
|
pub fn start(&self) -> usize {
|
|
self.start
|
|
}
|
|
|
|
pub fn end(&self) -> usize {
|
|
self.end
|
|
}
|
|
|
|
pub fn is_unknown(&self) -> bool {
|
|
self.start == 0 && self.end == 0
|
|
}
|
|
|
|
pub fn slice<'a>(&self, source: &'a str) -> &'a str {
|
|
&source[self.start..self.end]
|
|
}
|
|
}
|
|
|
|
impl language_reporting::ReportingSpan for Span {
|
|
fn with_start(&self, start: usize) -> Self {
|
|
if self.end < start {
|
|
Span::new(start, start)
|
|
} else {
|
|
Span::new(start, self.end)
|
|
}
|
|
}
|
|
|
|
fn with_end(&self, end: usize) -> Self {
|
|
if end < self.start {
|
|
Span::new(end, end)
|
|
} else {
|
|
Span::new(self.start, end)
|
|
}
|
|
}
|
|
|
|
fn start(&self) -> usize {
|
|
self.start
|
|
}
|
|
|
|
fn end(&self) -> usize {
|
|
self.end
|
|
}
|
|
}
|
|
|
|
pub trait IntoSpanned {
|
|
type Output: HasFallibleSpan;
|
|
|
|
fn into_spanned(self, span: impl Into<Span>) -> Self::Output;
|
|
}
|
|
|
|
impl<T: HasFallibleSpan> IntoSpanned for T {
|
|
type Output = T;
|
|
fn into_spanned(self, _span: impl Into<Span>) -> Self::Output {
|
|
self
|
|
}
|
|
}
|
|
|
|
pub trait HasSpan {
|
|
fn span(&self) -> Span;
|
|
}
|
|
|
|
impl<T, E> HasSpan for Result<T, E>
|
|
where
|
|
T: HasSpan,
|
|
{
|
|
fn span(&self) -> Span {
|
|
match self {
|
|
Result::Ok(val) => val.span(),
|
|
Result::Err(_) => Span::unknown(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T> HasSpan for Spanned<T> {
|
|
fn span(&self) -> Span {
|
|
self.span
|
|
}
|
|
}
|
|
|
|
pub trait HasFallibleSpan {
|
|
fn maybe_span(&self) -> Option<Span>;
|
|
}
|
|
|
|
impl HasFallibleSpan for bool {
|
|
fn maybe_span(&self) -> Option<Span> {
|
|
None
|
|
}
|
|
}
|
|
|
|
impl HasFallibleSpan for () {
|
|
fn maybe_span(&self) -> Option<Span> {
|
|
None
|
|
}
|
|
}
|
|
|
|
impl<T> HasFallibleSpan for T
|
|
where
|
|
T: HasSpan,
|
|
{
|
|
fn maybe_span(&self) -> Option<Span> {
|
|
Some(HasSpan::span(self))
|
|
}
|
|
}
|
|
|
|
impl PrettyDebugWithSource for Option<Span> {
|
|
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
|
|
match self {
|
|
None => b::description("no span"),
|
|
Some(span) => span.pretty_debug(source),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl HasFallibleSpan for Option<Span> {
|
|
fn maybe_span(&self) -> Option<Span> {
|
|
*self
|
|
}
|
|
}
|
|
|
|
impl PrettyDebugWithSource for Span {
|
|
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
|
|
b::typed(
|
|
"span",
|
|
b::keyword("for") + b::space() + b::description(format!("{:?}", self.slice(source))),
|
|
)
|
|
}
|
|
}
|
|
|
|
impl HasSpan for Span {
|
|
fn span(&self) -> Span {
|
|
*self
|
|
}
|
|
}
|
|
|
|
impl<T> PrettyDebugWithSource for Option<Spanned<T>>
|
|
where
|
|
Spanned<T>: PrettyDebugWithSource,
|
|
{
|
|
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
|
|
match self {
|
|
None => b::description("nothing"),
|
|
Some(v) => v.pretty_debug(v.span.slice(source)),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T> HasFallibleSpan for Option<Spanned<T>> {
|
|
fn maybe_span(&self) -> Option<Span> {
|
|
match self {
|
|
None => None,
|
|
Some(value) => Some(value.span),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T> PrettyDebugWithSource for Option<Tagged<T>>
|
|
where
|
|
Tagged<T>: PrettyDebugWithSource,
|
|
{
|
|
fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
|
|
match self {
|
|
None => b::description("nothing"),
|
|
Some(d) => d.pretty_debug(source),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T> HasFallibleSpan for Option<Tagged<T>> {
|
|
fn maybe_span(&self) -> Option<Span> {
|
|
match self {
|
|
None => None,
|
|
Some(value) => Some(value.tag.span),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T> HasSpan for Tagged<T> {
|
|
fn span(&self) -> Span {
|
|
self.tag.span
|
|
}
|
|
}
|