extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-08-09 16:15:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add experimental new parser (#1554)

Browse Source 
Move to an experimental new parser
This commit is contained in:
Jonathan Turner
2020-04-06 00:16:14 -07:00
committed by GitHub
parent 0a198b9bd0
commit c4daa2e40f
123 changed files with 2391 additions and 11585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/nu-cli/Cargo.toml
View File

@ -1,7 +1,7 @@
[package]
name = "nu-cli"
version = "0.12.0"
authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
authors = ["The Nu Project Contributors"]
description = "CLI for nushell"
edition = "2018"
license = "MIT"
@ -16,7 +16,6 @@ nu-protocol = { version = "0.12.0", path = "../nu-protocol" }
nu-errors = { version = "0.12.0", path = "../nu-errors" }
nu-parser = { version = "0.12.0", path = "../nu-parser" }
nu-value-ext = { version = "0.12.0", path = "../nu-value-ext" }
nu-macros = { version = "0.12.0", path = "../nu-macros" }
nu-test-support = { version = "0.12.0", path = "../nu-test-support" }
ansi_term = "0.12.1"

41
crates/nu-cli/src/cli.rs
View File

@ -10,13 +10,10 @@ use crate::prelude::*;
use futures_codec::FramedRead;
use nu_errors::ShellError;
use nu_parser::{
ClassifiedCommand, ClassifiedPipeline, ExternalCommand, PipelineShape, SpannedToken,
TokensIterator,
};
use nu_parser::{ClassifiedCommand, ExternalCommand};
use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
use log::{debug, log_enabled, trace};
use log::{debug, trace};
use rustyline::error::ReadlineError;
use rustyline::{
self, config::Configurer, config::EditMode, At, Cmd, ColorMode, CompletionType, Config, Editor,
@ -614,9 +611,9 @@ async fn process_line(
Ok(line) => {
let line = chomp_newline(line);
let result = match nu_parser::parse(&line) {
let result = match nu_parser::lite_parse(&line, 0) {
Err(err) => {
return LineResult::Error(line.to_string(), err);
return LineResult::Error(line.to_string(), err.into());
}
Ok(val) => val,
@ -625,7 +622,9 @@ async fn process_line(
debug!("=== Parsed ===");
debug!("{:#?}", result);
let pipeline = classify_pipeline(&result, &ctx, &Text::from(line));
let pipeline = nu_parser::classify_pipeline(&result, ctx.registry());
//println!("{:#?}", pipeline);
if let Some(failure) = pipeline.failed {
return LineResult::Error(line.to_string(), failure.into());
@ -642,9 +641,9 @@ async fn process_line(
ref name, ref args, ..
}) = pipeline.commands.list[0]
{
if dunce::canonicalize(name).is_ok()
&& PathBuf::from(name).is_dir()
&& ichwh::which(name).await.unwrap_or(None).is_none()
if dunce::canonicalize(&name).is_ok()
&& PathBuf::from(&name).is_dir()
&& ichwh::which(&name).await.unwrap_or(None).is_none()
&& args.list.is_empty()
{
// Here we work differently if we're in Windows because of the expected Windows behavior
@ -762,26 +761,6 @@ async fn process_line(
}
}
pub fn classify_pipeline(
pipeline: &SpannedToken,
context: &Context,
source: &Text,
) -> ClassifiedPipeline {
let pipeline_list = vec![pipeline.clone()];
let expand_context = context.expand_context(source);
let mut iterator = TokensIterator::new(&pipeline_list, expand_context, pipeline.span());
let result = iterator.expand_infallible(PipelineShape);
if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
outln!("");
let _ = ptree::print_tree(&iterator.expand_tracer().print(source.clone()));
outln!("");
}
result
}
pub fn print_err(err: ShellError, host: &dyn Host, source: &Text) {
let diag = err.into_diagnostic();

2
crates/nu-cli/src/commands/autoview.rs
View File

@ -282,7 +282,7 @@ fn create_default_command_args(context: &RunnableContextWithoutInput) -> RawComm
call_info: UnevaluatedCallInfo {
args: hir::Call {
head: Box::new(SpannedExpression::new(
Expression::Literal(Literal::String(span)),
Expression::Literal(Literal::String(String::new())),
span,
)),
positional: None,

17
crates/nu-cli/src/commands/cd.rs
View File

@ -1,7 +1,6 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use nu_errors::ShellError;
use nu_macros::signature;
use nu_protocol::{Signature, SyntaxShape};
pub struct Cd;
@ -12,17 +11,11 @@ impl WholeStreamCommand for Cd {
}
fn signature(&self) -> Signature {
signature! {
def cd {
"the directory to change to"
directory(optional Path) - "the directory to change to"
}
}
// Signature::build("cd").optional(
// "directory",
// SyntaxShape::Path,
// "the directory to change to",
// )
Signature::build("cd").optional(
"directory",
SyntaxShape::Path,
"the directory to change to",
)
}
fn usage(&self) -> &str {

2
crates/nu-cli/src/commands/classified/dynamic.rs
View File

@ -1,7 +1,7 @@
use derive_new::new;
use nu_parser::hir;
#[derive(new, Debug, Eq, PartialEq)]
#[derive(new, Debug)]
pub(crate) struct Command {
pub(crate) args: hir::Call,
}

2
crates/nu-cli/src/commands/classified/external.rs
View File

@ -6,7 +6,7 @@ use futures::stream::StreamExt;
use futures_codec::FramedRead;
use log::trace;
use nu_errors::ShellError;
use nu_parser::commands::classified::external::ExternalArg;
use nu_parser::ExternalArg;
use nu_parser::ExternalCommand;
use nu_protocol::{ColumnPath, Primitive, ShellTypeName, UntaggedValue, Value};
use nu_source::{Tag, Tagged};

4
crates/nu-cli/src/commands/classified/internal.rs
View File

@ -28,7 +28,7 @@ pub(crate) fn run_internal_command(
let result = {
context.run_command(
internal_command?,
command.name_tag.clone(),
Tag::unknown_anchor(command.name_span),
command.args.clone(),
&source,
objects,
@ -71,7 +71,7 @@ pub(crate) fn run_internal_command(
span: Span::unknown()
},
source: source.clone(),
name_tag: command.name_tag,
name_tag: Tag::unknown_anchor(command.name_span),
}
};
let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &context.registry);

8
crates/nu-cli/src/commands/classified/pipeline.rs
View File

@ -23,10 +23,9 @@ pub(crate) async fn run_pipeline(
return Err(ShellError::unimplemented("Dynamic commands"))
}
(Some(ClassifiedCommand::Expr(_)), _) | (_, Some(ClassifiedCommand::Expr(_))) => {
return Err(ShellError::unimplemented("Expression-only commands"))
}
// (Some(ClassifiedCommand::Expr(_)), _) | (_, Some(ClassifiedCommand::Expr(_))) => {
// return Err(ShellError::unimplemented("Expression-only commands"))
// }
(Some(ClassifiedCommand::Error(err)), _) => return Err(err.into()),
(_, Some(ClassifiedCommand::Error(err))) => return Err(err.clone().into()),
@ -43,6 +42,7 @@ pub(crate) async fn run_pipeline(
}
(None, _) => break,
_ => unimplemented!("Not yet implented cases in run_pipeline"),
};
}

2
crates/nu-cli/src/commands/config.rs
View File

@ -41,7 +41,7 @@ impl WholeStreamCommand for Config {
)
.named(
"set_into",
SyntaxShape::Member,
SyntaxShape::String,
"sets a variable from values in the pipeline",
Some('i'),
)

196
crates/nu-cli/src/commands/from_delimited_data.rs
View File

@ -1,12 +1,45 @@
use crate::prelude::*;
use csv::{ErrorKind, ReaderBuilder};
use nu_errors::ShellError;
use nu_parser::hir::syntax_shape::{ExpandContext, SignatureRegistry};
use nu_parser::utils::{parse_line_with_separator as parse, LineSeparatedShape};
use nu_parser::TokensIterator;
use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
use nu_source::nom_input;
use nu_protocol::{ReturnSuccess, TaggedDictBuilder, UntaggedValue, Value};
use derive_new::new;
fn from_delimited_string_to_value(
s: String,
headerless: bool,
separator: char,
tag: impl Into<Tag>,
) -> Result<Value, csv::Error> {
let mut reader = ReaderBuilder::new()
.has_headers(!headerless)
.delimiter(separator as u8)
.from_reader(s.as_bytes());
let tag = tag.into();
let headers = if headerless {
(1..=reader.headers()?.len())
.map(|i| format!("Column{}", i))
.collect::<Vec<String>>()
} else {
reader.headers()?.iter().map(String::from).collect()
};
let mut rows = vec![];
for row in reader.records() {
let mut tagged_row = TaggedDictBuilder::new(&tag);
for (value, header) in row?.iter().zip(headers.iter()) {
if let Ok(i) = value.parse::<i64>() {
tagged_row.insert_value(header, UntaggedValue::int(i).into_value(&tag))
} else if let Ok(f) = value.parse::<f64>() {
tagged_row.insert_value(header, UntaggedValue::decimal(f).into_value(&tag))
} else {
tagged_row.insert_value(header, UntaggedValue::string(value).into_value(&tag))
}
}
rows.push(tagged_row.into_value());
}
Ok(UntaggedValue::Table(rows).into_value(&tag))
}
pub fn from_delimited_data(
headerless: bool,
@ -20,20 +53,19 @@ pub fn from_delimited_data(
let concat_string = input.collect_string(name_tag.clone()).await?;
match from_delimited_string_to_value(concat_string.item, headerless, sep, name_tag.clone()) {
Ok(rows) => {
for row in rows {
match row {
Value { value: UntaggedValue::Table(list), .. } => {
for l in list {
yield ReturnSuccess::value(l);
}
}
x => yield ReturnSuccess::value(x),
Ok(x) => match x {
Value { value: UntaggedValue::Table(list), .. } => {
for l in list {
yield ReturnSuccess::value(l);
}
}
x => yield ReturnSuccess::value(x),
},
Err(err) => {
let line_one = format!("Could not parse as {}", format_name);
let line_one = match pretty_csv_error(err) {
Some(pretty) => format!("Could not parse as {} ({})", format_name,pretty),
None => format!("Could not parse as {}", format_name),
};
let line_two = format!("input cannot be parsed as {}", format_name);
yield Err(ShellError::labeled_error_with_secondary(
line_one,
@ -49,121 +81,25 @@ pub fn from_delimited_data(
Ok(stream.to_output_stream())
}
#[derive(Debug, Clone, new)]
pub struct EmptyRegistry {
#[new(default)]
signatures: indexmap::IndexMap<String, Signature>,
}
impl EmptyRegistry {}
impl SignatureRegistry for EmptyRegistry {
fn has(&self, _name: &str) -> bool {
false
}
fn get(&self, _name: &str) -> Option<Signature> {
None
}
fn clone_box(&self) -> Box<dyn SignatureRegistry> {
Box::new(self.clone())
}
}
fn from_delimited_string_to_value(
s: String,
headerless: bool,
sep: char,
tag: impl Into<Tag>,
) -> Result<Vec<Value>, ShellError> {
let tag = tag.into();
let mut entries = s.lines();
let mut fields = vec![];
let mut out = vec![];
if let Some(first_entry) = entries.next() {
let tokens = match parse(&sep.to_string(), nom_input(first_entry)) {
Ok((_, tokens)) => tokens,
Err(err) => return Err(ShellError::parse_error(err)),
};
let tokens_span = tokens.span;
let source: nu_source::Text = tokens_span.slice(&first_entry).into();
if !headerless {
fields = tokens
.item
.iter()
.filter(|token| !token.is_separator())
.map(|field| field.source(&source).to_string())
.collect::<Vec<_>>();
}
let registry = Box::new(EmptyRegistry::new());
let ctx = ExpandContext::new(registry, &source, None);
let mut iterator = TokensIterator::new(&tokens.item, ctx, tokens_span);
let (results, tokens_identified) = iterator.expand(LineSeparatedShape);
let results = results?;
let mut row = TaggedDictBuilder::new(&tag);
if headerless {
let fallback_columns = (1..=tokens_identified)
.map(|i| format!("Column{}", i))
.collect::<Vec<String>>();
for (idx, field) in results.into_iter().enumerate() {
let key = if headerless {
&fallback_columns[idx]
} else {
&fields[idx]
};
row.insert_value(key, field.into_value(&tag));
}
out.push(row.into_value())
}
}
for entry in entries {
let tokens = match parse(&sep.to_string(), nom_input(entry)) {
Ok((_, tokens)) => tokens,
Err(err) => return Err(ShellError::parse_error(err)),
};
let tokens_span = tokens.span;
let source: nu_source::Text = tokens_span.slice(&entry).into();
let registry = Box::new(EmptyRegistry::new());
let ctx = ExpandContext::new(registry, &source, None);
let mut iterator = TokensIterator::new(&tokens.item, ctx, tokens_span);
let (results, tokens_identified) = iterator.expand(LineSeparatedShape);
let results = results?;
let mut row = TaggedDictBuilder::new(&tag);
let fallback_columns = (1..=tokens_identified)
.map(|i| format!("Column{}", i))
.collect::<Vec<String>>();
for (idx, field) in results.into_iter().enumerate() {
let key = if headerless {
&fallback_columns[idx]
fn pretty_csv_error(err: csv::Error) -> Option<String> {
match err.kind() {
ErrorKind::UnequalLengths {
pos,
expected_len,
len,
} => {
if let Some(pos) = pos {
Some(format!(
"Line {}: expected {} fields, found {}",
pos.line(),
expected_len,
len
))
} else {
match fields.get(idx) {
Some(key) => key,
None => &fallback_columns[idx],
}
};
row.insert_value(key, field.into_value(&tag));
Some(format!("Expected {} fields, found {}", expected_len, len))
}
}
out.push(row.into_value())
ErrorKind::Seek => Some("Internal error while parsing csv".to_string()),
_ => None,
}
Ok(out)
}

2
crates/nu-cli/src/commands/histogram.rs
View File

@ -30,7 +30,7 @@ impl WholeStreamCommand for Histogram {
"the name of the column to graph by",
)
.rest(
SyntaxShape::Member,
SyntaxShape::String,
"column name to give the histogram's frequency column",
)
}

2
crates/nu-cli/src/commands/nth.rs
View File

@ -22,7 +22,7 @@ impl WholeStreamCommand for Nth {
Signature::build("nth")
.required(
"row number",
SyntaxShape::Any,
SyntaxShape::Int,
"the number of the row to return",
)
.rest(SyntaxShape::Any, "Optionally return more rows")

2
crates/nu-cli/src/commands/reject.rs
View File

@ -18,7 +18,7 @@ impl WholeStreamCommand for Reject {
}
fn signature(&self) -> Signature {
Signature::build("reject").rest(SyntaxShape::Member, "the names of columns to remove")
Signature::build("reject").rest(SyntaxShape::String, "the names of columns to remove")
}
fn usage(&self) -> &str {

2
crates/nu-cli/src/commands/rename.rs
View File

@ -26,7 +26,7 @@ impl WholeStreamCommand for Rename {
"the name of the column to rename for",
)
.rest(
SyntaxShape::Member,
SyntaxShape::String,
"Additional column name(s) to rename for",
)
}

2
crates/nu-cli/src/commands/split_column.rs
View File

@ -30,7 +30,7 @@ impl WholeStreamCommand for SplitColumn {
"the character that denotes what separates columns",
)
.switch("collapse-empty", "remove empty columns", Some('c'))
.rest(SyntaxShape::Member, "column names to give the new columns")
.rest(SyntaxShape::String, "column names to give the new columns")
}
fn usage(&self) -> &str {

13
crates/nu-cli/src/context.rs
View File

@ -4,7 +4,7 @@ use crate::shell::shell_manager::ShellManager;
use crate::stream::{InputStream, OutputStream};
use indexmap::IndexMap;
use nu_errors::ShellError;
use nu_parser::{hir, hir::syntax_shape::ExpandContext, hir::syntax_shape::SignatureRegistry};
use nu_parser::{hir, SignatureRegistry};
use nu_protocol::Signature;
use nu_source::{Tag, Text};
use parking_lot::Mutex;
@ -92,17 +92,6 @@ impl Context {
&self.registry
}
pub(crate) fn expand_context<'context>(
&'context self,
source: &'context Text,
) -> ExpandContext {
ExpandContext::new(
Box::new(self.registry.clone()),
source,
self.shell_manager.homedir(),
)
}
pub(crate) fn basic() -> Result<Context, Box<dyn Error>> {
let registry = CommandRegistry::new();

4
crates/nu-cli/src/data/base.rs
View File

@ -7,7 +7,7 @@ use chrono::{DateTime, Utc};
use derive_new::new;
use log::trace;
use nu_errors::ShellError;
use nu_parser::{hir, CompareOperator};
use nu_parser::hir;
use nu_protocol::{
Evaluate, EvaluateTrait, Primitive, Scope, ShellTypeName, SpannedTypeName, TaggedDictBuilder,
UntaggedValue, Value,
@ -23,7 +23,7 @@ use std::time::SystemTime;
#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new, Serialize)]
pub struct Operation {
pub(crate) left: Value,
pub(crate) operator: CompareOperator,
pub(crate) operator: hir::CompareOperator,
pub(crate) right: Value,
}

2
crates/nu-cli/src/data/primitive.rs
View File

@ -1,4 +1,4 @@
use nu_parser::Number;
use nu_parser::hir::Number;
use nu_protocol::Primitive;
pub fn number(number: impl Into<Number>) -> Primitive {

2
crates/nu-cli/src/data/value.rs
View File

@ -3,7 +3,7 @@ use crate::data::base::shape::{Column, InlineShape};
use crate::data::primitive::style_primitive;
use chrono::DateTime;
use nu_errors::ShellError;
use nu_parser::CompareOperator;
use nu_parser::hir::CompareOperator;
use nu_protocol::{Primitive, Type, UntaggedValue};
use nu_source::{DebugDocBuilder, PrettyDebug, Tagged};

2
crates/nu-cli/src/evaluate/evaluate_args.rs
View File

@ -36,7 +36,7 @@ pub(crate) fn evaluate_args(
hir::NamedValue::PresentSwitch(tag) => {
results.insert(name.clone(), UntaggedValue::boolean(true).into_value(tag));
}
hir::NamedValue::Value(expr) => {
hir::NamedValue::Value(_, expr) => {
results.insert(
name.clone(),
evaluate_baseline_expr(expr, registry, scope, source)?,

48
crates/nu-cli/src/evaluate/evaluator.rs
View File

@ -35,22 +35,27 @@ pub(crate) fn evaluate_baseline_expr(
Expression::Command(_) => evaluate_command(tag, scope, source),
Expression::ExternalCommand(external) => evaluate_external(external, scope, source),
Expression::Binary(binary) => {
let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?;
let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?;
let left = evaluate_baseline_expr(&binary.left, registry, scope, source)?;
let right = evaluate_baseline_expr(&binary.right, registry, scope, source)?;
trace!("left={:?} right={:?}", left.value, right.value);
match apply_operator(**binary.op(), &left, &right) {
Ok(result) => Ok(result.into_value(tag)),
Err((left_type, right_type)) => Err(ShellError::coerce_error(
left_type.spanned(binary.left().span),
right_type.spanned(binary.right().span),
)),
match binary.op.expr {
Expression::Literal(hir::Literal::Operator(op)) => {
match apply_operator(op, &left, &right) {
Ok(result) => Ok(result.into_value(tag)),
Err((left_type, right_type)) => Err(ShellError::coerce_error(
left_type.spanned(binary.left.span),
right_type.spanned(binary.right.span),
)),
}
}
_ => unreachable!(),
}
}
Expression::Range(range) => {
let left = range.left();
let right = range.right();
let left = &range.left;
let right = &range.right;
let left = evaluate_baseline_expr(left, registry, scope, source)?;
let right = evaluate_baseline_expr(right, registry, scope, source)?;
@ -85,10 +90,10 @@ pub(crate) fn evaluate_baseline_expr(
)))
.into_value(&tag)),
Expression::Path(path) => {
let value = evaluate_baseline_expr(path.head(), registry, scope, source)?;
let value = evaluate_baseline_expr(&path.head, registry, scope, source)?;
let mut item = value;
for member in path.tail() {
for member in &path.tail {
let next = item.get_data_by_member(member);
match next {
@ -123,28 +128,29 @@ pub(crate) fn evaluate_baseline_expr(
Ok(item.value.into_value(tag))
}
Expression::Boolean(_boolean) => unimplemented!(),
Expression::Garbage => unimplemented!(),
}
}
fn evaluate_literal(literal: &hir::Literal, span: Span, source: &Text) -> Value {
match &literal {
hir::Literal::ColumnPath(path) => {
let members = path
.iter()
.map(|member| member.to_path_member(source))
.collect();
let members = path.iter().map(|member| member.to_path_member()).collect();
UntaggedValue::Primitive(Primitive::ColumnPath(ColumnPath::new(members)))
.into_value(span)
}
hir::Literal::Number(int) => match int {
nu_parser::Number::Int(i) => UntaggedValue::int(i.clone()).into_value(span),
nu_parser::Number::Decimal(d) => UntaggedValue::decimal(d.clone()).into_value(span),
nu_parser::hir::Number::Int(i) => UntaggedValue::int(i.clone()).into_value(span),
nu_parser::hir::Number::Decimal(d) => {
UntaggedValue::decimal(d.clone()).into_value(span)
}
},
hir::Literal::Size(int, unit) => unit.compute(&int).into_value(span),
hir::Literal::String(tag) => UntaggedValue::string(tag.slice(source)).into_value(span),
hir::Literal::String(string) => UntaggedValue::string(string).into_value(span),
hir::Literal::GlobPattern(pattern) => UntaggedValue::pattern(pattern).into_value(span),
hir::Literal::Bare => UntaggedValue::string(span.slice(source)).into_value(span),
hir::Literal::Operator(_) => unimplemented!("Not sure what to do with operator yet"),
}
}
@ -157,7 +163,7 @@ fn evaluate_reference(
trace!("Evaluating {:?} with Scope {:?}", name, scope);
match name {
hir::Variable::It(_) => Ok(scope.it.value.clone().into_value(tag)),
hir::Variable::Other(inner) => match inner.slice(source) {
hir::Variable::Other(_, span) => match span.slice(source) {
x if x == "nu" => crate::evaluate::variables::nu(tag),
x => Ok(scope
.vars
@ -174,7 +180,7 @@ fn evaluate_external(
_source: &Text,
) -> Result<Value, ShellError> {
Err(ShellError::syntax_error(
"Unexpected external command".spanned(*external.name()),
"Unexpected external command".spanned(external.name.span),
))
}

2
crates/nu-cli/src/evaluate/operator.rs
View File

@ -1,5 +1,5 @@
use crate::data::value;
use nu_parser::CompareOperator;
use nu_parser::hir::CompareOperator;
use nu_protocol::{Primitive, ShellTypeName, UntaggedValue, Value};
use std::ops::Not;

1
crates/nu-cli/src/lib.rs
View File

@ -27,7 +27,6 @@ pub use crate::data::primitive;
pub use crate::data::value;
pub use crate::env::environment_syncer::EnvironmentSyncer;
pub use crate::env::host::BasicHost;
pub use nu_parser::TokenTreeBuilder;
pub use nu_value_ext::ValueExt;
pub use num_traits::cast::ToPrimitive;

4
crates/nu-cli/src/prelude.rs
View File

@ -93,8 +93,8 @@ pub(crate) use futures::stream::BoxStream;
pub(crate) use futures::{FutureExt, Stream, StreamExt};
pub(crate) use nu_protocol::{EvaluateTrait, MaybeOwned};
pub(crate) use nu_source::{
b, AnchorLocation, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span,
SpannedItem, Tag, TaggedItem, Text,
b, AnchorLocation, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, Span, SpannedItem, Tag,
TaggedItem, Text,
};
pub(crate) use nu_value_ext::ValueExt;
pub(crate) use num_bigint::BigInt;

70
crates/nu-cli/src/shell/completer.rs
View File

@ -1,8 +1,6 @@
use crate::context::CommandRegistry;
use derive_new::new;
use nu_parser::ExpandContext;
use nu_source::{HasSpan, Text};
use rustyline::completion::{Completer, FilenameCompleter};
use std::path::PathBuf;
@ -20,14 +18,6 @@ impl NuCompleter {
pos: usize,
context: &rustyline::Context,
) -> rustyline::Result<(usize, Vec<rustyline::completion::Pair>)> {
let text = Text::from(line);
let expand_context =
ExpandContext::new(Box::new(self.commands.clone()), &text, self.homedir.clone());
#[allow(unused)]
// smarter completions
let shapes = nu_parser::pipeline_shapes(line, expand_context);
let commands: Vec<String> = self.commands.names();
let line_chars: Vec<_> = line[..pos].chars().collect();
@ -44,7 +34,17 @@ impl NuCompleter {
// See if we're a flag
if pos > 0 && replace_pos < line_chars.len() && line_chars[replace_pos] == '-' {
completions = self.get_matching_arguments(&line_chars, line, replace_pos, pos);
if let Ok(lite_pipeline) = nu_parser::lite_parse(line, 0) {
completions = self.get_matching_arguments(
&lite_pipeline,
&line_chars,
line,
replace_pos,
pos,
);
} else {
completions = self.file_completer.complete(line, pos, context)?.1;
}
} else {
completions = self.file_completer.complete(line, pos, context)?.1;
@ -96,6 +96,7 @@ impl NuCompleter {
fn get_matching_arguments(
&self,
lite_parse: &nu_parser::LitePipeline,
line_chars: &[char],
line: &str,
replace_pos: usize,
@ -108,40 +109,23 @@ impl NuCompleter {
let replace_string = (replace_pos..pos).map(|_| " ").collect::<String>();
line_copy.replace_range(replace_pos..pos, &replace_string);
if let Ok(val) = nu_parser::parse(&line_copy) {
let source = Text::from(line);
let pipeline_list = vec![val.clone()];
let result = nu_parser::classify_pipeline(&lite_parse, &self.commands);
let expand_context = nu_parser::ExpandContext {
homedir: None,
registry: Box::new(self.commands.clone()),
source: &source,
};
for command in result.commands.list {
if let nu_parser::ClassifiedCommand::Internal(nu_parser::InternalCommand {
args, ..
}) = command
{
if replace_pos >= args.span.start() && replace_pos <= args.span.end() {
if let Some(named) = args.named {
for (name, _) in named.iter() {
let full_flag = format!("--{}", name);
let mut iterator =
nu_parser::TokensIterator::new(&pipeline_list, expand_context, val.span());
let result = iterator.expand_infallible(nu_parser::PipelineShape);
if result.failed.is_none() {
for command in result.commands.list {
if let nu_parser::ClassifiedCommand::Internal(nu_parser::InternalCommand {
args,
..
}) = command
{
if replace_pos >= args.span.start() && replace_pos <= args.span.end() {
if let Some(named) = args.named {
for (name, _) in named.iter() {
let full_flag = format!("--{}", name);
if full_flag.starts_with(&substring) {
matching_arguments.push(rustyline::completion::Pair {
display: full_flag.clone(),
replacement: full_flag,
});
}
}
if full_flag.starts_with(&substring) {
matching_arguments.push(rustyline::completion::Pair {
display: full_flag.clone(),
replacement: full_flag,
});
}
}
}

9
crates/nu-cli/src/shell/filesystem_shell.rs
View File

@ -10,7 +10,6 @@ use crate::shell::completer::NuCompleter;
use crate::shell::shell::Shell;
use crate::utils::FileStructure;
use nu_errors::ShellError;
use nu_parser::ExpandContext;
use nu_protocol::{Primitive, ReturnSuccess, UntaggedValue};
use rustyline::completion::FilenameCompleter;
use rustyline::hint::{Hinter, HistoryHinter};
@ -1149,13 +1148,7 @@ impl Shell for FilesystemShell {
self.completer.complete(line, pos, ctx)
}
fn hint(
&self,
line: &str,
pos: usize,
ctx: &rustyline::Context<'_>,
_expand_context: ExpandContext,
) -> Option<String> {
fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option<String> {
self.hinter.hint(line, pos, ctx)
}
}

9
crates/nu-cli/src/shell/help_shell.rs
View File

@ -8,7 +8,6 @@ use crate::data::command_dict;
use crate::prelude::*;
use crate::shell::shell::Shell;
use nu_errors::ShellError;
use nu_parser::ExpandContext;
use nu_protocol::{
Primitive, ReturnSuccess, ShellTypeName, TaggedDictBuilder, UntaggedValue, Value,
};
@ -249,13 +248,7 @@ impl Shell for HelpShell {
Ok((replace_pos, completions))
}
fn hint(
&self,
_line: &str,
_pos: usize,
_ctx: &rustyline::Context<'_>,
_context: ExpandContext,
) -> Option<String> {
fn hint(&self, _line: &str, _pos: usize, _ctx: &rustyline::Context<'_>) -> Option<String> {
None
}
}

187
crates/nu-cli/src/shell/helper.rs
View File

@ -1,9 +1,8 @@
use crate::context::Context;
use ansi_term::{Color, Style};
use log::log_enabled;
use nu_parser::{FlatShape, PipelineShape, ShapeResult, Token, TokensIterator};
use nu_protocol::{errln, outln};
use nu_source::{nom_input, HasSpan, Tag, Tagged, Text};
use nu_parser::hir::FlatShape;
use nu_parser::SignatureRegistry;
use nu_source::{Span, Spanned, Tag, Tagged};
use rustyline::completion::Completer;
use rustyline::error::ReadlineError;
use rustyline::highlight::Highlighter;
@ -38,10 +37,7 @@ impl Completer for Helper {
impl Hinter for Helper {
fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option<String> {
let text = Text::from(line);
self.context
.shell_manager
.hint(line, pos, ctx, self.context.expand_context(&text))
self.context.shell_manager.hint(line, pos, ctx)
}
}
@ -65,49 +61,19 @@ impl Highlighter for Helper {
}
fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> {
let tokens = nu_parser::pipeline(nom_input(line));
let lite_pipeline = nu_parser::lite_parse(line, 0);
match tokens {
match lite_pipeline {
Err(_) => Cow::Borrowed(line),
Ok((_rest, v)) => {
let pipeline = match v.as_pipeline() {
Err(_) => return Cow::Borrowed(line),
Ok(v) => v,
};
Ok(lp) => {
let classified =
nu_parser::classify_pipeline(&lp, &self.context.registry().clone_box());
let text = Text::from(line);
let expand_context = self.context.expand_context(&text);
let tokens = vec![Token::Pipeline(pipeline).into_spanned(v.span())];
let mut tokens = TokensIterator::new(&tokens[..], expand_context, v.span());
let shapes = {
// We just constructed a token list that only contains a pipeline, so it can't fail
let result = tokens.expand_infallible(PipelineShape);
if let Some(failure) = result.failed {
errln!(
"BUG: PipelineShape didn't find a pipeline :: {:#?}",
failure
);
}
tokens.finish_tracer();
tokens.state().shapes()
};
if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
outln!("");
let _ =
ptree::print_tree(&tokens.expand_tracer().clone().print(Text::from(line)));
outln!("");
}
let mut painter = Painter::new();
let shapes = nu_parser::shapes(&classified.commands);
let mut painter = Painter::new(line);
for shape in shapes {
painter.paint_shape(&shape, line);
painter.paint_shape(&shape);
}
Cow::Owned(painter.into_string())
@ -133,73 +99,94 @@ fn vec_tag<T>(input: Vec<Tagged<T>>) -> Option<Tag> {
}
struct Painter {
current: Style,
buffer: String,
original: Vec<u8>,
styles: Vec<Style>,
}
impl Painter {
fn new() -> Painter {
fn new(original: &str) -> Painter {
let bytes: Vec<u8> = original.bytes().collect();
let bytes_count = bytes.len();
Painter {
current: Style::default(),
buffer: String::new(),
original: bytes,
styles: vec![Color::White.normal(); bytes_count],
}
}
fn paint_shape(&mut self, shape: &Spanned<FlatShape>) {
let style = match &shape.item {
FlatShape::OpenDelimiter(_) => Color::White.normal(),
FlatShape::CloseDelimiter(_) => Color::White.normal(),
FlatShape::ItVariable | FlatShape::Keyword => Color::Purple.bold(),
FlatShape::Variable | FlatShape::Identifier => Color::Purple.normal(),
FlatShape::Type => Color::Blue.bold(),
FlatShape::CompareOperator => Color::Yellow.normal(),
FlatShape::DotDot => Color::Yellow.bold(),
FlatShape::Dot => Style::new().fg(Color::White),
FlatShape::InternalCommand => Color::Cyan.bold(),
FlatShape::ExternalCommand => Color::Cyan.normal(),
FlatShape::ExternalWord => Color::Green.bold(),
FlatShape::BareMember => Color::Yellow.bold(),
FlatShape::StringMember => Color::Yellow.bold(),
FlatShape::String => Color::Green.normal(),
FlatShape::Path => Color::Cyan.normal(),
FlatShape::GlobPattern => Color::Cyan.bold(),
FlatShape::Word => Color::Green.normal(),
FlatShape::Pipe => Color::Purple.bold(),
FlatShape::Flag => Color::Blue.bold(),
FlatShape::ShorthandFlag => Color::Blue.bold(),
FlatShape::Int => Color::Purple.bold(),
FlatShape::Decimal => Color::Purple.bold(),
FlatShape::Whitespace | FlatShape::Separator => Color::White.normal(),
FlatShape::Comment => Color::Green.bold(),
FlatShape::Garbage => Style::new().fg(Color::White).on(Color::Red),
FlatShape::Size { number, unit } => {
self.paint(Color::Purple.bold(), number);
self.paint(Color::Cyan.bold(), unit);
return;
}
};
self.paint(style, &shape.span);
}
fn paint(&mut self, style: Style, span: &Span) {
for pos in span.start()..span.end() {
self.styles[pos] = style;
}
}
fn into_string(self) -> String {
self.buffer
}
let mut idx_start = 0;
let mut idx_end = 1;
fn paint_shape(&mut self, shape: &ShapeResult, line: &str) {
let style = match &shape {
ShapeResult::Success(shape) => match shape.item {
FlatShape::OpenDelimiter(_) => Color::White.normal(),
FlatShape::CloseDelimiter(_) => Color::White.normal(),
FlatShape::ItVariable | FlatShape::Keyword => Color::Purple.bold(),
FlatShape::Variable | FlatShape::Identifier => Color::Purple.normal(),
FlatShape::Type => Color::Blue.bold(),
FlatShape::CompareOperator => Color::Yellow.normal(),
FlatShape::DotDot => Color::Yellow.bold(),
FlatShape::Dot => Style::new().fg(Color::White),
FlatShape::InternalCommand => Color::Cyan.bold(),
FlatShape::ExternalCommand => Color::Cyan.normal(),
FlatShape::ExternalWord => Color::Green.bold(),
FlatShape::BareMember => Color::Yellow.bold(),
FlatShape::StringMember => Color::Yellow.bold(),
FlatShape::String => Color::Green.normal(),
FlatShape::Path => Color::Cyan.normal(),
FlatShape::GlobPattern => Color::Cyan.bold(),
FlatShape::Word => Color::Green.normal(),
FlatShape::Pipe => Color::Purple.bold(),
FlatShape::Flag => Color::Blue.bold(),
FlatShape::ShorthandFlag => Color::Blue.bold(),
FlatShape::Int => Color::Purple.bold(),
FlatShape::Decimal => Color::Purple.bold(),
FlatShape::Whitespace | FlatShape::Separator => Color::White.normal(),
FlatShape::Comment => Color::Green.bold(),
FlatShape::Garbage => Style::new().fg(Color::White).on(Color::Red),
FlatShape::Size { number, unit } => {
let number = number.slice(line);
let unit = unit.slice(line);
if self.original.is_empty() {
String::new()
} else {
let mut builder = String::new();
self.paint(Color::Purple.bold(), number);
self.paint(Color::Cyan.bold(), unit);
return;
let mut current_style = self.styles[0];
while idx_end < self.styles.len() {
if self.styles[idx_end] != current_style {
// Emit, as we changed styles
let intermediate = String::from_utf8_lossy(&self.original[idx_start..idx_end]);
builder.push_str(&format!("{}", current_style.paint(intermediate)));
current_style = self.styles[idx_end];
idx_start = idx_end;
idx_end += 1;
} else {
idx_end += 1;
}
},
ShapeResult::Fallback { shape, .. } => match shape.item {
FlatShape::Whitespace | FlatShape::Separator => Color::White.normal(),
_ => Style::new().fg(Color::White).on(Color::Red),
},
};
}
self.paint(style, shape.span().slice(line));
}
let intermediate = String::from_utf8_lossy(&self.original[idx_start..idx_end]);
builder.push_str(&format!("{}", current_style.paint(intermediate)));
fn paint(&mut self, style: Style, body: &str) {
let infix = self.current.infix(style);
self.current = style;
self.buffer
.push_str(&format!("{}{}", infix, style.paint(body)));
builder
}
}
}

9
crates/nu-cli/src/shell/shell.rs
View File

@ -7,7 +7,6 @@ use crate::commands::rm::RemoveArgs;
use crate::prelude::*;
use crate::stream::OutputStream;
use nu_errors::ShellError;
use nu_parser::ExpandContext;
use std::path::PathBuf;
pub trait Shell: std::fmt::Debug {
@ -35,11 +34,5 @@ pub trait Shell: std::fmt::Debug {
ctx: &rustyline::Context<'_>,
) -> Result<(usize, Vec<rustyline::completion::Pair>), rustyline::error::ReadlineError>;
fn hint(
&self,
_line: &str,
_pos: usize,
_ctx: &rustyline::Context<'_>,
_context: ExpandContext,
) -> Option<String>;
fn hint(&self, _line: &str, _pos: usize, _ctx: &rustyline::Context<'_>) -> Option<String>;
}

5
crates/nu-cli/src/shell/shell_manager.rs
View File

@ -9,7 +9,6 @@ use crate::shell::filesystem_shell::FilesystemShell;
use crate::shell::shell::Shell;
use crate::stream::OutputStream;
use nu_errors::ShellError;
use nu_parser::ExpandContext;
use parking_lot::Mutex;
use std::error::Error;
use std::path::PathBuf;
@ -95,9 +94,9 @@ impl ShellManager {
line: &str,
pos: usize,
ctx: &rustyline::Context<'_>,
context: ExpandContext,
//context: ExpandContext,
) -> Option<String> {
self.shells.lock()[self.current_shell()].hint(line, pos, ctx, context)
self.shells.lock()[self.current_shell()].hint(line, pos, ctx)
}
pub fn next(&mut self) {

9
crates/nu-cli/src/shell/value_shell.rs
View File

@ -8,7 +8,6 @@ use crate::prelude::*;
use crate::shell::shell::Shell;
use crate::utils::ValueStructure;
use nu_errors::ShellError;
use nu_parser::ExpandContext;
use nu_protocol::{ReturnSuccess, ShellTypeName, UntaggedValue, Value};
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
@ -283,13 +282,7 @@ impl Shell for ValueShell {
Ok((replace_pos, completions))
}
fn hint(
&self,
_line: &str,
_pos: usize,
_ctx: &rustyline::Context<'_>,
_context: ExpandContext,
) -> Option<String> {
fn hint(&self, _line: &str, _pos: usize, _ctx: &rustyline::Context<'_>) -> Option<String> {
None
}
}

2
crates/nu-cli/src/utils/data_processing.rs
View File

@ -2,7 +2,7 @@ use crate::data::value::compare_values;
use crate::data::TaggedListBuilder;
use chrono::{DateTime, NaiveDate, Utc};
use nu_errors::ShellError;
use nu_parser::CompareOperator;
use nu_parser::hir::CompareOperator;
use nu_protocol::{Primitive, TaggedDictBuilder, UntaggedValue, Value};
use nu_source::{SpannedItem, Tag, Tagged, TaggedItem};
use nu_value_ext::{get_data_by_key, ValueExt};

26
crates/nu-cli/tests/commands/cd.rs
View File

@ -338,32 +338,6 @@ fn valuesystem_change_current_path_to_parent_path() {
})
}
#[test]
fn valuesystem_change_to_home_directory() {
Playground::setup("cd_test_14", |dirs, sandbox| {
sandbox.with_files(vec![FileWithContent(
"sample.toml",
r#"
[paquete]
el = "pollo loco"
"#,
)]);
let actual = nu!(
cwd: dirs.test(),
r#"
enter sample.toml
cd paquete
cd ~
pwd | echo $it
exit
"#
);
assert_eq!(PathBuf::from(actual), PathBuf::from("/"));
})
}
#[test]
fn valuesystem_change_to_a_path_containing_spaces() {
Playground::setup("cd_test_15", |dirs, sandbox| {

2
crates/nu-cli/tests/commands/sort_by.rs
View File

@ -37,5 +37,5 @@ fn sort_primitive_values() {
"#
));
assert_eq!(actual, "authors = [\"Yehuda Katz <wycats@gmail.com>\"]");
assert_eq!(actual, "authors = [\"The Nu Project Contributors\"]");
}

16
crates/nu-cli/tests/commands/sum.rs
View File

@ -6,19 +6,23 @@ use nu_test_support::{nu, pipeline};
fn all() {
Playground::setup("sum_test_1", |dirs, sandbox| {
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"meals.csv",
"meals.json",
r#"
description,calories
"1 large egg",90
"1 cup white rice",250
"1 tablespoon fish oil",108
{
meals: [
{description: "1 large egg", calories: 90},
{description: "1 cup white rice", calories: 250},
{description: "1 tablespoon fish oil", calories: 108}
]
}
"#,
)]);
let actual = nu!(
cwd: dirs.test(), pipeline(
r#"
open meals.csv
open meals.json
| get meals
| get calories
| sum
| echo $it

2
crates/nu-cli/tests/format_conversions/csv.rs
View File

@ -78,7 +78,7 @@ fn infers_types() {
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_cuatro_mosqueteros.csv",
r#"
first_name,last_name,rusty_luck
first_name,last_name,rusty_luck,d
Andrés,Robalino,1,d
Jonathan,Turner,1,d
Yehuda,Katz,1,d