Break up interdependencies of command crates (#9429)

# Description
Make sure that our different crates that contain commands can be
compiled in parallel.
This can under certain circumstances accelerate the compilation with
sufficient multithreading available.

## Details
- Move `help` commands from `nu-cmd-lang` back to `nu-command`
- This also makes sense as the commands are implemented in an
ANSI-terminal specific way
- Make `nu-cmd-lang` only a dev dependency for `nu-command`
- Change context creation helpers for `nu-cmd-extra` and
`nu-cmd-dataframe` to have a consistent api used in
`src/main.rs`:`get_engine_state()`
- `nu-command` now indepedent from `nu-cmd-extra` and `nu-cmd-dataframe`
that are now dependencies of `nu` directly. (change to internal
features)
- Fix tests that previously used `nu-command::create_default_context()`
with replacement functions

## From scratch compilation times:

just debug (dev) build and default features
```
cargo clean --profile dev && cargo build --timings
```

### before

![grafik](https://github.com/nushell/nushell/assets/15833959/e49f1f42-2e53-4a6c-bc23-625b686af1bc)

### after

![grafik](https://github.com/nushell/nushell/assets/15833959/8dec4723-e625-4a86-b91e-e6e808f64726)

# User-Facing Changes
None direct, only change to compilation on multithreaded jobs expected.

# Tests + Formatting
Tests that previously chose to use `nu-command` for their scope will
still use `nu-cmd-lang` + `nu-command` (command list in the granularity
at the time)
This commit is contained in:
Stefan Holderbach 2023-06-14 23:12:55 +02:00 committed by GitHub
parent b14bdd865f
commit 46eebc644c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 132 additions and 85 deletions

4
Cargo.lock generated
View File

@ -2652,6 +2652,7 @@ dependencies = [
"nu-ansi-term",
"nu-cli",
"nu-cmd-dataframe",
"nu-cmd-extra",
"nu-cmd-lang",
"nu-color-config",
"nu-command",
@ -2705,6 +2706,7 @@ dependencies = [
"log",
"miette",
"nu-ansi-term",
"nu-cmd-lang",
"nu-color-config",
"nu-command",
"nu-engine",
@ -2825,8 +2827,6 @@ dependencies = [
"native-tls",
"notify",
"nu-ansi-term",
"nu-cmd-dataframe",
"nu-cmd-extra",
"nu-cmd-lang",
"nu-color-config",
"nu-engine",

View File

@ -49,6 +49,7 @@ nu-cli = { path = "./crates/nu-cli", version = "0.81.1" }
nu-color-config = { path = "./crates/nu-color-config", version = "0.81.1" }
nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.81.1" }
nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.81.1", optional = true }
nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.81.1", optional = true }
nu-command = { path = "./crates/nu-command", version = "0.81.1" }
nu-engine = { path = "./crates/nu-engine", version = "0.81.1" }
nu-explore = { path = "./crates/nu-explore", version = "0.81.1" }
@ -112,6 +113,7 @@ plugin = [
default = ["plugin", "which-support", "trash-support", "sqlite"]
stable = ["default"]
wasi = ["nu-cmd-lang/wasi"]
# NOTE: individual features are also passed to `nu-cmd-lang` that uses them to generate the feature matrix in the `version` command
# Enable to statically link OpenSSL; otherwise the system version will be used. Not enabled by default because it takes a while to build
static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
@ -121,10 +123,10 @@ which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
# Extra feature for nushell
extra = ["nu-command/extra", "nu-cmd-lang/extra"]
extra = ["dep:nu-cmd-extra", "nu-cmd-lang/extra"]
# Dataframe feature for nushell
dataframe = ["nu-command/dataframe", "nu-cmd-lang/dataframe"]
dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
# SQLite commands for nushell
sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]

View File

@ -11,6 +11,7 @@ version = "0.81.1"
bench = false
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-test-support = { path = "../nu-test-support", version = "0.81.1" }
rstest = { version = "0.17.0", default-features = false }

View File

@ -26,7 +26,7 @@ pub fn add_cli_context(mut engine_state: EngineState) -> EngineState {
};
if let Err(err) = engine_state.merge_delta(delta) {
eprintln!("Error creating default context: {err:?}");
eprintln!("Error creating CLI command context: {err:?}");
}
engine_state

View File

@ -501,7 +501,8 @@ mod completer_tests {
#[test]
fn test_completion_helper() {
let mut engine_state = nu_command::create_default_context();
let mut engine_state =
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
// Custom additions
let delta = {

View File

@ -1,6 +1,5 @@
use std::path::PathBuf;
use nu_command::create_default_context;
use nu_engine::eval_block;
use nu_parser::parse;
use nu_protocol::{
@ -11,6 +10,10 @@ use nu_test_support::fs;
use reedline::Suggestion;
const SEP: char = std::path::MAIN_SEPARATOR;
fn create_default_context() -> EngineState {
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
}
// creates a new engine with the current path into the completions fixtures folder
pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
// Target folder inside assets

View File

@ -13,7 +13,6 @@ version = "0.81.1"
bench = false
[dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-engine = { path = "../nu-engine", version = "0.81.1" }
nu-parser = { path = "../nu-parser", version = "0.81.1" }
nu-protocol = { path = "../nu-protocol", version = "0.81.1" }
@ -66,4 +65,5 @@ dataframe = ["default"]
default = ["num", "polars", "sqlparser"]
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-test-support = { path = "../nu-test-support", version = "0.81.1" }

View File

@ -10,13 +10,24 @@ pub use expressions::add_expressions;
pub use lazy::add_lazy_decls;
pub use series::add_series_decls;
use nu_protocol::engine::StateWorkingSet;
use nu_protocol::engine::{EngineState, StateWorkingSet};
pub fn add_dataframe_decls(working_set: &mut StateWorkingSet) {
add_series_decls(working_set);
add_eager_decls(working_set);
add_expressions(working_set);
add_lazy_decls(working_set);
pub fn add_dataframe_context(mut engine_state: EngineState) -> EngineState {
let delta = {
let mut working_set = StateWorkingSet::new(&engine_state);
add_series_decls(&mut working_set);
add_eager_decls(&mut working_set);
add_expressions(&mut working_set);
add_lazy_decls(&mut working_set);
working_set.render()
};
if let Err(err) = engine_state.merge_delta(delta) {
eprintln!("Error creating dataframe command context: {err:?}");
}
engine_state
}
#[cfg(test)]

View File

@ -13,7 +13,6 @@ version = "0.81.1"
bench = false
[dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-engine = { path = "../nu-engine", version = "0.81.1" }
nu-parser = { path = "../nu-parser", version = "0.81.1" }
nu-protocol = { path = "../nu-protocol", version = "0.81.1" }
@ -27,4 +26,5 @@ extra = ["default"]
default = []
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-test-support = { path = "../nu-test-support", version = "0.81.1" }

View File

@ -1,9 +1,12 @@
mod bits;
use nu_protocol::engine::StateWorkingSet;
use nu_protocol::engine::{EngineState, StateWorkingSet};
pub fn add_extra_decls(working_set: &mut StateWorkingSet) {
macro_rules! bind_command {
pub fn add_extra_command_context(mut engine_state: EngineState) -> EngineState {
let delta = {
let mut working_set = StateWorkingSet::new(&engine_state);
macro_rules! bind_command {
( $command:expr ) => {
working_set.add_decl(Box::new($command));
};
@ -12,15 +15,23 @@ pub fn add_extra_decls(working_set: &mut StateWorkingSet) {
};
}
bind_command!(
bits::bits_::Bits,
bits::and::BitsAnd,
bits::not::BitsNot,
bits::or::BitsOr,
bits::xor::BitsXor,
bits::rotate_left::BitsRol,
bits::rotate_right::BitsRor,
bits::shift_left::BitsShl,
bits::shift_right::BitsShr
);
bind_command!(
bits::bits_::Bits,
bits::and::BitsAnd,
bits::not::BitsNot,
bits::or::BitsOr,
bits::xor::BitsXor,
bits::rotate_left::BitsRol,
bits::rotate_right::BitsRor,
bits::shift_left::BitsShl,
bits::shift_right::BitsShr
);
working_set.render()
};
if let Err(err) = engine_state.merge_delta(delta) {
eprintln!("Error creating extra command context: {err:?}");
}
engine_state
}

View File

@ -18,12 +18,6 @@ mod export_module;
mod export_use;
mod extern_;
mod for_;
pub mod help;
pub mod help_aliases;
pub mod help_commands;
pub mod help_externs;
pub mod help_modules;
mod help_operators;
mod hide;
mod hide_env;
mod if_;
@ -61,12 +55,6 @@ pub use export_module::ExportModule;
pub use export_use::ExportUse;
pub use extern_::Extern;
pub use for_::For;
pub use help::Help;
pub use help_aliases::HelpAliases;
pub use help_commands::HelpCommands;
pub use help_externs::HelpExterns;
pub use help_modules::HelpModules;
pub use help_operators::HelpOperators;
pub use hide::Hide;
pub use hide_env::HideEnv;
pub use if_::If;

View File

@ -36,12 +36,6 @@ pub fn create_default_context() -> EngineState {
ExportModule,
Extern,
For,
Help,
HelpAliases,
HelpCommands,
HelpModules,
HelpExterns,
HelpOperators,
Hide,
HideEnv,
If,

View File

@ -13,9 +13,6 @@ version = "0.81.1"
bench = false
[dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-cmd-dataframe = { path = "../nu-cmd-dataframe", version = "0.81.1", optional = true }
nu-cmd-extra = { path = "../nu-cmd-extra", version = "0.81.1", optional = true }
nu-color-config = { path = "../nu-color-config", version = "0.81.1" }
nu-engine = { path = "../nu-engine", version = "0.81.1" }
nu-glob = { path = "../nu-glob", version = "0.81.1" }
@ -117,17 +114,15 @@ features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_System
version = "0.48"
[features]
dataframe = ["dep:nu-cmd-dataframe"]
extra = ["dep:nu-cmd-extra"]
plugin = ["nu-parser/plugin"]
sqlite = [
"rusqlite",
] # TODO: given that rusqlite is included in reedline, should we just always include it?
sqlite = ["rusqlite"]
trash-support = ["trash"]
which-support = ["which"]
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
nu-test-support = { path = "../nu-test-support", version = "0.81.1" }
dirs-next = "2.0"
mockito = "1.0"
quickcheck = "1.0"

View File

@ -1,15 +1,10 @@
use nu_protocol::engine::{EngineState, StateWorkingSet};
use crate::*;
#[cfg(feature = "dataframe")]
use nu_cmd_dataframe::*;
#[cfg(feature = "extra")]
use nu_cmd_extra::*;
pub fn create_default_context() -> EngineState {
let mut engine_state = nu_cmd_lang::create_default_context();
use crate::{
help::{HelpAliases, HelpCommands, HelpExterns, HelpModules, HelpOperators},
*,
};
pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
let delta = {
let mut working_set = StateWorkingSet::new(&engine_state);
@ -24,12 +19,6 @@ pub fn create_default_context() -> EngineState {
// them only accessible if the correct input value category is used with the
// declaration
#[cfg(feature = "extra")]
add_extra_decls(&mut working_set);
#[cfg(feature = "dataframe")]
add_dataframe_decls(&mut working_set);
// Database-related
// Adds all related commands to query databases
#[cfg(feature = "sqlite")]
@ -138,6 +127,16 @@ pub fn create_default_context() -> EngineState {
Sys,
};
// Help
bind_command! {
Help,
HelpAliases,
HelpExterns,
HelpCommands,
HelpModules,
HelpOperators,
};
// Debug
bind_command! {
Ast,

View File

@ -1,5 +1,5 @@
use crate::help::highlight_search_string;
use itertools::Itertools;
use nu_cmd_lang::help::highlight_search_string;
use fancy_regex::Regex;
use nu_ansi_term::Style;

View File

@ -1,6 +1,6 @@
use crate::help_aliases::help_aliases;
use crate::help_commands::help_commands;
use crate::help_modules::help_modules;
use crate::help::help_aliases;
use crate::help::help_commands;
use crate::help::help_modules;
use fancy_regex::Regex;
use nu_ansi_term::Style;
use nu_engine::CallExt;

View File

@ -0,0 +1,18 @@
mod help_;
mod help_aliases;
mod help_commands;
mod help_externs;
mod help_modules;
mod help_operators;
pub use help_::Help;
pub use help_aliases::HelpAliases;
pub use help_commands::HelpCommands;
pub use help_externs::HelpExterns;
pub use help_modules::HelpModules;
pub use help_operators::HelpOperators;
pub(crate) use help_::{highlight_search_in_table, highlight_search_string};
pub(crate) use help_aliases::help_aliases;
pub(crate) use help_commands::help_commands;
pub(crate) use help_modules::help_modules;

View File

@ -13,6 +13,7 @@ mod filters;
mod formats;
mod generators;
mod hash;
mod help;
pub mod hook;
mod input_handler;
mod math;
@ -45,6 +46,7 @@ pub use filters::*;
pub use formats::*;
pub use generators::*;
pub use hash::*;
pub use help::*;
pub use hook::*;
pub use math::*;
pub use misc::*;

View File

@ -1,10 +1,16 @@
use nu_command::create_default_context;
use nu_protocol::{engine::StateWorkingSet, Category, Span};
use nu_protocol::{
engine::{EngineState, StateWorkingSet},
Category, Span,
};
use quickcheck_macros::quickcheck;
mod commands;
mod format_conversions;
fn create_default_context() -> EngineState {
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
}
#[quickcheck]
fn quickcheck_parse(data: String) -> bool {
let (tokens, err) = nu_parser::lex(data.as_bytes(), 0, b"", b"", true);
@ -24,7 +30,7 @@ fn quickcheck_parse(data: String) -> bool {
#[test]
fn signature_name_matches_command_name() {
let ctx = crate::create_default_context();
let ctx = create_default_context();
let decls = ctx.get_decls_sorted(true);
let mut failures = Vec::new();
@ -50,7 +56,7 @@ fn signature_name_matches_command_name() {
#[test]
fn commands_declare_input_output_types() {
let ctx = crate::create_default_context();
let ctx = create_default_context();
let decls = ctx.get_decls_sorted(true);
let mut failures = Vec::new();

View File

@ -19,8 +19,8 @@ use command::gather_commandline_args;
use log::Level;
use miette::Result;
use nu_cli::gather_parent_env_vars;
use nu_command::{create_default_context, get_init_cwd};
use nu_protocol::{report_error_new, Value};
use nu_command::get_init_cwd;
use nu_protocol::{engine::EngineState, report_error_new, Value};
use nu_protocol::{util::BufferedReader, PipelineData, RawStream};
use nu_std::load_standard_library;
use nu_utils::utils::perf;
@ -32,6 +32,17 @@ use std::{
sync::{atomic::AtomicBool, Arc},
};
fn get_engine_state() -> EngineState {
let engine_state = nu_cmd_lang::create_default_context();
let engine_state = nu_command::add_shell_command_context(engine_state);
#[cfg(feature = "extra")]
let engine_state = nu_cmd_extra::add_extra_command_context(engine_state);
#[cfg(feature = "dataframe")]
let engine_state = nu_cmd_dataframe::add_dataframe_context(engine_state);
let engine_state = nu_cli::add_cli_context(engine_state);
nu_explore::add_explore_context(engine_state)
}
fn main() -> Result<()> {
let entire_start_time = std::time::Instant::now();
let mut start_time = std::time::Instant::now();
@ -43,8 +54,7 @@ fn main() -> Result<()> {
// Get initial current working directory.
let init_cwd = get_init_cwd();
let mut engine_state =
nu_explore::add_explore_context(nu_cli::add_cli_context(create_default_context()));
let mut engine_state = get_engine_state();
// Custom additions
let delta = {

View File

@ -1,6 +1,5 @@
use std::io::{self, BufRead, Read, Write};
use nu_command::create_default_context;
use nu_command::hook::{eval_env_change_hook, eval_hook};
use nu_engine::eval_block;
use nu_parser::parse;
@ -168,12 +167,19 @@ fn outcome_ok(msg: String) -> ! {
std::process::exit(0);
}
/// Generate a minimal engine state with just `nu-cmd-lang`, `nu-command`, and `nu-cli` commands.
fn get_engine_state() -> EngineState {
let engine_state = nu_cmd_lang::create_default_context();
let engine_state = nu_command::add_shell_command_context(engine_state);
nu_cli::add_cli_context(engine_state)
}
pub fn nu_repl() {
//cwd: &str, source_lines: &[&str]) {
let cwd = std::env::current_dir().expect("Could not get current working directory.");
let source_lines = args();
let mut engine_state = nu_cli::add_cli_context(create_default_context());
let mut engine_state = get_engine_state();
let mut stack = Stack::new();
engine_state.add_env_var("PWD".into(), Value::test_string(cwd.to_string_lossy()));