extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-08-10 00:48:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

LazyRecord (#7619)

Browse Source 
This is an attempt to implement a new `Value::LazyRecord` variant for
performance reasons.

`LazyRecord` is like a regular `Record`, but it's possible to access
individual columns without evaluating other columns. I've implemented
`LazyRecord` for the special `$nu` variable; accessing `$nu` is
relatively slow because of all the information in `scope`, and [`$nu`
accounts for about 2/3 of Nu's startup time on
Linux](https://github.com/nushell/nushell/issues/6677#issuecomment-1364618122).

### Benchmarks

I ran some benchmarks on my desktop (Linux, 12900K) and the results are
very pleasing.

Nu's time to start up and run a command (`cargo build --release;
hyperfine 'target/release/nu -c "echo \"Hello, world!\""' --shell=none
--warmup 10`) goes from **8.8ms to 3.2ms, about 2.8x faster**.

Tests are also much faster! Running `cargo nextest` (with our very slow
`proptest` tests disabled) goes from **7.2s to 4.4s (1.6x faster)**,
because most tests involve launching a new instance of Nu.

### Design (updated)

I've added a new `LazyRecord` trait and added a `Value` variant wrapping
those trait objects, much like `CustomValue`. `LazyRecord`
implementations must implement these 2 functions:

```rust
// All column names
fn column_names(&self) -> Vec<&'static str>;

// Get 1 specific column value
fn get_column_value(&self, column: &str) -> Result<Value, ShellError>;
 ```

### Serializability

`Value` variants must implement `Serializable` and `Deserializable`, which poses some problems because I want to use unserializable things like `EngineState` in `LazyRecord`s. To work around this, I basically lie to the type system:

1. Add `#[typetag::serde(tag = "type")]` to `LazyRecord` to make it serializable
2. Any unserializable fields in `LazyRecord` implementations get marked with `#[serde(skip)]`
3. At the point where a `LazyRecord` normally would get serialized and sent to a plugin, I instead collect it into a regular `Value::Record` (which can be serialized)
This commit is contained in:
Reilly Wood
2023-01-18 19:27:26 -08:00
committed by GitHub
parent be32aeee70
commit 3b5172a8fa
19 changed files with 443 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/nu-engine/Cargo.toml
View File

@ -14,6 +14,7 @@ nu-glob = { path = "../nu-glob", version = "0.74.1" }
nu-utils = { path = "../nu-utils", version = "0.74.1" }
chrono = { version="0.4.23", features = ["std"], default-features = false }
serde = {version = "1.0.143", default-features = false }
sysinfo ="0.26.2"
[features]

154
crates/nu-engine/src/eval.rs
View File

@ -1,4 +1,4 @@
use crate::{current_dir_str, get_full_help, scope::create_scope};
use crate::{current_dir_str, get_full_help, nu_variable::NuVariable};
use nu_path::expand_path_with;
use nu_protocol::{
ast::{
@ -6,12 +6,11 @@ use nu_protocol::{
Operator, PathMember, PipelineElement, Redirection,
},
engine::{EngineState, Stack},
Config, HistoryFileFormat, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData,
Range, ShellError, Span, Spanned, Unit, Value, VarId, ENV_VARIABLE_ID,
Config, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData, Range, ShellError, Span,
Spanned, Unit, Value, VarId, ENV_VARIABLE_ID,
};
use nu_utils::stdout_write_all_and_flush;
use std::collections::HashMap;
use sysinfo::SystemExt;
pub fn eval_operator(op: &Expression) -> Result<Operator, ShellError> {
match op {
@ -1120,146 +1119,15 @@ pub fn eval_variable(
span: Span,
) -> Result<Value, ShellError> {
match var_id {
nu_protocol::NU_VARIABLE_ID => {
// $nu
let mut output_cols = vec![];
let mut output_vals = vec![];
if let Some(path) = engine_state.get_config_path("config-path") {
output_cols.push("config-path".into());
output_vals.push(Value::String {
val: path.to_string_lossy().to_string(),
span,
});
}
if let Some(path) = engine_state.get_config_path("env-path") {
output_cols.push("env-path".into());
output_vals.push(Value::String {
val: path.to_string_lossy().to_string(),
span,
});
}
if let Some(mut config_path) = nu_path::config_dir() {
config_path.push("nushell");
let mut env_config_path = config_path.clone();
let mut loginshell_path = config_path.clone();
let mut history_path = config_path.clone();
match engine_state.config.history_file_format {
HistoryFileFormat::Sqlite => {
history_path.push("history.sqlite3");
}
HistoryFileFormat::PlainText => {
history_path.push("history.txt");
}
}
// let mut history_path = config_files::get_history_path(); // todo: this should use the get_history_path method but idk where to put that function
output_cols.push("history-path".into());
output_vals.push(Value::String {
val: history_path.to_string_lossy().to_string(),
span,
});
if engine_state.get_config_path("config-path").is_none() {
config_path.push("config.nu");
output_cols.push("config-path".into());
output_vals.push(Value::String {
val: config_path.to_string_lossy().to_string(),
span,
});
}
if engine_state.get_config_path("env-path").is_none() {
env_config_path.push("env.nu");
output_cols.push("env-path".into());
output_vals.push(Value::String {
val: env_config_path.to_string_lossy().to_string(),
span,
});
}
loginshell_path.push("login.nu");
output_cols.push("loginshell-path".into());
output_vals.push(Value::String {
val: loginshell_path.to_string_lossy().to_string(),
span,
});
}
#[cfg(feature = "plugin")]
if let Some(path) = &engine_state.plugin_signatures {
if let Some(path_str) = path.to_str() {
output_cols.push("plugin-path".into());
output_vals.push(Value::String {
val: path_str.into(),
span,
});
}
}
output_cols.push("scope".into());
output_vals.push(create_scope(engine_state, stack, span)?);
if let Some(home_path) = nu_path::home_dir() {
if let Some(home_path_str) = home_path.to_str() {
output_cols.push("home-path".into());
output_vals.push(Value::String {
val: home_path_str.into(),
span,
})
}
}
let temp = std::env::temp_dir();
if let Some(temp_path) = temp.to_str() {
output_cols.push("temp-path".into());
output_vals.push(Value::String {
val: temp_path.into(),
span,
})
}
let pid = std::process::id();
output_cols.push("pid".into());
output_vals.push(Value::int(pid as i64, span));
let sys = sysinfo::System::new();
let ver = match sys.kernel_version() {
Some(v) => v,
None => "unknown".into(),
};
let os_record = Value::Record {
cols: vec![
"name".into(),
"arch".into(),
"family".into(),
"kernel_version".into(),
],
vals: vec![
Value::string(std::env::consts::OS, span),
Value::string(std::env::consts::ARCH, span),
Value::string(std::env::consts::FAMILY, span),
Value::string(ver, span),
],
// $nu
nu_protocol::NU_VARIABLE_ID => Ok(Value::LazyRecord {
val: Box::new(NuVariable {
engine_state: engine_state.clone(),
stack: stack.clone(),
span,
};
output_cols.push("os-info".into());
output_vals.push(os_record);
Ok(Value::Record {
cols: output_cols,
vals: output_vals,
span,
})
}
}),
span,
}),
ENV_VARIABLE_ID => {
let env_vars = stack.get_env_vars(engine_state);
let env_columns = env_vars.keys();

1
crates/nu-engine/src/lib.rs
View File

@ -4,6 +4,7 @@ pub mod documentation;
pub mod env;
mod eval;
mod glob_from;
mod nu_variable;
pub mod scope;
pub use call_ext::CallExt;

200
crates/nu-engine/src/nu_variable.rs Normal file
View File

@ -0,0 +1,200 @@
use crate::scope::create_scope;
use core::fmt;
use nu_protocol::{
engine::{EngineState, Stack},
LazyRecord, ShellError, Span, Value,
};
use serde::{Deserialize, Serialize};
use sysinfo::SystemExt;
// NuVariable: a LazyRecord for the special $nu variable
// $nu used to be a plain old Record, but LazyRecord lets us load different fields/columns lazily. This is important for performance;
// collecting all the information in $nu is expensive and unnecessary if you just want a subset of the data
// Note: NuVariable is not meaningfully serializable, this #[derive] is a lie to satisfy the type checker.
// Make sure to collect() the record before serializing it
#[derive(Serialize, Deserialize)]
pub struct NuVariable {
#[serde(skip)]
pub engine_state: EngineState,
#[serde(skip)]
pub stack: Stack,
pub span: Span,
}
impl LazyRecord for NuVariable {
fn column_names(&self) -> Vec<&'static str> {
let mut cols = vec!["config-path", "env-path", "history-path", "loginshell-path"];
#[cfg(feature = "plugin")]
if self.engine_state.plugin_signatures.is_some() {
cols.push("plugin-path");
}
cols.push("scope");
cols.push("home-path");
cols.push("temp-path");
cols.push("pid");
cols.push("os-info");
cols
}
fn get_column_value(&self, column: &str) -> Result<Value, ShellError> {
let err = |message: &str| -> Result<Value, ShellError> {
Err(ShellError::LazyRecordAccessFailed {
message: message.into(),
column_name: column.to_string(),
span: self.span,
})
};
match column {
"config-path" => {
if let Some(path) = self.engine_state.get_config_path("config-path") {
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else if let Some(mut path) = nu_path::config_dir() {
path.push("nushell/config.nu");
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else {
err("Could not get config directory")
}
}
"env-path" => {
if let Some(path) = self.engine_state.get_config_path("env-path") {
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else if let Some(mut path) = nu_path::config_dir() {
path.push("nushell/env.nu");
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else {
err("Could not get config directory")
}
}
"history-path" => {
if let Some(mut path) = nu_path::config_dir() {
path.push("nushell");
match self.engine_state.config.history_file_format {
nu_protocol::HistoryFileFormat::Sqlite => {
path.push("history.sqlite3");
}
nu_protocol::HistoryFileFormat::PlainText => {
path.push("history.txt");
}
}
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else {
err("Could not get config directory")
}
}
"loginshell-path" => {
if let Some(mut path) = nu_path::config_dir() {
path.push("nushell/login.nu");
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else {
err("Could not get config directory")
}
}
"plugin-path" => {
#[cfg(feature = "plugin")]
{
if let Some(path) = &self.engine_state.plugin_signatures {
Ok(Value::String {
val: path.to_string_lossy().to_string(),
span: self.span,
})
} else {
err("Could not get plugin signature location")
}
}
#[cfg(not(feature = "plugin"))]
{
err("Plugin feature not enabled")
}
}
"scope" => Ok(create_scope(&self.engine_state, &self.stack, self.span())?),
"home-path" => {
if let Some(home_path) = nu_path::home_dir() {
Ok(Value::String {
val: home_path.to_string_lossy().into(),
span: self.span(),
})
} else {
err("Could not get home path")
}
}
"temp-path" => {
let temp_path = std::env::temp_dir();
Ok(Value::String {
val: temp_path.to_string_lossy().into(),
span: self.span(),
})
}
"pid" => Ok(Value::int(std::process::id().into(), self.span())),
"os-info" => {
let sys = sysinfo::System::new();
let ver = match sys.kernel_version() {
Some(v) => v,
None => "unknown".into(),
};
let os_record = Value::Record {
cols: vec![
"name".into(),
"arch".into(),
"family".into(),
"kernel_version".into(),
],
vals: vec![
Value::string(std::env::consts::OS, self.span()),
Value::string(std::env::consts::ARCH, self.span()),
Value::string(std::env::consts::FAMILY, self.span()),
Value::string(ver, self.span()),
],
span: self.span(),
};
Ok(os_record)
}
_ => err(&format!("Could not find column '{column}'")),
}
}
fn span(&self) -> Span {
self.span
}
fn typetag_name(&self) -> &'static str {
"nu_variable"
}
fn typetag_deserialize(&self) {
unimplemented!("typetag_deserialize")
}
}
// manually implemented so we can skip engine_state which doesn't implement Debug
// FIXME: find a better way
impl fmt::Debug for NuVariable {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("NuVariable").finish()
}
}