extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-01-23 14:50:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

more closure serialization (#14698)

Browse Source 
# Description

This PR introduces a switch `--serialize` that allows serializing of
types that cannot be deserialized. Right now it only serializes closures
as strings in `to toml`, `to json`, `to nuon`, `to text`, some indirect
`to html` and `to yaml`.

A lot of the changes are just weaving the engine_state through calling
functions and the rest is just repetitive way of getting the closure
block span and grabbing the span's text.

In places where it has to report `<Closure 123>` I changed it to
`closure_123`. It always seemed like the `<>` were not very nushell-y.
This is still a breaking change.

I think this could also help with systematic translation of old config
to new config file.


# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the
tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->
This commit is contained in:
Darren Schroeder 2025-01-07 11:51:22 -06:00 committed by GitHub
parent 1f477c8eb1
commit dad956b2ee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 509 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
crates/nu-command/src/debug/explain.rs
View File

@ -154,7 +154,8 @@ fn get_arguments(
eval_expression_fn,
);
let arg_type = "expr";
let arg_value_name = debug_string_without_formatting(&evaluated_expression);
let arg_value_name =
debug_string_without_formatting(engine_state, &evaluated_expression);
let arg_value_type = &evaluated_expression.get_type().to_string();
let evaled_span = evaluated_expression.span();
let arg_value_name_span_start = evaled_span.start as i64;
@ -174,7 +175,8 @@ fn get_arguments(
let arg_type = "positional";
let evaluated_expression =
get_expression_as_value(engine_state, stack, inner_expr, eval_expression_fn);
let arg_value_name = debug_string_without_formatting(&evaluated_expression);
let arg_value_name =
debug_string_without_formatting(engine_state, &evaluated_expression);
let arg_value_type = &evaluated_expression.get_type().to_string();
let evaled_span = evaluated_expression.span();
let arg_value_name_span_start = evaled_span.start as i64;
@ -193,7 +195,8 @@ fn get_arguments(
let arg_type = "unknown";
let evaluated_expression =
get_expression_as_value(engine_state, stack, inner_expr, eval_expression_fn);
let arg_value_name = debug_string_without_formatting(&evaluated_expression);
let arg_value_name =
debug_string_without_formatting(engine_state, &evaluated_expression);
let arg_value_type = &evaluated_expression.get_type().to_string();
let evaled_span = evaluated_expression.span();
let arg_value_name_span_start = evaled_span.start as i64;
@ -212,7 +215,8 @@ fn get_arguments(
let arg_type = "spread";
let evaluated_expression =
get_expression_as_value(engine_state, stack, inner_expr, eval_expression_fn);
let arg_value_name = debug_string_without_formatting(&evaluated_expression);
let arg_value_name =
debug_string_without_formatting(engine_state, &evaluated_expression);
let arg_value_type = &evaluated_expression.get_type().to_string();
let evaled_span = evaluated_expression.span();
let arg_value_name_span_start = evaled_span.start as i64;
@ -245,7 +249,7 @@ fn get_expression_as_value(
}
}
pub fn debug_string_without_formatting(value: &Value) -> String {
pub fn debug_string_without_formatting(engine_state: &EngineState, value: &Value) -> String {
match value {
Value::Bool { val, .. } => val.to_string(),
Value::Int { val, .. } => val.to_string(),
@ -259,19 +263,31 @@ pub fn debug_string_without_formatting(value: &Value) -> String {
Value::List { vals: val, .. } => format!(
"[{}]",
val.iter()
.map(debug_string_without_formatting)
.map(|v| debug_string_without_formatting(engine_state, v))
.collect::<Vec<_>>()
.join(" ")
),
Value::Record { val, .. } => format!(
"{{{}}}",
val.iter()
.map(|(x, y)| format!("{}: {}", x, debug_string_without_formatting(y)))
.map(|(x, y)| format!(
"{}: {}",
x,
debug_string_without_formatting(engine_state, y)
))
.collect::<Vec<_>>()
.join(" ")
),
//TODO: It would be good to drill deeper into closures.
Value::Closure { val, .. } => format!("<Closure {}>", val.block_id.get()),
Value::Closure { val, .. } => {
let block = engine_state.get_block(val.block_id);
if let Some(span) = block.span {
let contents_bytes = engine_state.get_span_contents(span);
let contents_string = String::from_utf8_lossy(contents_bytes);
contents_string.to_string()
} else {
String::new()
}
}
Value::Nothing { .. } => String::new(),
Value::Error { error, .. } => format!("{error:?}"),
Value::Binary { val, .. } => format!("{val:?}"),
@ -280,7 +296,7 @@ pub fn debug_string_without_formatting(value: &Value) -> String {
// that critical here
Value::Custom { val, .. } => val
.to_base_value(value.span())
.map(|val| debug_string_without_formatting(&val))
.map(|val| debug_string_without_formatting(engine_state, &val))
.unwrap_or_else(|_| format!("<{}>", val.type_name())),
}
}

4
crates/nu-command/src/debug/inspect.rs
View File

@ -23,7 +23,7 @@ impl Command for Inspect {
fn run(
&self,
_engine_state: &EngineState,
engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
@ -40,7 +40,7 @@ impl Command for Inspect {
let (cols, _rows) = terminal_size().unwrap_or((0, 0));
let table = inspect_table::build_table(input_val, description, cols as usize);
let table = inspect_table::build_table(engine_state, input_val, description, cols as usize);
// Note that this is printed to stderr. The reason for this is so it doesn't disrupt the regular nushell
// tabular output. If we printed to stdout, nushell would get confused with two outputs.

56
crates/nu-command/src/debug/inspect_table.rs
View File

@ -1,19 +1,23 @@
// note: Seems like could be simplified
// IMHO: it shall not take 300+ lines :)
use self::{global_horizontal_char::SetHorizontalChar, set_widths::SetWidths};
use nu_protocol::engine::EngineState;
use nu_protocol::Value;
use nu_table::{string_width, string_wrap};
use tabled::{
grid::config::ColoredConfig,
settings::{peaker::Priority, width::Wrap, Settings, Style},
Table,
};
use self::{global_horizontal_char::SetHorizontalChar, set_widths::SetWidths};
pub fn build_table(value: Value, description: String, termsize: usize) -> String {
let (head, mut data) = util::collect_input(value);
pub fn build_table(
engine_state: &EngineState,
value: Value,
description: String,
termsize: usize,
) -> String {
let (head, mut data) = util::collect_input(engine_state, value);
let count_columns = head.len();
data.insert(0, head);
@ -195,10 +199,14 @@ fn push_empty_column(data: &mut Vec<Vec<String>>) {
mod util {
use crate::debug::explain::debug_string_without_formatting;
use nu_engine::get_columns;
use nu_protocol::engine::EngineState;
use nu_protocol::Value;
/// Try to build column names and a table grid.
pub fn collect_input(value: Value) -> (Vec<String>, Vec<Vec<String>>) {
pub fn collect_input(
engine_state: &EngineState,
value: Value,
) -> (Vec<String>, Vec<Vec<String>>) {
let span = value.span();
match value {
Value::Record { val: record, .. } => {
@ -210,7 +218,7 @@ mod util {
},
match vals
.into_iter()
.map(|s| debug_string_without_formatting(&s))
.map(|s| debug_string_without_formatting(engine_state, &s))
.collect::<Vec<String>>()
{
vals if vals.is_empty() => vec![],
@ -220,7 +228,7 @@ mod util {
}
Value::List { vals, .. } => {
let mut columns = get_columns(&vals);
let data = convert_records_to_dataset(&columns, vals);
let data = convert_records_to_dataset(engine_state, &columns, vals);
if columns.is_empty() {
columns = vec![String::from("")];
@ -232,7 +240,7 @@ mod util {
let lines = val
.lines()
.map(|line| Value::string(line.to_string(), span))
.map(|val| vec![debug_string_without_formatting(&val)])
.map(|val| vec![debug_string_without_formatting(engine_state, &val)])
.collect();
(vec![String::from("")], lines)
@ -240,47 +248,59 @@ mod util {
Value::Nothing { .. } => (vec![], vec![]),
value => (
vec![String::from("")],
vec![vec![debug_string_without_formatting(&value)]],
vec![vec![debug_string_without_formatting(engine_state, &value)]],
),
}
}
fn convert_records_to_dataset(cols: &[String], records: Vec<Value>) -> Vec<Vec<String>> {
fn convert_records_to_dataset(
engine_state: &EngineState,
cols: &[String],
records: Vec<Value>,
) -> Vec<Vec<String>> {
if !cols.is_empty() {
create_table_for_record(cols, &records)
create_table_for_record(engine_state, cols, &records)
} else if cols.is_empty() && records.is_empty() {
vec![]
} else if cols.len() == records.len() {
vec![records
.into_iter()
.map(|s| debug_string_without_formatting(&s))
.map(|s| debug_string_without_formatting(engine_state, &s))
.collect()]
} else {
records
.into_iter()
.map(|record| vec![debug_string_without_formatting(&record)])
.map(|record| vec![debug_string_without_formatting(engine_state, &record)])
.collect()
}
}
fn create_table_for_record(headers: &[String], items: &[Value]) -> Vec<Vec<String>> {
fn create_table_for_record(
engine_state: &EngineState,
headers: &[String],
items: &[Value],
) -> Vec<Vec<String>> {
let mut data = vec![Vec::new(); items.len()];
for (i, item) in items.iter().enumerate() {
let row = record_create_row(headers, item);
let row = record_create_row(engine_state, headers, item);
data[i] = row;
}
data
}
fn record_create_row(headers: &[String], item: &Value) -> Vec<String> {
fn record_create_row(
engine_state: &EngineState,
headers: &[String],
item: &Value,
) -> Vec<String> {
if let Value::Record { val, .. } = item {
headers
.iter()
.map(|col| {
val.get(col)
.map(debug_string_without_formatting)
.map(|v| debug_string_without_formatting(engine_state, v))
.unwrap_or_else(String::new)
})
.collect()

12
crates/nu-command/src/filters/uniq.rs
View File

@ -213,9 +213,15 @@ fn sort_attributes(val: Value) -> Value {
}
}
fn generate_key(item: &ValueCounter) -> Result<String, ShellError> {
fn generate_key(engine_state: &EngineState, item: &ValueCounter) -> Result<String, ShellError> {
let value = sort_attributes(item.val_to_compare.clone()); //otherwise, keys could be different for Records
nuon::to_nuon(&value, nuon::ToStyle::Raw, Some(Span::unknown()))
nuon::to_nuon(
engine_state,
&value,
nuon::ToStyle::Raw,
Some(Span::unknown()),
false,
)
}
fn generate_results_with_count(head: Span, uniq_values: Vec<ValueCounter>) -> Vec<Value> {
@ -264,7 +270,7 @@ pub fn uniq(
.try_fold(
HashMap::<String, ValueCounter>::new(),
|mut counter, item| {
let key = generate_key(&item);
let key = generate_key(engine_state, &item);
match key {
Ok(key) => {

52
crates/nu-command/src/formats/to/json.rs
View File

@ -25,6 +25,11 @@ impl Command for ToJson {
"specify indentation tab quantity",
Some('t'),
)
.switch(
"serialize",
"serialize nushell types that cannot be deserialized",
Some('s'),
)
.category(Category::Formats)
}
@ -42,12 +47,13 @@ impl Command for ToJson {
let raw = call.has_flag(engine_state, stack, "raw")?;
let use_tabs = call.get_flag(engine_state, stack, "tabs")?;
let indent = call.get_flag(engine_state, stack, "indent")?;
let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
let span = call.head;
// allow ranges to expand and turn into array
let input = input.try_expand_range()?;
let value = input.into_value(span)?;
let json_value = value_to_json_value(&value)?;
let json_value = value_to_json_value(engine_state, &value, serialize_types)?;
let json_result = if raw {
nu_json::to_string_raw(&json_value)
@ -105,7 +111,11 @@ impl Command for ToJson {
}
}
pub fn value_to_json_value(v: &Value) -> Result<nu_json::Value, ShellError> {
pub fn value_to_json_value(
engine_state: &EngineState,
v: &Value,
serialize_types: bool,
) -> Result<nu_json::Value, ShellError> {
let span = v.span();
Ok(match v {
Value::Bool { val, .. } => nu_json::Value::Bool(*val),
@ -127,31 +137,57 @@ pub fn value_to_json_value(v: &Value) -> Result<nu_json::Value, ShellError> {
.collect::<Result<Vec<nu_json::Value>, ShellError>>()?,
),
Value::List { vals, .. } => nu_json::Value::Array(json_list(vals)?),
Value::List { vals, .. } => {
nu_json::Value::Array(json_list(engine_state, vals, serialize_types)?)
}
Value::Error { error, .. } => return Err(*error.clone()),
Value::Closure { .. } | Value::Range { .. } => nu_json::Value::Null,
Value::Closure { val, .. } => {
if serialize_types {
let block = engine_state.get_block(val.block_id);
if let Some(span) = block.span {
let contents_bytes = engine_state.get_span_contents(span);
let contents_string = String::from_utf8_lossy(contents_bytes);
nu_json::Value::String(contents_string.to_string())
} else {
nu_json::Value::String(format!(
"unable to retrieve block contents for json block_id {}",
val.block_id.get()
))
}
} else {
nu_json::Value::Null
}
}
Value::Range { .. } => nu_json::Value::Null,
Value::Binary { val, .. } => {
nu_json::Value::Array(val.iter().map(|x| nu_json::Value::U64(*x as u64)).collect())
}
Value::Record { val, .. } => {
let mut m = nu_json::Map::new();
for (k, v) in &**val {
m.insert(k.clone(), value_to_json_value(v)?);
m.insert(
k.clone(),
value_to_json_value(engine_state, v, serialize_types)?,
);
}
nu_json::Value::Object(m)
}
Value::Custom { val, .. } => {
let collected = val.to_base_value(span)?;
value_to_json_value(&collected)?
value_to_json_value(engine_state, &collected, serialize_types)?
}
})
}
fn json_list(input: &[Value]) -> Result<Vec<nu_json::Value>, ShellError> {
fn json_list(
engine_state: &EngineState,
input: &[Value],
serialize_types: bool,
) -> Result<Vec<nu_json::Value>, ShellError> {
let mut out = vec![];
for value in input {
out.push(value_to_json_value(value)?);
out.push(value_to_json_value(engine_state, value, serialize_types)?);
}
Ok(out)

8
crates/nu-command/src/formats/to/nuon.rs
View File

@ -28,6 +28,11 @@ impl Command for ToNuon {
"specify indentation tab quantity",
Some('t'),
)
.switch(
"serialize",
"serialize nushell types that cannot be deserialized",
Some('s'),
)
.category(Category::Formats)
}
@ -47,6 +52,7 @@ impl Command for ToNuon {
.unwrap_or_default()
.with_content_type(Some("application/x-nuon".into()));
let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
let style = if call.has_flag(engine_state, stack, "raw")? {
nuon::ToStyle::Raw
} else if let Some(t) = call.get_flag(engine_state, stack, "tabs")? {
@ -60,7 +66,7 @@ impl Command for ToNuon {
let span = call.head;
let value = input.into_value(span)?;
match nuon::to_nuon(&value, style, Some(span)) {
match nuon::to_nuon(engine_state, &value, style, Some(span), serialize_types) {
Ok(serde_nuon_string) => Ok(Value::string(serde_nuon_string, span)
.into_pipeline_data_with_metadata(Some(metadata))),
_ => Ok(Value::error(

65
crates/nu-command/src/formats/to/text.rs
View File

@ -27,6 +27,11 @@ impl Command for ToText {
"Do not append a newline to the end of the text",
Some('n'),
)
.switch(
"serialize",
"serialize nushell types that cannot be deserialized",
Some('s'),
)
.category(Category::Formats)
}
@ -43,6 +48,7 @@ impl Command for ToText {
) -> Result<PipelineData, ShellError> {
let head = call.head;
let no_newline = call.has_flag(engine_state, stack, "no-newline")?;
let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
let input = input.try_expand_range()?;
let config = stack.get_config(engine_state);
@ -56,7 +62,8 @@ impl Command for ToText {
Value::Record { val, .. } => !val.is_empty(),
_ => false,
};
let mut str = local_into_string(value, LINE_ENDING, &config);
let mut str =
local_into_string(engine_state, value, LINE_ENDING, &config, serialize_types);
if add_trailing {
str.push_str(LINE_ENDING);
}
@ -70,6 +77,7 @@ impl Command for ToText {
let stream = if no_newline {
let mut first = true;
let mut iter = stream.into_inner();
let engine_state_clone = engine_state.clone();
ByteStream::from_fn(
span,
engine_state.signals().clone(),
@ -85,15 +93,28 @@ impl Command for ToText {
}
// TODO: write directly into `buf` instead of creating an intermediate
// string.
let str = local_into_string(val, LINE_ENDING, &config);
let str = local_into_string(
&engine_state_clone,
val,
LINE_ENDING,
&config,
serialize_types,
);
write!(buf, "{str}").err_span(head)?;
Ok(true)
},
)
} else {
let engine_state_clone = engine_state.clone();
ByteStream::from_iter(
stream.into_inner().map(move |val| {
let mut str = local_into_string(val, LINE_ENDING, &config);
let mut str = local_into_string(
&engine_state_clone,
val,
LINE_ENDING,
&config,
serialize_types,
);
str.push_str(LINE_ENDING);
str
}),
@ -137,7 +158,13 @@ impl Command for ToText {
}
}
fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
fn local_into_string(
engine_state: &EngineState,
value: Value,
separator: &str,
config: &Config,
serialize_types: bool,
) -> String {
let span = value.span();
match value {
Value::Bool { val, .. } => val.to_string(),
@ -153,16 +180,38 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
Value::Glob { val, .. } => val,
Value::List { vals: val, .. } => val
.into_iter()
.map(|x| local_into_string(x, ", ", config))
.map(|x| local_into_string(engine_state, x, ", ", config, serialize_types))
.collect::<Vec<_>>()
.join(separator),
Value::Record { val, .. } => val
.into_owned()
.into_iter()
.map(|(x, y)| format!("{}: {}", x, local_into_string(y, ", ", config)))
.map(|(x, y)| {
format!(
"{}: {}",
x,
local_into_string(engine_state, y, ", ", config, serialize_types)
)
})
.collect::<Vec<_>>()
.join(separator),
Value::Closure { val, .. } => format!("<Closure {}>", val.block_id.get()),
Value::Closure { val, .. } => {
if serialize_types {
let block = engine_state.get_block(val.block_id);
if let Some(span) = block.span {
let contents_bytes = engine_state.get_span_contents(span);
let contents_string = String::from_utf8_lossy(contents_bytes);
contents_string.to_string()
} else {
format!(
"unable to retrieve block contents for text block_id {}",
val.block_id.get()
)
}
} else {
format!("closure_{}", val.block_id.get())
}
}
Value::Nothing { .. } => String::new(),
Value::Error { error, .. } => format!("{error:?}"),
Value::Binary { val, .. } => format!("{val:?}"),
@ -171,7 +220,7 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
// that critical here
Value::Custom { val, .. } => val
.to_base_value(span)
.map(|val| local_into_string(val, separator, config))
.map(|val| local_into_string(engine_state, val, separator, config, serialize_types))
.unwrap_or_else(|_| format!("<{}>", val.type_name())),
}
}

65
crates/nu-command/src/formats/to/toml.rs
View File

@ -13,6 +13,11 @@ impl Command for ToToml {
fn signature(&self) -> Signature {
Signature::build("to toml")
.input_output_types(vec![(Type::record(), Type::String)])
.switch(
"serialize",
"serialize nushell types that cannot be deserialized",
Some('s'),
)
.category(Category::Formats)
}
@ -31,19 +36,24 @@ impl Command for ToToml {
fn run(
&self,
engine_state: &EngineState,
_stack: &mut Stack,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let head = call.head;
to_toml(engine_state, input, head)
let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
to_toml(engine_state, input, head, serialize_types)
}
}
// Helper method to recursively convert nu_protocol::Value -> toml::Value
// This shouldn't be called at the top-level
fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellError> {
let span = v.span();
fn helper(
engine_state: &EngineState,
v: &Value,
serialize_types: bool,
) -> Result<toml::Value, ShellError> {
Ok(match &v {
Value::Bool { val, .. } => toml::Value::Boolean(*val),
Value::Int { val, .. } => toml::Value::Integer(*val),
@ -56,15 +66,29 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellErr
Value::Record { val, .. } => {
let mut m = toml::map::Map::new();
for (k, v) in &**val {
m.insert(k.clone(), helper(engine_state, v)?);
m.insert(k.clone(), helper(engine_state, v, serialize_types)?);
}
toml::Value::Table(m)
}
Value::List { vals, .. } => toml::Value::Array(toml_list(engine_state, vals)?),
Value::Closure { .. } => {
let code = engine_state.get_span_contents(span);
let code = String::from_utf8_lossy(code).to_string();
toml::Value::String(code)
Value::List { vals, .. } => {
toml::Value::Array(toml_list(engine_state, vals, serialize_types)?)
}
Value::Closure { val, .. } => {
if serialize_types {
let block = engine_state.get_block(val.block_id);
if let Some(span) = block.span {
let contents_bytes = engine_state.get_span_contents(span);
let contents_string = String::from_utf8_lossy(contents_bytes);
toml::Value::String(contents_string.to_string())
} else {
toml::Value::String(format!(
"unable to retrieve block contents for toml block_id {}",
val.block_id.get()
))
}
} else {
toml::Value::String(format!("closure_{}", val.block_id.get()))
}
}
Value::Nothing { .. } => toml::Value::String("<Nothing>".to_string()),
Value::Error { error, .. } => return Err(*error.clone()),
@ -86,11 +110,15 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellErr
})
}
fn toml_list(engine_state: &EngineState, input: &[Value]) -> Result<Vec<toml::Value>, ShellError> {
fn toml_list(
engine_state: &EngineState,
input: &[Value],
serialize_types: bool,
) -> Result<Vec<toml::Value>, ShellError> {
let mut out = vec![];
for value in input {
out.push(helper(engine_state, value)?);
out.push(helper(engine_state, value, serialize_types)?);
}
Ok(out)
@ -129,9 +157,10 @@ fn value_to_toml_value(
engine_state: &EngineState,
v: &Value,
head: Span,
serialize_types: bool,
) -> Result<toml::Value, ShellError> {
match v {
Value::Record { .. } => helper(engine_state, v),
Value::Record { .. } | Value::Closure { .. } => helper(engine_state, v, serialize_types),
// Propagate existing errors
Value::Error { error, .. } => Err(*error.clone()),
_ => Err(ShellError::UnsupportedInput {
@ -147,11 +176,12 @@ fn to_toml(
engine_state: &EngineState,
input: PipelineData,
span: Span,
serialize_types: bool,
) -> Result<PipelineData, ShellError> {
let metadata = input.metadata();
let value = input.into_value(span)?;
let toml_value = value_to_toml_value(engine_state, &value, span)?;
let toml_value = value_to_toml_value(engine_state, &value, span, serialize_types)?;
match toml_value {
toml::Value::Array(ref vec) => match vec[..] {
[toml::Value::Table(_)] => toml_into_pipeline_data(
@ -218,6 +248,7 @@ mod tests {
#[test]
fn to_toml_creates_correct_date() {
let engine_state = EngineState::new();
let serialize_types = false;
let test_date = Value::date(
chrono::FixedOffset::east_opt(60 * 120)
@ -242,7 +273,7 @@ mod tests {
offset: Some(toml::value::Offset::Custom { minutes: 120 }),
});
let result = helper(&engine_state, &test_date);
let result = helper(&engine_state, &test_date, serialize_types);
assert!(result.is_ok_and(|res| res == reference_date));
}
@ -254,6 +285,7 @@ mod tests {
//
let engine_state = EngineState::new();
let serialize_types = false;
let mut m = indexmap::IndexMap::new();
m.insert("rust".to_owned(), Value::test_string("editor"));
@ -269,6 +301,7 @@ mod tests {
&engine_state,
&Value::record(m.into_iter().collect(), Span::test_data()),
Span::test_data(),
serialize_types,
)
.expect("Expected Ok from valid TOML dictionary");
assert_eq!(
@ -285,12 +318,14 @@ mod tests {
&engine_state,
&Value::test_string("not_valid"),
Span::test_data(),
serialize_types,
)
.expect_err("Expected non-valid toml (String) to cause error!");
value_to_toml_value(
&engine_state,
&Value::list(vec![Value::test_string("1")], Span::test_data()),
Span::test_data(),
serialize_types,
)
.expect_err("Expected non-valid toml (Table) to cause error!");
}

59
crates/nu-command/src/formats/to/yaml.rs
View File

@ -12,6 +12,11 @@ impl Command for ToYaml {
fn signature(&self) -> Signature {
Signature::build("to yaml")
.input_output_types(vec![(Type::Any, Type::String)])
.switch(
"serialize",
"serialize nushell types that cannot be deserialized",
Some('s'),
)
.category(Category::Formats)
}
@ -29,18 +34,24 @@ impl Command for ToYaml {
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let head = call.head;
let serialize_types = call.has_flag(engine_state, stack, "serialize")?;
let input = input.try_expand_range()?;
to_yaml(input, head)
to_yaml(engine_state, input, head, serialize_types)
}
}
pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
pub fn value_to_yaml_value(
engine_state: &EngineState,
v: &Value,
serialize_types: bool,
) -> Result<serde_yml::Value, ShellError> {
Ok(match &v {
Value::Bool { val, .. } => serde_yml::Value::Bool(*val),
Value::Int { val, .. } => serde_yml::Value::Number(serde_yml::Number::from(*val)),
@ -55,7 +66,10 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
Value::Record { val, .. } => {
let mut m = serde_yml::Mapping::new();
for (k, v) in &**val {
m.insert(serde_yml::Value::String(k.clone()), value_to_yaml_value(v)?);
m.insert(
serde_yml::Value::String(k.clone()),
value_to_yaml_value(engine_state, v, serialize_types)?,
);
}
serde_yml::Value::Mapping(m)
}
@ -63,12 +77,28 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
let mut out = vec![];
for value in vals {
out.push(value_to_yaml_value(value)?);
out.push(value_to_yaml_value(engine_state, value, serialize_types)?);
}
serde_yml::Value::Sequence(out)
}
Value::Closure { .. } => serde_yml::Value::Null,
Value::Closure { val, .. } => {
if serialize_types {
let block = engine_state.get_block(val.block_id);
if let Some(span) = block.span {
let contents_bytes = engine_state.get_span_contents(span);
let contents_string = String::from_utf8_lossy(contents_bytes);
serde_yml::Value::String(contents_string.to_string())
} else {
serde_yml::Value::String(format!(
"unable to retrieve block contents for yaml block_id {}",
val.block_id.get()
))
}
} else {
serde_yml::Value::Null
}
}
Value::Nothing { .. } => serde_yml::Value::Null,
Value::Error { error, .. } => return Err(*error.clone()),
Value::Binary { val, .. } => serde_yml::Value::Sequence(
@ -91,7 +121,12 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yml::Value, ShellError> {
})
}
fn to_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
fn to_yaml(
engine_state: &EngineState,
input: PipelineData,
head: Span,
serialize_types: bool,
) -> Result<PipelineData, ShellError> {
let metadata = input
.metadata()
.unwrap_or_default()
@ -99,7 +134,7 @@ fn to_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError>
.with_content_type(Some("application/yaml".into()));
let value = input.into_value(head)?;
let yaml_value = value_to_yaml_value(&value)?;
let yaml_value = value_to_yaml_value(engine_state, &value, serialize_types)?;
match serde_yml::to_string(&yaml_value) {
Ok(serde_yml_string) => {
Ok(Value::string(serde_yml_string, head)
@ -120,11 +155,9 @@ fn to_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError>
#[cfg(test)]
mod test {
use nu_cmd_lang::eval_pipeline_without_terminal_expression;
use crate::{Get, Metadata};
use super::*;
use crate::{Get, Metadata};
use nu_cmd_lang::eval_pipeline_without_terminal_expression;
#[test]
fn test_examples() {

18
crates/nu-command/src/network/http/client.rs
View File

@ -205,6 +205,7 @@ pub enum HttpBody {
// remove once all commands have been migrated
pub fn send_request(
engine_state: &EngineState,
request: Request,
http_body: HttpBody,
content_type: Option<String>,
@ -212,6 +213,9 @@ pub fn send_request(
signals: &Signals,
) -> Result<Response, ShellErrorOrRequestError> {
let request_url = request.url().to_string();
// hard code serialze_types to false because closures probably shouldn't be
// deserialized for send_request but it's required by send_json_request
let serialze_types = false;
match http_body {
HttpBody::None => {
@ -238,7 +242,15 @@ pub fn send_request(
};
match body_type {
BodyType::Json => send_json_request(&request_url, body, req, span, signals),
BodyType::Json => send_json_request(
engine_state,
&request_url,
body,
req,
span,
signals,
serialze_types,
),
BodyType::Form => send_form_request(&request_url, body, req, span, signals),
BodyType::Multipart => {
send_multipart_request(&request_url, body, req, span, signals)
@ -252,15 +264,17 @@ pub fn send_request(
}
fn send_json_request(
engine_state: &EngineState,
request_url: &str,
body: Value,
req: Request,
span: Span,
signals: &Signals,
serialize_types: bool,
) -> Result<Response, ShellErrorOrRequestError> {
match body {
Value::Int { .. } | Value::Float { .. } | Value::List { .. } | Value::Record { .. } => {
let data = value_to_json_value(&body)?;
let data = value_to_json_value(engine_state, &body, serialize_types)?;
send_cancellable_request(request_url, Box::new(|| req.send_json(data)), span, signals)
}
// If the body type is string, assume it is string json content.

1
crates/nu-command/src/network/http/delete.rs
View File

@ -214,6 +214,7 @@ fn helper(
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
args.data,
args.content_type,

1
crates/nu-command/src/network/http/get.rs
View File

@ -182,6 +182,7 @@ fn helper(
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
HttpBody::None,
None,

9
crates/nu-command/src/network/http/head.rs
View File

@ -155,7 +155,14 @@ fn helper(
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(request, HttpBody::None, None, call.head, signals);
let response = send_request(
engine_state,
request,
HttpBody::None,
None,
call.head,
signals,
);
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response_headers(span, response)
}

1
crates/nu-command/src/network/http/options.rs
View File

@ -161,6 +161,7 @@ fn helper(
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
HttpBody::None,
None,

1
crates/nu-command/src/network/http/patch.rs
View File

@ -216,6 +216,7 @@ fn helper(
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
args.data,
args.content_type,

1
crates/nu-command/src/network/http/post.rs
View File

@ -224,6 +224,7 @@ fn helper(
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
args.data,
args.content_type,

1
crates/nu-command/src/network/http/put.rs
View File

@ -215,6 +215,7 @@ fn helper(
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
args.data,
args.content_type,

14
crates/nu-command/tests/commands/database/into_sqlite.rs
View File

@ -1,6 +1,6 @@
use chrono::{DateTime, FixedOffset};
use nu_path::AbsolutePathBuf;
use nu_protocol::{ast::PathMember, record, Span, Value};
use nu_protocol::{ast::PathMember, engine::EngineState, record, Span, Value};
use nu_test_support::{
fs::{line_ending, Stub},
nu, pipeline,
@ -298,6 +298,9 @@ fn into_sqlite_existing_db_append() {
/// streaming pipeline instead of a simple value
#[test]
fn into_sqlite_big_insert() {
let engine_state = EngineState::new();
// don't serialize closures
let serialize_types = false;
Playground::setup("big_insert", |dirs, playground| {
const NUM_ROWS: usize = 10_000;
const NUON_FILE_NAME: &str = "data.nuon";
@ -330,7 +333,14 @@ fn into_sqlite_big_insert() {
)
.unwrap();
let nuon = nuon::to_nuon(&value, nuon::ToStyle::Raw, Some(Span::unknown())).unwrap()
let nuon = nuon::to_nuon(
&engine_state,
&value,
nuon::ToStyle::Raw,
Some(Span::unknown()),
serialize_types,
)
.unwrap()
+ &line_ending();
nuon_file.write_all(nuon.as_bytes()).unwrap();

1
crates/nu-command/tests/format_conversions/nuon.rs
View File

@ -278,6 +278,7 @@ fn from_nuon_datetime() {
}
#[test]
#[ignore]
fn to_nuon_errs_on_closure() {
let actual = nu!(pipeline(
r#"

2
crates/nu-protocol/src/value/mod.rs
View File

@ -908,7 +908,7 @@ impl Value {
.collect::<Vec<_>>()
.join(separator)
),
Value::Closure { val, .. } => format!("<Closure {}>", val.block_id.get()),
Value::Closure { val, .. } => format!("closure_{}", val.block_id.get()),
Value::Nothing { .. } => String::new(),
Value::Error { error, .. } => format!("{error:?}"),
Value::Binary { val, .. } => format!("{val:?}"),

94
crates/nuon/src/lib.rs
View File

@ -11,7 +11,7 @@ mod tests {
use chrono::DateTime;
use nu_protocol::{
ast::{CellPath, PathMember, RangeInclusion},
engine::Closure,
engine::{Closure, EngineState},
record, BlockId, IntRange, Range, Span, Value,
};
@ -25,11 +25,15 @@ mod tests {
/// an optional "middle" value can be given to test what the value is between `from nuon` and
/// `to nuon`.
fn nuon_end_to_end(input: &str, middle: Option<Value>) {
let engine_state = EngineState::new();
let val = from_nuon(input, None).unwrap();
if let Some(m) = middle {
assert_eq!(val, m);
}
assert_eq!(to_nuon(&val, ToStyle::Raw, None).unwrap(), input);
assert_eq!(
to_nuon(&engine_state, &val, ToStyle::Raw, None, false).unwrap(),
input
);
}
#[test]
@ -172,14 +176,19 @@ mod tests {
}
#[test]
#[ignore]
fn to_nuon_errs_on_closure() {
let engine_state = EngineState::new();
assert!(to_nuon(
&engine_state,
&Value::test_closure(Closure {
block_id: BlockId::new(0),
captures: vec![]
}),
ToStyle::Raw,
None,
false,
)
.unwrap_err()
.to_string()
@ -196,8 +205,17 @@ mod tests {
#[test]
fn binary_roundtrip() {
let engine_state = EngineState::new();
assert_eq!(
to_nuon(&from_nuon("0x[1f ff]", None).unwrap(), ToStyle::Raw, None).unwrap(),
to_nuon(
&engine_state,
&from_nuon("0x[1f ff]", None).unwrap(),
ToStyle::Raw,
None,
false,
)
.unwrap(),
"0x[1FFF]"
);
}
@ -237,40 +255,79 @@ mod tests {
#[test]
fn float_doesnt_become_int() {
let engine_state = EngineState::new();
assert_eq!(
to_nuon(&Value::test_float(1.0), ToStyle::Raw, None).unwrap(),
to_nuon(
&engine_state,
&Value::test_float(1.0),
ToStyle::Raw,
None,
false
)
.unwrap(),
"1.0"
);
}
#[test]
fn float_inf_parsed_properly() {
let engine_state = EngineState::new();
assert_eq!(
to_nuon(&Value::test_float(f64::INFINITY), ToStyle::Raw, None).unwrap(),
to_nuon(
&engine_state,
&Value::test_float(f64::INFINITY),
ToStyle::Raw,
None,
false,
)
.unwrap(),
"inf"
);
}
#[test]
fn float_neg_inf_parsed_properly() {
let engine_state = EngineState::new();
assert_eq!(
to_nuon(&Value::test_float(f64::NEG_INFINITY), ToStyle::Raw, None).unwrap(),
to_nuon(
&engine_state,
&Value::test_float(f64::NEG_INFINITY),
ToStyle::Raw,
None,
false,
)
.unwrap(),
"-inf"
);
}
#[test]
fn float_nan_parsed_properly() {
let engine_state = EngineState::new();
assert_eq!(
to_nuon(&Value::test_float(-f64::NAN), ToStyle::Raw, None).unwrap(),
to_nuon(
&engine_state,
&Value::test_float(-f64::NAN),
ToStyle::Raw,
None,
false,
)
.unwrap(),
"NaN"
);
}
#[test]
fn to_nuon_converts_columns_with_spaces() {
let engine_state = EngineState::new();
assert!(from_nuon(
&to_nuon(
&engine_state,
&Value::test_list(vec![
Value::test_record(record!(
"a" => Value::test_int(1),
@ -284,7 +341,8 @@ mod tests {
))
]),
ToStyle::Raw,
None
None,
false,
)
.unwrap(),
None,
@ -294,7 +352,15 @@ mod tests {
#[test]
fn to_nuon_quotes_empty_string() {
let res = to_nuon(&Value::test_string(""), ToStyle::Raw, None);
let engine_state = EngineState::new();
let res = to_nuon(
&engine_state,
&Value::test_string(""),
ToStyle::Raw,
None,
false,
);
assert!(res.is_ok());
assert_eq!(res.unwrap(), r#""""#);
}
@ -340,8 +406,11 @@ mod tests {
#[test]
fn does_not_quote_strings_unnecessarily() {
let engine_state = EngineState::new();
assert_eq!(
to_nuon(
&engine_state,
&Value::test_list(vec![
Value::test_record(record!(
"a" => Value::test_int(1),
@ -355,7 +424,8 @@ mod tests {
))
]),
ToStyle::Raw,
None
None,
false,
)
.unwrap(),
"[[a, b, \"c d\"]; [1, 2, 3], [4, 5, 6]]"
@ -363,12 +433,14 @@ mod tests {
assert_eq!(
to_nuon(
&engine_state,
&Value::test_record(record!(
"ro name" => Value::test_string("sam"),
"rank" => Value::test_int(10)
)),
ToStyle::Raw,
None
None,
false,
)
.unwrap(),
"{\"ro name\": sam, rank: 10}"

99
crates/nuon/src/to.rs
View File

@ -1,7 +1,6 @@
use core::fmt::Write;
use nu_engine::get_columns;
use nu_protocol::{Range, ShellError, Span, Value};
use nu_protocol::{engine::EngineState, Range, ShellError, Span, Value};
use nu_utils::{escape_quote_string, needs_quoting};
use std::ops::Bound;
@ -44,7 +43,13 @@ pub enum ToStyle {
/// > using this function in a command implementation such as [`to nuon`](https://www.nushell.sh/commands/docs/to_nuon.html).
///
/// also see [`super::from_nuon`] for the inverse operation
pub fn to_nuon(input: &Value, style: ToStyle, span: Option<Span>) -> Result<String, ShellError> {
pub fn to_nuon(
engine_state: &EngineState,
input: &Value,
style: ToStyle,
span: Option<Span>,
serialize_types: bool,
) -> Result<String, ShellError> {
let span = span.unwrap_or(Span::unknown());
let indentation = match style {
@ -53,16 +58,25 @@ pub fn to_nuon(input: &Value, style: ToStyle, span: Option<Span>) -> Result<Stri
ToStyle::Spaces(s) => Some(" ".repeat(s)),
};
let res = value_to_string(input, span, 0, indentation.as_deref())?;
let res = value_to_string(
engine_state,
input,
span,
0,
indentation.as_deref(),
serialize_types,
)?;
Ok(res)
}
fn value_to_string(
engine_state: &EngineState,
v: &Value,
span: Span,
depth: usize,
indent: Option<&str>,
serialize_types: bool,
) -> Result<String, ShellError> {
let (nl, sep) = get_true_separators(indent);
let idt = get_true_indentation(depth, indent);
@ -84,12 +98,25 @@ fn value_to_string(
}
Ok(format!("0x[{s}]"))
}
Value::Closure { .. } => Err(ShellError::UnsupportedInput {
msg: "closures are currently not nuon-compatible".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
}),
Value::Closure { val, .. } => {
if serialize_types {
let block = engine_state.get_block(val.block_id);
if let Some(span) = block.span {
let contents_bytes = engine_state.get_span_contents(span);
let contents_string = String::from_utf8_lossy(contents_bytes);
Ok(contents_string.to_string())
} else {
Ok(String::new())
}
} else {
Err(ShellError::UnsupportedInput {
msg: "closures are currently not nuon-compatible".into(),
input: "value originates from here".into(),
msg_span: span,
input_span: v.span(),
})
}
}
Value::Bool { val, .. } => {
if *val {
Ok("true".to_string())
@ -144,10 +171,12 @@ fn value_to_string(
if let Value::Record { val, .. } = val {
for val in val.values() {
row.push(value_to_string_without_quotes(
engine_state,
val,
span,
depth + 2,
indent,
serialize_types,
)?);
}
}
@ -165,7 +194,14 @@ fn value_to_string(
for val in vals {
collection.push(format!(
"{idt_po}{}",
value_to_string_without_quotes(val, span, depth + 1, indent,)?
value_to_string_without_quotes(
engine_state,
val,
span,
depth + 1,
indent,
serialize_types
)?
));
}
Ok(format!(
@ -178,18 +214,38 @@ fn value_to_string(
Value::Range { val, .. } => match **val {
Range::IntRange(range) => Ok(range.to_string()),
Range::FloatRange(range) => {
let start =
value_to_string(&Value::float(range.start(), span), span, depth + 1, indent)?;
let start = value_to_string(
engine_state,
&Value::float(range.start(), span),
span,
depth + 1,
indent,
serialize_types,
)?;
match range.end() {
Bound::Included(end) => Ok(format!(
"{}..{}",
start,
value_to_string(&Value::float(end, span), span, depth + 1, indent)?
value_to_string(
engine_state,
&Value::float(end, span),
span,
depth + 1,
indent,
serialize_types,
)?
)),
Bound::Excluded(end) => Ok(format!(
"{}..<{}",
start,
value_to_string(&Value::float(end, span), span, depth + 1, indent)?
value_to_string(
engine_state,
&Value::float(end, span),
span,
depth + 1,
indent,
serialize_types,
)?
)),
Bound::Unbounded => Ok(format!("{start}..",)),
}
@ -205,7 +261,14 @@ fn value_to_string(
};
collection.push(format!(
"{idt_po}{col}: {}",
value_to_string_without_quotes(val, span, depth + 1, indent)?
value_to_string_without_quotes(
engine_state,
val,
span,
depth + 1,
indent,
serialize_types
)?
));
}
Ok(format!(
@ -235,10 +298,12 @@ fn get_true_separators(indent: Option<&str>) -> (String, String) {
}
fn value_to_string_without_quotes(
engine_state: &EngineState,
v: &Value,
span: Span,
depth: usize,
indent: Option<&str>,
serialize_types: bool,
) -> Result<String, ShellError> {
match v {
Value::String { val, .. } => Ok({
@ -248,6 +313,6 @@ fn value_to_string_without_quotes(
val.clone()
}
}),
_ => value_to_string(v, span, depth, indent),
_ => value_to_string(engine_state, v, span, depth, indent, serialize_types),
}
}

22
tests/eval/mod.rs
View File

@ -103,7 +103,27 @@ fn literal_binary() {
#[test]
fn literal_closure() {
test_eval("{||}", Matches("<Closure"))
test_eval("{||}", Matches("closure_"))
}
#[test]
fn literal_closure_to_nuon() {
test_eval("{||} | to nuon --serialize", Eq("{||}"))
}
#[test]
fn literal_closure_to_json() {
test_eval("{||} | to json --serialize", Eq("\"{||}\""))
}
#[test]
fn literal_closure_to_toml() {
test_eval("{a: {||}} | to toml --serialize", Eq("a = \"{||}\""))
}
#[test]
fn literal_closure_to_yaml() {
test_eval("{||} | to yaml --serialize", Eq("'{||}'"))
}
#[test]