mirror of
https://github.com/nushell/nushell.git
synced 2025-07-19 07:33:17 +02:00
28e33587d9
# Description Increase default compression level for brotli on msgpackz to 3. This has the best compression time generally. Level 0 and 1 give weird results and sometimes cause extremely inflated outputs rather than being compressed. So far this hasn't really been a problem for the plugin registry file, but has been for other data. The `$example` is the web-app example from https://json.org/example.html Benchmarked with: ```nushell seq 0 11 | each { |level| let compressed = ($example | to msgpackz --quality $level) let time = (timeit { $example | to msgpackz --quality $level }) { level: $level time: $time length: ($compressed | bytes length) ratio: (($uncompressed_length | into float) / ($compressed | bytes length)) } } ``` ``` ╭────┬───────┬─────────────────┬────────┬───────╮ │ # │ level │ time │ length │ ratio │ ├────┼───────┼─────────────────┼────────┼───────┤ │ 0 │ 0 │ 4ms 611µs 875ns │ 3333 │ 0.72 │ │ 1 │ 1 │ 1ms 334µs 500ns │ 3333 │ 0.72 │ │ 2 │ 2 │ 190µs 333ns │ 1185 │ 2.02 │ │ 3 │ 3 │ 184µs 42ns │ 1128 │ 2.12 │ │ 4 │ 4 │ 245µs 83ns │ 1098 │ 2.18 │ │ 5 │ 5 │ 265µs 584ns │ 1040 │ 2.30 │ │ 6 │ 6 │ 270µs 792ns │ 1040 │ 2.30 │ │ 7 │ 7 │ 444µs 708ns │ 1040 │ 2.30 │ │ 8 │ 8 │ 1ms 801µs │ 1040 │ 2.30 │ │ 9 │ 9 │ 843µs 875ns │ 1037 │ 2.31 │ │ 10 │ 10 │ 4ms 128µs 375ns │ 984 │ 2.43 │ │ 11 │ 11 │ 6ms 352µs 834ns │ 986 │ 2.43 │ ╰────┴───────┴─────────────────┴────────┴───────╯ ``` cc @maxim-uvarov
89 lines
2.5 KiB
Rust
89 lines
2.5 KiB
Rust
use std::io::Write;
|
|
|
|
use nu_engine::command_prelude::*;
|
|
|
|
use super::msgpack::write_value;
|
|
|
|
const BUFFER_SIZE: usize = 65536;
|
|
const DEFAULT_QUALITY: u32 = 3; // 1 can be very bad
|
|
const DEFAULT_WINDOW_SIZE: u32 = 20;
|
|
|
|
#[derive(Clone)]
|
|
pub struct ToMsgpackz;
|
|
|
|
impl Command for ToMsgpackz {
|
|
fn name(&self) -> &str {
|
|
"to msgpackz"
|
|
}
|
|
|
|
fn signature(&self) -> Signature {
|
|
Signature::build(self.name())
|
|
.input_output_type(Type::Any, Type::Binary)
|
|
.named(
|
|
"quality",
|
|
SyntaxShape::Int,
|
|
"Quality of brotli compression (default 3)",
|
|
Some('q'),
|
|
)
|
|
.named(
|
|
"window-size",
|
|
SyntaxShape::Int,
|
|
"Window size for brotli compression (default 20)",
|
|
Some('w'),
|
|
)
|
|
.category(Category::Formats)
|
|
}
|
|
|
|
fn usage(&self) -> &str {
|
|
"Convert Nu values into brotli-compressed MessagePack."
|
|
}
|
|
|
|
fn extra_usage(&self) -> &str {
|
|
"This is the format used by the plugin registry file ($nu.plugin-path)."
|
|
}
|
|
|
|
fn run(
|
|
&self,
|
|
engine_state: &EngineState,
|
|
stack: &mut Stack,
|
|
call: &Call,
|
|
input: PipelineData,
|
|
) -> Result<PipelineData, ShellError> {
|
|
fn to_u32(n: Spanned<i64>) -> Result<Spanned<u32>, ShellError> {
|
|
u32::try_from(n.item)
|
|
.map_err(|err| ShellError::CantConvert {
|
|
to_type: "u32".into(),
|
|
from_type: "int".into(),
|
|
span: n.span,
|
|
help: Some(err.to_string()),
|
|
})
|
|
.map(|o| o.into_spanned(n.span))
|
|
}
|
|
|
|
let quality = call
|
|
.get_flag(engine_state, stack, "quality")?
|
|
.map(to_u32)
|
|
.transpose()?;
|
|
let window_size = call
|
|
.get_flag(engine_state, stack, "window-size")?
|
|
.map(to_u32)
|
|
.transpose()?;
|
|
|
|
let value_span = input.span().unwrap_or(call.head);
|
|
let value = input.into_value(value_span)?;
|
|
let mut out_buf = vec![];
|
|
let mut out = brotli::CompressorWriter::new(
|
|
&mut out_buf,
|
|
BUFFER_SIZE,
|
|
quality.map(|q| q.item).unwrap_or(DEFAULT_QUALITY),
|
|
window_size.map(|w| w.item).unwrap_or(DEFAULT_WINDOW_SIZE),
|
|
);
|
|
|
|
write_value(&mut out, &value, 0)?;
|
|
out.flush().err_span(call.head)?;
|
|
drop(out);
|
|
|
|
Ok(Value::binary(out_buf, call.head).into_pipeline_data())
|
|
}
|
|
}
|