From a7295c8f1b7eb085d6a862147885352d3fe332fa Mon Sep 17 00:00:00 2001 From: WindSoilder Date: Wed, 24 Aug 2022 00:49:51 +0800 Subject: [PATCH] Plugin: Add benchmark for different encoding protocol (#6384) * add MessagePack as a plugin protocol * tmp merge from remote * add benchmark * use less benchmark group, and add README for analysing benchmark result * update README * update README * rewrite * remove comment * rename * fmt Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> --- Cargo.lock | 125 +++++++++++++++++- crates/nu-plugin/Cargo.toml | 7 + crates/nu-plugin/README.md | 5 + crates/nu-plugin/benches/encoder_benchmark.rs | 76 +++++++++++ crates/nu-plugin/src/lib.rs | 2 +- 5 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 crates/nu-plugin/benches/encoder_benchmark.rs diff --git a/Cargo.lock b/Cargo.lock index 6b3bccdd0..06df1d6ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,6 +505,12 @@ version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82efa3b0ab5e7e32b786334b052560ec0094135f906975d7481651b9ecf31a6a" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.73" @@ -594,6 +600,17 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags", + "textwrap 0.11.0", + "unicode-width", +] + [[package]] name = "codepage" version = "0.1.1" @@ -708,6 +725,42 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "critical-section" version = "0.2.7" @@ -1536,6 +1589,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hamcrest2" version = "0.3.0" @@ -2264,7 +2323,7 @@ dependencies = [ "supports-hyperlinks", "supports-unicode", "terminal_size 0.1.17", - "textwrap", + "textwrap 0.15.0", "thiserror", "unicode-width", ] @@ -2760,6 +2819,7 @@ dependencies = [ "bincode", "byte-order", "capnp", + "criterion", "nu-engine", "nu-protocol", "rmp", @@ -3087,6 +3147,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -3440,6 +3506,34 @@ dependencies = [ "array-init-cursor", ] +[[package]] +name = "plotters" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "716b4eeb6c4a1d3ecc956f75b43ec2e8e8ba80026413e70a3f41fd3313d3492b" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + +[[package]] +name = "plotters-svg" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polars" version = "0.23.2" @@ -4369,6 +4463,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.143" @@ -4916,6 +5020,15 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "textwrap" version = "0.15.0" @@ -4991,6 +5104,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/crates/nu-plugin/Cargo.toml b/crates/nu-plugin/Cargo.toml index 52f9c9aba..b2ab8498e 100644 --- a/crates/nu-plugin/Cargo.toml +++ b/crates/nu-plugin/Cargo.toml @@ -18,3 +18,10 @@ byte-order = "0.3.0" rmp = "0.8.11" rmp-serde = "1.1.0" rmpv = "1.0.0" + +[dev-dependencies] +criterion = "0.3" + +[[bench]] +name = "encoder_benchmark" +harness = false diff --git a/crates/nu-plugin/README.md b/crates/nu-plugin/README.md index 6d2c4d1af..d7c5cf44d 100644 --- a/crates/nu-plugin/README.md +++ b/crates/nu-plugin/README.md @@ -16,3 +16,8 @@ The steps are as follows: 7. Modify the serialize/deserialize functions. Check the following PRs for details: * https://github.com/nushell/nushell/pull/4980 * https://github.com/nushell/nushell/pull/4920 + +## Benchmark +Here is a simple benchmark for different protocol for encoding/decoding nushell table, with different rows and columns. You can simply run `cargo bench` to run benchmark. + +The relative html report is in `target/criterion/report/index.html`. diff --git a/crates/nu-plugin/benches/encoder_benchmark.rs b/crates/nu-plugin/benches/encoder_benchmark.rs new file mode 100644 index 000000000..06b9de738 --- /dev/null +++ b/crates/nu-plugin/benches/encoder_benchmark.rs @@ -0,0 +1,76 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use nu_plugin::{EncodingType, PluginResponse}; +use nu_protocol::{Span, Value}; + +// generate a new table data with `row_cnt` rows, `col_cnt` columns. +fn new_test_data(row_cnt: usize, col_cnt: usize) -> Value { + let columns: Vec = (0..col_cnt).map(|x| format!("col_{x}")).collect(); + let vals: Vec = (0..col_cnt as i64).map(|i| Value::test_int(i)).collect(); + + Value::List { + vals: (0..row_cnt) + .map(|_| Value::test_record(columns.clone(), vals.clone())) + .collect(), + span: Span::test_data(), + } +} + +fn bench_encoding(c: &mut Criterion) { + let mut group = c.benchmark_group("Encoding"); + let test_cnt_pairs = [ + (100, 5), + (100, 10), + (100, 15), + (1000, 5), + (1000, 10), + (1000, 15), + (10000, 5), + (10000, 10), + (10000, 15), + ]; + for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() { + for fmt in ["capnp", "json", "msgpack"] { + group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| { + let mut res = vec![]; + let test_data = PluginResponse::Value(Box::new(new_test_data(row_cnt, col_cnt))); + let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap(); + b.iter(|| encoder.encode_response(&test_data, &mut res)) + }); + } + } + group.finish(); +} + +fn bench_decoding(c: &mut Criterion) { + let mut group = c.benchmark_group("Decoding"); + let test_cnt_pairs = [ + (100, 5), + (100, 10), + (100, 15), + (1000, 5), + (1000, 10), + (1000, 15), + (10000, 5), + (10000, 10), + (10000, 15), + ]; + for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() { + for fmt in ["capnp", "json", "msgpack"] { + group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| { + let mut res = vec![]; + let test_data = PluginResponse::Value(Box::new(new_test_data(row_cnt, col_cnt))); + let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap(); + encoder.encode_response(&test_data, &mut res).unwrap(); + let mut binary_data = std::io::Cursor::new(res); + b.iter(|| { + binary_data.set_position(0); + encoder.decode_response(&mut binary_data) + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_encoding, bench_decoding); +criterion_main!(benches); diff --git a/crates/nu-plugin/src/lib.rs b/crates/nu-plugin/src/lib.rs index 46f6e0da0..bdb321ed6 100644 --- a/crates/nu-plugin/src/lib.rs +++ b/crates/nu-plugin/src/lib.rs @@ -6,7 +6,7 @@ mod serializers; mod plugin_capnp; pub use plugin::{get_signature, serve_plugin, Plugin, PluginDeclaration}; -pub use protocol::{EvaluatedCall, LabeledError, PluginData}; +pub use protocol::{EvaluatedCall, LabeledError, PluginData, PluginResponse}; pub use serializers::{ capnp::CapnpSerializer, json::JsonSerializer, msgpack::MsgPackSerializer, EncodingType, };