Plugin: Add benchmark for different encoding protocol (#6384)

* add MessagePack as a plugin protocol

* tmp merge from remote

* add benchmark

* use less benchmark group, and add README for analysing benchmark result

* update README

* update README

* rewrite

* remove comment

* rename

* fmt

Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>

Cargo.lock

@@ -505,6 +505,12 @@ version = "0.14.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "82efa3b0ab5e7e32b786334b052560ec0094135f906975d7481651b9ecf31a6a"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.0.73"
@@ -594,6 +600,17 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "bitflags",
+ "textwrap 0.11.0",
+ "unicode-width",
+]
+
 [[package]]
 name = "codepage"
 version = "0.1.1"
@@ -708,6 +725,42 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
+dependencies = [
+ "atty",
+ "cast",
+ "clap",
+ "criterion-plot",
+ "csv",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_cbor",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
 [[package]]
 name = "critical-section"
 version = "0.2.7"
@@ -1536,6 +1589,12 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
 [[package]]
 name = "hamcrest2"
 version = "0.3.0"
@@ -2264,7 +2323,7 @@ dependencies = [
  "supports-hyperlinks",
  "supports-unicode",
  "terminal_size 0.1.17",
- "textwrap",
+ "textwrap 0.15.0",
  "thiserror",
  "unicode-width",
 ]
@@ -2760,6 +2819,7 @@ dependencies = [
  "bincode",
  "byte-order",
  "capnp",
+ "criterion",
  "nu-engine",
  "nu-protocol",
  "rmp",
@@ -3087,6 +3147,12 @@ version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
 [[package]]
 name = "opaque-debug"
 version = "0.3.0"
@@ -3440,6 +3506,34 @@ dependencies = [
  "array-init-cursor",
 ]
 
+[[package]]
+name = "plotters"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "716b4eeb6c4a1d3ecc956f75b43ec2e8e8ba80026413e70a3f41fd3313d3492b"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "polars"
 version = "0.23.2"
@@ -4369,6 +4463,16 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde_cbor"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
+dependencies = [
+ "half",
+ "serde",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.143"
@@ -4916,6 +5020,15 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
 
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
@@ -4991,6 +5104,16 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"

crates/nu-plugin/Cargo.toml

@@ -18,3 +18,10 @@ byte-order = "0.3.0"
 rmp = "0.8.11"
 rmp-serde = "1.1.0"
 rmpv = "1.0.0"
+
+[dev-dependencies]
+criterion = "0.3"
+
+[[bench]]
+name = "encoder_benchmark"
+harness = false

crates/nu-plugin/README.md

@@ -16,3 +16,8 @@ The steps are as follows:
 7. Modify the serialize/deserialize functions. Check the following PRs for details:
     * https://github.com/nushell/nushell/pull/4980
     * https://github.com/nushell/nushell/pull/4920
+
+## Benchmark
+Here is a simple benchmark for different protocol for encoding/decoding nushell table, with different rows and columns.  You can simply run `cargo bench` to run benchmark.
+
+The relative html report is in `target/criterion/report/index.html`.

crates/nu-plugin/benches/encoder_benchmark.rs

@@ -0,0 +1,76 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use nu_plugin::{EncodingType, PluginResponse};
+use nu_protocol::{Span, Value};
+
+// generate a new table data with `row_cnt` rows, `col_cnt` columns.
+fn new_test_data(row_cnt: usize, col_cnt: usize) -> Value {
+    let columns: Vec<String> = (0..col_cnt).map(|x| format!("col_{x}")).collect();
+    let vals: Vec<Value> = (0..col_cnt as i64).map(|i| Value::test_int(i)).collect();
+
+    Value::List {
+        vals: (0..row_cnt)
+            .map(|_| Value::test_record(columns.clone(), vals.clone()))
+            .collect(),
+        span: Span::test_data(),
+    }
+}
+
+fn bench_encoding(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Encoding");
+    let test_cnt_pairs = [
+        (100, 5),
+        (100, 10),
+        (100, 15),
+        (1000, 5),
+        (1000, 10),
+        (1000, 15),
+        (10000, 5),
+        (10000, 10),
+        (10000, 15),
+    ];
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
+        for fmt in ["capnp", "json", "msgpack"] {
+            group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| {
+                let mut res = vec![];
+                let test_data = PluginResponse::Value(Box::new(new_test_data(row_cnt, col_cnt)));
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
+                b.iter(|| encoder.encode_response(&test_data, &mut res))
+            });
+        }
+    }
+    group.finish();
+}
+
+fn bench_decoding(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Decoding");
+    let test_cnt_pairs = [
+        (100, 5),
+        (100, 10),
+        (100, 15),
+        (1000, 5),
+        (1000, 10),
+        (1000, 15),
+        (10000, 5),
+        (10000, 10),
+        (10000, 15),
+    ];
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
+        for fmt in ["capnp", "json", "msgpack"] {
+            group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| {
+                let mut res = vec![];
+                let test_data = PluginResponse::Value(Box::new(new_test_data(row_cnt, col_cnt)));
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
+                encoder.encode_response(&test_data, &mut res).unwrap();
+                let mut binary_data = std::io::Cursor::new(res);
+                b.iter(|| {
+                    binary_data.set_position(0);
+                    encoder.decode_response(&mut binary_data)
+                })
+            });
+        }
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_encoding, bench_decoding);
+criterion_main!(benches);

crates/nu-plugin/src/lib.rs

@@ -6,7 +6,7 @@ mod serializers;
 mod plugin_capnp;
 
 pub use plugin::{get_signature, serve_plugin, Plugin, PluginDeclaration};
-pub use protocol::{EvaluatedCall, LabeledError, PluginData};
+pub use protocol::{EvaluatedCall, LabeledError, PluginData, PluginResponse};
 pub use serializers::{
     capnp::CapnpSerializer, json::JsonSerializer, msgpack::MsgPackSerializer, EncodingType,
 };