From 9a274128ce49918fa59d46edb4eaaaa0ce8cc900 Mon Sep 17 00:00:00 2001 From: Reilly Wood <26268125+rgwood@users.noreply.github.com> Date: Tue, 10 Jan 2023 17:51:25 -0800 Subject: [PATCH] Combine benchmarks to speed up `cargo bench` build times (#7722) I've been using the new Criterion benchmarks and I noticed that they take a _long_ time to build before the benchmark can run. Turns out `cargo build` was building 3 separate benchmarking binaries with most of Nu's functionality in each one. As a simple temporary fix, I've moved all the benchmarks into a single file so that we only build 1 binary. ### Future work Would be nice to split the unrelated benchmarks out into modules, but when I did that a separate binary still got built for each one. I suspect Criterion's macros are doing something funny with module or file names. I've left a FIXME in the code to investigate this further. --- Cargo.toml | 10 +- benches/benchmarks.rs | 187 +++++++++++++++++++++++++++++++++++ benches/encoder_benchmark.rs | 76 -------------- benches/eval_benchmark.rs | 42 -------- benches/parser_benchmark.rs | 34 ------- 5 files changed, 188 insertions(+), 161 deletions(-) create mode 100644 benches/benchmarks.rs delete mode 100644 benches/encoder_benchmark.rs delete mode 100644 benches/eval_benchmark.rs delete mode 100644 benches/parser_benchmark.rs diff --git a/Cargo.toml b/Cargo.toml index fae3a67699..c6cc90d46c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,13 +146,5 @@ path = "src/main.rs" # Run all benchmarks with `cargo bench` # Run individual benchmarks like `cargo bench -- ` e.g. `cargo bench -- parse` [[bench]] -name = "encoder_benchmark" -harness = false - -[[bench]] -name = "eval_benchmark" -harness = false - -[[bench]] -name = "parser_benchmark" +name = "benchmarks" harness = false \ No newline at end of file diff --git a/benches/benchmarks.rs b/benches/benchmarks.rs new file mode 100644 index 0000000000..d7695e31f7 --- /dev/null +++ b/benches/benchmarks.rs @@ -0,0 +1,187 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use nu_cli::eval_source; +use nu_parser::parse; +use nu_plugin::{EncodingType, PluginResponse}; +use nu_protocol::{PipelineData, Span, Value}; +use nu_utils::{get_default_config, get_default_env}; + +// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking. +// When the *_benchmarks functions were in different files, `cargo bench` would build +// an executable for every single one - incredibly slowly. Would be nice to figure out +// a way to split things up again. + +fn parser_benchmarks(c: &mut Criterion) { + let mut engine_state = nu_command::create_default_context(); + // parsing config.nu breaks without PWD set + engine_state.add_env_var( + "PWD".into(), + Value::string("/some/dir".to_string(), Span::test_data()), + ); + + let default_env = get_default_env().as_bytes(); + c.bench_function("parse_default_env_file", |b| { + b.iter_batched( + || nu_protocol::engine::StateWorkingSet::new(&engine_state), + |mut working_set| parse(&mut working_set, None, default_env, false, &[]), + BatchSize::SmallInput, + ) + }); + + let default_config = get_default_config().as_bytes(); + c.bench_function("parse_default_config_file", |b| { + b.iter_batched( + || nu_protocol::engine::StateWorkingSet::new(&engine_state), + |mut working_set| parse(&mut working_set, None, default_config, false, &[]), + BatchSize::SmallInput, + ) + }); + + c.bench_function("eval default_env.nu", |b| { + b.iter(|| { + let mut engine_state = nu_command::create_default_context(); + let mut stack = nu_protocol::engine::Stack::new(); + eval_source( + &mut engine_state, + &mut stack, + get_default_env().as_bytes(), + "default_env.nu", + PipelineData::empty(), + ) + }) + }); + + c.bench_function("eval default_config.nu", |b| { + b.iter(|| { + let mut engine_state = nu_command::create_default_context(); + // parsing config.nu breaks without PWD set + engine_state.add_env_var( + "PWD".into(), + Value::string("/some/dir".to_string(), Span::test_data()), + ); + let mut stack = nu_protocol::engine::Stack::new(); + eval_source( + &mut engine_state, + &mut stack, + get_default_config().as_bytes(), + "default_config.nu", + PipelineData::empty(), + ) + }) + }); +} + +fn eval_benchmarks(c: &mut Criterion) { + c.bench_function("eval default_env.nu", |b| { + b.iter(|| { + let mut engine_state = nu_command::create_default_context(); + let mut stack = nu_protocol::engine::Stack::new(); + eval_source( + &mut engine_state, + &mut stack, + get_default_env().as_bytes(), + "default_env.nu", + PipelineData::empty(), + ) + }) + }); + + c.bench_function("eval default_config.nu", |b| { + b.iter(|| { + let mut engine_state = nu_command::create_default_context(); + // parsing config.nu breaks without PWD set + engine_state.add_env_var( + "PWD".into(), + Value::string("/some/dir".to_string(), Span::test_data()), + ); + let mut stack = nu_protocol::engine::Stack::new(); + eval_source( + &mut engine_state, + &mut stack, + get_default_config().as_bytes(), + "default_config.nu", + PipelineData::empty(), + ) + }) + }); +} + +// generate a new table data with `row_cnt` rows, `col_cnt` columns. +fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value { + let columns: Vec = (0..col_cnt).map(|x| format!("col_{x}")).collect(); + let vals: Vec = (0..col_cnt as i64).map(Value::test_int).collect(); + + Value::List { + vals: (0..row_cnt) + .map(|_| Value::test_record(columns.clone(), vals.clone())) + .collect(), + span: Span::test_data(), + } +} + +fn encoding_benchmarks(c: &mut Criterion) { + let mut group = c.benchmark_group("Encoding"); + let test_cnt_pairs = [ + (100, 5), + (100, 10), + (100, 15), + (1000, 5), + (1000, 10), + (1000, 15), + (10000, 5), + (10000, 10), + (10000, 15), + ]; + for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() { + for fmt in ["json", "msgpack"] { + group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| { + let mut res = vec![]; + let test_data = + PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt))); + let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap(); + b.iter(|| encoder.encode_response(&test_data, &mut res)) + }); + } + } + group.finish(); +} + +fn decoding_benchmarks(c: &mut Criterion) { + let mut group = c.benchmark_group("Decoding"); + let test_cnt_pairs = [ + (100, 5), + (100, 10), + (100, 15), + (1000, 5), + (1000, 10), + (1000, 15), + (10000, 5), + (10000, 10), + (10000, 15), + ]; + for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() { + for fmt in ["json", "msgpack"] { + group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| { + let mut res = vec![]; + let test_data = + PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt))); + let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap(); + encoder.encode_response(&test_data, &mut res).unwrap(); + let mut binary_data = std::io::Cursor::new(res); + b.iter(|| { + binary_data.set_position(0); + encoder.decode_response(&mut binary_data) + }) + }); + } + } + group.finish(); +} + +criterion_group!( + benches, + parser_benchmarks, + eval_benchmarks, + encoding_benchmarks, + decoding_benchmarks +); +criterion_main!(benches); diff --git a/benches/encoder_benchmark.rs b/benches/encoder_benchmark.rs deleted file mode 100644 index 63f3650e5f..0000000000 --- a/benches/encoder_benchmark.rs +++ /dev/null @@ -1,76 +0,0 @@ -use criterion::{criterion_group, criterion_main, Criterion}; -use nu_plugin::{EncodingType, PluginResponse}; -use nu_protocol::{Span, Value}; - -// generate a new table data with `row_cnt` rows, `col_cnt` columns. -fn new_test_data(row_cnt: usize, col_cnt: usize) -> Value { - let columns: Vec = (0..col_cnt).map(|x| format!("col_{x}")).collect(); - let vals: Vec = (0..col_cnt as i64).map(Value::test_int).collect(); - - Value::List { - vals: (0..row_cnt) - .map(|_| Value::test_record(columns.clone(), vals.clone())) - .collect(), - span: Span::test_data(), - } -} - -fn bench_encoding(c: &mut Criterion) { - let mut group = c.benchmark_group("Encoding"); - let test_cnt_pairs = [ - (100, 5), - (100, 10), - (100, 15), - (1000, 5), - (1000, 10), - (1000, 15), - (10000, 5), - (10000, 10), - (10000, 15), - ]; - for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() { - for fmt in ["json", "msgpack"] { - group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| { - let mut res = vec![]; - let test_data = PluginResponse::Value(Box::new(new_test_data(row_cnt, col_cnt))); - let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap(); - b.iter(|| encoder.encode_response(&test_data, &mut res)) - }); - } - } - group.finish(); -} - -fn bench_decoding(c: &mut Criterion) { - let mut group = c.benchmark_group("Decoding"); - let test_cnt_pairs = [ - (100, 5), - (100, 10), - (100, 15), - (1000, 5), - (1000, 10), - (1000, 15), - (10000, 5), - (10000, 10), - (10000, 15), - ]; - for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() { - for fmt in ["json", "msgpack"] { - group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| { - let mut res = vec![]; - let test_data = PluginResponse::Value(Box::new(new_test_data(row_cnt, col_cnt))); - let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap(); - encoder.encode_response(&test_data, &mut res).unwrap(); - let mut binary_data = std::io::Cursor::new(res); - b.iter(|| { - binary_data.set_position(0); - encoder.decode_response(&mut binary_data) - }) - }); - } - } - group.finish(); -} - -criterion_group!(benches, bench_encoding, bench_decoding); -criterion_main!(benches); diff --git a/benches/eval_benchmark.rs b/benches/eval_benchmark.rs deleted file mode 100644 index 6f0ec2e927..0000000000 --- a/benches/eval_benchmark.rs +++ /dev/null @@ -1,42 +0,0 @@ -use criterion::{criterion_group, criterion_main, Criterion}; -use nu_cli::eval_source; -use nu_protocol::{PipelineData, Span, Value}; -use nu_utils::{get_default_config, get_default_env}; - -fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("eval default_env.nu", |b| { - b.iter(|| { - let mut engine_state = nu_command::create_default_context(); - let mut stack = nu_protocol::engine::Stack::new(); - eval_source( - &mut engine_state, - &mut stack, - get_default_env().as_bytes(), - "default_env.nu", - PipelineData::empty(), - ) - }) - }); - - c.bench_function("eval default_config.nu", |b| { - b.iter(|| { - let mut engine_state = nu_command::create_default_context(); - // parsing config.nu breaks without PWD set - engine_state.add_env_var( - "PWD".into(), - Value::string("/some/dir".to_string(), Span::test_data()), - ); - let mut stack = nu_protocol::engine::Stack::new(); - eval_source( - &mut engine_state, - &mut stack, - get_default_config().as_bytes(), - "default_config.nu", - PipelineData::empty(), - ) - }) - }); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/benches/parser_benchmark.rs b/benches/parser_benchmark.rs deleted file mode 100644 index 6e939c4c1e..0000000000 --- a/benches/parser_benchmark.rs +++ /dev/null @@ -1,34 +0,0 @@ -use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; -use nu_parser::parse; -use nu_protocol::{Span, Value}; -use nu_utils::{get_default_config, get_default_env}; - -fn criterion_benchmark(c: &mut Criterion) { - let mut engine_state = nu_command::create_default_context(); - // parsing config.nu breaks without PWD set - engine_state.add_env_var( - "PWD".into(), - Value::string("/some/dir".to_string(), Span::test_data()), - ); - - let default_env = get_default_env().as_bytes(); - c.bench_function("parse_default_env_file", |b| { - b.iter_batched( - || nu_protocol::engine::StateWorkingSet::new(&engine_state), - |mut working_set| parse(&mut working_set, None, default_env, false, &[]), - BatchSize::SmallInput, - ) - }); - - let default_config = get_default_config().as_bytes(); - c.bench_function("parse_default_config_file", |b| { - b.iter_batched( - || nu_protocol::engine::StateWorkingSet::new(&engine_state), - |mut working_set| parse(&mut working_set, None, default_config, false, &[]), - BatchSize::SmallInput, - ) - }); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches);