Benchmark prototype

Cargo.lock

@@ -155,7 +155,6 @@ dependencies = [
  "predicates",
  "regex",
  "run_script",
- "rusty-fork",
  "semver",
  "serde",
  "serde_with",
@@ -1040,12 +1039,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "quick-error"
-version = "1.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
-
 [[package]]
 name = "quick-xml"
 version = "0.30.0"
@@ -1178,18 +1171,6 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "rusty-fork"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f"
-dependencies = [
- "fnv",
- "quick-error",
- "tempfile",
- "wait-timeout",
-]
-
 [[package]]
 name = "ryu"
 version = "1.0.11"

Cargo.toml

@@ -93,7 +93,6 @@ expect-test = "1.4.1"
 serial_test = { version = "2.0.0", default-features = false }
 predicates = "3.0.4"
 wait-timeout = "0.2.0"
-rusty-fork = "0.3.0"
 tempfile = "3.8.1"
 
 [target.'cfg(unix)'.dev-dependencies]

src/syntax_mapping.rs

@@ -128,10 +128,8 @@ impl<'a> SyntaxMapping<'a> {
 
 #[cfg(test)]
 mod tests {
-    use once_cell::sync::Lazy;
-    use rusty_fork::rusty_fork_test;
-
     use super::*;
+
     #[test]
     fn builtin_mappings_work() {
         let map = SyntaxMapping::new();
@@ -151,33 +149,6 @@ mod tests {
         let _mappings = map.builtin_mappings().collect::<Vec<_>>();
     }
 
-    // disabled for ARM builds because of an issue with `rusty_fork`
-    #[cfg(not(any(target_arch = "arm", target_arch = "aarch64")))]
-    // lazy initialisation test needs to be run on a separate instance because
-    // it will race with other tests
-    // see: https://github.com/rust-lang/rust/issues/47506
-    rusty_fork_test! {
-        #[test]
-        fn builtin_mappings_are_lazily_evaluated() {
-            let map = SyntaxMapping::new();
-
-            assert!(BUILTIN_MAPPINGS
-                .iter()
-                .all(|(matcher, _)| Lazy::get(matcher).is_none()));
-
-            // calling `builtin_mappings` should not trigger evaluation
-            let mappings_iter = map.builtin_mappings();
-            assert!(BUILTIN_MAPPINGS
-                .iter()
-                .all(|(matcher, _)| Lazy::get(matcher).is_none()));
-
-            let _mappings: Vec<_> = mappings_iter.collect();
-            assert!(BUILTIN_MAPPINGS
-                .iter()
-                .all(|(matcher, _)| Lazy::get(matcher).is_some()));
-        }
-    }
-
     #[test]
     fn builtin_mappings_matcher_only_compile_once() {
         let map = SyntaxMapping::new();

tests/benchmarks/run-benchmarks.sh

@@ -9,6 +9,13 @@ if ! command -v hyperfine > /dev/null 2>&1; then
 	exit 1
 fi
 
+# Check that jq is installed.
+if ! command -v jq > /dev/null 2>&1; then
+	echo "'jq' does not seem to be installed."
+	echo "You can get it here: https://jqlang.github.io/jq/download/"
+	exit 1
+fi
+
 # Check that python3 is installed.
 if ! command -v python3 > /dev/null 2>&1; then
 	echo "'python3' does not seem to be installed."
@@ -117,6 +124,49 @@ hyperfine \
 cat "$RESULT_DIR/startup-time-with-syntax-with-dependencies.md" >> "$REPORT"
 
 
+heading "Startup time with indeterminant syntax"
+hyperfine \
+	"$(printf "%q" "$BAT") --no-config startup-time-src/mystery-file" \
+	--command-name 'bat … mystery-file' \
+	--warmup "$WARMUP_COUNT" \
+    --runs $(($RUN_COUNT * 100)) \
+    --export-markdown "$RESULT_DIR/startup-time-with-indeterminant-syntax.md" \
+    --export-json "$RESULT_DIR/startup-time-with-indeterminant-syntax.json"
+cat "$RESULT_DIR/startup-time-with-indeterminant-syntax.md" >> "$REPORT"
+
+heading "Startup time with manually set syntax"
+hyperfine \
+	"$(printf "%q" "$BAT") --no-config --language=Dockerfile startup-time-src/mystery-file" \
+	--command-name 'bat … --language=Dockerfile mystery-file' \
+	--warmup "$WARMUP_COUNT" \
+    --runs $(($RUN_COUNT * 100)) \
+    --export-markdown "$RESULT_DIR/startup-time-with-manually-set-syntax.md" \
+    --export-json "$RESULT_DIR/startup-time-with-manually-set-syntax.json"
+cat "$RESULT_DIR/startup-time-with-manually-set-syntax.md" >> "$REPORT"
+
+heading "Startup time with mapped syntax"
+hyperfine \
+	"$(printf "%q" "$BAT") --no-config startup-time-src/Containerfile" \
+	--command-name 'bat … Containerfile' \
+	--warmup "$WARMUP_COUNT" \
+    --runs $(($RUN_COUNT * 100)) \
+    --export-markdown "$RESULT_DIR/startup-time-with-mapped-syntax.md" \
+    --export-json "$RESULT_DIR/startup-time-with-mapped-syntax.json"
+cat "$RESULT_DIR/startup-time-with-mapped-syntax.md" >> "$REPORT"
+
+AVG_TIMES=()
+for KIND in indeterminant manually-set mapped; do
+	JSON_SRC="$RESULT_DIR/startup-time-with-$KIND-syntax.json"
+	AVG=$(jq -r '.results[0].mean' "$JSON_SRC")
+	AVG_TIMES+=("$AVG")
+done
+# indeterminant should be slower, because it necessarily has to evaluate all rules
+# to ensure that nothing matches; manually-set and mapped should both be faster
+# because most or all GlobMatcher builds are skipped
+python3 -c "if ${AVG_TIMES[0]} < ${AVG_TIMES[1]}: print('WARN: indeterminant syntax has faster startup than manually set syntax!')"
+python3 -c "if ${AVG_TIMES[0]} < ${AVG_TIMES[2]}: print('WARN: indeterminant syntax has faster startup than mapped syntax!')"
+
+
 heading "Plain-text speed"
 hyperfine \
 	"$(printf "%q" "$BAT") --no-config --language=txt --style=plain highlighting-speed-src/numpy_test_multiarray.py" \

tests/benchmarks/startup-time-src/Containerfile

@@ -0,0 +1,3 @@
+FROM docker.io/alpine:latest
+COPY foo /root/bar
+RUN sleep 60

tests/benchmarks/startup-time-src/mystery-file

@@ -0,0 +1,3 @@
+FROM docker.io/alpine:latest
+COPY foo /root/bar
+RUN sleep 60