diff --git a/.azure/azure-pipelines.yml b/.azure/azure-pipelines.yml
index 50d9b3e1a7..469dc20fc0 100644
--- a/.azure/azure-pipelines.yml
+++ b/.azure/azure-pipelines.yml
@@ -44,13 +44,13 @@ steps:
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
       rustup component add rustfmt
     displayName: Install Rust
-  - bash: RUSTFLAGS="-D warnings" cargo test --all --features=stable
+  - bash: RUSTFLAGS="-D warnings" cargo test --all --features stable,test-bins
     condition: eq(variables['style'], 'unflagged')
     displayName: Run tests
   - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
     condition: eq(variables['style'], 'unflagged')
     displayName: Check clippy lints
-  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features=stable
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features stable,test-bins
     condition: eq(variables['style'], 'canary')
     displayName: Run tests
   - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
@@ -59,4 +59,4 @@ steps:
   - bash: cargo fmt --all -- --check
     condition: eq(variables['style'], 'fmt')
     displayName: Lint
-    
\ No newline at end of file
+    
diff --git a/Cargo.toml b/Cargo.toml
index 2417ecf887..c8c54dbb25 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -139,10 +139,10 @@ semver = {version = "0.9.0", optional = true}
 users = "0.9"
 
 [features]
-# Dummy executables
-nu-dummies = []
+# Test executables
+test-bins = []
 
-default = ["nu-dummies", "sys", "ps", "textview", "inc", "str"]
+default = ["sys", "ps", "textview", "inc", "str"]
 stable = ["default", "starship-prompt", "binaryview", "match", "tree", "average", "sum", "post", "fetch", "clipboard"]
 
 # Default
@@ -183,18 +183,18 @@ path = "src/lib.rs"
 
 [[bin]]
 name = "fail"
-path = "crates/nu-test-support/src/dummies/fail.rs"
-required-features = ["nu-dummies"]
+path = "crates/nu-test-support/src/bins/fail.rs"
+required-features = ["test-bins"]
 
 [[bin]]
 name = "chop"
-path = "crates/nu-test-support/src/dummies/chop.rs"
-required-features = ["nu-dummies"]
+path = "crates/nu-test-support/src/bins/chop.rs"
+required-features = ["test-bins"]
 
 [[bin]]
 name = "cococo"
-path = "crates/nu-test-support/src/dummies/cococo.rs"
-required-features = ["nu-dummies"]
+path = "crates/nu-test-support/src/bins/cococo.rs"
+required-features = ["test-bins"]
 
 # Core plugins that ship with `cargo install nu` by default
 # Currently, Cargo limits us to installing only one binary
diff --git a/crates/nu-test-support/src/dummies/chop.rs b/crates/nu-test-support/src/bins/chop.rs
similarity index 100%
rename from crates/nu-test-support/src/dummies/chop.rs
rename to crates/nu-test-support/src/bins/chop.rs
diff --git a/crates/nu-test-support/src/dummies/cococo.rs b/crates/nu-test-support/src/bins/cococo.rs
similarity index 100%
rename from crates/nu-test-support/src/dummies/cococo.rs
rename to crates/nu-test-support/src/bins/cococo.rs
diff --git a/crates/nu-test-support/src/dummies/fail.rs b/crates/nu-test-support/src/bins/fail.rs
similarity index 100%
rename from crates/nu-test-support/src/dummies/fail.rs
rename to crates/nu-test-support/src/bins/fail.rs