From 73bd8015c3ba50ceb5168a8b67bb1ac5d3b48dc1 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Sat, 19 Aug 2023 12:28:39 +0100 Subject: [PATCH] Automatically filter out secrets (#1182) I'd like to extend the regex list here very soon, but start off by automatically filtering out secrets. Do not store them in history! I've included regex for: 1. AWS key id 2. Github pat (old and new) 3. Slack oauth tokens (bot, user) 4. Slack webhooks 5. Stripe live/test keys Will need updating after #806 --- atuin-client/src/history.rs | 86 +++++++++++++++++++++++++++++ atuin-client/src/lib.rs | 1 + atuin-client/src/secrets.rs | 54 ++++++++++++++++++ atuin-client/src/settings.rs | 63 ++++++++++++++------- atuin/src/command/client/history.rs | 11 ++-- docs/docs/config/config.md | 14 +++++ 6 files changed, 201 insertions(+), 28 deletions(-) create mode 100644 atuin-client/src/secrets.rs diff --git a/atuin-client/src/history.rs b/atuin-client/src/history.rs index 441960c8..4d084786 100644 --- a/atuin-client/src/history.rs +++ b/atuin-client/src/history.rs @@ -3,6 +3,9 @@ use std::env; use chrono::Utc; use atuin_common::utils::uuid_v7; +use regex::RegexSet; + +use crate::{secrets::SECRET_PATTERNS, settings::Settings}; mod builder; @@ -185,4 +188,87 @@ impl History { pub fn success(&self) -> bool { self.exit == 0 || self.duration == -1 } + + pub fn should_save(&self, settings: &Settings) -> bool { + let secret_regex = SECRET_PATTERNS.iter().map(|f| f.1); + let secret_regex = RegexSet::new(secret_regex).expect("Failed to build secrets regex"); + + !(self.command.starts_with(' ') + || settings.history_filter.is_match(&self.command) + || settings.cwd_filter.is_match(&self.cwd) + || (secret_regex.is_match(&self.command)) && settings.secrets_filter) + } +} + +#[cfg(test)] +mod tests { + use regex::RegexSet; + + use crate::settings::Settings; + + use super::History; + + // Test that we don't save history where necessary + #[test] + fn privacy_test() { + let mut settings = Settings::default(); + settings.cwd_filter = RegexSet::new(["^/supasecret"]).unwrap(); + settings.history_filter = RegexSet::new(["^psql"]).unwrap(); + + let normal_command: History = History::capture() + .timestamp(chrono::Utc::now()) + .command("echo foo") + .cwd("/") + .build() + .into(); + + let with_space: History = History::capture() + .timestamp(chrono::Utc::now()) + .command(" echo bar") + .cwd("/") + .build() + .into(); + + let stripe_key: History = History::capture() + .timestamp(chrono::Utc::now()) + .command("curl foo.com/bar?key=sk_test_1234567890abcdefghijklmnop") + .cwd("/") + .build() + .into(); + + let secret_dir: History = History::capture() + .timestamp(chrono::Utc::now()) + .command("echo ohno") + .cwd("/supasecret") + .build() + .into(); + + let with_psql: History = History::capture() + .timestamp(chrono::Utc::now()) + .command("psql") + .cwd("/supasecret") + .build() + .into(); + + assert!(normal_command.should_save(&settings)); + assert!(!with_space.should_save(&settings)); + assert!(!stripe_key.should_save(&settings)); + assert!(!secret_dir.should_save(&settings)); + assert!(!with_psql.should_save(&settings)); + } + + #[test] + fn disable_secrets() { + let mut settings = Settings::default(); + settings.secrets_filter = false; + + let stripe_key: History = History::capture() + .timestamp(chrono::Utc::now()) + .command("curl foo.com/bar?key=sk_test_1234567890abcdefghijklmnop") + .cwd("/") + .build() + .into(); + + assert!(stripe_key.should_save(&settings)); + } } diff --git a/atuin-client/src/lib.rs b/atuin-client/src/lib.rs index 7ecfa894..05b69450 100644 --- a/atuin-client/src/lib.rs +++ b/atuin-client/src/lib.rs @@ -15,4 +15,5 @@ pub mod import; pub mod kv; pub mod ordering; pub mod record; +pub mod secrets; pub mod settings; diff --git a/atuin-client/src/secrets.rs b/atuin-client/src/secrets.rs new file mode 100644 index 00000000..ba6aee66 --- /dev/null +++ b/atuin-client/src/secrets.rs @@ -0,0 +1,54 @@ +// This file will probably trigger a lot of scanners. Sorry. + +// A list of (name, regex, test), where test should match against regex +pub static SECRET_PATTERNS: &[(&str, &str, &str)] = &[ + ( + "AWS Access Key ID", + "AKIA[0-9A-Z]{16}", + "AKIAIOSFODNN7EXAMPLE", + ), + ( + "GitHub PAT (old)", + "^ghp_[a-zA-Z0-9]{36}$", + "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", // legit, I expired it + ), + ( + "GitHub PAT (new)", + "^github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}$", + "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", // also legit, also expired + ), + ( + "Slack OAuth v2 bot", + "xoxb-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}", + "xoxb-17653672481-19874698323-pdFZKVeTuE8sk7oOcBrzbqgy", + ), + ( + "Slack OAuth v2 user token", + "xoxp-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}", + "xoxp-17653672481-19874698323-pdFZKVeTuE8sk7oOcBrzbqgy", + ), + ( + "Slack webhook", + "T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}", + "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX", + ), + ("Stripe test key", "sk_test_[0-9a-zA-Z]{24}", "sk_test_1234567890abcdefghijklmnop"), + ("Stripe live key", "sk_live_[0-9a-zA-Z]{24}", "sk_live_1234567890abcdefghijklmnop"), +]; + +#[cfg(test)] +mod tests { + use regex::Regex; + + use crate::secrets::SECRET_PATTERNS; + + #[test] + fn test_secrets() { + for (name, regex, test) in SECRET_PATTERNS { + let re = + Regex::new(regex).expect(format!("Failed to compile regex for {name}").as_str()); + + assert!(re.is_match(test), "{name} test failed!"); + } + } +} diff --git a/atuin-client/src/settings.rs b/atuin-client/src/settings.rs index 67050792..c68be0d5 100644 --- a/atuin-client/src/settings.rs +++ b/atuin-client/src/settings.rs @@ -7,7 +7,9 @@ use std::{ use atuin_common::record::HostId; use chrono::{prelude::*, Utc}; use clap::ValueEnum; -use config::{Config, Environment, File as ConfigFile, FileFormat}; +use config::{ + builder::DefaultState, Config, ConfigBuilder, Environment, File as ConfigFile, FileFormat, +}; use eyre::{eyre, Context, Result}; use fs_err::{create_dir_all, File}; use parse_duration::parse; @@ -168,6 +170,7 @@ pub struct Settings { pub history_filter: RegexSet, #[serde(with = "serde_regex", default = "RegexSet::empty")] pub cwd_filter: RegexSet, + pub secrets_filter: bool, pub workspaces: bool, pub ctrl_n_shortcuts: bool, @@ -330,32 +333,15 @@ impl Settings { None } - pub fn new() -> Result { - let config_dir = atuin_common::utils::config_dir(); - + pub fn builder() -> Result> { let data_dir = atuin_common::utils::data_dir(); - - create_dir_all(&config_dir) - .wrap_err_with(|| format!("could not create dir {config_dir:?}"))?; - create_dir_all(&data_dir).wrap_err_with(|| format!("could not create dir {data_dir:?}"))?; - - let mut config_file = if let Ok(p) = std::env::var("ATUIN_CONFIG_DIR") { - PathBuf::from(p) - } else { - let mut config_file = PathBuf::new(); - config_file.push(config_dir); - config_file - }; - - config_file.push("config.toml"); - let db_path = data_dir.join("history.db"); let record_store_path = data_dir.join("records.db"); let key_path = data_dir.join("key"); let session_path = data_dir.join("session"); - let mut config_builder = Config::builder() + Ok(Config::builder() .set_default("db_path", db_path.to_str())? .set_default("record_store_path", record_store_path.to_str())? .set_default("key_path", key_path.to_str())? @@ -384,11 +370,33 @@ impl Settings { .set_default("session_token", "")? .set_default("workspaces", false)? .set_default("ctrl_n_shortcuts", false)? + .set_default("secrets_filter", true)? .add_source( Environment::with_prefix("atuin") .prefix_separator("_") .separator("__"), - ); + )) + } + + pub fn new() -> Result { + let config_dir = atuin_common::utils::config_dir(); + let data_dir = atuin_common::utils::data_dir(); + + create_dir_all(&config_dir) + .wrap_err_with(|| format!("could not create dir {config_dir:?}"))?; + create_dir_all(&data_dir).wrap_err_with(|| format!("could not create dir {data_dir:?}"))?; + + let mut config_file = if let Ok(p) = std::env::var("ATUIN_CONFIG_DIR") { + PathBuf::from(p) + } else { + let mut config_file = PathBuf::new(); + config_file.push(config_dir); + config_file + }; + + config_file.push("config.toml"); + + let mut config_builder = Self::builder()?; config_builder = if config_file.exists() { config_builder.add_source(ConfigFile::new( @@ -433,3 +441,16 @@ impl Settings { Ok(settings) } } + +impl Default for Settings { + fn default() -> Self { + // if this panics something is very wrong, as the default config + // does not build or deserialize into the settings struct + Self::builder() + .expect("Could not build default") + .build() + .expect("Could not build config") + .try_deserialize() + .expect("Could not deserialize config") + } +} diff --git a/atuin/src/command/client/history.rs b/atuin/src/command/client/history.rs index f8154e57..44e621ec 100644 --- a/atuin/src/command/client/history.rs +++ b/atuin/src/command/client/history.rs @@ -191,16 +191,9 @@ impl Cmd { ) -> Result<()> { let command = command.join(" "); - if command.starts_with(' ') || settings.history_filter.is_match(&command) { - return Ok(()); - } - // It's better for atuin to silently fail here and attempt to // store whatever is ran, than to throw an error to the terminal let cwd = utils::get_current_dir(); - if !cwd.is_empty() && settings.cwd_filter.is_match(&cwd) { - return Ok(()); - } let h: History = History::capture() .timestamp(chrono::Utc::now()) @@ -209,6 +202,10 @@ impl Cmd { .build() .into(); + if !h.should_save(settings) { + return Ok(()); + } + // print the ID // we use this as the key for calling end println!("{}", h.id); diff --git a/docs/docs/config/config.md b/docs/docs/config/config.md index 50b1d7f2..e30ca2ae 100644 --- a/docs/docs/config/config.md +++ b/docs/docs/config/config.md @@ -250,6 +250,20 @@ history_filter = [ ] ``` +### secrets_filter + +``` +secrets_filter = true +``` + +Defaults to true. This matches history against a set of default regex, and will not save it if we get a match. Defaults include + +1. AWS key id +2. Github pat (old and new) +3. Slack oauth tokens (bot, user) +4. Slack webhooks +5. Stripe live/test keys + ## macOS Ctrl-n key shortcuts macOS does not have an Alt key, although terminal emulators can often be configured to map the Option key to be used as Alt. *However*, remapping Option this way may prevent typing some characters, such as using Option-3 to type `#` on the British English layout. For such a scenario, set the `ctrl_n_shortcuts` option to `true` in your config file to replace Alt-0 to Alt-9 shortcuts with Ctrl-0 to Ctrl-9 instead: