Automatically filter out secrets (#1182)

I'd like to extend the regex list here very soon, but start off by
automatically filtering out secrets. Do not store them in history!

I've included regex for:

1. AWS key id
2. Github pat (old and new)
3. Slack oauth tokens (bot, user)
4. Slack webhooks
5. Stripe live/test keys

Will need updating after #806

atuin-client/src/history.rs

@@ -3,6 +3,9 @@ use std::env;
 use chrono::Utc;
 
 use atuin_common::utils::uuid_v7;
+use regex::RegexSet;
+
+use crate::{secrets::SECRET_PATTERNS, settings::Settings};
 
 mod builder;
 
@@ -185,4 +188,87 @@ impl History {
     pub fn success(&self) -> bool {
         self.exit == 0 || self.duration == -1
     }
+
+    pub fn should_save(&self, settings: &Settings) -> bool {
+        let secret_regex = SECRET_PATTERNS.iter().map(|f| f.1);
+        let secret_regex = RegexSet::new(secret_regex).expect("Failed to build secrets regex");
+
+        !(self.command.starts_with(' ')
+            || settings.history_filter.is_match(&self.command)
+            || settings.cwd_filter.is_match(&self.cwd)
+            || (secret_regex.is_match(&self.command)) && settings.secrets_filter)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use regex::RegexSet;
+
+    use crate::settings::Settings;
+
+    use super::History;
+
+    // Test that we don't save history where necessary
+    #[test]
+    fn privacy_test() {
+        let mut settings = Settings::default();
+        settings.cwd_filter = RegexSet::new(["^/supasecret"]).unwrap();
+        settings.history_filter = RegexSet::new(["^psql"]).unwrap();
+
+        let normal_command: History = History::capture()
+            .timestamp(chrono::Utc::now())
+            .command("echo foo")
+            .cwd("/")
+            .build()
+            .into();
+
+        let with_space: History = History::capture()
+            .timestamp(chrono::Utc::now())
+            .command(" echo bar")
+            .cwd("/")
+            .build()
+            .into();
+
+        let stripe_key: History = History::capture()
+            .timestamp(chrono::Utc::now())
+            .command("curl foo.com/bar?key=sk_test_1234567890abcdefghijklmnop")
+            .cwd("/")
+            .build()
+            .into();
+
+        let secret_dir: History = History::capture()
+            .timestamp(chrono::Utc::now())
+            .command("echo ohno")
+            .cwd("/supasecret")
+            .build()
+            .into();
+
+        let with_psql: History = History::capture()
+            .timestamp(chrono::Utc::now())
+            .command("psql")
+            .cwd("/supasecret")
+            .build()
+            .into();
+
+        assert!(normal_command.should_save(&settings));
+        assert!(!with_space.should_save(&settings));
+        assert!(!stripe_key.should_save(&settings));
+        assert!(!secret_dir.should_save(&settings));
+        assert!(!with_psql.should_save(&settings));
+    }
+
+    #[test]
+    fn disable_secrets() {
+        let mut settings = Settings::default();
+        settings.secrets_filter = false;
+
+        let stripe_key: History = History::capture()
+            .timestamp(chrono::Utc::now())
+            .command("curl foo.com/bar?key=sk_test_1234567890abcdefghijklmnop")
+            .cwd("/")
+            .build()
+            .into();
+
+        assert!(stripe_key.should_save(&settings));
+    }
 }

atuin-client/src/lib.rs

@@ -15,4 +15,5 @@ pub mod import;
 pub mod kv;
 pub mod ordering;
 pub mod record;
+pub mod secrets;
 pub mod settings;

atuin-client/src/secrets.rs

@@ -0,0 +1,54 @@
+// This file will probably trigger a lot of scanners. Sorry.
+
+// A list of (name, regex, test), where test should match against regex
+pub static SECRET_PATTERNS: &[(&str, &str, &str)] = &[
+    (
+        "AWS Access Key ID",
+        "AKIA[0-9A-Z]{16}",
+        "AKIAIOSFODNN7EXAMPLE",
+    ),
+    (
+        "GitHub PAT (old)",
+        "^ghp_[a-zA-Z0-9]{36}$",
+        "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", // legit, I expired it
+    ),
+    (
+        "GitHub PAT (new)",
+        "^github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}$",
+        "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", // also legit, also expired
+    ),
+    (
+        "Slack OAuth v2 bot",
+        "xoxb-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}",
+        "xoxb-17653672481-19874698323-pdFZKVeTuE8sk7oOcBrzbqgy",
+    ),
+    (
+        "Slack OAuth v2 user token",
+        "xoxp-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}",
+        "xoxp-17653672481-19874698323-pdFZKVeTuE8sk7oOcBrzbqgy",
+    ),
+    (
+        "Slack webhook",
+        "T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}",
+        "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX",
+    ),
+    ("Stripe test key", "sk_test_[0-9a-zA-Z]{24}", "sk_test_1234567890abcdefghijklmnop"),
+    ("Stripe live key", "sk_live_[0-9a-zA-Z]{24}", "sk_live_1234567890abcdefghijklmnop"),
+];
+
+#[cfg(test)]
+mod tests {
+    use regex::Regex;
+
+    use crate::secrets::SECRET_PATTERNS;
+
+    #[test]
+    fn test_secrets() {
+        for (name, regex, test) in SECRET_PATTERNS {
+            let re =
+                Regex::new(regex).expect(format!("Failed to compile regex for {name}").as_str());
+
+            assert!(re.is_match(test), "{name} test failed!");
+        }
+    }
+}

atuin-client/src/settings.rs

@@ -7,7 +7,9 @@ use std::{
 use atuin_common::record::HostId;
 use chrono::{prelude::*, Utc};
 use clap::ValueEnum;
-use config::{Config, Environment, File as ConfigFile, FileFormat};
+use config::{
+    builder::DefaultState, Config, ConfigBuilder, Environment, File as ConfigFile, FileFormat,
+};
 use eyre::{eyre, Context, Result};
 use fs_err::{create_dir_all, File};
 use parse_duration::parse;
@@ -168,6 +170,7 @@ pub struct Settings {
     pub history_filter: RegexSet,
     #[serde(with = "serde_regex", default = "RegexSet::empty")]
     pub cwd_filter: RegexSet,
+    pub secrets_filter: bool,
     pub workspaces: bool,
     pub ctrl_n_shortcuts: bool,
 
@@ -330,32 +333,15 @@ impl Settings {
         None
     }
 
-    pub fn new() -> Result<Self> {
-        let config_dir = atuin_common::utils::config_dir();
-
+    pub fn builder() -> Result<ConfigBuilder<DefaultState>> {
         let data_dir = atuin_common::utils::data_dir();
-
-        create_dir_all(&config_dir)
-            .wrap_err_with(|| format!("could not create dir {config_dir:?}"))?;
-        create_dir_all(&data_dir).wrap_err_with(|| format!("could not create dir {data_dir:?}"))?;
-
-        let mut config_file = if let Ok(p) = std::env::var("ATUIN_CONFIG_DIR") {
-            PathBuf::from(p)
-        } else {
-            let mut config_file = PathBuf::new();
-            config_file.push(config_dir);
-            config_file
-        };
-
-        config_file.push("config.toml");
-
         let db_path = data_dir.join("history.db");
         let record_store_path = data_dir.join("records.db");
 
         let key_path = data_dir.join("key");
         let session_path = data_dir.join("session");
 
-        let mut config_builder = Config::builder()
+        Ok(Config::builder()
             .set_default("db_path", db_path.to_str())?
             .set_default("record_store_path", record_store_path.to_str())?
             .set_default("key_path", key_path.to_str())?
@@ -384,11 +370,33 @@ impl Settings {
             .set_default("session_token", "")?
             .set_default("workspaces", false)?
             .set_default("ctrl_n_shortcuts", false)?
+            .set_default("secrets_filter", true)?
             .add_source(
                 Environment::with_prefix("atuin")
                     .prefix_separator("_")
                     .separator("__"),
-            );
+            ))
+    }
+
+    pub fn new() -> Result<Self> {
+        let config_dir = atuin_common::utils::config_dir();
+        let data_dir = atuin_common::utils::data_dir();
+
+        create_dir_all(&config_dir)
+            .wrap_err_with(|| format!("could not create dir {config_dir:?}"))?;
+        create_dir_all(&data_dir).wrap_err_with(|| format!("could not create dir {data_dir:?}"))?;
+
+        let mut config_file = if let Ok(p) = std::env::var("ATUIN_CONFIG_DIR") {
+            PathBuf::from(p)
+        } else {
+            let mut config_file = PathBuf::new();
+            config_file.push(config_dir);
+            config_file
+        };
+
+        config_file.push("config.toml");
+
+        let mut config_builder = Self::builder()?;
 
         config_builder = if config_file.exists() {
             config_builder.add_source(ConfigFile::new(
@@ -433,3 +441,16 @@ impl Settings {
         Ok(settings)
     }
 }
+
+impl Default for Settings {
+    fn default() -> Self {
+        // if this panics something is very wrong, as the default config
+        // does not build or deserialize into the settings struct
+        Self::builder()
+            .expect("Could not build default")
+            .build()
+            .expect("Could not build config")
+            .try_deserialize()
+            .expect("Could not deserialize config")
+    }
+}

atuin/src/command/client/history.rs

@@ -191,16 +191,9 @@ impl Cmd {
     ) -> Result<()> {
         let command = command.join(" ");
 
-        if command.starts_with(' ') || settings.history_filter.is_match(&command) {
-            return Ok(());
-        }
-
         // It's better for atuin to silently fail here and attempt to
         // store whatever is ran, than to throw an error to the terminal
         let cwd = utils::get_current_dir();
-        if !cwd.is_empty() && settings.cwd_filter.is_match(&cwd) {
-            return Ok(());
-        }
 
         let h: History = History::capture()
             .timestamp(chrono::Utc::now())
@@ -209,6 +202,10 @@ impl Cmd {
             .build()
             .into();
 
+        if !h.should_save(settings) {
+            return Ok(());
+        }
+
         // print the ID
         // we use this as the key for calling end
         println!("{}", h.id);

docs/docs/config/config.md

@@ -250,6 +250,20 @@ history_filter = [
 ]
 ```
 
+### secrets_filter
+
+```
+secrets_filter = true
+```
+
+Defaults to true. This matches history against a set of default regex, and will not save it if we get a match. Defaults include
+
+1. AWS key id
+2. Github pat (old and new)
+3. Slack oauth tokens (bot, user)
+4. Slack webhooks
+5. Stripe live/test keys
+
 ## macOS <kbd>Ctrl-n</kbd> key shortcuts
 
 macOS does not have an <kbd>Alt</kbd> key, although terminal emulators can often be configured to map the <kbd>Option</kbd> key to be used as <kbd>Alt</kbd>. *However*, remapping <kbd>Option</kbd> this way may prevent typing some characters, such as using <kbd>Option-3</kbd> to type `#` on the British English layout. For such a scenario, set the `ctrl_n_shortcuts` option to `true` in your config file to replace <kbd>Alt-0</kbd> to <kbd>Alt-9</kbd> shortcuts with <kbd>Ctrl-0</kbd> to <kbd>Ctrl-9</kbd> instead: