slight refactor - allows caching crypto items

atuin-client/src/api_client.rs

@@ -1,7 +1,5 @@
 use std::collections::HashMap;
-use std::collections::HashSet;
 
-use atuin_common::api::EncryptionScheme;
 use chrono::Utc;
 use eyre::{bail, Result};
 use reqwest::{
@@ -11,22 +9,18 @@ use reqwest::{
 
 use atuin_common::api::{
     AddHistoryRequest, CountResponse, DeleteHistoryRequest, ErrorResponse, IndexResponse,
-    LoginRequest, LoginResponse, RegisterResponse, StatusResponse, SyncHistoryResponse,
+    LoginRequest, LoginResponse, RegisterResponse, StatusResponse, SyncHistoryItem,
+    SyncHistoryResponse,
 };
 use semver::Version;
 
-use crate::encryption::{key, xchacha20poly1305, xsalsa20poly1305legacy};
 use crate::settings::Settings;
 use crate::{history::History, sync::hash_str};
 
 static APP_USER_AGENT: &str = concat!("atuin/", env!("CARGO_PKG_VERSION"),);
 
-// TODO: remove all references to the encryption key from this
-// It should be handled *elsewhere*
-
 pub struct Client<'a> {
     sync_addr: &'a str,
-    key: key::Key,
     client: reqwest::Client,
 }
 
@@ -117,7 +111,6 @@ impl<'a> Client<'a> {
 
         Ok(Client {
             sync_addr: &settings.sync_address,
-            key: key::load(settings)?,
             client: reqwest::Client::builder()
                 .user_agent(APP_USER_AGENT)
                 .default_headers(headers)
@@ -160,8 +153,7 @@ impl<'a> Client<'a> {
         sync_ts: chrono::DateTime<Utc>,
         history_ts: chrono::DateTime<Utc>,
         host: Option<String>,
-        deleted: HashSet<String>,
-    ) -> Result<Vec<History>> {
+    ) -> Result<Vec<SyncHistoryItem>> {
         let host = match host {
             None => hash_str(&format!("{}:{}", whoami::hostname(), whoami::username())),
             Some(h) => h,
@@ -179,27 +171,7 @@ impl<'a> Client<'a> {
 
         let history = resp.json::<SyncHistoryResponse>().await?;
 
-        let mut output = Vec::with_capacity(history.history.len());
-        for entry in history.more_history {
-            let mut history = match entry.scheme {
-                Some(EncryptionScheme::XSalsa20Poly1305Legacy) | None => {
-                    xsalsa20poly1305legacy::decrypt(entry.data, &self.key)?
-                }
-                Some(EncryptionScheme::XChaCha20Poly1305) => {
-                    xchacha20poly1305::decrypt(&entry.data, &self.key, &entry.id)?
-                }
-                Some(EncryptionScheme::Unknown(x)) => {
-                    bail!("cannot decrypt '{x}' encryption scheme")
-                }
-            };
-            if deleted.contains(&entry.id) {
-                history.deleted_at = Some(Utc::now());
-                history.command.clear();
-            }
-            output.push(history);
-        }
-
-        Ok(output)
+        Ok(history.sync_history)
     }
 
     pub async fn post_history(&self, history: &[AddHistoryRequest]) -> Result<()> {

atuin-client/src/encryption.rs

@@ -74,7 +74,7 @@ pub mod key {
 // DO NOT MODIFY. We can't change old encryption schemes, only add new ones.
 pub mod xsalsa20poly1305legacy {
     use chrono::Utc;
-    use eyre::{eyre, Result};
+    use eyre::{bail, eyre, Result};
     use serde::{Deserialize, Serialize};
     use xsalsa20poly1305::{
         aead::{Nonce, OsRng},
@@ -99,54 +99,73 @@ pub mod xsalsa20poly1305legacy {
         pub hostname: String,
     }
 
-    use crate::history::History;
-
-    pub fn encrypt(history: &History, key: &Key) -> Result<String> {
-        // serialize with msgpack
-        let mut buf = rmp_serde::to_vec(history)?;
-
-        let nonce = XSalsa20Poly1305::generate_nonce(&mut OsRng);
-        XSalsa20Poly1305::new(key)
-            .encrypt_in_place(&nonce, &[], &mut buf)
-            .map_err(|_| eyre!("could not encrypt"))?;
-
-        let record = serde_json::to_string(&EncryptedHistory {
-            ciphertext: buf,
-            nonce,
-        })?;
-
-        Ok(record)
+    pub struct Client {
+        inner: XSalsa20Poly1305,
     }
 
-    pub fn decrypt(encrypted_history: String, key: &Key) -> Result<History> {
-        let mut decoded: EncryptedHistory = serde_json::from_str(&encrypted_history)?;
+    use crate::history::History;
 
-        XSalsa20Poly1305::new(key)
-            .decrypt_in_place(&decoded.nonce, &[], &mut decoded.ciphertext)
-            .map_err(|_| eyre!("could not decrypt"))?;
-        let plaintext = decoded.ciphertext;
+    impl Client {
+        pub fn new(key: &Key) -> Self {
+            Self {
+                inner: XSalsa20Poly1305::new(key),
+            }
+        }
 
-        let history = rmp_serde::from_slice(&plaintext);
+        pub fn encrypt(&self, history: &History) -> Result<String> {
+            // serialize with msgpack
+            let mut buf = rmp_serde::to_vec(&history)?;
 
-        // ugly hack because we broke things
-        let Ok(history) = history else {
-            // fallback to without deleted_at
-            let history: HistoryWithoutDelete = rmp_serde::from_slice(&plaintext)?;
+            let nonce = XSalsa20Poly1305::generate_nonce(&mut OsRng);
+            self.inner
+                .encrypt_in_place(&nonce, &[], &mut buf)
+                .map_err(|_| eyre!("could not encrypt"))?;
 
-            return Ok(History {
-                id: history.id,
-                cwd: history.cwd,
-                exit: history.exit,
-                command: history.command,
-                session: history.session,
-                duration: history.duration,
-                hostname: history.hostname,
-                timestamp: history.timestamp,
-                deleted_at: None,
-            });
-        };
+            let record = serde_json::to_string(&EncryptedHistory {
+                ciphertext: buf,
+                nonce,
+            })?;
 
-        Ok(history)
+            Ok(record)
+        }
+
+        pub fn decrypt(&self, encrypted_history: &str, id: &str) -> Result<History> {
+            let mut decoded: EncryptedHistory = serde_json::from_str(encrypted_history)?;
+
+            self.inner
+                .decrypt_in_place(&decoded.nonce, &[], &mut decoded.ciphertext)
+                .map_err(|_| eyre!("could not decrypt"))?;
+            let plaintext = decoded.ciphertext;
+
+            let history = rmp_serde::from_slice(&plaintext);
+
+            // ugly hack because we broke things
+            let history = match history {
+                Ok(history) => history,
+                Err(_) => {
+                    // fallback to without deleted_at
+                    let history: HistoryWithoutDelete = rmp_serde::from_slice(&plaintext)?;
+
+                    History {
+                        id: history.id,
+                        cwd: history.cwd,
+                        exit: history.exit,
+                        command: history.command,
+                        session: history.session,
+                        duration: history.duration,
+                        hostname: history.hostname,
+                        timestamp: history.timestamp,
+                        deleted_at: None,
+                    }
+                }
+            };
+
+            if history.id != id {
+                bail!("encryption integrity check failed")
+            }
+
+            Ok(history)
+        }
     }
 
     #[cfg(test)]
@@ -155,12 +174,12 @@ pub mod xsalsa20poly1305legacy {
 
         use crate::history::History;
 
-        use super::{decrypt, encrypt};
+        use super::Client;
 
         #[test]
         fn test_encrypt_decrypt() {
-            let key1 = XSalsa20Poly1305::generate_key(&mut OsRng);
-            let key2 = XSalsa20Poly1305::generate_key(&mut OsRng);
+            let key1 = Client::new(&XSalsa20Poly1305::generate_key(&mut OsRng));
+            let key2 = Client::new(&XSalsa20Poly1305::generate_key(&mut OsRng));
 
             let history = History::new(
                 chrono::Utc::now(),
@@ -173,20 +192,27 @@ pub mod xsalsa20poly1305legacy {
                 None,
             );
 
-            let e1 = encrypt(&history, &key1).unwrap();
-            let e2 = encrypt(&history, &key2).unwrap();
+            let e1 = key1.encrypt(&history).unwrap();
+            let e2 = key2.encrypt(&history).unwrap();
 
             assert_ne!(e1, e2);
 
             // test decryption works
             // this should pass
-            match decrypt(e1, &key1) {
+            match key1.decrypt(&e1, &history.id) {
                 Err(e) => panic!("failed to decrypt, got {}", e),
                 Ok(h) => assert_eq!(h, history),
             };
 
             // this should err
-            let _ = decrypt(e2, &key1).expect_err("expected an error decrypting with invalid key");
+            let _ = key1
+                .decrypt(&e2, &history.id)
+                .expect_err("expected an error decrypting with invalid key");
+
+            // this should err
+            let _ = key2
+                .decrypt(&e2, "bad id")
+                .expect_err("expected an error decrypting with incorrect id");
         }
     }
 }
@@ -222,64 +248,73 @@ pub mod xchacha20poly1305 {
 
     use crate::history::History;
 
-    fn content_key(key: &Key, id: &str) -> Result<chacha20poly1305::Key> {
-        let mut content_key = chacha20poly1305::Key::default();
-        Hkdf::<Sha256>::new(Some(b"history"), key)
-            .expand(id.as_bytes(), &mut content_key)
-            .map_err(|_| eyre!("could not derive encryption key"))?;
-        Ok(content_key)
+    pub struct Client {
+        inner: Hkdf<Sha256>,
     }
 
-    pub fn encrypt(history: History, key: &Key) -> Result<String> {
-        // a unique encryption key for this entry
-        let content_key = content_key(key, &history.id)?;
+    impl Client {
+        pub fn new(key: &Key) -> Self {
+            Self {
+                // constant 'salt' is important and actually helps with security, while helping to improving performance
+                // <https://soatok.blog/2021/11/17/understanding-hkdf/>
+                inner: Hkdf::<Sha256>::new(Some(b"history"), key),
+            }
+        }
 
-        let mut plaintext = rmp_serde::to_vec(&HistoryPlaintext {
-            duration: history.duration,
-            exit: history.exit,
-            command: history.command,
-            cwd: history.cwd,
-            session: history.session,
-            hostname: history.hostname,
-            timestamp: history.timestamp,
-        })?;
+        fn cipher(&self, id: &str) -> Result<XChaCha20Poly1305> {
+            let mut content_key = chacha20poly1305::Key::default();
+            self.inner
+                .expand(id.as_bytes(), &mut content_key)
+                .map_err(|_| eyre!("could not derive encryption key"))?;
+            Ok(XChaCha20Poly1305::new(&content_key))
+        }
 
-        let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng);
-        XChaCha20Poly1305::new(&content_key)
-            .encrypt_in_place(&nonce, history.id.as_bytes(), &mut plaintext)
-            .map_err(|_| eyre!("could not encrypt"))?;
+        pub fn encrypt(&self, history: History) -> Result<String> {
+            let mut plaintext = rmp_serde::to_vec(&HistoryPlaintext {
+                duration: history.duration,
+                exit: history.exit,
+                command: history.command,
+                cwd: history.cwd,
+                session: history.session,
+                hostname: history.hostname,
+                timestamp: history.timestamp,
+            })?;
 
-        let record = serde_json::to_string(&EncryptedHistory {
-            ciphertext: plaintext,
-            nonce,
-        })?;
+            let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng);
+            self.cipher(&history.id)?
+                .encrypt_in_place(&nonce, history.id.as_bytes(), &mut plaintext)
+                .map_err(|_| eyre!("could not encrypt"))?;
 
-        Ok(record)
-    }
+            let record = serde_json::to_string(&EncryptedHistory {
+                ciphertext: plaintext,
+                nonce,
+            })?;
 
-    pub fn decrypt(encrypted_history: &str, key: &Key, id: &str) -> Result<History> {
-        let content_key = content_key(key, id)?;
+            Ok(record)
+        }
 
-        let mut decoded: EncryptedHistory = serde_json::from_str(encrypted_history)?;
+        pub fn decrypt(&self, encrypted_history: &str, id: &str) -> Result<History> {
+            let mut decoded: EncryptedHistory = serde_json::from_str(encrypted_history)?;
 
-        XChaCha20Poly1305::new(&content_key)
-            .decrypt_in_place(&decoded.nonce, id.as_bytes(), &mut decoded.ciphertext)
-            .map_err(|_| eyre!("could not decrypt"))?;
-        let plaintext = decoded.ciphertext;
+            self.cipher(id)?
+                .decrypt_in_place(&decoded.nonce, id.as_bytes(), &mut decoded.ciphertext)
+                .map_err(|_| eyre!("could not decrypt"))?;
+            let plaintext = decoded.ciphertext;
 
-        let history: HistoryPlaintext = rmp_serde::from_slice(&plaintext)?;
+            let history: HistoryPlaintext = rmp_serde::from_slice(&plaintext)?;
 
-        Ok(History {
-            id: id.to_owned(),
-            cwd: history.cwd,
-            exit: history.exit,
-            command: history.command,
-            session: history.session,
-            duration: history.duration,
-            hostname: history.hostname,
-            timestamp: history.timestamp,
-            deleted_at: None,
-        })
+            Ok(History {
+                id: id.to_owned(),
+                cwd: history.cwd,
+                exit: history.exit,
+                command: history.command,
+                session: history.session,
+                duration: history.duration,
+                hostname: history.hostname,
+                timestamp: history.timestamp,
+                deleted_at: None,
+            })
+        }
     }
 
     #[cfg(test)]
@@ -289,11 +324,11 @@ pub mod xchacha20poly1305 {
 
         use crate::history::History;
 
-        use super::{decrypt, encrypt};
+        use super::Client;
 
         #[test]
         fn test_encrypt_decrypt() {
-            let key = XSalsa20Poly1305::generate_key(&mut OsRng);
+            let key = Client::new(&XSalsa20Poly1305::generate_key(&mut OsRng));
 
             let history1 = History::new(
                 chrono::Utc::now(),
@@ -311,24 +346,25 @@ pub mod xchacha20poly1305 {
             };
 
             // same contents, different id, different encryption key
-            let e1 = encrypt(history1.clone(), &key).unwrap();
-            let e2 = encrypt(history2.clone(), &key).unwrap();
+            let e1 = key.encrypt(history1.clone()).unwrap();
+            let e2 = key.encrypt(history2.clone()).unwrap();
 
             assert_ne!(e1, e2);
 
             // test decryption works
             // this should pass
-            match decrypt(&e1, &key, &history1.id) {
+            match key.decrypt(&e1, &history1.id) {
                 Err(e) => panic!("failed to decrypt, got {}", e),
                 Ok(h) => assert_eq!(h, history1),
             };
-            match decrypt(&e2, &key, &history2.id) {
+            match key.decrypt(&e2, &history2.id) {
                 Err(e) => panic!("failed to decrypt, got {}", e),
                 Ok(h) => assert_eq!(h, history2),
             };
 
             // this should err
-            let _ = decrypt(&e2, &key, &history1.id)
+            let _ = key
+                .decrypt(&e2, &history1.id)
                 .expect_err("expected an error decrypting with invalid key");
         }
     }

atuin-client/src/sync.rs

@@ -35,6 +35,7 @@ pub fn hash_str(string: &str) -> String {
 async fn sync_download(
     force: bool,
     client: &api_client::Client<'_>,
+    crypto: &Crypto,
     db: &mut (impl Database + Send),
 ) -> Result<(i64, i64)> {
     debug!("starting sync download");
@@ -43,7 +44,8 @@ async fn sync_download(
     let remote_count = remote_status.count;
 
     // useful to ensure we don't even save something that hasn't yet been synced + deleted
-    let remote_deleted = HashSet::from_iter(remote_status.deleted.clone());
+    let remote_deleted =
+        HashSet::<&str>::from_iter(remote_status.deleted.iter().map(String::as_str));
 
     let initial_local = db.history_count().await?;
     let mut local_count = initial_local;
@@ -58,16 +60,34 @@ async fn sync_download(
 
     let host = if force { Some(String::from("")) } else { None };
 
+    let mut page = Vec::new();
     while remote_count > local_count {
-        let page = client
-            .get_history(
-                last_sync,
-                last_timestamp,
-                host.clone(),
-                remote_deleted.clone(),
-            )
+        let encrypted_page = client
+            .get_history(last_sync, last_timestamp, host.clone())
             .await?;
 
+        page.clear();
+        page.reserve(encrypted_page.len());
+
+        for entry in encrypted_page {
+            let mut history = match entry.encryption {
+                Some(EncryptionScheme::XSalsa20Poly1305Legacy) | None => {
+                    crypto.salsa_legacy.decrypt(&entry.data, &entry.id)?
+                }
+                Some(EncryptionScheme::XChaCha20Poly1305) => {
+                    crypto.xchacha20.decrypt(&entry.data, &entry.id)?
+                }
+                Some(EncryptionScheme::Unknown(x)) => {
+                    bail!("cannot decrypt '{x}' encryption scheme")
+                }
+            };
+            if remote_deleted.contains(&*entry.id) {
+                history.deleted_at = Some(Utc::now());
+                history.command.clear();
+            }
+            page.push(history);
+        }
+
         db.save_bulk(&page).await?;
 
         local_count = db.history_count().await?;
@@ -113,6 +133,7 @@ async fn sync_upload(
     settings: &Settings,
     _force: bool,
     client: &api_client::Client<'_>,
+    crypto: &Crypto,
     db: &mut (impl Database + Send),
 ) -> Result<()> {
     debug!("starting sync upload");
@@ -127,10 +148,7 @@ async fn sync_upload(
 
     debug!("remote has {}, we have {}", remote_count, local_count);
 
-    let key = key::load(settings)?; // encryption key
-
     // first just try the most recent set
-
     let mut cursor = Utc::now();
 
     while local_count > remote_count {
@@ -146,12 +164,10 @@ async fn sync_upload(
                 id: i.id.clone(),
                 timestamp: i.timestamp,
                 hostname: hash_str(&i.hostname),
-                scheme: Some(settings.encryption_scheme.clone()),
+                encryption: Some(settings.encryption_scheme.clone()),
                 data: match &settings.encryption_scheme {
-                    EncryptionScheme::XSalsa20Poly1305Legacy => {
-                        xsalsa20poly1305legacy::encrypt(&i, &key)?
-                    }
-                    EncryptionScheme::XChaCha20Poly1305 => xchacha20poly1305::encrypt(i, &key)?,
+                    EncryptionScheme::XSalsa20Poly1305Legacy => crypto.salsa_legacy.encrypt(&i)?,
+                    EncryptionScheme::XChaCha20Poly1305 => crypto.xchacha20.encrypt(i)?,
                     EncryptionScheme::Unknown(x) => {
                         bail!("cannot encrypt with '{x}' encryption scheme")
                     }
@@ -185,10 +201,11 @@ async fn sync_upload(
 
 pub async fn sync(settings: &Settings, force: bool, db: &mut (impl Database + Send)) -> Result<()> {
     let client = api_client::Client::new(settings)?;
+    let crypto = Crypto::new(&key::load(settings)?);
 
-    sync_upload(settings, force, &client, db).await?;
+    sync_upload(settings, force, &client, &crypto, db).await?;
 
-    let download = sync_download(force, &client, db).await?;
+    let download = sync_download(force, &client, &crypto, db).await?;
 
     debug!("sync downloaded {}", download.0);
 
@@ -196,3 +213,17 @@ pub async fn sync(settings: &Settings, force: bool, db: &mut (impl Database + Se
 
     Ok(())
 }
+
+struct Crypto {
+    salsa_legacy: xsalsa20poly1305legacy::Client,
+    xchacha20: xchacha20poly1305::Client,
+}
+
+impl Crypto {
+    fn new(key: &key::Key) -> Self {
+        Self {
+            salsa_legacy: xsalsa20poly1305legacy::Client::new(key),
+            xchacha20: xchacha20poly1305::Client::new(key),
+        }
+    }
+}

atuin-common/src/api.rs

@@ -39,8 +39,7 @@ pub struct AddHistoryRequest {
     pub timestamp: chrono::DateTime<Utc>,
     pub data: String,
     pub hostname: String,
-    // the encryption scheme used
-    pub scheme: Option<EncryptionScheme>,
+    pub encryption: Option<EncryptionScheme>,
 }
 
 #[derive(Debug, Clone)]
@@ -106,8 +105,16 @@ pub struct SyncHistoryRequest {
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct SyncHistoryResponse {
+    /// deprecated
     pub history: Vec<String>,
-    pub more_history: Vec<AddHistoryRequest>,
+    pub sync_history: Vec<SyncHistoryItem>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SyncHistoryItem {
+    pub id: String,
+    pub data: String,
+    pub encryption: Option<EncryptionScheme>,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

atuin-server/src/handlers/history.rs

@@ -5,7 +5,6 @@ use axum::{
     http::HeaderMap,
     Json,
 };
-use chrono::{TimeZone, Utc};
 use http::StatusCode;
 use tracing::{debug, error, instrument};
 
@@ -90,14 +89,12 @@ pub async fn list<DB: Database>(
 
     let history: Vec<String> = entries.iter().map(|i| i.data.clone()).collect();
 
-    let more_history: Vec<AddHistoryRequest> = entries
+    let sync_history = entries
         .into_iter()
-        .map(|i| AddHistoryRequest {
+        .map(|i| SyncHistoryItem {
             id: i.client_id,
-            timestamp: Utc.from_utc_datetime(&i.timestamp),
             data: i.data,
-            hostname: i.hostname,
-            scheme: i.scheme.map(EncryptionScheme::from_string),
+            encryption: i.scheme.map(EncryptionScheme::from_string),
         })
         .collect();
 
@@ -109,7 +106,7 @@ pub async fn list<DB: Database>(
 
     Ok(Json(SyncHistoryResponse {
         history,
-        more_history,
+        sync_history,
     }))
 }