From 0da534d5249a9111f8e1af0ff517526e2c66cdb0 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Wed, 8 May 2024 17:28:52 +0100 Subject: [PATCH] fix: add incremental rebuild to daemon loop (#2010) --- Cargo.lock | 2 ++ crates/atuin-client/src/database.rs | 2 +- crates/atuin-client/src/history/store.rs | 2 +- crates/atuin-daemon/Cargo.toml | 2 ++ crates/atuin-daemon/src/server.rs | 9 ++++-- crates/atuin-daemon/src/server/sync.rs | 35 ++++++++++++++++++++---- 6 files changed, 43 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 419002e6..0a7ec77e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -322,6 +322,8 @@ name = "atuin-daemon" version = "0.1.0" dependencies = [ "atuin-client", + "atuin-dotfiles", + "atuin-history", "dashmap", "eyre", "prost", diff --git a/crates/atuin-client/src/database.rs b/crates/atuin-client/src/database.rs index 33156077..c443da37 100644 --- a/crates/atuin-client/src/database.rs +++ b/crates/atuin-client/src/database.rs @@ -120,7 +120,7 @@ pub trait Database: Send + Sync + 'static { // Intended for use on a developer machine and not a sync server. // TODO: implement IntoIterator -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Sqlite { pub pool: SqlitePool, } diff --git a/crates/atuin-client/src/history/store.rs b/crates/atuin-client/src/history/store.rs index fe2b7b92..b0341a26 100644 --- a/crates/atuin-client/src/history/store.rs +++ b/crates/atuin-client/src/history/store.rs @@ -12,7 +12,7 @@ use atuin_common::record::{DecryptedData, Host, HostId, Record, RecordId, Record use super::{History, HistoryId, HISTORY_TAG, HISTORY_VERSION}; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct HistoryStore { pub store: SqliteStore, pub host_id: HostId, diff --git a/crates/atuin-daemon/Cargo.toml b/crates/atuin-daemon/Cargo.toml index 5bcd1611..390cb28f 100644 --- a/crates/atuin-daemon/Cargo.toml +++ b/crates/atuin-daemon/Cargo.toml @@ -13,6 +13,8 @@ readme.workspace = true [dependencies] atuin-client = { path = "../atuin-client", version = "18.0.1" } +atuin-dotfiles = { path = "../atuin-dotfiles", version = "0.2.0" } +atuin-history = { path = "../atuin-history", version = "0.1.0" } time = { workspace = true } uuid = { workspace = true } diff --git a/crates/atuin-daemon/src/server.rs b/crates/atuin-daemon/src/server.rs index 42ef1701..72305737 100644 --- a/crates/atuin-daemon/src/server.rs +++ b/crates/atuin-daemon/src/server.rs @@ -166,7 +166,7 @@ pub async fn listen( let host_id = Settings::host_id().expect("failed to get host_id"); let history_store = HistoryStore::new(store.clone(), host_id, encryption_key); - let history = HistoryService::new(history_store, history_db); + let history = HistoryService::new(history_store.clone(), history_db.clone()); let socket = settings.daemon.socket_path.clone(); let uds = UnixListener::bind(socket.clone())?; @@ -175,7 +175,12 @@ pub async fn listen( tracing::info!("listening on unix socket {:?}", socket); // start services - tokio::spawn(sync::worker(settings.clone(), store)); + tokio::spawn(sync::worker( + settings.clone(), + store, + history_store, + history_db, + )); Server::builder() .add_service(HistoryServer::new(history)) diff --git a/crates/atuin-daemon/src/server/sync.rs b/crates/atuin-daemon/src/server/sync.rs index de34779c..ba037d4e 100644 --- a/crates/atuin-daemon/src/server/sync.rs +++ b/crates/atuin-daemon/src/server/sync.rs @@ -2,14 +2,32 @@ use eyre::Result; use rand::Rng; use tokio::time::{self, MissedTickBehavior}; +use atuin_client::database::Sqlite as HistoryDatabase; use atuin_client::{ + encryption, + history::store::HistoryStore, record::{sqlite_store::SqliteStore, sync}, settings::Settings, }; -pub async fn worker(settings: Settings, store: SqliteStore) -> Result<()> { +use atuin_dotfiles::store::{var::VarStore, AliasStore}; + +pub async fn worker( + settings: Settings, + store: SqliteStore, + history_store: HistoryStore, + history_db: HistoryDatabase, +) -> Result<()> { tracing::info!("booting sync worker"); + let encryption_key: [u8; 32] = encryption::load_key(&settings)?.into(); + let host_id = Settings::host_id().expect("failed to get host_id"); + let alias_store = AliasStore::new(store.clone(), host_id, encryption_key); + let var_store = VarStore::new(store.clone(), host_id, encryption_key); + + // Don't backoff by more than 30 mins (with a random jitter of up to 1 min) + let max_interval: f64 = 60.0 * 30.0 + rand::thread_rng().gen_range(0.0..60.0); + let mut ticker = time::interval(time::Duration::from_secs(settings.daemon.sync_frequency)); // IMPORTANT: without this, if we miss ticks because a sync takes ages or is otherwise delayed, @@ -24,13 +42,13 @@ pub async fn worker(settings: Settings, store: SqliteStore) -> Result<()> { if let Err(e) = res { tracing::error!("sync tick failed with {e}"); + let mut rng = rand::thread_rng(); - let new_interval = ticker.period().as_secs_f64() * rng.gen_range(2.0..2.2); + let mut new_interval = ticker.period().as_secs_f64() * rng.gen_range(2.0..2.2); - // Don't backoff by more than 30 mins - if new_interval > 60.0 * 30.0 { - continue; + if new_interval > max_interval { + new_interval = max_interval; } ticker = time::interval(time::Duration::from_secs(new_interval as u64)); @@ -46,6 +64,13 @@ pub async fn worker(settings: Settings, store: SqliteStore) -> Result<()> { "sync complete" ); + history_store + .incremental_build(&history_db, &downloaded) + .await?; + + alias_store.build().await?; + var_store.build().await?; + // Reset backoff on success if ticker.period().as_secs() != settings.daemon.sync_frequency { ticker = time::interval(time::Duration::from_secs(settings.daemon.sync_frequency));