feat: add background daemon (#2006)

* init daemon crate

* wip

* minimal functioning daemon, needs cleanup for sure

* better errors

* add signal cleanup

* logging

* things

* add sync worker

* move daemon crate

* 30s -> 5mins

* make clippy happy

* fix stuff maybe?

* fmt

* trim packages

* rate limit fix

* more protoc huh

* this makes no sense, why linux why

* can it install literally just curl

* windows in ci is slow, and all the newer things will not work there. disable the daemon feature and it will build

* add daemon feature

* maybe this

* ok wut where is protoc

* try setting protoc

* hm

* try copying protoc

* remove optional

* add cross config

* idk nix

* does nix want this?

* some random pkg I found does this

* uh oh

* hack, be gone!

* update contributing

.github/workflows/release.yaml

@@ -74,6 +74,11 @@ jobs:
         override: true
         profile: minimal # minimal component installation (ie, no documentation)
 
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+
     - name: Show version information (Rust, cargo, GCC)
       shell: bash
       run: |

.github/workflows/rust.yml

@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-14, windows-latest]
+        os: [ubuntu-latest, macos-14]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -24,6 +24,11 @@ jobs:
         with:
           toolchain: stable
 
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
       - uses: actions/cache@v3
         with:
           path: |
@@ -32,19 +37,6 @@ jobs:
             target
           key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: Install dependencies
-        if: matrix.os != 'macos-14' && matrix.os != 'windows-latest'
-        run: |
-          sudo apt update
-          sudo apt install libwebkit2gtk-4.1-dev \
-            build-essential \
-            curl \
-            wget \
-            file \
-            libssl-dev \
-            libayatana-appindicator3-dev \
-            librsvg2-dev
-
       - name: Run cargo build common
         run: cargo build -p atuin-common --locked --release
 
@@ -65,7 +57,6 @@ jobs:
         # warning: libelf.so.2, needed by <...>/libkvm.so, not found (try using -rpath or -rpath-link)
         target: [x86_64-unknown-illumos]
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v3
 
@@ -74,6 +65,11 @@ jobs:
         with:
           tool: cross
 
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
       - uses: actions/cache@v3
         with:
           path: |
@@ -92,12 +88,13 @@ jobs:
         run: cross build -p atuin-server --locked --target ${{ matrix.target }}
 
       - name: Run cross build main
-        run: cross build --all --locked --target ${{ matrix.target }}
+        run: |
+          cross build --all --locked --target ${{ matrix.target }}
 
   unit-test:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-14, windows-latest]
+        os: [ubuntu-latest, macos-14]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -108,18 +105,10 @@ jobs:
         with:
           toolchain: stable
 
-      - name: Install dependencies
+      - name: Install Protoc
-        if: matrix.os != 'macos-14' && matrix.os != 'windows-latest'
+        uses: arduino/setup-protoc@v3
-        run: |
+        with:
-          sudo apt update
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          sudo apt install libwebkit2gtk-4.1-dev \
-            build-essential \
-            curl \
-            wget \
-            file \
-            libssl-dev \
-            libayatana-appindicator3-dev \
-            librsvg2-dev
 
       - uses: taiki-e/install-action@v2
         name: Install nextest
@@ -140,7 +129,7 @@ jobs:
   check:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-14, windows-latest]
+        os: [ubuntu-latest, macos-14]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -151,18 +140,10 @@ jobs:
         with:
           toolchain: stable
 
-      - name: Install dependencies
+      - name: Install Protoc
-        if: matrix.os != 'macos-14' && matrix.os != 'windows-latest'
+        uses: arduino/setup-protoc@v3
-        run: |
+        with:
-          sudo apt update
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          sudo apt install libwebkit2gtk-4.1-dev \
-            build-essential \
-            curl \
-            wget \
-            file \
-            libssl-dev \
-            libayatana-appindicator3-dev \
-            librsvg2-dev
 
       - uses: actions/cache@v3
         with:
@@ -208,6 +189,11 @@ jobs:
         with:
           toolchain: stable
 
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
       - uses: taiki-e/install-action@v2
         name: Install nextest
         with:
@@ -238,18 +224,10 @@ jobs:
           toolchain: stable
           components: clippy
 
-      - name: Install dependencies
+      - name: Install Protoc
-        if: matrix.os != 'macos-14' && matrix.os != 'windows-latest'
+        uses: arduino/setup-protoc@v3
-        run: |
+        with:
-          sudo apt update
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          sudo apt install libwebkit2gtk-4.1-dev \
-            build-essential \
-            curl \
-            wget \
-            file \
-            libssl-dev \
-            libayatana-appindicator3-dev \
-            librsvg2-dev
 
       - uses: actions/cache@v3
         with:

.gitignore

@@ -6,3 +6,6 @@
 .vscode/
 result
 publish.sh
+
+ui/backend/target
+ui/backend/gen

CONTRIBUTING.md

@@ -2,7 +2,12 @@
 
 Thank you so much for considering contributing to Atuin! We really appreciate it <3
 
-Atuin doesn't require anything super special to develop - standard Rust tooling will do you just fine. We commit to supporting the latest stable version of Rust - nothing more, nothing less, no nightly.
+Development dependencies
+
+1. A rust toolchain ([rustup](https://rustup.rs) recommended)
+2. [Protobuf compiler](https://grpc.io/docs/protoc-installation/)
+
+We commit to supporting the latest stable version of Rust - nothing more, nothing less, no nightly.
 
 Before working on anything, we suggest taking a copy of your Atuin data directory (`~/.local/share/atuin` on most \*nix platforms). If anything goes wrong, you can always restore it!
 

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = [
-  "crates/*"
+  "crates/*",
 ]
 
 resolver = "2"
@@ -42,6 +42,12 @@ typed-builder = "0.18.2"
 pretty_assertions = "1.3.0"
 thiserror = "1.0"
 rustix = { version = "0.38.34", features = ["process", "fs"] }
+tower = "0.4"
+tracing = "0.1"
+
+[workspace.dependencies.tracing-subscriber]
+version = "0.3"
+features = ["ansi", "fmt", "registry", "env-filter"]
 
 [workspace.dependencies.reqwest]
 version = "0.11"

Cross.toml

@@ -0,0 +1,4 @@
+[build]
+pre-build = [
+  "apt update && apt install -y protobuf-compiler"
+]

atuin.nix

@@ -13,6 +13,7 @@
   Security,
   SystemConfiguration,
   AppKit,
+  protobuf,
 }:
 rustPlatform.buildRustPackage {
   name = "atuin";
@@ -27,7 +28,9 @@ rustPlatform.buildRustPackage {
 
   nativeBuildInputs = [installShellFiles];
 
-  buildInputs = lib.optionals stdenv.isDarwin [libiconv Security SystemConfiguration AppKit];
+  buildInputs = lib.optionals stdenv.isDarwin [libiconv Security SystemConfiguration AppKit protobuf];
+
+  env.PROTOC = lib.getExe' protobuf "protoc";
 
   postInstall = ''
     installShellCompletion --cmd atuin \

crates/atuin-client/Cargo.toml

@@ -13,8 +13,9 @@ repository = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [features]
-default = ["sync"]
+default = ["sync", "daemon"]
 sync = ["urlencoding", "reqwest", "sha2", "hex"]
+daemon = []
 check-update = []
 
 [dependencies]

crates/atuin-client/src/database.rs

@@ -120,6 +120,7 @@ pub trait Database: Send + Sync + 'static {
 
 // Intended for use on a developer machine and not a sync server.
 // TODO: implement IntoIterator
+#[derive(Debug)]
 pub struct Sqlite {
     pub pool: SqlitePool,
 }

crates/atuin-client/src/history.rs

@@ -303,6 +303,48 @@ impl History {
         builder::HistoryCaptured::builder()
     }
 
+    /// Builder for a history entry that is captured via hook, and sent to the daemon.
+    ///
+    /// This builder is used only at the `start` step of the hook,
+    /// so it doesn't have any fields which are known only after
+    /// the command is finished, such as `exit` or `duration`.
+    ///
+    /// It does, however, include information that can usually be inferred.
+    ///
+    /// This is because the daemon we are sending a request to lacks the context of the command
+    ///
+    /// ## Examples
+    /// ```rust
+    /// use atuin_client::history::History;
+    ///
+    /// let history: History = History::daemon()
+    ///     .timestamp(time::OffsetDateTime::now_utc())
+    ///     .command("ls -la")
+    ///     .cwd("/home/user")
+    ///     .session("018deb6e8287781f9973ef40e0fde76b")
+    ///     .hostname("computer:ellie")
+    ///     .build()
+    ///     .into();
+    /// ```
+    ///
+    /// Command without any required info cannot be captured, which is forced at compile time:
+    ///
+    /// ```compile_fail
+    /// use atuin_client::history::History;
+    ///
+    /// // this will not compile because `hostname` is missing
+    /// let history: History = History::daemon()
+    ///     .timestamp(time::OffsetDateTime::now_utc())
+    ///     .command("ls -la")
+    ///     .cwd("/home/user")
+    ///     .session("018deb6e8287781f9973ef40e0fde76b")
+    ///     .build()
+    ///     .into();
+    /// ```
+    pub fn daemon() -> builder::HistoryDaemonCaptureBuilder {
+        builder::HistoryDaemonCapture::builder()
+    }
+
     /// Builder for a history entry that is imported from the database.
     ///
     /// All fields are required, as they are all present in the database.

crates/atuin-client/src/history/builder.rs

@@ -97,3 +97,36 @@ impl From<HistoryFromDb> for History {
         }
     }
 }
+
+/// Builder for a history entry that is captured via hook and sent to the daemon
+///
+/// This builder is similar to Capture, but we just require more information up front.
+/// For the old setup, we could just rely on History::new to read some of the missing
+/// data. This is no longer the case.
+#[derive(Debug, Clone, TypedBuilder)]
+pub struct HistoryDaemonCapture {
+    timestamp: time::OffsetDateTime,
+    #[builder(setter(into))]
+    command: String,
+    #[builder(setter(into))]
+    cwd: String,
+    #[builder(setter(into))]
+    session: String,
+    #[builder(setter(into))]
+    hostname: String,
+}
+
+impl From<HistoryDaemonCapture> for History {
+    fn from(captured: HistoryDaemonCapture) -> Self {
+        History::new(
+            captured.timestamp,
+            captured.command,
+            captured.cwd,
+            -1,
+            -1,
+            Some(captured.session),
+            Some(captured.hostname),
+            None,
+        )
+    }
+}

crates/atuin-client/src/settings.rs

@@ -342,6 +342,19 @@ pub struct Preview {
     pub strategy: PreviewStrategy,
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Daemon {
+    /// Use the daemon to sync
+    /// If enabled, requires a running daemon with `atuin daemon`
+    pub enabled: bool,
+
+    /// The daemon will handle sync on an interval. How often to sync, in seconds.
+    pub sync_frequency: u64,
+
+    /// The path to the unix socket used by the daemon
+    pub socket_path: String,
+}
+
 impl Default for Preview {
     fn default() -> Self {
         Self {
@@ -350,6 +363,16 @@ impl Default for Preview {
     }
 }
 
+impl Default for Daemon {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            sync_frequency: 300,
+            socket_path: "".to_string(),
+        }
+    }
+}
+
 // The preview height strategy also takes max_preview_height into account.
 #[derive(Clone, Debug, Deserialize, Copy, PartialEq, Eq, ValueEnum, Serialize)]
 pub enum PreviewStrategy {
@@ -428,6 +451,9 @@ pub struct Settings {
     #[serde(default)]
     pub dotfiles: dotfiles::Settings,
 
+    #[serde(default)]
+    pub daemon: Daemon,
+
     // This is automatically loaded when settings is created. Do not set in
     // config! Keep secrets and settings apart.
     #[serde(skip)]
@@ -622,6 +648,7 @@ impl Settings {
         let data_dir = atuin_common::utils::data_dir();
         let db_path = data_dir.join("history.db");
         let record_store_path = data_dir.join("records.db");
+        let socket_path = data_dir.join("atuin.sock");
 
         let key_path = data_dir.join("key");
         let session_path = data_dir.join("session");
@@ -643,6 +670,7 @@ impl Settings {
             .set_default("style", "auto")?
             .set_default("inline_height", 0)?
             .set_default("show_preview", true)?
+            .set_default("preview.strategy", "auto")?
             .set_default("max_preview_height", 4)?
             .set_default("show_help", true)?
             .set_default("show_tabs", true)?
@@ -675,6 +703,9 @@ impl Settings {
             .set_default("keymap_cursor", HashMap::<String, String>::new())?
             .set_default("smart_sort", false)?
             .set_default("store_failed", true)?
+            .set_default("daemon.sync_frequency", 300)?
+            .set_default("daemon.enabled", false)?
+            .set_default("daemon.socket_path", socket_path.to_str())?
             .set_default(
                 "prefers_reduced_motion",
                 std::env::var("NO_MOTION")

crates/atuin-daemon/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "atuin-daemon"
+edition = "2021"
+version = "0.1.0"
+authors.workspace = true
+rust-version.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+readme.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+atuin-client = { path = "../atuin-client", version = "18.0.1" }
+
+time = { workspace = true }
+uuid = { workspace = true }
+tokio = { workspace = true }
+tower = { workspace = true }
+eyre = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+
+dashmap = "5.5.3"
+tonic-types = "0.11.0"
+tonic = "0.11"
+prost = "0.12"
+prost-types = "0.12"
+tokio-stream = {version="0.1.14", features=["net"]}
+rand.workspace = true
+
+[build-dependencies]
+tonic-build = "0.11"

crates/atuin-daemon/build.rs

@@ -0,0 +1,4 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    tonic_build::compile_protos("./proto/history.proto")?;
+    Ok(())
+}

crates/atuin-daemon/proto/history.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+package history;
+
+import "google/protobuf/timestamp.proto";
+
+message StartHistoryRequest {
+  // If people are still using my software in ~530 years, they can figure out a u128 migration
+  uint64 timestamp = 1; // nanosecond unix epoch
+  string command = 2;
+  string cwd = 3;
+  string session = 4;
+  string hostname = 5;
+}
+
+message EndHistoryRequest {
+  string id = 1;
+  int64 exit = 2;
+  uint64 duration = 3;
+}
+
+message StartHistoryReply {
+  string id = 1;
+}
+
+message EndHistoryReply {
+  string id = 1;
+  uint64 idx = 2;
+}
+
+service History {
+  rpc StartHistory(StartHistoryRequest) returns (StartHistoryReply);
+  rpc EndHistory(EndHistoryRequest) returns (EndHistoryReply);
+}

crates/atuin-daemon/src/client.rs

@@ -0,0 +1,60 @@
+use eyre::{eyre, Result};
+use tokio::net::UnixStream;
+use tonic::transport::{Channel, Endpoint, Uri};
+use tower::service_fn;
+
+use atuin_client::history::History;
+
+use crate::history::{
+    history_client::HistoryClient as HistoryServiceClient, EndHistoryRequest, StartHistoryRequest,
+};
+
+pub struct HistoryClient {
+    client: HistoryServiceClient<Channel>,
+}
+
+// Wrap the grpc client
+impl HistoryClient {
+    pub async fn new(path: String) -> Result<Self> {
+        let channel = Endpoint::try_from("http://atuin_local_daemon:0")?
+            .connect_with_connector(service_fn(move |_: Uri| {
+                let path = path.to_string();
+
+                UnixStream::connect(path)
+            }))
+            .await
+            .map_err(|_| eyre!("failed to connect to local atuin daemon. Is it running?"))?;
+
+        let client = HistoryServiceClient::new(channel);
+
+        Ok(HistoryClient { client })
+    }
+
+    pub async fn start_history(&mut self, h: History) -> Result<String> {
+        let req = StartHistoryRequest {
+            command: h.command,
+            cwd: h.cwd,
+            hostname: h.hostname,
+            session: h.session,
+            timestamp: h.timestamp.unix_timestamp_nanos() as u64,
+        };
+
+        let resp = self.client.start_history(req).await?;
+
+        Ok(resp.into_inner().id)
+    }
+
+    pub async fn end_history(
+        &mut self,
+        id: String,
+        duration: u64,
+        exit: i64,
+    ) -> Result<(String, u64)> {
+        let req = EndHistoryRequest { id, duration, exit };
+
+        let resp = self.client.end_history(req).await?;
+        let resp = resp.into_inner();
+
+        Ok((resp.id, resp.idx))
+    }
+}

crates/atuin-daemon/src/history.rs

@@ -0,0 +1 @@
+tonic::include_proto!("history");

crates/atuin-daemon/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod client;
+pub mod history;
+pub mod server;

crates/atuin-daemon/src/server.rs

@@ -0,0 +1,186 @@
+use eyre::WrapErr;
+
+use atuin_client::encryption;
+use atuin_client::history::store::HistoryStore;
+use atuin_client::record::sqlite_store::SqliteStore;
+use atuin_client::settings::Settings;
+use std::path::PathBuf;
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tracing::{instrument, Level};
+
+use atuin_client::database::{Database, Sqlite as HistoryDatabase};
+use atuin_client::history::{History, HistoryId};
+use dashmap::DashMap;
+use eyre::Result;
+use tokio::net::UnixListener;
+use tokio_stream::wrappers::UnixListenerStream;
+use tonic::{transport::Server, Request, Response, Status};
+
+use crate::history::history_server::{History as HistorySvc, HistoryServer};
+
+use crate::history::{EndHistoryReply, EndHistoryRequest, StartHistoryReply, StartHistoryRequest};
+
+mod sync;
+
+#[derive(Debug)]
+pub struct HistoryService {
+    // A store for WIP history
+    // This is history that has not yet been completed, aka a command that's current running.
+    running: Arc<DashMap<HistoryId, History>>,
+    store: HistoryStore,
+    history_db: HistoryDatabase,
+}
+
+impl HistoryService {
+    pub fn new(store: HistoryStore, history_db: HistoryDatabase) -> Self {
+        Self {
+            running: Arc::new(DashMap::new()),
+            store,
+            history_db,
+        }
+    }
+}
+
+#[tonic::async_trait()]
+impl HistorySvc for HistoryService {
+    #[instrument(skip_all, level = Level::INFO)]
+    async fn start_history(
+        &self,
+        request: Request<StartHistoryRequest>,
+    ) -> Result<Response<StartHistoryReply>, Status> {
+        let running = self.running.clone();
+        let req = request.into_inner();
+
+        let timestamp =
+            OffsetDateTime::from_unix_timestamp_nanos(req.timestamp as i128).map_err(|_| {
+                Status::invalid_argument(
+                    "failed to parse timestamp as unix time (expected nanos since epoch)",
+                )
+            })?;
+
+        let h: History = History::daemon()
+            .timestamp(timestamp)
+            .command(req.command)
+            .cwd(req.cwd)
+            .session(req.session)
+            .hostname(req.hostname)
+            .build()
+            .into();
+
+        // The old behaviour had us inserting half-finished history records into the database
+        // The new behaviour no longer allows that.
+        // History that's running is stored in-memory by the daemon, and only committed when
+        // complete.
+        // If anyone relied on the old behaviour, we could perhaps insert to the history db here
+        // too. I'd rather keep it pure, unless that ends up being the case.
+        let id = h.id.clone();
+        tracing::info!(id = id.to_string(), "start history");
+        running.insert(id.clone(), h);
+
+        let reply = StartHistoryReply { id: id.to_string() };
+
+        Ok(Response::new(reply))
+    }
+
+    #[instrument(skip_all, level = Level::INFO)]
+    async fn end_history(
+        &self,
+        request: Request<EndHistoryRequest>,
+    ) -> Result<Response<EndHistoryReply>, Status> {
+        let running = self.running.clone();
+        let req = request.into_inner();
+
+        let id = HistoryId(req.id);
+
+        if let Some((_, mut history)) = running.remove(&id) {
+            history.exit = req.exit;
+            history.duration = match req.duration {
+                0 => i64::try_from(
+                    (OffsetDateTime::now_utc() - history.timestamp).whole_nanoseconds(),
+                )
+                .expect("failed to convert calculated duration to i64"),
+                value => i64::try_from(value).expect("failed to get i64 duration"),
+            };
+
+            // Perhaps allow the incremental build to handle this entirely.
+            self.history_db
+                .save(&history)
+                .await
+                .map_err(|e| Status::internal(format!("failed to write to db: {e:?}")))?;
+
+            tracing::info!(
+                id = id.0.to_string(),
+                duration = history.duration,
+                "end history"
+            );
+
+            let (id, idx) =
+                self.store.push(history).await.map_err(|e| {
+                    Status::internal(format!("failed to push record to store: {e:?}"))
+                })?;
+
+            let reply = EndHistoryReply {
+                id: id.0.to_string(),
+                idx,
+            };
+
+            return Ok(Response::new(reply));
+        }
+
+        Err(Status::not_found(format!(
+            "could not find history with id: {id}"
+        )))
+    }
+}
+
+async fn shutdown_signal(socket: PathBuf) {
+    let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+        .expect("failed to register sigterm handler");
+    let mut int = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
+        .expect("failed to register sigint handler");
+
+    tokio::select! {
+        _  = term.recv() => {},
+        _  = int.recv() => {},
+    }
+
+    eprintln!("Removing socket...");
+    std::fs::remove_file(socket).expect("failed to remove socket");
+    eprintln!("Shutting down...");
+}
+
+// break the above down when we end up with multiple services
+
+/// Listen on a unix socket
+/// Pass the path to the socket
+pub async fn listen(
+    settings: Settings,
+    store: SqliteStore,
+    history_db: HistoryDatabase,
+) -> Result<()> {
+    let encryption_key: [u8; 32] = encryption::load_key(&settings)
+        .context("could not load encryption key")?
+        .into();
+
+    let host_id = Settings::host_id().expect("failed to get host_id");
+    let history_store = HistoryStore::new(store.clone(), host_id, encryption_key);
+
+    let history = HistoryService::new(history_store, history_db);
+
+    let socket = settings.daemon.socket_path.clone();
+    let uds = UnixListener::bind(socket.clone())?;
+    let uds_stream = UnixListenerStream::new(uds);
+
+    tracing::info!("listening on unix socket {:?}", socket);
+
+    // start services
+    tokio::spawn(sync::worker(settings.clone(), store));
+
+    Server::builder()
+        .add_service(HistoryServer::new(history))
+        .serve_with_incoming_shutdown(uds_stream, shutdown_signal(socket.into()))
+        .await?;
+
+    Ok(())
+}

crates/atuin-daemon/src/server/sync.rs

@@ -0,0 +1,55 @@
+use eyre::Result;
+use rand::Rng;
+use tokio::time::{self, MissedTickBehavior};
+
+use atuin_client::{
+    record::{sqlite_store::SqliteStore, sync},
+    settings::Settings,
+};
+
+pub async fn worker(settings: Settings, store: SqliteStore) -> Result<()> {
+    tracing::info!("booting sync worker");
+
+    let mut ticker = time::interval(time::Duration::from_secs(settings.daemon.sync_frequency));
+
+    // IMPORTANT: without this, if we miss ticks because a sync takes ages or is otherwise delayed,
+    // we may end up running a lot of syncs in a hot loop. No bueno!
+    ticker.set_missed_tick_behavior(MissedTickBehavior::Skip);
+
+    loop {
+        ticker.tick().await;
+        tracing::info!("sync worker tick");
+
+        let res = sync::sync(&settings, &store).await;
+
+        if let Err(e) = res {
+            tracing::error!("sync tick failed with {e}");
+            let mut rng = rand::thread_rng();
+
+            let new_interval = ticker.period().as_secs_f64() * rng.gen_range(2.0..2.2);
+
+            // Don't backoff by more than 30 mins
+            if new_interval > 60.0 * 30.0 {
+                continue;
+            }
+
+            ticker = time::interval(time::Duration::from_secs(new_interval as u64));
+            ticker.reset_after(time::Duration::from_secs(new_interval as u64));
+
+            tracing::error!("backing off, next sync tick in {new_interval}");
+        } else {
+            let (uploaded, downloaded) = res.unwrap();
+
+            tracing::info!(
+                uploaded = ?uploaded,
+                downloaded = ?downloaded,
+                "sync complete"
+            );
+
+            // Reset backoff on success
+            if ticker.period().as_secs() != settings.daemon.sync_frequency {
+                ticker = time::interval(time::Duration::from_secs(settings.daemon.sync_frequency));
+            }
+        }
+    }
+}

crates/atuin-server/Cargo.toml

@@ -28,7 +28,7 @@ async-trait = { workspace = true }
 axum = "0.7.4"
 axum-server = { version = "0.6.0", features = ["tls-rustls"] }
 fs-err = { workspace = true }
-tower = "0.4"
+tower = { workspace = true }
 tower-http = { version = "0.5.1", features = ["trace"] }
 reqwest = { workspace = true }
 rustls = "0.21"

crates/atuin/Cargo.toml

@@ -33,10 +33,11 @@ buildflags = ["--release"]
 atuin = { path = "/usr/bin/atuin" }
 
 [features]
-default = ["client", "sync", "server", "clipboard", "check-update"]
+default = ["client", "sync", "server", "clipboard", "check-update", "daemon"]
 client = ["atuin-client"]
 sync = ["atuin-client/sync"]
-server = ["atuin-server", "atuin-server-postgres", "tracing-subscriber"]
+daemon = ["atuin-client/daemon"]
+server = ["atuin-server", "atuin-server-postgres"]
 clipboard = ["cli-clipboard"]
 check-update = ["atuin-client/check-update"]
 
@@ -47,6 +48,7 @@ atuin-client = { path = "../atuin-client", version = "18.2.0", optional = true,
 atuin-common = { path = "../atuin-common", version = "18.2.0" }
 atuin-dotfiles = { path = "../atuin-dotfiles", version = "0.2.0" }
 atuin-history = { path = "../atuin-history", version = "0.1.0" }
+atuin-daemon = { path = "../atuin-daemon", version = "0.1.0" }
 
 log = { workspace = true }
 env_logger = "0.11.2"
@@ -78,6 +80,7 @@ fuzzy-matcher = "0.3.7"
 colored = "2.0.4"
 ratatui = "0.26"
 tracing = "0.1"
+tracing-subscriber = { workspace = true }
 uuid = { workspace = true }
 unicode-segmentation = "1.11.0"
 serde_yaml = "0.9.32"
@@ -87,11 +90,5 @@ regex="1.10.4"
 [target.'cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))'.dependencies]
 cli-clipboard = { version = "0.4.0", optional = true }
 
-[dependencies.tracing-subscriber]
-version = "0.3"
-default-features = false
-features = ["ansi", "fmt", "registry", "env-filter"]
-optional = true
-
 [dev-dependencies]
 tracing-tree = "0.3"

crates/atuin/src/command/client.rs

@@ -4,7 +4,7 @@ use clap::Subcommand;
 use eyre::{Result, WrapErr};
 
 use atuin_client::{database::Sqlite, record::sqlite_store::SqliteStore, settings::Settings};
-use env_logger::Builder;
+use tracing_subscriber::{filter::EnvFilter, fmt, prelude::*};
 
 #[cfg(feature = "sync")]
 mod sync;
@@ -12,6 +12,9 @@ mod sync;
 #[cfg(feature = "sync")]
 mod account;
 
+#[cfg(feature = "daemon")]
+mod daemon;
+
 mod default_config;
 mod doctor;
 mod dotfiles;
@@ -73,6 +76,10 @@ pub enum Cmd {
     #[command()]
     Doctor,
 
+    #[cfg(feature = "daemon")]
+    #[command()]
+    Daemon,
+
     /// Print example configuration
     #[command()]
     DefaultConfig,
@@ -94,10 +101,12 @@ impl Cmd {
     }
 
     async fn run_inner(self, mut settings: Settings) -> Result<()> {
-        Builder::new()
+        let filter =
-            .filter_level(log::LevelFilter::Off)
+            EnvFilter::from_env("ATUIN_LOG").add_directive("sqlx_sqlite::regexp=off".parse()?);
-            .filter_module("sqlx_sqlite::regexp", log::LevelFilter::Off)
+
-            .parse_env("ATUIN_LOG")
+        tracing_subscriber::registry()
+            .with(fmt::layer())
+            .with(filter)
             .init();
 
         tracing::trace!(command = ?self, "client command");
@@ -139,6 +148,9 @@ impl Cmd {
                 default_config::run();
                 Ok(())
             }
+
+            #[cfg(feature = "daemon")]
+            Self::Daemon => daemon::run(settings, sqlite_store, db).await,
         }
     }
 }

crates/atuin/src/command/client/daemon.rs

@@ -0,0 +1,10 @@
+use eyre::Result;
+
+use atuin_client::{database::Sqlite, record::sqlite_store::SqliteStore, settings::Settings};
+use atuin_daemon::server::listen;
+
+pub async fn run(settings: Settings, store: SqliteStore, history_db: Sqlite) -> Result<()> {
+    listen(settings, store, history_db).await?;
+
+    Ok(())
+}

crates/atuin/src/command/client/history.rs

@@ -314,6 +314,20 @@ impl Cmd {
             return Ok(());
         }
 
+        if settings.daemon.enabled {
+            let resp =
+                atuin_daemon::client::HistoryClient::new(settings.daemon.socket_path.clone())
+                    .await?
+                    .start_history(h)
+                    .await?;
+
+            // print the ID
+            // we use this as the key for calling end
+            println!("{resp}");
+
+            return Ok(());
+        }
+
         // print the ID
         // we use this as the key for calling end
         println!("{}", h.id);
@@ -332,6 +346,18 @@ impl Cmd {
         exit: i64,
         duration: Option<u64>,
     ) -> Result<()> {
+        // If the daemon is enabled, use it. Ignore the rest.
+        // We will need to keep the old code around for a while.
+        // At the very least, while this is opt-in
+        if settings.daemon.enabled {
+            atuin_daemon::client::HistoryClient::new(settings.daemon.socket_path.clone())
+                .await?
+                .end_history(id.to_string(), duration.unwrap_or(0), exit)
+                .await?;
+
+            return Ok(());
+        }
+
         if id.trim() == "" {
             return Ok(());
         }

ui/.gitignore

@@ -22,5 +22,6 @@ dist-ssr
 *.njsproj
 *.sln
 *.sw?
+.vite
 
 gen