SQLite History MVP with timestamp, duration, working directory, exit status metadata (#5721)

This PR adds support for an SQLite history via nushell/reedline#401

The SQLite history is enabled by setting history_file_format: "sqlite" in config.nu.

* somewhat working sqlite history
* Hook up history command
* Fix error in SQlitebacked with empty lines

When entering an empty line there previously was the "No command run"
error with `SqliteBackedHistory` during addition of the metadata

May be considered a temporary fix

Co-authored-by: sholderbach <sholderbach@users.noreply.github.com>

crates/nu-cli/Cargo.toml

@@ -17,7 +17,7 @@ nu-parser = { path = "../nu-parser", version = "0.63.1"  }
 nu-protocol = { path = "../nu-protocol", version = "0.63.1"  }
 nu-utils = { path = "../nu-utils", version = "0.63.1"  }
 nu-ansi-term = "0.46.0"
-reedline = { git = "https://github.com/nushell/reedline", branch = "main", features = ["bashisms"]}
+reedline = { version = "0.6.0", features = ["bashisms", "sqlite"]}
 nu-color-config = { path = "../nu-color-config", version = "0.63.1"  }
 crossterm = "0.23.0"
 miette = { version = "4.5.0", features = ["fancy"] }
@@ -26,6 +26,8 @@ fuzzy-matcher = "0.3.7"
 
 log = "0.4"
 is_executable = "1.0.1"
+chrono = "0.4.19"
+sysinfo = "0.24.1"
 
 [features]
 plugin = []

crates/nu-cli/src/config_files.rs

@@ -2,12 +2,15 @@ use crate::util::{eval_source, report_error};
 #[cfg(feature = "plugin")]
 use log::info;
 use nu_protocol::engine::{EngineState, Stack, StateDelta, StateWorkingSet};
-use nu_protocol::{PipelineData, Span};
+use nu_protocol::{HistoryFileFormat, PipelineData, Span};
 use std::path::PathBuf;
 
 #[cfg(feature = "plugin")]
 const PLUGIN_FILE: &str = "plugin.nu";
 
+const HISTORY_FILE_TXT: &str = "history.txt";
+const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
+
 #[cfg(feature = "plugin")]
 pub fn read_plugin_file(
     engine_state: &mut EngineState,
@@ -84,3 +87,14 @@ pub fn eval_config_contents(
         }
     }
 }
+
+pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> Option<PathBuf> {
+    nu_path::config_dir().map(|mut history_path| {
+        history_path.push(storage_path);
+        history_path.push(match mode {
+            HistoryFileFormat::PlainText => HISTORY_FILE_TXT,
+            HistoryFileFormat::Sqlite => HISTORY_FILE_SQLITE,
+        });
+        history_path
+    })
+}

crates/nu-cli/src/repl.rs

@@ -12,12 +12,12 @@ use nu_engine::{convert_env_values, eval_block};
 use nu_parser::lex;
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    BlockId, PipelineData, PositionalArg, ShellError, Span, Value,
+    BlockId, HistoryFileFormat, PipelineData, PositionalArg, ShellError, Span, Value,
 };
-use reedline::{DefaultHinter, Emacs, Vi};
+use reedline::{DefaultHinter, Emacs, SqliteBackedHistory, Vi};
 use std::io::{self, Write};
-use std::path::PathBuf;
 use std::{sync::atomic::Ordering, time::Instant};
+use sysinfo::SystemExt;
 
 const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
 const PRE_EXECUTE_MARKER: &str = "\x1b]133;C\x1b\\";
@@ -27,7 +27,7 @@ const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
 pub fn evaluate_repl(
     engine_state: &mut EngineState,
     stack: &mut Stack,
-    history_path: Option<PathBuf>,
+    nushell_path: &str,
     is_perf_true: bool,
 ) -> Result<()> {
     use reedline::{FileBackedHistory, Reedline, Signal};
@@ -85,20 +85,32 @@ pub fn evaluate_repl(
         info!("setup reedline {}:{}:{}", file!(), line!(), column!());
     }
     let mut line_editor = Reedline::create();
+    let history_path = crate::config_files::get_history_path(
+        nushell_path,
+        engine_state.config.history_file_format,
+    );
     if let Some(history_path) = history_path.as_deref() {
         if is_perf_true {
             info!("setup history {}:{}:{}", file!(), line!(), column!());
         }
-        let history = Box::new(
-            FileBackedHistory::with_file(
-                config.max_history_size as usize,
-                history_path.to_path_buf(),
-            )
-            .into_diagnostic()?,
-        );
+
+        let history: Box<dyn reedline::History> = match engine_state.config.history_file_format {
+            HistoryFileFormat::PlainText => Box::new(
+                FileBackedHistory::with_file(
+                    config.max_history_size as usize,
+                    history_path.to_path_buf(),
+                )
+                .into_diagnostic()?,
+            ),
+            HistoryFileFormat::Sqlite => Box::new(
+                SqliteBackedHistory::with_file(history_path.to_path_buf()).into_diagnostic()?,
+            ),
+        };
         line_editor = line_editor.with_history(history);
     };
 
+    let sys = sysinfo::System::new();
+
     loop {
         if is_perf_true {
             info!(
@@ -300,6 +312,20 @@ pub fn evaluate_repl(
 
         match input {
             Ok(Signal::Success(s)) => {
+                let history_supports_meta =
+                    matches!(config.history_file_format, HistoryFileFormat::Sqlite);
+                if history_supports_meta && !s.is_empty() {
+                    line_editor
+                        .update_last_command_context(&|mut c| {
+                            c.start_timestamp = Some(chrono::Utc::now());
+                            c.hostname = sys.host_name();
+
+                            c.cwd = Some(StateWorkingSet::new(engine_state).get_cwd());
+                            c
+                        })
+                        .into_diagnostic()?; // todo: don't stop repl if error here?
+                }
+
                 // Right before we start running the code the user gave us,
                 // fire the "pre_execution" hook
                 if let Some(hook) = &config.hooks.pre_execution {
@@ -401,11 +427,12 @@ pub fn evaluate_repl(
                         PipelineData::new(Span::new(0, 0)),
                     );
                 }
+                let cmd_duration = start_time.elapsed();
 
                 stack.add_env_var(
                     "CMD_DURATION_MS".into(),
                     Value::String {
-                        val: format!("{}", start_time.elapsed().as_millis()),
+                        val: format!("{}", cmd_duration.as_millis()),
                         span: Span { start: 0, end: 0 },
                     },
                 );
@@ -418,6 +445,18 @@ pub fn evaluate_repl(
                     engine_state.add_env_var("PWD".into(), cwd);
                 }
 
+                if history_supports_meta && !s.is_empty() {
+                    line_editor
+                        .update_last_command_context(&|mut c| {
+                            c.duration = Some(cmd_duration);
+                            c.exit_status = stack
+                                .get_env_var(engine_state, "LAST_EXIT_CODE")
+                                .and_then(|e| e.as_i64().ok());
+                            c
+                        })
+                        .into_diagnostic()?; // todo: don't stop repl if error here?
+                }
+
                 if shell_integration {
                     // FIXME: use variant with exit code, if apropriate
                     run_ansi_sequence(CMD_FINISHED_MARKER)?;

crates/nu-command/Cargo.toml

@@ -83,7 +83,7 @@ unicode-segmentation = "1.8.0"
 url = "2.2.1"
 uuid = { version = "0.8.2", features = ["v4"] }
 which = { version = "4.2.2", optional = true }
-reedline = { git = "https://github.com/nushell/reedline", branch = "main", features = ["bashisms"]}
+reedline = { version = "0.6.0", features = ["bashisms", "sqlite"]}
 wax = { version =  "0.4.0", features = ["diagnostics"] }
 rusqlite = { version = "0.27.0", features = ["bundled"], optional = true }
 sqlparser = { version = "0.16.0", features = ["serde"], optional = true }

crates/nu-command/src/core_commands/history.rs

@@ -1,14 +1,13 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
-    Category, Example, IntoInterruptiblePipelineData, PipelineData, ShellError, Signature, Value,
+    Category, Example, HistoryFileFormat, IntoInterruptiblePipelineData, PipelineData, ShellError,
+    Signature, Value,
+};
+use reedline::{
+    FileBackedHistory, History as ReedlineHistory, SearchDirection, SearchQuery,
+    SqliteBackedHistory,
 };
-
-const NEWLINE_ESCAPE_CODE: &str = "<\\n>";
-
-fn decode_newlines(escaped: &str) -> String {
-    escaped.replace(NEWLINE_ESCAPE_CODE, "\n")
-}
 
 #[derive(Clone)]
 pub struct History;
@@ -36,44 +35,74 @@ impl Command for History {
         _input: PipelineData,
     ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
         let head = call.head;
+        // todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
         if let Some(config_path) = nu_path::config_dir() {
             let clear = call.has_flag("clear");
             let ctrlc = engine_state.ctrlc.clone();
 
             let mut history_path = config_path;
             history_path.push("nushell");
-            history_path.push("history.txt");
+            match engine_state.config.history_file_format {
+                HistoryFileFormat::Sqlite => {
+                    history_path.push("history.sqlite3");
+                }
+                HistoryFileFormat::PlainText => {
+                    history_path.push("history.txt");
+                }
+            }
 
             if clear {
                 let _ = std::fs::remove_file(history_path);
+                // TODO: FIXME also clear the auxiliary files when using sqlite
                 Ok(PipelineData::new(head))
             } else {
-                let contents = std::fs::read_to_string(history_path);
+                let history_reader: Option<Box<dyn ReedlineHistory>> =
+                    match engine_state.config.history_file_format {
+                        HistoryFileFormat::Sqlite => SqliteBackedHistory::with_file(history_path)
+                            .map(|inner| {
+                                let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                                boxed
+                            })
+                            .ok(),
 
-                if let Ok(contents) = contents {
-                    Ok(contents
-                        .lines()
-                        .enumerate()
-                        .map(move |(index, command)| Value::Record {
-                            cols: vec!["command".to_string(), "index".to_string()],
-                            vals: vec![
-                                Value::String {
-                                    val: decode_newlines(command),
-                                    span: head,
-                                },
-                                Value::Int {
-                                    val: index as i64,
-                                    span: head,
-                                },
-                            ],
-                            span: head,
+                        HistoryFileFormat::PlainText => FileBackedHistory::with_file(
+                            engine_state.config.max_history_size as usize,
+                            history_path,
+                        )
+                        .map(|inner| {
+                            let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                            boxed
                         })
-                        .collect::<Vec<_>>()
-                        .into_iter()
-                        .into_pipeline_data(ctrlc))
-                } else {
-                    Err(ShellError::FileNotFound(head))
-                }
+                        .ok(),
+                    };
+
+                let data = history_reader
+                    .and_then(|h| {
+                        h.search(SearchQuery::everything(SearchDirection::Forward))
+                            .ok()
+                    })
+                    .map(move |entries| {
+                        entries
+                            .into_iter()
+                            .enumerate()
+                            .map(move |(idx, entry)| Value::Record {
+                                cols: vec!["command".to_string(), "index".to_string()],
+                                vals: vec![
+                                    Value::String {
+                                        val: entry.command_line,
+                                        span: head,
+                                    },
+                                    Value::Int {
+                                        val: idx as i64,
+                                        span: head,
+                                    },
+                                ],
+                                span: head,
+                            })
+                    })
+                    .ok_or(ShellError::FileNotFound(head))?
+                    .into_pipeline_data(ctrlc);
+                Ok(data)
             }
         } else {
             Err(ShellError::FileNotFound(head))

crates/nu-engine/src/eval.rs

@@ -1239,6 +1239,7 @@ pub fn eval_variable(
                 let mut history_path = config_path.clone();
 
                 history_path.push("history.txt");
+                // let mut history_path = config_files::get_history_path(); // todo: this should use the get_history_path method but idk where to put that function
 
                 output_cols.push("history-path".into());
                 output_vals.push(Value::String {

crates/nu-protocol/src/config.rs

@@ -66,6 +66,7 @@ pub struct Config {
     pub edit_mode: String,
     pub max_history_size: i64,
     pub sync_history_on_enter: bool,
+    pub history_file_format: HistoryFileFormat,
     pub log_level: String,
     pub keybindings: Vec<ParsedKeybinding>,
     pub menus: Vec<ParsedMenu>,
@@ -98,6 +99,7 @@ impl Default for Config {
             edit_mode: "emacs".into(),
             max_history_size: i64::MAX,
             sync_history_on_enter: true,
+            history_file_format: HistoryFileFormat::PlainText,
             log_level: String::new(),
             keybindings: Vec::new(),
             menus: Vec::new(),
@@ -125,6 +127,14 @@ pub enum FooterMode {
     Auto,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, Copy)]
+pub enum HistoryFileFormat {
+    /// Store history as an SQLite database with additional context
+    Sqlite,
+    /// store history as a plain text file where every line is one command (without any context such as timestamps)
+    PlainText,
+}
+
 impl Value {
     pub fn into_config(self) -> Result<Config, ShellError> {
         let v = self.as_record();
@@ -248,6 +258,23 @@ impl Value {
                             eprintln!("$config.edit_mode is not a string")
                         }
                     }
+                    "history_file_format" => {
+                        if let Ok(b) = value.as_string() {
+                            let val_str = b.to_lowercase();
+                            config.history_file_format = match val_str.as_ref() {
+                                "sqlite" => HistoryFileFormat::Sqlite,
+                                "plaintext" => HistoryFileFormat::PlainText,
+                                _ => {
+                                    eprintln!(
+                                        "unrecognized $config.history_file_format '{val_str}'"
+                                    );
+                                    HistoryFileFormat::PlainText
+                                }
+                            };
+                        } else {
+                            eprintln!("$config.history_file_format is not a string")
+                        }
+                    }
                     "max_history_size" => {
                         if let Ok(i) = value.as_i64() {
                             config.max_history_size = i;