Refactor nu-cli/env* (#3041)

* Revert "History, more test coverage improvements, and refactorings. (#3217)"

This reverts commit 8fc8fc89aa.

* Add tests

* Refactor .nu-env

* Change logic of Config write to logic of read()

* Fix reload always appends to old vars

* Fix reload always takes last_modified of global config

* Add reload_config in evaluation context

* Reload config after writing to it in cfg set / cfg set_into

* Add --no-history to cli options

* Use --no-history in tests

* Add comment about maybe_print_errors

* Get ctrl_exit var from context.global_config

* Use context.global_config in command "config"

* Add Readme in engine how env vars are now handled

* Update docs from autoenv command

* Move history_path from engine to nu_data

* Move load history out of if

* No let before return

* Add import for indexmap

crates/nu-data/src/config.rs

@@ -1,18 +1,23 @@
 mod conf;
+mod config_trust;
+mod local_config;
 mod nuconfig;
 pub mod path;
 
-pub mod tests;
-
 pub use conf::Conf;
+pub use config_trust::is_file_trusted;
+pub use config_trust::read_trusted;
+pub use config_trust::Trusted;
+pub use local_config::loadable_cfg_exists_in_dir;
+pub use local_config::LocalConfigDiff;
 pub use nuconfig::NuConfig;
 
 use indexmap::IndexMap;
 use log::trace;
 use nu_errors::{CoerceInto, ShellError};
 use nu_protocol::{
-    Dictionary, Primitive, ShellTypeName, TaggedDictBuilder, UnspannedPathMember, UntaggedValue,
-    Value,
+    ConfigPath, Dictionary, Primitive, ShellTypeName, TaggedDictBuilder, UnspannedPathMember,
+    UntaggedValue, Value,
 };
 use nu_source::{SpannedItem, Tag, TaggedItem};
 use std::fs::{self, OpenOptions};
@@ -186,7 +191,7 @@ pub fn default_path_for(file: &Option<PathBuf>) -> Result<PathBuf, ShellError> {
     let file: &Path = file
         .as_ref()
         .map(AsRef::as_ref)
-        .unwrap_or_else(|| self::path::DEFAULT_CONFIG_LOCATION.as_ref());
+        .unwrap_or_else(|| "config.toml".as_ref());
     filename.push(file);
 
     Ok(filename)
@@ -297,14 +302,11 @@ pub fn config(tag: impl Into<Tag>) -> Result<IndexMap<String, Value>, ShellError
 }
 
 pub fn write(config: &IndexMap<String, Value>, at: &Option<PathBuf>) -> Result<(), ShellError> {
-    let filename = &mut default_path()?;
+    let filename = default_path()?;
+
     let filename = match at {
         None => filename,
-        Some(file) => {
-            filename.pop();
-            filename.push(file);
-            filename
-        }
+        Some(ref file) => file.clone(),
     };
 
     let contents = value_to_toml_value(
@@ -325,3 +327,7 @@ fn touch(path: &Path) -> io::Result<()> {
         Err(e) => Err(e),
     }
 }
+
+pub fn cfg_path_to_scope_tag(cfg_path: &ConfigPath) -> String {
+    cfg_path.get_path().to_string_lossy().to_string()
+}

crates/nu-data/src/config/conf.rs

@@ -1,22 +1,17 @@
+use nu_errors::ShellError;
 use nu_protocol::Value;
-use std::any::Any;
-use std::fmt::Debug;
+use std::{fmt::Debug, path::PathBuf};
 
 pub trait Conf: Debug + Send {
-    fn as_any(&self) -> &dyn Any;
     fn is_modified(&self) -> Result<bool, Box<dyn std::error::Error>>;
     fn var(&self, key: &str) -> Option<Value>;
     fn env(&self) -> Option<Value>;
-    fn path(&self) -> Option<Value>;
-    fn reload(&mut self);
+    fn path(&self) -> Result<Option<Vec<PathBuf>>, ShellError>;
     fn clone_box(&self) -> Box<dyn Conf>;
+    fn reload(&mut self);
 }
 
 impl Conf for Box<dyn Conf> {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn is_modified(&self) -> Result<bool, Box<dyn std::error::Error>> {
         (**self).is_modified()
     }
@@ -29,10 +24,6 @@ impl Conf for Box<dyn Conf> {
         (**self).env()
     }
 
-    fn path(&self) -> Option<Value> {
-        (**self).path()
-    }
-
     fn reload(&mut self) {
         (**self).reload();
     }
@@ -40,4 +31,8 @@ impl Conf for Box<dyn Conf> {
     fn clone_box(&self) -> Box<dyn Conf> {
         (**self).clone_box()
     }
+
+    fn path(&self) -> Result<Option<Vec<PathBuf>>, ShellError> {
+        (**self).path()
+    }
 }

crates/nu-data/src/config/config_trust.rs

@@ -0,0 +1,44 @@
+use serde::Deserialize;
+use serde::Serialize;
+use sha2::{Digest, Sha256};
+use std::{io::Read, path::Path, path::PathBuf};
+
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+
+#[derive(Deserialize, Serialize, Debug, Default)]
+pub struct Trusted {
+    pub files: IndexMap<String, Vec<u8>>,
+}
+
+impl Trusted {
+    pub fn new() -> Self {
+        Trusted {
+            files: IndexMap::new(),
+        }
+    }
+}
+
+pub fn is_file_trusted(nu_env_file: &Path, content: &[u8]) -> Result<bool, ShellError> {
+    let contentdigest = Sha256::digest(&content).as_slice().to_vec();
+    let nufile = std::fs::canonicalize(nu_env_file)?;
+
+    let trusted = read_trusted()?;
+    Ok(trusted.files.get(&nufile.to_string_lossy().to_string()) == Some(&contentdigest))
+}
+
+pub fn read_trusted() -> Result<Trusted, ShellError> {
+    let config_path = crate::config::default_path_for(&Some(PathBuf::from("nu-env.toml")))?;
+
+    let mut file = std::fs::OpenOptions::new()
+        .read(true)
+        .create(true)
+        .write(true)
+        .open(config_path)
+        .map_err(|_| ShellError::untagged_runtime_error("Couldn't open nu-env.toml"))?;
+    let mut doc = String::new();
+    file.read_to_string(&mut doc)?;
+
+    let allowed = toml::de::from_str(doc.as_str()).unwrap_or_else(|_| Trusted::new());
+    Ok(allowed)
+}

crates/nu-data/src/config/local_config.rs

@@ -0,0 +1,150 @@
+use nu_errors::ShellError;
+use std::path::{Path, PathBuf};
+
+static LOCAL_CFG_FILE_NAME: &str = ".nu-env";
+
+pub struct LocalConfigDiff {
+    pub cfgs_to_load: Vec<PathBuf>,
+    pub cfgs_to_unload: Vec<PathBuf>,
+}
+
+/// Finds all local configs between `from` up to `to`.
+/// Every config seen while going up the filesystem (e.G. from `/foo` to `/foo/bar`) is returned
+/// as a config to load
+/// Every config seen while going down the filesystem (e.G. from `/foo/bar` to `/foo/bar`) is
+/// returned as a config to unload
+/// If both paths are unrelated to each other, (e.G. windows paths as: `C:/foo` and `D:/bar`)
+/// this function first walks `from` completly down the filesystem and then it walks up until `to`.
+///
+/// Both paths are required to be absolute.
+impl LocalConfigDiff {
+    pub fn between(from: PathBuf, to: PathBuf) -> (LocalConfigDiff, Vec<ShellError>) {
+        let common_prefix = common_path::common_path(&from, &to);
+        let (cfgs_to_unload, err_down) = walk_down(&from, &common_prefix);
+        let (cfgs_to_load, err_up) = walk_up(&common_prefix, &to);
+
+        (
+            LocalConfigDiff {
+                cfgs_to_load,
+                cfgs_to_unload,
+            },
+            err_down.into_iter().chain(err_up).collect(),
+        )
+    }
+}
+
+///Walks from the first parameter down the filesystem to the second parameter. Marking all
+///configs found in directories on the way as to remove.
+///If to is None, this method walks from the first paramter down to the beginning of the
+///filesystem
+///Returns tuple of (configs to remove, errors from io).
+fn walk_down(
+    from_inclusive: &Path,
+    to_exclusive: &Option<PathBuf>,
+) -> (Vec<PathBuf>, Vec<ShellError>) {
+    let mut all_err = vec![];
+    let mut all_cfgs_to_unload = vec![];
+    for dir in from_inclusive.ancestors().take_while(|cur_path| {
+        if let Some(until_path) = to_exclusive {
+            //Stop before `to_exclusive`
+            *cur_path != until_path
+        } else {
+            //No end, walk all the way down
+            true
+        }
+    }) {
+        match local_cfg_should_be_unloaded(dir.to_path_buf()) {
+            Ok(Some(cfg)) => all_cfgs_to_unload.push(cfg),
+            Err(e) => all_err.push(e),
+            _ => {}
+        }
+    }
+
+    (all_cfgs_to_unload, all_err)
+}
+
+///Walks from the first parameter up the filesystem to the second parameter, returns all configs
+///found in directories on the way to load.
+///Returns combined errors from checking directories on the way
+///If from is None, this method walks from the beginning of the second parameter up to the
+///second parameter
+fn walk_up(
+    from_exclusive: &Option<PathBuf>,
+    to_inclusive: &Path,
+) -> (Vec<PathBuf>, Vec<ShellError>) {
+    let mut all_err = vec![];
+    let mut all_cfgs_to_load = vec![];
+
+    //skip all paths until (inclusive) from (or 0 if from is None)
+    let skip_ahead = from_exclusive
+        .as_ref()
+        .map(|p| p.ancestors().count())
+        .unwrap_or(0);
+    //We have to traverse ancestors in reverse order (apply lower directories first)
+    //ancestors() does not yield iter with .rev() method. So we store all ancestors
+    //and then iterate over the vec
+    let dirs: Vec<_> = to_inclusive.ancestors().map(Path::to_path_buf).collect();
+    for dir in dirs.iter().rev().skip(skip_ahead) {
+        match loadable_cfg_exists_in_dir(dir.clone()) {
+            Ok(Some(cfg)) => all_cfgs_to_load.push(cfg),
+            Err(e) => all_err.push(e),
+            _ => {}
+        }
+    }
+
+    (all_cfgs_to_load, all_err)
+}
+
+fn is_existent_local_cfg(cfg_file_path: &Path) -> Result<bool, ShellError> {
+    if !cfg_file_path.exists() || cfg_file_path.parent() == super::default_path()?.parent() {
+        //Don't treat global cfg as local one
+        Ok(false)
+    } else {
+        Ok(true)
+    }
+}
+
+fn is_trusted_local_cfg_content(cfg_file_path: &Path, content: &[u8]) -> Result<bool, ShellError> {
+    //This checks whether user used `autoenv trust` to mark this cfg as secure
+    if !super::is_file_trusted(&cfg_file_path, &content)? {
+        //Notify user about present config, but not trusted
+        Err(ShellError::untagged_runtime_error(
+                format!("{:?} is untrusted. Run 'autoenv trust {:?}' to trust it.\nThis needs to be done after each change to the file.",
+                    cfg_file_path, cfg_file_path.parent().unwrap_or_else(|| &Path::new("")))))
+    } else {
+        Ok(true)
+    }
+}
+
+fn local_cfg_should_be_unloaded<P: AsRef<Path>>(cfg_dir: P) -> Result<Option<PathBuf>, ShellError> {
+    let mut cfg = cfg_dir.as_ref().to_path_buf();
+    cfg.push(LOCAL_CFG_FILE_NAME);
+    if is_existent_local_cfg(&cfg)? {
+        //No need to compute whether content is good. If it is not loaded before, unloading does
+        //nothing
+        Ok(Some(cfg))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Checks whether a local_cfg exists in cfg_dir and returns:
+/// Ok(Some(cfg_path)) if cfg exists and is good to load
+/// Ok(None) if no cfg exists
+/// Err(error) if cfg exits, but is not good to load
+pub fn loadable_cfg_exists_in_dir(mut cfg_dir: PathBuf) -> Result<Option<PathBuf>, ShellError> {
+    cfg_dir.push(LOCAL_CFG_FILE_NAME);
+    let cfg_path = cfg_dir;
+
+    if !is_existent_local_cfg(&cfg_path)? {
+        return Ok(None);
+    }
+
+    let content = std::fs::read(&cfg_path)?;
+
+    if !is_trusted_local_cfg_content(&cfg_path, &content)? {
+        return Ok(None);
+    }
+
+    Ok(Some(cfg_path))
+}

crates/nu-data/src/config/nuconfig.rs

@@ -1,23 +1,18 @@
 use crate::config::{last_modified, read, Conf, Status};
 use indexmap::IndexMap;
+use nu_errors::ShellError;
 use nu_protocol::Value;
 use nu_source::Tag;
-use std::any::Any;
-use std::fmt::Debug;
-use std::path::PathBuf;
+use std::{fmt::Debug, path::PathBuf};
 
 #[derive(Debug, Clone, Default)]
 pub struct NuConfig {
-    pub source_file: Option<std::path::PathBuf>,
     pub vars: IndexMap<String, Value>,
+    pub file_path: PathBuf,
     pub modified_at: Status,
 }
 
 impl Conf for NuConfig {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn is_modified(&self) -> Result<bool, Box<dyn std::error::Error>> {
         self.is_modified()
     }
@@ -30,17 +25,15 @@ impl Conf for NuConfig {
         self.env()
     }
 
-    fn path(&self) -> Option<Value> {
+    fn path(&self) -> Result<Option<Vec<PathBuf>>, ShellError> {
         self.path()
     }
 
     fn reload(&mut self) {
-        let vars = &mut self.vars;
+        if let Ok(variables) = read(Tag::unknown(), &Some(self.file_path.clone())) {
+            self.vars = variables;
 
-        if let Ok(variables) = read(Tag::unknown(), &self.source_file) {
-            vars.extend(variables);
-
-            self.modified_at = if let Ok(status) = last_modified(&None) {
+            self.modified_at = if let Ok(status) = last_modified(&Some(self.file_path.clone())) {
                 status
             } else {
                 Status::Unavailable
@@ -54,25 +47,20 @@ impl Conf for NuConfig {
 }
 
 impl NuConfig {
-    pub fn with(config_file: Option<std::ffi::OsString>) -> NuConfig {
-        match &config_file {
-            None => NuConfig::new(),
-            Some(_) => {
-                let source_file = config_file.map(std::path::PathBuf::from);
+    pub fn load(cfg_file_path: Option<PathBuf>) -> Result<NuConfig, ShellError> {
+        let vars = read(Tag::unknown(), &cfg_file_path)?;
+        let modified_at = NuConfig::get_last_modified(&cfg_file_path);
+        let file_path = if let Some(file_path) = cfg_file_path {
+            file_path
+        } else {
+            crate::config::default_path()?
+        };
 
-                let vars = if let Ok(variables) = read(Tag::unknown(), &source_file) {
-                    variables
-                } else {
-                    IndexMap::default()
-                };
-
-                NuConfig {
-                    source_file: source_file.clone(),
-                    vars,
-                    modified_at: NuConfig::get_last_modified(&source_file),
-                }
-            }
-        }
+        Ok(NuConfig {
+            file_path,
+            vars,
+            modified_at,
+        })
     }
 
     pub fn new() -> NuConfig {
@@ -81,18 +69,19 @@ impl NuConfig {
         } else {
             IndexMap::default()
         };
+        let path = if let Ok(path) = crate::config::default_path() {
+            path
+        } else {
+            PathBuf::new()
+        };
 
         NuConfig {
-            source_file: None,
             vars,
             modified_at: NuConfig::get_last_modified(&None),
+            file_path: path,
         }
     }
 
-    pub fn history_path(&self) -> PathBuf {
-        super::path::history(self)
-    }
-
     pub fn get_last_modified(config_file: &Option<std::path::PathBuf>) -> Status {
         if let Ok(status) = last_modified(config_file) {
             status
@@ -104,17 +93,15 @@ impl NuConfig {
     pub fn is_modified(&self) -> Result<bool, Box<dyn std::error::Error>> {
         let modified_at = &self.modified_at;
 
-        Ok(
-            match (NuConfig::get_last_modified(&self.source_file), modified_at) {
-                (Status::LastModified(left), Status::LastModified(right)) => {
-                    let left = left.duration_since(std::time::UNIX_EPOCH)?;
-                    let right = (*right).duration_since(std::time::UNIX_EPOCH)?;
+        Ok(match (NuConfig::get_last_modified(&None), modified_at) {
+            (Status::LastModified(left), Status::LastModified(right)) => {
+                let left = left.duration_since(std::time::UNIX_EPOCH)?;
+                let right = (*right).duration_since(std::time::UNIX_EPOCH)?;
 
-                    left != right
-                }
-                (_, _) => false,
-            },
-        )
+                left != right
+            }
+            (_, _) => false,
+        })
     }
 
     pub fn var(&self, key: &str) -> Option<Value> {
@@ -127,6 +114,19 @@ impl NuConfig {
         None
     }
 
+    /// Return environment variables as map
+    pub fn env_map(&self) -> IndexMap<String, String> {
+        let mut result = IndexMap::new();
+        if let Some(variables) = self.env() {
+            for var in variables.row_entries() {
+                if let Ok(value) = var.1.as_string() {
+                    result.insert(var.0.clone(), value);
+                }
+            }
+        }
+        result
+    }
+
     pub fn env(&self) -> Option<Value> {
         let vars = &self.vars;
 
@@ -137,17 +137,43 @@ impl NuConfig {
         None
     }
 
-    pub fn path(&self) -> Option<Value> {
+    pub fn path(&self) -> Result<Option<Vec<PathBuf>>, ShellError> {
         let vars = &self.vars;
 
-        if let Some(env_vars) = vars.get("path") {
-            return Some(env_vars.clone());
+        if let Some(path) = vars.get("path").or_else(|| vars.get("PATH")) {
+            path
+                .table_entries()
+                .map(|p| {
+                    p.as_string().map(PathBuf::from).map_err(|_| {
+                        ShellError::untagged_runtime_error("Could not format path entry as string!\nPath entry from config won't be added")
+                    })
+                })
+            .collect::<Result<Vec<PathBuf>, ShellError>>().map(Some)
+        } else {
+            Ok(None)
         }
+    }
 
-        if let Some(env_vars) = vars.get("PATH") {
-            return Some(env_vars.clone());
+    fn load_scripts_if_present(&self, scripts_name: &str) -> Result<Vec<String>, ShellError> {
+        if let Some(array) = self.var(scripts_name) {
+            if !array.is_table() {
+                Err(ShellError::untagged_runtime_error(format!(
+                    "expected an array of strings as {} commands",
+                    scripts_name
+                )))
+            } else {
+                array.table_entries().map(Value::as_string).collect()
+            }
+        } else {
+            Ok(vec![])
         }
+    }
 
-        None
+    pub fn exit_scripts(&self) -> Result<Vec<String>, ShellError> {
+        self.load_scripts_if_present("on_exit")
+    }
+
+    pub fn startup_scripts(&self) -> Result<Vec<String>, ShellError> {
+        self.load_scripts_if_present("startup")
     }
 }

crates/nu-data/src/config/path.rs

@@ -1,32 +1,25 @@
-use crate::config::NuConfig;
+use crate::config::Conf;
 use std::path::PathBuf;
 
-pub const DEFAULT_CONFIG_LOCATION: &str = "config.toml";
-const DEFAULT_HISTORY_LOCATION: &str = "history.txt";
+const DEFAULT_LOCATION: &str = "history.txt";
 
-pub fn history(config: &NuConfig) -> PathBuf {
-    let default_path = crate::config::user_data()
+pub fn default_history_path() -> PathBuf {
+    crate::config::user_data()
         .map(|mut p| {
-            p.push(DEFAULT_HISTORY_LOCATION);
+            p.push(DEFAULT_LOCATION);
             p
         })
-        .unwrap_or_else(|_| PathBuf::from(DEFAULT_HISTORY_LOCATION));
+        .unwrap_or_else(|_| PathBuf::from(DEFAULT_LOCATION))
+}
 
-    let path = &config.var("history-path");
+pub fn history_path(config: &dyn Conf) -> PathBuf {
+    let default_history_path = default_history_path();
 
-    path.as_ref().map_or(default_path.clone(), |custom_path| {
-        match custom_path.as_string() {
+    config.var("history-path").map_or(
+        default_history_path.clone(),
+        |custom_path| match custom_path.as_string() {
             Ok(path) => PathBuf::from(path),
-            Err(_) => default_path,
-        }
-    })
-}
-
-pub fn source_file(config: &NuConfig) -> PathBuf {
-    match &config.source_file {
-        Some(path) => PathBuf::from(path),
-        None => {
-            crate::config::default_path().unwrap_or_else(|_| PathBuf::from(DEFAULT_CONFIG_LOCATION))
-        }
-    }
+            Err(_) => default_history_path,
+        },
+    )
 }

crates/nu-data/src/config/tests.rs

@@ -1,6 +1,5 @@
 use crate::config::{Conf, NuConfig, Status};
 use nu_protocol::Value;
-use std::any::Any;
 use std::path::{Path, PathBuf};
 
 #[derive(Debug, Clone)]
@@ -10,10 +9,6 @@ pub struct FakeConfig {
 }
 
 impl Conf for FakeConfig {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn is_modified(&self) -> Result<bool, Box<dyn std::error::Error>> {
         self.is_modified()
     }