Use only $nu.env.PWD for getting the current directory (#587)

* Use only $nu.env.PWD for getting current directory

Because setting and reading to/from std::env changes the global state
shich is problematic if we call `cd` from multiple threads (e.g., in a
`par-each` block).

With this change, when engine-q starts, it will either inherit existing
PWD env var, or create a new one from `std::env::current_dir()`.
Otherwise, everything that needs the current directory will get it from
`$nu.env.PWD`. Each spawned external command will get its current
directory per-process which should be thread-safe.

One thing left to do is to patch nu-path for this as well since it uses
`std::env::current_dir()` in its expansions.

* Rename nu-path functions

*_with is not *_relative which should be more descriptive and frees
"with" for use in a followup commit.

* Clone stack every each iter; Fix some commands

Cloning the stack each iteration of `each` makes sure we're not reusing
PWD between iterations.

Some fixes in commands to make them use the new PWD.

* Post-rebase cleanup, fmt, clippy

* Change back _relative to _with in nu-path funcs

Didn't use the idea I had for the new "_with".

* Remove leftover current_dir from rebase

* Add cwd sync at merge_delta()

This makes sure the parser and completer always have up-to-date cwd.

* Always pass absolute path to glob in ls

* Do not allow PWD a relative path; Allow recovery

Makes it possible to recover PWD by proceeding with the REPL cycle.

* Clone stack in each also for byte/string stream

* (WIP) Start moving env variables to engine state

* (WIP) Move env vars to engine state (ugly)

Quick and dirty code.

* (WIP) Remove unused mut and args; Fmt

* (WIP) Fix dataframe tests

* (WIP) Fix missing args after rebase

* (WIP) Clone only env vars, not the whole stack

* (WIP) Add env var clone to `for` loop as well

* Minor edits

* Refactor merge_delta() to include stack merging.

Less error-prone than doing it manually.

* Clone env for each `update` command iteration

* Mark env var hidden only when found in eng. state

* Fix clippt warnings

* Add TODO about env var reading

* Do not clone empty environment in loops

* Remove extra cwd collection

* Split current_dir() into str and path; Fix autocd

* Make completions respect PWD env var

crates/nu-protocol/src/engine/engine_state.rs

@@ -1,4 +1,4 @@
-use super::Command;
+use super::{Command, Stack};
 use crate::{
     ast::Block, BlockId, DeclId, Example, Overlay, OverlayId, ShellError, Signature, Span, Type,
     VarId,
@@ -9,6 +9,10 @@ use std::{
     sync::{atomic::AtomicBool, Arc},
 };
 
+use crate::Value;
+
+use std::path::Path;
+
 #[cfg(feature = "plugin")]
 use std::path::PathBuf;
 
@@ -140,6 +144,7 @@ pub struct EngineState {
     overlays: im::Vector<Overlay>,
     pub scope: im::Vector<ScopeFrame>,
     pub ctrlc: Option<Arc<AtomicBool>>,
+    pub env_vars: im::HashMap<String, Value>,
     #[cfg(feature = "plugin")]
     pub plugin_signatures: Option<PathBuf>,
 }
@@ -167,6 +172,7 @@ impl EngineState {
             overlays: im::vector![],
             scope: im::vector![ScopeFrame::new()],
             ctrlc: None,
+            env_vars: im::HashMap::new(),
             #[cfg(feature = "plugin")]
             plugin_signatures: None,
         }
@@ -179,7 +185,12 @@ impl EngineState {
     ///
     /// When we want to preserve what the parser has created, we can take its output (the `StateDelta`) and
     /// use this function to merge it into the global state.
-    pub fn merge_delta(&mut self, mut delta: StateDelta) -> Result<(), ShellError> {
+    pub fn merge_delta(
+        &mut self,
+        mut delta: StateDelta,
+        stack: Option<&mut Stack>,
+        cwd: impl AsRef<Path>,
+    ) -> Result<(), ShellError> {
         // Take the mutable reference and extend the permanent state from the working set
         self.files.extend(delta.files);
         self.file_contents.extend(delta.file_contents);
@@ -216,6 +227,16 @@ impl EngineState {
             }
         }
 
+        if let Some(stack) = stack {
+            for mut env_scope in stack.env_vars.drain(..) {
+                for (k, v) in env_scope.drain() {
+                    self.env_vars.insert(k, v);
+                }
+            }
+        }
+
+        std::env::set_current_dir(cwd)?;
+
         Ok(())
     }
 
@@ -1209,7 +1230,8 @@ mod engine_state_tests {
             working_set.render()
         };
 
-        engine_state.merge_delta(delta)?;
+        let cwd = std::env::current_dir().expect("Could not get current working directory.");
+        engine_state.merge_delta(delta, None, &cwd)?;
 
         assert_eq!(engine_state.num_files(), 2);
         assert_eq!(&engine_state.files[0].0, "test.nu");

crates/nu-protocol/src/engine/stack.rs

@@ -1,5 +1,6 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
+use crate::engine::EngineState;
 use crate::{Config, ShellError, Value, VarId, CONFIG_VARIABLE_ID};
 
 /// A runtime value stack used during evaluation
@@ -25,6 +26,9 @@ pub struct Stack {
     pub vars: HashMap<VarId, Value>,
     /// Environment variables arranged as a stack to be able to recover values from parent scopes
     pub env_vars: Vec<HashMap<String, Value>>,
+    /// Tells which environment variables from engine state are hidden. We don't need to track the
+    /// env vars in the stack since we can just delete them.
+    pub env_hidden: HashSet<String>,
 }
 
 impl Default for Stack {
@@ -38,6 +42,17 @@ impl Stack {
         Stack {
             vars: HashMap::new(),
             env_vars: vec![],
+            env_hidden: HashSet::new(),
+        }
+    }
+
+    pub fn with_env(&mut self, env_vars: &[HashMap<String, Value>], env_hidden: &HashSet<String>) {
+        if env_vars.iter().any(|scope| !scope.is_empty()) {
+            self.env_vars = env_vars.to_owned();
+        }
+
+        if !env_hidden.is_empty() {
+            self.env_hidden = env_hidden.clone();
         }
     }
 
@@ -54,6 +69,9 @@ impl Stack {
     }
 
     pub fn add_env_var(&mut self, var: String, value: Value) {
+        // if the env var was hidden, let's activate it again
+        self.env_hidden.remove(&var);
+
         if let Some(scope) = self.env_vars.last_mut() {
             scope.insert(var, value);
         } else {
@@ -85,8 +103,15 @@ impl Stack {
     }
 
     /// Flatten the env var scope frames into one frame
-    pub fn get_env_vars(&self) -> HashMap<String, Value> {
-        let mut result = HashMap::new();
+    pub fn get_env_vars(&self, engine_state: &EngineState) -> HashMap<String, Value> {
+        // TODO: We're collecting im::HashMap to HashMap here. It might make sense to make these
+        // the same data structure.
+        let mut result: HashMap<String, Value> = engine_state
+            .env_vars
+            .iter()
+            .filter(|(k, _)| !self.env_hidden.contains(*k))
+            .map(|(k, v)| (k.clone(), v.clone()))
+            .collect();
 
         for scope in &self.env_vars {
             result.extend(scope.clone());
@@ -95,24 +120,37 @@ impl Stack {
         result
     }
 
-    pub fn get_env_var(&self, name: &str) -> Option<Value> {
+    pub fn get_env_var(&self, engine_state: &EngineState, name: &str) -> Option<Value> {
         for scope in self.env_vars.iter().rev() {
             if let Some(v) = scope.get(name) {
                 return Some(v.clone());
             }
         }
 
-        None
+        if self.env_hidden.contains(name) {
+            None
+        } else {
+            engine_state.env_vars.get(name).cloned()
+        }
     }
 
-    pub fn remove_env_var(&mut self, name: &str) -> Option<Value> {
+    pub fn remove_env_var(&mut self, engine_state: &EngineState, name: &str) -> Option<Value> {
         for scope in self.env_vars.iter_mut().rev() {
             if let Some(v) = scope.remove(name) {
                 return Some(v);
             }
         }
 
-        None
+        if self.env_hidden.contains(name) {
+            // the environment variable is already hidden
+            None
+        } else if let Some(val) = engine_state.env_vars.get(name) {
+            // the environment variable was found in the engine state => mark it as hidden
+            self.env_hidden.insert(name.to_string());
+            Some(val.clone())
+        } else {
+            None
+        }
     }
 
     pub fn get_config(&self) -> Result<Config, ShellError> {

crates/nu-protocol/src/shell_error.rs

@@ -96,12 +96,9 @@ pub enum ShellError {
     VariableNotFoundAtRuntime(#[label = "variable not found"] Span),
 
     #[error("Environment variable not found")]
-    #[diagnostic(code(nu::shell::variable_not_found), url(docsrs))]
+    #[diagnostic(code(nu::shell::env_variable_not_found), url(docsrs))]
     EnvVarNotFoundAtRuntime(#[label = "environment variable not found"] Span),
 
-    // #[error("Environment variable is not a string")]
-    // #[diagnostic(code(nu::shell::variable_not_found), url(docsrs))]
-    // EnvVarNotAString(#[label = "does not evaluate to a string"] Span),
     #[error("Not found.")]
     #[diagnostic(code(nu::parser::not_found), url(docsrs))]
     NotFound(#[label = "did not find anything under this name"] Span),
@@ -185,7 +182,7 @@ pub enum ShellError {
     PluginFailedToDecode(String),
 
     #[error("I/O error")]
-    #[diagnostic(code(nu::shell::io_error), url(docsrs))]
+    #[diagnostic(code(nu::shell::io_error), url(docsrs), help("{0}"))]
     IOError(String),
 
     #[error("Directory not found")]