Move from source to source-env (#6277)

* start working on source-env

* WIP

* Get most tests working, still one to go

* Fix file-relative paths; Report parser error

* Fix merge conflicts; Restore source as deprecated

* Tests: Use source-env; Remove redundant tests

* Fmt

* Respect hidden env vars

* Fix file-relative eval for source-env

* Add file-relative eval to "overlay use"

* Use FILE_PWD only in source-env and "overlay use"

* Ignore new tests for now

This will be another issue

* Throw an error if setting FILE_PWD manually

* Fix source-related test failures

* Fix nu-check to respect FILE_PWD

* Fix corrupted spans in source-env shell errors

* Fix up some references to old source

* Remove deprecation message

* Re-introduce deleted tests

Co-authored-by: kubouch <kubouch@gmail.com>

crates/nu-command/src/core_commands/mod.rs

@@ -25,7 +25,6 @@ mod let_;
 mod metadata;
 mod module;
 pub(crate) mod overlay;
-mod source;
 mod use_;
 mod version;
 
@@ -56,7 +55,6 @@ pub use let_::Let;
 pub use metadata::Metadata;
 pub use module::Module;
 pub use overlay::*;
-pub use source::Source;
 pub use use_::Use;
 pub use version::Version;
 #[cfg(feature = "plugin")]

crates/nu-command/src/core_commands/overlay/use_.rs

@@ -1,7 +1,9 @@
-use nu_engine::{eval_block, redirect_env, CallExt};
+use nu_engine::{eval_block, find_in_dirs_env, redirect_env, CallExt};
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
 
 use std::path::Path;
 
@@ -79,7 +81,7 @@ impl Command for OverlayUse {
             .find_overlay(name_arg.item.as_bytes())
             .is_some()
         {
-            (name_arg.item, name_arg.span)
+            (name_arg.item.clone(), name_arg.span)
         } else if let Some(os_str) = Path::new(&name_arg.item).file_stem() {
             if let Some(name) = os_str.to_str() {
                 (name.to_string(), name_arg.span)
@@ -131,6 +133,22 @@ impl Command for OverlayUse {
 
                     // Evaluate the export-env block (if any) and keep its environment
                     if let Some(block_id) = module.env_block {
+                        let maybe_path =
+                            find_in_dirs_env(&name_arg.item, engine_state, caller_stack)?;
+
+                        if let Some(path) = &maybe_path {
+                            // Set the currently evaluated directory, if the argument is a valid path
+                            let mut parent = path.clone();
+                            parent.pop();
+
+                            let file_pwd = Value::String {
+                                val: parent.to_string_lossy().to_string(),
+                                span: call.head,
+                            };
+
+                            caller_stack.add_env_var("FILE_PWD".to_string(), file_pwd);
+                        }
+
                         let block = engine_state.get_block(block_id);
                         let mut callee_stack = caller_stack.gather_captures(&block.captures);
 
@@ -143,7 +161,13 @@ impl Command for OverlayUse {
                             call.redirect_stderr,
                         );
 
+                        // Merge the block's environment to the current stack
                         redirect_env(engine_state, caller_stack, &callee_stack);
+
+                        if maybe_path.is_some() {
+                            // Remove the file-relative PWD, if the argument is a valid path
+                            caller_stack.remove_env_var(engine_state, "FILE_PWD");
+                        }
                     }
                 }
             }

crates/nu-command/src/default_context.rs

@@ -59,7 +59,6 @@ pub fn create_default_context() -> EngineState {
             Let,
             Metadata,
             Module,
-            Source,
             Use,
             Version,
         };
@@ -358,6 +357,7 @@ pub fn create_default_context() -> EngineState {
             ExportEnv,
             LetEnv,
             LoadEnv,
+            SourceEnv,
             WithEnv,
             ConfigNu,
             ConfigEnv,
@@ -432,6 +432,7 @@ pub fn create_default_context() -> EngineState {
         // Deprecated
         bind_command! {
             HashBase64,
+            Source,
             StrDatetimeDeprecated,
             StrDecimalDeprecated,
             StrIntDeprecated,

crates/nu-command/src/deprecated/mod.rs

@@ -1,5 +1,6 @@
 mod deprecated_commands;
 mod hash_base64;
+mod source;
 mod str_datetime;
 mod str_decimal;
 mod str_find_replace;
@@ -7,6 +8,7 @@ mod str_int;
 
 pub use deprecated_commands::*;
 pub use hash_base64::HashBase64;
+pub use source::Source;
 pub use str_datetime::StrDatetimeDeprecated;
 pub use str_decimal::StrDecimalDeprecated;
 pub use str_find_replace::StrFindReplaceDeprecated;

crates/nu-command/src/env/let_env.rs

@@ -1,7 +1,9 @@
 use nu_engine::{current_dir, eval_expression_with_input, CallExt};
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, SyntaxShape, Value};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
 
 #[derive(Clone)]
 pub struct LetEnv;
@@ -34,7 +36,7 @@ impl Command for LetEnv {
         input: PipelineData,
     ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
         // TODO: find and require the crossplatform restrictions on environment names
-        let env_var = call.req(engine_state, stack, 0)?;
+        let env_var: Spanned<String> = call.req(engine_state, stack, 0)?;
 
         let keyword_expr = call
             .positional_nth(1)
@@ -47,19 +49,26 @@ impl Command for LetEnv {
                 .0
                 .into_value(call.head);
 
-        if env_var == "PWD" {
+        if env_var.item == "FILE_PWD" {
+            return Err(ShellError::AutomaticEnvVarSetManually(
+                env_var.item,
+                env_var.span,
+            ));
+        }
+
+        if env_var.item == "PWD" {
             let cwd = current_dir(engine_state, stack)?;
             let rhs = rhs.as_string()?;
             let rhs = nu_path::expand_path_with(rhs, cwd);
             stack.add_env_var(
-                env_var,
+                env_var.item,
                 Value::String {
                     val: rhs.to_string_lossy().to_string(),
                     span: call.head,
                 },
             );
         } else {
-            stack.add_env_var(env_var, rhs);
+            stack.add_env_var(env_var.item, rhs);
         }
         Ok(PipelineData::new(call.head))
     }

crates/nu-command/src/env/load_env.rs

@@ -38,6 +38,10 @@ impl Command for LoadEnv {
         match arg {
             Some((cols, vals)) => {
                 for (env_var, rhs) in cols.into_iter().zip(vals) {
+                    if env_var == "FILE_PWD" {
+                        return Err(ShellError::AutomaticEnvVarSetManually(env_var, call.head));
+                    }
+
                     if env_var == "PWD" {
                         let cwd = current_dir(engine_state, stack)?;
                         let rhs = rhs.as_string()?;
@@ -58,6 +62,10 @@ impl Command for LoadEnv {
             None => match input {
                 PipelineData::Value(Value::Record { cols, vals, .. }, ..) => {
                     for (env_var, rhs) in cols.into_iter().zip(vals) {
+                        if env_var == "FILE_PWD" {
+                            return Err(ShellError::AutomaticEnvVarSetManually(env_var, call.head));
+                        }
+
                         if env_var == "PWD" {
                             let cwd = current_dir(engine_state, stack)?;
                             let rhs = rhs.as_string()?;

crates/nu-command/src/env/mod.rs

@@ -3,6 +3,7 @@ mod env_command;
 mod export_env;
 mod let_env;
 mod load_env;
+mod source_env;
 mod with_env;
 
 pub use config::ConfigEnv;
@@ -13,4 +14,5 @@ pub use env_command::Env;
 pub use export_env::ExportEnv;
 pub use let_env::LetEnv;
 pub use load_env::LoadEnv;
+pub use source_env::SourceEnv;
 pub use with_env::WithEnv;

crates/nu-command/src/env/source_env.rs

@@ -0,0 +1,142 @@
+use std::path::PathBuf;
+
+use nu_engine::{eval_block, find_in_dirs_env, redirect_env, CallExt};
+use nu_parser::parse;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack, StateWorkingSet};
+use nu_protocol::{
+    Category, CliError, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
+
+/// Source a file for environment variables.
+#[derive(Clone)]
+pub struct SourceEnv;
+
+impl Command for SourceEnv {
+    fn name(&self) -> &str {
+        "source-env"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("source-env")
+            .required(
+                "filename",
+                SyntaxShape::String, // type is string to avoid automatically canonicalizing the path
+                "the filepath to the script file to source the environment from",
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "Source the environment from a source file into the current environment."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        caller_stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let source_filename: Spanned<String> = call.req(engine_state, caller_stack, 0)?;
+
+        if let Some(path) = find_in_dirs_env(&source_filename.item, engine_state, caller_stack)? {
+            if let Ok(content) = std::fs::read_to_string(&path) {
+                let mut parent = PathBuf::from(&path);
+                parent.pop();
+
+                let mut new_engine_state = engine_state.clone();
+
+                let (block, delta) = {
+                    let mut working_set = StateWorkingSet::new(&new_engine_state);
+
+                    // Set the currently parsed directory
+                    working_set.currently_parsed_cwd = Some(parent.clone());
+
+                    let (block, err) = parse(&mut working_set, None, content.as_bytes(), true, &[]);
+
+                    if let Some(err) = err {
+                        // Because the error span points at new_engine_state, we must create the error message now
+                        let msg = format!(
+                            r#"Found this parser error: {:?}"#,
+                            CliError(&err, &working_set)
+                        );
+
+                        return Err(ShellError::GenericError(
+                            "Failed to parse content".to_string(),
+                            "cannot parse this file".to_string(),
+                            Some(source_filename.span),
+                            Some(msg),
+                            vec![],
+                        ));
+                    } else {
+                        (block, working_set.render())
+                    }
+                };
+
+                // Merge parser changes to a temporary engine state
+                new_engine_state.merge_delta(delta)?;
+
+                // Set the currently evaluated directory
+                let file_pwd = Value::String {
+                    val: parent.to_string_lossy().to_string(),
+                    span: call.head,
+                };
+
+                caller_stack.add_env_var("FILE_PWD".to_string(), file_pwd);
+
+                // Evaluate the parsed file's block
+                let mut callee_stack = caller_stack.gather_captures(&block.captures);
+
+                let result = eval_block(
+                    &new_engine_state,
+                    &mut callee_stack,
+                    &block,
+                    input,
+                    true,
+                    true,
+                );
+
+                let result = if let Err(err) = result {
+                    // Because the error span points at new_engine_state, we must create the error message now
+                    let working_set = StateWorkingSet::new(&new_engine_state);
+
+                    let msg = format!(
+                        r#"Found this shell error: {:?}"#,
+                        CliError(&err, &working_set)
+                    );
+
+                    Err(ShellError::GenericError(
+                        "Failed to evaluate content".to_string(),
+                        "cannot evaluate this file".to_string(),
+                        Some(source_filename.span),
+                        Some(msg),
+                        vec![],
+                    ))
+                } else {
+                    result
+                };
+
+                // Merge the block's environment to the current stack
+                redirect_env(engine_state, caller_stack, &callee_stack);
+
+                // Remove the file-relative PWD
+                caller_stack.remove_env_var(engine_state, "FILE_PWD");
+
+                result
+            } else {
+                Err(ShellError::FileNotFound(source_filename.span))
+            }
+        } else {
+            Err(ShellError::FileNotFound(source_filename.span))
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Sources the environment from foo.nu in the current context",
+            example: r#"source-env foo.nu"#,
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/nu_check.rs

@@ -1,4 +1,4 @@
-use nu_engine::{current_dir, CallExt};
+use nu_engine::{find_in_dirs_env, CallExt};
 use nu_parser::{parse, parse_module_block, unescape_unquote_string};
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack, StateWorkingSet};
@@ -6,8 +6,6 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value,
 };
 
-use std::path::Path;
-
 #[derive(Clone)]
 pub struct NuCheck;
 
@@ -18,7 +16,8 @@ impl Command for NuCheck {
 
     fn signature(&self) -> Signature {
         Signature::build("nu-check")
-            .optional("path", SyntaxShape::Filepath, "File path to parse")
+            // type is string to avoid automatically canonicalizing the path
+            .optional("path", SyntaxShape::String, "File path to parse")
             .switch("as-module", "Parse content as module", Some('m'))
             .switch("debug", "Show error messages", Some('d'))
             .switch("all", "Parse content as script first, returns result if success, otherwise, try with module", Some('a'))
@@ -102,13 +101,23 @@ impl Command for NuCheck {
                 }
             }
             _ => {
-                if path.is_some() {
-                    let path = match find_path(path, engine_state, stack, call.head) {
-                        Ok(path) => path,
+                if let Some(path_str) = path {
+                    // look up the path as relative to FILE_PWD or inside NU_LIB_DIRS (same process as source-env)
+                    let path = match find_in_dirs_env(&path_str.item, engine_state, stack) {
+                        Ok(path) => {
+                            if let Some(path) = path {
+                                path
+                            } else {
+                                return Err(ShellError::FileNotFound(path_str.span));
+                            }
+                        }
                         Err(error) => return Err(error),
                     };
 
-                    let ext: Vec<_> = path.rsplitn(2, '.').collect();
+                    // get the expanded path as a string
+                    let path_str = path.to_string_lossy().to_string();
+
+                    let ext: Vec<_> = path_str.rsplitn(2, '.').collect();
                     if ext[0] != "nu" {
                         return Err(ShellError::GenericError(
                             "Cannot parse input".to_string(),
@@ -120,8 +129,7 @@ impl Command for NuCheck {
                     }
 
                     // Change currently parsed directory
-                    let prev_currently_parsed_cwd = if let Some(parent) = Path::new(&path).parent()
-                    {
+                    let prev_currently_parsed_cwd = if let Some(parent) = path.parent() {
                         let prev = working_set.currently_parsed_cwd.clone();
 
                         working_set.currently_parsed_cwd = Some(parent.into());
@@ -132,11 +140,11 @@ impl Command for NuCheck {
                     };
 
                     let result = if is_all {
-                        heuristic_parse_file(path, &mut working_set, call, is_debug)
+                        heuristic_parse_file(path_str, &mut working_set, call, is_debug)
                     } else if is_module {
-                        parse_file_module(path, &mut working_set, call, is_debug)
+                        parse_file_module(path_str, &mut working_set, call, is_debug)
                     } else {
-                        parse_file_script(path, &mut working_set, call, is_debug)
+                        parse_file_script(path_str, &mut working_set, call, is_debug)
                     };
 
                     // Restore the currently parsed directory back
@@ -202,46 +210,6 @@ impl Command for NuCheck {
     }
 }
 
-fn find_path(
-    path: Option<Spanned<String>>,
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    span: Span,
-) -> Result<String, ShellError> {
-    let cwd = current_dir(engine_state, stack)?;
-
-    let path = match path {
-        Some(s) => {
-            let path_no_whitespace = &s.item.trim_end_matches(|x| matches!(x, '\x09'..='\x0d'));
-
-            let path = match nu_path::canonicalize_with(path_no_whitespace, &cwd) {
-                Ok(p) => {
-                    if !p.is_file() {
-                        return Err(ShellError::GenericError(
-                            "Cannot parse input".to_string(),
-                            "Path is not a file".to_string(),
-                            Some(s.span),
-                            None,
-                            Vec::new(),
-                        ));
-                    } else {
-                        p
-                    }
-                }
-
-                Err(_) => {
-                    return Err(ShellError::FileNotFound(s.span));
-                }
-            };
-            path.to_string_lossy().to_string()
-        }
-        None => {
-            return Err(ShellError::NotFound(span));
-        }
-    };
-    Ok(path)
-}
-
 fn heuristic_parse(
     working_set: &mut StateWorkingSet,
     filename: Option<&str>,