Allow custom lib dir path for sourcing nu script libraries. (#3940)

Given we can write nu scripts. As the codebase grows, splitting into many smaller nu scripts is necessary.

In general, when we work with paths and files we seem to face quite a few difficulties. Here we just tackle one of them and it involves sourcing
files that also source other nu files and so forth. The current working directory becomes important here and being on a different directory
when sourcing scripts will not work. Mostly because we expand the path on the current working directory and parse the files when a source command
call is done.

For the moment, we introduce a `lib_dirs` configuration value and, unfortunately, introduce a new dependency in `nu-parser` (`nu-data`) to get
a handle of the configuration file to retrieve it. This should give clues and ideas as the new parser engine continues (introduce a way to also know paths)

With this PR we can do the following:

Let's assume we want to write a nu library called `my_library`. We will have the code in a directory called `project`: The file structure will looks like this:

```
project/my_library.nu
project/my_library/hello.nu
project/my_library/name.nu
```

This "pattern" works well, that is, when creating a library have a directory named `my_library` and next to it a `my_library.nu` file. Filling them like this:

```

source my_library/hello.nu
source my_library/name.nu
```

```

def hello [] {
  "hello world"
}
```

```

def name [] {
  "Nu"
end
```

Assuming this `project` directory is stored at `/path/to/lib/project`, we can do:

```
config set lib_dirs ['path/to/lib/project']
```

Given we have this `lib_dirs` configuration value, we can be anywhere while using Nu and do the following:

```
source my_library.nu

echo (hello) (name)

```

Cargo.lock

@@ -3494,6 +3494,7 @@ dependencies = [
  "indexmap",
  "itertools",
  "log",
+ "nu-data",
  "nu-errors",
  "nu-path",
  "nu-protocol",

crates/nu-command/src/commands/core_commands/source.rs

@@ -6,7 +6,7 @@ use nu_path::expand_path;
 use nu_protocol::{Signature, SyntaxShape};
 use nu_source::Tagged;
 
-use std::{borrow::Cow, path::Path};
+use std::{borrow::Cow, path::Path, path::PathBuf};
 
 pub struct Source;
 
@@ -32,7 +32,7 @@ impl WholeStreamCommand for Source {
         "Runs a script file in the current context."
     }
 
-    fn run_with_actions(&self, args: CommandArgs) -> Result<ActionStream, ShellError> {
+    fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
         source(args)
     }
 
@@ -41,14 +41,58 @@ impl WholeStreamCommand for Source {
     }
 }
 
-pub fn source(args: CommandArgs) -> Result<ActionStream, ShellError> {
+pub fn source(args: CommandArgs) -> Result<OutputStream, ShellError> {
     let ctx = &args.context;
     let filename: Tagged<String> = args.req(0)?;
 
+    let source_file = Path::new(&filename.item);
+
     // Note: this is a special case for setting the context from a command
     // In this case, if we don't set it now, we'll lose the scope that this
     // variable should be set into.
-    let contents = std::fs::read_to_string(&expand_path(Cow::Borrowed(Path::new(&filename.item))));
+
+    let lib_dirs = &ctx
+        .configs()
+        .lock()
+        .global_config
+        .as_ref()
+        .map(|configuration| match configuration.var("lib_dirs") {
+            Some(paths) => paths
+                .table_entries()
+                .cloned()
+                .map(|path| path.as_string())
+                .collect(),
+            None => vec![],
+        });
+
+    if let Some(dir) = lib_dirs {
+        for lib_path in dir {
+            match lib_path {
+                Ok(name) => {
+                    let path = PathBuf::from(name).join(source_file);
+
+                    if let Ok(contents) =
+                        std::fs::read_to_string(&expand_path(Cow::Borrowed(path.as_path())))
+                    {
+                        let result = script::run_script_standalone(contents, true, ctx, false);
+
+                        if let Err(err) = result {
+                            ctx.error(err);
+                        }
+                        return Ok(OutputStream::empty());
+                    }
+                }
+                Err(reason) => {
+                    ctx.error(reason.clone());
+                }
+            }
+        }
+    }
+
+    let path = Path::new(source_file);
+
+    let contents = std::fs::read_to_string(&expand_path(Cow::Borrowed(path)));
+
     match contents {
         Ok(contents) => {
             let result = script::run_script_standalone(contents, true, ctx, false);
@@ -56,7 +100,7 @@ pub fn source(args: CommandArgs) -> Result<ActionStream, ShellError> {
             if let Err(err) = result {
                 ctx.error(err);
             }
-            Ok(ActionStream::empty())
+            Ok(OutputStream::empty())
         }
         Err(_) => {
             ctx.error(ShellError::labeled_error(
@@ -65,7 +109,7 @@ pub fn source(args: CommandArgs) -> Result<ActionStream, ShellError> {
                 filename.span(),
             ));
 
-            Ok(ActionStream::empty())
+            Ok(OutputStream::empty())
         }
     }
 }

crates/nu-command/tests/commands/source.rs

@@ -1,6 +1,57 @@
-use nu_test_support::fs::Stub::FileWithContent;
+use nu_test_support::fs::{AbsolutePath, DisplayPath, Stub::FileWithContent};
 use nu_test_support::nu;
-use nu_test_support::playground::Playground;
+use nu_test_support::pipeline as input;
+use nu_test_support::playground::{says, Playground};
+
+use hamcrest2::assert_that;
+use hamcrest2::prelude::*;
+
+#[should_panic]
+#[test]
+fn sources_also_files_under_custom_lib_dirs_path() {
+    Playground::setup("source_test_1", |dirs, nu| {
+        let file = AbsolutePath::new(dirs.test().join("config.toml"));
+        let library_path = AbsolutePath::new(dirs.test().join("lib"));
+
+        nu.with_config(&file);
+        nu.with_files(vec![FileWithContent(
+            "config.toml",
+            &format!(
+                r#"
+                lib_dirs = ["{}"]
+                skip_welcome_message = true
+            "#,
+                library_path.display_path()
+            ),
+        )]);
+
+        nu.within("lib").with_files(vec![FileWithContent(
+            "my_library.nu",
+            r#"
+                source my_library/main.nu
+            "#,
+        )]);
+        nu.within("lib/my_library").with_files(vec![FileWithContent(
+            "main.nu",
+            r#"
+            def hello [] {
+                echo "hello nu"
+            }
+            "#,
+        )]);
+
+        assert_that!(
+            nu.pipeline(&input(
+                r#"
+                source my_library.nu ;
+
+                hello
+                "#,
+            )),
+            says().stdout("hello nu")
+        );
+    })
+}
 
 fn try_source_foo_with_double_quotes_in(testdir: &str, playdir: &str) {
     Playground::setup(playdir, |dirs, sandbox| {

crates/nu-parser/Cargo.toml

@@ -20,7 +20,9 @@ itertools = "0.10.0"
 smart-default = "0.6.0"
 
 dunce = "1.0.1"
+
 nu-errors = { version = "0.36.0", path="../nu-errors" }
+nu-data = { version = "0.36.0", path="../nu-data" }
 nu-path = { version = "0.36.0", path="../nu-path" }
 nu-protocol = { version = "0.36.0", path="../nu-protocol" }
 nu-source = { version = "0.36.0", path="../nu-source" }

crates/nu-parser/src/parse.rs

@@ -1,8 +1,5 @@
 use std::borrow::Cow;
-use std::{
-    path::{Path, PathBuf},
-    sync::Arc,
-};
+use std::{path::PathBuf, sync::Arc};
 
 use bigdecimal::BigDecimal;
 use indexmap::IndexMap;
@@ -18,6 +15,7 @@ use nu_protocol::{NamedType, PositionalType, Signature, SyntaxShape, UnspannedPa
 use nu_source::{HasSpan, Span, Spanned, SpannedItem};
 use num_bigint::BigInt;
 
+use crate::parse::source::parse_source_internal;
 use crate::{lex::lexer::NewlineMode, parse::def::parse_parameter};
 use crate::{
     lex::lexer::{lex, parse_block},
@@ -38,6 +36,7 @@ use self::{
 };
 
 mod def;
+mod source;
 mod util;
 
 pub use self::util::garbage;
@@ -1871,48 +1870,9 @@ fn parse_call(
         let (mut internal_command, err) = parse_internal_command(&lite_cmd, scope, &signature, 0);
 
         if internal_command.name == "source" {
-            if lite_cmd.parts.len() != 2 {
-                return (
-                    Some(ClassifiedCommand::Internal(internal_command)),
-                    Some(ParseError::argument_error(
-                        lite_cmd.parts[0].clone(),
-                        ArgumentError::MissingMandatoryPositional("a path for sourcing".into()),
-                    )),
-                );
-            }
-            if lite_cmd.parts[1].item.starts_with('$') {
-                return (
-                    Some(ClassifiedCommand::Internal(internal_command)),
-                    Some(ParseError::mismatch(
-                        "a filepath constant",
-                        lite_cmd.parts[1].clone(),
-                    )),
-                );
-            }
+            let err = parse_source_internal(&lite_cmd, &internal_command, scope).err();
 
-            let script_path = if let Some(ref positional_args) = internal_command.args.positional {
-                if let Expression::FilePath(ref p) = positional_args[0].expr {
-                    p
-                } else {
-                    Path::new(&lite_cmd.parts[1].item)
-                }
-            } else {
-                Path::new(&lite_cmd.parts[1].item)
-            };
-
-            if let Ok(contents) =
-                std::fs::read_to_string(&expand_path(Cow::Borrowed(Path::new(script_path))))
-            {
-                let _ = parse(&contents, 0, scope);
-            } else {
-                return (
-                    Some(ClassifiedCommand::Internal(internal_command)),
-                    Some(ParseError::argument_error(
-                        lite_cmd.parts[1].clone(),
-                        ArgumentError::BadValue("can't load source file".into()),
-                    )),
-                );
-            }
+            return (Some(ClassifiedCommand::Internal(internal_command)), err);
         } else if lite_cmd.parts[0].item == "alias" || lite_cmd.parts[0].item == "unalias" {
             let error = parse_alias(&lite_cmd, scope);
             if error.is_none() {

crates/nu-parser/src/parse/source.rs

@@ -0,0 +1,93 @@
+use crate::{lex::tokens::LiteCommand, ParserScope};
+use nu_errors::{ArgumentError, ParseError};
+use nu_path::expand_path;
+use nu_protocol::hir::{Expression, InternalCommand};
+
+use std::borrow::Cow;
+use std::path::Path;
+use std::path::PathBuf;
+
+pub fn parse_source_internal(
+    lite_cmd: &LiteCommand,
+    command: &InternalCommand,
+    scope: &dyn ParserScope,
+) -> Result<(), ParseError> {
+    if lite_cmd.parts.len() != 2 {
+        return Err(ParseError::argument_error(
+            lite_cmd.parts[0].clone(),
+            ArgumentError::MissingMandatoryPositional("a path for sourcing".into()),
+        ));
+    }
+
+    if lite_cmd.parts[1].item.starts_with('$') {
+        return Err(ParseError::mismatch(
+            "a filepath constant",
+            lite_cmd.parts[1].clone(),
+        ));
+    }
+
+    // look for source files in lib dirs first
+    // if not files are found, try the current path
+    // first file found wins.
+    find_source_file(lite_cmd, command, scope)
+}
+
+fn find_source_file(
+    lite_cmd: &LiteCommand,
+    command: &InternalCommand,
+    scope: &dyn ParserScope,
+) -> Result<(), ParseError> {
+    let file = if let Some(ref positional_args) = command.args.positional {
+        if let Expression::FilePath(ref p) = positional_args[0].expr {
+            p
+        } else {
+            Path::new(&lite_cmd.parts[1].item)
+        }
+    } else {
+        Path::new(&lite_cmd.parts[1].item)
+    };
+
+    let lib_dirs = nu_data::config::config(nu_source::Tag::unknown())
+        .ok()
+        .as_ref()
+        .map(|configuration| match configuration.get("lib_dirs") {
+            Some(paths) => paths
+                .table_entries()
+                .cloned()
+                .map(|path| path.as_string())
+                .collect(),
+            None => vec![],
+        });
+
+    if let Some(dir) = lib_dirs {
+        for lib_path in dir.into_iter().flatten() {
+            let path = PathBuf::from(lib_path).join(&file);
+
+            if let Ok(contents) =
+                std::fs::read_to_string(&expand_path(Cow::Borrowed(path.as_path())))
+            {
+                return parse(&contents, 0, scope);
+            }
+        }
+    }
+
+    let path = Path::new(&file);
+
+    let contents = std::fs::read_to_string(&expand_path(Cow::Borrowed(path)));
+
+    match contents {
+        Ok(contents) => parse(&contents, 0, scope),
+        Err(_) => Err(ParseError::argument_error(
+            lite_cmd.parts[1].clone(),
+            ArgumentError::BadValue("can't load source file".into()),
+        )),
+    }
+}
+
+pub fn parse(input: &str, span_offset: usize, scope: &dyn ParserScope) -> Result<(), ParseError> {
+    if let (_, Some(parse_error)) = super::parse(input, span_offset, scope) {
+        Err(parse_error)
+    } else {
+        Ok(())
+    }
+}