Split nu-cli into nu-cli/nu-engine (#2898)

We split off the evaluation engine part of nu-cli into its own crate. This helps improve build times for nu-cli by 17% in my tests. It also helps us see a bit better what's the core engine portion vs the part specific to the interactive CLI piece.

There's more than can be done here, but I think it's a good start in the right direction.

crates/nu-engine/src/filesystem/dir_info.rs

@@ -0,0 +1,267 @@
+use filesize::file_real_size_fast;
+use glob::Pattern;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{UntaggedValue, Value};
+use nu_source::Tag;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+pub struct DirBuilder {
+    pub tag: Tag,
+    pub min: Option<u64>,
+    pub deref: bool,
+    pub exclude: Option<Pattern>,
+    pub all: bool,
+}
+
+impl DirBuilder {
+    pub fn new(
+        tag: Tag,
+        min: Option<u64>,
+        deref: bool,
+        exclude: Option<Pattern>,
+        all: bool,
+    ) -> DirBuilder {
+        DirBuilder {
+            tag,
+            min,
+            deref,
+            exclude,
+            all,
+        }
+    }
+}
+
+pub struct DirInfo {
+    dirs: Vec<DirInfo>,
+    files: Vec<FileInfo>,
+    errors: Vec<ShellError>,
+    size: u64,
+    blocks: u64,
+    path: PathBuf,
+    tag: Tag,
+}
+
+pub struct FileInfo {
+    path: PathBuf,
+    size: u64,
+    blocks: Option<u64>,
+    tag: Tag,
+}
+
+impl FileInfo {
+    pub fn new(path: impl Into<PathBuf>, deref: bool, tag: Tag) -> Result<Self, ShellError> {
+        let path = path.into();
+        let m = if deref {
+            std::fs::metadata(&path)
+        } else {
+            std::fs::symlink_metadata(&path)
+        };
+
+        match m {
+            Ok(d) => {
+                let block_size = file_real_size_fast(&path, &d).ok();
+
+                Ok(FileInfo {
+                    path,
+                    blocks: block_size,
+                    size: d.len(),
+                    tag,
+                })
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+}
+
+impl DirInfo {
+    pub fn new(
+        path: impl Into<PathBuf>,
+        params: &DirBuilder,
+        depth: Option<u64>,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Self {
+        let path = path.into();
+
+        let mut s = Self {
+            dirs: Vec::new(),
+            errors: Vec::new(),
+            files: Vec::new(),
+            size: 0,
+            blocks: 0,
+            tag: params.tag.clone(),
+            path,
+        };
+
+        match std::fs::read_dir(&s.path) {
+            Ok(d) => {
+                for f in d {
+                    if ctrl_c.load(Ordering::SeqCst) {
+                        break;
+                    }
+
+                    match f {
+                        Ok(i) => match i.file_type() {
+                            Ok(t) if t.is_dir() => {
+                                s = s.add_dir(i.path(), depth, &params, ctrl_c.clone())
+                            }
+                            Ok(_t) => s = s.add_file(i.path(), &params),
+                            Err(e) => s = s.add_error(e.into()),
+                        },
+                        Err(e) => s = s.add_error(e.into()),
+                    }
+                }
+            }
+            Err(e) => s = s.add_error(e.into()),
+        }
+        s
+    }
+
+    fn add_dir(
+        mut self,
+        path: impl Into<PathBuf>,
+        mut depth: Option<u64>,
+        params: &DirBuilder,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Self {
+        if let Some(current) = depth {
+            if let Some(new) = current.checked_sub(1) {
+                depth = Some(new);
+            } else {
+                return self;
+            }
+        }
+
+        let d = DirInfo::new(path, &params, depth, ctrl_c);
+        self.size += d.size;
+        self.blocks += d.blocks;
+        self.dirs.push(d);
+        self
+    }
+
+    fn add_file(mut self, f: impl Into<PathBuf>, params: &DirBuilder) -> Self {
+        let f = f.into();
+        let include = params
+            .exclude
+            .as_ref()
+            .map_or(true, |x| !x.matches_path(&f));
+        if include {
+            match FileInfo::new(f, params.deref, self.tag.clone()) {
+                Ok(file) => {
+                    let inc = params.min.map_or(true, |s| file.size >= s);
+                    if inc {
+                        self.size += file.size;
+                        self.blocks += file.blocks.unwrap_or(0);
+                        if params.all {
+                            self.files.push(file);
+                        }
+                    }
+                }
+                Err(e) => self = self.add_error(e),
+            }
+        }
+        self
+    }
+
+    fn add_error(mut self, e: ShellError) -> Self {
+        self.errors.push(e);
+        self
+    }
+
+    pub fn get_size(&self) -> u64 {
+        self.size
+    }
+}
+
+impl From<DirInfo> for Value {
+    fn from(d: DirInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::filepath(d.path).into_value(&d.tag),
+        );
+
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::filesize(d.size).into_value(&d.tag),
+        );
+
+        r.insert(
+            "physical".to_string(),
+            UntaggedValue::filesize(d.blocks).into_value(&d.tag),
+        );
+
+        r.insert("directories".to_string(), value_from_vec(d.dirs, &d.tag));
+
+        r.insert("files".to_string(), value_from_vec(d.files, &d.tag));
+
+        if !d.errors.is_empty() {
+            let v = UntaggedValue::Table(
+                d.errors
+                    .into_iter()
+                    .map(move |e| UntaggedValue::Error(e).into_untagged_value())
+                    .collect::<Vec<Value>>(),
+            )
+            .into_value(&d.tag);
+
+            r.insert("errors".to_string(), v);
+        }
+
+        Value {
+            value: UntaggedValue::row(r),
+            tag: d.tag,
+        }
+    }
+}
+
+impl From<FileInfo> for Value {
+    fn from(f: FileInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::filepath(f.path).into_value(&f.tag),
+        );
+
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::filesize(f.size).into_value(&f.tag),
+        );
+
+        let b = f
+            .blocks
+            .map(UntaggedValue::filesize)
+            .unwrap_or_else(UntaggedValue::nothing)
+            .into_value(&f.tag);
+
+        r.insert("physical".to_string(), b);
+
+        r.insert(
+            "directories".to_string(),
+            UntaggedValue::nothing().into_value(&f.tag),
+        );
+
+        r.insert(
+            "files".to_string(),
+            UntaggedValue::nothing().into_value(&f.tag),
+        );
+
+        UntaggedValue::row(r).into_value(&f.tag)
+    }
+}
+
+fn value_from_vec<V>(vec: Vec<V>, tag: &Tag) -> Value
+where
+    V: Into<Value>,
+{
+    if vec.is_empty() {
+        UntaggedValue::nothing()
+    } else {
+        let values = vec.into_iter().map(Into::into).collect::<Vec<Value>>();
+        UntaggedValue::Table(values)
+    }
+    .into_value(tag)
+}

crates/nu-engine/src/filesystem/mod.rs

@@ -0,0 +1,4 @@
+pub(crate) mod dir_info;
+pub mod filesystem_shell;
+pub mod path;
+pub(crate) mod utils;

crates/nu-engine/src/filesystem/path.rs

@@ -0,0 +1,114 @@
+use std::io;
+use std::path::{Component, Path, PathBuf};
+
+pub fn absolutize<P, Q>(relative_to: P, path: Q) -> PathBuf
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let path = if path.as_ref() == Path::new(".") {
+        // Joining a Path with '.' appends a '.' at the end, making the prompt
+        // more ugly - so we don't do anything, which should result in an equal
+        // path on all supported systems.
+        relative_to.as_ref().to_owned()
+    } else {
+        relative_to.as_ref().join(path)
+    };
+
+    let (relative_to, path) = {
+        let components: Vec<_> = path.components().collect();
+        let separator = components
+            .iter()
+            .enumerate()
+            .find(|(_, c)| c == &&Component::CurDir || c == &&Component::ParentDir);
+
+        if let Some((index, _)) = separator {
+            let (absolute, relative) = components.split_at(index);
+            let absolute: PathBuf = absolute.iter().collect();
+            let relative: PathBuf = relative.iter().collect();
+
+            (absolute, relative)
+        } else {
+            (relative_to.as_ref().to_path_buf(), path)
+        }
+    };
+
+    let path = if path.is_relative() {
+        let mut result = relative_to;
+        path.components().for_each(|component| match component {
+            Component::ParentDir => {
+                result.pop();
+            }
+            Component::Normal(normal) => result.push(normal),
+            _ => {}
+        });
+
+        result
+    } else {
+        path
+    };
+
+    dunce::simplified(&path).to_path_buf()
+}
+
+pub fn canonicalize<P, Q>(relative_to: P, path: Q) -> io::Result<PathBuf>
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let absolutized = absolutize(&relative_to, path);
+    let path = match std::fs::read_link(&absolutized) {
+        Ok(resolved) => {
+            let parent = absolutized.parent().unwrap_or(&absolutized);
+            absolutize(parent, resolved)
+        }
+
+        Err(e) => {
+            if absolutized.exists() {
+                absolutized
+            } else {
+                return Err(e);
+            }
+        }
+    };
+
+    Ok(dunce::simplified(&path).to_path_buf())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io;
+
+    #[test]
+    fn absolutize_two_dots() {
+        let relative_to = Path::new("/foo/bar");
+        let path = Path::new("..");
+
+        assert_eq!(
+            PathBuf::from("/foo"), // missing path
+            absolutize(relative_to, path)
+        );
+    }
+
+    #[test]
+    fn canonicalize_should_succeed() -> io::Result<()> {
+        let relative_to = Path::new("/foo/bar");
+        let path = Path::new("../..");
+
+        assert_eq!(
+            PathBuf::from("/"), // existing path
+            canonicalize(relative_to, path)?,
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn canonicalize_should_fail() {
+        let relative_to = Path::new("/foo/bar/baz"); // '/foo' is missing
+        let path = Path::new("../..");
+
+        assert!(canonicalize(relative_to, path).is_err());
+    }
+}

crates/nu-engine/src/filesystem/utils.rs

@@ -0,0 +1,210 @@
+use crate::filesystem::path::canonicalize;
+use nu_errors::ShellError;
+use std::path::{Path, PathBuf};
+
+#[derive(Default)]
+pub struct FileStructure {
+    pub resources: Vec<Res>,
+}
+
+impl FileStructure {
+    pub fn new() -> FileStructure {
+        FileStructure {
+            resources: Vec::<Res>::new(),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn contains_more_than_one_file(&self) -> bool {
+        self.resources.len() > 1
+    }
+
+    #[allow(dead_code)]
+    pub fn contains_files(&self) -> bool {
+        !self.resources.is_empty()
+    }
+
+    pub fn paths_applying_with<F>(
+        &mut self,
+        to: F,
+    ) -> Result<Vec<(PathBuf, PathBuf)>, Box<dyn std::error::Error>>
+    where
+        F: Fn((PathBuf, usize)) -> Result<(PathBuf, PathBuf), Box<dyn std::error::Error>>,
+    {
+        self.resources
+            .iter()
+            .map(|f| (PathBuf::from(&f.loc), f.at))
+            .map(|f| to(f))
+            .collect()
+    }
+
+    pub fn walk_decorate(&mut self, start_path: &Path) -> Result<(), ShellError> {
+        self.resources = Vec::<Res>::new();
+        self.build(start_path, 0)?;
+        self.resources.sort();
+
+        Ok(())
+    }
+
+    fn build(&mut self, src: &Path, lvl: usize) -> Result<(), ShellError> {
+        let source = canonicalize(std::env::current_dir()?, src)?;
+
+        if source.is_dir() {
+            for entry in std::fs::read_dir(src)? {
+                let entry = entry?;
+                let path = entry.path();
+
+                if path.is_dir() {
+                    self.build(&path, lvl + 1)?;
+                }
+
+                self.resources.push(Res {
+                    loc: path.to_path_buf(),
+                    at: lvl,
+                });
+            }
+        } else {
+            self.resources.push(Res {
+                loc: source,
+                at: lvl,
+            });
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Eq, Ord, PartialEq, PartialOrd)]
+pub struct Res {
+    pub at: usize,
+    pub loc: PathBuf,
+}
+
+impl Res {}
+
+#[cfg(test)]
+mod tests {
+    use super::{FileStructure, Res};
+    use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value, ValueResource, ValueStructure};
+    use nu_source::Tag;
+    use nu_test_support::{fs::Stub::EmptyFile, playground::Playground};
+    use std::path::PathBuf;
+
+    fn structured_sample_record(key: &str, value: &str) -> Value {
+        let mut record = TaggedDictBuilder::new(Tag::unknown());
+        record.insert_untagged(key, UntaggedValue::string(value));
+        record.into_value()
+    }
+
+    fn sample_nushell_source_code() -> Value {
+        /*
+            src
+             commands
+              plugins => "sys.rs"
+             tests
+              helpers => "mod.rs"
+        */
+
+        let mut src = TaggedDictBuilder::new(Tag::unknown());
+        let mut record = TaggedDictBuilder::new(Tag::unknown());
+
+        record.insert_value("commands", structured_sample_record("plugins", "sys.rs"));
+        record.insert_value("tests", structured_sample_record("helpers", "mod.rs"));
+        src.insert_value("src", record.into_value());
+
+        src.into_value()
+    }
+
+    #[test]
+    fn prepares_and_decorates_value_filesystemlike_sources() {
+        let mut res = ValueStructure::new();
+
+        res.walk_decorate(&sample_nushell_source_code())
+            .expect("Can not decorate values traversal.");
+
+        assert_eq!(
+            res.resources,
+            vec![
+                ValueResource {
+                    loc: PathBuf::from("src"),
+                    at: 0,
+                },
+                ValueResource {
+                    loc: PathBuf::from("commands"),
+                    at: 1,
+                },
+                ValueResource {
+                    loc: PathBuf::from("tests"),
+                    at: 1,
+                },
+                ValueResource {
+                    loc: PathBuf::from("helpers"),
+                    at: 2,
+                },
+                ValueResource {
+                    loc: PathBuf::from("plugins"),
+                    at: 2,
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn recognizes_if_path_exists_in_value_filesystemlike_sources() {
+        let mut res = ValueStructure::new();
+
+        res.walk_decorate(&sample_nushell_source_code())
+            .expect("Can not decorate values traversal.");
+
+        assert!(res.exists(&PathBuf::from("/")));
+
+        assert!(res.exists(&PathBuf::from("src/commands/plugins")));
+        assert!(res.exists(&PathBuf::from("src/commands")));
+        assert!(res.exists(&PathBuf::from("src/tests")));
+        assert!(res.exists(&PathBuf::from("src/tests/helpers")));
+        assert!(res.exists(&PathBuf::from("src")));
+
+        assert!(res.exists(&PathBuf::from("/src/commands/plugins")));
+        assert!(res.exists(&PathBuf::from("/src/commands")));
+        assert!(res.exists(&PathBuf::from("/src/tests")));
+        assert!(res.exists(&PathBuf::from("/src/tests/helpers")));
+        assert!(res.exists(&PathBuf::from("/src")));
+
+        assert!(!res.exists(&PathBuf::from("/not_valid")));
+        assert!(!res.exists(&PathBuf::from("/src/not_valid")));
+    }
+
+    #[test]
+    fn prepares_and_decorates_filesystem_source_files() {
+        Playground::setup("file_structure_test", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("sample.ini"),
+                EmptyFile("sample.eml"),
+                EmptyFile("cargo_sample.toml"),
+            ]);
+
+            let mut res = FileStructure::new();
+
+            res.walk_decorate(&dirs.test())
+                .expect("Can not decorate files traversal.");
+
+            assert_eq!(
+                res.resources,
+                vec![
+                    Res {
+                        loc: dirs.test().join("cargo_sample.toml"),
+                        at: 0
+                    },
+                    Res {
+                        loc: dirs.test().join("sample.eml"),
+                        at: 0
+                    },
+                    Res {
+                        loc: dirs.test().join("sample.ini"),
+                        at: 0
+                    }
+                ]
+            );
+        })
+    }
+}