Enable reloading changes to a submodule (#13170)

# Description

Fixes: https://github.com/nushell/nushell/issues/12099

Currently if user run `use voice.nu`, and file is unchanged, then run
`use voice.nu` again. nushell will use the module directly, even if
submodule inside `voice.nu` is changed.

After discussed with @kubouch, I think it's ok to re-parse the module
file when:
1. It exports sub modules which are defined by a file
2. It uses other modules which are defined by a file

## About the change:
To achieve the behavior, we need to add 2 attributes to `Module`:
1. `imported_modules`: it tracks the other modules is imported by the
givem `module`, e.g: `use foo.nu`
2. `file`: the path of a module, if a module is defined by a file, it
will be `Some(path)`, or else it will be `None`.

After the change:

    use voice.nu always read the file and parse it.
    use voice will still use the module which is saved in EngineState.

# User-Facing Changes

use `xxx.nu` will read the file and parse it if it exports submodules or
uses submodules

# Tests + Formatting

Done

---------

Co-authored-by: Jakub Žádník <kubouch@gmail.com>

crates/nu-protocol/src/lib.rs

@@ -11,6 +11,7 @@ mod example;
 mod id;
 mod lev_distance;
 mod module;
+pub mod parser_path;
 mod pipeline;
 #[cfg(feature = "plugin")]
 mod plugin;

crates/nu-protocol/src/module.rs

@@ -1,8 +1,9 @@
 use crate::{
-    ast::ImportPatternMember, engine::StateWorkingSet, BlockId, DeclId, ModuleId, ParseError, Span,
-    Value, VarId,
+    ast::ImportPatternMember, engine::StateWorkingSet, BlockId, DeclId, FileId, ModuleId,
+    ParseError, Span, Value, VarId,
 };
 
+use crate::parser_path::ParserPath;
 use indexmap::IndexMap;
 
 pub struct ResolvedImportPattern {
@@ -35,6 +36,8 @@ pub struct Module {
     pub env_block: Option<BlockId>, // `export-env { ... }` block
     pub main: Option<DeclId>,       // `export def main`
     pub span: Option<Span>,
+    pub imported_modules: Vec<ModuleId>, // use other_module.nu
+    pub file: Option<(ParserPath, FileId)>,
 }
 
 impl Module {
@@ -47,6 +50,8 @@ impl Module {
             env_block: None,
             main: None,
             span: None,
+            imported_modules: vec![],
+            file: None,
         }
     }
 
@@ -59,6 +64,8 @@ impl Module {
             env_block: None,
             main: None,
             span: Some(span),
+            imported_modules: vec![],
+            file: None,
         }
     }
 
@@ -82,6 +89,12 @@ impl Module {
         self.env_block = Some(block_id);
     }
 
+    pub fn track_imported_modules(&mut self, module_id: &[ModuleId]) {
+        for m in module_id {
+            self.imported_modules.push(*m)
+        }
+    }
+
     pub fn has_decl(&self, name: &[u8]) -> bool {
         if name == self.name && self.main.is_some() {
             return true;
@@ -90,6 +103,9 @@ impl Module {
         self.decls.contains_key(name)
     }
 
+    /// Resolve `members` from given module, which is indicated by `self_id` to import.
+    ///
+    /// When resolving, all modules are recorded in `imported_modules`.
     pub fn resolve_import_pattern(
         &self,
         working_set: &StateWorkingSet,
@@ -97,7 +113,9 @@ impl Module {
         members: &[ImportPatternMember],
         name_override: Option<&[u8]>, // name under the module was stored (doesn't have to be the same as self.name)
         backup_span: Span,
+        imported_modules: &mut Vec<ModuleId>,
     ) -> (ResolvedImportPattern, Vec<ParseError>) {
+        imported_modules.push(self_id);
         let final_name = name_override.unwrap_or(&self.name).to_vec();
 
         let (head, rest) = if let Some((head, rest)) = members.split_first() {
@@ -112,8 +130,14 @@ impl Module {
                 let submodule = working_set.get_module(*id);
                 let span = submodule.span.or(self.span).unwrap_or(backup_span);
 
-                let (sub_results, sub_errors) =
-                    submodule.resolve_import_pattern(working_set, *id, &[], None, span);
+                let (sub_results, sub_errors) = submodule.resolve_import_pattern(
+                    working_set,
+                    *id,
+                    &[],
+                    None,
+                    span,
+                    imported_modules,
+                );
                 errors.extend(sub_errors);
 
                 for (sub_name, sub_decl_id) in sub_results.decls {
@@ -212,6 +236,7 @@ impl Module {
                         rest,
                         None,
                         self.span.unwrap_or(backup_span),
+                        imported_modules,
                     )
                 } else {
                     (
@@ -234,6 +259,7 @@ impl Module {
                         &[],
                         None,
                         self.span.unwrap_or(backup_span),
+                        imported_modules,
                     );
                     decls.extend(sub_results.decls);
 
@@ -287,6 +313,7 @@ impl Module {
                             rest,
                             None,
                             self.span.unwrap_or(backup_span),
+                            imported_modules,
                         );
 
                         decls.extend(sub_results.decls);

crates/nu-protocol/src/parser_path.rs

@@ -0,0 +1,152 @@
+use crate::engine::{StateWorkingSet, VirtualPath};
+use std::{
+    ffi::OsStr,
+    path::{Path, PathBuf},
+};
+
+/// An abstraction over a PathBuf that can have virtual paths (files and directories). Virtual
+/// paths always exist and represent a way to ship Nushell code inside the binary without requiring
+/// paths to be present in the file system.
+///
+/// Created from VirtualPath found in the engine state.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ParserPath {
+    RealPath(PathBuf),
+    VirtualFile(PathBuf, usize),
+    VirtualDir(PathBuf, Vec<ParserPath>),
+}
+
+impl ParserPath {
+    pub fn is_dir(&self) -> bool {
+        match self {
+            ParserPath::RealPath(p) => p.is_dir(),
+            ParserPath::VirtualFile(..) => false,
+            ParserPath::VirtualDir(..) => true,
+        }
+    }
+
+    pub fn is_file(&self) -> bool {
+        match self {
+            ParserPath::RealPath(p) => p.is_file(),
+            ParserPath::VirtualFile(..) => true,
+            ParserPath::VirtualDir(..) => false,
+        }
+    }
+
+    pub fn exists(&self) -> bool {
+        match self {
+            ParserPath::RealPath(p) => p.exists(),
+            ParserPath::VirtualFile(..) => true,
+            ParserPath::VirtualDir(..) => true,
+        }
+    }
+
+    pub fn path(&self) -> &Path {
+        match self {
+            ParserPath::RealPath(p) => p,
+            ParserPath::VirtualFile(p, _) => p,
+            ParserPath::VirtualDir(p, _) => p,
+        }
+    }
+
+    pub fn path_buf(self) -> PathBuf {
+        match self {
+            ParserPath::RealPath(p) => p,
+            ParserPath::VirtualFile(p, _) => p,
+            ParserPath::VirtualDir(p, _) => p,
+        }
+    }
+
+    pub fn parent(&self) -> Option<&Path> {
+        match self {
+            ParserPath::RealPath(p) => p.parent(),
+            ParserPath::VirtualFile(p, _) => p.parent(),
+            ParserPath::VirtualDir(p, _) => p.parent(),
+        }
+    }
+
+    pub fn read_dir(&self) -> Option<Vec<ParserPath>> {
+        match self {
+            ParserPath::RealPath(p) => p.read_dir().ok().map(|read_dir| {
+                read_dir
+                    .flatten()
+                    .map(|dir_entry| ParserPath::RealPath(dir_entry.path()))
+                    .collect()
+            }),
+            ParserPath::VirtualFile(..) => None,
+            ParserPath::VirtualDir(_, files) => Some(files.clone()),
+        }
+    }
+
+    pub fn file_stem(&self) -> Option<&OsStr> {
+        self.path().file_stem()
+    }
+
+    pub fn extension(&self) -> Option<&OsStr> {
+        self.path().extension()
+    }
+
+    pub fn join(self, path: impl AsRef<Path>) -> ParserPath {
+        match self {
+            ParserPath::RealPath(p) => ParserPath::RealPath(p.join(path)),
+            ParserPath::VirtualFile(p, file_id) => ParserPath::VirtualFile(p.join(path), file_id),
+            ParserPath::VirtualDir(p, entries) => {
+                let new_p = p.join(path);
+                let mut pp = ParserPath::RealPath(new_p.clone());
+                for entry in entries {
+                    if new_p == entry.path() {
+                        pp = entry.clone();
+                    }
+                }
+                pp
+            }
+        }
+    }
+
+    pub fn open<'a>(
+        &'a self,
+        working_set: &'a StateWorkingSet,
+    ) -> std::io::Result<Box<dyn std::io::Read + 'a>> {
+        match self {
+            ParserPath::RealPath(p) => {
+                std::fs::File::open(p).map(|f| Box::new(f) as Box<dyn std::io::Read>)
+            }
+            ParserPath::VirtualFile(_, file_id) => working_set
+                .get_contents_of_file(*file_id)
+                .map(|bytes| Box::new(bytes) as Box<dyn std::io::Read>)
+                .ok_or(std::io::ErrorKind::NotFound.into()),
+
+            ParserPath::VirtualDir(..) => Err(std::io::ErrorKind::NotFound.into()),
+        }
+    }
+
+    pub fn read<'a>(&'a self, working_set: &'a StateWorkingSet) -> Option<Vec<u8>> {
+        self.open(working_set)
+            .and_then(|mut reader| {
+                let mut vec = vec![];
+                reader.read_to_end(&mut vec)?;
+                Ok(vec)
+            })
+            .ok()
+    }
+
+    pub fn from_virtual_path(
+        working_set: &StateWorkingSet,
+        name: &str,
+        virtual_path: &VirtualPath,
+    ) -> Self {
+        match virtual_path {
+            VirtualPath::File(file_id) => ParserPath::VirtualFile(PathBuf::from(name), *file_id),
+            VirtualPath::Dir(entries) => ParserPath::VirtualDir(
+                PathBuf::from(name),
+                entries
+                    .iter()
+                    .map(|virtual_path_id| {
+                        let (virt_name, virt_path) = working_set.get_virtual_path(*virtual_path_id);
+                        ParserPath::from_virtual_path(working_set, virt_name, virt_path)
+                    })
+                    .collect(),
+            ),
+        }
+    }
+}