From 9e3d6c3bfd4fd1869c5949fd8985f18b39e11660 Mon Sep 17 00:00:00 2001 From: JT <547158+jntrnr@users.noreply.github.com> Date: Mon, 10 Apr 2023 08:55:47 +1200 Subject: [PATCH] Only add the std lib files once (#8830) # Description We were seeing duplicate entries for the std lib files, and this PR addresses that. Each file should now only be added once. Note: they are still parsed twice because it's hard to recover the module from the output of `parse` but a bit of clever hacking in a future PR might be able to do that. # User-Facing Changes _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --- .../src/completions/command_completions.rs | 2 +- crates/nu-command/src/system/nu_check.rs | 6 ++--- crates/nu-command/tests/main.rs | 2 +- crates/nu-parser/src/parse_keywords.rs | 24 +++++++------------ crates/nu-parser/src/parser.rs | 7 +++--- crates/nu-protocol/src/engine/engine_state.rs | 22 ++++++++++++++++- crates/nu-std/src/lib.rs | 8 +++---- 7 files changed, 39 insertions(+), 32 deletions(-) diff --git a/crates/nu-cli/src/completions/command_completions.rs b/crates/nu-cli/src/completions/command_completions.rs index 80737f3fa..336946210 100644 --- a/crates/nu-cli/src/completions/command_completions.rs +++ b/crates/nu-cli/src/completions/command_completions.rs @@ -299,7 +299,7 @@ mod command_completions_tests { let delta = { let mut working_set = StateWorkingSet::new(&engine_state); - working_set.add_file("child.nu".into(), input); + let _ = working_set.add_file("child.nu".into(), input); working_set.render() }; diff --git a/crates/nu-command/src/system/nu_check.rs b/crates/nu-command/src/system/nu_check.rs index f5e854936..0ad65f1d5 100644 --- a/crates/nu-command/src/system/nu_check.rs +++ b/crates/nu-command/src/system/nu_check.rs @@ -314,11 +314,9 @@ fn parse_module( ) -> Result { let filename = filename.unwrap_or_else(|| "empty".to_string()); - let start = working_set.next_span_start(); - working_set.add_file(filename.clone(), contents); - let end = working_set.next_span_start(); + let file_id = working_set.add_file(filename.clone(), contents); + let new_span = working_set.get_span_for_file(file_id); - let new_span = Span::new(start, end); let starting_error_count = working_set.parse_errors.len(); parse_module_block(working_set, new_span, filename.as_bytes()); diff --git a/crates/nu-command/tests/main.rs b/crates/nu-command/tests/main.rs index 057f6e9a7..5ecefec90 100644 --- a/crates/nu-command/tests/main.rs +++ b/crates/nu-command/tests/main.rs @@ -13,7 +13,7 @@ fn quickcheck_parse(data: String) -> bool { let context = create_default_context(); { let mut working_set = StateWorkingSet::new(&context); - working_set.add_file("quickcheck".into(), data.as_bytes()); + let _ = working_set.add_file("quickcheck".into(), data.as_bytes()); let _ = nu_parser::parse_block(&mut working_set, &tokens, false, false); } diff --git a/crates/nu-parser/src/parse_keywords.rs b/crates/nu-parser/src/parse_keywords.rs index ce5e90338..f158d821f 100644 --- a/crates/nu-parser/src/parse_keywords.rs +++ b/crates/nu-parser/src/parse_keywords.rs @@ -1719,9 +1719,8 @@ pub fn parse_use(working_set: &mut StateWorkingSet, spans: &[Span]) -> (Pipeline }; if let Ok(contents) = std::fs::read(&module_path) { - let span_start = working_set.next_span_start(); - working_set.add_file(module_filename, &contents); - let span_end = working_set.next_span_start(); + let file_id = working_set.add_file(module_filename, &contents); + let new_span = working_set.get_span_for_file(file_id); // Change the currently parsed directory let prev_currently_parsed_cwd = if let Some(parent) = module_path.parent() { @@ -1738,11 +1737,8 @@ pub fn parse_use(working_set: &mut StateWorkingSet, spans: &[Span]) -> (Pipeline working_set.parsed_module_files.push(module_path); // Parse the module - let (block, module, module_comments) = parse_module_block( - working_set, - Span::new(span_start, span_end), - module_name.as_bytes(), - ); + let (block, module, module_comments) = + parse_module_block(working_set, new_span, module_name.as_bytes()); // Remove the file from the stack of parsed module files working_set.parsed_module_files.pop(); @@ -2245,9 +2241,8 @@ pub fn parse_overlay_use(working_set: &mut StateWorkingSet, call: Box) -> }; if let Ok(contents) = std::fs::read(&module_path) { - let span_start = working_set.next_span_start(); - working_set.add_file(module_filename, &contents); - let span_end = working_set.next_span_start(); + let file_id = working_set.add_file(module_filename, &contents); + let new_span = working_set.get_span_for_file(file_id); // Change currently parsed directory let prev_currently_parsed_cwd = @@ -2261,11 +2256,8 @@ pub fn parse_overlay_use(working_set: &mut StateWorkingSet, call: Box) -> working_set.currently_parsed_cwd.clone() }; - let (block, module, module_comments) = parse_module_block( - working_set, - Span::new(span_start, span_end), - overlay_name.as_bytes(), - ); + let (block, module, module_comments) = + parse_module_block(working_set, new_span, overlay_name.as_bytes()); // Restore the currently parsed directory back working_set.currently_parsed_cwd = prev_currently_parsed_cwd; diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 6dae201d6..41d278c47 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -5896,16 +5896,15 @@ pub fn parse( contents: &[u8], scoped: bool, ) -> Block { - let span_offset = working_set.next_span_start(); - let name = match fname { Some(fname) => fname.to_string(), None => "source".to_string(), }; - working_set.add_file(name, contents); + let file_id = working_set.add_file(name, contents); + let new_span = working_set.get_span_for_file(file_id); - let (output, err) = lex(contents, span_offset, &[], &[], false); + let (output, err) = lex(contents, new_span.start, &[], &[], false); if let Some(err) = err { working_set.error(err) } diff --git a/crates/nu-protocol/src/engine/engine_state.rs b/crates/nu-protocol/src/engine/engine_state.rs index 0ac5ac3b4..dfed7272c 100644 --- a/crates/nu-protocol/src/engine/engine_state.rs +++ b/crates/nu-protocol/src/engine/engine_state.rs @@ -1345,7 +1345,18 @@ impl<'a> StateWorkingSet<'a> { "".into() } + #[must_use] pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize { + // First, look for the file to see if we already have it + for (idx, (fname, file_start, file_end)) in self.files().enumerate() { + if fname == &filename { + let prev_contents = self.get_span_contents(Span::new(*file_start, *file_end)); + if prev_contents == contents { + return idx; + } + } + } + let next_span_start = self.next_span_start(); let next_span_end = next_span_start + contents.len(); @@ -1360,6 +1371,15 @@ impl<'a> StateWorkingSet<'a> { self.num_files() - 1 } + pub fn get_span_for_file(&self, file_id: usize) -> Span { + let result = self + .files() + .nth(file_id) + .expect("internal error: could not find source for previously parsed file"); + + Span::new(result.1, result.2) + } + pub fn get_span_contents(&self, span: Span) -> &[u8] { let permanent_end = self.permanent_state.next_span_start(); if permanent_end <= span.start { @@ -2113,7 +2133,7 @@ mod engine_state_tests { let delta = { let mut working_set = StateWorkingSet::new(&engine_state); - working_set.add_file("child.nu".into(), &[]); + let _ = working_set.add_file("child.nu".into(), &[]); working_set.render() }; diff --git a/crates/nu-std/src/lib.rs b/crates/nu-std/src/lib.rs index 674805b15..7c847cb49 100644 --- a/crates/nu-std/src/lib.rs +++ b/crates/nu-std/src/lib.rs @@ -7,12 +7,10 @@ fn add_file( name: &String, content: &[u8], ) -> (Module, Vec) { - let start = working_set.next_span_start(); - working_set.add_file(name.clone(), content); - let end = working_set.next_span_start(); + let file_id = working_set.add_file(name.clone(), content); + let new_span = working_set.get_span_for_file(file_id); - let (_, module, comments) = - parse_module_block(working_set, Span::new(start, end), name.as_bytes()); + let (_, module, comments) = parse_module_block(working_set, new_span, name.as_bytes()); if let Some(err) = working_set.parse_errors.first() { report_error(working_set, err);