feat(lsp): use lsp-textdocument to handle utf16 position (#14742)

<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx

you can also mention related issues, PRs or discussions!
-->

# Description
<!--
Thank you for improving Nushell. Please, check our [contributing
guide](../CONTRIBUTING.md) and talk to the core team before making major
changes.

Description of your pull request goes here. **Provide examples and/or
screenshots** if your changes affect the user experience.
-->

This PR replaces `ropey` with `lsp-textdocument` for easier utf16
position handling.
As a side effect, if fixes the following crashing bug:

1. create a `foo.nu` file with errors in it
2. in `bar.nu`, add code `use foo.nu *`

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

* <s>Diagnostics are now triggered only with document open/save, that's
my personal preference. Changing back to previous behavior is easy if
you guys have other concerns.</s>
* UTF-8 position encoding is not supported by lsp-textdocument, but
that's not an issue, since the previous utf-8 ropey implementation is
buggy when used in real scenarios in a text editor.

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the
tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

No new tests added, removed some utf-8 related ones.

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

Cargo.lock

@@ -1964,6 +1964,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "fluent-uri"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -3195,16 +3204,26 @@ dependencies = [
 ]
 
 [[package]]
-name = "lsp-types"
-version = "0.95.1"
+name = "lsp-textdocument"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e34d33a8e9b006cd3fc4fe69a921affa097bae4bb65f76271f4644f9a334365"
+checksum = "d8dc223af95101fe950a871d4d567b6f98a1ecfcee5861f4b57644581aaa980d"
+dependencies = [
+ "lsp-types",
+ "serde_json",
+]
+
+[[package]]
+name = "lsp-types"
+version = "0.97.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53353550a17c04ac46c585feb189c2db82154fc84b79c7a66c96c2c644f66071"
 dependencies = [
  "bitflags 1.3.2",
+ "fluent-uri",
  "serde",
  "serde_json",
  "serde_repr",
- "url",
 ]
 
 [[package]]
@@ -3924,6 +3943,7 @@ dependencies = [
  "assert-json-diff",
  "crossbeam-channel",
  "lsp-server",
+ "lsp-textdocument",
  "lsp-types",
  "miette",
  "nu-cli",
@@ -3933,9 +3953,9 @@ dependencies = [
  "nu-protocol",
  "nu-test-support",
  "reedline",
- "ropey",
  "serde",
  "serde_json",
+ "url",
 ]
 
 [[package]]
@@ -6247,16 +6267,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "ropey"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5"
-dependencies = [
- "smallvec",
- "str_indices",
-]
-
 [[package]]
 name = "roxmltree"
 version = "0.20.0"
@@ -7015,12 +7025,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
-[[package]]
-name = "str_indices"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6"
-
 [[package]]
 name = "streaming-decompression"
 version = "0.1.2"
@@ -7800,7 +7804,6 @@ dependencies = [
  "form_urlencoded",
  "idna",
  "percent-encoding",
- "serde",
 ]
 
 [[package]]

Cargo.toml

@@ -103,7 +103,8 @@ log = "0.4"
 lru = "0.12"
 lscolors = { version = "0.17", default-features = false }
 lsp-server = "0.7.5"
-lsp-types = { version = "0.95.0", features = ["proposed"] }
+lsp-types = { version = "0.97.0", features = ["proposed"] }
+lsp-textdocument = "0.4.0"
 mach2 = "0.4"
 md5 = { version = "0.10", package = "md-5" }
 miette = "7.3"
@@ -139,10 +140,8 @@ rand_chacha = "0.3.1"
 ratatui = "0.26"
 rayon = "1.10"
 reedline = "0.38.0"
-regex = "1.9.5"
 rmp = "0.8"
 rmp-serde = "1.3"
-ropey = "1.6.1"
 roxmltree = "0.20"
 rstest = { version = "0.23", default-features = false }
 rusqlite = "0.31"
@@ -330,4 +329,4 @@ bench = false
 # Run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`
 [[bench]]
 name = "benchmarks"
-harness = false
+harness = false

crates/nu-lsp/Cargo.toml

@@ -15,12 +15,13 @@ nu-protocol = { path = "../nu-protocol", version = "0.101.1" }
 reedline = { workspace = true }
 
 crossbeam-channel = { workspace = true }
-lsp-types = { workspace = true }
 lsp-server = { workspace = true }
+lsp-types = { workspace = true }
+lsp-textdocument = { workspace = true }
 miette = { workspace = true }
-ropey = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+url = { workspace = true }
 
 [dev-dependencies]
 nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.101.1" }
@@ -30,4 +31,4 @@ nu-test-support = { path = "../nu-test-support", version = "0.101.1" }
 assert-json-diff = "2.0"
 
 [lints]
-workspace = true
+workspace = true

crates/nu-lsp/src/diagnostics.rs

@@ -1,41 +1,24 @@
 use crate::LanguageServer;
 use lsp_types::{
     notification::{Notification, PublishDiagnostics},
-    Diagnostic, DiagnosticSeverity, PublishDiagnosticsParams, Url,
+    Diagnostic, DiagnosticSeverity, PublishDiagnosticsParams, Uri,
 };
 use miette::{IntoDiagnostic, Result};
-use nu_parser::parse;
-use nu_protocol::{
-    engine::{EngineState, StateWorkingSet},
-    Span, Value,
-};
+use nu_protocol::Value;
 
 impl LanguageServer {
-    pub(crate) fn publish_diagnostics_for_file(
-        &self,
-        uri: Url,
-        engine_state: &mut EngineState,
-    ) -> Result<()> {
+    pub(crate) fn publish_diagnostics_for_file(&mut self, uri: Uri) -> Result<()> {
+        let mut engine_state = self.engine_state.clone();
         let cwd = std::env::current_dir().expect("Could not get current working directory.");
         engine_state.add_env_var("PWD".into(), Value::test_string(cwd.to_string_lossy()));
         engine_state.generate_nu_constant();
 
-        let mut working_set = StateWorkingSet::new(engine_state);
-
-        let Some((rope_of_file, file_path)) = self.rope(&uri) else {
+        let Some((_, offset, working_set, file)) =
+            self.update_engine_state(&mut engine_state, &uri)
+        else {
             return Ok(());
         };
 
-        let contents = rope_of_file.bytes().collect::<Vec<u8>>();
-        let offset = working_set.next_span_start();
-        working_set.files.push(file_path.into(), Span::unknown())?;
-        parse(
-            &mut working_set,
-            Some(&file_path.to_string_lossy()),
-            &contents,
-            false,
-        );
-
         let mut diagnostics = PublishDiagnosticsParams {
             uri,
             diagnostics: Vec::new(),
@@ -46,12 +29,7 @@ impl LanguageServer {
             let message = err.to_string();
 
             diagnostics.diagnostics.push(Diagnostic {
-                range: Self::span_to_range(
-                    &err.span(),
-                    rope_of_file,
-                    offset,
-                    &self.position_encoding,
-                ),
+                range: Self::span_to_range(&err.span(), file, offset),
                 severity: Some(DiagnosticSeverity::ERROR),
                 message,
                 ..Default::default()
@@ -70,20 +48,20 @@ impl LanguageServer {
 #[cfg(test)]
 mod tests {
     use assert_json_diff::assert_json_eq;
-    use lsp_types::Url;
     use nu_test_support::fs::fixtures;
 
+    use crate::path_to_uri;
     use crate::tests::{initialize_language_server, open_unchecked, update};
 
     #[test]
     fn publish_diagnostics_variable_does_not_exists() {
-        let (client_connection, _recv) = initialize_language_server(None);
+        let (client_connection, _recv) = initialize_language_server();
 
         let mut script = fixtures();
         script.push("lsp");
         script.push("diagnostics");
         script.push("var.nu");
-        let script = Url::from_file_path(script).unwrap();
+        let script = path_to_uri(&script);
 
         let notification = open_unchecked(&client_connection, script.clone());
 
@@ -108,13 +86,13 @@ mod tests {
 
     #[test]
     fn publish_diagnostics_fixed_unknown_variable() {
-        let (client_connection, _recv) = initialize_language_server(None);
+        let (client_connection, _recv) = initialize_language_server();
 
         let mut script = fixtures();
         script.push("lsp");
         script.push("diagnostics");
         script.push("var.nu");
-        let script = Url::from_file_path(script).unwrap();
+        let script = path_to_uri(&script);
 
         open_unchecked(&client_connection, script.clone());
         let notification = update(

crates/nu-lsp/src/notification.rs

@@ -1,10 +1,7 @@
 use lsp_types::{
-    notification::{
-        DidChangeTextDocument, DidCloseTextDocument, DidOpenTextDocument, Notification,
-    },
-    DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, Url,
+    notification::{DidChangeTextDocument, DidOpenTextDocument, DidSaveTextDocument, Notification},
+    DidChangeTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams, Uri,
 };
-use ropey::Rope;
 
 use crate::LanguageServer;
 
@@ -12,107 +9,52 @@ impl LanguageServer {
     pub(crate) fn handle_lsp_notification(
         &mut self,
         notification: lsp_server::Notification,
-    ) -> Option<Url> {
+    ) -> Option<Uri> {
+        self.docs
+            .listen(notification.method.as_str(), &notification.params);
         match notification.method.as_str() {
-            DidOpenTextDocument::METHOD => Self::handle_notification_payload::<
-                DidOpenTextDocumentParams,
-                _,
-            >(notification, |param| {
-                if let Ok(file_path) = param.text_document.uri.to_file_path() {
-                    let rope = Rope::from_str(&param.text_document.text);
-                    self.ropes.insert(file_path, rope);
-                    Some(param.text_document.uri)
-                } else {
-                    None
-                }
-            }),
+            DidOpenTextDocument::METHOD => {
+                let params: DidOpenTextDocumentParams =
+                    serde_json::from_value(notification.params.clone())
+                        .expect("Expect receive DidOpenTextDocumentParams");
+                Some(params.text_document.uri)
+            }
+            DidSaveTextDocument::METHOD => {
+                let params: DidSaveTextDocumentParams =
+                    serde_json::from_value(notification.params.clone())
+                        .expect("Expect receive DidSaveTextDocumentParams");
+                Some(params.text_document.uri)
+            }
             DidChangeTextDocument::METHOD => {
-                Self::handle_notification_payload::<DidChangeTextDocumentParams, _>(
-                    notification,
-                    |params| self.update_rope(params),
-                )
+                let params: DidChangeTextDocumentParams =
+                    serde_json::from_value(notification.params.clone())
+                        .expect("Expect receive DidChangeTextDocumentParams");
+                Some(params.text_document.uri)
             }
-            DidCloseTextDocument::METHOD => Self::handle_notification_payload::<
-                DidCloseTextDocumentParams,
-                _,
-            >(notification, |param| {
-                if let Ok(file_path) = param.text_document.uri.to_file_path() {
-                    self.ropes.remove(&file_path);
-                }
-                None
-            }),
             _ => None,
         }
     }
-
-    fn handle_notification_payload<P, H>(
-        notification: lsp_server::Notification,
-        mut param_handler: H,
-    ) -> Option<Url>
-    where
-        P: serde::de::DeserializeOwned,
-        H: FnMut(P) -> Option<Url>,
-    {
-        if let Ok(params) = serde_json::from_value::<P>(notification.params) {
-            param_handler(params)
-        } else {
-            None
-        }
-    }
-
-    fn update_rope(&mut self, params: DidChangeTextDocumentParams) -> Option<Url> {
-        if let Ok(file_path) = params.text_document.uri.to_file_path() {
-            for content_change in params.content_changes.into_iter() {
-                let entry = self.ropes.entry(file_path.clone());
-                match content_change.range {
-                    Some(range) => {
-                        entry.and_modify(|rope| {
-                            let start = Self::lsp_position_to_location(
-                                &range.start,
-                                rope,
-                                &self.position_encoding,
-                            );
-                            let end = Self::lsp_position_to_location(
-                                &range.end,
-                                rope,
-                                &self.position_encoding,
-                            );
-
-                            rope.remove(start..end);
-                            rope.insert(start, &content_change.text);
-                        });
-                    }
-                    None => {
-                        entry.and_modify(|r| *r = Rope::from_str(&content_change.text));
-                    }
-                }
-            }
-
-            Some(params.text_document.uri)
-        } else {
-            None
-        }
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use assert_json_diff::assert_json_eq;
     use lsp_server::Message;
-    use lsp_types::{Range, Url};
+    use lsp_types::Range;
     use nu_test_support::fs::fixtures;
 
+    use crate::path_to_uri;
     use crate::tests::{hover, initialize_language_server, open, open_unchecked, update};
 
     #[test]
     fn hover_correct_documentation_on_let() {
-        let (client_connection, _recv) = initialize_language_server(None);
+        let (client_connection, _recv) = initialize_language_server();
 
         let mut script = fixtures();
         script.push("lsp");
         script.push("hover");
         script.push("var.nu");
-        let script = Url::from_file_path(script).unwrap();
+        let script = path_to_uri(&script);
 
         open_unchecked(&client_connection, script.clone());
 
@@ -136,13 +78,13 @@ mod tests {
 
     #[test]
     fn hover_on_command_after_full_content_change() {
-        let (client_connection, _recv) = initialize_language_server(None);
+        let (client_connection, _recv) = initialize_language_server();
 
         let mut script = fixtures();
         script.push("lsp");
         script.push("hover");
         script.push("command.nu");
-        let script = Url::from_file_path(script).unwrap();
+        let script = path_to_uri(&script);
 
         open_unchecked(&client_connection, script.clone());
         update(
@@ -177,13 +119,13 @@ hello"#,
 
     #[test]
     fn hover_on_command_after_partial_content_change() {
-        let (client_connection, _recv) = initialize_language_server(None);
+        let (client_connection, _recv) = initialize_language_server();
 
         let mut script = fixtures();
         script.push("lsp");
         script.push("hover");
         script.push("command.nu");
-        let script = Url::from_file_path(script).unwrap();
+        let script = path_to_uri(&script);
 
         open_unchecked(&client_connection, script.clone());
         update(
@@ -222,13 +164,13 @@ hello"#,
 
     #[test]
     fn open_document_with_utf_char() {
-        let (client_connection, _recv) = initialize_language_server(None);
+        let (client_connection, _recv) = initialize_language_server();
 
         let mut script = fixtures();
         script.push("lsp");
         script.push("notifications");
         script.push("issue_11522.nu");
-        let script = Url::from_file_path(script).unwrap();
+        let script = path_to_uri(&script);
 
         let result = open(&client_connection, script);
 

src/main.rs

@@ -460,7 +460,7 @@ fn main() -> Result<()> {
             );
         }
 
-        LanguageServer::initialize_stdio_connection()?.serve_requests(engine_state)?
+        LanguageServer::initialize_stdio_connection(engine_state)?.serve_requests()?
     } else if let Some(commands) = parsed_nu_cli_args.commands.clone() {
         run_commands(
             &mut engine_state,