Fix editor config for reedline and config nu/env (#10535)

# Description
This merges @horasal 's changes from #10246 and #10269

Closes #10205
Closes #8714

Fixes the bug that editor paths with spaces are unusable

Closes #10210 
Closes #10269


# User-Facing Changes
You can now either pass a string with the name of the executable or a
list with the executable and any flags to
`$env.config.buffer_editor`/`$env.EDITOR`/`$env.VISUAL`

Both the external buffer editor of reedline (by default bound to
`Ctrl-o`) and the commands `config nu` and `config env` will respect
those variables in the following order:
1. `$env.config.buffer_editor`
2. `$env.EDITOR`
3. `$env.VISUAL`

Example:
```
$env.EDITOR = "nvim"                      # The system-wide EDITOR is neovim
$env.config.buffer_editor = ["vim" "-p2"] # Force vim to open two tabs (not particularly useful)
$env.config.buffer_editor = null          # Unset `buffer_editor` -> Uses `$env.EDITOR` ergo nvim
```
# Tests + Formatting
None

---------

Co-authored-by: Horasal <1991933+horasal@users.noreply.github.com>

crates/nu-cmd-base/src/util.rs

@@ -55,3 +55,72 @@ pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> {
 
     Ok((start, end))
 }
+
+const HELP_MSG: &str = "Nushell's config file can be found with the command: $nu.config-path. \
+For more help: (https://nushell.sh/book/configuration.html#configurations-with-built-in-commands)";
+
+fn get_editor_commandline(
+    value: &Value,
+    var_name: &str,
+) -> Result<(String, Vec<String>), ShellError> {
+    match value {
+        Value::String { val, .. } if !val.is_empty() => Ok((val.to_string(), Vec::new())),
+        Value::List { vals, .. } if !vals.is_empty() => {
+            let mut editor_cmd = vals.iter().map(|l| l.as_string());
+            match editor_cmd.next().transpose()? {
+                Some(editor) if !editor.is_empty() => {
+                    let params = editor_cmd.collect::<Result<_, ShellError>>()?;
+                    Ok((editor, params))
+                }
+                _ => Err(ShellError::GenericError(
+                    "Editor executable is missing".into(),
+                    "Set the first element to an executable".into(),
+                    Some(value.span()),
+                    Some(HELP_MSG.into()),
+                    vec![],
+                )),
+            }
+        }
+        Value::String { .. } | Value::List { .. } => Err(ShellError::GenericError(
+            format!("{var_name} should be a non-empty string or list<String>"),
+            "Specify an executable here".into(),
+            Some(value.span()),
+            Some(HELP_MSG.into()),
+            vec![],
+        )),
+        x => Err(ShellError::CantConvert {
+            to_type: "string or list<string>".into(),
+            from_type: x.get_type().to_string(),
+            span: value.span(),
+            help: None,
+        }),
+    }
+}
+
+pub fn get_editor(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    span: Span,
+) -> Result<(String, Vec<String>), ShellError> {
+    let config = engine_state.get_config();
+    let env_vars = stack.get_env_vars(engine_state);
+
+    if let Ok(buff_editor) =
+        get_editor_commandline(&config.buffer_editor, "$env.config.buffer_editor")
+    {
+        Ok(buff_editor)
+    } else if let Some(value) = env_vars.get("EDITOR") {
+        get_editor_commandline(value, "$env.EDITOR")
+    } else if let Some(value) = env_vars.get("VISUAL") {
+        get_editor_commandline(value, "$env.VISUAL")
+    } else {
+        Err(ShellError::GenericError(
+            "No editor configured".into(),
+            "Please specify one via `$env.config.buffer_editor` or `$env.EDITOR`/`$env.VISUAL`"
+                .into(),
+            Some(span),
+            Some(HELP_MSG.into()),
+            vec![],
+        ))
+    }
+}