Local socket mode and foreground terminal control for plugins (#12448)

# Description

Adds support for running plugins using local socket communication
instead of stdio. This will be an optional thing that not all plugins
have to support.

This frees up stdio for use to make plugins that use stdio to create
terminal UIs, cc @amtoine, @fdncred.

This uses the [`interprocess`](https://crates.io/crates/interprocess)
crate (298 stars, MIT license, actively maintained), which seems to be
the best option for cross-platform local socket support in Rust. On
Windows, a local socket name is provided. On Unixes, it's a path. The
socket name is kept to a relatively small size because some operating
systems have pretty strict limits on the whole path (~100 chars), so on
macOS for example we prefer `/tmp/nu.{pid}.{hash64}.sock` where the hash
includes the plugin filename and timestamp to be unique enough.

This also adds an API for moving plugins in and out of the foreground
group, which is relevant for Unixes where direct terminal control
depends on that.

TODO:

- [x] Generate local socket path according to OS conventions
- [x] Add support for passing `--local-socket` to the plugin executable
instead of `--stdio`, and communicating over that instead
- [x] Test plugins that were broken, including
[amtoine/nu_plugin_explore](https://github.com/amtoine/nu_plugin_explore)
- [x] Automatically upgrade to using local sockets when supported,
falling back if it doesn't work, transparently to the user without any
visible error messages
  - Added protocol feature: `LocalSocket`
- [x] Reset preferred mode to `None` on `register`
- [x] Allow plugins to detect whether they're running on a local socket
and can use stdio freely, so that TUI plugins can just produce an error
message otherwise
  - Implemented via `EngineInterface::is_using_stdio()`
- [x] Clean up foreground state when plugin command exits on the engine
side too, not just whole plugin
- [x] Make sure tests for failure cases work as intended
  - `nu_plugin_stress_internals` added

# User-Facing Changes
- TUI plugins work
- Non-Rust plugins could optionally choose to use this
- This might behave differently, so will need to test it carefully
across different operating systems

# Tests + Formatting
- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting
- [ ] Document local socket option in plugin contrib docs
- [ ] Document how to do a terminal UI plugin in plugin contrib docs
- [ ] Document: `EnterForeground` engine call
- [ ] Document: `LeaveForeground` engine call
- [ ] Document: `LocalSocket` protocol feature

crates/nu-system/src/foreground.rs

@@ -1,16 +1,11 @@
 use std::{
     io,
     process::{Child, Command},
+    sync::{atomic::AtomicU32, Arc},
 };
 
 #[cfg(unix)]
-use std::{
-    io::IsTerminal,
-    sync::{
-        atomic::{AtomicU32, Ordering},
-        Arc,
-    },
-};
+use std::{io::IsTerminal, sync::atomic::Ordering};
 
 #[cfg(unix)]
 pub use foreground_pgroup::stdin_fd;
@@ -97,6 +92,139 @@ impl Drop for ForegroundChild {
     }
 }
 
+/// Keeps a specific already existing process in the foreground as long as the [`ForegroundGuard`].
+/// If the process needs to be spawned in the foreground, use [`ForegroundChild`] instead. This is
+/// used to temporarily bring plugin processes into the foreground.
+///
+/// # OS-specific behavior
+/// ## Unix
+///
+/// If there is already a foreground external process running, spawned with [`ForegroundChild`],
+/// this expects the process ID to remain in the process group created by the [`ForegroundChild`]
+/// for the lifetime of the guard, and keeps the terminal controlling process group set to that. If
+/// there is no foreground external process running, this sets the foreground process group to the
+/// plugin's process ID. The process group that is expected can be retrieved with [`.pgrp()`] if
+/// different from the plugin process ID.
+///
+/// ## Other systems
+///
+/// It does nothing special on non-unix systems.
+#[derive(Debug)]
+pub struct ForegroundGuard {
+    #[cfg(unix)]
+    pgrp: Option<u32>,
+    #[cfg(unix)]
+    pipeline_state: Arc<(AtomicU32, AtomicU32)>,
+}
+
+impl ForegroundGuard {
+    /// Move the given process to the foreground.
+    #[cfg(unix)]
+    pub fn new(
+        pid: u32,
+        pipeline_state: &Arc<(AtomicU32, AtomicU32)>,
+    ) -> std::io::Result<ForegroundGuard> {
+        use nix::unistd::{self, Pid};
+
+        let pid_nix = Pid::from_raw(pid as i32);
+        let (pgrp, pcnt) = pipeline_state.as_ref();
+
+        // Might have to retry due to race conditions on the atomics
+        loop {
+            // Try to give control to the child, if there isn't currently a foreground group
+            if pgrp
+                .compare_exchange(0, pid, Ordering::SeqCst, Ordering::SeqCst)
+                .is_ok()
+            {
+                let _ = pcnt.fetch_add(1, Ordering::SeqCst);
+
+                // We don't need the child to change process group. Make the guard now so that if there
+                // is an error, it will be cleaned up
+                let guard = ForegroundGuard {
+                    pgrp: None,
+                    pipeline_state: pipeline_state.clone(),
+                };
+
+                log::trace!("Giving control of the terminal to the plugin group, pid={pid}");
+
+                // Set the terminal controlling process group to the child process
+                unistd::tcsetpgrp(unsafe { stdin_fd() }, pid_nix)?;
+
+                return Ok(guard);
+            } else if pcnt
+                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
+                    // Avoid a race condition: only increment if count is > 0
+                    if count > 0 {
+                        Some(count + 1)
+                    } else {
+                        None
+                    }
+                })
+                .is_ok()
+            {
+                // We successfully added another count to the foreground process group, which means
+                // we only need to tell the child process to join this one
+                let pgrp = pgrp.load(Ordering::SeqCst);
+                log::trace!(
+                    "Will ask the plugin pid={pid} to join pgrp={pgrp} for control of the \
+                    terminal"
+                );
+                return Ok(ForegroundGuard {
+                    pgrp: Some(pgrp),
+                    pipeline_state: pipeline_state.clone(),
+                });
+            } else {
+                // The state has changed, we'll have to retry
+                continue;
+            }
+        }
+    }
+
+    /// Move the given process to the foreground.
+    #[cfg(not(unix))]
+    pub fn new(
+        pid: u32,
+        pipeline_state: &Arc<(AtomicU32, AtomicU32)>,
+    ) -> std::io::Result<ForegroundGuard> {
+        let _ = (pid, pipeline_state);
+        Ok(ForegroundGuard {})
+    }
+
+    /// If the child process is expected to join a different process group to be in the foreground,
+    /// this returns `Some(pgrp)`. This only ever returns `Some` on Unix.
+    pub fn pgrp(&self) -> Option<u32> {
+        #[cfg(unix)]
+        {
+            self.pgrp
+        }
+        #[cfg(not(unix))]
+        {
+            None
+        }
+    }
+
+    /// This should only be called once by `Drop`
+    fn reset_internal(&mut self) {
+        #[cfg(unix)]
+        {
+            log::trace!("Leaving the foreground group");
+
+            let (pgrp, pcnt) = self.pipeline_state.as_ref();
+            if pcnt.fetch_sub(1, Ordering::SeqCst) == 1 {
+                // Clean up if we are the last one around
+                pgrp.store(0, Ordering::SeqCst);
+                foreground_pgroup::reset()
+            }
+        }
+    }
+}
+
+impl Drop for ForegroundGuard {
+    fn drop(&mut self) {
+        self.reset_internal();
+    }
+}
+
 // It's a simpler version of fish shell's external process handling.
 #[cfg(unix)]
 mod foreground_pgroup {

crates/nu-system/src/lib.rs

@@ -9,7 +9,7 @@ mod windows;
 
 #[cfg(unix)]
 pub use self::foreground::stdin_fd;
-pub use self::foreground::ForegroundChild;
+pub use self::foreground::{ForegroundChild, ForegroundGuard};
 #[cfg(any(target_os = "android", target_os = "linux"))]
 pub use self::linux::*;
 #[cfg(target_os = "macos")]