Enable macOS foreground process handling (#9909)

# Description
Currently, foreground process management is disabled for macOS, since
the original code had issues (see #7068).
This PR re-enables process management on macOS in combination with the
changes from #9693.

# User-Facing Changes
Fixes hang on exit for nested nushells on macOS (issue #9859). Nushell
should now manage processes in the same way on macOS and other unix
systems.

src/terminal.rs

@@ -1,19 +1,39 @@
 #[cfg(unix)]
 pub(crate) fn acquire_terminal(interactive: bool) {
     use is_terminal::IsTerminal;
-    use nix::sys::signal::{signal, SigHandler, Signal};
+    use nix::{
+        errno::Errno,
+        sys::signal::{signal, SigHandler, Signal},
+        unistd,
+    };
 
-    if !std::io::stdin().is_terminal() {
-        return;
-    }
+    if interactive && std::io::stdin().is_terminal() {
+        // see also: https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
 
-    take_control(interactive);
+        take_control();
 
-    unsafe {
-        // SIGINT and SIGQUIT have special handling above
-        signal(Signal::SIGTSTP, SigHandler::SigIgn).expect("signal ignore");
-        signal(Signal::SIGTTIN, SigHandler::SigIgn).expect("signal ignore");
-        signal(Signal::SIGTTOU, SigHandler::SigIgn).expect("signal ignore");
+        unsafe {
+            // SIGINT and SIGQUIT have special handling above
+            signal(Signal::SIGTSTP, SigHandler::SigIgn).expect("signal ignore");
+            signal(Signal::SIGTTIN, SigHandler::SigIgn).expect("signal ignore");
+            signal(Signal::SIGTTOU, SigHandler::SigIgn).expect("signal ignore");
+        }
+
+        // Put ourselves in our own process group, if not already
+        let shell_pgid = unistd::getpid();
+        match unistd::setpgid(shell_pgid, shell_pgid) {
+            // setpgid returns EPERM if we are the session leader (e.g., as a login shell).
+            // The other cases that return EPERM cannot happen, since we gave our own pid.
+            // See: setpgid(2)
+            // Therefore, it is safe to ignore EPERM.
+            Ok(()) | Err(Errno::EPERM) => (),
+            Err(_) => {
+                eprintln!("ERROR: failed to put nushell in its own process group");
+                std::process::exit(1);
+            }
+        }
+        // Set our possibly new pgid to be in control of terminal
+        let _ = unistd::tcsetpgrp(nix::libc::STDIN_FILENO, shell_pgid);
     }
 }
 
@@ -22,7 +42,7 @@ pub(crate) fn acquire_terminal(_: bool) {}
 
 // Inspired by fish's acquire_tty_or_exit
 #[cfg(unix)]
-fn take_control(interactive: bool) {
+fn take_control() {
     use nix::{
         errno::Errno,
         sys::signal::{self, SaFlags, SigAction, SigHandler, SigSet, Signal},
@@ -59,32 +79,23 @@ fn take_control(interactive: bool) {
         }
     }
 
-    let mut success = false;
     for _ in 0..4096 {
         match unistd::tcgetpgrp(nix::libc::STDIN_FILENO) {
             Ok(owner_pgid) if owner_pgid == shell_pgid => {
-                success = true;
-                break;
+                // success
+                return;
             }
             Ok(owner_pgid) if owner_pgid == Pid::from_raw(0) => {
                 // Zero basically means something like "not owned" and we can just take it
                 let _ = unistd::tcsetpgrp(nix::libc::STDIN_FILENO, shell_pgid);
             }
             Err(Errno::ENOTTY) => {
-                if !interactive {
-                    // that's fine
-                    return;
-                }
                 eprintln!("ERROR: no TTY for interactive shell");
                 std::process::exit(1);
             }
             _ => {
                 // fish also has other heuristics than "too many attempts" for the orphan check, but they're optional
-                if signal::killpg(Pid::from_raw(-shell_pgid.as_raw()), Signal::SIGTTIN).is_err() {
-                    if !interactive {
-                        // that's fine
-                        return;
-                    }
+                if signal::killpg(shell_pgid, Signal::SIGTTIN).is_err() {
                     eprintln!("ERROR: failed to SIGTTIN ourselves");
                     std::process::exit(1);
                 }
@@ -92,8 +103,6 @@ fn take_control(interactive: bool) {
         }
     }
 
-    if !success && interactive {
-        eprintln!("ERROR: failed take control of the terminal, we might be orphaned");
-        std::process::exit(1);
-    }
+    eprintln!("ERROR: failed take control of the terminal, we might be orphaned");
+    std::process::exit(1);
 }