Do not attempt to take control of terminal in non-interactive mode (#9693)

# Description
Fixes a regression from #9681 where nushell will attempt to place itself
into the background or take control of the terminal even in
non-interactive mode.

Using the same
[reference](https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html)
from #6584:

>A subshell that runs *interactively* has to ensure that it has been
placed in the foreground...

>A subshell that runs *non-interactively* cannot and should not support
job control.

`fish`
[code](54fa1ad6ec/src/reader.cpp (L4862))
also seems to follow this.

This *partially* fixes
[9026](https://github.com/nushell/nushell/issues/9026). That is, nushell
will no longer set the foreground process group in non-interactive mode.

src/main.rs

@@ -81,7 +81,10 @@ fn main() -> Result<()> {
     let parsed_nu_cli_args = parse_commandline_args(&args_to_nushell.join(" "), &mut engine_state)
         .unwrap_or_else(|_| std::process::exit(1));
 
-    engine_state.is_interactive = parsed_nu_cli_args.interactive_shell.is_some();
+    // keep this condition in sync with the branches at the end
+    engine_state.is_interactive = parsed_nu_cli_args.interactive_shell.is_some()
+        || (parsed_nu_cli_args.commands.is_none() && script_name.is_empty());
+
     engine_state.is_login = parsed_nu_cli_args.login_shell.is_some();
 
     let use_color = engine_state.get_config().use_ansi_coloring;
@@ -142,8 +145,7 @@ fn main() -> Result<()> {
     );
 
     start_time = std::time::Instant::now();
-    // keep this condition in sync with the branches below
-    acquire_terminal(parsed_nu_cli_args.commands.is_none() && script_name.is_empty());
+    acquire_terminal(engine_state.is_interactive);
     perf(
         "acquire_terminal",
         start_time,
@@ -289,7 +291,6 @@ fn main() -> Result<()> {
             input,
         )
     } else {
-        engine_state.is_interactive = true;
         run_repl(&mut engine_state, parsed_nu_cli_args, entire_start_time)
     }
 }

src/terminal.rs

@@ -1,19 +1,38 @@
 #[cfg(unix)]
 pub(crate) fn acquire_terminal(interactive: bool) {
     use is_terminal::IsTerminal;
-    use nix::sys::signal::{signal, SigHandler, Signal};
+    use nix::{
+        errno::Errno,
+        sys::signal::{signal, SigHandler, Signal},
+        unistd,
+    };
 
-    if !std::io::stdin().is_terminal() {
-        return;
-    }
+    if interactive && std::io::stdin().is_terminal() {
+        // see also: https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
 
-    take_control(interactive);
+        take_control();
 
-    unsafe {
-        // SIGINT and SIGQUIT have special handling above
-        signal(Signal::SIGTSTP, SigHandler::SigIgn).expect("signal ignore");
-        signal(Signal::SIGTTIN, SigHandler::SigIgn).expect("signal ignore");
-        signal(Signal::SIGTTOU, SigHandler::SigIgn).expect("signal ignore");
+        unsafe {
+            // SIGINT and SIGQUIT have special handling above
+            signal(Signal::SIGTSTP, SigHandler::SigIgn).expect("signal ignore");
+            signal(Signal::SIGTTIN, SigHandler::SigIgn).expect("signal ignore");
+            signal(Signal::SIGTTOU, SigHandler::SigIgn).expect("signal ignore");
+        }
+
+        // Put ourselves in our own process group and take control of terminal, if not already
+        let shell_pgid = unistd::getpid();
+        match unistd::setpgid(shell_pgid, shell_pgid) {
+            // setpgid returns EPERM if we are the session leader (e.g., as a login shell).
+            // The other cases that return EPERM cannot happen, since we gave our own pid.
+            // See: setpgid(2)
+            // Therefore, it is safe to ignore EPERM.
+            Ok(()) | Err(Errno::EPERM) => (),
+            Err(_) => {
+                eprintln!("ERROR: failed to put nushell in its own process group");
+                std::process::exit(1);
+            }
+        }
+        let _ = unistd::tcsetpgrp(nix::libc::STDIN_FILENO, shell_pgid);
     }
 }
 
@@ -22,7 +41,7 @@ pub(crate) fn acquire_terminal(_: bool) {}
 
 // Inspired by fish's acquire_tty_or_exit
 #[cfg(unix)]
-fn take_control(interactive: bool) {
+fn take_control() {
     use nix::{
         errno::Errno,
         sys::signal::{self, SaFlags, SigAction, SigHandler, SigSet, Signal},
@@ -59,40 +78,30 @@ fn take_control(interactive: bool) {
         }
     }
 
-    let mut success = false;
     for _ in 0..4096 {
         match unistd::tcgetpgrp(nix::libc::STDIN_FILENO) {
             Ok(owner_pgid) if owner_pgid == shell_pgid => {
-                success = true;
-                break;
+                // success
+                return;
             }
             Ok(owner_pgid) if owner_pgid == Pid::from_raw(0) => {
                 // Zero basically means something like "not owned" and we can just take it
                 let _ = unistd::tcsetpgrp(nix::libc::STDIN_FILENO, shell_pgid);
             }
             Err(Errno::ENOTTY) => {
-                if !interactive {
-                    // that's fine
-                    return;
-                }
                 eprintln!("ERROR: no TTY for interactive shell");
                 std::process::exit(1);
             }
             _ => {
                 // fish also has other heuristics than "too many attempts" for the orphan check, but they're optional
-                if signal::killpg(Pid::from_raw(-shell_pgid.as_raw()), Signal::SIGTTIN).is_err() {
-                    if !interactive {
-                        // that's fine
-                        return;
-                    }
+                if signal::killpg(shell_pgid, Signal::SIGTTIN).is_err() {
                     eprintln!("ERROR: failed to SIGTTIN ourselves");
                     std::process::exit(1);
                 }
             }
         }
     }
-    if !success && interactive {
-        eprintln!("ERROR: failed take control of the terminal, we might be orphaned");
-        std::process::exit(1);
-    }
+
+    eprintln!("ERROR: failed take control of the terminal, we might be orphaned");
+    std::process::exit(1);
 }