Jobs (#14883)

# Description

This is an attempt to improve the nushell situation with regard to issue
#247.

This PR implements:
- [X] spawning jobs: `job spawn { do_background_thing }`
Jobs will be implemented as threads and not forks, to maintain a
consistent behavior between unix and windows.

- [X] listing running jobs: `job list`
This should allow users to list what background tasks they currently
have running.

- [X] killing jobs: `job kill <id>`
- [X] interupting nushell code in the job's background thread
- [X] interrupting the job's currently-running process, if any.

Things that should be taken into consideration for implementation:
- [X] (unix-only) Handling `TSTP` signals while executing code and
turning the current program into a background job, and unfreezing them
in foreground `job unfreeze`.

- [X] Ensuring processes spawned by background jobs get distinct process
groups from the nushell shell itself

This PR originally aimed to implement some of the following, but it is
probably ideal to be left for another PR (scope creep)
- Disowning external process jobs (`job dispatch`)
- Inter job communication (`job send/recv`)

Roadblocks encountered so far:
- Nushell does some weird terminal sequence magics which make so that
when a background process or thread prints something to stderr and the
prompt is idle, the stderr output ends up showing up weirdly

crates/nu-cli/src/repl.rs

@@ -20,6 +20,7 @@ use nu_cmd_base::util::get_editor;
 use nu_color_config::StyleComputer;
 #[allow(deprecated)]
 use nu_engine::env_to_strings;
+use nu_engine::exit::cleanup_exit;
 use nu_parser::{lex, parse, trim_quotes_str};
 use nu_protocol::shell_error::io::IoError;
 use nu_protocol::{
@@ -36,6 +37,7 @@ use reedline::{
     CursorConfig, CwdAwareHinter, DefaultCompleter, EditCommand, Emacs, FileBackedHistory,
     HistorySessionId, Reedline, SqliteBackedHistory, Vi,
 };
+use std::sync::atomic::Ordering;
 use std::{
     collections::HashMap,
     env::temp_dir,
@@ -692,7 +694,11 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
             );
 
             println!();
-            return (false, stack, line_editor);
+
+            cleanup_exit((), engine_state, 0);
+
+            // if cleanup_exit didn't exit, we should keep running
+            return (true, stack, line_editor);
         }
         Err(err) => {
             let message = err.to_string();
@@ -930,6 +936,9 @@ fn do_run_cmd(
     trace!("eval source: {}", s);
 
     let mut cmds = s.split_whitespace();
+
+    let had_warning_before = engine_state.exit_warning_given.load(Ordering::SeqCst);
+
     if let Some("exit") = cmds.next() {
         let mut working_set = StateWorkingSet::new(engine_state);
         let _ = parse(&mut working_set, None, s.as_bytes(), false);
@@ -938,13 +947,11 @@ fn do_run_cmd(
             match cmds.next() {
                 Some(s) => {
                     if let Ok(n) = s.parse::<i32>() {
-                        drop(line_editor);
-                        std::process::exit(n);
+                        return cleanup_exit(line_editor, engine_state, n);
                     }
                 }
                 None => {
-                    drop(line_editor);
-                    std::process::exit(0);
+                    return cleanup_exit(line_editor, engine_state, 0);
                 }
             }
         }
@@ -963,6 +970,14 @@ fn do_run_cmd(
         false,
     );
 
+    // if there was a warning before, and we got to this point, it means
+    // the possible call to cleanup_exit did not occur.
+    if had_warning_before && engine_state.is_interactive {
+        engine_state
+            .exit_warning_given
+            .store(false, Ordering::SeqCst);
+    }
+
     line_editor
 }
 

crates/nu-command/src/default_context.rs

@@ -448,8 +448,17 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
         // Experimental
         bind_command! {
             IsAdmin,
+            JobSpawn,
+            JobList,
+            JobKill,
+            Job,
         };
 
+        #[cfg(unix)]
+        bind_command! {
+            JobUnfreeze,
+        }
+
         // Removed
         bind_command! {
             LetEnv,

crates/nu-command/src/env/config/config_.rs

@@ -106,6 +106,7 @@ pub(super) fn start_editor(
     let child = ForegroundChild::spawn(
         command,
         engine_state.is_interactive,
+        engine_state.is_background_job(),
         &engine_state.pipeline_externals_state,
     );
 
@@ -119,7 +120,7 @@ pub(super) fn start_editor(
     })?;
 
     // Wrap the output into a `PipelineData::ByteStream`.
-    let child = nu_protocol::process::ChildProcess::new(child, None, false, call.head)?;
+    let child = nu_protocol::process::ChildProcess::new(child, None, false, call.head, None)?;
     Ok(PipelineData::ByteStream(
         ByteStream::child(child, call.head),
         None,

crates/nu-command/src/experimental/job.rs

@@ -0,0 +1,34 @@
+use nu_engine::{command_prelude::*, get_full_help};
+
+#[derive(Clone)]
+pub struct Job;
+
+impl Command for Job {
+    fn name(&self) -> &str {
+        "job"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("job")
+            .category(Category::Strings)
+            .input_output_types(vec![(Type::Nothing, Type::String)])
+    }
+
+    fn description(&self) -> &str {
+        "Various commands for working with background jobs."
+    }
+
+    fn extra_description(&self) -> &str {
+        "You must use one of the following subcommands. Using this command as-is will only produce this help message."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data())
+    }
+}

crates/nu-command/src/experimental/job_kill.rs

@@ -0,0 +1,72 @@
+use nu_engine::command_prelude::*;
+use nu_protocol::JobId;
+
+#[derive(Clone)]
+pub struct JobKill;
+
+impl Command for JobKill {
+    fn name(&self) -> &str {
+        "job kill"
+    }
+
+    fn description(&self) -> &str {
+        "Kill a background job."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("job kill")
+            .category(Category::Experimental)
+            .required("id", SyntaxShape::Int, "The id of the job to kill.")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .allow_variants_without_examples(true)
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["halt", "stop", "end", "close"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+
+        let id_arg: Spanned<i64> = call.req(engine_state, stack, 0)?;
+
+        if id_arg.item < 0 {
+            return Err(ShellError::NeedsPositiveValue { span: id_arg.span });
+        }
+
+        let id: JobId = JobId::new(id_arg.item as usize);
+
+        let mut jobs = engine_state.jobs.lock().expect("jobs lock is poisoned!");
+
+        if jobs.lookup(id).is_none() {
+            return Err(ShellError::JobNotFound {
+                id: id.get(),
+                span: head,
+            });
+        }
+
+        jobs.kill_and_remove(id).map_err(|err| {
+            ShellError::Io(IoError::new_internal(
+                err.kind(),
+                "Failed to kill the requested job",
+                nu_protocol::location!(),
+            ))
+        })?;
+
+        Ok(Value::nothing(head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            example: "let id = job spawn { sleep 10sec }; job kill $id",
+            description: "Kill a newly spawned job",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/experimental/job_list.rs

@@ -0,0 +1,75 @@
+use nu_engine::command_prelude::*;
+use nu_protocol::engine::{FrozenJob, Job};
+
+#[derive(Clone)]
+pub struct JobList;
+
+impl Command for JobList {
+    fn name(&self) -> &str {
+        "job list"
+    }
+
+    fn description(&self) -> &str {
+        "List background jobs."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("job list")
+            .category(Category::Experimental)
+            .input_output_types(vec![(Type::Nothing, Type::table())])
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["background", "jobs"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+
+        let jobs = engine_state.jobs.lock().expect("jobs lock is poisoned!");
+
+        let values = jobs
+            .iter()
+            .map(|(id, job)| {
+                let record = record! {
+                    "id" => Value::int(id.get() as i64, head),
+                    "type" => match job {
+                        Job::Thread(_) => Value::string("thread", head),
+                        Job::Frozen(_) => Value::string("frozen", head),
+                    },
+                    "pids" => match job {
+                        Job::Thread(job) => Value::list(
+                            job.collect_pids()
+                                .into_iter()
+                                .map(|it| Value::int(it as i64, head))
+                                .collect::<Vec<Value>>(),
+                            head,
+                        ),
+
+                        Job::Frozen(FrozenJob { unfreeze }) => {
+                            Value::list(vec![ Value::int(unfreeze.pid() as i64, head) ], head)
+                        }
+                    }
+                };
+
+                Value::record(record, head)
+            })
+            .collect::<Vec<Value>>();
+
+        Ok(Value::list(values, head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            example: "job list",
+            description: "List all background jobs",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/experimental/job_spawn.rs

@@ -0,0 +1,126 @@
+use std::{
+    sync::{
+        atomic::{AtomicBool, AtomicU32},
+        Arc,
+    },
+    thread,
+};
+
+use nu_engine::{command_prelude::*, ClosureEvalOnce};
+use nu_protocol::{
+    engine::{Closure, Job, ThreadJob},
+    report_shell_error, Signals,
+};
+
+#[derive(Clone)]
+pub struct JobSpawn;
+
+impl Command for JobSpawn {
+    fn name(&self) -> &str {
+        "job spawn"
+    }
+
+    fn description(&self) -> &str {
+        "Spawn a background job and retrieve its ID."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("job spawn")
+            .category(Category::Experimental)
+            .input_output_types(vec![(Type::Nothing, Type::Int)])
+            .required(
+                "closure",
+                SyntaxShape::Closure(Some(vec![SyntaxShape::Any])),
+                "The closure to run in another thread.",
+            )
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["background", "bg", "&"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+
+        let closure: Closure = call.req(engine_state, stack, 0)?;
+
+        let mut job_state = engine_state.clone();
+        job_state.is_interactive = false;
+
+        let job_stack = stack.clone();
+
+        // the new job should have its ctrl-c independent of foreground
+        let job_signals = Signals::new(Arc::new(AtomicBool::new(false)));
+        job_state.set_signals(job_signals.clone());
+
+        // the new job has a separate process group state for its processes
+        job_state.pipeline_externals_state = Arc::new((AtomicU32::new(0), AtomicU32::new(0)));
+
+        job_state.exit_warning_given = Arc::new(AtomicBool::new(false));
+
+        let jobs = job_state.jobs.clone();
+        let mut jobs = jobs.lock().expect("jobs lock is poisoned!");
+
+        let id = {
+            let thread_job = ThreadJob::new(job_signals);
+            job_state.current_thread_job = Some(thread_job.clone());
+            jobs.add_job(Job::Thread(thread_job))
+        };
+
+        let result = thread::Builder::new()
+            .name(format!("background job {}", id.get()))
+            .spawn(move || {
+                ClosureEvalOnce::new(&job_state, &job_stack, closure)
+                    .run_with_input(Value::nothing(head).into_pipeline_data())
+                    .and_then(|data| data.into_value(head))
+                    .unwrap_or_else(|err| {
+                        if !job_state.signals().interrupted() {
+                            report_shell_error(&job_state, &err);
+                        }
+
+                        Value::nothing(head)
+                    });
+
+                {
+                    let mut jobs = job_state.jobs.lock().expect("jobs lock is poisoned!");
+
+                    jobs.remove_job(id);
+                }
+            });
+
+        match result {
+            Ok(_) => Ok(Value::int(id.get() as i64, head).into_pipeline_data()),
+            Err(err) => {
+                jobs.remove_job(id);
+                Err(ShellError::Io(IoError::new_with_additional_context(
+                    err.kind(),
+                    call.head,
+                    None,
+                    "Failed to spawn thread for job",
+                )))
+            }
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            example: "job spawn { sleep 5sec; rm evidence.pdf }",
+            description: "Spawn a background job to do some time consuming work",
+            result: None,
+        }]
+    }
+
+    fn extra_description(&self) -> &str {
+        r#"Executes the provided closure in a background thread
+and registers this task in the background job table, which can be retrieved with `job list`.
+
+This command returns the job id of the newly created job.
+            "#
+    }
+}

crates/nu-command/src/experimental/job_unfreeze.rs

@@ -0,0 +1,160 @@
+use nu_engine::command_prelude::*;
+use nu_protocol::{
+    engine::{FrozenJob, Job, ThreadJob},
+    process::check_ok,
+    shell_error, JobId,
+};
+use nu_system::{kill_by_pid, ForegroundWaitStatus};
+
+#[derive(Clone)]
+pub struct JobUnfreeze;
+
+impl Command for JobUnfreeze {
+    fn name(&self) -> &str {
+        "job unfreeze"
+    }
+
+    fn description(&self) -> &str {
+        "Unfreeze a frozen process job in foreground."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("job unfreeze")
+            .category(Category::Experimental)
+            .optional("id", SyntaxShape::Int, "The process id to unfreeze.")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .allow_variants_without_examples(true)
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["fg"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+
+        let option_id: Option<Spanned<i64>> = call.opt(engine_state, stack, 0)?;
+
+        let mut jobs = engine_state.jobs.lock().expect("jobs lock is poisoned!");
+
+        if let Some(id_arg) = option_id {
+            if id_arg.item < 0 {
+                return Err(ShellError::NeedsPositiveValue { span: id_arg.span });
+            }
+        }
+
+        let id = option_id
+            .map(|it| JobId::new(it.item as usize))
+            .or_else(|| jobs.most_recent_frozen_job_id())
+            .ok_or_else(|| ShellError::NoFrozenJob { span: head })?;
+
+        let job = match jobs.lookup(id) {
+            None => {
+                return Err(ShellError::JobNotFound {
+                    id: id.get(),
+                    span: head,
+                })
+            }
+            Some(Job::Thread(ThreadJob { .. })) => {
+                return Err(ShellError::JobNotFrozen {
+                    id: id.get(),
+                    span: head,
+                })
+            }
+            Some(Job::Frozen(FrozenJob { .. })) => jobs
+                .remove_job(id)
+                .expect("job was supposed to be in job list"),
+        };
+
+        drop(jobs);
+
+        unfreeze_job(engine_state, id, job, head)?;
+
+        Ok(Value::nothing(head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                example: "job unfreeze",
+                description: "Unfreeze the latest frozen job",
+                result: None,
+            },
+            Example {
+                example: "job unfreeze 4",
+                description: "Unfreeze a specific frozen job by its PID",
+                result: None,
+            },
+        ]
+    }
+
+    fn extra_description(&self) -> &str {
+        r#"When a running process is frozen (with the SIGTSTP signal or with the Ctrl-Z key on unix),
+a background job gets registered for this process, which can then be resumed using this command."#
+    }
+}
+
+fn unfreeze_job(
+    state: &EngineState,
+    old_id: JobId,
+    job: Job,
+    span: Span,
+) -> Result<(), ShellError> {
+    match job {
+        Job::Thread(ThreadJob { .. }) => Err(ShellError::JobNotFrozen {
+            id: old_id.get(),
+            span,
+        }),
+
+        Job::Frozen(FrozenJob { unfreeze: handle }) => {
+            let pid = handle.pid();
+
+            if let Some(thread_job) = &state.current_thread_job {
+                if !thread_job.try_add_pid(pid) {
+                    kill_by_pid(pid.into()).map_err(|err| {
+                        ShellError::Io(IoError::new_internal(
+                            err.kind(),
+                            "job was interrupted; could not kill foreground process",
+                            nu_protocol::location!(),
+                        ))
+                    })?;
+                }
+            }
+
+            let result = handle.unfreeze(
+                state
+                    .is_interactive
+                    .then(|| state.pipeline_externals_state.clone()),
+            );
+
+            if let Some(thread_job) = &state.current_thread_job {
+                thread_job.remove_pid(pid);
+            }
+
+            match result {
+                Ok(ForegroundWaitStatus::Frozen(handle)) => {
+                    let mut jobs = state.jobs.lock().expect("jobs lock is poisoned!");
+
+                    jobs.add_job_with_id(old_id, Job::Frozen(FrozenJob { unfreeze: handle }))
+                        .expect("job was supposed to be removed");
+
+                    Ok(())
+                }
+
+                Ok(ForegroundWaitStatus::Finished(status)) => check_ok(status, false, span),
+
+                Err(err) => Err(ShellError::Io(IoError::new_internal(
+                    shell_error::io::ErrorKind::Std(err.kind()),
+                    "Failed to unfreeze foreground process",
+                    nu_protocol::location!(),
+                ))),
+            }
+        }
+    }
+}

crates/nu-command/src/experimental/mod.rs

@@ -1,3 +1,18 @@
 mod is_admin;
+mod job;
+mod job_kill;
+mod job_list;
+mod job_spawn;
+
+#[cfg(unix)]
+mod job_unfreeze;
 
 pub use is_admin::IsAdmin;
+pub use job::Job;
+pub use job_kill::JobKill;
+pub use job_list::JobList;
+
+pub use job_spawn::JobSpawn;
+
+#[cfg(unix)]
+pub use job_unfreeze::JobUnfreeze;

crates/nu-command/src/platform/kill.rs

@@ -1,5 +1,6 @@
 use nu_engine::command_prelude::*;
-use std::process::{Command as CommandSys, Stdio};
+use nu_system::build_kill_command;
+use std::process::Stdio;
 
 #[derive(Clone)]
 pub struct Kill;
@@ -56,70 +57,36 @@ impl Command for Kill {
         let signal: Option<Spanned<i64>> = call.get_flag(engine_state, stack, "signal")?;
         let quiet: bool = call.has_flag(engine_state, stack, "quiet")?;
 
-        let mut cmd = if cfg!(windows) {
-            let mut cmd = CommandSys::new("taskkill");
-
-            if force {
-                cmd.arg("/F");
-            }
-
-            cmd.arg("/PID");
-            cmd.arg(pid.to_string());
-
-            // each pid must written as `/PID 0` otherwise
-            // taskkill will act as `killall` unix command
-            for id in &rest {
-                cmd.arg("/PID");
-                cmd.arg(id.to_string());
-            }
-
-            cmd
-        } else {
-            let mut cmd = CommandSys::new("kill");
-            if force {
-                if let Some(Spanned {
+        if cfg!(unix) {
+            if let (
+                true,
+                Some(Spanned {
                     item: _,
                     span: signal_span,
-                }) = signal
-                {
-                    return Err(ShellError::IncompatibleParameters {
-                        left_message: "force".to_string(),
-                        left_span: call.get_flag_span(stack, "force").ok_or_else(|| {
-                            ShellError::GenericError {
-                                error: "Flag error".into(),
-                                msg: "flag force not found".into(),
-                                span: Some(call.head),
-                                help: None,
-                                inner: vec![],
-                            }
-                        })?,
-                        right_message: "signal".to_string(),
-                        right_span: Span::merge(
-                            call.get_flag_span(stack, "signal").ok_or_else(|| {
-                                ShellError::GenericError {
-                                    error: "Flag error".into(),
-                                    msg: "flag signal not found".into(),
-                                    span: Some(call.head),
-                                    help: None,
-                                    inner: vec![],
-                                }
-                            })?,
-                            signal_span,
-                        ),
-                    });
-                }
-                cmd.arg("-9");
-            } else if let Some(signal_value) = signal {
-                cmd.arg(format!("-{}", signal_value.item));
+                }),
+            ) = (force, signal)
+            {
+                return Err(ShellError::IncompatibleParameters {
+                    left_message: "force".to_string(),
+                    left_span: call
+                        .get_flag_span(stack, "force")
+                        .expect("Had flag force, but didn't have span for flag"),
+                    right_message: "signal".to_string(),
+                    right_span: Span::merge(
+                        call.get_flag_span(stack, "signal")
+                            .expect("Had flag signal, but didn't have span for flag"),
+                        signal_span,
+                    ),
+                });
             }
-
-            cmd.arg(pid.to_string());
-
-            cmd.args(rest.iter().map(move |id| id.to_string()));
-
-            cmd
         };
 
+        let mut cmd = build_kill_command(
+            force,
+            std::iter::once(pid).chain(rest),
+            signal.map(|spanned| spanned.item as u32),
+        );
+
         // pipe everything to null
         if quiet {
             cmd.stdin(Stdio::null())

crates/nu-command/src/shells/exit.rs

@@ -1,4 +1,4 @@
-use nu_engine::command_prelude::*;
+use nu_engine::{command_prelude::*, exit::cleanup_exit};
 
 #[derive(Clone)]
 pub struct Exit;
@@ -36,11 +36,11 @@ impl Command for Exit {
     ) -> Result<PipelineData, ShellError> {
         let exit_code: Option<i64> = call.opt(engine_state, stack, 0)?;
 
-        if let Some(exit_code) = exit_code {
-            std::process::exit(exit_code as i32);
-        }
+        let exit_code = exit_code.map_or(0, |it| it as i32);
 
-        std::process::exit(0);
+        cleanup_exit((), engine_state, exit_code);
+
+        Ok(Value::nothing(call.head).into_pipeline_data())
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/system/run_external.rs

@@ -2,10 +2,13 @@ use nu_cmd_base::hook::eval_hook;
 use nu_engine::{command_prelude::*, env_to_strings};
 use nu_path::{dots::expand_ndots_safe, expand_tilde, AbsolutePath};
 use nu_protocol::{
-    did_you_mean, process::ChildProcess, shell_error::io::IoError, ByteStream, NuGlob, OutDest,
-    Signals, UseAnsiColoring,
+    did_you_mean,
+    engine::{FrozenJob, Job},
+    process::{ChildProcess, PostWaitCallback},
+    shell_error::io::IoError,
+    ByteStream, NuGlob, OutDest, Signals, UseAnsiColoring,
 };
-use nu_system::ForegroundChild;
+use nu_system::{kill_by_pid, ForegroundChild, ForegroundWaitStatus};
 use nu_utils::IgnoreCaseExt;
 use pathdiff::diff_paths;
 #[cfg(windows)]
@@ -204,12 +207,28 @@ impl Command for External {
             command.stderr(writer);
             Some(reader)
         } else {
-            command.stdout(
-                Stdio::try_from(stdout).map_err(|err| IoError::new(err.kind(), call.head, None))?,
-            );
-            command.stderr(
-                Stdio::try_from(stderr).map_err(|err| IoError::new(err.kind(), call.head, None))?,
-            );
+            if engine_state.is_background_job()
+                && matches!(stdout, OutDest::Inherit | OutDest::Print)
+            {
+                command.stdout(Stdio::null());
+            } else {
+                command.stdout(
+                    Stdio::try_from(stdout)
+                        .map_err(|err| IoError::new(err.kind(), call.head, None))?,
+                );
+            }
+
+            if engine_state.is_background_job()
+                && matches!(stderr, OutDest::Inherit | OutDest::Print)
+            {
+                command.stderr(Stdio::null());
+            } else {
+                command.stderr(
+                    Stdio::try_from(stderr)
+                        .map_err(|err| IoError::new(err.kind(), call.head, None))?,
+                );
+            }
+
             None
         };
 
@@ -248,6 +267,7 @@ impl Command for External {
         let child = ForegroundChild::spawn(
             command,
             engine_state.is_interactive,
+            engine_state.is_background_job(),
             &engine_state.pipeline_externals_state,
         );
 
@@ -259,6 +279,18 @@ impl Command for External {
             )
         })?;
 
+        if let Some(thread_job) = &engine_state.current_thread_job {
+            if !thread_job.try_add_pid(child.pid()) {
+                kill_by_pid(child.pid().into()).map_err(|err| {
+                    ShellError::Io(IoError::new_internal(
+                        err.kind(),
+                        "Could not spawn external stdin worker",
+                        nu_protocol::location!(),
+                    ))
+                })?;
+            }
+        }
+
         // If we need to copy data into the child process, do it now.
         if let Some(data) = data_to_copy_into_stdin {
             let stdin = child.as_mut().stdin.take().expect("stdin is piped");
@@ -279,12 +311,28 @@ impl Command for External {
                 })?;
         }
 
+        let jobs = engine_state.jobs.clone();
+        let this_job = engine_state.current_thread_job.clone();
+        let child_pid = child.pid();
+
         // Wrap the output into a `PipelineData::ByteStream`.
         let mut child = ChildProcess::new(
             child,
             merged_stream,
             matches!(stderr, OutDest::Pipe),
             call.head,
+            // handle wait statuses for job control
+            Some(PostWaitCallback(Box::new(move |status| {
+                if let Some(this_job) = this_job {
+                    this_job.remove_pid(child_pid);
+                }
+
+                if let ForegroundWaitStatus::Frozen(unfreeze) = status {
+                    let mut jobs = jobs.lock().expect("jobs lock is poisoned!");
+
+                    jobs.add_job(Job::Frozen(FrozenJob { unfreeze }));
+                }
+            }))),
         )?;
 
         if matches!(stdout, OutDest::Pipe | OutDest::PipeSeparate)

crates/nu-command/tests/commands/job.rs

@@ -0,0 +1,217 @@
+use nu_test_support::{nu, playground::Playground};
+
+#[test]
+fn jobs_do_run() {
+    Playground::setup("job_test_1", |dirs, sandbox| {
+        sandbox.with_files(&[]);
+
+        let actual = nu!(
+            cwd: dirs.root(),
+            r#"
+            rm -f a.txt;
+            job spawn { sleep 200ms; 'a' | save a.txt };
+            let before = 'a.txt' | path exists;
+            sleep 400ms;
+            let after = 'a.txt' | path exists;
+            [$before, $after] | to nuon"#
+        );
+        assert_eq!(actual.out, "[false, true]");
+    })
+}
+
+#[test]
+fn first_job_id_is_one() {
+    let actual = nu!(r#"job spawn {} | to nuon"#);
+
+    assert_eq!(actual.out, "1");
+}
+
+#[test]
+fn job_list_adds_jobs_correctly() {
+    let actual = nu!(format!(
+        r#"
+            let list0 = job list | get id;
+            let job1 = job spawn {{ sleep 20ms }};
+            let list1 = job list | get id;
+            let job2 = job spawn {{ sleep 20ms }};
+            let list2 = job list | get id;
+            let job3 = job spawn {{ sleep 20ms }};
+            let list3 = job list | get id;
+            [({}), ({}), ({}), ({})] | to nuon
+            "#,
+        "$list0 == []",
+        "$list1 == [$job1]",
+        "($list2 | sort) == ([$job1, $job2] | sort)",
+        "($list3 | sort) == ([$job1, $job2, $job3] | sort)"
+    ));
+
+    assert_eq!(actual.out, "[true, true, true, true]");
+}
+
+#[test]
+fn jobs_get_removed_from_list_after_termination() {
+    let actual = nu!(format!(
+        r#"
+            let job = job spawn {{ sleep 0.5sec }};
+
+            let list0 = job list | get id;
+
+            sleep 1sec
+
+            let list1 = job list | get id;
+
+            [({}) ({})] | to nuon
+            "#,
+        "$list0 == [$job]", "$list1 == []",
+    ));
+
+    assert_eq!(actual.out, "[true, true]");
+}
+
+#[test]
+fn job_list_shows_pids() {
+    let actual = nu!(format!(
+        r#"
+            let job1 = job spawn {{ nu -c "sleep 1sec" | nu -c "sleep 2sec" }};
+            sleep 500ms;
+            let list0 = job list | where id == $job1 | first | get pids;
+            sleep 1sec;
+            let list1 = job list | where id == $job1 | first | get pids;
+            [({}), ({}), ({})] | to nuon
+            "#,
+        "($list0 | length) == 2", "($list1 | length) == 1", "$list1.0 in $list0",
+    ));
+
+    assert_eq!(actual.out, "[true, true, true]");
+}
+
+#[test]
+fn killing_job_removes_it_from_table() {
+    let actual = nu!(format!(
+        r#"
+            let job1 = job spawn {{ sleep 100ms }}
+            let job2 = job spawn {{ sleep 100ms }}
+            let job3 = job spawn {{ sleep 100ms }}
+
+            let list_before = job list | get id
+
+            job kill $job1
+            let list_after_kill_1 = job list | get id
+
+            job kill $job2
+            let list_after_kill_2 = job list | get id
+
+            job kill $job3
+            let list_after_kill_3 = job list | get id
+            
+            [({}) ({}) ({}) ({})] | to nuon
+            "#,
+        "($list_before | sort) == ([$job1 $job2 $job3] | sort)",
+        "($list_after_kill_1 | sort) == ([$job2 $job3] | sort)",
+        "($list_after_kill_2 | sort) == ([$job3] | sort)",
+        "$list_after_kill_3 == []",
+    ));
+
+    assert_eq!(actual.out, "[true, true, true, true]");
+}
+
+#[test]
+fn killing_job_kills_pids() {
+    let actual = nu!(format!(
+        r#"
+            let job1 = job spawn {{ nu -c "sleep 1sec" | nu -c "sleep 1sec" }}
+
+            sleep 25ms
+
+            let pids = job list | where id == $job1 | get pids
+
+            let child_pids_before = ps | where ppid == $nu.pid
+
+            job kill $job1
+            
+            sleep 25ms
+
+            let child_pids_after = ps | where ppid == $nu.pid
+
+            [({}) ({})] | to nuon
+            "#,
+        "($child_pids_before | length) == 2", "$child_pids_after == []",
+    ));
+
+    assert_eq!(actual.out, "[true, true]");
+}
+
+#[test]
+fn exiting_nushell_kills_jobs() {
+    let actual = nu!(r#"
+            let result = nu -c "let job = job spawn { nu -c 'sleep 1sec' };
+                   sleep 100ms;
+                   let child_pid = job list | where id == $job | get pids | first;
+                   [$nu.pid $child_pid] | to nuon"
+
+            let info = $result | from nuon
+            let child_pid = $info.0
+            let grandchild_pid = $info.1
+
+            ps | where pid == $grandchild_pid | filter { $in.ppid in [$child_pid, 1] } | length | to nuon
+            "#);
+    assert_eq!(actual.out, "0");
+}
+
+#[cfg(unix)]
+#[test]
+fn jobs_get_group_id_right() {
+    let actual = nu!(r#"
+            let job1 = job spawn { nu -c "sleep 0.5sec" | nu -c "sleep 0.5sec"; }
+
+            sleep 25ms
+
+            let pids = job list | where id == $job1 | first | get pids
+            
+            let pid1 = $pids.0
+            let pid2 = $pids.1
+
+            let groups = ^ps -ax -o pid,pgid | from ssv -m 1 | update PID {|it| $it.PID | into int} | update PGID {|it| $it.PGID | into int}
+
+            let my_group = $groups | where PID == $nu.pid | first | get PGID
+            let group1 = $groups | where PID == $pid1 | first | get PGID
+            let group2 = $groups | where PID == $pid2 | first | get PGID
+
+            [($my_group != $group1) ($my_group != $group2) ($group1 == $group2)] | to nuon
+            "#,);
+
+    assert_eq!(actual.out, "[true, true, true]");
+}
+
+#[test]
+fn job_extern_output_is_silent() {
+    let actual = nu!(r#" job spawn { nu -c "'hi'" }; sleep 1sec"#);
+    assert_eq!(actual.out, "");
+    assert_eq!(actual.err, "");
+}
+
+#[test]
+fn job_print_is_not_silent() {
+    let actual = nu!(r#" job spawn { print "hi" }; sleep 1sec"#);
+    assert_eq!(actual.out, "hi");
+    assert_eq!(actual.err, "");
+}
+
+#[test]
+fn job_extern_into_value_is_not_silent() {
+    let actual = nu!(r#" job spawn { print (nu -c "'hi'") }; sleep 1sec"#);
+    assert_eq!(actual.out, "hi");
+    assert_eq!(actual.err, "");
+}
+
+#[test]
+fn job_extern_into_pipe_is_not_silent() {
+    let actual = nu!(r#"
+        job spawn { 
+            print (nu -c "10" | nu --stdin -c "($in | into int) + 1")
+        }
+        sleep 1sec"#);
+
+    assert_eq!(actual.out, "11");
+    assert_eq!(actual.err, "");
+}

crates/nu-command/tests/commands/mod.rs

@@ -120,6 +120,7 @@ mod ulimit;
 mod window;
 
 mod debug;
+mod job;
 mod umkdir;
 mod uname;
 mod uniq;

crates/nu-engine/src/exit.rs

@@ -0,0 +1,37 @@
+use std::sync::atomic::Ordering;
+
+use nu_protocol::engine::EngineState;
+
+/// Exit the process or clean jobs if appropriate.
+///
+/// Drops `tag` and exits the current process if there are no running jobs, or if `exit_warning_given` is true.
+/// When running in an interactive session, warns the user if there
+/// were jobs and sets `exit_warning_given` instead, returning `tag` itself in that case.
+///
+// Currently, this `tag` argument exists mostly so that a LineEditor can be dropped before exiting the process.
+pub fn cleanup_exit<T>(tag: T, engine_state: &EngineState, exit_code: i32) -> T {
+    let mut jobs = engine_state.jobs.lock().expect("failed to lock job table");
+
+    if engine_state.is_interactive
+        && jobs.iter().next().is_some()
+        && !engine_state.exit_warning_given.load(Ordering::SeqCst)
+    {
+        let job_count = jobs.iter().count();
+
+        println!("There are still background jobs running ({}).", job_count);
+
+        println!("Running `exit` a second time will kill all of them.");
+
+        engine_state
+            .exit_warning_given
+            .store(true, Ordering::SeqCst);
+
+        return tag;
+    }
+
+    let _ = jobs.kill_all();
+
+    drop(tag);
+
+    std::process::exit(exit_code);
+}

crates/nu-engine/src/lib.rs

@@ -9,6 +9,7 @@ pub mod env;
 mod eval;
 mod eval_helpers;
 mod eval_ir;
+pub mod exit;
 mod glob_from;
 pub mod scope;
 

crates/nu-protocol/src/engine/engine_state.rs

@@ -31,6 +31,8 @@ type PoisonDebuggerError<'a> = PoisonError<MutexGuard<'a, Box<dyn Debugger>>>;
 #[cfg(feature = "plugin")]
 use crate::{PluginRegistryFile, PluginRegistryItem, RegisteredPlugin};
 
+use super::{Jobs, ThreadJob};
+
 #[derive(Clone, Debug)]
 pub enum VirtualPath {
     File(FileId),
@@ -111,6 +113,19 @@ pub struct EngineState {
     startup_time: i64,
     is_debugging: IsDebugging,
     pub debugger: Arc<Mutex<Box<dyn Debugger>>>,
+
+    pub jobs: Arc<Mutex<Jobs>>,
+
+    // The job being executed with this engine state, or None if main thread
+    pub current_thread_job: Option<ThreadJob>,
+
+    // When there are background jobs running, the interactive behavior of `exit` changes depending on
+    // the value of this flag:
+    // - if this is false, then a warning about running jobs is shown and `exit` enables this flag
+    // - if this is true, then `exit` will `std::process::exit`
+    //
+    // This ensures that running exit twice will terminate the program correctly
+    pub exit_warning_given: Arc<AtomicBool>,
 }
 
 // The max number of compiled regexes to keep around in a LRU cache, arbitrarily chosen
@@ -180,6 +195,9 @@ impl EngineState {
             startup_time: -1,
             is_debugging: IsDebugging::new(false),
             debugger: Arc::new(Mutex::new(Box::new(NoopDebugger))),
+            jobs: Arc::new(Mutex::new(Jobs::default())),
+            current_thread_job: None,
+            exit_warning_given: Arc::new(AtomicBool::new(false)),
         }
     }
 
@@ -1036,6 +1054,9 @@ impl EngineState {
                 cursor_pos: 0,
             }));
         }
+        if Mutex::is_poisoned(&self.jobs) {
+            self.jobs = Arc::new(Mutex::new(Jobs::default()));
+        }
         if Mutex::is_poisoned(&self.regex_cache) {
             self.regex_cache = Arc::new(Mutex::new(LruCache::new(
                 NonZeroUsize::new(REGEX_CACHE_SIZE).expect("tried to create cache of size zero"),
@@ -1056,6 +1077,11 @@ impl EngineState {
             .position(|sp| sp == &span)
             .map(SpanId::new)
     }
+
+    // Determines whether the current state is being held by a background job
+    pub fn is_background_job(&self) -> bool {
+        self.current_thread_job.is_some()
+    }
 }
 
 impl GetSpan for &EngineState {

crates/nu-protocol/src/engine/jobs.rs

@@ -0,0 +1,219 @@
+use std::{
+    collections::{HashMap, HashSet},
+    sync::{Arc, Mutex},
+};
+
+use nu_system::{kill_by_pid, UnfreezeHandle};
+
+use crate::Signals;
+
+use crate::JobId;
+
+pub struct Jobs {
+    next_job_id: usize,
+
+    // this is the ID of the most recently added frozen job in the jobs table.
+    // the methods of this struct must ensure the invariant of this always
+    // being None or pointing to a valid job in the table
+    last_frozen_job_id: Option<JobId>,
+    jobs: HashMap<JobId, Job>,
+}
+
+impl Default for Jobs {
+    fn default() -> Self {
+        Self {
+            next_job_id: 1,
+            last_frozen_job_id: None,
+            jobs: HashMap::default(),
+        }
+    }
+}
+
+impl Jobs {
+    pub fn iter(&self) -> impl Iterator<Item = (JobId, &Job)> {
+        self.jobs.iter().map(|(k, v)| (*k, v))
+    }
+
+    pub fn lookup(&self, id: JobId) -> Option<&Job> {
+        self.jobs.get(&id)
+    }
+
+    pub fn remove_job(&mut self, id: JobId) -> Option<Job> {
+        if self.last_frozen_job_id.is_some_and(|last| id == last) {
+            self.last_frozen_job_id = None;
+        }
+
+        self.jobs.remove(&id)
+    }
+
+    fn assign_last_frozen_id_if_frozen(&mut self, id: JobId, job: &Job) {
+        if let Job::Frozen(_) = job {
+            self.last_frozen_job_id = Some(id);
+        }
+    }
+
+    pub fn add_job(&mut self, job: Job) -> JobId {
+        let this_id = JobId::new(self.next_job_id);
+
+        self.assign_last_frozen_id_if_frozen(this_id, &job);
+
+        self.jobs.insert(this_id, job);
+        self.next_job_id += 1;
+
+        this_id
+    }
+
+    pub fn most_recent_frozen_job_id(&mut self) -> Option<JobId> {
+        self.last_frozen_job_id
+    }
+
+    // this is useful when you want to remove a job from the list and add it back later
+    pub fn add_job_with_id(&mut self, id: JobId, job: Job) -> Result<(), &'static str> {
+        self.assign_last_frozen_id_if_frozen(id, &job);
+
+        if let std::collections::hash_map::Entry::Vacant(e) = self.jobs.entry(id) {
+            e.insert(job);
+            Ok(())
+        } else {
+            Err("job already exists")
+        }
+    }
+
+    /// This function tries to forcefully kill a job from this job table,
+    /// removes it from the job table. It always succeeds in removing the job
+    /// from the table, but may fail in killing the job's active processes.
+    pub fn kill_and_remove(&mut self, id: JobId) -> std::io::Result<()> {
+        if let Some(job) = self.jobs.get(&id) {
+            let err = job.kill();
+
+            self.remove_job(id);
+
+            err?
+        }
+
+        Ok(())
+    }
+
+    /// This function tries to forcefully kill all the background jobs and
+    /// removes all of them from the job table.
+    ///
+    /// It returns an error if any of the job killing attempts fails, but always
+    /// succeeds in removing the jobs from the table.
+    pub fn kill_all(&mut self) -> std::io::Result<()> {
+        self.last_frozen_job_id = None;
+
+        self.jobs.clear();
+
+        let first_err = self
+            .iter()
+            .map(|(_, job)| job.kill().err())
+            .fold(None, |acc, x| acc.or(x));
+
+        if let Some(err) = first_err {
+            Err(err)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+pub enum Job {
+    Thread(ThreadJob),
+    Frozen(FrozenJob),
+}
+
+// A thread job represents a job that is currently executing as a background thread in nushell.
+// This is an Arc-y type, cloning it does not uniquely clone the information of this particular
+// job.
+
+// Although rust's documentation does not document the acquire-release semantics of Mutex, this
+// is a direct undocumentented requirement of its soundness, and is thus assumed by this
+// implementaation.
+// see issue https://github.com/rust-lang/rust/issues/126239.
+#[derive(Clone)]
+pub struct ThreadJob {
+    signals: Signals,
+    pids: Arc<Mutex<HashSet<u32>>>,
+}
+
+impl ThreadJob {
+    pub fn new(signals: Signals) -> Self {
+        ThreadJob {
+            signals,
+            pids: Arc::new(Mutex::new(HashSet::default())),
+        }
+    }
+
+    /// Tries to add the provided pid to the active pid set of the current job.
+    ///
+    /// Returns true if the pid was added successfully, or false if the
+    /// current job is interrupted.
+    pub fn try_add_pid(&self, pid: u32) -> bool {
+        let mut pids = self.pids.lock().expect("PIDs lock was poisoned");
+
+        // note: this signals check must occur after the pids lock has been locked.
+        if self.signals.interrupted() {
+            false
+        } else {
+            pids.insert(pid);
+            true
+        }
+    }
+
+    pub fn collect_pids(&self) -> Vec<u32> {
+        let lock = self.pids.lock().expect("PID lock was poisoned");
+
+        lock.iter().copied().collect()
+    }
+
+    pub fn kill(&self) -> std::io::Result<()> {
+        // it's okay to make this interrupt outside of the mutex, since it has acquire-release
+        // semantics.
+
+        self.signals.trigger();
+
+        let mut pids = self.pids.lock().expect("PIDs lock was poisoned");
+
+        for pid in pids.iter() {
+            kill_by_pid((*pid).into())?;
+        }
+
+        pids.clear();
+
+        Ok(())
+    }
+
+    pub fn remove_pid(&self, pid: u32) {
+        let mut pids = self.pids.lock().expect("PID lock was poisoned");
+
+        pids.remove(&pid);
+    }
+}
+
+impl Job {
+    pub fn kill(&self) -> std::io::Result<()> {
+        match self {
+            Job::Thread(thread_job) => thread_job.kill(),
+            Job::Frozen(frozen_job) => frozen_job.kill(),
+        }
+    }
+}
+
+pub struct FrozenJob {
+    pub unfreeze: UnfreezeHandle,
+}
+
+impl FrozenJob {
+    pub fn kill(&self) -> std::io::Result<()> {
+        #[cfg(unix)]
+        {
+            kill_by_pid(self.unfreeze.pid() as i64)
+        }
+
+        // it doesn't happen outside unix.
+        #[cfg(not(unix))]
+        {
+            Ok(())
+        }
+    }
+}

crates/nu-protocol/src/engine/mod.rs

@@ -8,6 +8,7 @@ mod command;
 mod description;
 mod engine_state;
 mod error_handler;
+mod jobs;
 mod overlay;
 mod pattern_match;
 mod sequence;
@@ -26,6 +27,7 @@ pub use capture_block::*;
 pub use command::*;
 pub use engine_state::*;
 pub use error_handler::*;
+pub use jobs::*;
 pub use overlay::*;
 pub use pattern_match::*;
 pub use sequence::*;

crates/nu-protocol/src/errors/shell_error/mod.rs

@@ -1326,6 +1326,40 @@ On Windows, this would be %USERPROFILE%\AppData\Roaming"#
         span: Option<Span>,
     },
 
+    #[error("Job {id} not found")]
+    #[diagnostic(
+        code(nu::shell::job_not_found),
+        help(
+            "The operation could not be completed, there is no job currently running with this id"
+        )
+    )]
+    JobNotFound {
+        id: usize,
+        #[label = "job not found"]
+        span: Span,
+    },
+
+    #[error("No frozen job to unfreeze")]
+    #[diagnostic(
+        code(nu::shell::no_frozen_job),
+        help("There is currently no frozen job to unfreeze")
+    )]
+    NoFrozenJob {
+        #[label = "no frozen job"]
+        span: Span,
+    },
+
+    #[error("Job {id} is not frozen")]
+    #[diagnostic(
+        code(nu::shell::os_disabled),
+        help("You tried to unfreeze a job which is not frozen")
+    )]
+    JobNotFrozen {
+        id: usize,
+        #[label = "job not frozen"]
+        span: Span,
+    },
+
     #[error(transparent)]
     #[diagnostic(transparent)]
     ChainedError(ChainedError),

crates/nu-protocol/src/id.rs

@@ -94,6 +94,8 @@ pub mod marker {
     pub struct Span;
     #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
     pub struct Reg;
+    #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+    pub struct Job;
 }
 
 pub type VarId = Id<marker::Var>;
@@ -104,6 +106,7 @@ pub type OverlayId = Id<marker::Overlay>;
 pub type FileId = Id<marker::File>;
 pub type VirtualPathId = Id<marker::VirtualPath>;
 pub type SpanId = Id<marker::Span>;
+pub type JobId = Id<marker::Job>;
 
 /// An ID for an [IR](crate::ir) register.
 ///

crates/nu-protocol/src/process/child.rs

@@ -1,5 +1,6 @@
 use crate::{byte_stream::convert_file, shell_error::io::IoError, ShellError, Span};
-use nu_system::{ExitStatus, ForegroundChild};
+use nu_system::{ExitStatus, ForegroundChild, ForegroundWaitStatus};
+
 use os_pipe::PipeReader;
 use std::{
     fmt::Debug,
@@ -8,7 +9,7 @@ use std::{
     thread,
 };
 
-fn check_ok(status: ExitStatus, ignore_error: bool, span: Span) -> Result<(), ShellError> {
+pub fn check_ok(status: ExitStatus, ignore_error: bool, span: Span) -> Result<(), ShellError> {
     match status {
         ExitStatus::Exited(exit_code) => {
             if ignore_error {
@@ -165,12 +166,21 @@ pub struct ChildProcess {
     span: Span,
 }
 
+pub struct PostWaitCallback(pub Box<dyn FnOnce(ForegroundWaitStatus) + Send>);
+
+impl Debug for PostWaitCallback {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<wait_callback>")
+    }
+}
+
 impl ChildProcess {
     pub fn new(
         mut child: ForegroundChild,
         reader: Option<PipeReader>,
         swap: bool,
         span: Span,
+        callback: Option<PostWaitCallback>,
     ) -> Result<Self, ShellError> {
         let (stdout, stderr) = if let Some(combined) = reader {
             (Some(combined), None)
@@ -190,7 +200,32 @@ impl ChildProcess {
 
         thread::Builder::new()
             .name("exit status waiter".into())
-            .spawn(move || exit_status_sender.send(child.wait()))
+            .spawn(move || {
+                let matched = match child.wait() {
+                    // there are two possible outcomes when we `wait` for a process to finish:
+                    // 1. the process finishes as usual
+                    // 2. (unix only) the process gets signaled with SIGTSTP
+                    //
+                    // in the second case, although the process may still be alive in a
+                    // cryonic state, we explicitly treat as it has finished with exit code 0
+                    // for the sake of the current pipeline
+                    Ok(wait_status) => {
+                        let next = match &wait_status {
+                            ForegroundWaitStatus::Frozen(_) => ExitStatus::Exited(0),
+                            ForegroundWaitStatus::Finished(exit_status) => *exit_status,
+                        };
+
+                        if let Some(callback) = callback {
+                            (callback.0)(wait_status);
+                        }
+
+                        Ok(next)
+                    }
+                    Err(err) => Err(err),
+                };
+
+                exit_status_sender.send(matched)
+            })
             .map_err(|err| {
                 IoError::new_with_additional_context(
                     err.kind(),

crates/nu-system/src/foreground.rs

@@ -1,10 +1,8 @@
-#[cfg(unix)]
-use std::io::prelude::*;
-use std::{
-    io,
-    process::{Child, Command},
-    sync::{atomic::AtomicU32, Arc},
-};
+use std::sync::{atomic::AtomicU32, Arc};
+
+use std::io;
+
+use std::process::{Child, Command};
 
 use crate::ExitStatus;
 
@@ -12,7 +10,7 @@ use crate::ExitStatus;
 use std::{io::IsTerminal, sync::atomic::Ordering};
 
 #[cfg(unix)]
-pub use foreground_pgroup::stdin_fd;
+pub use child_pgroup::stdin_fd;
 
 #[cfg(unix)]
 use nix::{sys::signal, sys::wait, unistd::Pid};
@@ -37,6 +35,10 @@ pub struct ForegroundChild {
     inner: Child,
     #[cfg(unix)]
     pipeline_state: Option<Arc<(AtomicU32, AtomicU32)>>,
+
+    // this is unix-only since we don't have to deal with process groups in windows
+    #[cfg(unix)]
+    interactive: bool,
 }
 
 impl ForegroundChild {
@@ -49,16 +51,22 @@ impl ForegroundChild {
     pub fn spawn(
         mut command: Command,
         interactive: bool,
+        background: bool,
         pipeline_state: &Arc<(AtomicU32, AtomicU32)>,
     ) -> io::Result<Self> {
-        if interactive && io::stdin().is_terminal() {
+        let interactive = interactive && io::stdin().is_terminal();
+
+        let uses_dedicated_process_group = interactive || background;
+
+        if uses_dedicated_process_group {
             let (pgrp, pcnt) = pipeline_state.as_ref();
             let existing_pgrp = pgrp.load(Ordering::SeqCst);
-            foreground_pgroup::prepare_command(&mut command, existing_pgrp);
+            child_pgroup::prepare_command(&mut command, existing_pgrp, background);
             command
                 .spawn()
                 .map(|child| {
-                    foreground_pgroup::set(&child, existing_pgrp);
+                    child_pgroup::set(&child, existing_pgrp, background);
+
                     let _ = pcnt.fetch_add(1, Ordering::SeqCst);
                     if existing_pgrp == 0 {
                         pgrp.store(child.id(), Ordering::SeqCst);
@@ -66,67 +74,121 @@ impl ForegroundChild {
                     Self {
                         inner: child,
                         pipeline_state: Some(pipeline_state.clone()),
+                        interactive,
                     }
                 })
                 .inspect_err(|_e| {
-                    foreground_pgroup::reset();
+                    if interactive {
+                        child_pgroup::reset();
+                    }
                 })
         } else {
             command.spawn().map(|child| Self {
                 inner: child,
                 pipeline_state: None,
+                interactive,
             })
         }
     }
 
-    pub fn wait(&mut self) -> io::Result<ExitStatus> {
+    pub fn wait(&mut self) -> io::Result<ForegroundWaitStatus> {
         #[cfg(unix)]
         {
-            // the child may be stopped multiple times, we loop until it exits
-            loop {
-                let child_pid = Pid::from_raw(self.inner.id() as i32);
-                let status = wait::waitpid(child_pid, Some(wait::WaitPidFlag::WUNTRACED));
-                match status {
-                    Err(e) => {
-                        drop(self.inner.stdin.take());
-                        return Err(e.into());
-                    }
-                    Ok(wait::WaitStatus::Exited(_, status)) => {
-                        drop(self.inner.stdin.take());
-                        return Ok(ExitStatus::Exited(status));
-                    }
-                    Ok(wait::WaitStatus::Signaled(_, signal, core_dumped)) => {
-                        drop(self.inner.stdin.take());
-                        return Ok(ExitStatus::Signaled {
-                            signal: signal as i32,
-                            core_dumped,
-                        });
-                    }
-                    Ok(wait::WaitStatus::Stopped(_, _)) => {
-                        println!("nushell currently does not support background jobs");
-                        // acquire terminal in order to be able to read from stdin
-                        foreground_pgroup::reset();
-                        let mut stdin = io::stdin();
-                        let mut stdout = io::stdout();
-                        write!(stdout, "press any key to continue")?;
-                        stdout.flush()?;
-                        stdin.read_exact(&mut [0u8])?;
-                        // bring child's pg back into foreground and continue it
-                        if let Some(state) = self.pipeline_state.as_ref() {
-                            let existing_pgrp = state.0.load(Ordering::SeqCst);
-                            foreground_pgroup::set(&self.inner, existing_pgrp);
-                        }
-                        signal::killpg(child_pid, signal::SIGCONT)?;
-                    }
-                    Ok(_) => {
-                        // keep waiting
-                    }
-                };
-            }
+            let child_pid = Pid::from_raw(self.inner.id() as i32);
+
+            unix_wait(child_pid).inspect(|result| {
+                if let (true, ForegroundWaitStatus::Frozen(_)) = (self.interactive, result) {
+                    child_pgroup::reset();
+                }
+            })
         }
         #[cfg(not(unix))]
         self.as_mut().wait().map(Into::into)
     }
+
+    pub fn pid(&self) -> u32 {
+        self.inner.id()
+    }
+}
+
+#[cfg(unix)]
+fn unix_wait(child_pid: Pid) -> std::io::Result<ForegroundWaitStatus> {
+    use ForegroundWaitStatus::*;
+
+    // the child may be stopped multiple times, we loop until it exits
+    loop {
+        let status = wait::waitpid(child_pid, Some(wait::WaitPidFlag::WUNTRACED));
+        match status {
+            Err(e) => {
+                return Err(e.into());
+            }
+            Ok(wait::WaitStatus::Exited(_, status)) => {
+                return Ok(Finished(ExitStatus::Exited(status)));
+            }
+            Ok(wait::WaitStatus::Signaled(_, signal, core_dumped)) => {
+                return Ok(Finished(ExitStatus::Signaled {
+                    signal: signal as i32,
+                    core_dumped,
+                }));
+            }
+            Ok(wait::WaitStatus::Stopped(_, _)) => {
+                return Ok(Frozen(UnfreezeHandle { child_pid }));
+            }
+            Ok(_) => {
+                // keep waiting
+            }
+        };
+    }
+}
+
+pub enum ForegroundWaitStatus {
+    Finished(ExitStatus),
+    Frozen(UnfreezeHandle),
+}
+
+impl From<std::process::ExitStatus> for ForegroundWaitStatus {
+    fn from(status: std::process::ExitStatus) -> Self {
+        ForegroundWaitStatus::Finished(status.into())
+    }
+}
+
+#[derive(Debug)]
+pub struct UnfreezeHandle {
+    #[cfg(unix)]
+    child_pid: Pid,
+}
+
+impl UnfreezeHandle {
+    #[cfg(unix)]
+    pub fn unfreeze(
+        self,
+        pipeline_state: Option<Arc<(AtomicU32, AtomicU32)>>,
+    ) -> io::Result<ForegroundWaitStatus> {
+        // bring child's process group back into foreground and continue it
+
+        // we only keep the guard for its drop impl
+        let _guard = pipeline_state.map(|pipeline_state| {
+            ForegroundGuard::new(self.child_pid.as_raw() as u32, &pipeline_state)
+        });
+
+        if let Err(err) = signal::killpg(self.child_pid, signal::SIGCONT) {
+            return Err(err.into());
+        }
+
+        let child_pid = self.child_pid;
+
+        unix_wait(child_pid)
+    }
+
+    pub fn pid(&self) -> u32 {
+        #[cfg(unix)]
+        {
+            self.child_pid.as_raw() as u32
+        }
+
+        #[cfg(not(unix))]
+        0
+    }
 }
 
 impl AsMut<Child> for ForegroundChild {
@@ -141,7 +203,10 @@ impl Drop for ForegroundChild {
         if let Some((pgrp, pcnt)) = self.pipeline_state.as_deref() {
             if pcnt.fetch_sub(1, Ordering::SeqCst) == 1 {
                 pgrp.store(0, Ordering::SeqCst);
-                foreground_pgroup::reset()
+
+                if self.interactive {
+                    child_pgroup::reset()
+                }
             }
         }
     }
@@ -149,7 +214,7 @@ impl Drop for ForegroundChild {
 
 /// Keeps a specific already existing process in the foreground as long as the [`ForegroundGuard`].
 /// If the process needs to be spawned in the foreground, use [`ForegroundChild`] instead. This is
-/// used to temporarily bring plugin processes into the foreground.
+/// used to temporarily bring frozen and plugin processes into the foreground.
 ///
 /// # OS-specific behavior
 /// ## Unix
@@ -158,8 +223,8 @@ impl Drop for ForegroundChild {
 /// this expects the process ID to remain in the process group created by the [`ForegroundChild`]
 /// for the lifetime of the guard, and keeps the terminal controlling process group set to that.
 /// If there is no foreground external process running, this sets the foreground process group to
-/// the plugin's process ID. The process group that is expected can be retrieved with
-/// [`.pgrp()`](Self::pgrp) if different from the plugin process ID.
+/// the provided process ID. The process group that is expected can be retrieved with
+/// [`.pgrp()`](Self::pgrp) if different from the provided process ID.
 ///
 /// ## Other systems
 ///
@@ -200,7 +265,7 @@ impl ForegroundGuard {
                     pipeline_state: pipeline_state.clone(),
                 };
 
-                log::trace!("Giving control of the terminal to the plugin group, pid={pid}");
+                log::trace!("Giving control of the terminal to the process group, pid={pid}");
 
                 // Set the terminal controlling process group to the child process
                 unistd::tcsetpgrp(unsafe { stdin_fd() }, pid_nix)?;
@@ -221,7 +286,7 @@ impl ForegroundGuard {
                 // we only need to tell the child process to join this one
                 let pgrp = pgrp.load(Ordering::SeqCst);
                 log::trace!(
-                    "Will ask the plugin pid={pid} to join pgrp={pgrp} for control of the \
+                    "Will ask the process pid={pid} to join pgrp={pgrp} for control of the \
                     terminal"
                 );
                 return Ok(ForegroundGuard {
@@ -268,7 +333,7 @@ impl ForegroundGuard {
             if pcnt.fetch_sub(1, Ordering::SeqCst) == 1 {
                 // Clean up if we are the last one around
                 pgrp.store(0, Ordering::SeqCst);
-                foreground_pgroup::reset()
+                child_pgroup::reset()
             }
         }
     }
@@ -282,7 +347,7 @@ impl Drop for ForegroundGuard {
 
 // It's a simpler version of fish shell's external process handling.
 #[cfg(unix)]
-mod foreground_pgroup {
+mod child_pgroup {
     use nix::{
         sys::signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal},
         unistd::{self, Pid},
@@ -306,7 +371,7 @@ mod foreground_pgroup {
         unsafe { BorrowedFd::borrow_raw(nix::libc::STDIN_FILENO) }
     }
 
-    pub fn prepare_command(external_command: &mut Command, existing_pgrp: u32) {
+    pub fn prepare_command(external_command: &mut Command, existing_pgrp: u32, background: bool) {
         unsafe {
             // Safety:
             // POSIX only allows async-signal-safe functions to be called.
@@ -320,19 +385,13 @@ mod foreground_pgroup {
                 // According to glibc's job control manual:
                 // https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
                 // This has to be done *both* in the parent and here in the child due to race conditions.
-                set_foreground_pid(Pid::this(), existing_pgrp);
+                set_foreground_pid(Pid::this(), existing_pgrp, background);
 
-                // Reset signal handlers for child, sync with `terminal.rs`
+                // `terminal.rs` makes the shell process ignore some signals,
+                //  so we set them to their default behavior for our child
                 let default = SigAction::new(SigHandler::SigDfl, SaFlags::empty(), SigSet::empty());
-                // SIGINT has special handling
+
                 let _ = sigaction(Signal::SIGQUIT, &default);
-                // We don't support background jobs, so keep some signals blocked for now
-                // let _ = sigaction(Signal::SIGTTIN, &default);
-                // let _ = sigaction(Signal::SIGTTOU, &default);
-                // We do need to reset SIGTSTP though, since some TUI
-                // applications implement their own Ctrl-Z handling, and
-                // ForegroundChild::wait() needs to be able to react to the
-                // child being stopped.
                 let _ = sigaction(Signal::SIGTSTP, &default);
                 let _ = sigaction(Signal::SIGTERM, &default);
 
@@ -341,11 +400,15 @@ mod foreground_pgroup {
         }
     }
 
-    pub fn set(process: &Child, existing_pgrp: u32) {
-        set_foreground_pid(Pid::from_raw(process.id() as i32), existing_pgrp);
+    pub fn set(process: &Child, existing_pgrp: u32, background: bool) {
+        set_foreground_pid(
+            Pid::from_raw(process.id() as i32),
+            existing_pgrp,
+            background,
+        );
     }
 
-    fn set_foreground_pid(pid: Pid, existing_pgrp: u32) {
+    fn set_foreground_pid(pid: Pid, existing_pgrp: u32, background: bool) {
         // Safety: needs to be async-signal-safe.
         // `setpgid` and `tcsetpgrp` are async-signal-safe.
 
@@ -357,7 +420,10 @@ mod foreground_pgroup {
             Pid::from_raw(existing_pgrp as i32)
         };
         let _ = unistd::setpgid(pid, pgrp);
-        let _ = unistd::tcsetpgrp(unsafe { stdin_fd() }, pgrp);
+
+        if !background {
+            let _ = unistd::tcsetpgrp(unsafe { stdin_fd() }, pgrp);
+        }
     }
 
     /// Reset the foreground process group to the shell

crates/nu-system/src/lib.rs

@@ -1,6 +1,7 @@
 #![doc = include_str!("../README.md")]
 mod exit_status;
 mod foreground;
+mod util;
 
 #[cfg(target_os = "freebsd")]
 mod freebsd;
@@ -17,7 +18,11 @@ mod windows;
 pub use self::exit_status::ExitStatus;
 #[cfg(unix)]
 pub use self::foreground::stdin_fd;
-pub use self::foreground::{ForegroundChild, ForegroundGuard};
+pub use self::foreground::{
+    ForegroundChild, ForegroundGuard, ForegroundWaitStatus, UnfreezeHandle,
+};
+
+pub use self::util::*;
 
 #[cfg(target_os = "freebsd")]
 pub use self::freebsd::*;

crates/nu-system/src/util.rs

@@ -0,0 +1,54 @@
+use std::io;
+use std::process::Command as CommandSys;
+
+/// Tries to forcefully kill a process by its PID
+pub fn kill_by_pid(pid: i64) -> io::Result<()> {
+    let mut cmd = build_kill_command(true, std::iter::once(pid), None);
+
+    let output = cmd.output()?;
+
+    if !output.status.success() {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            "failed to kill process",
+        ));
+    }
+
+    Ok(())
+}
+
+/// Create a `std::process::Command` for the current target platform, for killing
+/// the processes with the given PIDs
+pub fn build_kill_command(
+    force: bool,
+    pids: impl Iterator<Item = i64>,
+    signal: Option<u32>,
+) -> CommandSys {
+    if cfg!(windows) {
+        let mut cmd = CommandSys::new("taskkill");
+
+        if force {
+            cmd.arg("/F");
+        }
+
+        // each pid must written as `/PID 0` otherwise
+        // taskkill will act as `killall` unix command
+        for id in pids {
+            cmd.arg("/PID");
+            cmd.arg(id.to_string());
+        }
+
+        cmd
+    } else {
+        let mut cmd = CommandSys::new("kill");
+        if let Some(signal_value) = signal {
+            cmd.arg(format!("-{}", signal_value));
+        } else if force {
+            cmd.arg("-9");
+        }
+
+        cmd.args(pids.map(move |id| id.to_string()));
+
+        cmd
+    }
+}

src/main.rs

@@ -22,7 +22,7 @@ use command::gather_commandline_args;
 use log::{trace, Level};
 use miette::Result;
 use nu_cli::gather_parent_env_vars;
-use nu_engine::convert_env_values;
+use nu_engine::{convert_env_values, exit::cleanup_exit};
 use nu_lsp::LanguageServer;
 use nu_path::canonicalize_with;
 use nu_protocol::{
@@ -479,6 +479,8 @@ fn main() -> Result<()> {
             input,
             entire_start_time,
         );
+
+        cleanup_exit(0, &engine_state, 0);
     } else if !script_name.is_empty() {
         run_file(
             &mut engine_state,
@@ -489,6 +491,8 @@ fn main() -> Result<()> {
             args_to_script,
             input,
         );
+
+        cleanup_exit(0, &engine_state, 0);
     } else {
         // Environment variables that apply only when in REPL
         engine_state.add_env_var("PROMPT_INDICATOR".to_string(), Value::test_string("> "));
@@ -524,7 +528,9 @@ fn main() -> Result<()> {
             stack,
             parsed_nu_cli_args,
             entire_start_time,
-        )?
+        )?;
+
+        cleanup_exit(0, &engine_state, 0);
     }
 
     Ok(())