Split the plugin crate (#12563)

# Description

This breaks `nu-plugin` up into four crates:

- `nu-plugin-protocol`: just the type definitions for the protocol, no
I/O. If someone wanted to wire up something more bare metal, maybe for
async I/O, they could use this.
- `nu-plugin-core`: the shared stuff between engine/plugin. Less stable
interface.
- `nu-plugin-engine`: everything required for the engine to talk to
plugins. Less stable interface.
- `nu-plugin`: everything required for the plugin to talk to the engine,
what plugin developers use. Should be the most stable interface.

No changes are made to the interface exposed by `nu-plugin` - it should
all still be there. Re-exports from `nu-plugin-protocol` or
`nu-plugin-core` are used as required. Plugins shouldn't ever have to
use those crates directly.

This should be somewhat faster to compile as `nu-plugin-engine` and
`nu-plugin` can compile in parallel, and the engine doesn't need
`nu-plugin` and plugins don't need `nu-plugin-engine` (except for test
support), so that should reduce what needs to be compiled too.

The only significant change here other than splitting stuff up was to
break the `source` out of `PluginCustomValue` and create a new
`PluginCustomValueWithSource` type that contains that instead. One bonus
of that is we get rid of the option and it's now more type-safe, but it
also means that the logic for that stuff (actually running the plugin
for custom value ops) can live entirely within the `nu-plugin-engine`
crate.

# User-Facing Changes
- New crates.
- Added `local-socket` feature for `nu` to try to make it possible to
compile without that support if needed.

# Tests + Formatting
- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

Cargo.lock

@@ -2876,7 +2876,9 @@ dependencies = [
  "nu-lsp",
  "nu-parser",
  "nu-path",
- "nu-plugin",
+ "nu-plugin-core",
+ "nu-plugin-engine",
+ "nu-plugin-protocol",
  "nu-protocol",
  "nu-std",
  "nu-system",
@@ -2923,7 +2925,7 @@ dependencies = [
  "nu-engine",
  "nu-parser",
  "nu-path",
- "nu-plugin",
+ "nu-plugin-engine",
  "nu-protocol",
  "nu-test-support",
  "nu-utils",
@@ -3017,7 +3019,7 @@ dependencies = [
  "itertools 0.12.1",
  "nu-engine",
  "nu-path",
- "nu-plugin",
+ "nu-plugin-engine",
  "nu-protocol",
 ]
 
@@ -3221,7 +3223,7 @@ dependencies = [
  "log",
  "nu-engine",
  "nu-path",
- "nu-plugin",
+ "nu-plugin-engine",
  "nu-protocol",
  "rstest",
  "serde_json",
@@ -3240,24 +3242,58 @@ dependencies = [
 name = "nu-plugin"
 version = "0.92.3"
 dependencies = [
- "bincode",
- "interprocess",
  "log",
- "miette",
  "nix",
  "nu-engine",
+ "nu-plugin-core",
+ "nu-plugin-protocol",
  "nu-protocol",
- "nu-system",
- "nu-utils",
- "rmp-serde",
- "semver",
  "serde",
- "serde_json",
  "thiserror",
  "typetag",
+]
+
+[[package]]
+name = "nu-plugin-core"
+version = "0.92.3"
+dependencies = [
+ "interprocess",
+ "log",
+ "nu-plugin-protocol",
+ "nu-protocol",
+ "rmp-serde",
+ "serde",
+ "serde_json",
  "windows 0.54.0",
 ]
 
+[[package]]
+name = "nu-plugin-engine"
+version = "0.92.3"
+dependencies = [
+ "log",
+ "nu-engine",
+ "nu-plugin-core",
+ "nu-plugin-protocol",
+ "nu-protocol",
+ "nu-system",
+ "serde",
+ "typetag",
+ "windows 0.54.0",
+]
+
+[[package]]
+name = "nu-plugin-protocol"
+version = "0.92.3"
+dependencies = [
+ "bincode",
+ "nu-protocol",
+ "nu-utils",
+ "semver",
+ "serde",
+ "typetag",
+]
+
 [[package]]
 name = "nu-plugin-test-support"
 version = "0.92.3"
@@ -3267,6 +3303,9 @@ dependencies = [
  "nu-engine",
  "nu-parser",
  "nu-plugin",
+ "nu-plugin-core",
+ "nu-plugin-engine",
+ "nu-plugin-protocol",
  "nu-protocol",
  "serde",
  "similar",
@@ -4980,9 +5019,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
 
 [[package]]
 name = "rmp"
-version = "0.8.13"
+version = "0.8.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bddb316f4b9cae1a3e89c02f1926d557d1142d0d2e684b038c11c1b77705229a"
+checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4"
 dependencies = [
  "byteorder",
  "num-traits",

Cargo.toml

@@ -41,6 +41,9 @@ members = [
   "crates/nu-pretty-hex",
   "crates/nu-protocol",
   "crates/nu-plugin",
+  "crates/nu-plugin-core",
+  "crates/nu-plugin-engine",
+  "crates/nu-plugin-protocol",
   "crates/nu-plugin-test-support",
   "crates/nu_plugin_inc",
   "crates/nu_plugin_gstat",
@@ -90,6 +93,7 @@ heck = "0.5.0"
 human-date-parser = "0.1.1"
 indexmap = "2.2"
 indicatif = "0.17"
+interprocess = "1.2.1"
 is_executable = "1.0"
 itertools = "0.12"
 libc = "0.2"
@@ -184,7 +188,7 @@ nu-explore = { path = "./crates/nu-explore", version = "0.92.3" }
 nu-lsp = { path = "./crates/nu-lsp/", version = "0.92.3" }
 nu-parser = { path = "./crates/nu-parser", version = "0.92.3" }
 nu-path = { path = "./crates/nu-path", version = "0.92.3" }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.92.3" }
+nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.92.3" }
 nu-protocol = { path = "./crates/nu-protocol", version = "0.92.3" }
 nu-std = { path = "./crates/nu-std", version = "0.92.3" }
 nu-system = { path = "./crates/nu-system", version = "0.92.3" }
@@ -218,6 +222,8 @@ nix = { workspace = true, default-features = false, features = [
 
 [dev-dependencies]
 nu-test-support = { path = "./crates/nu-test-support", version = "0.92.3" }
+nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.92.3" }
+nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.92.3" }
 assert_cmd = "2.0"
 dirs-next = { workspace = true }
 divan = "0.1.14"
@@ -228,7 +234,7 @@ tempfile = { workspace = true }
 
 [features]
 plugin = [
-  "nu-plugin",
+  "nu-plugin-engine",
   "nu-cmd-plugin",
   "nu-cli/plugin",
   "nu-parser/plugin",

benches/benchmarks.rs

@@ -1,6 +1,7 @@
 use nu_cli::{eval_source, evaluate_commands};
 use nu_parser::parse;
-use nu_plugin::{Encoder, EncodingType, PluginCallResponse, PluginOutput};
+use nu_plugin_core::{Encoder, EncodingType};
+use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
 use nu_protocol::{
     engine::{EngineState, Stack},
     eval_const::create_nu_constant,

crates/nu-cli/Cargo.toml

@@ -21,7 +21,7 @@ nu-cmd-base = { path = "../nu-cmd-base", version = "0.92.3" }
 nu-engine = { path = "../nu-engine", version = "0.92.3" }
 nu-path = { path = "../nu-path", version = "0.92.3" }
 nu-parser = { path = "../nu-parser", version = "0.92.3" }
-nu-plugin = { path = "../nu-plugin", version = "0.92.3", optional = true }
+nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.92.3", optional = true }
 nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
 nu-utils = { path = "../nu-utils", version = "0.92.3" }
 nu-color-config = { path = "../nu-color-config", version = "0.92.3" }
@@ -45,5 +45,5 @@ uuid = { workspace = true, features = ["v4"] }
 which = { workspace = true }
 
 [features]
-plugin = ["nu-plugin"]
+plugin = ["nu-plugin-engine"]
 system-clipboard = ["reedline/system_clipboard"]

crates/nu-cli/src/config_files.rs

@@ -150,7 +150,7 @@ pub fn read_plugin_file(
 
         let mut working_set = StateWorkingSet::new(engine_state);
 
-        nu_plugin::load_plugin_file(&mut working_set, &contents, span);
+        nu_plugin_engine::load_plugin_file(&mut working_set, &contents, span);
 
         if let Err(err) = engine_state.merge_delta(working_set.render()) {
             report_error_new(engine_state, &err);

crates/nu-cmd-plugin/Cargo.toml

@@ -13,7 +13,7 @@ version = "0.92.3"
 nu-engine = { path = "../nu-engine", version = "0.92.3" }
 nu-path = { path = "../nu-path", version = "0.92.3" }
 nu-protocol = { path = "../nu-protocol", version = "0.92.3", features = ["plugin"] }
-nu-plugin = { path = "../nu-plugin", version = "0.92.3" }
+nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.92.3" }
 
 itertools = { workspace = true }
 

crates/nu-cmd-plugin/src/commands/plugin/add.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use nu_engine::{command_prelude::*, current_dir};
-use nu_plugin::{GetPlugin, PersistentPlugin};
+use nu_plugin_engine::{GetPlugin, PersistentPlugin};
 use nu_protocol::{PluginGcConfig, PluginIdentity, PluginRegistryItem, RegisteredPlugin};
 
 use crate::util::{get_plugin_dirs, modify_plugin_file};

crates/nu-parser/Cargo.toml

@@ -14,7 +14,7 @@ bench = false
 [dependencies]
 nu-engine = { path = "../nu-engine", version = "0.92.3" }
 nu-path = { path = "../nu-path", version = "0.92.3" }
-nu-plugin = { path = "../nu-plugin", optional = true, version = "0.92.3" }
+nu-plugin-engine = { path = "../nu-plugin-engine", optional = true, version = "0.92.3" }
 nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
 
 bytesize = { workspace = true }
@@ -27,4 +27,4 @@ serde_json = { workspace = true }
 rstest = { workspace = true, default-features = false }
 
 [features]
-plugin = ["nu-plugin"]
+plugin = ["nu-plugin-engine"]

crates/nu-parser/src/parse_keywords.rs

@@ -3555,7 +3555,7 @@ pub fn parse_where(working_set: &mut StateWorkingSet, lite_command: &LiteCommand
 /// `register` is deprecated and will be removed in 0.94. Use `plugin add` and `plugin use` instead.
 #[cfg(feature = "plugin")]
 pub fn parse_register(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) -> Pipeline {
-    use nu_plugin::{get_signature, PluginDeclaration};
+    use nu_plugin_engine::PluginDeclaration;
     use nu_protocol::{
         engine::Stack, ErrSpan, ParseWarning, PluginIdentity, PluginRegistryItem, PluginSignature,
         RegisteredPlugin,
@@ -3714,7 +3714,7 @@ pub fn parse_register(working_set: &mut StateWorkingSet, lite_command: &LiteComm
         // Create the plugin identity. This validates that the plugin name starts with `nu_plugin_`
         let identity = PluginIdentity::new(path, shell).err_span(path_span)?;
 
-        let plugin = nu_plugin::add_plugin_to_working_set(working_set, &identity)
+        let plugin = nu_plugin_engine::add_plugin_to_working_set(working_set, &identity)
             .map_err(|err| err.wrap(working_set, call.head))?;
 
         let signatures = signature.map_or_else(
@@ -3731,7 +3731,11 @@ pub fn parse_register(working_set: &mut StateWorkingSet, lite_command: &LiteComm
                     )
                 })?;
 
-                let signatures = get_signature(plugin.clone(), get_envs).map_err(|err| {
+                let signatures = plugin
+                    .clone()
+                    .get(get_envs)
+                    .and_then(|p| p.get_signature())
+                    .map_err(|err| {
                         log::warn!("Error getting signatures: {err:?}");
                         ParseError::LabeledError(
                             "Error getting signatures".into(),
@@ -3863,7 +3867,7 @@ pub fn parse_plugin_use(working_set: &mut StateWorkingSet, call: Box<Call>) -> P
             })?;
 
         // Now add the signatures to the working set
-        nu_plugin::load_plugin_registry_item(working_set, plugin_item, Some(call.head))
+        nu_plugin_engine::load_plugin_registry_item(working_set, plugin_item, Some(call.head))
             .map_err(|err| err.wrap(working_set, call.head))?;
 
         Ok(())

crates/nu-plugin-core/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "Shared internal functionality to support Nushell plugins"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-plugin-core"
+edition = "2021"
+license = "MIT"
+name = "nu-plugin-core"
+version = "0.92.3"
+
+[lib]
+bench = false
+
+[dependencies]
+nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
+nu-plugin-protocol = { path = "../nu-plugin-protocol", version = "0.92.3" }
+
+rmp-serde = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+log = { workspace = true }
+interprocess = { workspace = true, optional = true }
+
+[features]
+default = ["local-socket"]
+local-socket = ["interprocess"]
+
+[target.'cfg(target_os = "windows")'.dependencies]
+windows = { workspace = true }

crates/nu-plugin-core/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-plugin-core/README.md

@@ -0,0 +1,3 @@
+# nu-plugin-core
+
+This crate provides functionality that is shared by the [Nushell](https://nushell.sh/) engine and plugins.

crates/nu-plugin-core/src/communication_mode/mod.rs

@@ -13,8 +13,15 @@ mod local_socket;
 #[cfg(feature = "local-socket")]
 use local_socket::*;
 
+/// The type of communication used between the plugin and the engine.
+///
+/// `Stdio` is required to be supported by all plugins, and is attempted initially. If the
+/// `local-socket` feature is enabled and the plugin supports it, `LocalSocket` may be attempted.
+///
+/// Local socket communication has the benefit of not tying up stdio, so it's more compatible with
+/// plugins that want to take user input from the terminal in some way.
 #[derive(Debug, Clone)]
-pub(crate) enum CommunicationMode {
+pub enum CommunicationMode {
     /// Communicate using `stdin` and `stdout`.
     Stdio,
     /// Communicate using an operating system-specific local socket.
@@ -115,8 +122,15 @@ impl CommunicationMode {
     }
 }
 
-pub(crate) enum PreparedServerCommunication {
+/// The result of [`CommunicationMode::serve()`], which acts as an intermediate stage for
+/// communication modes that require some kind of socket binding to occur before the client process
+/// can be started. Call [`.connect()`] once the client process has been started.
+///
+/// The socket may be cleaned up on `Drop` if applicable.
+pub enum PreparedServerCommunication {
+    /// Will take stdin and stdout from the process on [`.connect()`].
     Stdio,
+    /// Contains the listener to accept connections on. On Unix, the socket is unlinked on `Drop`.
     #[cfg(feature = "local-socket")]
     LocalSocket {
         #[cfg_attr(windows, allow(dead_code))] // not used on Windows
@@ -214,7 +228,8 @@ impl Drop for PreparedServerCommunication {
     }
 }
 
-pub(crate) enum ServerCommunicationIo {
+/// The required streams for communication from the engine side, i.e. the server in socket terms.
+pub enum ServerCommunicationIo {
     Stdio(ChildStdin, ChildStdout),
     #[cfg(feature = "local-socket")]
     LocalSocket {
@@ -223,7 +238,8 @@ pub(crate) enum ServerCommunicationIo {
     },
 }
 
-pub(crate) enum ClientCommunicationIo {
+/// The required streams for communication from the plugin side, i.e. the client in socket terms.
+pub enum ClientCommunicationIo {
     Stdio(Stdin, Stdout),
     #[cfg(feature = "local-socket")]
     LocalSocket {

crates/nu-plugin-core/src/interface/mod.rs

@@ -1,11 +1,7 @@
 //! Implements the stream multiplexing interface for both the plugin side and the engine side.
 
-use crate::{
+use nu_plugin_protocol::{
-    plugin::Encoder,
-    protocol::{
     ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, RawStreamInfo, StreamMessage,
-    },
-    sequence::Sequence,
 };
 use nu_protocol::{ListStream, PipelineData, RawStream, ShellError};
 use std::{
@@ -17,18 +13,13 @@ use std::{
     thread,
 };
 
-mod stream;
+pub mod stream;
 
-mod engine;
+use crate::{util::Sequence, Encoder};
-pub use engine::{EngineInterface, EngineInterfaceManager, ReceivedPluginCall};
-
-mod plugin;
-pub use plugin::{PluginInterface, PluginInterfaceManager};
 
 use self::stream::{StreamManager, StreamManagerHandle, StreamWriter, WriteStreamMessage};
 
-#[cfg(test)]
+pub mod test_util;
-mod test_util;
 
 #[cfg(test)]
 mod tests;
@@ -42,9 +33,6 @@ const LIST_STREAM_HIGH_PRESSURE: i32 = 100;
 const RAW_STREAM_HIGH_PRESSURE: i32 = 50;
 
 /// Read input/output from the stream.
-///
-/// This is not a public API.
-#[doc(hidden)]
 pub trait PluginRead<T> {
     /// Returns `Ok(None)` on end of stream.
     fn read(&mut self) -> Result<Option<T>, ShellError>;
@@ -72,9 +60,6 @@ where
 /// Write input/output to the stream.
 ///
 /// The write should be atomic, without interference from other threads.
-///
-/// This is not a public API.
-#[doc(hidden)]
 pub trait PluginWrite<T>: Send + Sync {
     fn write(&self, data: &T) -> Result<(), ShellError>;
 
@@ -146,15 +131,13 @@ where
     }
 }
 
-/// An interface manager handles I/O and state management for communication between a plugin and the
+/// An interface manager handles I/O and state management for communication between a plugin and
-/// engine. See [`PluginInterfaceManager`] for communication from the engine side to a plugin, or
+/// the engine. See `PluginInterfaceManager` in `nu-plugin-engine` for communication from the engine
-/// [`EngineInterfaceManager`] for communication from the plugin side to the engine.
+/// side to a plugin, or `EngineInterfaceManager` in `nu-plugin` for communication from the plugin
+/// side to the engine.
 ///
 /// There is typically one [`InterfaceManager`] consuming input from a background thread, and
 /// managing shared state.
-///
-/// This is not a public API.
-#[doc(hidden)]
 pub trait InterfaceManager {
     /// The corresponding interface type.
     type Interface: Interface + 'static;
@@ -233,13 +216,10 @@ pub trait InterfaceManager {
 }
 
 /// An interface provides an API for communicating with a plugin or the engine and facilitates
-/// stream I/O. See [`PluginInterface`] for the API from the engine side to a plugin, or
+/// stream I/O. See `PluginInterface` in `nu-plugin-engine` for the API from the engine side to a
-/// [`EngineInterface`] for the API from the plugin side to the engine.
+/// plugin, or `EngineInterface` in `nu-plugin` for the API from the plugin side to the engine.
 ///
 /// There can be multiple copies of the interface managed by a single [`InterfaceManager`].
-///
-/// This is not a public API.
-#[doc(hidden)]
 pub trait Interface: Clone + Send {
     /// The output message type, which must be capable of encapsulating a [`StreamMessage`].
     type Output: From<StreamMessage>;
@@ -253,7 +233,7 @@ pub trait Interface: Clone + Send {
     /// Flush the output buffer, so messages are visible to the other side.
     fn flush(&self) -> Result<(), ShellError>;
 
-    /// Get the sequence for generating new [`StreamId`](crate::protocol::StreamId)s.
+    /// Get the sequence for generating new [`StreamId`](nu_plugin_protocol::StreamId)s.
     fn stream_id_sequence(&self) -> &Sequence;
 
     /// Get the [`StreamManagerHandle`] for doing stream operations.
@@ -384,7 +364,7 @@ where
     W: WriteStreamMessage + Send + 'static,
 {
     /// Write all of the data in each of the streams. This method waits for completion.
-    pub(crate) fn write(self) -> Result<(), ShellError> {
+    pub fn write(self) -> Result<(), ShellError> {
         match self {
             // If no stream was contained in the PipelineData, do nothing.
             PipelineDataWriter::None => Ok(()),
@@ -442,7 +422,7 @@ where
 
     /// Write all of the data in each of the streams. This method returns immediately; any necessary
     /// write will happen in the background. If a thread was spawned, its handle is returned.
-    pub(crate) fn write_background(
+    pub fn write_background(
         self,
     ) -> Result<Option<thread::JoinHandle<Result<(), ShellError>>>, ShellError> {
         match self {

crates/nu-plugin-core/src/interface/stream/mod.rs

@@ -1,4 +1,4 @@
-use crate::protocol::{StreamData, StreamId, StreamMessage};
+use nu_plugin_protocol::{StreamData, StreamId, StreamMessage};
 use nu_protocol::{ShellError, Span, Value};
 use std::{
     collections::{btree_map, BTreeMap},
@@ -25,7 +25,7 @@ mod tests;
 /// For each message read, it sends [`StreamMessage::Ack`] to the writer. When dropped,
 /// it sends [`StreamMessage::Drop`].
 #[derive(Debug)]
-pub(crate) struct StreamReader<T, W>
+pub struct StreamReader<T, W>
 where
     W: WriteStreamMessage,
 {
@@ -43,7 +43,7 @@ where
     W: WriteStreamMessage,
 {
     /// Create a new StreamReader from parts
-    pub(crate) fn new(
+    fn new(
         id: StreamId,
         receiver: mpsc::Receiver<Result<Option<StreamData>, ShellError>>,
         writer: W,
@@ -61,7 +61,7 @@ where
     /// * the channel couldn't be received from
     /// * an error was sent on the channel
     /// * the message received couldn't be converted to `T`
-    pub(crate) fn recv(&mut self) -> Result<Option<T>, ShellError> {
+    pub fn recv(&mut self) -> Result<Option<T>, ShellError> {
         let connection_lost = || ShellError::GenericError {
             error: "Stream ended unexpectedly".into(),
             msg: "connection lost before explicit end of stream".into(),
@@ -146,7 +146,7 @@ where
 }
 
 /// Values that can contain a `ShellError` to signal an error has occurred.
-pub(crate) trait FromShellError {
+pub trait FromShellError {
     fn from_shell_error(err: ShellError) -> Self;
 }
 
@@ -179,7 +179,7 @@ impl<W> StreamWriter<W>
 where
     W: WriteStreamMessage,
 {
-    pub(crate) fn new(id: StreamId, signal: Arc<StreamWriterSignal>, writer: W) -> StreamWriter<W> {
+    fn new(id: StreamId, signal: Arc<StreamWriterSignal>, writer: W) -> StreamWriter<W> {
         StreamWriter {
             id,
             signal,
@@ -190,7 +190,7 @@ where
 
     /// Check if the stream was dropped from the other end. Recommended to do this before calling
     /// [`.write()`], especially in a loop.
-    pub(crate) fn is_dropped(&self) -> Result<bool, ShellError> {
+    pub fn is_dropped(&self) -> Result<bool, ShellError> {
         self.signal.is_dropped()
     }
 
@@ -198,7 +198,7 @@ where
     ///
     /// Error if something failed with the write, or if [`.end()`] was already called
     /// previously.
-    pub(crate) fn write(&mut self, data: impl Into<StreamData>) -> Result<(), ShellError> {
+    pub fn write(&mut self, data: impl Into<StreamData>) -> Result<(), ShellError> {
         if !self.ended {
             self.writer
                 .write_stream_message(StreamMessage::Data(self.id, data.into()))?;
@@ -232,10 +232,7 @@ where
     ///
     /// Returns `Ok(true)` if the iterator was fully consumed, or `Ok(false)` if a drop interrupted
     /// the stream from the other side.
-    pub(crate) fn write_all<T>(
+    pub fn write_all<T>(&mut self, data: impl IntoIterator<Item = T>) -> Result<bool, ShellError>
-        &mut self,
-        data: impl IntoIterator<Item = T>,
-    ) -> Result<bool, ShellError>
     where
         T: Into<StreamData>,
     {
@@ -257,7 +254,7 @@ where
 
     /// End the stream. Recommend doing this instead of relying on `Drop` so that you can catch the
     /// error.
-    pub(crate) fn end(&mut self) -> Result<(), ShellError> {
+    pub fn end(&mut self) -> Result<(), ShellError> {
         if !self.ended {
             // Set the flag first so we don't double-report in the Drop
             self.ended = true;
@@ -285,13 +282,13 @@ where
 /// Stores stream state for a writer, and can be blocked on to wait for messages to be acknowledged.
 /// A key part of managing stream lifecycle and flow control.
 #[derive(Debug)]
-pub(crate) struct StreamWriterSignal {
+pub struct StreamWriterSignal {
     mutex: Mutex<StreamWriterSignalState>,
     change_cond: Condvar,
 }
 
 #[derive(Debug)]
-pub(crate) struct StreamWriterSignalState {
+pub struct StreamWriterSignalState {
     /// Stream has been dropped and consumer is no longer interested in any messages.
     dropped: bool,
     /// Number of messages that have been sent without acknowledgement.
@@ -306,7 +303,7 @@ impl StreamWriterSignal {
     /// If `notify_sent()` is called more than `high_pressure_mark` times, it will wait until
     /// `notify_acknowledge()` is called by another thread enough times to bring the number of
     /// unacknowledged sent messages below that threshold.
-    pub(crate) fn new(high_pressure_mark: i32) -> StreamWriterSignal {
+    fn new(high_pressure_mark: i32) -> StreamWriterSignal {
         assert!(high_pressure_mark > 0);
 
         StreamWriterSignal {
@@ -327,12 +324,12 @@ impl StreamWriterSignal {
 
     /// True if the stream was dropped and the consumer is no longer interested in it. Indicates
     /// that no more messages should be sent, other than `End`.
-    pub(crate) fn is_dropped(&self) -> Result<bool, ShellError> {
+    pub fn is_dropped(&self) -> Result<bool, ShellError> {
         Ok(self.lock()?.dropped)
     }
 
     /// Notify the writers that the stream has been dropped, so they can stop writing.
-    pub(crate) fn set_dropped(&self) -> Result<(), ShellError> {
+    pub fn set_dropped(&self) -> Result<(), ShellError> {
         let mut state = self.lock()?;
         state.dropped = true;
         // Unblock the writers so they can terminate
@@ -343,7 +340,7 @@ impl StreamWriterSignal {
     /// Track that a message has been sent. Returns `Ok(true)` if more messages can be sent,
     /// or `Ok(false)` if the high pressure mark has been reached and [`.wait_for_drain()`] should
     /// be called to block.
-    pub(crate) fn notify_sent(&self) -> Result<bool, ShellError> {
+    pub fn notify_sent(&self) -> Result<bool, ShellError> {
         let mut state = self.lock()?;
         state.unacknowledged =
             state
@@ -357,7 +354,7 @@ impl StreamWriterSignal {
     }
 
     /// Wait for acknowledgements before sending more data. Also returns if the stream is dropped.
-    pub(crate) fn wait_for_drain(&self) -> Result<(), ShellError> {
+    pub fn wait_for_drain(&self) -> Result<(), ShellError> {
         let mut state = self.lock()?;
         while !state.dropped && state.unacknowledged >= state.high_pressure_mark {
             state = self
@@ -372,7 +369,7 @@ impl StreamWriterSignal {
 
     /// Notify the writers that a message has been acknowledged, so they can continue to write
     /// if they were waiting.
-    pub(crate) fn notify_acknowledged(&self) -> Result<(), ShellError> {
+    pub fn notify_acknowledged(&self) -> Result<(), ShellError> {
         let mut state = self.lock()?;
         state.unacknowledged =
             state
@@ -417,7 +414,7 @@ pub struct StreamManager {
 
 impl StreamManager {
     /// Create a new StreamManager.
-    pub(crate) fn new() -> StreamManager {
+    pub fn new() -> StreamManager {
         StreamManager {
             state: Default::default(),
         }
@@ -428,14 +425,14 @@ impl StreamManager {
     }
 
     /// Create a new handle to the StreamManager for registering streams.
-    pub(crate) fn get_handle(&self) -> StreamManagerHandle {
+    pub fn get_handle(&self) -> StreamManagerHandle {
         StreamManagerHandle {
             state: Arc::downgrade(&self.state),
         }
     }
 
     /// Process a stream message, and update internal state accordingly.
-    pub(crate) fn handle_message(&self, message: StreamMessage) -> Result<(), ShellError> {
+    pub fn handle_message(&self, message: StreamMessage) -> Result<(), ShellError> {
         let mut state = self.lock()?;
         match message {
             StreamMessage::Data(id, data) => {
@@ -492,7 +489,7 @@ impl StreamManager {
     }
 
     /// Broadcast an error to all stream readers. This is useful for error propagation.
-    pub(crate) fn broadcast_read_error(&self, error: ShellError) -> Result<(), ShellError> {
+    pub fn broadcast_read_error(&self, error: ShellError) -> Result<(), ShellError> {
         let state = self.lock()?;
         for channel in state.reading_streams.values() {
             // Ignore send errors.
@@ -517,6 +514,12 @@ impl StreamManager {
     }
 }
 
+impl Default for StreamManager {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Drop for StreamManager {
     fn drop(&mut self) {
         if let Err(err) = self.drop_all_writers() {
@@ -557,7 +560,7 @@ impl StreamManagerHandle {
     /// Register a new stream for reading, and return a [`StreamReader`] that can be used to iterate
     /// on the values received. A [`StreamMessage`] writer is required for writing control messages
     /// back to the producer.
-    pub(crate) fn read_stream<T, W>(
+    pub fn read_stream<T, W>(
         &self,
         id: StreamId,
         writer: W,
@@ -591,7 +594,7 @@ impl StreamManagerHandle {
     /// The `high_pressure_mark` value controls how many messages can be written without receiving
     /// an acknowledgement before any further attempts to write will wait for the consumer to
     /// acknowledge them. This prevents overwhelming the reader.
-    pub(crate) fn write_stream<W>(
+    pub fn write_stream<W>(
         &self,
         id: StreamId,
         writer: W,

crates/nu-plugin-core/src/interface/stream/tests.rs

@@ -7,7 +7,7 @@ use std::{
 };
 
 use super::{StreamManager, StreamReader, StreamWriter, StreamWriterSignal, WriteStreamMessage};
-use crate::protocol::{StreamData, StreamMessage};
+use nu_plugin_protocol::{StreamData, StreamMessage};
 use nu_protocol::{ShellError, Value};
 
 // Should be long enough to definitely complete any quick operation, but not so long that tests are

crates/nu-plugin-core/src/interface/test_util.rs

@@ -1,20 +1,22 @@
-use super::{EngineInterfaceManager, PluginInterfaceManager, PluginRead, PluginWrite};
-use crate::{plugin::PluginSource, protocol::PluginInput, PluginOutput};
 use nu_protocol::ShellError;
 use std::{
     collections::VecDeque,
     sync::{Arc, Mutex},
 };
 
+use crate::{PluginRead, PluginWrite};
+
+const FAILED: &str = "failed to lock TestCase";
+
 /// Mock read/write helper for the engine and plugin interfaces.
 #[derive(Debug, Clone)]
-pub(crate) struct TestCase<I, O> {
+pub struct TestCase<I, O> {
     r#in: Arc<Mutex<TestData<I>>>,
     out: Arc<Mutex<TestData<O>>>,
 }
 
 #[derive(Debug)]
-pub(crate) struct TestData<T> {
+pub struct TestData<T> {
     data: VecDeque<T>,
     error: Option<ShellError>,
     flushed: bool,
@@ -32,7 +34,7 @@ impl<T> Default for TestData<T> {
 
 impl<I, O> PluginRead<I> for TestCase<I, O> {
     fn read(&mut self) -> Result<Option<I>, ShellError> {
-        let mut lock = self.r#in.lock().unwrap();
+        let mut lock = self.r#in.lock().expect(FAILED);
         if let Some(err) = lock.error.take() {
             Err(err)
         } else {
@@ -47,7 +49,7 @@ where
     O: Send + Clone,
 {
     fn write(&self, data: &O) -> Result<(), ShellError> {
-        let mut lock = self.out.lock().unwrap();
+        let mut lock = self.out.lock().expect(FAILED);
         lock.flushed = false;
 
         if let Some(err) = lock.error.take() {
@@ -59,7 +61,7 @@ where
     }
 
     fn flush(&self) -> Result<(), ShellError> {
-        let mut lock = self.out.lock().unwrap();
+        let mut lock = self.out.lock().expect(FAILED);
         lock.flushed = true;
         Ok(())
     }
@@ -67,7 +69,7 @@ where
 
 #[allow(dead_code)]
 impl<I, O> TestCase<I, O> {
-    pub(crate) fn new() -> TestCase<I, O> {
+    pub fn new() -> TestCase<I, O> {
         TestCase {
             r#in: Default::default(),
             out: Default::default(),
@@ -75,66 +77,58 @@ impl<I, O> TestCase<I, O> {
     }
 
     /// Clear the read buffer.
-    pub(crate) fn clear(&self) {
+    pub fn clear(&self) {
-        self.r#in.lock().unwrap().data.truncate(0);
+        self.r#in.lock().expect(FAILED).data.truncate(0);
     }
 
     /// Add input that will be read by the interface.
-    pub(crate) fn add(&self, input: impl Into<I>) {
+    pub fn add(&self, input: impl Into<I>) {
-        self.r#in.lock().unwrap().data.push_back(input.into());
+        self.r#in.lock().expect(FAILED).data.push_back(input.into());
     }
 
     /// Add multiple inputs that will be read by the interface.
-    pub(crate) fn extend(&self, inputs: impl IntoIterator<Item = I>) {
+    pub fn extend(&self, inputs: impl IntoIterator<Item = I>) {
-        self.r#in.lock().unwrap().data.extend(inputs);
+        self.r#in.lock().expect(FAILED).data.extend(inputs);
     }
 
     /// Return an error from the next read operation.
-    pub(crate) fn set_read_error(&self, err: ShellError) {
+    pub fn set_read_error(&self, err: ShellError) {
-        self.r#in.lock().unwrap().error = Some(err);
+        self.r#in.lock().expect(FAILED).error = Some(err);
     }
 
     /// Return an error from the next write operation.
-    pub(crate) fn set_write_error(&self, err: ShellError) {
+    pub fn set_write_error(&self, err: ShellError) {
-        self.out.lock().unwrap().error = Some(err);
+        self.out.lock().expect(FAILED).error = Some(err);
     }
 
     /// Get the next output that was written.
-    pub(crate) fn next_written(&self) -> Option<O> {
+    pub fn next_written(&self) -> Option<O> {
-        self.out.lock().unwrap().data.pop_front()
+        self.out.lock().expect(FAILED).data.pop_front()
     }
 
     /// Iterator over written data.
-    pub(crate) fn written(&self) -> impl Iterator<Item = O> + '_ {
+    pub fn written(&self) -> impl Iterator<Item = O> + '_ {
         std::iter::from_fn(|| self.next_written())
     }
 
     /// Returns true if the writer was flushed after the last write operation.
-    pub(crate) fn was_flushed(&self) -> bool {
+    pub fn was_flushed(&self) -> bool {
-        self.out.lock().unwrap().flushed
+        self.out.lock().expect(FAILED).flushed
     }
 
     /// Returns true if the reader has unconsumed reads.
-    pub(crate) fn has_unconsumed_read(&self) -> bool {
+    pub fn has_unconsumed_read(&self) -> bool {
-        !self.r#in.lock().unwrap().data.is_empty()
+        !self.r#in.lock().expect(FAILED).data.is_empty()
     }
 
     /// Returns true if the writer has unconsumed writes.
-    pub(crate) fn has_unconsumed_write(&self) -> bool {
+    pub fn has_unconsumed_write(&self) -> bool {
-        !self.out.lock().unwrap().data.is_empty()
+        !self.out.lock().expect(FAILED).data.is_empty()
     }
 }
 
-impl TestCase<PluginOutput, PluginInput> {
+impl<I, O> Default for TestCase<I, O> {
-    /// Create a new [`PluginInterfaceManager`] that writes to this test case.
+    fn default() -> Self {
-    pub(crate) fn plugin(&self, name: &str) -> PluginInterfaceManager {
+        Self::new()
-        PluginInterfaceManager::new(PluginSource::new_fake(name).into(), None, self.clone())
-    }
-}
-
-impl TestCase<PluginInput, PluginOutput> {
-    /// Create a new [`EngineInterfaceManager`] that writes to this test case.
-    pub(crate) fn engine(&self) -> EngineInterfaceManager {
-        EngineInterfaceManager::new(self.clone())
     }
 }

crates/nu-plugin-core/src/interface/tests.rs

@@ -0,0 +1,573 @@
+use crate::util::Sequence;
+
+use super::{
+    stream::{StreamManager, StreamManagerHandle},
+    test_util::TestCase,
+    Interface, InterfaceManager, PluginRead, PluginWrite,
+};
+use nu_plugin_protocol::{
+    ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, PluginInput, PluginOutput,
+    RawStreamInfo, StreamData, StreamMessage,
+};
+use nu_protocol::{
+    DataSource, ListStream, PipelineData, PipelineMetadata, RawStream, ShellError, Span, Value,
+};
+use std::{path::Path, sync::Arc};
+
+fn test_metadata() -> PipelineMetadata {
+    PipelineMetadata {
+        data_source: DataSource::FilePath("/test/path".into()),
+    }
+}
+
+#[derive(Debug)]
+struct TestInterfaceManager {
+    stream_manager: StreamManager,
+    test: TestCase<PluginInput, PluginOutput>,
+    seq: Arc<Sequence>,
+}
+
+#[derive(Debug, Clone)]
+struct TestInterface {
+    stream_manager_handle: StreamManagerHandle,
+    test: TestCase<PluginInput, PluginOutput>,
+    seq: Arc<Sequence>,
+}
+
+impl TestInterfaceManager {
+    fn new(test: &TestCase<PluginInput, PluginOutput>) -> TestInterfaceManager {
+        TestInterfaceManager {
+            stream_manager: StreamManager::new(),
+            test: test.clone(),
+            seq: Arc::new(Sequence::default()),
+        }
+    }
+
+    fn consume_all(&mut self) -> Result<(), ShellError> {
+        while let Some(msg) = self.test.read()? {
+            self.consume(msg)?;
+        }
+        Ok(())
+    }
+}
+
+impl InterfaceManager for TestInterfaceManager {
+    type Interface = TestInterface;
+    type Input = PluginInput;
+
+    fn get_interface(&self) -> Self::Interface {
+        TestInterface {
+            stream_manager_handle: self.stream_manager.get_handle(),
+            test: self.test.clone(),
+            seq: self.seq.clone(),
+        }
+    }
+
+    fn consume(&mut self, input: Self::Input) -> Result<(), ShellError> {
+        match input {
+            PluginInput::Data(..)
+            | PluginInput::End(..)
+            | PluginInput::Drop(..)
+            | PluginInput::Ack(..) => self.consume_stream_message(
+                input
+                    .try_into()
+                    .expect("failed to convert message to StreamMessage"),
+            ),
+            _ => unimplemented!(),
+        }
+    }
+
+    fn stream_manager(&self) -> &StreamManager {
+        &self.stream_manager
+    }
+
+    fn prepare_pipeline_data(&self, data: PipelineData) -> Result<PipelineData, ShellError> {
+        Ok(data.set_metadata(Some(test_metadata())))
+    }
+}
+
+impl Interface for TestInterface {
+    type Output = PluginOutput;
+    type DataContext = ();
+
+    fn write(&self, output: Self::Output) -> Result<(), ShellError> {
+        self.test.write(&output)
+    }
+
+    fn flush(&self) -> Result<(), ShellError> {
+        Ok(())
+    }
+
+    fn stream_id_sequence(&self) -> &Sequence {
+        &self.seq
+    }
+
+    fn stream_manager_handle(&self) -> &StreamManagerHandle {
+        &self.stream_manager_handle
+    }
+
+    fn prepare_pipeline_data(
+        &self,
+        data: PipelineData,
+        _context: &(),
+    ) -> Result<PipelineData, ShellError> {
+        // Add an arbitrary check to the data to verify this is being called
+        match data {
+            PipelineData::Value(Value::Binary { .. }, None) => Err(ShellError::NushellFailed {
+                msg: "TEST can't send binary".into(),
+            }),
+            _ => Ok(data),
+        }
+    }
+}
+
+#[test]
+fn read_pipeline_data_empty() -> Result<(), ShellError> {
+    let manager = TestInterfaceManager::new(&TestCase::new());
+    let header = PipelineDataHeader::Empty;
+
+    assert!(matches!(
+        manager.read_pipeline_data(header, None)?,
+        PipelineData::Empty
+    ));
+    Ok(())
+}
+
+#[test]
+fn read_pipeline_data_value() -> Result<(), ShellError> {
+    let manager = TestInterfaceManager::new(&TestCase::new());
+    let value = Value::test_int(4);
+    let header = PipelineDataHeader::Value(value.clone());
+
+    match manager.read_pipeline_data(header, None)? {
+        PipelineData::Value(read_value, _) => assert_eq!(value, read_value),
+        PipelineData::ListStream(_, _) => panic!("unexpected ListStream"),
+        PipelineData::ExternalStream { .. } => panic!("unexpected ExternalStream"),
+        PipelineData::Empty => panic!("unexpected Empty"),
+    }
+
+    Ok(())
+}
+
+#[test]
+fn read_pipeline_data_list_stream() -> Result<(), ShellError> {
+    let test = TestCase::new();
+    let mut manager = TestInterfaceManager::new(&test);
+
+    let data = (0..100).map(Value::test_int).collect::<Vec<_>>();
+
+    for value in &data {
+        test.add(StreamMessage::Data(7, value.clone().into()));
+    }
+    test.add(StreamMessage::End(7));
+
+    let header = PipelineDataHeader::ListStream(ListStreamInfo { id: 7 });
+
+    let pipe = manager.read_pipeline_data(header, None)?;
+    assert!(
+        matches!(pipe, PipelineData::ListStream(..)),
+        "unexpected PipelineData: {pipe:?}"
+    );
+
+    // need to consume input
+    manager.consume_all()?;
+
+    let mut count = 0;
+    for (expected, read) in data.into_iter().zip(pipe) {
+        assert_eq!(expected, read);
+        count += 1;
+    }
+    assert_eq!(100, count);
+
+    assert!(test.has_unconsumed_write());
+
+    Ok(())
+}
+
+#[test]
+fn read_pipeline_data_external_stream() -> Result<(), ShellError> {
+    let test = TestCase::new();
+    let mut manager = TestInterfaceManager::new(&test);
+
+    let iterations = 100;
+    let out_pattern = b"hello".to_vec();
+    let err_pattern = vec![5, 4, 3, 2];
+
+    test.add(StreamMessage::Data(14, Value::test_int(1).into()));
+    for _ in 0..iterations {
+        test.add(StreamMessage::Data(
+            12,
+            StreamData::Raw(Ok(out_pattern.clone())),
+        ));
+        test.add(StreamMessage::Data(
+            13,
+            StreamData::Raw(Ok(err_pattern.clone())),
+        ));
+    }
+    test.add(StreamMessage::End(12));
+    test.add(StreamMessage::End(13));
+    test.add(StreamMessage::End(14));
+
+    let test_span = Span::new(10, 13);
+    let header = PipelineDataHeader::ExternalStream(ExternalStreamInfo {
+        span: test_span,
+        stdout: Some(RawStreamInfo {
+            id: 12,
+            is_binary: false,
+            known_size: Some((out_pattern.len() * iterations) as u64),
+        }),
+        stderr: Some(RawStreamInfo {
+            id: 13,
+            is_binary: true,
+            known_size: None,
+        }),
+        exit_code: Some(ListStreamInfo { id: 14 }),
+        trim_end_newline: true,
+    });
+
+    let pipe = manager.read_pipeline_data(header, None)?;
+
+    // need to consume input
+    manager.consume_all()?;
+
+    match pipe {
+        PipelineData::ExternalStream {
+            stdout,
+            stderr,
+            exit_code,
+            span,
+            metadata,
+            trim_end_newline,
+        } => {
+            let stdout = stdout.expect("stdout is None");
+            let stderr = stderr.expect("stderr is None");
+            let exit_code = exit_code.expect("exit_code is None");
+            assert_eq!(test_span, span);
+            assert!(
+                metadata.is_some(),
+                "expected metadata to be Some due to prepare_pipeline_data()"
+            );
+            assert!(trim_end_newline);
+
+            assert!(!stdout.is_binary);
+            assert!(stderr.is_binary);
+
+            assert_eq!(
+                Some((out_pattern.len() * iterations) as u64),
+                stdout.known_size
+            );
+            assert_eq!(None, stderr.known_size);
+
+            // check the streams
+            let mut count = 0;
+            for chunk in stdout.stream {
+                assert_eq!(out_pattern, chunk?);
+                count += 1;
+            }
+            assert_eq!(iterations, count, "stdout length");
+            let mut count = 0;
+
+            for chunk in stderr.stream {
+                assert_eq!(err_pattern, chunk?);
+                count += 1;
+            }
+            assert_eq!(iterations, count, "stderr length");
+
+            assert_eq!(vec![Value::test_int(1)], exit_code.collect::<Vec<_>>());
+        }
+        _ => panic!("unexpected PipelineData: {pipe:?}"),
+    }
+
+    // Don't need to check exactly what was written, just be sure that there is some output
+    assert!(test.has_unconsumed_write());
+
+    Ok(())
+}
+
+#[test]
+fn read_pipeline_data_ctrlc() -> Result<(), ShellError> {
+    let manager = TestInterfaceManager::new(&TestCase::new());
+    let header = PipelineDataHeader::ListStream(ListStreamInfo { id: 0 });
+    let ctrlc = Default::default();
+    match manager.read_pipeline_data(header, Some(&ctrlc))? {
+        PipelineData::ListStream(
+            ListStream {
+                ctrlc: stream_ctrlc,
+                ..
+            },
+            _,
+        ) => {
+            assert!(Arc::ptr_eq(&ctrlc, &stream_ctrlc.expect("ctrlc not set")));
+            Ok(())
+        }
+        _ => panic!("Unexpected PipelineData, should have been ListStream"),
+    }
+}
+
+#[test]
+fn read_pipeline_data_prepared_properly() -> Result<(), ShellError> {
+    let manager = TestInterfaceManager::new(&TestCase::new());
+    let header = PipelineDataHeader::ListStream(ListStreamInfo { id: 0 });
+    match manager.read_pipeline_data(header, None)? {
+        PipelineData::ListStream(_, meta) => match meta {
+            Some(PipelineMetadata { data_source }) => match data_source {
+                DataSource::FilePath(path) => {
+                    assert_eq!(Path::new("/test/path"), path);
+                    Ok(())
+                }
+                _ => panic!("wrong metadata: {data_source:?}"),
+            },
+            None => panic!("metadata not set"),
+        },
+        _ => panic!("Unexpected PipelineData, should have been ListStream"),
+    }
+}
+
+#[test]
+fn write_pipeline_data_empty() -> Result<(), ShellError> {
+    let test = TestCase::new();
+    let manager = TestInterfaceManager::new(&test);
+    let interface = manager.get_interface();
+
+    let (header, writer) = interface.init_write_pipeline_data(PipelineData::Empty, &())?;
+
+    assert!(matches!(header, PipelineDataHeader::Empty));
+
+    writer.write()?;
+
+    assert!(
+        !test.has_unconsumed_write(),
+        "Empty shouldn't write any stream messages, test: {test:#?}"
+    );
+
+    Ok(())
+}
+
+#[test]
+fn write_pipeline_data_value() -> Result<(), ShellError> {
+    let test = TestCase::new();
+    let manager = TestInterfaceManager::new(&test);
+    let interface = manager.get_interface();
+    let value = Value::test_int(7);
+
+    let (header, writer) =
+        interface.init_write_pipeline_data(PipelineData::Value(value.clone(), None), &())?;
+
+    match header {
+        PipelineDataHeader::Value(read_value) => assert_eq!(value, read_value),
+        _ => panic!("unexpected header: {header:?}"),
+    }
+
+    writer.write()?;
+
+    assert!(
+        !test.has_unconsumed_write(),
+        "Value shouldn't write any stream messages, test: {test:#?}"
+    );
+
+    Ok(())
+}
+
+#[test]
+fn write_pipeline_data_prepared_properly() {
+    let manager = TestInterfaceManager::new(&TestCase::new());
+    let interface = manager.get_interface();
+
+    // Sending a binary should be an error in our test scenario
+    let value = Value::test_binary(vec![7, 8]);
+
+    match interface.init_write_pipeline_data(PipelineData::Value(value, None), &()) {
+        Ok(_) => panic!("prepare_pipeline_data was not called"),
+        Err(err) => {
+            assert_eq!(
+                ShellError::NushellFailed {
+                    msg: "TEST can't send binary".into()
+                }
+                .to_string(),
+                err.to_string()
+            );
+        }
+    }
+}
+
+#[test]
+fn write_pipeline_data_list_stream() -> Result<(), ShellError> {
+    let test = TestCase::new();
+    let manager = TestInterfaceManager::new(&test);
+    let interface = manager.get_interface();
+
+    let values = vec![
+        Value::test_int(40),
+        Value::test_bool(false),
+        Value::test_string("this is a test"),
+    ];
+
+    // Set up pipeline data for a list stream
+    let pipe = PipelineData::ListStream(
+        ListStream::from_stream(values.clone().into_iter(), None),
+        None,
+    );
+
+    let (header, writer) = interface.init_write_pipeline_data(pipe, &())?;
+
+    let info = match header {
+        PipelineDataHeader::ListStream(info) => info,
+        _ => panic!("unexpected header: {header:?}"),
+    };
+
+    writer.write()?;
+
+    // Now make sure the stream messages have been written
+    for value in values {
+        match test.next_written().expect("unexpected end of stream") {
+            PluginOutput::Data(id, data) => {
+                assert_eq!(info.id, id, "Data id");
+                match data {
+                    StreamData::List(read_value) => assert_eq!(value, read_value, "Data value"),
+                    _ => panic!("unexpected Data: {data:?}"),
+                }
+            }
+            other => panic!("unexpected output: {other:?}"),
+        }
+    }
+
+    match test.next_written().expect("unexpected end of stream") {
+        PluginOutput::End(id) => {
+            assert_eq!(info.id, id, "End id");
+        }
+        other => panic!("unexpected output: {other:?}"),
+    }
+
+    assert!(!test.has_unconsumed_write());
+
+    Ok(())
+}
+
+#[test]
+fn write_pipeline_data_external_stream() -> Result<(), ShellError> {
+    let test = TestCase::new();
+    let manager = TestInterfaceManager::new(&test);
+    let interface = manager.get_interface();
+
+    let stdout_bufs = vec![
+        b"hello".to_vec(),
+        b"world".to_vec(),
+        b"these are tests".to_vec(),
+    ];
+    let stdout_len = stdout_bufs.iter().map(|b| b.len() as u64).sum::<u64>();
+    let stderr_bufs = vec![b"error messages".to_vec(), b"go here".to_vec()];
+    let exit_code = Value::test_int(7);
+
+    let span = Span::new(400, 500);
+
+    // Set up pipeline data for an external stream
+    let pipe = PipelineData::ExternalStream {
+        stdout: Some(RawStream::new(
+            Box::new(stdout_bufs.clone().into_iter().map(Ok)),
+            None,
+            span,
+            Some(stdout_len),
+        )),
+        stderr: Some(RawStream::new(
+            Box::new(stderr_bufs.clone().into_iter().map(Ok)),
+            None,
+            span,
+            None,
+        )),
+        exit_code: Some(ListStream::from_stream(
+            std::iter::once(exit_code.clone()),
+            None,
+        )),
+        span,
+        metadata: None,
+        trim_end_newline: true,
+    };
+
+    let (header, writer) = interface.init_write_pipeline_data(pipe, &())?;
+
+    let info = match header {
+        PipelineDataHeader::ExternalStream(info) => info,
+        _ => panic!("unexpected header: {header:?}"),
+    };
+
+    writer.write()?;
+
+    let stdout_info = info.stdout.as_ref().expect("stdout info is None");
+    let stderr_info = info.stderr.as_ref().expect("stderr info is None");
+    let exit_code_info = info.exit_code.as_ref().expect("exit code info is None");
+
+    assert_eq!(span, info.span);
+    assert!(info.trim_end_newline);
+
+    assert_eq!(Some(stdout_len), stdout_info.known_size);
+    assert_eq!(None, stderr_info.known_size);
+
+    // Now make sure the stream messages have been written
+    let mut stdout_iter = stdout_bufs.into_iter();
+    let mut stderr_iter = stderr_bufs.into_iter();
+    let mut exit_code_iter = std::iter::once(exit_code);
+
+    let mut stdout_ended = false;
+    let mut stderr_ended = false;
+    let mut exit_code_ended = false;
+
+    // There's no specific order these messages must come in with respect to how the streams are
+    // interleaved, but all of the data for each stream must be in its original order, and the
+    // End must come after all Data
+    for msg in test.written() {
+        match msg {
+            PluginOutput::Data(id, data) => {
+                if id == stdout_info.id {
+                    let result: Result<Vec<u8>, ShellError> =
+                        data.try_into().expect("wrong data in stdout stream");
+                    assert_eq!(
+                        stdout_iter.next().expect("too much data in stdout"),
+                        result.expect("unexpected error in stdout stream")
+                    );
+                } else if id == stderr_info.id {
+                    let result: Result<Vec<u8>, ShellError> =
+                        data.try_into().expect("wrong data in stderr stream");
+                    assert_eq!(
+                        stderr_iter.next().expect("too much data in stderr"),
+                        result.expect("unexpected error in stderr stream")
+                    );
+                } else if id == exit_code_info.id {
+                    let code: Value = data.try_into().expect("wrong data in stderr stream");
+                    assert_eq!(
+                        exit_code_iter.next().expect("too much data in stderr"),
+                        code
+                    );
+                } else {
+                    panic!("unrecognized stream id: {id}");
+                }
+            }
+            PluginOutput::End(id) => {
+                if id == stdout_info.id {
+                    assert!(!stdout_ended, "double End of stdout");
+                    assert!(stdout_iter.next().is_none(), "unexpected end of stdout");
+                    stdout_ended = true;
+                } else if id == stderr_info.id {
+                    assert!(!stderr_ended, "double End of stderr");
+                    assert!(stderr_iter.next().is_none(), "unexpected end of stderr");
+                    stderr_ended = true;
+                } else if id == exit_code_info.id {
+                    assert!(!exit_code_ended, "double End of exit_code");
+                    assert!(
+                        exit_code_iter.next().is_none(),
+                        "unexpected end of exit_code"
+                    );
+                    exit_code_ended = true;
+                } else {
+                    panic!("unrecognized stream id: {id}");
+                }
+            }
+            other => panic!("unexpected output: {other:?}"),
+        }
+    }
+
+    assert!(stdout_ended, "stdout did not End");
+    assert!(stderr_ended, "stderr did not End");
+    assert!(exit_code_ended, "exit_code did not End");
+
+    Ok(())
+}

crates/nu-plugin-core/src/lib.rs

@@ -0,0 +1,24 @@
+//! Functionality and types shared between the plugin and the engine, other than protocol types.
+//!
+//! If you are writing a plugin, you probably don't need this crate. We will make fewer guarantees
+//! for the stability of the interface of this crate than for `nu_plugin`.
+
+pub mod util;
+
+mod communication_mode;
+mod interface;
+mod serializers;
+
+pub use communication_mode::{
+    ClientCommunicationIo, CommunicationMode, PreparedServerCommunication, ServerCommunicationIo,
+};
+pub use interface::{
+    stream::{FromShellError, StreamManager, StreamManagerHandle, StreamReader, StreamWriter},
+    Interface, InterfaceManager, PipelineDataWriter, PluginRead, PluginWrite,
+};
+pub use serializers::{
+    json::JsonSerializer, msgpack::MsgPackSerializer, Encoder, EncodingType, PluginEncoder,
+};
+
+#[doc(hidden)]
+pub use interface::test_util as interface_test_util;

crates/nu-plugin-core/src/serializers/json.rs

@@ -1,10 +1,9 @@
-use crate::{
+use nu_plugin_protocol::{PluginInput, PluginOutput};
-    plugin::{Encoder, PluginEncoder},
-    protocol::{PluginInput, PluginOutput},
-};
 use nu_protocol::ShellError;
 use serde::Deserialize;
 
+use crate::{Encoder, PluginEncoder};
+
 /// A `PluginEncoder` that enables the plugin to communicate with Nushell with JSON
 /// serialized data.
 ///

crates/nu-plugin-core/src/serializers/mod.rs

@@ -0,0 +1,71 @@
+use nu_plugin_protocol::{PluginInput, PluginOutput};
+use nu_protocol::ShellError;
+
+pub mod json;
+pub mod msgpack;
+
+#[cfg(test)]
+mod tests;
+
+/// Encoder for a specific message type. Usually implemented on [`PluginInput`]
+/// and [`PluginOutput`].
+pub trait Encoder<T>: Clone + Send + Sync {
+    /// Serialize a value in the [`PluginEncoder`]s format
+    ///
+    /// Returns [`ShellError::IOError`] if there was a problem writing, or
+    /// [`ShellError::PluginFailedToEncode`] for a serialization error.
+    fn encode(&self, data: &T, writer: &mut impl std::io::Write) -> Result<(), ShellError>;
+
+    /// Deserialize a value from the [`PluginEncoder`]'s format
+    ///
+    /// Returns `None` if there is no more output to receive.
+    ///
+    /// Returns [`ShellError::IOError`] if there was a problem reading, or
+    /// [`ShellError::PluginFailedToDecode`] for a deserialization error.
+    fn decode(&self, reader: &mut impl std::io::BufRead) -> Result<Option<T>, ShellError>;
+}
+
+/// Encoding scheme that defines a plugin's communication protocol with Nu
+pub trait PluginEncoder: Encoder<PluginInput> + Encoder<PluginOutput> {
+    /// The name of the encoder (e.g., `json`)
+    fn name(&self) -> &str;
+}
+
+/// Enum that supports all of the plugin serialization formats.
+#[derive(Clone, Copy, Debug)]
+pub enum EncodingType {
+    Json(json::JsonSerializer),
+    MsgPack(msgpack::MsgPackSerializer),
+}
+
+impl EncodingType {
+    /// Determine the plugin encoding type from the provided byte string (either `b"json"` or
+    /// `b"msgpack"`).
+    pub fn try_from_bytes(bytes: &[u8]) -> Option<Self> {
+        match bytes {
+            b"json" => Some(Self::Json(json::JsonSerializer {})),
+            b"msgpack" => Some(Self::MsgPack(msgpack::MsgPackSerializer {})),
+            _ => None,
+        }
+    }
+}
+
+impl<T> Encoder<T> for EncodingType
+where
+    json::JsonSerializer: Encoder<T>,
+    msgpack::MsgPackSerializer: Encoder<T>,
+{
+    fn encode(&self, data: &T, writer: &mut impl std::io::Write) -> Result<(), ShellError> {
+        match self {
+            EncodingType::Json(encoder) => encoder.encode(data, writer),
+            EncodingType::MsgPack(encoder) => encoder.encode(data, writer),
+        }
+    }
+
+    fn decode(&self, reader: &mut impl std::io::BufRead) -> Result<Option<T>, ShellError> {
+        match self {
+            EncodingType::Json(encoder) => encoder.decode(reader),
+            EncodingType::MsgPack(encoder) => encoder.decode(reader),
+        }
+    }
+}

crates/nu-plugin-core/src/serializers/msgpack.rs

@@ -1,12 +1,11 @@
 use std::io::ErrorKind;
 
-use crate::{
+use nu_plugin_protocol::{PluginInput, PluginOutput};
-    plugin::{Encoder, PluginEncoder},
-    protocol::{PluginInput, PluginOutput},
-};
 use nu_protocol::ShellError;
 use serde::Deserialize;
 
+use crate::{Encoder, PluginEncoder};
+
 /// A `PluginEncoder` that enables the plugin to communicate with Nushell with MsgPack
 /// serialized data.
 ///

crates/nu-plugin-core/src/serializers/tests.rs

@@ -1,6 +1,6 @@
 macro_rules! generate_tests {
     ($encoder:expr) => {
-        use crate::protocol::{
+        use nu_plugin_protocol::{
             CallInfo, CustomValueOp, EvaluatedCall, PipelineDataHeader, PluginCall,
             PluginCallResponse, PluginCustomValue, PluginInput, PluginOption, PluginOutput,
             StreamData,
@@ -178,7 +178,7 @@ macro_rules! generate_tests {
 
             let custom_value_op = PluginCall::CustomValueOp(
                 Spanned {
-                    item: PluginCustomValue::new("Foo".into(), data.clone(), false, None),
+                    item: PluginCustomValue::new("Foo".into(), data.clone(), false),
                     span,
                 },
                 CustomValueOp::ToBaseValue,
@@ -321,12 +321,7 @@ macro_rules! generate_tests {
             let span = Span::new(2, 30);
 
             let value = Value::custom(
-                Box::new(PluginCustomValue::new(
+                Box::new(PluginCustomValue::new(name.into(), data.clone(), true)),
-                    name.into(),
-                    data.clone(),
-                    true,
-                    None,
-                )),
                 span,
             );
 

crates/nu-plugin-core/src/util/mod.rs

@@ -0,0 +1,7 @@
+mod sequence;
+mod waitable;
+mod with_custom_values_in;
+
+pub use sequence::Sequence;
+pub use waitable::*;
+pub use with_custom_values_in::with_custom_values_in;

crates/nu-plugin-core/src/util/sequence.rs

@@ -8,7 +8,7 @@ pub struct Sequence(AtomicUsize);
 impl Sequence {
     /// Return the next available id from a sequence, returning an error on overflow
     #[track_caller]
-    pub(crate) fn next(&self) -> Result<usize, ShellError> {
+    pub fn next(&self) -> Result<usize, ShellError> {
         // It's totally safe to use Relaxed ordering here, as there aren't other memory operations
         // that depend on this value having been set for safety
         //

crates/nu-plugin-core/src/util/with_custom_values_in.rs

@@ -6,7 +6,7 @@ use nu_protocol::{CustomValue, IntoSpanned, ShellError, Spanned, Value};
 /// `LazyRecord`s will be collected to plain values for completeness.
 pub fn with_custom_values_in<E>(
     value: &mut Value,
-    mut f: impl FnMut(Spanned<&mut (dyn CustomValue + '_)>) -> Result<(), E>,
+    mut f: impl FnMut(Spanned<&mut Box<dyn CustomValue>>) -> Result<(), E>,
 ) -> Result<(), E>
 where
     E: From<ShellError>,
@@ -16,7 +16,7 @@ where
         match value {
             Value::Custom { val, .. } => {
                 // Operate on a CustomValue.
-                f(val.as_mut().into_spanned(span))
+                f(val.into_spanned(span))
             }
             // LazyRecord would be a problem for us, since it could return something else the
             // next time, and we have to collect it anyway to serialize it. Collect it in place,
@@ -32,7 +32,7 @@ where
 
 #[test]
 fn find_custom_values() {
-    use crate::protocol::test_util::test_plugin_custom_value;
+    use nu_plugin_protocol::test_util::test_plugin_custom_value;
     use nu_protocol::{engine::Closure, record, LazyRecord, Span};
 
     #[derive(Debug, Clone)]

crates/nu-plugin-engine/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "Functionality for running Nushell plugins from a Nushell engine"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-plugin-engine"
+edition = "2021"
+license = "MIT"
+name = "nu-plugin-engine"
+version = "0.92.3"
+
+[lib]
+bench = false
+
+[dependencies]
+nu-engine = { path = "../nu-engine", version = "0.92.3" }
+nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
+nu-system = { path = "../nu-system", version = "0.92.3" }
+nu-plugin-protocol = { path = "../nu-plugin-protocol", version = "0.92.3" }
+nu-plugin-core = { path = "../nu-plugin-core", version = "0.92.3", default-features = false }
+
+serde = { workspace = true }
+log = { workspace = true }
+
+[dev-dependencies]
+typetag = "0.2"
+
+[features]
+default = ["local-socket"]
+local-socket = ["nu-plugin-core/local-socket"]
+
+[target.'cfg(target_os = "windows")'.dependencies]
+windows = { workspace = true, features = [
+  # For setting process creation flags
+  "Win32_System_Threading",
+] }

crates/nu-plugin-engine/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-plugin-engine/README.md

@@ -0,0 +1,3 @@
+# nu-plugin-engine
+
+This crate provides functionality for the [Nushell](https://nushell.sh/) engine to spawn and interact with plugins.

crates/nu-plugin-engine/src/context.rs

@@ -15,9 +15,6 @@ use std::{
 };
 
 /// Object safe trait for abstracting operations required of the plugin context.
-///
-/// This is not a public API.
-#[doc(hidden)]
 pub trait PluginExecutionContext: Send + Sync {
     /// A span pointing to the command being executed
     fn span(&self) -> Span;
@@ -55,9 +52,6 @@ pub trait PluginExecutionContext: Send + Sync {
 }
 
 /// The execution context of a plugin command. Can be borrowed.
-///
-/// This is not a public API.
-#[doc(hidden)]
 pub struct PluginExecutionCommandContext<'a> {
     identity: Arc<PluginIdentity>,
     engine_state: Cow<'a, EngineState>,

crates/nu-plugin-engine/src/declaration.rs

@@ -1,10 +1,11 @@
-use super::{GetPlugin, PluginExecutionCommandContext, PluginSource};
-use crate::protocol::{CallInfo, EvaluatedCall};
 use nu_engine::{command_prelude::*, get_eval_expression};
+use nu_plugin_protocol::{CallInfo, EvaluatedCall};
 use nu_protocol::{PluginIdentity, PluginSignature};
 use std::sync::Arc;
 
-#[doc(hidden)] // Note: not for plugin authors / only used in nu-parser
+use crate::{GetPlugin, PluginExecutionCommandContext, PluginSource};
+
+/// The command declaration proxy used within the engine for all plugin commands.
 #[derive(Clone)]
 pub struct PluginDeclaration {
     name: String,

crates/nu-plugin-engine/src/init.rs

@@ -0,0 +1,306 @@
+use std::{
+    io::{BufReader, BufWriter},
+    path::Path,
+    process::Child,
+    sync::{Arc, Mutex},
+};
+
+#[cfg(unix)]
+use std::os::unix::process::CommandExt;
+#[cfg(windows)]
+use std::os::windows::process::CommandExt;
+
+use nu_plugin_core::{
+    CommunicationMode, EncodingType, InterfaceManager, PreparedServerCommunication,
+    ServerCommunicationIo,
+};
+use nu_protocol::{
+    engine::StateWorkingSet, report_error_new, PluginIdentity, PluginRegistryFile,
+    PluginRegistryItem, PluginRegistryItemData, RegisteredPlugin, ShellError, Span,
+};
+
+use crate::{
+    PersistentPlugin, PluginDeclaration, PluginGc, PluginInterface, PluginInterfaceManager,
+    PluginSource,
+};
+
+pub(crate) const OUTPUT_BUFFER_SIZE: usize = 8192;
+
+/// Spawn the command for a plugin, in the given `mode`. After spawning, it can be passed to
+/// [`make_plugin_interface()`] to get a [`PluginInterface`].
+pub fn create_command(
+    path: &Path,
+    mut shell: Option<&Path>,
+    mode: &CommunicationMode,
+) -> std::process::Command {
+    log::trace!("Starting plugin: {path:?}, shell = {shell:?}, mode = {mode:?}");
+
+    let mut shell_args = vec![];
+
+    if shell.is_none() {
+        // We only have to do this for things that are not executable by Rust's Command API on
+        // Windows. They do handle bat/cmd files for us, helpfully.
+        //
+        // Also include anything that wouldn't be executable with a shebang, like JAR files.
+        shell = match path.extension().and_then(|e| e.to_str()) {
+            Some("sh") => {
+                if cfg!(unix) {
+                    // We don't want to override what might be in the shebang if this is Unix, since
+                    // some scripts will have a shebang specifying bash even if they're .sh
+                    None
+                } else {
+                    Some(Path::new("sh"))
+                }
+            }
+            Some("nu") => {
+                shell_args.push("--stdin");
+                Some(Path::new("nu"))
+            }
+            Some("py") => Some(Path::new("python")),
+            Some("rb") => Some(Path::new("ruby")),
+            Some("jar") => {
+                shell_args.push("-jar");
+                Some(Path::new("java"))
+            }
+            _ => None,
+        };
+    }
+
+    let mut process = if let Some(shell) = shell {
+        let mut process = std::process::Command::new(shell);
+        process.args(shell_args);
+        process.arg(path);
+
+        process
+    } else {
+        std::process::Command::new(path)
+    };
+
+    process.args(mode.args());
+
+    // Setup I/O according to the communication mode
+    mode.setup_command_io(&mut process);
+
+    // The plugin should be run in a new process group to prevent Ctrl-C from stopping it
+    #[cfg(unix)]
+    process.process_group(0);
+    #[cfg(windows)]
+    process.creation_flags(windows::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP.0);
+
+    // In order to make bugs with improper use of filesystem without getting the engine current
+    // directory more obvious, the plugin always starts in the directory of its executable
+    if let Some(dirname) = path.parent() {
+        process.current_dir(dirname);
+    }
+
+    process
+}
+
+/// Create a plugin interface from a spawned child process.
+///
+/// `comm` determines the communication type the process was spawned with, and whether stdio will
+/// be taken from the child.
+pub fn make_plugin_interface(
+    mut child: Child,
+    comm: PreparedServerCommunication,
+    source: Arc<PluginSource>,
+    pid: Option<u32>,
+    gc: Option<PluginGc>,
+) -> Result<PluginInterface, ShellError> {
+    match comm.connect(&mut child)? {
+        ServerCommunicationIo::Stdio(stdin, stdout) => make_plugin_interface_with_streams(
+            stdout,
+            stdin,
+            move || {
+                let _ = child.wait();
+            },
+            source,
+            pid,
+            gc,
+        ),
+        #[cfg(feature = "local-socket")]
+        ServerCommunicationIo::LocalSocket { read_out, write_in } => {
+            make_plugin_interface_with_streams(
+                read_out,
+                write_in,
+                move || {
+                    let _ = child.wait();
+                },
+                source,
+                pid,
+                gc,
+            )
+        }
+    }
+}
+
+/// Create a plugin interface from low-level components.
+///
+/// - `after_close` is called to clean up after the `reader` ends.
+/// - `source` is required so that custom values produced by the plugin can spawn it.
+/// - `pid` may be provided for process management (e.g. `EnterForeground`).
+/// - `gc` may be provided for communication with the plugin's GC (e.g. `SetGcDisabled`).
+pub fn make_plugin_interface_with_streams(
+    mut reader: impl std::io::Read + Send + 'static,
+    writer: impl std::io::Write + Send + 'static,
+    after_close: impl FnOnce() + Send + 'static,
+    source: Arc<PluginSource>,
+    pid: Option<u32>,
+    gc: Option<PluginGc>,
+) -> Result<PluginInterface, ShellError> {
+    let encoder = get_plugin_encoding(&mut reader)?;
+
+    let reader = BufReader::with_capacity(OUTPUT_BUFFER_SIZE, reader);
+    let writer = BufWriter::with_capacity(OUTPUT_BUFFER_SIZE, writer);
+
+    let mut manager =
+        PluginInterfaceManager::new(source.clone(), pid, (Mutex::new(writer), encoder));
+    manager.set_garbage_collector(gc);
+
+    let interface = manager.get_interface();
+    interface.hello()?;
+
+    // Spawn the reader on a new thread. We need to be able to read messages at the same time that
+    // we write, because we are expected to be able to handle multiple messages coming in from the
+    // plugin at any time, including stream messages like `Drop`.
+    std::thread::Builder::new()
+        .name(format!(
+            "plugin interface reader ({})",
+            source.identity.name()
+        ))
+        .spawn(move || {
+            if let Err(err) = manager.consume_all((reader, encoder)) {
+                log::warn!("Error in PluginInterfaceManager: {err}");
+            }
+            // If the loop has ended, drop the manager so everyone disconnects and then run
+            // after_close
+            drop(manager);
+            after_close();
+        })
+        .map_err(|err| ShellError::PluginFailedToLoad {
+            msg: format!("Failed to spawn thread for plugin: {err}"),
+        })?;
+
+    Ok(interface)
+}
+
+/// Determine the plugin's encoding from a freshly opened stream.
+///
+/// The plugin is expected to send a 1-byte length and either `json` or `msgpack`, so this reads
+/// that and determines the right length.
+pub fn get_plugin_encoding(
+    child_stdout: &mut impl std::io::Read,
+) -> Result<EncodingType, ShellError> {
+    let mut length_buf = [0u8; 1];
+    child_stdout
+        .read_exact(&mut length_buf)
+        .map_err(|e| ShellError::PluginFailedToLoad {
+            msg: format!("unable to get encoding from plugin: {e}"),
+        })?;
+
+    let mut buf = vec![0u8; length_buf[0] as usize];
+    child_stdout
+        .read_exact(&mut buf)
+        .map_err(|e| ShellError::PluginFailedToLoad {
+            msg: format!("unable to get encoding from plugin: {e}"),
+        })?;
+
+    EncodingType::try_from_bytes(&buf).ok_or_else(|| {
+        let encoding_for_debug = String::from_utf8_lossy(&buf);
+        ShellError::PluginFailedToLoad {
+            msg: format!("get unsupported plugin encoding: {encoding_for_debug}"),
+        }
+    })
+}
+
+/// Load the definitions from the plugin file into the engine state
+pub fn load_plugin_file(
+    working_set: &mut StateWorkingSet,
+    plugin_registry_file: &PluginRegistryFile,
+    span: Option<Span>,
+) {
+    for plugin in &plugin_registry_file.plugins {
+        // Any errors encountered should just be logged.
+        if let Err(err) = load_plugin_registry_item(working_set, plugin, span) {
+            report_error_new(working_set.permanent_state, &err)
+        }
+    }
+}
+
+/// Load a definition from the plugin file into the engine state
+pub fn load_plugin_registry_item(
+    working_set: &mut StateWorkingSet,
+    plugin: &PluginRegistryItem,
+    span: Option<Span>,
+) -> Result<Arc<PersistentPlugin>, ShellError> {
+    let identity =
+        PluginIdentity::new(plugin.filename.clone(), plugin.shell.clone()).map_err(|_| {
+            ShellError::GenericError {
+                error: "Invalid plugin filename in plugin registry file".into(),
+                msg: "loaded from here".into(),
+                span,
+                help: Some(format!(
+                    "the filename for `{}` is not a valid nushell plugin: {}",
+                    plugin.name,
+                    plugin.filename.display()
+                )),
+                inner: vec![],
+            }
+        })?;
+
+    match &plugin.data {
+        PluginRegistryItemData::Valid { commands } => {
+            let plugin = add_plugin_to_working_set(working_set, &identity)?;
+
+            // Ensure that the plugin is reset. We're going to load new signatures, so we want to
+            // make sure the running plugin reflects those new signatures, and it's possible that it
+            // doesn't.
+            plugin.reset()?;
+
+            // Create the declarations from the commands
+            for signature in commands {
+                let decl = PluginDeclaration::new(plugin.clone(), signature.clone());
+                working_set.add_decl(Box::new(decl));
+            }
+            Ok(plugin)
+        }
+        PluginRegistryItemData::Invalid => Err(ShellError::PluginRegistryDataInvalid {
+            plugin_name: identity.name().to_owned(),
+            span,
+            add_command: identity.add_command(),
+        }),
+    }
+}
+
+/// Find [`PersistentPlugin`] with the given `identity` in the `working_set`, or construct it
+/// if it doesn't exist.
+///
+/// The garbage collection config is always found and set in either case.
+pub fn add_plugin_to_working_set(
+    working_set: &mut StateWorkingSet,
+    identity: &PluginIdentity,
+) -> Result<Arc<PersistentPlugin>, ShellError> {
+    // Find garbage collection config for the plugin
+    let gc_config = working_set
+        .get_config()
+        .plugin_gc
+        .get(identity.name())
+        .clone();
+
+    // Add it to / get it from the working set
+    let plugin = working_set.find_or_create_plugin(identity, || {
+        Arc::new(PersistentPlugin::new(identity.clone(), gc_config.clone()))
+    });
+
+    plugin.set_gc_config(&gc_config);
+
+    // Downcast the plugin to `PersistentPlugin` - we generally expect this to succeed.
+    // The trait object only exists so that nu-protocol can contain plugins without knowing
+    // anything about their implementation, but we only use `PersistentPlugin` in practice.
+    plugin
+        .as_any()
+        .downcast()
+        .map_err(|_| ShellError::NushellFailed {
+            msg: "encountered unexpected RegisteredPlugin type".into(),
+        })
+}

crates/nu-plugin-engine/src/interface/mod.rs

@@ -1,18 +1,14 @@
 //! Interface used by the engine to communicate with the plugin.
 
-use super::{
+use nu_plugin_core::{
-    stream::{StreamManager, StreamManagerHandle},
+    util::{with_custom_values_in, Sequence, Waitable, WaitableMut},
-    Interface, InterfaceManager, PipelineDataWriter, PluginRead, PluginWrite,
+    Interface, InterfaceManager, PipelineDataWriter, PluginRead, PluginWrite, StreamManager,
+    StreamManagerHandle,
 };
-use crate::{
+use nu_plugin_protocol::{
-    plugin::{context::PluginExecutionContext, gc::PluginGc, process::PluginProcess, PluginSource},
+    CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, EvaluatedCall, Ordering,
-    protocol::{
-        CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, Ordering,
     PluginCall, PluginCallId, PluginCallResponse, PluginCustomValue, PluginInput, PluginOption,
     PluginOutput, ProtocolInfo, StreamId, StreamMessage,
-    },
-    sequence::Sequence,
-    util::{with_custom_values_in, Waitable, WaitableMut},
 };
 use nu_protocol::{
     ast::Operator, CustomValue, IntoInterruptiblePipelineData, IntoSpanned, ListStream,
@@ -23,6 +19,11 @@ use std::{
     sync::{atomic::AtomicBool, mpsc, Arc, OnceLock},
 };
 
+use crate::{
+    process::PluginProcess, PluginCustomValueWithSource, PluginExecutionContext, PluginGc,
+    PluginSource,
+};
+
 #[cfg(test)]
 mod tests;
 
@@ -113,8 +114,8 @@ struct PluginCallState {
     /// them in memory so they can be dropped at the end of the call. We hold the sender as well so
     /// we can generate the CurrentCallState.
     keep_plugin_custom_values: (
-        mpsc::Sender<PluginCustomValue>,
+        mpsc::Sender<PluginCustomValueWithSource>,
-        mpsc::Receiver<PluginCustomValue>,
+        mpsc::Receiver<PluginCustomValueWithSource>,
     ),
     /// Number of streams that still need to be read from the plugin call response
     remaining_streams_to_read: i32,
@@ -131,10 +132,7 @@ impl Drop for PluginCallState {
 }
 
 /// Manages reading and dispatching messages for [`PluginInterface`]s.
-///
-/// This is not a public API.
 #[derive(Debug)]
-#[doc(hidden)]
 pub struct PluginInterfaceManager {
     /// Shared state
     state: Arc<PluginInterfaceState>,
@@ -557,7 +555,10 @@ impl InterfaceManager for PluginInterfaceManager {
                             } => {
                                 for arg in positional.iter_mut() {
                                     // Add source to any plugin custom values in the arguments
-                                    PluginCustomValue::add_source_in(arg, &self.state.source)?;
+                                    PluginCustomValueWithSource::add_source_in(
+                                        arg,
+                                        &self.state.source,
+                                    )?;
                                 }
                                 Ok(engine_call)
                             }
@@ -586,7 +587,7 @@ impl InterfaceManager for PluginInterfaceManager {
         match data {
             PipelineData::Value(ref mut value, _) => {
                 with_custom_values_in(value, |custom_value| {
-                    PluginCustomValue::add_source(custom_value.item, &self.state.source);
+                    PluginCustomValueWithSource::add_source(custom_value.item, &self.state.source);
                     Ok::<_, ShellError>(())
                 })?;
                 Ok(data)
@@ -595,7 +596,7 @@ impl InterfaceManager for PluginInterfaceManager {
                 let source = self.state.source.clone();
                 Ok(stream
                     .map(move |mut value| {
-                        let _ = PluginCustomValue::add_source_in(&mut value, &source);
+                        let _ = PluginCustomValueWithSource::add_source_in(&mut value, &source);
                         value
                     })
                     .into_pipeline_data_with_metadata(meta, ctrlc))
@@ -718,12 +719,7 @@ impl PluginInterface {
             PluginCall::CustomValueOp(value, op) => {
                 (PluginCall::CustomValueOp(value, op), Default::default())
             }
-            PluginCall::Run(CallInfo {
+            PluginCall::Run(CallInfo { name, call, input }) => {
-                name,
-                mut call,
-                input,
-            }) => {
-                state.prepare_call_args(&mut call, &self.state.source)?;
                 let (header, writer) = self.init_write_pipeline_data(input, &state)?;
                 (
                     PluginCall::Run(CallInfo {
@@ -945,12 +941,16 @@ impl PluginInterface {
     /// Do a custom value op that expects a value response (i.e. most of them)
     fn custom_value_op_expecting_value(
         &self,
-        value: Spanned<PluginCustomValue>,
+        value: Spanned<PluginCustomValueWithSource>,
         op: CustomValueOp,
     ) -> Result<Value, ShellError> {
         let op_name = op.name();
         let span = value.span;
-        let call = PluginCall::CustomValueOp(value, op);
+
+        // Check that the value came from the right source
+        value.item.verify_source(span, &self.state.source)?;
+
+        let call = PluginCall::CustomValueOp(value.map(|cv| cv.without_source()), op);
         match self.plugin_call(call, None)? {
             PluginCallResponse::PipelineData(out_data) => Ok(out_data.into_value(span)),
             PluginCallResponse::Error(err) => Err(err.into()),
@@ -963,7 +963,7 @@ impl PluginInterface {
     /// Collapse a custom value to its base value.
     pub fn custom_value_to_base_value(
         &self,
-        value: Spanned<PluginCustomValue>,
+        value: Spanned<PluginCustomValueWithSource>,
     ) -> Result<Value, ShellError> {
         self.custom_value_op_expecting_value(value, CustomValueOp::ToBaseValue)
     }
@@ -971,7 +971,7 @@ impl PluginInterface {
     /// Follow a numbered cell path on a custom value - e.g. `value.0`.
     pub fn custom_value_follow_path_int(
         &self,
-        value: Spanned<PluginCustomValue>,
+        value: Spanned<PluginCustomValueWithSource>,
         index: Spanned<usize>,
     ) -> Result<Value, ShellError> {
         self.custom_value_op_expecting_value(value, CustomValueOp::FollowPathInt(index))
@@ -980,7 +980,7 @@ impl PluginInterface {
     /// Follow a named cell path on a custom value - e.g. `value.column`.
     pub fn custom_value_follow_path_string(
         &self,
-        value: Spanned<PluginCustomValue>,
+        value: Spanned<PluginCustomValueWithSource>,
         column_name: Spanned<String>,
     ) -> Result<Value, ShellError> {
         self.custom_value_op_expecting_value(value, CustomValueOp::FollowPathString(column_name))
@@ -989,13 +989,16 @@ impl PluginInterface {
     /// Invoke comparison logic for custom values.
     pub fn custom_value_partial_cmp(
         &self,
-        value: PluginCustomValue,
+        value: PluginCustomValueWithSource,
         other_value: Value,
     ) -> Result<Option<Ordering>, ShellError> {
+        // Check that the value came from the right source
+        value.verify_source(Span::unknown(), &self.state.source)?;
+
         // Note: the protocol is always designed to have a span with the custom value, but this
         // operation doesn't support one.
         let call = PluginCall::CustomValueOp(
-            value.into_spanned(Span::unknown()),
+            value.without_source().into_spanned(Span::unknown()),
             CustomValueOp::PartialCmp(other_value),
         );
         match self.plugin_call(call, None)? {
@@ -1010,7 +1013,7 @@ impl PluginInterface {
     /// Invoke functionality for an operator on a custom value.
     pub fn custom_value_operation(
         &self,
-        left: Spanned<PluginCustomValue>,
+        left: Spanned<PluginCustomValueWithSource>,
         operator: Spanned<Operator>,
         right: Value,
     ) -> Result<Value, ShellError> {
@@ -1118,9 +1121,6 @@ struct WritePluginCallResult {
 }
 
 /// State related to the current plugin call being executed.
-///
-/// This is not a public API.
-#[doc(hidden)]
 #[derive(Default, Clone)]
 pub struct CurrentCallState {
     /// Sender for context, which should be sent if the plugin call returned a stream so that
@@ -1128,7 +1128,7 @@ pub struct CurrentCallState {
     context_tx: Option<mpsc::Sender<Context>>,
     /// Sender for a channel that retains plugin custom values that need to stay alive for the
     /// duration of a plugin call.
-    keep_plugin_custom_values_tx: Option<mpsc::Sender<PluginCustomValue>>,
+    keep_plugin_custom_values_tx: Option<mpsc::Sender<PluginCustomValueWithSource>>,
     /// The plugin call entered the foreground: this should be cleaned up automatically when the
     /// plugin call returns.
     entered_foreground: bool,
@@ -1141,18 +1141,21 @@ impl CurrentCallState {
     /// shouldn't be dropped immediately.
     fn prepare_custom_value(
         &self,
-        custom_value: Spanned<&mut (dyn CustomValue + '_)>,
+        custom_value: Spanned<&mut Box<dyn CustomValue>>,
         source: &PluginSource,
     ) -> Result<(), ShellError> {
         // Ensure we can use it
-        PluginCustomValue::verify_source(custom_value.as_deref(), source)?;
+        PluginCustomValueWithSource::verify_source_of_custom_value(
+            custom_value.as_deref().map(|cv| &**cv),
+            source,
+        )?;
 
         // Check whether we need to keep it
         if let Some(keep_tx) = &self.keep_plugin_custom_values_tx {
             if let Some(custom_value) = custom_value
                 .item
                 .as_any()
-                .downcast_ref::<PluginCustomValue>()
+                .downcast_ref::<PluginCustomValueWithSource>()
             {
                 if custom_value.notify_on_drop() {
                     log::trace!("Keeping custom value for drop later: {:?}", custom_value);
@@ -1164,6 +1167,10 @@ impl CurrentCallState {
                 }
             }
         }
+
+        // Strip the source from it so it can be serialized
+        PluginCustomValueWithSource::remove_source(&mut *custom_value.item);
+
         Ok(())
     }
 
@@ -1177,7 +1184,7 @@ impl CurrentCallState {
     /// Prepare call arguments for write.
     fn prepare_call_args(
         &self,
-        call: &mut crate::EvaluatedCall,
+        call: &mut EvaluatedCall,
         source: &PluginSource,
     ) -> Result<(), ShellError> {
         for arg in call.positional.iter_mut() {
@@ -1199,11 +1206,7 @@ impl CurrentCallState {
         match call {
             PluginCall::Signature => Ok(()),
             PluginCall::Run(CallInfo { call, .. }) => self.prepare_call_args(call, source),
-            PluginCall::CustomValueOp(custom_value, op) => {
+            PluginCall::CustomValueOp(_, op) => {
-                // `source` isn't present on Dropped.
-                if !matches!(op, CustomValueOp::Dropped) {
-                    self.prepare_custom_value(custom_value.as_mut().map(|r| r as &mut _), source)?;
-                }
                 // Handle anything within the op.
                 match op {
                     CustomValueOp::ToBaseValue => Ok(()),

crates/nu-plugin-engine/src/interface/tests.rs

@@ -2,21 +2,17 @@ use super::{
     Context, PluginCallState, PluginInterface, PluginInterfaceManager, ReceivedPluginCallMessage,
 };
 use crate::{
-    plugin::{
+    context::PluginExecutionBogusContext, interface::CurrentCallState,
-        context::PluginExecutionBogusContext,
+    plugin_custom_value_with_source::WithSource, test_util::*, PluginCustomValueWithSource,
-        interface::{plugin::CurrentCallState, test_util::TestCase, Interface, InterfaceManager},
     PluginSource,
-    },
+};
-    protocol::{
+use nu_plugin_core::{interface_test_util::TestCase, Interface, InterfaceManager};
-        test_util::{
+use nu_plugin_protocol::{
-            expected_test_custom_value, test_plugin_custom_value,
+    test_util::{expected_test_custom_value, test_plugin_custom_value},
-            test_plugin_custom_value_with_source,
+    CallInfo, CustomValueOp, EngineCall, EngineCallResponse, EvaluatedCall, ExternalStreamInfo,
-        },
+    ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallId, PluginCallResponse,
-        CallInfo, CustomValueOp, EngineCall, EngineCallResponse, ExternalStreamInfo,
+    PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, RawStreamInfo,
-        ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallId, PluginCustomValue,
+    StreamData, StreamMessage,
-        PluginInput, Protocol, ProtocolInfo, RawStreamInfo, StreamData, StreamMessage,
-    },
-    EvaluatedCall, PluginCallResponse, PluginOutput,
 };
 use nu_protocol::{
     ast::{Math, Operator},
@@ -666,17 +662,13 @@ fn manager_prepare_pipeline_data_adds_source_to_values() -> Result<(), ShellErro
         .into_iter()
         .next()
         .expect("prepared pipeline data is empty");
-    let custom_value: &PluginCustomValue = value
+    let custom_value: &PluginCustomValueWithSource = value
         .as_custom_value()?
         .as_any()
         .downcast_ref()
-        .expect("custom value is not a PluginCustomValue");
+        .expect("{value:?} is not a PluginCustomValueWithSource");
 
-    if let Some(source) = custom_value.source() {
+    assert_eq!("test", custom_value.source().name());
-        assert_eq!("test", source.name());
-    } else {
-        panic!("source was not set");
-    }
 
     Ok(())
 }
@@ -696,17 +688,13 @@ fn manager_prepare_pipeline_data_adds_source_to_list_streams() -> Result<(), She
         .into_iter()
         .next()
         .expect("prepared pipeline data is empty");
-    let custom_value: &PluginCustomValue = value
+    let custom_value: &PluginCustomValueWithSource = value
         .as_custom_value()?
         .as_any()
         .downcast_ref()
-        .expect("custom value is not a PluginCustomValue");
+        .expect("{value:?} is not a PluginCustomValueWithSource");
 
-    if let Some(source) = custom_value.source() {
+    assert_eq!("test", custom_value.source().name());
-        assert_eq!("test", source.name());
-    } else {
-        panic!("source was not set");
-    }
 
     Ok(())
 }
@@ -792,7 +780,7 @@ fn interface_write_plugin_call_writes_custom_value_op() -> Result<(), ShellError
     let result = interface.write_plugin_call(
         PluginCall::CustomValueOp(
             Spanned {
-                item: test_plugin_custom_value_with_source(),
+                item: test_plugin_custom_value(),
                 span: Span::test_data(),
             },
             CustomValueOp::ToBaseValue,
@@ -1113,13 +1101,12 @@ fn interface_custom_value_to_base_value() -> Result<(), ShellError> {
 fn normal_values(interface: &PluginInterface) -> Vec<Value> {
     vec![
         Value::test_int(5),
-        Value::test_custom_value(Box::new(PluginCustomValue::new(
+        Value::test_custom_value(Box::new(
-            "SomeTest".into(),
+            PluginCustomValue::new("SomeTest".into(), vec![1, 2, 3], false).with_source(
-            vec![1, 2, 3],
-            false,
                 // Has the same source, so it should be accepted
-            Some(interface.state.source.clone()),
+                interface.state.source.clone(),
-        ))),
+            ),
+        )),
     ]
 }
 
@@ -1173,15 +1160,12 @@ fn bad_custom_values() -> Vec<Value> {
             "SomeTest".into(),
             vec![1, 2, 3],
             false,
-            None,
         ))),
         // Has a different source, so it should be rejected
-        Value::test_custom_value(Box::new(PluginCustomValue::new(
+        Value::test_custom_value(Box::new(
-            "SomeTest".into(),
+            PluginCustomValue::new("SomeTest".into(), vec![1, 2, 3], false)
-            vec![1, 2, 3],
+                .with_source(PluginSource::new_fake("pluto").into()),
-            false,
+        )),
-            Some(PluginSource::new_fake("pluto").into()),
-        ))),
     ]
 }
 
@@ -1227,7 +1211,7 @@ fn prepare_custom_value_verifies_source() {
     let span = Span::test_data();
     let source = Arc::new(PluginSource::new_fake("test"));
 
-    let mut val = test_plugin_custom_value();
+    let mut val: Box<dyn CustomValue> = Box::new(test_plugin_custom_value());
     assert!(CurrentCallState::default()
         .prepare_custom_value(
             Spanned {
@@ -1238,7 +1222,8 @@ fn prepare_custom_value_verifies_source() {
         )
         .is_err());
 
-    let mut val = test_plugin_custom_value().with_source(Some(source.clone()));
+    let mut val: Box<dyn CustomValue> =
+        Box::new(test_plugin_custom_value().with_source(source.clone()));
     assert!(CurrentCallState::default()
         .prepare_custom_value(
             Spanned {
@@ -1289,8 +1274,10 @@ fn prepare_custom_value_sends_to_keep_channel_if_drop_notify() -> Result<(), She
         ..Default::default()
     };
     // Try with a custom val that has drop check set
-    let mut drop_val = PluginCustomValue::serialize_from_custom_value(&DropCustomVal, span)?
+    let mut drop_val: Box<dyn CustomValue> = Box::new(
-        .with_source(Some(source.clone()));
+        PluginCustomValue::serialize_from_custom_value(&DropCustomVal, span)?
+            .with_source(source.clone()),
+    );
     state.prepare_custom_value(
         Spanned {
             item: &mut drop_val,
@@ -1301,7 +1288,8 @@ fn prepare_custom_value_sends_to_keep_channel_if_drop_notify() -> Result<(), She
     // Check that the custom value was actually sent
     assert!(rx.try_recv().is_ok());
     // Now try with one that doesn't have it
-    let mut not_drop_val = test_plugin_custom_value().with_source(Some(source.clone()));
+    let mut not_drop_val: Box<dyn CustomValue> =
+        Box::new(test_plugin_custom_value().with_source(source.clone()));
     state.prepare_custom_value(
         Spanned {
             item: &mut not_drop_val,
@@ -1321,10 +1309,10 @@ fn prepare_plugin_call_run() {
     let source = Arc::new(PluginSource::new_fake("test"));
     let other_source = Arc::new(PluginSource::new_fake("other"));
     let cv_ok = test_plugin_custom_value()
-        .with_source(Some(source.clone()))
+        .with_source(source.clone())
         .into_value(span);
     let cv_bad = test_plugin_custom_value()
-        .with_source(Some(other_source))
+        .with_source(other_source)
         .into_value(span);
 
     let fixtures = [
@@ -1414,9 +1402,9 @@ fn prepare_plugin_call_custom_value_op() {
     let span = Span::test_data();
     let source = Arc::new(PluginSource::new_fake("test"));
     let other_source = Arc::new(PluginSource::new_fake("other"));
-    let cv_ok = test_plugin_custom_value().with_source(Some(source.clone()));
+    let cv_ok = test_plugin_custom_value().with_source(source.clone());
     let cv_ok_val = cv_ok.clone_value(span);
-    let cv_bad = test_plugin_custom_value().with_source(Some(other_source));
+    let cv_bad = test_plugin_custom_value().with_source(other_source);
     let cv_bad_val = cv_bad.clone_value(span);
 
     let fixtures = [
@@ -1424,17 +1412,7 @@ fn prepare_plugin_call_custom_value_op() {
             true, // should succeed
             PluginCall::CustomValueOp::<PipelineData>(
                 Spanned {
-                    item: cv_ok.clone(),
+                    item: cv_ok.clone().without_source(),
-                    span,
-                },
-                CustomValueOp::ToBaseValue,
-            ),
-        ),
-        (
-            false, // should fail
-            PluginCall::CustomValueOp(
-                Spanned {
-                    item: cv_bad.clone(),
                     span,
                 },
                 CustomValueOp::ToBaseValue,
@@ -1455,7 +1433,7 @@ fn prepare_plugin_call_custom_value_op() {
             true, // should succeed
             PluginCall::CustomValueOp::<PipelineData>(
                 Spanned {
-                    item: cv_ok.clone(),
+                    item: cv_ok.clone().without_source(),
                     span,
                 },
                 CustomValueOp::PartialCmp(cv_ok_val.clone()),
@@ -1465,7 +1443,7 @@ fn prepare_plugin_call_custom_value_op() {
             false, // should fail
             PluginCall::CustomValueOp(
                 Spanned {
-                    item: cv_ok.clone(),
+                    item: cv_ok.clone().without_source(),
                     span,
                 },
                 CustomValueOp::PartialCmp(cv_bad_val.clone()),
@@ -1475,7 +1453,7 @@ fn prepare_plugin_call_custom_value_op() {
             true, // should succeed
             PluginCall::CustomValueOp::<PipelineData>(
                 Spanned {
-                    item: cv_ok.clone(),
+                    item: cv_ok.clone().without_source(),
                     span,
                 },
                 CustomValueOp::Operation(
@@ -1488,7 +1466,7 @@ fn prepare_plugin_call_custom_value_op() {
             false, // should fail
             PluginCall::CustomValueOp(
                 Spanned {
-                    item: cv_ok.clone(),
+                    item: cv_ok.clone().without_source(),
                     span,
                 },
                 CustomValueOp::Operation(

crates/nu-plugin-engine/src/lib.rs

@@ -0,0 +1,24 @@
+//! Provides functionality for running Nushell plugins from a Nushell engine.
+
+mod context;
+mod declaration;
+mod gc;
+mod init;
+mod interface;
+mod persistent;
+mod plugin_custom_value_with_source;
+mod process;
+mod source;
+mod util;
+
+#[cfg(test)]
+mod test_util;
+
+pub use context::{PluginExecutionCommandContext, PluginExecutionContext};
+pub use declaration::PluginDeclaration;
+pub use gc::PluginGc;
+pub use init::*;
+pub use interface::{PluginInterface, PluginInterfaceManager};
+pub use persistent::{GetPlugin, PersistentPlugin};
+pub use plugin_custom_value_with_source::{PluginCustomValueWithSource, WithSource};
+pub use source::PluginSource;

crates/nu-plugin-engine/src/persistent.rs

@@ -1,7 +1,10 @@
-use super::{
+use crate::{
-    communication_mode::CommunicationMode, create_command, gc::PluginGc, make_plugin_interface,
+    init::{create_command, make_plugin_interface},
-    PluginInterface, PluginSource,
+    PluginGc,
 };
+
+use super::{PluginInterface, PluginSource};
+use nu_plugin_core::CommunicationMode;
 use nu_protocol::{
     engine::{EngineState, Stack},
     PluginGcConfig, PluginIdentity, RegisteredPlugin, ShellError,
@@ -14,9 +17,6 @@ use std::{
 /// A box that can keep a plugin that was spawned persistent for further uses. The plugin may or
 /// may not be currently running. [`.get()`] gets the currently running plugin, or spawns it if it's
 /// not running.
-///
-/// Note: used in the parser, not for plugin authors
-#[doc(hidden)]
 #[derive(Debug)]
 pub struct PersistentPlugin {
     /// Identity (filename, shell, name) of the plugin
@@ -69,7 +69,7 @@ impl PersistentPlugin {
     ///
     /// Will call `envs` to get environment variables to spawn the plugin if the plugin needs to be
     /// spawned.
-    pub(crate) fn get(
+    pub fn get(
         self: Arc<Self>,
         envs: impl FnOnce() -> Result<HashMap<String, String>, ShellError>,
     ) -> Result<PluginInterface, ShellError> {
@@ -194,7 +194,7 @@ impl PersistentPlugin {
         if mutable.preferred_mode.is_none()
             && interface
                 .protocol_info()?
-                .supports_feature(&crate::protocol::Feature::LocalSocket)
+                .supports_feature(&nu_plugin_protocol::Feature::LocalSocket)
         {
             log::trace!(
                 "{}: Attempting to upgrade to local socket mode",
@@ -289,9 +289,6 @@ impl RegisteredPlugin for PersistentPlugin {
 }
 
 /// Anything that can produce a plugin interface.
-///
-/// This is not a public interface.
-#[doc(hidden)]
 pub trait GetPlugin: RegisteredPlugin {
     /// Retrieve or spawn a [`PluginInterface`]. The `context` may be used for determining
     /// environment variables to launch the plugin with.

crates/nu-plugin-engine/src/plugin_custom_value_with_source/mod.rs

@@ -0,0 +1,274 @@
+use std::{cmp::Ordering, sync::Arc};
+
+use nu_plugin_core::util::with_custom_values_in;
+use nu_plugin_protocol::PluginCustomValue;
+use nu_protocol::{ast::Operator, CustomValue, IntoSpanned, ShellError, Span, Spanned, Value};
+use serde::Serialize;
+
+use crate::{PluginInterface, PluginSource};
+
+#[cfg(test)]
+mod tests;
+
+/// Wraps a [`PluginCustomValue`] together with its [`PluginSource`], so that the [`CustomValue`]
+/// methods can be implemented by calling the plugin, and to ensure that any custom values sent to a
+/// plugin came from it originally.
+#[derive(Debug, Clone)]
+pub struct PluginCustomValueWithSource {
+    inner: PluginCustomValue,
+
+    /// Which plugin the custom value came from. This is not sent over the serialization boundary.
+    source: Arc<PluginSource>,
+}
+
+impl PluginCustomValueWithSource {
+    /// Wrap a [`PluginCustomValue`] together with its source.
+    pub fn new(inner: PluginCustomValue, source: Arc<PluginSource>) -> PluginCustomValueWithSource {
+        PluginCustomValueWithSource { inner, source }
+    }
+
+    /// Create a [`Value`] containing this custom value.
+    pub fn into_value(self, span: Span) -> Value {
+        Value::custom(Box::new(self), span)
+    }
+
+    /// Which plugin the custom value came from. This provides a direct reference to be able to get
+    /// a plugin interface in order to make a call, when needed.
+    pub fn source(&self) -> &Arc<PluginSource> {
+        &self.source
+    }
+
+    /// Unwrap the [`PluginCustomValueWithSource`], discarding the source.
+    pub fn without_source(self) -> PluginCustomValue {
+        // Because of the `Drop` implementation, we can't destructure this.
+        self.inner.clone()
+    }
+
+    /// Helper to get the plugin to implement an op
+    fn get_plugin(&self, span: Option<Span>, for_op: &str) -> Result<PluginInterface, ShellError> {
+        let wrap_err = |err: ShellError| ShellError::GenericError {
+            error: format!(
+                "Unable to spawn plugin `{}` to {for_op}",
+                self.source.name()
+            ),
+            msg: err.to_string(),
+            span,
+            help: None,
+            inner: vec![err],
+        };
+
+        self.source
+            .clone()
+            .persistent(span)
+            .and_then(|p| p.get_plugin(None))
+            .map_err(wrap_err)
+    }
+
+    /// Add a [`PluginSource`] to the given [`CustomValue`] if it is a [`PluginCustomValue`].
+    pub fn add_source(value: &mut Box<dyn CustomValue>, source: &Arc<PluginSource>) {
+        if let Some(custom_value) = value.as_any().downcast_ref::<PluginCustomValue>() {
+            *value = Box::new(custom_value.clone().with_source(source.clone()));
+        }
+    }
+
+    /// Add a [`PluginSource`] to all [`PluginCustomValue`]s within the value, recursively.
+    pub fn add_source_in(value: &mut Value, source: &Arc<PluginSource>) -> Result<(), ShellError> {
+        with_custom_values_in(value, |custom_value| {
+            Self::add_source(custom_value.item, source);
+            Ok::<_, ShellError>(())
+        })
+    }
+
+    /// Remove a [`PluginSource`] from the given [`CustomValue`] if it is a
+    /// [`PluginCustomValueWithSource`]. This will turn it back into a [`PluginCustomValue`].
+    pub fn remove_source(value: &mut Box<dyn CustomValue>) {
+        if let Some(custom_value) = value.as_any().downcast_ref::<PluginCustomValueWithSource>() {
+            *value = Box::new(custom_value.clone().without_source());
+        }
+    }
+
+    /// Remove the [`PluginSource`] from all [`PluginCustomValue`]s within the value, recursively.
+    pub fn remove_source_in(value: &mut Value) -> Result<(), ShellError> {
+        with_custom_values_in(value, |custom_value| {
+            Self::remove_source(custom_value.item);
+            Ok::<_, ShellError>(())
+        })
+    }
+
+    /// Check that `self` came from the given `source`, and return an `error` if not.
+    pub fn verify_source(&self, span: Span, source: &PluginSource) -> Result<(), ShellError> {
+        if self.source.is_compatible(source) {
+            Ok(())
+        } else {
+            Err(ShellError::CustomValueIncorrectForPlugin {
+                name: self.name().to_owned(),
+                span,
+                dest_plugin: source.name().to_owned(),
+                src_plugin: Some(self.source.name().to_owned()),
+            })
+        }
+    }
+
+    /// Check that a [`CustomValue`] is a [`PluginCustomValueWithSource`] that came from the given
+    /// `source`, and return an error if not.
+    pub fn verify_source_of_custom_value(
+        value: Spanned<&dyn CustomValue>,
+        source: &PluginSource,
+    ) -> Result<(), ShellError> {
+        if let Some(custom_value) = value
+            .item
+            .as_any()
+            .downcast_ref::<PluginCustomValueWithSource>()
+        {
+            custom_value.verify_source(value.span, source)
+        } else {
+            // Only PluginCustomValueWithSource can be sent
+            Err(ShellError::CustomValueIncorrectForPlugin {
+                name: value.item.type_name(),
+                span: value.span,
+                dest_plugin: source.name().to_owned(),
+                src_plugin: None,
+            })
+        }
+    }
+}
+
+impl std::ops::Deref for PluginCustomValueWithSource {
+    type Target = PluginCustomValue;
+
+    fn deref(&self) -> &PluginCustomValue {
+        &self.inner
+    }
+}
+
+/// This `Serialize` implementation always produces an error. Strip the source before sending.
+impl Serialize for PluginCustomValueWithSource {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        use serde::ser::Error;
+        Err(Error::custom(
+            "can't serialize PluginCustomValueWithSource, remove the source first",
+        ))
+    }
+}
+
+impl CustomValue for PluginCustomValueWithSource {
+    fn clone_value(&self, span: Span) -> Value {
+        self.clone().into_value(span)
+    }
+
+    fn type_name(&self) -> String {
+        self.name().to_owned()
+    }
+
+    fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
+        self.get_plugin(Some(span), "get base value")?
+            .custom_value_to_base_value(self.clone().into_spanned(span))
+    }
+
+    fn follow_path_int(
+        &self,
+        self_span: Span,
+        index: usize,
+        path_span: Span,
+    ) -> Result<Value, ShellError> {
+        self.get_plugin(Some(self_span), "follow cell path")?
+            .custom_value_follow_path_int(
+                self.clone().into_spanned(self_span),
+                index.into_spanned(path_span),
+            )
+    }
+
+    fn follow_path_string(
+        &self,
+        self_span: Span,
+        column_name: String,
+        path_span: Span,
+    ) -> Result<Value, ShellError> {
+        self.get_plugin(Some(self_span), "follow cell path")?
+            .custom_value_follow_path_string(
+                self.clone().into_spanned(self_span),
+                column_name.into_spanned(path_span),
+            )
+    }
+
+    fn partial_cmp(&self, other: &Value) -> Option<Ordering> {
+        self.get_plugin(Some(other.span()), "perform comparison")
+            .and_then(|plugin| {
+                // We're passing Span::unknown() here because we don't have one, and it probably
+                // shouldn't matter here and is just a consequence of the API
+                plugin.custom_value_partial_cmp(self.clone(), other.clone())
+            })
+            .unwrap_or_else(|err| {
+                // We can't do anything with the error other than log it.
+                log::warn!(
+                    "Error in partial_cmp on plugin custom value (source={source:?}): {err}",
+                    source = self.source
+                );
+                None
+            })
+            .map(|ordering| ordering.into())
+    }
+
+    fn operation(
+        &self,
+        lhs_span: Span,
+        operator: Operator,
+        op_span: Span,
+        right: &Value,
+    ) -> Result<Value, ShellError> {
+        self.get_plugin(Some(lhs_span), "invoke operator")?
+            .custom_value_operation(
+                self.clone().into_spanned(lhs_span),
+                operator.into_spanned(op_span),
+                right.clone(),
+            )
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
+    #[doc(hidden)]
+    fn typetag_name(&self) -> &'static str {
+        "PluginCustomValueWithSource"
+    }
+
+    #[doc(hidden)]
+    fn typetag_deserialize(&self) {}
+}
+
+impl Drop for PluginCustomValueWithSource {
+    fn drop(&mut self) {
+        // If the custom value specifies notify_on_drop and this is the last copy, we need to let
+        // the plugin know about it if we can.
+        if self.notify_on_drop() && self.inner.ref_count() == 1 {
+            self.get_plugin(None, "drop")
+                // While notifying drop, we don't need a copy of the source
+                .and_then(|plugin| plugin.custom_value_dropped(self.inner.clone()))
+                .unwrap_or_else(|err| {
+                    // We shouldn't do anything with the error except log it
+                    let name = self.name();
+                    log::warn!("Failed to notify drop of custom value ({name}): {err}")
+                });
+        }
+    }
+}
+
+/// Helper trait for adding a source to a [`PluginCustomValue`]
+pub trait WithSource {
+    /// Add a source to a plugin custom value
+    fn with_source(self, source: Arc<PluginSource>) -> PluginCustomValueWithSource;
+}
+
+impl WithSource for PluginCustomValue {
+    fn with_source(self, source: Arc<PluginSource>) -> PluginCustomValueWithSource {
+        PluginCustomValueWithSource::new(self, source)
+    }
+}

crates/nu-plugin-engine/src/plugin_custom_value_with_source/tests.rs

@@ -0,0 +1,198 @@
+use std::sync::Arc;
+
+use nu_plugin_protocol::test_util::{test_plugin_custom_value, TestCustomValue};
+use nu_protocol::{engine::Closure, record, CustomValue, IntoSpanned, ShellError, Span, Value};
+
+use crate::{
+    test_util::test_plugin_custom_value_with_source, PluginCustomValueWithSource, PluginSource,
+};
+
+use super::WithSource;
+
+#[test]
+fn add_source_in_at_root() -> Result<(), ShellError> {
+    let mut val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
+    let source = Arc::new(PluginSource::new_fake("foo"));
+    PluginCustomValueWithSource::add_source_in(&mut val, &source)?;
+
+    let custom_value = val.as_custom_value()?;
+    let plugin_custom_value: &PluginCustomValueWithSource = custom_value
+        .as_any()
+        .downcast_ref()
+        .expect("not PluginCustomValueWithSource");
+    assert_eq!(
+        Arc::as_ptr(&source),
+        Arc::as_ptr(&plugin_custom_value.source)
+    );
+    Ok(())
+}
+
+fn check_record_custom_values(
+    val: &Value,
+    keys: &[&str],
+    mut f: impl FnMut(&str, &dyn CustomValue) -> Result<(), ShellError>,
+) -> Result<(), ShellError> {
+    let record = val.as_record()?;
+    for key in keys {
+        let val = record
+            .get(key)
+            .unwrap_or_else(|| panic!("record does not contain '{key}'"));
+        let custom_value = val
+            .as_custom_value()
+            .unwrap_or_else(|_| panic!("'{key}' not custom value"));
+        f(key, custom_value)?;
+    }
+    Ok(())
+}
+
+#[test]
+fn add_source_in_nested_record() -> Result<(), ShellError> {
+    let orig_custom_val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
+    let mut val = Value::test_record(record! {
+        "foo" => orig_custom_val.clone(),
+        "bar" => orig_custom_val.clone(),
+    });
+    let source = Arc::new(PluginSource::new_fake("foo"));
+    PluginCustomValueWithSource::add_source_in(&mut val, &source)?;
+
+    check_record_custom_values(&val, &["foo", "bar"], |key, custom_value| {
+        let plugin_custom_value: &PluginCustomValueWithSource = custom_value
+            .as_any()
+            .downcast_ref()
+            .unwrap_or_else(|| panic!("'{key}' not PluginCustomValueWithSource"));
+        assert_eq!(
+            Arc::as_ptr(&source),
+            Arc::as_ptr(&plugin_custom_value.source),
+            "'{key}' source not set correctly"
+        );
+        Ok(())
+    })
+}
+
+fn check_list_custom_values(
+    val: &Value,
+    indices: impl IntoIterator<Item = usize>,
+    mut f: impl FnMut(usize, &dyn CustomValue) -> Result<(), ShellError>,
+) -> Result<(), ShellError> {
+    let list = val.as_list()?;
+    for index in indices {
+        let val = list
+            .get(index)
+            .unwrap_or_else(|| panic!("[{index}] not present in list"));
+        let custom_value = val
+            .as_custom_value()
+            .unwrap_or_else(|_| panic!("[{index}] not custom value"));
+        f(index, custom_value)?;
+    }
+    Ok(())
+}
+
+#[test]
+fn add_source_in_nested_list() -> Result<(), ShellError> {
+    let orig_custom_val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
+    let mut val = Value::test_list(vec![orig_custom_val.clone(), orig_custom_val.clone()]);
+    let source = Arc::new(PluginSource::new_fake("foo"));
+    PluginCustomValueWithSource::add_source_in(&mut val, &source)?;
+
+    check_list_custom_values(&val, 0..=1, |index, custom_value| {
+        let plugin_custom_value: &PluginCustomValueWithSource = custom_value
+            .as_any()
+            .downcast_ref()
+            .unwrap_or_else(|| panic!("[{index}] not PluginCustomValueWithSource"));
+        assert_eq!(
+            Arc::as_ptr(&source),
+            Arc::as_ptr(&plugin_custom_value.source),
+            "[{index}] source not set correctly"
+        );
+        Ok(())
+    })
+}
+
+fn check_closure_custom_values(
+    val: &Value,
+    indices: impl IntoIterator<Item = usize>,
+    mut f: impl FnMut(usize, &dyn CustomValue) -> Result<(), ShellError>,
+) -> Result<(), ShellError> {
+    let closure = val.as_closure()?;
+    for index in indices {
+        let val = closure
+            .captures
+            .get(index)
+            .unwrap_or_else(|| panic!("[{index}] not present in closure"));
+        let custom_value = val
+            .1
+            .as_custom_value()
+            .unwrap_or_else(|_| panic!("[{index}] not custom value"));
+        f(index, custom_value)?;
+    }
+    Ok(())
+}
+
+#[test]
+fn add_source_in_nested_closure() -> Result<(), ShellError> {
+    let orig_custom_val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
+    let mut val = Value::test_closure(Closure {
+        block_id: 0,
+        captures: vec![(0, orig_custom_val.clone()), (1, orig_custom_val.clone())],
+    });
+    let source = Arc::new(PluginSource::new_fake("foo"));
+    PluginCustomValueWithSource::add_source_in(&mut val, &source)?;
+
+    check_closure_custom_values(&val, 0..=1, |index, custom_value| {
+        let plugin_custom_value: &PluginCustomValueWithSource = custom_value
+            .as_any()
+            .downcast_ref()
+            .unwrap_or_else(|| panic!("[{index}] not PluginCustomValueWithSource"));
+        assert_eq!(
+            Arc::as_ptr(&source),
+            Arc::as_ptr(&plugin_custom_value.source),
+            "[{index}] source not set correctly"
+        );
+        Ok(())
+    })
+}
+
+#[test]
+fn verify_source_error_message() -> Result<(), ShellError> {
+    let span = Span::new(5, 7);
+    let ok_val = test_plugin_custom_value_with_source();
+    let native_val = TestCustomValue(32);
+    let foreign_val =
+        test_plugin_custom_value().with_source(Arc::new(PluginSource::new_fake("other")));
+    let source = PluginSource::new_fake("test");
+
+    PluginCustomValueWithSource::verify_source_of_custom_value(
+        (&ok_val as &dyn CustomValue).into_spanned(span),
+        &source,
+    )
+    .expect("ok_val should be verified ok");
+
+    for (val, src_plugin) in [
+        (&native_val as &dyn CustomValue, None),
+        (&foreign_val as &dyn CustomValue, Some("other")),
+    ] {
+        let error = PluginCustomValueWithSource::verify_source_of_custom_value(
+            val.into_spanned(span),
+            &source,
+        )
+        .expect_err(&format!(
+            "a custom value from {src_plugin:?} should result in an error"
+        ));
+        if let ShellError::CustomValueIncorrectForPlugin {
+            name,
+            span: err_span,
+            dest_plugin,
+            src_plugin: err_src_plugin,
+        } = error
+        {
+            assert_eq!("TestCustomValue", name, "error.name from {src_plugin:?}");
+            assert_eq!(span, err_span, "error.span from {src_plugin:?}");
+            assert_eq!("test", dest_plugin, "error.dest_plugin from {src_plugin:?}");
+            assert_eq!(src_plugin, err_src_plugin.as_deref(), "error.src_plugin");
+        } else {
+            panic!("the error returned should be CustomValueIncorrectForPlugin");
+        }
+    }
+
+    Ok(())
+}

crates/nu-plugin-engine/src/source.rs

@@ -4,10 +4,7 @@ use std::sync::{Arc, Weak};
 
 /// The source of a custom value or plugin command. Includes a weak reference to the persistent
 /// plugin so it can be retrieved.
-///
-/// This is not a public interface.
 #[derive(Debug, Clone)]
-#[doc(hidden)]
 pub struct PluginSource {
     /// The identity of the plugin
     pub(crate) identity: Arc<PluginIdentity>,
@@ -30,8 +27,7 @@ impl PluginSource {
     /// Create a new fake source with a fake identity, for testing
     ///
     /// Warning: [`.persistent()`] will always return an error.
-    #[cfg(test)]
+    pub fn new_fake(name: &str) -> PluginSource {
-    pub(crate) fn new_fake(name: &str) -> PluginSource {
         PluginSource {
             identity: PluginIdentity::new_fake(name).into(),
             persistent: Weak::<crate::PersistentPlugin>::new(),
@@ -40,9 +36,6 @@ impl PluginSource {
 
     /// Try to upgrade the persistent reference, and return an error referencing `span` as the
     /// object that referenced it otherwise
-    ///
-    /// This is not a public API.
-    #[doc(hidden)]
     pub fn persistent(&self, span: Option<Span>) -> Result<Arc<dyn GetPlugin>, ShellError> {
         self.persistent
             .upgrade()

crates/nu-plugin-engine/src/test_util.rs

@@ -0,0 +1,24 @@
+use std::sync::Arc;
+
+use nu_plugin_core::interface_test_util::TestCase;
+use nu_plugin_protocol::{test_util::test_plugin_custom_value, PluginInput, PluginOutput};
+
+use crate::{PluginCustomValueWithSource, PluginInterfaceManager, PluginSource};
+
+pub trait TestCaseExt {
+    /// Create a new [`PluginInterfaceManager`] that writes to this test case.
+    fn plugin(&self, name: &str) -> PluginInterfaceManager;
+}
+
+impl TestCaseExt for TestCase<PluginOutput, PluginInput> {
+    fn plugin(&self, name: &str) -> PluginInterfaceManager {
+        PluginInterfaceManager::new(PluginSource::new_fake(name).into(), None, self.clone())
+    }
+}
+
+pub fn test_plugin_custom_value_with_source() -> PluginCustomValueWithSource {
+    PluginCustomValueWithSource::new(
+        test_plugin_custom_value(),
+        Arc::new(PluginSource::new_fake("test")),
+    )
+}

crates/nu-plugin-engine/src/util/mod.rs

@@ -0,0 +1,3 @@
+mod mutable_cow;
+
+pub use mutable_cow::MutableCow;

crates/nu-plugin-protocol/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "Protocol type definitions for Nushell plugins"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-plugin-protocol"
+edition = "2021"
+license = "MIT"
+name = "nu-plugin-protocol"
+version = "0.92.3"
+
+[lib]
+bench = false
+
+[dependencies]
+nu-protocol = { path = "../nu-protocol", version = "0.92.3", features = ["plugin"] }
+nu-utils = { path = "../nu-utils", version = "0.92.3" }
+
+bincode = "1.3"
+serde = { workspace = true, features = ["derive"] }
+semver = "1.0"
+typetag = "0.2"

crates/nu-plugin-protocol/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-plugin-protocol/README.md

@@ -0,0 +1,5 @@
+# nu-plugin-protocol
+
+This crate provides serde-compatible types for implementing the [Nushell plugin protocol](https://www.nushell.sh/contributor-book/plugin_protocol_reference.html). It is primarily used by the `nu-plugin` family of crates, but can also be used separately as well.
+
+The specifics of I/O and serialization are not included in this crate. Use `serde_json` and/or `rmp-serde` (with the `named` serialization) to turn the types in this crate into data in the wire format.

crates/nu-plugin-protocol/src/evaluated_call.rs

@@ -7,10 +7,10 @@ use serde::{Deserialize, Serialize};
 
 /// A representation of the plugin's invocation command including command line args
 ///
-/// The `EvaluatedCall` contains information about the way a [`Plugin`](crate::Plugin) was invoked
+/// The `EvaluatedCall` contains information about the way a `Plugin` was invoked representing the
-/// representing the [`Span`] corresponding to the invocation as well as the arguments
+/// [`Span`] corresponding to the invocation as well as the arguments it was invoked with. It is
-/// it was invoked with. It is one of three items passed to [`run()`](crate::PluginCommand::run()) along with
+/// one of the items passed to `PluginCommand::run()`, along with the plugin reference, the engine
-/// `name` which command that was invoked and a [`Value`] that represents the input.
+/// interface, and a [`Value`] that represents the input.
 ///
 /// The evaluated call is used with the Plugins because the plugin doesn't have
 /// access to the Stack and the EngineState the way a built in command might. For that
@@ -27,7 +27,8 @@ pub struct EvaluatedCall {
 }
 
 impl EvaluatedCall {
-    pub(crate) fn try_from_call(
+    /// Try to create an [`EvaluatedCall`] from a command `Call`.
+    pub fn try_from_call(
         call: &Call,
         engine_state: &EngineState,
         stack: &mut Stack,
@@ -62,7 +63,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --foo`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -78,7 +79,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --bar`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -94,7 +95,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --foo=true`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -110,7 +111,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --foo=false`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -126,7 +127,7 @@ impl EvaluatedCall {
     /// Invoked with wrong type as `my_command --foo=1`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -163,7 +164,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --foo 123`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -184,7 +185,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -214,7 +215,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command a b c`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -244,7 +245,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --foo 123`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -261,7 +262,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --bar 123`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -278,7 +279,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command --foo abc`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,
@@ -305,7 +306,7 @@ impl EvaluatedCall {
     /// Invoked as `my_command zero one two three`:
     /// ```
     /// # use nu_protocol::{Spanned, Span, Value};
-    /// # use nu_plugin::EvaluatedCall;
+    /// # use nu_plugin_protocol::EvaluatedCall;
     /// # let null_span = Span::new(0, 0);
     /// # let call = EvaluatedCall {
     /// #     head: null_span,

crates/nu-plugin-protocol/src/lib.rs

@@ -1,3 +1,14 @@
+//! Type definitions, including full `Serialize` and `Deserialize` implementations, for the protocol
+//! used for communication between the engine and a plugin.
+//!
+//! See the [plugin protocol reference](https://www.nushell.sh/contributor-book/plugin_protocol_reference.html)
+//! for more details on what exactly is being specified here.
+//!
+//! Plugins accept messages of [`PluginInput`] and send messages back of [`PluginOutput`]. This
+//! crate explicitly avoids implementing any functionality that depends on I/O, so the exact
+//! byte-level encoding scheme is not implemented here. See the protocol ref or `nu_plugin_core` for
+//! more details on how that works.
+
 mod evaluated_call;
 mod plugin_custom_value;
 mod protocol_info;
@@ -5,8 +16,10 @@ mod protocol_info;
 #[cfg(test)]
 mod tests;
 
-#[cfg(test)]
+/// Things that can help with protocol-related tests. Not part of the public API, just used by other
-pub(crate) mod test_util;
+/// nushell crates.
+#[doc(hidden)]
+pub mod test_util;
 
 use nu_protocol::{
     ast::Operator, engine::Closure, Config, LabeledError, PipelineData, PluginSignature, RawStream,
@@ -44,7 +57,7 @@ pub struct CallInfo<D> {
 
 impl<D> CallInfo<D> {
     /// Convert the type of `input` from `D` to `T`.
-    pub(crate) fn map_data<T>(
+    pub fn map_data<T>(
         self,
         f: impl FnOnce(D) -> Result<T, ShellError>,
     ) -> Result<CallInfo<T>, ShellError> {
@@ -77,7 +90,7 @@ pub enum PipelineDataHeader {
 
 impl PipelineDataHeader {
     /// Return a list of stream IDs embedded in the header
-    pub(crate) fn stream_ids(&self) -> Vec<StreamId> {
+    pub fn stream_ids(&self) -> Vec<StreamId> {
         match self {
             PipelineDataHeader::Empty => vec![],
             PipelineDataHeader::Value(_) => vec![],
@@ -124,7 +137,7 @@ pub struct RawStreamInfo {
 }
 
 impl RawStreamInfo {
-    pub(crate) fn new(id: StreamId, stream: &RawStream) -> Self {
+    pub fn new(id: StreamId, stream: &RawStream) -> Self {
         RawStreamInfo {
             id,
             is_binary: stream.is_binary,
@@ -144,7 +157,7 @@ pub enum PluginCall<D> {
 impl<D> PluginCall<D> {
     /// Convert the data type from `D` to `T`. The function will not be called if the variant does
     /// not contain data.
-    pub(crate) fn map_data<T>(
+    pub fn map_data<T>(
         self,
         f: impl FnOnce(D) -> Result<T, ShellError>,
     ) -> Result<PluginCall<T>, ShellError> {
@@ -187,7 +200,7 @@ pub enum CustomValueOp {
 
 impl CustomValueOp {
     /// Get the name of the op, for error messages.
-    pub(crate) fn name(&self) -> &'static str {
+    pub fn name(&self) -> &'static str {
         match self {
             CustomValueOp::ToBaseValue => "to_base_value",
             CustomValueOp::FollowPathInt(_) => "follow_path_int",
@@ -200,10 +213,7 @@ impl CustomValueOp {
 }
 
 /// Any data sent to the plugin
-///
-/// Note: exported for internal use, not public.
 #[derive(Serialize, Deserialize, Debug, Clone)]
-#[doc(hidden)]
 pub enum PluginInput {
     /// This must be the first message. Indicates supported protocol
     Hello(ProtocolInfo),
@@ -326,10 +336,7 @@ pub enum StreamMessage {
 }
 
 /// Response to a [`PluginCall`]. The type parameter determines the output type for pipeline data.
-///
-/// Note: exported for internal use, not public.
 #[derive(Serialize, Deserialize, Debug, Clone)]
-#[doc(hidden)]
 pub enum PluginCallResponse<D> {
     Error(LabeledError),
     Signature(Vec<PluginSignature>),
@@ -340,7 +347,7 @@ pub enum PluginCallResponse<D> {
 impl<D> PluginCallResponse<D> {
     /// Convert the data type from `D` to `T`. The function will not be called if the variant does
     /// not contain data.
-    pub(crate) fn map_data<T>(
+    pub fn map_data<T>(
         self,
         f: impl FnOnce(D) -> Result<T, ShellError>,
     ) -> Result<PluginCallResponse<T>, ShellError> {
@@ -366,7 +373,7 @@ impl PluginCallResponse<PipelineDataHeader> {
 
 impl PluginCallResponse<PipelineData> {
     /// Does this response have a stream?
-    pub(crate) fn has_stream(&self) -> bool {
+    pub fn has_stream(&self) -> bool {
         match self {
             PluginCallResponse::PipelineData(data) => match data {
                 PipelineData::Empty => false,
@@ -385,7 +392,7 @@ pub enum PluginOption {
     /// Send `GcDisabled(true)` to stop the plugin from being automatically garbage collected, or
     /// `GcDisabled(false)` to enable it again.
     ///
-    /// See [`EngineInterface::set_gc_disabled`] for more information.
+    /// See `EngineInterface::set_gc_disabled()` in `nu-plugin` for more information.
     GcDisabled(bool),
 }
 
@@ -418,10 +425,7 @@ impl From<Ordering> for std::cmp::Ordering {
 }
 
 /// Information received from the plugin
-///
-/// Note: exported for internal use, not public.
 #[derive(Serialize, Deserialize, Debug, Clone)]
-#[doc(hidden)]
 pub enum PluginOutput {
     /// This must be the first message. Indicates supported protocol
     Hello(ProtocolInfo),
@@ -536,7 +540,7 @@ impl<D> EngineCall<D> {
 
     /// Convert the data type from `D` to `T`. The function will not be called if the variant does
     /// not contain data.
-    pub(crate) fn map_data<T>(
+    pub fn map_data<T>(
         self,
         f: impl FnOnce(D) -> Result<T, ShellError>,
     ) -> Result<EngineCall<T>, ShellError> {
@@ -581,7 +585,7 @@ pub enum EngineCallResponse<D> {
 impl<D> EngineCallResponse<D> {
     /// Convert the data type from `D` to `T`. The function will not be called if the variant does
     /// not contain data.
-    pub(crate) fn map_data<T>(
+    pub fn map_data<T>(
         self,
         f: impl FnOnce(D) -> Result<T, ShellError>,
     ) -> Result<EngineCallResponse<T>, ShellError> {
@@ -596,12 +600,12 @@ impl<D> EngineCallResponse<D> {
 
 impl EngineCallResponse<PipelineData> {
     /// Build an [`EngineCallResponse::PipelineData`] from a [`Value`]
-    pub(crate) fn value(value: Value) -> EngineCallResponse<PipelineData> {
+    pub fn value(value: Value) -> EngineCallResponse<PipelineData> {
         EngineCallResponse::PipelineData(PipelineData::Value(value, None))
     }
 
     /// An [`EngineCallResponse::PipelineData`] with [`PipelineData::Empty`]
-    pub(crate) const fn empty() -> EngineCallResponse<PipelineData> {
+    pub const fn empty() -> EngineCallResponse<PipelineData> {
         EngineCallResponse::PipelineData(PipelineData::Empty)
     }
 }

crates/nu-plugin-protocol/src/plugin_custom_value/mod.rs

@@ -0,0 +1,236 @@
+use std::cmp::Ordering;
+
+use nu_protocol::{ast::Operator, CustomValue, ShellError, Span, Value};
+use nu_utils::SharedCow;
+
+use serde::{Deserialize, Serialize};
+
+#[cfg(test)]
+mod tests;
+
+/// An opaque container for a custom value that is handled fully by a plugin.
+///
+/// This is the only type of custom value that is allowed to cross the plugin serialization
+/// boundary.
+///
+/// The plugin is responsible for ensuring that local plugin custom values are converted to and from
+/// [`PluginCustomValue`] on the boundary.
+///
+/// The engine is responsible for adding tracking the source of the custom value, ensuring that only
+/// [`PluginCustomValue`] is contained within any values sent, and that the source of any values
+/// sent matches the plugin it is being sent to.
+///
+/// Most of the [`CustomValue`] methods on this type will result in a panic. The source must be
+/// added (see `nu_plugin_engine::PluginCustomValueWithSource`) in order to implement the
+/// functionality via plugin calls.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct PluginCustomValue(SharedCow<SharedContent>);
+
+/// Content shared across copies of a plugin custom value.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct SharedContent {
+    /// The name of the type of the custom value as defined by the plugin (`type_name()`)
+    name: String,
+    /// The bincoded representation of the custom value on the plugin side
+    data: Vec<u8>,
+    /// True if the custom value should notify the source if all copies of it are dropped.
+    ///
+    /// This is not serialized if `false`, since most custom values don't need it.
+    #[serde(default, skip_serializing_if = "is_false")]
+    notify_on_drop: bool,
+}
+
+fn is_false(b: &bool) -> bool {
+    !b
+}
+
+#[typetag::serde]
+impl CustomValue for PluginCustomValue {
+    fn clone_value(&self, span: Span) -> Value {
+        self.clone().into_value(span)
+    }
+
+    fn type_name(&self) -> String {
+        self.name().to_owned()
+    }
+
+    fn to_base_value(&self, _span: Span) -> Result<Value, ShellError> {
+        panic!("to_base_value() not available on plugin custom value without source");
+    }
+
+    fn follow_path_int(
+        &self,
+        _self_span: Span,
+        _index: usize,
+        _path_span: Span,
+    ) -> Result<Value, ShellError> {
+        panic!("follow_path_int() not available on plugin custom value without source");
+    }
+
+    fn follow_path_string(
+        &self,
+        _self_span: Span,
+        _column_name: String,
+        _path_span: Span,
+    ) -> Result<Value, ShellError> {
+        panic!("follow_path_string() not available on plugin custom value without source");
+    }
+
+    fn partial_cmp(&self, _other: &Value) -> Option<Ordering> {
+        panic!("partial_cmp() not available on plugin custom value without source");
+    }
+
+    fn operation(
+        &self,
+        _lhs_span: Span,
+        _operator: Operator,
+        _op_span: Span,
+        _right: &Value,
+    ) -> Result<Value, ShellError> {
+        panic!("operation() not available on plugin custom value without source");
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+}
+
+impl PluginCustomValue {
+    /// Create a new [`PluginCustomValue`].
+    pub fn new(name: String, data: Vec<u8>, notify_on_drop: bool) -> PluginCustomValue {
+        PluginCustomValue(SharedCow::new(SharedContent {
+            name,
+            data,
+            notify_on_drop,
+        }))
+    }
+
+    /// Create a [`Value`] containing this custom value.
+    pub fn into_value(self, span: Span) -> Value {
+        Value::custom(Box::new(self), span)
+    }
+
+    /// The name of the type of the custom value as defined by the plugin (`type_name()`)
+    pub fn name(&self) -> &str {
+        &self.0.name
+    }
+
+    /// The bincoded representation of the custom value on the plugin side
+    pub fn data(&self) -> &[u8] {
+        &self.0.data
+    }
+
+    /// True if the custom value should notify the source if all copies of it are dropped.
+    pub fn notify_on_drop(&self) -> bool {
+        self.0.notify_on_drop
+    }
+
+    /// Count the number of shared copies of this [`PluginCustomValue`].
+    pub fn ref_count(&self) -> usize {
+        SharedCow::ref_count(&self.0)
+    }
+
+    /// Serialize a custom value into a [`PluginCustomValue`]. This should only be done on the
+    /// plugin side.
+    pub fn serialize_from_custom_value(
+        custom_value: &dyn CustomValue,
+        span: Span,
+    ) -> Result<PluginCustomValue, ShellError> {
+        let name = custom_value.type_name();
+        let notify_on_drop = custom_value.notify_plugin_on_drop();
+        bincode::serialize(custom_value)
+            .map(|data| PluginCustomValue::new(name, data, notify_on_drop))
+            .map_err(|err| ShellError::CustomValueFailedToEncode {
+                msg: err.to_string(),
+                span,
+            })
+    }
+
+    /// Deserialize a [`PluginCustomValue`] into a `Box<dyn CustomValue>`. This should only be done
+    /// on the plugin side.
+    pub fn deserialize_to_custom_value(
+        &self,
+        span: Span,
+    ) -> Result<Box<dyn CustomValue>, ShellError> {
+        bincode::deserialize::<Box<dyn CustomValue>>(self.data()).map_err(|err| {
+            ShellError::CustomValueFailedToDecode {
+                msg: err.to_string(),
+                span,
+            }
+        })
+    }
+    /// Convert all plugin-native custom values to [`PluginCustomValue`] within the given `value`,
+    /// recursively. This should only be done on the plugin side.
+    pub fn serialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
+        value.recurse_mut(&mut |value| {
+            let span = value.span();
+            match value {
+                Value::Custom { ref val, .. } => {
+                    if val.as_any().downcast_ref::<PluginCustomValue>().is_some() {
+                        // Already a PluginCustomValue
+                        Ok(())
+                    } else {
+                        let serialized = Self::serialize_from_custom_value(&**val, span)?;
+                        *value = Value::custom(Box::new(serialized), span);
+                        Ok(())
+                    }
+                }
+                // Collect LazyRecord before proceeding
+                Value::LazyRecord { ref val, .. } => {
+                    *value = val.collect()?;
+                    Ok(())
+                }
+                _ => Ok(()),
+            }
+        })
+    }
+
+    /// Convert all [`PluginCustomValue`]s to plugin-native custom values within the given `value`,
+    /// recursively. This should only be done on the plugin side.
+    pub fn deserialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
+        value.recurse_mut(&mut |value| {
+            let span = value.span();
+            match value {
+                Value::Custom { ref val, .. } => {
+                    if let Some(val) = val.as_any().downcast_ref::<PluginCustomValue>() {
+                        let deserialized = val.deserialize_to_custom_value(span)?;
+                        *value = Value::custom(deserialized, span);
+                        Ok(())
+                    } else {
+                        // Already not a PluginCustomValue
+                        Ok(())
+                    }
+                }
+                // Collect LazyRecord before proceeding
+                Value::LazyRecord { ref val, .. } => {
+                    *value = val.collect()?;
+                    Ok(())
+                }
+                _ => Ok(()),
+            }
+        })
+    }
+
+    /// Render any custom values in the `Value` using `to_base_value()`
+    pub fn render_to_base_value_in(value: &mut Value) -> Result<(), ShellError> {
+        value.recurse_mut(&mut |value| {
+            let span = value.span();
+            match value {
+                Value::Custom { ref val, .. } => {
+                    *value = val.to_base_value(span)?;
+                    Ok(())
+                }
+                // Collect LazyRecord before proceeding
+                Value::LazyRecord { ref val, .. } => {
+                    *value = val.collect()?;
+                    Ok(())
+                }
+                _ => Ok(()),
+            }
+        })
+    }
+}

crates/nu-plugin-protocol/src/plugin_custom_value/tests.rs

@@ -1,13 +1,63 @@
+use crate::test_util::{expected_test_custom_value, test_plugin_custom_value, TestCustomValue};
+
 use super::PluginCustomValue;
-use crate::{
+use nu_protocol::{engine::Closure, record, CustomValue, ShellError, Span, Value};
-    plugin::PluginSource,
+
-    protocol::test_util::{
+fn check_record_custom_values(
-        expected_test_custom_value, test_plugin_custom_value, test_plugin_custom_value_with_source,
+    val: &Value,
-        TestCustomValue,
+    keys: &[&str],
-    },
+    mut f: impl FnMut(&str, &dyn CustomValue) -> Result<(), ShellError>,
-};
+) -> Result<(), ShellError> {
-use nu_protocol::{engine::Closure, record, CustomValue, IntoSpanned, ShellError, Span, Value};
+    let record = val.as_record()?;
-use std::sync::Arc;
+    for key in keys {
+        let val = record
+            .get(key)
+            .unwrap_or_else(|| panic!("record does not contain '{key}'"));
+        let custom_value = val
+            .as_custom_value()
+            .unwrap_or_else(|_| panic!("'{key}' not custom value"));
+        f(key, custom_value)?;
+    }
+    Ok(())
+}
+
+fn check_list_custom_values(
+    val: &Value,
+    indices: impl IntoIterator<Item = usize>,
+    mut f: impl FnMut(usize, &dyn CustomValue) -> Result<(), ShellError>,
+) -> Result<(), ShellError> {
+    let list = val.as_list()?;
+    for index in indices {
+        let val = list
+            .get(index)
+            .unwrap_or_else(|| panic!("[{index}] not present in list"));
+        let custom_value = val
+            .as_custom_value()
+            .unwrap_or_else(|_| panic!("[{index}] not custom value"));
+        f(index, custom_value)?;
+    }
+    Ok(())
+}
+
+fn check_closure_custom_values(
+    val: &Value,
+    indices: impl IntoIterator<Item = usize>,
+    mut f: impl FnMut(usize, &dyn CustomValue) -> Result<(), ShellError>,
+) -> Result<(), ShellError> {
+    let closure = val.as_closure()?;
+    for index in indices {
+        let val = closure
+            .captures
+            .get(index)
+            .unwrap_or_else(|| panic!("[{index}] not present in closure"));
+        let custom_value = val
+            .1
+            .as_custom_value()
+            .unwrap_or_else(|_| panic!("[{index}] not custom value"));
+        f(index, custom_value)?;
+    }
+    Ok(())
+}
 
 #[test]
 fn serialize_deserialize() -> Result<(), ShellError> {
@@ -15,7 +65,6 @@ fn serialize_deserialize() -> Result<(), ShellError> {
     let span = Span::test_data();
     let serialized = PluginCustomValue::serialize_from_custom_value(&original_value, span)?;
     assert_eq!(original_value.type_name(), serialized.name());
-    assert!(serialized.source.is_none());
     let deserialized = serialized.deserialize_to_custom_value(span)?;
     let downcasted = deserialized
         .as_any()
@@ -39,190 +88,6 @@ fn expected_serialize_output() -> Result<(), ShellError> {
     Ok(())
 }
 
-#[test]
-fn add_source_in_at_root() -> Result<(), ShellError> {
-    let mut val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
-    let source = Arc::new(PluginSource::new_fake("foo"));
-    PluginCustomValue::add_source_in(&mut val, &source)?;
-
-    let custom_value = val.as_custom_value()?;
-    let plugin_custom_value: &PluginCustomValue = custom_value
-        .as_any()
-        .downcast_ref()
-        .expect("not PluginCustomValue");
-    assert_eq!(
-        Some(Arc::as_ptr(&source)),
-        plugin_custom_value.source.as_ref().map(Arc::as_ptr)
-    );
-    Ok(())
-}
-
-fn check_record_custom_values(
-    val: &Value,
-    keys: &[&str],
-    mut f: impl FnMut(&str, &dyn CustomValue) -> Result<(), ShellError>,
-) -> Result<(), ShellError> {
-    let record = val.as_record()?;
-    for key in keys {
-        let val = record
-            .get(key)
-            .unwrap_or_else(|| panic!("record does not contain '{key}'"));
-        let custom_value = val
-            .as_custom_value()
-            .unwrap_or_else(|_| panic!("'{key}' not custom value"));
-        f(key, custom_value)?;
-    }
-    Ok(())
-}
-
-#[test]
-fn add_source_in_nested_record() -> Result<(), ShellError> {
-    let orig_custom_val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
-    let mut val = Value::test_record(record! {
-        "foo" => orig_custom_val.clone(),
-        "bar" => orig_custom_val.clone(),
-    });
-    let source = Arc::new(PluginSource::new_fake("foo"));
-    PluginCustomValue::add_source_in(&mut val, &source)?;
-
-    check_record_custom_values(&val, &["foo", "bar"], |key, custom_value| {
-        let plugin_custom_value: &PluginCustomValue = custom_value
-            .as_any()
-            .downcast_ref()
-            .unwrap_or_else(|| panic!("'{key}' not PluginCustomValue"));
-        assert_eq!(
-            Some(Arc::as_ptr(&source)),
-            plugin_custom_value.source.as_ref().map(Arc::as_ptr),
-            "'{key}' source not set correctly"
-        );
-        Ok(())
-    })
-}
-
-fn check_list_custom_values(
-    val: &Value,
-    indices: impl IntoIterator<Item = usize>,
-    mut f: impl FnMut(usize, &dyn CustomValue) -> Result<(), ShellError>,
-) -> Result<(), ShellError> {
-    let list = val.as_list()?;
-    for index in indices {
-        let val = list
-            .get(index)
-            .unwrap_or_else(|| panic!("[{index}] not present in list"));
-        let custom_value = val
-            .as_custom_value()
-            .unwrap_or_else(|_| panic!("[{index}] not custom value"));
-        f(index, custom_value)?;
-    }
-    Ok(())
-}
-
-#[test]
-fn add_source_in_nested_list() -> Result<(), ShellError> {
-    let orig_custom_val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
-    let mut val = Value::test_list(vec![orig_custom_val.clone(), orig_custom_val.clone()]);
-    let source = Arc::new(PluginSource::new_fake("foo"));
-    PluginCustomValue::add_source_in(&mut val, &source)?;
-
-    check_list_custom_values(&val, 0..=1, |index, custom_value| {
-        let plugin_custom_value: &PluginCustomValue = custom_value
-            .as_any()
-            .downcast_ref()
-            .unwrap_or_else(|| panic!("[{index}] not PluginCustomValue"));
-        assert_eq!(
-            Some(Arc::as_ptr(&source)),
-            plugin_custom_value.source.as_ref().map(Arc::as_ptr),
-            "[{index}] source not set correctly"
-        );
-        Ok(())
-    })
-}
-
-fn check_closure_custom_values(
-    val: &Value,
-    indices: impl IntoIterator<Item = usize>,
-    mut f: impl FnMut(usize, &dyn CustomValue) -> Result<(), ShellError>,
-) -> Result<(), ShellError> {
-    let closure = val.as_closure()?;
-    for index in indices {
-        let val = closure
-            .captures
-            .get(index)
-            .unwrap_or_else(|| panic!("[{index}] not present in closure"));
-        let custom_value = val
-            .1
-            .as_custom_value()
-            .unwrap_or_else(|_| panic!("[{index}] not custom value"));
-        f(index, custom_value)?;
-    }
-    Ok(())
-}
-
-#[test]
-fn add_source_in_nested_closure() -> Result<(), ShellError> {
-    let orig_custom_val = Value::test_custom_value(Box::new(test_plugin_custom_value()));
-    let mut val = Value::test_closure(Closure {
-        block_id: 0,
-        captures: vec![(0, orig_custom_val.clone()), (1, orig_custom_val.clone())],
-    });
-    let source = Arc::new(PluginSource::new_fake("foo"));
-    PluginCustomValue::add_source_in(&mut val, &source)?;
-
-    check_closure_custom_values(&val, 0..=1, |index, custom_value| {
-        let plugin_custom_value: &PluginCustomValue = custom_value
-            .as_any()
-            .downcast_ref()
-            .unwrap_or_else(|| panic!("[{index}] not PluginCustomValue"));
-        assert_eq!(
-            Some(Arc::as_ptr(&source)),
-            plugin_custom_value.source.as_ref().map(Arc::as_ptr),
-            "[{index}] source not set correctly"
-        );
-        Ok(())
-    })
-}
-
-#[test]
-fn verify_source_error_message() -> Result<(), ShellError> {
-    let span = Span::new(5, 7);
-    let ok_val = test_plugin_custom_value_with_source();
-    let native_val = TestCustomValue(32);
-    let foreign_val = {
-        let mut val = test_plugin_custom_value();
-        val.source = Some(Arc::new(PluginSource::new_fake("other")));
-        val
-    };
-    let source = PluginSource::new_fake("test");
-
-    PluginCustomValue::verify_source((&ok_val as &dyn CustomValue).into_spanned(span), &source)
-        .expect("ok_val should be verified ok");
-
-    for (val, src_plugin) in [
-        (&native_val as &dyn CustomValue, None),
-        (&foreign_val as &dyn CustomValue, Some("other")),
-    ] {
-        let error = PluginCustomValue::verify_source(val.into_spanned(span), &source).expect_err(
-            &format!("a custom value from {src_plugin:?} should result in an error"),
-        );
-        if let ShellError::CustomValueIncorrectForPlugin {
-            name,
-            span: err_span,
-            dest_plugin,
-            src_plugin: err_src_plugin,
-        } = error
-        {
-            assert_eq!("TestCustomValue", name, "error.name from {src_plugin:?}");
-            assert_eq!(span, err_span, "error.span from {src_plugin:?}");
-            assert_eq!("test", dest_plugin, "error.dest_plugin from {src_plugin:?}");
-            assert_eq!(src_plugin, err_src_plugin.as_deref(), "error.src_plugin");
-        } else {
-            panic!("the error returned should be CustomValueIncorrectForPlugin");
-        }
-    }
-
-    Ok(())
-}
-
 #[test]
 fn serialize_in_root() -> Result<(), ShellError> {
     let span = Span::new(4, 10);
@@ -238,7 +103,6 @@ fn serialize_in_root() -> Result<(), ShellError> {
             test_plugin_custom_value().data(),
             plugin_custom_value.data()
         );
-        assert!(plugin_custom_value.source.is_none());
     } else {
         panic!("Failed to downcast to PluginCustomValue");
     }

crates/nu-plugin-protocol/src/protocol_info.rs

@@ -9,7 +9,7 @@ pub struct ProtocolInfo {
     /// The name of the protocol being implemented. Only one protocol is supported. This field
     /// can be safely ignored, because not matching is a deserialization error
     pub protocol: Protocol,
-    /// The semantic version of the protocol. This should be the version of the `nu-plugin`
+    /// The semantic version of the protocol. This should be the version of the `nu-plugin-protocol`
     /// crate
     pub version: String,
     /// Supported optional features. This helps to maintain semver compatibility when adding new

crates/nu-plugin-protocol/src/test_util.rs

@@ -1,12 +1,12 @@
-use super::PluginCustomValue;
+use crate::PluginCustomValue;
-use crate::plugin::PluginSource;
 use nu_protocol::{CustomValue, ShellError, Span, Value};
 use serde::{Deserialize, Serialize};
 
+/// A custom value that can be used for testing.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct TestCustomValue(pub i32);
+pub struct TestCustomValue(pub i32);
 
-#[typetag::serde]
+#[typetag::serde(name = "nu_plugin_protocol::test_util::TestCustomValue")]
 impl CustomValue for TestCustomValue {
     fn clone_value(&self, span: Span) -> Value {
         Value::custom(Box::new(self.clone()), span)
@@ -29,17 +29,15 @@ impl CustomValue for TestCustomValue {
     }
 }
 
-pub(crate) fn test_plugin_custom_value() -> PluginCustomValue {
+/// A [`TestCustomValue`] serialized as a [`PluginCustomValue`].
+pub fn test_plugin_custom_value() -> PluginCustomValue {
     let data = bincode::serialize(&expected_test_custom_value() as &dyn CustomValue)
         .expect("bincode serialization of the expected_test_custom_value() failed");
 
-    PluginCustomValue::new("TestCustomValue".into(), data, false, None)
+    PluginCustomValue::new("TestCustomValue".into(), data, false)
 }
 
-pub(crate) fn expected_test_custom_value() -> TestCustomValue {
+/// The expected [`TestCustomValue`] that [`test_plugin_custom_value()`] should deserialize into.
+pub fn expected_test_custom_value() -> TestCustomValue {
     TestCustomValue(-1)
 }
-
-pub(crate) fn test_plugin_custom_value_with_source() -> PluginCustomValue {
-    test_plugin_custom_value().with_source(Some(PluginSource::new_fake("test").into()))
-}

crates/nu-plugin-test-support/Cargo.toml

@@ -6,6 +6,9 @@ license = "MIT"
 description = "Testing support for Nushell plugins"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-plugin-test-support"
 
+[lib]
+bench = false
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -13,6 +16,9 @@ nu-engine = { path = "../nu-engine", version = "0.92.3", features = ["plugin"] }
 nu-protocol = { path = "../nu-protocol", version = "0.92.3", features = ["plugin"] }
 nu-parser = { path = "../nu-parser", version = "0.92.3", features = ["plugin"] }
 nu-plugin = { path = "../nu-plugin", version = "0.92.3" }
+nu-plugin-core = { path = "../nu-plugin-core", version = "0.92.3" }
+nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.92.3" }
+nu-plugin-protocol = { path = "../nu-plugin-protocol", version = "0.92.3" }
 nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.92.3" }
 nu-ansi-term = { workspace = true }
 similar = "2.5"

crates/nu-plugin-test-support/src/fake_persistent_plugin.rs

@@ -3,7 +3,7 @@ use std::{
     sync::{Arc, OnceLock},
 };
 
-use nu_plugin::{GetPlugin, PluginInterface};
+use nu_plugin_engine::{GetPlugin, PluginInterface};
 use nu_protocol::{
     engine::{EngineState, Stack},
     PluginGcConfig, PluginIdentity, RegisteredPlugin, ShellError,

crates/nu-plugin-test-support/src/fake_register.rs

@@ -1,6 +1,7 @@
 use std::{ops::Deref, sync::Arc};
 
-use nu_plugin::{create_plugin_signature, Plugin, PluginDeclaration};
+use nu_plugin::{create_plugin_signature, Plugin};
+use nu_plugin_engine::PluginDeclaration;
 use nu_protocol::{engine::StateWorkingSet, RegisteredPlugin, ShellError};
 
 use crate::{fake_persistent_plugin::FakePersistentPlugin, spawn_fake_plugin::spawn_fake_plugin};

crates/nu-plugin-test-support/src/plugin_test.rs

@@ -4,7 +4,9 @@ use nu_ansi_term::Style;
 use nu_cmd_lang::create_default_context;
 use nu_engine::eval_block;
 use nu_parser::parse;
-use nu_plugin::{Plugin, PluginCommand, PluginCustomValue, PluginSource};
+use nu_plugin::{Plugin, PluginCommand};
+use nu_plugin_engine::{PluginCustomValueWithSource, PluginSource, WithSource};
+use nu_plugin_protocol::PluginCustomValue;
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
@@ -135,13 +137,14 @@ impl PluginTest {
         // Serialize custom values in the input
         let source = self.source.clone();
         let input = input.map(
-            move |mut value| match PluginCustomValue::serialize_custom_values_in(&mut value) {
+            move |mut value| {
-                Ok(()) => {
+                let result = PluginCustomValue::serialize_custom_values_in(&mut value)
                     // Make sure to mark them with the source so they pass correctly, too.
-                    let _ = PluginCustomValue::add_source_in(&mut value, &source);
+                    .and_then(|_| PluginCustomValueWithSource::add_source_in(&mut value, &source));
-                    value
+                match result {
-                }
+                    Ok(()) => value,
                     Err(err) => Value::error(err, value.span()),
+                }
             },
             None,
         )?;
@@ -151,7 +154,9 @@ impl PluginTest {
         eval_block::<WithoutDebug>(&self.engine_state, &mut stack, &block, input)?.map(
             |mut value| {
                 // Make sure to deserialize custom values
-                match PluginCustomValue::deserialize_custom_values_in(&mut value) {
+                let result = PluginCustomValueWithSource::remove_source_in(&mut value)
+                    .and_then(|_| PluginCustomValue::deserialize_custom_values_in(&mut value));
+                match result {
                     Ok(()) => value,
                     Err(err) => Value::error(err, value.span()),
                 }
@@ -284,12 +289,12 @@ impl PluginTest {
         match (a, b) {
             (Value::Custom { val, .. }, _) => {
                 // We have to serialize both custom values before handing them to the plugin
-                let mut serialized =
+                let serialized =
-                    PluginCustomValue::serialize_from_custom_value(val.as_ref(), a.span())?;
+                    PluginCustomValue::serialize_from_custom_value(val.as_ref(), a.span())?
-                serialized.set_source(Some(self.source.clone()));
+                        .with_source(self.source.clone());
                 let mut b_serialized = b.clone();
                 PluginCustomValue::serialize_custom_values_in(&mut b_serialized)?;
-                PluginCustomValue::add_source_in(&mut b_serialized, &self.source)?;
+                PluginCustomValueWithSource::add_source_in(&mut b_serialized, &self.source)?;
                 // Now get the plugin reference and execute the comparison
                 let persistent = self.source.persistent(None)?.get_plugin(None)?;
                 let ordering = persistent.custom_value_partial_cmp(serialized, b_serialized)?;
@@ -354,8 +359,8 @@ impl PluginTest {
         val: &dyn CustomValue,
         span: Span,
     ) -> Result<Value, ShellError> {
-        let mut serialized = PluginCustomValue::serialize_from_custom_value(val, span)?;
+        let serialized = PluginCustomValue::serialize_from_custom_value(val, span)?
-        serialized.set_source(Some(self.source.clone()));
+            .with_source(self.source.clone());
         let persistent = self.source.persistent(None)?.get_plugin(None)?;
         persistent.custom_value_to_base_value(serialized.into_spanned(span))
     }

crates/nu-plugin-test-support/src/spawn_fake_plugin.rs

@@ -1,9 +1,9 @@
 use std::sync::{mpsc, Arc};
 
-use nu_plugin::{
+use nu_plugin::Plugin;
-    InterfaceManager, Plugin, PluginInput, PluginInterfaceManager, PluginOutput, PluginRead,
+use nu_plugin_core::{InterfaceManager, PluginRead, PluginWrite};
-    PluginSource, PluginWrite,
+use nu_plugin_engine::{PluginInterfaceManager, PluginSource};
-};
+use nu_plugin_protocol::{PluginInput, PluginOutput};
 use nu_protocol::{PluginIdentity, ShellError};
 
 use crate::fake_persistent_plugin::FakePersistentPlugin;

crates/nu-plugin/Cargo.toml

@@ -13,30 +13,20 @@ bench = false
 [dependencies]
 nu-engine = { path = "../nu-engine", version = "0.92.3" }
 nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
-nu-system = { path = "../nu-system", version = "0.92.3" }
+nu-plugin-protocol = { path = "../nu-plugin-protocol", version = "0.92.3" }
-nu-utils = { path = "../nu-utils", version = "0.92.3" }
+nu-plugin-core = { path = "../nu-plugin-core", version = "0.92.3", default-features = false }
 
-bincode = "1.3"
+log = { workspace = true }
-rmp-serde = "1.2"
-serde = { workspace = true }
-serde_json = { workspace = true }
-log = "0.4"
-miette = { workspace = true }
-semver = "1.0"
-typetag = "0.2"
 thiserror = "1.0"
-interprocess = { version = "1.2.1", optional = true }
+
+[dev-dependencies]
+serde = { workspace = true }
+typetag = "0.2"
 
 [features]
 default = ["local-socket"]
-local-socket = ["interprocess"]
+local-socket = ["nu-plugin-core/local-socket"]
 
 [target.'cfg(target_family = "unix")'.dependencies]
 # For setting the process group ID (EnterForeground / LeaveForeground)
 nix = { workspace = true, default-features = false, features = ["process"] }
-
-[target.'cfg(target_os = "windows")'.dependencies]
-windows = { workspace = true, features = [
-  # For setting process creation flags
-  "Win32_System_Threading",
-] }

crates/nu-plugin/src/lib.rs

@@ -64,35 +64,16 @@
 //! [Plugin Example](https://github.com/nushell/nushell/tree/main/crates/nu_plugin_example)
 //! that demonstrates the full range of plugin capabilities.
 mod plugin;
-mod protocol;
-mod sequence;
-mod serializers;
 
-pub use plugin::{
+#[cfg(test)]
-    serve_plugin, EngineInterface, Plugin, PluginCommand, PluginEncoder, PluginRead, PluginWrite,
+mod test_util;
-    SimplePluginCommand,
-};
-pub use protocol::EvaluatedCall;
-pub use serializers::{json::JsonSerializer, msgpack::MsgPackSerializer};
 
-// Used by other nu crates.
+pub use plugin::{serve_plugin, EngineInterface, Plugin, PluginCommand, SimplePluginCommand};
-#[doc(hidden)]
-pub use plugin::{
-    add_plugin_to_working_set, create_plugin_signature, get_signature, load_plugin_file,
-    load_plugin_registry_item, serve_plugin_io, EngineInterfaceManager, GetPlugin, Interface,
-    InterfaceManager, PersistentPlugin, PluginDeclaration, PluginExecutionCommandContext,
-    PluginExecutionContext, PluginInterface, PluginInterfaceManager, PluginSource,
-    ServePluginError,
-};
-#[doc(hidden)]
-pub use protocol::{PluginCustomValue, PluginInput, PluginOutput};
-#[doc(hidden)]
-pub use serializers::EncodingType;
-#[doc(hidden)]
-pub mod util;
 
-// Used by external benchmarks.
+// Re-exports. Consider semver implications carefully.
+pub use nu_plugin_core::{JsonSerializer, MsgPackSerializer, PluginEncoder};
+pub use nu_plugin_protocol::EvaluatedCall;
+
+// Required by other internal crates.
 #[doc(hidden)]
-pub use plugin::Encoder;
+pub use plugin::{create_plugin_signature, serve_plugin_io};
-#[doc(hidden)]
-pub use protocol::PluginCallResponse;

crates/nu-plugin/src/plugin/interface/mod.rs

@@ -1,16 +1,14 @@
 //! Interface used by the plugin to communicate with the engine.
 
-use super::{
+use nu_plugin_core::{
-    stream::{StreamManager, StreamManagerHandle},
+    util::{Sequence, Waitable, WaitableMut},
-    Interface, InterfaceManager, PipelineDataWriter, PluginRead, PluginWrite, Sequence,
+    Interface, InterfaceManager, PipelineDataWriter, PluginRead, PluginWrite, StreamManager,
+    StreamManagerHandle,
 };
-use crate::{
+use nu_plugin_protocol::{
-    protocol::{
+    CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, Ordering, PluginCall,
-        CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, Ordering,
+    PluginCallId, PluginCallResponse, PluginCustomValue, PluginInput, PluginOption, PluginOutput,
-        PluginCall, PluginCallId, PluginCallResponse, PluginCustomValue, PluginInput, PluginOption,
+    ProtocolInfo,
-        PluginOutput, ProtocolInfo,
-    },
-    util::{Waitable, WaitableMut},
 };
 use nu_protocol::{
     engine::Closure, Config, IntoInterruptiblePipelineData, LabeledError, ListStream, PipelineData,

crates/nu-plugin/src/plugin/mod.rs

@@ -1,249 +1,36 @@
-use crate::{
-    plugin::interface::ReceivedPluginCall,
-    protocol::{CallInfo, CustomValueOp, PluginCustomValue, PluginInput, PluginOutput},
-    EncodingType,
-};
-
 use std::{
     cmp::Ordering,
     collections::HashMap,
     env,
     ffi::OsString,
-    io::{BufReader, BufWriter},
     ops::Deref,
     panic::AssertUnwindSafe,
-    path::Path,
+    sync::mpsc::{self, TrySendError},
-    process::{Child, Command as CommandSys},
-    sync::{
-        mpsc::{self, TrySendError},
-        Arc, Mutex,
-    },
     thread,
 };
 
 use nu_engine::documentation::get_flags_section;
+use nu_plugin_core::{
+    ClientCommunicationIo, CommunicationMode, InterfaceManager, PluginEncoder, PluginRead,
+    PluginWrite,
+};
+use nu_plugin_protocol::{CallInfo, CustomValueOp, PluginCustomValue, PluginInput, PluginOutput};
 use nu_protocol::{
-    ast::Operator, engine::StateWorkingSet, report_error_new, CustomValue, IntoSpanned,
+    ast::Operator, CustomValue, IntoSpanned, LabeledError, PipelineData, ShellError, Spanned, Value,
-    LabeledError, PipelineData, PluginIdentity, PluginRegistryFile, PluginRegistryItem,
-    PluginRegistryItemData, PluginSignature, RegisteredPlugin, ShellError, Span, Spanned, Value,
 };
 use thiserror::Error;
 
-#[cfg(unix)]
+use self::{command::render_examples, interface::ReceivedPluginCall};
-use std::os::unix::process::CommandExt;
-#[cfg(windows)]
-use std::os::windows::process::CommandExt;
-
-pub use self::interface::{PluginRead, PluginWrite};
-use self::{
-    command::render_examples,
-    communication_mode::{
-        ClientCommunicationIo, CommunicationMode, PreparedServerCommunication,
-        ServerCommunicationIo,
-    },
-    gc::PluginGc,
-};
 
 mod command;
-mod communication_mode;
-mod context;
-mod declaration;
-mod gc;
 mod interface;
-mod persistent;
-mod process;
-mod source;
 
 pub use command::{create_plugin_signature, PluginCommand, SimplePluginCommand};
-pub use declaration::PluginDeclaration;
+pub use interface::{EngineInterface, EngineInterfaceManager};
-pub use interface::{
-    EngineInterface, EngineInterfaceManager, Interface, InterfaceManager, PluginInterface,
-    PluginInterfaceManager,
-};
-pub use persistent::{GetPlugin, PersistentPlugin};
-
-pub use context::{PluginExecutionCommandContext, PluginExecutionContext};
-pub use source::PluginSource;
 
+#[allow(dead_code)]
 pub(crate) const OUTPUT_BUFFER_SIZE: usize = 8192;
 
-/// Encoder for a specific message type. Usually implemented on [`PluginInput`]
-/// and [`PluginOutput`].
-#[doc(hidden)]
-pub trait Encoder<T>: Clone + Send + Sync {
-    /// Serialize a value in the [`PluginEncoder`]s format
-    ///
-    /// Returns [`ShellError::IOError`] if there was a problem writing, or
-    /// [`ShellError::PluginFailedToEncode`] for a serialization error.
-    #[doc(hidden)]
-    fn encode(&self, data: &T, writer: &mut impl std::io::Write) -> Result<(), ShellError>;
-
-    /// Deserialize a value from the [`PluginEncoder`]'s format
-    ///
-    /// Returns `None` if there is no more output to receive.
-    ///
-    /// Returns [`ShellError::IOError`] if there was a problem reading, or
-    /// [`ShellError::PluginFailedToDecode`] for a deserialization error.
-    #[doc(hidden)]
-    fn decode(&self, reader: &mut impl std::io::BufRead) -> Result<Option<T>, ShellError>;
-}
-
-/// Encoding scheme that defines a plugin's communication protocol with Nu
-pub trait PluginEncoder: Encoder<PluginInput> + Encoder<PluginOutput> {
-    /// The name of the encoder (e.g., `json`)
-    fn name(&self) -> &str;
-}
-
-fn create_command(path: &Path, mut shell: Option<&Path>, mode: &CommunicationMode) -> CommandSys {
-    log::trace!("Starting plugin: {path:?}, shell = {shell:?}, mode = {mode:?}");
-
-    let mut shell_args = vec![];
-
-    if shell.is_none() {
-        // We only have to do this for things that are not executable by Rust's Command API on
-        // Windows. They do handle bat/cmd files for us, helpfully.
-        //
-        // Also include anything that wouldn't be executable with a shebang, like JAR files.
-        shell = match path.extension().and_then(|e| e.to_str()) {
-            Some("sh") => {
-                if cfg!(unix) {
-                    // We don't want to override what might be in the shebang if this is Unix, since
-                    // some scripts will have a shebang specifying bash even if they're .sh
-                    None
-                } else {
-                    Some(Path::new("sh"))
-                }
-            }
-            Some("nu") => {
-                shell_args.push("--stdin");
-                Some(Path::new("nu"))
-            }
-            Some("py") => Some(Path::new("python")),
-            Some("rb") => Some(Path::new("ruby")),
-            Some("jar") => {
-                shell_args.push("-jar");
-                Some(Path::new("java"))
-            }
-            _ => None,
-        };
-    }
-
-    let mut process = if let Some(shell) = shell {
-        let mut process = std::process::Command::new(shell);
-        process.args(shell_args);
-        process.arg(path);
-
-        process
-    } else {
-        std::process::Command::new(path)
-    };
-
-    process.args(mode.args());
-
-    // Setup I/O according to the communication mode
-    mode.setup_command_io(&mut process);
-
-    // The plugin should be run in a new process group to prevent Ctrl-C from stopping it
-    #[cfg(unix)]
-    process.process_group(0);
-    #[cfg(windows)]
-    process.creation_flags(windows::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP.0);
-
-    // In order to make bugs with improper use of filesystem without getting the engine current
-    // directory more obvious, the plugin always starts in the directory of its executable
-    if let Some(dirname) = path.parent() {
-        process.current_dir(dirname);
-    }
-
-    process
-}
-
-fn make_plugin_interface(
-    mut child: Child,
-    comm: PreparedServerCommunication,
-    source: Arc<PluginSource>,
-    pid: Option<u32>,
-    gc: Option<PluginGc>,
-) -> Result<PluginInterface, ShellError> {
-    match comm.connect(&mut child)? {
-        ServerCommunicationIo::Stdio(stdin, stdout) => make_plugin_interface_with_streams(
-            stdout,
-            stdin,
-            move || {
-                let _ = child.wait();
-            },
-            source,
-            pid,
-            gc,
-        ),
-        #[cfg(feature = "local-socket")]
-        ServerCommunicationIo::LocalSocket { read_out, write_in } => {
-            make_plugin_interface_with_streams(
-                read_out,
-                write_in,
-                move || {
-                    let _ = child.wait();
-                },
-                source,
-                pid,
-                gc,
-            )
-        }
-    }
-}
-
-fn make_plugin_interface_with_streams(
-    mut reader: impl std::io::Read + Send + 'static,
-    writer: impl std::io::Write + Send + 'static,
-    after_close: impl FnOnce() + Send + 'static,
-    source: Arc<PluginSource>,
-    pid: Option<u32>,
-    gc: Option<PluginGc>,
-) -> Result<PluginInterface, ShellError> {
-    let encoder = get_plugin_encoding(&mut reader)?;
-
-    let reader = BufReader::with_capacity(OUTPUT_BUFFER_SIZE, reader);
-    let writer = BufWriter::with_capacity(OUTPUT_BUFFER_SIZE, writer);
-
-    let mut manager =
-        PluginInterfaceManager::new(source.clone(), pid, (Mutex::new(writer), encoder));
-    manager.set_garbage_collector(gc);
-
-    let interface = manager.get_interface();
-    interface.hello()?;
-
-    // Spawn the reader on a new thread. We need to be able to read messages at the same time that
-    // we write, because we are expected to be able to handle multiple messages coming in from the
-    // plugin at any time, including stream messages like `Drop`.
-    std::thread::Builder::new()
-        .name(format!(
-            "plugin interface reader ({})",
-            source.identity.name()
-        ))
-        .spawn(move || {
-            if let Err(err) = manager.consume_all((reader, encoder)) {
-                log::warn!("Error in PluginInterfaceManager: {err}");
-            }
-            // If the loop has ended, drop the manager so everyone disconnects and then run
-            // after_close
-            drop(manager);
-            after_close();
-        })
-        .map_err(|err| ShellError::PluginFailedToLoad {
-            msg: format!("Failed to spawn thread for plugin: {err}"),
-        })?;
-
-    Ok(interface)
-}
-
-#[doc(hidden)] // Note: not for plugin authors / only used in nu-parser
-pub fn get_signature(
-    plugin: Arc<PersistentPlugin>,
-    envs: impl FnOnce() -> Result<HashMap<String, String>, ShellError>,
-) -> Result<Vec<PluginSignature>, ShellError> {
-    plugin.get(envs)?.get_signature()
-}
-
 /// The API for a Nushell plugin
 ///
 /// A plugin defines multiple commands, which are added to the engine when the user calls
@@ -499,6 +286,9 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
             read_in,
             mut write_out,
         }) => {
+            use std::io::{BufReader, BufWriter};
+            use std::sync::Mutex;
+
             tell_nushell_encoding(&mut write_out, &encoder)
                 .expect("failed to tell nushell encoding");
 
@@ -895,119 +685,3 @@ fn print_help(plugin: &impl Plugin, encoder: impl PluginEncoder) {
 
     println!("{help}")
 }
-
-pub fn get_plugin_encoding(
-    child_stdout: &mut impl std::io::Read,
-) -> Result<EncodingType, ShellError> {
-    let mut length_buf = [0u8; 1];
-    child_stdout
-        .read_exact(&mut length_buf)
-        .map_err(|e| ShellError::PluginFailedToLoad {
-            msg: format!("unable to get encoding from plugin: {e}"),
-        })?;
-
-    let mut buf = vec![0u8; length_buf[0] as usize];
-    child_stdout
-        .read_exact(&mut buf)
-        .map_err(|e| ShellError::PluginFailedToLoad {
-            msg: format!("unable to get encoding from plugin: {e}"),
-        })?;
-
-    EncodingType::try_from_bytes(&buf).ok_or_else(|| {
-        let encoding_for_debug = String::from_utf8_lossy(&buf);
-        ShellError::PluginFailedToLoad {
-            msg: format!("get unsupported plugin encoding: {encoding_for_debug}"),
-        }
-    })
-}
-
-/// Load the definitions from the plugin file into the engine state
-#[doc(hidden)]
-pub fn load_plugin_file(
-    working_set: &mut StateWorkingSet,
-    plugin_registry_file: &PluginRegistryFile,
-    span: Option<Span>,
-) {
-    for plugin in &plugin_registry_file.plugins {
-        // Any errors encountered should just be logged.
-        if let Err(err) = load_plugin_registry_item(working_set, plugin, span) {
-            report_error_new(working_set.permanent_state, &err)
-        }
-    }
-}
-
-/// Load a definition from the plugin file into the engine state
-#[doc(hidden)]
-pub fn load_plugin_registry_item(
-    working_set: &mut StateWorkingSet,
-    plugin: &PluginRegistryItem,
-    span: Option<Span>,
-) -> Result<Arc<PersistentPlugin>, ShellError> {
-    let identity =
-        PluginIdentity::new(plugin.filename.clone(), plugin.shell.clone()).map_err(|_| {
-            ShellError::GenericError {
-                error: "Invalid plugin filename in plugin registry file".into(),
-                msg: "loaded from here".into(),
-                span,
-                help: Some(format!(
-                    "the filename for `{}` is not a valid nushell plugin: {}",
-                    plugin.name,
-                    plugin.filename.display()
-                )),
-                inner: vec![],
-            }
-        })?;
-
-    match &plugin.data {
-        PluginRegistryItemData::Valid { commands } => {
-            let plugin = add_plugin_to_working_set(working_set, &identity)?;
-
-            // Ensure that the plugin is reset. We're going to load new signatures, so we want to
-            // make sure the running plugin reflects those new signatures, and it's possible that it
-            // doesn't.
-            plugin.reset()?;
-
-            // Create the declarations from the commands
-            for signature in commands {
-                let decl = PluginDeclaration::new(plugin.clone(), signature.clone());
-                working_set.add_decl(Box::new(decl));
-            }
-            Ok(plugin)
-        }
-        PluginRegistryItemData::Invalid => Err(ShellError::PluginRegistryDataInvalid {
-            plugin_name: identity.name().to_owned(),
-            span,
-            add_command: identity.add_command(),
-        }),
-    }
-}
-
-#[doc(hidden)]
-pub fn add_plugin_to_working_set(
-    working_set: &mut StateWorkingSet,
-    identity: &PluginIdentity,
-) -> Result<Arc<PersistentPlugin>, ShellError> {
-    // Find garbage collection config for the plugin
-    let gc_config = working_set
-        .get_config()
-        .plugin_gc
-        .get(identity.name())
-        .clone();
-
-    // Add it to / get it from the working set
-    let plugin = working_set.find_or_create_plugin(identity, || {
-        Arc::new(PersistentPlugin::new(identity.clone(), gc_config.clone()))
-    });
-
-    plugin.set_gc_config(&gc_config);
-
-    // Downcast the plugin to `PersistentPlugin` - we generally expect this to succeed.
-    // The trait object only exists so that nu-protocol can contain plugins without knowing
-    // anything about their implementation, but we only use `PersistentPlugin` in practice.
-    plugin
-        .as_any()
-        .downcast()
-        .map_err(|_| ShellError::NushellFailed {
-            msg: "encountered unexpected RegisteredPlugin type".into(),
-        })
-}

crates/nu-plugin/src/protocol/plugin_custom_value.rs

@@ -1,402 +0,0 @@
-use std::cmp::Ordering;
-use std::sync::Arc;
-
-use crate::{
-    plugin::{PluginInterface, PluginSource},
-    util::with_custom_values_in,
-};
-use nu_protocol::{ast::Operator, CustomValue, IntoSpanned, ShellError, Span, Spanned, Value};
-use nu_utils::SharedCow;
-
-use serde::{Deserialize, Serialize};
-
-#[cfg(test)]
-mod tests;
-
-/// An opaque container for a custom value that is handled fully by a plugin
-///
-/// This is the only type of custom value that is allowed to cross the plugin serialization
-/// boundary.
-///
-/// [`EngineInterface`](crate::interface::EngineInterface) is responsible for ensuring
-/// that local plugin custom values are converted to and from [`PluginCustomData`] on the boundary.
-///
-/// [`PluginInterface`](crate::interface::PluginInterface) is responsible for adding the
-/// appropriate [`PluginSource`](crate::plugin::PluginSource), ensuring that only
-/// [`PluginCustomData`] is contained within any values sent, and that the `source` of any
-/// values sent matches the plugin it is being sent to.
-///
-/// This is not a public API.
-#[derive(Clone, Debug, Serialize, Deserialize)]
-#[doc(hidden)]
-pub struct PluginCustomValue {
-    #[serde(flatten)]
-    shared: SharedCow<SharedContent>,
-
-    /// Which plugin the custom value came from. This is not defined on the plugin side. The engine
-    /// side is responsible for maintaining it, and it is not sent over the serialization boundary.
-    #[serde(skip, default)]
-    source: Option<Arc<PluginSource>>,
-}
-
-/// Content shared across copies of a plugin custom value.
-#[derive(Clone, Debug, Serialize, Deserialize)]
-struct SharedContent {
-    /// The name of the type of the custom value as defined by the plugin (`type_name()`)
-    name: String,
-    /// The bincoded representation of the custom value on the plugin side
-    data: Vec<u8>,
-    /// True if the custom value should notify the source if all copies of it are dropped.
-    ///
-    /// This is not serialized if `false`, since most custom values don't need it.
-    #[serde(default, skip_serializing_if = "is_false")]
-    notify_on_drop: bool,
-}
-
-fn is_false(b: &bool) -> bool {
-    !b
-}
-
-impl PluginCustomValue {
-    pub fn into_value(self, span: Span) -> Value {
-        Value::custom(Box::new(self), span)
-    }
-}
-
-#[typetag::serde]
-impl CustomValue for PluginCustomValue {
-    fn clone_value(&self, span: Span) -> Value {
-        self.clone().into_value(span)
-    }
-
-    fn type_name(&self) -> String {
-        self.name().to_owned()
-    }
-
-    fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
-        self.get_plugin(Some(span), "get base value")?
-            .custom_value_to_base_value(self.clone().into_spanned(span))
-    }
-
-    fn follow_path_int(
-        &self,
-        self_span: Span,
-        index: usize,
-        path_span: Span,
-    ) -> Result<Value, ShellError> {
-        self.get_plugin(Some(self_span), "follow cell path")?
-            .custom_value_follow_path_int(
-                self.clone().into_spanned(self_span),
-                index.into_spanned(path_span),
-            )
-    }
-
-    fn follow_path_string(
-        &self,
-        self_span: Span,
-        column_name: String,
-        path_span: Span,
-    ) -> Result<Value, ShellError> {
-        self.get_plugin(Some(self_span), "follow cell path")?
-            .custom_value_follow_path_string(
-                self.clone().into_spanned(self_span),
-                column_name.into_spanned(path_span),
-            )
-    }
-
-    fn partial_cmp(&self, other: &Value) -> Option<Ordering> {
-        self.get_plugin(Some(other.span()), "perform comparison")
-            .and_then(|plugin| {
-                // We're passing Span::unknown() here because we don't have one, and it probably
-                // shouldn't matter here and is just a consequence of the API
-                plugin.custom_value_partial_cmp(self.clone(), other.clone())
-            })
-            .unwrap_or_else(|err| {
-                // We can't do anything with the error other than log it.
-                log::warn!(
-                    "Error in partial_cmp on plugin custom value (source={source:?}): {err}",
-                    source = self.source
-                );
-                None
-            })
-            .map(|ordering| ordering.into())
-    }
-
-    fn operation(
-        &self,
-        lhs_span: Span,
-        operator: Operator,
-        op_span: Span,
-        right: &Value,
-    ) -> Result<Value, ShellError> {
-        self.get_plugin(Some(lhs_span), "invoke operator")?
-            .custom_value_operation(
-                self.clone().into_spanned(lhs_span),
-                operator.into_spanned(op_span),
-                right.clone(),
-            )
-    }
-
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
-    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
-        self
-    }
-}
-
-impl PluginCustomValue {
-    /// Create a new [`PluginCustomValue`].
-    pub(crate) fn new(
-        name: String,
-        data: Vec<u8>,
-        notify_on_drop: bool,
-        source: Option<Arc<PluginSource>>,
-    ) -> PluginCustomValue {
-        PluginCustomValue {
-            shared: SharedCow::new(SharedContent {
-                name,
-                data,
-                notify_on_drop,
-            }),
-            source,
-        }
-    }
-
-    /// The name of the type of the custom value as defined by the plugin (`type_name()`)
-    pub fn name(&self) -> &str {
-        &self.shared.name
-    }
-
-    /// The bincoded representation of the custom value on the plugin side
-    pub fn data(&self) -> &[u8] {
-        &self.shared.data
-    }
-
-    /// True if the custom value should notify the source if all copies of it are dropped.
-    pub fn notify_on_drop(&self) -> bool {
-        self.shared.notify_on_drop
-    }
-
-    /// Which plugin the custom value came from. This is not defined on the plugin side. The engine
-    /// side is responsible for maintaining it, and it is not sent over the serialization boundary.
-    pub fn source(&self) -> &Option<Arc<PluginSource>> {
-        &self.source
-    }
-
-    /// Set the [`PluginSource`] for this [`PluginCustomValue`].
-    pub fn set_source(&mut self, source: Option<Arc<PluginSource>>) {
-        self.source = source;
-    }
-
-    /// Create the [`PluginCustomValue`] with the given source.
-    #[cfg(test)]
-    pub(crate) fn with_source(mut self, source: Option<Arc<PluginSource>>) -> PluginCustomValue {
-        self.source = source;
-        self
-    }
-
-    /// Helper to get the plugin to implement an op
-    fn get_plugin(&self, span: Option<Span>, for_op: &str) -> Result<PluginInterface, ShellError> {
-        let wrap_err = |err: ShellError| ShellError::GenericError {
-            error: format!(
-                "Unable to spawn plugin `{}` to {for_op}",
-                self.source
-                    .as_ref()
-                    .map(|s| s.name())
-                    .unwrap_or("<unknown>")
-            ),
-            msg: err.to_string(),
-            span,
-            help: None,
-            inner: vec![err],
-        };
-
-        let source = self.source.clone().ok_or_else(|| {
-            wrap_err(ShellError::NushellFailed {
-                msg: "The plugin source for the custom value was not set".into(),
-            })
-        })?;
-
-        source
-            .persistent(span)
-            .and_then(|p| p.get_plugin(None))
-            .map_err(wrap_err)
-    }
-
-    /// Serialize a custom value into a [`PluginCustomValue`]. This should only be done on the
-    /// plugin side.
-    pub fn serialize_from_custom_value(
-        custom_value: &dyn CustomValue,
-        span: Span,
-    ) -> Result<PluginCustomValue, ShellError> {
-        let name = custom_value.type_name();
-        let notify_on_drop = custom_value.notify_plugin_on_drop();
-        bincode::serialize(custom_value)
-            .map(|data| PluginCustomValue::new(name, data, notify_on_drop, None))
-            .map_err(|err| ShellError::CustomValueFailedToEncode {
-                msg: err.to_string(),
-                span,
-            })
-    }
-
-    /// Deserialize a [`PluginCustomValue`] into a `Box<dyn CustomValue>`. This should only be done
-    /// on the plugin side.
-    pub fn deserialize_to_custom_value(
-        &self,
-        span: Span,
-    ) -> Result<Box<dyn CustomValue>, ShellError> {
-        bincode::deserialize::<Box<dyn CustomValue>>(self.data()).map_err(|err| {
-            ShellError::CustomValueFailedToDecode {
-                msg: err.to_string(),
-                span,
-            }
-        })
-    }
-
-    /// Add a [`PluginSource`] to the given [`CustomValue`] if it is a [`PluginCustomValue`].
-    pub fn add_source(value: &mut dyn CustomValue, source: &Arc<PluginSource>) {
-        if let Some(custom_value) = value.as_mut_any().downcast_mut::<PluginCustomValue>() {
-            custom_value.set_source(Some(source.clone()));
-        }
-    }
-
-    /// Add a [`PluginSource`] to all [`PluginCustomValue`]s within the value, recursively.
-    pub fn add_source_in(value: &mut Value, source: &Arc<PluginSource>) -> Result<(), ShellError> {
-        with_custom_values_in(value, |custom_value| {
-            Self::add_source(custom_value.item, source);
-            Ok::<_, ShellError>(())
-        })
-    }
-
-    /// Check that a [`CustomValue`] is a [`PluginCustomValue`] that come from the given `source`,
-    /// and return an error if not.
-    ///
-    /// This method will collapse `LazyRecord` in-place as necessary to make the guarantee,
-    /// since `LazyRecord` could return something different the next time it is called.
-    pub fn verify_source(
-        value: Spanned<&dyn CustomValue>,
-        source: &PluginSource,
-    ) -> Result<(), ShellError> {
-        if let Some(custom_value) = value.item.as_any().downcast_ref::<PluginCustomValue>() {
-            if custom_value
-                .source
-                .as_ref()
-                .map(|s| s.is_compatible(source))
-                .unwrap_or(false)
-            {
-                Ok(())
-            } else {
-                Err(ShellError::CustomValueIncorrectForPlugin {
-                    name: custom_value.name().to_owned(),
-                    span: value.span,
-                    dest_plugin: source.name().to_owned(),
-                    src_plugin: custom_value.source.as_ref().map(|s| s.name().to_owned()),
-                })
-            }
-        } else {
-            // Only PluginCustomValues can be sent
-            Err(ShellError::CustomValueIncorrectForPlugin {
-                name: value.item.type_name(),
-                span: value.span,
-                dest_plugin: source.name().to_owned(),
-                src_plugin: None,
-            })
-        }
-    }
-
-    /// Convert all plugin-native custom values to [`PluginCustomValue`] within the given `value`,
-    /// recursively. This should only be done on the plugin side.
-    pub fn serialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
-        value.recurse_mut(&mut |value| {
-            let span = value.span();
-            match value {
-                Value::Custom { ref val, .. } => {
-                    if val.as_any().downcast_ref::<PluginCustomValue>().is_some() {
-                        // Already a PluginCustomValue
-                        Ok(())
-                    } else {
-                        let serialized = Self::serialize_from_custom_value(&**val, span)?;
-                        *value = Value::custom(Box::new(serialized), span);
-                        Ok(())
-                    }
-                }
-                // Collect LazyRecord before proceeding
-                Value::LazyRecord { ref val, .. } => {
-                    *value = val.collect()?;
-                    Ok(())
-                }
-                _ => Ok(()),
-            }
-        })
-    }
-
-    /// Convert all [`PluginCustomValue`]s to plugin-native custom values within the given `value`,
-    /// recursively. This should only be done on the plugin side.
-    pub fn deserialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
-        value.recurse_mut(&mut |value| {
-            let span = value.span();
-            match value {
-                Value::Custom { ref val, .. } => {
-                    if let Some(val) = val.as_any().downcast_ref::<PluginCustomValue>() {
-                        let deserialized = val.deserialize_to_custom_value(span)?;
-                        *value = Value::custom(deserialized, span);
-                        Ok(())
-                    } else {
-                        // Already not a PluginCustomValue
-                        Ok(())
-                    }
-                }
-                // Collect LazyRecord before proceeding
-                Value::LazyRecord { ref val, .. } => {
-                    *value = val.collect()?;
-                    Ok(())
-                }
-                _ => Ok(()),
-            }
-        })
-    }
-
-    /// Render any custom values in the `Value` using `to_base_value()`
-    pub fn render_to_base_value_in(value: &mut Value) -> Result<(), ShellError> {
-        value.recurse_mut(&mut |value| {
-            let span = value.span();
-            match value {
-                Value::Custom { ref val, .. } => {
-                    *value = val.to_base_value(span)?;
-                    Ok(())
-                }
-                // Collect LazyRecord before proceeding
-                Value::LazyRecord { ref val, .. } => {
-                    *value = val.collect()?;
-                    Ok(())
-                }
-                _ => Ok(()),
-            }
-        })
-    }
-}
-
-impl Drop for PluginCustomValue {
-    fn drop(&mut self) {
-        // If the custom value specifies notify_on_drop and this is the last copy, we need to let
-        // the plugin know about it if we can.
-        if self.source.is_some() && self.notify_on_drop() && SharedCow::ref_count(&self.shared) == 1
-        {
-            self.get_plugin(None, "drop")
-                // While notifying drop, we don't need a copy of the source
-                .and_then(|plugin| {
-                    plugin.custom_value_dropped(PluginCustomValue {
-                        shared: self.shared.clone(),
-                        source: None,
-                    })
-                })
-                .unwrap_or_else(|err| {
-                    // We shouldn't do anything with the error except log it
-                    let name = self.name();
-                    log::warn!("Failed to notify drop of custom value ({name}): {err}")
-                });
-        }
-    }
-}

crates/nu-plugin/src/serializers/mod.rs

@@ -1,45 +0,0 @@
-use crate::plugin::Encoder;
-use nu_protocol::ShellError;
-
-pub mod json;
-pub mod msgpack;
-
-#[cfg(test)]
-mod tests;
-
-#[doc(hidden)]
-#[derive(Clone, Copy, Debug)]
-pub enum EncodingType {
-    Json(json::JsonSerializer),
-    MsgPack(msgpack::MsgPackSerializer),
-}
-
-impl EncodingType {
-    pub fn try_from_bytes(bytes: &[u8]) -> Option<Self> {
-        match bytes {
-            b"json" => Some(Self::Json(json::JsonSerializer {})),
-            b"msgpack" => Some(Self::MsgPack(msgpack::MsgPackSerializer {})),
-            _ => None,
-        }
-    }
-}
-
-impl<T> Encoder<T> for EncodingType
-where
-    json::JsonSerializer: Encoder<T>,
-    msgpack::MsgPackSerializer: Encoder<T>,
-{
-    fn encode(&self, data: &T, writer: &mut impl std::io::Write) -> Result<(), ShellError> {
-        match self {
-            EncodingType::Json(encoder) => encoder.encode(data, writer),
-            EncodingType::MsgPack(encoder) => encoder.encode(data, writer),
-        }
-    }
-
-    fn decode(&self, reader: &mut impl std::io::BufRead) -> Result<Option<T>, ShellError> {
-        match self {
-            EncodingType::Json(encoder) => encoder.decode(reader),
-            EncodingType::MsgPack(encoder) => encoder.decode(reader),
-        }
-    }
-}

crates/nu-plugin/src/test_util.rs

@@ -0,0 +1,15 @@
+use nu_plugin_core::interface_test_util::TestCase;
+use nu_plugin_protocol::{PluginInput, PluginOutput};
+
+use crate::plugin::EngineInterfaceManager;
+
+pub trait TestCaseExt {
+    /// Create a new [`EngineInterfaceManager`] that writes to this test case.
+    fn engine(&self) -> EngineInterfaceManager;
+}
+
+impl TestCaseExt for TestCase<PluginInput, PluginOutput> {
+    fn engine(&self) -> EngineInterfaceManager {
+        EngineInterfaceManager::new(self.clone())
+    }
+}

crates/nu-plugin/src/util/mod.rs

@@ -1,7 +0,0 @@
-mod mutable_cow;
-mod waitable;
-mod with_custom_values_in;
-
-pub(crate) use mutable_cow::*;
-pub use waitable::*;
-pub use with_custom_values_in::*;

crates/nu_plugin_stress_internals/Cargo.toml

@@ -16,4 +16,4 @@ bench = false
 # assumptions about the serialized format
 serde = { workspace = true }
 serde_json = { workspace = true }
-interprocess = "1.2.1"
+interprocess = { workspace = true }

src/main.rs

@@ -391,7 +391,7 @@ fn main() -> Result<()> {
 
     #[cfg(feature = "plugin")]
     if let Some(plugins) = &parsed_nu_cli_args.plugins {
-        use nu_plugin::{GetPlugin, PluginDeclaration};
+        use nu_plugin_engine::{GetPlugin, PluginDeclaration};
         use nu_protocol::{engine::StateWorkingSet, ErrSpan, PluginIdentity};
 
         // Load any plugins specified with --plugins
@@ -409,7 +409,7 @@ fn main() -> Result<()> {
                 .map_err(ShellError::from)?;
 
             // Create the plugin and add it to the working set
-            let plugin = nu_plugin::add_plugin_to_working_set(&mut working_set, &identity)?;
+            let plugin = nu_plugin_engine::add_plugin_to_working_set(&mut working_set, &identity)?;
 
             // Spawn the plugin to get its signatures, and then add the commands to the working set
             for signature in plugin.clone().get_plugin(None)?.get_signature()? {