holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add test support crate for plugin developers (#12259)

Browse Source 
# Description

Adds a `nu-plugin-test-support` crate with an interface that supports
testing plugins.

Unlike in reality, these plugins run in the same process on separate
threads. This will allow
testing aspects of the plugin internal state and handling serialized
plugin custom values easily.
We still serialize their custom values and all of the engine to plugin
logic is still in play, so
from a logical perspective this should still expose any bugs that would
have been caused by that.
The only difference is that it doesn't run in a different process, and
doesn't try to serialize
everything to the final wire format for stdin/stdout.

TODO still:

- [x] Clean up warnings about private types exposed in trait definition
- [x] Automatically deserialize plugin custom values in the result so
they can be inspected
- [x] Automatic plugin examples test function
- [x] Write a bit more documentation
- [x] More tests
- [x] Add MIT License file to new crate

# User-Facing Changes

Plugin developers get a nice way to test their plugins.

# Tests + Formatting
Run the tests with `cargo test -p nu-plugin-test-support --
--show-output` to see some examples of what the failing test output for
examples can look like. I used the `difference` crate (MIT licensed) to
make it look nice.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting

- [ ] Add a section to the book about testing
- [ ] Test some of the example plugins this way
- [ ] Add example tests to nu_plugin_template so plugin developers have
something to start with
This commit is contained in:
Devyn Cairns
2024-03-23 11:29:54 -07:00
committed by GitHub
parent ff41cf91ef
commit c79c43d2f8
31 changed files with 1171 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
crates/nu-plugin/src/lib.rs
View File

@ -62,14 +62,21 @@ mod serializers;
mod util;
pub use plugin::{
serve_plugin, EngineInterface, Plugin, PluginCommand, PluginEncoder, SimplePluginCommand,
serve_plugin, EngineInterface, Plugin, PluginCommand, PluginEncoder, PluginRead, PluginWrite,
SimplePluginCommand,
};
pub use protocol::EvaluatedCall;
pub use serializers::{json::JsonSerializer, msgpack::MsgPackSerializer};
// Used by other nu crates.
#[doc(hidden)]
pub use plugin::{get_signature, PersistentPlugin, PluginDeclaration};
pub use plugin::{
get_signature, serve_plugin_io, EngineInterfaceManager, GetPlugin, Interface, InterfaceManager,
PersistentPlugin, PluginDeclaration, PluginExecutionCommandContext, PluginExecutionContext,
PluginInterface, PluginInterfaceManager, PluginSource, ServePluginError,
};
#[doc(hidden)]
pub use protocol::{PluginCustomValue, PluginInput, PluginOutput};
#[doc(hidden)]
pub use serializers::EncodingType;
@ -77,4 +84,4 @@ pub use serializers::EncodingType;
#[doc(hidden)]
pub use plugin::Encoder;
#[doc(hidden)]
pub use protocol::{PluginCallResponse, PluginOutput};
pub use protocol::PluginCallResponse;

10
crates/nu-plugin/src/plugin/context.rs
View File

@ -14,7 +14,10 @@ use nu_protocol::{
use crate::util::MutableCow;
/// Object safe trait for abstracting operations required of the plugin context.
pub(crate) trait PluginExecutionContext: Send + Sync {
///
/// This is not a public API.
#[doc(hidden)]
pub trait PluginExecutionContext: Send + Sync {
/// The interrupt signal, if present
fn ctrlc(&self) -> Option<&Arc<AtomicBool>>;
/// Get engine configuration
@ -43,7 +46,10 @@ pub(crate) trait PluginExecutionContext: Send + Sync {
}
/// The execution context of a plugin command. Can be borrowed.
pub(crate) struct PluginExecutionCommandContext<'a> {
///
/// This is not a public API.
#[doc(hidden)]
pub struct PluginExecutionCommandContext<'a> {
identity: Arc<PluginIdentity>,
engine_state: Cow<'a, EngineState>,
stack: MutableCow<'a, Stack>,

14
crates/nu-plugin/src/plugin/declaration.rs
View File

@ -1,4 +1,4 @@
use super::{PersistentPlugin, PluginExecutionCommandContext, PluginSource};
use super::{GetPlugin, PluginExecutionCommandContext, PluginSource};
use crate::protocol::{CallInfo, EvaluatedCall};
use std::sync::Arc;
@ -6,7 +6,7 @@ use nu_engine::get_eval_expression;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{ast::Call, PluginSignature, Signature};
use nu_protocol::{Example, PipelineData, PluginIdentity, RegisteredPlugin, ShellError};
use nu_protocol::{Example, PipelineData, PluginIdentity, ShellError};
#[doc(hidden)] // Note: not for plugin authors / only used in nu-parser
#[derive(Clone)]
@ -17,7 +17,7 @@ pub struct PluginDeclaration {
}
impl PluginDeclaration {
pub fn new(plugin: &Arc<PersistentPlugin>, signature: PluginSignature) -> Self {
pub fn new(plugin: Arc<dyn GetPlugin>, signature: PluginSignature) -> Self {
Self {
name: signature.sig.name.clone(),
signature,
@ -88,13 +88,7 @@ impl Command for PluginDeclaration {
.and_then(|p| {
// Set the garbage collector config from the local config before running
p.set_gc_config(engine_config.plugin_gc.get(p.identity().name()));
p.get(|| {
// We need the current environment variables for `python` based plugins. Or
// we'll likely have a problem when a plugin is implemented in a virtual Python
// environment.
let stack = &mut stack.start_capture();
nu_engine::env::env_to_strings(engine_state, stack)
})
p.get_plugin(Some((engine_state, stack)))
})
.map_err(|err| {
let decl = engine_state.get_decl(call.decl_id);

27
crates/nu-plugin/src/plugin/interface.rs
View File

@ -22,11 +22,10 @@ use crate::{
mod stream;
mod engine;
pub use engine::EngineInterface;
pub(crate) use engine::{EngineInterfaceManager, ReceivedPluginCall};
pub use engine::{EngineInterface, EngineInterfaceManager, ReceivedPluginCall};
mod plugin;
pub(crate) use plugin::{PluginInterface, PluginInterfaceManager};
pub use plugin::{PluginInterface, PluginInterfaceManager};
use self::stream::{StreamManager, StreamManagerHandle, StreamWriter, WriteStreamMessage};
@ -45,7 +44,10 @@ const LIST_STREAM_HIGH_PRESSURE: i32 = 100;
const RAW_STREAM_HIGH_PRESSURE: i32 = 50;
/// Read input/output from the stream.
pub(crate) trait PluginRead<T> {
///
/// This is not a public API.
#[doc(hidden)]
pub trait PluginRead<T> {
/// Returns `Ok(None)` on end of stream.
fn read(&mut self) -> Result<Option<T>, ShellError>;
}
@ -72,7 +74,10 @@ where
/// Write input/output to the stream.
///
/// The write should be atomic, without interference from other threads.
pub(crate) trait PluginWrite<T>: Send + Sync {
///
/// This is not a public API.
#[doc(hidden)]
pub trait PluginWrite<T>: Send + Sync {
fn write(&self, data: &T) -> Result<(), ShellError>;
/// Flush any internal buffers, if applicable.
@ -136,7 +141,10 @@ where
///
/// There is typically one [`InterfaceManager`] consuming input from a background thread, and
/// managing shared state.
pub(crate) trait InterfaceManager {
///
/// This is not a public API.
#[doc(hidden)]
pub trait InterfaceManager {
/// The corresponding interface type.
type Interface: Interface + 'static;
@ -218,7 +226,10 @@ pub(crate) trait InterfaceManager {
/// [`EngineInterface`] for the API from the plugin side to the engine.
///
/// There can be multiple copies of the interface managed by a single [`InterfaceManager`].
pub(crate) trait Interface: Clone + Send {
///
/// This is not a public API.
#[doc(hidden)]
pub trait Interface: Clone + Send {
/// The output message type, which must be capable of encapsulating a [`StreamMessage`].
type Output: From<StreamMessage>;
@ -338,7 +349,7 @@ where
/// [`PipelineDataWriter::write()`] to write all of the data contained within the streams.
#[derive(Default)]
#[must_use]
pub(crate) enum PipelineDataWriter<W: WriteStreamMessage> {
pub enum PipelineDataWriter<W: WriteStreamMessage> {
#[default]
None,
ListStream(StreamWriter<W>, ListStream),

10
crates/nu-plugin/src/plugin/interface/engine.rs
View File

@ -30,8 +30,11 @@ use crate::sequence::Sequence;
/// With each call, an [`EngineInterface`] is included that can be provided to the plugin code
/// and should be used to send the response. The interface sent includes the [`PluginCallId`] for
/// sending associated messages with the correct context.
///
/// This is not a public API.
#[derive(Debug)]
pub(crate) enum ReceivedPluginCall {
#[doc(hidden)]
pub enum ReceivedPluginCall {
Signature {
engine: EngineInterface,
},
@ -76,8 +79,11 @@ impl std::fmt::Debug for EngineInterfaceState {
}
/// Manages reading and dispatching messages for [`EngineInterface`]s.
///
/// This is not a public API.
#[derive(Debug)]
pub(crate) struct EngineInterfaceManager {
#[doc(hidden)]
pub struct EngineInterfaceManager {
/// Shared state
state: Arc<EngineInterfaceState>,
/// Channel to send received PluginCalls to. This is removed after `Goodbye` is received.

40
crates/nu-plugin/src/plugin/interface/plugin.rs
View File

@ -104,8 +104,11 @@ struct PluginCallState {
}
/// Manages reading and dispatching messages for [`PluginInterface`]s.
///
/// This is not a public API.
#[derive(Debug)]
pub(crate) struct PluginInterfaceManager {
#[doc(hidden)]
pub struct PluginInterfaceManager {
/// Shared state
state: Arc<PluginInterfaceState>,
/// Manages stream messages and state
@ -125,7 +128,7 @@ pub(crate) struct PluginInterfaceManager {
}
impl PluginInterfaceManager {
pub(crate) fn new(
pub fn new(
source: Arc<PluginSource>,
writer: impl PluginWrite<PluginInput> + 'static,
) -> PluginInterfaceManager {
@ -152,7 +155,7 @@ impl PluginInterfaceManager {
/// Add a garbage collector to this plugin. The manager will notify the garbage collector about
/// the state of the plugin so that it can be automatically cleaned up if the plugin is
/// inactive.
pub(crate) fn set_garbage_collector(&mut self, gc: Option<PluginGc>) {
pub fn set_garbage_collector(&mut self, gc: Option<PluginGc>) {
self.gc = gc;
}
@ -359,14 +362,14 @@ impl PluginInterfaceManager {
/// True if there are no other copies of the state (which would mean there are no interfaces
/// and no stream readers/writers)
pub(crate) fn is_finished(&self) -> bool {
pub fn is_finished(&self) -> bool {
Arc::strong_count(&self.state) < 2
}
/// Loop on input from the given reader as long as `is_finished()` is false
///
/// Any errors will be propagated to all read streams automatically.
pub(crate) fn consume_all(
pub fn consume_all(
&mut self,
mut reader: impl PluginRead<PluginOutput>,
) -> Result<(), ShellError> {
@ -545,8 +548,11 @@ impl InterfaceManager for PluginInterfaceManager {
}
/// A reference through which a plugin can be interacted with during execution.
///
/// This is not a public API.
#[derive(Debug, Clone)]
pub(crate) struct PluginInterface {
#[doc(hidden)]
pub struct PluginInterface {
/// Shared state
state: Arc<PluginInterfaceState>,
/// Handle to stream manager
@ -557,7 +563,7 @@ pub(crate) struct PluginInterface {
impl PluginInterface {
/// Write the protocol info. This should be done after initialization
pub(crate) fn hello(&self) -> Result<(), ShellError> {
pub fn hello(&self) -> Result<(), ShellError> {
self.write(PluginInput::Hello(ProtocolInfo::default()))?;
self.flush()
}
@ -567,14 +573,14 @@ impl PluginInterface {
///
/// Note that this is automatically called when the last existing `PluginInterface` is dropped.
/// You probably do not need to call this manually.
pub(crate) fn goodbye(&self) -> Result<(), ShellError> {
pub fn goodbye(&self) -> Result<(), ShellError> {
self.write(PluginInput::Goodbye)?;
self.flush()
}
/// Write an [`EngineCallResponse`]. Writes the full stream contained in any [`PipelineData`]
/// before returning.
pub(crate) fn write_engine_call_response(
pub fn write_engine_call_response(
&self,
id: EngineCallId,
response: EngineCallResponse<PipelineData>,
@ -782,7 +788,7 @@ impl PluginInterface {
}
/// Get the command signatures from the plugin.
pub(crate) fn get_signature(&self) -> Result<Vec<PluginSignature>, ShellError> {
pub fn get_signature(&self) -> Result<Vec<PluginSignature>, ShellError> {
match self.plugin_call(PluginCall::Signature, None)? {
PluginCallResponse::Signature(sigs) => Ok(sigs),
PluginCallResponse::Error(err) => Err(err.into()),
@ -793,7 +799,7 @@ impl PluginInterface {
}
/// Run the plugin with the given call and execution context.
pub(crate) fn run(
pub fn run(
&self,
call: CallInfo<PipelineData>,
context: &mut dyn PluginExecutionContext,
@ -826,7 +832,7 @@ impl PluginInterface {
}
/// Collapse a custom value to its base value.
pub(crate) fn custom_value_to_base_value(
pub fn custom_value_to_base_value(
&self,
value: Spanned<PluginCustomValue>,
) -> Result<Value, ShellError> {
@ -834,7 +840,7 @@ impl PluginInterface {
}
/// Follow a numbered cell path on a custom value - e.g. `value.0`.
pub(crate) fn custom_value_follow_path_int(
pub fn custom_value_follow_path_int(
&self,
value: Spanned<PluginCustomValue>,
index: Spanned<usize>,
@ -843,7 +849,7 @@ impl PluginInterface {
}
/// Follow a named cell path on a custom value - e.g. `value.column`.
pub(crate) fn custom_value_follow_path_string(
pub fn custom_value_follow_path_string(
&self,
value: Spanned<PluginCustomValue>,
column_name: Spanned<String>,
@ -852,7 +858,7 @@ impl PluginInterface {
}
/// Invoke comparison logic for custom values.
pub(crate) fn custom_value_partial_cmp(
pub fn custom_value_partial_cmp(
&self,
value: PluginCustomValue,
mut other_value: Value,
@ -874,7 +880,7 @@ impl PluginInterface {
}
/// Invoke functionality for an operator on a custom value.
pub(crate) fn custom_value_operation(
pub fn custom_value_operation(
&self,
left: Spanned<PluginCustomValue>,
operator: Spanned<Operator>,
@ -885,7 +891,7 @@ impl PluginInterface {
}
/// Notify the plugin about a dropped custom value.
pub(crate) fn custom_value_dropped(&self, value: PluginCustomValue) -> Result<(), ShellError> {
pub fn custom_value_dropped(&self, value: PluginCustomValue) -> Result<(), ShellError> {
// Note: the protocol is always designed to have a span with the custom value, but this
// operation doesn't support one.
self.custom_value_op_expecting_value(

20
crates/nu-plugin/src/plugin/interface/stream.rs
View File

@ -170,7 +170,7 @@ impl<T> FromShellError for Result<T, ShellError> {
///
/// The `signal` contained
#[derive(Debug)]
pub(crate) struct StreamWriter<W: WriteStreamMessage> {
pub struct StreamWriter<W: WriteStreamMessage> {
id: StreamId,
signal: Arc<StreamWriterSignal>,
writer: W,
@ -308,7 +308,7 @@ impl StreamWriterSignal {
/// If `notify_sent()` is called more than `high_pressure_mark` times, it will wait until
/// `notify_acknowledge()` is called by another thread enough times to bring the number of
/// unacknowledged sent messages below that threshold.
pub fn new(high_pressure_mark: i32) -> StreamWriterSignal {
pub(crate) fn new(high_pressure_mark: i32) -> StreamWriterSignal {
assert!(high_pressure_mark > 0);
StreamWriterSignal {
@ -329,12 +329,12 @@ impl StreamWriterSignal {
/// True if the stream was dropped and the consumer is no longer interested in it. Indicates
/// that no more messages should be sent, other than `End`.
pub fn is_dropped(&self) -> Result<bool, ShellError> {
pub(crate) fn is_dropped(&self) -> Result<bool, ShellError> {
Ok(self.lock()?.dropped)
}
/// Notify the writers that the stream has been dropped, so they can stop writing.
pub fn set_dropped(&self) -> Result<(), ShellError> {
pub(crate) fn set_dropped(&self) -> Result<(), ShellError> {
let mut state = self.lock()?;
state.dropped = true;
// Unblock the writers so they can terminate
@ -345,7 +345,7 @@ impl StreamWriterSignal {
/// Track that a message has been sent. Returns `Ok(true)` if more messages can be sent,
/// or `Ok(false)` if the high pressure mark has been reached and [`.wait_for_drain()`] should
/// be called to block.
pub fn notify_sent(&self) -> Result<bool, ShellError> {
pub(crate) fn notify_sent(&self) -> Result<bool, ShellError> {
let mut state = self.lock()?;
state.unacknowledged =
state
@ -359,7 +359,7 @@ impl StreamWriterSignal {
}
/// Wait for acknowledgements before sending more data. Also returns if the stream is dropped.
pub fn wait_for_drain(&self) -> Result<(), ShellError> {
pub(crate) fn wait_for_drain(&self) -> Result<(), ShellError> {
let mut state = self.lock()?;
while !state.dropped && state.unacknowledged >= state.high_pressure_mark {
state = self
@ -374,7 +374,7 @@ impl StreamWriterSignal {
/// Notify the writers that a message has been acknowledged, so they can continue to write
/// if they were waiting.
pub fn notify_acknowledged(&self) -> Result<(), ShellError> {
pub(crate) fn notify_acknowledged(&self) -> Result<(), ShellError> {
let mut state = self.lock()?;
state.unacknowledged =
state
@ -390,7 +390,7 @@ impl StreamWriterSignal {
}
/// A sink for a [`StreamMessage`]
pub(crate) trait WriteStreamMessage {
pub trait WriteStreamMessage {
fn write_stream_message(&mut self, msg: StreamMessage) -> Result<(), ShellError>;
fn flush(&mut self) -> Result<(), ShellError>;
}
@ -413,7 +413,7 @@ impl StreamManagerState {
}
#[derive(Debug)]
pub(crate) struct StreamManager {
pub struct StreamManager {
state: Arc<Mutex<StreamManagerState>>,
}
@ -532,7 +532,7 @@ impl Drop for StreamManager {
/// Streams can be registered for reading, returning a [`StreamReader`], or for writing, returning
/// a [`StreamWriter`].
#[derive(Debug, Clone)]
pub(crate) struct StreamManagerHandle {
pub struct StreamManagerHandle {
state: Weak<Mutex<StreamManagerState>>,
}

243
crates/nu-plugin/src/plugin/mod.rs
View File

@ -1,5 +1,6 @@
use nu_engine::documentation::get_flags_section;
use nu_protocol::LabeledError;
use thiserror::Error;
use std::cmp::Ordering;
use std::collections::HashMap;
@ -13,7 +14,7 @@ use std::{env, thread};
use std::sync::mpsc::TrySendError;
use std::sync::{mpsc, Arc, Mutex};
use crate::plugin::interface::{EngineInterfaceManager, ReceivedPluginCall};
use crate::plugin::interface::ReceivedPluginCall;
use crate::protocol::{CallInfo, CustomValueOp, PluginCustomValue, PluginInput, PluginOutput};
use crate::EncodingType;
@ -29,6 +30,7 @@ use nu_protocol::{
};
use self::gc::PluginGc;
pub use self::interface::{PluginRead, PluginWrite};
mod command;
mod context;
@ -40,14 +42,14 @@ mod source;
pub use command::{PluginCommand, SimplePluginCommand};
pub use declaration::PluginDeclaration;
pub use interface::EngineInterface;
pub use persistent::PersistentPlugin;
pub use interface::{
EngineInterface, EngineInterfaceManager, Interface, InterfaceManager, PluginInterface,
PluginInterfaceManager,
};
pub use persistent::{GetPlugin, PersistentPlugin};
pub(crate) use context::PluginExecutionCommandContext;
pub(crate) use interface::PluginInterface;
pub(crate) use source::PluginSource;
use interface::{InterfaceManager, PluginInterfaceManager};
pub use context::{PluginExecutionCommandContext, PluginExecutionContext};
pub use source::PluginSource;
pub(crate) const OUTPUT_BUFFER_SIZE: usize = 8192;
@ -440,19 +442,6 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
std::process::exit(1)
}
// Build commands map, to make running a command easier
let mut commands: HashMap<String, _> = HashMap::new();
for command in plugin.commands() {
if let Some(previous) = commands.insert(command.signature().sig.name.clone(), command) {
eprintln!(
"Plugin `{plugin_name}` warning: command `{}` shadowed by another command with the \
same name. Check your command signatures",
previous.signature().sig.name
);
}
}
// tell nushell encoding.
//
// 1 byte
@ -471,7 +460,114 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
.expect("Failed to tell nushell my encoding when flushing stdout");
}
let mut manager = EngineInterfaceManager::new((stdout, encoder.clone()));
let encoder_clone = encoder.clone();
let result = serve_plugin_io(
plugin,
&plugin_name,
move || (std::io::stdin().lock(), encoder_clone),
move || (std::io::stdout(), encoder),
);
match result {
Ok(()) => (),
// Write unreported errors to the console
Err(ServePluginError::UnreportedError(err)) => {
eprintln!("Plugin `{plugin_name}` error: {err}");
std::process::exit(1);
}
Err(_) => std::process::exit(1),
}
}
/// An error from [`serve_plugin_io()`]
#[derive(Debug, Error)]
pub enum ServePluginError {
/// An error occurred that could not be reported to the engine.
#[error("{0}")]
UnreportedError(#[source] ShellError),
/// An error occurred that could be reported to the engine.
#[error("{0}")]
ReportedError(#[source] ShellError),
/// A version mismatch occurred.
#[error("{0}")]
Incompatible(#[source] ShellError),
/// An I/O error occurred.
#[error("{0}")]
IOError(#[source] ShellError),
/// A thread spawning error occurred.
#[error("{0}")]
ThreadSpawnError(#[source] std::io::Error),
}
impl From<ShellError> for ServePluginError {
fn from(error: ShellError) -> Self {
match error {
ShellError::IOError { .. } => ServePluginError::IOError(error),
ShellError::PluginFailedToLoad { .. } => ServePluginError::Incompatible(error),
_ => ServePluginError::UnreportedError(error),
}
}
}
/// Convert result error to ReportedError if it can be reported to the engine.
trait TryToReport {
type T;
fn try_to_report(self, engine: &EngineInterface) -> Result<Self::T, ServePluginError>;
}
impl<T, E> TryToReport for Result<T, E>
where
E: Into<ServePluginError>,
{
type T = T;
fn try_to_report(self, engine: &EngineInterface) -> Result<T, ServePluginError> {
self.map_err(|e| match e.into() {
ServePluginError::UnreportedError(err) => {
if engine.write_response(Err(err.clone())).is_ok() {
ServePluginError::ReportedError(err)
} else {
ServePluginError::UnreportedError(err)
}
}
other => other,
})
}
}
/// Serve a plugin on the given input & output.
///
/// Unlike [`serve_plugin`], this doesn't assume total control over the process lifecycle / stdin /
/// stdout, and can be used for more advanced use cases.
///
/// This is not a public API.
#[doc(hidden)]
pub fn serve_plugin_io<I, O>(
plugin: &impl Plugin,
plugin_name: &str,
input: impl FnOnce() -> I + Send + 'static,
output: impl FnOnce() -> O + Send + 'static,
) -> Result<(), ServePluginError>
where
I: PluginRead<PluginInput> + 'static,
O: PluginWrite<PluginOutput> + 'static,
{
let (error_tx, error_rx) = mpsc::channel();
// Build commands map, to make running a command easier
let mut commands: HashMap<String, _> = HashMap::new();
for command in plugin.commands() {
if let Some(previous) = commands.insert(command.signature().sig.name.clone(), command) {
eprintln!(
"Plugin `{plugin_name}` warning: command `{}` shadowed by another command with the \
same name. Check your command signatures",
previous.signature().sig.name
);
}
}
let mut manager = EngineInterfaceManager::new(output());
let call_receiver = manager
.take_plugin_call_receiver()
// This expect should be totally safe, as we just created the manager
@ -480,54 +576,22 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
// We need to hold on to the interface to keep the manager alive. We can drop it at the end
let interface = manager.get_interface();
// Try an operation that could result in ShellError. Exit if an I/O error is encountered.
// Try to report the error to nushell otherwise, and failing that, panic.
macro_rules! try_or_report {
($interface:expr, $expr:expr) => (match $expr {
Ok(val) => val,
// Just exit if there is an I/O error. Most likely this just means that nushell
// interrupted us. If not, the error probably happened on the other side too, so we
// don't need to also report it.
Err(ShellError::IOError { .. }) => std::process::exit(1),
// If there is another error, try to send it to nushell and then exit.
Err(err) => {
let _ = $interface.write_response(Err(err.clone())).unwrap_or_else(|_| {
// If we can't send it to nushell, panic with it so at least we get the output
panic!("Plugin `{plugin_name}`: {}", err)
});
std::process::exit(1)
}
})
}
// Send Hello message
try_or_report!(interface, interface.hello());
interface.hello()?;
let plugin_name_clone = plugin_name.clone();
// Spawn the reader thread
std::thread::Builder::new()
.name("engine interface reader".into())
.spawn(move || {
if let Err(err) = manager.consume_all((std::io::stdin().lock(), encoder)) {
// Do our best to report the read error. Most likely there is some kind of
// incompatibility between the plugin and nushell, so it makes more sense to try to
// report it on stderr than to send something.
//
// Don't report a `PluginFailedToLoad` error, as it's probably just from Hello
// version mismatch which the engine side would also report.
if !matches!(err, ShellError::PluginFailedToLoad { .. }) {
eprintln!("Plugin `{plugin_name_clone}` read error: {err}");
{
// Spawn the reader thread
let error_tx = error_tx.clone();
std::thread::Builder::new()
.name("engine interface reader".into())
.spawn(move || {
// Report the error on the channel if we get an error
if let Err(err) = manager.consume_all(input()) {
let _ = error_tx.send(ServePluginError::from(err));
}
std::process::exit(1);
}
})
.unwrap_or_else(|err| {
// If we fail to spawn the reader thread, we should exit
eprintln!("Plugin `{plugin_name}` failed to launch: {err}");
std::process::exit(1);
});
})
.map_err(ServePluginError::ThreadSpawnError)?;
}
// Handle each Run plugin call on a thread
thread::scope(|scope| {
@ -545,8 +609,11 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
};
let write_result = engine
.write_response(result)
.and_then(|writer| writer.write());
try_or_report!(engine, write_result);
.and_then(|writer| writer.write())
.try_to_report(&engine);
if let Err(err) = write_result {
let _ = error_tx.send(err);
}
};
// As an optimization: create one thread that can be reused for Run calls in sequence
@ -558,13 +625,14 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
run(engine, call);
}
})
.unwrap_or_else(|err| {
// If we fail to spawn the runner thread, we should exit
eprintln!("Plugin `{plugin_name}` failed to launch: {err}");
std::process::exit(1);
});
.map_err(ServePluginError::ThreadSpawnError)?;
for plugin_call in call_receiver {
// Check for pending errors
if let Ok(error) = error_rx.try_recv() {
return Err(error);
}
match plugin_call {
// Sending the signature back to nushell to create the declaration definition
ReceivedPluginCall::Signature { engine } => {
@ -572,7 +640,7 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
.values()
.map(|command| command.signature())
.collect();
try_or_report!(engine, engine.write_signature(sigs));
engine.write_signature(sigs).try_to_report(&engine)?;
}
// Run the plugin on a background thread, handling any input or output streams
ReceivedPluginCall::Run { engine, call } => {
@ -582,14 +650,10 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
// If the primary thread isn't ready, spawn a secondary thread to do it
Err(TrySendError::Full((engine, call)))
| Err(TrySendError::Disconnected((engine, call))) => {
let engine_clone = engine.clone();
try_or_report!(
engine_clone,
thread::Builder::new()
.name("plugin runner (secondary)".into())
.spawn_scoped(scope, move || run(engine, call))
.map_err(ShellError::from)
);
thread::Builder::new()
.name("plugin runner (secondary)".into())
.spawn_scoped(scope, move || run(engine, call))
.map_err(ServePluginError::ThreadSpawnError)?;
}
}
}
@ -599,14 +663,23 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static)
custom_value,
op,
} => {
try_or_report!(engine, custom_value_op(plugin, &engine, custom_value, op));
custom_value_op(plugin, &engine, custom_value, op).try_to_report(&engine)?;
}
}
}
});
Ok::<_, ServePluginError>(())
})?;
// This will stop the manager
drop(interface);
// Receive any error left on the channel
if let Ok(err) = error_rx.try_recv() {
Err(err)
} else {
Ok(())
}
}
fn custom_value_op(

45
crates/nu-plugin/src/plugin/persistent.rs
View File

@ -3,7 +3,10 @@ use std::{
sync::{Arc, Mutex},
};
use nu_protocol::{PluginGcConfig, PluginIdentity, RegisteredPlugin, ShellError};
use nu_protocol::{
engine::{EngineState, Stack},
PluginGcConfig, PluginIdentity, RegisteredPlugin, ShellError,
};
use super::{create_command, gc::PluginGc, make_plugin_interface, PluginInterface, PluginSource};
@ -81,6 +84,9 @@ impl PersistentPlugin {
//
// We hold the lock the whole time to prevent others from trying to spawn and ending
// up with duplicate plugins
//
// TODO: We should probably store the envs somewhere, in case we have to launch without
// envs (e.g. from a custom value)
let new_running = self.clone().spawn(envs()?, &mutable.gc_config)?;
let interface = new_running.interface.clone();
mutable.running = Some(new_running);
@ -126,7 +132,7 @@ impl PersistentPlugin {
let pid = child.id();
let interface =
make_plugin_interface(child, Arc::new(PluginSource::new(&self)), Some(gc.clone()))?;
make_plugin_interface(child, Arc::new(PluginSource::new(self)), Some(gc.clone()))?;
Ok(RunningPlugin { pid, interface, gc })
}
@ -193,3 +199,38 @@ impl RegisteredPlugin for PersistentPlugin {
self
}
}
/// Anything that can produce a plugin interface.
///
/// This is not a public interface.
#[doc(hidden)]
pub trait GetPlugin: RegisteredPlugin {
/// Retrieve or spawn a [`PluginInterface`]. The `context` may be used for determining
/// environment variables to launch the plugin with.
fn get_plugin(
self: Arc<Self>,
context: Option<(&EngineState, &mut Stack)>,
) -> Result<PluginInterface, ShellError>;
}
impl GetPlugin for PersistentPlugin {
fn get_plugin(
self: Arc<Self>,
context: Option<(&EngineState, &mut Stack)>,
) -> Result<PluginInterface, ShellError> {
self.get(|| {
// Get envs from the context if provided.
let envs = context
.map(|(engine_state, stack)| {
// We need the current environment variables for `python` based plugins. Or
// we'll likely have a problem when a plugin is implemented in a virtual Python
// environment.
let stack = &mut stack.start_capture();
nu_engine::env::env_to_strings(engine_state, stack)
})
.transpose()?;
Ok(envs.into_iter().flatten())
})
}
}

26
crates/nu-plugin/src/plugin/source.rs
View File

@ -1,26 +1,31 @@
use std::sync::{Arc, Weak};
use nu_protocol::{PluginIdentity, RegisteredPlugin, ShellError, Span};
use nu_protocol::{PluginIdentity, ShellError, Span};
use super::PersistentPlugin;
use super::GetPlugin;
/// The source of a custom value or plugin command. Includes a weak reference to the persistent
/// plugin so it can be retrieved.
///
/// This is not a public interface.
#[derive(Debug, Clone)]
pub(crate) struct PluginSource {
#[doc(hidden)]
pub struct PluginSource {
/// The identity of the plugin
pub(crate) identity: Arc<PluginIdentity>,
/// A weak reference to the persistent plugin that might hold an interface to the plugin.
///
/// This is weak to avoid cyclic references, but it does mean we might fail to upgrade if
/// the engine state lost the [`PersistentPlugin`] at some point.
pub(crate) persistent: Weak<PersistentPlugin>,
pub(crate) persistent: Weak<dyn GetPlugin>,
}
impl PluginSource {
/// Create from an `Arc<PersistentPlugin>`
pub(crate) fn new(plugin: &Arc<PersistentPlugin>) -> PluginSource {
/// Create from an implementation of `GetPlugin`
pub fn new(plugin: Arc<dyn GetPlugin>) -> PluginSource {
PluginSource {
identity: plugin.identity().clone().into(),
persistent: Arc::downgrade(plugin),
persistent: Arc::downgrade(&plugin),
}
}
@ -31,16 +36,13 @@ impl PluginSource {
pub(crate) fn new_fake(name: &str) -> PluginSource {
PluginSource {
identity: PluginIdentity::new_fake(name).into(),
persistent: Weak::new(),
persistent: Weak::<crate::PersistentPlugin>::new(),
}
}
/// Try to upgrade the persistent reference, and return an error referencing `span` as the
/// object that referenced it otherwise
pub(crate) fn persistent(
&self,
span: Option<Span>,
) -> Result<Arc<PersistentPlugin>, ShellError> {
pub(crate) fn persistent(&self, span: Option<Span>) -> Result<Arc<dyn GetPlugin>, ShellError> {
self.persistent
.upgrade()
.ok_or_else(|| ShellError::GenericError {

3
crates/nu-plugin/src/protocol/mod.rs
View File

@ -192,7 +192,10 @@ impl CustomValueOp {
}
/// Any data sent to the plugin
///
/// Note: exported for internal use, not public.
#[derive(Serialize, Deserialize, Debug, Clone)]
#[doc(hidden)]
pub enum PluginInput {
/// This must be the first message. Indicates supported protocol
Hello(ProtocolInfo),

14
crates/nu-plugin/src/protocol/plugin_custom_value.rs
View File

@ -20,7 +20,10 @@ mod tests;
/// appropriate [`PluginSource`](crate::plugin::PluginSource), ensuring that only
/// [`PluginCustomData`] is contained within any values sent, and that the `source` of any
/// values sent matches the plugin it is being sent to.
///
/// This is not a public API.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[doc(hidden)]
pub struct PluginCustomValue {
#[serde(flatten)]
shared: SerdeArc<SharedContent>,
@ -197,12 +200,9 @@ impl PluginCustomValue {
})
})?;
// Envs probably should be passed here, but it's likely that the plugin is already running
let empty_envs = std::iter::empty::<(&str, &str)>();
source
.persistent(span)
.and_then(|p| p.get(|| Ok(empty_envs)))
.and_then(|p| p.get_plugin(None))
.map_err(wrap_err)
}
@ -237,7 +237,7 @@ impl PluginCustomValue {
}
/// Add a [`PluginSource`] to all [`PluginCustomValue`]s within a value, recursively.
pub(crate) fn add_source(value: &mut Value, source: &Arc<PluginSource>) {
pub fn add_source(value: &mut Value, source: &Arc<PluginSource>) {
// This can't cause an error.
let _: Result<(), Infallible> = value.recurse_mut(&mut |value| {
let span = value.span();
@ -318,7 +318,7 @@ impl PluginCustomValue {
/// Convert all plugin-native custom values to [`PluginCustomValue`] within the given `value`,
/// recursively. This should only be done on the plugin side.
pub(crate) fn serialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
pub fn serialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
value.recurse_mut(&mut |value| {
let span = value.span();
match value {
@ -344,7 +344,7 @@ impl PluginCustomValue {
/// Convert all [`PluginCustomValue`]s to plugin-native custom values within the given `value`,
/// recursively. This should only be done on the plugin side.
pub(crate) fn deserialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
pub fn deserialize_custom_values_in(value: &mut Value) -> Result<(), ShellError> {
value.recurse_mut(&mut |value| {
let span = value.span();
match value {

2
crates/nu-plugin/src/sequence.rs
View File

@ -4,7 +4,7 @@ use nu_protocol::ShellError;
/// Implements an atomically incrementing sequential series of numbers
#[derive(Debug, Default)]
pub(crate) struct Sequence(AtomicUsize);
pub struct Sequence(AtomicUsize);
impl Sequence {
/// Return the next available id from a sequence, returning an error on overflow