Add CustomValue support to plugins (#6070)

* Skeleton implementation

Lots and lots of TODOs

* Bootstrap simple CustomValue plugin support test

* Create nu_plugin_custom_value

* Skeleton for nu_plugin_custom_values

* Return a custom value from plugin

* Encode CustomValues from plugin calls as PluginResponse::PluginData

* Add new PluginCall variant CollapseCustomValue

* Handle CollapseCustomValue plugin calls

* Add CallInput::Data variant to CallInfo inputs

* Handle CallInfo with CallInput::Data plugin calls

* Send CallInput::Data if Value is PluginCustomValue from plugin calls

* Remove unnecessary boxing of plugins CallInfo

* Add fields needed to collapse PluginCustomValue to it

* Document PluginCustomValue and its purpose

* Impl collapsing using plugin calls in PluginCustomValue::to_base_value

* Implement proper typetag based deserialization for CoolCustomValue

* Test demonstrating that passing back a custom value to plugin works

* Added a failing test for describing plugin CustomValues

* Support describe for PluginCustomValues

- Add name to PluginResponse::PluginData
  - Also turn it into a struct for clarity
- Add name to PluginCustomValue
- Return name field from PluginCustomValue

* Demonstrate that plugins can create and handle multiple CustomValues

* Add bincode to nu-plugin dependencies

This is for demonstration purposes, any schemaless binary seralization
format will work. I picked bincode since it's the most popular for Rust
but there are defintely better options out there for this usecase

* serde_json::Value -> Vec<u8>

* Update capnp schema for new CallInfo.input field

* Move call_input capnp serialization and deserialization into new file

* Deserialize Value's span from Value itself instead of passing call.head

I am not sure if this was correct and I am breaking it or if it was a
bug, I don't fully understand how nu creates and uses Spans. What should
reuse spans and what should recreate new ones?
But yeah it felt weird that the Value's Span was being ignored since in
the json serializer just uses the Value's Span

* Add call_info value round trip test

* Add capnp CallInput::Data serialization and deserialization support

* Add CallInfo::CollapseCustomValue to capnp schema

* Add capnp PluginCall::CollapseCustomValue serialization and deserialization support

* Add PluginResponse::PluginData to capnp schema

* Add capnp PluginResponse::PluginData serialization and deserialization support

* Switch plugins::custom_values tests to capnp

Both json and capnp would work now! Sadly I can't choose both at the
same time :(

* Add missing JsonSerializer round trip tests

* Handle plugin returning PluginData as a response to CollapseCustomValue

* Refactor plugin calling into a reusable function

Many less levels of indentation now!

* Export PluginData from nu_plugin

So plugins can create their very own serve_plugin with whatever
CustomValue behavior they may desire

* Error if CustomValue cannot be handled by Plugin

crates/nu-plugin/src/protocol/mod.rs

@@ -1,21 +1,32 @@
 mod evaluated_call;
+mod plugin_custom_value;
+mod plugin_data;
 
 pub use evaluated_call::EvaluatedCall;
 use nu_protocol::{ShellError, Signature, Span, Value};
+pub use plugin_custom_value::PluginCustomValue;
+pub use plugin_data::PluginData;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct CallInfo {
     pub name: String,
     pub call: EvaluatedCall,
-    pub input: Value,
+    pub input: CallInput,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+pub enum CallInput {
+    Value(Value),
+    Data(PluginData),
 }
 
 // Information sent to the plugin
 #[derive(Serialize, Deserialize, Debug)]
 pub enum PluginCall {
     Signature,
-    CallInfo(Box<CallInfo>),
+    CallInfo(CallInfo),
+    CollapseCustomValue(PluginData),
 }
 
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Debug)]
@@ -88,4 +99,5 @@ pub enum PluginResponse {
     Error(LabeledError),
     Signature(Vec<Signature>),
     Value(Box<Value>),
+    PluginData(String, PluginData),
 }

crates/nu-plugin/src/protocol/plugin_custom_value.rs

@@ -0,0 +1,126 @@
+use std::path::PathBuf;
+
+use nu_protocol::{CustomValue, ShellError, Value};
+use serde::Serialize;
+
+use crate::{
+    plugin::{call_plugin, create_command},
+    EncodingType,
+};
+
+use super::{PluginCall, PluginData, PluginResponse};
+
+/// An opaque container for a custom value that is handled fully by a plugin
+///
+/// This is constructed by the main nushell engine when it receives [`PluginResponse::PluginData`]
+/// it stores that data as well as metadata related to the plugin to be able to call the plugin
+/// later.
+/// Since the data in it is opaque to the engine, there are only two final destinations for it:
+/// either it will be sent back to the plugin that generated it across a pipeline, or it will be
+/// sent to the plugin with a request to collapse it into a base value
+#[derive(Clone, Debug, Serialize)]
+pub struct PluginCustomValue {
+    /// The name of the custom value as defined by the plugin
+    pub name: String,
+    pub data: Vec<u8>,
+    pub filename: PathBuf,
+
+    // PluginCustomValue must implement Serialize because all CustomValues must implement Serialize
+    // However, the main place where values are serialized and deserialized is when they are being
+    // sent between plugins and nushell's main engine. PluginCustomValue is never meant to be sent
+    // between that boundary
+    #[serde(skip)]
+    pub shell: Option<PathBuf>,
+    #[serde(skip)]
+    pub encoding: EncodingType,
+    #[serde(skip)]
+    pub source: String,
+}
+
+impl CustomValue for PluginCustomValue {
+    fn clone_value(&self, span: nu_protocol::Span) -> nu_protocol::Value {
+        Value::CustomValue {
+            val: Box::new(self.clone()),
+            span,
+        }
+    }
+
+    fn value_string(&self) -> String {
+        self.name.clone()
+    }
+
+    fn to_base_value(
+        &self,
+        span: nu_protocol::Span,
+    ) -> Result<nu_protocol::Value, nu_protocol::ShellError> {
+        let mut plugin_cmd = create_command(&self.filename, &self.shell);
+
+        let mut child = plugin_cmd.spawn().map_err(|err| {
+            ShellError::GenericError(
+                format!(
+                    "Unable to spawn plugin for {} to get base value",
+                    self.source
+                ),
+                format!("{}", err),
+                Some(span),
+                None,
+                Vec::new(),
+            )
+        })?;
+
+        let plugin_call = PluginCall::CollapseCustomValue(PluginData {
+            data: self.data.clone(),
+            span,
+        });
+
+        let response = call_plugin(&mut child, plugin_call, &self.encoding, span).map_err(|err| {
+            ShellError::GenericError(
+                format!(
+                    "Unable to decode call for {} to get base value",
+                    self.source
+                ),
+                format!("{}", err),
+                Some(span),
+                None,
+                Vec::new(),
+            )
+        });
+
+        let value = match response {
+            Ok(PluginResponse::Value(value)) => Ok(*value),
+            Ok(PluginResponse::PluginData(..)) => Err(ShellError::GenericError(
+                "Plugin misbehaving".into(),
+                "Plugin returned custom data as a response to a collapse call".into(),
+                Some(span),
+                None,
+                Vec::new(),
+            )),
+            Ok(PluginResponse::Error(err)) => Err(err.into()),
+            Ok(PluginResponse::Signature(..)) => Err(ShellError::GenericError(
+                "Plugin missing value".into(),
+                "Received a signature from plugin instead of value".into(),
+                Some(span),
+                None,
+                Vec::new(),
+            )),
+            Err(err) => Err(err),
+        };
+
+        // We need to call .wait() on the child, or we'll risk summoning the zombie horde
+        let _ = child.wait();
+
+        value
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn typetag_name(&self) -> &'static str {
+        "PluginCustomValue"
+    }
+
+    fn typetag_deserialize(&self) {
+        unimplemented!("typetag_deserialize")
+    }
+}

crates/nu-plugin/src/protocol/plugin_data.rs

@@ -0,0 +1,8 @@
+use nu_protocol::Span;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+pub struct PluginData {
+    pub data: Vec<u8>,
+    pub span: Span,
+}