add MessagePack as a plugin protocol (#6370)

crates/nu-command/src/core_commands/register.rs

@@ -24,7 +24,7 @@ impl Command for Register {
             .required_named(
                 "encoding",
                 SyntaxShape::String,
-                "Encoding used to communicate with plugin. Options: [capnp, json]",
+                "Encoding used to communicate with plugin. Options: [capnp, json, msgpack]",
                 Some('e'),
             )
             .optional(

crates/nu-parser/src/parse_keywords.rs

@@ -2802,7 +2802,7 @@ pub fn parse_register(
                             ParseError::IncorrectValue(
                                 "wrong encoding".into(),
                                 expr.span,
-                                "Encodings available: capnp and json".into(),
+                                "Encodings available: capnp, json, and msgpack".into(),
                             )
                         })
                 })

crates/nu-plugin/Cargo.toml

@@ -12,5 +12,9 @@ bincode = "1.3.3"
 capnp = "0.14.3"
 nu-protocol = { path = "../nu-protocol", version = "0.67.1"  }
 nu-engine = { path = "../nu-engine", version = "0.67.1"  }
-serde = {version = "1.0.130", features = ["derive"]}
+serde = {version = "1.0.143", features = ["derive"]}
 serde_json = { version = "1.0"}
+byte-order = "0.3.0"
+rmp = "0.8.11"
+rmp-serde = "1.1.0"
+rmpv = "1.0.0"

crates/nu-plugin/src/lib.rs

@@ -7,4 +7,6 @@ mod plugin_capnp;
 
 pub use plugin::{get_signature, serve_plugin, Plugin, PluginDeclaration};
 pub use protocol::{EvaluatedCall, LabeledError, PluginData};
-pub use serializers::{capnp::CapnpSerializer, json::JsonSerializer, EncodingType};
+pub use serializers::{
+    capnp::CapnpSerializer, json::JsonSerializer, msgpack::MsgPackSerializer, EncodingType,
+};

crates/nu-plugin/src/serializers/mod.rs

@@ -1,17 +1,18 @@
-use nu_protocol::ShellError;
-
 use crate::{
     plugin::PluginEncoder,
     protocol::{PluginCall, PluginResponse},
 };
+use nu_protocol::ShellError;
 
 pub mod capnp;
 pub mod json;
+pub mod msgpack;
 
 #[derive(Clone, Debug)]
 pub enum EncodingType {
     Capnp(capnp::CapnpSerializer),
     Json(json::JsonSerializer),
+    MsgPack(msgpack::MsgPackSerializer),
 }
 
 impl EncodingType {
@@ -19,6 +20,7 @@ impl EncodingType {
         match bytes {
             b"capnp" => Some(Self::Capnp(capnp::CapnpSerializer {})),
             b"json" => Some(Self::Json(json::JsonSerializer {})),
+            b"msgpack" => Some(Self::MsgPack(msgpack::MsgPackSerializer {})),
             _ => None,
         }
     }
@@ -31,6 +33,7 @@ impl EncodingType {
         match self {
             EncodingType::Capnp(encoder) => encoder.encode_call(plugin_call, writer),
             EncodingType::Json(encoder) => encoder.encode_call(plugin_call, writer),
+            EncodingType::MsgPack(encoder) => encoder.encode_call(plugin_call, writer),
         }
     }
 
@@ -41,6 +44,7 @@ impl EncodingType {
         match self {
             EncodingType::Capnp(encoder) => encoder.decode_call(reader),
             EncodingType::Json(encoder) => encoder.decode_call(reader),
+            EncodingType::MsgPack(encoder) => encoder.decode_call(reader),
         }
     }
 
@@ -52,6 +56,7 @@ impl EncodingType {
         match self {
             EncodingType::Capnp(encoder) => encoder.encode_response(plugin_response, writer),
             EncodingType::Json(encoder) => encoder.encode_response(plugin_response, writer),
+            EncodingType::MsgPack(encoder) => encoder.encode_response(plugin_response, writer),
         }
     }
 
@@ -62,6 +67,7 @@ impl EncodingType {
         match self {
             EncodingType::Capnp(encoder) => encoder.decode_response(reader),
             EncodingType::Json(encoder) => encoder.decode_response(reader),
+            EncodingType::MsgPack(encoder) => encoder.decode_response(reader),
         }
     }
 
@@ -69,6 +75,7 @@ impl EncodingType {
         match self {
             Self::Capnp(_) => "capnp",
             Self::Json(_) => "json",
+            Self::MsgPack(_) => "msgpack",
         }
     }
 }

crates/nu-plugin/src/serializers/msgpack.rs

@@ -0,0 +1,360 @@
+use crate::{plugin::PluginEncoder, protocol::PluginResponse};
+use nu_protocol::ShellError;
+
+#[derive(Clone, Debug)]
+pub struct MsgPackSerializer;
+
+impl PluginEncoder for MsgPackSerializer {
+    fn name(&self) -> &str {
+        "MsgPack Serializer"
+    }
+
+    fn encode_call(
+        &self,
+        plugin_call: &crate::protocol::PluginCall,
+        writer: &mut impl std::io::Write,
+    ) -> Result<(), nu_protocol::ShellError> {
+        rmp_serde::encode::write(writer, plugin_call)
+            .map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
+    }
+
+    fn decode_call(
+        &self,
+        reader: &mut impl std::io::BufRead,
+    ) -> Result<crate::protocol::PluginCall, nu_protocol::ShellError> {
+        rmp_serde::from_read(reader)
+            .map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
+    }
+
+    fn encode_response(
+        &self,
+        plugin_response: &PluginResponse,
+        writer: &mut impl std::io::Write,
+    ) -> Result<(), ShellError> {
+        rmp_serde::encode::write(writer, plugin_response)
+            .map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
+    }
+
+    fn decode_response(
+        &self,
+        reader: &mut impl std::io::BufRead,
+    ) -> Result<PluginResponse, ShellError> {
+        rmp_serde::from_read(reader)
+            .map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::protocol::{
+        CallInfo, CallInput, EvaluatedCall, LabeledError, PluginCall, PluginData, PluginResponse,
+    };
+    use nu_protocol::{Signature, Span, Spanned, SyntaxShape, Value};
+
+    #[test]
+    fn callinfo_round_trip_signature() {
+        let plugin_call = PluginCall::Signature;
+        let encoder = MsgPackSerializer {};
+
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_call(&plugin_call, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_call(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginCall::Signature => {}
+            PluginCall::CallInfo(_) => panic!("decoded into wrong value"),
+            PluginCall::CollapseCustomValue(_) => panic!("decoded into wrong value"),
+        }
+    }
+
+    #[test]
+    fn callinfo_round_trip_callinfo() {
+        let name = "test".to_string();
+
+        let input = Value::Bool {
+            val: false,
+            span: Span { start: 1, end: 20 },
+        };
+
+        let call = EvaluatedCall {
+            head: Span { start: 0, end: 10 },
+            positional: vec![
+                Value::Float {
+                    val: 1.0,
+                    span: Span { start: 0, end: 10 },
+                },
+                Value::String {
+                    val: "something".into(),
+                    span: Span { start: 0, end: 10 },
+                },
+            ],
+            named: vec![(
+                Spanned {
+                    item: "name".to_string(),
+                    span: Span { start: 0, end: 10 },
+                },
+                Some(Value::Float {
+                    val: 1.0,
+                    span: Span { start: 0, end: 10 },
+                }),
+            )],
+        };
+
+        let plugin_call = PluginCall::CallInfo(CallInfo {
+            name: name.clone(),
+            call: call.clone(),
+            input: CallInput::Value(input.clone()),
+        });
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_call(&plugin_call, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_call(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginCall::Signature => panic!("returned wrong call type"),
+            PluginCall::CallInfo(call_info) => {
+                assert_eq!(name, call_info.name);
+                assert_eq!(CallInput::Value(input), call_info.input);
+                assert_eq!(call.head, call_info.call.head);
+                assert_eq!(call.positional.len(), call_info.call.positional.len());
+
+                call.positional
+                    .iter()
+                    .zip(call_info.call.positional.iter())
+                    .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+                call.named
+                    .iter()
+                    .zip(call_info.call.named.iter())
+                    .for_each(|(lhs, rhs)| {
+                        // Comparing the keys
+                        assert_eq!(lhs.0.item, rhs.0.item);
+
+                        match (&lhs.1, &rhs.1) {
+                            (None, None) => {}
+                            (Some(a), Some(b)) => assert_eq!(a, b),
+                            _ => panic!("not matching values"),
+                        }
+                    });
+            }
+            PluginCall::CollapseCustomValue(_) => panic!("returned wrong call type"),
+        }
+    }
+
+    #[test]
+    fn callinfo_round_trip_collapsecustomvalue() {
+        let data = vec![1, 2, 3, 4, 5, 6, 7];
+        let span = Span { start: 0, end: 20 };
+
+        let collapse_custom_value = PluginCall::CollapseCustomValue(PluginData {
+            data: data.clone(),
+            span,
+        });
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_call(&collapse_custom_value, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_call(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginCall::Signature => panic!("returned wrong call type"),
+            PluginCall::CallInfo(_) => panic!("returned wrong call type"),
+            PluginCall::CollapseCustomValue(plugin_data) => {
+                assert_eq!(data, plugin_data.data);
+                assert_eq!(span, plugin_data.span);
+            }
+        }
+    }
+
+    #[test]
+    fn response_round_trip_signature() {
+        let signature = Signature::build("nu-plugin")
+            .required("first", SyntaxShape::String, "first required")
+            .required("second", SyntaxShape::Int, "second required")
+            .required_named("first-named", SyntaxShape::String, "first named", Some('f'))
+            .required_named(
+                "second-named",
+                SyntaxShape::String,
+                "second named",
+                Some('s'),
+            )
+            .rest("remaining", SyntaxShape::Int, "remaining");
+
+        let response = PluginResponse::Signature(vec![signature.clone()]);
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_response(&response, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_response(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginResponse::Error(_) => panic!("returned wrong call type"),
+            PluginResponse::Value(_) => panic!("returned wrong call type"),
+            PluginResponse::PluginData(..) => panic!("returned wrong call type"),
+            PluginResponse::Signature(returned_signature) => {
+                assert!(returned_signature.len() == 1);
+                assert_eq!(signature.name, returned_signature[0].name);
+                assert_eq!(signature.usage, returned_signature[0].usage);
+                assert_eq!(signature.extra_usage, returned_signature[0].extra_usage);
+                assert_eq!(signature.is_filter, returned_signature[0].is_filter);
+
+                signature
+                    .required_positional
+                    .iter()
+                    .zip(returned_signature[0].required_positional.iter())
+                    .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+                signature
+                    .optional_positional
+                    .iter()
+                    .zip(returned_signature[0].optional_positional.iter())
+                    .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+                signature
+                    .named
+                    .iter()
+                    .zip(returned_signature[0].named.iter())
+                    .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+                assert_eq!(
+                    signature.rest_positional,
+                    returned_signature[0].rest_positional,
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn response_round_trip_value() {
+        let value = Value::Int {
+            val: 10,
+            span: Span { start: 2, end: 30 },
+        };
+
+        let response = PluginResponse::Value(Box::new(value.clone()));
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_response(&response, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_response(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginResponse::Error(_) => panic!("returned wrong call type"),
+            PluginResponse::Signature(_) => panic!("returned wrong call type"),
+            PluginResponse::PluginData(..) => panic!("returned wrong call type"),
+            PluginResponse::Value(returned_value) => {
+                assert_eq!(&value, returned_value.as_ref())
+            }
+        }
+    }
+
+    #[test]
+    fn response_round_trip_plugin_data() {
+        let name = "test".to_string();
+
+        let data = vec![1, 2, 3, 4, 5];
+        let span = Span { start: 2, end: 30 };
+
+        let response = PluginResponse::PluginData(
+            name.clone(),
+            PluginData {
+                data: data.clone(),
+                span,
+            },
+        );
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_response(&response, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_response(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginResponse::Error(_) => panic!("returned wrong call type"),
+            PluginResponse::Signature(_) => panic!("returned wrong call type"),
+            PluginResponse::Value(_) => panic!("returned wrong call type"),
+            PluginResponse::PluginData(returned_name, returned_plugin_data) => {
+                assert_eq!(name, returned_name);
+                assert_eq!(data, returned_plugin_data.data);
+                assert_eq!(span, returned_plugin_data.span);
+            }
+        }
+    }
+
+    #[test]
+    fn response_round_trip_error() {
+        let error = LabeledError {
+            label: "label".into(),
+            msg: "msg".into(),
+            span: Some(Span { start: 2, end: 30 }),
+        };
+        let response = PluginResponse::Error(error.clone());
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_response(&response, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_response(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginResponse::Error(msg) => assert_eq!(error, msg),
+            PluginResponse::Signature(_) => panic!("returned wrong call type"),
+            PluginResponse::Value(_) => panic!("returned wrong call type"),
+            PluginResponse::PluginData(..) => panic!("returned wrong call type"),
+        }
+    }
+
+    #[test]
+    fn response_round_trip_error_none() {
+        let error = LabeledError {
+            label: "label".into(),
+            msg: "msg".into(),
+            span: None,
+        };
+        let response = PluginResponse::Error(error.clone());
+
+        let encoder = MsgPackSerializer {};
+        let mut buffer: Vec<u8> = Vec::new();
+        encoder
+            .encode_response(&response, &mut buffer)
+            .expect("unable to serialize message");
+        let returned = encoder
+            .decode_response(&mut buffer.as_slice())
+            .expect("unable to deserialize message");
+
+        match returned {
+            PluginResponse::Error(msg) => assert_eq!(error, msg),
+            PluginResponse::Signature(_) => panic!("returned wrong call type"),
+            PluginResponse::Value(_) => panic!("returned wrong call type"),
+            PluginResponse::PluginData(..) => panic!("returned wrong call type"),
+        }
+    }
+}

crates/nu_plugin_custom_values/src/main.rs

@@ -2,7 +2,7 @@ mod cool_custom_value;
 mod second_custom_value;
 
 use cool_custom_value::CoolCustomValue;
-use nu_plugin::{serve_plugin, CapnpSerializer, Plugin};
+use nu_plugin::{serve_plugin, MsgPackSerializer, Plugin};
 use nu_plugin::{EvaluatedCall, LabeledError};
 use nu_protocol::{Category, ShellError, Signature, Value};
 use second_custom_value::SecondCustomValue;
@@ -74,5 +74,5 @@ impl CustomValuePlugin {
 }
 
 fn main() {
-    serve_plugin(&mut CustomValuePlugin, CapnpSerializer {})
+    serve_plugin(&mut CustomValuePlugin, MsgPackSerializer {})
 }

crates/nu_plugin_example/src/main.rs

@@ -1,4 +1,4 @@
-use nu_plugin::{serve_plugin, CapnpSerializer};
+use nu_plugin::{serve_plugin, MsgPackSerializer};
 use nu_plugin_example::Example;
 
 fn main() {
@@ -6,7 +6,7 @@ fn main() {
     // used to encode and decode the messages. The available options are
     // CapnpSerializer and JsonSerializer. Both are defined in the serializer
     // folder in nu-plugin.
-    serve_plugin(&mut Example {}, CapnpSerializer {})
+    serve_plugin(&mut Example {}, MsgPackSerializer {})
 
     // Note
     // When creating plugins in other languages one needs to consider how a plugin

crates/nu_plugin_gstat/README.md

@@ -10,4 +10,4 @@ To install:
 
 To register (from inside Nushell):
 ```
-> register <path to installed plugin> --encoding json
+> register <path to installed plugin> --encoding msgpack

crates/nu_plugin_gstat/src/main.rs

@@ -1,6 +1,6 @@
-use nu_plugin::{serve_plugin, JsonSerializer};
+use nu_plugin::{serve_plugin, MsgPackSerializer};
 use nu_plugin_gstat::GStat;
 
 fn main() {
-    serve_plugin(&mut GStat::new(), JsonSerializer {})
+    serve_plugin(&mut GStat::new(), MsgPackSerializer {})
 }