holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add MessagePack as a plugin protocol (#6370)

Browse Source
This commit is contained in:
Darren Schroeder 2022-08-21 06:13:38 -05:00 committed by GitHub
parent 56ce10347e
commit 5337a6dffa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 444 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Cargo.lock generated
View File

@ -419,6 +419,12 @@ version = "3.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
[[package]]
name = "byte-order"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b021a13e4bf34a5679ada4609a01337ae82f2c4c97493b9d8cbf8aa9af9bd0f4"
[[package]]
name = "byte-slice-cast"
version = "1.2.1"
@ -2752,9 +2758,13 @@ name = "nu-plugin"
version = "0.67.1"
dependencies = [
"bincode",
"byte-order",
"capnp",
"nu-engine",
"nu-protocol",
"rmp",
"rmp-serde",
"rmpv",
"serde",
"serde_json",
]
@ -3233,6 +3243,12 @@ dependencies = [
"regex",
]
[[package]]
name = "paste"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22"
[[package]]
name = "pathdiff"
version = "0.2.1"
@ -4043,6 +4059,38 @@ dependencies = [
"regex",
]
[[package]]
name = "rmp"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f"
dependencies = [
"byteorder",
"num-traits",
"paste",
]
[[package]]
name = "rmp-serde"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "rmpv"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de8813b3a2f95c5138fe5925bfb8784175d88d6bff059ba8ce090aa891319754"
dependencies = [
"num-traits",
"rmp",
]
[[package]]
name = "roxmltree"
version = "0.14.1"
@ -4314,18 +4362,18 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.140"
version = "1.0.143"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03"
checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.140"
version = "1.0.143"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da"
checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391"
dependencies = [
"proc-macro2",
"quote",

2
crates/nu-command/src/core_commands/register.rs
View File

@ -24,7 +24,7 @@ impl Command for Register {
.required_named(
"encoding",
SyntaxShape::String,
"Encoding used to communicate with plugin. Options: [capnp, json]",
"Encoding used to communicate with plugin. Options: [capnp, json, msgpack]",
Some('e'),
)
.optional(

2
crates/nu-parser/src/parse_keywords.rs
View File

@ -2802,7 +2802,7 @@ pub fn parse_register(
ParseError::IncorrectValue(
"wrong encoding".into(),
expr.span,
"Encodings available: capnp and json".into(),
"Encodings available: capnp, json, and msgpack".into(),
)
})
})

6
crates/nu-plugin/Cargo.toml
View File

@ -12,5 +12,9 @@ bincode = "1.3.3"
capnp = "0.14.3"
nu-protocol = { path = "../nu-protocol", version = "0.67.1" }
nu-engine = { path = "../nu-engine", version = "0.67.1" }
serde = {version = "1.0.130", features = ["derive"]}
serde = {version = "1.0.143", features = ["derive"]}
serde_json = { version = "1.0"}
byte-order = "0.3.0"
rmp = "0.8.11"
rmp-serde = "1.1.0"
rmpv = "1.0.0"

4
crates/nu-plugin/src/lib.rs
View File

@ -7,4 +7,6 @@ mod plugin_capnp;
pub use plugin::{get_signature, serve_plugin, Plugin, PluginDeclaration};
pub use protocol::{EvaluatedCall, LabeledError, PluginData};
pub use serializers::{capnp::CapnpSerializer, json::JsonSerializer, EncodingType};
pub use serializers::{
capnp::CapnpSerializer, json::JsonSerializer, msgpack::MsgPackSerializer, EncodingType,
};

11
crates/nu-plugin/src/serializers/mod.rs
View File

@ -1,17 +1,18 @@
use nu_protocol::ShellError;
use crate::{
plugin::PluginEncoder,
protocol::{PluginCall, PluginResponse},
};
use nu_protocol::ShellError;
pub mod capnp;
pub mod json;
pub mod msgpack;
#[derive(Clone, Debug)]
pub enum EncodingType {
Capnp(capnp::CapnpSerializer),
Json(json::JsonSerializer),
MsgPack(msgpack::MsgPackSerializer),
}
impl EncodingType {
@ -19,6 +20,7 @@ impl EncodingType {
match bytes {
b"capnp" => Some(Self::Capnp(capnp::CapnpSerializer {})),
b"json" => Some(Self::Json(json::JsonSerializer {})),
b"msgpack" => Some(Self::MsgPack(msgpack::MsgPackSerializer {})),
_ => None,
}
}
@ -31,6 +33,7 @@ impl EncodingType {
match self {
EncodingType::Capnp(encoder) => encoder.encode_call(plugin_call, writer),
EncodingType::Json(encoder) => encoder.encode_call(plugin_call, writer),
EncodingType::MsgPack(encoder) => encoder.encode_call(plugin_call, writer),
}
}
@ -41,6 +44,7 @@ impl EncodingType {
match self {
EncodingType::Capnp(encoder) => encoder.decode_call(reader),
EncodingType::Json(encoder) => encoder.decode_call(reader),
EncodingType::MsgPack(encoder) => encoder.decode_call(reader),
}
}
@ -52,6 +56,7 @@ impl EncodingType {
match self {
EncodingType::Capnp(encoder) => encoder.encode_response(plugin_response, writer),
EncodingType::Json(encoder) => encoder.encode_response(plugin_response, writer),
EncodingType::MsgPack(encoder) => encoder.encode_response(plugin_response, writer),
}
}
@ -62,6 +67,7 @@ impl EncodingType {
match self {
EncodingType::Capnp(encoder) => encoder.decode_response(reader),
EncodingType::Json(encoder) => encoder.decode_response(reader),
EncodingType::MsgPack(encoder) => encoder.decode_response(reader),
}
}
@ -69,6 +75,7 @@ impl EncodingType {
match self {
Self::Capnp(_) => "capnp",
Self::Json(_) => "json",
Self::MsgPack(_) => "msgpack",
}
}
}

360
crates/nu-plugin/src/serializers/msgpack.rs Normal file
View File

@ -0,0 +1,360 @@
use crate::{plugin::PluginEncoder, protocol::PluginResponse};
use nu_protocol::ShellError;
#[derive(Clone, Debug)]
pub struct MsgPackSerializer;
impl PluginEncoder for MsgPackSerializer {
fn name(&self) -> &str {
"MsgPack Serializer"
}
fn encode_call(
&self,
plugin_call: &crate::protocol::PluginCall,
writer: &mut impl std::io::Write,
) -> Result<(), nu_protocol::ShellError> {
rmp_serde::encode::write(writer, plugin_call)
.map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
}
fn decode_call(
&self,
reader: &mut impl std::io::BufRead,
) -> Result<crate::protocol::PluginCall, nu_protocol::ShellError> {
rmp_serde::from_read(reader)
.map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
}
fn encode_response(
&self,
plugin_response: &PluginResponse,
writer: &mut impl std::io::Write,
) -> Result<(), ShellError> {
rmp_serde::encode::write(writer, plugin_response)
.map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
}
fn decode_response(
&self,
reader: &mut impl std::io::BufRead,
) -> Result<PluginResponse, ShellError> {
rmp_serde::from_read(reader)
.map_err(|err| ShellError::PluginFailedToEncode(err.to_string()))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::protocol::{
CallInfo, CallInput, EvaluatedCall, LabeledError, PluginCall, PluginData, PluginResponse,
};
use nu_protocol::{Signature, Span, Spanned, SyntaxShape, Value};
#[test]
fn callinfo_round_trip_signature() {
let plugin_call = PluginCall::Signature;
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_call(&plugin_call, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_call(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginCall::Signature => {}
PluginCall::CallInfo(_) => panic!("decoded into wrong value"),
PluginCall::CollapseCustomValue(_) => panic!("decoded into wrong value"),
}
}
#[test]
fn callinfo_round_trip_callinfo() {
let name = "test".to_string();
let input = Value::Bool {
val: false,
span: Span { start: 1, end: 20 },
};
let call = EvaluatedCall {
head: Span { start: 0, end: 10 },
positional: vec![
Value::Float {
val: 1.0,
span: Span { start: 0, end: 10 },
},
Value::String {
val: "something".into(),
span: Span { start: 0, end: 10 },
},
],
named: vec![(
Spanned {
item: "name".to_string(),
span: Span { start: 0, end: 10 },
},
Some(Value::Float {
val: 1.0,
span: Span { start: 0, end: 10 },
}),
)],
};
let plugin_call = PluginCall::CallInfo(CallInfo {
name: name.clone(),
call: call.clone(),
input: CallInput::Value(input.clone()),
});
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_call(&plugin_call, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_call(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginCall::Signature => panic!("returned wrong call type"),
PluginCall::CallInfo(call_info) => {
assert_eq!(name, call_info.name);
assert_eq!(CallInput::Value(input), call_info.input);
assert_eq!(call.head, call_info.call.head);
assert_eq!(call.positional.len(), call_info.call.positional.len());
call.positional
.iter()
.zip(call_info.call.positional.iter())
.for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
call.named
.iter()
.zip(call_info.call.named.iter())
.for_each(|(lhs, rhs)| {
// Comparing the keys
assert_eq!(lhs.0.item, rhs.0.item);
match (&lhs.1, &rhs.1) {
(None, None) => {}
(Some(a), Some(b)) => assert_eq!(a, b),
_ => panic!("not matching values"),
}
});
}
PluginCall::CollapseCustomValue(_) => panic!("returned wrong call type"),
}
}
#[test]
fn callinfo_round_trip_collapsecustomvalue() {
let data = vec![1, 2, 3, 4, 5, 6, 7];
let span = Span { start: 0, end: 20 };
let collapse_custom_value = PluginCall::CollapseCustomValue(PluginData {
data: data.clone(),
span,
});
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_call(&collapse_custom_value, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_call(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginCall::Signature => panic!("returned wrong call type"),
PluginCall::CallInfo(_) => panic!("returned wrong call type"),
PluginCall::CollapseCustomValue(plugin_data) => {
assert_eq!(data, plugin_data.data);
assert_eq!(span, plugin_data.span);
}
}
}
#[test]
fn response_round_trip_signature() {
let signature = Signature::build("nu-plugin")
.required("first", SyntaxShape::String, "first required")
.required("second", SyntaxShape::Int, "second required")
.required_named("first-named", SyntaxShape::String, "first named", Some('f'))
.required_named(
"second-named",
SyntaxShape::String,
"second named",
Some('s'),
)
.rest("remaining", SyntaxShape::Int, "remaining");
let response = PluginResponse::Signature(vec![signature.clone()]);
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_response(&response, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_response(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginResponse::Error(_) => panic!("returned wrong call type"),
PluginResponse::Value(_) => panic!("returned wrong call type"),
PluginResponse::PluginData(..) => panic!("returned wrong call type"),
PluginResponse::Signature(returned_signature) => {
assert!(returned_signature.len() == 1);
assert_eq!(signature.name, returned_signature[0].name);
assert_eq!(signature.usage, returned_signature[0].usage);
assert_eq!(signature.extra_usage, returned_signature[0].extra_usage);
assert_eq!(signature.is_filter, returned_signature[0].is_filter);
signature
.required_positional
.iter()
.zip(returned_signature[0].required_positional.iter())
.for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
signature
.optional_positional
.iter()
.zip(returned_signature[0].optional_positional.iter())
.for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
signature
.named
.iter()
.zip(returned_signature[0].named.iter())
.for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
assert_eq!(
signature.rest_positional,
returned_signature[0].rest_positional,
);
}
}
}
#[test]
fn response_round_trip_value() {
let value = Value::Int {
val: 10,
span: Span { start: 2, end: 30 },
};
let response = PluginResponse::Value(Box::new(value.clone()));
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_response(&response, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_response(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginResponse::Error(_) => panic!("returned wrong call type"),
PluginResponse::Signature(_) => panic!("returned wrong call type"),
PluginResponse::PluginData(..) => panic!("returned wrong call type"),
PluginResponse::Value(returned_value) => {
assert_eq!(&value, returned_value.as_ref())
}
}
}
#[test]
fn response_round_trip_plugin_data() {
let name = "test".to_string();
let data = vec![1, 2, 3, 4, 5];
let span = Span { start: 2, end: 30 };
let response = PluginResponse::PluginData(
name.clone(),
PluginData {
data: data.clone(),
span,
},
);
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_response(&response, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_response(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginResponse::Error(_) => panic!("returned wrong call type"),
PluginResponse::Signature(_) => panic!("returned wrong call type"),
PluginResponse::Value(_) => panic!("returned wrong call type"),
PluginResponse::PluginData(returned_name, returned_plugin_data) => {
assert_eq!(name, returned_name);
assert_eq!(data, returned_plugin_data.data);
assert_eq!(span, returned_plugin_data.span);
}
}
}
#[test]
fn response_round_trip_error() {
let error = LabeledError {
label: "label".into(),
msg: "msg".into(),
span: Some(Span { start: 2, end: 30 }),
};
let response = PluginResponse::Error(error.clone());
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_response(&response, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_response(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginResponse::Error(msg) => assert_eq!(error, msg),
PluginResponse::Signature(_) => panic!("returned wrong call type"),
PluginResponse::Value(_) => panic!("returned wrong call type"),
PluginResponse::PluginData(..) => panic!("returned wrong call type"),
}
}
#[test]
fn response_round_trip_error_none() {
let error = LabeledError {
label: "label".into(),
msg: "msg".into(),
span: None,
};
let response = PluginResponse::Error(error.clone());
let encoder = MsgPackSerializer {};
let mut buffer: Vec<u8> = Vec::new();
encoder
.encode_response(&response, &mut buffer)
.expect("unable to serialize message");
let returned = encoder
.decode_response(&mut buffer.as_slice())
.expect("unable to deserialize message");
match returned {
PluginResponse::Error(msg) => assert_eq!(error, msg),
PluginResponse::Signature(_) => panic!("returned wrong call type"),
PluginResponse::Value(_) => panic!("returned wrong call type"),
PluginResponse::PluginData(..) => panic!("returned wrong call type"),
}
}
}

4
crates/nu_plugin_custom_values/src/main.rs
View File

@ -2,7 +2,7 @@ mod cool_custom_value;
mod second_custom_value;
use cool_custom_value::CoolCustomValue;
use nu_plugin::{serve_plugin, CapnpSerializer, Plugin};
use nu_plugin::{serve_plugin, MsgPackSerializer, Plugin};
use nu_plugin::{EvaluatedCall, LabeledError};
use nu_protocol::{Category, ShellError, Signature, Value};
use second_custom_value::SecondCustomValue;
@ -74,5 +74,5 @@ impl CustomValuePlugin {
}
fn main() {
serve_plugin(&mut CustomValuePlugin, CapnpSerializer {})
serve_plugin(&mut CustomValuePlugin, MsgPackSerializer {})
}

4
crates/nu_plugin_example/src/main.rs
View File

@ -1,4 +1,4 @@
use nu_plugin::{serve_plugin, CapnpSerializer};
use nu_plugin::{serve_plugin, MsgPackSerializer};
use nu_plugin_example::Example;
fn main() {
@ -6,7 +6,7 @@ fn main() {
// used to encode and decode the messages. The available options are
// CapnpSerializer and JsonSerializer. Both are defined in the serializer
// folder in nu-plugin.
serve_plugin(&mut Example {}, CapnpSerializer {})
serve_plugin(&mut Example {}, MsgPackSerializer {})
// Note
// When creating plugins in other languages one needs to consider how a plugin

2
crates/nu_plugin_gstat/README.md
View File

@ -10,4 +10,4 @@ To install:
To register (from inside Nushell):
```
> register <path to installed plugin> --encoding json
> register <path to installed plugin> --encoding msgpack

4
crates/nu_plugin_gstat/src/main.rs
View File

@ -1,6 +1,6 @@
use nu_plugin::{serve_plugin, JsonSerializer};
use nu_plugin::{serve_plugin, MsgPackSerializer};
use nu_plugin_gstat::GStat;
fn main() {
serve_plugin(&mut GStat::new(), JsonSerializer {})
serve_plugin(&mut GStat::new(), MsgPackSerializer {})
}

12
tests/plugins/custom_values.rs
View File

@ -4,7 +4,7 @@ use nu_test_support::nu_with_plugins;
fn can_get_custom_value_from_plugin_and_instantly_collapse_it() {
let actual = nu_with_plugins!(
cwd: "tests",
plugin: ("capnp", "nu_plugin_custom_values"),
plugin: ("msgpack", "nu_plugin_custom_values"),
"custom-value generate"
);
@ -15,7 +15,7 @@ fn can_get_custom_value_from_plugin_and_instantly_collapse_it() {
fn can_get_custom_value_from_plugin_and_pass_it_over() {
let actual = nu_with_plugins!(
cwd: "tests",
plugin: ("capnp", "nu_plugin_custom_values"),
plugin: ("msgpack", "nu_plugin_custom_values"),
"custom-value generate | custom-value update"
);
@ -29,7 +29,7 @@ fn can_get_custom_value_from_plugin_and_pass_it_over() {
fn can_generate_and_updated_multiple_types_of_custom_values() {
let actual = nu_with_plugins!(
cwd: "tests",
plugin: ("capnp", "nu_plugin_custom_values"),
plugin: ("msgpack", "nu_plugin_custom_values"),
"custom-value generate2 | custom-value update"
);
@ -43,7 +43,7 @@ fn can_generate_and_updated_multiple_types_of_custom_values() {
fn can_get_describe_plugin_custom_values() {
let actual = nu_with_plugins!(
cwd: "tests",
plugin: ("capnp", "nu_plugin_custom_values"),
plugin: ("msgpack", "nu_plugin_custom_values"),
"custom-value generate | describe"
);
@ -58,7 +58,7 @@ fn can_get_describe_plugin_custom_values() {
fn fails_if_passing_engine_custom_values_to_plugins() {
let actual = nu_with_plugins!(
cwd: "tests/fixtures/formats",
plugin: ("capnp", "nu_plugin_custom_values"),
plugin: ("msgpack", "nu_plugin_custom_values"),
"open-db sample.db | custom-value update"
);
@ -72,7 +72,7 @@ fn fails_if_passing_custom_values_across_plugins() {
let actual = nu_with_plugins!(
cwd: "tests",
plugins: [
("capnp", "nu_plugin_custom_values"),
("msgpack", "nu_plugin_custom_values"),
("json", "nu_plugin_inc")
],
"custom-value generate | inc --major"