Merge pull request #476 from pmeredit/topic/to_bson

Implement to-bson

src/cli.rs

@@ -162,6 +162,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(Reverse),
             whole_stream_command(Trim),
             whole_stream_command(ToArray),
+            whole_stream_command(ToBSON),
             whole_stream_command(ToCSV),
             whole_stream_command(ToJSON),
             whole_stream_command(ToTOML),

src/commands.rs

@@ -48,6 +48,7 @@ crate mod split_row;
 crate mod table;
 crate mod tags;
 crate mod to_array;
+crate mod to_bson;
 crate mod to_csv;
 crate mod to_json;
 crate mod to_toml;
@@ -104,6 +105,7 @@ crate use split_row::SplitRow;
 crate use table::Table;
 crate use tags::Tags;
 crate use to_array::ToArray;
+crate use to_bson::ToBSON;
 crate use to_csv::ToCSV;
 crate use to_json::ToJSON;
 crate use to_toml::ToTOML;

src/commands/from_bson.rs

@@ -58,6 +58,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value
             );
             collected.into_tagged_value()
         }
+        // TODO: Add Int32 to nushell?
         Bson::I32(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag),
         Bson::I64(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag),
         Bson::JavaScriptCode(js) => {
@@ -104,7 +105,14 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value
             );
             collected.into_tagged_value()
         }
-        Bson::ObjectId(obj_id) => Value::Primitive(Primitive::String(obj_id.to_hex())).tagged(tag),
+        Bson::ObjectId(obj_id) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$object_id".to_string(),
+                Value::Primitive(Primitive::String(obj_id.to_hex())).tagged(tag),
+            );
+            collected.into_tagged_value()
+        }
         Bson::UtcDatetime(dt) => Value::Primitive(Primitive::Date(*dt)).tagged(tag),
         Bson::Symbol(s) => {
             let mut collected = TaggedDictBuilder::new(tag);

src/commands/to_bson.rs

@@ -0,0 +1,231 @@
+use crate::commands::WholeStreamCommand;
+use crate::object::{Dictionary, Primitive, Value};
+use crate::prelude::*;
+use bson::{encode_document, oid::ObjectId, spec::BinarySubtype, Bson, Document};
+use std::convert::TryInto;
+
+pub struct ToBSON;
+
+impl WholeStreamCommand for ToBSON {
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_bson(args, registry)
+    }
+
+    fn name(&self) -> &str {
+        "to-bson"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-bson")
+    }
+}
+
+pub fn value_to_bson_value(v: &Value) -> Bson {
+    match v {
+        Value::Primitive(Primitive::Boolean(b)) => Bson::Boolean(*b),
+        Value::Primitive(Primitive::Bytes(b)) => Bson::I64(*b as i64),
+        Value::Primitive(Primitive::Date(d)) => Bson::UtcDatetime(*d),
+        Value::Primitive(Primitive::EndOfStream) => Bson::Null,
+        Value::Primitive(Primitive::BeginningOfStream) => Bson::Null,
+        Value::Primitive(Primitive::Float(f)) => Bson::FloatingPoint(f.into_inner()),
+        Value::Primitive(Primitive::Int(i)) => Bson::I64(*i),
+        Value::Primitive(Primitive::Nothing) => Bson::Null,
+        Value::Primitive(Primitive::String(s)) => Bson::String(s.clone()),
+        Value::Primitive(Primitive::Path(s)) => Bson::String(s.display().to_string()),
+        Value::List(l) => Bson::Array(l.iter().map(|x| value_to_bson_value(x)).collect()),
+        Value::Block(_) => Bson::Null,
+        Value::Binary(b) => Bson::Binary(BinarySubtype::Generic, b.clone()),
+        Value::Object(o) => object_value_to_bson(o),
+    }
+}
+
+// object_value_to_bson handles all Objects, even those that correspond to special
+// types (things like regex or javascript code).
+fn object_value_to_bson(o: &Dictionary) -> Bson {
+    let mut it = o.entries.iter();
+    if it.len() > 2 {
+        return generic_object_value_to_bson(o);
+    }
+    match it.next() {
+        Some((regex, tagged_regex_value)) if regex == "$regex" => match it.next() {
+            Some((options, tagged_opts_value)) if options == "$options" => {
+                let r: Result<String, _> = tagged_regex_value.try_into();
+                let opts: Result<String, _> = tagged_opts_value.try_into();
+                if r.is_err() || opts.is_err() {
+                    generic_object_value_to_bson(o)
+                } else {
+                    Bson::RegExp(r.unwrap(), opts.unwrap())
+                }
+            }
+            _ => generic_object_value_to_bson(o),
+        },
+        Some((javascript, tagged_javascript_value)) if javascript == "$javascript" => {
+            match it.next() {
+                Some((scope, tagged_scope_value)) if scope == "$scope" => {
+                    let js: Result<String, _> = tagged_javascript_value.try_into();
+                    let s: Result<&Dictionary, _> = tagged_scope_value.try_into();
+                    if js.is_err() || s.is_err() {
+                        generic_object_value_to_bson(o)
+                    } else {
+                        if let Bson::Document(doc) = object_value_to_bson(s.unwrap()) {
+                            Bson::JavaScriptCodeWithScope(js.unwrap(), doc)
+                        } else {
+                            generic_object_value_to_bson(o)
+                        }
+                    }
+                }
+                None => {
+                    let js: Result<String, _> = tagged_javascript_value.try_into();
+                    if js.is_err() {
+                        generic_object_value_to_bson(o)
+                    } else {
+                        Bson::JavaScriptCode(js.unwrap())
+                    }
+                }
+                _ => generic_object_value_to_bson(o),
+            }
+        }
+        Some((timestamp, tagged_timestamp_value)) if timestamp == "$timestamp" => {
+            let ts: Result<i64, _> = tagged_timestamp_value.try_into();
+            if ts.is_err() {
+                generic_object_value_to_bson(o)
+            } else {
+                Bson::TimeStamp(ts.unwrap())
+            }
+        }
+        Some((binary_subtype, tagged_binary_subtype_value))
+            if binary_subtype == "$binary_subtype" =>
+        {
+            match it.next() {
+                Some((binary, tagged_bin_value)) if binary == "$binary" => {
+                    let bst = get_binary_subtype(tagged_binary_subtype_value);
+                    let bin: Result<Vec<u8>, _> = tagged_bin_value.try_into();
+                    if bst.is_none() || bin.is_err() {
+                        generic_object_value_to_bson(o)
+                    } else {
+                        Bson::Binary(bst.unwrap(), bin.unwrap())
+                    }
+                }
+                _ => generic_object_value_to_bson(o),
+            }
+        }
+        Some((object_id, tagged_object_id_value)) if object_id == "$object_id" => {
+            let obj_id: Result<String, _> = tagged_object_id_value.try_into();
+            if obj_id.is_err() {
+                generic_object_value_to_bson(o)
+            } else {
+                let obj_id = ObjectId::with_string(&obj_id.unwrap());
+                if obj_id.is_err() {
+                    generic_object_value_to_bson(o)
+                } else {
+                    Bson::ObjectId(obj_id.unwrap())
+                }
+            }
+        }
+        Some((symbol, tagged_symbol_value)) if symbol == "$symbol" => {
+            let sym: Result<String, _> = tagged_symbol_value.try_into();
+            if sym.is_err() {
+                generic_object_value_to_bson(o)
+            } else {
+                Bson::Symbol(sym.unwrap())
+            }
+        }
+        _ => generic_object_value_to_bson(o),
+    }
+}
+
+fn get_binary_subtype<'a>(tagged_value: &'a Tagged<Value>) -> Option<BinarySubtype> {
+    match tagged_value.item() {
+        Value::Primitive(Primitive::String(s)) => Some(match s.as_ref() {
+            "generic" => BinarySubtype::Generic,
+            "function" => BinarySubtype::Function,
+            "binary_old" => BinarySubtype::BinaryOld,
+            "uuid_old" => BinarySubtype::UuidOld,
+            "uuid" => BinarySubtype::Uuid,
+            "md5" => BinarySubtype::Md5,
+            _ => unreachable!(),
+        }),
+        Value::Primitive(Primitive::Int(i)) => Some(BinarySubtype::UserDefined(*i as u8)),
+        _ => None,
+    }
+}
+
+// generic_object_value_bson handles any Object that does not
+// correspond to a special bson type (things like regex or javascript code).
+fn generic_object_value_to_bson(o: &Dictionary) -> Bson {
+    let mut doc = Document::new();
+    for (k, v) in o.entries.iter() {
+        doc.insert(k.clone(), value_to_bson_value(v));
+    }
+    Bson::Document(doc)
+}
+
+fn shell_encode_document(
+    writer: &mut Vec<u8>,
+    doc: Document,
+    span: Span,
+) -> Result<(), ShellError> {
+    match encode_document(writer, &doc) {
+        Err(e) => Err(ShellError::labeled_error(
+            format!("Failed to encode document due to: {:?}", e),
+            "requires BSON-compatible document",
+            span,
+        )),
+        _ => Ok(()),
+    }
+}
+
+fn bson_value_to_bytes(bson: Bson, span: Span) -> Result<Vec<u8>, ShellError> {
+    let mut out = Vec::new();
+    match bson {
+        Bson::Array(a) => {
+            for v in a.into_iter() {
+                match v {
+                    Bson::Document(d) => shell_encode_document(&mut out, d, span)?,
+                    _ => {
+                        return Err(ShellError::labeled_error(
+                            format!("All top level values must be Documents, got {:?}", v),
+                            "requires BSON-compatible document",
+                            span,
+                        ))
+                    }
+                }
+            }
+        }
+        Bson::Document(d) => shell_encode_document(&mut out, d, span)?,
+        _ => {
+            return Err(ShellError::labeled_error(
+                format!("All top level values must be Documents, got {:?}", bson),
+                "requires BSON-compatible document",
+                span,
+            ))
+        }
+    }
+    Ok(out)
+}
+
+fn to_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let name_span = args.name_span();
+    let out = args.input;
+
+    Ok(out
+        .values
+        .map(
+            move |a| match bson_value_to_bytes(value_to_bson_value(&a), name_span) {
+                Ok(x) => ReturnSuccess::value(Value::Binary(x).simple_spanned(name_span)),
+                _ => Err(ShellError::labeled_error_with_secondary(
+                    "Expected an object with BSON-compatible structure from pipeline",
+                    "requires BSON-compatible input: Must be Array or Object",
+                    name_span,
+                    format!("{} originates from here", a.item.type_name()),
+                    a.span(),
+                )),
+            },
+        )
+        .to_output_stream())
+}

src/object/base.rs

@@ -243,6 +243,48 @@ impl std::convert::TryFrom<&'a Tagged<Value>> for i64 {
     }
 }
 
+impl std::convert::TryFrom<&'a Tagged<Value>> for String {
+    type Error = ShellError;
+
+    fn try_from(value: &'a Tagged<Value>) -> Result<String, ShellError> {
+        match value.item() {
+            Value::Primitive(Primitive::String(s)) => Ok(s.clone()),
+            v => Err(ShellError::type_error(
+                "String",
+                value.copy_span(v.type_name()),
+            )),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&'a Tagged<Value>> for Vec<u8> {
+    type Error = ShellError;
+
+    fn try_from(value: &'a Tagged<Value>) -> Result<Vec<u8>, ShellError> {
+        match value.item() {
+            Value::Binary(b) => Ok(b.clone()),
+            v => Err(ShellError::type_error(
+                "Binary",
+                value.copy_span(v.type_name()),
+            )),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&'a Tagged<Value>> for &'a crate::object::Dictionary {
+    type Error = ShellError;
+
+    fn try_from(value: &'a Tagged<Value>) -> Result<&'a crate::object::Dictionary, ShellError> {
+        match value.item() {
+            Value::Object(d) => Ok(d),
+            v => Err(ShellError::type_error(
+                "Dictionary",
+                value.copy_span(v.type_name()),
+            )),
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize)]
 pub enum Switch {
     Present,

tests/command_open_tests.rs

@@ -28,7 +28,7 @@ fn open_can_parse_bson_1() {
     nu!(
         output,
         cwd("tests/fixtures/formats"),
-        "open sample.bson | nth 0 | get b | echo $it"
+        "open sample.bson | get root | nth 0 | get b | echo $it"
     );
 
     assert_eq!(output, "hello");
@@ -39,7 +39,7 @@ fn open_can_parse_bson_2() {
     nu!(
         output,
         cwd("tests/fixtures/formats"),
-        "open sample.bson | nth 6 | get b | get '$binary_subtype' | echo $it "
+        "open sample.bson | get root | nth 6 | get b | get '$binary_subtype' | echo $it "
     );
 
     assert_eq!(output, "function");

tests/filters_test.rs

@@ -106,6 +106,17 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() {
     assert_eq!(output, "markup");
 }
 
+#[test]
+fn can_convert_json_text_to_bson_and_back_into_table() {
+    nu!(
+        output,
+        cwd("tests/fixtures/formats"),
+        "open sample.bson | to-bson | from-bson | get root | nth 1 | get b | echo $it"
+    );
+
+    assert_eq!(output, "whel");
+}
+
 #[test]
 fn can_convert_table_to_toml_text_and_from_toml_text_back_into_table() {
     nu!(