Merge pull request #398 from pmeredit/topic/support_bson

Topic/support bson
This commit is contained in:
Andrés N. Robalino 2019-08-25 11:58:28 -05:00 committed by GitHub
commit de930daf33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 330 additions and 4 deletions

53
Cargo.lock generated
View File

@ -149,6 +149,25 @@ name = "block"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bson"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
"hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
"linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"md5 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"try_from 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bstr"
version = "0.2.6"
@ -1214,6 +1233,15 @@ name = "hex"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "hostname"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
"winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "http"
version = "0.1.18"
@ -1510,6 +1538,11 @@ name = "matches"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "md5"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr"
version = "2.2.1"
@ -1650,6 +1683,7 @@ dependencies = [
"ansi_term 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"battery 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bson 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-unit 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2562,6 +2596,7 @@ name = "serde_json"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2900,6 +2935,11 @@ dependencies = [
"serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "try_from"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "typenum"
version = "1.10.0"
@ -3186,6 +3226,14 @@ dependencies = [
"winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winutil"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "x11"
version = "2.18.1"
@ -3254,6 +3302,7 @@ dependencies = [
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
"checksum blake2b_simd 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "461f4b879a8eb70c1debf7d0788a9a5ff15f1ea9d25925fea264ef4258bed6b2"
"checksum block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
"checksum bson 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8984b7b33b1f8ac97468df3cefa76c7035abb0786473aa2a437dea0c72855702"
"checksum bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e0a692f1c740e7e821ca71a22cf99b9b2322dfa94d10f71443befb1797b3946a"
"checksum bumpalo 2.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2cd43d82f27d68911e6ee11ee791fb248f138f5d69424dc02e098d4f152b0b05"
"checksum byte-unit 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90139954ec9776c4832d44f212e558ccdacbe915a881bf3de3a1a487fa8d1e87"
@ -3364,6 +3413,7 @@ dependencies = [
"checksum heim-runtime 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b3304dc68b138eb6e9b6c79785dd911306a4bca66757a88373cb034a4dfe3a4d"
"checksum heim-virt 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6f68f73f66e6f00404d7b8ed97b458778f6ccafe1ecd65af797ec36f2003bf90"
"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
"checksum hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb46a83a85e824ef93669c8b390009623863b5c195d1ba747292c0c72f94e"
"checksum http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "372bcb56f939e449117fb0869c2e8fd8753a8223d92a172c6e808cf123a5b6e4"
"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114"
"checksum ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
@ -3400,6 +3450,7 @@ dependencies = [
"checksum mach 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
"checksum malloc_buf 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"
"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
"checksum md5 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48"
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
"checksum memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce6075db033bbbb7ee5a0bbd3a3186bbae616f57fb001c485c7ff77955f8177f"
@ -3548,6 +3599,7 @@ dependencies = [
"checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926"
"checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f"
"checksum toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c7aabe75941d914b72bf3e5d3932ed92ce0664d49d8432305a8b547c37227724"
"checksum try_from 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "923a7ee3e97dbfe8685261beb4511cc9620a1252405d02693d43169729570111"
"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
"checksum unicase 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a84e5511b2a947f3ae965dcb29b13b7b1691b6e7332cf5dbc1744138d5acb7f6"
"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
@ -3585,6 +3637,7 @@ dependencies = [
"checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba"
"checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e"
"checksum x11 2.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39697e3123f715483d311b5826e254b6f3cfebdd83cf7ef3358f579c3d68e235"
"checksum x11-clipboard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "89bd49c06c9eb5d98e6ba6536cf64ac9f7ee3a009b2f53996d405b3944f6bcea"
"checksum xcb 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5e917a3f24142e9ff8be2414e36c649d47d6cc2ba81f16201cdef96e533e02de"

View File

@ -37,6 +37,7 @@ bytes = "0.4.12"
log = "0.4.8"
pretty_env_logger = "0.3.1"
serde = { version = "1.0.98", features = ["derive"] }
bson = "0.13.0"
serde_json = "1.0.40"
serde-hjson = "0.9.1"
serde_yaml = "0.8"

View File

@ -173,6 +173,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(FromArray),
whole_stream_command(FromCSV),
whole_stream_command(FromINI),
whole_stream_command(FromBSON),
whole_stream_command(FromJSON),
whole_stream_command(FromTOML),
whole_stream_command(FromXML),

View File

@ -15,6 +15,7 @@ crate mod enter;
crate mod exit;
crate mod first;
crate mod from_array;
crate mod from_bson;
crate mod from_csv;
crate mod from_ini;
crate mod from_json;
@ -70,6 +71,7 @@ crate use enter::Enter;
crate use exit::Exit;
crate use first::First;
crate use from_array::FromArray;
crate use from_bson::FromBSON;
crate use from_csv::FromCSV;
crate use from_ini::FromINI;
crate use from_json::FromJSON;

View File

@ -176,7 +176,7 @@ impl InternalCommand {
match contents {
Value::Primitive(Primitive::String(string)) => {
let value = crate::commands::open::parse_as_value(
let value = crate::commands::open::parse_string_as_value(
file_extension,
string,
contents_tag,

204
src/commands/from_bson.rs Normal file
View File

@ -0,0 +1,204 @@
use crate::commands::WholeStreamCommand;
use crate::object::base::OF64;
use crate::object::{Primitive, TaggedDictBuilder, Value};
use crate::prelude::*;
use bson::{decode_document, Bson, spec::BinarySubtype};
pub struct FromBSON;
impl WholeStreamCommand for FromBSON {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_bson(args, registry)
}
fn name(&self) -> &str {
"from-bson"
}
fn signature(&self) -> Signature {
Signature::build("from-bson")
}
}
fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value> {
let tag = tag.into();
match v {
Bson::FloatingPoint(n) => Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag),
Bson::String(s) => Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
Bson::Array(a) => Value::List(
a.iter()
.map(|x| convert_bson_value_to_nu_value(x, tag))
.collect(),
)
.tagged(tag),
Bson::Document(doc) => {
let mut collected = TaggedDictBuilder::new(tag);
for (k, v) in doc.iter() {
collected.insert_tagged(k.clone(), convert_bson_value_to_nu_value(v, tag));
}
collected.into_tagged_value()
}
Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
Bson::RegExp(r, opts) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(
"$regex".to_string(),
Value::Primitive(Primitive::String(String::from(r))).tagged(tag),
);
collected.insert_tagged(
"$options".to_string(),
Value::Primitive(Primitive::String(String::from(opts))).tagged(tag),
);
collected.into_tagged_value()
}
Bson::I32(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag),
Bson::I64(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag),
Bson::JavaScriptCode(js) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(
"$javascript".to_string(),
Value::Primitive(Primitive::String(String::from(js))).tagged(tag),
);
collected.into_tagged_value()
}
Bson::JavaScriptCodeWithScope(js, doc) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(
"$javascript".to_string(),
Value::Primitive(Primitive::String(String::from(js))).tagged(tag),
);
collected.insert_tagged(
"$scope".to_string(),
convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag),
);
collected.into_tagged_value()
}
Bson::TimeStamp(ts) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(
"$timestamp".to_string(),
Value::Primitive(Primitive::Int(*ts as i64)).tagged(tag),
);
collected.into_tagged_value()
}
Bson::Binary(bst, bytes) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(
"$binary_subtype".to_string(),
match bst {
BinarySubtype::UserDefined(u) => Value::Primitive(Primitive::Int(*u as i64)),
_ => Value::Primitive(Primitive::String(binary_subtype_to_string(*bst))),
}.tagged(tag)
);
collected.insert_tagged(
"$binary".to_string(),
Value::Binary(bytes.to_owned()).tagged(tag),
);
collected.into_tagged_value()
}
Bson::ObjectId(obj_id) => Value::Primitive(Primitive::String(obj_id.to_hex())).tagged(tag),
Bson::UtcDatetime(dt) => Value::Primitive(Primitive::Date(*dt)).tagged(tag),
Bson::Symbol(s) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(
"$symbol".to_string(),
Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
);
collected.into_tagged_value()
}
}
}
fn binary_subtype_to_string(bst: BinarySubtype) -> String {
match bst {
BinarySubtype::Generic => "generic",
BinarySubtype::Function => "function",
BinarySubtype::BinaryOld => "binary_old",
BinarySubtype::UuidOld => "uuid_old",
BinarySubtype::Uuid => "uuid",
BinarySubtype::Md5 => "md5",
_ => unreachable!(),
}.to_string()
}
#[derive(Debug)]
struct BytesReader {
pos: usize,
inner: Vec<u8>,
}
impl BytesReader {
fn new(bytes: Vec<u8>) -> BytesReader {
BytesReader {
pos: 0,
inner: bytes,
}
}
}
impl std::io::Read for BytesReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let src: &mut &[u8] = &mut self.inner[self.pos..].as_ref();
let diff = src.read(buf)?;
self.pos += diff;
Ok(diff)
}
}
pub fn from_bson_bytes_to_value(
bytes: Vec<u8>,
tag: impl Into<Tag>,
) -> bson::DecoderResult<Tagged<Value>> {
let mut docs = Vec::new();
let mut b_reader = BytesReader::new(bytes);
while let Ok(v) = decode_document(&mut b_reader) {
docs.push(Bson::Document(v));
}
Ok(convert_bson_value_to_nu_value(&Bson::Array(docs), tag))
}
fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let span = args.name_span();
let input = args.input;
let stream = async_stream_block! {
let values: Vec<Tagged<Value>> = input.values.collect().await;
for value in values {
let value_tag = value.tag();
match value.item {
Value::Binary(vb) =>
match from_bson_bytes_to_value(vb, span) {
Ok(x) => yield ReturnSuccess::value(x),
Err(_) => {
yield Err(ShellError::labeled_error_with_secondary(
"Could not parse as BSON",
"input cannot be parsed as BSON",
span,
"value originates from here",
value_tag.span,
))
}
}
_ => yield Err(ShellError::labeled_error_with_secondary(
"Expected a string from pipeline",
"requires string input",
span,
"value originates from here",
value_tag.span,
)),
}
}
};
Ok(stream.to_output_stream())
}

View File

@ -59,9 +59,12 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
let file_extension = if has_raw {
None
} else {
file_extension
// If the extension could not be determined via mimetype, try to use the path
// extension. Some file types do not declare their mimetypes (such as bson files).
file_extension.or(path_str.split('.').last().map(String::from))
};
if let Some(uuid) = contents_tag.origin {
// If we have loaded something, track its source
yield ReturnSuccess::action(CommandAction::AddSpanSource(
@ -72,7 +75,7 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
match contents {
Value::Primitive(Primitive::String(string)) => {
let value = parse_as_value(file_extension, string, contents_tag, name_span).unwrap();
let value = parse_string_as_value(file_extension, string, contents_tag, name_span).unwrap();
match value {
Tagged {
@ -86,7 +89,21 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
x => yield ReturnSuccess::value(x),
}
}
Value::Binary(binary) => {
let value = parse_binary_as_value(file_extension, binary, contents_tag, name_span).unwrap();
match value {
Tagged {
item: Value::List(list),
..
} => {
for elem in list {
yield ReturnSuccess::value(elem);
}
}
x => yield ReturnSuccess::value(x),
}
}
other => yield ReturnSuccess::value(other.tagged(contents_tag)),
};
};
@ -402,7 +419,7 @@ fn read_be_u16(input: &[u8]) -> Option<Vec<u16>> {
}
}
pub fn parse_as_value(
pub fn parse_string_as_value(
extension: Option<String>,
contents: String,
contents_tag: Tag,
@ -477,3 +494,25 @@ pub fn parse_as_value(
_ => Ok(Value::string(contents).tagged(contents_tag)),
}
}
pub fn parse_binary_as_value(
extension: Option<String>,
contents: Vec<u8>,
contents_tag: Tag,
name_span: Span,
) -> Result<Tagged<Value>, ShellError> {
match extension {
Some(x) if x == "bson" => {
crate::commands::from_bson::from_bson_bytes_to_value(contents, contents_tag).map_err(
move |_| {
ShellError::labeled_error(
"Could not open as BSON",
"could not open as BSON",
name_span,
)
},
)
}
_ => Ok(Value::Binary(contents).tagged(contents_tag)),
}
}

View File

@ -196,6 +196,10 @@ mod tests {
loc: fixtures().join("jonathan.xml"),
at: 0
},
Res {
loc: fixtures().join("sample.bson"),
at: 0
},
Res {
loc: fixtures().join("sample.ini"),
at: 0

View File

@ -24,6 +24,28 @@ fn open_can_parse_csv() {
assert_eq!(output, "SPAIN");
}
#[test]
fn open_can_parse_bson_1() {
nu!(
output,
cwd("tests/fixtures/formats"),
"open sample.bson | nth 0 | get b | echo $it"
);
assert_eq!(output, "hello");
}
#[test]
fn open_can_parse_bson_2() {
nu!(
output,
cwd("tests/fixtures/formats"),
"open sample.bson | nth 6 | get b | get '$binary_subtype' | echo $it "
);
assert_eq!(output, "function");
}
#[test]
fn open_can_parse_toml() {
nu!(

BIN
tests/fixtures/formats/sample.bson vendored Normal file

Binary file not shown.