extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-08-19 13:11:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add wasm support (#2199)

Browse Source 
* Working towards a PoC for wasm

* Move bson and sqlite to plugins

* proof of concept now working

* tests are green

* Add CI test for --no-default-features

* Fix some tests

* Fix clippy and windows build

* More fixes

* Fix the windows build

* Fix the windows test
This commit is contained in:
Jonathan Turner
2020-07-18 13:59:23 +12:00
committed by GitHub
parent dbe0effd67
commit d8594a62c2
59 changed files with 781 additions and 448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
crates/nu-cli/src/commands/classified/external.rs
View File

@@ -26,7 +26,7 @@ pub(crate) async fn run_external_command(
) -> Result<InputStream, ShellError> {
trace!(target: "nu::run::external", "-> {}", command.name);
if !did_find_command(&command.name).await {
if !did_find_command(&command.name) {
return Err(ShellError::labeled_error(
"Command not found",
"command not found",
@@ -70,7 +70,18 @@ async fn run_with_stdin(
let process_args = command_args
.iter()
.map(|arg| {
let arg = expand_tilde(arg.deref(), dirs::home_dir);
let home_dir;
#[cfg(feature = "dirs")]
{
home_dir = dirs::home_dir;
}
#[cfg(not(feature = "dirs"))]
{
home_dir = || Some(std::path::PathBuf::from("/"));
}
let arg = expand_tilde(arg.deref(), home_dir);
#[cfg(not(windows))]
{
@@ -407,13 +418,19 @@ fn spawn(
}
}
async fn did_find_command(name: &str) -> bool {
#[cfg(not(windows))]
pub fn did_find_command(#[allow(unused)] name: &str) -> bool {
#[cfg(not(feature = "which"))]
{
// we can't perform this check, so just assume it can be found
true
}
#[cfg(all(feature = "which", unix))]
{
which::which(name).is_ok()
}
#[cfg(windows)]
#[cfg(all(feature = "which", windows))]
{
if which::which(name).is_ok() {
true
@@ -483,11 +500,17 @@ fn shell_os_paths() -> Vec<std::path::PathBuf> {
mod tests {
use super::{
add_quotes, argument_contains_whitespace, argument_is_quoted, expand_tilde, remove_quotes,
run_external_command, Context, InputStream,
};
#[cfg(feature = "which")]
use super::{run_external_command, Context, InputStream};
#[cfg(feature = "which")]
use futures::executor::block_on;
#[cfg(feature = "which")]
use nu_errors::ShellError;
#[cfg(feature = "which")]
use nu_protocol::Scope;
#[cfg(feature = "which")]
use nu_test_support::commands::ExternalBuilder;
// async fn read(mut stream: OutputStream) -> Option<Value> {
@@ -503,6 +526,7 @@ mod tests {
// }
// }
#[cfg(feature = "which")]
async fn non_existent_run() -> Result<(), ShellError> {
let cmd = ExternalBuilder::for_name("i_dont_exist.exe").build();
@@ -542,6 +566,7 @@ mod tests {
// block_on(failure_run())
// }
#[cfg(feature = "which")]
#[test]
fn identifies_command_not_found() -> Result<(), ShellError> {
block_on(non_existent_run())

244
crates/nu-cli/src/commands/from_bson.rs
View File

@@ -1,244 +0,0 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use bson::{decode_document, spec::BinarySubtype, Bson};
use nu_errors::{ExpectedRange, ShellError};
use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
use nu_source::SpannedItem;
use std::str::FromStr;
pub struct FromBSON;
#[async_trait]
impl WholeStreamCommand for FromBSON {
fn name(&self) -> &str {
"from bson"
}
fn signature(&self) -> Signature {
Signature::build("from bson")
}
fn usage(&self) -> &str {
"Parse binary as .bson and create table."
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_bson(args, registry).await
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Convert bson data to a table",
example: "open file.bin | from bson",
result: None,
}]
}
}
fn bson_array(input: &[Bson], tag: Tag) -> Result<Vec<Value>, ShellError> {
let mut out = vec![];
for value in input {
out.push(convert_bson_value_to_nu_value(value, &tag)?);
}
Ok(out)
}
fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value, ShellError> {
let tag = tag.into();
let span = tag.span;
Ok(match v {
Bson::FloatingPoint(n) => UntaggedValue::Primitive(Primitive::from(*n)).into_value(&tag),
Bson::String(s) => {
UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(&tag)
}
Bson::Array(a) => UntaggedValue::Table(bson_array(a, tag.clone())?).into_value(&tag),
Bson::Document(doc) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
for (k, v) in doc.iter() {
collected.insert_value(k.clone(), convert_bson_value_to_nu_value(v, &tag)?);
}
collected.into_value()
}
Bson::Boolean(b) => UntaggedValue::Primitive(Primitive::Boolean(*b)).into_value(&tag),
Bson::Null => UntaggedValue::Primitive(Primitive::Nothing).into_value(&tag),
Bson::RegExp(r, opts) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$regex".to_string(),
UntaggedValue::Primitive(Primitive::String(String::from(r))).into_value(&tag),
);
collected.insert_value(
"$options".to_string(),
UntaggedValue::Primitive(Primitive::String(String::from(opts))).into_value(&tag),
);
collected.into_value()
}
Bson::I32(n) => UntaggedValue::int(*n).into_value(&tag),
Bson::I64(n) => UntaggedValue::int(*n).into_value(&tag),
Bson::Decimal128(n) => {
// TODO: this really isn't great, and we should update this to do a higher
// fidelity translation
let decimal = BigDecimal::from_str(&format!("{}", n)).map_err(|_| {
ShellError::range_error(
ExpectedRange::BigDecimal,
&n.spanned(span),
"converting BSON Decimal128 to BigDecimal".to_owned(),
)
})?;
UntaggedValue::Primitive(Primitive::Decimal(decimal)).into_value(&tag)
}
Bson::JavaScriptCode(js) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$javascript".to_string(),
UntaggedValue::Primitive(Primitive::String(String::from(js))).into_value(&tag),
);
collected.into_value()
}
Bson::JavaScriptCodeWithScope(js, doc) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$javascript".to_string(),
UntaggedValue::Primitive(Primitive::String(String::from(js))).into_value(&tag),
);
collected.insert_value(
"$scope".to_string(),
convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag)?,
);
collected.into_value()
}
Bson::TimeStamp(ts) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$timestamp".to_string(),
UntaggedValue::int(*ts).into_value(&tag),
);
collected.into_value()
}
Bson::Binary(bst, bytes) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$binary_subtype".to_string(),
match bst {
BinarySubtype::UserDefined(u) => UntaggedValue::int(*u),
_ => {
UntaggedValue::Primitive(Primitive::String(binary_subtype_to_string(*bst)))
}
}
.into_value(&tag),
);
collected.insert_value(
"$binary".to_string(),
UntaggedValue::Primitive(Primitive::Binary(bytes.to_owned())).into_value(&tag),
);
collected.into_value()
}
Bson::ObjectId(obj_id) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$object_id".to_string(),
UntaggedValue::Primitive(Primitive::String(obj_id.to_hex())).into_value(&tag),
);
collected.into_value()
}
Bson::UtcDatetime(dt) => UntaggedValue::Primitive(Primitive::Date(*dt)).into_value(&tag),
Bson::Symbol(s) => {
let mut collected = TaggedDictBuilder::new(tag.clone());
collected.insert_value(
"$symbol".to_string(),
UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(&tag),
);
collected.into_value()
}
})
}
fn binary_subtype_to_string(bst: BinarySubtype) -> String {
match bst {
BinarySubtype::Generic => "generic",
BinarySubtype::Function => "function",
BinarySubtype::BinaryOld => "binary_old",
BinarySubtype::UuidOld => "uuid_old",
BinarySubtype::Uuid => "uuid",
BinarySubtype::Md5 => "md5",
_ => unreachable!(),
}
.to_string()
}
#[derive(Debug)]
struct BytesReader {
pos: usize,
inner: Vec<u8>,
}
impl BytesReader {
fn new(bytes: Vec<u8>) -> BytesReader {
BytesReader {
pos: 0,
inner: bytes,
}
}
}
impl std::io::Read for BytesReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let src: &mut &[u8] = &mut self.inner[self.pos..].as_ref();
let diff = src.read(buf)?;
self.pos += diff;
Ok(diff)
}
}
pub fn from_bson_bytes_to_value(bytes: Vec<u8>, tag: impl Into<Tag>) -> Result<Value, ShellError> {
let mut docs = Vec::new();
let mut b_reader = BytesReader::new(bytes);
while let Ok(v) = decode_document(&mut b_reader) {
docs.push(Bson::Document(v));
}
convert_bson_value_to_nu_value(&Bson::Array(docs), tag)
}
async fn from_bson(
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
let args = args.evaluate_once(&registry).await?;
let tag = args.name_tag();
let input = args.input;
let bytes = input.collect_binary(tag.clone()).await?;
match from_bson_bytes_to_value(bytes.item, tag.clone()) {
Ok(x) => Ok(OutputStream::one(ReturnSuccess::value(x))),
Err(_) => Err(ShellError::labeled_error_with_secondary(
"Could not parse as BSON",
"input cannot be parsed as BSON",
tag.clone(),
"value originates from here",
bytes.tag,
)),
}
}
#[cfg(test)]
mod tests {
use super::FromBSON;
#[test]
fn examples_work_as_expected() {
use crate::examples::test as test_examples;
test_examples(FromBSON {})
}
}

181
crates/nu-cli/src/commands/from_sqlite.rs
View File

@@ -1,181 +0,0 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use nu_errors::ShellError;
use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value};
use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
use std::io::Write;
use std::path::Path;
pub struct FromSQLite;
#[async_trait]
impl WholeStreamCommand for FromSQLite {
fn name(&self) -> &str {
"from sqlite"
}
fn signature(&self) -> Signature {
Signature::build("from sqlite")
}
fn usage(&self) -> &str {
"Parse binary data as sqlite .db and create table."
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry).await
}
}
pub struct FromDB;
#[async_trait]
impl WholeStreamCommand for FromDB {
fn name(&self) -> &str {
"from db"
}
fn signature(&self) -> Signature {
Signature::build("from db")
}
fn usage(&self) -> &str {
"Parse binary data as db and create table."
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry).await
}
}
pub fn convert_sqlite_file_to_nu_value(
path: &Path,
tag: impl Into<Tag> + Clone,
) -> Result<Value, rusqlite::Error> {
let conn = Connection::open(path)?;
let mut meta_out = Vec::new();
let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
while let Some(meta_row) = meta_rows.next()? {
let table_name: String = meta_row.get(0)?;
let mut meta_dict = TaggedDictBuilder::new(tag.clone());
let mut out = Vec::new();
let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?;
let mut table_rows = table_stmt.query(NO_PARAMS)?;
while let Some(table_row) = table_rows.next()? {
out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?)
}
meta_dict.insert_value(
"table_name".to_string(),
UntaggedValue::Primitive(Primitive::String(table_name)).into_value(tag.clone()),
);
meta_dict.insert_value(
"table_values",
UntaggedValue::Table(out).into_value(tag.clone()),
);
meta_out.push(meta_dict.into_value());
}
let tag = tag.into();
Ok(UntaggedValue::Table(meta_out).into_value(tag))
}
fn convert_sqlite_row_to_nu_value(
row: &Row,
tag: impl Into<Tag> + Clone,
) -> Result<Value, rusqlite::Error> {
let mut collected = TaggedDictBuilder::new(tag.clone());
for (i, c) in row.columns().iter().enumerate() {
collected.insert_value(
c.name().to_string(),
convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
);
}
Ok(collected.into_value())
}
fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Value {
match value {
ValueRef::Null => {
UntaggedValue::Primitive(Primitive::String(String::from(""))).into_value(tag)
}
ValueRef::Integer(i) => UntaggedValue::int(i).into_value(tag),
ValueRef::Real(f) => UntaggedValue::decimal(f).into_value(tag),
ValueRef::Text(s) => {
// this unwrap is safe because we know the ValueRef is Text.
UntaggedValue::Primitive(Primitive::String(String::from_utf8_lossy(s).to_string()))
.into_value(tag)
}
ValueRef::Blob(u) => UntaggedValue::binary(u.to_owned()).into_value(tag),
}
}
pub fn from_sqlite_bytes_to_value(
mut bytes: Vec<u8>,
tag: impl Into<Tag> + Clone,
) -> Result<Value, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
tempfile.write_all(bytes.as_mut_slice())?;
match convert_sqlite_file_to_nu_value(tempfile.path(), tag) {
Ok(value) => Ok(value),
Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
}
}
async fn from_sqlite(
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
let args = args.evaluate_once(&registry).await?;
let tag = args.name_tag();
let input = args.input;
let bytes = input.collect_binary(tag.clone()).await?;
match from_sqlite_bytes_to_value(bytes.item, tag.clone()) {
Ok(x) => match x {
Value {
value: UntaggedValue::Table(list),
..
} => Ok(futures::stream::iter(list).to_output_stream()),
_ => Ok(OutputStream::one(x)),
},
Err(err) => {
println!("{:?}", err);
Err(ShellError::labeled_error_with_secondary(
"Could not parse as SQLite",
"input cannot be parsed as SQLite",
&tag,
"value originates from here",
bytes.tag,
))
}
}
}
#[cfg(test)]
mod tests {
use super::FromSQLite;
#[test]
fn examples_work_as_expected() {
use crate::examples::test as test_examples;
test_examples(FromSQLite {})
}
}

2
crates/nu-cli/src/commands/random/mod.rs
View File

@@ -2,10 +2,12 @@ pub mod command;
pub mod bool;
pub mod dice;
#[cfg(feature = "uuid_crate")]
pub mod uuid;
pub use command::Command as Random;
pub use self::bool::SubCommand as RandomBool;
pub use dice::SubCommand as RandomDice;
#[cfg(feature = "uuid_crate")]
pub use uuid::SubCommand as RandomUUID;

2
crates/nu-cli/src/commands/run_external.rs
View File

@@ -157,7 +157,7 @@ async fn maybe_autocd_dir<'a>(cmd: &ExternalCommand, ctx: &mut Context) -> Optio
|| (cmd.args.is_empty()
&& PathBuf::from(name).is_dir()
&& dunce::canonicalize(name).is_ok()
&& which::which(&name).is_err())
&& !crate::commands::classified::external::did_find_command(&name))
{
Some(name)
} else {

323
crates/nu-cli/src/commands/to_bson.rs
View File

@@ -1,323 +0,0 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use bson::{encode_document, oid::ObjectId, spec::BinarySubtype, Bson, Document};
use nu_errors::{CoerceInto, ShellError};
use nu_protocol::{
Dictionary, Primitive, ReturnSuccess, Signature, SpannedTypeName, UnspannedPathMember,
UntaggedValue, Value,
};
use std::convert::TryInto;
pub struct ToBSON;
#[async_trait]
impl WholeStreamCommand for ToBSON {
fn name(&self) -> &str {
"to bson"
}
fn signature(&self) -> Signature {
Signature::build("to bson")
}
fn usage(&self) -> &str {
"Convert table into .bson text."
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_bson(args, registry).await
}
fn is_binary(&self) -> bool {
true
}
}
pub fn value_to_bson_value(v: &Value) -> Result<Bson, ShellError> {
Ok(match &v.value {
UntaggedValue::Primitive(Primitive::Boolean(b)) => Bson::Boolean(*b),
// FIXME: What about really big decimals?
UntaggedValue::Primitive(Primitive::Filesize(decimal)) => Bson::FloatingPoint(
(decimal)
.to_f64()
.expect("Unimplemented BUG: What about big decimals?"),
),
UntaggedValue::Primitive(Primitive::Duration(i)) => Bson::String(i.to_string()),
UntaggedValue::Primitive(Primitive::Date(d)) => Bson::UtcDatetime(*d),
UntaggedValue::Primitive(Primitive::EndOfStream) => Bson::Null,
UntaggedValue::Primitive(Primitive::BeginningOfStream) => Bson::Null,
UntaggedValue::Primitive(Primitive::Decimal(d)) => {
Bson::FloatingPoint(d.to_f64().ok_or_else(|| {
ShellError::labeled_error(
"Could not convert value to decimal",
"could not convert to decimal",
&v.tag,
)
})?)
}
UntaggedValue::Primitive(Primitive::Int(i)) => {
Bson::I64(i.tagged(&v.tag).coerce_into("converting to BSON")?)
}
UntaggedValue::Primitive(Primitive::Nothing) => Bson::Null,
UntaggedValue::Primitive(Primitive::String(s)) => Bson::String(s.clone()),
UntaggedValue::Primitive(Primitive::Line(s)) => Bson::String(s.clone()),
UntaggedValue::Primitive(Primitive::ColumnPath(path)) => Bson::Array(
path.iter()
.map(|x| match &x.unspanned {
UnspannedPathMember::String(string) => Ok(Bson::String(string.clone())),
UnspannedPathMember::Int(int) => Ok(Bson::I64(
int.tagged(&v.tag).coerce_into("converting to BSON")?,
)),
})
.collect::<Result<Vec<Bson>, ShellError>>()?,
),
UntaggedValue::Primitive(Primitive::Pattern(p)) => Bson::String(p.clone()),
UntaggedValue::Primitive(Primitive::Path(s)) => Bson::String(s.display().to_string()),
UntaggedValue::Table(l) => Bson::Array(
l.iter()
.map(|x| value_to_bson_value(x))
.collect::<Result<_, _>>()?,
),
UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => Bson::Null,
UntaggedValue::Error(e) => return Err(e.clone()),
UntaggedValue::Primitive(Primitive::Binary(b)) => {
Bson::Binary(BinarySubtype::Generic, b.clone())
}
UntaggedValue::Row(o) => object_value_to_bson(o)?,
})
}
// object_value_to_bson handles all Objects, even those that correspond to special
// types (things like regex or javascript code).
fn object_value_to_bson(o: &Dictionary) -> Result<Bson, ShellError> {
let mut it = o.entries.iter();
if it.len() > 2 {
return generic_object_value_to_bson(o);
}
match it.next() {
Some((regex, tagged_regex_value)) if regex == "$regex" => match it.next() {
Some((options, tagged_opts_value)) if options == "$options" => {
let r: Result<String, _> = tagged_regex_value.try_into();
let opts: Result<String, _> = tagged_opts_value.try_into();
match (r, opts) {
(Ok(r), Ok(opts)) => Ok(Bson::RegExp(r, opts)),
_ => generic_object_value_to_bson(o),
}
}
_ => generic_object_value_to_bson(o),
},
Some((javascript, tagged_javascript_value)) if javascript == "$javascript" => {
match it.next() {
Some((scope, tagged_scope_value)) if scope == "$scope" => {
let js: Result<String, _> = tagged_javascript_value.try_into();
let s: Result<&Dictionary, _> = tagged_scope_value.try_into();
match (js, s) {
(Ok(js), Ok(s)) => {
if let Bson::Document(doc) = object_value_to_bson(s)? {
Ok(Bson::JavaScriptCodeWithScope(js, doc))
} else {
generic_object_value_to_bson(o)
}
}
_ => generic_object_value_to_bson(o),
}
}
None => {
let js: Result<String, _> = tagged_javascript_value.try_into();
match js {
Err(_) => generic_object_value_to_bson(o),
Ok(v) => Ok(Bson::JavaScriptCode(v)),
}
}
_ => generic_object_value_to_bson(o),
}
}
Some((timestamp, tagged_timestamp_value)) if timestamp == "$timestamp" => {
let ts: Result<i64, _> = tagged_timestamp_value.try_into();
if let Ok(ts) = ts {
Ok(Bson::TimeStamp(ts))
} else {
generic_object_value_to_bson(o)
}
}
Some((binary_subtype, tagged_binary_subtype_value))
if binary_subtype == "$binary_subtype" =>
{
match it.next() {
Some((binary, tagged_bin_value)) if binary == "$binary" => {
let bst = get_binary_subtype(tagged_binary_subtype_value);
let bin: Result<Vec<u8>, _> = tagged_bin_value.try_into();
match (bin, bst) {
(Ok(bin), Ok(v)) => Ok(Bson::Binary(v, bin)),
_ => generic_object_value_to_bson(o),
}
}
_ => generic_object_value_to_bson(o),
}
}
Some((object_id, tagged_object_id_value)) if object_id == "$object_id" => {
let obj_id: Result<String, _> = tagged_object_id_value.try_into();
if let Ok(obj_id) = obj_id {
let obj_id = ObjectId::with_string(&obj_id);
if let Ok(obj_id) = obj_id {
Ok(Bson::ObjectId(obj_id))
} else {
generic_object_value_to_bson(o)
}
} else {
generic_object_value_to_bson(o)
}
}
Some((symbol, tagged_symbol_value)) if symbol == "$symbol" => {
let sym: Result<String, _> = tagged_symbol_value.try_into();
if let Ok(sym) = sym {
Ok(Bson::Symbol(sym))
} else {
generic_object_value_to_bson(o)
}
}
_ => generic_object_value_to_bson(o),
}
}
fn get_binary_subtype(tagged_value: &Value) -> Result<BinarySubtype, ShellError> {
match &tagged_value.value {
UntaggedValue::Primitive(Primitive::String(s)) => Ok(match s.as_ref() {
"generic" => BinarySubtype::Generic,
"function" => BinarySubtype::Function,
"binary_old" => BinarySubtype::BinaryOld,
"uuid_old" => BinarySubtype::UuidOld,
"uuid" => BinarySubtype::Uuid,
"md5" => BinarySubtype::Md5,
_ => unreachable!(),
}),
UntaggedValue::Primitive(Primitive::Int(i)) => Ok(BinarySubtype::UserDefined(
i.tagged(&tagged_value.tag)
.coerce_into("converting to BSON binary subtype")?,
)),
_ => Err(ShellError::type_error(
"bson binary",
tagged_value.spanned_type_name(),
)),
}
}
// generic_object_value_bson handles any Object that does not
// correspond to a special bson type (things like regex or javascript code).
fn generic_object_value_to_bson(o: &Dictionary) -> Result<Bson, ShellError> {
let mut doc = Document::new();
for (k, v) in o.entries.iter() {
doc.insert(k.clone(), value_to_bson_value(v)?);
}
Ok(Bson::Document(doc))
}
fn shell_encode_document(writer: &mut Vec<u8>, doc: Document, tag: Tag) -> Result<(), ShellError> {
match encode_document(writer, &doc) {
Err(e) => Err(ShellError::labeled_error(
format!("Failed to encode document due to: {:?}", e),
"requires BSON-compatible document",
tag,
)),
_ => Ok(()),
}
}
fn bson_value_to_bytes(bson: Bson, tag: Tag) -> Result<Vec<u8>, ShellError> {
let mut out = Vec::new();
match bson {
Bson::Array(a) => {
for v in a.into_iter() {
match v {
Bson::Document(d) => shell_encode_document(&mut out, d, tag.clone())?,
_ => {
return Err(ShellError::labeled_error(
format!("All top level values must be Documents, got {:?}", v),
"requires BSON-compatible document",
&tag,
))
}
}
}
}
Bson::Document(d) => shell_encode_document(&mut out, d, tag)?,
_ => {
return Err(ShellError::labeled_error(
format!("All top level values must be Documents, got {:?}", bson),
"requires BSON-compatible document",
tag,
))
}
}
Ok(out)
}
async fn to_bson(
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
let args = args.evaluate_once(&registry).await?;
let name_tag = args.name_tag();
let name_span = name_tag.span;
let input: Vec<Value> = args.input.collect().await;
let to_process_input = match input.len() {
x if x > 1 => {
let tag = input[0].tag.clone();
vec![Value {
value: UntaggedValue::Table(input),
tag,
}]
}
1 => input,
_ => vec![],
};
Ok(futures::stream::iter(to_process_input.into_iter().map(
move |value| match value_to_bson_value(&value) {
Ok(bson_value) => {
let value_span = value.tag.span;
match bson_value_to_bytes(bson_value, name_tag.clone()) {
Ok(x) => ReturnSuccess::value(UntaggedValue::binary(x).into_value(&name_tag)),
_ => Err(ShellError::labeled_error_with_secondary(
"Expected a table with BSON-compatible structure from pipeline",
"requires BSON-compatible input",
name_span,
"originates from here".to_string(),
value_span,
)),
}
}
_ => Err(ShellError::labeled_error(
"Expected a table with BSON-compatible structure from pipeline",
"requires BSON-compatible input",
&name_tag,
)),
},
))
.to_output_stream())
}
#[cfg(test)]
mod tests {
use super::ToBSON;
#[test]
fn examples_work_as_expected() {
use crate::examples::test as test_examples;
test_examples(ToBSON {})
}
}

235
crates/nu-cli/src/commands/to_sqlite.rs
View File

@@ -1,235 +0,0 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use hex::encode;
use nu_errors::ShellError;
use nu_protocol::{Dictionary, Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
use rusqlite::{Connection, NO_PARAMS};
use std::io::Read;
pub struct ToSQLite;
#[async_trait]
impl WholeStreamCommand for ToSQLite {
fn name(&self) -> &str {
"to sqlite"
}
fn signature(&self) -> Signature {
Signature::build("to sqlite")
}
fn usage(&self) -> &str {
"Convert table to sqlite .db binary data"
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry).await
}
fn is_binary(&self) -> bool {
true
}
}
pub struct ToDB;
#[async_trait]
impl WholeStreamCommand for ToDB {
fn name(&self) -> &str {
"to db"
}
fn signature(&self) -> Signature {
Signature::build("to db")
}
fn usage(&self) -> &str {
"Convert table to db data"
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry).await
}
fn is_binary(&self) -> bool {
true
}
}
fn comma_concat(acc: String, current: String) -> String {
if acc == "" {
current
} else {
format!("{}, {}", acc, current)
}
}
fn get_columns(rows: &[Value]) -> Result<String, std::io::Error> {
match &rows[0].value {
UntaggedValue::Row(d) => Ok(d
.entries
.iter()
.map(|(k, _v)| k.clone())
.fold("".to_string(), comma_concat)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
}
}
fn nu_value_to_sqlite_string(v: Value) -> String {
match &v.value {
UntaggedValue::Primitive(p) => match p {
Primitive::Nothing => "NULL".into(),
Primitive::Int(i) => format!("{}", i),
Primitive::Duration(i) => format!("{}", i),
Primitive::Decimal(f) => format!("{}", f),
Primitive::Filesize(u) => format!("{}", u),
Primitive::Pattern(s) => format!("'{}'", s.replace("'", "''")),
Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
Primitive::Line(s) => format!("'{}'", s.replace("'", "''")),
Primitive::Boolean(true) => "1".into(),
Primitive::Boolean(_) => "0".into(),
Primitive::Date(d) => format!("'{}'", d),
Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
Primitive::Binary(u) => format!("x'{}'", encode(u)),
Primitive::BeginningOfStream
| Primitive::EndOfStream
| Primitive::ColumnPath(_)
| Primitive::Range(_) => "NULL".into(),
},
_ => "NULL".into(),
}
}
fn get_insert_values(rows: Vec<Value>) -> Result<String, std::io::Error> {
let values: Result<Vec<_>, _> = rows
.into_iter()
.map(|value| match value.value {
UntaggedValue::Row(d) => Ok(format!(
"({})",
d.entries
.iter()
.map(|(_k, v)| nu_value_to_sqlite_string(v.clone()))
.fold("".to_string(), comma_concat)
)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
})
.collect();
let values = values?;
Ok(values.into_iter().fold("".to_string(), comma_concat))
}
fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
let table_name = match table.entries.get("table_name") {
Some(Value {
value: UntaggedValue::Primitive(Primitive::String(table_name)),
..
}) => table_name,
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table name",
))
}
};
let (columns, insert_values) = match table.entries.get("table_values") {
Some(Value {
value: UntaggedValue::Table(l),
..
}) => (get_columns(l), get_insert_values(l.to_vec())),
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table values",
))
}
};
let create = format!("create table {}({})", table_name, columns?);
let insert = format!("insert into {} values {}", table_name, insert_values?);
Ok((create, insert))
}
fn sqlite_input_stream_to_bytes(values: Vec<Value>) -> Result<Value, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
let conn = match Connection::open(tempfile.path()) {
Ok(conn) => conn,
Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
};
let tag = values[0].tag.clone();
for value in values.into_iter() {
match &value.value {
UntaggedValue::Row(d) => {
let (create, insert) = generate_statements(d.to_owned())?;
match conn
.execute(&create, NO_PARAMS)
.and_then(|_| conn.execute(&insert, NO_PARAMS))
{
Ok(_) => (),
Err(e) => {
outln!("{}", create);
outln!("{}", insert);
outln!("{:?}", e);
return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
}
}
}
other => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Expected row, found {:?}", other),
))
}
}
}
let mut out = Vec::new();
tempfile.read_to_end(&mut out)?;
Ok(UntaggedValue::binary(out).into_value(tag))
}
async fn to_sqlite(
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
let args = args.evaluate_once(&registry).await?;
let name_tag = args.name_tag();
let input: Vec<Value> = args.input.collect().await;
match sqlite_input_stream_to_bytes(input) {
Ok(out) => Ok(OutputStream::one(ReturnSuccess::value(out))),
_ => Err(ShellError::labeled_error(
"Expected a table with SQLite-compatible structure from pipeline",
"requires SQLite-compatible input",
name_tag,
)),
}
}
#[cfg(test)]
mod tests {
use super::ToSQLite;
#[test]
fn examples_work_as_expected() {
use crate::examples::test as test_examples;
test_examples(ToSQLite {})
}
}

18
crates/nu-cli/src/commands/which_.rs
View File

@@ -60,6 +60,7 @@ macro_rules! entry_builtin {
};
}
#[allow(unused)]
macro_rules! entry_path {
($arg:expr, $path:expr, $tag:expr) => {
entry(
@@ -99,13 +100,16 @@ async fn which(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputSt
}
}
if let Ok(paths) = ichwh::which_all(&item).await {
for path in paths {
output.push(ReturnSuccess::value(entry_path!(
item,
path.into(),
application.tag.clone()
)));
#[cfg(feature = "ichwh")]
{
if let Ok(paths) = ichwh::which_all(&item).await {
for path in paths {
output.push(ReturnSuccess::value(entry_path!(
item,
path.into(),
application.tag.clone()
)));
}
}
}