holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge master

Browse Source
This commit is contained in:
Jonathan Turner 2019-08-31 13:30:41 +12:00
commit 2cec8168c7
12 changed files with 543 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
Cargo.lock generated
View File

@ -715,6 +715,16 @@ dependencies = [
"synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fixedbitset"
version = "0.1.9"
@ -1308,6 +1318,16 @@ dependencies = [
"libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libsqlite3-sys"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libz-sys"
version = "1.0.25"
@ -1358,6 +1378,14 @@ dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lzw"
version = "0.10.0"
@ -1576,6 +1604,7 @@ dependencies = [
"git2 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
"glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"heim 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
"hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.22.1 (registry+https://github.com/rust-lang/crates.io-index)",
"indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1596,6 +1625,7 @@ dependencies = [
"rawkey 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustyline 5.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2244,6 +2274,20 @@ dependencies = [
"xmlparser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rusqlite"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rust-argon2"
version = "0.5.0"
@ -3183,6 +3227,8 @@ dependencies = [
"checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
"checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
"checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
"checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
"checksum flate2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "550934ad4808d5d39365e5d61727309bf18b3b02c6c56b729cb92e7dd84bc3d8"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@ -3242,12 +3288,14 @@ dependencies = [
"checksum libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)" = "d44e80633f007889c7eff624b709ab43c92d708caad982295768a7b13ca3b5eb"
"checksum libgit2-sys 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4c179ed6d19cd3a051e68c177fbbc214e79ac4724fac3a850ec9f3d3eb8a5578"
"checksum libnghttp2-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02254d44f4435dd79e695f2c2b83cd06a47919adea30216ceaf0c57ca0a72463"
"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25"
"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
"checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c"
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
"checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
"checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084"
"checksum macaddr 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ff4752cb15cffb3e68f7dcb22e0818ac871f8c98fb07a634a81f41fb202a09f"
"checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
@ -3342,6 +3390,7 @@ dependencies = [
"checksum render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68ed587df09cfb7ce1bc6fe8f77e24db219f222c049326ccbfb948ec67e31664"
"checksum result 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560"
"checksum roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "153c367ce9fb8ef7afe637ef92bd083ba0f88b03ef3fcf0287d40be05ae0a61c"
"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051"
"checksum rust-argon2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81ed8d04228b44a740c8d46ff872a28e50fff3d659f307ab4da2cc502e019ff3"
"checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
"checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af"

5
Cargo.toml
View File

@ -62,6 +62,7 @@ mime = "0.3.13"
regex = "1.2.1"
pretty-hex = "0.1.0"
neso = { version = "0.5.0", optional = true }
hex = "0.3.2"
crossterm = "0.10.2"
tempfile = "3.1.0"
image = "0.22.1"
@ -82,6 +83,10 @@ pin-utils = "0.1.0-alpha.4"
[features]
raw-key = ["rawkey", "neso"]
[dependencies.rusqlite]
version = "0.20.0"
features = ["bundled", "blob"]
[dev-dependencies]
pretty_assertions = "0.6.1"

6
src/cli.rs
View File

@ -17,7 +17,7 @@ use crate::prelude::*;
use log::{debug, trace};
use regex::Regex;
use rustyline::error::ReadlineError;
use rustyline::{self, ColorMode, Config, Editor, config::Configurer, config::EditMode};
use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
use std::env;
use std::error::Error;
use std::io::{BufRead, BufReader, Write};
@ -179,6 +179,8 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(ToBSON),
whole_stream_command(ToCSV),
whole_stream_command(ToJSON),
whole_stream_command(ToSQLite),
whole_stream_command(ToDB),
whole_stream_command(ToTOML),
whole_stream_command(ToTSV),
whole_stream_command(ToYAML),
@ -193,6 +195,8 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(FromINI),
whole_stream_command(FromBSON),
whole_stream_command(FromJSON),
whole_stream_command(FromDB),
whole_stream_command(FromSQLite),
whole_stream_command(FromTOML),
whole_stream_command(FromXML),
whole_stream_command(FromYAML),

7
src/commands.rs
View File

@ -19,6 +19,7 @@ pub(crate) mod from_bson;
pub(crate) mod from_csv;
pub(crate) mod from_ini;
pub(crate) mod from_json;
pub(crate) mod from_sqlite;
pub(crate) mod from_toml;
pub(crate) mod from_tsv;
pub(crate) mod from_xml;
@ -52,6 +53,7 @@ pub(crate) mod to_array;
pub(crate) mod to_bson;
pub(crate) mod to_csv;
pub(crate) mod to_json;
pub(crate) mod to_sqlite;
pub(crate) mod to_toml;
pub(crate) mod to_tsv;
pub(crate) mod to_yaml;
@ -67,6 +69,7 @@ pub(crate) use command::{
per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
UnevaluatedCallInfo, WholeStreamCommand,
};
pub(crate) use config::Config;
pub(crate) use cp::Cpy;
pub(crate) use date::Date;
@ -79,6 +82,8 @@ pub(crate) use from_bson::FromBSON;
pub(crate) use from_csv::FromCSV;
pub(crate) use from_ini::FromINI;
pub(crate) use from_json::FromJSON;
pub(crate) use from_sqlite::FromDB;
pub(crate) use from_sqlite::FromSQLite;
pub(crate) use from_toml::FromTOML;
pub(crate) use from_tsv::FromTSV;
pub(crate) use from_xml::FromXML;
@ -112,6 +117,8 @@ pub(crate) use to_array::ToArray;
pub(crate) use to_bson::ToBSON;
pub(crate) use to_csv::ToCSV;
pub(crate) use to_json::ToJSON;
pub(crate) use to_sqlite::ToDB;
pub(crate) use to_sqlite::ToSQLite;
pub(crate) use to_toml::ToTOML;
pub(crate) use to_tsv::ToTSV;
pub(crate) use to_yaml::ToYAML;

2
src/commands/from_bson.rs
View File

@ -45,7 +45,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value
collected.into_tagged_value()
}
Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
Bson::RegExp(r, opts) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(

4
src/commands/from_json.rs
View File

@ -32,9 +32,7 @@ fn convert_json_value_to_nu_value(v: &serde_hjson::Value, tag: impl Into<Tag>) -
let tag = tag.into();
match v {
serde_hjson::Value::Null => {
Value::Primitive(Primitive::String(String::from(""))).tagged(tag)
}
serde_hjson::Value::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
serde_hjson::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
serde_hjson::Value::F64(n) => {
Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag)

159
src/commands/from_sqlite.rs Normal file
View File

@ -0,0 +1,159 @@
use crate::commands::WholeStreamCommand;
use crate::errors::ShellError;
use crate::object::base::OF64;
use crate::object::{Primitive, TaggedDictBuilder, Value};
use crate::prelude::*;
use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
use std::io::Write;
use std::path::Path;
pub struct FromSQLite;
impl WholeStreamCommand for FromSQLite {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry)
}
fn name(&self) -> &str {
"from-sqlite"
}
fn signature(&self) -> Signature {
Signature::build("from-sqlite")
}
}
pub struct FromDB;
impl WholeStreamCommand for FromDB {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry)
}
fn name(&self) -> &str {
"from-db"
}
fn signature(&self) -> Signature {
Signature::build("from-db")
}
}
pub fn convert_sqlite_file_to_nu_value(
path: &Path,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, rusqlite::Error> {
let conn = Connection::open(path)?;
let mut meta_out = Vec::new();
let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
while let Some(meta_row) = meta_rows.next()? {
let table_name: String = meta_row.get(0)?;
let mut meta_dict = TaggedDictBuilder::new(tag.clone());
let mut out = Vec::new();
let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?;
let mut table_rows = table_stmt.query(NO_PARAMS)?;
while let Some(table_row) = table_rows.next()? {
out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?)
}
meta_dict.insert_tagged(
"table_name".to_string(),
Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()),
);
meta_dict.insert_tagged("table_values", Value::List(out).tagged(tag.clone()));
meta_out.push(meta_dict.into_tagged_value());
}
let tag = tag.into();
Ok(Value::List(meta_out).tagged(tag))
}
fn convert_sqlite_row_to_nu_value(
row: &Row,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, rusqlite::Error> {
let mut collected = TaggedDictBuilder::new(tag.clone());
for (i, c) in row.columns().iter().enumerate() {
collected.insert_tagged(
c.name().to_string(),
convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
);
}
return Ok(collected.into_tagged_value());
}
fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Tagged<Value> {
match value {
ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
ValueRef::Integer(i) => Value::Primitive(Primitive::Int(i)).tagged(tag),
ValueRef::Real(f) => Value::Primitive(Primitive::Float(OF64::from(f))).tagged(tag),
t @ ValueRef::Text(_) => {
// this unwrap is safe because we know the ValueRef is Text.
Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag)
}
ValueRef::Blob(u) => Value::Binary(u.to_owned()).tagged(tag),
}
}
pub fn from_sqlite_bytes_to_value(
mut bytes: Vec<u8>,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
tempfile.write_all(bytes.as_mut_slice())?;
match convert_sqlite_file_to_nu_value(tempfile.path(), tag) {
Ok(value) => Ok(value),
Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
}
}
fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let span = args.name_span();
let input = args.input;
let stream = async_stream_block! {
let values: Vec<Tagged<Value>> = input.values.collect().await;
for value in values {
let value_tag = value.tag();
match value.item {
Value::Binary(vb) =>
match from_sqlite_bytes_to_value(vb, span) {
Ok(x) => yield ReturnSuccess::value(x),
Err(_) => {
yield Err(ShellError::labeled_error_with_secondary(
"Could not parse as SQLite",
"input cannot be parsed as SQLite",
span,
"value originates from here",
value_tag.span,
))
}
}
_ => yield Err(ShellError::labeled_error_with_secondary(
"Expected a string from pipeline",
"requires string input",
span,
"value originates from here",
value_tag.span,
)),
}
}
};
Ok(stream.to_output_stream())
}

203
src/commands/to_sqlite.rs Normal file
View File

@ -0,0 +1,203 @@
use crate::commands::WholeStreamCommand;
use crate::object::{Dictionary, Primitive, Value};
use crate::prelude::*;
use hex::encode;
use rusqlite::{Connection, NO_PARAMS};
use std::io::Read;
pub struct ToSQLite;
impl WholeStreamCommand for ToSQLite {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry)
}
fn name(&self) -> &str {
"to-sqlite"
}
fn signature(&self) -> Signature {
Signature::build("to-sqlite")
}
}
pub struct ToDB;
impl WholeStreamCommand for ToDB {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry)
}
fn name(&self) -> &str {
"to-db"
}
fn signature(&self) -> Signature {
Signature::build("to-db")
}
}
fn comma_concat(acc: String, current: String) -> String {
if acc == "" {
current
} else {
format!("{}, {}", acc, current)
}
}
fn get_columns(rows: &Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
match &rows[0].item {
Value::Object(d) => Ok(d
.entries
.iter()
.map(|(k, _v)| k.clone())
.fold("".to_string(), comma_concat)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
}
}
fn nu_value_to_sqlite_string(v: Value) -> String {
match v {
Value::Binary(u) => format!("x'{}'", encode(u)),
Value::Primitive(p) => match p {
Primitive::Nothing => "NULL".into(),
Primitive::Int(i) => format!("{}", i),
Primitive::Float(f) => format!("{}", f.into_inner()),
Primitive::Bytes(u) => format!("{}", u),
Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
Primitive::Boolean(true) => "1".into(),
Primitive::Boolean(_) => "0".into(),
Primitive::Date(d) => format!("'{}'", d),
Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
Primitive::BeginningOfStream => "NULL".into(),
Primitive::EndOfStream => "NULL".into(),
},
_ => "NULL".into(),
}
}
fn get_insert_values(rows: Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
let values: Result<Vec<_>, _> = rows
.into_iter()
.map(|value| match value.item {
Value::Object(d) => Ok(format!(
"({})",
d.entries
.iter()
.map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone()))
.fold("".to_string(), comma_concat)
)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
})
.collect();
let values = values?;
Ok(values.into_iter().fold("".to_string(), comma_concat))
}
fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
let table_name = match table.entries.get("table_name") {
Some(Tagged {
item: Value::Primitive(Primitive::String(table_name)),
..
}) => table_name,
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table name",
))
}
};
let (columns, insert_values) = match table.entries.get("table_values") {
Some(Tagged {
item: Value::List(l),
..
}) => (get_columns(l), get_insert_values(l.to_vec())),
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table values",
))
}
};
let create = format!("create table {}({})", table_name, columns?);
let insert = format!("insert into {} values {}", table_name, insert_values?);
Ok((create, insert))
}
fn sqlite_input_stream_to_bytes(
values: Vec<Tagged<Value>>,
) -> Result<Tagged<Value>, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
let conn = match Connection::open(tempfile.path()) {
Ok(conn) => conn,
Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
};
let tag = values[0].tag.clone();
for value in values.into_iter() {
match value.item() {
Value::Object(d) => {
let (create, insert) = generate_statements(d.to_owned())?;
match conn
.execute(&create, NO_PARAMS)
.and_then(|_| conn.execute(&insert, NO_PARAMS))
{
Ok(_) => (),
Err(e) => {
println!("{}", create);
println!("{}", insert);
println!("{:?}", e);
return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
}
}
}
other => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Expected object, found {:?}", other),
))
}
}
}
let mut out = Vec::new();
tempfile.read_to_end(&mut out)?;
Ok(Value::Binary(out).tagged(tag))
}
fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let name_span = args.name_span();
let stream = async_stream_block! {
let values: Vec<_> = args.input.into_vec().await;
match sqlite_input_stream_to_bytes(values) {
Ok(out) => {
yield ReturnSuccess::value(out)
}
Err(_) => {
yield Err(ShellError::labeled_error(
"Expected an object with SQLite-compatible structure from pipeline",
"requires SQLite-compatible input",
name_span,
))
}
};
};
Ok(stream.to_output_stream())
}

4
src/utils.rs
View File

@ -266,6 +266,10 @@ mod tests {
loc: fixtures().join("sample.bson"),
at: 0
},
Res {
loc: fixtures().join("sample.db"),
at: 0
},
Res {
loc: fixtures().join("sample.ini"),
at: 0

48
tests/command_open_tests.rs
View File

@ -6,23 +6,22 @@ use helpers::{Playground, Stub::*};
#[test]
fn recognizes_csv() {
Playground::setup("open_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"nu.zion.csv",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"nu.zion.csv",
r#"
author,lang,source
Jonathan Turner,Rust,New Zealand
Andres N. Robalino,Rust,Ecuador
Yehuda Katz,Rust,Estados Unidos
"#
"#,
)]);
let actual = nu!(
cwd: dirs.test(), h::pipeline(
r#"
open nu.zion.csv
| where author == "Andres N. Robalino"
| get source
open nu.zion.csv
| where author == "Andres N. Robalino"
| get source
| echo $it
"#
));
@ -46,11 +45,11 @@ fn open_can_parse_bson_2() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.bson
| get root
| nth 6
| get b
| get '$binary_subtype'
open sample.bson
| get root
| nth 6
| get b
| get '$binary_subtype'
| echo $it
"#
));
@ -58,6 +57,21 @@ fn open_can_parse_bson_2() {
assert_eq!(actual, "function");
}
#[test]
fn open_can_parse_sqlite() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.db
| get table_values
| nth 2
| get x
| echo $it"#
));
assert_eq!(actual, "hello");
}
#[test]
fn open_can_parse_toml() {
let actual = nu!(
@ -74,8 +88,8 @@ fn open_can_parse_tsv() {
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open caco3_plastics.tsv
| first 1
| get origin
| first 1
| get origin
| echo $it
"#
));
@ -88,8 +102,8 @@ fn open_can_parse_json() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sgml_description.json
| get glossary.GlossDiv.GlossList.GlossEntry.GlossSee
open sgml_description.json
| get glossary.GlossDiv.GlossList.GlossEntry.GlossSee
| echo $it
"#
));

139
tests/filters_test.rs
View File

@ -16,14 +16,13 @@ fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() {
#[test]
fn converts_structured_table_to_csv_text() {
Playground::setup("filter_to_csv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
importer,shipper,tariff_item,name,origin
Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain
Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia
"#
"#,
)]);
let actual = nu!(
@ -47,14 +46,13 @@ fn converts_structured_table_to_csv_text() {
#[test]
fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() {
Playground::setup("filter_to_csv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
importer,shipper,tariff_item,name,origin
Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain
Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia
"#
"#,
)]);
let actual = nu!(
@ -76,10 +74,9 @@ fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() {
#[test]
fn converts_from_csv_text_to_structured_table() {
Playground::setup("filter_from_csv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first_name,last_name,rusty_luck
Andrés,Robalino,1
Jonathan,Turner,1
@ -106,10 +103,9 @@ fn converts_from_csv_text_to_structured_table() {
#[test]
fn converts_from_csv_text_skipping_headers_to_structured_table() {
Playground::setup("filter_from_csv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first_name,last_name,rusty_luck
Andrés,Robalino,1
Jonathan,Turner,1
@ -152,10 +148,9 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() {
#[test]
fn converts_from_json_text_to_structured_table() {
Playground::setup("filter_from_json_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
{
"katz": [
{"name": "Yehuda", "rusty_luck": 1},
@ -173,17 +168,15 @@ fn converts_from_json_text_to_structured_table() {
);
assert_eq!(actual, "4");
})
}
#[test]
fn converts_from_json_text_recognizing_objects_independendtly_to_structured_table() {
Playground::setup("filter_from_json_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
{"name": "Yehuda", "rusty_luck": 1}
{"name": "Jonathan", "rusty_luck": 1}
{"name": "Andres", "rusty_luck": 1}
@ -209,10 +202,9 @@ fn converts_from_json_text_recognizing_objects_independendtly_to_structured_tabl
#[test]
fn converts_structured_table_to_json_text() {
Playground::setup("filter_to_json_test", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"sample.txt",
r#"
JonAndrehudaTZ,3
GorbyPuff,100
"#,
@ -250,14 +242,13 @@ fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table() {
#[test]
fn converts_structured_table_to_tsv_text() {
Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
importer shipper tariff_item name origin
Plasticos Rival Reverte 2509000000 Calcium carbonate Spain
Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia
"#
"#,
)]);
let actual = nu!(
@ -281,14 +272,13 @@ fn converts_structured_table_to_tsv_text() {
#[test]
fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() {
Playground::setup("filter_to_tsv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
importer shipper tariff_item name origin
Plasticos Rival Reverte 2509000000 Calcium carbonate Spain
Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia
"#
"#,
)]);
let actual = nu!(
@ -310,10 +300,9 @@ fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() {
#[test]
fn converts_from_tsv_text_to_structured_table() {
Playground::setup("filter_from_tsv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first Name Last Name rusty_luck
Andrés Robalino 1
Jonathan Turner 1
@ -340,10 +329,9 @@ fn converts_from_tsv_text_to_structured_table() {
#[test]
fn converts_from_tsv_text_skipping_headers_to_structured_table() {
Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first Name Last Name rusty_luck
Andrés Robalino 1
Jonathan Turner 1
@ -354,11 +342,11 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() {
let actual = nu!(
cwd: dirs.test(), h::pipeline(
r#"
open los_tres_amigos.txt
| from-tsv --headerless
| get Column3
| str --to-int
| sum
open los_tres_amigos.txt
| from-tsv --headerless
| get Column3
| str --to-int
| sum
| echo $it
"#
));
@ -368,21 +356,50 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() {
}
#[test]
fn can_convert_json_text_to_bson_and_back_into_table() {
fn can_convert_table_to_bson_and_back_into_table() {
let actual = nu!(
cwd: "tests/fixtures/formats",
"open sample.bson | to-bson | from-bson | get root | nth 1 | get b | echo $it"
);
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.bson
| to-bson
| from-bson
| get root
| nth 1
| get b
| echo $it"#
));
assert_eq!(actual, "whel");
}
#[test]
fn can_convert_table_to_sqlite_and_back_into_table() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.db
| to-sqlite
| from-sqlite
| get table_values
| nth 2
| get x
| echo $it"#
));
assert_eq!(actual, "hello");
}
#[test]
fn can_convert_table_to_toml_text_and_from_toml_text_back_into_table() {
let actual = nu!(
cwd: "tests/fixtures/formats",
"open cargo_sample.toml | to-toml | from-toml | get package.name | echo $it"
);
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open cargo_sample.toml
| to-toml
| from-toml
| get package.name
| echo $it"#
));
assert_eq!(actual, "nu");
}

BIN
tests/fixtures/formats/sample.db vendored Normal file
View File

Binary file not shown.