extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-12-22 07:02:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add SQLite support

Browse Source
This commit is contained in:
Patrick Meredith 2019-08-27 17:45:18 -04:00
parent 4cde96bcc4
commit 3d147d1143
12 changed files with 507 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
Cargo.lock generated
View File

@ -715,6 +715,16 @@ dependencies = [
"synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fixedbitset"
version = "0.1.9"
@ -1308,6 +1318,16 @@ dependencies = [
"libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libsqlite3-sys"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libz-sys"
version = "1.0.25"
@ -1358,6 +1378,14 @@ dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lzw"
version = "0.10.0"
@ -1576,6 +1604,7 @@ dependencies = [
"git2 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
"glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"heim 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
"hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.22.1 (registry+https://github.com/rust-lang/crates.io-index)",
"indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1596,6 +1625,7 @@ dependencies = [
"rawkey 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustyline 5.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2244,6 +2274,20 @@ dependencies = [
"xmlparser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rusqlite"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rust-argon2"
version = "0.5.0"
@ -3183,6 +3227,8 @@ dependencies = [
"checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
"checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
"checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
"checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
"checksum flate2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "550934ad4808d5d39365e5d61727309bf18b3b02c6c56b729cb92e7dd84bc3d8"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@ -3242,12 +3288,14 @@ dependencies = [
"checksum libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)" = "d44e80633f007889c7eff624b709ab43c92d708caad982295768a7b13ca3b5eb"
"checksum libgit2-sys 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4c179ed6d19cd3a051e68c177fbbc214e79ac4724fac3a850ec9f3d3eb8a5578"
"checksum libnghttp2-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02254d44f4435dd79e695f2c2b83cd06a47919adea30216ceaf0c57ca0a72463"
"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25"
"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
"checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c"
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
"checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
"checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084"
"checksum macaddr 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ff4752cb15cffb3e68f7dcb22e0818ac871f8c98fb07a634a81f41fb202a09f"
"checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
@ -3342,6 +3390,7 @@ dependencies = [
"checksum render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68ed587df09cfb7ce1bc6fe8f77e24db219f222c049326ccbfb948ec67e31664"
"checksum result 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560"
"checksum roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "153c367ce9fb8ef7afe637ef92bd083ba0f88b03ef3fcf0287d40be05ae0a61c"
"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051"
"checksum rust-argon2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81ed8d04228b44a740c8d46ff872a28e50fff3d659f307ab4da2cc502e019ff3"
"checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
"checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af"

5
Cargo.toml
View File

@ -62,6 +62,7 @@ mime = "0.3.13"
regex = "1.2.1"
pretty-hex = "0.1.0"
neso = { version = "0.5.0", optional = true }
hex = "0.3.2"
crossterm = "0.10.2"
tempfile = "3.1.0"
image = "0.22.1"
@ -82,6 +83,10 @@ pin-utils = "0.1.0-alpha.4"
[features]
raw-key = ["rawkey", "neso"]
[dependencies.rusqlite]
version = "0.20.0"
features = ["bundled", "blob"]
[dev-dependencies]
pretty_assertions = "0.6.1"

2
src/cli.rs
View File

@ -179,6 +179,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(ToBSON),
whole_stream_command(ToCSV),
whole_stream_command(ToJSON),
whole_stream_command(ToSQLite),
whole_stream_command(ToTOML),
whole_stream_command(ToTSV),
whole_stream_command(ToYAML),
@ -193,6 +194,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(FromINI),
whole_stream_command(FromBSON),
whole_stream_command(FromJSON),
whole_stream_command(FromSQLite),
whole_stream_command(FromTOML),
whole_stream_command(FromXML),
whole_stream_command(FromYAML),

5
src/commands.rs
View File

@ -19,6 +19,7 @@ pub(crate) mod from_bson;
pub(crate) mod from_csv;
pub(crate) mod from_ini;
pub(crate) mod from_json;
pub(crate) mod from_sqlite;
pub(crate) mod from_toml;
pub(crate) mod from_tsv;
pub(crate) mod from_xml;
@ -52,6 +53,7 @@ pub(crate) mod to_array;
pub(crate) mod to_bson;
pub(crate) mod to_csv;
pub(crate) mod to_json;
pub(crate) mod to_sqlite;
pub(crate) mod to_toml;
pub(crate) mod to_tsv;
pub(crate) mod to_yaml;
@ -67,6 +69,7 @@ pub(crate) use command::{
per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
UnevaluatedCallInfo, WholeStreamCommand,
};
pub(crate) use config::Config;
pub(crate) use cp::Cpy;
pub(crate) use date::Date;
@ -79,6 +82,7 @@ pub(crate) use from_bson::FromBSON;
pub(crate) use from_csv::FromCSV;
pub(crate) use from_ini::FromINI;
pub(crate) use from_json::FromJSON;
pub(crate) use from_sqlite::FromSQLite;
pub(crate) use from_toml::FromTOML;
pub(crate) use from_tsv::FromTSV;
pub(crate) use from_xml::FromXML;
@ -111,6 +115,7 @@ pub(crate) use to_array::ToArray;
pub(crate) use to_bson::ToBSON;
pub(crate) use to_csv::ToCSV;
pub(crate) use to_json::ToJSON;
pub(crate) use to_sqlite::ToSQLite;
pub(crate) use to_toml::ToTOML;
pub(crate) use to_tsv::ToTSV;
pub(crate) use to_yaml::ToYAML;

2
src/commands/from_bson.rs
View File

@ -45,7 +45,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value
collected.into_tagged_value()
}
Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
Bson::RegExp(r, opts) => {
let mut collected = TaggedDictBuilder::new(tag);
collected.insert_tagged(

139
src/commands/from_sqlite.rs Normal file
View File

@ -0,0 +1,139 @@
use crate::commands::WholeStreamCommand;
use crate::errors::ShellError;
use crate::object::base::OF64;
use crate::object::{Primitive, TaggedDictBuilder, Value};
use crate::prelude::*;
use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
use std::io::Write;
use std::path::Path;
pub struct FromSQLite;
impl WholeStreamCommand for FromSQLite {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
from_sqlite(args, registry)
}
fn name(&self) -> &str {
"from-sqlite"
}
fn signature(&self) -> Signature {
Signature::build("from-sqlite")
}
}
pub fn convert_sqlite_file_to_nu_value(
path: &Path,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, rusqlite::Error> {
let conn = Connection::open(path)?;
let mut meta_out = Vec::new();
let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
while let Some(meta_row) = meta_rows.next()? {
let table_name: String = meta_row.get(0)?;
let mut meta_dict = TaggedDictBuilder::new(tag.clone());
let mut out = Vec::new();
let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?;
let mut table_rows = table_stmt.query(NO_PARAMS)?;
while let Some(table_row) = table_rows.next()? {
out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?)
}
meta_dict.insert_tagged(
"table_name".to_string(),
Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()),
);
meta_dict.insert_tagged("table_values", Value::List(out).tagged(tag.clone()));
meta_out.push(meta_dict.into_tagged_value());
}
let tag = tag.into();
Ok(Value::List(meta_out).tagged(tag))
}
fn convert_sqlite_row_to_nu_value(
row: &Row,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, rusqlite::Error> {
let mut collected = TaggedDictBuilder::new(tag.clone());
for (i, c) in row.columns().iter().enumerate() {
collected.insert_tagged(
c.name().to_string(),
convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
);
}
return Ok(collected.into_tagged_value());
}
fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Tagged<Value> {
match value {
ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
ValueRef::Integer(i) => Value::Primitive(Primitive::Int(i)).tagged(tag),
ValueRef::Real(f) => Value::Primitive(Primitive::Float(OF64::from(f))).tagged(tag),
t @ ValueRef::Text(_) => {
// this unwrap is safe because we know the ValueRef is Text.
Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag)
}
ValueRef::Blob(u) => Value::Binary(u.to_owned()).tagged(tag),
}
}
pub fn from_sqlite_bytes_to_value(
mut bytes: Vec<u8>,
tag: impl Into<Tag> + Clone,
) -> Result<Tagged<Value>, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
tempfile.write_all(bytes.as_mut_slice())?;
match convert_sqlite_file_to_nu_value(tempfile.path(), tag) {
Ok(value) => Ok(value),
Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
}
}
fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let span = args.name_span();
let input = args.input;
let stream = async_stream_block! {
let values: Vec<Tagged<Value>> = input.values.collect().await;
for value in values {
let value_tag = value.tag();
match value.item {
Value::Binary(vb) =>
match from_sqlite_bytes_to_value(vb, span) {
Ok(x) => yield ReturnSuccess::value(x),
Err(_) => {
yield Err(ShellError::labeled_error_with_secondary(
"Could not parse as SQLite",
"input cannot be parsed as SQLite",
span,
"value originates from here",
value_tag.span,
))
}
}
_ => yield Err(ShellError::labeled_error_with_secondary(
"Expected a string from pipeline",
"requires string input",
span,
"value originates from here",
value_tag.span,
)),
}
}
};
Ok(stream.to_output_stream())
}

10
src/commands/open.rs
View File

@ -527,6 +527,16 @@ pub fn parse_binary_as_value(
},
)
}
Some(ref x) if x == "db" => {
crate::commands::from_sqlite::from_sqlite_bytes_to_value(contents, contents_tag)
.map_err(move |_| {
ShellError::labeled_error(
"Could not open as SQLite",
"could not open as SQLite",
name_span,
)
})
}
_ => Ok(Value::Binary(contents).tagged(contents_tag)),
}
}

183
src/commands/to_sqlite.rs Normal file
View File

@ -0,0 +1,183 @@
use crate::commands::WholeStreamCommand;
use crate::object::{Dictionary, Primitive, Value};
use crate::prelude::*;
use hex::encode;
use rusqlite::{Connection, NO_PARAMS};
use std::io::Read;
pub struct ToSQLite;
impl WholeStreamCommand for ToSQLite {
fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
to_sqlite(args, registry)
}
fn name(&self) -> &str {
"to-sqlite"
}
fn signature(&self) -> Signature {
Signature::build("to-sqlite")
}
}
fn comma_concat(acc: String, current: String) -> String {
if acc == "" {
current
} else {
format!("{}, {}", acc, current)
}
}
fn get_columns(rows: &Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
match &rows[0].item {
Value::Object(d) => Ok(d
.entries
.iter()
.map(|(k, _v)| k.clone())
.fold("".to_string(), comma_concat)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
}
}
fn nu_value_to_sqlite_string(v: Value) -> String {
match v {
Value::Binary(u) => format!("x'{}'", encode(u)),
Value::Primitive(p) => match p {
Primitive::Nothing => "NULL".into(),
Primitive::Int(i) => format!("{}", i),
Primitive::Float(f) => format!("{}", f.into_inner()),
Primitive::Bytes(u) => format!("{}", u),
Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
Primitive::Boolean(true) => "1".into(),
Primitive::Boolean(_) => "0".into(),
Primitive::Date(d) => format!("'{}'", d),
Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
Primitive::BeginningOfStream => "NULL".into(),
Primitive::EndOfStream => "NULL".into(),
},
_ => "NULL".into(),
}
}
fn get_insert_values(rows: Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
let values: Result<Vec<_>, _> = rows
.into_iter()
.map(|value| match value.item {
Value::Object(d) => Ok(format!(
"({})",
d.entries
.iter()
.map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone()))
.fold("".to_string(), comma_concat)
)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table column names",
)),
})
.collect();
let values = values?;
Ok(values.into_iter().fold("".to_string(), comma_concat))
}
fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
let table_name = match table.entries.get("table_name") {
Some(Tagged {
item: Value::Primitive(Primitive::String(table_name)),
..
}) => table_name,
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table name",
))
}
};
let (columns, insert_values) = match table.entries.get("table_values") {
Some(Tagged {
item: Value::List(l),
..
}) => (get_columns(l), get_insert_values(l.to_vec())),
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Could not find table values",
))
}
};
let create = format!("create table {}({})", table_name, columns?);
let insert = format!("insert into {} values {}", table_name, insert_values?);
Ok((create, insert))
}
fn sqlite_input_stream_to_bytes(
values: Vec<Tagged<Value>>,
) -> Result<Tagged<Value>, std::io::Error> {
// FIXME: should probably write a sqlite virtual filesystem
// that will allow us to use bytes as a file to avoid this
// write out, but this will require C code. Might be
// best done as a PR to rusqlite.
let mut tempfile = tempfile::NamedTempFile::new()?;
let conn = match Connection::open(tempfile.path()) {
Ok(conn) => conn,
Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
};
let tag = values[0].tag.clone();
for value in values.into_iter() {
match value.item() {
Value::Object(d) => {
let (create, insert) = generate_statements(d.to_owned())?;
match conn
.execute(&create, NO_PARAMS)
.and_then(|_| conn.execute(&insert, NO_PARAMS))
{
Ok(_) => (),
Err(e) => {
println!("{}", create);
println!("{}", insert);
println!("{:?}", e);
return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
}
}
}
other => {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Expected object, found {:?}", other),
))
}
}
}
let mut out = Vec::new();
tempfile.read_to_end(&mut out)?;
Ok(Value::Binary(out).tagged(tag))
}
fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
let args = args.evaluate_once(registry)?;
let name_span = args.name_span();
let stream = async_stream_block! {
let values: Vec<_> = args.input.into_vec().await;
match sqlite_input_stream_to_bytes(values) {
Ok(out) => {
yield ReturnSuccess::value(out)
}
Err(_) => {
yield Err(ShellError::labeled_error(
"Expected an object with SQLite-compatible structure from pipeline",
"requires SQLite-compatible input",
name_span,
))
}
};
};
Ok(stream.to_output_stream())
}

4
src/utils.rs
View File

@ -266,6 +266,10 @@ mod tests {
loc: fixtures().join("sample.bson"),
at: 0
},
Res {
loc: fixtures().join("sample.db"),
at: 0
},
Res {
loc: fixtures().join("sample.ini"),
at: 0

48
tests/command_open_tests.rs
View File

@ -6,23 +6,22 @@ use helpers::{Playground, Stub::*};
#[test]
fn recognizes_csv() {
Playground::setup("open_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"nu.zion.csv",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"nu.zion.csv",
r#"
author,lang,source
Jonathan Turner,Rust,New Zealand
Andres N. Robalino,Rust,Ecuador
Yehuda Katz,Rust,Estados Unidos
"#
"#,
)]);
let actual = nu!(
cwd: dirs.test(), h::pipeline(
r#"
open nu.zion.csv
| where author == "Andres N. Robalino"
| get source
open nu.zion.csv
| where author == "Andres N. Robalino"
| get source
| echo $it
"#
));
@ -46,11 +45,11 @@ fn open_can_parse_bson_2() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.bson
| get root
| nth 6
| get b
| get '$binary_subtype'
open sample.bson
| get root
| nth 6
| get b
| get '$binary_subtype'
| echo $it
"#
));
@ -58,6 +57,21 @@ fn open_can_parse_bson_2() {
assert_eq!(actual, "function");
}
#[test]
fn open_can_parse_sqlite() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.db
| get table_values
| nth 2
| get x
| echo $it"#
));
assert_eq!(actual, "hello");
}
#[test]
fn open_can_parse_toml() {
let actual = nu!(
@ -74,8 +88,8 @@ fn open_can_parse_tsv() {
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open caco3_plastics.tsv
| first 1
| get origin
| first 1
| get origin
| echo $it
"#
));
@ -88,8 +102,8 @@ fn open_can_parse_json() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sgml_description.json
| get glossary.GlossDiv.GlossList.GlossEntry.GlossSee
open sgml_description.json
| get glossary.GlossDiv.GlossList.GlossEntry.GlossSee
| echo $it
"#
));

139
tests/filters_test.rs
View File

@ -16,14 +16,13 @@ fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() {
#[test]
fn converts_structured_table_to_csv_text() {
Playground::setup("filter_to_csv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
importer,shipper,tariff_item,name,origin
Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain
Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia
"#
"#,
)]);
let actual = nu!(
@ -47,14 +46,13 @@ fn converts_structured_table_to_csv_text() {
#[test]
fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() {
Playground::setup("filter_to_csv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"csv_text_sample.txt",
r#"
importer,shipper,tariff_item,name,origin
Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain
Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia
"#
"#,
)]);
let actual = nu!(
@ -76,10 +74,9 @@ fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() {
#[test]
fn converts_from_csv_text_to_structured_table() {
Playground::setup("filter_from_csv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first_name,last_name,rusty_luck
Andrés,Robalino,1
Jonathan,Turner,1
@ -106,10 +103,9 @@ fn converts_from_csv_text_to_structured_table() {
#[test]
fn converts_from_csv_text_skipping_headers_to_structured_table() {
Playground::setup("filter_from_csv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first_name,last_name,rusty_luck
Andrés,Robalino,1
Jonathan,Turner,1
@ -152,10 +148,9 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() {
#[test]
fn converts_from_json_text_to_structured_table() {
Playground::setup("filter_from_json_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
{
"katz": [
{"name": "Yehuda", "rusty_luck": 1},
@ -173,17 +168,15 @@ fn converts_from_json_text_to_structured_table() {
);
assert_eq!(actual, "4");
})
}
#[test]
fn converts_from_json_text_recognizing_objects_independendtly_to_structured_table() {
Playground::setup("filter_from_json_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"katz.txt",
r#"
{"name": "Yehuda", "rusty_luck": 1}
{"name": "Jonathan", "rusty_luck": 1}
{"name": "Andres", "rusty_luck": 1}
@ -209,10 +202,9 @@ fn converts_from_json_text_recognizing_objects_independendtly_to_structured_tabl
#[test]
fn converts_structured_table_to_json_text() {
Playground::setup("filter_to_json_test", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"sample.txt",
r#"
JonAndrehudaTZ,3
GorbyPuff,100
"#,
@ -250,14 +242,13 @@ fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table() {
#[test]
fn converts_structured_table_to_tsv_text() {
Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
importer shipper tariff_item name origin
Plasticos Rival Reverte 2509000000 Calcium carbonate Spain
Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia
"#
"#,
)]);
let actual = nu!(
@ -281,14 +272,13 @@ fn converts_structured_table_to_tsv_text() {
#[test]
fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() {
Playground::setup("filter_to_tsv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"tsv_text_sample.txt",
r#"
importer shipper tariff_item name origin
Plasticos Rival Reverte 2509000000 Calcium carbonate Spain
Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia
"#
"#,
)]);
let actual = nu!(
@ -310,10 +300,9 @@ fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() {
#[test]
fn converts_from_tsv_text_to_structured_table() {
Playground::setup("filter_from_tsv_test_1", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first Name Last Name rusty_luck
Andrés Robalino 1
Jonathan Turner 1
@ -340,10 +329,9 @@ fn converts_from_tsv_text_to_structured_table() {
#[test]
fn converts_from_tsv_text_skipping_headers_to_structured_table() {
Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| {
sandbox
.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_amigos.txt",
r#"
first Name Last Name rusty_luck
Andrés Robalino 1
Jonathan Turner 1
@ -354,11 +342,11 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() {
let actual = nu!(
cwd: dirs.test(), h::pipeline(
r#"
open los_tres_amigos.txt
| from-tsv --headerless
| get Column3
| str --to-int
| sum
open los_tres_amigos.txt
| from-tsv --headerless
| get Column3
| str --to-int
| sum
| echo $it
"#
));
@ -368,21 +356,50 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() {
}
#[test]
fn can_convert_json_text_to_bson_and_back_into_table() {
fn can_convert_table_to_bson_and_back_into_table() {
let actual = nu!(
cwd: "tests/fixtures/formats",
"open sample.bson | to-bson | from-bson | get root | nth 1 | get b | echo $it"
);
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.bson
| to-bson
| from-bson
| get root
| nth 1
| get b
| echo $it"#
));
assert_eq!(actual, "whel");
}
#[test]
fn can_convert_table_to_sqlite_and_back_into_table() {
let actual = nu!(
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open sample.db
| to-sqlite
| from-sqlite
| get table_values
| nth 2
| get x
| echo $it"#
));
assert_eq!(actual, "hello");
}
#[test]
fn can_convert_table_to_toml_text_and_from_toml_text_back_into_table() {
let actual = nu!(
cwd: "tests/fixtures/formats",
"open cargo_sample.toml | to-toml | from-toml | get package.name | echo $it"
);
cwd: "tests/fixtures/formats", h::pipeline(
r#"
open cargo_sample.toml
| to-toml
| from-toml
| get package.name
| echo $it"#
));
assert_eq!(actual, "nu");
}

BIN
tests/fixtures/formats/sample.db vendored Normal file
View File

Binary file not shown.