holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move some from xxx commands to plugin (#7942)

Browse Source 
# Description

From nushell 0.8 philosophy:
https://github.com/nushell/nushell.github.io/blob/main/contributor-book/philosophy_0_80.md#core-categories

> The following categories should be moved to plugins:
Uncommon format support

So this pr is trying to move following commands to plugin:
- [X] from eml
- [x] from ics
- [x] from ini
- [x] from vcf

And we can have a new plugin handles for these formatting, currently
it's implemented here:

https://github.com/WindSoilder/nu_plugin_format

The command usage should be the same to original command.

If it's ok, the plugin can support more formats like
[parquet](https://github.com/fdncred/nu_plugin_from_parquet), or [EDN
format](https://github.com/nushell/nushell/issues/6415), or something
else.

Just create a draft pr to show what's the blueprint looks like, and is
it a good direction to move forward?

# User-Facing Changes

_(List of all changes that impact the user experience here. This helps
us keep track of breaking changes.)_

# Tests + Formatting

Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect` to check that you're using the standard code
style
- `cargo test --workspace` to check that all tests pass

# After Submitting

If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
This commit is contained in:
WindSoilder 2023-02-13 20:42:08 +08:00 committed by GitHub
parent 4468dc835c
commit 208ffdc1da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 734 additions and 828 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Cargo.lock generated
View File

@ -2735,7 +2735,6 @@ dependencies = [
"digest",
"dirs-next",
"dtparse",
"eml-parser",
"encoding_rs",
"fancy-regex",
"filesize",
@ -2743,7 +2742,6 @@ dependencies = [
"fs_extra",
"hamcrest2",
"htmlescape",
"ical",
"indexmap",
"indicatif",
"is-root",
@ -2792,7 +2790,6 @@ dependencies = [
"rstest",
"rusqlite",
"rust-embed",
"rust-ini",
"same-file",
"serde",
"serde_urlencoded",
@ -2993,6 +2990,7 @@ dependencies = [
"num-format",
"once_cell",
"tempfile",
"which",
]
[[package]]
@ -3025,6 +3023,19 @@ dependencies = [
"nu-protocol",
]
[[package]]
name = "nu_plugin_formats"
version = "0.1.0"
dependencies = [
"eml-parser",
"ical",
"indexmap",
"nu-engine",
"nu-plugin",
"nu-protocol",
"rust-ini",
]
[[package]]
name = "nu_plugin_gstat"
version = "0.75.1"

1
Cargo.toml
View File

@ -35,6 +35,7 @@ members = [
"crates/nu_plugin_example",
"crates/nu_plugin_query",
"crates/nu_plugin_custom_values",
"crates/nu_plugin_formats",
"crates/nu-utils",
]

1
build-all.nu
View File

@ -12,6 +12,7 @@ let plugins = [
nu_plugin_query,
nu_plugin_example,
nu_plugin_custom_values,
nu_plugin_formats,
]
for plugin in $plugins {

2
crates/nu-command/Cargo.toml
View File

@ -46,14 +46,12 @@ csv = "1.1.6"
dialoguer = { default-features = false, version = "0.10.3" }
digest = { default-features = false, version = "0.10.0" }
dtparse = "1.2.0"
eml-parser = "0.1.0"
encoding_rs = "0.8.30"
fancy-regex = "0.11.0"
filesize = "0.2.0"
filetime = "0.2.15"
fs_extra = "1.3.0"
htmlescape = "0.3.1"
ical = "0.8.0"
indexmap = { version = "1.7", features = ["serde-1"] }
indicatif = "0.17.2"
is-root = "0.1.2"

4
crates/nu-command/src/default_context.rs
View File

@ -328,9 +328,6 @@ pub fn create_default_context() -> EngineState {
bind_command! {
From,
FromCsv,
FromEml,
FromIcs,
FromIni,
FromJson,
FromNuon,
FromOds,
@ -338,7 +335,6 @@ pub fn create_default_context() -> EngineState {
FromToml,
FromTsv,
FromUrl,
FromVcf,
FromXlsx,
FromXml,
FromYaml,

247
crates/nu-command/src/formats/from/eml.rs
View File

@ -1,247 +0,0 @@
use ::eml_parser::eml::*;
use ::eml_parser::EmlParser;
use indexmap::map::IndexMap;
use nu_engine::CallExt;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::Category;
use nu_protocol::{
Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
};
#[derive(Clone)]
pub struct FromEml;
const DEFAULT_BODY_PREVIEW: usize = 50;
impl Command for FromEml {
fn name(&self) -> &str {
"from eml"
}
fn signature(&self) -> Signature {
Signature::build("from eml")
.input_output_types(vec![(Type::String, Type::Record(vec![]))])
.named(
"preview-body",
SyntaxShape::Int,
"How many bytes of the body to preview",
Some('b'),
)
.category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .eml and create record."
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let head = call.head;
let preview_body: Option<Spanned<i64>> =
call.get_flag(engine_state, stack, "preview-body")?;
from_eml(input, preview_body, head)
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Convert eml structured data into record",
example: "'From: test@email.com
Subject: Welcome
To: someone@somewhere.com
Test' | from eml",
result: Some(Value::Record {
cols: vec![
"Subject".to_string(),
"From".to_string(),
"To".to_string(),
"Body".to_string(),
],
vals: vec![
Value::test_string("Welcome"),
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("test@email.com"),
],
span: Span::test_data(),
},
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("someone@somewhere.com"),
],
span: Span::test_data(),
},
Value::test_string("Test"),
],
span: Span::test_data(),
}),
},
Example {
description: "Convert eml structured data into record",
example: "'From: test@email.com
Subject: Welcome
To: someone@somewhere.com
Test' | from eml -b 1",
result: Some(Value::Record {
cols: vec![
"Subject".to_string(),
"From".to_string(),
"To".to_string(),
"Body".to_string(),
],
vals: vec![
Value::test_string("Welcome"),
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("test@email.com"),
],
span: Span::test_data(),
},
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("someone@somewhere.com"),
],
span: Span::test_data(),
},
Value::test_string("T"),
],
span: Span::test_data(),
}),
},
]
}
}
fn emailaddress_to_value(span: Span, email_address: &EmailAddress) -> Value {
let (n, a) = match email_address {
EmailAddress::AddressOnly { address } => (
Value::nothing(span),
Value::String {
val: address.to_string(),
span,
},
),
EmailAddress::NameAndEmailAddress { name, address } => (
Value::String {
val: name.to_string(),
span,
},
Value::String {
val: address.to_string(),
span,
},
),
};
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![n, a],
span,
}
}
fn headerfieldvalue_to_value(head: Span, value: &HeaderFieldValue) -> Value {
use HeaderFieldValue::*;
match value {
SingleEmailAddress(address) => emailaddress_to_value(head, address),
MultipleEmailAddresses(addresses) => Value::List {
vals: addresses
.iter()
.map(|a| emailaddress_to_value(head, a))
.collect(),
span: head,
},
Unstructured(s) => Value::string(s, head),
Empty => Value::nothing(head),
}
}
fn from_eml(
input: PipelineData,
preview_body: Option<Spanned<i64>>,
head: Span,
) -> Result<PipelineData, ShellError> {
let (value, _span, metadata, ..) = input.collect_string_strict(head)?;
let body_preview = preview_body
.map(|b| b.item as usize)
.unwrap_or(DEFAULT_BODY_PREVIEW);
let eml = EmlParser::from_string(value)
.with_body_preview(body_preview)
.parse()
.map_err(|_| {
ShellError::CantConvert("structured eml data".into(), "string".into(), head, None)
})?;
let mut collected = IndexMap::new();
if let Some(subj) = eml.subject {
collected.insert(
"Subject".to_string(),
Value::String {
val: subj,
span: head,
},
);
}
if let Some(from) = eml.from {
collected.insert("From".to_string(), headerfieldvalue_to_value(head, &from));
}
if let Some(to) = eml.to {
collected.insert("To".to_string(), headerfieldvalue_to_value(head, &to));
}
for HeaderField { name, value } in &eml.headers {
collected.insert(name.to_string(), headerfieldvalue_to_value(head, value));
}
if let Some(body) = eml.body {
collected.insert(
"Body".to_string(),
Value::String {
val: body,
span: head,
},
);
}
Ok(PipelineData::Value(
Value::from(Spanned {
item: collected,
span: head,
}),
metadata,
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromEml {})
}
}

157
crates/nu-command/src/formats/from/ini.rs
View File

@ -1,157 +0,0 @@
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Type, Value,
};
#[derive(Clone)]
pub struct FromIni;
impl Command for FromIni {
fn name(&self) -> &str {
"from ini"
}
fn signature(&self) -> Signature {
Signature::build("from ini")
.input_output_types(vec![(Type::String, Type::Record(vec![]))])
.category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .ini and create record"
}
fn examples(&self) -> Vec<Example> {
vec![Example {
example: "'[foo]
a=1
b=2' | from ini",
description: "Converts ini formatted string to record",
result: Some(Value::Record {
cols: vec!["foo".to_string()],
vals: vec![Value::Record {
cols: vec!["a".to_string(), "b".to_string()],
vals: vec![Value::test_string("1"), Value::test_string("2")],
span: Span::test_data(),
}],
span: Span::test_data(),
}),
}]
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let head = call.head;
from_ini(input, head)
}
}
pub fn from_ini_string_to_value(
s: String,
span: Span,
val_span: Span,
) -> Result<Value, ShellError> {
let ini_config: Result<ini::Ini, ini::ParseError> = ini::Ini::load_from_str(&s);
match ini_config {
Ok(config) => {
let mut sections: Vec<String> = Vec::new();
let mut sections_key_value_pairs: Vec<Value> = Vec::new();
for (section, properties) in config.iter() {
let mut keys_for_section: Vec<String> = Vec::new();
let mut values_for_section: Vec<Value> = Vec::new();
// section
match section {
Some(section_name) => {
sections.push(section_name.to_owned());
}
None => {
sections.push(String::new());
}
}
// section's key value pairs
for (key, value) in properties.iter() {
keys_for_section.push(key.to_owned());
values_for_section.push(Value::String {
val: value.to_owned(),
span,
});
}
// section with its key value pairs
sections_key_value_pairs.push(Value::Record {
cols: keys_for_section,
vals: values_for_section,
span,
});
}
// all sections with all its key value pairs
Ok(Value::Record {
cols: sections,
vals: sections_key_value_pairs,
span,
})
}
Err(err) => Err(ShellError::UnsupportedInput(
format!("Could not load ini: {err}"),
"value originates from here".into(),
span,
val_span,
)),
}
}
fn from_ini(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
let (concat_string, span, metadata) = input.collect_string_strict(head)?;
match from_ini_string_to_value(concat_string, head, span) {
Ok(x) => Ok(x.into_pipeline_data_with_metadata(metadata)),
Err(other) => Err(other),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromIni {})
}
#[test]
fn read_ini_config_passes() {
let ini_test_config = r"
min-width=450
max-width=820
[normal]
sound-file=/usr/share/sounds/freedesktop/stereo/dialog-information.oga
[critical]
border-color=FAB387ff
default-timeout=20
sound-file=/usr/share/sounds/freedesktop/stereo/dialog-warning.oga
";
let result = from_ini_string_to_value(
ini_test_config.to_owned(),
Span::test_data(),
Span::test_data(),
);
assert!(result.is_ok());
}
}

8
crates/nu-command/src/formats/from/mod.rs
View File

@ -1,9 +1,6 @@
mod command;
mod csv;
mod delimited;
mod eml;
mod ics;
mod ini;
mod json;
mod nuon;
mod ods;
@ -11,7 +8,6 @@ mod ssv;
mod toml;
mod tsv;
mod url;
mod vcf;
mod xlsx;
mod xml;
mod yaml;
@ -19,16 +15,12 @@ mod yaml;
pub use self::csv::FromCsv;
pub use self::toml::FromToml;
pub use self::url::FromUrl;
pub use crate::formats::from::ini::FromIni;
pub use command::From;
pub use eml::FromEml;
pub use ics::FromIcs;
pub use json::FromJson;
pub use nuon::FromNuon;
pub use ods::FromOds;
pub use ssv::FromSsv;
pub use tsv::FromTsv;
pub use vcf::FromVcf;
pub use xlsx::FromXlsx;
pub use xml::FromXml;
pub use yaml::FromYaml;

206
crates/nu-command/src/formats/from/vcf.rs
View File

@ -1,206 +0,0 @@
use ical::parser::vcard::component::*;
use ical::property::Property;
use indexmap::map::IndexMap;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, Type,
Value,
};
#[derive(Clone)]
pub struct FromVcf;
impl Command for FromVcf {
fn name(&self) -> &str {
"from vcf"
}
fn signature(&self) -> Signature {
Signature::build("from vcf")
.input_output_types(vec![(Type::String, Type::Table(vec![]))])
.category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .vcf and create table."
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let head = call.head;
from_vcf(input, head)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
example: "'BEGIN:VCARD
N:Foo
FN:Bar
EMAIL:foo@bar.com
END:VCARD' | from vcf",
description: "Converts ics formatted string to table",
result: Some(Value::List {
vals: vec![Value::Record {
cols: vec!["properties".to_string()],
vals: vec![Value::List {
vals: vec![
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::test_string("N"),
Value::test_string("Foo"),
Value::Nothing {
span: Span::test_data(),
},
],
span: Span::test_data(),
},
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::test_string("FN"),
Value::test_string("Bar"),
Value::Nothing {
span: Span::test_data(),
},
],
span: Span::test_data(),
},
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::test_string("EMAIL"),
Value::test_string("foo@bar.com"),
Value::Nothing {
span: Span::test_data(),
},
],
span: Span::test_data(),
},
],
span: Span::test_data(),
}],
span: Span::test_data(),
}],
span: Span::test_data(),
}),
}]
}
}
fn from_vcf(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
let (input_string, span, metadata) = input.collect_string_strict(head)?;
let input_string = input_string
.lines()
.map(|x| x.trim().to_string())
.collect::<Vec<_>>()
.join("\n");
let input_bytes = input_string.as_bytes();
let cursor = std::io::Cursor::new(input_bytes);
let parser = ical::VcardParser::new(cursor);
let iter = parser.map(move |contact| match contact {
Ok(c) => contact_to_value(c, head),
Err(e) => Value::Error {
error: ShellError::UnsupportedInput(
format!("input cannot be parsed as .vcf ({e})"),
"value originates from here".into(),
head,
span,
),
},
});
let collected: Vec<_> = iter.collect();
Ok(Value::List {
vals: collected,
span: head,
}
.into_pipeline_data_with_metadata(metadata))
}
fn contact_to_value(contact: VcardContact, span: Span) -> Value {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(contact.properties, span),
);
Value::from(Spanned { item: row, span })
}
fn properties_to_value(properties: Vec<Property>, span: Span) -> Value {
Value::List {
vals: properties
.into_iter()
.map(|prop| {
let mut row = IndexMap::new();
let name = Value::String {
val: prop.name,
span,
};
let value = match prop.value {
Some(val) => Value::String { val, span },
None => Value::Nothing { span },
};
let params = match prop.params {
Some(param_list) => params_to_value(param_list, span),
None => Value::Nothing { span },
};
row.insert("name".to_string(), name);
row.insert("value".to_string(), value);
row.insert("params".to_string(), params);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn params_to_value(params: Vec<(String, Vec<String>)>, span: Span) -> Value {
let mut row = IndexMap::new();
for (param_name, param_values) in params {
let values: Vec<Value> = param_values
.into_iter()
.map(|val| Value::string(val, span))
.collect();
let values = Value::List { vals: values, span };
row.insert(param_name, values);
}
Value::from(Spanned { item: row, span })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromVcf {})
}
}

20
crates/nu-command/tests/commands/open.rs
View File

@ -188,26 +188,6 @@ fn parses_xml() {
)
}
#[test]
fn parses_ini() {
let actual = nu!(
cwd: "tests/fixtures/formats",
"open sample.ini | get SectionOne.integer"
);
assert_eq!(actual.out, "1234")
}
#[test]
fn parses_utf16_ini() {
let actual = nu!(
cwd: "tests/fixtures/formats",
"open ./utf16.ini --raw | decode utf-16 | from ini | rename info | get info | get IconIndex"
);
assert_eq!(actual.out, "-236")
}
#[cfg(feature = "dataframe")]
#[test]
fn parses_arrow_ipc() {

3
crates/nu-command/tests/format_conversions/mod.rs
View File

@ -1,8 +1,6 @@
mod bson;
mod csv;
mod eml;
mod html;
mod ics;
mod json;
mod markdown;
mod nuon;
@ -11,7 +9,6 @@ mod ssv;
mod toml;
mod tsv;
mod url;
mod vcf;
mod xlsx;
mod xml;
mod yaml;

1
crates/nu-test-support/Cargo.toml
View File

@ -17,6 +17,7 @@ nu-glob = { path = "../nu-glob", version = "0.75.1" }
nu-utils = { path="../nu-utils", version = "0.75.1" }
once_cell = "1.16.0"
num-format = "0.4.3"
which = "4.3.0"
getset = "0.1.1"
tempfile = "3.2.0"

5
crates/nu-test-support/src/fs.rs
View File

@ -207,6 +207,11 @@ pub fn executable_path() -> PathBuf {
path
}
pub fn installed_nu_path() -> PathBuf {
let path = std::env::var_os(crate::NATIVE_PATH_ENV_VAR);
which::which_in("nu", path, ".").unwrap_or_else(|_| executable_path())
}
pub fn root() -> PathBuf {
let manifest_dir = if let Ok(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
PathBuf::from(manifest_dir)

8
crates/nu-test-support/src/macros.rs
View File

@ -269,7 +269,13 @@ macro_rules! nu_with_plugins {
let commands = format!("{registrations}{}", $command);
let target_cwd = $crate::fs::in_directory(&$cwd);
let mut process = match Command::new($crate::fs::executable_path())
// In plugin testing, we need to use installed nushell to drive
// plugin commands.
let mut executable_path = $crate::fs::executable_path();
if !executable_path.exists() {
executable_path = $crate::fs::installed_nu_path();
}
let mut process = match Command::new(executable_path)
.current_dir(&target_cwd)
.env("PWD", &target_cwd) // setting PWD is enough to set cwd
.arg("--commands")

16
crates/nu_plugin_formats/Cargo.toml Normal file
View File

@ -0,0 +1,16 @@
[package]
name = "nu_plugin_formats"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
nu-plugin = { path = "../nu-plugin", version = "0.75.1" }
nu-protocol = { path = "../nu-protocol", version = "0.75.1", features = ["plugin"] }
nu-engine = { path = "../nu-engine", version = "0.75.1" }
indexmap = { version = "1.7", features = ["serde-1"] }
eml-parser = "0.1.0"
ical = "0.8.0"
rust-ini = "0.18.0"

18
crates/nu_plugin_formats/README.md Normal file
View File

@ -0,0 +1,18 @@
# nu_plugin_formats
A nushell plugin to convert data to nushell tables.
# support commands:
1. from eml - original ported from nushell core.
2. from ics - original ported from nushell core.
3. from ini - original ported from nushell core.
4. from vcf - original ported from nushell core.
# Prerequisite
`nushell`, It's a nushell plugin, so you need it.
# Usage
1. compile the binary: `cargo build`
2. register plugin(assume it's compiled in ./target/debug/):
```
register ./target/debug/nu_plugin_formats
```

190
crates/nu_plugin_formats/src/from/eml.rs Normal file
View File

@ -0,0 +1,190 @@
use eml_parser::eml::*;
use eml_parser::EmlParser;
use indexmap::map::IndexMap;
use nu_plugin::{EvaluatedCall, LabeledError};
use nu_protocol::{PluginExample, ShellError, Span, Spanned, Value};
const DEFAULT_BODY_PREVIEW: usize = 50;
pub const CMD_NAME: &str = "from eml";
pub fn from_eml_call(call: &EvaluatedCall, input: &Value) -> Result<Value, LabeledError> {
let preview_body: usize = call
.get_flag::<i64>("preview-body")?
.map(|l| if l < 0 { 0 } else { l as usize })
.unwrap_or(DEFAULT_BODY_PREVIEW);
from_eml(input, preview_body, call.head)
}
pub fn examples() -> Vec<PluginExample> {
vec![
PluginExample {
description: "Convert eml structured data into record".into(),
example: "'From: test@email.com
Subject: Welcome
To: someone@somewhere.com
Test' | from eml"
.into(),
result: Some(Value::Record {
cols: vec![
"Subject".to_string(),
"From".to_string(),
"To".to_string(),
"Body".to_string(),
],
vals: vec![
Value::test_string("Welcome"),
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("test@email.com"),
],
span: Span::test_data(),
},
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("someone@somewhere.com"),
],
span: Span::test_data(),
},
Value::test_string("Test"),
],
span: Span::test_data(),
}),
},
PluginExample {
description: "Convert eml structured data into record".into(),
example: "'From: test@email.com
Subject: Welcome
To: someone@somewhere.com
Test' | from eml -b 1"
.into(),
result: Some(Value::Record {
cols: vec![
"Subject".to_string(),
"From".to_string(),
"To".to_string(),
"Body".to_string(),
],
vals: vec![
Value::test_string("Welcome"),
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("test@email.com"),
],
span: Span::test_data(),
},
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::test_data()),
Value::test_string("someone@somewhere.com"),
],
span: Span::test_data(),
},
Value::test_string("T"),
],
span: Span::test_data(),
}),
},
]
}
fn emailaddress_to_value(span: Span, email_address: &EmailAddress) -> Value {
let (n, a) = match email_address {
EmailAddress::AddressOnly { address } => (
Value::nothing(span),
Value::String {
val: address.to_string(),
span,
},
),
EmailAddress::NameAndEmailAddress { name, address } => (
Value::String {
val: name.to_string(),
span,
},
Value::String {
val: address.to_string(),
span,
},
),
};
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![n, a],
span,
}
}
fn headerfieldvalue_to_value(head: Span, value: &HeaderFieldValue) -> Value {
use HeaderFieldValue::*;
match value {
SingleEmailAddress(address) => emailaddress_to_value(head, address),
MultipleEmailAddresses(addresses) => Value::List {
vals: addresses
.iter()
.map(|a| emailaddress_to_value(head, a))
.collect(),
span: head,
},
Unstructured(s) => Value::string(s, head),
Empty => Value::nothing(head),
}
}
fn from_eml(input: &Value, body_preview: usize, head: Span) -> Result<Value, LabeledError> {
let value = input.as_string()?;
let eml = EmlParser::from_string(value)
.with_body_preview(body_preview)
.parse()
.map_err(|_| {
ShellError::CantConvert("structured eml data".into(), "string".into(), head, None)
})?;
let mut collected = IndexMap::new();
if let Some(subj) = eml.subject {
collected.insert(
"Subject".to_string(),
Value::String {
val: subj,
span: head,
},
);
}
if let Some(from) = eml.from {
collected.insert("From".to_string(), headerfieldvalue_to_value(head, &from));
}
if let Some(to) = eml.to {
collected.insert("To".to_string(), headerfieldvalue_to_value(head, &to));
}
for HeaderField { name, value } in &eml.headers {
collected.insert(name.to_string(), headerfieldvalue_to_value(head, value));
}
if let Some(body) = eml.body {
collected.insert(
"Body".to_string(),
Value::String {
val: body,
span: head,
},
);
}
Ok(Value::from(Spanned {
item: collected,
span: head,
}))
}

125
crates/nu-command/src/formats/from/ics.rs → crates/nu_plugin_formats/src/from/ics.rs
View File

@ -1,49 +1,54 @@
extern crate ical;
use ical::parser::ical::component::*;
use ical::property::Property;
use indexmap::map::IndexMap;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, Type,
Value,
};
use nu_plugin::{EvaluatedCall, LabeledError};
use nu_protocol::{PluginExample, ShellError, Span, Spanned, Value};
use std::io::BufReader;
#[derive(Clone)]
pub struct FromIcs;
pub const CMD_NAME: &str = "from ics";
impl Command for FromIcs {
fn name(&self) -> &str {
"from ics"
}
fn signature(&self) -> Signature {
Signature::build("from ics")
.input_output_types(vec![(Type::String, Type::Table(vec![]))])
.category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .ics and create table."
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
pub fn from_ics_call(call: &EvaluatedCall, input: &Value) -> Result<Value, LabeledError> {
let span = input.span().unwrap_or(call.head);
let input_string = input.as_string()?;
let head = call.head;
from_ics(input, head)
let input_string = input_string
.lines()
.map(|x| x.trim().to_string())
.collect::<Vec<_>>()
.join("\n");
let input_bytes = input_string.as_bytes();
let buf_reader = BufReader::new(input_bytes);
let parser = ical::IcalParser::new(buf_reader);
let mut output = vec![];
for calendar in parser {
match calendar {
Ok(c) => output.push(calendar_to_value(c, head)),
Err(e) => output.push(Value::Error {
error: ShellError::UnsupportedInput(
format!("input cannot be parsed as .ics ({e})"),
"value originates from here".into(),
head,
span,
),
}),
}
}
Ok(Value::List {
vals: output,
span: head,
})
}
fn examples(&self) -> Vec<Example> {
vec![Example {
pub fn examples() -> Vec<PluginExample> {
vec![PluginExample {
example: "'BEGIN:VCALENDAR
END:VCALENDAR' | from ics",
description: "Converts ics formatted string to table",
END:VCALENDAR' | from ics"
.into(),
description: "Converts ics formatted string to table".into(),
result: Some(Value::List {
vals: vec![Value::Record {
cols: vec![
@ -91,42 +96,6 @@ END:VCALENDAR' | from ics",
}),
}]
}
}
fn from_ics(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
let (input_string, span, metadata) = input.collect_string_strict(head)?;
let input_string = input_string
.lines()
.map(|x| x.trim().to_string())
.collect::<Vec<_>>()
.join("\n");
let input_bytes = input_string.as_bytes();
let buf_reader = BufReader::new(input_bytes);
let parser = ical::IcalParser::new(buf_reader);
let mut output = vec![];
for calendar in parser {
match calendar {
Ok(c) => output.push(calendar_to_value(c, head)),
Err(e) => output.push(Value::Error {
error: ShellError::UnsupportedInput(
format!("input cannot be parsed as .ics ({e})"),
"value originates from here".into(),
head,
span,
),
}),
}
}
Ok(Value::List {
vals: output,
span: head,
}
.into_pipeline_data_with_metadata(metadata))
}
fn calendar_to_value(calendar: IcalCalendar, span: Span) -> Value {
let mut row = IndexMap::new();
@ -323,15 +292,3 @@ fn params_to_value(params: Vec<(String, Vec<String>)>, span: Span) -> Value {
Value::from(Spanned { item: row, span })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromIcs {})
}
}

82
crates/nu_plugin_formats/src/from/ini.rs Normal file
View File

@ -0,0 +1,82 @@
use nu_plugin::{EvaluatedCall, LabeledError};
use nu_protocol::{PluginExample, ShellError, Span, Value};
pub const CMD_NAME: &str = "from ini";
pub fn from_ini_call(call: &EvaluatedCall, input: &Value) -> Result<Value, LabeledError> {
let span = input.span().unwrap_or(call.head);
let input_string = input.as_string()?;
let head = call.head;
let ini_config: Result<ini::Ini, ini::ParseError> = ini::Ini::load_from_str(&input_string);
match ini_config {
Ok(config) => {
let mut sections: Vec<String> = Vec::new();
let mut sections_key_value_pairs: Vec<Value> = Vec::new();
for (section, properties) in config.iter() {
let mut keys_for_section: Vec<String> = Vec::new();
let mut values_for_section: Vec<Value> = Vec::new();
// section
match section {
Some(section_name) => {
sections.push(section_name.to_owned());
}
None => {
sections.push(String::new());
}
}
// section's key value pairs
for (key, value) in properties.iter() {
keys_for_section.push(key.to_owned());
values_for_section.push(Value::String {
val: value.to_owned(),
span,
});
}
// section with its key value pairs
sections_key_value_pairs.push(Value::Record {
cols: keys_for_section,
vals: values_for_section,
span,
});
}
// all sections with all its key value pairs
Ok(Value::Record {
cols: sections,
vals: sections_key_value_pairs,
span,
})
}
Err(err) => Err(ShellError::UnsupportedInput(
format!("Could not load ini: {err}"),
"value originates from here".into(),
head,
span,
)
.into()),
}
}
pub fn examples() -> Vec<PluginExample> {
vec![PluginExample {
example: "'[foo]
a=1
b=2' | from ini"
.into(),
description: "Converts ini formatted string to record".into(),
result: Some(Value::Record {
cols: vec!["foo".to_string()],
vals: vec![Value::Record {
cols: vec!["a".to_string(), "b".to_string()],
vals: vec![Value::test_string("1"), Value::test_string("2")],
span: Span::test_data(),
}],
span: Span::test_data(),
}),
}]
}

4
crates/nu_plugin_formats/src/from/mod.rs Normal file
View File

@ -0,0 +1,4 @@
pub mod eml;
pub mod ics;
pub mod ini;
pub mod vcf;

164
crates/nu_plugin_formats/src/from/vcf.rs Normal file
View File

@ -0,0 +1,164 @@
use ical::parser::vcard::component::*;
use ical::property::Property;
use indexmap::map::IndexMap;
use nu_plugin::{EvaluatedCall, LabeledError};
use nu_protocol::{PluginExample, ShellError, Span, Spanned, Value};
pub const CMD_NAME: &str = "from vcf";
pub fn from_vcf_call(call: &EvaluatedCall, input: &Value) -> Result<Value, LabeledError> {
let span = input.span().unwrap_or(call.head);
let input_string = input.as_string()?;
let head = call.head;
let input_string = input_string
.lines()
.map(|x| x.trim().to_string())
.collect::<Vec<_>>()
.join("\n");
let input_bytes = input_string.as_bytes();
let cursor = std::io::Cursor::new(input_bytes);
let parser = ical::VcardParser::new(cursor);
let iter = parser.map(move |contact| match contact {
Ok(c) => contact_to_value(c, head),
Err(e) => Value::Error {
error: ShellError::UnsupportedInput(
format!("input cannot be parsed as .vcf ({e})"),
"value originates from here".into(),
head,
span,
),
},
});
let collected: Vec<_> = iter.collect();
Ok(Value::List {
vals: collected,
span: head,
})
}
pub fn examples() -> Vec<PluginExample> {
vec![PluginExample {
example: "'BEGIN:VCARD
N:Foo
FN:Bar
EMAIL:foo@bar.com
END:VCARD' | from vcf"
.into(),
description: "Converts ics formatted string to table".into(),
result: Some(Value::List {
vals: vec![Value::Record {
cols: vec!["properties".to_string()],
vals: vec![Value::List {
vals: vec![
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::test_string("N"),
Value::test_string("Foo"),
Value::Nothing {
span: Span::test_data(),
},
],
span: Span::test_data(),
},
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::test_string("FN"),
Value::test_string("Bar"),
Value::Nothing {
span: Span::test_data(),
},
],
span: Span::test_data(),
},
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::test_string("EMAIL"),
Value::test_string("foo@bar.com"),
Value::Nothing {
span: Span::test_data(),
},
],
span: Span::test_data(),
},
],
span: Span::test_data(),
}],
span: Span::test_data(),
}],
span: Span::test_data(),
}),
}]
}
fn contact_to_value(contact: VcardContact, span: Span) -> Value {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(contact.properties, span),
);
Value::from(Spanned { item: row, span })
}
fn properties_to_value(properties: Vec<Property>, span: Span) -> Value {
Value::List {
vals: properties
.into_iter()
.map(|prop| {
let mut row = IndexMap::new();
let name = Value::String {
val: prop.name,
span,
};
let value = match prop.value {
Some(val) => Value::String { val, span },
None => Value::Nothing { span },
};
let params = match prop.params {
Some(param_list) => params_to_value(param_list, span),
None => Value::Nothing { span },
};
row.insert("name".to_string(), name);
row.insert("value".to_string(), value);
row.insert("params".to_string(), params);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn params_to_value(params: Vec<(String, Vec<String>)>, span: Span) -> Value {
let mut row = IndexMap::new();
for (param_name, param_values) in params {
let values: Vec<Value> = param_values
.into_iter()
.map(|val| Value::string(val, span))
.collect();
let values = Value::List { vals: values, span };
row.insert(param_name, values);
}
Value::from(Spanned { item: row, span })
}

55
crates/nu_plugin_formats/src/lib.rs Normal file
View File

@ -0,0 +1,55 @@
mod from;
use from::{eml, ics, ini, vcf};
use nu_plugin::{EvaluatedCall, LabeledError, Plugin};
use nu_protocol::{Category, PluginSignature, SyntaxShape, Type, Value};
pub struct FromCmds;
impl Plugin for FromCmds {
fn signature(&self) -> Vec<PluginSignature> {
vec![
PluginSignature::build(eml::CMD_NAME)
.input_output_types(vec![(Type::String, Type::Record(vec![]))])
.named(
"preview-body",
SyntaxShape::Int,
"How many bytes of the body to preview",
Some('b'),
)
.plugin_examples(eml::examples())
.category(Category::Formats),
PluginSignature::build(ics::CMD_NAME)
.input_output_types(vec![(Type::String, Type::Table(vec![]))])
.plugin_examples(ics::examples())
.category(Category::Formats),
PluginSignature::build(vcf::CMD_NAME)
.input_output_types(vec![(Type::String, Type::Table(vec![]))])
.plugin_examples(vcf::examples())
.category(Category::Formats),
PluginSignature::build(ini::CMD_NAME)
.input_output_types(vec![(Type::String, Type::Record(vec![]))])
.plugin_examples(ini::examples())
.category(Category::Formats),
]
}
fn run(
&mut self,
name: &str,
call: &EvaluatedCall,
input: &Value,
) -> Result<Value, LabeledError> {
match name {
eml::CMD_NAME => eml::from_eml_call(call, input),
ics::CMD_NAME => ics::from_ics_call(call, input),
vcf::CMD_NAME => vcf::from_vcf_call(call, input),
ini::CMD_NAME => ini::from_ini_call(call, input),
_ => Err(LabeledError {
label: "Plugin call with wrong name signature".into(),
msg: "the signature used to call the plugin does not match any name in the plugin signature vector".into(),
span: Some(call.head),
}),
}
}
}

6
crates/nu_plugin_formats/src/main.rs Normal file
View File

@ -0,0 +1,6 @@
use nu_plugin::{serve_plugin, MsgPackSerializer};
use nu_plugin_formats::FromCmds;
fn main() {
serve_plugin(&mut FromCmds, MsgPackSerializer {})
}

3
install-all.ps1
View File

@ -15,7 +15,8 @@ $NU_PLUGINS = @(
'nu_plugin_gstat',
'nu_plugin_inc',
'nu_plugin_query',
'nu_plugin_custom_values'
'nu_plugin_custom_values',
'nu_plugin_formats'
)
foreach ( $plugin in $NU_PLUGINS) {

1
install-all.sh
View File

@ -17,6 +17,7 @@ NU_PLUGINS=(
'nu_plugin_query'
'nu_plugin_example'
'nu_plugin_custom_values'
'nu_plugin_formats'
)
for plugin in "${NU_PLUGINS[@]}"

68
crates/nu-command/tests/format_conversions/eml.rs → tests/plugins/formats/eml.rs
View File

@ -1,32 +1,22 @@
use nu_test_support::{nu, pipeline};
use nu_test_support::nu_with_plugins;
const TEST_CWD: &str = "tests/fixtures/formats";
// Note: the tests can only run successfully if nushell binary is in `target/debug/`
// The To field in this email is just "to@example.com", which gets parsed out as the Address. The Name is empty.
#[test]
fn from_eml_get_to_field() {
let actual = nu!(
let actual = nu_with_plugins!(
cwd: TEST_CWD,
pipeline(
r#"
open sample.eml
| get To
| get Address
"#
)
plugin: ("nu_plugin_formats"),
"open sample.eml | get To.Address"
);
assert_eq!(actual.out, "to@example.com");
let actual = nu!(
let actual = nu_with_plugins!(
cwd: TEST_CWD,
pipeline(
r#"
open sample.eml
| get To
| get Name
"#
)
plugin: ("nu_plugin_formats"),
"open sample.eml | get To | get Name"
);
assert_eq!(actual.out, "");
@ -35,28 +25,18 @@ fn from_eml_get_to_field() {
// The Reply-To field in this email is "replyto@example.com" <replyto@example.com>, meaning both the Name and Address values are identical.
#[test]
fn from_eml_get_replyto_field() {
let actual = nu!(
let actual = nu_with_plugins!(
cwd: TEST_CWD,
pipeline(
r#"
open sample.eml
| get Reply-To
| get Address
"#
)
plugin: ("nu_plugin_formats"),
"open sample.eml | get Reply-To | get Address"
);
assert_eq!(actual.out, "replyto@example.com");
let actual = nu!(
let actual = nu_with_plugins!(
cwd: TEST_CWD,
pipeline(
r#"
open sample.eml
| get Reply-To
| get Name
"#
)
plugin: ("nu_plugin_formats"),
"open sample.eml | get Reply-To | get Name"
);
assert_eq!(actual.out, "replyto@example.com");
@ -64,14 +44,10 @@ fn from_eml_get_replyto_field() {
#[test]
fn from_eml_get_subject_field() {
let actual = nu!(
let actual = nu_with_plugins!(
cwd: TEST_CWD,
pipeline(
r#"
open sample.eml
| get Subject
"#
)
plugin: ("nu_plugin_formats"),
"open sample.eml | get Subject"
);
assert_eq!(actual.out, "Test Message");
@ -79,14 +55,10 @@ fn from_eml_get_subject_field() {
#[test]
fn from_eml_get_another_header_field() {
let actual = nu!(
let actual = nu_with_plugins!(
cwd: TEST_CWD,
pipeline(
r#"
open sample.eml
| get MIME-Version
"#
)
plugin: ("nu_plugin_formats"),
"open sample.eml | get MIME-Version"
);
assert_eq!(actual.out, "1.0");

24
crates/nu-command/tests/format_conversions/ics.rs → tests/plugins/formats/ics.rs
View File

@ -1,6 +1,6 @@
use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
use nu_test_support::nu_with_plugins;
use nu_test_support::playground::Playground;
use nu_test_support::{nu, pipeline};
#[test]
fn infers_types() {
@ -42,15 +42,13 @@ fn infers_types() {
END:VCALENDAR
"#,
)]);
let cwd = dirs.test();
let actual = nu!(
cwd: dirs.test(), pipeline(
r#"
open calendar.ics
| get events.0
| length
"#
));
let actual = nu_with_plugins!(
cwd: cwd,
plugin: ("nu_plugin_formats"),
"open calendar.ics | get events.0 | length"
);
assert_eq!(actual.out, "2");
})
@ -81,8 +79,10 @@ fn from_ics_text_to_table() {
"#,
)]);
let actual = nu!(
cwd: dirs.test(), pipeline(
let cwd = dirs.test();
let actual = nu_with_plugins!(
cwd: cwd,
plugin: ("nu_plugin_formats"),
r#"
open calendar.txt
| from ics
@ -92,7 +92,7 @@ fn from_ics_text_to_table() {
| first
| get value
"#
));
);
assert_eq!(actual.out, "Maryland Game");
})

55
tests/plugins/formats/ini.rs Normal file
View File

@ -0,0 +1,55 @@
use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
use nu_test_support::nu_with_plugins;
use nu_test_support::playground::Playground;
const TEST_CWD: &str = "tests/fixtures/formats";
#[test]
fn parses_ini() {
let actual = nu_with_plugins!(
cwd: TEST_CWD,
plugin: ("nu_plugin_formats"),
"open sample.ini | get SectionOne.integer"
);
assert_eq!(actual.out, "1234")
}
#[test]
fn parses_utf16_ini() {
let actual = nu_with_plugins!(
cwd: TEST_CWD,
plugin: ("nu_plugin_formats"),
"open ./utf16.ini --raw | decode utf-16 | from ini | rename info | get info | get IconIndex"
);
assert_eq!(actual.out, "-236")
}
#[test]
fn read_ini_with_missing_session() {
Playground::setup("from ini with missiong session", |dirs, sandbox| {
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"some_missing.ini",
r#"
min-width=450
max-width=820
[normal]
sound-file=/usr/share/sounds/freedesktop/stereo/dialog-information.oga
[critical]
border-color=FAB387ff
default-timeout=20
sound-file=/usr/share/sounds/freedesktop/stereo/dialog-warning.oga
"#,
)]);
let cwd = dirs.test();
let actual = nu_with_plugins!(
cwd: cwd,
plugin: ("nu_plugin_formats"),
r#"open some_missing.ini | get "".min-width "#
);
assert_eq!(actual.out, "450");
})
}

4
tests/plugins/formats/mod.rs Normal file
View File

@ -0,0 +1,4 @@
mod eml;
mod ics;
mod ini;
mod vcf;

23
crates/nu-command/tests/format_conversions/vcf.rs → tests/plugins/formats/vcf.rs
View File

@ -1,6 +1,6 @@
use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
use nu_test_support::nu_with_plugins;
use nu_test_support::playground::Playground;
use nu_test_support::{nu, pipeline};
#[test]
fn infers_types() {
@ -31,13 +31,12 @@ fn infers_types() {
"#,
)]);
let actual = nu!(
cwd: dirs.test(), pipeline(
r#"
open contacts.vcf
| length
"#
));
let cwd = dirs.test();
let actual = nu_with_plugins!(
cwd: cwd,
plugin: ("nu_plugin_formats"),
"open contacts.vcf | length"
);
assert_eq!(actual.out, "2");
})
@ -65,8 +64,10 @@ fn from_vcf_text_to_table() {
"#,
)]);
let actual = nu!(
cwd: dirs.test(), pipeline(
let cwd = dirs.test();
let actual = nu_with_plugins!(
cwd: cwd,
plugin: ("nu_plugin_formats"),
r#"
open contacts.txt
| from vcf
@ -75,7 +76,7 @@ fn from_vcf_text_to_table() {
| first
| get value
"#
));
);
assert_eq!(actual.out, "john.doe99@gmail.com");
})

1
tests/plugins/mod.rs
View File

@ -1,2 +1,3 @@
mod core_inc;
mod custom_values;
mod formats;

1
uninstall-all.sh
View File

@ -12,6 +12,7 @@ NU_PLUGINS=(
'nu_plugin_gstat'
'nu_plugin_query'
'nu_plugin_example'
'nu_plugin_formats'
)
cargo uninstall nu