Ensure stable plugins get installed. (#1373)

Cargo.toml

@@ -228,6 +228,42 @@ name = "nu_plugin_core_sys"
 path = "src/plugins/nu_plugin_core_sys.rs"
 required-features = ["sys"]
 
+# Stable plugins
+[[bin]]
+name = "nu_plugin_stable_average"
+path = "src/plugins/nu_plugin_stable_average.rs"
+required-features = ["average"]
+
+[[bin]]
+name = "nu_plugin_stable_fetch"
+path = "src/plugins/nu_plugin_stable_fetch.rs"
+required-features = ["fetch"]
+
+[[bin]]
+name = "nu_plugin_stable_binaryview"
+path = "src/plugins/nu_plugin_stable_binaryview.rs"
+required-features = ["binaryview"]
+
+[[bin]]
+name = "nu_plugin_stable_match"
+path = "src/plugins/nu_plugin_stable_match.rs"
+required-features = ["match"]
+
+[[bin]]
+name = "nu_plugin_stable_post"
+path = "src/plugins/nu_plugin_stable_post.rs"
+required-features = ["post"]
+
+[[bin]]
+name = "nu_plugin_stable_sum"
+path = "src/plugins/nu_plugin_stable_sum.rs"
+required-features = ["sum"]
+
+[[bin]]
+name = "nu_plugin_stable_tree"
+path = "src/plugins/nu_plugin_stable_tree.rs"
+required-features = ["tree"]
+
 # Main nu binary
 [[bin]]
 name = "nu"

crates/nu-source/src/meta.rs

@@ -262,7 +262,18 @@ impl From<&std::ops::Range<usize>> for Span {
 
 /// The set of metadata that can be associated with a value
 #[derive(
-    Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, new,
+    Debug,
+    Default,
+    Clone,
+    PartialEq,
+    Eq,
+    Ord,
+    PartialOrd,
+    Serialize,
+    Deserialize,
+    Hash,
+    Getters,
+    new,
 )]
 pub struct Tag {
     /// The original source for this value
@@ -335,6 +346,14 @@ impl From<&Tag> for Span {
 }
 
 impl Tag {
+    /// Creates a default `Tag' with unknown `Span` position and no `AnchorLocation`
+    pub fn default() -> Self {
+        Tag {
+            anchor: None,
+            span: Span::unknown(),
+        }
+    }
+
     /// Creates a `Tag` from the given `Span` with no `AnchorLocation`
     pub fn unknown_anchor(span: Span) -> Tag {
         Tag { anchor: None, span }
@@ -470,7 +489,9 @@ pub fn span_for_spanned_list(mut iter: impl Iterator<Item = Span>) -> Span {
 ///
 /// `Span`s are combined with `AnchorLocation`s to form another type of metadata, a `Tag`.
 /// A `Span`'s end position must be greater than or equal to its start position.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+#[derive(
+    Debug, Default, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash,
+)]
 pub struct Span {
     start: usize,
     end: usize,
@@ -492,6 +513,11 @@ impl From<Option<Span>> for Span {
 }
 
 impl Span {
+    /// Creates a default new `Span` that has 0 start and 0 end.
+    pub fn default() -> Self {
+        Span::unknown()
+    }
+
     /// Creates a new `Span` that has 0 start and 0 end.
     pub fn unknown() -> Span {
         Span::new(0, 0)

crates/nu_plugin_average/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "An average value plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_average/src/average.rs

@@ -0,0 +1,68 @@
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, UntaggedValue, Value};
+
+#[derive(Debug, Default)]
+pub struct Average {
+    pub total: Option<Value>,
+    pub count: u64,
+}
+
+impl Average {
+    pub fn new() -> Average {
+        Average {
+            total: None,
+            count: 0,
+        }
+    }
+
+    pub fn average(&mut self, value: Value) -> Result<(), ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Nothing) => Ok(()),
+            UntaggedValue::Primitive(Primitive::Int(i)) => match &self.total {
+                Some(Value {
+                    value: UntaggedValue::Primitive(Primitive::Int(j)),
+                    tag,
+                }) => {
+                    self.total = Some(UntaggedValue::int(i + j).into_value(tag));
+                    self.count += 1;
+                    Ok(())
+                }
+                None => {
+                    self.total = Some(value.clone());
+                    self.count += 1;
+                    Ok(())
+                }
+                _ => Err(ShellError::labeled_error(
+                    "Could calculate average of non-integer or unrelated types",
+                    "source",
+                    value.tag,
+                )),
+            },
+            UntaggedValue::Primitive(Primitive::Bytes(b)) => match &self.total {
+                Some(Value {
+                    value: UntaggedValue::Primitive(Primitive::Bytes(j)),
+                    tag,
+                }) => {
+                    self.total = Some(UntaggedValue::bytes(b + j).into_value(tag));
+                    self.count += 1;
+                    Ok(())
+                }
+                None => {
+                    self.total = Some(value);
+                    self.count += 1;
+                    Ok(())
+                }
+                _ => Err(ShellError::labeled_error(
+                    "Could calculate average of non-integer or unrelated types",
+                    "source",
+                    value.tag,
+                )),
+            },
+            x => Err(ShellError::labeled_error(
+                format!("Unrecognized type in stream: {:?}", x),
+                "source",
+                value.tag,
+            )),
+        }
+    }
+}

crates/nu_plugin_average/src/lib.rs

@@ -0,0 +1,4 @@
+mod average;
+mod nu;
+
+pub use average::Average;

crates/nu_plugin_average/src/main.rs

@@ -1,117 +1,6 @@
-use nu_errors::{CoerceInto, ShellError};
+use nu_plugin::serve_plugin;
-use nu_plugin::{serve_plugin, Plugin};
+use nu_plugin_average::Average;
-use nu_protocol::{
-    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, UntaggedValue, Value,
-};
-use nu_source::TaggedItem;
-
-#[derive(Debug)]
-struct Average {
-    total: Option<Value>,
-    count: u64,
-}
-
-impl Average {
-    fn new() -> Average {
-        Average {
-            total: None,
-            count: 0,
-        }
-    }
-
-    fn average(&mut self, value: Value) -> Result<(), ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Nothing) => Ok(()),
-            UntaggedValue::Primitive(Primitive::Int(i)) => match &self.total {
-                Some(Value {
-                    value: UntaggedValue::Primitive(Primitive::Int(j)),
-                    tag,
-                }) => {
-                    self.total = Some(UntaggedValue::int(i + j).into_value(tag));
-                    self.count += 1;
-                    Ok(())
-                }
-                None => {
-                    self.total = Some(value.clone());
-                    self.count += 1;
-                    Ok(())
-                }
-                _ => Err(ShellError::labeled_error(
-                    "Could calculate average of non-integer or unrelated types",
-                    "source",
-                    value.tag,
-                )),
-            },
-            UntaggedValue::Primitive(Primitive::Bytes(b)) => match &self.total {
-                Some(Value {
-                    value: UntaggedValue::Primitive(Primitive::Bytes(j)),
-                    tag,
-                }) => {
-                    self.total = Some(UntaggedValue::bytes(b + j).into_value(tag));
-                    self.count += 1;
-                    Ok(())
-                }
-                None => {
-                    self.total = Some(value);
-                    self.count += 1;
-                    Ok(())
-                }
-                _ => Err(ShellError::labeled_error(
-                    "Could calculate average of non-integer or unrelated types",
-                    "source",
-                    value.tag,
-                )),
-            },
-            x => Err(ShellError::labeled_error(
-                format!("Unrecognized type in stream: {:?}", x),
-                "source",
-                value.tag,
-            )),
-        }
-    }
-}
-
-impl Plugin for Average {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("average")
-            .desc("Compute the average of a column of numerical values.")
-            .filter())
-    }
-
-    fn begin_filter(&mut self, _: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
-        Ok(vec![])
-    }
-
-    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
-        self.average(input)?;
-        Ok(vec![])
-    }
-
-    fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
-        match self.total {
-            None => Ok(vec![]),
-            Some(ref inner) => match &inner.value {
-                UntaggedValue::Primitive(Primitive::Int(i)) => {
-                    let total: u64 = i
-                        .tagged(inner.tag.clone())
-                        .coerce_into("converting for average")?;
-                    let avg = total as f64 / self.count as f64;
-                    let primitive_value: UntaggedValue = Primitive::from(avg).into();
-                    let value = primitive_value.into_value(inner.tag.clone());
-                    Ok(vec![ReturnSuccess::value(value)])
-                }
-                UntaggedValue::Primitive(Primitive::Bytes(bytes)) => {
-                    let avg = *bytes as f64 / self.count as f64;
-                    let primitive_value: UntaggedValue = UntaggedValue::bytes(avg as u64);
-                    let tagged_value = primitive_value.into_value(inner.tag.clone());
-                    Ok(vec![ReturnSuccess::value(tagged_value)])
-                }
-                _ => Ok(vec![]),
-            },
-        }
-    }
-}
 
 fn main() {
-    serve_plugin(&mut Average::new());
+    serve_plugin(&mut Average::new())
 }

crates/nu_plugin_average/src/nu/mod.rs

@@ -0,0 +1,48 @@
+use crate::Average;
+use nu_errors::{CoerceInto, ShellError};
+use nu_plugin::Plugin;
+use nu_protocol::{
+    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, UntaggedValue, Value,
+};
+use nu_source::TaggedItem;
+
+impl Plugin for Average {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("average")
+            .desc("Compute the average of a column of numerical values.")
+            .filter())
+    }
+
+    fn begin_filter(&mut self, _: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        self.average(input)?;
+        Ok(vec![])
+    }
+
+    fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
+        match self.total {
+            None => Ok(vec![]),
+            Some(ref inner) => match &inner.value {
+                UntaggedValue::Primitive(Primitive::Int(i)) => {
+                    let total: u64 = i
+                        .tagged(inner.tag.clone())
+                        .coerce_into("converting for average")?;
+                    let avg = total as f64 / self.count as f64;
+                    let primitive_value: UntaggedValue = Primitive::from(avg).into();
+                    let value = primitive_value.into_value(inner.tag.clone());
+                    Ok(vec![ReturnSuccess::value(value)])
+                }
+                UntaggedValue::Primitive(Primitive::Bytes(bytes)) => {
+                    let avg = *bytes as f64 / self.count as f64;
+                    let primitive_value: UntaggedValue = UntaggedValue::bytes(avg as u64);
+                    let tagged_value = primitive_value.into_value(inner.tag.clone());
+                    Ok(vec![ReturnSuccess::value(tagged_value)])
+                }
+                _ => Ok(vec![]),
+            },
+        }
+    }
+}

crates/nu_plugin_binaryview/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "A binary viewer plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 ansi_term = "0.12.1"
 crossterm = { version = "0.14.2" }

crates/nu_plugin_binaryview/src/binaryview.rs

@@ -0,0 +1,402 @@
+use crossterm::{style::Attribute, ExecutableCommand};
+use nu_protocol::outln;
+use nu_source::AnchorLocation;
+use pretty_hex::*;
+
+#[derive(Default)]
+pub struct BinaryView;
+
+impl BinaryView {
+    pub fn new() -> BinaryView {
+        BinaryView
+    }
+}
+
+pub fn view_binary(
+    b: &[u8],
+    source: Option<&AnchorLocation>,
+    lores_mode: bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    if b.len() > 3 {
+        if let (0x4e, 0x45, 0x53) = (b[0], b[1], b[2]) {
+            view_contents_interactive(b, source, lores_mode)?;
+            return Ok(());
+        }
+    }
+    view_contents(b, source, lores_mode)?;
+    Ok(())
+}
+
+pub struct RenderContext {
+    pub width: usize,
+    pub height: usize,
+    pub frame_buffer: Vec<(u8, u8, u8)>,
+    pub since_last_button: Vec<usize>,
+    pub lores_mode: bool,
+}
+
+impl RenderContext {
+    pub fn blank(lores_mode: bool) -> RenderContext {
+        RenderContext {
+            width: 0,
+            height: 0,
+            frame_buffer: vec![],
+            since_last_button: vec![0; 8],
+            lores_mode,
+        }
+    }
+    pub fn clear(&mut self) {
+        self.frame_buffer = vec![(0, 0, 0); self.width * self.height as usize];
+    }
+
+    fn render_to_screen_lores(&mut self) -> Result<(), Box<dyn std::error::Error>> {
+        let mut prev_color: Option<(u8, u8, u8)> = None;
+        let mut prev_count = 1;
+
+        let _ = std::io::stdout().execute(crossterm::cursor::MoveTo(0, 0));
+
+        for pixel in &self.frame_buffer {
+            match prev_color {
+                Some(c) if c == *pixel => {
+                    prev_count += 1;
+                }
+                Some(c) => {
+                    print!(
+                        "{}",
+                        ansi_term::Colour::RGB(c.0, c.1, c.2)
+                            .paint((0..prev_count).map(|_| "█").collect::<String>())
+                    );
+                    prev_color = Some(*pixel);
+                    prev_count = 1;
+                }
+                _ => {
+                    prev_color = Some(*pixel);
+                    prev_count = 1;
+                }
+            }
+        }
+
+        if prev_count > 0 {
+            if let Some(color) = prev_color {
+                print!(
+                    "{}",
+                    ansi_term::Colour::RGB(color.0, color.1, color.2)
+                        .paint((0..prev_count).map(|_| "█").collect::<String>())
+                );
+            }
+        }
+        outln!("{}", Attribute::Reset);
+        Ok(())
+    }
+    fn render_to_screen_hires(&mut self) -> Result<(), Box<dyn std::error::Error>> {
+        let mut prev_fg: Option<(u8, u8, u8)> = None;
+        let mut prev_bg: Option<(u8, u8, u8)> = None;
+        let mut prev_count = 1;
+
+        let mut pos = 0;
+        let fb_len = self.frame_buffer.len();
+
+        let _ = std::io::stdout().execute(crossterm::cursor::MoveTo(0, 0));
+
+        while pos < (fb_len - self.width) {
+            let top_pixel = self.frame_buffer[pos];
+            let bottom_pixel = self.frame_buffer[pos + self.width];
+
+            match (prev_fg, prev_bg) {
+                (Some(c), Some(d)) if c == top_pixel && d == bottom_pixel => {
+                    prev_count += 1;
+                }
+                (Some(c), Some(d)) => {
+                    print!(
+                        "{}",
+                        ansi_term::Colour::RGB(c.0, c.1, c.2)
+                            .on(ansi_term::Colour::RGB(d.0, d.1, d.2,))
+                            .paint((0..prev_count).map(|_| "▀").collect::<String>())
+                    );
+                    prev_fg = Some(top_pixel);
+                    prev_bg = Some(bottom_pixel);
+                    prev_count = 1;
+                }
+                _ => {
+                    prev_fg = Some(top_pixel);
+                    prev_bg = Some(bottom_pixel);
+                    prev_count = 1;
+                }
+            }
+            pos += 1;
+            if pos % self.width == 0 {
+                pos += self.width;
+            }
+        }
+        if prev_count > 0 {
+            if let (Some(c), Some(d)) = (prev_fg, prev_bg) {
+                print!(
+                    "{}",
+                    ansi_term::Colour::RGB(c.0, c.1, c.2)
+                        .on(ansi_term::Colour::RGB(d.0, d.1, d.2,))
+                        .paint((0..prev_count).map(|_| "▀").collect::<String>())
+                );
+            }
+        }
+        outln!("{}", Attribute::Reset);
+        Ok(())
+    }
+    pub fn flush(&mut self) -> Result<(), Box<dyn std::error::Error>> {
+        if self.lores_mode {
+            self.render_to_screen_lores()
+        } else {
+            self.render_to_screen_hires()
+        }
+    }
+    pub fn update(&mut self) -> Result<(), Box<dyn std::error::Error>> {
+        let terminal_size = crossterm::terminal::size().unwrap_or_else(|_| (80, 24));
+
+        if (self.width != terminal_size.0 as usize) || (self.height != terminal_size.1 as usize) {
+            let _ = std::io::stdout().execute(crossterm::cursor::Hide);
+
+            self.width = terminal_size.0 as usize;
+            self.height = if self.lores_mode {
+                terminal_size.1 as usize - 1
+            } else {
+                (terminal_size.1 as usize - 1) * 2
+            };
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+struct RawImageBuffer {
+    dimensions: (u64, u64),
+    colortype: image::ColorType,
+    buffer: Vec<u8>,
+}
+
+fn load_from_png_buffer(buffer: &[u8]) -> Result<RawImageBuffer, Box<dyn std::error::Error>> {
+    use image::ImageDecoder;
+
+    let decoder = image::png::PNGDecoder::new(buffer)?;
+
+    let dimensions = decoder.dimensions();
+    let colortype = decoder.colortype();
+    let buffer = decoder.read_image()?;
+
+    Ok(RawImageBuffer {
+        dimensions,
+        colortype,
+        buffer,
+    })
+}
+
+fn load_from_jpg_buffer(buffer: &[u8]) -> Result<RawImageBuffer, Box<dyn std::error::Error>> {
+    use image::ImageDecoder;
+
+    let decoder = image::jpeg::JPEGDecoder::new(buffer)?;
+
+    let dimensions = decoder.dimensions();
+    let colortype = decoder.colortype();
+    let buffer = decoder.read_image()?;
+
+    Ok(RawImageBuffer {
+        dimensions,
+        colortype,
+        buffer,
+    })
+}
+
+pub fn view_contents(
+    buffer: &[u8],
+    _source: Option<&AnchorLocation>,
+    lores_mode: bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut raw_image_buffer = load_from_png_buffer(buffer);
+
+    if raw_image_buffer.is_err() {
+        raw_image_buffer = load_from_jpg_buffer(buffer);
+    }
+
+    if raw_image_buffer.is_err() {
+        //Not yet supported
+        outln!("{:?}", buffer.hex_dump());
+        return Ok(());
+    }
+    let raw_image_buffer = raw_image_buffer?;
+
+    let mut render_context: RenderContext = RenderContext::blank(lores_mode);
+    let _ = render_context.update();
+    render_context.clear();
+
+    match raw_image_buffer.colortype {
+        image::ColorType::RGBA(8) => {
+            let img = image::ImageBuffer::<image::Rgba<u8>, Vec<u8>>::from_vec(
+                raw_image_buffer.dimensions.0 as u32,
+                raw_image_buffer.dimensions.1 as u32,
+                raw_image_buffer.buffer,
+            )
+            .ok_or("Cannot convert image data")?;
+
+            let resized_img = image::imageops::resize(
+                &img,
+                render_context.width as u32,
+                render_context.height as u32,
+                image::FilterType::Lanczos3,
+            );
+
+            let mut count = 0;
+            for pixel in resized_img.pixels() {
+                use image::Pixel;
+                let rgb = pixel.to_rgb();
+                render_context.frame_buffer[count] = (rgb[0], rgb[1], rgb[2]);
+                count += 1;
+            }
+        }
+        image::ColorType::RGB(8) => {
+            let img = image::ImageBuffer::<image::Rgb<u8>, Vec<u8>>::from_vec(
+                raw_image_buffer.dimensions.0 as u32,
+                raw_image_buffer.dimensions.1 as u32,
+                raw_image_buffer.buffer,
+            )
+            .ok_or("Cannot convert image data")?;
+
+            let resized_img = image::imageops::resize(
+                &img,
+                render_context.width as u32,
+                render_context.height as u32,
+                image::FilterType::Lanczos3,
+            );
+
+            let mut count = 0;
+            for pixel in resized_img.pixels() {
+                use image::Pixel;
+                let rgb = pixel.to_rgb();
+                render_context.frame_buffer[count] = (rgb[0], rgb[1], rgb[2]);
+                count += 1;
+            }
+        }
+        _ => {
+            //Not yet supported
+            outln!("{:?}", buffer.hex_dump());
+            return Ok(());
+        }
+    }
+
+    render_context.flush()?;
+
+    let _ = std::io::stdout().execute(crossterm::cursor::Show);
+
+    let _ = crossterm::terminal::disable_raw_mode();
+
+    Ok(())
+}
+
+pub fn view_contents_interactive(
+    buffer: &[u8],
+    source: Option<&AnchorLocation>,
+    lores_mode: bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    use rawkey::{KeyCode, RawKey};
+
+    let sav_path = if let Some(AnchorLocation::File(f)) = source {
+        let mut path = std::path::PathBuf::from(f);
+        path.set_extension("sav");
+        Some(path)
+    } else {
+        None
+    };
+
+    let mut nes = neso::Nes::new(0.0);
+    let rawkey = RawKey::new();
+    nes.load_rom(&buffer);
+
+    if let Some(ref sav_path) = sav_path {
+        if let Ok(contents) = std::fs::read(sav_path) {
+            let _ = nes.load_state(&contents);
+        }
+    }
+
+    nes.reset();
+
+    if let Ok(_raw) = crossterm::terminal::enable_raw_mode() {
+        let mut render_context: RenderContext = RenderContext::blank(lores_mode);
+
+        let buttons = vec![
+            KeyCode::Alt,
+            KeyCode::LeftControl,
+            KeyCode::Tab,
+            KeyCode::BackSpace,
+            KeyCode::UpArrow,
+            KeyCode::DownArrow,
+            KeyCode::LeftArrow,
+            KeyCode::RightArrow,
+        ];
+
+        let _ = std::io::stdout().execute(crossterm::cursor::Hide);
+
+        'gameloop: loop {
+            let _ = render_context.update();
+            nes.step_frame();
+
+            let image_buffer = nes.image_buffer();
+
+            let slice = unsafe { std::slice::from_raw_parts(image_buffer, 256 * 240 * 4) };
+            let img = image::ImageBuffer::<image::Rgba<u8>, &[u8]>::from_raw(256, 240, slice)
+                .ok_or("Cannot convert image data")?;
+            let resized_img = image::imageops::resize(
+                &img,
+                render_context.width as u32,
+                render_context.height as u32,
+                image::FilterType::Lanczos3,
+            );
+
+            render_context.clear();
+
+            let mut count = 0;
+            for pixel in resized_img.pixels() {
+                use image::Pixel;
+                let rgb = pixel.to_rgb();
+
+                render_context.frame_buffer[count] = (rgb[0], rgb[1], rgb[2]);
+                count += 1;
+            }
+            render_context.flush()?;
+
+            if rawkey.is_pressed(rawkey::KeyCode::Escape) {
+                break 'gameloop;
+            } else {
+                for (idx, button) in buttons.iter().enumerate() {
+                    if rawkey.is_pressed(*button) {
+                        nes.press_button(0, idx as u8);
+                    } else {
+                        nes.release_button(0, idx as u8);
+                    }
+                }
+                loop {
+                    let x = crossterm::event::poll(std::time::Duration::from_secs(0));
+                    match x {
+                        Ok(true) => {
+                            // Swallow the events so we don't queue them into the line editor
+                            let _ = crossterm::event::read();
+                        }
+                        _ => {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    if let Some(ref sav_path) = sav_path {
+        let buffer = nes.save_state();
+        if let Ok(buffer) = buffer {
+            let _ = std::fs::write(sav_path, buffer);
+        }
+    }
+
+    let _ = std::io::stdout().execute(crossterm::cursor::Show);
+
+    let _screen = crossterm::terminal::disable_raw_mode();
+
+    Ok(())
+}

crates/nu_plugin_binaryview/src/lib.rs

@@ -0,0 +1,4 @@
+mod binaryview;
+mod nu;
+
+pub use binaryview::BinaryView;

crates/nu_plugin_binaryview/src/main.rs

@@ -1,424 +1,6 @@
-use crossterm::{style::Attribute, ExecutableCommand};
+use nu_plugin::serve_plugin;
-use nu_errors::ShellError;
+use nu_plugin_binaryview::BinaryView;
-use nu_plugin::{serve_plugin, Plugin};
-use nu_protocol::{outln, CallInfo, Primitive, Signature, UntaggedValue, Value};
-use nu_source::AnchorLocation;
-use pretty_hex::*;
-
-struct BinaryView;
-
-impl BinaryView {
-    fn new() -> BinaryView {
-        BinaryView
-    }
-}
-
-impl Plugin for BinaryView {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("binaryview")
-            .desc("Autoview of binary data.")
-            .switch("lores", "use low resolution output mode"))
-    }
-
-    fn sink(&mut self, call_info: CallInfo, input: Vec<Value>) {
-        for v in input {
-            let value_anchor = v.anchor();
-            if let UntaggedValue::Primitive(Primitive::Binary(b)) = &v.value {
-                let _ = view_binary(&b, value_anchor.as_ref(), call_info.args.has("lores"));
-            }
-        }
-    }
-}
-
-fn view_binary(
-    b: &[u8],
-    source: Option<&AnchorLocation>,
-    lores_mode: bool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    if b.len() > 3 {
-        if let (0x4e, 0x45, 0x53) = (b[0], b[1], b[2]) {
-            view_contents_interactive(b, source, lores_mode)?;
-            return Ok(());
-        }
-    }
-    view_contents(b, source, lores_mode)?;
-    Ok(())
-}
-
-pub struct RenderContext {
-    pub width: usize,
-    pub height: usize,
-    pub frame_buffer: Vec<(u8, u8, u8)>,
-    pub since_last_button: Vec<usize>,
-    pub lores_mode: bool,
-}
-
-impl RenderContext {
-    pub fn blank(lores_mode: bool) -> RenderContext {
-        RenderContext {
-            width: 0,
-            height: 0,
-            frame_buffer: vec![],
-            since_last_button: vec![0; 8],
-            lores_mode,
-        }
-    }
-    pub fn clear(&mut self) {
-        self.frame_buffer = vec![(0, 0, 0); self.width * self.height as usize];
-    }
-
-    fn render_to_screen_lores(&mut self) -> Result<(), Box<dyn std::error::Error>> {
-        let mut prev_color: Option<(u8, u8, u8)> = None;
-        let mut prev_count = 1;
-
-        let _ = std::io::stdout().execute(crossterm::cursor::MoveTo(0, 0));
-
-        for pixel in &self.frame_buffer {
-            match prev_color {
-                Some(c) if c == *pixel => {
-                    prev_count += 1;
-                }
-                Some(c) => {
-                    print!(
-                        "{}",
-                        ansi_term::Colour::RGB(c.0, c.1, c.2)
-                            .paint((0..prev_count).map(|_| "█").collect::<String>())
-                    );
-                    prev_color = Some(*pixel);
-                    prev_count = 1;
-                }
-                _ => {
-                    prev_color = Some(*pixel);
-                    prev_count = 1;
-                }
-            }
-        }
-
-        if prev_count > 0 {
-            if let Some(color) = prev_color {
-                print!(
-                    "{}",
-                    ansi_term::Colour::RGB(color.0, color.1, color.2)
-                        .paint((0..prev_count).map(|_| "█").collect::<String>())
-                );
-            }
-        }
-        outln!("{}", Attribute::Reset);
-        Ok(())
-    }
-    fn render_to_screen_hires(&mut self) -> Result<(), Box<dyn std::error::Error>> {
-        let mut prev_fg: Option<(u8, u8, u8)> = None;
-        let mut prev_bg: Option<(u8, u8, u8)> = None;
-        let mut prev_count = 1;
-
-        let mut pos = 0;
-        let fb_len = self.frame_buffer.len();
-
-        let _ = std::io::stdout().execute(crossterm::cursor::MoveTo(0, 0));
-
-        while pos < (fb_len - self.width) {
-            let top_pixel = self.frame_buffer[pos];
-            let bottom_pixel = self.frame_buffer[pos + self.width];
-
-            match (prev_fg, prev_bg) {
-                (Some(c), Some(d)) if c == top_pixel && d == bottom_pixel => {
-                    prev_count += 1;
-                }
-                (Some(c), Some(d)) => {
-                    print!(
-                        "{}",
-                        ansi_term::Colour::RGB(c.0, c.1, c.2)
-                            .on(ansi_term::Colour::RGB(d.0, d.1, d.2,))
-                            .paint((0..prev_count).map(|_| "▀").collect::<String>())
-                    );
-                    prev_fg = Some(top_pixel);
-                    prev_bg = Some(bottom_pixel);
-                    prev_count = 1;
-                }
-                _ => {
-                    prev_fg = Some(top_pixel);
-                    prev_bg = Some(bottom_pixel);
-                    prev_count = 1;
-                }
-            }
-            pos += 1;
-            if pos % self.width == 0 {
-                pos += self.width;
-            }
-        }
-        if prev_count > 0 {
-            if let (Some(c), Some(d)) = (prev_fg, prev_bg) {
-                print!(
-                    "{}",
-                    ansi_term::Colour::RGB(c.0, c.1, c.2)
-                        .on(ansi_term::Colour::RGB(d.0, d.1, d.2,))
-                        .paint((0..prev_count).map(|_| "▀").collect::<String>())
-                );
-            }
-        }
-        outln!("{}", Attribute::Reset);
-        Ok(())
-    }
-    pub fn flush(&mut self) -> Result<(), Box<dyn std::error::Error>> {
-        if self.lores_mode {
-            self.render_to_screen_lores()
-        } else {
-            self.render_to_screen_hires()
-        }
-    }
-    pub fn update(&mut self) -> Result<(), Box<dyn std::error::Error>> {
-        let terminal_size = crossterm::terminal::size().unwrap_or_else(|_| (80, 24));
-
-        if (self.width != terminal_size.0 as usize) || (self.height != terminal_size.1 as usize) {
-            let _ = std::io::stdout().execute(crossterm::cursor::Hide);
-
-            self.width = terminal_size.0 as usize;
-            self.height = if self.lores_mode {
-                terminal_size.1 as usize - 1
-            } else {
-                (terminal_size.1 as usize - 1) * 2
-            };
-        }
-
-        Ok(())
-    }
-}
-
-#[derive(Debug)]
-struct RawImageBuffer {
-    dimensions: (u64, u64),
-    colortype: image::ColorType,
-    buffer: Vec<u8>,
-}
-
-fn load_from_png_buffer(buffer: &[u8]) -> Result<RawImageBuffer, Box<dyn std::error::Error>> {
-    use image::ImageDecoder;
-
-    let decoder = image::png::PNGDecoder::new(buffer)?;
-
-    let dimensions = decoder.dimensions();
-    let colortype = decoder.colortype();
-    let buffer = decoder.read_image()?;
-
-    Ok(RawImageBuffer {
-        dimensions,
-        colortype,
-        buffer,
-    })
-}
-
-fn load_from_jpg_buffer(buffer: &[u8]) -> Result<RawImageBuffer, Box<dyn std::error::Error>> {
-    use image::ImageDecoder;
-
-    let decoder = image::jpeg::JPEGDecoder::new(buffer)?;
-
-    let dimensions = decoder.dimensions();
-    let colortype = decoder.colortype();
-    let buffer = decoder.read_image()?;
-
-    Ok(RawImageBuffer {
-        dimensions,
-        colortype,
-        buffer,
-    })
-}
-
-pub fn view_contents(
-    buffer: &[u8],
-    _source: Option<&AnchorLocation>,
-    lores_mode: bool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let mut raw_image_buffer = load_from_png_buffer(buffer);
-
-    if raw_image_buffer.is_err() {
-        raw_image_buffer = load_from_jpg_buffer(buffer);
-    }
-
-    if raw_image_buffer.is_err() {
-        //Not yet supported
-        outln!("{:?}", buffer.hex_dump());
-        return Ok(());
-    }
-    let raw_image_buffer = raw_image_buffer?;
-
-    let mut render_context: RenderContext = RenderContext::blank(lores_mode);
-    let _ = render_context.update();
-    render_context.clear();
-
-    match raw_image_buffer.colortype {
-        image::ColorType::RGBA(8) => {
-            let img = image::ImageBuffer::<image::Rgba<u8>, Vec<u8>>::from_vec(
-                raw_image_buffer.dimensions.0 as u32,
-                raw_image_buffer.dimensions.1 as u32,
-                raw_image_buffer.buffer,
-            )
-            .ok_or("Cannot convert image data")?;
-
-            let resized_img = image::imageops::resize(
-                &img,
-                render_context.width as u32,
-                render_context.height as u32,
-                image::FilterType::Lanczos3,
-            );
-
-            let mut count = 0;
-            for pixel in resized_img.pixels() {
-                use image::Pixel;
-                let rgb = pixel.to_rgb();
-                render_context.frame_buffer[count] = (rgb[0], rgb[1], rgb[2]);
-                count += 1;
-            }
-        }
-        image::ColorType::RGB(8) => {
-            let img = image::ImageBuffer::<image::Rgb<u8>, Vec<u8>>::from_vec(
-                raw_image_buffer.dimensions.0 as u32,
-                raw_image_buffer.dimensions.1 as u32,
-                raw_image_buffer.buffer,
-            )
-            .ok_or("Cannot convert image data")?;
-
-            let resized_img = image::imageops::resize(
-                &img,
-                render_context.width as u32,
-                render_context.height as u32,
-                image::FilterType::Lanczos3,
-            );
-
-            let mut count = 0;
-            for pixel in resized_img.pixels() {
-                use image::Pixel;
-                let rgb = pixel.to_rgb();
-                render_context.frame_buffer[count] = (rgb[0], rgb[1], rgb[2]);
-                count += 1;
-            }
-        }
-        _ => {
-            //Not yet supported
-            outln!("{:?}", buffer.hex_dump());
-            return Ok(());
-        }
-    }
-
-    render_context.flush()?;
-
-    let _ = std::io::stdout().execute(crossterm::cursor::Show);
-
-    let _ = crossterm::terminal::disable_raw_mode();
-
-    Ok(())
-}
-
-pub fn view_contents_interactive(
-    buffer: &[u8],
-    source: Option<&AnchorLocation>,
-    lores_mode: bool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    use rawkey::{KeyCode, RawKey};
-
-    let sav_path = if let Some(AnchorLocation::File(f)) = source {
-        let mut path = std::path::PathBuf::from(f);
-        path.set_extension("sav");
-        Some(path)
-    } else {
-        None
-    };
-
-    let mut nes = neso::Nes::new(0.0);
-    let rawkey = RawKey::new();
-    nes.load_rom(&buffer);
-
-    if let Some(ref sav_path) = sav_path {
-        if let Ok(contents) = std::fs::read(sav_path) {
-            let _ = nes.load_state(&contents);
-        }
-    }
-
-    nes.reset();
-
-    if let Ok(_raw) = crossterm::terminal::enable_raw_mode() {
-        let mut render_context: RenderContext = RenderContext::blank(lores_mode);
-
-        let buttons = vec![
-            KeyCode::Alt,
-            KeyCode::LeftControl,
-            KeyCode::Tab,
-            KeyCode::BackSpace,
-            KeyCode::UpArrow,
-            KeyCode::DownArrow,
-            KeyCode::LeftArrow,
-            KeyCode::RightArrow,
-        ];
-
-        let _ = std::io::stdout().execute(crossterm::cursor::Hide);
-
-        'gameloop: loop {
-            let _ = render_context.update();
-            nes.step_frame();
-
-            let image_buffer = nes.image_buffer();
-
-            let slice = unsafe { std::slice::from_raw_parts(image_buffer, 256 * 240 * 4) };
-            let img = image::ImageBuffer::<image::Rgba<u8>, &[u8]>::from_raw(256, 240, slice)
-                .ok_or("Cannot convert image data")?;
-            let resized_img = image::imageops::resize(
-                &img,
-                render_context.width as u32,
-                render_context.height as u32,
-                image::FilterType::Lanczos3,
-            );
-
-            render_context.clear();
-
-            let mut count = 0;
-            for pixel in resized_img.pixels() {
-                use image::Pixel;
-                let rgb = pixel.to_rgb();
-
-                render_context.frame_buffer[count] = (rgb[0], rgb[1], rgb[2]);
-                count += 1;
-            }
-            render_context.flush()?;
-
-            if rawkey.is_pressed(rawkey::KeyCode::Escape) {
-                break 'gameloop;
-            } else {
-                for (idx, button) in buttons.iter().enumerate() {
-                    if rawkey.is_pressed(*button) {
-                        nes.press_button(0, idx as u8);
-                    } else {
-                        nes.release_button(0, idx as u8);
-                    }
-                }
-                loop {
-                    let x = crossterm::event::poll(std::time::Duration::from_secs(0));
-                    match x {
-                        Ok(true) => {
-                            // Swallow the events so we don't queue them into the line editor
-                            let _ = crossterm::event::read();
-                        }
-                        _ => {
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    if let Some(ref sav_path) = sav_path {
-        let buffer = nes.save_state();
-        if let Ok(buffer) = buffer {
-            let _ = std::fs::write(sav_path, buffer);
-        }
-    }
-
-    let _ = std::io::stdout().execute(crossterm::cursor::Show);
-
-    let _screen = crossterm::terminal::disable_raw_mode();
-
-    Ok(())
-}
 
 fn main() {
-    serve_plugin(&mut BinaryView::new());
+    serve_plugin(&mut BinaryView::new())
 }

crates/nu_plugin_binaryview/src/nu/mod.rs

@@ -0,0 +1,23 @@
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{CallInfo, Primitive, Signature, UntaggedValue, Value};
+
+use crate::binaryview::view_binary;
+use crate::BinaryView;
+
+impl Plugin for BinaryView {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("binaryview")
+            .desc("Autoview of binary data.")
+            .switch("lores", "use low resolution output mode"))
+    }
+
+    fn sink(&mut self, call_info: CallInfo, input: Vec<Value>) {
+        for v in input {
+            let value_anchor = v.anchor();
+            if let UntaggedValue::Primitive(Primitive::Binary(b)) = &v.value {
+                let _ = view_binary(&b, value_anchor.as_ref(), call_info.args.has("lores"));
+            }
+        }
+    }
+}

crates/nu_plugin_fetch/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "A URL fetch plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_fetch/src/fetch.rs

@@ -0,0 +1,258 @@
+use mime::Mime;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, CommandAction, ReturnSuccess, ReturnValue, UntaggedValue, Value};
+use nu_source::{AnchorLocation, Span, Tag};
+use std::path::PathBuf;
+use std::str::FromStr;
+use surf::mime;
+
+#[derive(Default)]
+pub struct Fetch {
+    pub path: Option<Value>,
+    pub has_raw: bool,
+}
+
+impl Fetch {
+    pub fn new() -> Fetch {
+        Fetch {
+            path: None,
+            has_raw: false,
+        }
+    }
+
+    pub fn setup(&mut self, call_info: CallInfo) -> ReturnValue {
+        self.path = Some(
+            match call_info.args.nth(0).ok_or_else(|| {
+                ShellError::labeled_error(
+                    "No file or directory specified",
+                    "for command",
+                    &call_info.name_tag,
+                )
+            })? {
+                file => file.clone(),
+            },
+        );
+
+        self.has_raw = call_info.args.has("raw");
+
+        ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value())
+    }
+}
+
+pub async fn fetch_helper(path: &Value, has_raw: bool, row: Value) -> ReturnValue {
+    let path_buf = path.as_path()?;
+    let path_str = path_buf.display().to_string();
+
+    //FIXME: this is a workaround because plugins don't yet support per-item iteration
+    let path_str = if path_str == "$it" {
+        let path_buf = row.as_path()?;
+        path_buf.display().to_string()
+    } else {
+        path_str
+    };
+
+    let path_span = path.tag.span;
+
+    let result = fetch(&path_str, path_span).await;
+
+    if let Err(e) = result {
+        return Err(e);
+    }
+    let (file_extension, contents, contents_tag) = result?;
+
+    let file_extension = if has_raw {
+        None
+    } else {
+        // If the extension could not be determined via mimetype, try to use the path
+        // extension. Some file types do not declare their mimetypes (such as bson files).
+        file_extension.or_else(|| path_str.split('.').last().map(String::from))
+    };
+
+    let tagged_contents = contents.retag(&contents_tag);
+
+    if let Some(extension) = file_extension {
+        Ok(ReturnSuccess::Action(CommandAction::AutoConvert(
+            tagged_contents,
+            extension,
+        )))
+    } else {
+        ReturnSuccess::value(tagged_contents)
+    }
+}
+
+pub async fn fetch(
+    location: &str,
+    span: Span,
+) -> Result<(Option<String>, UntaggedValue, Tag), ShellError> {
+    if url::Url::parse(location).is_err() {
+        return Err(ShellError::labeled_error(
+            "Incomplete or incorrect url",
+            "expected a full url",
+            span,
+        ));
+    }
+
+    let response = surf::get(location).await;
+    match response {
+        Ok(mut r) => match r.headers().get("content-type") {
+            Some(content_type) => {
+                let content_type = Mime::from_str(content_type).map_err(|_| {
+                    ShellError::labeled_error(
+                        format!("MIME type unknown: {}", content_type),
+                        "given unknown MIME type",
+                        span,
+                    )
+                })?;
+                match (content_type.type_(), content_type.subtype()) {
+                    (mime::APPLICATION, mime::XML) => Ok((
+                        Some("xml".to_string()),
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load text from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
+                        },
+                    )),
+                    (mime::APPLICATION, mime::JSON) => Ok((
+                        Some("json".to_string()),
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load text from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
+                        },
+                    )),
+                    (mime::APPLICATION, mime::OCTET_STREAM) => {
+                        let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load binary file",
+                                "could not load",
+                                span,
+                            )
+                        })?;
+                        Ok((
+                            None,
+                            UntaggedValue::binary(buf),
+                            Tag {
+                                span,
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                            },
+                        ))
+                    }
+                    (mime::IMAGE, mime::SVG) => Ok((
+                        Some("svg".to_string()),
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load svg from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
+                        },
+                    )),
+                    (mime::IMAGE, image_ty) => {
+                        let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load image file",
+                                "could not load",
+                                span,
+                            )
+                        })?;
+                        Ok((
+                            Some(image_ty.to_string()),
+                            UntaggedValue::binary(buf),
+                            Tag {
+                                span,
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                            },
+                        ))
+                    }
+                    (mime::TEXT, mime::HTML) => Ok((
+                        Some("html".to_string()),
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load text from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
+                        },
+                    )),
+                    (mime::TEXT, mime::PLAIN) => {
+                        let path_extension = url::Url::parse(location)
+                            .map_err(|_| {
+                                ShellError::labeled_error(
+                                    format!("Cannot parse URL: {}", location),
+                                    "cannot parse",
+                                    span,
+                                )
+                            })?
+                            .path_segments()
+                            .and_then(|segments| segments.last())
+                            .and_then(|name| if name.is_empty() { None } else { Some(name) })
+                            .and_then(|name| {
+                                PathBuf::from(name)
+                                    .extension()
+                                    .map(|name| name.to_string_lossy().to_string())
+                            });
+
+                        Ok((
+                            path_extension,
+                            UntaggedValue::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    span,
+                                )
+                            })?),
+                            Tag {
+                                span,
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                            },
+                        ))
+                    }
+                    (ty, sub_ty) => Ok((
+                        None,
+                        UntaggedValue::string(format!(
+                            "Not yet supported MIME type: {} {}",
+                            ty, sub_ty
+                        )),
+                        Tag {
+                            span,
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
+                        },
+                    )),
+                }
+            }
+            None => Ok((
+                None,
+                UntaggedValue::string("No content type found".to_owned()),
+                Tag {
+                    span,
+                    anchor: Some(AnchorLocation::Url(location.to_string())),
+                },
+            )),
+        },
+        Err(_) => Err(ShellError::labeled_error(
+            "URL could not be opened",
+            "url not found",
+            span,
+        )),
+    }
+}

crates/nu_plugin_fetch/src/lib.rs

@@ -0,0 +1,4 @@
+mod fetch;
+mod nu;
+
+pub use fetch::Fetch;

crates/nu_plugin_fetch/src/main.rs

@@ -1,299 +1,6 @@
-use futures::executor::block_on;
+use nu_plugin::serve_plugin;
-use mime::Mime;
+use nu_plugin_fetch::Fetch;
-use nu_errors::ShellError;
-use nu_plugin::{serve_plugin, Plugin};
-use nu_protocol::{
-    CallInfo, CommandAction, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue,
-    Value,
-};
-use nu_source::{AnchorLocation, Span, Tag};
-use std::path::PathBuf;
-use std::str::FromStr;
-use surf::mime;
-
-struct Fetch {
-    path: Option<Value>,
-    has_raw: bool,
-}
-
-impl Fetch {
-    fn new() -> Fetch {
-        Fetch {
-            path: None,
-            has_raw: false,
-        }
-    }
-
-    fn setup(&mut self, call_info: CallInfo) -> ReturnValue {
-        self.path = Some(
-            match call_info.args.nth(0).ok_or_else(|| {
-                ShellError::labeled_error(
-                    "No file or directory specified",
-                    "for command",
-                    &call_info.name_tag,
-                )
-            })? {
-                file => file.clone(),
-            },
-        );
-
-        self.has_raw = call_info.args.has("raw");
-
-        ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value())
-    }
-}
-
-impl Plugin for Fetch {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("fetch")
-            .desc("Load from a URL into a cell, convert to table if possible (avoid by appending '--raw')")
-            .required(
-                "path",
-                SyntaxShape::Path,
-                "the URL to fetch the contents from",
-            )
-            .switch("raw", "fetch contents as text rather than a table")
-            .filter())
-    }
-
-    fn begin_filter(&mut self, callinfo: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
-        self.setup(callinfo)?;
-        Ok(vec![])
-    }
-
-    fn filter(&mut self, value: Value) -> Result<Vec<ReturnValue>, ShellError> {
-        Ok(vec![block_on(fetch_helper(
-            &self.path.clone().ok_or_else(|| {
-                ShellError::labeled_error(
-                    "internal error: path not set",
-                    "path not set",
-                    &value.tag,
-                )
-            })?,
-            self.has_raw,
-            value,
-        ))])
-    }
-}
 
 fn main() {
-    serve_plugin(&mut Fetch::new());
+    serve_plugin(&mut Fetch::new())
-}
-
-async fn fetch_helper(path: &Value, has_raw: bool, row: Value) -> ReturnValue {
-    let path_buf = path.as_path()?;
-    let path_str = path_buf.display().to_string();
-
-    //FIXME: this is a workaround because plugins don't yet support per-item iteration
-    let path_str = if path_str == "$it" {
-        let path_buf = row.as_path()?;
-        path_buf.display().to_string()
-    } else {
-        path_str
-    };
-
-    let path_span = path.tag.span;
-
-    let result = fetch(&path_str, path_span).await;
-
-    if let Err(e) = result {
-        return Err(e);
-    }
-    let (file_extension, contents, contents_tag) = result?;
-
-    let file_extension = if has_raw {
-        None
-    } else {
-        // If the extension could not be determined via mimetype, try to use the path
-        // extension. Some file types do not declare their mimetypes (such as bson files).
-        file_extension.or_else(|| path_str.split('.').last().map(String::from))
-    };
-
-    let tagged_contents = contents.retag(&contents_tag);
-
-    if let Some(extension) = file_extension {
-        Ok(ReturnSuccess::Action(CommandAction::AutoConvert(
-            tagged_contents,
-            extension,
-        )))
-    } else {
-        ReturnSuccess::value(tagged_contents)
-    }
-}
-
-pub async fn fetch(
-    location: &str,
-    span: Span,
-) -> Result<(Option<String>, UntaggedValue, Tag), ShellError> {
-    if url::Url::parse(location).is_err() {
-        return Err(ShellError::labeled_error(
-            "Incomplete or incorrect url",
-            "expected a full url",
-            span,
-        ));
-    }
-
-    let response = surf::get(location).await;
-    match response {
-        Ok(mut r) => match r.headers().get("content-type") {
-            Some(content_type) => {
-                let content_type = Mime::from_str(content_type).map_err(|_| {
-                    ShellError::labeled_error(
-                        format!("MIME type unknown: {}", content_type),
-                        "given unknown MIME type",
-                        span,
-                    )
-                })?;
-                match (content_type.type_(), content_type.subtype()) {
-                    (mime::APPLICATION, mime::XML) => Ok((
-                        Some("xml".to_string()),
-                        UntaggedValue::string(r.body_string().await.map_err(|_| {
-                            ShellError::labeled_error(
-                                "Could not load text from remote url",
-                                "could not load",
-                                span,
-                            )
-                        })?),
-                        Tag {
-                            span,
-                            anchor: Some(AnchorLocation::Url(location.to_string())),
-                        },
-                    )),
-                    (mime::APPLICATION, mime::JSON) => Ok((
-                        Some("json".to_string()),
-                        UntaggedValue::string(r.body_string().await.map_err(|_| {
-                            ShellError::labeled_error(
-                                "Could not load text from remote url",
-                                "could not load",
-                                span,
-                            )
-                        })?),
-                        Tag {
-                            span,
-                            anchor: Some(AnchorLocation::Url(location.to_string())),
-                        },
-                    )),
-                    (mime::APPLICATION, mime::OCTET_STREAM) => {
-                        let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
-                            ShellError::labeled_error(
-                                "Could not load binary file",
-                                "could not load",
-                                span,
-                            )
-                        })?;
-                        Ok((
-                            None,
-                            UntaggedValue::binary(buf),
-                            Tag {
-                                span,
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                            },
-                        ))
-                    }
-                    (mime::IMAGE, mime::SVG) => Ok((
-                        Some("svg".to_string()),
-                        UntaggedValue::string(r.body_string().await.map_err(|_| {
-                            ShellError::labeled_error(
-                                "Could not load svg from remote url",
-                                "could not load",
-                                span,
-                            )
-                        })?),
-                        Tag {
-                            span,
-                            anchor: Some(AnchorLocation::Url(location.to_string())),
-                        },
-                    )),
-                    (mime::IMAGE, image_ty) => {
-                        let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
-                            ShellError::labeled_error(
-                                "Could not load image file",
-                                "could not load",
-                                span,
-                            )
-                        })?;
-                        Ok((
-                            Some(image_ty.to_string()),
-                            UntaggedValue::binary(buf),
-                            Tag {
-                                span,
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                            },
-                        ))
-                    }
-                    (mime::TEXT, mime::HTML) => Ok((
-                        Some("html".to_string()),
-                        UntaggedValue::string(r.body_string().await.map_err(|_| {
-                            ShellError::labeled_error(
-                                "Could not load text from remote url",
-                                "could not load",
-                                span,
-                            )
-                        })?),
-                        Tag {
-                            span,
-                            anchor: Some(AnchorLocation::Url(location.to_string())),
-                        },
-                    )),
-                    (mime::TEXT, mime::PLAIN) => {
-                        let path_extension = url::Url::parse(location)
-                            .map_err(|_| {
-                                ShellError::labeled_error(
-                                    format!("Cannot parse URL: {}", location),
-                                    "cannot parse",
-                                    span,
-                                )
-                            })?
-                            .path_segments()
-                            .and_then(|segments| segments.last())
-                            .and_then(|name| if name.is_empty() { None } else { Some(name) })
-                            .and_then(|name| {
-                                PathBuf::from(name)
-                                    .extension()
-                                    .map(|name| name.to_string_lossy().to_string())
-                            });
-
-                        Ok((
-                            path_extension,
-                            UntaggedValue::string(r.body_string().await.map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load text from remote url",
-                                    "could not load",
-                                    span,
-                                )
-                            })?),
-                            Tag {
-                                span,
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                            },
-                        ))
-                    }
-                    (ty, sub_ty) => Ok((
-                        None,
-                        UntaggedValue::string(format!(
-                            "Not yet supported MIME type: {} {}",
-                            ty, sub_ty
-                        )),
-                        Tag {
-                            span,
-                            anchor: Some(AnchorLocation::Url(location.to_string())),
-                        },
-                    )),
-                }
-            }
-            None => Ok((
-                None,
-                UntaggedValue::string("No content type found".to_owned()),
-                Tag {
-                    span,
-                    anchor: Some(AnchorLocation::Url(location.to_string())),
-                },
-            )),
-        },
-        Err(_) => Err(ShellError::labeled_error(
-            "URL could not be opened",
-            "url not found",
-            span,
-        )),
-    }
 }

crates/nu_plugin_fetch/src/nu/mod.rs

@@ -0,0 +1,40 @@
+use futures::executor::block_on;
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{CallInfo, ReturnValue, Signature, SyntaxShape, Value};
+
+use crate::fetch::fetch_helper;
+use crate::Fetch;
+
+impl Plugin for Fetch {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("fetch")
+            .desc("Load from a URL into a cell, convert to table if possible (avoid by appending '--raw')")
+            .required(
+                "path",
+                SyntaxShape::Path,
+                "the URL to fetch the contents from",
+            )
+            .switch("raw", "fetch contents as text rather than a table")
+            .filter())
+    }
+
+    fn begin_filter(&mut self, callinfo: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        self.setup(callinfo)?;
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, value: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![block_on(fetch_helper(
+            &self.path.clone().ok_or_else(|| {
+                ShellError::labeled_error(
+                    "internal error: path not set",
+                    "path not set",
+                    &value.tag,
+                )
+            })?,
+            self.has_raw,
+            value,
+        ))])
+    }
+}

crates/nu_plugin_match/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "A regex match plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_match/src/lib.rs

@@ -0,0 +1,4 @@
+mod match_;
+mod nu;
+
+pub use match_::Match;

crates/nu_plugin_match/src/main.rs

@@ -1,112 +1,5 @@
-use nu_errors::ShellError;
+use nu_plugin::serve_plugin;
-use nu_plugin::{serve_plugin, Plugin};
+use nu_plugin_match::Match;
-use nu_protocol::{
-    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value,
-};
-
-use regex::Regex;
-
-struct Match {
-    column: String,
-    regex: Regex,
-}
-
-impl Match {
-    #[allow(clippy::trivial_regex)]
-    fn new() -> Result<Self, Box<dyn std::error::Error>> {
-        Ok(Match {
-            column: String::new(),
-            regex: Regex::new("")?,
-        })
-    }
-}
-
-impl Plugin for Match {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("match")
-            .desc("filter rows by regex")
-            .required("member", SyntaxShape::Member, "the column name to match")
-            .required("regex", SyntaxShape::String, "the regex to match with")
-            .filter())
-    }
-    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
-        if let Some(args) = call_info.args.positional {
-            match &args[0] {
-                Value {
-                    value: UntaggedValue::Primitive(Primitive::String(s)),
-                    ..
-                } => {
-                    self.column = s.clone();
-                }
-                Value { tag, .. } => {
-                    return Err(ShellError::labeled_error(
-                        "Unrecognized type in params",
-                        "value",
-                        tag,
-                    ));
-                }
-            }
-            match &args[1] {
-                Value {
-                    value: UntaggedValue::Primitive(Primitive::String(s)),
-                    tag,
-                } => {
-                    self.regex = Regex::new(s).map_err(|_| {
-                        ShellError::labeled_error(
-                            "Internal error while creating regex",
-                            "internal error created by pattern",
-                            tag,
-                        )
-                    })?;
-                }
-                Value { tag, .. } => {
-                    return Err(ShellError::labeled_error(
-                        "Unrecognized type in params",
-                        "unexpected value",
-                        tag,
-                    ));
-                }
-            }
-        }
-        Ok(vec![])
-    }
-
-    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
-        let flag: bool;
-        match &input {
-            Value {
-                value: UntaggedValue::Row(dict),
-                tag,
-            } => {
-                if let Some(val) = dict.entries.get(&self.column) {
-                    if let Ok(s) = val.as_string() {
-                        flag = self.regex.is_match(&s);
-                    } else {
-                        return Err(ShellError::labeled_error(
-                            "expected string",
-                            "value",
-                            val.tag(),
-                        ));
-                    }
-                } else {
-                    return Err(ShellError::labeled_error(
-                        format!("column not in row! {:?} {:?}", &self.column, dict),
-                        "row",
-                        tag,
-                    ));
-                }
-            }
-            Value { tag, .. } => {
-                return Err(ShellError::labeled_error("Expected row", "value", tag));
-            }
-        }
-        if flag {
-            Ok(vec![Ok(ReturnSuccess::Value(input))])
-        } else {
-            Ok(vec![])
-        }
-    }
-}
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     serve_plugin(&mut Match::new()?);

crates/nu_plugin_match/src/match_.rs

@@ -0,0 +1,16 @@
+use regex::Regex;
+
+pub struct Match {
+    pub column: String,
+    pub regex: Regex,
+}
+
+impl Match {
+    #[allow(clippy::trivial_regex)]
+    pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
+        Ok(Match {
+            column: String::new(),
+            regex: Regex::new("")?,
+        })
+    }
+}

crates/nu_plugin_match/src/nu/mod.rs

@@ -0,0 +1,96 @@
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{
+    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value,
+};
+
+use crate::Match;
+use regex::Regex;
+
+impl Plugin for Match {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("match")
+            .desc("filter rows by regex")
+            .required("member", SyntaxShape::Member, "the column name to match")
+            .required("regex", SyntaxShape::String, "the regex to match with")
+            .filter())
+    }
+
+    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        if let Some(args) = call_info.args.positional {
+            match &args[0] {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::String(s)),
+                    ..
+                } => {
+                    self.column = s.clone();
+                }
+                Value { tag, .. } => {
+                    return Err(ShellError::labeled_error(
+                        "Unrecognized type in params",
+                        "value",
+                        tag,
+                    ));
+                }
+            }
+            match &args[1] {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::String(s)),
+                    tag,
+                } => {
+                    self.regex = Regex::new(s).map_err(|_| {
+                        ShellError::labeled_error(
+                            "Internal error while creating regex",
+                            "internal error created by pattern",
+                            tag,
+                        )
+                    })?;
+                }
+                Value { tag, .. } => {
+                    return Err(ShellError::labeled_error(
+                        "Unrecognized type in params",
+                        "unexpected value",
+                        tag,
+                    ));
+                }
+            }
+        }
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        let flag: bool;
+        match &input {
+            Value {
+                value: UntaggedValue::Row(dict),
+                tag,
+            } => {
+                if let Some(val) = dict.entries.get(&self.column) {
+                    if let Ok(s) = val.as_string() {
+                        flag = self.regex.is_match(&s);
+                    } else {
+                        return Err(ShellError::labeled_error(
+                            "expected string",
+                            "value",
+                            val.tag(),
+                        ));
+                    }
+                } else {
+                    return Err(ShellError::labeled_error(
+                        format!("column not in row! {:?} {:?}", &self.column, dict),
+                        "row",
+                        tag,
+                    ));
+                }
+            }
+            Value { tag, .. } => {
+                return Err(ShellError::labeled_error("Expected row", "value", tag));
+            }
+        }
+        if flag {
+            Ok(vec![Ok(ReturnSuccess::Value(input))])
+        } else {
+            Ok(vec![])
+        }
+    }
+}

crates/nu_plugin_post/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "An HTTP post plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_post/src/lib.rs

@@ -0,0 +1,4 @@
+mod nu;
+mod post;
+
+pub use post::Post;

crates/nu_plugin_post/src/main.rs

@@ -1,569 +1,6 @@
-use base64::encode;
+use nu_plugin::serve_plugin;
-use futures::executor::block_on;
+use nu_plugin_post::Post;
-use mime::Mime;
-use nu_errors::{CoerceInto, ShellError};
-use nu_plugin::{serve_plugin, Plugin};
-use nu_protocol::{
-    CallInfo, CommandAction, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape,
-    UnspannedPathMember, UntaggedValue, Value,
-};
-use nu_source::{AnchorLocation, Tag, TaggedItem};
-use num_traits::cast::ToPrimitive;
-use std::path::PathBuf;
-use std::str::FromStr;
-use surf::mime;
-
-#[derive(Clone)]
-pub enum HeaderKind {
-    ContentType(String),
-    ContentLength(String),
-}
-
-struct Post {
-    path: Option<Value>,
-    has_raw: bool,
-    body: Option<Value>,
-    user: Option<String>,
-    password: Option<String>,
-    headers: Vec<HeaderKind>,
-    tag: Tag,
-}
-
-impl Post {
-    fn new() -> Post {
-        Post {
-            path: None,
-            has_raw: false,
-            body: None,
-            user: None,
-            password: None,
-            headers: vec![],
-            tag: Tag::unknown(),
-        }
-    }
-
-    fn setup(&mut self, call_info: CallInfo) -> ReturnValue {
-        self.path = Some(
-            match call_info.args.nth(0).ok_or_else(|| {
-                ShellError::labeled_error(
-                    "No file or directory specified",
-                    "for command",
-                    &call_info.name_tag,
-                )
-            })? {
-                file => file.clone(),
-            },
-        );
-
-        self.has_raw = call_info.args.has("raw");
-
-        self.body = match call_info.args.nth(1).ok_or_else(|| {
-            ShellError::labeled_error("No body specified", "for command", &call_info.name_tag)
-        })? {
-            file => Some(file.clone()),
-        };
-
-        self.user = match call_info.args.get("user") {
-            Some(user) => Some(user.as_string()?),
-            None => None,
-        };
-
-        self.password = match call_info.args.get("password") {
-            Some(password) => Some(password.as_string()?),
-            None => None,
-        };
-
-        self.headers = get_headers(&call_info)?;
-
-        self.tag = call_info.name_tag;
-
-        ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value())
-    }
-}
-
-impl Plugin for Post {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("post")
-            .desc("Post content to a url and retrieve data as a table if possible.")
-            .required("path", SyntaxShape::Any, "the URL to post to")
-            .required("body", SyntaxShape::Any, "the contents of the post body")
-            .named("user", SyntaxShape::Any, "the username when authenticating")
-            .named(
-                "password",
-                SyntaxShape::Any,
-                "the password when authenticating",
-            )
-            .named(
-                "content-type",
-                SyntaxShape::Any,
-                "the MIME type of content to post",
-            )
-            .named(
-                "content-length",
-                SyntaxShape::Any,
-                "the length of the content being posted",
-            )
-            .switch("raw", "return values as a string instead of a table")
-            .filter())
-    }
-
-    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
-        self.setup(call_info)?;
-        Ok(vec![])
-    }
-
-    fn filter(&mut self, row: Value) -> Result<Vec<ReturnValue>, ShellError> {
-        Ok(vec![block_on(post_helper(
-            &self.path.clone().ok_or_else(|| {
-                ShellError::labeled_error("expected a 'path'", "expected a 'path'", &self.tag)
-            })?,
-            self.has_raw,
-            &self.body.clone().ok_or_else(|| {
-                ShellError::labeled_error("expected a 'body'", "expected a 'body'", &self.tag)
-            })?,
-            self.user.clone(),
-            self.password.clone(),
-            &self.headers.clone(),
-            row,
-        ))])
-    }
-}
 
 fn main() {
     serve_plugin(&mut Post::new());
 }
-
-async fn post_helper(
-    path: &Value,
-    has_raw: bool,
-    body: &Value,
-    user: Option<String>,
-    password: Option<String>,
-    headers: &[HeaderKind],
-    row: Value,
-) -> ReturnValue {
-    let path_tag = path.tag.clone();
-    let path_str = path.as_string()?.to_string();
-
-    //FIXME: this is a workaround because plugins don't yet support per-item iteration
-    let path_str = if path_str == "$it" {
-        let path_buf = row.as_path()?;
-        path_buf.display().to_string()
-    } else {
-        path_str
-    };
-
-    //FIXME: this is a workaround because plugins don't yet support per-item iteration
-    let body = if let Ok(x) = body.as_string() {
-        if x == "$it" {
-            &row
-        } else {
-            body
-        }
-    } else {
-        body
-    };
-
-    let (file_extension, contents, contents_tag) =
-        post(&path_str, &body, user, password, &headers, path_tag.clone()).await?;
-
-    let file_extension = if has_raw {
-        None
-    } else {
-        // If the extension could not be determined via mimetype, try to use the path
-        // extension. Some file types do not declare their mimetypes (such as bson files).
-        file_extension.or_else(|| path_str.split('.').last().map(String::from))
-    };
-
-    let tagged_contents = contents.into_value(&contents_tag);
-
-    if let Some(extension) = file_extension {
-        Ok(ReturnSuccess::Action(CommandAction::AutoConvert(
-            tagged_contents,
-            extension,
-        )))
-    } else {
-        ReturnSuccess::value(tagged_contents)
-    }
-}
-
-pub async fn post(
-    location: &str,
-    body: &Value,
-    user: Option<String>,
-    password: Option<String>,
-    headers: &[HeaderKind],
-    tag: Tag,
-) -> Result<(Option<String>, UntaggedValue, Tag), ShellError> {
-    if location.starts_with("http:") || location.starts_with("https:") {
-        let login = match (user, password) {
-            (Some(user), Some(password)) => Some(encode(&format!("{}:{}", user, password))),
-            (Some(user), _) => Some(encode(&format!("{}:", user))),
-            _ => None,
-        };
-        let response = match body {
-            Value {
-                value: UntaggedValue::Primitive(Primitive::String(body_str)),
-                ..
-            } => {
-                let mut s = surf::post(location).body_string(body_str.to_string());
-                if let Some(login) = login {
-                    s = s.set_header("Authorization", format!("Basic {}", login));
-                }
-
-                for h in headers {
-                    s = match h {
-                        HeaderKind::ContentType(ct) => s.set_header("Content-Type", ct),
-                        HeaderKind::ContentLength(cl) => s.set_header("Content-Length", cl),
-                    };
-                }
-                s.await
-            }
-            Value {
-                value: UntaggedValue::Primitive(Primitive::Binary(b)),
-                ..
-            } => {
-                let mut s = surf::post(location).body_bytes(b);
-                if let Some(login) = login {
-                    s = s.set_header("Authorization", format!("Basic {}", login));
-                }
-                s.await
-            }
-            Value { value, tag } => {
-                match value_to_json_value(&value.clone().into_untagged_value()) {
-                    Ok(json_value) => match serde_json::to_string(&json_value) {
-                        Ok(result_string) => {
-                            let mut s = surf::post(location).body_string(result_string);
-
-                            if let Some(login) = login {
-                                s = s.set_header("Authorization", format!("Basic {}", login));
-                            }
-                            s.await
-                        }
-                        _ => {
-                            return Err(ShellError::labeled_error(
-                                "Could not automatically convert table",
-                                "needs manual conversion",
-                                tag,
-                            ));
-                        }
-                    },
-                    _ => {
-                        return Err(ShellError::labeled_error(
-                            "Could not automatically convert table",
-                            "needs manual conversion",
-                            tag,
-                        ));
-                    }
-                }
-            }
-        };
-        match response {
-            Ok(mut r) => match r.headers().get("content-type") {
-                Some(content_type) => {
-                    let content_type = Mime::from_str(content_type).map_err(|_| {
-                        ShellError::labeled_error(
-                            format!("Unknown MIME type: {}", content_type),
-                            "unknown MIME type",
-                            &tag,
-                        )
-                    })?;
-                    match (content_type.type_(), content_type.subtype()) {
-                        (mime::APPLICATION, mime::XML) => Ok((
-                            Some("xml".to_string()),
-                            UntaggedValue::string(r.body_string().await.map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load text from remote url",
-                                    "could not load",
-                                    &tag,
-                                )
-                            })?),
-                            Tag {
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                                span: tag.span,
-                            },
-                        )),
-                        (mime::APPLICATION, mime::JSON) => Ok((
-                            Some("json".to_string()),
-                            UntaggedValue::string(r.body_string().await.map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load text from remote url",
-                                    "could not load",
-                                    &tag,
-                                )
-                            })?),
-                            Tag {
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                                span: tag.span,
-                            },
-                        )),
-                        (mime::APPLICATION, mime::OCTET_STREAM) => {
-                            let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load binary file",
-                                    "could not load",
-                                    &tag,
-                                )
-                            })?;
-                            Ok((
-                                None,
-                                UntaggedValue::binary(buf),
-                                Tag {
-                                    anchor: Some(AnchorLocation::Url(location.to_string())),
-                                    span: tag.span,
-                                },
-                            ))
-                        }
-                        (mime::IMAGE, image_ty) => {
-                            let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load image file",
-                                    "could not load",
-                                    &tag,
-                                )
-                            })?;
-                            Ok((
-                                Some(image_ty.to_string()),
-                                UntaggedValue::binary(buf),
-                                Tag {
-                                    anchor: Some(AnchorLocation::Url(location.to_string())),
-                                    span: tag.span,
-                                },
-                            ))
-                        }
-                        (mime::TEXT, mime::HTML) => Ok((
-                            Some("html".to_string()),
-                            UntaggedValue::string(r.body_string().await.map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load text from remote url",
-                                    "could not load",
-                                    &tag,
-                                )
-                            })?),
-                            Tag {
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                                span: tag.span,
-                            },
-                        )),
-                        (mime::TEXT, mime::PLAIN) => {
-                            let path_extension = url::Url::parse(location)
-                                .map_err(|_| {
-                                    ShellError::labeled_error(
-                                        format!("could not parse URL: {}", location),
-                                        "could not parse URL",
-                                        &tag,
-                                    )
-                                })?
-                                .path_segments()
-                                .and_then(|segments| segments.last())
-                                .and_then(|name| if name.is_empty() { None } else { Some(name) })
-                                .and_then(|name| {
-                                    PathBuf::from(name)
-                                        .extension()
-                                        .map(|name| name.to_string_lossy().to_string())
-                                });
-
-                            Ok((
-                                path_extension,
-                                UntaggedValue::string(r.body_string().await.map_err(|_| {
-                                    ShellError::labeled_error(
-                                        "Could not load text from remote url",
-                                        "could not load",
-                                        &tag,
-                                    )
-                                })?),
-                                Tag {
-                                    anchor: Some(AnchorLocation::Url(location.to_string())),
-                                    span: tag.span,
-                                },
-                            ))
-                        }
-                        (ty, sub_ty) => Ok((
-                            None,
-                            UntaggedValue::string(format!(
-                                "Not yet supported MIME type: {} {}",
-                                ty, sub_ty
-                            )),
-                            Tag {
-                                anchor: Some(AnchorLocation::Url(location.to_string())),
-                                span: tag.span,
-                            },
-                        )),
-                    }
-                }
-                None => Ok((
-                    None,
-                    UntaggedValue::string("No content type found".to_owned()),
-                    Tag {
-                        anchor: Some(AnchorLocation::Url(location.to_string())),
-                        span: tag.span,
-                    },
-                )),
-            },
-            Err(_) => Err(ShellError::labeled_error(
-                "URL could not be opened",
-                "url not found",
-                tag,
-            )),
-        }
-    } else {
-        Err(ShellError::labeled_error(
-            "Expected a url",
-            "needs a url",
-            tag,
-        ))
-    }
-}
-
-// FIXME FIXME FIXME
-// Ultimately, we don't want to duplicate to-json here, but we need to because there isn't an easy way to call into it, yet
-pub fn value_to_json_value(v: &Value) -> Result<serde_json::Value, ShellError> {
-    Ok(match &v.value {
-        UntaggedValue::Primitive(Primitive::Boolean(b)) => serde_json::Value::Bool(*b),
-        UntaggedValue::Primitive(Primitive::Bytes(b)) => serde_json::Value::Number(
-            serde_json::Number::from(b.to_u64().expect("What about really big numbers")),
-        ),
-        UntaggedValue::Primitive(Primitive::Duration(secs)) => {
-            serde_json::Value::Number(serde_json::Number::from(*secs))
-        }
-        UntaggedValue::Primitive(Primitive::Date(d)) => serde_json::Value::String(d.to_string()),
-        UntaggedValue::Primitive(Primitive::EndOfStream) => serde_json::Value::Null,
-        UntaggedValue::Primitive(Primitive::BeginningOfStream) => serde_json::Value::Null,
-        UntaggedValue::Primitive(Primitive::Decimal(f)) => serde_json::Value::Number(
-            serde_json::Number::from_f64(
-                f.to_f64().expect("TODO: What about really big decimals?"),
-            )
-            .ok_or_else(|| {
-                ShellError::labeled_error(
-                    "Can not convert big decimal to f64",
-                    "cannot convert big decimal to f64",
-                    &v.tag,
-                )
-            })?,
-        ),
-        UntaggedValue::Primitive(Primitive::Int(i)) => {
-            serde_json::Value::Number(serde_json::Number::from(CoerceInto::<i64>::coerce_into(
-                i.tagged(&v.tag),
-                "converting to JSON number",
-            )?))
-        }
-        UntaggedValue::Primitive(Primitive::Nothing) => serde_json::Value::Null,
-        UntaggedValue::Primitive(Primitive::Pattern(s)) => serde_json::Value::String(s.clone()),
-        UntaggedValue::Primitive(Primitive::String(s)) => serde_json::Value::String(s.clone()),
-        UntaggedValue::Primitive(Primitive::Line(s)) => serde_json::Value::String(s.clone()),
-        UntaggedValue::Primitive(Primitive::ColumnPath(path)) => serde_json::Value::Array(
-            path.iter()
-                .map(|x| match &x.unspanned {
-                    UnspannedPathMember::String(string) => {
-                        Ok(serde_json::Value::String(string.clone()))
-                    }
-                    UnspannedPathMember::Int(int) => Ok(serde_json::Value::Number(
-                        serde_json::Number::from(CoerceInto::<i64>::coerce_into(
-                            int.tagged(&v.tag),
-                            "converting to JSON number",
-                        )?),
-                    )),
-                })
-                .collect::<Result<Vec<serde_json::Value>, ShellError>>()?,
-        ),
-        UntaggedValue::Primitive(Primitive::Path(s)) => {
-            serde_json::Value::String(s.display().to_string())
-        }
-
-        UntaggedValue::Table(l) => serde_json::Value::Array(json_list(l)?),
-        UntaggedValue::Error(e) => return Err(e.clone()),
-        UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => {
-            serde_json::Value::Null
-        }
-        UntaggedValue::Primitive(Primitive::Binary(b)) => {
-            let mut output = vec![];
-
-            for item in b.iter() {
-                output.push(serde_json::Value::Number(
-                    serde_json::Number::from_f64(*item as f64).ok_or_else(|| {
-                        ShellError::labeled_error(
-                            "Cannot create number from from f64",
-                            "cannot created number from f64",
-                            &v.tag,
-                        )
-                    })?,
-                ));
-            }
-            serde_json::Value::Array(output)
-        }
-        UntaggedValue::Row(o) => {
-            let mut m = serde_json::Map::new();
-            for (k, v) in o.entries.iter() {
-                m.insert(k.clone(), value_to_json_value(v)?);
-            }
-            serde_json::Value::Object(m)
-        }
-    })
-}
-
-fn json_list(input: &[Value]) -> Result<Vec<serde_json::Value>, ShellError> {
-    let mut out = vec![];
-
-    for value in input {
-        out.push(value_to_json_value(value)?);
-    }
-
-    Ok(out)
-}
-
-fn get_headers(call_info: &CallInfo) -> Result<Vec<HeaderKind>, ShellError> {
-    let mut headers = vec![];
-
-    match extract_header_value(&call_info, "content-type") {
-        Ok(h) => {
-            if let Some(ct) = h {
-                headers.push(HeaderKind::ContentType(ct))
-            }
-        }
-        Err(e) => {
-            return Err(e);
-        }
-    };
-
-    match extract_header_value(&call_info, "content-length") {
-        Ok(h) => {
-            if let Some(cl) = h {
-                headers.push(HeaderKind::ContentLength(cl))
-            }
-        }
-        Err(e) => {
-            return Err(e);
-        }
-    };
-
-    Ok(headers)
-}
-
-fn extract_header_value(call_info: &CallInfo, key: &str) -> Result<Option<String>, ShellError> {
-    if call_info.args.has(key) {
-        let tagged = call_info.args.get(key);
-        let val = match tagged {
-            Some(Value {
-                value: UntaggedValue::Primitive(Primitive::String(s)),
-                ..
-            }) => s.clone(),
-            Some(Value { tag, .. }) => {
-                return Err(ShellError::labeled_error(
-                    format!("{} not in expected format.  Expected string.", key),
-                    "post error",
-                    tag,
-                ));
-            }
-            _ => {
-                return Err(ShellError::labeled_error(
-                    format!("{} not in expected format.  Expected string.", key),
-                    "post error",
-                    Tag::unknown(),
-                ));
-            }
-        };
-        return Ok(Some(val));
-    }
-
-    Ok(None)
-}

crates/nu_plugin_post/src/nu/mod.rs

@@ -0,0 +1,55 @@
+use futures::executor::block_on;
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{CallInfo, ReturnValue, Signature, SyntaxShape, Value};
+
+use crate::post::post_helper;
+use crate::Post;
+
+impl Plugin for Post {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("post")
+            .desc("Post content to a url and retrieve data as a table if possible.")
+            .required("path", SyntaxShape::Any, "the URL to post to")
+            .required("body", SyntaxShape::Any, "the contents of the post body")
+            .named("user", SyntaxShape::Any, "the username when authenticating")
+            .named(
+                "password",
+                SyntaxShape::Any,
+                "the password when authenticating",
+            )
+            .named(
+                "content-type",
+                SyntaxShape::Any,
+                "the MIME type of content to post",
+            )
+            .named(
+                "content-length",
+                SyntaxShape::Any,
+                "the length of the content being posted",
+            )
+            .switch("raw", "return values as a string instead of a table")
+            .filter())
+    }
+
+    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        self.setup(call_info)?;
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, row: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![block_on(post_helper(
+            &self.path.clone().ok_or_else(|| {
+                ShellError::labeled_error("expected a 'path'", "expected a 'path'", &self.tag)
+            })?,
+            self.has_raw,
+            &self.body.clone().ok_or_else(|| {
+                ShellError::labeled_error("expected a 'body'", "expected a 'body'", &self.tag)
+            })?,
+            self.user.clone(),
+            self.password.clone(),
+            &self.headers.clone(),
+            row,
+        ))])
+    }
+}

crates/nu_plugin_post/src/post.rs

@@ -0,0 +1,516 @@
+use base64::encode;
+use mime::Mime;
+use nu_errors::{CoerceInto, ShellError};
+use nu_protocol::{
+    CallInfo, CommandAction, Primitive, ReturnSuccess, ReturnValue, UnspannedPathMember,
+    UntaggedValue, Value,
+};
+use nu_source::{AnchorLocation, Tag, TaggedItem};
+use num_traits::cast::ToPrimitive;
+use std::path::PathBuf;
+use std::str::FromStr;
+use surf::mime;
+
+#[derive(Clone)]
+pub enum HeaderKind {
+    ContentType(String),
+    ContentLength(String),
+}
+
+#[derive(Default)]
+pub struct Post {
+    pub path: Option<Value>,
+    pub has_raw: bool,
+    pub body: Option<Value>,
+    pub user: Option<String>,
+    pub password: Option<String>,
+    pub headers: Vec<HeaderKind>,
+    pub tag: Tag,
+}
+
+impl Post {
+    pub fn new() -> Post {
+        Post {
+            path: None,
+            has_raw: false,
+            body: None,
+            user: None,
+            password: None,
+            headers: vec![],
+            tag: Tag::default(),
+        }
+    }
+
+    pub fn setup(&mut self, call_info: CallInfo) -> ReturnValue {
+        self.path = Some(
+            match call_info.args.nth(0).ok_or_else(|| {
+                ShellError::labeled_error(
+                    "No file or directory specified",
+                    "for command",
+                    &call_info.name_tag,
+                )
+            })? {
+                file => file.clone(),
+            },
+        );
+
+        self.has_raw = call_info.args.has("raw");
+
+        self.body = match call_info.args.nth(1).ok_or_else(|| {
+            ShellError::labeled_error("No body specified", "for command", &call_info.name_tag)
+        })? {
+            file => Some(file.clone()),
+        };
+
+        self.user = match call_info.args.get("user") {
+            Some(user) => Some(user.as_string()?),
+            None => None,
+        };
+
+        self.password = match call_info.args.get("password") {
+            Some(password) => Some(password.as_string()?),
+            None => None,
+        };
+
+        self.headers = get_headers(&call_info)?;
+
+        self.tag = call_info.name_tag;
+
+        ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value())
+    }
+}
+
+pub async fn post_helper(
+    path: &Value,
+    has_raw: bool,
+    body: &Value,
+    user: Option<String>,
+    password: Option<String>,
+    headers: &[HeaderKind],
+    row: Value,
+) -> ReturnValue {
+    let path_tag = path.tag.clone();
+    let path_str = path.as_string()?.to_string();
+
+    //FIXME: this is a workaround because plugins don't yet support per-item iteration
+    let path_str = if path_str == "$it" {
+        let path_buf = row.as_path()?;
+        path_buf.display().to_string()
+    } else {
+        path_str
+    };
+
+    //FIXME: this is a workaround because plugins don't yet support per-item iteration
+    let body = if let Ok(x) = body.as_string() {
+        if x == "$it" {
+            &row
+        } else {
+            body
+        }
+    } else {
+        body
+    };
+
+    let (file_extension, contents, contents_tag) =
+        post(&path_str, &body, user, password, &headers, path_tag.clone()).await?;
+
+    let file_extension = if has_raw {
+        None
+    } else {
+        // If the extension could not be determined via mimetype, try to use the path
+        // extension. Some file types do not declare their mimetypes (such as bson files).
+        file_extension.or_else(|| path_str.split('.').last().map(String::from))
+    };
+
+    let tagged_contents = contents.into_value(&contents_tag);
+
+    if let Some(extension) = file_extension {
+        Ok(ReturnSuccess::Action(CommandAction::AutoConvert(
+            tagged_contents,
+            extension,
+        )))
+    } else {
+        ReturnSuccess::value(tagged_contents)
+    }
+}
+
+pub async fn post(
+    location: &str,
+    body: &Value,
+    user: Option<String>,
+    password: Option<String>,
+    headers: &[HeaderKind],
+    tag: Tag,
+) -> Result<(Option<String>, UntaggedValue, Tag), ShellError> {
+    if location.starts_with("http:") || location.starts_with("https:") {
+        let login = match (user, password) {
+            (Some(user), Some(password)) => Some(encode(&format!("{}:{}", user, password))),
+            (Some(user), _) => Some(encode(&format!("{}:", user))),
+            _ => None,
+        };
+        let response = match body {
+            Value {
+                value: UntaggedValue::Primitive(Primitive::String(body_str)),
+                ..
+            } => {
+                let mut s = surf::post(location).body_string(body_str.to_string());
+                if let Some(login) = login {
+                    s = s.set_header("Authorization", format!("Basic {}", login));
+                }
+
+                for h in headers {
+                    s = match h {
+                        HeaderKind::ContentType(ct) => s.set_header("Content-Type", ct),
+                        HeaderKind::ContentLength(cl) => s.set_header("Content-Length", cl),
+                    };
+                }
+                s.await
+            }
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Binary(b)),
+                ..
+            } => {
+                let mut s = surf::post(location).body_bytes(b);
+                if let Some(login) = login {
+                    s = s.set_header("Authorization", format!("Basic {}", login));
+                }
+                s.await
+            }
+            Value { value, tag } => {
+                match value_to_json_value(&value.clone().into_untagged_value()) {
+                    Ok(json_value) => match serde_json::to_string(&json_value) {
+                        Ok(result_string) => {
+                            let mut s = surf::post(location).body_string(result_string);
+
+                            if let Some(login) = login {
+                                s = s.set_header("Authorization", format!("Basic {}", login));
+                            }
+                            s.await
+                        }
+                        _ => {
+                            return Err(ShellError::labeled_error(
+                                "Could not automatically convert table",
+                                "needs manual conversion",
+                                tag,
+                            ));
+                        }
+                    },
+                    _ => {
+                        return Err(ShellError::labeled_error(
+                            "Could not automatically convert table",
+                            "needs manual conversion",
+                            tag,
+                        ));
+                    }
+                }
+            }
+        };
+        match response {
+            Ok(mut r) => match r.headers().get("content-type") {
+                Some(content_type) => {
+                    let content_type = Mime::from_str(content_type).map_err(|_| {
+                        ShellError::labeled_error(
+                            format!("Unknown MIME type: {}", content_type),
+                            "unknown MIME type",
+                            &tag,
+                        )
+                    })?;
+                    match (content_type.type_(), content_type.subtype()) {
+                        (mime::APPLICATION, mime::XML) => Ok((
+                            Some("xml".to_string()),
+                            UntaggedValue::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    &tag,
+                                )
+                            })?),
+                            Tag {
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                                span: tag.span,
+                            },
+                        )),
+                        (mime::APPLICATION, mime::JSON) => Ok((
+                            Some("json".to_string()),
+                            UntaggedValue::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    &tag,
+                                )
+                            })?),
+                            Tag {
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                                span: tag.span,
+                            },
+                        )),
+                        (mime::APPLICATION, mime::OCTET_STREAM) => {
+                            let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load binary file",
+                                    "could not load",
+                                    &tag,
+                                )
+                            })?;
+                            Ok((
+                                None,
+                                UntaggedValue::binary(buf),
+                                Tag {
+                                    anchor: Some(AnchorLocation::Url(location.to_string())),
+                                    span: tag.span,
+                                },
+                            ))
+                        }
+                        (mime::IMAGE, image_ty) => {
+                            let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load image file",
+                                    "could not load",
+                                    &tag,
+                                )
+                            })?;
+                            Ok((
+                                Some(image_ty.to_string()),
+                                UntaggedValue::binary(buf),
+                                Tag {
+                                    anchor: Some(AnchorLocation::Url(location.to_string())),
+                                    span: tag.span,
+                                },
+                            ))
+                        }
+                        (mime::TEXT, mime::HTML) => Ok((
+                            Some("html".to_string()),
+                            UntaggedValue::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    &tag,
+                                )
+                            })?),
+                            Tag {
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                                span: tag.span,
+                            },
+                        )),
+                        (mime::TEXT, mime::PLAIN) => {
+                            let path_extension = url::Url::parse(location)
+                                .map_err(|_| {
+                                    ShellError::labeled_error(
+                                        format!("could not parse URL: {}", location),
+                                        "could not parse URL",
+                                        &tag,
+                                    )
+                                })?
+                                .path_segments()
+                                .and_then(|segments| segments.last())
+                                .and_then(|name| if name.is_empty() { None } else { Some(name) })
+                                .and_then(|name| {
+                                    PathBuf::from(name)
+                                        .extension()
+                                        .map(|name| name.to_string_lossy().to_string())
+                                });
+
+                            Ok((
+                                path_extension,
+                                UntaggedValue::string(r.body_string().await.map_err(|_| {
+                                    ShellError::labeled_error(
+                                        "Could not load text from remote url",
+                                        "could not load",
+                                        &tag,
+                                    )
+                                })?),
+                                Tag {
+                                    anchor: Some(AnchorLocation::Url(location.to_string())),
+                                    span: tag.span,
+                                },
+                            ))
+                        }
+                        (ty, sub_ty) => Ok((
+                            None,
+                            UntaggedValue::string(format!(
+                                "Not yet supported MIME type: {} {}",
+                                ty, sub_ty
+                            )),
+                            Tag {
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
+                                span: tag.span,
+                            },
+                        )),
+                    }
+                }
+                None => Ok((
+                    None,
+                    UntaggedValue::string("No content type found".to_owned()),
+                    Tag {
+                        anchor: Some(AnchorLocation::Url(location.to_string())),
+                        span: tag.span,
+                    },
+                )),
+            },
+            Err(_) => Err(ShellError::labeled_error(
+                "URL could not be opened",
+                "url not found",
+                tag,
+            )),
+        }
+    } else {
+        Err(ShellError::labeled_error(
+            "Expected a url",
+            "needs a url",
+            tag,
+        ))
+    }
+}
+
+// FIXME FIXME FIXME
+// Ultimately, we don't want to duplicate to-json here, but we need to because there isn't an easy way to call into it, yet
+pub fn value_to_json_value(v: &Value) -> Result<serde_json::Value, ShellError> {
+    Ok(match &v.value {
+        UntaggedValue::Primitive(Primitive::Boolean(b)) => serde_json::Value::Bool(*b),
+        UntaggedValue::Primitive(Primitive::Bytes(b)) => serde_json::Value::Number(
+            serde_json::Number::from(b.to_u64().expect("What about really big numbers")),
+        ),
+        UntaggedValue::Primitive(Primitive::Duration(secs)) => {
+            serde_json::Value::Number(serde_json::Number::from(*secs))
+        }
+        UntaggedValue::Primitive(Primitive::Date(d)) => serde_json::Value::String(d.to_string()),
+        UntaggedValue::Primitive(Primitive::EndOfStream) => serde_json::Value::Null,
+        UntaggedValue::Primitive(Primitive::BeginningOfStream) => serde_json::Value::Null,
+        UntaggedValue::Primitive(Primitive::Decimal(f)) => serde_json::Value::Number(
+            serde_json::Number::from_f64(
+                f.to_f64().expect("TODO: What about really big decimals?"),
+            )
+            .ok_or_else(|| {
+                ShellError::labeled_error(
+                    "Can not convert big decimal to f64",
+                    "cannot convert big decimal to f64",
+                    &v.tag,
+                )
+            })?,
+        ),
+        UntaggedValue::Primitive(Primitive::Int(i)) => {
+            serde_json::Value::Number(serde_json::Number::from(CoerceInto::<i64>::coerce_into(
+                i.tagged(&v.tag),
+                "converting to JSON number",
+            )?))
+        }
+        UntaggedValue::Primitive(Primitive::Nothing) => serde_json::Value::Null,
+        UntaggedValue::Primitive(Primitive::Pattern(s)) => serde_json::Value::String(s.clone()),
+        UntaggedValue::Primitive(Primitive::String(s)) => serde_json::Value::String(s.clone()),
+        UntaggedValue::Primitive(Primitive::Line(s)) => serde_json::Value::String(s.clone()),
+        UntaggedValue::Primitive(Primitive::ColumnPath(path)) => serde_json::Value::Array(
+            path.iter()
+                .map(|x| match &x.unspanned {
+                    UnspannedPathMember::String(string) => {
+                        Ok(serde_json::Value::String(string.clone()))
+                    }
+                    UnspannedPathMember::Int(int) => Ok(serde_json::Value::Number(
+                        serde_json::Number::from(CoerceInto::<i64>::coerce_into(
+                            int.tagged(&v.tag),
+                            "converting to JSON number",
+                        )?),
+                    )),
+                })
+                .collect::<Result<Vec<serde_json::Value>, ShellError>>()?,
+        ),
+        UntaggedValue::Primitive(Primitive::Path(s)) => {
+            serde_json::Value::String(s.display().to_string())
+        }
+
+        UntaggedValue::Table(l) => serde_json::Value::Array(json_list(l)?),
+        UntaggedValue::Error(e) => return Err(e.clone()),
+        UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => {
+            serde_json::Value::Null
+        }
+        UntaggedValue::Primitive(Primitive::Binary(b)) => {
+            let mut output = vec![];
+
+            for item in b.iter() {
+                output.push(serde_json::Value::Number(
+                    serde_json::Number::from_f64(*item as f64).ok_or_else(|| {
+                        ShellError::labeled_error(
+                            "Cannot create number from from f64",
+                            "cannot created number from f64",
+                            &v.tag,
+                        )
+                    })?,
+                ));
+            }
+            serde_json::Value::Array(output)
+        }
+        UntaggedValue::Row(o) => {
+            let mut m = serde_json::Map::new();
+            for (k, v) in o.entries.iter() {
+                m.insert(k.clone(), value_to_json_value(v)?);
+            }
+            serde_json::Value::Object(m)
+        }
+    })
+}
+
+fn json_list(input: &[Value]) -> Result<Vec<serde_json::Value>, ShellError> {
+    let mut out = vec![];
+
+    for value in input {
+        out.push(value_to_json_value(value)?);
+    }
+
+    Ok(out)
+}
+
+fn get_headers(call_info: &CallInfo) -> Result<Vec<HeaderKind>, ShellError> {
+    let mut headers = vec![];
+
+    match extract_header_value(&call_info, "content-type") {
+        Ok(h) => {
+            if let Some(ct) = h {
+                headers.push(HeaderKind::ContentType(ct))
+            }
+        }
+        Err(e) => {
+            return Err(e);
+        }
+    };
+
+    match extract_header_value(&call_info, "content-length") {
+        Ok(h) => {
+            if let Some(cl) = h {
+                headers.push(HeaderKind::ContentLength(cl))
+            }
+        }
+        Err(e) => {
+            return Err(e);
+        }
+    };
+
+    Ok(headers)
+}
+
+fn extract_header_value(call_info: &CallInfo, key: &str) -> Result<Option<String>, ShellError> {
+    if call_info.args.has(key) {
+        let tagged = call_info.args.get(key);
+        let val = match tagged {
+            Some(Value {
+                value: UntaggedValue::Primitive(Primitive::String(s)),
+                ..
+            }) => s.clone(),
+            Some(Value { tag, .. }) => {
+                return Err(ShellError::labeled_error(
+                    format!("{} not in expected format.  Expected string.", key),
+                    "post error",
+                    tag,
+                ));
+            }
+            _ => {
+                return Err(ShellError::labeled_error(
+                    format!("{} not in expected format.  Expected string.", key),
+                    "post error",
+                    Tag::unknown(),
+                ));
+            }
+        };
+        return Ok(Some(val));
+    }
+
+    Ok(None)
+}

crates/nu_plugin_sum/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "A simple summation plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_sum/src/lib.rs

@@ -0,0 +1,4 @@
+mod nu;
+mod sum;
+
+pub use sum::Sum;

crates/nu_plugin_sum/src/main.rs

@@ -1,94 +1,5 @@
-use nu_errors::ShellError;
+use nu_plugin::serve_plugin;
-use nu_plugin::{serve_plugin, Plugin};
+use nu_plugin_sum::Sum;
-use nu_protocol::{
-    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, UntaggedValue, Value,
-};
-
-struct Sum {
-    total: Option<Value>,
-}
-impl Sum {
-    fn new() -> Sum {
-        Sum { total: None }
-    }
-
-    fn sum(&mut self, value: Value) -> Result<(), ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Nothing) => Ok(()),
-            UntaggedValue::Primitive(Primitive::Int(i)) => {
-                match &self.total {
-                    Some(Value {
-                        value: UntaggedValue::Primitive(Primitive::Int(j)),
-                        tag,
-                    }) => {
-                        //TODO: handle overflow
-                        self.total = Some(UntaggedValue::int(i + j).into_value(tag));
-                        Ok(())
-                    }
-                    None => {
-                        self.total = Some(value.clone());
-                        Ok(())
-                    }
-                    _ => Err(ShellError::labeled_error(
-                        "Could not sum non-integer or unrelated types",
-                        "source",
-                        value.tag,
-                    )),
-                }
-            }
-            UntaggedValue::Primitive(Primitive::Bytes(b)) => {
-                match &self.total {
-                    Some(Value {
-                        value: UntaggedValue::Primitive(Primitive::Bytes(j)),
-                        tag,
-                    }) => {
-                        //TODO: handle overflow
-                        self.total = Some(UntaggedValue::bytes(b + j).into_value(tag));
-                        Ok(())
-                    }
-                    None => {
-                        self.total = Some(value);
-                        Ok(())
-                    }
-                    _ => Err(ShellError::labeled_error(
-                        "Could not sum non-integer or unrelated types",
-                        "source",
-                        value.tag,
-                    )),
-                }
-            }
-            x => Err(ShellError::labeled_error(
-                format!("Unrecognized type in stream: {:?}", x),
-                "source",
-                value.tag,
-            )),
-        }
-    }
-}
-
-impl Plugin for Sum {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("sum")
-            .desc("Sum a column of values.")
-            .filter())
-    }
-
-    fn begin_filter(&mut self, _: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
-        Ok(vec![])
-    }
-
-    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
-        self.sum(input)?;
-        Ok(vec![])
-    }
-
-    fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
-        match self.total {
-            None => Ok(vec![]),
-            Some(ref v) => Ok(vec![ReturnSuccess::value(v.clone())]),
-        }
-    }
-}
 
 fn main() {
     serve_plugin(&mut Sum::new());

crates/nu_plugin_sum/src/nu/mod.rs

@@ -0,0 +1,29 @@
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{CallInfo, ReturnSuccess, ReturnValue, Signature, Value};
+
+use crate::Sum;
+
+impl Plugin for Sum {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("sum")
+            .desc("Sum a column of values.")
+            .filter())
+    }
+
+    fn begin_filter(&mut self, _: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        self.sum(input)?;
+        Ok(vec![])
+    }
+
+    fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
+        match self.total {
+            None => Ok(vec![]),
+            Some(ref v) => Ok(vec![ReturnSuccess::value(v.clone())]),
+        }
+    }
+}

crates/nu_plugin_sum/src/sum.rs

@@ -0,0 +1,66 @@
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, UntaggedValue, Value};
+
+#[derive(Default)]
+pub struct Sum {
+    pub total: Option<Value>,
+}
+
+impl Sum {
+    pub fn new() -> Sum {
+        Sum { total: None }
+    }
+
+    pub fn sum(&mut self, value: Value) -> Result<(), ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Nothing) => Ok(()),
+            UntaggedValue::Primitive(Primitive::Int(i)) => {
+                match &self.total {
+                    Some(Value {
+                        value: UntaggedValue::Primitive(Primitive::Int(j)),
+                        tag,
+                    }) => {
+                        //TODO: handle overflow
+                        self.total = Some(UntaggedValue::int(i + j).into_value(tag));
+                        Ok(())
+                    }
+                    None => {
+                        self.total = Some(value.clone());
+                        Ok(())
+                    }
+                    _ => Err(ShellError::labeled_error(
+                        "Could not sum non-integer or unrelated types",
+                        "source",
+                        value.tag,
+                    )),
+                }
+            }
+            UntaggedValue::Primitive(Primitive::Bytes(b)) => {
+                match &self.total {
+                    Some(Value {
+                        value: UntaggedValue::Primitive(Primitive::Bytes(j)),
+                        tag,
+                    }) => {
+                        //TODO: handle overflow
+                        self.total = Some(UntaggedValue::bytes(b + j).into_value(tag));
+                        Ok(())
+                    }
+                    None => {
+                        self.total = Some(value);
+                        Ok(())
+                    }
+                    _ => Err(ShellError::labeled_error(
+                        "Could not sum non-integer or unrelated types",
+                        "source",
+                        value.tag,
+                    )),
+                }
+            }
+            x => Err(ShellError::labeled_error(
+                format!("Unrecognized type in stream: {:?}", x),
+                "source",
+                value.tag,
+            )),
+        }
+    }
+}

crates/nu_plugin_sys/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "A system info plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_textview/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "Text viewer plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_tree/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2018"
 description = "Tree viewer plugin for Nushell"
 license = "MIT"
 
+[lib]
+doctest = false
+
 [dependencies]
 nu-plugin = { path = "../nu-plugin", version = "0.9.0" }
 nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu_plugin_tree/src/lib.rs

@@ -0,0 +1,4 @@
+mod nu;
+mod tree;
+
+pub use tree::TreeViewer;

crates/nu_plugin_tree/src/main.rs

@@ -1,100 +1,5 @@
-use derive_new::new;
+use nu_plugin::serve_plugin;
-use nu_errors::ShellError;
+use nu_plugin_tree::TreeViewer;
-use nu_plugin::{serve_plugin, Plugin};
-use nu_protocol::{format_primitive, CallInfo, Signature, UntaggedValue, Value};
-use ptree::item::StringItem;
-use ptree::output::print_tree_with;
-use ptree::print_config::PrintConfig;
-use ptree::style::{Color, Style};
-use ptree::TreeBuilder;
-
-#[derive(new)]
-pub struct TreeView {
-    tree: StringItem,
-}
-
-impl TreeView {
-    fn from_value_helper(value: &UntaggedValue, mut builder: &mut TreeBuilder) {
-        match value {
-            UntaggedValue::Primitive(p) => {
-                let _ = builder.add_empty_child(format_primitive(p, None));
-            }
-            UntaggedValue::Row(o) => {
-                for (k, v) in o.entries.iter() {
-                    builder = builder.begin_child(k.clone());
-                    Self::from_value_helper(v, builder);
-                    builder = builder.end_child();
-                }
-            }
-            UntaggedValue::Table(l) => {
-                for elem in l.iter() {
-                    Self::from_value_helper(elem, builder);
-                }
-            }
-            _ => {}
-        }
-    }
-
-    fn from_value(value: &Value) -> TreeView {
-        let descs = value.data_descriptors();
-
-        let mut tree = TreeBuilder::new("".to_string());
-        let mut builder = &mut tree;
-
-        for desc in descs {
-            let value = match &value.value {
-                UntaggedValue::Row(d) => d.get_data(&desc).borrow().clone(),
-                _ => value.clone(),
-            };
-            builder = builder.begin_child(desc.clone());
-            Self::from_value_helper(&value, &mut builder);
-            builder = builder.end_child();
-            //entries.push((desc.name.clone(), value.borrow().copy()))
-        }
-
-        TreeView::new(builder.build())
-    }
-
-    fn render_view(&self) -> Result<(), ShellError> {
-        // Set up the print configuration
-        let config = {
-            let mut config = PrintConfig::from_env();
-            config.branch = Style {
-                foreground: Some(Color::Green),
-                dimmed: true,
-                ..Style::default()
-            };
-            config.leaf = Style {
-                bold: true,
-                ..Style::default()
-            };
-            config.indent = 4;
-            config
-        };
-
-        // Print out the tree using custom formatting
-        print_tree_with(&self.tree, &config)?;
-
-        Ok(())
-    }
-}
-
-struct TreeViewer;
-
-impl Plugin for TreeViewer {
-    fn config(&mut self) -> Result<Signature, ShellError> {
-        Ok(Signature::build("tree").desc("View the contents of the pipeline as a tree."))
-    }
-
-    fn sink(&mut self, _call_info: CallInfo, input: Vec<Value>) {
-        if !input.is_empty() {
-            for i in input.iter() {
-                let view = TreeView::from_value(&i);
-                let _ = view.render_view();
-            }
-        }
-    }
-}
 
 fn main() {
     serve_plugin(&mut TreeViewer);

crates/nu_plugin_tree/src/nu/mod.rs

@@ -0,0 +1,21 @@
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{CallInfo, Signature, Value};
+
+use crate::tree::TreeView;
+use crate::TreeViewer;
+
+impl Plugin for TreeViewer {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("tree").desc("View the contents of the pipeline as a tree."))
+    }
+
+    fn sink(&mut self, _call_info: CallInfo, input: Vec<Value>) {
+        if !input.is_empty() {
+            for i in input.iter() {
+                let view = TreeView::from_value(&i);
+                let _ = view.render_view();
+            }
+        }
+    }
+}

crates/nu_plugin_tree/src/tree.rs

@@ -0,0 +1,80 @@
+use derive_new::new;
+use nu_errors::ShellError;
+use nu_protocol::{format_primitive, UntaggedValue, Value};
+use ptree::item::StringItem;
+use ptree::output::print_tree_with;
+use ptree::print_config::PrintConfig;
+use ptree::style::{Color, Style};
+use ptree::TreeBuilder;
+
+pub struct TreeViewer;
+#[derive(new)]
+pub struct TreeView {
+    tree: StringItem,
+}
+
+impl TreeView {
+    fn from_value_helper(value: &UntaggedValue, mut builder: &mut TreeBuilder) {
+        match value {
+            UntaggedValue::Primitive(p) => {
+                let _ = builder.add_empty_child(format_primitive(p, None));
+            }
+            UntaggedValue::Row(o) => {
+                for (k, v) in o.entries.iter() {
+                    builder = builder.begin_child(k.clone());
+                    Self::from_value_helper(v, builder);
+                    builder = builder.end_child();
+                }
+            }
+            UntaggedValue::Table(l) => {
+                for elem in l.iter() {
+                    Self::from_value_helper(elem, builder);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    pub fn from_value(value: &Value) -> TreeView {
+        let descs = value.data_descriptors();
+
+        let mut tree = TreeBuilder::new("".to_string());
+        let mut builder = &mut tree;
+
+        for desc in descs {
+            let value = match &value.value {
+                UntaggedValue::Row(d) => d.get_data(&desc).borrow().clone(),
+                _ => value.clone(),
+            };
+            builder = builder.begin_child(desc.clone());
+            Self::from_value_helper(&value, &mut builder);
+            builder = builder.end_child();
+            //entries.push((desc.name.clone(), value.borrow().copy()))
+        }
+
+        TreeView::new(builder.build())
+    }
+
+    pub fn render_view(&self) -> Result<(), ShellError> {
+        // Set up the print configuration
+        let config = {
+            let mut config = PrintConfig::from_env();
+            config.branch = Style {
+                foreground: Some(Color::Green),
+                dimmed: true,
+                ..Style::default()
+            };
+            config.leaf = Style {
+                bold: true,
+                ..Style::default()
+            };
+            config.indent = 4;
+            config
+        };
+
+        // Print out the tree using custom formatting
+        print_tree_with(&self.tree, &config)?;
+
+        Ok(())
+    }
+}

src/plugins/nu_plugin_stable_average.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_average::Average;
+
+fn main() {
+    serve_plugin(&mut Average::new());
+}

src/plugins/nu_plugin_stable_binaryview.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_binaryview::BinaryView;
+
+fn main() {
+    serve_plugin(&mut BinaryView::new());
+}

src/plugins/nu_plugin_stable_fetch.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_fetch::Fetch;
+
+fn main() {
+    serve_plugin(&mut Fetch::new());
+}

src/plugins/nu_plugin_stable_match.rs

@@ -0,0 +1,7 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_match::Match;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    serve_plugin(&mut Match::new()?);
+    Ok(())
+}

src/plugins/nu_plugin_stable_post.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_post::Post;
+
+fn main() {
+    serve_plugin(&mut Post::new());
+}

src/plugins/nu_plugin_stable_sum.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_sum::Sum;
+
+fn main() {
+    serve_plugin(&mut Sum::new());
+}

src/plugins/nu_plugin_stable_tree.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_tree::TreeViewer;
+
+fn main() {
+    serve_plugin(&mut TreeViewer);
+}

tests/commands/cd.rs

@@ -24,7 +24,7 @@ fn filesystem_change_from_current_directory_using_absolute_path() {
         let actual = nu!(
             cwd: dirs.test(),
             r#"
-                cd "{}"
+                cd {}
                 pwd | echo $it
             "#,
             dirs.formats()

tests/commands/cp.rs

@@ -8,7 +8,7 @@ fn copies_a_file() {
     Playground::setup("cp_test_1", |dirs, _| {
         nu!(
             cwd: dirs.root(),
-            "cp \"{}\" cp_test_1/sample.ini",
+            "cp {} cp_test_1/sample.ini",
             dirs.formats().join("sample.ini")
         );
 

tests/commands/save.rs

@@ -37,7 +37,7 @@ fn writes_out_csv() {
 
         nu!(
             cwd: dirs.root(),
-            "open \"{}/cargo_sample.toml\" | get package | save save_test_2/cargo_sample.csv",
+            "open {}/cargo_sample.toml | get package | save save_test_2/cargo_sample.csv",
             dirs.formats()
         );