Fetch content from S3 (#2328)

* fetch content from s3 resource

* remove submodule

* fix clippy

* update Cargo.lock

* fix s3 plugin dependency version

Cargo.toml

@@ -40,6 +40,7 @@ nu_plugin_textview = {version = "0.18.1", path = "./crates/nu_plugin_textview",
 nu_plugin_to_bson = {version = "0.18.1", path = "./crates/nu_plugin_to_bson", optional = true}
 nu_plugin_to_sqlite = {version = "0.18.1", path = "./crates/nu_plugin_to_sqlite", optional = true}
 nu_plugin_tree = {version = "0.18.1", path = "./crates/nu_plugin_tree", optional = true}
+nu_plugin_s3 = { version = "0.18.1", path = "./crates/nu_plugin_s3", optional=true }
 
 crossterm = {version = "0.17.5", optional = true}
 semver = {version = "0.10.0", optional = true}
@@ -76,7 +77,7 @@ default = [
   "term-support",
   "uuid-support",
 ]
-stable = ["default", "binaryview", "match", "tree", "post", "fetch", "clipboard-cli", "trash-support", "start", "starship-prompt", "bson", "sqlite"]
+stable = ["default", "binaryview", "match", "tree", "post", "fetch", "clipboard-cli", "trash-support", "start", "starship-prompt", "bson", "sqlite", "s3"]
 
 # Default
 inc = ["semver", "nu_plugin_inc"]
@@ -94,6 +95,7 @@ sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"]
 start = ["nu_plugin_start"]
 trace = ["nu-parser/trace"]
 tree = ["nu_plugin_tree"]
+s3 = ["nu_plugin_s3"]
 
 clipboard-cli = ["nu-cli/clipboard-cli"]
 ctrlc-support = ["nu-cli/ctrlc"]
@@ -160,6 +162,11 @@ name = "nu_plugin_stable_start"
 path = "src/plugins/nu_plugin_stable_start.rs"
 required-features = ["start"]
 
+[[bin]]
+name = "nu_plugin_stable_s3"
+path = "src/plugins/nu_plugin_stable_s3.rs"
+required-features = ["s3"]
+
 # Main nu binary
 [[bin]]
 name = "nu"

crates/nu_plugin_s3/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "nu_plugin_s3"
+version = "0.18.1"
+authors = ["The Nu Project Contributors"]
+edition = "2018"
+description = "An S3 plugin for Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-plugin = { path = "../nu-plugin", version = "0.18.1" }
+nu-protocol = { path = "../nu-protocol", version = "0.18.1" }
+nu-source = { path = "../nu-source", version = "0.18.1" }
+nu-errors = { path = "../nu-errors", version = "0.18.1" }
+futures = { version = "0.3", features = ["compat", "io-compat"] }
+s3handler = "0.5.0"
+
+[build-dependencies]

crates/nu_plugin_s3/README.md

@@ -0,0 +1,9 @@
+Nu Plugin S3
+---
+
+An S3 plugin for nu shell, it can load the content of S3 objects and convert into table
+
+#### Snapshot
+In following example, the return content from httpbin is saved before as an object in AWS S3.
+![snapshot](https://raw.githubusercontent.com/yanganto/nu_plugin_s3/master/demo.png)
+

crates/nu_plugin_s3/src/handler.rs

@@ -0,0 +1,134 @@
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, CommandAction, ReturnSuccess, ReturnValue, UntaggedValue, Value};
+use nu_source::{AnchorLocation, Tag};
+use s3handler::{CredentialConfig, Handler as S3Handler};
+
+pub struct Handler {
+    pub resource: Option<Value>,
+    pub tag: Tag,
+    pub has_raw: bool,
+    pub config: CredentialConfig,
+}
+
+impl Handler {
+    pub fn new() -> Handler {
+        Handler {
+            tag: Tag::unknown(),
+            config: CredentialConfig {
+                host: String::new(),
+                access_key: String::new(),
+                secret_key: String::new(),
+                user: None,
+                region: None,
+                s3_type: None,
+                secure: None,
+            },
+            resource: None,
+            has_raw: false,
+        }
+    }
+
+    pub fn setup(&mut self, call_info: CallInfo) -> ReturnValue {
+        self.resource = {
+            let r = call_info.args.nth(0).ok_or_else(|| {
+                ShellError::labeled_error(
+                    "No obj or directory specified",
+                    "for command",
+                    &call_info.name_tag,
+                )
+            })?;
+            Some(r.clone())
+        };
+        self.tag = call_info.name_tag.clone();
+        self.has_raw = call_info.args.has("raw");
+
+        if let Some(e) = call_info.args.get("endpoint") {
+            self.config.host = e.as_string()?
+        } else {
+            return Err(ShellError::labeled_error(
+                "No endpoint provided",
+                "for command",
+                &call_info.name_tag,
+            ));
+        }
+
+        if let Some(access_key) = call_info.args.get("access_key") {
+            self.config.access_key = access_key.as_string()?
+        } else {
+            return Err(ShellError::labeled_error(
+                "No access key provided",
+                "for command",
+                &call_info.name_tag,
+            ));
+        }
+
+        if let Some(secret_key) = call_info.args.get("secret_key") {
+            self.config.secret_key = secret_key.as_string()?
+        } else {
+            return Err(ShellError::labeled_error(
+                "No secret key provided",
+                "for command",
+                &call_info.name_tag,
+            ));
+        }
+
+        if let Some(region) = call_info.args.get("region") {
+            self.config.region = Some(region.as_string()?)
+        }
+
+        ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value())
+    }
+}
+
+impl Default for Handler {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub async fn s3_helper(resource: &Value, has_raw: bool, config: &CredentialConfig) -> ReturnValue {
+    let resource_str = resource.as_string()?;
+    let mut handler = S3Handler::from(config);
+    let (output, content_type) = handler
+        .cat(&resource_str)
+        .map_err(|e| ShellError::unexpected(e.to_string()))?;
+
+    let extension = if has_raw {
+        None
+    } else {
+        fn get_accept_ext(s: String) -> Option<String> {
+            if s.contains("json") {
+                Some("json".to_string())
+            } else if s.contains("xml") {
+                Some("xml".to_string())
+            } else if s.contains("svg") {
+                Some("svg".to_string())
+            } else if s.contains("html") {
+                Some("html".to_string())
+            } else {
+                None
+            }
+        }
+        // If the extension could not provide when uploading,
+        // try to use the resource extension.
+        content_type.and_then(get_accept_ext).or_else(|| {
+            resource_str
+                .split('.')
+                .last()
+                .map(String::from)
+                .and_then(get_accept_ext)
+        })
+    };
+
+    if let Some(e) = extension {
+        Ok(ReturnSuccess::Action(CommandAction::AutoConvert(
+            UntaggedValue::string(output).into_value(Tag {
+                span: resource.tag.span,
+                anchor: Some(AnchorLocation::Url(resource_str)),
+            }),
+            e,
+        )))
+    } else {
+        ReturnSuccess::value(UntaggedValue::string(output))
+    }
+}

crates/nu_plugin_s3/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod handler;
+mod nu;
+
+pub use handler::Handler;

crates/nu_plugin_s3/src/main.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_s3::handler;
+
+fn main() {
+    serve_plugin(&mut handler::Handler::new())
+}

crates/nu_plugin_s3/src/nu/mod.rs

@@ -0,0 +1,60 @@
+use futures::executor::block_on;
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{CallInfo, ReturnValue, Signature, SyntaxShape};
+
+use crate::handler;
+use crate::handler::s3_helper;
+
+impl Plugin for handler::Handler {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("s3")
+            .desc("Load S3 resource into a cell, convert to table if possible (avoid by appending '--raw' or '-R')")
+            .required(
+                "RESOURCE",
+                SyntaxShape::String,
+                "the RESOURCE to fetch the contents from",
+            )
+            .named(
+                "endpoint",
+                SyntaxShape::Any,
+                "the enpoint info for the S3 resource, i.g., s3.ap-northeast-1.amazonaws.com or 10.1.1.1",
+                Some('e'),
+            )
+            .named(
+                "access_key",
+                SyntaxShape::Any,
+                "the accessy key when authenticating",
+                Some('a'),
+            )
+            .named(
+                "secret_key",
+                SyntaxShape::Any,
+                "the secret key when authenticating",
+                Some('s'),
+            )
+            .named(
+                "region",
+                SyntaxShape::Any,
+                "the region of the resource, default will use us-east-1",
+                Some('r'),
+            )
+            .switch("raw", "fetch contents as text rather than a table", Some('R'))
+            .filter())
+    }
+
+    fn begin_filter(&mut self, callinfo: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        self.setup(callinfo)?;
+        Ok(vec![block_on(s3_helper(
+            &self.resource.clone().ok_or_else(|| {
+                ShellError::labeled_error(
+                    "internal error: resource not set",
+                    "resource not set",
+                    &self.tag,
+                )
+            })?,
+            self.has_raw,
+            &self.config,
+        ))])
+    }
+}

src/plugins/nu_plugin_stable_s3.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_s3::Handler;
+
+fn main() {
+    serve_plugin(&mut Handler::new());
+}