From 9e167713b39a738e827fabaee62e47d6df2d47ab Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 31 Aug 2019 06:27:15 +1200 Subject: [PATCH 1/2] Add post command --- Cargo.lock | 1 + Cargo.toml | 1 + src/cli.rs | 3 +- src/commands.rs | 2 + src/commands/post.rs | 248 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 src/commands/post.rs diff --git a/Cargo.lock b/Cargo.lock index 02440e0c37..c8c912821e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1549,6 +1549,7 @@ version = "0.2.0" dependencies = [ "ansi_term 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "battery 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", "bson 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "byte-unit 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 08a1272db1..66b21fe37d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ dunce = "1.0.0" indexmap = { version = "1.0.2", features = ["serde-1"] } chrono-humanize = "0.0.11" byte-unit = "3.0.1" +base64 = "0.10.1" ordered-float = {version = "1.0.2", features = ["serde"]} futures-preview = { version = "=0.3.0-alpha.18", features = ["compat", "io-compat"] } futures-async-stream = "=0.1.0-alpha.5" diff --git a/src/cli.rs b/src/cli.rs index e01afe89ff..1f8eb2712e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -17,7 +17,7 @@ use crate::prelude::*; use log::{debug, trace}; use regex::Regex; use rustyline::error::ReadlineError; -use rustyline::{self, ColorMode, Config, Editor, config::Configurer, config::EditMode}; +use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor}; use std::env; use std::error::Error; use std::io::{BufRead, BufReader, Write}; @@ -200,6 +200,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(Get), per_item_command(Remove), per_item_command(Open), + per_item_command(Post), per_item_command(Where), whole_stream_command(Config), whole_stream_command(SkipWhile), diff --git a/src/commands.rs b/src/commands.rs index 12cd6d56ee..00bb1faf9b 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -34,6 +34,7 @@ pub(crate) mod nth; pub(crate) mod open; pub(crate) mod pick; pub(crate) mod plugin; +pub(crate) mod post; pub(crate) mod prev; pub(crate) mod ps; pub(crate) mod reject; @@ -93,6 +94,7 @@ pub(crate) use next::Next; pub(crate) use nth::Nth; pub(crate) use open::Open; pub(crate) use pick::Pick; +pub(crate) use post::Post; pub(crate) use prev::Previous; pub(crate) use ps::PS; pub(crate) use reject::Reject; diff --git a/src/commands/post.rs b/src/commands/post.rs new file mode 100644 index 0000000000..7a6d99774f --- /dev/null +++ b/src/commands/post.rs @@ -0,0 +1,248 @@ +use crate::context::SpanSource; +use crate::errors::ShellError; +use crate::object::{Primitive, Value}; +use crate::parser::hir::SyntaxType; +use crate::parser::registry::Signature; +use crate::prelude::*; +use base64::encode; +use mime::Mime; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use surf::mime; +use uuid::Uuid; +pub struct Post; + +impl PerItemCommand for Post { + fn name(&self) -> &str { + "post" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .required("path", SyntaxType::Any) + .required("body", SyntaxType::Any) + .named("user", SyntaxType::Any) + .named("password", SyntaxType::Any) + .switch("raw") + } + + fn run( + &self, + call_info: &CallInfo, + _registry: &CommandRegistry, + shell_manager: &ShellManager, + _input: Tagged, + ) -> Result { + run(call_info, shell_manager) + } +} + +fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result { + let cwd = PathBuf::from(shell_manager.path()); + let full_path = PathBuf::from(cwd); + + let path = match call_info + .args + .nth(0) + .ok_or_else(|| ShellError::string(&format!("No file or directory specified")))? + { + file => file, + }; + let body = match call_info + .args + .nth(1) + .ok_or_else(|| ShellError::string(&format!("No body specified")))? + { + file => file, + }; + let path_str = path.as_string()?; + let body_str = body.as_string()?; + let path_span = path.span(); + let name_span = call_info.name_span; + let has_raw = call_info.args.has("raw"); + let user = call_info.args.get("user").map(|x| x.as_string().unwrap()); + let password = call_info + .args + .get("password") + .map(|x| x.as_string().unwrap()); + + //r#"{"query": "query { viewer { name, } }"}"#.to_string() + let stream = async_stream_block! { + let (file_extension, contents, contents_tag, span_source) = + post(&path_str, body_str, user, password, path_span).await.unwrap(); + + //println!("{:?}", contents); + + yield ReturnSuccess::value(contents.tagged(contents_tag)); + }; + + Ok(stream.to_output_stream()) +} + +pub async fn post( + location: &str, + body: String, + user: Option, + password: Option, + span: Span, +) -> Result<(Option, Value, Tag, SpanSource), ShellError> { + if location.starts_with("http:") || location.starts_with("https:") { + let login = encode(&format!("{}:{}", user.unwrap(), password.unwrap())); + let response = surf::post(location) + .body_string(body) + .set_header("Authorization", format!("Basic {}", login)) + .await; + match response { + Ok(mut r) => match r.headers().get("content-type") { + Some(content_type) => { + let content_type = Mime::from_str(content_type).unwrap(); + match (content_type.type_(), content_type.subtype()) { + (mime::APPLICATION, mime::XML) => Ok(( + Some("xml".to_string()), + Value::string(r.body_string().await.map_err(|_| { + ShellError::labeled_error( + "Could not load text from remote url", + "could not load", + span, + ) + })?), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )), + (mime::APPLICATION, mime::JSON) => Ok(( + Some("json".to_string()), + Value::string(r.body_string().await.map_err(|_| { + ShellError::labeled_error( + "Could not load text from remote url", + "could not load", + span, + ) + })?), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )), + (mime::APPLICATION, mime::OCTET_STREAM) => { + let buf: Vec = r.body_bytes().await.map_err(|_| { + ShellError::labeled_error( + "Could not load binary file", + "could not load", + span, + ) + })?; + Ok(( + None, + Value::Binary(buf), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )) + } + (mime::IMAGE, image_ty) => { + let buf: Vec = r.body_bytes().await.map_err(|_| { + ShellError::labeled_error( + "Could not load image file", + "could not load", + span, + ) + })?; + Ok(( + Some(image_ty.to_string()), + Value::Binary(buf), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )) + } + (mime::TEXT, mime::HTML) => Ok(( + Some("html".to_string()), + Value::string(r.body_string().await.map_err(|_| { + ShellError::labeled_error( + "Could not load text from remote url", + "could not load", + span, + ) + })?), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )), + (mime::TEXT, mime::PLAIN) => { + let path_extension = url::Url::parse(location) + .unwrap() + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|name| if name.is_empty() { None } else { Some(name) }) + .and_then(|name| { + PathBuf::from(name) + .extension() + .map(|name| name.to_string_lossy().to_string()) + }); + + Ok(( + path_extension, + Value::string(r.body_string().await.map_err(|_| { + ShellError::labeled_error( + "Could not load text from remote url", + "could not load", + span, + ) + })?), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )) + } + (ty, sub_ty) => Ok(( + None, + Value::string(format!( + "Not yet supported MIME type: {} {}", + ty, sub_ty + )), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )), + } + } + None => Ok(( + None, + Value::string(format!("No content type found")), + Tag { + span, + origin: Some(Uuid::new_v4()), + }, + SpanSource::Url(location.to_string()), + )), + }, + Err(_) => { + return Err(ShellError::labeled_error( + "URL could not be opened", + "url not found", + span, + )); + } + } + } else { + Err(ShellError::labeled_error( + "Expected a url", + "needs a url", + span, + )) + } +} From ad18c7f61a80af394630b7d6c13f8ccccc051c26 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Sat, 31 Aug 2019 16:08:59 +1200 Subject: [PATCH 2/2] Finish magic post and magic receive --- src/commands/post.rs | 174 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 151 insertions(+), 23 deletions(-) diff --git a/src/commands/post.rs b/src/commands/post.rs index 7a6d99774f..adab330248 100644 --- a/src/commands/post.rs +++ b/src/commands/post.rs @@ -1,12 +1,13 @@ +use crate::commands::UnevaluatedCallInfo; use crate::context::SpanSource; use crate::errors::ShellError; -use crate::object::{Primitive, Value}; +use crate::object::Value; use crate::parser::hir::SyntaxType; use crate::parser::registry::Signature; use crate::prelude::*; use base64::encode; use mime::Mime; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::str::FromStr; use surf::mime; use uuid::Uuid; @@ -29,51 +30,106 @@ impl PerItemCommand for Post { fn run( &self, call_info: &CallInfo, - _registry: &CommandRegistry, - shell_manager: &ShellManager, + registry: &CommandRegistry, + raw_args: &RawCommandArgs, _input: Tagged, ) -> Result { - run(call_info, shell_manager) + run(call_info, registry, raw_args) } } -fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result { - let cwd = PathBuf::from(shell_manager.path()); - let full_path = PathBuf::from(cwd); - +fn run( + call_info: &CallInfo, + registry: &CommandRegistry, + raw_args: &RawCommandArgs, +) -> Result { + let call_info = call_info.clone(); let path = match call_info .args .nth(0) - .ok_or_else(|| ShellError::string(&format!("No file or directory specified")))? + .ok_or_else(|| ShellError::string(&format!("No url specified")))? { - file => file, + file => file.clone(), }; let body = match call_info .args .nth(1) .ok_or_else(|| ShellError::string(&format!("No body specified")))? { - file => file, + file => file.clone(), }; let path_str = path.as_string()?; - let body_str = body.as_string()?; + //let body_str = body.as_string()?; let path_span = path.span(); - let name_span = call_info.name_span; let has_raw = call_info.args.has("raw"); let user = call_info.args.get("user").map(|x| x.as_string().unwrap()); let password = call_info .args .get("password") .map(|x| x.as_string().unwrap()); + let registry = registry.clone(); + let raw_args = raw_args.clone(); - //r#"{"query": "query { viewer { name, } }"}"#.to_string() let stream = async_stream_block! { let (file_extension, contents, contents_tag, span_source) = - post(&path_str, body_str, user, password, path_span).await.unwrap(); + post(&path_str, &body, user, password, path_span, ®istry, &raw_args).await.unwrap(); - //println!("{:?}", contents); + let file_extension = if has_raw { + None + } else { + // If the extension could not be determined via mimetype, try to use the path + // extension. Some file types do not declare their mimetypes (such as bson files). + file_extension.or(path_str.split('.').last().map(String::from)) + }; - yield ReturnSuccess::value(contents.tagged(contents_tag)); + if let Some(uuid) = contents_tag.origin { + // If we have loaded something, track its source + yield ReturnSuccess::action(CommandAction::AddSpanSource( + uuid, + span_source, + )); + } + + let tagged_contents = contents.tagged(contents_tag); + + if let Some(extension) = file_extension { + let command_name = format!("from-{}", extension); + if let Some(converter) = registry.get_command(&command_name) { + let new_args = RawCommandArgs { + host: raw_args.host, + shell_manager: raw_args.shell_manager, + call_info: UnevaluatedCallInfo { + args: crate::parser::hir::Call { + head: raw_args.call_info.args.head, + positional: None, + named: None + }, + source: raw_args.call_info.source, + source_map: raw_args.call_info.source_map, + name_span: raw_args.call_info.name_span, + } + }; + let mut result = converter.run(new_args.with_input(vec![tagged_contents]), ®istry); + let result_vec: Vec> = result.drain_vec().await; + for res in result_vec { + match res { + Ok(ReturnSuccess::Value(Tagged { item: Value::List(list), ..})) => { + for l in list { + yield Ok(ReturnSuccess::Value(l)); + } + } + Ok(ReturnSuccess::Value(Tagged { item, .. })) => { + yield Ok(ReturnSuccess::Value(Tagged { item: item, tag: contents_tag })); + } + x => yield x, + } + } + } else { + yield ReturnSuccess::value(tagged_contents); + } + } else { + yield ReturnSuccess::value(tagged_contents); + } }; Ok(stream.to_output_stream()) @@ -81,17 +137,89 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result, user: Option, password: Option, span: Span, + registry: &CommandRegistry, + raw_args: &RawCommandArgs, ) -> Result<(Option, Value, Tag, SpanSource), ShellError> { + let registry = registry.clone(); + let raw_args = raw_args.clone(); if location.starts_with("http:") || location.starts_with("https:") { let login = encode(&format!("{}:{}", user.unwrap(), password.unwrap())); - let response = surf::post(location) - .body_string(body) - .set_header("Authorization", format!("Basic {}", login)) - .await; + let response = match body { + Tagged { + item: Value::Primitive(Primitive::String(body_str)), + .. + } => { + surf::post(location) + .body_string(body_str.to_string()) + .set_header("Authorization", format!("Basic {}", login)) + .await + } + Tagged { + item: Value::Binary(b), + .. + } => { + surf::post(location) + .body_bytes(b) + .set_header("Authorization", format!("Basic {}", login)) + .await + } + Tagged { item, tag } => { + if let Some(converter) = registry.get_command("to-json") { + let new_args = RawCommandArgs { + host: raw_args.host, + shell_manager: raw_args.shell_manager, + call_info: UnevaluatedCallInfo { + args: crate::parser::hir::Call { + head: raw_args.call_info.args.head, + positional: None, + named: None, + }, + source: raw_args.call_info.source, + source_map: raw_args.call_info.source_map, + name_span: raw_args.call_info.name_span, + }, + }; + let mut result = converter.run( + new_args.with_input(vec![item.clone().tagged(tag.clone())]), + ®istry, + ); + let result_vec: Vec> = + result.drain_vec().await; + let mut result_string = String::new(); + for res in result_vec { + match res { + Ok(ReturnSuccess::Value(Tagged { + item: Value::Primitive(Primitive::String(s)), + .. + })) => { + result_string.push_str(&s); + } + _ => { + return Err(ShellError::labeled_error( + "Save could not successfully save", + "unexpected data during save", + span, + )); + } + } + } + surf::post(location) + .body_string(result_string) + .set_header("Authorization", format!("Basic {}", login)) + .await + } else { + return Err(ShellError::labeled_error( + "Could not automatically convert table", + "needs manual conversion", + tag.span, + )); + } + } + }; match response { Ok(mut r) => match r.headers().get("content-type") { Some(content_type) => {