Move fetch to extra and clean up some code (#664)

* Move fetch to extra

* Move byte stream code to a function instead of copying it twice

* Fix formatting issues

* Make fetch a default command

* Fix formatting

Cargo.toml

@@ -59,7 +59,6 @@ pretty_assertions = "1.0.0"
 
 [features]
 plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
-fetch-support = ["nu-command/fetch"]
 default = [
 	"plugin",
 	"inc",

crates/nu-command/Cargo.toml

@@ -68,7 +68,7 @@ sha2 = "0.10.0"
 base64 = "0.13.0"
 encoding_rs = "0.8.30"
 num = { version = "0.4.0", optional = true }
-reqwest = {version = "0.11", features = ["blocking"], optional = true }
+reqwest = {version = "0.11", features = ["blocking"] }
 mime = "0.3.16"
 
 [target.'cfg(unix)'.dependencies]
@@ -88,7 +88,6 @@ features = [
 trash-support = ["trash"]
 plugin = ["nu-parser/plugin"]
 dataframe = ["polars", "num"]
-fetch = ["reqwest"]
 
 [build-dependencies]
 shadow-rs = "0.8.1"

crates/nu-command/src/default_context.rs

@@ -271,6 +271,7 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
 
         // Network
         bind_command! {
+            Fetch,
             Url,
             UrlHost,
             UrlPath,
@@ -304,9 +305,6 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
         #[cfg(feature = "plugin")]
         bind_command!(Register);
 
-        #[cfg(feature = "fetch")]
-        bind_command!(Fetch);
-
         // This is a WIP proof of concept
         // bind_command!(ListGitBranches, Git, GitCheckout, Source);
 

crates/nu-command/src/network/fetch.rs

@@ -7,6 +7,7 @@ use nu_protocol::ByteStream;
 use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
+use reqwest::blocking::Response;
 
 use std::io::{BufRead, BufReader, Read};
 
@@ -162,98 +163,61 @@ fn helper(
     }
 
     match request.send() {
-        Ok(resp) => {
+        Ok(resp) => match resp.headers().get("content-type") {
-            // let temp = std::fs::File::create("temp_dwl.txt")?;
+            Some(content_type) => {
-            // let mut b = BufWriter::new(temp);
+                let content_type = content_type.to_str().map_err(|e| {
-            // let _bytes = resp.copy_to(&mut b);
+                    ShellError::LabeledError(e.to_string(), "MIME type were invalid".to_string())
-            // let temp1 = std::fs::File::open("temp_dwl.txt")?;
+                })?;
-            // let a = BufReader::new(temp1);
+                let content_type = mime::Mime::from_str(content_type).map_err(|_| {
-
+                    ShellError::LabeledError(
-            // TODO I guess we should check if all bytes were written/read...
+                        format!("MIME type unknown: {}", content_type),
-            match resp.headers().get("content-type") {
+                        "given unknown MIME type".to_string(),
-                Some(content_type) => {
+                    )
-                    let content_type = content_type.to_str().map_err(|e| {
+                })?;
-                        ShellError::LabeledError(
+                let ext = match (content_type.type_(), content_type.subtype()) {
-                            e.to_string(),
+                    (mime::TEXT, mime::PLAIN) => {
-                            "MIME type were invalid".to_string(),
+                        let path_extension = url::Url::parse(&requested_url)
-                        )
+                            .map_err(|_| {
-                    })?;
+                                ShellError::LabeledError(
-                    let content_type = mime::Mime::from_str(content_type).map_err(|_| {
+                                    format!("Cannot parse URL: {}", requested_url),
-                        ShellError::LabeledError(
+                                    "cannot parse".to_string(),
-                            format!("MIME type unknown: {}", content_type),
+                                )
-                            "given unknown MIME type".to_string(),
+                            })?
-                        )
+                            .path_segments()
-                    })?;
+                            .and_then(|segments| segments.last())
-                    let ext = match (content_type.type_(), content_type.subtype()) {
+                            .and_then(|name| if name.is_empty() { None } else { Some(name) })
-                        (mime::TEXT, mime::PLAIN) => {
+                            .and_then(|name| {
-                            let path_extension = url::Url::parse(&requested_url)
+                                PathBuf::from(name)
-                                .map_err(|_| {
+                                    .extension()
-                                    ShellError::LabeledError(
+                                    .map(|name| name.to_string_lossy().to_string())
-                                        format!("Cannot parse URL: {}", requested_url),
+                            });
-                                        "cannot parse".to_string(),
+                        path_extension
-                                    )
-                                })?
-                                .path_segments()
-                                .and_then(|segments| segments.last())
-                                .and_then(|name| if name.is_empty() { None } else { Some(name) })
-                                .and_then(|name| {
-                                    PathBuf::from(name)
-                                        .extension()
-                                        .map(|name| name.to_string_lossy().to_string())
-                                });
-                            path_extension
-                        }
-                        _ => Some(content_type.subtype().to_string()),
-                    };
-
-                    let buffered_input = BufReader::new(resp);
-
-                    let output = PipelineData::ByteStream(
-                        ByteStream {
-                            stream: Box::new(BufferedReader {
-                                input: buffered_input,
-                            }),
-                            ctrlc: engine_state.ctrlc.clone(),
-                        },
-                        span,
-                        None,
-                    );
-
-                    if raw {
-                        return Ok(output);
                     }
+                    _ => Some(content_type.subtype().to_string()),
+                };
 
-                    if let Some(ext) = ext {
+                let output = response_to_buffer(resp, engine_state, span);
-                        match engine_state.find_decl(format!("from {}", ext).as_bytes()) {
+
-                            Some(converter_id) => engine_state.get_decl(converter_id).run(
+                if raw {
-                                engine_state,
+                    return Ok(output);
-                                stack,
-                                &Call::new(),
-                                output,
-                            ),
-                            None => Ok(output),
-                        }
-                    } else {
-                        Ok(output)
-                    }
                 }
-                None => {
-                    let buffered_input = BufReader::new(resp);
 
-                    let output = PipelineData::ByteStream(
+                if let Some(ext) = ext {
-                        ByteStream {
+                    match engine_state.find_decl(format!("from {}", ext).as_bytes()) {
-                            stream: Box::new(BufferedReader {
+                        Some(converter_id) => engine_state.get_decl(converter_id).run(
-                                input: buffered_input,
+                            engine_state,
-                            }),
+                            stack,
-                            ctrlc: engine_state.ctrlc.clone(),
+                            &Call::new(),
-                        },
+                            output,
-                        span,
+                        ),
-                        None,
+                        None => Ok(output),
-                    );
+                    }
+                } else {
                     Ok(output)
                 }
             }
-        }
+            None => Ok(response_to_buffer(resp, engine_state, span)),
+        },
         Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure(
             format!("Request to {} has timed out", requested_url),
             span,
@@ -327,6 +291,25 @@ impl<R: Read> Iterator for BufferedReader<R> {
     }
 }
 
+fn response_to_buffer(
+    response: Response,
+    engine_state: &EngineState,
+    span: Span,
+) -> nu_protocol::PipelineData {
+    let buffered_input = BufReader::new(response);
+
+    PipelineData::ByteStream(
+        ByteStream {
+            stream: Box::new(BufferedReader {
+                input: buffered_input,
+            }),
+            ctrlc: engine_state.ctrlc.clone(),
+        },
+        span,
+        None,
+    )
+}
+
 // Only panics if the user agent is invalid but we define it statically so either
 // it always or never fails
 #[allow(clippy::unwrap_used)]

crates/nu-command/src/network/mod.rs

@@ -1,7 +1,5 @@
-#[cfg(feature = "fetch")]
 mod fetch;
 mod url;
 
 pub use self::url::*;
-#[cfg(feature = "fetch")]
 pub use fetch::SubCommand as Fetch;