Move fetch to extra and clean up some code (#664)

* Move fetch to extra

* Move byte stream code to a function instead of copying it twice

* Fix formatting issues

* Make fetch a default command

* Fix formatting

Cargo.toml

@@ -59,7 +59,6 @@ pretty_assertions = "1.0.0"
 
 [features]
 plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
-fetch-support = ["nu-command/fetch"]
 default = [
 	"plugin",
 	"inc",

crates/nu-command/Cargo.toml

@@ -68,7 +68,7 @@ sha2 = "0.10.0"
 base64 = "0.13.0"
 encoding_rs = "0.8.30"
 num = { version = "0.4.0", optional = true }
-reqwest = {version = "0.11", features = ["blocking"], optional = true }
+reqwest = {version = "0.11", features = ["blocking"] }
 mime = "0.3.16"
 
 [target.'cfg(unix)'.dependencies]
@@ -88,7 +88,6 @@ features = [
 trash-support = ["trash"]
 plugin = ["nu-parser/plugin"]
 dataframe = ["polars", "num"]
-fetch = ["reqwest"]
 
 [build-dependencies]
 shadow-rs = "0.8.1"

crates/nu-command/src/default_context.rs

@@ -271,6 +271,7 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
 
         // Network
         bind_command! {
+            Fetch,
             Url,
             UrlHost,
             UrlPath,
@@ -304,9 +305,6 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
         #[cfg(feature = "plugin")]
         bind_command!(Register);
 
-        #[cfg(feature = "fetch")]
-        bind_command!(Fetch);
-
         // This is a WIP proof of concept
         // bind_command!(ListGitBranches, Git, GitCheckout, Source);
 

crates/nu-command/src/network/fetch.rs

@@ -7,6 +7,7 @@ use nu_protocol::ByteStream;
 use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
+use reqwest::blocking::Response;
 
 use std::io::{BufRead, BufReader, Read};
 
@@ -162,21 +163,10 @@ fn helper(
     }
 
     match request.send() {
-        Ok(resp) => {
+        Ok(resp) => match resp.headers().get("content-type") {
-            // let temp = std::fs::File::create("temp_dwl.txt")?;
-            // let mut b = BufWriter::new(temp);
-            // let _bytes = resp.copy_to(&mut b);
-            // let temp1 = std::fs::File::open("temp_dwl.txt")?;
-            // let a = BufReader::new(temp1);
-
-            // TODO I guess we should check if all bytes were written/read...
-            match resp.headers().get("content-type") {
             Some(content_type) => {
                 let content_type = content_type.to_str().map_err(|e| {
-                        ShellError::LabeledError(
+                    ShellError::LabeledError(e.to_string(), "MIME type were invalid".to_string())
-                            e.to_string(),
-                            "MIME type were invalid".to_string(),
-                        )
                 })?;
                 let content_type = mime::Mime::from_str(content_type).map_err(|_| {
                     ShellError::LabeledError(
@@ -206,18 +196,7 @@ fn helper(
                     _ => Some(content_type.subtype().to_string()),
                 };
 
-                    let buffered_input = BufReader::new(resp);
+                let output = response_to_buffer(resp, engine_state, span);
-
-                    let output = PipelineData::ByteStream(
-                        ByteStream {
-                            stream: Box::new(BufferedReader {
-                                input: buffered_input,
-                            }),
-                            ctrlc: engine_state.ctrlc.clone(),
-                        },
-                        span,
-                        None,
-                    );
 
                 if raw {
                     return Ok(output);
@@ -237,23 +216,8 @@ fn helper(
                     Ok(output)
                 }
             }
-                None => {
+            None => Ok(response_to_buffer(resp, engine_state, span)),
-                    let buffered_input = BufReader::new(resp);
-
-                    let output = PipelineData::ByteStream(
-                        ByteStream {
-                            stream: Box::new(BufferedReader {
-                                input: buffered_input,
-                            }),
-                            ctrlc: engine_state.ctrlc.clone(),
         },
-                        span,
-                        None,
-                    );
-                    Ok(output)
-                }
-            }
-        }
         Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure(
             format!("Request to {} has timed out", requested_url),
             span,
@@ -327,6 +291,25 @@ impl<R: Read> Iterator for BufferedReader<R> {
     }
 }
 
+fn response_to_buffer(
+    response: Response,
+    engine_state: &EngineState,
+    span: Span,
+) -> nu_protocol::PipelineData {
+    let buffered_input = BufReader::new(response);
+
+    PipelineData::ByteStream(
+        ByteStream {
+            stream: Box::new(BufferedReader {
+                input: buffered_input,
+            }),
+            ctrlc: engine_state.ctrlc.clone(),
+        },
+        span,
+        None,
+    )
+}
+
 // Only panics if the user agent is invalid but we define it statically so either
 // it always or never fails
 #[allow(clippy::unwrap_used)]

crates/nu-command/src/network/mod.rs

@@ -1,7 +1,5 @@
-#[cfg(feature = "fetch")]
 mod fetch;
 mod url;
 
 pub use self::url::*;
-#[cfg(feature = "fetch")]
 pub use fetch::SubCommand as Fetch;