Take advantage of streaming.

It's now possible to download huge files with HTTPie, and it's often faster than curl and wget!

README.rst

@@ -183,7 +183,7 @@ Note that when the **output is redirected** (like the examples above), HTTPie
 applies a different set of defaults than for a console output. Namely, colors
 aren't used (unless ``--pretty`` is set) and only the response body
 is printed (unless ``--print`` options specified). It is a convenience
-that allows for things like the one above or downloading (smallish) binary
+that allows for things like the one above or downloading binary
 files without having to set any flags:
 
 .. code-block:: shell
@@ -373,6 +373,7 @@ Changelog
 =========
 
 * `0.2.7dev`_
+    * Support for efficient large file downloads.
     * Response body is fetched only when needed (e.g., not with ``--headers``).
     * Updated Solarized color scheme.
     * Windows: Added ``--output FILE`` to store output into a file

httpie/core.py

@@ -16,7 +16,7 @@ import requests.auth
 from requests.compat import str
 
 from .models import HTTPRequest, HTTPResponse, Environment
-from .output import OutputProcessor, format
+from .output import OutputProcessor, formatted_stream
 from .input import (OUT_REQ_BODY, OUT_REQ_HEAD,
                     OUT_RESP_HEAD, OUT_RESP_BODY)
 from .cli import parser
@@ -51,7 +51,6 @@ def get_response(args, env):
             # the `Content-Type` for us.
             args.headers['Content-Type'] = FORM
 
-    try:
     credentials = None
     if args.auth:
         credentials = {
@@ -79,17 +78,8 @@ def get_response(args, env):
         params=args.params,
     )
 
-    except (KeyboardInterrupt, SystemExit):
-        env.stderr.write('\n')
-        sys.exit(1)
-    except Exception as e:
-        if args.debug:
-            raise
-        env.stderr.write(str(repr(e) + '\n'))
-        sys.exit(1)
 
-
+def output_stream(args, env, request, response):
-def get_output(args, env, request, response):
     """Format parts of the `request`-`response` exchange
      according to `args` and `env` and return `bytes`.
 
@@ -99,32 +89,38 @@ def get_output(args, env, request, response):
     prettifier = (OutputProcessor(env, pygments_style=args.style)
                   if args.prettify else None)
 
-    if (OUT_REQ_HEAD in args.output_options
+    with_request = (OUT_REQ_HEAD in args.output_options
-        or OUT_REQ_BODY in args.output_options):
+                    or OUT_REQ_BODY in args.output_options)
-        exchange.append(format(
+    with_response = (OUT_RESP_HEAD in args.output_options
+                     or OUT_RESP_BODY in args.output_options)
+
+    if with_request:
+        request_iter = formatted_stream(
             msg=HTTPRequest(request),
             env=env,
             prettifier=prettifier,
             with_headers=OUT_REQ_HEAD in args.output_options,
-            with_body=OUT_REQ_BODY in args.output_options
+            with_body=OUT_REQ_BODY in args.output_options)
-        ))
 
-    if (OUT_RESP_HEAD in args.output_options
+        for chunk in request_iter:
-        or OUT_RESP_BODY in args.output_options):
+            yield chunk
-        exchange.append(format(
+
+        if with_response:
+            yield b'\n\n\n'
+
+    if with_response:
+        response_iter = formatted_stream(
             msg=HTTPResponse(response),
             env=env,
             prettifier=prettifier,
             with_headers=OUT_RESP_HEAD in args.output_options,
             with_body=OUT_RESP_BODY in args.output_options)
-        )
 
-    output = b'\n\n\n'.join(exchange)
+        for chunk in response_iter:
+            yield chunk
 
     if env.stdout_isatty:
-        output += b'\n\n'
+        yield b'\n\n'
-
-    return output
 
 
 def get_exist_status(code, allow_redirects=False):
@@ -155,10 +151,9 @@ def main(args=sys.argv[1:], env=Environment()):
             ' Please use `--output FILE\' instead.\n')
         return 1
 
+    try:
         args = parser.parse_args(args=args, env=env)
-
         response = get_response(args, env)
-
         status = 0
 
         if args.check_status:
@@ -169,11 +164,24 @@ def main(args=sys.argv[1:], env=Environment()):
                     response.raw.status, response.raw.reason)
                 env.stderr.write(err)
 
-    output = get_output(args, env, response.request, response)
-
         try:
-        env.stdout.buffer.write(output)
+            # We are writing bytes so we use buffer on Python 3
+            buffer = env.stdout.buffer
         except AttributeError:
-        env.stdout.write(output)
+            buffer = env.stdout
+
+        for chunk in output_stream(args, env, response.request, response):
+            buffer.write(chunk)
+            if env.stdout_isatty:
+                env.stdout.flush()
+
+    except (KeyboardInterrupt, SystemExit):
+        env.stderr.write('\n')
+        return 1
+    except Exception as e:
+        if '--debug' in args:
+            raise
+        env.stderr.write(str(repr(e) + '\n'))
+        return 1
 
     return status

httpie/input.py

@@ -8,6 +8,7 @@ import json
 import argparse
 import mimetypes
 import getpass
+from io import BytesIO
 
 try:
     from collections import OrderedDict
@@ -424,8 +425,9 @@ def parse_items(items, data=None, headers=None, files=None, params=None):
             target = params
         elif item.sep == SEP_FILES:
             try:
+                with open(os.path.expanduser(value), 'rb') as f:
                     value = (os.path.basename(value),
-                         open(os.path.expanduser(value), 'rb'))
+                             BytesIO(f.read()))
             except IOError as e:
                 raise ParseError(
                     'Invalid argument "%s": %s' % (item.orig, e))

httpie/models.py

@@ -57,6 +57,10 @@ class HTTPMessage(object):
 class HTTPResponse(HTTPMessage):
     """A `requests.models.Response` wrapper."""
 
+    def __iter__(self):
+        mb = 1024 * 1000
+        return self._orig.iter_content(chunk_size=2 * mb)
+
     @property
     def line(self):
         """Return Status-Line"""
@@ -85,6 +89,9 @@ class HTTPResponse(HTTPMessage):
 class HTTPRequest(HTTPMessage):
     """A `requests.models.Request` wrapper."""
 
+    def __iter__(self):
+        yield self.body
+
     @property
     def line(self):
         """Return Request-Line"""

httpie/output.py

@@ -26,11 +26,12 @@ BINARY_SUPPRESSED_NOTICE = (
 )
 
 
-def format(msg, prettifier=None, with_headers=True, with_body=True,
+def formatted_stream(msg, prettifier=None, with_headers=True, with_body=True,
                      env=Environment()):
-    """Return `bytes` representation of a `models.HTTPMessage`.
+    """Return an iterator yielding `bytes` representing `msg`
+    (a `models.HTTPMessage` subclass).
 
-    Sometimes the body contains binary data so we always return `bytes`.
+    The body can be binary so we always yield `bytes`.
 
     If `prettifier` is set or the output is a terminal then a binary
     body is not included in the output and is replaced with notice.
@@ -41,7 +42,6 @@ def format(msg, prettifier=None, with_headers=True, with_body=True,
     then we prefer readability over precision.
 
     """
-
     # Output encoding.
     if env.stdout_isatty:
         # Use encoding suitable for the terminal. Unsupported characters
@@ -59,46 +59,42 @@ def format(msg, prettifier=None, with_headers=True, with_body=True,
     if prettifier:
         env.init_colors()
 
-    #noinspection PyArgumentList
-    output = bytearray()
-
     if with_headers:
         headers = '\n'.join([msg.line, msg.headers])
 
         if prettifier:
             headers = prettifier.process_headers(headers)
 
-        output.extend(
+        yield headers.encode(output_encoding, errors).strip()
-            headers.encode(output_encoding, errors).strip())
 
-        if with_body and msg.body:
+    if with_body:
-            output.extend(b'\n\n')
 
-    if with_body and msg.body:
+        prefix = b'\n\n' if with_headers else None
-
-        body = msg.body
 
         if not (env.stdout_isatty or prettifier):
             # Verbatim body even if it's binary.
-            pass
+            for body_chunk in msg:
-        else:
+                if prefix:
+                    yield prefix
+                    prefix = None
+                yield body_chunk
+        elif msg.body:
             try:
-                body = body.decode(msg.encoding)
+                body = msg.body.decode(msg.encoding)
             except UnicodeDecodeError:
                 # Suppress binary data.
                 body = BINARY_SUPPRESSED_NOTICE.encode(output_encoding)
                 if not with_headers:
-                    output.extend(b'\n')
+                    yield b'\n'
             else:
                 if prettifier and msg.content_type:
                     body = prettifier.process_body(
                         body, msg.content_type).strip()
 
                 body = body.encode(output_encoding, errors)
-
+            if prefix:
-        output.extend(body)
+                yield prefix
-
+            yield body
-    return bytes(output)
 
 
 class HTTPLexer(lexer.RegexLexer):

tests/tests.py

@@ -22,9 +22,9 @@ To make it run faster and offline you can::
 import os
 import sys
 import json
-import tempfile
 import unittest
 import argparse
+import tempfile
 try:
     from urllib.request import urlopen
 except ImportError:
@@ -43,7 +43,7 @@ sys.path.insert(0, os.path.realpath(os.path.join(TESTS_ROOT, '..')))
 
 from httpie import input
 from httpie.models import Environment
-from httpie.core import main, get_output
+from httpie.core import main, output_stream
 from httpie.output import BINARY_SUPPRESSED_NOTICE
 from httpie.input import ParseError
 
@@ -104,7 +104,7 @@ def http(*args, **kwargs):
     except (Exception, SystemExit) as e:
         sys.stderr.write(stderr.read())
         raise
-
+    else:
         stdout.seek(0)
         stderr.seek(0)
 
@@ -135,11 +135,12 @@ def http(*args, **kwargs):
 
         r.stderr = stderr.read()
         r.exit_status = exit_status
-
+        return r
+    finally:
         stdout.close()
         stderr.close()
 
-    return r
+
 
 
 class BaseTestCase(unittest.TestCase):
@@ -591,8 +592,9 @@ class MultipartFormDataFileUploadTest(BaseTestCase):
 class BinaryRequestDataTest(BaseTestCase):
 
     def test_binary_stdin(self):
+        with open(TEST_BIN_FILE_PATH, 'rb') as stdin:
             env = Environment(
-            stdin=open(TEST_BIN_FILE_PATH, 'rb'),
+                stdin=stdin,
                 stdin_isatty=False,
                 stdout_isatty=False
             )
@@ -700,37 +702,24 @@ class RequestBodyFromFilePathTest(BaseTestCase):
 
     def test_request_body_from_file_by_path_no_field_name_allowed(self):
         env = Environment(stdin_isatty=True)
-        try:
+        r = http(
-            http(
             'POST',
             httpbin('/post'),
             'field-name@' + TEST_FILE_PATH,
             env=env
         )
-        except SystemExit:
+        self.assertIn('perhaps you meant --form?', r.stderr)
-            env.stderr.seek(0)
-            stderr = env.stderr.read()
-            self.assertIn('perhaps you meant --form?', stderr)
-        else:
-            self.fail('validation did not work')
 
     def test_request_body_from_file_by_path_no_data_items_allowed(self):
         env = Environment(stdin_isatty=True)
-        try:
+        r = http(
-            http(
             'POST',
             httpbin('/post'),
             '@' + TEST_FILE_PATH,
             'foo=bar',
             env=env
         )
-        except SystemExit:
+        self.assertIn('cannot be mixed', r.stderr)
-            env.stderr.seek(0)
-            self.assertIn(
-                'cannot be mixed',
-                env.stderr.read())
-        else:
-            self.fail('validation did not work')
 
 
 class AuthTest(BaseTestCase):
@@ -852,10 +841,12 @@ class FakeWindowsTest(BaseTestCase):
 
     def test_output_file_pretty_not_allowed_on_windows(self):
         env = Environment(
-            is_windows=True, stdout_isatty=True, stdin_isatty=True)
+            is_windows=True,
+            stdout_isatty=True,
+            stdin_isatty=True
+        )
 
-        try:
+        r = http(
-            http(
             '--output',
             os.path.join(tempfile.gettempdir(), '__httpie_test_output__'),
             '--pretty',
@@ -863,13 +854,8 @@ class FakeWindowsTest(BaseTestCase):
             httpbin('/get'),
             env=env
         )
-        except SystemExit:
-            env.stderr.seek(0)
-            err = env.stderr.read()
         self.assertIn(
-                'Only terminal output can be prettified on Windows', err)
+            'Only terminal output can be prettified on Windows', r.stderr)
-        else:
-            self.fail('validation did not work')
 
 
 #################################################################
@@ -1077,7 +1063,8 @@ class UnicodeOutputTestCase(BaseTestCase):
         args.style = 'default'
 
         # colorized output contains escape sequences
-        output = get_output(args, Environment(), response.request, response).decode('utf8')
+        output = output_stream(args, Environment(), response.request, response)
+        output = b''.join(output).decode('utf8')
         for key, value in response_dict.items():
             self.assertIn(key, output)
             self.assertIn(value, output)