v0.2.7

`--stream` can be used to enable streaming also with `--pretty` and to ensure
a more frequent output flushing.

.gitignore

@@ -3,3 +3,7 @@ httpie.egg-info
 build
 *.pyc
 .tox
+README.html
+.coverage
+htmlcov
+

AUTHORS.rst

@@ -0,0 +1,29 @@
+==============
+HTTPie authors
+==============
+
+* `Jakub Roztocil <https://github.com/jkbr>`_
+
+
+Patches and ideas
+-----------------
+
+* `Hank Gay <https://github.com/gthank>`_
+* `Jake Basile <https://github.com/jakebasile>`_
+* `Vladimir Berkutov <https://github.com/dair-targ>`_
+* `Jakob Kramer <https://github.com/gandaro>`_
+* `Chris Faulkner <https://github.com/faulkner>`_
+* `Alen Mujezinovic <https://github.com/flashingpumpkin>`_
+* `Praful Mathur <https://github.com/tictactix>`_
+* `Marc Abramowitz <https://github.com/msabramo>`_
+* `Ismail Badawi <https://github.com/isbadawi>`_
+* `Laurent Bachelier <https://github.com/laurentb>`_
+* `Isman Firmansyah <https://github.com/iromli>`_
+* `Simon Olofsson <https://github.com/simono>`_
+* `Churkin Oleg <https://github.com/Bahus>`_
+* `Jökull Sólberg Auðunsson <https://github.com/jokull>`_
+* `Matthew M. Boedicker <https://github.com/mmb>`_
+* `marblar <https://github.com/marblar>`_
+* `Tomek Wójcik <https://github.com/tomekwojcik>`_
+* `Davey Shafik <https://github.com/dshafik>`_
+* `cido <https://github.com/cido>`_

httpie/__init__.py

@@ -3,5 +3,14 @@ HTTPie - cURL for humans.
 
 """
 __author__ = 'Jakub Roztocil'
-__version__ = '0.2.6'
+__version__ = '0.2.7'
 __licence__ = 'BSD'
+
+
+class EXIT:
+    OK = 0
+    ERROR = 1
+    # Used only when requested:
+    ERROR_HTTP_3XX = 3
+    ERROR_HTTP_4XX = 4
+    ERROR_HTTP_5XX = 5

httpie/cli.py

@@ -3,9 +3,13 @@
 NOTE: the CLI interface may change before reaching v1.0.
 
 """
+import argparse
+
+from requests.compat import is_windows
+
 from . import __doc__
 from . import __version__
-from .output import AVAILABLE_STYLES
+from .output import AVAILABLE_STYLES, DEFAULT_STYLE
 from .input import (Parser, AuthCredentialsArgType, KeyValueArgType,
                     PRETTIFY_STDOUT_TTY_ONLY,
                     SEP_PROXY, SEP_CREDENTIALS, SEP_GROUP_ITEMS,
@@ -50,11 +54,19 @@ group_type.add_argument(
 # Output options.
 #############################################
 
+
 parser.add_argument(
-    '--traceback', action='store_true', default=False,
+    '--output', '-o', type=argparse.FileType('w+b'),
-    help=_('''
+    metavar='FILE',
-        Print exception traceback should one occur.
+    help= argparse.SUPPRESS if not is_windows else _(
-    ''')
+        '''
+        Save output to FILE.
+        This option is a replacement for piping output to FILE,
+        which would on Windows result into corrupted data
+        being saved.
+
+        '''
+    )
 )
 
 prettify = parser.add_mutually_exclusive_group(required=False)
@@ -119,16 +131,31 @@ output_options.add_argument(
 )
 
 parser.add_argument(
-    '--style', '-s', dest='style', default='solarized', metavar='STYLE',
+    '--style', '-s', dest='style', default=DEFAULT_STYLE, metavar='STYLE',
     choices=AVAILABLE_STYLES,
     help=_('''
-        Output coloring style, one of %s. Defaults to solarized.
+        Output coloring style, one of %s. Defaults to "%s".
         For this option to work properly, please make sure that the
         $TERM environment variable is set to "xterm-256color" or similar
         (e.g., via `export TERM=xterm-256color' in your ~/.bashrc).
-    ''') % ', '.join(sorted(AVAILABLE_STYLES))
+    ''') % (', '.join(sorted(AVAILABLE_STYLES)), DEFAULT_STYLE)
 )
 
+parser.add_argument('--stream', '-S', action='store_true', default=False, help=_(
+    '''
+    Always stream the output by line, i.e., behave like `tail -f'.
+
+    Without --stream and with --pretty (either set or implied),
+    HTTPie fetches the whole response before it outputs the processed data.
+
+    Set this option when you want to continuously display a prettified
+    long-lived response, such as one from the Twitter streaming API.
+
+    It is useful also without --pretty: It ensures that the output is flushed
+    more often and in smaller chunks.
+
+    '''
+))
 parser.add_argument(
     '--check-status', default=False, action='store_true',
     help=_('''
@@ -199,6 +226,13 @@ parser.add_argument(
         (Use socket.setdefaulttimeout() as fallback).
     ''')
 )
+parser.add_argument(
+    '--debug', action='store_true', default=False,
+    help=_('''
+        Prints exception traceback should one occur and other
+        information useful for debugging HTTPie itself.
+    ''')
+)
 
 
 # Positional arguments.

httpie/core.py

@@ -3,37 +3,38 @@
 Invocation flow:
 
     1. Read, validate and process the input (args, `stdin`).
-    2. Create a request and send it, get the response.
+    2. Create and send a request.
-    3. Process and format the requested parts of the request-response exchange.
+    3. Stream, and possibly process and format, the requested parts
-    4. Write to `stdout` and exit.
+       of the request-response exchange.
+    4. Simultaneously write to `stdout`
+    5. Exit.
 
 """
 import sys
 import json
+import errno
 
 import requests
 import requests.auth
 from requests.compat import str
 
-from .models import HTTPMessage, Environment
-from .output import OutputProcessor
-from .input import (PRETTIFY_STDOUT_TTY_ONLY,
-                    OUT_REQ_BODY, OUT_REQ_HEAD,
-                    OUT_RESP_HEAD, OUT_RESP_BODY)
 from .cli import parser
+from .models import Environment
+from .output import output_stream, write
+from . import EXIT
 
 
-TYPE_FORM = 'application/x-www-form-urlencoded; charset=utf-8'
+FORM = 'application/x-www-form-urlencoded; charset=utf-8'
-TYPE_JSON = 'application/json; charset=utf-8'
+JSON = 'application/json; charset=utf-8'
 
 
-def get_response(args, env):
+def get_response(args):
     """Send the request and return a `request.Response`."""
 
     auto_json = args.data and not args.form
     if args.json or auto_json:
-        if 'Content-Type' not in args.headers:
+        if 'Content-Type' not in args.headers and args.data:
-            args.headers['Content-Type'] = TYPE_JSON
+            args.headers['Content-Type'] = JSON
 
         if 'Accept' not in args.headers:
             # Default Accept to JSON as well.
@@ -48,100 +49,44 @@ def get_response(args, env):
         if not args.files and 'Content-Type' not in args.headers:
             # If sending files, `requests` will set
             # the `Content-Type` for us.
-            args.headers['Content-Type'] = TYPE_FORM
+            args.headers['Content-Type'] = FORM
 
-    try:
+    credentials = None
-        credentials = None
+    if args.auth:
-        if args.auth:
+        credentials = {
-            credentials = {
+            'basic': requests.auth.HTTPBasicAuth,
-                'basic': requests.auth.HTTPBasicAuth,
+            'digest': requests.auth.HTTPDigestAuth,
-                'digest': requests.auth.HTTPDigestAuth,
+        }[args.auth_type](args.auth.key, args.auth.value)
-            }[args.auth_type](args.auth.key, args.auth.value)
 
-        return requests.request(
+    return requests.request(
-            method=args.method.lower(),
+        prefetch=False,
-            url=args.url if '://' in args.url else 'http://%s' % args.url,
+        method=args.method.lower(),
-            headers=args.headers,
+        url=args.url,
-            data=args.data,
+        headers=args.headers,
-            verify={'yes': True, 'no': False}.get(args.verify, args.verify),
+        data=args.data,
-            timeout=args.timeout,
+        verify={'yes': True, 'no': False}.get(args.verify, args.verify),
-            auth=credentials,
+        timeout=args.timeout,
-            proxies=dict((p.key, p.value) for p in args.proxy),
+        auth=credentials,
-            files=args.files,
+        proxies=dict((p.key, p.value) for p in args.proxy),
-            allow_redirects=args.allow_redirects,
+        files=args.files,
-            params=args.params,
+        allow_redirects=args.allow_redirects,
-        )
+        params=args.params,
-
+    )
-    except (KeyboardInterrupt, SystemExit):
-        env.stderr.write('\n')
-        sys.exit(1)
-    except Exception as e:
-        if args.traceback:
-            raise
-        env.stderr.write(str(e.message) + '\n')
-        sys.exit(1)
-
-
-def get_output(args, env, request, response):
-    """Format parts of the `request`-`response` exchange
-     according to `args` and `env` and return a `unicode`.
-
-    """
-    do_prettify = (args.prettify is True
-                   or (args.prettify == PRETTIFY_STDOUT_TTY_ONLY
-                       and env.stdout_isatty))
-
-    do_output_request = (OUT_REQ_HEAD in args.output_options
-                         or OUT_REQ_BODY in args.output_options)
-
-    do_output_response = (OUT_RESP_HEAD in args.output_options
-                          or OUT_RESP_BODY in args.output_options)
-
-    prettifier = None
-    if do_prettify:
-        prettifier = OutputProcessor(
-            env, pygments_style=args.style)
-
-    buf = []
-
-    if do_output_request:
-        req_msg = HTTPMessage.from_request(request)
-        req = req_msg.format(
-            prettifier=prettifier,
-            with_headers=OUT_REQ_HEAD in args.output_options,
-            with_body=OUT_REQ_BODY in args.output_options
-        )
-        buf.append(req)
-        buf.append('\n')
-        if do_output_response:
-            buf.append('\n')
-
-    if do_output_response:
-        resp_msg = HTTPMessage.from_response(response)
-        resp = resp_msg.format(
-            prettifier=prettifier,
-            with_headers=OUT_RESP_HEAD in args.output_options,
-            with_body=OUT_RESP_BODY in args.output_options
-        )
-        buf.append(resp)
-        buf.append('\n')
-
-    return ''.join(buf)
 
 
 def get_exist_status(code, allow_redirects=False):
     """Translate HTTP status code to exit status."""
     if 300 <= code <= 399 and not allow_redirects:
         # Redirect
-        return 3
+        return EXIT.ERROR_HTTP_3XX
     elif 400 <= code <= 499:
         # Client Error
-        return 4
+        return EXIT.ERROR_HTTP_4XX
     elif 500 <= code <= 599:
         # Server Error
-        return 5
+        return EXIT.ERROR_HTTP_5XX
     else:
-        return 0
+        return EXIT.OK
 
 
 def main(args=sys.argv[1:], env=Environment()):
@@ -150,22 +95,50 @@ def main(args=sys.argv[1:], env=Environment()):
     Return exit status.
 
     """
-    args = parser.parse_args(args=args, env=env)
-    response = get_response(args, env)
 
-    status = 0
+    def error(msg, *args):
+        msg = msg % args
+        env.stderr.write('\nhttp: error: %s\n' % msg)
 
-    if args.check_status:
+    debug = '--debug' in args
-        status = get_exist_status(response.status_code,
+    status = EXIT.OK
-                                  args.allow_redirects)
-        if status and not env.stdout_isatty:
-            err = 'http error: %s %s\n' % (
-                response.raw.status, response.raw.reason)
-            env.stderr.write(err.encode('utf8'))
 
-    output = get_output(args, env, response.request, response)
+    try:
-    output_bytes = output.encode('utf8')
+        args = parser.parse_args(args=args, env=env)
-    f = getattr(env.stdout, 'buffer', env.stdout)
+        response = get_response(args)
-    f.write(output_bytes)
+
+        if args.check_status:
+            status = get_exist_status(response.status_code,
+                                      args.allow_redirects)
+            if status and not env.stdout_isatty:
+                error('%s %s', response.raw.status, response.raw.reason)
+
+        stream = output_stream(args, env, response.request, response)
+
+        try:
+            write(stream=stream,
+                  outfile=env.stdout,
+                  flush=env.stdout_isatty or args.stream)
+
+        except IOError as e:
+            if not debug and e.errno == errno.EPIPE:
+                # Ignore broken pipes unless --debug.
+                env.stderr.write('\n')
+            else:
+                raise
+
+    except (KeyboardInterrupt, SystemExit):
+        if debug:
+            raise
+        env.stderr.write('\n')
+        status = EXIT.ERROR
+
+    except Exception as e:
+        # TODO: distinguish between expected and unexpected errors.
+        #       network errors vs. bugs, etc.
+        if debug:
+            raise
+        error('%s: %s', type(e).__name__, str(e))
+        status = EXIT.ERROR
 
     return status

httpie/input.py

@@ -8,6 +8,7 @@ import json
 import argparse
 import mimetypes
 import getpass
+from io import BytesIO
 
 try:
     from collections import OrderedDict
@@ -15,13 +16,15 @@ except ImportError:
     OrderedDict = dict
 
 from requests.structures import CaseInsensitiveDict
-from requests.compat import str
+from requests.compat import str, urlparse
 
 from . import __version__
 
 
 HTTP_POST = 'POST'
 HTTP_GET = 'GET'
+HTTP = 'http://'
+HTTPS = 'https://'
 
 
 # Various separators used in args
@@ -90,8 +93,18 @@ class Parser(argparse.ArgumentParser):
     #noinspection PyMethodOverriding
     def parse_args(self, env, args=None, namespace=None):
 
+        self.env = env
+
+        if env.is_windows and not env.stdout_isatty:
+            self.error('Output redirection is not supported on Windows.'
+                  ' Please use `--output FILE\' instead.')
+
         args = super(Parser, self).parse_args(args, namespace)
 
+        if args.output:
+            env.stdout = args.output
+            env.stdout_isatty = False
+
         self._process_output_options(args, env)
         self._guess_method(args, env)
         self._parse_items(args)
@@ -99,22 +112,41 @@ class Parser(argparse.ArgumentParser):
         if not env.stdin_isatty:
             self._body_from_file(args, env.stdin)
 
+        if not (args.url.startswith(HTTP) or args.url.startswith(HTTPS)):
+            scheme = HTTPS if env.progname == 'https' else HTTP
+            args.url = scheme + args.url
+
         if args.auth and not args.auth.has_password():
             # Stdin already read (if not a tty) so it's save to prompt.
-            args.auth.prompt_password()
+            args.auth.prompt_password(urlparse(args.url).netloc)
+
+        if args.prettify == PRETTIFY_STDOUT_TTY_ONLY:
+            args.prettify = env.stdout_isatty
+        elif args.prettify and env.is_windows:
+            self.error('Only terminal output can be prettified on Windows.')
 
         return args
 
-    def _body_from_file(self, args, f):
+    def _print_message(self, message, file=None):
-        """Use the content of `f` as the `request.data`.
+        # Sneak in our stderr/stdout.
+        file = {
+            sys.stdout: self.env.stdout,
+            sys.stderr: self.env.stderr,
+            None: self.env.stderr
+        }.get(file, file)
 
-        There can only be one source of request data.
+        super(Parser, self)._print_message(message, file)
+
+    def _body_from_file(self, args, fd):
+        """There can only be one source of request data.
+
+        Bytes are always read.
 
         """
         if args.data:
             self.error('Request body (from stdin or a file) and request '
                        'data (key=value) cannot be mixed.')
-        args.data = f.read()
+        args.data = getattr(fd, 'buffer', fd).read()
 
     def _guess_method(self, args, env):
         """Set `args.method` if not specified to either POST or GET
@@ -139,7 +171,7 @@ class Parser(argparse.ArgumentParser):
                     0, KeyValueArgType(*SEP_GROUP_ITEMS).__call__(args.url))
 
             except argparse.ArgumentTypeError as e:
-                if args.traceback:
+                if args.debug:
                     raise
                 self.error(e.message)
 
@@ -169,29 +201,23 @@ class Parser(argparse.ArgumentParser):
                         files=args.files,
                         params=args.params)
         except ParseError as e:
-            if args.traceback:
+            if args.debug:
                 raise
             self.error(e.message)
 
         if args.files and not args.form:
             # `http url @/path/to/file`
-            # It's not --form so the file contents will be used as the
+            file_fields = list(args.files.keys())
-            # body of the requests. Also, we try to detect the appropriate
+            if file_fields != ['']:
-            # Content-Type.
-            if len(args.files) > 1:
                 self.error(
-                    'Only one file can be specified unless'
+                    'Invalid file fields (perhaps you meant --form?): %s'
-                    ' --form is used. File fields: %s'
+                    % ','.join(file_fields))
-                    % ','.join(args.files.keys()))
 
-            f = list(args.files.values())[0]
+            fn, fd = args.files['']
-            self._body_from_file(args, f)
-
-            # Reset files
             args.files = {}
-
+            self._body_from_file(args, fd)
             if 'Content-Type' not in args.headers:
-                mime, encoding = mimetypes.guess_type(f.name, strict=False)
+                mime, encoding = mimetypes.guess_type(fn, strict=False)
                 if mime:
                     content_type = mime
                     if encoding:
@@ -327,9 +353,10 @@ class AuthCredentials(KeyValue):
     def has_password(self):
         return self.value is not None
 
-    def prompt_password(self):
+    def prompt_password(self, host):
         try:
-            self.value = self._getpass("Password for user '%s': " % self.key)
+            self.value = self._getpass(
+                'http: password for %s@%s: ' % (self.key, host))
         except (EOFError, KeyboardInterrupt):
             sys.stderr.write('\n')
             sys.exit(0)
@@ -406,12 +433,12 @@ def parse_items(items, data=None, headers=None, files=None, params=None):
             target = params
         elif item.sep == SEP_FILES:
             try:
-                value = open(os.path.expanduser(item.value), 'r')
+                with open(os.path.expanduser(value), 'rb') as f:
+                    value = (os.path.basename(value),
+                             BytesIO(f.read()))
             except IOError as e:
                 raise ParseError(
                     'Invalid argument "%s": %s' % (item.orig, e))
-            if not key:
-                key = os.path.basename(value.name)
             target = files
 
         elif item.sep in [SEP_DATA, SEP_DATA_RAW_JSON]:

httpie/models.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from requests.compat import urlparse, is_windows
+from requests.compat import urlparse, is_windows, bytes, str
 
 
 class Environment(object):
@@ -10,130 +10,175 @@ class Environment(object):
     and allows for mocking.
 
     """
-    stdin_isatty = sys.stdin.isatty()
+
-    stdin = sys.stdin
+    #noinspection PyUnresolvedReferences
+    is_windows = is_windows
+
+    progname = os.path.basename(sys.argv[0])
+    if progname not in ['http', 'https']:
+        progname = 'http'
 
     if is_windows:
-        # `colorama` patches `sys.stdout` so its initialization
+        import colorama.initialise
-        # needs to happen before the default environment is set.
+        colorama.initialise.init()
-        import colorama
-        colorama.init()
-        del colorama
 
+    stdin_isatty = sys.stdin.isatty()
+    stdin = sys.stdin
     stdout_isatty = sys.stdout.isatty()
     stdout = sys.stdout
-
     stderr = sys.stderr
 
     # Can be set to 0 to disable colors completely.
     colors = 256 if '256color' in os.environ.get('TERM', '') else 88
 
     def __init__(self, **kwargs):
+        assert all(hasattr(type(self), attr)
+                   for attr in kwargs.keys())
         self.__dict__.update(**kwargs)
 
 
 class HTTPMessage(object):
-    """Model representing an HTTP message."""
+    """Abstract class for HTTP messages."""
 
-    def __init__(self, line, headers, body, content_type=None):
+    def __init__(self, orig):
-        # {Request,Status}-Line
+        self._orig = orig
-        self.line = line
-        self.headers = headers
-        self.body = body
-        self.content_type = content_type
 
-    def format(self, prettifier=None, with_headers=True, with_body=True):
+    def iter_body(self, chunk_size):
-        """Return a `unicode` representation of `self`. """
+        """Return an iterator over the body."""
-        pretty = prettifier is not None
+        raise NotImplementedError()
-        bits = []
 
-        if with_headers:
+    def iter_lines(self, chunk_size):
-            bits.append(self.line)
+        """Return an iterator over the body yielding (`line`, `line_feed`)."""
-            bits.append(self.headers)
+        raise NotImplementedError()
-            if pretty:
-                bits = [
-                    prettifier.process_headers('\n'.join(bits))
-                ]
-            if with_body and self.body:
-                bits.append('\n')
 
-        if with_body and self.body:
+    @property
-            if pretty and self.content_type:
+    def headers(self):
-                bits.append(prettifier.process_body(
+        """Return a `str` with the message's headers."""
-                    self.body, self.content_type))
+        raise NotImplementedError()
-            else:
-                bits.append(self.body)
 
-        return '\n'.join(bit.strip() for bit in bits)
+    @property
+    def encoding(self):
+        """Return a `str` with the message's encoding, if known."""
+        raise NotImplementedError()
 
-    @staticmethod
+    @property
-    def from_request(request):
+    def body(self):
-        """Make an `HTTPMessage` from `requests.models.Request`."""
+        """Return a `bytes` with the message's body."""
+        raise NotImplementedError()
 
-        url = urlparse(request.url)
+    @property
+    def content_type(self):
+        """Return the message content type."""
+        ct = self._orig.headers.get('Content-Type', '')
+        if isinstance(ct, bytes):
+            ct = ct.decode()
+        return ct
+
+
+class HTTPResponse(HTTPMessage):
+    """A :class:`requests.models.Response` wrapper."""
+
+    def iter_body(self, chunk_size=1):
+        return self._orig.iter_content(chunk_size=chunk_size)
+
+    def iter_lines(self, chunk_size):
+        for line in self._orig.iter_lines(chunk_size):
+            yield line, b'\n'
+
+    @property
+    def headers(self):
+        original = self._orig.raw._original_response
+        status_line = 'HTTP/{version} {status} {reason}'.format(
+             version='.'.join(str(original.version)),
+             status=original.status,
+             reason=original.reason
+         )
+        headers = str(original.msg)
+        return '\n'.join([status_line, headers]).strip()
+
+    @property
+    def encoding(self):
+        return self._orig.encoding or 'utf8'
+
+    @property
+    def body(self):
+        # Only now the response body is fetched.
+        # Shouldn't be touched unless the body is actually needed.
+        return self._orig.content
+
+
+class HTTPRequest(HTTPMessage):
+    """A :class:`requests.models.Request` wrapper."""
+
+    def iter_body(self, chunk_size):
+        yield self.body
+
+    def iter_lines(self, chunk_size):
+        yield self.body, b''
+
+    @property
+    def headers(self):
+        """Return Request-Line"""
+        url = urlparse(self._orig.url)
 
         # Querystring
         qs = ''
-        if url.query or request.params:
+        if url.query or self._orig.params:
             qs = '?'
             if url.query:
                 qs += url.query
             # Requests doesn't make params part of ``request.url``.
-            if request.params:
+            if self._orig.params:
                 if url.query:
                     qs += '&'
                 #noinspection PyUnresolvedReferences
-                qs += type(request)._encode_params(request.params)
+                qs += type(self._orig)._encode_params(self._orig.params)
 
         # Request-Line
         request_line = '{method} {path}{query} HTTP/1.1'.format(
-            method=request.method,
+            method=self._orig.method,
             path=url.path or '/',
             query=qs
         )
 
-        # Headers
+        headers = dict(self._orig.headers)
-        headers = dict(request.headers)
+
-        content_type = headers.get('Content-Type')
         if 'Host' not in headers:
-            headers['Host'] = url.netloc
+            headers['Host'] = urlparse(self._orig.url).netloc
-        headers = '\n'.join(
-            str('%s: %s') % (name, value)
-            for name, value
-            in headers.items()
-        )
 
-        # Body
+        headers = ['%s: %s' % (name, value)
-        try:
+                   for name, value in headers.items()]
-            body = request.data
-        except AttributeError:
-            # requests < 0.12.1
-            body = request._enc_data
-        if isinstance(body, dict):
-            #noinspection PyUnresolvedReferences
-            body = type(request)._encode_params(body)
 
-        return HTTPMessage(
+        headers.insert(0, request_line)
-            line=request_line,
-            headers=headers,
-            body=body,
-            content_type=content_type
-        )
 
-    @classmethod
+        return '\n'.join(headers).strip()
-    def from_response(cls, response):
+
-        """Make an `HTTPMessage` from `requests.models.Response`."""
+    @property
-        encoding = response.encoding or 'ISO-8859-1'
+    def encoding(self):
-        original = response.raw._original_response
+        return 'utf8'
-        response_headers = response.headers
+
-        status_line = 'HTTP/{version} {status} {reason}'.format(
+    @property
-            version='.'.join(str(original.version)),
+    def body(self):
-            status=original.status,
+        """Reconstruct and return the original request body bytes."""
-            reason=original.reason
+        if self._orig.files:
-        )
+            # TODO: would be nice if we didn't need to encode the files again
-        body = response.content.decode(encoding) if response.content else ''
+            # FIXME: Also the boundary header doesn't match the one used.
-        return cls(
+            for fn, fd in self._orig.files.values():
-            line=status_line,
+                # Rewind the files as they have already been read before.
-            headers=str(original.msg),
+                fd.seek(0)
-            body=body,
+            body, _ = self._orig._encode_files(self._orig.files)
-            content_type=response_headers.get('Content-Type'))
+        else:
+            try:
+                body = self._orig.data
+            except AttributeError:
+                # requests < 0.12.1
+                body = self._orig._enc_data
+
+            if isinstance(body, dict):
+                #noinspection PyUnresolvedReferences
+                body = type(self._orig)._encode_params(body)
+
+            if isinstance(body, str):
+                body = body.encode('utf8')
+
+        return body

httpie/output.py

@@ -1,31 +1,291 @@
-"""Output processing and formatting.
+"""Output streaming, processing and formatting.
 
 """
-import re
 import json
+from functools import partial
+from itertools import chain
 
 import pygments
 from pygments import token, lexer
 from pygments.styles import get_style_by_name, STYLE_MAP
-from pygments.lexers import get_lexer_for_mimetype
+from pygments.lexers import get_lexer_for_mimetype, get_lexer_by_name
 from pygments.formatters.terminal import TerminalFormatter
 from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.util import ClassNotFound
 from requests.compat import is_windows
 
-from . import solarized
+from .solarized import Solarized256Style
+from .models import HTTPRequest, HTTPResponse, Environment
+from .input import (OUT_REQ_BODY, OUT_REQ_HEAD,
+                    OUT_RESP_HEAD, OUT_RESP_BODY)
 
 
-DEFAULT_STYLE = 'solarized'
+# Colors on Windows via colorama aren't that great and fruity
-AVAILABLE_STYLES = [DEFAULT_STYLE] + list(STYLE_MAP.keys())
+# seems to give the best result there.
+DEFAULT_STYLE = 'solarized' if not is_windows else 'fruity'
 
+#noinspection PySetFunctionToLiteral
+AVAILABLE_STYLES = set([DEFAULT_STYLE]) | set(STYLE_MAP.keys())
+
+
+BINARY_SUPPRESSED_NOTICE = (
+    b'\n'
+    b'+-----------------------------------------+\n'
+    b'| NOTE: binary data not shown in terminal |\n'
+    b'+-----------------------------------------+'
+)
+
+
+class BinarySuppressedError(Exception):
+    """An error indicating that the body is binary and won't be written,
+     e.g., for terminal output)."""
+
+    message = BINARY_SUPPRESSED_NOTICE
+
+
+###############################################################################
+# Output Streams
+###############################################################################
+
+
+def write(stream, outfile, flush):
+    """Write the output stream."""
+    try:
+        # Writing bytes so we use the buffer interface (Python 3).
+        buf = outfile.buffer
+    except AttributeError:
+        buf = outfile
+
+    for chunk in stream:
+        buf.write(chunk)
+        if flush:
+            outfile.flush()
+
+
+def output_stream(args, env, request, response):
+    """Build and return a chain of iterators over the `request`-`response`
+    exchange each of which yields `bytes` chunks.
+
+    """
+
+    Stream = make_stream(env, args)
+
+    req_h = OUT_REQ_HEAD in args.output_options
+    req_b = OUT_REQ_BODY in args.output_options
+    resp_h = OUT_RESP_HEAD in args.output_options
+    resp_b = OUT_RESP_BODY  in args.output_options
+
+    req = req_h or req_b
+    resp = resp_h or resp_b
+
+    output = []
+
+    if req:
+        output.append(Stream(
+            msg=HTTPRequest(request),
+            with_headers=req_h,
+            with_body=req_b))
+
+    if req and resp:
+        output.append([b'\n\n\n'])
+
+    if resp:
+        output.append(Stream(
+            msg=HTTPResponse(response),
+            with_headers=resp_h,
+            with_body=resp_b))
+
+    if env.stdout_isatty:
+        output.append([b'\n\n'])
+
+    return chain(*output)
+
+
+def make_stream(env, args):
+    """Pick the right stream type based on `env` and `args`.
+    Wrap it in a partial with the type-specific args so that
+    we don't need to think what stream we are dealing with.
+
+    """
+    if not env.stdout_isatty and not args.prettify:
+        Stream = partial(
+            RawStream,
+            chunk_size=RawStream.CHUNK_SIZE_BY_LINE
+                       if args.stream
+                       else RawStream.CHUNK_SIZE)
+    elif args.prettify:
+        Stream = partial(
+            PrettyStream if args.stream else BufferedPrettyStream,
+            processor=OutputProcessor(env, pygments_style=args.style),
+            env=env)
+    else:
+        Stream = partial(EncodedStream, env=env)
+
+    return Stream
+
+
+class BaseStream(object):
+    """Base HTTP message stream class."""
+
+    def __init__(self, msg, with_headers=True, with_body=True):
+        """
+        :param msg: a :class:`models.HTTPMessage` subclass
+        :param with_headers: if `True`, headers will be included
+        :param with_body: if `True`, body will be included
+
+        """
+        self.msg = msg
+        self.with_headers = with_headers
+        self.with_body = with_body
+
+    def _headers(self):
+        """Return the headers' bytes."""
+        return self.msg.headers.encode('ascii')
+
+    def _body(self):
+        """Return an iterator over the message body."""
+        raise NotImplementedError()
+
+    def __iter__(self):
+        """Return an iterator over `self.msg`."""
+        if self.with_headers:
+            yield self._headers()
+
+        if self.with_body:
+            it = self._body()
+
+            try:
+                if self.with_headers:
+                    # Yield the headers/body separator only if needed.
+                    chunk = next(it)
+                    if chunk:
+                        yield b'\n\n'
+                        yield chunk
+
+                for chunk in it:
+                    yield chunk
+
+            except BinarySuppressedError as e:
+                if self.with_headers:
+                    yield b'\n'
+                yield e.message
+
+
+class RawStream(BaseStream):
+    """The message is streamed in chunks with no processing."""
+
+    CHUNK_SIZE = 1024 * 100
+    CHUNK_SIZE_BY_LINE = 1024 * 5
+
+    def __init__(self, chunk_size=CHUNK_SIZE, **kwargs):
+        super(RawStream, self).__init__(**kwargs)
+        self.chunk_size = chunk_size
+
+    def _body(self):
+        return self.msg.iter_body(self.chunk_size)
+
+
+class EncodedStream(BaseStream):
+    """Encoded HTTP message stream.
+
+    The message bytes are converted to an encoding suitable for
+    `self.env.stdout`. Unicode errors are replaced and binary data
+    is suppressed. The body is always streamed by line.
+
+    """
+    CHUNK_SIZE = 1024 * 5
+    def __init__(self, env=Environment(), **kwargs):
+
+        super(EncodedStream, self).__init__(**kwargs)
+
+        if env.stdout_isatty:
+            # Use the encoding supported by the terminal.
+            output_encoding = getattr(env.stdout, 'encoding', None)
+        else:
+            # Preserve the message encoding.
+            output_encoding = self.msg.encoding
+
+        # Default to utf8 when unsure.
+        self.output_encoding = output_encoding or 'utf8'
+
+    def _body(self):
+
+        for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
+
+            if b'\0' in line:
+                raise BinarySuppressedError()
+
+            yield line.decode(self.msg.encoding)\
+                      .encode(self.output_encoding, 'replace') + lf
+
+
+class PrettyStream(EncodedStream):
+    """In addition to :class:`EncodedStream` behaviour, this stream applies
+    content processing.
+
+    Useful for long-lived HTTP responses that stream by lines
+    such as the Twitter streaming API.
+
+    """
+
+    CHUNK_SIZE = 1024 * 5
+
+    def __init__(self, processor, **kwargs):
+        super(PrettyStream, self).__init__(**kwargs)
+        self.processor = processor
+
+    def _headers(self):
+        return self.processor.process_headers(
+            self.msg.headers).encode(self.output_encoding)
+
+    def _body(self):
+        for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
+            if b'\0' in line:
+                raise BinarySuppressedError()
+            yield self._process_body(line) + lf
+
+    def _process_body(self, chunk):
+        return (self.processor
+                    .process_body(
+                        chunk.decode(self.msg.encoding, 'replace'),
+                        self.msg.content_type)
+                    .encode(self.output_encoding, 'replace'))
+
+
+class BufferedPrettyStream(PrettyStream):
+    """The same as :class:`PrettyStream` except that the body is fully
+    fetched before it's processed.
+
+    Suitable regular HTTP responses.
+
+    """
+
+    CHUNK_SIZE = 1024 * 10
+
+    def _body(self):
+
+        #noinspection PyArgumentList
+        # Read the whole body before prettifying it,
+        # but bail out immediately if the body is binary.
+        body = bytearray()
+        for chunk in self.msg.iter_body(self.CHUNK_SIZE):
+            if b'\0' in chunk:
+                raise BinarySuppressedError()
+            body.extend(chunk)
+
+        yield self._process_body(body)
+
+
+###############################################################################
+# Processing
+###############################################################################
 
 class HTTPLexer(lexer.RegexLexer):
     """Simplified HTTP lexer for Pygments.
 
     It only operates on headers and provides a stronger contrast between
     their names and values than the original one bundled with Pygments
-    (`pygments.lexers.text import HttpLexer`), especially when
+    (:class:`pygments.lexers.text import HttpLexer`), especially when
     Solarized color scheme is used.
 
     """
@@ -34,7 +294,6 @@ class HTTPLexer(lexer.RegexLexer):
     filenames = ['*.http']
     tokens = {
         'root': [
-
             # Request-Line
             (r'([A-Z]+)( +)([^ ]+)( +)(HTTP)(/)(\d+\.\d+)',
              lexer.bygroups(
@@ -46,7 +305,6 @@ class HTTPLexer(lexer.RegexLexer):
                 token.Operator,
                 token.Number
              )),
-
             # Response Status-Line
             (r'(HTTP)(/)(\d+\.\d+)( +)(\d{3})( +)(.+)',
              lexer.bygroups(
@@ -58,7 +316,6 @@ class HTTPLexer(lexer.RegexLexer):
                  token.Text,
                  token.Name.Exception,  # Reason
              )),
-
             # Header
             (r'(.*?)( *)(:)( *)(.+)', lexer.bygroups(
                 token.Name.Attribute, # Name
@@ -71,54 +328,85 @@ class HTTPLexer(lexer.RegexLexer):
 
 
 class BaseProcessor(object):
+    """Base, noop output processor class."""
 
     enabled = True
 
     def __init__(self, env, **kwargs):
+        """
+        :param env:
+            an class:`Environment` instance
+        :param kwargs:
+            additional keyword argument that some processor might require.
+
+        """
         self.env = env
         self.kwargs = kwargs
 
     def process_headers(self, headers):
+        """Return processed `headers`
+
+        :param headers:
+            The headers as text.
+
+        """
         return headers
 
-    def process_body(self, content, content_type):
+    def process_body(self, content, content_type, subtype):
+        """Return processed `content`.
+
+        :param content:
+            The body content as text
+
+        :param content_type:
+            Full content type, e.g., 'application/atom+xml'.
+
+        :param subtype:
+            E.g. 'xml'.
+
+        """
         return content
 
 
 class JSONProcessor(BaseProcessor):
+    """JSON body processor."""
 
-    def process_body(self, content, content_type):
+    def process_body(self, content, content_type, subtype):
-        if content_type == 'application/json':
+        if subtype == 'json':
             try:
-                # Indent and sort the JSON data.
+                # Indent the JSON data, sort keys by name, and
-                content = json.dumps(
+                # avoid unicode escapes to improve readability.
-                    json.loads(content),
+                content = json.dumps(json.loads(content),
-                    sort_keys=True,
+                                     sort_keys=True,
-                    ensure_ascii=False,
+                                     ensure_ascii=False,
-                    indent=4,
+                                     indent=4)
-                )
             except ValueError:
-                # Invalid JSON - we don't care.
+                # Invalid JSON but we don't care.
                 pass
         return content
 
 
 class PygmentsProcessor(BaseProcessor):
+    """A processor that applies syntax-highlighting using Pygments
+    to the headers, and to the body as well if its content type is recognized.
 
+    """
     def __init__(self, *args, **kwargs):
         super(PygmentsProcessor, self).__init__(*args, **kwargs)
 
+        # Cache that speeds up when we process streamed body by line.
+        self.lexers_by_type = {}
+
         if not self.env.colors:
             self.enabled = False
             return
 
         try:
-            style = get_style_by_name(
+            style = get_style_by_name(self.kwargs['pygments_style'])
-                self.kwargs.get('pygments_style', DEFAULT_STYLE))
         except ClassNotFound:
-            style = solarized.SolarizedStyle
+            style = Solarized256Style
 
-        if is_windows or self.env.colors == 256:
+        if self.env.is_windows or self.env.colors == 256:
             fmt_class = Terminal256Formatter
         else:
             fmt_class = TerminalFormatter
@@ -126,27 +414,45 @@ class PygmentsProcessor(BaseProcessor):
 
     def process_headers(self, headers):
         return pygments.highlight(
-            headers, HTTPLexer(), self.formatter)
+            headers, HTTPLexer(), self.formatter).strip()
 
-    def process_body(self, content, content_type):
+    def process_body(self, content, content_type, subtype):
         try:
-            lexer = get_lexer_for_mimetype(content_type)
+            lexer = self.lexers_by_type.get(content_type)
+            if not lexer:
+                try:
+                    lexer = get_lexer_for_mimetype(content_type)
+                except ClassNotFound:
+                    lexer = get_lexer_by_name(subtype)
+                self.lexers_by_type[content_type] = lexer
         except ClassNotFound:
             pass
         else:
             content = pygments.highlight(content, lexer, self.formatter)
-        return content
+        return content.strip()
+
+
+class HeadersProcessor(BaseProcessor):
+    """Sorts headers by name retaining relative order of multiple headers
+    with the same name.
+
+    """
+    def process_headers(self, headers):
+        lines = headers.splitlines()
+        headers = sorted(lines[1:], key=lambda h: h.split(':')[0])
+        return '\n'.join(lines[:1] + headers)
 
 
 class OutputProcessor(object):
+    """A delegate class that invokes the actual processors."""
 
     installed_processors = [
         JSONProcessor,
+        HeadersProcessor,
         PygmentsProcessor
     ]
 
     def __init__(self, env, **kwargs):
-        self.env = env
         processors = [
             cls(env, **kwargs)
             for cls in self.installed_processors
@@ -155,22 +461,16 @@ class OutputProcessor(object):
 
     def process_headers(self, headers):
         for processor in self.processors:
-         headers = processor.process_headers(headers)
+            headers = processor.process_headers(headers)
         return headers
 
     def process_body(self, content, content_type):
+        # e.g., 'application/atom+xml'
         content_type = content_type.split(';')[0]
-
+        # e.g., 'xml'
-        application_match = re.match(
+        subtype = content_type.split('/')[-1].split('+')[-1]
-            r'application/(.+\+)(json|xml)$',
-            content_type
-        )
-        if application_match:
-            # Strip vendor and extensions from Content-Type
-            vendor, extension = application_match.groups()
-            content_type = content_type.replace(vendor, '')
 
         for processor in self.processors:
-            content = processor.process_body(content, content_type)
+            content = processor.process_body(content, content_type, subtype)
 
         return content

httpie/solarized.py

@@ -1,74 +1,57 @@
-"""
+# -*- coding: utf-8 -*-
-A Pygments_ style based on the dark background variant of Solarized_.
-
-.. _Pygments: http://pygments.org/
-.. _Solarized: http://ethanschoonover.com/solarized
-
-Copyright (c) 2011 Hank Gay
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
 
 """
+    solarized256
+    ------------
+
+    A Pygments style inspired by Solarized's 256 color mode.
+
+    :copyright: (c) 2011 by Hank Gay, (c) 2012 by John Mastro.
+    :license: BSD, see LICENSE for more details.
+"""
+
 from pygments.style import Style
-from pygments.token import (Token, Comment, Name, Keyword, Generic, Number,
+from pygments.token import Token, Comment, Name, Keyword, Generic, Number, \
-                            Operator, String)
+    Operator, String
+
+BASE03  = "#1c1c1c"
+BASE02  = "#262626"
+BASE01  = "#4e4e4e"
+BASE00  = "#585858"
+BASE0   = "#808080"
+BASE1   = "#8a8a8a"
+BASE2   = "#d7d7af"
+BASE3   = "#ffffd7"
+YELLOW  = "#af8700"
+ORANGE  = "#d75f00"
+RED     = "#af0000"
+MAGENTA = "#af005f"
+VIOLET  = "#5f5faf"
+BLUE    = "#0087ff"
+CYAN    = "#00afaf"
+GREEN   = "#5f8700"
 
 
-BASE03 = '#002B36'
+class Solarized256Style(Style):
-BASE02 = '#073642'
-BASE01 = '#586E75'
-BASE00 = '#657B83'
-BASE0 = '#839496'
-BASE1 = '#93A1A1'
-BASE2 = '#EEE8D5'
-BASE3 = '#FDF6E3'
-YELLOW = '#B58900'
-ORANGE = '#CB4B16'
-RED = '#DC322F'
-MAGENTA = '#D33682'
-VIOLET = '#6C71C4'
-BLUE = '#268BD2'
-CYAN = '#2AA198'
-GREEN = '#859900'
-
-
-class SolarizedStyle(Style):
     background_color = BASE03
     styles = {
         Keyword: GREEN,
         Keyword.Constant: ORANGE,
         Keyword.Declaration: BLUE,
-        #Keyword.Namespace
+        Keyword.Namespace: ORANGE,
         #Keyword.Pseudo
         Keyword.Reserved: BLUE,
         Keyword.Type: RED,
 
         #Name
         Name.Attribute: BASE1,
-        Name.Builtin: YELLOW,
+        Name.Builtin: BLUE,
         Name.Builtin.Pseudo: BLUE,
         Name.Class: BLUE,
         Name.Constant: ORANGE,
         Name.Decorator: BLUE,
         Name.Entity: ORANGE,
-        Name.Exception: ORANGE,
+        Name.Exception: YELLOW,
         Name.Function: BLUE,
         #Name.Label
         #Name.Namespace
@@ -84,10 +67,10 @@ class SolarizedStyle(Style):
         String: CYAN,
         String.Backtick: BASE01,
         String.Char: CYAN,
-        String.Doc: BASE1,
+        String.Doc: CYAN,
         #String.Double
-        String.Escape: ORANGE,
+        String.Escape: RED,
-        String.Heredoc: BASE1,
+        String.Heredoc: CYAN,
         #String.Interpol
         #String.Other
         String.Regex: RED,
@@ -100,8 +83,8 @@ class SolarizedStyle(Style):
         #Number.Integer.Long
         #Number.Oct
 
-        Operator: GREEN,
+        Operator: BASE1,
-        #Operator.Word
+        Operator.Word: GREEN,
 
         #Punctuation: ORANGE,