Fixed changelog link.

Expunge user:pass@... from Host header.

.gitignore

@@ -3,3 +3,8 @@ httpie.egg-info
 build
 *.pyc
 .tox
+README.html
+.coverage
+htmlcov
+.idea
+.DS_Store

.travis.yml

@@ -3,7 +3,7 @@ python:
   - 2.6
   - 2.7
   - pypy
-  - 3.2
+  - 3.3
 script: python setup.py test
 install:
   - pip install . --use-mirrors

AUTHORS.rst

@@ -0,0 +1,32 @@
+==============
+HTTPie authors
+==============
+
+* `Jakub Roztocil <https://github.com/jkbr>`_
+
+
+Patches and ideas
+-----------------
+
+* `Cláudia T. Delgado <https://github.com/claudiatd>`_ (logo)
+* `Hank Gay <https://github.com/gthank>`_
+* `Jake Basile <https://github.com/jakebasile>`_
+* `Vladimir Berkutov <https://github.com/dair-targ>`_
+* `Jakob Kramer <https://github.com/gandaro>`_
+* `Chris Faulkner <https://github.com/faulkner>`_
+* `Alen Mujezinovic <https://github.com/flashingpumpkin>`_
+* `Praful Mathur <https://github.com/tictactix>`_
+* `Marc Abramowitz <https://github.com/msabramo>`_
+* `Ismail Badawi <https://github.com/isbadawi>`_
+* `Laurent Bachelier <https://github.com/laurentb>`_
+* `Isman Firmansyah <https://github.com/iromli>`_
+* `Simon Olofsson <https://github.com/simono>`_
+* `Churkin Oleg <https://github.com/Bahus>`_
+* `Jökull Sólberg Auðunsson <https://github.com/jokull>`_
+* `Matthew M. Boedicker <https://github.com/mmb>`_
+* `marblar <https://github.com/marblar>`_
+* `Tomek Wójcik <https://github.com/tomekwojcik>`_
+* `Davey Shafik <https://github.com/dshafik>`_
+* `cido <https://github.com/cido>`_
+* `Justin Bonnar <https://github.com/jargonjustin>`_
+* `Nathan LaFreniere <https://github.com/nlf>`_

httpie/__init__.py

@@ -1,7 +1,19 @@
 """
-HTTPie - cURL for humans.
+HTTPie - a CLI, cURL-like tool for humans.
 
 """
 __author__ = 'Jakub Roztocil'
-__version__ = '0.2.6'
+__version__ = '0.8.0'
 __licence__ = 'BSD'
+
+
+class ExitStatus:
+    """Exit status code constants."""
+    OK = 0
+    ERROR = 1
+    ERROR_TIMEOUT = 2
+
+    # Used only when requested with --check-status:
+    ERROR_HTTP_3XX = 3
+    ERROR_HTTP_4XX = 4
+    ERROR_HTTP_5XX = 5

httpie/cli.py

@@ -3,238 +3,546 @@
 NOTE: the CLI interface may change before reaching v1.0.
 
 """
+from textwrap import dedent, wrap
+#noinspection PyCompatibility
+from argparse import (RawDescriptionHelpFormatter, FileType,
+                      OPTIONAL, ZERO_OR_MORE, SUPPRESS)
+
 from . import __doc__
 from . import __version__
-from .output import AVAILABLE_STYLES
+from .plugins.builtin import BuiltinAuthPlugin
+from .plugins import plugin_manager
+from .sessions import DEFAULT_SESSIONS_DIR
+from .output import AVAILABLE_STYLES, DEFAULT_STYLE
 from .input import (Parser, AuthCredentialsArgType, KeyValueArgType,
-                    PRETTIFY_STDOUT_TTY_ONLY,
-                    SEP_PROXY, SEP_CREDENTIALS, SEP_GROUP_ITEMS,
+                    SEP_PROXY, SEP_CREDENTIALS, SEP_GROUP_ALL_ITEMS,
                     OUT_REQ_HEAD, OUT_REQ_BODY, OUT_RESP_HEAD,
-                    OUT_RESP_BODY, OUTPUT_OPTIONS)
+                    OUT_RESP_BODY, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
+                    PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, SessionNameValidator)
 
 
-def _(text):
-    """Normalize whitespace."""
-    return ' '.join(text.strip().split())
+class HTTPieHelpFormatter(RawDescriptionHelpFormatter):
+    """A nicer help formatter.
+
+    Help for arguments can be indented and contain new lines.
+    It will be de-dented and arguments in the help
+    will be separated by a blank line for better readability.
 
 
-parser = Parser(description='%s <http://httpie.org>' % __doc__.strip())
-parser.add_argument('--version', action='version', version=__version__)
+    """
+    def __init__(self, max_help_position=6, *args, **kwargs):
+        # A smaller indent for args help.
+        kwargs['max_help_position'] = max_help_position
+        super(HTTPieHelpFormatter, self).__init__(*args, **kwargs)
+
+    def _split_lines(self, text, width):
+        text = dedent(text).strip() + '\n\n'
+        return text.splitlines()
+
+parser = Parser(
+    formatter_class=HTTPieHelpFormatter,
+    description='%s <http://httpie.org>' % __doc__.strip(),
+    epilog=dedent("""
+    For every --OPTION there is also a --no-OPTION that reverts OPTION
+    to its default value.
+
+    Suggestions and bug reports are greatly appreciated:
+
+        https://github.com/jkbr/httpie/issues
+
+    """)
+)
 
 
+#######################################################################
+# Positional arguments.
+#######################################################################
+
+positional = parser.add_argument_group(
+    title='Positional arguments',
+    description=dedent("""
+    These arguments come after any flags and in the order they are listed here.
+    Only URL is required.
+
+    """)
+)
+positional.add_argument(
+    'method',
+    metavar='METHOD',
+    nargs=OPTIONAL,
+    default=None,
+    help="""
+    The HTTP method to be used for the request (GET, POST, PUT, DELETE, ...).
+
+    This argument can be omitted in which case HTTPie will use POST if there
+    is some data to be sent, otherwise GET:
+
+        $ http example.org               # => GET
+        $ http example.org hello=world   # => POST
+
+    """
+)
+positional.add_argument(
+    'url',
+    metavar='URL',
+    help="""
+    The scheme defaults to 'http://' if the URL does not include one.
+
+    You can also use a shorthand for localhost
+
+        $ http :3000                    # => http://localhost:3000
+        $ http :/foo                    # => http://localhost/foo
+
+    """
+)
+positional.add_argument(
+    'items',
+    metavar='REQUEST_ITEM',
+    nargs=ZERO_OR_MORE,
+    type=KeyValueArgType(*SEP_GROUP_ALL_ITEMS),
+    help=r"""
+    Optional key-value pairs to be included in the request. The separator used
+    determines the type:
+
+    ':' HTTP headers:
+
+        Referer:http://httpie.org  Cookie:foo=bar  User-Agent:bacon/1.0
+
+    '==' URL parameters to be appended to the request URI:
+
+        search==httpie
+
+    '=' Data fields to be serialized into a JSON object (with --json, -j)
+        or form data (with --form, -f):
+
+        name=HTTPie  language=Python  description='CLI HTTP client'
+
+    ':=' Non-string JSON data fields (only with --json, -j):
+
+        awesome:=true  amount:=42  colors:='["red", "green", "blue"]'
+
+    '@' Form file fields (only with --form, -f):
+
+        cs@~/Documents/CV.pdf
+
+    '=@' A data field like '=', but takes a file path and embeds its content:
+
+         essay=@Documents/essay.txt
+
+    ':=@' A raw JSON field like ':=', but takes a file path and embeds its content:
+
+        package:=@./package.json
+
+    You can use a backslash to escape a colliding separator in the field name:
+
+        field-name-with\:colon=value
+
+    """
+)
+
+
+#######################################################################
 # Content type.
-#############################################
+#######################################################################
 
-group_type = parser.add_mutually_exclusive_group(required=False)
-group_type.add_argument(
-    '--json', '-j', action='store_true',
-    help=_('''
-        (default) Data items from the command
-        line are serialized as a JSON object.
-        The Content-Type and Accept headers
-        are set to application/json (if not specified).
-    ''')
+content_type = parser.add_argument_group(
+    title='Predefined content types',
+    description=None
 )
-group_type.add_argument(
-    '--form', '-f', action='store_true',
-    help=_('''
-        Data items from the command line are serialized as form fields.
-        The Content-Type is set to application/x-www-form-urlencoded
-        (if not specified).
-        The presence of any file fields results
-        into a multipart/form-data request.
-     ''')
+
+content_type.add_argument(
+    '--json', '-j',
+    action='store_true',
+    help="""
+    (default) Data items from the command line are serialized as a JSON object.
+    The Content-Type and Accept headers are set to application/json
+    (if not specified).
+
+    """
+)
+content_type.add_argument(
+    '--form', '-f',
+    action='store_true',
+    help="""
+    Data items from the command line are serialized as form fields.
+
+    The Content-Type is set to application/x-www-form-urlencoded (if not
+    specified). The presence of any file fields results in a
+    multipart/form-data request.
+
+    """
 )
 
 
-# Output options.
-#############################################
+#######################################################################
+# Output processing
+#######################################################################
 
-parser.add_argument(
-    '--traceback', action='store_true', default=False,
-    help=_('''
-        Print exception traceback should one occur.
-    ''')
-)
+output_processing = parser.add_argument_group(title='Output processing')
 
-prettify = parser.add_mutually_exclusive_group(required=False)
-prettify.add_argument(
-    '--pretty', dest='prettify', action='store_true',
-    default=PRETTIFY_STDOUT_TTY_ONLY,
-    help=_('''
-        If stdout is a terminal, the response is prettified
-        by default (colorized and indented if it is JSON).
-        This flag ensures prettifying even when stdout is redirected.
-    ''')
-)
-prettify.add_argument(
-    '--ugly', '-u', dest='prettify', action='store_false',
-    help=_('''
-        Do not prettify the response.
-    ''')
-)
+output_processing.add_argument(
+    '--pretty',
+    dest='prettify',
+    default=PRETTY_STDOUT_TTY_ONLY,
+    choices=sorted(PRETTY_MAP.keys()),
+    help="""
+    Controls output processing. The value can be "none" to not prettify
+    the output (default for redirected output), "all" to apply both colors
+    and formatting (default for terminal output), "colors", or "format".
 
-output_options = parser.add_mutually_exclusive_group(required=False)
-output_options.add_argument('--print', '-p', dest='output_options',
-    help=_('''
-        String specifying what the output should contain:
-        "{request_headers}" stands for the request headers,  and
-        "{request_body}" for the request body.
-        "{response_headers}" stands for the response headers and
-        "{response_body}" for response the body.
-        The default behaviour is "hb" (i.e., the response
-        headers and body is printed), if standard output is not redirected.
-        If the output is piped to another program or to a file,
-        then only the body is printed by default.
-    '''.format(
-        request_headers=OUT_REQ_HEAD,
-        request_body=OUT_REQ_BODY,
-        response_headers=OUT_RESP_HEAD,
-        response_body=OUT_RESP_BODY,
-    ))
+    """
 )
-output_options.add_argument(
-    '--verbose', '-v', dest='output_options',
-    action='store_const', const=''.join(OUTPUT_OPTIONS),
-    help=_('''
-        Print the whole request as well as the response.
-        Shortcut for --print={0}.
-    '''.format(''.join(OUTPUT_OPTIONS)))
-)
-output_options.add_argument(
-    '--headers', '-h', dest='output_options',
-    action='store_const', const=OUT_RESP_HEAD,
-    help=_('''
-        Print only the response headers.
-        Shortcut for --print={0}.
-    '''.format(OUT_RESP_HEAD))
-)
-output_options.add_argument(
-    '--body', '-b', dest='output_options',
-    action='store_const', const=OUT_RESP_BODY,
-    help=_('''
-        Print only the response body.
-        Shortcut for --print={0}.
-    '''.format(OUT_RESP_BODY))
-)
-
-parser.add_argument(
-    '--style', '-s', dest='style', default='solarized', metavar='STYLE',
+output_processing.add_argument(
+    '--style', '-s',
+    dest='style',
+    metavar='STYLE',
+    default=DEFAULT_STYLE,
     choices=AVAILABLE_STYLES,
-    help=_('''
-        Output coloring style, one of %s. Defaults to solarized.
-        For this option to work properly, please make sure that the
-        $TERM environment variable is set to "xterm-256color" or similar
-        (e.g., via `export TERM=xterm-256color' in your ~/.bashrc).
-    ''') % ', '.join(sorted(AVAILABLE_STYLES))
+    help="""
+    Output coloring style (default is "{default}"). One of:
+
+{available}
+
+    For this option to work properly, please make sure that the $TERM
+    environment variable is set to "xterm-256color" or similar
+    (e.g., via `export TERM=xterm-256color' in your ~/.bashrc).
+
+    """
+    .format(
+        default=DEFAULT_STYLE,
+        available='\n'.join(
+            '{0: >20}'.format(line.strip())
+            for line in
+            wrap(' '.join(sorted(AVAILABLE_STYLES)), 60)
+        ),
+    )
 )
 
-parser.add_argument(
-    '--check-status', default=False, action='store_true',
-    help=_('''
-        By default, HTTPie exits with 0 when no network or other fatal
-        errors occur.
 
-        This flag instructs HTTPie to also check the HTTP status code and
-        exit with an error if the status indicates one.
+#######################################################################
+# Output options
+#######################################################################
+output_options = parser.add_argument_group(title='Output options')
 
-        When the server replies with a 4xx (Client Error) or 5xx
-        (Server Error) status code, HTTPie exits with 4 or 5 respectively.
-        If the response is a 3xx (Redirect) and --allow-redirects
-        hasn't been set, then the exit status is 3.
+output_options.add_argument(
+    '--print', '-p',
+    dest='output_options',
+    metavar='WHAT',
+    help="""
+    String specifying what the output should contain:
 
-        Also an error message is written to stderr if stdout is redirected.
+        '{req_head}' request headers
+        '{req_body}' request body
+        '{res_head}' response headers
+        '{res_body}' response body
 
-    ''')
+    The default behaviour is '{default}' (i.e., the response headers and body
+    is printed), if standard output is not redirected. If the output is piped
+    to another program or to a file, then only the response body is printed
+    by default.
+
+    """
+    .format(
+        req_head=OUT_REQ_HEAD,
+        req_body=OUT_REQ_BODY,
+        res_head=OUT_RESP_HEAD,
+        res_body=OUT_RESP_BODY,
+        default=OUTPUT_OPTIONS_DEFAULT,
+    )
 )
+output_options.add_argument(
+    '--verbose', '-v',
+    dest='output_options',
+    action='store_const',
+    const=''.join(OUTPUT_OPTIONS),
+    help="""
+    Print the whole request as well as the response. Shortcut for --print={0}.
+
+    """
+    .format(''.join(OUTPUT_OPTIONS))
+)
+output_options.add_argument(
+    '--headers', '-h',
+    dest='output_options',
+    action='store_const',
+    const=OUT_RESP_HEAD,
+    help="""
+    Print only the response headers. Shortcut for --print={0}.
+
+    """
+    .format(OUT_RESP_HEAD)
+)
+output_options.add_argument(
+    '--body', '-b',
+    dest='output_options',
+    action='store_const',
+    const=OUT_RESP_BODY,
+    help="""
+    Print only the response body. Shortcut for --print={0}.
+
+    """
+    .format(OUT_RESP_BODY)
+)
+
+output_options.add_argument(
+    '--stream', '-S',
+    action='store_true',
+    default=False,
+    help="""
+    Always stream the output by line, i.e., behave like `tail -f'.
+
+    Without --stream and with --pretty (either set or implied),
+    HTTPie fetches the whole response before it outputs the processed data.
+
+    Set this option when you want to continuously display a prettified
+    long-lived response, such as one from the Twitter streaming API.
+
+    It is useful also without --pretty: It ensures that the output is flushed
+    more often and in smaller chunks.
+
+    """
+)
+output_options.add_argument(
+    '--output', '-o',
+    type=FileType('a+b'),
+    dest='output_file',
+    metavar='FILE',
+    help="""
+    Save output to FILE. If --download is set, then only the response body is
+    saved to the file. Other parts of the HTTP exchange are printed to stderr.
+
+    """
+
+)
+
+output_options.add_argument(
+    '--download', '-d',
+    action='store_true',
+    default=False,
+    help="""
+    Do not print the response body to stdout. Rather, download it and store it
+    in a file. The filename is guessed unless specified with --output
+    [filename]. This action is similar to the default behaviour of wget.
+
+    """
+)
+
+output_options.add_argument(
+    '--continue', '-c',
+    dest='download_resume',
+    action='store_true',
+    default=False,
+    help="""
+    Resume an interrupted download. Note that the --output option needs to be
+    specified as well.
+
+    """
+)
+
+
+#######################################################################
+# Sessions
+#######################################################################
+
+sessions = parser.add_argument_group(title='Sessions')\
+                 .add_mutually_exclusive_group(required=False)
+
+session_name_validator = SessionNameValidator(
+    'Session name contains invalid characters.'
+)
+
+sessions.add_argument(
+    '--session',
+    metavar='SESSION_NAME_OR_PATH',
+    type=session_name_validator,
+    help="""
+    Create, or reuse and update a session. Within a session, custom headers,
+    auth credential, as well as any cookies sent by the server persist between
+    requests.
+
+    Session files are stored in:
+
+        {session_dir}/<HOST>/<SESSION_NAME>.json.
+
+    """
+    .format(session_dir=DEFAULT_SESSIONS_DIR)
+)
+sessions.add_argument(
+    '--session-read-only',
+    metavar='SESSION_NAME_OR_PATH',
+    type=session_name_validator,
+    help="""
+    Create or read a session without updating it form the request/response
+    exchange.
+
+    """
+)
+
+
+#######################################################################
+# Authentication
+#######################################################################
 
 # ``requests.request`` keyword arguments.
-parser.add_argument(
+auth = parser.add_argument_group(title='Authentication')
+auth.add_argument(
     '--auth', '-a',
+    metavar='USER[:PASS]',
     type=AuthCredentialsArgType(SEP_CREDENTIALS),
-    help=_('''
-        username:password.
-        If only the username is provided (-a username),
-        HTTPie will prompt for the password.
-    '''),
+    help="""
+    If only the username is provided (-a username), HTTPie will prompt
+    for the password.
+
+    """,
 )
 
-parser.add_argument(
-    '--auth-type', choices=['basic', 'digest'], default='basic',
-    help=_('''
-        The authentication mechanism to be used.
-        Defaults to "basic".
-    ''')
+_auth_plugins = plugin_manager.get_auth_plugins()
+auth.add_argument(
+    '--auth-type',
+    choices=[plugin.auth_type for plugin in _auth_plugins],
+    default=_auth_plugins[0].auth_type,
+    help="""
+    The authentication mechanism to be used. Defaults to "{default}".
+
+    {types}
+
+    """
+    .format(default=_auth_plugins[0].auth_type, types='\n    '.join(
+        '"{type}": {name}{package}{description}'.format(
+            type=plugin.auth_type,
+            name=plugin.name,
+            package=(
+                '' if issubclass(plugin, BuiltinAuthPlugin)
+                else ' (provided by %s)' % plugin.package_name
+            ),
+            description=(
+                '' if not plugin.description else
+                '\n      ' + ('\n      '.join(wrap(plugin.description)))
+            )
+        )
+        for plugin in _auth_plugins
+    )),
 )
 
-parser.add_argument(
-    '--verify', default='yes',
-    help=_('''
-        Set to "no" to skip checking the host\'s SSL certificate.
-        You can also pass the  path to a CA_BUNDLE
-        file for private certs. You can also set
-        the REQUESTS_CA_BUNDLE  environment variable.
-        Defaults to "yes".
-    ''')
-)
-parser.add_argument(
-    '--proxy', default=[], action='append',
+
+#######################################################################
+# Network
+#######################################################################
+
+network = parser.add_argument_group(title='Network')
+
+network.add_argument(
+    '--proxy',
+    default=[],
+    action='append',
+    metavar='PROTOCOL:PROXY_URL',
     type=KeyValueArgType(SEP_PROXY),
-    help=_('''
-        String mapping protocol to the URL of the proxy
-        (e.g. http:foo.bar:3128).
-    ''')
+    help="""
+    String mapping protocol to the URL of the proxy
+    (e.g. http:http://foo.bar:3128). You can specify multiple proxies with
+    different protocols.
+
+    """
 )
-parser.add_argument(
-    '--allow-redirects', default=False, action='store_true',
-    help=_('''
-        Set this flag if full redirects are allowed
-        (e.g. re-POST-ing of data at new ``Location``)
-    ''')
+network.add_argument(
+    '--follow',
+    default=False,
+    action='store_true',
+    help="""
+    Set this flag if full redirects are allowed (e.g. re-POST-ing of data at
+    new Location).
+
+    """
 )
-parser.add_argument(
-    '--timeout', type=float,
-    help=_('''
-        Float describes the timeout of the request
-        (Use socket.setdefaulttimeout() as fallback).
-    ''')
+network.add_argument(
+    '--verify',
+    default='yes',
+    help="""
+    Set to "no" to skip checking the host's SSL certificate. You can also pass
+    the  path to a CA_BUNDLE file for private certs. You can also set the
+    REQUESTS_CA_BUNDLE  environment variable. Defaults to "yes".
+
+    """
+)
+
+network.add_argument(
+    '--timeout',
+    type=float,
+    default=30,
+    metavar='SECONDS',
+    help="""
+    The connection timeout of the request in seconds. The default value is
+    30 seconds.
+
+    """
+)
+network.add_argument(
+    '--check-status',
+    default=False,
+    action='store_true',
+    help="""
+    By default, HTTPie exits with 0 when no network or other fatal errors
+    occur. This flag instructs HTTPie to also check the HTTP status code and
+    exit with an error if the status indicates one.
+
+    When the server replies with a 4xx (Client Error) or 5xx (Server Error)
+    status code, HTTPie exits with 4 or 5 respectively. If the response is a
+    3xx (Redirect) and --follow hasn't been set, then the exit status is 3.
+    Also an error message is written to stderr if stdout is redirected.
+
+    """
 )
 
 
-# Positional arguments.
-#############################################
+#######################################################################
+# Troubleshooting
+#######################################################################
 
-parser.add_argument(
-    'method', metavar='METHOD',
-    nargs='?',
-    default=None,
-    help=_('''
-        The HTTP method to be used for the request
-        (GET, POST, PUT, DELETE, PATCH, ...).
-        If this argument is omitted, then HTTPie
-        will guess the HTTP method. If there is some
-        data to be sent, then it will be POST, otherwise GET.
-    ''')
+troubleshooting = parser.add_argument_group(title='Troubleshooting')
+
+troubleshooting.add_argument(
+    '--ignore-stdin',
+    action='store_true',
+    default=False,
+    help="""
+    Do not attempt to read stdin.
+
+    """
 )
-parser.add_argument(
-    'url', metavar='URL',
-    help=_('''
-        The protocol defaults to http:// if the
-        URL does not include one.
-    ''')
+troubleshooting.add_argument(
+    '--help',
+    action='help',
+    default=SUPPRESS,
+    help="""
+    Show this help message and exit.
+
+    """
 )
-parser.add_argument(
-    'items', nargs='*',
-    metavar='ITEM',
-    type=KeyValueArgType(*SEP_GROUP_ITEMS),
-    help=_('''
-        A key-value pair whose type is defined by the
-        separator used. It can be an HTTP header (header:value),
-        a data field to be used in the request body (field_name=value),
-        a raw JSON data field (field_name:=value),
-        a query parameter (name==value),
-        or a file field (field_name@/path/to/file).
-        You can use a backslash to escape a colliding
-        separator in the field name.
-    ''')
+troubleshooting.add_argument(
+    '--version',
+    action='version',
+    version=__version__,
+    help="""
+    Show version and exit.
+
+    """
+)
+troubleshooting.add_argument(
+    '--traceback',
+    action='store_true',
+    default=False,
+    help="""
+    Prints exception traceback should one occur.
+
+    """
+)
+troubleshooting.add_argument(
+    '--debug',
+    action='store_true',
+    default=False,
+    help="""
+    Prints exception traceback should one occur, and also other information
+    that is useful for debugging HTTPie itself and for reporting bugs.
+
+    """
 )

httpie/client.py

@@ -0,0 +1,94 @@
+import json
+import sys
+from pprint import pformat
+
+import requests
+
+from . import sessions
+from . import __version__
+from .plugins import plugin_manager
+
+
+FORM = 'application/x-www-form-urlencoded; charset=utf-8'
+JSON = 'application/json; charset=utf-8'
+DEFAULT_UA = 'HTTPie/%s' % __version__
+
+
+def get_response(args, config_dir):
+    """Send the request and return a `request.Response`."""
+
+    requests_kwargs = get_requests_kwargs(args)
+
+    if args.debug:
+        sys.stderr.write('\n>>> requests.request(%s)\n\n'
+                         % pformat(requests_kwargs))
+
+    if not args.session and not args.session_read_only:
+        response = requests.request(**requests_kwargs)
+    else:
+        response = sessions.get_response(
+            args=args,
+            config_dir=config_dir,
+            session_name=args.session or args.session_read_only,
+            requests_kwargs=requests_kwargs,
+            read_only=bool(args.session_read_only),
+        )
+
+    return response
+
+
+def get_requests_kwargs(args):
+    """Translate our `args` into `requests.request` keyword arguments."""
+
+    implicit_headers = {
+        'User-Agent': DEFAULT_UA
+    }
+
+    auto_json = args.data and not args.form
+    # FIXME: Accept is set to JSON with `http url @./file.txt`.
+    if args.json or auto_json:
+        implicit_headers['Accept'] = 'application/json'
+        if args.json or (auto_json and args.data):
+            implicit_headers['Content-Type'] = JSON
+
+        if isinstance(args.data, dict):
+            if args.data:
+                args.data = json.dumps(args.data)
+            else:
+                # We need to set data to an empty string to prevent requests
+                # from assigning an empty list to `response.request.data`.
+                args.data = ''
+
+    elif args.form and not args.files:
+        # If sending files, `requests` will set
+        # the `Content-Type` for us.
+        implicit_headers['Content-Type'] = FORM
+
+    for name, value in implicit_headers.items():
+        if name not in args.headers:
+            args.headers[name] = value
+
+    credentials = None
+    if args.auth:
+        auth_plugin = plugin_manager.get_auth_plugin(args.auth_type)()
+        credentials = auth_plugin.get_auth(args.auth.key, args.auth.value)
+
+    kwargs = {
+        'stream': True,
+        'method': args.method.lower(),
+        'url': args.url,
+        'headers': args.headers,
+        'data': args.data,
+        'verify': {
+            'yes': True,
+            'no': False
+        }.get(args.verify, args.verify),
+        'timeout': args.timeout,
+        'auth': credentials,
+        'proxies': dict((p.key, p.value) for p in args.proxy),
+        'files': args.files,
+        'allow_redirects': args.follow,
+        'params': args.params,
+    }
+
+    return kwargs

httpie/compat.py

@@ -0,0 +1,19 @@
+"""
+Python 2/3 compatibility.
+
+"""
+#noinspection PyUnresolvedReferences
+from requests.compat import (
+    is_windows,
+    bytes,
+    str,
+    is_py3,
+    is_py26,
+)
+
+try:
+    #noinspection PyUnresolvedReferences,PyCompatibility
+    from urllib.parse import urlsplit
+except ImportError:
+    #noinspection PyUnresolvedReferences,PyCompatibility
+    from urlparse import urlsplit

httpie/config.py

@@ -0,0 +1,100 @@
+import os
+import json
+import errno
+
+from . import __version__
+from .compat import is_windows
+
+
+DEFAULT_CONFIG_DIR = os.environ.get(
+    'HTTPIE_CONFIG_DIR',
+    os.path.expanduser('~/.httpie') if not is_windows else
+    os.path.expandvars(r'%APPDATA%\\httpie')
+)
+
+
+class BaseConfigDict(dict):
+
+    name = None
+    helpurl = None
+    about = None
+    directory = DEFAULT_CONFIG_DIR
+
+    def __init__(self, directory=None, *args, **kwargs):
+        super(BaseConfigDict, self).__init__(*args, **kwargs)
+        if directory:
+            self.directory = directory
+
+    def __getattr__(self, item):
+        return self[item]
+
+    def _get_path(self):
+        """Return the config file path without side-effects."""
+        return os.path.join(self.directory, self.name + '.json')
+
+    @property
+    def path(self):
+        """Return the config file path creating basedir, if needed."""
+        path = self._get_path()
+        try:
+            os.makedirs(os.path.dirname(path), mode=0o700)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        return path
+
+    @property
+    def is_new(self):
+        return not os.path.exists(self._get_path())
+
+    def load(self):
+        try:
+            with open(self.path, 'rt') as f:
+                try:
+                    data = json.load(f)
+                except ValueError as e:
+                    raise ValueError(
+                        'Invalid %s JSON: %s [%s]' %
+                        (type(self).__name__, e.message, self.path)
+                    )
+                self.update(data)
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    def save(self):
+        self['__meta__'] = {
+            'httpie': __version__
+        }
+        if self.helpurl:
+            self['__meta__']['help'] = self.helpurl
+
+        if self.about:
+            self['__meta__']['about'] = self.about
+
+        with open(self.path, 'w') as f:
+            json.dump(self, f, indent=4, sort_keys=True, ensure_ascii=True)
+            f.write('\n')
+
+    def delete(self):
+        try:
+            os.unlink(self.path)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+
+class Config(BaseConfigDict):
+
+    name = 'config'
+    helpurl = 'https://github.com/jkbr/httpie#config'
+    about = 'HTTPie configuration file'
+
+    DEFAULTS = {
+        'implicit_content_type': 'json',
+        'default_options': []
+    }
+
+    def __init__(self, *args, **kwargs):
+        super(Config, self).__init__(*args, **kwargs)
+        self.update(self.DEFAULTS)

httpie/core.py

@@ -3,169 +3,169 @@
 Invocation flow:
 
     1. Read, validate and process the input (args, `stdin`).
-    2. Create a request and send it, get the response.
-    3. Process and format the requested parts of the request-response exchange.
-    4. Write to `stdout` and exit.
+    2. Create and send a request.
+    3. Stream, and possibly process and format, the requested parts
+       of the request-response exchange.
+    4. Simultaneously write to `stdout`
+    5. Exit.
 
 """
 import sys
-import json
+import errno
 
 import requests
-import requests.auth
-from requests.compat import str
+from httpie import __version__ as httpie_version
+from requests import __version__ as requests_version
+from pygments import __version__ as pygments_version
 
-from .models import HTTPMessage, Environment
-from .output import OutputProcessor
-from .input import (PRETTIFY_STDOUT_TTY_ONLY,
-                    OUT_REQ_BODY, OUT_REQ_HEAD,
-                    OUT_RESP_HEAD, OUT_RESP_BODY)
-from .cli import parser
+from .compat import str, is_py3
+from .client import get_response
+from .downloads import Download
+from .models import Environment
+from .output import build_output_stream, write, write_with_colors_win_py3
+from . import ExitStatus
+from .plugins import plugin_manager
 
 
-TYPE_FORM = 'application/x-www-form-urlencoded; charset=utf-8'
-TYPE_JSON = 'application/json; charset=utf-8'
-
-
-def get_response(args, env):
-    """Send the request and return a `request.Response`."""
-
-    auto_json = args.data and not args.form
-    if args.json or auto_json:
-        if 'Content-Type' not in args.headers:
-            args.headers['Content-Type'] = TYPE_JSON
-
-        if 'Accept' not in args.headers:
-            # Default Accept to JSON as well.
-            args.headers['Accept'] = 'application/json'
-
-        if isinstance(args.data, dict):
-            # If not empty, serialize the data `dict` parsed from arguments.
-            # Otherwise set it to `None` avoid sending "{}".
-            args.data = json.dumps(args.data) if args.data else None
-
-    elif args.form:
-        if not args.files and 'Content-Type' not in args.headers:
-            # If sending files, `requests` will set
-            # the `Content-Type` for us.
-            args.headers['Content-Type'] = TYPE_FORM
-
-    try:
-        credentials = None
-        if args.auth:
-            credentials = {
-                'basic': requests.auth.HTTPBasicAuth,
-                'digest': requests.auth.HTTPDigestAuth,
-            }[args.auth_type](args.auth.key, args.auth.value)
-
-        return requests.request(
-            method=args.method.lower(),
-            url=args.url if '://' in args.url else 'http://%s' % args.url,
-            headers=args.headers,
-            data=args.data,
-            verify={'yes': True, 'no': False}.get(args.verify, args.verify),
-            timeout=args.timeout,
-            auth=credentials,
-            proxies=dict((p.key, p.value) for p in args.proxy),
-            files=args.files,
-            allow_redirects=args.allow_redirects,
-            params=args.params,
-        )
-
-    except (KeyboardInterrupt, SystemExit):
-        env.stderr.write('\n')
-        sys.exit(1)
-    except Exception as e:
-        if args.traceback:
-            raise
-        env.stderr.write(str(e.message) + '\n')
-        sys.exit(1)
-
-
-def get_output(args, env, request, response):
-    """Format parts of the `request`-`response` exchange
-     according to `args` and `env` and return a `unicode`.
-
-    """
-    do_prettify = (args.prettify is True
-                   or (args.prettify == PRETTIFY_STDOUT_TTY_ONLY
-                       and env.stdout_isatty))
-
-    do_output_request = (OUT_REQ_HEAD in args.output_options
-                         or OUT_REQ_BODY in args.output_options)
-
-    do_output_response = (OUT_RESP_HEAD in args.output_options
-                          or OUT_RESP_BODY in args.output_options)
-
-    prettifier = None
-    if do_prettify:
-        prettifier = OutputProcessor(
-            env, pygments_style=args.style)
-
-    buf = []
-
-    if do_output_request:
-        req_msg = HTTPMessage.from_request(request)
-        req = req_msg.format(
-            prettifier=prettifier,
-            with_headers=OUT_REQ_HEAD in args.output_options,
-            with_body=OUT_REQ_BODY in args.output_options
-        )
-        buf.append(req)
-        buf.append('\n')
-        if do_output_response:
-            buf.append('\n')
-
-    if do_output_response:
-        resp_msg = HTTPMessage.from_response(response)
-        resp = resp_msg.format(
-            prettifier=prettifier,
-            with_headers=OUT_RESP_HEAD in args.output_options,
-            with_body=OUT_RESP_BODY in args.output_options
-        )
-        buf.append(resp)
-        buf.append('\n')
-
-    return ''.join(buf)
-
-
-def get_exist_status(code, allow_redirects=False):
-    """Translate HTTP status code to exit status."""
-    if 300 <= code <= 399 and not allow_redirects:
+def get_exit_status(http_status, follow=False):
+    """Translate HTTP status code to exit status code."""
+    if 300 <= http_status <= 399 and not follow:
         # Redirect
-        return 3
-    elif 400 <= code <= 499:
+        return ExitStatus.ERROR_HTTP_3XX
+    elif 400 <= http_status <= 499:
         # Client Error
-        return 4
-    elif 500 <= code <= 599:
+        return ExitStatus.ERROR_HTTP_4XX
+    elif 500 <= http_status <= 599:
         # Server Error
-        return 5
+        return ExitStatus.ERROR_HTTP_5XX
     else:
-        return 0
+        return ExitStatus.OK
+
+
+def print_debug_info(env):
+    sys.stderr.writelines([
+        'HTTPie %s\n' % httpie_version,
+        'HTTPie data: %s\n' % env.config.directory,
+        'Requests %s\n' % requests_version,
+        'Pygments %s\n' % pygments_version,
+        'Python %s %s\n' % (sys.version, sys.platform)
+    ])
 
 
 def main(args=sys.argv[1:], env=Environment()):
     """Run the main program and write the output to ``env.stdout``.
 
-    Return exit status.
+    Return exit status code.
 
     """
-    args = parser.parse_args(args=args, env=env)
-    response = get_response(args, env)
+    plugin_manager.load_installed_plugins()
+    from .cli import parser
 
-    status = 0
+    if env.config.default_options:
+        args = env.config.default_options + args
 
-    if args.check_status:
-        status = get_exist_status(response.status_code,
-                                  args.allow_redirects)
-        if status and not env.stdout_isatty:
-            err = 'http error: %s %s\n' % (
-                response.raw.status, response.raw.reason)
-            env.stderr.write(err.encode('utf8'))
+    def error(msg, *args, **kwargs):
+        msg = msg % args
+        level = kwargs.get('level', 'error')
+        env.stderr.write('\nhttp: %s: %s\n' % (level, msg))
 
-    output = get_output(args, env, response.request, response)
-    output_bytes = output.encode('utf8')
-    f = getattr(env.stdout, 'buffer', env.stdout)
-    f.write(output_bytes)
+    debug = '--debug' in args
+    traceback = debug or '--traceback' in args
+    exit_status = ExitStatus.OK
 
-    return status
+    if debug:
+        print_debug_info(env)
+        if args == ['--debug']:
+            return exit_status
+
+    download = None
+
+    try:
+        args = parser.parse_args(args=args, env=env)
+
+        if args.download:
+            args.follow = True  # --download implies --follow.
+            download = Download(
+                output_file=args.output_file,
+                progress_file=env.stderr,
+                resume=args.download_resume
+            )
+            download.pre_request(args.headers)
+
+        response = get_response(args, config_dir=env.config.directory)
+
+        if args.check_status or download:
+
+            exit_status = get_exit_status(
+                http_status=response.status_code,
+                follow=args.follow
+            )
+
+            if not env.stdout_isatty and exit_status != ExitStatus.OK:
+                error('HTTP %s %s',
+                      response.raw.status,
+                      response.raw.reason,
+                      level='warning')
+
+        write_kwargs = {
+            'stream': build_output_stream(
+                args, env, response.request, response),
+
+            # This will in fact be `stderr` with `--download`
+            'outfile': env.stdout,
+
+            'flush': env.stdout_isatty or args.stream
+        }
+
+        try:
+
+            if env.is_windows and is_py3 and 'colors' in args.prettify:
+                write_with_colors_win_py3(**write_kwargs)
+            else:
+                write(**write_kwargs)
+
+            if download and exit_status == ExitStatus.OK:
+                # Response body download.
+                download_stream, download_to = download.start(response)
+                write(
+                    stream=download_stream,
+                    outfile=download_to,
+                    flush=False,
+                )
+                download.finish()
+                if download.interrupted:
+                    exit_status = ExitStatus.ERROR
+                    error('Incomplete download: size=%d; downloaded=%d' % (
+                        download.status.total_size,
+                        download.status.downloaded
+                    ))
+
+        except IOError as e:
+            if not traceback and e.errno == errno.EPIPE:
+                # Ignore broken pipes unless --traceback.
+                env.stderr.write('\n')
+            else:
+                raise
+    except (KeyboardInterrupt, SystemExit):
+        if traceback:
+            raise
+        env.stderr.write('\n')
+        exit_status = ExitStatus.ERROR
+
+    except requests.Timeout:
+        exit_status = ExitStatus.ERROR_TIMEOUT
+        error('Request timed out (%ss).', args.timeout)
+
+    except Exception as e:
+        # TODO: Better distinction between expected and unexpected errors.
+        #       Network errors vs. bugs, etc.
+        if traceback:
+            raise
+        error('%s: %s', type(e).__name__, str(e))
+        exit_status = ExitStatus.ERROR
+
+    finally:
+        if download and not download.finished:
+            download.failed()
+
+    return exit_status

httpie/downloads.py

@@ -0,0 +1,427 @@
+# coding=utf-8
+"""
+Download mode implementation.
+
+"""
+from __future__ import division
+import os
+import re
+import sys
+import mimetypes
+import threading
+from time import sleep, time
+from mailbox import Message
+
+from .output import RawStream
+from .models import HTTPResponse
+from .utils import humanize_bytes
+from .compat import urlsplit
+
+
+PARTIAL_CONTENT = 206
+
+
+CLEAR_LINE = '\r\033[K'
+PROGRESS = (
+    '{percentage: 6.2f} %'
+    ' {downloaded: >10}'
+    ' {speed: >10}/s'
+    ' {eta: >8} ETA'
+)
+PROGRESS_NO_CONTENT_LENGTH = '{downloaded: >10} {speed: >10}/s'
+SUMMARY = 'Done. {downloaded} in {time:0.5f}s ({speed}/s)\n'
+SPINNER = '|/-\\'
+
+
+class ContentRangeError(ValueError):
+    pass
+
+
+def parse_content_range(content_range, resumed_from):
+    """
+    Parse and validate Content-Range header.
+
+    <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>
+
+    :param content_range: the value of a Content-Range response header
+                          eg. "bytes 21010-47021/47022"
+    :param resumed_from: first byte pos. from the Range request header
+    :return: total size of the response body when fully downloaded.
+
+    """
+    if content_range is None:
+        raise ContentRangeError('Missing Content-Range')
+
+    pattern = (
+        '^bytes (?P<first_byte_pos>\d+)-(?P<last_byte_pos>\d+)'
+        '/(\*|(?P<instance_length>\d+))$'
+    )
+    match = re.match(pattern, content_range)
+
+    if not match:
+        raise ContentRangeError(
+            'Invalid Content-Range format %r' % content_range)
+
+    content_range_dict = match.groupdict()
+    first_byte_pos = int(content_range_dict['first_byte_pos'])
+    last_byte_pos = int(content_range_dict['last_byte_pos'])
+    instance_length = (
+        int(content_range_dict['instance_length'])
+        if content_range_dict['instance_length']
+        else None
+    )
+
+    # "A byte-content-range-spec with a byte-range-resp-spec whose
+    # last- byte-pos value is less than its first-byte-pos value,
+    # or whose instance-length value is less than or equal to its
+    # last-byte-pos value, is invalid. The recipient of an invalid
+    # byte-content-range- spec MUST ignore it and any content
+    # transferred along with it."
+    if (first_byte_pos >= last_byte_pos
+            or (instance_length is not None
+                and instance_length <= last_byte_pos)):
+        raise ContentRangeError(
+            'Invalid Content-Range returned: %r' % content_range)
+
+    if (first_byte_pos != resumed_from
+        or (instance_length is not None
+            and last_byte_pos + 1 != instance_length)):
+        # Not what we asked for.
+        raise ContentRangeError(
+            'Unexpected Content-Range returned (%r)'
+            ' for the requested Range ("bytes=%d-")'
+            % (content_range, resumed_from)
+        )
+
+    return last_byte_pos + 1
+
+
+def filename_from_content_disposition(content_disposition):
+    """
+    Extract and validate filename from a Content-Disposition header.
+
+    :param content_disposition: Content-Disposition value
+    :return: the filename if present and valid, otherwise `None`
+
+    """
+    # attachment; filename=jkbr-httpie-0.4.1-20-g40bd8f6.tar.gz
+
+    msg = Message('Content-Disposition: %s' % content_disposition)
+    filename = msg.get_filename()
+    if filename:
+        # Basic sanitation.
+        filename = os.path.basename(filename).lstrip('.').strip()
+        if filename:
+            return filename
+
+
+def filename_from_url(url, content_type):
+    fn = urlsplit(url).path.rstrip('/')
+    fn = os.path.basename(fn) if fn else 'index'
+    if '.' not in fn and content_type:
+        content_type = content_type.split(';')[0]
+        if content_type == 'text/plain':
+            # mimetypes returns '.ksh'
+            ext = '.txt'
+        else:
+            ext = mimetypes.guess_extension(content_type)
+
+        if ext == '.htm':  # Python 3
+            ext = '.html'
+
+        if ext:
+            fn += ext
+
+    return fn
+
+
+def get_unique_filename(fn, exists=os.path.exists):
+    attempt = 0
+    while True:
+        suffix = '-' + str(attempt) if attempt > 0 else ''
+        if not exists(fn + suffix):
+            return fn + suffix
+        attempt += 1
+
+
+class Download(object):
+
+    def __init__(self, output_file=None,
+                 resume=False, progress_file=sys.stderr):
+        """
+        :param resume: Should the download resume if partial download
+                       already exists.
+        :type resume: bool
+
+        :param output_file: The file to store response body in. If not
+                            provided, it will be guessed from the response.
+        :type output_file: file
+
+        :param progress_file: Where to report download progress.
+        :type progress_file: file
+
+        """
+        self._output_file = output_file
+        self._resume = resume
+        self._resumed_from = 0
+        self.finished = False
+
+        self.status = Status()
+        self._progress_reporter = ProgressReporterThread(
+            status=self.status,
+            output=progress_file
+        )
+
+    def pre_request(self, request_headers):
+        """Called just before the HTTP request is sent.
+
+        Might alter `request_headers`.
+
+        :type request_headers: dict
+
+        """
+        # Disable content encoding so that we can resume, etc.
+        request_headers['Accept-Encoding'] = None
+        if self._resume:
+            bytes_have = os.path.getsize(self._output_file.name)
+            if bytes_have:
+                # Set ``Range`` header to resume the download
+                # TODO: Use "If-Range: mtime" to make sure it's fresh?
+                request_headers['Range'] = 'bytes=%d-' % bytes_have
+                self._resumed_from = bytes_have
+
+    def start(self, response):
+        """
+        Initiate and return a stream for `response` body  with progress
+        callback attached. Can be called only once.
+
+        :param response: Initiated response object with headers already fetched
+        :type response: requests.models.Response
+
+        :return: RawStream, output_file
+
+        """
+        assert not self.status.time_started
+
+        try:
+            total_size = int(response.headers['Content-Length'])
+        except (KeyError, ValueError, TypeError):
+            total_size = None
+
+        if self._output_file:
+            if self._resume and response.status_code == PARTIAL_CONTENT:
+                total_size = parse_content_range(
+                    response.headers.get('Content-Range'),
+                    self._resumed_from
+                )
+
+            else:
+                self._resumed_from = 0
+                try:
+                    self._output_file.seek(0)
+                    self._output_file.truncate()
+                except IOError:
+                    pass  # stdout
+        else:
+            # TODO: Should the filename be taken from response.history[0].url?
+            # Output file not specified. Pick a name that doesn't exist yet.
+            fn = None
+            if 'Content-Disposition' in response.headers:
+                fn = filename_from_content_disposition(
+                    response.headers['Content-Disposition'])
+            if not fn:
+                fn = filename_from_url(
+                    url=response.url,
+                    content_type=response.headers.get('Content-Type'),
+                )
+            self._output_file = open(get_unique_filename(fn), mode='a+b')
+
+        self.status.started(
+            resumed_from=self._resumed_from,
+            total_size=total_size
+        )
+
+        stream = RawStream(
+            msg=HTTPResponse(response),
+            with_headers=False,
+            with_body=True,
+            on_body_chunk_downloaded=self.chunk_downloaded,
+            chunk_size=1024 * 8
+        )
+
+        self._progress_reporter.output.write(
+            'Downloading %sto "%s"\n' % (
+                (humanize_bytes(total_size) + ' '
+                 if total_size is not None
+                 else ''),
+                self._output_file.name
+            )
+        )
+        self._progress_reporter.start()
+
+        return stream, self._output_file
+
+    def finish(self):
+        assert not self.finished
+        self.finished = True
+        self.status.finished()
+
+    def failed(self):
+        self._progress_reporter.stop()
+
+    @property
+    def interrupted(self):
+        return (
+            self.finished
+            and self.status.total_size
+            and self.status.total_size != self.status.downloaded
+        )
+
+    def chunk_downloaded(self, chunk):
+        """
+        A download progress callback.
+
+        :param chunk: A chunk of response body data that has just
+                      been downloaded and written to the output.
+        :type chunk: bytes
+
+        """
+        self.status.chunk_downloaded(len(chunk))
+
+
+class Status(object):
+    """Holds details about the downland status."""
+
+    def __init__(self):
+        self.downloaded = 0
+        self.total_size = None
+        self.resumed_from = 0
+        self.time_started = None
+        self.time_finished = None
+
+    def started(self, resumed_from=0, total_size=None):
+        assert self.time_started is None
+        if total_size is not None:
+            self.total_size = total_size
+        self.downloaded = self.resumed_from = resumed_from
+        self.time_started = time()
+
+    def chunk_downloaded(self, size):
+        assert self.time_finished is None
+        self.downloaded += size
+
+    @property
+    def has_finished(self):
+        return self.time_finished is not None
+
+    def finished(self):
+        assert self.time_started is not None
+        assert self.time_finished is None
+        self.time_finished = time()
+
+
+class ProgressReporterThread(threading.Thread):
+    """
+    Reports download progress based on its status.
+
+    Uses threading to periodically update the status (speed, ETA, etc.).
+
+    """
+    def __init__(self, status, output, tick=.1, update_interval=1):
+        """
+
+        :type status: Status
+        :type output: file
+        """
+        super(ProgressReporterThread, self).__init__()
+        self.status = status
+        self.output = output
+        self._tick = tick
+        self._update_interval = update_interval
+        self._spinner_pos = 0
+        self._status_line = ''
+        self._prev_bytes = 0
+        self._prev_time = time()
+        self._should_stop = threading.Event()
+
+    def stop(self):
+        """Stop reporting on next tick."""
+        self._should_stop.set()
+
+    def run(self):
+        while not self._should_stop.is_set():
+            if self.status.has_finished:
+                self.sum_up()
+                break
+
+            self.report_speed()
+            sleep(self._tick)
+
+    def report_speed(self):
+
+        now = time()
+
+        if now - self._prev_time >= self._update_interval:
+            downloaded = self.status.downloaded
+            try:
+                speed = ((downloaded - self._prev_bytes)
+                         / (now - self._prev_time))
+            except ZeroDivisionError:
+                speed = 0
+
+            if not self.status.total_size:
+                self._status_line = PROGRESS_NO_CONTENT_LENGTH.format(
+                    downloaded=humanize_bytes(downloaded),
+                    speed=humanize_bytes(speed),
+                )
+            else:
+                try:
+                    percentage = downloaded / self.status.total_size * 100
+                except ZeroDivisionError:
+                    percentage = 0
+
+                if not speed:
+                    eta = '-:--:--'
+                else:
+                    s = int((self.status.total_size - downloaded) / speed)
+                    h, s = divmod(s, 60 * 60)
+                    m, s = divmod(s, 60)
+                    eta = '{0}:{1:0>2}:{2:0>2}'.format(h, m, s)
+
+                self._status_line = PROGRESS.format(
+                    percentage=percentage,
+                    downloaded=humanize_bytes(downloaded),
+                    speed=humanize_bytes(speed),
+                    eta=eta,
+                )
+
+            self._prev_time = now
+            self._prev_bytes = downloaded
+
+        self.output.write(
+            CLEAR_LINE
+            + ' '
+            + SPINNER[self._spinner_pos]
+            + ' '
+            + self._status_line
+        )
+        self.output.flush()
+
+        self._spinner_pos = (self._spinner_pos + 1
+                             if self._spinner_pos + 1 != len(SPINNER)
+                             else 0)
+
+    def sum_up(self):
+        actually_downloaded = (self.status.downloaded
+                               - self.status.resumed_from)
+        time_taken = self.status.time_finished - self.status.time_started
+
+        self.output.write(CLEAR_LINE)
+        self.output.write(SUMMARY.format(
+            downloaded=humanize_bytes(actually_downloaded),
+            total=(self.status.total_size
+                   and humanize_bytes(self.status.total_size)),
+            speed=humanize_bytes(actually_downloaded / time_taken),
+            time=time_taken,
+        ))
+        self.output.flush()

httpie/input.py

@@ -5,23 +5,29 @@ import os
 import sys
 import re
 import json
-import argparse
 import mimetypes
 import getpass
+from io import BytesIO
+#noinspection PyCompatibility
+from argparse import ArgumentParser, ArgumentTypeError, ArgumentError
 
 try:
     from collections import OrderedDict
 except ImportError:
     OrderedDict = dict
 
+# TODO: Use MultiDict for headers once added to `requests`.
+# https://github.com/jkbr/httpie/issues/130
 from requests.structures import CaseInsensitiveDict
-from requests.compat import str
 
-from . import __version__
+from .compat import urlsplit, str
+from .sessions import VALID_SESSION_NAME_PATTERN
 
 
 HTTP_POST = 'POST'
 HTTP_GET = 'GET'
+HTTP = 'http://'
+HTTPS = 'https://'
 
 
 # Various separators used in args
@@ -31,22 +37,40 @@ SEP_PROXY = ':'
 SEP_DATA = '='
 SEP_DATA_RAW_JSON = ':='
 SEP_FILES = '@'
+SEP_DATA_EMBED_FILE = '=@'
+SEP_DATA_EMBED_RAW_JSON_FILE = ':=@'
 SEP_QUERY = '=='
 
 # Separators that become request data
 SEP_GROUP_DATA_ITEMS = frozenset([
     SEP_DATA,
     SEP_DATA_RAW_JSON,
-    SEP_FILES
+    SEP_FILES,
+    SEP_DATA_EMBED_FILE,
+    SEP_DATA_EMBED_RAW_JSON_FILE
+])
+
+# Separators for items whose value is a filename to be embedded
+SEP_GROUP_DATA_EMBED_ITEMS = frozenset([
+    SEP_DATA_EMBED_FILE,
+    SEP_DATA_EMBED_RAW_JSON_FILE,
+])
+
+# Separators for raw JSON items
+SEP_GROUP_RAW_JSON_ITEMS = frozenset([
+    SEP_DATA_RAW_JSON,
+    SEP_DATA_EMBED_RAW_JSON_FILE,
 ])
 
 # Separators allowed in ITEM arguments
-SEP_GROUP_ITEMS = frozenset([
+SEP_GROUP_ALL_ITEMS = frozenset([
     SEP_HEADERS,
     SEP_QUERY,
     SEP_DATA,
     SEP_DATA_RAW_JSON,
-    SEP_FILES
+    SEP_FILES,
+    SEP_DATA_EMBED_FILE,
+    SEP_DATA_EMBED_RAW_JSON_FILE,
 ])
 
 
@@ -63,15 +87,22 @@ OUTPUT_OPTIONS = frozenset([
     OUT_RESP_BODY
 ])
 
+# Pretty
+PRETTY_MAP = {
+    'all': ['format', 'colors'],
+    'colors': ['colors'],
+    'format': ['format'],
+    'none': []
+}
+PRETTY_STDOUT_TTY_ONLY = object()
+
 
 # Defaults
 OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
 OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
-PRETTIFY_STDOUT_TTY_ONLY = object()
-DEFAULT_UA = 'HTTPie/%s' % __version__
 
 
-class Parser(argparse.ArgumentParser):
+class Parser(ArgumentParser):
     """Adds additional logic to `argparse.ArgumentParser`.
 
     Handles all input (CLI args, file args, stdin), applies defaults,
@@ -82,135 +113,284 @@ class Parser(argparse.ArgumentParser):
     def __init__(self, *args, **kwargs):
         kwargs['add_help'] = False
         super(Parser, self).__init__(*args, **kwargs)
-        # Help only as --help (-h is used for --headers).
-        self.add_argument('--help',
-            action='help', default=argparse.SUPPRESS,
-            help=argparse._('show this help message and exit'))
 
     #noinspection PyMethodOverriding
     def parse_args(self, env, args=None, namespace=None):
 
-        args = super(Parser, self).parse_args(args, namespace)
+        self.env = env
+        self.args, no_options = super(Parser, self)\
+            .parse_known_args(args, namespace)
 
-        self._process_output_options(args, env)
-        self._guess_method(args, env)
-        self._parse_items(args)
+        if self.args.debug:
+            self.args.traceback = True
 
-        if not env.stdin_isatty:
-            self._body_from_file(args, env.stdin)
+        # Arguments processing and environment setup.
+        self._apply_no_options(no_options)
+        self._apply_config()
+        self._validate_download_options()
+        self._setup_standard_streams()
+        self._process_output_options()
+        self._process_pretty_options()
+        self._guess_method()
+        self._parse_items()
+        if not self.args.ignore_stdin and not env.stdin_isatty:
+            self._body_from_file(self.env.stdin)
+        if not (self.args.url.startswith((HTTP, HTTPS))):
+            # Default to 'https://' if invoked as `https args`.
+            scheme = HTTPS if self.env.progname == 'https' else HTTP
 
-        if args.auth and not args.auth.has_password():
-            # Stdin already read (if not a tty) so it's save to prompt.
-            args.auth.prompt_password()
+            # See if we're using curl style shorthand for localhost (:3000/foo)
+            shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
+            if shorthand:
+                port = shorthand.group(1)
+                rest = shorthand.group(2)
+                self.args.url = scheme + 'localhost'
+                if port:
+                    self.args.url += ':' + port
+                self.args.url += rest
+            else:
+                self.args.url = scheme + self.args.url
+        self._process_auth()
 
-        return args
+        return self.args
 
-    def _body_from_file(self, args, f):
-        """Use the content of `f` as the `request.data`.
+    # noinspection PyShadowingBuiltins
+    def _print_message(self, message, file=None):
+        # Sneak in our stderr/stdout.
+        file = {
+            sys.stdout: self.env.stdout,
+            sys.stderr: self.env.stderr,
+            None: self.env.stderr
+        }.get(file, file)
 
-        There can only be one source of request data.
+        super(Parser, self)._print_message(message, file)
+
+    def _setup_standard_streams(self):
+        """
+        Modify `env.stdout` and `env.stdout_isatty` based on args, if needed.
 
         """
-        if args.data:
+        if not self.env.stdout_isatty and self.args.output_file:
+            self.error('Cannot use --output, -o with redirected output.')
+
+        # FIXME: Come up with a cleaner solution.
+        if self.args.download:
+
+            if not self.env.stdout_isatty:
+                # Use stdout as tge download output file.
+                self.args.output_file = self.env.stdout
+
+            # With `--download`, we write everything that would normally go to
+            # `stdout` to `stderr` instead. Let's replace the stream so that
+            # we don't have to use many `if`s throughout the codebase.
+            # The response body will be treated separately.
+            self.env.stdout = self.env.stderr
+            self.env.stdout_isatty = self.env.stderr_isatty
+
+        elif self.args.output_file:
+            # When not `--download`ing, then `--output` simply replaces
+            # `stdout`. The file is opened for appending, which isn't what
+            # we want in this case.
+            self.args.output_file.seek(0)
+            self.args.output_file.truncate()
+
+            self.env.stdout = self.args.output_file
+            self.env.stdout_isatty = False
+
+    def _apply_config(self):
+        if (not self.args.json
+                and self.env.config.implicit_content_type == 'form'):
+            self.args.form = True
+
+    def _process_auth(self):
+        """
+        If only a username provided via --auth, then ask for a password.
+        Or, take credentials from the URL, if provided.
+
+        """
+        url = urlsplit(self.args.url)
+
+        if self.args.auth:
+            if not self.args.auth.has_password():
+                # Stdin already read (if not a tty) so it's save to prompt.
+                if self.args.ignore_stdin:
+                    self.error('Unable to prompt for passwords because'
+                               ' --ignore-stdin is set.')
+                self.args.auth.prompt_password(url.netloc)
+
+        elif url.username is not None:
+            # Handle http://username:password@hostname/
+            username, password = url.username, url.password
+            self.args.auth = AuthCredentials(
+                key=username,
+                value=password,
+                sep=SEP_CREDENTIALS,
+                orig=SEP_CREDENTIALS.join([username, password])
+            )
+
+    def _apply_no_options(self, no_options):
+        """For every `--no-OPTION` in `no_options`, set `args.OPTION` to
+        its default value. This allows for un-setting of options, e.g.,
+        specified in config.
+
+        """
+        invalid = []
+
+        for option in no_options:
+            if not option.startswith('--no-'):
+                invalid.append(option)
+                continue
+
+            # --no-option => --option
+            inverted = '--' + option[5:]
+            for action in self._actions:
+                if inverted in action.option_strings:
+                    setattr(self.args, action.dest, action.default)
+                    break
+            else:
+                invalid.append(option)
+
+        if invalid:
+            msg = 'unrecognized arguments: %s'
+            self.error(msg % ' '.join(invalid))
+
+    def _body_from_file(self, fd):
+        """There can only be one source of request data.
+
+        Bytes are always read.
+
+        """
+        if self.args.data:
             self.error('Request body (from stdin or a file) and request '
                        'data (key=value) cannot be mixed.')
-        args.data = f.read()
+        self.args.data = getattr(fd, 'buffer', fd).read()
 
-    def _guess_method(self, args, env):
+    def _guess_method(self):
         """Set `args.method` if not specified to either POST or GET
         based on whether the request has data or not.
 
         """
-        if args.method is None:
+        if self.args.method is None:
             # Invoked as `http URL'.
-            assert not args.items
-            if not env.stdin_isatty:
-                args.method = HTTP_POST
+            assert not self.args.items
+            if not self.args.ignore_stdin and not self.env.stdin_isatty:
+                self.args.method = HTTP_POST
             else:
-                args.method = HTTP_GET
+                self.args.method = HTTP_GET
 
         # FIXME: False positive, e.g., "localhost" matches but is a valid URL.
-        elif not re.match('^[a-zA-Z]+$', args.method):
+        elif not re.match('^[a-zA-Z]+$', self.args.method):
             # Invoked as `http URL item+'. The URL is now in `args.method`
             # and the first ITEM is now incorrectly in `args.url`.
             try:
                 # Parse the URL as an ITEM and store it as the first ITEM arg.
-                args.items.insert(
-                    0, KeyValueArgType(*SEP_GROUP_ITEMS).__call__(args.url))
+                self.args.items.insert(
+                    0,
+                    KeyValueArgType(*SEP_GROUP_ALL_ITEMS).__call__(self.args.url)
+                )
 
-            except argparse.ArgumentTypeError as e:
-                if args.traceback:
+            except ArgumentTypeError as e:
+                if self.args.traceback:
                     raise
                 self.error(e.message)
 
             else:
                 # Set the URL correctly
-                args.url = args.method
+                self.args.url = self.args.method
                 # Infer the method
-                has_data = not env.stdin_isatty or any(
-                    item.sep in SEP_GROUP_DATA_ITEMS for item in args.items)
-                args.method = HTTP_POST if has_data else HTTP_GET
+                has_data = (
+                    (not self.args.ignore_stdin and
+                     not self.env.stdin_isatty) or any(
+                        item.sep in SEP_GROUP_DATA_ITEMS
+                        for item in self.args.items
+                    )
+                )
+                self.args.method = HTTP_POST if has_data else HTTP_GET
 
-    def _parse_items(self, args):
-        """Parse `args.items` into `args.headers`, `args.data`,
-        `args.`, and `args.files`.
+    def _parse_items(self):
+        """Parse `args.items` into `args.headers`, `args.data`, `args.params`,
+         and `args.files`.
 
         """
-        args.headers = CaseInsensitiveDict()
-        args.headers['User-Agent'] = DEFAULT_UA
-        args.data = ParamDict() if args.form else OrderedDict()
-        args.files = OrderedDict()
-        args.params = ParamDict()
+        self.args.headers = CaseInsensitiveDict()
+        self.args.data = ParamDict() if self.args.form else OrderedDict()
+        self.args.files = OrderedDict()
+        self.args.params = ParamDict()
 
         try:
-            parse_items(items=args.items,
-                        headers=args.headers,
-                        data=args.data,
-                        files=args.files,
-                        params=args.params)
+            parse_items(items=self.args.items,
+                        headers=self.args.headers,
+                        data=self.args.data,
+                        files=self.args.files,
+                        params=self.args.params)
         except ParseError as e:
-            if args.traceback:
+            if self.args.traceback:
                 raise
             self.error(e.message)
 
-        if args.files and not args.form:
+        if self.args.files and not self.args.form:
             # `http url @/path/to/file`
-            # It's not --form so the file contents will be used as the
-            # body of the requests. Also, we try to detect the appropriate
-            # Content-Type.
-            if len(args.files) > 1:
+            file_fields = list(self.args.files.keys())
+            if file_fields != ['']:
                 self.error(
-                    'Only one file can be specified unless'
-                    ' --form is used. File fields: %s'
-                    % ','.join(args.files.keys()))
+                    'Invalid file fields (perhaps you meant --form?): %s'
+                    % ','.join(file_fields))
 
-            f = list(args.files.values())[0]
-            self._body_from_file(args, f)
+            fn, fd = self.args.files['']
+            self.args.files = {}
 
-            # Reset files
-            args.files = {}
+            self._body_from_file(fd)
 
-            if 'Content-Type' not in args.headers:
-                mime, encoding = mimetypes.guess_type(f.name, strict=False)
+            if 'Content-Type' not in self.args.headers:
+                mime, encoding = mimetypes.guess_type(fn, strict=False)
                 if mime:
                     content_type = mime
                     if encoding:
                         content_type = '%s; charset=%s' % (mime, encoding)
-                    args.headers['Content-Type'] = content_type
+                    self.args.headers['Content-Type'] = content_type
 
-    def _process_output_options(self, args, env):
-        """Apply defaults to output options or validate the provided ones.
+    def _process_output_options(self):
+        """Apply defaults to output options, or validate the provided ones.
 
         The default output options are stdout-type-sensitive.
 
         """
-        if not args.output_options:
-            args.output_options = (OUTPUT_OPTIONS_DEFAULT if env.stdout_isatty
-                                else OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED)
+        if not self.args.output_options:
+            self.args.output_options = (
+                OUTPUT_OPTIONS_DEFAULT
+                if self.env.stdout_isatty
+                else OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
+            )
 
-        unknown = set(args.output_options) - OUTPUT_OPTIONS
-        if unknown:
-            self.error('Unknown output options: %s' % ','.join(unknown))
+        unknown_output_options = set(self.args.output_options) - OUTPUT_OPTIONS
+        if unknown_output_options:
+            self.error(
+                'Unknown output options: %s' % ','.join(unknown_output_options)
+            )
+
+        if self.args.download and OUT_RESP_BODY in self.args.output_options:
+            # Response body is always downloaded with --download and it goes
+            # through a different routine, so we remove it.
+            self.args.output_options = str(
+                set(self.args.output_options) - set(OUT_RESP_BODY))
+
+    def _process_pretty_options(self):
+        if self.args.prettify == PRETTY_STDOUT_TTY_ONLY:
+            self.args.prettify = PRETTY_MAP[
+                'all' if self.env.stdout_isatty else 'none']
+        elif self.args.prettify and self.env.is_windows:
+            self.error('Only terminal output can be colorized on Windows.')
+        else:
+            # noinspection PyTypeChecker
+            self.args.prettify = PRETTY_MAP[self.args.prettify]
+
+    def _validate_download_options(self):
+        if not self.args.download:
+            if self.args.download_resume:
+                self.error('--continue only works with --download')
+        if self.args.download_resume and not (
+                self.args.download and self.args.output_file):
+            self.error('--continue requires --output to be specified')
 
 
 class ParseError(Exception):
@@ -230,6 +410,19 @@ class KeyValue(object):
         return self.__dict__ == other.__dict__
 
 
+class SessionNameValidator(object):
+
+    def __init__(self, error_message):
+        self.error_message = error_message
+
+    def __call__(self, value):
+        # Session name can be a path or just a name.
+        if (os.path.sep not in value
+                and not VALID_SESSION_NAME_PATTERN.search(value)):
+            raise ArgumentError(None, self.error_message)
+        return value
+
+
 class KeyValueArgType(object):
     """A key-value pair argument type used with `argparse`.
 
@@ -260,8 +453,8 @@ class KeyValueArgType(object):
             """Tokenize `s`. There are only two token types - strings
             and escaped characters:
 
-            >>> tokenize(r'foo\=bar\\baz')
-            ['foo', Escaped('='), 'bar', Escaped('\\'), 'baz']
+            tokenize(r'foo\=bar\\baz')
+            => ['foo', Escaped('='), 'bar', Escaped('\\'), 'baz']
 
             """
             tokens = ['']
@@ -310,7 +503,7 @@ class KeyValueArgType(object):
                 break
 
         else:
-            raise argparse.ArgumentTypeError(
+            raise ArgumentTypeError(
                 '"%s" is not a valid value' % string)
 
         return self.key_value_class(
@@ -327,9 +520,10 @@ class AuthCredentials(KeyValue):
     def has_password(self):
         return self.value is not None
 
-    def prompt_password(self):
+    def prompt_password(self, host):
         try:
-            self.value = self._getpass("Password for user '%s': " % self.key)
+            self.value = self._getpass(
+                'http: password for %s@%s: ' % (self.key, host))
         except (EOFError, KeyboardInterrupt):
             sys.stderr.write('\n')
             sys.exit(0)
@@ -348,7 +542,7 @@ class AuthCredentialsArgType(KeyValueArgType):
         """
         try:
             return super(AuthCredentialsArgType, self).__call__(string)
-        except argparse.ArgumentTypeError:
+        except ArgumentTypeError:
             # No password provided, will prompt for it later.
             return self.key_value_class(
                 key=string,
@@ -370,9 +564,6 @@ class ParamDict(OrderedDict):
         data and URL params.
 
         """
-        # NOTE: Won't work when used for form data with multiple values
-        # for a field and a file field is present:
-        # https://github.com/kennethreitz/requests/issues/737
         if key not in self:
             super(ParamDict, self).__setitem__(key, value)
         else:
@@ -396,9 +587,7 @@ def parse_items(items, data=None, headers=None, files=None, params=None):
         params = ParamDict()
 
     for item in items:
-
         value = item.value
-        key = item.key
 
         if item.sep == SEP_HEADERS:
             target = headers
@@ -406,25 +595,38 @@ def parse_items(items, data=None, headers=None, files=None, params=None):
             target = params
         elif item.sep == SEP_FILES:
             try:
-                value = open(os.path.expanduser(item.value), 'r')
+                with open(os.path.expanduser(value), 'rb') as f:
+                    value = (os.path.basename(value),
+                             BytesIO(f.read()))
             except IOError as e:
-                raise ParseError(
-                    'Invalid argument "%s": %s' % (item.orig, e))
-            if not key:
-                key = os.path.basename(value.name)
+                raise ParseError('"%s": %s' % (item.orig, e))
             target = files
 
-        elif item.sep in [SEP_DATA, SEP_DATA_RAW_JSON]:
-            if item.sep == SEP_DATA_RAW_JSON:
+        elif item.sep in SEP_GROUP_DATA_ITEMS:
+
+            if item.sep in SEP_GROUP_DATA_EMBED_ITEMS:
                 try:
-                    value = json.loads(item.value)
-                except ValueError:
-                    raise ParseError('"%s" is not valid JSON' % item.orig)
+                    with open(os.path.expanduser(value), 'rb') as f:
+                        value = f.read().decode('utf8')
+                except IOError as e:
+                    raise ParseError('"%s": %s' % (item.orig, e))
+                except UnicodeDecodeError:
+                    raise ParseError(
+                        '"%s": cannot embed the content of "%s",'
+                        ' not a UTF8 or ASCII-encoded text file'
+                        % (item.orig, item.value)
+                    )
+
+            if item.sep in SEP_GROUP_RAW_JSON_ITEMS:
+                try:
+                    value = json.loads(value)
+                except ValueError as e:
+                    raise ParseError('"%s": %s' % (item.orig, e))
             target = data
 
         else:
             raise TypeError(item)
 
-        target[key] = value
+        target[item.key] = value
 
     return headers, data, files, params

httpie/models.py

@@ -1,6 +1,8 @@
 import os
 import sys
-from requests.compat import urlparse, is_windows
+
+from .config import DEFAULT_CONFIG_DIR, Config
+from .compat import urlsplit, is_windows, bytes, str
 
 
 class Environment(object):
@@ -10,130 +12,166 @@ class Environment(object):
     and allows for mocking.
 
     """
-    stdin_isatty = sys.stdin.isatty()
-    stdin = sys.stdin
 
-    if is_windows:
-        # `colorama` patches `sys.stdout` so its initialization
-        # needs to happen before the default environment is set.
-        import colorama
-        colorama.init()
-        del colorama
+    is_windows = is_windows
 
-    stdout_isatty = sys.stdout.isatty()
-    stdout = sys.stdout
+    progname = os.path.basename(sys.argv[0])
+    if progname not in ['http', 'https']:
+        progname = 'http'
 
-    stderr = sys.stderr
+    config_dir = DEFAULT_CONFIG_DIR
 
     # Can be set to 0 to disable colors completely.
     colors = 256 if '256color' in os.environ.get('TERM', '') else 88
 
+    stdin = sys.stdin
+    stdin_isatty = sys.stdin.isatty()
+
+    stdout_isatty = sys.stdout.isatty()
+    stderr_isatty = sys.stderr.isatty()
+    if is_windows:
+        # noinspection PyUnresolvedReferences
+        from colorama.initialise import wrap_stream
+        stdout = wrap_stream(sys.stdout, convert=None,
+                             strip=None, autoreset=True, wrap=True)
+        stderr = wrap_stream(sys.stderr, convert=None,
+                             strip=None, autoreset=True, wrap=True)
+    else:
+        stdout = sys.stdout
+        stderr = sys.stderr
+
     def __init__(self, **kwargs):
+        assert all(hasattr(type(self), attr)
+                   for attr in kwargs.keys())
         self.__dict__.update(**kwargs)
 
+    @property
+    def config(self):
+        if not hasattr(self, '_config'):
+            self._config = Config(directory=self.config_dir)
+            if self._config.is_new:
+                self._config.save()
+            else:
+                self._config.load()
+        return self._config
+
 
 class HTTPMessage(object):
-    """Model representing an HTTP message."""
+    """Abstract class for HTTP messages."""
 
-    def __init__(self, line, headers, body, content_type=None):
-        # {Request,Status}-Line
-        self.line = line
-        self.headers = headers
-        self.body = body
-        self.content_type = content_type
+    def __init__(self, orig):
+        self._orig = orig
 
-    def format(self, prettifier=None, with_headers=True, with_body=True):
-        """Return a `unicode` representation of `self`. """
-        pretty = prettifier is not None
-        bits = []
+    def iter_body(self, chunk_size):
+        """Return an iterator over the body."""
+        raise NotImplementedError()
 
-        if with_headers:
-            bits.append(self.line)
-            bits.append(self.headers)
-            if pretty:
-                bits = [
-                    prettifier.process_headers('\n'.join(bits))
-                ]
-            if with_body and self.body:
-                bits.append('\n')
+    def iter_lines(self, chunk_size):
+        """Return an iterator over the body yielding (`line`, `line_feed`)."""
+        raise NotImplementedError()
 
-        if with_body and self.body:
-            if pretty and self.content_type:
-                bits.append(prettifier.process_body(
-                    self.body, self.content_type))
-            else:
-                bits.append(self.body)
+    @property
+    def headers(self):
+        """Return a `str` with the message's headers."""
+        raise NotImplementedError()
 
-        return '\n'.join(bit.strip() for bit in bits)
+    @property
+    def encoding(self):
+        """Return a `str` with the message's encoding, if known."""
+        raise NotImplementedError()
 
-    @staticmethod
-    def from_request(request):
-        """Make an `HTTPMessage` from `requests.models.Request`."""
+    @property
+    def body(self):
+        """Return a `bytes` with the message's body."""
+        raise NotImplementedError()
 
-        url = urlparse(request.url)
+    @property
+    def content_type(self):
+        """Return the message content type."""
+        return self._orig.headers.get('Content-Type', '')
 
-        # Querystring
-        qs = ''
-        if url.query or request.params:
-            qs = '?'
-            if url.query:
-                qs += url.query
-            # Requests doesn't make params part of ``request.url``.
-            if request.params:
-                if url.query:
-                    qs += '&'
-                #noinspection PyUnresolvedReferences
-                qs += type(request)._encode_params(request.params)
 
-        # Request-Line
-        request_line = '{method} {path}{query} HTTP/1.1'.format(
-            method=request.method,
-            path=url.path or '/',
-            query=qs
-        )
+class HTTPResponse(HTTPMessage):
+    """A :class:`requests.models.Response` wrapper."""
 
-        # Headers
-        headers = dict(request.headers)
-        content_type = headers.get('Content-Type')
-        if 'Host' not in headers:
-            headers['Host'] = url.netloc
-        headers = '\n'.join(
-            str('%s: %s') % (name, value)
-            for name, value
-            in headers.items()
-        )
+    def iter_body(self, chunk_size=1):
+        return self._orig.iter_content(chunk_size=chunk_size)
 
-        # Body
-        try:
-            body = request.data
-        except AttributeError:
-            # requests < 0.12.1
-            body = request._enc_data
-        if isinstance(body, dict):
-            #noinspection PyUnresolvedReferences
-            body = type(request)._encode_params(body)
+    def iter_lines(self, chunk_size):
+        return ((line, b'\n') for line in self._orig.iter_lines(chunk_size))
 
-        return HTTPMessage(
-            line=request_line,
-            headers=headers,
-            body=body,
-            content_type=content_type
-        )
-
-    @classmethod
-    def from_response(cls, response):
-        """Make an `HTTPMessage` from `requests.models.Response`."""
-        encoding = response.encoding or 'ISO-8859-1'
-        original = response.raw._original_response
-        response_headers = response.headers
+    #noinspection PyProtectedMember
+    @property
+    def headers(self):
+        original = self._orig.raw._original_response
         status_line = 'HTTP/{version} {status} {reason}'.format(
             version='.'.join(str(original.version)),
             status=original.status,
             reason=original.reason
         )
-        body = response.content.decode(encoding) if response.content else ''
-        return cls(
-            line=status_line,
-            headers=str(original.msg),
-            body=body,
-            content_type=response_headers.get('Content-Type'))
+        headers = [status_line]
+        try:
+            # `original.msg` is a `http.client.HTTPMessage` on Python 3
+            # `_headers` is a 2-tuple
+            headers.extend(
+                '%s: %s' % header for header in original.msg._headers)
+        except AttributeError:
+            # and a `httplib.HTTPMessage` on Python 2.x
+            # `headers` is a list of `name: val<CRLF>`.
+            headers.extend(h.strip() for h in original.msg.headers)
+
+        return '\r\n'.join(headers)
+
+    @property
+    def encoding(self):
+        return self._orig.encoding or 'utf8'
+
+    @property
+    def body(self):
+        # Only now the response body is fetched.
+        # Shouldn't be touched unless the body is actually needed.
+        return self._orig.content
+
+
+class HTTPRequest(HTTPMessage):
+    """A :class:`requests.models.Request` wrapper."""
+
+    def iter_body(self, chunk_size):
+        yield self.body
+
+    def iter_lines(self, chunk_size):
+        yield self.body, b''
+
+    @property
+    def headers(self):
+        url = urlsplit(self._orig.url)
+
+        request_line = '{method} {path}{query} HTTP/1.1'.format(
+            method=self._orig.method,
+            path=url.path or '/',
+            query='?' + url.query if url.query else ''
+        )
+
+        headers = dict(self._orig.headers)
+
+        if 'Host' not in headers:
+            headers['Host'] = url.netloc.split('@')[-1]
+
+        headers = ['%s: %s' % (name, value)
+                   for name, value in headers.items()]
+
+        headers.insert(0, request_line)
+
+        return '\r\n'.join(headers).strip()
+
+    @property
+    def encoding(self):
+        return 'utf8'
+
+    @property
+    def body(self):
+        body = self._orig.body
+        if isinstance(body, str):
+            # Happens with JSON/form request data parsed from the command line.
+            body = body.encode('utf8')
+        return body or b''

httpie/output.py

@@ -1,31 +1,312 @@
-"""Output processing and formatting.
+"""Output streaming, processing and formatting.
 
 """
-import re
 import json
+import xml.dom.minidom
+from functools import partial
+from itertools import chain
 
 import pygments
 from pygments import token, lexer
 from pygments.styles import get_style_by_name, STYLE_MAP
-from pygments.lexers import get_lexer_for_mimetype
+from pygments.lexers import get_lexer_for_mimetype, get_lexer_by_name
 from pygments.formatters.terminal import TerminalFormatter
 from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.util import ClassNotFound
-from requests.compat import is_windows
 
-from . import solarized
+from .compat import is_windows
+from .solarized import Solarized256Style
+from .models import HTTPRequest, HTTPResponse, Environment
+from .input import (OUT_REQ_BODY, OUT_REQ_HEAD,
+                    OUT_RESP_HEAD, OUT_RESP_BODY)
 
 
-DEFAULT_STYLE = 'solarized'
-AVAILABLE_STYLES = [DEFAULT_STYLE] + list(STYLE_MAP.keys())
+# The default number of spaces to indent when pretty printing
+DEFAULT_INDENT = 4
 
+# Colors on Windows via colorama don't look that
+# great and fruity seems to give the best result there.
+AVAILABLE_STYLES = set(STYLE_MAP.keys())
+AVAILABLE_STYLES.add('solarized')
+DEFAULT_STYLE = 'solarized' if not is_windows else 'fruity'
+
+
+BINARY_SUPPRESSED_NOTICE = (
+    b'\n'
+    b'+-----------------------------------------+\n'
+    b'| NOTE: binary data not shown in terminal |\n'
+    b'+-----------------------------------------+'
+)
+
+
+class BinarySuppressedError(Exception):
+    """An error indicating that the body is binary and won't be written,
+     e.g., for terminal output)."""
+
+    message = BINARY_SUPPRESSED_NOTICE
+
+
+###############################################################################
+# Output Streams
+###############################################################################
+
+
+def write(stream, outfile, flush):
+    """Write the output stream."""
+    try:
+        # Writing bytes so we use the buffer interface (Python 3).
+        buf = outfile.buffer
+    except AttributeError:
+        buf = outfile
+
+    for chunk in stream:
+        buf.write(chunk)
+        if flush:
+            outfile.flush()
+
+
+def write_with_colors_win_py3(stream, outfile, flush):
+    """Like `write`, but colorized chunks are written as text
+    directly to `outfile` to ensure it gets processed by colorama.
+    Applies only to Windows with Python 3 and colorized terminal output.
+
+    """
+    color = b'\x1b['
+    encoding = outfile.encoding
+    for chunk in stream:
+        if color in chunk:
+            outfile.write(chunk.decode(encoding))
+        else:
+            outfile.buffer.write(chunk)
+        if flush:
+            outfile.flush()
+
+
+def build_output_stream(args, env, request, response):
+    """Build and return a chain of iterators over the `request`-`response`
+    exchange each of which yields `bytes` chunks.
+
+    """
+
+    req_h = OUT_REQ_HEAD in args.output_options
+    req_b = OUT_REQ_BODY in args.output_options
+    resp_h = OUT_RESP_HEAD in args.output_options
+    resp_b = OUT_RESP_BODY in args.output_options
+    req = req_h or req_b
+    resp = resp_h or resp_b
+
+    output = []
+    Stream = get_stream_type(env, args)
+
+    if req:
+        output.append(Stream(
+            msg=HTTPRequest(request),
+            with_headers=req_h,
+            with_body=req_b))
+
+    if req_b and resp:
+        # Request/Response separator.
+        output.append([b'\n\n'])
+
+    if resp:
+        output.append(Stream(
+            msg=HTTPResponse(response),
+            with_headers=resp_h,
+            with_body=resp_b))
+
+    if env.stdout_isatty and resp_b:
+        # Ensure a blank line after the response body.
+        # For terminal output only.
+        output.append([b'\n\n'])
+
+    return chain(*output)
+
+
+def get_stream_type(env, args):
+    """Pick the right stream type based on `env` and `args`.
+    Wrap it in a partial with the type-specific args so that
+    we don't need to think what stream we are dealing with.
+
+    """
+    if not env.stdout_isatty and not args.prettify:
+        Stream = partial(
+            RawStream,
+            chunk_size=RawStream.CHUNK_SIZE_BY_LINE
+                       if args.stream
+                       else RawStream.CHUNK_SIZE
+        )
+    elif args.prettify:
+        Stream = partial(
+            PrettyStream if args.stream else BufferedPrettyStream,
+            env=env,
+            processor=OutputProcessor(
+                env=env, groups=args.prettify, pygments_style=args.style),
+        )
+    else:
+        Stream = partial(EncodedStream, env=env)
+
+    return Stream
+
+
+class BaseStream(object):
+    """Base HTTP message output stream class."""
+
+    def __init__(self, msg, with_headers=True, with_body=True,
+                 on_body_chunk_downloaded=None):
+        """
+        :param msg: a :class:`models.HTTPMessage` subclass
+        :param with_headers: if `True`, headers will be included
+        :param with_body: if `True`, body will be included
+
+        """
+        assert with_headers or with_body
+        self.msg = msg
+        self.with_headers = with_headers
+        self.with_body = with_body
+        self.on_body_chunk_downloaded = on_body_chunk_downloaded
+
+    def _get_headers(self):
+        """Return the headers' bytes."""
+        return self.msg.headers.encode('ascii')
+
+    def _iter_body(self):
+        """Return an iterator over the message body."""
+        raise NotImplementedError()
+
+    def __iter__(self):
+        """Return an iterator over `self.msg`."""
+        if self.with_headers:
+            yield self._get_headers()
+            yield b'\r\n\r\n'
+
+        if self.with_body:
+            try:
+                for chunk in self._iter_body():
+                    yield chunk
+                    if self.on_body_chunk_downloaded:
+                        self.on_body_chunk_downloaded(chunk)
+            except BinarySuppressedError as e:
+                if self.with_headers:
+                    yield b'\n'
+                yield e.message
+
+
+class RawStream(BaseStream):
+    """The message is streamed in chunks with no processing."""
+
+    CHUNK_SIZE = 1024 * 100
+    CHUNK_SIZE_BY_LINE = 1
+
+    def __init__(self, chunk_size=CHUNK_SIZE, **kwargs):
+        super(RawStream, self).__init__(**kwargs)
+        self.chunk_size = chunk_size
+
+    def _iter_body(self):
+        return self.msg.iter_body(self.chunk_size)
+
+
+class EncodedStream(BaseStream):
+    """Encoded HTTP message stream.
+
+    The message bytes are converted to an encoding suitable for
+    `self.env.stdout`. Unicode errors are replaced and binary data
+    is suppressed. The body is always streamed by line.
+
+    """
+    CHUNK_SIZE = 1
+
+    def __init__(self, env=Environment(), **kwargs):
+
+        super(EncodedStream, self).__init__(**kwargs)
+
+        if env.stdout_isatty:
+            # Use the encoding supported by the terminal.
+            output_encoding = getattr(env.stdout, 'encoding', None)
+        else:
+            # Preserve the message encoding.
+            output_encoding = self.msg.encoding
+
+        # Default to utf8 when unsure.
+        self.output_encoding = output_encoding or 'utf8'
+
+    def _iter_body(self):
+
+        for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
+
+            if b'\0' in line:
+                raise BinarySuppressedError()
+
+            yield line.decode(self.msg.encoding)\
+                      .encode(self.output_encoding, 'replace') + lf
+
+
+class PrettyStream(EncodedStream):
+    """In addition to :class:`EncodedStream` behaviour, this stream applies
+    content processing.
+
+    Useful for long-lived HTTP responses that stream by lines
+    such as the Twitter streaming API.
+
+    """
+
+    CHUNK_SIZE = 1
+
+    def __init__(self, processor, **kwargs):
+        super(PrettyStream, self).__init__(**kwargs)
+        self.processor = processor
+
+    def _get_headers(self):
+        return self.processor.process_headers(
+            self.msg.headers).encode(self.output_encoding)
+
+    def _iter_body(self):
+        for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
+            if b'\0' in line:
+                raise BinarySuppressedError()
+            yield self._process_body(line) + lf
+
+    def _process_body(self, chunk):
+        return (self.processor
+                    .process_body(
+                        content=chunk.decode(self.msg.encoding, 'replace'),
+                        content_type=self.msg.content_type,
+                        encoding=self.msg.encoding)
+                    .encode(self.output_encoding, 'replace'))
+
+
+class BufferedPrettyStream(PrettyStream):
+    """The same as :class:`PrettyStream` except that the body is fully
+    fetched before it's processed.
+
+    Suitable regular HTTP responses.
+
+    """
+
+    CHUNK_SIZE = 1024 * 10
+
+    def _iter_body(self):
+
+        # Read the whole body before prettifying it,
+        # but bail out immediately if the body is binary.
+        body = bytearray()
+        for chunk in self.msg.iter_body(self.CHUNK_SIZE):
+            if b'\0' in chunk:
+                raise BinarySuppressedError()
+            body.extend(chunk)
+
+        yield self._process_body(body)
+
+
+###############################################################################
+# Processing
+###############################################################################
 
 class HTTPLexer(lexer.RegexLexer):
     """Simplified HTTP lexer for Pygments.
 
     It only operates on headers and provides a stronger contrast between
     their names and values than the original one bundled with Pygments
-    (`pygments.lexers.text import HttpLexer`), especially when
+    (:class:`pygments.lexers.text import HttpLexer`), especially when
     Solarized color scheme is used.
 
     """
@@ -34,19 +315,17 @@ class HTTPLexer(lexer.RegexLexer):
     filenames = ['*.http']
     tokens = {
         'root': [
-
             # Request-Line
             (r'([A-Z]+)( +)([^ ]+)( +)(HTTP)(/)(\d+\.\d+)',
              lexer.bygroups(
-                token.Name.Function,
-                token.Text,
-                token.Name.Namespace,
-                token.Text,
-                token.Keyword.Reserved,
-                token.Operator,
-                token.Number
+                 token.Name.Function,
+                 token.Text,
+                 token.Name.Namespace,
+                 token.Text,
+                 token.Keyword.Reserved,
+                 token.Operator,
+                 token.Number
              )),
-
             # Response Status-Line
             (r'(HTTP)(/)(\d+\.\d+)( +)(\d{3})( +)(.+)',
              lexer.bygroups(
@@ -58,56 +337,98 @@ class HTTPLexer(lexer.RegexLexer):
                  token.Text,
                  token.Name.Exception,  # Reason
              )),
-
             # Header
             (r'(.*?)( *)(:)( *)(.+)', lexer.bygroups(
-                token.Name.Attribute, # Name
+                token.Name.Attribute,  # Name
                 token.Text,
                 token.Operator,  # Colon
                 token.Text,
                 token.String  # Value
             ))
-    ]}
+        ]
+    }
 
 
 class BaseProcessor(object):
+    """Base, noop output processor class."""
 
     enabled = True
 
-    def __init__(self, env, **kwargs):
+    def __init__(self, env=Environment(), **kwargs):
+        """
+        :param env: an class:`Environment` instance
+        :param kwargs: additional keyword argument that some
+                       processor might require.
+
+        """
         self.env = env
         self.kwargs = kwargs
 
     def process_headers(self, headers):
+        """Return processed `headers`
+
+        :param headers: The headers as text.
+
+        """
         return headers
 
-    def process_body(self, content, content_type):
+    def process_body(self, content, content_type, subtype, encoding):
+        """Return processed `content`.
+
+        :param content: The body content as text
+        :param content_type: Full content type, e.g., 'application/atom+xml'.
+        :param subtype: E.g. 'xml'.
+        :param encoding: The original content encoding.
+
+        """
         return content
 
 
 class JSONProcessor(BaseProcessor):
+    """JSON body processor."""
 
-    def process_body(self, content, content_type):
-        if content_type == 'application/json':
+    def process_body(self, content, content_type, subtype, encoding):
+        if subtype == 'json':
             try:
-                # Indent and sort the JSON data.
-                content = json.dumps(
-                    json.loads(content),
-                    sort_keys=True,
-                    ensure_ascii=False,
-                    indent=4,
-                )
+                # Indent the JSON data, sort keys by name, and
+                # avoid unicode escapes to improve readability.
+                content = json.dumps(json.loads(content),
+                                     sort_keys=True,
+                                     ensure_ascii=False,
+                                     indent=DEFAULT_INDENT)
             except ValueError:
-                # Invalid JSON - we don't care.
+                # Invalid JSON but we don't care.
+                pass
+        return content
+
+
+class XMLProcessor(BaseProcessor):
+    """XML body processor."""
+    # TODO: tests
+
+    def process_body(self, content, content_type, subtype, encoding):
+        if subtype == 'xml':
+            try:
+                # Pretty print the XML
+                doc = xml.dom.minidom.parseString(content.encode(encoding))
+                content = doc.toprettyxml(indent=' ' * DEFAULT_INDENT)
+            except xml.parsers.expat.ExpatError:
+                # Ignore invalid XML errors (skips attempting to pretty print)
                 pass
         return content
 
 
 class PygmentsProcessor(BaseProcessor):
+    """A processor that applies syntax-highlighting using Pygments
+    to the headers, and to the body as well if its content type is recognized.
 
+    """
     def __init__(self, *args, **kwargs):
         super(PygmentsProcessor, self).__init__(*args, **kwargs)
 
+        # Cache that speeds up when we process streamed body by line.
+        self.lexers_by_type = {}
+
         if not self.env.colors:
             self.enabled = False
             return
@@ -116,9 +437,9 @@ class PygmentsProcessor(BaseProcessor):
             style = get_style_by_name(
                 self.kwargs.get('pygments_style', DEFAULT_STYLE))
         except ClassNotFound:
-            style = solarized.SolarizedStyle
+            style = Solarized256Style
 
-        if is_windows or self.env.colors == 256:
+        if self.env.is_windows or self.env.colors == 256:
             fmt_class = Terminal256Formatter
         else:
             fmt_class = TerminalFormatter
@@ -126,51 +447,80 @@ class PygmentsProcessor(BaseProcessor):
 
     def process_headers(self, headers):
         return pygments.highlight(
-            headers, HTTPLexer(), self.formatter)
+            headers, HTTPLexer(), self.formatter).strip()
 
-    def process_body(self, content, content_type):
+    def process_body(self, content, content_type, subtype, encoding):
         try:
-            lexer = get_lexer_for_mimetype(content_type)
+            lexer = self.lexers_by_type.get(content_type)
+            if not lexer:
+                try:
+                    lexer = get_lexer_for_mimetype(content_type)
+                except ClassNotFound:
+                    lexer = get_lexer_by_name(subtype)
+                self.lexers_by_type[content_type] = lexer
         except ClassNotFound:
             pass
         else:
             content = pygments.highlight(content, lexer, self.formatter)
-        return content
+        return content.strip()
+
+
+class HeadersProcessor(BaseProcessor):
+    """Sorts headers by name retaining relative order of multiple headers
+    with the same name.
+
+    """
+    def process_headers(self, headers):
+        lines = headers.splitlines()
+        headers = sorted(lines[1:], key=lambda h: h.split(':')[0])
+        return '\r\n'.join(lines[:1] + headers)
 
 
 class OutputProcessor(object):
+    """A delegate class that invokes the actual processors."""
 
-    installed_processors = [
-        JSONProcessor,
-        PygmentsProcessor
-    ]
-
-    def __init__(self, env, **kwargs):
-        self.env = env
-        processors = [
-            cls(env, **kwargs)
-            for cls in self.installed_processors
+    installed_processors = {
+        'format': [
+            HeadersProcessor,
+            JSONProcessor,
+            XMLProcessor
+        ],
+        'colors': [
+            PygmentsProcessor
         ]
-        self.processors = [p for p in processors if p.enabled]
+    }
+
+    def __init__(self, groups, env=Environment(), **kwargs):
+        """
+        :param env: a :class:`models.Environment` instance
+        :param groups: the groups of processors to be applied
+        :param kwargs: additional keyword arguments for processors
+
+        """
+        self.processors = []
+        for group in groups:
+            for cls in self.installed_processors[group]:
+                processor = cls(env, **kwargs)
+                if processor.enabled:
+                    self.processors.append(processor)
 
     def process_headers(self, headers):
         for processor in self.processors:
-         headers = processor.process_headers(headers)
+            headers = processor.process_headers(headers)
         return headers
 
-    def process_body(self, content, content_type):
+    def process_body(self, content, content_type, encoding):
+        # e.g., 'application/atom+xml'
         content_type = content_type.split(';')[0]
-
-        application_match = re.match(
-            r'application/(.+\+)(json|xml)$',
-            content_type
-        )
-        if application_match:
-            # Strip vendor and extensions from Content-Type
-            vendor, extension = application_match.groups()
-            content_type = content_type.replace(vendor, '')
+        # e.g., 'xml'
+        subtype = content_type.split('/')[-1].split('+')[-1]
 
         for processor in self.processors:
-            content = processor.process_body(content, content_type)
+            content = processor.process_body(
+                content,
+                content_type,
+                subtype,
+                encoding
+            )
 
         return content

httpie/plugins/__init__.py

@@ -0,0 +1,9 @@
+from .base import AuthPlugin
+from .manager import PluginManager
+from .builtin import BasicAuthPlugin, DigestAuthPlugin
+
+
+plugin_manager = PluginManager()
+plugin_manager.register(BasicAuthPlugin)
+plugin_manager.register(DigestAuthPlugin)
+

httpie/plugins/base.py

@@ -0,0 +1,28 @@
+class AuthPlugin(object):
+    """
+    Base auth plugin class.
+
+    See <https://github.com/jkbr/httpie-ntlm> for an example auth plugin.
+
+    """
+
+    # The value that should be passed to --auth-type
+    # to use this auth plugin. Eg. "my-auth"
+    auth_type = None
+
+    # The name of the plugin, eg. "My auth".
+    name = None
+
+    # Optional short description. Will be be shown in the help
+    # under --auth-type.
+    description = None
+
+    # This be set automatically once the plugin has been loaded.
+    package_name = None
+
+    def get_auth(self, username, password):
+        """
+        Return a ``requests.auth.AuthBase`` subclass instance.
+
+        """
+        raise NotImplementedError()

httpie/plugins/builtin.py

@@ -0,0 +1,26 @@
+import requests.auth
+
+from .base import AuthPlugin
+
+
+class BuiltinAuthPlugin(AuthPlugin):
+
+    package_name = '(builtin)'
+
+
+class BasicAuthPlugin(BuiltinAuthPlugin):
+
+    name = 'Basic HTTP auth'
+    auth_type = 'basic'
+
+    def get_auth(self, username, password):
+        return requests.auth.HTTPBasicAuth(username, password)
+
+
+class DigestAuthPlugin(BuiltinAuthPlugin):
+
+    name = 'Digest HTTP auth'
+    auth_type = 'digest'
+
+    def get_auth(self, username, password):
+        return requests.auth.HTTPDigestAuth(username, password)

httpie/plugins/manager.py

@@ -0,0 +1,35 @@
+from pkg_resources import iter_entry_points
+
+
+ENTRY_POINT_NAMES = [
+    'httpie.plugins.auth.v1'
+]
+
+
+class PluginManager(object):
+
+    def __init__(self):
+        self._plugins = []
+
+    def __iter__(self):
+        return iter(self._plugins)
+
+    def register(self, plugin):
+        self._plugins.append(plugin)
+
+    def get_auth_plugins(self):
+        return list(self._plugins)
+
+    def get_auth_plugin_mapping(self):
+        return dict((plugin.auth_type, plugin) for plugin in self)
+
+    def get_auth_plugin(self, auth_type):
+        return self.get_auth_plugin_mapping()[auth_type]
+
+    def load_installed_plugins(self):
+
+        for entry_point_name in ENTRY_POINT_NAMES:
+            for entry_point in iter_entry_points(entry_point_name):
+                plugin = entry_point.load()
+                plugin.package_name = entry_point.dist.key
+                self.register(entry_point.load())

httpie/sessions.py

@@ -0,0 +1,153 @@
+"""Persistent, JSON-serialized sessions.
+
+"""
+import re
+import os
+
+import requests
+from requests.cookies import RequestsCookieJar, create_cookie
+
+from .compat import urlsplit
+from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
+from httpie.plugins import plugin_manager
+
+
+SESSIONS_DIR_NAME = 'sessions'
+DEFAULT_SESSIONS_DIR = os.path.join(DEFAULT_CONFIG_DIR, SESSIONS_DIR_NAME)
+VALID_SESSION_NAME_PATTERN = re.compile('^[a-zA-Z0-9_.-]+$')
+# Request headers starting with these prefixes won't be stored in sessions.
+# They are specific to each request.
+# http://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Requests
+SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-']
+
+
+def get_response(session_name, requests_kwargs, config_dir, args,
+                 read_only=False):
+    """Like `client.get_response`, but applies permanent
+    aspects of the session to the request.
+
+    """
+    if os.path.sep in session_name:
+        path = os.path.expanduser(session_name)
+    else:
+        hostname = (
+            requests_kwargs['headers'].get('Host', None)
+            or urlsplit(requests_kwargs['url']).netloc.split('@')[-1]
+        )
+
+        assert re.match('^[a-zA-Z0-9_.:-]+$', hostname)
+
+        # host:port => host_port
+        hostname = hostname.replace(':', '_')
+        path = os.path.join(config_dir,
+                            SESSIONS_DIR_NAME,
+                            hostname,
+                            session_name + '.json')
+
+    session = Session(path)
+    session.load()
+
+    request_headers = requests_kwargs.get('headers', {})
+    requests_kwargs['headers'] = dict(session.headers, **request_headers)
+    session.update_headers(request_headers)
+
+    if args.auth:
+        session.auth = {
+            'type': args.auth_type,
+            'username': args.auth.key,
+            'password': args.auth.value,
+        }
+    elif session.auth:
+        requests_kwargs['auth'] = session.auth
+
+    requests_session = requests.Session()
+    requests_session.cookies = session.cookies
+
+    try:
+        response = requests_session.request(**requests_kwargs)
+    except Exception:
+        raise
+    else:
+        # Existing sessions with `read_only=True` don't get updated.
+        if session.is_new or not read_only:
+            session.cookies = requests_session.cookies
+            session.save()
+        return response
+
+
+class Session(BaseConfigDict):
+    helpurl = 'https://github.com/jkbr/httpie#sessions'
+    about = 'HTTPie session file'
+
+    def __init__(self, path, *args, **kwargs):
+        super(Session, self).__init__(*args, **kwargs)
+        self._path = path
+        self['headers'] = {}
+        self['cookies'] = {}
+        self['auth'] = {
+            'type': None,
+            'username': None,
+            'password': None
+        }
+
+    def _get_path(self):
+        return self._path
+
+    def update_headers(self, request_headers):
+        """
+        Update the session headers with the request ones while ignoring
+        certain name prefixes.
+
+        :type request_headers: dict
+
+        """
+        for name, value in request_headers.items():
+
+            if name == 'User-Agent' and value.startswith('HTTPie/'):
+                continue
+
+            for prefix in SESSION_IGNORED_HEADER_PREFIXES:
+                if name.lower().startswith(prefix.lower()):
+                    break
+            else:
+                self['headers'][name] = value
+
+    @property
+    def headers(self):
+        return self['headers']
+
+    @property
+    def cookies(self):
+        jar = RequestsCookieJar()
+        for name, cookie_dict in self['cookies'].items():
+            jar.set_cookie(create_cookie(
+                name, cookie_dict.pop('value'), **cookie_dict))
+        jar.clear_expired_cookies()
+        return jar
+
+    @cookies.setter
+    def cookies(self, jar):
+        """
+        :type jar: CookieJar
+        """
+        # http://docs.python.org/2/library/cookielib.html#cookie-objects
+        stored_attrs = ['value', 'path', 'secure', 'expires']
+        self['cookies'] = {}
+        for cookie in jar:
+            self['cookies'][cookie.name] = dict(
+                (attname, getattr(cookie, attname))
+                for attname in stored_attrs
+            )
+
+    @property
+    def auth(self):
+        auth = self.get('auth', None)
+        if not auth or not auth['type']:
+            return
+        auth_plugin = plugin_manager.get_auth_plugin(auth['type'])()
+        return auth_plugin.get_auth(auth['username'], auth['password'])
+
+    @auth.setter
+    def auth(self, auth):
+        assert set(['type', 'username', 'password']) == set(auth.keys())
+        self['auth'] = auth

httpie/solarized.py

@@ -1,74 +1,57 @@
-"""
-A Pygments_ style based on the dark background variant of Solarized_.
-
-.. _Pygments: http://pygments.org/
-.. _Solarized: http://ethanschoonover.com/solarized
-
-Copyright (c) 2011 Hank Gay
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
+# -*- coding: utf-8 -*-
 
 """
+    solarized256
+    ------------
+
+    A Pygments style inspired by Solarized's 256 color mode.
+
+    :copyright: (c) 2011 by Hank Gay, (c) 2012 by John Mastro.
+    :license: BSD, see LICENSE for more details.
+"""
+
 from pygments.style import Style
-from pygments.token import (Token, Comment, Name, Keyword, Generic, Number,
-                            Operator, String)
+from pygments.token import Token, Comment, Name, Keyword, Generic, Number, \
+    Operator, String
+
+BASE03  = "#1c1c1c"
+BASE02  = "#262626"
+BASE01  = "#4e4e4e"
+BASE00  = "#585858"
+BASE0   = "#808080"
+BASE1   = "#8a8a8a"
+BASE2   = "#d7d7af"
+BASE3   = "#ffffd7"
+YELLOW  = "#af8700"
+ORANGE  = "#d75f00"
+RED     = "#af0000"
+MAGENTA = "#af005f"
+VIOLET  = "#5f5faf"
+BLUE    = "#0087ff"
+CYAN    = "#00afaf"
+GREEN   = "#5f8700"
 
 
-BASE03 = '#002B36'
-BASE02 = '#073642'
-BASE01 = '#586E75'
-BASE00 = '#657B83'
-BASE0 = '#839496'
-BASE1 = '#93A1A1'
-BASE2 = '#EEE8D5'
-BASE3 = '#FDF6E3'
-YELLOW = '#B58900'
-ORANGE = '#CB4B16'
-RED = '#DC322F'
-MAGENTA = '#D33682'
-VIOLET = '#6C71C4'
-BLUE = '#268BD2'
-CYAN = '#2AA198'
-GREEN = '#859900'
-
-
-class SolarizedStyle(Style):
+class Solarized256Style(Style):
     background_color = BASE03
     styles = {
         Keyword: GREEN,
         Keyword.Constant: ORANGE,
         Keyword.Declaration: BLUE,
-        #Keyword.Namespace
+        Keyword.Namespace: ORANGE,
         #Keyword.Pseudo
         Keyword.Reserved: BLUE,
         Keyword.Type: RED,
 
         #Name
         Name.Attribute: BASE1,
-        Name.Builtin: YELLOW,
+        Name.Builtin: BLUE,
         Name.Builtin.Pseudo: BLUE,
         Name.Class: BLUE,
         Name.Constant: ORANGE,
         Name.Decorator: BLUE,
         Name.Entity: ORANGE,
-        Name.Exception: ORANGE,
+        Name.Exception: YELLOW,
         Name.Function: BLUE,
         #Name.Label
         #Name.Namespace
@@ -84,10 +67,10 @@ class SolarizedStyle(Style):
         String: CYAN,
         String.Backtick: BASE01,
         String.Char: CYAN,
-        String.Doc: BASE1,
+        String.Doc: CYAN,
         #String.Double
-        String.Escape: ORANGE,
-        String.Heredoc: BASE1,
+        String.Escape: RED,
+        String.Heredoc: CYAN,
         #String.Interpol
         #String.Other
         String.Regex: RED,
@@ -100,8 +83,8 @@ class SolarizedStyle(Style):
         #Number.Integer.Long
         #Number.Oct
 
-        Operator: GREEN,
-        #Operator.Word
+        Operator: BASE1,
+        Operator.Word: GREEN,
 
         #Punctuation: ORANGE,
 

httpie/utils.py

@@ -0,0 +1,46 @@
+from __future__ import division
+
+
+def humanize_bytes(n, precision=2):
+    # Author: Doug Latornell
+    # Licence: MIT
+    # URL: http://code.activestate.com/recipes/577081/
+    """Return a humanized string representation of a number of bytes.
+
+    Assumes `from __future__ import division`.
+
+    >>> humanize_bytes(1)
+    '1 byte'
+    >>> humanize_bytes(1024)
+    '1.0 kB'
+    >>> humanize_bytes(1024 * 123)
+    '123.0 kB'
+    >>> humanize_bytes(1024 * 12342)
+    '12.1 MB'
+    >>> humanize_bytes(1024 * 12342, 2)
+    '12.05 MB'
+    >>> humanize_bytes(1024 * 1234, 2)
+    '1.21 MB'
+    >>> humanize_bytes(1024 * 1234 * 1111, 2)
+    '1.31 GB'
+    >>> humanize_bytes(1024 * 1234 * 1111, 1)
+    '1.3 GB'
+
+    """
+    abbrevs = [
+        (1 << 50, 'PB'),
+        (1 << 40, 'TB'),
+        (1 << 30, 'GB'),
+        (1 << 20, 'MB'),
+        (1 << 10, 'kB'),
+        (1, 'B')
+    ]
+
+    if n == 1:
+        return '1 B'
+
+    for factor, suffix in abbrevs:
+        if n >= factor:
+            break
+
+    return '%.*f %s' % (precision, n / factor, suffix)

requirements-dev.txt

@@ -0,0 +1,3 @@
+tox
+git+git://github.com/kennethreitz/httpbin.git@7c96875e87a448f08fb1981e85eb79e77d592d98
+docutils

setup.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import codecs
 from setuptools import setup
 import httpie
 
@@ -10,29 +11,36 @@ if sys.argv[-1] == 'test':
 
 
 requirements = [
-    # Debian has only requests==0.10.1 and httpie.deb depends on that.
-    'requests>=0.10.1',
+    'requests>=2.0.0',
     'Pygments>=1.5'
 ]
-if sys.version_info[:2] in ((2, 6), (3, 1)):
-    # argparse has been added in Python 3.2 / 2.7
+try:
+    #noinspection PyUnresolvedReferences
+    import argparse
+except ImportError:
     requirements.append('argparse>=1.2.1')
+
 if 'win32' in str(sys.platform).lower():
     # Terminal colors for Windows
     requirements.append('colorama>=0.2.4')
 
 
+def long_description():
+    with codecs.open('README.rst', encoding='utf8') as f:
+        return f.read()
+
+
 setup(
     name='httpie',
     version=httpie.__version__,
     description=httpie.__doc__.strip(),
-    long_description=open('README.rst').read(),
+    long_description=long_description(),
     url='http://httpie.org/',
     download_url='https://github.com/jkbr/httpie',
     author=httpie.__author__,
     author_email='jakub@roztocil.name',
     license=httpie.__licence__,
-    packages=['httpie'],
+    packages=['httpie', 'httpie.plugins'],
     entry_points={
         'console_scripts': [
             'http = httpie.__main__:main',

tests/fixtures/test.json

@@ -0,0 +1,3 @@
+{
+    "hello": "world"
+}

tox.ini

@@ -4,16 +4,10 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py26, py27, py32, pypy
+envlist = py26, py27, py33, pypy
 
 [testenv]
 commands = {envpython} setup.py test
 
 [testenv:py26]
 deps = argparse
-
-[testenv:py30]
-deps = argparse
-
-[testenv:py31]
-deps = argparse