Cleanup

make test fails when finding .rst files from site packages installed
in the virtual environment

.editorconfig

@@ -1,4 +1,4 @@
-# http://editorconfig.org
+# https://editorconfig.org
 root = true
 
 [*]

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: https://paypal.me/roztocil

.github/workflows/build.yml

@@ -0,0 +1,37 @@
+name: Build
+on: [push]
+jobs:
+  extras:
+    # Run coverage and extra tests only once
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - run: pip install --upgrade pip
+      - run: make install
+      - run: make pycodestyle
+      - run: make test-cover
+      - run: make codecov-upload
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_REPO_TOKEN }}
+      - run: make test-dist
+  test:
+    # Run core HTTPie tests everywhere
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        python-version: [3.6, 3.7, 3.8]
+        exclude:
+          - os: windows-latest
+            python-version: 3.8
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python -m pip install --upgrade pip
+      - run: pip install --upgrade --editable .
+      - run: python setup.py test

.gitignore

@@ -1,13 +1,142 @@
 .DS_Store
 .idea/
-__pycache__/
-dist/
-build/
 *.egg-info
 .cache/
-.tox/
-.coverage
 *.pyc
-*.egg
 htmlcov
+
+
+##############################################################################
+# The bellow is GitHub template for Python project. gitignore.
+# <https://github.com/github/gitignore/blob/master/Python.gitignore>
+##############################################################################
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
 .pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

.travis.yml

@@ -1,96 +0,0 @@
-# <https://travis-ci.org/jakubroztocil/httpie>
-sudo: false
-language: python
-os:
-  - linux
-env:
-  global:
-    - NEWEST_PYTHON=3.7
-python:
-  # <https://docs.travis-ci.com/user/languages/python/>
-
-  - 2.7
-
-  # Python 3.4 fails installing packages
-  # <https://travis-ci.org/jakubroztocil/httpie/jobs/403263566#L636>
-  # - 3.4
-
-  - 3.5
-  - 3.6
-  # - 3.7  # is done in the matrix below as described in travis-ci/travis-ci#9069
-  - pypy
-
-  # pypy3 currently fails because of a Flask issue
-  # - pypy3
-
-cache: pip
-matrix:
-  include:
-    # Add manually defined OS X builds
-    # <https://docs.travis-ci.com/user/multi-os/#Python-example-(unsupported-languages)>
-    - os: osx
-      language: generic
-      env:
-        # Stock OSX Python
-        - TOXENV=py27-osx-builtin
-        - BREW_PYTHON_PACKAGE=
-    - os: osx
-      language: generic
-      env:
-        # Latest Python 2.7 from Homebrew
-        - TOXENV=py27
-        - BREW_PYTHON_PACKAGE=python@2
-    - os: osx
-      language: generic
-      env:
-        # Latest Python 3.x from Homebrew
-        - TOXENV=py37  # <= needs to be kept up-to-date to reflect latest minor version
-        - BREW_PYTHON_PACKAGE=python@3
-    # Travis Python 3.7 must run sudo on 
-    - os: linux
-      python: 3.7
-      env: TOXENV=py37
-      sudo: true    # Required for Python 3.7
-      dist: xenial  # Required for Python 3.7
-    # Add a codestyle-only build
-    - os: linux
-      python: 3.6
-      env: CODESTYLE_ONLY=true
-install:
-    - |
-      if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
-          if [[ -n "$BREW_PYTHON_PACKAGE" ]]; then
-              brew update
-              if ! brew list --versions "$BREW_PYTHON_PACKAGE" >/dev/null; then
-                  brew install "$BREW_PYTHON_PACKAGE"
-              elif ! brew outdated "$BREW_PYTHON_PACKAGE"; then
-                  brew upgrade "$BREW_PYTHON_PACKAGE"
-              fi
-          fi
-          sudo pip2 install tox
-      fi
-script:
-    - |
-      if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
-          if [[ $CODESTYLE_ONLY ]]; then
-              make pycodestyle
-          else
-              make test
-          fi
-      else
-          PATH="/usr/local/bin:$PATH" tox -e "$TOXENV"
-      fi
-after_success:
-    - |
-      if [[ $TRAVIS_PYTHON_VERSION == $NEWEST_PYTHON && $TRAVIS_OS_NAME == 'linux' ]]; then
-          make coveralls
-      fi
-notifications:
-  webhooks:
-    # options: [always|never|change] default: always
-    on_success: always
-    on_failure: always
-    on_start: always
-    urls:
-      # https://gitter.im/jkbrzt/httpie
-      - https://webhooks.gitter.im/e/c42fcd359a110d02830b

AUTHORS.rst

@@ -36,5 +36,7 @@ Patches and ideas
 * `Dennis Brakhane <https://github.com/brakhane>`_
 * `Matt Layman <https://github.com/mblayman>`_
 * `Edward Yang <https://github.com/honorabrutroll>`_
+* `Aleksandr Vinokurov <https://github.com/aleksandr-vin>`_
+* `Jeff Byrnes <https://github.com/jeffbyrnes>`_
 
 

CHANGELOG.rst

@@ -2,8 +2,72 @@
 Change Log
 ==========
 
-This document records all notable changes to `HTTPie <http://httpie.org>`_.
-This project adheres to `Semantic Versioning <http://semver.org/>`_.
+This document records all notable changes to `HTTPie <https://httpie.org>`_.
+This project adheres to `Semantic Versioning <https://semver.org/>`_.
+
+
+`2.0.0`_ (2020-01-12)
+-------------------------
+* Removed Python 2.7 support (`EOL Jan 2020 <https://www.python.org/doc/sunset-python-2/>`_).
+* Added ``--offline`` to allow building an HTTP request and printing it but not
+  actually sending it over the network.
+* Replaced the old collect-all-then-process handling of HTTP communication
+  with one-by-one processing of each HTTP request or response as they become
+  available. This means that you can see headers immediately,
+  see what is being send even when the request fails, etc.
+* Removed automatic config file creation to avoid concurrency issues.
+* Removed the default 30-second connection ``--timeout`` limit.
+* Removed Python’s default limit of 100 response headers.
+* Added ``--max-headers`` to allow setting the max header limit.
+* Added ``--compress`` to allow request body compression.
+* Added ``--ignore-netrc`` to allow bypassing credentials from ``.netrc``.
+* Added ``https`` alias command with ``https://`` as the default scheme.
+* Added ``$ALL_PROXY`` documentation.
+* Added type annotations throughout the codebase.
+* Added ``tests/`` to the PyPi package for the convenience of
+  downstream package maintainers.
+* Fixed an error when ``stdin`` was a closed fd.
+* Improved ``--debug`` output formatting.
+
+
+`1.0.3`_ (2019-08-26)
+---------------------
+
+* Fixed CVE-2019-10751 — the way the output filename is generated for
+  ``--download`` requests without ``--output`` resulting in a redirect has
+  been changed to only consider the initial URL as the base for the generated
+  filename, and not the final one. This fixes a potential security issue under
+  the following scenario:
+
+  1. A ``--download`` request with no explicit ``--output`` is made (e.g.,
+     ``$ http -d example.org/file.txt``), instructing httpie to
+     `generate the output filename <https://httpie.org/doc#downloaded-filename>`_
+     from the ``Content-Disposition`` response header, or from the URL if the header
+     is not provided.
+  2. The server handling the request has been modified by an attacker and
+     instead of the expected response the URL returns a redirect to another
+     URL, e.g., ``attacker.example.org/.bash_profile``, whose response does
+     not provide  a ``Content-Disposition`` header (i.e., the base for the
+     generated filename becomes ``.bash_profile`` instead of ``file.txt``).
+  3. Your current directory doesn’t already contain ``.bash_profile``
+     (i.e., no unique suffix is added to the generated filename).
+  4. You don’t notice the potentially unexpected output filename
+     as reported by httpie in the console output
+     (e.g., ``Downloading 100.00 B to ".bash_profile"``).
+
+  Reported by Raul Onitza and Giulio Comi.
+
+
+`1.0.2`_ (2018-11-14)
+-------------------------
+
+* Fixed tests for installation with pyOpenSSL.
+
+
+`1.0.1`_ (2018-11-14)
+-------------------------
+
+* Removed external URL calls from tests.
 
 
 `1.0.0`_ (2018-11-02)
@@ -308,13 +372,13 @@ This project adheres to `Semantic Versioning <http://semver.org/>`_.
 * Many improvements and bug fixes
 
 
-`0.1`_ (2012-02-25)
--------------------
+`0.1.0`_ (2012-02-25)
+---------------------
 
 * Initial public release
 
 
-.. _`0.1`: https://github.com/jakubroztocil/httpie/commit/b966efa
+.. _`0.1.0`: https://github.com/jakubroztocil/httpie/commit/b966efa
 .. _0.1.4: https://github.com/jakubroztocil/httpie/compare/b966efa...0.1.4
 .. _0.1.5: https://github.com/jakubroztocil/httpie/compare/0.1.4...0.1.5
 .. _0.1.6: https://github.com/jakubroztocil/httpie/compare/0.1.5...0.1.6
@@ -341,3 +405,8 @@ This project adheres to `Semantic Versioning <http://semver.org/>`_.
 .. _0.9.8: https://github.com/jakubroztocil/httpie/compare/0.9.6...0.9.8
 .. _0.9.9: https://github.com/jakubroztocil/httpie/compare/0.9.8...0.9.9
 .. _1.0.0: https://github.com/jakubroztocil/httpie/compare/0.9.9...1.0.0
+.. _1.0.1: https://github.com/jakubroztocil/httpie/compare/1.0.0...1.0.1
+.. _1.0.2: https://github.com/jakubroztocil/httpie/compare/1.0.1...1.0.2
+.. _1.0.3: https://github.com/jakubroztocil/httpie/compare/1.0.2...1.0.3
+.. _2.0.0: https://github.com/jakubroztocil/httpie/compare/1.0.3...2.0.0
+.. _2.1.0-dev: https://github.com/jakubroztocil/httpie/compare/2.0.0...master

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at jakub@roztocil.co. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.rst

@@ -29,25 +29,77 @@ is a bigger one, it's always good to discuss before you start working on
 it.
 
 
-Creating Development Environment
+Development Environment
 --------------------------------
 
+
+Getting the code
+****************
+
 Go to https://github.com/jakubroztocil/httpie and fork the project repository.
 
 
 .. code-block:: bash
 
-    git clone https://github.com/<YOU>/httpie
+    # Clone your fork
+    git clone git@github.com:<YOU>/httpie.git
 
+    # Enter the project directory
     cd httpie
 
+    # Create a branch for your changes
     git checkout -b my_topical_branch
 
-    # (Recommended: create a new virtualenv)
 
-    # Install dev. requirements and also HTTPie (in editable mode
-    # so that the `http' command will point to your working copy):
-    make init
+Setup
+*****
+
+The `Makefile`_ contains a bunch of tasks to get you started. Just run
+the following command, which:
+
+
+* Creates an isolated Python virtual environment inside ``./venv``
+  (via the standard library `venv`_ tool);
+* installs all dependencies and also installs HTTPie
+  (in editable mode so that the ``http`` command will point to your
+  working copy).
+* and runs tests (It is the same as running ``make install test``).
+
+
+.. code-block:: bash
+
+    make
+
+
+
+Python virtual environment
+**************************
+
+Activate the Python virtual environment—created via the ``make install``
+task during `setup`_—for your active shell session using the following command:
+
+.. code-block:: bash
+
+    source venv/bin/activate
+
+(If you use ``virtualenvwrapper``, you can also use ``workon httpie`` to
+activate the environment — we have created a symlink for you. It’s a bit of
+a hack but it works™.)
+
+You should now see ``(httpie)`` next to your shell prompt, and
+the ``http`` should point to you development copy:
+
+.. code-block::
+
+    (httpie) ~/Code/httpie $ which http
+    /Users/jakub/Code/httpie/venv/bin/http
+    (httpie) ~/Code/httpie $ http --version
+    2.0.0-dev
+
+(Btw, you don’t need to activate the virtual environment if you just want
+run some of the ``make`` tasks. You can also invoke the development
+version of HTTPie directly with ``./venv/bin/http`` without having to activate
+the environment first. The same goes for ``./venv/bin/py.test``, etc.).
 
 
 Making Changes
@@ -57,36 +109,47 @@ Please make sure your changes conform to `Style Guide for Python Code`_ (PEP8)
 and that ``make pycodestyle`` passes.
 
 
-Testing
--------
+Testing & CI
+------------
 
-Before opening a pull requests, please make sure the `test suite`_ passes
-in all of the `supported Python environments`_. You should also add tests
-for any new features and bug fixes.
+Please add tests for any new features and bug fixes.
 
-HTTPie uses `pytest`_ and `Tox`_ for testing.
+When you open a pull request,
+`Github Actions <https://github.com/jakubroztocil/httpie/actions>`_
+will automatically run HTTPie’s `test suite`_ against your code
+so please make sure all checks pass.
 
 
-Running all tests:
-******************
+Running tests locally
+*********************
+
+HTTPie uses the `pytest`_ runner. It also uses `Tox`_ which allows you to run
+tests on multiple Python versions even when testing locally.
+
 
 .. code-block:: bash
 
-    # Run all tests on the current Python interpreter with coverage
+    # Run tests on the current Python interpreter with coverage.
     make test
 
+    # Run tests with coverage
+    make test-cover
+
     # Run all tests in all of the supported and available Pythons via Tox
     make test-tox
 
-    # Run all tests for code as well as packaging, etc.
-    make test-all
-
     # Test PEP8 compliance
     make pycodestyle
 
+    # Run extended tests — for code as well as .rst files syntax, packaging, etc.
+    make test-all
 
-Running specific tests:
-***********************
+
+Running specific tests
+**********************
+
+After you have activated your virtual environment (see `setup`_), you
+can run specific tests from the terminal:
 
 .. code-block:: bash
 
@@ -104,7 +167,9 @@ Running specific tests:
 -----
 
 See `Makefile`_ for additional development utilities.
-Don't forget to add yourself to `AUTHORS`_!
+
+
+Finally, don't forget to add yourself to `AUTHORS`_!
 
 
 .. _Tox: http://tox.testrun.org
@@ -112,6 +177,7 @@ Don't forget to add yourself to `AUTHORS`_!
 .. _existing issues: https://github.com/jakubroztocil/httpie/issues?state=open
 .. _AUTHORS: https://github.com/jakubroztocil/httpie/blob/master/AUTHORS.rst
 .. _Makefile: https://github.com/jakubroztocil/httpie/blob/master/Makefile
-.. _pytest: http://pytest.org/
-.. _Style Guide for Python Code: http://python.org/dev/peps/pep-0008/
+.. _venv: https://docs.python.org/3/library/venv.html
+.. _pytest: https://pytest.org/
+.. _Style Guide for Python Code: https://python.org/dev/peps/pep-0008/
 .. _test suite: https://github.com/jakubroztocil/httpie/tree/master/tests

LICENSE

@@ -1,4 +1,4 @@
-Copyright © 2012-2017 Jakub Roztocil <jakub@roztocil.co>
+Copyright © 2012-2019 Jakub Roztocil <jakub@roztocil.co>
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
@@ -10,14 +10,14 @@ modification, are permitted provided that the following conditions are met:
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
 
-    3. Neither the name of The author nor the names of its contributors may
-       be used to endorse or promote products derived from this software
+    3. Neither the name of the copyright holder nor the names of its contributors
+       may be used to endorse or promote products derived from this software
        without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE FOR
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON

MANIFEST.in

@@ -2,3 +2,6 @@ include LICENSE
 include README.rst
 include CHANGELOG.rst
 include AUTHORS.rst
+
+# <https://github.com/jakubroztocil/httpie/issues/182>
+recursive-include tests/ *

Makefile

@@ -2,44 +2,91 @@
 # See ./CONTRIBUTING.rst
 ###############################################################################
 
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 VERSION=$(shell grep __version__ httpie/__init__.py)
-REQUIREMENTS="requirements-dev.txt"
-TAG="\n\n\033[0;32m\#\#\# "
-END=" \#\#\# \033[0m\n"
+REQUIREMENTS=requirements-dev.txt
+H1="\n\n\033[0;32m\#\#\# "
+H1END=" \#\#\# \033[0m\n"
 
 
-all: test
+# Only used to create our venv.
+SYSTEM_PYTHON=python3
+
+VENV_ROOT=venv
+VENV_BIN=$(VENV_ROOT)/bin
+VENV_PIP=$(VENV_BIN)/pip3
+VENV_PYTHON=$(VENV_BIN)/python
 
 
-init: uninstall-httpie
-	@echo $(TAG)Installing dev requirements$(END)
-	pip install --upgrade -r $(REQUIREMENTS)
+export PATH := $(VENV_BIN):$(PATH)
 
-	@echo $(TAG)Installing HTTPie$(END)
-	pip install --upgrade --editable .
+
+all: uninstall-httpie install test
+
+
+install: venv
+	@echo $(H1)Installing dev requirements$(H1END)
+	$(VENV_PIP) install --upgrade -r $(REQUIREMENTS)
+
+	@echo $(H1)Installing HTTPie$(H1END)
+	$(VENV_PIP) install --upgrade --editable .
 
 	@echo
 
 clean:
-	@echo $(TAG)Cleaning up$(END)
+	@echo $(H1)Cleaning up$(H1END)
+	rm -rf $(VENV_ROOT)
+	# Remove symlink for virtualenvwrapper, if we’ve created one.
+	[ -n "$(WORKON_HOME)" -a -L "$(WORKON_HOME)/httpie" -a -f "$(WORKON_HOME)/httpie" ] && rm $(WORKON_HOME)/httpie || true
 	rm -rf .tox *.egg dist build .coverage .cache .pytest_cache httpie.egg-info
-	find . -name '__pycache__' -delete -print -o -name '*.pyc' -delete -print
+	find . -name '__pycache__' -delete -o -name '*.pyc' -delete
 	@echo
 
 
+venv:
+	@echo $(H1)Creating a Python environment $(VENV_ROOT) $(H1END)
+
+	$(SYSTEM_PYTHON) -m venv --prompt httpie $(VENV_ROOT)
+
+	@echo
+	@echo done.
+	@echo
+	@echo To active it manually, run:
+	@echo
+	@echo "    source $(VENV_BIN)/activate"
+	@echo
+	@echo '(learn more: https://docs.python.org/3/library/venv.html)'
+	@echo
+	@if [ -n "$(WORKON_HOME)" ]; then \
+		echo $(ROOT_DIR) >  $(VENV_ROOT)/.project; \
+		if [ ! -d $(WORKON_HOME)/httpie -a ! -L $(WORKON_HOME)/httpie ]; then \
+			ln -s $(ROOT_DIR)/$(VENV_ROOT) $(WORKON_HOME)/httpie ; \
+			echo ''; \
+			echo 'Since you use virtualenvwrapper, we created a symlink'; \
+			echo 'so you can also use "workon httpie" to activate the venv.'; \
+			echo ''; \
+		fi; \
+	fi
+
+
 ###############################################################################
 # Testing
 ###############################################################################
 
 
-test: init
-	@echo $(TAG)Running tests on the current Python interpreter with coverage $(END)
-	py.test --cov ./httpie --cov ./tests --doctest-modules --verbose ./httpie ./tests
+test:
+	@echo $(H1)Running tests$(HEADER_EXTRA)$(H1END)
+	$(VENV_BIN)/py.test $(COV) ./httpie $(COV) ./tests --doctest-modules --verbose ./httpie ./tests
 	@echo
 
 
+test-cover: COV=--cov
+test-cover: HEADER_EXTRA=' (with coverage)'
+test-cover: test
+
+
 # test-all is meant to test everything — even this Makefile
-test-all: uninstall-all clean init test test-tox test-dist pycodestyle
+test-all: clean install test test-tox test-dist pycodestyle
 	@echo
 
 
@@ -47,37 +94,41 @@ test-dist: test-sdist test-bdist-wheel
 	@echo
 
 
-test-tox: init
-	@echo $(TAG)Running tests on all Pythons via Tox$(END)
-	tox
+test-tox: uninstall-httpie install
+	@echo $(H1)Running tests on all Pythons via Tox$(H1END)
+	$(VENV_BIN)/tox
 	@echo
 
 
-test-sdist: clean uninstall-httpie
-	@echo $(TAG)Testing sdist build an installation$(END)
-	python setup.py sdist
-	pip install --force-reinstall --upgrade dist/*.gz
-	which http
+test-sdist: clean venv
+	@echo $(H1)Testing sdist build an installation$(H1END)
+	$(VENV_PYTHON) setup.py sdist
+	$(VENV_PIP) install --force-reinstall --upgrade dist/*.gz
+	$(VENV_BIN)/http --version
 	@echo
 
 
-test-bdist-wheel: clean uninstall-httpie
-	@echo $(TAG)Testing wheel build an installation$(END)
-	python setup.py bdist_wheel
-	pip install --force-reinstall --upgrade dist/*.whl
-	which http
+test-bdist-wheel: clean venv
+	@echo $(H1)Testing wheel build an installation$(H1END)
+	$(VENV_PIP) install wheel
+	$(VENV_PYTHON) setup.py bdist_wheel
+	$(VENV_PIP) install --force-reinstall --upgrade dist/*.whl
+	$(VENV_BIN)/http --version
 	@echo
 
 
 pycodestyle:
-	which pycodestyle || pip install pycodestyle
-	pycodestyle
+	@echo $(H1)Running pycodestyle$(H1END)
+	@[ -f $(VENV_BIN)/pycodestyle ] || $(VENV_PIP) install pycodestyle
+	$(VENV_BIN)/pycodestyle httpie/ tests/ extras/ *.py
 	@echo
 
 
-coveralls:
-	which coveralls || pip install python-coveralls
-	coveralls
+codecov-upload:
+	@echo $(H1)Running codecov$(H1END)
+	@[ -f $(VENV_BIN)/codecov ] || $(VENV_PIP) install codecov
+	# $(VENV_BIN)/codecov --required
+	$(VENV_BIN)/codecov
 	@echo
 
 
@@ -90,12 +141,11 @@ publish: test-all publish-no-test
 
 
 publish-no-test:
-	@echo $(TAG)Testing wheel build an installation$(END)
+	@echo $(H1)Testing wheel build an installation$(H1END)
 	@echo "$(VERSION)"
 	@echo "$(VERSION)" | grep -q "dev" && echo '!!!Not publishing dev version!!!' && exit 1 || echo ok
-	python setup.py register
-	python setup.py sdist upload
-	python setup.py bdist_wheel upload
+	$(VENV_PYTHON) setup.py sdist bdist_wheel
+	$(VENV_BIN)/twine upload dist/*
 	@echo
 
 
@@ -105,29 +155,40 @@ publish-no-test:
 ###############################################################################
 
 uninstall-httpie:
-	@echo $(TAG)Uninstalling httpie$(END)
-	- pip uninstall --yes httpie &2>/dev/null
+	@echo $(H1)Uninstalling httpie$(H1END)
+	- $(VENV_PIP) uninstall --yes httpie &2>/dev/null
 
 	@echo "Verifying…"
-	cd .. && ! python -m httpie --version &2>/dev/null
+	cd .. && ! $(VENV_PYTHON) -m httpie --version &2>/dev/null
 
 	@echo "Done"
 	@echo
 
 
-uninstall-all: uninstall-httpie
+###############################################################################
+# Docs
+###############################################################################
 
-	@echo $(TAG)Uninstalling httpie requirements$(END)
-	- pip uninstall --yes pygments requests
-
-	@echo $(TAG)Uninstalling development requirements$(END)
-	- pip uninstall --yes -r $(REQUIREMENTS)
+pdf:
+	# NOTE: rst2pdf needs to be installed manually and against a Python 2
+	@echo "Converting README.rst to PDF…"
+	rst2pdf \
+		--strip-elements-with-class=no-pdf \
+		README.rst \
+		-o README.pdf
+	@echo "Done"
+	@echo
 
 
 ###############################################################################
-# Utils
+# Homebrew
 ###############################################################################
 
+brew-deps:
+	extras/brew-deps.py
 
-homebrew-formula-vars:
-	extras/get-homebrew-formula-vars.py
+brew-test:
+	- brew uninstall httpie
+	brew install --build-from-source ./extras/httpie.rb
+	brew test httpie
+	brew audit --strict httpie

README.rst

@@ -9,18 +9,17 @@ colorized output. HTTPie can be used for testing, debugging, and
 generally interacting with HTTP servers.
 
 
-.. class:: no-web
+.. class:: no-web no-pdf
 
-    .. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.png
-        :alt: HTTPie compared to cURL
-        :width: 100%
-        :align: center
+    |pypi| |build| |coverage| |downloads| |gitter|
 
 
 .. class:: no-web no-pdf
 
-|pypi| |unix_build| |coverage| |gitter|
-
+    .. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.gif
+        :alt: HTTPie in action
+        :width: 100%
+        :align: center
 
 
 .. contents::
@@ -32,6 +31,7 @@ generally interacting with HTTP servers.
 Main features
 =============
 
+
 * Expressive and intuitive syntax
 * Formatted and colorized terminal output
 * Built-in JSON support
@@ -41,13 +41,20 @@ Main features
 * Custom headers
 * Persistent sessions
 * Wget-like downloads
-* Python 2.7 and 3.x support
 * Linux, macOS and Windows support
 * Plugins
 * Documentation
 * Test coverage
 
 
+.. class:: no-web
+
+    .. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.png
+        :alt: HTTPie compared to cURL
+        :width: 100%
+        :align: center
+
+
 Installation
 ============
 
@@ -56,7 +63,7 @@ macOS
 -----
 
 
-On macOS, HTTPie can be installed via `Homebrew <http://brew.sh/>`_
+On macOS, HTTPie can be installed via `Homebrew <https://brew.sh/>`_
 (recommended):
 
 .. code-block:: bash
@@ -119,12 +126,7 @@ and always provides the latest version) is to use `pip`_:
 Python version
 --------------
 
-Although Python 2.7 is supported as well, it is strongly recommended to
-install HTTPie against the latest Python 3.x whenever possible. That will
-ensure that some of the newer HTTP features, such as
-`SNI (Server Name Indication)`_, work out of the box.
-Python 3 is the default for Homebrew installations starting with version 0.9.4.
-To see which version HTTPie uses, run ``http --debug``.
+Python version 3.6 or greater is required.
 
 
 Unstable version
@@ -134,7 +136,10 @@ You can also install the latest unreleased development version directly from
 the ``master`` branch on GitHub.  It is a work-in-progress of a future stable
 release so the experience might be not as smooth.
 
-|unix_build|
+
+.. class:: no-pdf
+
+|build|
 
 
 On macOS you can install it with Homebrew:
@@ -289,20 +294,22 @@ Querystring parameters
 
 If you find yourself manually constructing URLs with querystring parameters
 on the terminal, you may appreciate the ``param==value`` syntax for appending
-URL parameters. With that, you don't have to worry about escaping the ``&``
-separators for your shell. Also, special characters in parameter values,
-will also automatically escaped (HTTPie otherwise expects the URL to be
-already escaped). To search for ``HTTPie logo`` on Google Images you could use
-this command:
+URL parameters.
+
+With that, you don't have to worry about escaping the ``&``
+separators for your shell. Additionally, any special characters in the
+parameter name or value get automatically URL-escaped
+(as opposed to parameters specified in the full URL, which HTTPie doesn’t
+modify).
 
 .. code-block:: bash
 
-    $ http www.google.com search=='HTTPie logo' tbm==isch
+    $ http https://api.github.com/search/repositories q==httpie per_page==1
 
 
 .. code-block:: http
 
-    GET /?search=HTTPie+logo&tbm=isch HTTP/1.1
+    GET /search/repositories?q=httpie&per_page=1 HTTP/1.1
 
 
 
@@ -346,17 +353,33 @@ If the port is omitted, then port 80 is assumed.
     Host: localhost
 
 
-Custom default scheme
+Other default schemes
 ---------------------
 
-You can use the ``--default-scheme <URL_SCHEME>`` option to create
-shortcuts for other protocols than HTTP:
+When HTTPie is invoked as ``https`` then the default scheme is ``https://``
+(``$ https example.org`` will make a request to ``https://example.org``).
+
+You can also use the ``--default-scheme <URL_SCHEME>`` option to create
+shortcuts for other protocols than HTTP (possibly supported via plugins).
+Example for the `httpie-unixsocket <https://github.com/httpie/httpie-unixsocket>`_ plugin:
 
 .. code-block:: bash
 
-    $ alias https='http --default-scheme=https'
+    # Before
+    $ http http+unix://%2Fvar%2Frun%2Fdocker.sock/info
 
 
+.. code-block:: bash
+
+    # Create an alias
+    $ alias http-unix='http --default-scheme="http+unix"'
+
+
+.. code-block:: bash
+
+    # Now the scheme can be omitted
+    $ http-unix %2Fvar%2Frun%2Fdocker.sock/info
+
 Request items
 =============
 
@@ -516,7 +539,7 @@ fields using ``=@`` and ``:=@``:
         "married": false,
         "name": "John",
         "bookmarks": {
-            "HTTPie": "http://httpie.org",
+            "HTTPie": "https://httpie.org",
         }
     }
 
@@ -589,7 +612,7 @@ To set custom headers you can use the ``Header:Value`` notation:
 .. code-block:: bash
 
     $ http example.org  User-Agent:Bacon/1.0  'Cookie:valued-visitor=yes;foo=bar'  \
-        X-Foo:Bar  Referer:http://httpie.org/
+        X-Foo:Bar  Referer:https://httpie.org/
 
 
 .. code-block:: http
@@ -599,7 +622,7 @@ To set custom headers you can use the ``Header:Value`` notation:
     Accept-Encoding: gzip, deflate
     Cookie: valued-visitor=yes;foo=bar
     Host: example.org
-    Referer: http://httpie.org/
+    Referer: https://httpie.org/
     User-Agent: Bacon/1.0
     X-Foo: Bar
 
@@ -643,6 +666,19 @@ To send a header with an empty value, use ``Header;``:
     $ http httpbin.org/headers 'Header;'
 
 
+Limiting response headers
+-------------------------
+
+The ``--max-headers=n`` options allows you to control the number of headers
+HTTPie reads before giving up (the default ``0``, i.e., there’s no limit).
+
+
+.. code-block:: bash
+
+    $ http --max-headers=100 httpbin.org/get
+
+
+
 Cookies
 =======
 
@@ -742,7 +778,10 @@ Password prompt
 ``.netrc``
 ----------
 
-Authentication information from your ``~/.netrc`` file is honored as well:
+Authentication information from your ``~/.netrc``
+file is by default honored as well.
+
+For example:
 
 .. code-block:: bash
 
@@ -751,10 +790,20 @@ Authentication information from your ``~/.netrc`` file is honored as well:
     login httpie
     password test
 
+.. code-block:: bash
+
     $ http httpbin.org/basic-auth/httpie/test
     HTTP/1.1 200 OK
     [...]
 
+This can be disabled with the ``--ignore-netrc`` option:
+
+.. code-block:: bash
+
+    $ http --ignore-netrc httpbin.org/basic-auth/httpie/test
+    HTTP/1.1 401 UNAUTHORIZED
+    [...]
+
 
 Auth plugins
 ------------
@@ -846,10 +895,10 @@ With Basic authentication:
 Environment variables
 ---------------------
 
-You can also configure proxies by environment variables ``HTTP_PROXY`` and
-``HTTPS_PROXY``, and the underlying Requests library will pick them up as well.
-If you want to disable proxies configured through the environment variables for
-certain hosts, you can specify them in ``NO_PROXY``.
+You can also configure proxies by environment variables ``ALL_PROXY``,
+``HTTP_PROXY`` and ``HTTPS_PROXY``, and the underlying Requests library will
+pick them up as well. If you want to disable proxies configured through
+the environment variables for certain hosts, you can specify them in ``NO_PROXY``.
 
 In your ``~/.bash_profile``:
 
@@ -863,7 +912,9 @@ In your ``~/.bash_profile``:
 SOCKS
 -----
 
-Homebrew-installed HTTPie comes with SOCKS proxy support out of the box. To enable SOCKS proxy support for non-Homebrew  installations, you'll need to install ``requests[socks]`` manually using ``pip``:
+Homebrew-installed HTTPie comes with SOCKS proxy support out of the box.
+To enable SOCKS proxy support for non-Homebrew  installations, you'll
+might need to install ``requests[socks]`` manually using ``pip``:
 
 
 .. code-block:: bash
@@ -936,26 +987,6 @@ available set of protocols may vary depending on your OpenSSL installation.)
     $ http --ssl=ssl3 https://vulnerable.example.org
 
 
-SNI (Server Name Indication)
-----------------------------
-
-If you use HTTPie with `Python version`_ lower than 2.7.9
-(can be verified with ``http --debug``) and need to talk to servers that
-use SNI (Server Name Indication) you need to install some additional
-dependencies:
-
-.. code-block:: bash
-
-    $ pip install --upgrade requests[security]
-
-
-You can use the following command to test SNI support:
-
-.. code-block:: bash
-
-    $ http https://sni.velox.ch
-
-
 Output options
 ==============
 
@@ -1103,6 +1134,13 @@ You can use ``echo`` for simple data:
     $ echo '{"name": "John"}' | http PATCH example.com/person/1 X-API-Token:123
 
 
+You can also use a Bash *here string*:
+
+.. code-block:: bash
+
+    $ http example.com/ <<<'{"name": "John"}'
+
+
 You can even pipe web services together using HTTPie:
 
 .. code-block:: bash
@@ -1297,13 +1335,25 @@ is being saved to a file.
     Done. 251.30 kB in 2.73862s (91.76 kB/s)
 
 
-Downloaded file name
+Downloaded filename
 --------------------
 
-If not provided via ``--output, -o``, the output filename will be determined
-from ``Content-Disposition`` (if available), or from the URL and
-``Content-Type``. If the guessed filename already exists, HTTPie adds a unique
-suffix to it.
+There are three mutually exclusive ways through which HTTPie determines
+the output filename (with decreasing priority):
+
+1. You can explicitly provide it via ``--output, -o``.
+   The file gets overwritten if it already exists
+   (or appended to with ``--continue, -c``).
+2. The server may specify the filename in the optional ``Content-Disposition``
+   response header. Any leading dots are stripped from a server-provided filename.
+3. The last resort HTTPie uses is to generate the filename from a combination
+   of the request URL and the response ``Content-Type``.
+   The initial URL is always used as the basis for
+   the generated filename — even if there has been one or more redirects.
+
+
+To prevent data loss by overwriting, HTTPie adds a unique numerical suffix to the
+filename when necessary (unless specified with ``--output, -o``).
 
 
 Piping while downloading
@@ -1439,7 +1489,8 @@ To create or reuse a different session, simple specify a different name:
 
     $ http --session=user2 -a user2:password example.org X-Bar:Foo
 
-Named sessions' data is stored in JSON files in the directory
+Named sessions’s data is stored in JSON files in the the ``sessions``
+subdirectory of the `config`_ directory:
 ``~/.httpie/sessions/<host>/<name>.json``
 (``%APPDATA%\httpie\sessions\<host>\<name>.json`` on Windows).
 
@@ -1469,46 +1520,61 @@ exchange once it is created, specify the session name via
 Config
 ======
 
-HTTPie uses a simple JSON config file.
+HTTPie uses a simple ``config.json`` file. The file doesn’t exist by default
+but you can create it manually.
 
 
-
-Config file location
---------------------
-
+Config file directory
+---------------------
 
 The default location of the configuration file is ``~/.httpie/config.json``
-(or ``%APPDATA%\httpie\config.json`` on Windows). The config directory
-location can be changed by setting the ``HTTPIE_CONFIG_DIR``
-environment variable. To view the exact location run ``http --debug``.
+(or ``%APPDATA%\httpie\config.json`` on Windows).
+
+The config directory can be changed by setting the ``$HTTPIE_CONFIG_DIR``
+environment variable:
+
+.. code-block:: bash
+
+    $ export HTTPIE_CONFIG_DIR=/tmp/httpie
+    $ http example.org
+
+To view the exact location run ``http --debug``.
+
 
 Configurable options
 --------------------
 
-The JSON file contains an object with the following keys:
+Currently HTTPie offers a single configurable option:
 
 
 ``default_options``
 ~~~~~~~~~~~~~~~~~~~
 
-
 An ``Array`` (by default empty) of default options that should be applied to
 every invocation of HTTPie.
 
-For instance, you can use this option to change the default style and output
-options: ``"default_options": ["--style=fruity", "--body"]`` Another useful
-default option could be ``"--session=default"`` to make HTTPie always
-use `sessions`_ (one named ``default`` will automatically be used).
-Or you could change the implicit request content type from JSON to form by
-adding ``--form`` to the list.
+For instance, you can use this config option to change your default color theme:
 
 
-``__meta__``
-~~~~~~~~~~~~
+.. code-block:: bash
 
-HTTPie automatically stores some of its metadata here. Please do not change.
+    $ cat ~/.httpie/config.json
 
 
+.. code-block:: json
+
+    {
+        "default_options": [
+          "--style=fruity"
+        ]
+    }
+
+
+Even though it is technically possible to include there any of HTTPie’s
+options, it is not recommended to modify the default behaviour in a way
+that would break your compatibility with the wider world as that can
+generate a lot of confusion.
+
 
 Un-setting previously specified options
 ---------------------------------------
@@ -1550,7 +1616,7 @@ Best practices
 --------------
 
 The default behaviour of automatically reading ``stdin`` is typically not
-desirable during non-interactive invocations. You most likely want
+desirable during non-interactive invocations. You most likely want to
 use the ``--ignore-stdin`` option to disable it.
 
 It is a common gotcha that without this option HTTPie seemingly hangs.
@@ -1561,8 +1627,8 @@ expecting that the request body will be passed through.
 And since there's no data nor ``EOF``, it will be stuck. So unless you're
 piping some data to HTTPie, this flag should be used in scripts.
 
-Also, it might be good to override the default ``30`` second ``--timeout`` to
-something that suits you.
+Also, it might be good to set a connection ``--timeout`` limit to prevent
+your program from hanging if the server never responds.
 
 
 
@@ -1626,7 +1692,7 @@ Please use the following support channels:
   to ask questions, discuss features, and for general discussion.
 * `StackOverflow <https://stackoverflow.com>`_
   to ask questions (please make sure to use the
-  `httpie <http://stackoverflow.com/questions/tagged/httpie>`_ tag).
+  `httpie <https://stackoverflow.com/questions/tagged/httpie>`_ tag).
 * Tweet directly to `@clihttp <https://twitter.com/clihttp>`_.
 * You can also tweet directly to `@jakubroztocil`_.
 
@@ -1639,9 +1705,9 @@ Dependencies
 
 Under the hood, HTTPie uses these two amazing libraries:
 
-* `Requests <http://python-requests.org>`_
+* `Requests <https://python-requests.org>`_
   — Python HTTP library for humans
-* `Pygments <http://pygments.org/>`_
+* `Pygments <https://pygments.org/>`_
   — Python syntax highlighter
 
 
@@ -1682,7 +1748,9 @@ See `CHANGELOG <https://github.com/jakubroztocil/httpie/blob/master/CHANGELOG.rs
 Artwork
 -------
 
-See `claudiatd/httpie-artwork`_
+* `Logo <https://github.com/claudiatd/httpie-artwork>`_ by `Cláudia Delgado <https://github.com/claudiatd>`_.
+* `Animation <https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.gif>`_ by `Allen Smith <https://github.com/loranallensmith>`_ of GitHub.
+
 
 
 Licence
@@ -1700,25 +1768,29 @@ have contributed.
 
 
 .. _pip: https://pip.pypa.io/en/stable/installing/
-.. _Github API: http://developer.github.com/v3/issues/comments/#create-a-comment
+.. _Github API: https://developer.github.com/v3/issues/comments/#create-a-comment
 .. _these fine people: https://github.com/jakubroztocil/httpie/contributors
 .. _Jakub Roztocil: https://roztocil.co
 .. _@jakubroztocil: https://twitter.com/jakubroztocil
-.. _claudiatd/httpie-artwork: https://github.com/claudiatd/httpie-artwork
 
 
 .. |pypi| image:: https://img.shields.io/pypi/v/httpie.svg?style=flat-square&label=latest%20stable%20version
     :target: https://pypi.python.org/pypi/httpie
     :alt: Latest version released on PyPi
 
-.. |coverage| image:: https://img.shields.io/coveralls/jakubroztocil/httpie/master.svg?style=flat-square&label=coverage
-    :target: https://coveralls.io/r/jakubroztocil/httpie?branch=master
+.. |coverage| image:: https://img.shields.io/codecov/c/github/jakubroztocil/httpie?style=flat-square
+    :target: https://codecov.io/gh/jakubroztocil/httpie
     :alt: Test coverage
 
-.. |unix_build| image:: https://img.shields.io/travis/jakubroztocil/httpie/master.svg?style=flat-square&label=unix%20build
-    :target: http://travis-ci.org/jakubroztocil/httpie
-    :alt: Build status of the master branch on Mac/Linux
+.. |build| image:: https://github.com/jakubroztocil/httpie/workflows/Build/badge.svg
+    :target: https://github.com/jakubroztocil/httpie/actions
+    :alt: Build status of the master branch on Mac/Linux/Windows
 
 .. |gitter| image:: https://img.shields.io/gitter/room/jkbrzt/httpie.svg?style=flat-square
     :target: https://gitter.im/jkbrzt/httpie
     :alt: Chat on Gitter
+
+.. |downloads| image:: https://pepy.tech/badge/httpie
+    :target: https://pepy.tech/project/httpie
+    :alt: Download count
+

extras/brew-deps.py

@@ -1,9 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
-Generate URLs and file hashes to be included in the Homebrew formula
-after a new release of HTTPie has been published on PyPi.
+Generate Ruby code with URLs and file hashes for packages from PyPi
+(i.e., httpie itself as well as its dependencies) to be included
+in the Homebrew formula after a new release of HTTPie has been published
+on PyPi.
 
-https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb
+<https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb>
 
 """
 import hashlib
@@ -12,13 +14,18 @@ import requests
 
 PACKAGES = [
     'httpie',
+    'Pygments',
     'requests',
-    'pygments',
+    'certifi',
+    'urllib3',
+    'idna',
+    'chardet',
+    'PySocks',
 ]
 
 
 def get_package_meta(package_name):
-    api_url = 'https://pypi.python.org/pypi/{}/json'.format(package_name)
+    api_url = f'https://pypi.python.org/pypi/{package_name}/json'
     resp = requests.get(api_url).json()
     hasher = hashlib.sha256()
     for release in resp['urls']:
@@ -31,8 +38,7 @@ def get_package_meta(package_name):
                 'sha256': hasher.hexdigest(),
             }
     else:
-        raise RuntimeError(
-            '{}: download not found: {}'.format(package_name, resp))
+        raise RuntimeError(f'{package_name}: download not found: {resp}')
 
 
 def main():
@@ -50,7 +56,7 @@ def main():
         print('    url "{url}"'.format(url=dep_meta['url']))
         print('    sha256 "{sha256}"'.format(sha256=dep_meta['sha256']))
         print('  end')
-        print()
+        print('')
 
 
 if __name__ == '__main__':

extras/httpie.rb

@@ -5,40 +5,60 @@
 # https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb
 #
 class Httpie < Formula
+  include Language::Python::Virtualenv
+
   desc "User-friendly cURL replacement (command-line HTTP client)"
   homepage "https://httpie.org/"
-
-  url "https://pypi.python.org/packages/85/95/7ccea3ae7fd1185e21629f6d14fa9c896d6250bb15fb492efa91edc741a2/httpie-0.9.8.tar.gz"
-  sha256 "515870b15231530f56fe2164190581748e8799b66ef0fe36ec9da3396f0df6e1"
-
+  url "https://files.pythonhosted.org/packages/d5/a4/ab61c1dbfdef33c7b7f5f7df0d79eb5cd55a106601a4acc17f983f320b4a/httpie-1.0.3.tar.gz"
+  sha256 "6d1b6e21da7d3ec030ae95536d4032c1129bdaf9de4adc72c596b87e5f646e80"
   head "https://github.com/jakubroztocil/httpie.git"
 
-  depends_on :python3
-
-  resource "requests" do
-    url "https://pypi.python.org/packages/d9/03/155b3e67fe35fe5b6f4227a8d9e96a14fda828b18199800d161bcefc1359/requests-2.12.3.tar.gz"
-    sha256 "de5d266953875e9647e37ef7bfe6ef1a46ff8ddfe61b5b3652edf7ea717ee2b2"
+  bottle do
+    cellar :any_skip_relocation
+    sha256 "158258be68ac93de13860be2bef02da6fd8b68aa24b2e6609bcff1ec3f93e7a0" => :mojave
+    sha256 "54352116b6fa2c3bd65f26136fdcb57986dbff8a52de5febf7aea59c126d29e1" => :high_sierra
+    sha256 "9cce71768fe388808e11b26d651b44a6b54219f5406845b4273b5099f5c1f76f" => :sierra
   end
 
-  resource "pygments" do
-    url "https://pypi.python.org/packages/b8/67/ab177979be1c81bc99c8d0592ef22d547e70bb4c6815c383286ed5dec504/Pygments-2.1.3.tar.gz"
-    sha256 "88e4c8a91b2af5962bfa5ea2447ec6dd357018e86e94c7d14bd8cacbc5b55d81"
+  depends_on "python"
+
+  resource "Pygments" do
+    url "https://files.pythonhosted.org/packages/7e/ae/26808275fc76bf2832deb10d3a3ed3107bc4de01b85dcccbe525f2cd6d1e/Pygments-2.4.2.tar.gz"
+    sha256 "881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"
+  end
+
+  resource "requests" do
+    url "https://files.pythonhosted.org/packages/01/62/ddcf76d1d19885e8579acb1b1df26a852b03472c0e46d2b959a714c90608/requests-2.22.0.tar.gz"
+    sha256 "11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"
+  end
+
+  resource "certifi" do
+    url "https://files.pythonhosted.org/packages/c5/67/5d0548226bcc34468e23a0333978f0e23d28d0b3f0c71a151aef9c3f7680/certifi-2019.6.16.tar.gz"
+    sha256 "945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+  end
+
+  resource "urllib3" do
+    url "https://files.pythonhosted.org/packages/4c/13/2386233f7ee40aa8444b47f7463338f3cbdf00c316627558784e3f542f07/urllib3-1.25.3.tar.gz"
+    sha256 "dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+  end
+
+  resource "idna" do
+    url "https://files.pythonhosted.org/packages/ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7/idna-2.8.tar.gz"
+    sha256 "c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407"
+  end
+
+  resource "chardet" do
+    url "https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz"
+    sha256 "84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"
+  end
+
+  resource "PySocks" do
+    url "https://files.pythonhosted.org/packages/15/ab/35824cfdee1aac662e3298275fa1e6cbedb52126d1785f8977959b769ccf/PySocks-1.7.0.tar.gz"
+    sha256 "d9031ea45fdfacbe59a99273e9f0448ddb33c1580fe3831c1b09557c5718977c"
   end
 
   def install
-    pyver = Language::Python.major_minor_version "python3"
-    ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python#{pyver}/site-packages"
-    %w[pygments requests].each do |r|
-      resource(r).stage do
-        system "python3", *Language::Python.setup_install_args(libexec/"vendor")
-      end
-    end
-
-    ENV.prepend_create_path "PYTHONPATH", libexec/"lib/python#{pyver}/site-packages"
-    system "python3", *Language::Python.setup_install_args(libexec)
-
-    bin.install Dir["#{libexec}/bin/*"]
-    bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"])
+    virtualenv_install_with_resources
   end
 
   test do

httpie/__init__.py

@@ -2,31 +2,7 @@
 HTTPie - a CLI, cURL-like tool for humans.
 
 """
-__version__ = '1.0.0'
+
+__version__ = '2.0.0'
 __author__ = 'Jakub Roztocil'
 __licence__ = 'BSD'
-
-
-class ExitStatus:
-    """Program exit code constants."""
-    SUCCESS = 0
-    ERROR = 1
-    PLUGIN_ERROR = 7
-
-    # 128+2 SIGINT <http://www.tldp.org/LDP/abs/html/exitcodes.html>
-    ERROR_CTRL_C = 130
-
-    ERROR_TIMEOUT = 2
-    ERROR_TOO_MANY_REDIRECTS = 6
-
-    # Used only when requested with --check-status:
-    ERROR_HTTP_3XX = 3
-    ERROR_HTTP_4XX = 4
-    ERROR_HTTP_5XX = 5
-
-
-EXIT_STATUS_LABELS = {
-    value: key
-    for key, value in ExitStatus.__dict__.items()
-    if key.isupper()
-}

httpie/__main__.py

@@ -8,10 +8,12 @@ import sys
 def main():
     try:
         from .core import main
-        sys.exit(main())
+        exit_status = main()
     except KeyboardInterrupt:
-        from . import ExitStatus
-        sys.exit(ExitStatus.ERROR_CTRL_C)
+        from httpie.status import ExitStatus
+        exit_status = ExitStatus.ERROR_CTRL_C
+
+    sys.exit(exit_status.value)
 
 
 if __name__ == '__main__':

httpie/cli/argparser.py

@@ -0,0 +1,387 @@
+import argparse
+import errno
+import os
+import re
+import sys
+from argparse import RawDescriptionHelpFormatter
+from textwrap import dedent
+from urllib.parse import urlsplit
+
+from httpie.cli.argtypes import AuthCredentials, KeyValueArgType, parse_auth
+from httpie.cli.constants import (
+    HTTP_GET, HTTP_POST, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
+    OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED, OUT_RESP_BODY, PRETTY_MAP,
+    PRETTY_STDOUT_TTY_ONLY, SEPARATOR_CREDENTIALS, SEPARATOR_GROUP_ALL_ITEMS,
+    SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
+    OUTPUT_OPTIONS_DEFAULT_OFFLINE,
+)
+from httpie.cli.exceptions import ParseError
+from httpie.cli.requestitems import RequestItems
+from httpie.context import Environment
+from httpie.plugins import plugin_manager
+from httpie.utils import ExplicitNullAuth, get_content_type
+
+
+class HTTPieHelpFormatter(RawDescriptionHelpFormatter):
+    """A nicer help formatter.
+
+    Help for arguments can be indented and contain new lines.
+    It will be de-dented and arguments in the help
+    will be separated by a blank line for better readability.
+
+
+    """
+
+    def __init__(self, max_help_position=6, *args, **kwargs):
+        # A smaller indent for args help.
+        kwargs['max_help_position'] = max_help_position
+        super().__init__(*args, **kwargs)
+
+    def _split_lines(self, text, width):
+        text = dedent(text).strip() + '\n\n'
+        return text.splitlines()
+
+
+class HTTPieArgumentParser(argparse.ArgumentParser):
+    """Adds additional logic to `argparse.ArgumentParser`.
+
+    Handles all input (CLI args, file args, stdin), applies defaults,
+    and performs extra validation.
+
+    """
+
+    def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs):
+        kwargs['add_help'] = False
+        super().__init__(*args, formatter_class=formatter_class, **kwargs)
+        self.env = None
+        self.args = None
+        self.has_stdin_data = False
+
+    # noinspection PyMethodOverriding
+    def parse_args(
+        self,
+        env: Environment,
+        args=None,
+        namespace=None
+    ) -> argparse.Namespace:
+        self.env = env
+        self.args, no_options = super().parse_known_args(args, namespace)
+
+        if self.args.debug:
+            self.args.traceback = True
+
+        self.has_stdin_data = (
+            self.env.stdin
+            and not self.args.ignore_stdin
+            and not self.env.stdin_isatty
+        )
+
+        # Arguments processing and environment setup.
+        self._apply_no_options(no_options)
+        self._validate_download_options()
+        self._setup_standard_streams()
+        self._process_output_options()
+        self._process_pretty_options()
+        self._guess_method()
+        self._parse_items()
+
+        if self.has_stdin_data:
+            self._body_from_file(self.env.stdin)
+        if not URL_SCHEME_RE.match(self.args.url):
+            if os.path.basename(env.program_name) == 'https':
+                scheme = 'https://'
+            else:
+                scheme = self.args.default_scheme + "://"
+
+            # See if we're using curl style shorthand for localhost (:3000/foo)
+            shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
+            if shorthand:
+                port = shorthand.group(1)
+                rest = shorthand.group(2)
+                self.args.url = scheme + 'localhost'
+                if port:
+                    self.args.url += ':' + port
+                self.args.url += rest
+            else:
+                self.args.url = scheme + self.args.url
+        self._process_auth()
+
+        return self.args
+
+    # noinspection PyShadowingBuiltins
+    def _print_message(self, message, file=None):
+        # Sneak in our stderr/stdout.
+        file = {
+            sys.stdout: self.env.stdout,
+            sys.stderr: self.env.stderr,
+            None: self.env.stderr
+        }.get(file, file)
+        if not hasattr(file, 'buffer') and isinstance(message, str):
+            message = message.encode(self.env.stdout_encoding)
+        super()._print_message(message, file)
+
+    def _setup_standard_streams(self):
+        """
+        Modify `env.stdout` and `env.stdout_isatty` based on args, if needed.
+
+        """
+        self.args.output_file_specified = bool(self.args.output_file)
+        if self.args.download:
+            # FIXME: Come up with a cleaner solution.
+            if not self.args.output_file and not self.env.stdout_isatty:
+                # Use stdout as the download output file.
+                self.args.output_file = self.env.stdout
+            # With `--download`, we write everything that would normally go to
+            # `stdout` to `stderr` instead. Let's replace the stream so that
+            # we don't have to use many `if`s throughout the codebase.
+            # The response body will be treated separately.
+            self.env.stdout = self.env.stderr
+            self.env.stdout_isatty = self.env.stderr_isatty
+        elif self.args.output_file:
+            # When not `--download`ing, then `--output` simply replaces
+            # `stdout`. The file is opened for appending, which isn't what
+            # we want in this case.
+            self.args.output_file.seek(0)
+            try:
+                self.args.output_file.truncate()
+            except IOError as e:
+                if e.errno == errno.EINVAL:
+                    # E.g. /dev/null on Linux.
+                    pass
+                else:
+                    raise
+            self.env.stdout = self.args.output_file
+            self.env.stdout_isatty = False
+
+    def _process_auth(self):
+        # TODO: refactor
+        self.args.auth_plugin = None
+        default_auth_plugin = plugin_manager.get_auth_plugins()[0]
+        auth_type_set = self.args.auth_type is not None
+        url = urlsplit(self.args.url)
+
+        if self.args.auth is None and not auth_type_set:
+            if url.username is not None:
+                # Handle http://username:password@hostname/
+                username = url.username
+                password = url.password or ''
+                self.args.auth = AuthCredentials(
+                    key=username,
+                    value=password,
+                    sep=SEPARATOR_CREDENTIALS,
+                    orig=SEPARATOR_CREDENTIALS.join([username, password])
+                )
+
+        if self.args.auth is not None or auth_type_set:
+            if not self.args.auth_type:
+                self.args.auth_type = default_auth_plugin.auth_type
+            plugin = plugin_manager.get_auth_plugin(self.args.auth_type)()
+
+            if plugin.auth_require and self.args.auth is None:
+                self.error('--auth required')
+
+            plugin.raw_auth = self.args.auth
+            self.args.auth_plugin = plugin
+            already_parsed = isinstance(self.args.auth, AuthCredentials)
+
+            if self.args.auth is None or not plugin.auth_parse:
+                self.args.auth = plugin.get_auth()
+            else:
+                if already_parsed:
+                    # from the URL
+                    credentials = self.args.auth
+                else:
+                    credentials = parse_auth(self.args.auth)
+
+                if (not credentials.has_password()
+                        and plugin.prompt_password):
+                    if self.args.ignore_stdin:
+                        # Non-tty stdin read by now
+                        self.error(
+                            'Unable to prompt for passwords because'
+                            ' --ignore-stdin is set.'
+                        )
+                    credentials.prompt_password(url.netloc)
+                self.args.auth = plugin.get_auth(
+                    username=credentials.key,
+                    password=credentials.value,
+                )
+        if not self.args.auth and self.args.ignore_netrc:
+            # Set a no-op auth to force requests to ignore .netrc
+            # <https://github.com/psf/requests/issues/2773#issuecomment-174312831>
+            self.args.auth = ExplicitNullAuth()
+
+    def _apply_no_options(self, no_options):
+        """For every `--no-OPTION` in `no_options`, set `args.OPTION` to
+        its default value. This allows for un-setting of options, e.g.,
+        specified in config.
+
+        """
+        invalid = []
+
+        for option in no_options:
+            if not option.startswith('--no-'):
+                invalid.append(option)
+                continue
+
+            # --no-option => --option
+            inverted = '--' + option[5:]
+            for action in self._actions:
+                if inverted in action.option_strings:
+                    setattr(self.args, action.dest, action.default)
+                    break
+            else:
+                invalid.append(option)
+
+        if invalid:
+            msg = 'unrecognized arguments: %s'
+            self.error(msg % ' '.join(invalid))
+
+    def _body_from_file(self, fd):
+        """There can only be one source of request data.
+
+        Bytes are always read.
+
+        """
+        if self.args.data:
+            self.error('Request body (from stdin or a file) and request '
+                       'data (key=value) cannot be mixed. Pass '
+                       '--ignore-stdin to let key/value take priority.')
+        self.args.data = getattr(fd, 'buffer', fd).read()
+
+    def _guess_method(self):
+        """Set `args.method` if not specified to either POST or GET
+        based on whether the request has data or not.
+
+        """
+        if self.args.method is None:
+            # Invoked as `http URL'.
+            assert not self.args.request_items
+            if self.has_stdin_data:
+                self.args.method = HTTP_POST
+            else:
+                self.args.method = HTTP_GET
+
+        # FIXME: False positive, e.g., "localhost" matches but is a valid URL.
+        elif not re.match('^[a-zA-Z]+$', self.args.method):
+            # Invoked as `http URL item+'. The URL is now in `args.method`
+            # and the first ITEM is now incorrectly in `args.url`.
+            try:
+                # Parse the URL as an ITEM and store it as the first ITEM arg.
+                self.args.request_items.insert(0, KeyValueArgType(
+                    *SEPARATOR_GROUP_ALL_ITEMS).__call__(self.args.url))
+
+            except argparse.ArgumentTypeError as e:
+                if self.args.traceback:
+                    raise
+                self.error(e.args[0])
+
+            else:
+                # Set the URL correctly
+                self.args.url = self.args.method
+                # Infer the method
+                has_data = (
+                    self.has_stdin_data
+                    or any(
+                        item.sep in SEPARATOR_GROUP_DATA_ITEMS
+                        for item in self.args.request_items)
+                )
+                self.args.method = HTTP_POST if has_data else HTTP_GET
+
+    def _parse_items(self):
+        """
+        Parse `args.request_items` into `args.headers`, `args.data`,
+        `args.params`, and `args.files`.
+
+        """
+        try:
+            request_items = RequestItems.from_args(
+                request_item_args=self.args.request_items,
+                as_form=self.args.form,
+            )
+        except ParseError as e:
+            if self.args.traceback:
+                raise
+            self.error(e.args[0])
+        else:
+            self.args.headers = request_items.headers
+            self.args.data = request_items.data
+            self.args.files = request_items.files
+            self.args.params = request_items.params
+
+        if self.args.files and not self.args.form:
+            # `http url @/path/to/file`
+            file_fields = list(self.args.files.keys())
+            if file_fields != ['']:
+                self.error(
+                    'Invalid file fields (perhaps you meant --form?): %s'
+                    % ','.join(file_fields))
+
+            fn, fd, ct = self.args.files['']
+            self.args.files = {}
+
+            self._body_from_file(fd)
+
+            if 'Content-Type' not in self.args.headers:
+                content_type = get_content_type(fn)
+                if content_type:
+                    self.args.headers['Content-Type'] = content_type
+
+    def _process_output_options(self):
+        """Apply defaults to output options, or validate the provided ones.
+
+        The default output options are stdout-type-sensitive.
+
+        """
+
+        def check_options(value, option):
+            unknown = set(value) - OUTPUT_OPTIONS
+            if unknown:
+                self.error('Unknown output options: {0}={1}'.format(
+                    option,
+                    ','.join(unknown)
+                ))
+
+        if self.args.verbose:
+            self.args.all = True
+
+        if self.args.output_options is None:
+            if self.args.verbose:
+                self.args.output_options = ''.join(OUTPUT_OPTIONS)
+            elif self.args.offline:
+                self.args.output_options = OUTPUT_OPTIONS_DEFAULT_OFFLINE
+            elif not self.env.stdout_isatty:
+                self.args.output_options = OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
+            else:
+                self.args.output_options = OUTPUT_OPTIONS_DEFAULT
+
+        if self.args.output_options_history is None:
+            self.args.output_options_history = self.args.output_options
+
+        check_options(self.args.output_options, '--print')
+        check_options(self.args.output_options_history, '--history-print')
+
+        if self.args.download and OUT_RESP_BODY in self.args.output_options:
+            # Response body is always downloaded with --download and it goes
+            # through a different routine, so we remove it.
+            self.args.output_options = str(
+                set(self.args.output_options) - set(OUT_RESP_BODY))
+
+    def _process_pretty_options(self):
+        if self.args.prettify == PRETTY_STDOUT_TTY_ONLY:
+            self.args.prettify = PRETTY_MAP[
+                'all' if self.env.stdout_isatty else 'none']
+        elif (self.args.prettify and self.env.is_windows
+              and self.args.output_file):
+            self.error('Only terminal output can be colorized on Windows.')
+        else:
+            # noinspection PyTypeChecker
+            self.args.prettify = PRETTY_MAP[self.args.prettify]
+
+    def _validate_download_options(self):
+        if not self.args.download:
+            if self.args.download_resume:
+                self.error('--continue only works with --download')
+        if self.args.download_resume and not (
+                self.args.download and self.args.output_file):
+            self.error('--continue requires --output to be specified')

httpie/cli/argtypes.py

@@ -0,0 +1,183 @@
+import argparse
+import getpass
+import os
+import sys
+from typing import Union, List, Optional
+
+from httpie.cli.constants import SEPARATOR_CREDENTIALS
+from httpie.sessions import VALID_SESSION_NAME_PATTERN
+
+
+class KeyValueArg:
+    """Base key-value pair parsed from CLI."""
+
+    def __init__(self, key: str, value: Optional[str], sep: str, orig: str):
+        self.key = key
+        self.value = value
+        self.sep = sep
+        self.orig = orig
+
+    def __eq__(self, other: 'KeyValueArg'):
+        return self.__dict__ == other.__dict__
+
+    def __repr__(self):
+        return repr(self.__dict__)
+
+
+class SessionNameValidator:
+
+    def __init__(self, error_message: str):
+        self.error_message = error_message
+
+    def __call__(self, value: str) -> str:
+        # Session name can be a path or just a name.
+        if (os.path.sep not in value
+                and not VALID_SESSION_NAME_PATTERN.search(value)):
+            raise argparse.ArgumentError(None, self.error_message)
+        return value
+
+
+class Escaped(str):
+    """Represents an escaped character."""
+
+    def __repr__(self):
+        return f"Escaped({repr(str(self))})"
+
+
+class KeyValueArgType:
+    """A key-value pair argument type used with `argparse`.
+
+    Parses a key-value arg and constructs a `KeyValueArg` instance.
+    Used for headers, form data, and other key-value pair types.
+
+    """
+
+    key_value_class = KeyValueArg
+
+    def __init__(self, *separators: str):
+        self.separators = separators
+        self.special_characters = set('\\')
+        for separator in separators:
+            self.special_characters.update(separator)
+
+    def __call__(self, s: str) -> KeyValueArg:
+        """Parse raw string arg  and return `self.key_value_class` instance.
+
+        The best of `self.separators` is determined (first found, longest).
+        Back slash escaped characters aren't considered as separators
+        (or parts thereof). Literal back slash characters have to be escaped
+        as well (r'\\').
+
+        """
+        tokens = self.tokenize(s)
+
+        # Sorting by length ensures that the longest one will be
+        # chosen as it will overwrite any shorter ones starting
+        # at the same position in the `found` dictionary.
+        separators = sorted(self.separators, key=len)
+
+        for i, token in enumerate(tokens):
+
+            if isinstance(token, Escaped):
+                continue
+
+            found = {}
+            for sep in separators:
+                pos = token.find(sep)
+                if pos != -1:
+                    found[pos] = sep
+
+            if found:
+                # Starting first, longest separator found.
+                sep = found[min(found.keys())]
+
+                key, value = token.split(sep, 1)
+
+                # Any preceding tokens are part of the key.
+                key = ''.join(tokens[:i]) + key
+
+                # Any following tokens are part of the value.
+                value += ''.join(tokens[i + 1:])
+
+                break
+
+        else:
+            raise argparse.ArgumentTypeError(f'{s!r} is not a valid value')
+
+        return self.key_value_class(key=key, value=value, sep=sep, orig=s)
+
+    def tokenize(self, s: str) -> List[Union[str, Escaped]]:
+        r"""Tokenize the raw arg string
+
+        There are only two token types - strings and escaped characters:
+
+        >>> KeyValueArgType('=').tokenize(r'foo\=bar\\baz')
+        ['foo', Escaped('='), 'bar', Escaped('\\'), 'baz']
+
+        """
+        tokens = ['']
+        characters = iter(s)
+        for char in characters:
+            if char == '\\':
+                char = next(characters, '')
+                if char not in self.special_characters:
+                    tokens[-1] += '\\' + char
+                else:
+                    tokens.extend([Escaped(char), ''])
+            else:
+                tokens[-1] += char
+        return tokens
+
+
+class AuthCredentials(KeyValueArg):
+    """Represents parsed credentials."""
+
+    def has_password(self) -> bool:
+        return self.value is not None
+
+    def prompt_password(self, host: str):
+        prompt_text = f'http: password for {self.key}@{host}: '
+        try:
+            self.value = self._getpass(prompt_text)
+        except (EOFError, KeyboardInterrupt):
+            sys.stderr.write('\n')
+            sys.exit(0)
+
+    @staticmethod
+    def _getpass(prompt):
+        # To allow easy mocking.
+        return getpass.getpass(str(prompt))
+
+
+class AuthCredentialsArgType(KeyValueArgType):
+    """A key-value arg type that parses credentials."""
+
+    key_value_class = AuthCredentials
+
+    def __call__(self, s):
+        """Parse credentials from `s`.
+
+        ("username" or "username:password").
+
+        """
+        try:
+            return super().__call__(s)
+        except argparse.ArgumentTypeError:
+            # No password provided, will prompt for it later.
+            return self.key_value_class(
+                key=s,
+                value=None,
+                sep=SEPARATOR_CREDENTIALS,
+                orig=s
+            )
+
+
+parse_auth = AuthCredentialsArgType(SEPARATOR_CREDENTIALS)
+
+
+def readable_file_arg(filename):
+    try:
+        with open(filename, 'rb'):
+            return filename
+    except IOError as ex:
+        raise argparse.ArgumentTypeError(f'{filename}: {ex.args[1]}')

httpie/cli/constants.py

@@ -0,0 +1,103 @@
+"""Parsing and processing of CLI input (args, auth credentials, files, stdin).
+
+"""
+import re
+import ssl
+
+
+# TODO: Use MultiDict for headers once added to `requests`.
+# <https://github.com/jakubroztocil/httpie/issues/130>
+
+
+# ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+# <https://tools.ietf.org/html/rfc3986#section-3.1>
+URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)
+
+HTTP_POST = 'POST'
+HTTP_GET = 'GET'
+
+# Various separators used in args
+SEPARATOR_HEADER = ':'
+SEPARATOR_HEADER_EMPTY = ';'
+SEPARATOR_CREDENTIALS = ':'
+SEPARATOR_PROXY = ':'
+SEPARATOR_DATA_STRING = '='
+SEPARATOR_DATA_RAW_JSON = ':='
+SEPARATOR_FILE_UPLOAD = '@'
+SEPARATOR_DATA_EMBED_FILE_CONTENTS = '=@'
+SEPARATOR_DATA_EMBED_RAW_JSON_FILE = ':=@'
+SEPARATOR_QUERY_PARAM = '=='
+
+# Separators that become request data
+SEPARATOR_GROUP_DATA_ITEMS = frozenset({
+    SEPARATOR_DATA_STRING,
+    SEPARATOR_DATA_RAW_JSON,
+    SEPARATOR_FILE_UPLOAD,
+    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
+    SEPARATOR_DATA_EMBED_RAW_JSON_FILE
+})
+
+# Separators for items whose value is a filename to be embedded
+SEPARATOR_GROUP_DATA_EMBED_ITEMS = frozenset({
+    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
+    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
+})
+
+# Separators for raw JSON items
+SEPARATOR_GROUP_RAW_JSON_ITEMS = frozenset([
+    SEPARATOR_DATA_RAW_JSON,
+    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
+])
+
+# Separators allowed in ITEM arguments
+SEPARATOR_GROUP_ALL_ITEMS = frozenset({
+    SEPARATOR_HEADER,
+    SEPARATOR_HEADER_EMPTY,
+    SEPARATOR_QUERY_PARAM,
+    SEPARATOR_DATA_STRING,
+    SEPARATOR_DATA_RAW_JSON,
+    SEPARATOR_FILE_UPLOAD,
+    SEPARATOR_DATA_EMBED_FILE_CONTENTS,
+    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
+})
+
+# Output options
+OUT_REQ_HEAD = 'H'
+OUT_REQ_BODY = 'B'
+OUT_RESP_HEAD = 'h'
+OUT_RESP_BODY = 'b'
+
+OUTPUT_OPTIONS = frozenset({
+    OUT_REQ_HEAD,
+    OUT_REQ_BODY,
+    OUT_RESP_HEAD,
+    OUT_RESP_BODY
+})
+
+# Pretty
+PRETTY_MAP = {
+    'all': ['format', 'colors'],
+    'colors': ['colors'],
+    'format': ['format'],
+    'none': []
+}
+PRETTY_STDOUT_TTY_ONLY = object()
+
+# Defaults
+OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
+OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
+OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY
+
+SSL_VERSION_ARG_MAPPING = {
+    'ssl2.3': 'PROTOCOL_SSLv23',
+    'ssl3': 'PROTOCOL_SSLv3',
+    'tls1': 'PROTOCOL_TLSv1',
+    'tls1.1': 'PROTOCOL_TLSv1_1',
+    'tls1.2': 'PROTOCOL_TLSv1_2',
+    'tls1.3': 'PROTOCOL_TLSv1_3',
+}
+SSL_VERSION_ARG_MAPPING = {
+    cli_arg: getattr(ssl, ssl_constant)
+    for cli_arg, ssl_constant in SSL_VERSION_ARG_MAPPING.items()
+    if hasattr(ssl, ssl_constant)
+}

httpie/cli/definition.py

@@ -1,56 +1,31 @@
-"""CLI arguments definition.
-
-NOTE: the CLI interface may change before reaching v1.0.
+"""
+CLI arguments definition.
 
 """
-# noinspection PyCompatibility
-from argparse import (
-    RawDescriptionHelpFormatter, FileType,
-    OPTIONAL, ZERO_OR_MORE, SUPPRESS
-)
+from argparse import (FileType, OPTIONAL, SUPPRESS, ZERO_OR_MORE)
 from textwrap import dedent, wrap
 
 from httpie import __doc__, __version__
-from httpie.input import (
-    HTTPieArgumentParser, KeyValueArgType,
-    SEP_PROXY, SEP_GROUP_ALL_ITEMS,
-    OUT_REQ_HEAD, OUT_REQ_BODY, OUT_RESP_HEAD,
-    OUT_RESP_BODY, OUTPUT_OPTIONS,
-    OUTPUT_OPTIONS_DEFAULT, PRETTY_MAP,
-    PRETTY_STDOUT_TTY_ONLY, SessionNameValidator,
-    readable_file_arg, SSL_VERSION_ARG_MAPPING
+from httpie.cli.argparser import HTTPieArgumentParser
+from httpie.cli.argtypes import (
+    KeyValueArgType, SessionNameValidator, readable_file_arg,
+)
+from httpie.cli.constants import (
+    OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT, OUT_REQ_BODY, OUT_REQ_HEAD,
+    OUT_RESP_BODY, OUT_RESP_HEAD, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY,
+    SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, SSL_VERSION_ARG_MAPPING,
 )
 from httpie.output.formatters.colors import (
-    AVAILABLE_STYLES, DEFAULT_STYLE, AUTO_STYLE
+    AUTO_STYLE, AVAILABLE_STYLES, DEFAULT_STYLE,
 )
 from httpie.plugins import plugin_manager
 from httpie.plugins.builtin import BuiltinAuthPlugin
 from httpie.sessions import DEFAULT_SESSIONS_DIR
 
 
-class HTTPieHelpFormatter(RawDescriptionHelpFormatter):
-    """A nicer help formatter.
-
-    Help for arguments can be indented and contain new lines.
-    It will be de-dented and arguments in the help
-    will be separated by a blank line for better readability.
-
-
-    """
-    def __init__(self, max_help_position=6, *args, **kwargs):
-        # A smaller indent for args help.
-        kwargs['max_help_position'] = max_help_position
-        super(HTTPieHelpFormatter, self).__init__(*args, **kwargs)
-
-    def _split_lines(self, text, width):
-        text = dedent(text).strip() + '\n\n'
-        return text.splitlines()
-
-
 parser = HTTPieArgumentParser(
     prog='http',
-    formatter_class=HTTPieHelpFormatter,
-    description='%s <http://httpie.org>' % __doc__.strip(),
+    description='%s <https://httpie.org>' % __doc__.strip(),
     epilog=dedent("""
     For every --OPTION there is also a --no-OPTION that reverts OPTION
     to its default value.
@@ -62,7 +37,6 @@ parser = HTTPieArgumentParser(
     """),
 )
 
-
 #######################################################################
 # Positional arguments.
 #######################################################################
@@ -76,7 +50,7 @@ positional = parser.add_argument_group(
     """)
 )
 positional.add_argument(
-    'method',
+    dest='method',
     metavar='METHOD',
     nargs=OPTIONAL,
     default=None,
@@ -92,7 +66,7 @@ positional.add_argument(
     """
 )
 positional.add_argument(
-    'url',
+    dest='url',
     metavar='URL',
     help="""
     The scheme defaults to 'http://' if the URL does not include one.
@@ -106,11 +80,11 @@ positional.add_argument(
     """
 )
 positional.add_argument(
-    'items',
+    dest='request_items',
     metavar='REQUEST_ITEM',
     nargs=ZERO_OR_MORE,
     default=None,
-    type=KeyValueArgType(*SEP_GROUP_ALL_ITEMS),
+    type=KeyValueArgType(*SEPARATOR_GROUP_ALL_ITEMS),
     help=r"""
     Optional key-value pairs to be included in the request. The separator used
     determines the type:
@@ -151,7 +125,6 @@ positional.add_argument(
     """
 )
 
-
 #######################################################################
 # Content type.
 #######################################################################
@@ -184,6 +157,28 @@ content_type.add_argument(
     """
 )
 
+#######################################################################
+# Content processing.
+#######################################################################
+
+content_processing = parser.add_argument_group(
+    title='Content Processing Options',
+    description=None
+)
+
+content_processing.add_argument(
+    '--compress', '-x',
+    action='count',
+    default=0,
+    help="""
+    Content compressed (encoded) with Deflate algorithm.
+    The Content-Encoding header is set to deflate.
+
+    Compression is skipped if it appears that compression ratio is
+    negative. Compression can be forced by repeating the argument.
+
+    """
+)
 
 #######################################################################
 # Output processing
@@ -230,7 +225,6 @@ output_processing.add_argument(
     )
 )
 
-
 #######################################################################
 # Output options
 #######################################################################
@@ -240,49 +234,40 @@ output_options.add_argument(
     '--print', '-p',
     dest='output_options',
     metavar='WHAT',
-    help="""
+    help=f"""
     String specifying what the output should contain:
 
-        '{req_head}' request headers
-        '{req_body}' request body
-        '{res_head}' response headers
-        '{res_body}' response body
+        '{OUT_REQ_HEAD}' request headers
+        '{OUT_REQ_BODY}' request body
+        '{OUT_RESP_HEAD}' response headers
+        '{OUT_RESP_BODY}' response body
 
-    The default behaviour is '{default}' (i.e., the response headers and body
-    is printed), if standard output is not redirected. If the output is piped
-    to another program or to a file, then only the response body is printed
-    by default.
+    The default behaviour is '{OUTPUT_OPTIONS_DEFAULT}' (i.e., the response
+    headers and body is printed), if standard output is not redirected.
+    If the output is piped to another program or to a file, then only the
+    response body is printed by default.
 
     """
-    .format(
-        req_head=OUT_REQ_HEAD,
-        req_body=OUT_REQ_BODY,
-        res_head=OUT_RESP_HEAD,
-        res_body=OUT_RESP_BODY,
-        default=OUTPUT_OPTIONS_DEFAULT,
-    )
 )
 output_options.add_argument(
     '--headers', '-h',
     dest='output_options',
     action='store_const',
     const=OUT_RESP_HEAD,
-    help="""
-    Print only the response headers. Shortcut for --print={0}.
+    help=f"""
+    Print only the response headers. Shortcut for --print={OUT_RESP_HEAD}.
 
     """
-    .format(OUT_RESP_HEAD)
 )
 output_options.add_argument(
     '--body', '-b',
     dest='output_options',
     action='store_const',
     const=OUT_RESP_BODY,
-    help="""
-    Print only the response body. Shortcut for --print={0}.
+    help=f"""
+    Print only the response body. Shortcut for --print={OUT_RESP_BODY}.
 
     """
-    .format(OUT_RESP_BODY)
 )
 
 output_options.add_argument(
@@ -294,8 +279,7 @@ output_options.add_argument(
     any intermediary requests/responses (such as redirects).
     It's a shortcut for: --all --print={0}
 
-    """
-    .format(''.join(OUTPUT_OPTIONS))
+    """.format(''.join(OUTPUT_OPTIONS))
 )
 output_options.add_argument(
     '--all',
@@ -377,13 +361,12 @@ output_options.add_argument(
     """
 )
 
-
 #######################################################################
 # Sessions
 #######################################################################
 
-sessions = parser.add_argument_group(title='Sessions')\
-                 .add_mutually_exclusive_group(required=False)
+sessions = parser.add_argument_group(title='Sessions') \
+    .add_mutually_exclusive_group(required=False)
 
 session_name_validator = SessionNameValidator(
     'Session name contains invalid characters.'
@@ -393,17 +376,16 @@ sessions.add_argument(
     '--session',
     metavar='SESSION_NAME_OR_PATH',
     type=session_name_validator,
-    help="""
+    help=f"""
     Create, or reuse and update a session. Within a session, custom headers,
     auth credential, as well as any cookies sent by the server persist between
     requests.
 
     Session files are stored in:
 
-        {session_dir}/<HOST>/<SESSION_NAME>.json.
+        {DEFAULT_SESSIONS_DIR}/<HOST>/<SESSION_NAME>.json.
 
     """
-    .format(session_dir=DEFAULT_SESSIONS_DIR)
 )
 sessions.add_argument(
     '--session-read-only',
@@ -434,7 +416,7 @@ auth.add_argument(
 )
 
 
-class _AuthTypeLazyChoices(object):
+class _AuthTypeLazyChoices:
     # Needed for plugin testing
 
     def __contains__(self, item):
@@ -454,8 +436,7 @@ auth.add_argument(
 
     {types}
 
-    """
-    .format(default=_auth_plugins[0].auth_type, types='\n    '.join(
+    """.format(default=_auth_plugins[0].auth_type, types='\n    '.join(
         '"{type}": {name}{package}{description}'.format(
             type=plugin.auth_type,
             name=plugin.name,
@@ -471,7 +452,15 @@ auth.add_argument(
         for plugin in _auth_plugins
     )),
 )
+auth.add_argument(
+    '--ignore-netrc',
+    default=False,
+    action='store_true',
+    help="""
+    Ignore credentials from .netrc.
 
+    """,
+)
 
 #######################################################################
 # Network
@@ -479,16 +468,25 @@ auth.add_argument(
 
 network = parser.add_argument_group(title='Network')
 
+network.add_argument(
+    '--offline',
+    default=False,
+    action='store_true',
+    help="""
+    Build the request and print it but don’t actually send it.
+    """
+)
 network.add_argument(
     '--proxy',
     default=[],
     action='append',
     metavar='PROTOCOL:PROXY_URL',
-    type=KeyValueArgType(SEP_PROXY),
+    type=KeyValueArgType(SEPARATOR_PROXY),
     help="""
     String mapping protocol to the URL of the proxy
     (e.g. http:http://foo.bar:3128). You can specify multiple proxies with
-    different protocols.
+    different protocols. The environment variables $ALL_PROXY, $HTTP_PROXY,
+    and $HTTPS_proxy are supported as well.
 
     """
 )
@@ -512,14 +510,29 @@ network.add_argument(
     """
 )
 
+network.add_argument(
+    '--max-headers',
+    type=int,
+    default=0,
+    help="""
+    The maximum number of response headers to be read before giving up
+    (default 0, i.e., no limit).
+
+    """
+)
+
 network.add_argument(
     '--timeout',
     type=float,
-    default=30,
+    default=0,
     metavar='SECONDS',
     help="""
-    The connection timeout of the request in seconds. The default value is
-    30 seconds.
+    The connection timeout of the request in seconds.
+    The default value is 0, i.e., there is no timeout limit.
+    This is not a time limit on the entire response download;
+    rather, an error is reported if the server has not issued a response for
+    timeout seconds (more precisely, if no bytes have been received on
+    the underlying socket for timeout seconds).
 
     """
 )
@@ -540,7 +553,6 @@ network.add_argument(
     """
 )
 
-
 #######################################################################
 # SSL
 #######################################################################

httpie/cli/dicts.py

@@ -0,0 +1,53 @@
+from collections import OrderedDict
+
+from requests.structures import CaseInsensitiveDict
+
+
+class RequestHeadersDict(CaseInsensitiveDict):
+    """
+    Headers are case-insensitive and multiple values are currently not supported.
+
+    """
+
+
+class RequestJSONDataDict(OrderedDict):
+    pass
+
+
+class MultiValueOrderedDict(OrderedDict):
+    """Multi-value dict for URL parameters and form data."""
+
+    def __setitem__(self, key, value):
+        """
+        If `key` is assigned more than once, `self[key]` holds a
+        `list` of all the values.
+
+        This allows having multiple fields with the same name in form
+        data and URL params.
+
+        """
+        assert not isinstance(value, list)
+        if key not in self:
+            super().__setitem__(key, value)
+        else:
+            if not isinstance(self[key], list):
+                super().__setitem__(key, [self[key]])
+            self[key].append(value)
+
+
+class RequestQueryParamsDict(MultiValueOrderedDict):
+    pass
+
+
+class RequestDataDict(MultiValueOrderedDict):
+
+    def items(self):
+        for key, values in super(MultiValueOrderedDict, self).items():
+            if not isinstance(values, list):
+                values = [values]
+            for value in values:
+                yield key, value
+
+
+class RequestFilesDict(RequestDataDict):
+    pass

httpie/cli/exceptions.py

@@ -0,0 +1,2 @@
+class ParseError(Exception):
+    pass

httpie/cli/requestitems.py

@@ -0,0 +1,149 @@
+import os
+from io import BytesIO
+from typing import Callable, Dict, IO, List, Optional, Tuple, Union
+
+from httpie.cli.argtypes import KeyValueArg
+from httpie.cli.constants import (
+    SEPARATOR_DATA_EMBED_FILE_CONTENTS, SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
+    SEPARATOR_DATA_RAW_JSON, SEPARATOR_DATA_STRING, SEPARATOR_FILE_UPLOAD,
+    SEPARATOR_HEADER, SEPARATOR_HEADER_EMPTY, SEPARATOR_QUERY_PARAM,
+)
+from httpie.cli.dicts import (
+    RequestDataDict, RequestFilesDict, RequestHeadersDict, RequestJSONDataDict,
+    RequestQueryParamsDict,
+)
+from httpie.cli.exceptions import ParseError
+from httpie.utils import (get_content_type, load_json_preserve_order)
+
+
+class RequestItems:
+
+    def __init__(self, as_form=False):
+        self.headers = RequestHeadersDict()
+        self.data = RequestDataDict() if as_form else RequestJSONDataDict()
+        self.files = RequestFilesDict()
+        self.params = RequestQueryParamsDict()
+
+    @classmethod
+    def from_args(
+        cls,
+        request_item_args: List[KeyValueArg],
+        as_form=False,
+    ) -> 'RequestItems':
+        instance = cls(as_form=as_form)
+        rules: Dict[str, Tuple[Callable, dict]] = {
+            SEPARATOR_HEADER: (
+                process_header_arg,
+                instance.headers,
+            ),
+            SEPARATOR_HEADER_EMPTY: (
+                process_empty_header_arg,
+                instance.headers,
+            ),
+            SEPARATOR_QUERY_PARAM: (
+                process_query_param_arg,
+                instance.params,
+            ),
+            SEPARATOR_FILE_UPLOAD: (
+                process_file_upload_arg,
+                instance.files,
+            ),
+            SEPARATOR_DATA_STRING: (
+                process_data_item_arg,
+                instance.data,
+            ),
+            SEPARATOR_DATA_EMBED_FILE_CONTENTS: (
+                process_data_embed_file_contents_arg,
+                instance.data,
+            ),
+            SEPARATOR_DATA_RAW_JSON: (
+                process_data_raw_json_embed_arg,
+                instance.data,
+            ),
+            SEPARATOR_DATA_EMBED_RAW_JSON_FILE: (
+                process_data_embed_raw_json_file_arg,
+                instance.data,
+            ),
+        }
+
+        for arg in request_item_args:
+            processor_func, target_dict = rules[arg.sep]
+            target_dict[arg.key] = processor_func(arg)
+
+        return instance
+
+
+JSONType = Union[str, bool, int, list, dict]
+
+
+def process_header_arg(arg: KeyValueArg) -> Optional[str]:
+    return arg.value or None
+
+
+def process_empty_header_arg(arg: KeyValueArg) -> str:
+    if arg.value:
+        raise ParseError(
+            'Invalid item "%s" '
+            '(to specify an empty header use `Header;`)'
+            % arg.orig
+        )
+    return arg.value
+
+
+def process_query_param_arg(arg: KeyValueArg) -> str:
+    return arg.value
+
+
+def process_file_upload_arg(arg: KeyValueArg) -> Tuple[str, IO, str]:
+    filename = arg.value
+    try:
+        with open(os.path.expanduser(filename), 'rb') as f:
+            contents = f.read()
+    except IOError as e:
+        raise ParseError('"%s": %s' % (arg.orig, e))
+    return (
+        os.path.basename(filename),
+        BytesIO(contents),
+        get_content_type(filename),
+    )
+
+
+def process_data_item_arg(arg: KeyValueArg) -> str:
+    return arg.value
+
+
+def process_data_embed_file_contents_arg(arg: KeyValueArg) -> str:
+    return load_text_file(arg)
+
+
+def process_data_embed_raw_json_file_arg(arg: KeyValueArg) -> JSONType:
+    contents = load_text_file(arg)
+    value = load_json(arg, contents)
+    return value
+
+
+def process_data_raw_json_embed_arg(arg: KeyValueArg) -> JSONType:
+    value = load_json(arg, arg.value)
+    return value
+
+
+def load_text_file(item: KeyValueArg) -> str:
+    path = item.value
+    try:
+        with open(os.path.expanduser(path), 'rb') as f:
+            return f.read().decode()
+    except IOError as e:
+        raise ParseError('"%s": %s' % (item.orig, e))
+    except UnicodeDecodeError:
+        raise ParseError(
+            '"%s": cannot embed the content of "%s",'
+            ' not a UTF8 or ASCII-encoded text file'
+            % (item.orig, item.value)
+        )
+
+
+def load_json(arg: KeyValueArg, contents: str) -> JSONType:
+    try:
+        return load_json_preserve_order(contents)
+    except ValueError as e:
+        raise ParseError('"%s": %s' % (arg.orig, e))

httpie/client.py

@@ -1,113 +1,202 @@
+import argparse
+import http.client
 import json
 import sys
+import zlib
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterable, Union
 
 import requests
 from requests.adapters import HTTPAdapter
-from requests.structures import CaseInsensitiveDict
 
-from httpie import sessions
 from httpie import __version__
-from httpie.compat import str
-from httpie.input import SSL_VERSION_ARG_MAPPING
+from httpie.cli.constants import SSL_VERSION_ARG_MAPPING
+from httpie.cli.dicts import RequestHeadersDict
 from httpie.plugins import plugin_manager
-from httpie.utils import repr_dict_nice
+from httpie.sessions import get_httpie_session
+from httpie.utils import repr_dict
+
 
 try:
-    # https://urllib3.readthedocs.io/en/latest/security.html
     # noinspection PyPackageRequirements
     import urllib3
+    # <https://urllib3.readthedocs.io/en/latest/security.html>
     urllib3.disable_warnings()
 except (ImportError, AttributeError):
-    # In some rare cases, the user may have an old version of the requests
-    # or urllib3, and there is no method called "disable_warnings." In these
-    # cases, we don't need to call the method.
-    # They may get some noisy output but execution shouldn't die. Move on.
     pass
 
 
 FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8'
 JSON_CONTENT_TYPE = 'application/json'
-JSON_ACCEPT = '{0}, */*'.format(JSON_CONTENT_TYPE)
-DEFAULT_UA = 'HTTPie/%s' % __version__
+JSON_ACCEPT = f'{JSON_CONTENT_TYPE}, */*'
+DEFAULT_UA = f'HTTPie/{__version__}'
 
 
-class HTTPieHTTPAdapter(HTTPAdapter):
+def collect_messages(
+    args: argparse.Namespace,
+    config_dir: Path,
+) -> Iterable[Union[requests.PreparedRequest, requests.Response]]:
+    httpie_session = None
+    httpie_session_headers = None
+    if args.session or args.session_read_only:
+        httpie_session = get_httpie_session(
+            config_dir=config_dir,
+            session_name=args.session or args.session_read_only,
+            host=args.headers.get('Host'),
+            url=args.url,
+        )
+        httpie_session_headers = httpie_session.headers
+
+    request_kwargs = make_request_kwargs(
+        args=args,
+        base_headers=httpie_session_headers,
+    )
+    send_kwargs = make_send_kwargs(args)
+    send_kwargs_mergeable_from_env = make_send_kwargs_mergeable_from_env(args)
+    requests_session = build_requests_session(
+        ssl_version=args.ssl_version,
+    )
+
+    if httpie_session:
+        httpie_session.update_headers(request_kwargs['headers'])
+        requests_session.cookies = httpie_session.cookies
+        if args.auth_plugin:
+            # Save auth from CLI to HTTPie session.
+            httpie_session.auth = {
+                'type': args.auth_plugin.auth_type,
+                'raw_auth': args.auth_plugin.raw_auth,
+            }
+        elif httpie_session.auth:
+            # Apply auth from HTTPie session
+            request_kwargs['auth'] = httpie_session.auth
+
+    if args.debug:
+        # TODO: reflect the split between request and send kwargs.
+        dump_request(request_kwargs)
+
+    request = requests.Request(**request_kwargs)
+    prepared_request = requests_session.prepare_request(request)
+    if args.compress and prepared_request.body:
+        compress_body(prepared_request, always=args.compress > 1)
+    response_count = 0
+    while prepared_request:
+        yield prepared_request
+        if not args.offline:
+            send_kwargs_merged = requests_session.merge_environment_settings(
+                url=prepared_request.url,
+                **send_kwargs_mergeable_from_env,
+            )
+            with max_headers(args.max_headers):
+                response = requests_session.send(
+                    request=prepared_request,
+                    **send_kwargs_merged,
+                    **send_kwargs,
+                )
+            response_count += 1
+            if response.next:
+                if args.max_redirects and response_count == args.max_redirects:
+                    raise requests.TooManyRedirects
+                if args.follow:
+                    prepared_request = response.next
+                    if args.all:
+                        yield response
+                    continue
+            yield response
+        break
+
+    if httpie_session:
+        if httpie_session.is_new() or not args.session_read_only:
+            httpie_session.cookies = requests_session.cookies
+            httpie_session.save()
+
+
+# noinspection PyProtectedMember
+@contextmanager
+def max_headers(limit):
+    # <https://github.com/jakubroztocil/httpie/issues/802>
+    orig = http.client._MAXHEADERS
+    http.client._MAXHEADERS = limit or float('Inf')
+    try:
+        yield
+    finally:
+        http.client._MAXHEADERS = orig
+
+
+def compress_body(request: requests.PreparedRequest, always: bool):
+    deflater = zlib.compressobj()
+    body_bytes = (
+        request.body
+        if isinstance(request.body, bytes)
+        else request.body.encode()
+    )
+    deflated_data = deflater.compress(body_bytes)
+    deflated_data += deflater.flush()
+    is_economical = len(deflated_data) < len(body_bytes)
+    if is_economical or always:
+        request.body = deflated_data
+        request.headers['Content-Encoding'] = 'deflate'
+        request.headers['Content-Length'] = str(len(deflated_data))
+
+
+class HTTPieHTTPSAdapter(HTTPAdapter):
 
     def __init__(self, ssl_version=None, **kwargs):
         self._ssl_version = ssl_version
-        super(HTTPieHTTPAdapter, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
     def init_poolmanager(self, *args, **kwargs):
         kwargs['ssl_version'] = self._ssl_version
-        super(HTTPieHTTPAdapter, self).init_poolmanager(*args, **kwargs)
+        super().init_poolmanager(*args, **kwargs)
 
 
-def get_requests_session(ssl_version):
+def build_requests_session(
+    ssl_version: str = None,
+) -> requests.Session:
     requests_session = requests.Session()
-    requests_session.mount(
-        'https://',
-        HTTPieHTTPAdapter(ssl_version=ssl_version)
-    )
-    for cls in plugin_manager.get_transport_plugins():
-        transport_plugin = cls()
-        requests_session.mount(prefix=transport_plugin.prefix,
-                               adapter=transport_plugin.get_adapter())
+
+    # Install our adapter.
+    requests_session.mount('https://', HTTPieHTTPSAdapter(
+        ssl_version=(
+            SSL_VERSION_ARG_MAPPING[ssl_version]
+            if ssl_version else None
+        )
+    ))
+
+    # Install adapters from plugins.
+    for plugin_cls in plugin_manager.get_transport_plugins():
+        transport_plugin = plugin_cls()
+        requests_session.mount(
+            prefix=transport_plugin.prefix,
+            adapter=transport_plugin.get_adapter(),
+        )
+
     return requests_session
 
 
-def get_response(args, config_dir):
-    """Send the request and return a `request.Response`."""
-
-    ssl_version = None
-    if args.ssl_version:
-        ssl_version = SSL_VERSION_ARG_MAPPING[args.ssl_version]
-
-    requests_session = get_requests_session(ssl_version)
-    requests_session.max_redirects = args.max_redirects
-
-    if not args.session and not args.session_read_only:
-        kwargs = get_requests_kwargs(args)
-        if args.debug:
-            dump_request(kwargs)
-        response = requests_session.request(**kwargs)
-    else:
-        response = sessions.get_response(
-            requests_session=requests_session,
-            args=args,
-            config_dir=config_dir,
-            session_name=args.session or args.session_read_only,
-            read_only=bool(args.session_read_only),
-        )
-
-    return response
+def dump_request(kwargs: dict):
+    sys.stderr.write(
+        f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n')
 
 
-def dump_request(kwargs):
-    sys.stderr.write('\n>>> requests.request(**%s)\n\n'
-                     % repr_dict_nice(kwargs))
-
-
-def finalize_headers(headers):
-    final_headers = {}
+def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
+    final_headers = RequestHeadersDict()
     for name, value in headers.items():
         if value is not None:
-
-            # >leading or trailing LWS MAY be removed without
-            # >changing the semantics of the field value"
-            # -https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html
+            # “leading or trailing LWS MAY be removed without
+            # changing the semantics of the field value”
+            # <https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html>
             # Also, requests raises `InvalidHeader` for leading spaces.
             value = value.strip()
-
             if isinstance(value, str):
-                # See: https://github.com/jakubroztocil/httpie/issues/212
+                # See <https://github.com/jakubroztocil/httpie/issues/212>
                 value = value.encode('utf8')
-
         final_headers[name] = value
     return final_headers
 
 
-def get_default_headers(args):
-    default_headers = CaseInsensitiveDict({
+def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict:
+    default_headers = RequestHeadersDict({
         'User-Agent': DEFAULT_UA
     })
 
@@ -124,9 +213,40 @@ def get_default_headers(args):
     return default_headers
 
 
-def get_requests_kwargs(args, base_headers=None):
+def make_send_kwargs(args: argparse.Namespace) -> dict:
+    kwargs = {
+        'timeout': args.timeout or None,
+        'allow_redirects': False,
+    }
+    return kwargs
+
+
+def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
+    cert = None
+    if args.cert:
+        cert = args.cert
+        if args.cert_key:
+            cert = cert, args.cert_key
+    kwargs = {
+        'proxies': {p.key: p.value for p in args.proxy},
+        'stream': True,
+        'verify': {
+            'yes': True,
+            'true': True,
+            'no': False,
+            'false': False,
+        }.get(args.verify.lower(), args.verify),
+        'cert': cert,
+    }
+    return kwargs
+
+
+def make_request_kwargs(
+    args: argparse.Namespace,
+    base_headers: RequestHeadersDict = None
+) -> dict:
     """
-    Translate our `args` into `requests.request` keyword arguments.
+    Translate our `args` into `requests.Request` keyword arguments.
 
     """
     # Serialize JSON data, if needed.
@@ -141,37 +261,20 @@ def get_requests_kwargs(args, base_headers=None):
             data = ''
 
     # Finalize headers.
-    headers = get_default_headers(args)
+    headers = make_default_headers(args)
     if base_headers:
         headers.update(base_headers)
     headers.update(args.headers)
     headers = finalize_headers(headers)
 
-    cert = None
-    if args.cert:
-        cert = args.cert
-        if args.cert_key:
-            cert = cert, args.cert_key
-
     kwargs = {
-        'stream': True,
         'method': args.method.lower(),
         'url': args.url,
         'headers': headers,
         'data': data,
-        'verify': {
-            'yes': True,
-            'true': True,
-            'no': False,
-            'false': False,
-        }.get(args.verify.lower(), args.verify),
-        'cert': cert,
-        'timeout': args.timeout,
         'auth': args.auth,
-        'proxies': {p.key: p.value for p in args.proxy},
-        'files': args.files,
-        'allow_redirects': args.follow,
         'params': args.params,
+        'files': args.files,
     }
 
     return kwargs

httpie/compat.py

@@ -1,39 +1,4 @@
-"""
-Python 2.7, and 3.x compatibility.
-
-"""
 import sys
 
 
-is_py2 = sys.version_info[0] == 2
-is_py27 = sys.version_info[:2] == (2, 7)
-is_py3 = sys.version_info[0] == 3
-is_pypy = 'pypy' in sys.version.lower()
 is_windows = 'win32' in str(sys.platform).lower()
-
-
-if is_py2:
-    # noinspection PyShadowingBuiltins
-    bytes = str
-    # noinspection PyUnresolvedReferences,PyShadowingBuiltins
-    str = unicode
-elif is_py3:
-    # noinspection PyShadowingBuiltins
-    str = str
-    # noinspection PyShadowingBuiltins
-    bytes = bytes
-
-
-try:  # pragma: no cover
-    # noinspection PyUnresolvedReferences,PyCompatibility
-    from urllib.parse import urlsplit
-except ImportError:  # pragma: no cover
-    # noinspection PyUnresolvedReferences,PyCompatibility
-    from urlparse import urlsplit
-
-try:  # pragma: no cover
-    # noinspection PyCompatibility
-    from urllib.request import urlopen
-except ImportError:  # pragma: no cover
-    # noinspection PyCompatibility,PyUnresolvedReferences
-    from urllib2 import urlopen

httpie/config.py

@@ -1,61 +1,59 @@
-import os
-import json
 import errno
+import json
+import os
+from pathlib import Path
+from typing import Union
 
 from httpie import __version__
 from httpie.compat import is_windows
 
 
-DEFAULT_CONFIG_DIR = str(os.environ.get(
+DEFAULT_CONFIG_DIR = Path(os.environ.get(
     'HTTPIE_CONFIG_DIR',
     os.path.expanduser('~/.httpie') if not is_windows else
     os.path.expandvars(r'%APPDATA%\\httpie')
 ))
 
 
-class BaseConfigDict(dict):
+class ConfigFileError(Exception):
+    pass
 
+
+class BaseConfigDict(dict):
     name = None
     helpurl = None
     about = None
 
-    def __getattr__(self, item):
-        return self[item]
+    def __init__(self, path: Path):
+        super().__init__()
+        self.path = path
 
-    def _get_path(self):
-        """Return the config file path without side-effects."""
-        raise NotImplementedError()
-
-    @property
-    def path(self):
-        """Return the config file path creating basedir, if needed."""
-        path = self._get_path()
+    def ensure_directory(self):
         try:
-            os.makedirs(os.path.dirname(path), mode=0o700)
+            self.path.parent.mkdir(mode=0o700, parents=True)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
-        return path
 
-    def is_new(self):
-        return not os.path.exists(self._get_path())
+    def is_new(self) -> bool:
+        return not self.path.exists()
 
     def load(self):
+        config_type = type(self).__name__.lower()
         try:
-            with open(self.path, 'rt') as f:
+            with self.path.open('rt') as f:
                 try:
                     data = json.load(f)
                 except ValueError as e:
-                    raise ValueError(
-                        'Invalid %s JSON: %s [%s]' %
-                        (type(self).__name__, str(e), self.path)
+                    raise ConfigFileError(
+                        f'invalid {config_type} file: {e} [{self.path}]'
                     )
                 self.update(data)
         except IOError as e:
             if e.errno != errno.ENOENT:
-                raise
+                raise ConfigFileError(f'cannot read {config_type} file: {e}')
 
-    def save(self):
+    def save(self, fail_silently=False):
         self['__meta__'] = {
             'httpie': __version__
         }
@@ -65,48 +63,41 @@ class BaseConfigDict(dict):
         if self.about:
             self['__meta__']['about'] = self.about
 
-        with open(self.path, 'w') as f:
-            json.dump(self, f, indent=4, sort_keys=True, ensure_ascii=True)
-            f.write('\n')
+        self.ensure_directory()
+
+        try:
+            with self.path.open('w') as f:
+                json.dump(
+                    obj=self,
+                    fp=f,
+                    indent=4,
+                    sort_keys=True,
+                    ensure_ascii=True,
+                )
+                f.write('\n')
+        except IOError:
+            if not fail_silently:
+                raise
 
     def delete(self):
         try:
-            os.unlink(self.path)
+            self.path.unlink()
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
 
 class Config(BaseConfigDict):
-
-    name = 'config'
-    helpurl = 'https://httpie.org/doc#config'
-    about = 'HTTPie configuration file'
-
+    FILENAME = 'config.json'
     DEFAULTS = {
         'default_options': []
     }
 
-    def __init__(self, directory=DEFAULT_CONFIG_DIR):
-        super(Config, self).__init__()
+    def __init__(self, directory: Union[str, Path] = DEFAULT_CONFIG_DIR):
+        self.directory = Path(directory)
+        super().__init__(path=self.directory / self.FILENAME)
         self.update(self.DEFAULTS)
-        self.directory = directory
 
-    def load(self):
-        super(Config, self).load()
-        self._migrate_implicit_content_type()
-
-    def _get_path(self):
-        return os.path.join(self.directory, self.name + '.json')
-
-    def _migrate_implicit_content_type(self):
-        """Migrate the removed implicit_content_type config option"""
-        try:
-            implicit_content_type = self.pop('implicit_content_type')
-        except KeyError:
-            self.save()
-        else:
-            if implicit_content_type == 'form':
-                self['default_options'].insert(0, '--form')
-            self.save()
-            self.load()
+    @property
+    def default_options(self) -> list:
+        return self['default_options']

httpie/context.py

@@ -1,16 +1,21 @@
+import os
 import sys
+from pathlib import Path
+from typing import Union, IO, Optional
+
+
 try:
     import curses
 except ImportError:
     curses = None  # Compiled w/o curses
 
 from httpie.compat import is_windows
-from httpie.config import DEFAULT_CONFIG_DIR, Config
+from httpie.config import DEFAULT_CONFIG_DIR, Config, ConfigFileError
 
-from httpie.utils import repr_dict_nice
+from httpie.utils import repr_dict
 
 
-class Environment(object):
+class Environment:
     """
     Information about the execution context
     (standard streams, config directory, etc).
@@ -20,17 +25,18 @@ class Environment(object):
     is used by the test suite to simulate various scenarios.
 
     """
-    is_windows = is_windows
-    config_dir = DEFAULT_CONFIG_DIR
-    stdin = sys.stdin
-    stdin_isatty = stdin.isatty()
-    stdin_encoding = None
-    stdout = sys.stdout
-    stdout_isatty = stdout.isatty()
-    stdout_encoding = None
-    stderr = sys.stderr
-    stderr_isatty = stderr.isatty()
+    is_windows: bool = is_windows
+    config_dir: Path = DEFAULT_CONFIG_DIR
+    stdin: Optional[IO] = sys.stdin  # `None` when closed fd (#791)
+    stdin_isatty: bool = stdin.isatty() if stdin else False
+    stdin_encoding: str = None
+    stdout: IO = sys.stdout
+    stdout_isatty: bool = stdout.isatty()
+    stdout_encoding: str = None
+    stderr: IO = sys.stderr
+    stderr_isatty: bool = stderr.isatty()
     colors = 256
+    program_name: str = 'http'
     if not is_windows:
         if curses:
             try:
@@ -61,7 +67,7 @@ class Environment(object):
         self.__dict__.update(**kwargs)
 
         # Keyword arguments > stream.encoding > default utf8
-        if self.stdin_encoding is None:
+        if self.stdin and self.stdin_encoding is None:
             self.stdin_encoding = getattr(
                 self.stdin, 'encoding', None) or 'utf8'
         if self.stdout_encoding is None:
@@ -70,30 +76,39 @@ class Environment(object):
                 # noinspection PyUnresolvedReferences
                 from colorama import AnsiToWin32
                 if isinstance(self.stdout, AnsiToWin32):
+                    # noinspection PyUnresolvedReferences
                     actual_stdout = self.stdout.wrapped
             self.stdout_encoding = getattr(
                 actual_stdout, 'encoding', None) or 'utf8'
 
-    @property
-    def config(self):
-        if not hasattr(self, '_config'):
-            self._config = Config(directory=self.config_dir)
-            if self._config.is_new():
-                self._config.save()
-            else:
-                self._config.load()
-        return self._config
-
     def __str__(self):
         defaults = dict(type(self).__dict__)
         actual = dict(defaults)
         actual.update(self.__dict__)
         actual['config'] = self.config
-        return repr_dict_nice(dict(
-            (key, value)
+        return repr_dict({
+            key: value
             for key, value in actual.items()
-            if not key.startswith('_'))
-        )
+            if not key.startswith('_')
+        })
 
     def __repr__(self):
-        return '<{0} {1}>'.format(type(self).__name__, str(self))
+        return f'<{type(self).__name__} {self}>'
+
+    _config: Config = None
+
+    @property
+    def config(self) -> Config:
+        config = self._config
+        if not config:
+            self._config = config = Config(directory=self.config_dir)
+            if not config.is_new():
+                try:
+                    config.load()
+                except ConfigFileError as e:
+                    self.log_error(e, level='warning')
+        return config
+
+    def log_error(self, msg, level='error'):
+        assert level in ['error', 'warning']
+        self.stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n')

httpie/core.py

@@ -1,175 +1,26 @@
-"""This module provides the main functionality of HTTPie.
-
-Invocation flow:
-
-  1. Read, validate and process the input (args, `stdin`).
-  2. Create and send a request.
-  3. Stream, and possibly process and format, the parts
-     of the request-response exchange selected by output options.
-  4. Simultaneously write to `stdout`
-  5. Exit.
-
-"""
-import sys
-import errno
+import argparse
+import os
 import platform
+import sys
+from typing import List, Union
 
 import requests
-from requests import __version__ as requests_version
 from pygments import __version__ as pygments_version
+from requests import __version__ as requests_version
 
-from httpie import __version__ as httpie_version, ExitStatus
-from httpie.compat import str, bytes, is_py3
-from httpie.client import get_response
-from httpie.downloads import Downloader
+from httpie import __version__ as httpie_version
+from httpie.client import collect_messages
 from httpie.context import Environment
+from httpie.downloads import Downloader
+from httpie.output.writer import write_message, write_stream
 from httpie.plugins import plugin_manager
-from httpie.output.streams import (
-    build_output_stream,
-    write_stream,
-    write_stream_with_colors_win_py3
-)
+from httpie.status import ExitStatus, http_status_to_exit_status
 
 
-def get_exit_status(http_status, follow=False):
-    """Translate HTTP status code to exit status code."""
-    if 300 <= http_status <= 399 and not follow:
-        # Redirect
-        return ExitStatus.ERROR_HTTP_3XX
-    elif 400 <= http_status <= 499:
-        # Client Error
-        return ExitStatus.ERROR_HTTP_4XX
-    elif 500 <= http_status <= 599:
-        # Server Error
-        return ExitStatus.ERROR_HTTP_5XX
-    else:
-        return ExitStatus.SUCCESS
-
-
-def print_debug_info(env):
-    env.stderr.writelines([
-        'HTTPie %s\n' % httpie_version,
-        'Requests %s\n' % requests_version,
-        'Pygments %s\n' % pygments_version,
-        'Python %s\n%s\n' % (sys.version, sys.executable),
-        '%s %s' % (platform.system(), platform.release()),
-    ])
-    env.stderr.write('\n\n')
-    env.stderr.write(repr(env))
-    env.stderr.write('\n')
-
-
-def decode_args(args, stdin_encoding):
-    """
-    Convert all bytes args to str
-    by decoding them using stdin encoding.
-
-    """
-    return [
-        arg.decode(stdin_encoding)
-        if type(arg) == bytes else arg
-        for arg in args
-    ]
-
-
-def program(args, env, log_error):
-    """
-    The main program without error handling
-
-    :param args: parsed args (argparse.Namespace)
-    :type env: Environment
-    :param log_error: error log function
-    :return: status code
-
-    """
-    exit_status = ExitStatus.SUCCESS
-    downloader = None
-    show_traceback = args.debug or args.traceback
-
-    try:
-        if args.download:
-            args.follow = True  # --download implies --follow.
-            downloader = Downloader(
-                output_file=args.output_file,
-                progress_file=env.stderr,
-                resume=args.download_resume
-            )
-            downloader.pre_request(args.headers)
-
-        final_response = get_response(args, config_dir=env.config.directory)
-        if args.all:
-            responses = final_response.history + [final_response]
-        else:
-            responses = [final_response]
-
-        for response in responses:
-
-            if args.check_status or downloader:
-                exit_status = get_exit_status(
-                    http_status=response.status_code,
-                    follow=args.follow
-                )
-                if not env.stdout_isatty and exit_status != ExitStatus.SUCCESS:
-                    log_error(
-                        'HTTP %s %s', response.raw.status, response.raw.reason,
-                        level='warning'
-                    )
-
-            write_stream_kwargs = {
-                'stream': build_output_stream(
-                    args=args,
-                    env=env,
-                    request=response.request,
-                    response=response,
-                    output_options=(
-                        args.output_options
-                        if response is final_response
-                        else args.output_options_history
-                    )
-                ),
-                # NOTE: `env.stdout` will in fact be `stderr` with `--download`
-                'outfile': env.stdout,
-                'flush': env.stdout_isatty or args.stream
-            }
-            try:
-                if env.is_windows and is_py3 and 'colors' in args.prettify:
-                    write_stream_with_colors_win_py3(**write_stream_kwargs)
-                else:
-                    write_stream(**write_stream_kwargs)
-            except IOError as e:
-                if not show_traceback and e.errno == errno.EPIPE:
-                    # Ignore broken pipes unless --traceback.
-                    env.stderr.write('\n')
-                else:
-                    raise
-
-        if downloader and exit_status == ExitStatus.SUCCESS:
-            # Last response body download.
-            download_stream, download_to = downloader.start(final_response)
-            write_stream(
-                stream=download_stream,
-                outfile=download_to,
-                flush=False,
-            )
-            downloader.finish()
-            if downloader.interrupted:
-                exit_status = ExitStatus.ERROR
-                log_error('Incomplete download: size=%d; downloaded=%d' % (
-                    downloader.status.total_size,
-                    downloader.status.downloaded
-                ))
-        return exit_status
-
-    finally:
-        if downloader and not downloader.finished:
-            downloader.failed()
-
-        if (not isinstance(args, list) and args.output_file
-                and args.output_file_specified):
-            args.output_file.close()
-
-
-def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
+def main(
+    args: List[Union[str, bytes]] = sys.argv,
+    env=Environment(),
+) -> ExitStatus:
     """
     The main function.
 
@@ -179,23 +30,16 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
     Return exit status code.
 
     """
-    args = decode_args(args, env.stdin_encoding)
+    program_name, *args = args
+    env.program_name = os.path.basename(program_name)
+    args = decode_raw_args(args, env.stdin_encoding)
     plugin_manager.load_installed_plugins()
 
-    def log_error(msg, *args, **kwargs):
-        msg = msg % args
-        level = kwargs.get('level', 'error')
-        assert level in ['error', 'warning']
-        env.stderr.write('\nhttp: %s: %s\n' % (level, msg))
-
-    from httpie.cli import parser
+    from httpie.cli.definition import parser
 
     if env.config.default_options:
         args = env.config.default_options + args
 
-    if custom_log_error:
-        log_error = custom_log_error
-
     include_debug_info = '--debug' in args
     include_traceback = include_debug_info or '--traceback' in args
 
@@ -207,7 +51,10 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
     exit_status = ExitStatus.SUCCESS
 
     try:
-        parsed_args = parser.parse_args(args=args, env=env)
+        parsed_args = parser.parse_args(
+            args=args,
+            env=env,
+        )
     except KeyboardInterrupt:
         env.stderr.write('\n')
         if include_traceback:
@@ -224,7 +71,6 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
             exit_status = program(
                 args=parsed_args,
                 env=env,
-                log_error=log_error,
             )
         except KeyboardInterrupt:
             env.stderr.write('\n')
@@ -239,22 +85,132 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
                 exit_status = ExitStatus.ERROR
         except requests.Timeout:
             exit_status = ExitStatus.ERROR_TIMEOUT
-            log_error('Request timed out (%ss).', parsed_args.timeout)
+            env.log_error(f'Request timed out ({parsed_args.timeout}s).')
         except requests.TooManyRedirects:
             exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
-            log_error('Too many redirects (--max-redirects=%s).',
-                      parsed_args.max_redirects)
+            env.log_error(
+                f'Too many redirects'
+                f' (--max-redirects=parsed_args.max_redirects).'
+            )
         except Exception as e:
             # TODO: Further distinction between expected and unexpected errors.
             msg = str(e)
             if hasattr(e, 'request'):
                 request = e.request
                 if hasattr(request, 'url'):
-                    msg += ' while doing %s request to URL: %s' % (
-                        request.method, request.url)
-            log_error('%s: %s', type(e).__name__, msg)
+                    msg = (
+                        f'{msg} while doing a {request.method}'
+                        f' request to URL: {request.url}'
+                    )
+            env.log_error(f'{type(e).__name__}: {msg}')
             if include_traceback:
                 raise
             exit_status = ExitStatus.ERROR
 
     return exit_status
+
+
+def program(
+    args: argparse.Namespace,
+    env: Environment,
+) -> ExitStatus:
+    """
+    The main program without error handling.
+
+    """
+    exit_status = ExitStatus.SUCCESS
+    downloader = None
+
+    try:
+        if args.download:
+            args.follow = True  # --download implies --follow.
+            downloader = Downloader(
+                output_file=args.output_file,
+                progress_file=env.stderr,
+                resume=args.download_resume
+            )
+            downloader.pre_request(args.headers)
+
+        initial_request = None
+        final_response = None
+
+        for message in collect_messages(args, env.config.directory):
+            write_message(
+                requests_message=message,
+                env=env,
+                args=args,
+            )
+            if isinstance(message, requests.PreparedRequest):
+                if not initial_request:
+                    initial_request = message
+            else:
+                final_response = message
+                if args.check_status or downloader:
+                    exit_status = http_status_to_exit_status(
+                        http_status=message.status_code,
+                        follow=args.follow
+                    )
+                    if (not env.stdout_isatty
+                            and exit_status != ExitStatus.SUCCESS):
+                        env.log_error(
+                            f'HTTP {message.raw.status} {message.raw.reason}',
+                            level='warning'
+                        )
+
+        if downloader and exit_status == ExitStatus.SUCCESS:
+            # Last response body download.
+            download_stream, download_to = downloader.start(
+                initial_url=initial_request.url,
+                final_response=final_response,
+            )
+            write_stream(
+                stream=download_stream,
+                outfile=download_to,
+                flush=False,
+            )
+            downloader.finish()
+            if downloader.interrupted:
+                exit_status = ExitStatus.ERROR
+                env.log_error(
+                    'Incomplete download: size=%d; downloaded=%d' % (
+                        downloader.status.total_size,
+                        downloader.status.downloaded
+                    ))
+        return exit_status
+
+    finally:
+        if downloader and not downloader.finished:
+            downloader.failed()
+
+        if (not isinstance(args, list) and args.output_file
+                and args.output_file_specified):
+            args.output_file.close()
+
+
+def print_debug_info(env: Environment):
+    env.stderr.writelines([
+        f'HTTPie {httpie_version}\n',
+        f'Requests {requests_version}\n',
+        f'Pygments {pygments_version}\n',
+        f'Python {sys.version}\n{sys.executable}\n',
+        f'{platform.system()} {platform.release()}',
+    ])
+    env.stderr.write('\n\n')
+    env.stderr.write(repr(env))
+    env.stderr.write('\n')
+
+
+def decode_raw_args(
+    args: List[Union[str, bytes]],
+    stdin_encoding: str
+) -> List[str]:
+    """
+    Convert all bytes args to str
+    by decoding them using stdin encoding.
+
+    """
+    return [
+        arg.decode(stdin_encoding)
+        if type(arg) == bytes else arg
+        for arg in args
+    ]

httpie/downloads.py

@@ -4,24 +4,27 @@ Download mode implementation.
 
 """
 from __future__ import division
+
+import errno
+import mimetypes
 import os
 import re
 import sys
-import errno
-import mimetypes
 import threading
-from time import sleep, time
 from mailbox import Message
+from time import sleep, time
+from typing import IO, Optional, Tuple
+from urllib.parse import urlsplit
+
+import requests
 
-from httpie.output.streams import RawStream
 from httpie.models import HTTPResponse
+from httpie.output.streams import RawStream
 from httpie.utils import humanize_bytes
-from httpie.compat import urlsplit
 
 
 PARTIAL_CONTENT = 206
 
-
 CLEAR_LINE = '\r\033[K'
 PROGRESS = (
     '{percentage: 6.2f} %'
@@ -38,11 +41,11 @@ class ContentRangeError(ValueError):
     pass
 
 
-def parse_content_range(content_range, resumed_from):
+def parse_content_range(content_range: str, resumed_from: int) -> int:
     """
     Parse and validate Content-Range header.
 
-    <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>
+    <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>
 
     :param content_range: the value of a Content-Range response header
                           eg. "bytes 21010-47021/47022"
@@ -79,14 +82,14 @@ def parse_content_range(content_range, resumed_from):
     # byte-content-range- spec MUST ignore it and any content
     # transferred along with it."
     if (first_byte_pos >= last_byte_pos
-            or (instance_length is not None
-                and instance_length <= last_byte_pos)):
+        or (instance_length is not None
+            and instance_length <= last_byte_pos)):
         raise ContentRangeError(
             'Invalid Content-Range returned: %r' % content_range)
 
     if (first_byte_pos != resumed_from
-            or (instance_length is not None
-                and last_byte_pos + 1 != instance_length)):
+        or (instance_length is not None
+            and last_byte_pos + 1 != instance_length)):
         # Not what we asked for.
         raise ContentRangeError(
             'Unexpected Content-Range returned (%r)'
@@ -97,7 +100,9 @@ def parse_content_range(content_range, resumed_from):
     return last_byte_pos + 1
 
 
-def filename_from_content_disposition(content_disposition):
+def filename_from_content_disposition(
+    content_disposition: str
+) -> Optional[str]:
     """
     Extract and validate filename from a Content-Disposition header.
 
@@ -116,7 +121,7 @@ def filename_from_content_disposition(content_disposition):
             return filename
 
 
-def filename_from_url(url, content_type):
+def filename_from_url(url: str, content_type: Optional[str]) -> str:
     fn = urlsplit(url).path.rstrip('/')
     fn = os.path.basename(fn) if fn else 'index'
     if '.' not in fn and content_type:
@@ -136,7 +141,7 @@ def filename_from_url(url, content_type):
     return fn
 
 
-def trim_filename(filename, max_len):
+def trim_filename(filename: str, max_len: int) -> str:
     if len(filename) > max_len:
         trim_by = len(filename) - max_len
         name, ext = os.path.splitext(filename)
@@ -147,7 +152,7 @@ def trim_filename(filename, max_len):
     return filename
 
 
-def get_filename_max_length(directory):
+def get_filename_max_length(directory: str) -> int:
     max_len = 255
     try:
         pathconf = os.pathconf
@@ -162,14 +167,14 @@ def get_filename_max_length(directory):
     return max_len
 
 
-def trim_filename_if_needed(filename, directory='.', extra=0):
+def trim_filename_if_needed(filename: str, directory='.', extra=0) -> str:
     max_len = get_filename_max_length(directory) - extra
     if len(filename) > max_len:
         filename = trim_filename(filename, max_len)
     return filename
 
 
-def get_unique_filename(filename, exists=os.path.exists):
+def get_unique_filename(filename: str, exists=os.path.exists) -> str:
     attempt = 0
     while True:
         suffix = '-' + str(attempt) if attempt > 0 else ''
@@ -180,10 +185,14 @@ def get_unique_filename(filename, exists=os.path.exists):
         attempt += 1
 
 
-class Downloader(object):
+class Downloader:
 
-    def __init__(self, output_file=None,
-                 resume=False, progress_file=sys.stderr):
+    def __init__(
+        self,
+        output_file: IO = None,
+        resume: bool = False,
+        progress_file: IO = sys.stderr
+    ):
         """
         :param resume: Should the download resume if partial download
                        already exists.
@@ -195,24 +204,21 @@ class Downloader(object):
         :param progress_file: Where to report download progress.
 
         """
+        self.finished = False
+        self.status = DownloadStatus()
         self._output_file = output_file
         self._resume = resume
         self._resumed_from = 0
-        self.finished = False
-
-        self.status = Status()
         self._progress_reporter = ProgressReporterThread(
             status=self.status,
             output=progress_file
         )
 
-    def pre_request(self, request_headers):
+    def pre_request(self, request_headers: dict):
         """Called just before the HTTP request is sent.
 
         Might alter `request_headers`.
 
-        :type request_headers: dict
-
         """
         # Ask the server not to encode the content so that we can resume, etc.
         request_headers['Accept-Encoding'] = 'identity'
@@ -224,13 +230,17 @@ class Downloader(object):
                 request_headers['Range'] = 'bytes=%d-' % bytes_have
                 self._resumed_from = bytes_have
 
-    def start(self, response):
+    def start(
+        self,
+        initial_url: str,
+        final_response: requests.Response
+    ) -> Tuple[RawStream, IO]:
         """
         Initiate and return a stream for `response` body  with progress
         callback attached. Can be called only once.
 
-        :param response: Initiated response object with headers already fetched
-        :type response: requests.models.Response
+        :param initial_url: The original requested URL
+        :param final_response: Initiated response object with headers already fetched
 
         :return: RawStream, output_file
 
@@ -240,14 +250,20 @@ class Downloader(object):
         # FIXME: some servers still might sent Content-Encoding: gzip
         # <https://github.com/jakubroztocil/httpie/issues/423>
         try:
-            total_size = int(response.headers['Content-Length'])
+            total_size = int(final_response.headers['Content-Length'])
         except (KeyError, ValueError, TypeError):
             total_size = None
 
-        if self._output_file:
-            if self._resume and response.status_code == PARTIAL_CONTENT:
+        if not self._output_file:
+            self._output_file = self._get_output_file_from_response(
+                initial_url=initial_url,
+                final_response=final_response,
+            )
+        else:
+            # `--output, -o` provided
+            if self._resume and final_response.status_code == PARTIAL_CONTENT:
                 total_size = parse_content_range(
-                    response.headers.get('Content-Range'),
+                    final_response.headers.get('Content-Range'),
                     self._resumed_from
                 )
 
@@ -258,19 +274,6 @@ class Downloader(object):
                     self._output_file.truncate()
                 except IOError:
                     pass  # stdout
-        else:
-            # TODO: Should the filename be taken from response.history[0].url?
-            # Output file not specified. Pick a name that doesn't exist yet.
-            filename = None
-            if 'Content-Disposition' in response.headers:
-                filename = filename_from_content_disposition(
-                    response.headers['Content-Disposition'])
-            if not filename:
-                filename = filename_from_url(
-                    url=response.url,
-                    content_type=response.headers.get('Content-Type'),
-                )
-            self._output_file = open(get_unique_filename(filename), mode='a+b')
 
         self.status.started(
             resumed_from=self._resumed_from,
@@ -278,7 +281,7 @@ class Downloader(object):
         )
 
         stream = RawStream(
-            msg=HTTPResponse(response),
+            msg=HTTPResponse(final_response),
             with_headers=False,
             with_body=True,
             on_body_chunk_downloaded=self.chunk_downloaded,
@@ -306,14 +309,14 @@ class Downloader(object):
         self._progress_reporter.stop()
 
     @property
-    def interrupted(self):
+    def interrupted(self) -> bool:
         return (
             self.finished
             and self.status.total_size
             and self.status.total_size != self.status.downloaded
         )
 
-    def chunk_downloaded(self, chunk):
+    def chunk_downloaded(self, chunk: bytes):
         """
         A download progress callback.
 
@@ -324,9 +327,27 @@ class Downloader(object):
         """
         self.status.chunk_downloaded(len(chunk))
 
+    @staticmethod
+    def _get_output_file_from_response(
+        initial_url: str,
+        final_response: requests.Response,
+    ) -> IO:
+        # Output file not specified. Pick a name that doesn't exist yet.
+        filename = None
+        if 'Content-Disposition' in final_response.headers:
+            filename = filename_from_content_disposition(
+                final_response.headers['Content-Disposition'])
+        if not filename:
+            filename = filename_from_url(
+                url=initial_url,
+                content_type=final_response.headers.get('Content-Type'),
+            )
+        unique_filename = get_unique_filename(filename)
+        return open(unique_filename, mode='a+b')
 
-class Status(object):
-    """Holds details about the downland status."""
+
+class DownloadStatus:
+    """Holds details about the download status."""
 
     def __init__(self):
         self.downloaded = 0
@@ -362,13 +383,15 @@ class ProgressReporterThread(threading.Thread):
     Uses threading to periodically update the status (speed, ETA, etc.).
 
     """
-    def __init__(self, status, output, tick=.1, update_interval=1):
-        """
 
-        :type status: Status
-        :type output: file
-        """
-        super(ProgressReporterThread, self).__init__()
+    def __init__(
+        self,
+        status: DownloadStatus,
+        output: IO,
+        tick=.1,
+        update_interval=1
+    ):
+        super().__init__()
         self.status = status
         self.output = output
         self._tick = tick

httpie/input.py

@@ -1,758 +0,0 @@
-"""Parsing and processing of CLI input (args, auth credentials, files, stdin).
-
-"""
-import os
-import ssl
-import sys
-import re
-import errno
-import mimetypes
-import getpass
-from io import BytesIO
-from collections import namedtuple, Iterable, OrderedDict
-# noinspection PyCompatibility
-from argparse import ArgumentParser, ArgumentTypeError, ArgumentError
-
-# TODO: Use MultiDict for headers once added to `requests`.
-# https://github.com/jakubroztocil/httpie/issues/130
-from httpie.plugins import plugin_manager
-from requests.structures import CaseInsensitiveDict
-
-from httpie.compat import urlsplit, str, is_pypy, is_py27
-from httpie.sessions import VALID_SESSION_NAME_PATTERN
-from httpie.utils import load_json_preserve_order
-
-
-# ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
-# <http://tools.ietf.org/html/rfc3986#section-3.1>
-URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)
-
-HTTP_POST = 'POST'
-HTTP_GET = 'GET'
-
-
-# Various separators used in args
-SEP_HEADERS = ':'
-SEP_HEADERS_EMPTY = ';'
-SEP_CREDENTIALS = ':'
-SEP_PROXY = ':'
-SEP_DATA = '='
-SEP_DATA_RAW_JSON = ':='
-SEP_FILES = '@'
-SEP_DATA_EMBED_FILE = '=@'
-SEP_DATA_EMBED_RAW_JSON_FILE = ':=@'
-SEP_QUERY = '=='
-
-# Separators that become request data
-SEP_GROUP_DATA_ITEMS = frozenset([
-    SEP_DATA,
-    SEP_DATA_RAW_JSON,
-    SEP_FILES,
-    SEP_DATA_EMBED_FILE,
-    SEP_DATA_EMBED_RAW_JSON_FILE
-])
-
-# Separators for items whose value is a filename to be embedded
-SEP_GROUP_DATA_EMBED_ITEMS = frozenset([
-    SEP_DATA_EMBED_FILE,
-    SEP_DATA_EMBED_RAW_JSON_FILE,
-])
-
-# Separators for raw JSON items
-SEP_GROUP_RAW_JSON_ITEMS = frozenset([
-    SEP_DATA_RAW_JSON,
-    SEP_DATA_EMBED_RAW_JSON_FILE,
-])
-
-# Separators allowed in ITEM arguments
-SEP_GROUP_ALL_ITEMS = frozenset([
-    SEP_HEADERS,
-    SEP_HEADERS_EMPTY,
-    SEP_QUERY,
-    SEP_DATA,
-    SEP_DATA_RAW_JSON,
-    SEP_FILES,
-    SEP_DATA_EMBED_FILE,
-    SEP_DATA_EMBED_RAW_JSON_FILE,
-])
-
-
-# Output options
-OUT_REQ_HEAD = 'H'
-OUT_REQ_BODY = 'B'
-OUT_RESP_HEAD = 'h'
-OUT_RESP_BODY = 'b'
-
-OUTPUT_OPTIONS = frozenset([
-    OUT_REQ_HEAD,
-    OUT_REQ_BODY,
-    OUT_RESP_HEAD,
-    OUT_RESP_BODY
-])
-
-# Pretty
-PRETTY_MAP = {
-    'all': ['format', 'colors'],
-    'colors': ['colors'],
-    'format': ['format'],
-    'none': []
-}
-PRETTY_STDOUT_TTY_ONLY = object()
-
-
-# Defaults
-OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
-OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
-
-
-SSL_VERSION_ARG_MAPPING = {
-    'ssl2.3': 'PROTOCOL_SSLv23',
-    'ssl3': 'PROTOCOL_SSLv3',
-    'tls1': 'PROTOCOL_TLSv1',
-    'tls1.1': 'PROTOCOL_TLSv1_1',
-    'tls1.2': 'PROTOCOL_TLSv1_2',
-    'tls1.3': 'PROTOCOL_TLSv1_3',
-}
-SSL_VERSION_ARG_MAPPING = {
-    cli_arg: getattr(ssl, ssl_constant)
-    for cli_arg, ssl_constant in SSL_VERSION_ARG_MAPPING.items()
-    if hasattr(ssl, ssl_constant)
-}
-
-
-class HTTPieArgumentParser(ArgumentParser):
-    """Adds additional logic to `argparse.ArgumentParser`.
-
-    Handles all input (CLI args, file args, stdin), applies defaults,
-    and performs extra validation.
-
-    """
-
-    def __init__(self, *args, **kwargs):
-        kwargs['add_help'] = False
-        super(HTTPieArgumentParser, self).__init__(*args, **kwargs)
-
-    # noinspection PyMethodOverriding
-    def parse_args(self, env, args=None, namespace=None):
-
-        self.env = env
-        self.args, no_options = super(HTTPieArgumentParser, self)\
-            .parse_known_args(args, namespace)
-
-        if self.args.debug:
-            self.args.traceback = True
-
-        # Arguments processing and environment setup.
-        self._apply_no_options(no_options)
-        self._validate_download_options()
-        self._setup_standard_streams()
-        self._process_output_options()
-        self._process_pretty_options()
-        self._guess_method()
-        self._parse_items()
-        if not self.args.ignore_stdin and not env.stdin_isatty:
-            self._body_from_file(self.env.stdin)
-        if not URL_SCHEME_RE.match(self.args.url):
-            scheme = self.args.default_scheme + "://"
-
-            # See if we're using curl style shorthand for localhost (:3000/foo)
-            shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
-            if shorthand:
-                port = shorthand.group(1)
-                rest = shorthand.group(2)
-                self.args.url = scheme + 'localhost'
-                if port:
-                    self.args.url += ':' + port
-                self.args.url += rest
-            else:
-                self.args.url = scheme + self.args.url
-        self._process_auth()
-
-        return self.args
-
-    # noinspection PyShadowingBuiltins
-    def _print_message(self, message, file=None):
-        # Sneak in our stderr/stdout.
-        file = {
-            sys.stdout: self.env.stdout,
-            sys.stderr: self.env.stderr,
-            None: self.env.stderr
-        }.get(file, file)
-        if not hasattr(file, 'buffer') and isinstance(message, str):
-            message = message.encode(self.env.stdout_encoding)
-        super(HTTPieArgumentParser, self)._print_message(message, file)
-
-    def _setup_standard_streams(self):
-        """
-        Modify `env.stdout` and `env.stdout_isatty` based on args, if needed.
-
-        """
-        self.args.output_file_specified = bool(self.args.output_file)
-        if self.args.download:
-            # FIXME: Come up with a cleaner solution.
-            if not self.args.output_file and not self.env.stdout_isatty:
-                # Use stdout as the download output file.
-                self.args.output_file = self.env.stdout
-            # With `--download`, we write everything that would normally go to
-            # `stdout` to `stderr` instead. Let's replace the stream so that
-            # we don't have to use many `if`s throughout the codebase.
-            # The response body will be treated separately.
-            self.env.stdout = self.env.stderr
-            self.env.stdout_isatty = self.env.stderr_isatty
-        elif self.args.output_file:
-            # When not `--download`ing, then `--output` simply replaces
-            # `stdout`. The file is opened for appending, which isn't what
-            # we want in this case.
-            self.args.output_file.seek(0)
-            try:
-                self.args.output_file.truncate()
-            except IOError as e:
-                if e.errno == errno.EINVAL:
-                    # E.g. /dev/null on Linux.
-                    pass
-                else:
-                    raise
-            self.env.stdout = self.args.output_file
-            self.env.stdout_isatty = False
-
-    def _process_auth(self):
-        # TODO: refactor
-        self.args.auth_plugin = None
-        default_auth_plugin = plugin_manager.get_auth_plugins()[0]
-        auth_type_set = self.args.auth_type is not None
-        url = urlsplit(self.args.url)
-
-        if self.args.auth is None and not auth_type_set:
-            if url.username is not None:
-                # Handle http://username:password@hostname/
-                username = url.username
-                password = url.password or ''
-                self.args.auth = AuthCredentials(
-                    key=username,
-                    value=password,
-                    sep=SEP_CREDENTIALS,
-                    orig=SEP_CREDENTIALS.join([username, password])
-                )
-
-        if self.args.auth is not None or auth_type_set:
-            if not self.args.auth_type:
-                self.args.auth_type = default_auth_plugin.auth_type
-            plugin = plugin_manager.get_auth_plugin(self.args.auth_type)()
-
-            if plugin.auth_require and self.args.auth is None:
-                self.error('--auth required')
-
-            plugin.raw_auth = self.args.auth
-            self.args.auth_plugin = plugin
-            already_parsed = isinstance(self.args.auth, AuthCredentials)
-
-            if self.args.auth is None or not plugin.auth_parse:
-                self.args.auth = plugin.get_auth()
-            else:
-                if already_parsed:
-                    # from the URL
-                    credentials = self.args.auth
-                else:
-                    credentials = parse_auth(self.args.auth)
-
-                if (not credentials.has_password()
-                        and plugin.prompt_password):
-                    if self.args.ignore_stdin:
-                        # Non-tty stdin read by now
-                        self.error(
-                            'Unable to prompt for passwords because'
-                            ' --ignore-stdin is set.'
-                        )
-                    credentials.prompt_password(url.netloc)
-                self.args.auth = plugin.get_auth(
-                    username=credentials.key,
-                    password=credentials.value,
-                )
-
-    def _apply_no_options(self, no_options):
-        """For every `--no-OPTION` in `no_options`, set `args.OPTION` to
-        its default value. This allows for un-setting of options, e.g.,
-        specified in config.
-
-        """
-        invalid = []
-
-        for option in no_options:
-            if not option.startswith('--no-'):
-                invalid.append(option)
-                continue
-
-            # --no-option => --option
-            inverted = '--' + option[5:]
-            for action in self._actions:
-                if inverted in action.option_strings:
-                    setattr(self.args, action.dest, action.default)
-                    break
-            else:
-                invalid.append(option)
-
-        if invalid:
-            msg = 'unrecognized arguments: %s'
-            self.error(msg % ' '.join(invalid))
-
-    def _body_from_file(self, fd):
-        """There can only be one source of request data.
-
-        Bytes are always read.
-
-        """
-        if self.args.data:
-            self.error('Request body (from stdin or a file) and request '
-                       'data (key=value) cannot be mixed. Pass '
-                       '--ignore-stdin to let key/value take priority.')
-        self.args.data = getattr(fd, 'buffer', fd).read()
-
-    def _guess_method(self):
-        """Set `args.method` if not specified to either POST or GET
-        based on whether the request has data or not.
-
-        """
-        if self.args.method is None:
-            # Invoked as `http URL'.
-            assert not self.args.items
-            if not self.args.ignore_stdin and not self.env.stdin_isatty:
-                self.args.method = HTTP_POST
-            else:
-                self.args.method = HTTP_GET
-
-        # FIXME: False positive, e.g., "localhost" matches but is a valid URL.
-        elif not re.match('^[a-zA-Z]+$', self.args.method):
-            # Invoked as `http URL item+'. The URL is now in `args.method`
-            # and the first ITEM is now incorrectly in `args.url`.
-            try:
-                # Parse the URL as an ITEM and store it as the first ITEM arg.
-                self.args.items.insert(0, KeyValueArgType(
-                    *SEP_GROUP_ALL_ITEMS).__call__(self.args.url))
-
-            except ArgumentTypeError as e:
-                if self.args.traceback:
-                    raise
-                self.error(e.args[0])
-
-            else:
-                # Set the URL correctly
-                self.args.url = self.args.method
-                # Infer the method
-                has_data = (
-                    (not self.args.ignore_stdin and not self.env.stdin_isatty)
-                    or any(
-                        item.sep in SEP_GROUP_DATA_ITEMS
-                        for item in self.args.items
-                    )
-                )
-                self.args.method = HTTP_POST if has_data else HTTP_GET
-
-    def _parse_items(self):
-        """Parse `args.items` into `args.headers`, `args.data`, `args.params`,
-         and `args.files`.
-
-        """
-        try:
-            items = parse_items(
-                items=self.args.items,
-                data_class=ParamsDict if self.args.form else OrderedDict
-            )
-        except ParseError as e:
-            if self.args.traceback:
-                raise
-            self.error(e.args[0])
-        else:
-            self.args.headers = items.headers
-            self.args.data = items.data
-            self.args.files = items.files
-            self.args.params = items.params
-
-        if self.args.files and not self.args.form:
-            # `http url @/path/to/file`
-            file_fields = list(self.args.files.keys())
-            if file_fields != ['']:
-                self.error(
-                    'Invalid file fields (perhaps you meant --form?): %s'
-                    % ','.join(file_fields))
-
-            fn, fd, ct = self.args.files['']
-            self.args.files = {}
-
-            self._body_from_file(fd)
-
-            if 'Content-Type' not in self.args.headers:
-                content_type = get_content_type(fn)
-                if content_type:
-                    self.args.headers['Content-Type'] = content_type
-
-    def _process_output_options(self):
-        """Apply defaults to output options, or validate the provided ones.
-
-        The default output options are stdout-type-sensitive.
-
-        """
-        def check_options(value, option):
-            unknown = set(value) - OUTPUT_OPTIONS
-            if unknown:
-                self.error('Unknown output options: {0}={1}'.format(
-                    option,
-                    ','.join(unknown)
-                ))
-
-        if self.args.verbose:
-            self.args.all = True
-
-        if self.args.output_options is None:
-            if self.args.verbose:
-                self.args.output_options = ''.join(OUTPUT_OPTIONS)
-            else:
-                self.args.output_options = (
-                    OUTPUT_OPTIONS_DEFAULT
-                    if self.env.stdout_isatty
-                    else OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
-                )
-
-        if self.args.output_options_history is None:
-            self.args.output_options_history = self.args.output_options
-
-        check_options(self.args.output_options, '--print')
-        check_options(self.args.output_options_history, '--history-print')
-
-        if self.args.download and OUT_RESP_BODY in self.args.output_options:
-            # Response body is always downloaded with --download and it goes
-            # through a different routine, so we remove it.
-            self.args.output_options = str(
-                set(self.args.output_options) - set(OUT_RESP_BODY))
-
-    def _process_pretty_options(self):
-        if self.args.prettify == PRETTY_STDOUT_TTY_ONLY:
-            self.args.prettify = PRETTY_MAP[
-                'all' if self.env.stdout_isatty else 'none']
-        elif (self.args.prettify and self.env.is_windows
-                and self.args.output_file):
-            self.error('Only terminal output can be colorized on Windows.')
-        else:
-            # noinspection PyTypeChecker
-            self.args.prettify = PRETTY_MAP[self.args.prettify]
-
-    def _validate_download_options(self):
-        if not self.args.download:
-            if self.args.download_resume:
-                self.error('--continue only works with --download')
-        if self.args.download_resume and not (
-                self.args.download and self.args.output_file):
-            self.error('--continue requires --output to be specified')
-
-
-class ParseError(Exception):
-    pass
-
-
-class KeyValue(object):
-    """Base key-value pair parsed from CLI."""
-
-    def __init__(self, key, value, sep, orig):
-        self.key = key
-        self.value = value
-        self.sep = sep
-        self.orig = orig
-
-    def __eq__(self, other):
-        return self.__dict__ == other.__dict__
-
-    def __repr__(self):
-        return repr(self.__dict__)
-
-
-class SessionNameValidator(object):
-
-    def __init__(self, error_message):
-        self.error_message = error_message
-
-    def __call__(self, value):
-        # Session name can be a path or just a name.
-        if (os.path.sep not in value
-                and not VALID_SESSION_NAME_PATTERN.search(value)):
-            raise ArgumentError(None, self.error_message)
-        return value
-
-
-class KeyValueArgType(object):
-    """A key-value pair argument type used with `argparse`.
-
-    Parses a key-value arg and constructs a `KeyValue` instance.
-    Used for headers, form data, and other key-value pair types.
-
-    """
-
-    key_value_class = KeyValue
-
-    def __init__(self, *separators):
-        self.separators = separators
-        self.special_characters = set('\\')
-        for separator in separators:
-            self.special_characters.update(separator)
-
-    def __call__(self, string):
-        """Parse `string` and return `self.key_value_class()` instance.
-
-        The best of `self.separators` is determined (first found, longest).
-        Back slash escaped characters aren't considered as separators
-        (or parts thereof). Literal back slash characters have to be escaped
-        as well (r'\\').
-
-        """
-
-        class Escaped(str):
-            """Represents an escaped character."""
-
-        def tokenize(string):
-            r"""Tokenize `string`. There are only two token types - strings
-            and escaped characters:
-
-            tokenize(r'foo\=bar\\baz')
-            => ['foo', Escaped('='), 'bar', Escaped('\\'), 'baz']
-
-            """
-            tokens = ['']
-            characters = iter(string)
-            for char in characters:
-                if char == '\\':
-                    char = next(characters, '')
-                    if char not in self.special_characters:
-                        tokens[-1] += '\\' + char
-                    else:
-                        tokens.extend([Escaped(char), ''])
-                else:
-                    tokens[-1] += char
-            return tokens
-
-        tokens = tokenize(string)
-
-        # Sorting by length ensures that the longest one will be
-        # chosen as it will overwrite any shorter ones starting
-        # at the same position in the `found` dictionary.
-        separators = sorted(self.separators, key=len)
-
-        for i, token in enumerate(tokens):
-
-            if isinstance(token, Escaped):
-                continue
-
-            found = {}
-            for sep in separators:
-                pos = token.find(sep)
-                if pos != -1:
-                    found[pos] = sep
-
-            if found:
-                # Starting first, longest separator found.
-                sep = found[min(found.keys())]
-
-                key, value = token.split(sep, 1)
-
-                # Any preceding tokens are part of the key.
-                key = ''.join(tokens[:i]) + key
-
-                # Any following tokens are part of the value.
-                value += ''.join(tokens[i + 1:])
-
-                break
-
-        else:
-            raise ArgumentTypeError(
-                u'"%s" is not a valid value' % string)
-
-        return self.key_value_class(
-            key=key, value=value, sep=sep, orig=string)
-
-
-class AuthCredentials(KeyValue):
-    """Represents parsed credentials."""
-
-    def _getpass(self, prompt):
-        # To allow mocking.
-        return getpass.getpass(str(prompt))
-
-    def has_password(self):
-        return self.value is not None
-
-    def prompt_password(self, host):
-        try:
-            self.value = self._getpass(
-                'http: password for %s@%s: ' % (self.key, host))
-        except (EOFError, KeyboardInterrupt):
-            sys.stderr.write('\n')
-            sys.exit(0)
-
-
-class AuthCredentialsArgType(KeyValueArgType):
-    """A key-value arg type that parses credentials."""
-
-    key_value_class = AuthCredentials
-
-    def __call__(self, string):
-        """Parse credentials from `string`.
-
-        ("username" or "username:password").
-
-        """
-        try:
-            return super(AuthCredentialsArgType, self).__call__(string)
-        except ArgumentTypeError:
-            # No password provided, will prompt for it later.
-            return self.key_value_class(
-                key=string,
-                value=None,
-                sep=SEP_CREDENTIALS,
-                orig=string
-            )
-
-
-parse_auth = AuthCredentialsArgType(SEP_CREDENTIALS)
-
-
-class RequestItemsDict(OrderedDict):
-    """Multi-value dict for URL parameters and form data."""
-
-    if is_pypy and is_py27:
-        # Manually set keys when initialized with an iterable as PyPy
-        # doesn't call __setitem__ in such case (pypy3 does).
-        def __init__(self, *args, **kwargs):
-            if len(args) == 1 and isinstance(args[0], Iterable):
-                super(RequestItemsDict, self).__init__(**kwargs)
-                for k, v in args[0]:
-                    self[k] = v
-            else:
-                super(RequestItemsDict, self).__init__(*args, **kwargs)
-
-    # noinspection PyMethodOverriding
-    def __setitem__(self, key, value):
-        """ If `key` is assigned more than once, `self[key]` holds a
-        `list` of all the values.
-
-        This allows having multiple fields with the same name in form
-        data and URL params.
-
-        """
-        assert not isinstance(value, list)
-        if key not in self:
-            super(RequestItemsDict, self).__setitem__(key, value)
-        else:
-            if not isinstance(self[key], list):
-                super(RequestItemsDict, self).__setitem__(key, [self[key]])
-            self[key].append(value)
-
-
-class ParamsDict(RequestItemsDict):
-    pass
-
-
-class DataDict(RequestItemsDict):
-
-    def items(self):
-        for key, values in super(RequestItemsDict, self).items():
-            if not isinstance(values, list):
-                values = [values]
-            for value in values:
-                yield key, value
-
-
-RequestItems = namedtuple('RequestItems',
-                          ['headers', 'data', 'files', 'params'])
-
-
-def get_content_type(filename):
-    """
-    Return the content type for ``filename`` in format appropriate
-    for Content-Type headers, or ``None`` if the file type is unknown
-    to ``mimetypes``.
-
-    """
-    mime, encoding = mimetypes.guess_type(filename, strict=False)
-    if mime:
-        content_type = mime
-        if encoding:
-            content_type = '%s; charset=%s' % (mime, encoding)
-        return content_type
-
-
-def parse_items(items,
-                headers_class=CaseInsensitiveDict,
-                data_class=OrderedDict,
-                files_class=DataDict,
-                params_class=ParamsDict):
-    """Parse `KeyValue` `items` into `data`, `headers`, `files`,
-    and `params`.
-
-    """
-    headers = []
-    data = []
-    files = []
-    params = []
-    for item in items:
-        value = item.value
-        if item.sep == SEP_HEADERS:
-            if value == '':
-                # No value => unset the header
-                value = None
-            target = headers
-        elif item.sep == SEP_HEADERS_EMPTY:
-            if item.value:
-                raise ParseError(
-                    'Invalid item "%s" '
-                    '(to specify an empty header use `Header;`)'
-                    % item.orig
-                )
-            target = headers
-        elif item.sep == SEP_QUERY:
-            target = params
-        elif item.sep == SEP_FILES:
-            try:
-                with open(os.path.expanduser(value), 'rb') as f:
-                    value = (os.path.basename(value),
-                             BytesIO(f.read()),
-                             get_content_type(value))
-            except IOError as e:
-                raise ParseError('"%s": %s' % (item.orig, e))
-            target = files
-
-        elif item.sep in SEP_GROUP_DATA_ITEMS:
-
-            if item.sep in SEP_GROUP_DATA_EMBED_ITEMS:
-                try:
-                    with open(os.path.expanduser(value), 'rb') as f:
-                        value = f.read().decode('utf8')
-                except IOError as e:
-                    raise ParseError('"%s": %s' % (item.orig, e))
-                except UnicodeDecodeError:
-                    raise ParseError(
-                        '"%s": cannot embed the content of "%s",'
-                        ' not a UTF8 or ASCII-encoded text file'
-                        % (item.orig, item.value)
-                    )
-
-            if item.sep in SEP_GROUP_RAW_JSON_ITEMS:
-                try:
-                    value = load_json_preserve_order(value)
-                except ValueError as e:
-                    raise ParseError('"%s": %s' % (item.orig, e))
-            target = data
-
-        else:
-            raise TypeError(item)
-
-        target.append((item.key, value))
-
-    return RequestItems(headers_class(headers),
-                        data_class(data),
-                        files_class(files),
-                        params_class(params))
-
-
-def readable_file_arg(filename):
-    try:
-        open(filename, 'rb')
-    except IOError as ex:
-        raise ArgumentTypeError('%s: %s' % (filename, ex.args[1]))
-    return filename

httpie/models.py

@@ -1,37 +1,38 @@
-from httpie.compat import urlsplit, str
+from typing import Iterable, Optional
+from urllib.parse import urlsplit
 
 
-class HTTPMessage(object):
+class HTTPMessage:
     """Abstract class for HTTP messages."""
 
     def __init__(self, orig):
         self._orig = orig
 
-    def iter_body(self, chunk_size):
+    def iter_body(self, chunk_size: int) -> Iterable[bytes]:
         """Return an iterator over the body."""
         raise NotImplementedError()
 
-    def iter_lines(self, chunk_size):
+    def iter_lines(self, chunk_size: int) -> Iterable[bytes]:
         """Return an iterator over the body yielding (`line`, `line_feed`)."""
         raise NotImplementedError()
 
     @property
-    def headers(self):
+    def headers(self) -> str:
         """Return a `str` with the message's headers."""
         raise NotImplementedError()
 
     @property
-    def encoding(self):
+    def encoding(self) -> Optional[str]:
         """Return a `str` with the message's encoding, if known."""
         raise NotImplementedError()
 
     @property
-    def body(self):
+    def body(self) -> bytes:
         """Return a `bytes` with the message's body."""
         raise NotImplementedError()
 
     @property
-    def content_type(self):
+    def content_type(self) -> str:
         """Return the message content type."""
         ct = self._orig.headers.get('Content-Type', '')
         if not isinstance(ct, str):
@@ -60,11 +61,7 @@ class HTTPResponse(HTTPMessage):
             20: '2',
         }[original.version]
 
-        status_line = 'HTTP/{version} {status} {reason}'.format(
-            version=version,
-            status=original.status,
-            reason=original.reason
-        )
+        status_line = f'HTTP/{version} {original.status} {original.reason}'
         headers = [status_line]
         try:
             # `original.msg` is a `http.client.HTTPMessage` on Python 3

httpie/output/formatters/colors.py

@@ -1,18 +1,22 @@
 from __future__ import absolute_import
+
 import json
+from typing import Optional, Type
 
 import pygments.lexer
-import pygments.token
-import pygments.styles
 import pygments.lexers
 import pygments.style
+import pygments.styles
+import pygments.token
 from pygments.formatters.terminal import TerminalFormatter
 from pygments.formatters.terminal256 import Terminal256Formatter
+from pygments.lexer import Lexer
 from pygments.lexers.special import TextLexer
 from pygments.lexers.text import HttpLexer as PygmentsHttpLexer
 from pygments.util import ClassNotFound
 
 from httpie.compat import is_windows
+from httpie.context import Environment
 from httpie.plugins import FormatterPlugin
 
 
@@ -24,7 +28,6 @@ if is_windows:
     # great and fruity seems to give the best result there.
     DEFAULT_STYLE = 'fruity'
 
-
 AVAILABLE_STYLES = set(pygments.styles.get_all_styles())
 AVAILABLE_STYLES.add(SOLARIZED_STYLE)
 AVAILABLE_STYLES.add(AUTO_STYLE)
@@ -40,9 +43,14 @@ class ColorFormatter(FormatterPlugin):
     """
     group_name = 'colors'
 
-    def __init__(self, env, explicit_json=False,
-                 color_scheme=DEFAULT_STYLE, **kwargs):
-        super(ColorFormatter, self).__init__(**kwargs)
+    def __init__(
+        self,
+        env: Environment,
+        explicit_json=False,
+        color_scheme=DEFAULT_STYLE,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
 
         if not env.colors:
             self.enabled = False
@@ -63,14 +71,14 @@ class ColorFormatter(FormatterPlugin):
         self.formatter = formatter
         self.http_lexer = http_lexer
 
-    def format_headers(self, headers):
+    def format_headers(self, headers: str) -> str:
         return pygments.highlight(
             code=headers,
             lexer=self.http_lexer,
             formatter=self.formatter,
         ).strip()
 
-    def format_body(self, body, mime):
+    def format_body(self, body: str, mime: str) -> str:
         lexer = self.get_lexer_for_body(mime, body)
         if lexer:
             body = pygments.highlight(
@@ -80,22 +88,29 @@ class ColorFormatter(FormatterPlugin):
             )
         return body.strip()
 
-    def get_lexer_for_body(self, mime, body):
+    def get_lexer_for_body(
+        self, mime: str,
+        body: str
+    ) -> Optional[Type[Lexer]]:
         return get_lexer(
             mime=mime,
             explicit_json=self.explicit_json,
             body=body,
         )
 
-    def get_style_class(self, color_scheme):
+    @staticmethod
+    def get_style_class(color_scheme: str) -> Type[pygments.style.Style]:
         try:
             return pygments.styles.get_style_by_name(color_scheme)
         except ClassNotFound:
             return Solarized256Style
 
 
-def get_lexer(mime, explicit_json=False, body=''):
-
+def get_lexer(
+    mime: str,
+    explicit_json=False,
+    body=''
+) -> Optional[Type[Lexer]]:
     # Build candidate mime type and lexer names.
     mime_types, lexer_names = [mime], []
     type_, subtype = mime.split('/', 1)

httpie/output/formatters/headers.py

@@ -3,7 +3,7 @@ from httpie.plugins import FormatterPlugin
 
 class HeadersFormatter(FormatterPlugin):
 
-    def format_headers(self, headers):
+    def format_headers(self, headers: str) -> str:
         """
         Sorts headers by name while retaining relative
         order of multiple headers with the same name.

httpie/output/formatters/json.py

@@ -9,7 +9,7 @@ DEFAULT_INDENT = 4
 
 class JSONFormatter(FormatterPlugin):
 
-    def format_body(self, body, mime):
+    def format_body(self, body: str, mime: str) -> str:
         maybe_json = [
             'json',
             'javascript',

httpie/output/processing.py

@@ -1,6 +1,7 @@
 import re
+from typing import Optional, List
 
-from httpie.plugins import plugin_manager
+from httpie.plugins import plugin_manager, ConverterPlugin
 from httpie.context import Environment
 
 
@@ -11,19 +12,20 @@ def is_valid_mime(mime):
     return mime and MIME_RE.match(mime)
 
 
-class Conversion(object):
+class Conversion:
 
-    def get_converter(self, mime):
+    @staticmethod
+    def get_converter(mime: str) -> Optional[ConverterPlugin]:
         if is_valid_mime(mime):
             for converter_class in plugin_manager.get_converters():
                 if converter_class.supports(mime):
                     return converter_class(mime)
 
 
-class Formatting(object):
+class Formatting:
     """A delegate class that invokes the actual processors."""
 
-    def __init__(self, groups, env=Environment(), **kwargs):
+    def __init__(self, groups: List[str], env=Environment(), **kwargs):
         """
         :param groups: names of processor groups to be applied
         :param env: Environment
@@ -38,12 +40,12 @@ class Formatting(object):
                 if p.enabled:
                     self.enabled_plugins.append(p)
 
-    def format_headers(self, headers):
+    def format_headers(self, headers: str) -> str:
         for p in self.enabled_plugins:
             headers = p.format_headers(headers)
         return headers
 
-    def format_body(self, content, mime):
+    def format_body(self, content: str, mime: str) -> str:
         if is_valid_mime(mime):
             for p in self.enabled_plugins:
                 content = p.format_body(content, mime)

httpie/output/streams.py

@@ -1,12 +1,9 @@
 from itertools import chain
-from functools import partial
+from typing import Callable, Iterable, Union
 
-from httpie.compat import str
 from httpie.context import Environment
-from httpie.models import HTTPRequest, HTTPResponse
-from httpie.input import (OUT_REQ_BODY, OUT_REQ_HEAD,
-                          OUT_RESP_HEAD, OUT_RESP_BODY)
-from httpie.output.processing import Formatting, Conversion
+from httpie.models import HTTPMessage
+from httpie.output.processing import Conversion, Formatting
 
 
 BINARY_SUPPRESSED_NOTICE = (
@@ -24,112 +21,16 @@ class BinarySuppressedError(Exception):
     message = BINARY_SUPPRESSED_NOTICE
 
 
-def write_stream(stream, outfile, flush):
-    """Write the output stream."""
-    try:
-        # Writing bytes so we use the buffer interface (Python 3).
-        buf = outfile.buffer
-    except AttributeError:
-        buf = outfile
-
-    for chunk in stream:
-        buf.write(chunk)
-        if flush:
-            outfile.flush()
-
-
-def write_stream_with_colors_win_py3(stream, outfile, flush):
-    """Like `write`, but colorized chunks are written as text
-    directly to `outfile` to ensure it gets processed by colorama.
-    Applies only to Windows with Python 3 and colorized terminal output.
-
-    """
-    color = b'\x1b['
-    encoding = outfile.encoding
-    for chunk in stream:
-        if color in chunk:
-            outfile.write(chunk.decode(encoding))
-        else:
-            outfile.buffer.write(chunk)
-        if flush:
-            outfile.flush()
-
-
-def build_output_stream(args, env, request, response, output_options):
-    """Build and return a chain of iterators over the `request`-`response`
-    exchange each of which yields `bytes` chunks.
-
-    """
-    req_h = OUT_REQ_HEAD in output_options
-    req_b = OUT_REQ_BODY in output_options
-    resp_h = OUT_RESP_HEAD in output_options
-    resp_b = OUT_RESP_BODY in output_options
-    req = req_h or req_b
-    resp = resp_h or resp_b
-
-    output = []
-    Stream = get_stream_type(env, args)
-
-    if req:
-        output.append(Stream(
-            msg=HTTPRequest(request),
-            with_headers=req_h,
-            with_body=req_b))
-
-    if req_b and resp:
-        # Request/Response separator.
-        output.append([b'\n\n'])
-
-    if resp:
-        output.append(Stream(
-            msg=HTTPResponse(response),
-            with_headers=resp_h,
-            with_body=resp_b))
-
-    if env.stdout_isatty and resp_b:
-        # Ensure a blank line after the response body.
-        # For terminal output only.
-        output.append([b'\n\n'])
-
-    return chain(*output)
-
-
-def get_stream_type(env, args):
-    """Pick the right stream type based on `env` and `args`.
-    Wrap it in a partial with the type-specific args so that
-    we don't need to think what stream we are dealing with.
-
-    """
-    if not env.stdout_isatty and not args.prettify:
-        Stream = partial(
-            RawStream,
-            chunk_size=RawStream.CHUNK_SIZE_BY_LINE
-            if args.stream
-            else RawStream.CHUNK_SIZE
-        )
-    elif args.prettify:
-        Stream = partial(
-            PrettyStream if args.stream else BufferedPrettyStream,
-            env=env,
-            conversion=Conversion(),
-            formatting=Formatting(
-                env=env,
-                groups=args.prettify,
-                color_scheme=args.style,
-                explicit_json=args.json,
-            ),
-        )
-    else:
-        Stream = partial(EncodedStream, env=env)
-
-    return Stream
-
-
-class BaseStream(object):
+class BaseStream:
     """Base HTTP message output stream class."""
 
-    def __init__(self, msg, with_headers=True, with_body=True,
-                 on_body_chunk_downloaded=None):
+    def __init__(
+        self,
+        msg: HTTPMessage,
+        with_headers=True,
+        with_body=True,
+        on_body_chunk_downloaded: Callable[[bytes], None] = None
+    ):
         """
         :param msg: a :class:`models.HTTPMessage` subclass
         :param with_headers: if `True`, headers will be included
@@ -142,15 +43,15 @@ class BaseStream(object):
         self.with_body = with_body
         self.on_body_chunk_downloaded = on_body_chunk_downloaded
 
-    def get_headers(self):
+    def get_headers(self) -> bytes:
         """Return the headers' bytes."""
         return self.msg.headers.encode('utf8')
 
-    def iter_body(self):
+    def iter_body(self) -> Iterable[bytes]:
         """Return an iterator over the message body."""
         raise NotImplementedError()
 
-    def __iter__(self):
+    def __iter__(self) -> Iterable[bytes]:
         """Return an iterator over `self.msg`."""
         if self.with_headers:
             yield self.get_headers()
@@ -175,10 +76,10 @@ class RawStream(BaseStream):
     CHUNK_SIZE_BY_LINE = 1
 
     def __init__(self, chunk_size=CHUNK_SIZE, **kwargs):
-        super(RawStream, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.chunk_size = chunk_size
 
-    def iter_body(self):
+    def iter_body(self) -> Iterable[bytes]:
         return self.msg.iter_body(self.chunk_size)
 
 
@@ -193,26 +94,20 @@ class EncodedStream(BaseStream):
     CHUNK_SIZE = 1
 
     def __init__(self, env=Environment(), **kwargs):
-
-        super(EncodedStream, self).__init__(**kwargs)
-
+        super().__init__(**kwargs)
         if env.stdout_isatty:
             # Use the encoding supported by the terminal.
             output_encoding = env.stdout_encoding
         else:
             # Preserve the message encoding.
             output_encoding = self.msg.encoding
-
         # Default to utf8 when unsure.
         self.output_encoding = output_encoding or 'utf8'
 
-    def iter_body(self):
-
+    def iter_body(self) -> Iterable[bytes]:
         for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
-
             if b'\0' in line:
                 raise BinarySuppressedError()
-
             yield line.decode(self.msg.encoding) \
                       .encode(self.output_encoding, 'replace') + lf
 
@@ -228,17 +123,21 @@ class PrettyStream(EncodedStream):
 
     CHUNK_SIZE = 1
 
-    def __init__(self, conversion, formatting, **kwargs):
-        super(PrettyStream, self).__init__(**kwargs)
+    def __init__(
+        self, conversion: Conversion,
+        formatting: Formatting,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
         self.formatting = formatting
         self.conversion = conversion
         self.mime = self.msg.content_type.split(';')[0]
 
-    def get_headers(self):
+    def get_headers(self) -> bytes:
         return self.formatting.format_headers(
             self.msg.headers).encode(self.output_encoding)
 
-    def iter_body(self):
+    def iter_body(self) -> Iterable[bytes]:
         first_chunk = True
         iter_lines = self.msg.iter_lines(self.CHUNK_SIZE)
         for line, lf in iter_lines:
@@ -259,7 +158,7 @@ class PrettyStream(EncodedStream):
             yield self.process_body(line) + lf
             first_chunk = False
 
-    def process_body(self, chunk):
+    def process_body(self, chunk: Union[str, bytes]) -> bytes:
         if not isinstance(chunk, str):
             # Text when a converter has been used,
             # otherwise it will always be bytes.
@@ -278,7 +177,7 @@ class BufferedPrettyStream(PrettyStream):
 
     CHUNK_SIZE = 1024 * 10
 
-    def iter_body(self):
+    def iter_body(self) -> Iterable[bytes]:
         # Read the whole body before prettifying it,
         # but bail out immediately if the body is binary.
         converter = None

httpie/output/writer.py

@@ -0,0 +1,163 @@
+import argparse
+import errno
+from typing import Union, IO, TextIO, Tuple, Type
+
+import requests
+
+from httpie.context import Environment
+from httpie.models import HTTPRequest, HTTPResponse
+from httpie.output.processing import Conversion, Formatting
+from httpie.output.streams import (
+    RawStream, PrettyStream,
+    BufferedPrettyStream, EncodedStream,
+    BaseStream,
+)
+from httpie.cli.constants import (
+    OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD,
+)
+
+
+def write_message(
+    requests_message: Union[requests.PreparedRequest, requests.Response],
+    env: Environment,
+    args: argparse.Namespace,
+):
+    output_options_by_message_type = {
+        requests.PreparedRequest: {
+            'with_headers': OUT_REQ_HEAD in args.output_options,
+            'with_body': OUT_REQ_BODY in args.output_options,
+        },
+        requests.Response: {
+            'with_headers': OUT_RESP_HEAD in args.output_options,
+            'with_body': OUT_RESP_BODY in args.output_options,
+        },
+    }
+    output_options = output_options_by_message_type[type(requests_message)]
+    if not any(output_options.values()):
+        return
+    write_stream_kwargs = {
+        'stream': build_output_stream_for_message(
+            args=args,
+            env=env,
+            requests_message=requests_message,
+            **output_options,
+        ),
+        # NOTE: `env.stdout` will in fact be `stderr` with `--download`
+        'outfile': env.stdout,
+        'flush': env.stdout_isatty or args.stream
+    }
+    try:
+        if env.is_windows and 'colors' in args.prettify:
+            write_stream_with_colors_win_py3(**write_stream_kwargs)
+        else:
+            write_stream(**write_stream_kwargs)
+    except IOError as e:
+        show_traceback = args.debug or args.traceback
+        if not show_traceback and e.errno == errno.EPIPE:
+            # Ignore broken pipes unless --traceback.
+            env.stderr.write('\n')
+        else:
+            raise
+
+
+def write_stream(
+    stream: BaseStream,
+    outfile: Union[IO, TextIO],
+    flush: bool
+):
+    """Write the output stream."""
+    try:
+        # Writing bytes so we use the buffer interface (Python 3).
+        buf = outfile.buffer
+    except AttributeError:
+        buf = outfile
+
+    for chunk in stream:
+        buf.write(chunk)
+        if flush:
+            outfile.flush()
+
+
+def write_stream_with_colors_win_py3(
+    stream: 'BaseStream',
+    outfile: TextIO,
+    flush: bool
+):
+    """Like `write`, but colorized chunks are written as text
+    directly to `outfile` to ensure it gets processed by colorama.
+    Applies only to Windows with Python 3 and colorized terminal output.
+
+    """
+    color = b'\x1b['
+    encoding = outfile.encoding
+    for chunk in stream:
+        if color in chunk:
+            outfile.write(chunk.decode(encoding))
+        else:
+            outfile.buffer.write(chunk)
+        if flush:
+            outfile.flush()
+
+
+def build_output_stream_for_message(
+    args: argparse.Namespace,
+    env: Environment,
+    requests_message: Union[requests.PreparedRequest, requests.Response],
+    with_headers: bool,
+    with_body: bool,
+):
+    stream_class, stream_kwargs = get_stream_type_and_kwargs(
+        env=env,
+        args=args,
+    )
+    message_class = {
+        requests.PreparedRequest: HTTPRequest,
+        requests.Response: HTTPResponse,
+    }[type(requests_message)]
+    yield from stream_class(
+        msg=message_class(requests_message),
+        with_headers=with_headers,
+        with_body=with_body,
+        **stream_kwargs,
+    )
+    if env.stdout_isatty and with_body:
+        # Ensure a blank line after the response body.
+        # For terminal output only.
+        yield b'\n\n'
+
+
+def get_stream_type_and_kwargs(
+    env: Environment,
+    args: argparse.Namespace
+) -> Tuple[Type['BaseStream'], dict]:
+    """Pick the right stream type and kwargs for it based on `env` and `args`.
+
+    """
+    if not env.stdout_isatty and not args.prettify:
+        stream_class = RawStream
+        stream_kwargs = {
+            'chunk_size': (
+                RawStream.CHUNK_SIZE_BY_LINE
+                if args.stream
+                else RawStream.CHUNK_SIZE
+            )
+        }
+    elif args.prettify:
+        stream_class = PrettyStream if args.stream else BufferedPrettyStream
+        stream_kwargs = {
+            'env': env,
+            'conversion': Conversion(),
+            'formatting': Formatting(
+                env=env,
+                groups=args.prettify,
+                color_scheme=args.style,
+                explicit_json=args.json,
+            )
+        }
+    else:
+        stream_class = EncodedStream
+        stream_kwargs = {
+            'env': env
+        }
+
+    return stream_class, stream_kwargs

httpie/plugins/__init__.py

@@ -1,6 +1,6 @@
 """
 WARNING: The plugin API is still work in progress and will
-         probably be completely reworked by v1.0.0.
+         probably be completely reworked in the future.
 
 """
 from httpie.plugins.base import (
@@ -15,8 +15,10 @@ from httpie.output.formatters.colors import ColorFormatter
 
 
 plugin_manager = PluginManager()
-plugin_manager.register(BasicAuthPlugin,
-                        DigestAuthPlugin)
-plugin_manager.register(HeadersFormatter,
-                        JSONFormatter,
-                        ColorFormatter)
+plugin_manager.register(
+    BasicAuthPlugin,
+    DigestAuthPlugin,
+    HeadersFormatter,
+    JSONFormatter,
+    ColorFormatter,
+)

httpie/plugins/base.py

@@ -1,4 +1,4 @@
-class BasePlugin(object):
+class BasePlugin:
 
     # The name of the plugin, eg. "My auth".
     name = None
@@ -59,7 +59,7 @@ class AuthPlugin(BasePlugin):
 class TransportPlugin(BasePlugin):
     """
 
-    http://docs.python-requests.org/en/latest/user/advanced/#transport-adapters
+    https://2.python-requests.org/en/latest/user/advanced/#transport-adapters
 
     """
 
@@ -75,7 +75,7 @@ class TransportPlugin(BasePlugin):
         raise NotImplementedError()
 
 
-class ConverterPlugin(object):
+class ConverterPlugin(BasePlugin):
 
     def __init__(self, mime):
         self.mime = mime
@@ -88,7 +88,8 @@ class ConverterPlugin(object):
         raise NotImplementedError
 
 
-class FormatterPlugin(object):
+class FormatterPlugin(BasePlugin):
+    group_name = 'format'
 
     def __init__(self, **kwargs):
         """
@@ -100,7 +101,7 @@ class FormatterPlugin(object):
         self.enabled = True
         self.kwargs = kwargs
 
-    def format_headers(self, headers):
+    def format_headers(self, headers: str) -> str:
         """Return processed `headers`
 
         :param headers: The headers as text.
@@ -108,7 +109,7 @@ class FormatterPlugin(object):
         """
         return headers
 
-    def format_body(self, content, mime):
+    def format_body(self, content: str, mime: str) -> str:
         """Return processed `content`.
 
         :param mime: E.g., 'application/atom+xml'.

httpie/plugins/builtin.py

@@ -7,37 +7,38 @@ from httpie.plugins.base import AuthPlugin
 
 # noinspection PyAbstractClass
 class BuiltinAuthPlugin(AuthPlugin):
-
     package_name = '(builtin)'
 
 
 class HTTPBasicAuth(requests.auth.HTTPBasicAuth):
 
-    def __call__(self, r):
+    def __call__(
+        self,
+        request: requests.PreparedRequest
+    ) -> requests.PreparedRequest:
         """
         Override username/password serialization to allow unicode.
 
         See https://github.com/jakubroztocil/httpie/issues/212
 
         """
-        r.headers['Authorization'] = type(self).make_header(
+        request.headers['Authorization'] = type(self).make_header(
             self.username, self.password).encode('latin1')
-        return r
+        return request
 
     @staticmethod
-    def make_header(username, password):
+    def make_header(username: str, password: str) -> str:
         credentials = u'%s:%s' % (username, password)
         token = b64encode(credentials.encode('utf8')).strip().decode('latin1')
         return 'Basic %s' % token
 
 
 class BasicAuthPlugin(BuiltinAuthPlugin):
-
     name = 'Basic HTTP auth'
     auth_type = 'basic'
 
     # noinspection PyMethodOverriding
-    def get_auth(self, username, password):
+    def get_auth(self, username: str, password: str) -> HTTPBasicAuth:
         return HTTPBasicAuth(username, password)
 
 
@@ -47,5 +48,9 @@ class DigestAuthPlugin(BuiltinAuthPlugin):
     auth_type = 'digest'
 
     # noinspection PyMethodOverriding
-    def get_auth(self, username, password):
+    def get_auth(
+        self,
+        username: str,
+        password: str
+    ) -> requests.auth.HTTPDigestAuth:
         return requests.auth.HTTPDigestAuth(username, password)

httpie/plugins/manager.py

@@ -1,7 +1,11 @@
 from itertools import groupby
+from operator import attrgetter
+from typing import Dict, List, Type
+
 from pkg_resources import iter_entry_points
-from httpie.plugins import AuthPlugin, FormatterPlugin, ConverterPlugin
-from httpie.plugins.base import TransportPlugin
+
+from httpie.plugins import AuthPlugin, ConverterPlugin, FormatterPlugin
+from httpie.plugins.base import BasePlugin, TransportPlugin
 
 
 ENTRY_POINT_NAMES = [
@@ -12,20 +16,17 @@ ENTRY_POINT_NAMES = [
 ]
 
 
-class PluginManager(object):
+class PluginManager(list):
 
-    def __init__(self):
-        self._plugins = []
-
-    def __iter__(self):
-        return iter(self._plugins)
-
-    def register(self, *plugins):
+    def register(self, *plugins: Type[BasePlugin]):
         for plugin in plugins:
-            self._plugins.append(plugin)
+            self.append(plugin)
 
-    def unregister(self, plugin):
-        self._plugins.remove(plugin)
+    def unregister(self, plugin: Type[BasePlugin]):
+        self.remove(plugin)
+
+    def filter(self, by_type=Type[BasePlugin]):
+        return [plugin for plugin in self if issubclass(plugin, by_type)]
 
     def load_installed_plugins(self):
         for entry_point_name in ENTRY_POINT_NAMES:
@@ -35,33 +36,34 @@ class PluginManager(object):
                 self.register(entry_point.load())
 
     # Auth
-    def get_auth_plugins(self):
-        return [plugin for plugin in self if issubclass(plugin, AuthPlugin)]
+    def get_auth_plugins(self) -> List[Type[AuthPlugin]]:
+        return self.filter(AuthPlugin)
 
-    def get_auth_plugin_mapping(self):
-        return {plugin.auth_type: plugin for plugin in self.get_auth_plugins()}
+    def get_auth_plugin_mapping(self) -> Dict[str, Type[AuthPlugin]]:
+        return {
+            plugin.auth_type: plugin for plugin in self.get_auth_plugins()
+        }
 
-    def get_auth_plugin(self, auth_type):
+    def get_auth_plugin(self, auth_type: str) -> Type[AuthPlugin]:
         return self.get_auth_plugin_mapping()[auth_type]
 
     # Output processing
-    def get_formatters(self):
-        return [plugin for plugin in self
-                if issubclass(plugin, FormatterPlugin)]
+    def get_formatters(self) -> List[Type[FormatterPlugin]]:
+        return self.filter(FormatterPlugin)
 
-    def get_formatters_grouped(self):
-        groups = {}
-        for group_name, group in groupby(
-                self.get_formatters(),
-                key=lambda p: getattr(p, 'group_name', 'format')):
-            groups[group_name] = list(group)
-        return groups
+    def get_formatters_grouped(self) -> Dict[str, List[Type[FormatterPlugin]]]:
+        return {
+            group_name: list(group)
+            for group_name, group
+            in groupby(self.get_formatters(), key=attrgetter('group_name'))
+        }
 
-    def get_converters(self):
-        return [plugin for plugin in self
-                if issubclass(plugin, ConverterPlugin)]
+    def get_converters(self) -> List[Type[ConverterPlugin]]:
+        return self.filter(ConverterPlugin)
 
     # Adapters
-    def get_transport_plugins(self):
-        return [plugin for plugin in self
-                if issubclass(plugin, TransportPlugin)]
+    def get_transport_plugins(self) -> List[Type[TransportPlugin]]:
+        return self.filter(TransportPlugin)
+
+    def __repr__(self):
+        return f'<PluginManager: {list(self)}>'

httpie/sessions.py

@@ -1,85 +1,60 @@
-"""Persistent, JSON-serialized sessions.
+"""
+Persistent, JSON-serialized sessions.
 
 """
-import re
 import os
+import re
+from pathlib import Path
+from typing import Optional, Union
+from urllib.parse import urlsplit
 
+from requests.auth import AuthBase
 from requests.cookies import RequestsCookieJar, create_cookie
 
-from httpie.compat import urlsplit
+from httpie.cli.dicts import RequestHeadersDict
 from httpie.config import BaseConfigDict, DEFAULT_CONFIG_DIR
 from httpie.plugins import plugin_manager
 
 
 SESSIONS_DIR_NAME = 'sessions'
-DEFAULT_SESSIONS_DIR = os.path.join(DEFAULT_CONFIG_DIR, SESSIONS_DIR_NAME)
+DEFAULT_SESSIONS_DIR = DEFAULT_CONFIG_DIR / SESSIONS_DIR_NAME
 VALID_SESSION_NAME_PATTERN = re.compile('^[a-zA-Z0-9_.-]+$')
 # Request headers starting with these prefixes won't be stored in sessions.
 # They are specific to each request.
-# http://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Requests
+# <https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Requests>
 SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-']
 
 
-def get_response(requests_session, session_name,
-                 config_dir, args, read_only=False):
-    """Like `client.get_responses`, but applies permanent
-    aspects of the session to the request.
-
-    """
-    from .client import get_requests_kwargs, dump_request
+def get_httpie_session(
+    config_dir: Path,
+    session_name: str,
+    host: Optional[str],
+    url: str,
+) -> 'Session':
     if os.path.sep in session_name:
         path = os.path.expanduser(session_name)
     else:
-        hostname = (args.headers.get('Host', None)
-                    or urlsplit(args.url).netloc.split('@')[-1])
+        hostname = host or urlsplit(url).netloc.split('@')[-1]
         if not hostname:
             # HACK/FIXME: httpie-unixsocket's URLs have no hostname.
             hostname = 'localhost'
 
         # host:port => host_port
         hostname = hostname.replace(':', '_')
-        path = os.path.join(config_dir,
-                            SESSIONS_DIR_NAME,
-                            hostname,
-                            session_name + '.json')
-
+        path = (
+            config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json'
+        )
     session = Session(path)
     session.load()
-
-    kwargs = get_requests_kwargs(args, base_headers=session.headers)
-    if args.debug:
-        dump_request(kwargs)
-    session.update_headers(kwargs['headers'])
-
-    if args.auth_plugin:
-        session.auth = {
-            'type': args.auth_plugin.auth_type,
-            'raw_auth': args.auth_plugin.raw_auth,
-        }
-    elif session.auth:
-        kwargs['auth'] = session.auth
-
-    requests_session.cookies = session.cookies
-
-    try:
-        response = requests_session.request(**kwargs)
-    except Exception:
-        raise
-    else:
-        # Existing sessions with `read_only=True` don't get updated.
-        if session.is_new() or not read_only:
-            session.cookies = requests_session.cookies
-            session.save()
-        return response
+    return session
 
 
 class Session(BaseConfigDict):
     helpurl = 'https://httpie.org/doc#sessions'
     about = 'HTTPie session file'
 
-    def __init__(self, path, *args, **kwargs):
-        super(Session, self).__init__(*args, **kwargs)
-        self._path = path
+    def __init__(self, path: Union[str, Path]):
+        super().__init__(path=Path(path))
         self['headers'] = {}
         self['cookies'] = {}
         self['auth'] = {
@@ -88,21 +63,17 @@ class Session(BaseConfigDict):
             'password': None
         }
 
-    def _get_path(self):
-        return self._path
-
-    def update_headers(self, request_headers):
+    def update_headers(self, request_headers: RequestHeadersDict):
         """
         Update the session headers with the request ones while ignoring
         certain name prefixes.
 
-        :type request_headers: dict
-
         """
+        headers = self.headers
         for name, value in request_headers.items():
 
             if value is None:
-                continue  # Ignore explicitely unset headers
+                continue  # Ignore explicitly unset headers
 
             value = value.decode('utf8')
             if name == 'User-Agent' and value.startswith('HTTPie/'):
@@ -112,14 +83,16 @@ class Session(BaseConfigDict):
                 if name.lower().startswith(prefix.lower()):
                     break
             else:
-                self['headers'][name] = value
+                headers[name] = value
+
+        self['headers'] = dict(headers)
 
     @property
-    def headers(self):
-        return self['headers']
+    def headers(self) -> RequestHeadersDict:
+        return RequestHeadersDict(self['headers'])
 
     @property
-    def cookies(self):
+    def cookies(self) -> RequestsCookieJar:
         jar = RequestsCookieJar()
         for name, cookie_dict in self['cookies'].items():
             jar.set_cookie(create_cookie(
@@ -128,11 +101,8 @@ class Session(BaseConfigDict):
         return jar
 
     @cookies.setter
-    def cookies(self, jar):
-        """
-        :type jar: CookieJar
-        """
-        # http://docs.python.org/2/library/cookielib.html#cookie-objects
+    def cookies(self, jar: RequestsCookieJar):
+        # <https://docs.python.org/2/library/cookielib.html#cookie-objects>
         stored_attrs = ['value', 'path', 'secure', 'expires']
         self['cookies'] = {}
         for cookie in jar:
@@ -142,7 +112,7 @@ class Session(BaseConfigDict):
             }
 
     @property
-    def auth(self):
+    def auth(self) -> Optional[AuthBase]:
         auth = self.get('auth', None)
         if not auth or not auth['type']:
             return
@@ -161,7 +131,7 @@ class Session(BaseConfigDict):
             }
         else:
             if plugin.auth_parse:
-                from httpie.input import parse_auth
+                from httpie.cli.argtypes import parse_auth
                 parsed = parse_auth(plugin.raw_auth)
                 credentials = {
                     'username': parsed.key,
@@ -171,6 +141,6 @@ class Session(BaseConfigDict):
         return plugin.get_auth(**credentials)
 
     @auth.setter
-    def auth(self, auth):
-        assert set(['type', 'raw_auth']) == set(auth.keys())
+    def auth(self, auth: dict):
+        assert {'type', 'raw_auth'} == auth.keys()
         self['auth'] = auth

httpie/status.py

@@ -0,0 +1,40 @@
+from enum import IntEnum, unique
+
+
+@unique
+class ExitStatus(IntEnum):
+    """Program exit status code constants."""
+    SUCCESS = 0
+    ERROR = 1
+    ERROR_TIMEOUT = 2
+
+    # See --check-status
+    ERROR_HTTP_3XX = 3
+    ERROR_HTTP_4XX = 4
+    ERROR_HTTP_5XX = 5
+
+    ERROR_TOO_MANY_REDIRECTS = 6
+    PLUGIN_ERROR = 7
+    # 128+2 SIGINT
+    # <http://www.tldp.org/LDP/abs/html/exitcodes.html>
+    ERROR_CTRL_C = 130
+
+
+def http_status_to_exit_status(http_status: int, follow=False) -> ExitStatus:
+    """
+    Translate HTTP status code to exit status code.
+
+    (Relevant only when invoked with --check-status or --download.)
+
+    """
+    if 300 <= http_status <= 399 and not follow:
+        # Redirect
+        return ExitStatus.ERROR_HTTP_3XX
+    elif 400 <= http_status <= 499:
+        # Client Error
+        return ExitStatus.ERROR_HTTP_4XX
+    elif 500 <= http_status <= 599:
+        # Server Error
+        return ExitStatus.ERROR_HTTP_5XX
+    else:
+        return ExitStatus.SUCCESS

httpie/utils.py

@@ -1,32 +1,24 @@
 from __future__ import division
 import json
+import mimetypes
 from collections import OrderedDict
+from pprint import pformat
+
+import requests.auth
 
 
 def load_json_preserve_order(s):
     return json.loads(s, object_pairs_hook=OrderedDict)
 
 
-def repr_dict_nice(d):
-    def prepare_dict(d):
-        for k, v in d.items():
-            if isinstance(v, dict):
-                v = dict(prepare_dict(v))
-            elif isinstance(v, bytes):
-                v = v.decode('utf8')
-            elif not isinstance(v, (int, str)):
-                v = repr(v)
-            yield k, v
-    return json.dumps(
-        dict(prepare_dict(d)),
-        indent=4, sort_keys=True,
-    )
+def repr_dict(d: dict) -> str:
+    return pformat(d)
 
 
 def humanize_bytes(n, precision=2):
     # Author: Doug Latornell
     # Licence: MIT
-    # URL: http://code.activestate.com/recipes/577081/
+    # URL: https://code.activestate.com/recipes/577081/
     """Return a humanized string representation of a number of bytes.
 
     Assumes `from __future__ import division`.
@@ -67,3 +59,27 @@ def humanize_bytes(n, precision=2):
 
     # noinspection PyUnboundLocalVariable
     return '%.*f %s' % (precision, n / factor, suffix)
+
+
+class ExplicitNullAuth(requests.auth.AuthBase):
+    """Forces requests to ignore the ``.netrc``.
+    <https://github.com/psf/requests/issues/2773#issuecomment-174312831>
+    """
+
+    def __call__(self, r):
+        return r
+
+
+def get_content_type(filename):
+    """
+    Return the content type for ``filename`` in format appropriate
+    for Content-Type headers, or ``None`` if the file type is unknown
+    to ``mimetypes``.
+
+    """
+    mime, encoding = mimetypes.guess_type(filename, strict=False)
+    if mime:
+        content_type = mime
+        if encoding:
+            content_type = '%s; charset=%s' % (mime, encoding)
+        return content_type

requirements-dev.txt

@@ -6,3 +6,4 @@ pytest-httpbin>=0.0.6
 docutils
 wheel
 pycodestyle
+twine

setup.py

@@ -35,8 +35,8 @@ tests_require = [
 
 
 install_requires = [
-    'requests>=2.18.4',
-    'Pygments>=2.1.3'
+    'requests>=2.22.0',
+    'Pygments>=2.5.2',
 ]
 
 
@@ -57,7 +57,7 @@ if 'bdist_wheel' not in sys.argv:
 
 # bdist_wheel
 extras_require = {
-    # http://wheel.readthedocs.io/en/latest/#defining-conditional-dependencies
+    # https://wheel.readthedocs.io/en/latest/#defining-conditional-dependencies
     'python_version == "3.0" or python_version == "3.1"': ['argparse>=1.2.1'],
     ':sys_platform == "win32"': ['colorama>=0.2.4'],
 }
@@ -73,7 +73,7 @@ setup(
     version=httpie.__version__,
     description=httpie.__doc__.strip(),
     long_description=long_description(),
-    url='http://httpie.org/',
+    url='https://httpie.org/',
     download_url='https://github.com/jakubroztocil/httpie',
     author=httpie.__author__,
     author_email='jakub@roztocil.co',
@@ -82,6 +82,7 @@ setup(
     entry_points={
         'console_scripts': [
             'http = httpie.__main__:main',
+            'https = httpie.__main__:main',
         ],
     },
     extras_require=extras_require,
@@ -91,14 +92,9 @@ setup(
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.1',
-        'Programming Language :: Python :: 3.2',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Environment :: Console',
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',

tests/test_auth.py

@@ -2,9 +2,11 @@
 import mock
 import pytest
 
+from httpie.plugins.builtin import HTTPBasicAuth
+from httpie.utils import ExplicitNullAuth
 from utils import http, add_auth, HTTP_OK, MockEnvironment
-import httpie.input
-import httpie.cli
+import httpie.cli.constants
+import httpie.cli.definition
 
 
 def test_basic_auth(httpbin_both):
@@ -22,7 +24,7 @@ def test_digest_auth(httpbin_both, argument_name):
     assert r.json == {'authenticated': True, 'user': 'user'}
 
 
-@mock.patch('httpie.input.AuthCredentials._getpass',
+@mock.patch('httpie.cli.argtypes.AuthCredentials._getpass',
             new=lambda self, prompt: 'password')
 def test_password_prompt(httpbin):
     r = http('--auth', 'user',
@@ -58,7 +60,7 @@ def test_only_username_in_url(url):
     https://github.com/jakubroztocil/httpie/issues/242
 
     """
-    args = httpie.cli.parser.parse_args(args=[url], env=MockEnvironment())
+    args = httpie.cli.definition.parser.parse_args(args=[url], env=MockEnvironment())
     assert args.auth
     assert args.auth.username == 'username'
     assert args.auth.password == ''
@@ -69,7 +71,39 @@ def test_missing_auth(httpbin):
         '--auth-type=basic',
         'GET',
         httpbin + '/basic-auth/user/password',
-        error_exit_ok=True
+        tolerate_error_exit_status=True
     )
     assert HTTP_OK not in r
     assert '--auth required' in r.stderr
+
+
+def test_netrc(httpbin_both):
+    with mock.patch('requests.sessions.get_netrc_auth') as get_netrc_auth:
+        get_netrc_auth.return_value = ('httpie', 'password')
+        r = http(httpbin_both + '/basic-auth/httpie/password')
+        assert get_netrc_auth.call_count == 1
+        assert HTTP_OK in r
+
+
+def test_ignore_netrc(httpbin_both):
+    with mock.patch('requests.sessions.get_netrc_auth') as get_netrc_auth:
+        get_netrc_auth.return_value = ('httpie', 'password')
+        r = http('--ignore-netrc', httpbin_both + '/basic-auth/httpie/password')
+        assert get_netrc_auth.call_count == 0
+        assert 'HTTP/1.1 401 UNAUTHORIZED' in r
+
+
+def test_ignore_netrc_null_auth():
+    args = httpie.cli.definition.parser.parse_args(
+        args=['--ignore-netrc', 'example.org'],
+        env=MockEnvironment(),
+    )
+    assert isinstance(args.auth, ExplicitNullAuth)
+
+
+def test_ignore_netrc_together_with_auth():
+    args = httpie.cli.definition.parser.parse_args(
+        args=['--ignore-netrc', '--auth=username:password', 'example.org'],
+        env=MockEnvironment(),
+    )
+    assert isinstance(args.auth, HTTPBasicAuth)

tests/test_auth_plugins.py

@@ -1,6 +1,6 @@
 from mock import mock
 
-from httpie.input import SEP_CREDENTIALS
+from httpie.cli.constants import SEPARATOR_CREDENTIALS
 from httpie.plugins import AuthPlugin, plugin_manager
 from utils import http, HTTP_OK
 
@@ -83,7 +83,7 @@ def test_auth_plugin_require_auth_false_and_auth_provided(httpbin):
         auth_require = False
 
         def get_auth(self, username=None, password=None):
-            assert self.raw_auth == USERNAME + SEP_CREDENTIALS + PASSWORD
+            assert self.raw_auth == USERNAME + SEPARATOR_CREDENTIALS + PASSWORD
             assert username == USERNAME
             assert password == PASSWORD
             return basic_auth()
@@ -95,7 +95,7 @@ def test_auth_plugin_require_auth_false_and_auth_provided(httpbin):
             '--auth-type',
             Plugin.auth_type,
             '--auth',
-            USERNAME + SEP_CREDENTIALS + PASSWORD,
+            USERNAME + SEPARATOR_CREDENTIALS + PASSWORD,
         )
         assert HTTP_OK in r
         assert r.json == AUTH_OK
@@ -103,7 +103,7 @@ def test_auth_plugin_require_auth_false_and_auth_provided(httpbin):
         plugin_manager.unregister(Plugin)
 
 
-@mock.patch('httpie.input.AuthCredentials._getpass',
+@mock.patch('httpie.cli.argtypes.AuthCredentials._getpass',
             new=lambda self, prompt: 'UNEXPECTED_PROMPT_RESPONSE')
 def test_auth_plugin_prompt_password_false(httpbin):
 

tests/test_binary.py

@@ -1,6 +1,7 @@
 """Tests for dealing with binary request and response data."""
+import requests
+
 from fixtures import BIN_FILE_PATH, BIN_FILE_CONTENT, BIN_FILE_PATH_ARG
-from httpie.compat import urlopen
 from httpie.output.streams import BINARY_SUPPRESSED_NOTICE
 from utils import MockEnvironment, http
 
@@ -31,25 +32,19 @@ class TestBinaryRequestData:
 
 
 class TestBinaryResponseData:
-    url = 'http://www.google.com/favicon.ico'
 
-    @property
-    def bindata(self):
-        if not hasattr(self, '_bindata'):
-            self._bindata = urlopen(self.url).read()
-        return self._bindata
-
-    def test_binary_suppresses_when_terminal(self):
-        r = http('GET', self.url)
+    def test_binary_suppresses_when_terminal(self, httpbin):
+        r = http('GET', httpbin + '/bytes/1024?seed=1')
         assert BINARY_SUPPRESSED_NOTICE.decode() in r
 
-    def test_binary_suppresses_when_not_terminal_but_pretty(self):
+    def test_binary_suppresses_when_not_terminal_but_pretty(self, httpbin):
         env = MockEnvironment(stdin_isatty=True, stdout_isatty=False)
-        r = http('--pretty=all', 'GET', self.url,
-                 env=env)
+        r = http('--pretty=all', 'GET', httpbin + '/bytes/1024?seed=1', env=env)
         assert BINARY_SUPPRESSED_NOTICE.decode() in r
 
-    def test_binary_included_and_correct_when_suitable(self):
+    def test_binary_included_and_correct_when_suitable(self, httpbin):
         env = MockEnvironment(stdin_isatty=True, stdout_isatty=False)
-        r = http('GET', self.url, env=env)
-        assert r == self.bindata
+        url = httpbin + '/bytes/1024?seed=1'
+        r = http('GET', url, env=env)
+        expected = requests.get(url).content
+        assert r == expected

tests/test_cli.py

@@ -1,42 +1,42 @@
 """CLI argument parsing related tests."""
-import json
-# noinspection PyCompatibility
 import argparse
+import json
 
 import pytest
 from requests.exceptions import InvalidSchema
 
-from httpie import input
-from httpie.input import KeyValue, KeyValueArgType, DataDict
-from httpie import ExitStatus
-from httpie.cli import parser
-from utils import MockEnvironment, http, HTTP_OK
+import httpie.cli.argparser
 from fixtures import (
-    FILE_PATH_ARG, JSON_FILE_PATH_ARG,
-    JSON_FILE_CONTENT, FILE_CONTENT, FILE_PATH
+    FILE_CONTENT, FILE_PATH, FILE_PATH_ARG, JSON_FILE_CONTENT,
+    JSON_FILE_PATH_ARG,
 )
+from httpie.status import ExitStatus
+from httpie.cli import constants
+from httpie.cli.definition import parser
+from httpie.cli.argtypes import KeyValueArg, KeyValueArgType
+from httpie.cli.requestitems import RequestItems
+from utils import HTTP_OK, MockEnvironment, http
 
 
 class TestItemParsing:
-
-    key_value = KeyValueArgType(*input.SEP_GROUP_ALL_ITEMS)
+    key_value_arg = KeyValueArgType(*constants.SEPARATOR_GROUP_ALL_ITEMS)
 
     def test_invalid_items(self):
         items = ['no-separator']
         for item in items:
-            pytest.raises(argparse.ArgumentTypeError, self.key_value, item)
+            pytest.raises(argparse.ArgumentTypeError, self.key_value_arg, item)
 
     def test_escape_separator(self):
-        items = input.parse_items([
+        items = RequestItems.from_args([
             # headers
-            self.key_value(r'foo\:bar:baz'),
-            self.key_value(r'jack\@jill:hill'),
+            self.key_value_arg(r'foo\:bar:baz'),
+            self.key_value_arg(r'jack\@jill:hill'),
 
             # data
-            self.key_value(r'baz\=bar=foo'),
+            self.key_value_arg(r'baz\=bar=foo'),
 
             # files
-            self.key_value(r'bar\@baz@%s' % FILE_PATH_ARG),
+            self.key_value_arg(r'bar\@baz@%s' % FILE_PATH_ARG),
         ])
         # `requests.structures.CaseInsensitiveDict` => `dict`
         headers = dict(items.headers._store.values())
@@ -45,7 +45,9 @@ class TestItemParsing:
             'foo:bar': 'baz',
             'jack@jill': 'hill',
         }
-        assert items.data == {'baz=bar': 'foo'}
+        assert items.data == {
+            'baz=bar': 'foo'
+        }
         assert 'bar@baz' in items.files
 
     @pytest.mark.parametrize(('string', 'key', 'sep', 'value'), [
@@ -54,31 +56,34 @@ class TestItemParsing:
         ('path\\==c:\\windows', 'path=', '=', 'c:\\windows'),
     ])
     def test_backslash_before_non_special_character_does_not_escape(
-            self, string, key, sep, value):
-        expected = KeyValue(orig=string, key=key, sep=sep, value=value)
-        actual = self.key_value(string)
+        self, string, key, sep, value
+    ):
+        expected = KeyValueArg(orig=string, key=key, sep=sep, value=value)
+        actual = self.key_value_arg(string)
         assert actual == expected
 
     def test_escape_longsep(self):
-        items = input.parse_items([
-            self.key_value(r'bob\:==foo'),
+        items = RequestItems.from_args([
+            self.key_value_arg(r'bob\:==foo'),
         ])
-        assert items.params == {'bob:': 'foo'}
+        assert items.params == {
+            'bob:': 'foo'
+        }
 
     def test_valid_items(self):
-        items = input.parse_items([
-            self.key_value('string=value'),
-            self.key_value('Header:value'),
-            self.key_value('Unset-Header:'),
-            self.key_value('Empty-Header;'),
-            self.key_value('list:=["a", 1, {}, false]'),
-            self.key_value('obj:={"a": "b"}'),
-            self.key_value('ed='),
-            self.key_value('bool:=true'),
-            self.key_value('file@' + FILE_PATH_ARG),
-            self.key_value('query==value'),
-            self.key_value('string-embed=@' + FILE_PATH_ARG),
-            self.key_value('raw-json-embed:=@' + JSON_FILE_PATH_ARG),
+        items = RequestItems.from_args([
+            self.key_value_arg('string=value'),
+            self.key_value_arg('Header:value'),
+            self.key_value_arg('Unset-Header:'),
+            self.key_value_arg('Empty-Header;'),
+            self.key_value_arg('list:=["a", 1, {}, false]'),
+            self.key_value_arg('obj:={"a": "b"}'),
+            self.key_value_arg('ed='),
+            self.key_value_arg('bool:=true'),
+            self.key_value_arg('file@' + FILE_PATH_ARG),
+            self.key_value_arg('query==value'),
+            self.key_value_arg('string-embed=@' + FILE_PATH_ARG),
+            self.key_value_arg('raw-json-embed:=@' + JSON_FILE_PATH_ARG),
         ])
 
         # Parsed headers
@@ -99,12 +104,16 @@ class TestItemParsing:
             "string": "value",
             "bool": True,
             "list": ["a", 1, {}, False],
-            "obj": {"a": "b"},
+            "obj": {
+                "a": "b"
+            },
             "string-embed": FILE_CONTENT,
         }
 
         # Parsed query string parameters
-        assert items.params == {'query': 'value'}
+        assert items.params == {
+            'query': 'value'
+        }
 
         # Parsed file fields
         assert 'file' in items.files
@@ -112,17 +121,19 @@ class TestItemParsing:
                 decode('utf8') == FILE_CONTENT)
 
     def test_multiple_file_fields_with_same_field_name(self):
-        items = input.parse_items([
-            self.key_value('file_field@' + FILE_PATH_ARG),
-            self.key_value('file_field@' + FILE_PATH_ARG),
+        items = RequestItems.from_args([
+            self.key_value_arg('file_field@' + FILE_PATH_ARG),
+            self.key_value_arg('file_field@' + FILE_PATH_ARG),
         ])
         assert len(items.files['file_field']) == 2
 
     def test_multiple_text_fields_with_same_field_name(self):
-        items = input.parse_items(
-            [self.key_value('text_field=a'),
-             self.key_value('text_field=b')],
-            data_class=DataDict
+        items = RequestItems.from_args(
+            request_item_args=[
+                self.key_value_arg('text_field=a'),
+                self.key_value_arg('text_field=b')
+            ],
+            as_form=True,
         )
         assert items.data['text_field'] == ['a', 'b']
         assert list(items.data.items()) == [
@@ -206,92 +217,80 @@ class TestLocalhostShorthand:
 class TestArgumentParser:
 
     def setup_method(self, method):
-        self.parser = input.HTTPieArgumentParser()
+        self.parser = httpie.cli.argparser.HTTPieArgumentParser()
 
     def test_guess_when_method_set_and_valid(self):
         self.parser.args = argparse.Namespace()
         self.parser.args.method = 'GET'
         self.parser.args.url = 'http://example.com/'
-        self.parser.args.items = []
+        self.parser.args.request_items = []
         self.parser.args.ignore_stdin = False
-
         self.parser.env = MockEnvironment()
-
         self.parser._guess_method()
-
         assert self.parser.args.method == 'GET'
         assert self.parser.args.url == 'http://example.com/'
-        assert self.parser.args.items == []
+        assert self.parser.args.request_items == []
 
     def test_guess_when_method_not_set(self):
         self.parser.args = argparse.Namespace()
         self.parser.args.method = None
         self.parser.args.url = 'http://example.com/'
-        self.parser.args.items = []
+        self.parser.args.request_items = []
         self.parser.args.ignore_stdin = False
         self.parser.env = MockEnvironment()
-
         self.parser._guess_method()
-
         assert self.parser.args.method == 'GET'
         assert self.parser.args.url == 'http://example.com/'
-        assert self.parser.args.items == []
+        assert self.parser.args.request_items == []
 
     def test_guess_when_method_set_but_invalid_and_data_field(self):
         self.parser.args = argparse.Namespace()
         self.parser.args.method = 'http://example.com/'
         self.parser.args.url = 'data=field'
-        self.parser.args.items = []
+        self.parser.args.request_items = []
         self.parser.args.ignore_stdin = False
         self.parser.env = MockEnvironment()
         self.parser._guess_method()
-
         assert self.parser.args.method == 'POST'
         assert self.parser.args.url == 'http://example.com/'
-        assert self.parser.args.items == [
-            KeyValue(key='data',
-                     value='field',
-                     sep='=',
-                     orig='data=field')
+        assert self.parser.args.request_items == [
+            KeyValueArg(key='data',
+                        value='field',
+                        sep='=',
+                        orig='data=field')
         ]
 
     def test_guess_when_method_set_but_invalid_and_header_field(self):
         self.parser.args = argparse.Namespace()
         self.parser.args.method = 'http://example.com/'
         self.parser.args.url = 'test:header'
-        self.parser.args.items = []
+        self.parser.args.request_items = []
         self.parser.args.ignore_stdin = False
-
         self.parser.env = MockEnvironment()
-
         self.parser._guess_method()
-
         assert self.parser.args.method == 'GET'
         assert self.parser.args.url == 'http://example.com/'
-        assert self.parser.args.items, [
-            KeyValue(key='test',
-                     value='header',
-                     sep=':',
-                     orig='test:header')
+        assert self.parser.args.request_items, [
+            KeyValueArg(key='test',
+                        value='header',
+                        sep=':',
+                        orig='test:header')
         ]
 
     def test_guess_when_method_set_but_invalid_and_item_exists(self):
         self.parser.args = argparse.Namespace()
         self.parser.args.method = 'http://example.com/'
         self.parser.args.url = 'new_item=a'
-        self.parser.args.items = [
-            KeyValue(
+        self.parser.args.request_items = [
+            KeyValueArg(
                 key='old_item', value='b', sep='=', orig='old_item=b')
         ]
         self.parser.args.ignore_stdin = False
-
         self.parser.env = MockEnvironment()
-
         self.parser._guess_method()
-
-        assert self.parser.args.items, [
-            KeyValue(key='new_item', value='a', sep='=', orig='new_item=a'),
-            KeyValue(
+        assert self.parser.args.request_items, [
+            KeyValueArg(key='new_item', value='a', sep='=', orig='new_item=a'),
+            KeyValueArg(
                 key='old_item', value='b', sep='=', orig='old_item=b'),
         ]
 
@@ -304,13 +303,13 @@ class TestNoOptions:
 
     def test_invalid_no_options(self, httpbin):
         r = http('--no-war', 'GET', httpbin.url + '/get',
-                 error_exit_ok=True)
-        assert r.exit_status == 1
+                 tolerate_error_exit_status=True)
+        assert r.exit_status == ExitStatus.ERROR
         assert 'unrecognized arguments: --no-war' in r.stderr
         assert 'GET /get HTTP/1.1' not in r
 
 
-class TestIgnoreStdin:
+class TestStdin:
 
     def test_ignore_stdin(self, httpbin):
         with open(FILE_PATH) as f:
@@ -323,10 +322,14 @@ class TestIgnoreStdin:
 
     def test_ignore_stdin_cannot_prompt_password(self, httpbin):
         r = http('--ignore-stdin', '--auth=no-password', httpbin.url + '/get',
-                 error_exit_ok=True)
+                 tolerate_error_exit_status=True)
         assert r.exit_status == ExitStatus.ERROR
         assert 'because --ignore-stdin' in r.stderr
 
+    def test_stdin_closed(self, httpbin):
+        r = http(httpbin + '/get', env=MockEnvironment(stdin=None))
+        assert HTTP_OK in r
+
 
 class TestSchemes:
 
@@ -342,6 +345,10 @@ class TestSchemes:
         with pytest.raises(InvalidSchema):
             http('bah', '--default=scheme=foo+bar-BAZ.123')
 
-    def test_default_scheme(self, httpbin_secure):
+    def test_default_scheme_option(self, httpbin_secure):
         url = '{0}:{1}'.format(httpbin_secure.host, httpbin_secure.port)
         assert HTTP_OK in http(url, '--default-scheme=https')
+
+    def test_scheme_when_invoked_as_https(self, httpbin_secure):
+        url = '{0}:{1}'.format(httpbin_secure.host, httpbin_secure.port)
+        assert HTTP_OK in http(url, program_name='https')

tests/test_compress.py

@@ -0,0 +1,110 @@
+"""
+We test against httpbin which doesn't return the request data in a
+consistent way:
+
+1. Non-form requests: the `data` field contains base64 encoded version of
+our zlib-encoded request data.
+
+2. Form requests: `form` contains a messed up version of the data.
+
+"""
+import base64
+import zlib
+
+from fixtures import FILE_PATH, FILE_CONTENT
+from utils import http, HTTP_OK, MockEnvironment
+
+
+def assert_decompressed_equal(base64_compressed_data, expected_str):
+    compressed_data = base64.b64decode(
+        base64_compressed_data.split(',', 1)[1])
+    data = zlib.decompress(compressed_data)
+    actual_str = data.decode()
+
+    # FIXME: contains a trailing linebreak with an uploaded file
+    actual_str = actual_str.rstrip()
+
+    assert actual_str == expected_str
+
+
+def test_compress_skip_negative_ratio(httpbin_both):
+    r = http(
+        '--compress',
+        httpbin_both + '/post',
+        'foo=bar',
+    )
+    assert HTTP_OK in r
+    assert 'Content-Encoding' not in r.json['headers']
+    assert r.json['json'] == {'foo': 'bar'}
+
+
+def test_compress_force_with_negative_ratio(httpbin_both):
+    r = http(
+        '--compress',
+        '--compress',
+        httpbin_both + '/post',
+        'foo=bar',
+    )
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert_decompressed_equal(r.json['data'], '{"foo": "bar"}')
+
+
+def test_compress_json(httpbin_both):
+    r = http(
+        '--compress',
+        '--compress',
+        httpbin_both + '/post',
+        'foo=bar',
+    )
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert_decompressed_equal(r.json['data'], '{"foo": "bar"}')
+    assert r.json['json'] is None
+
+
+def test_compress_form(httpbin_both):
+    r = http(
+        '--form',
+        '--compress',
+        '--compress',
+        httpbin_both + '/post',
+        'foo=bar',
+    )
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert r.json['data'] == ""
+    assert '"foo": "bar"' not in r
+
+
+def test_compress_stdin(httpbin_both):
+    with open(FILE_PATH) as f:
+        env = MockEnvironment(stdin=f, stdin_isatty=False)
+        r = http(
+            '--compress',
+            '--compress',
+            'PATCH',
+            httpbin_both + '/patch',
+            env=env,
+        )
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert_decompressed_equal(r.json['data'], FILE_CONTENT.strip())
+    assert not r.json['json']
+
+
+def test_compress_file(httpbin_both):
+    r = http(
+        '--form',
+        '--compress',
+        '--compress',
+        'PUT',
+        httpbin_both + '/put',
+        'file@' + FILE_PATH,
+    )
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert r.json['headers']['Content-Type'].startswith(
+        'multipart/form-data; boundary=')
+    assert r.json['files'] == {}
+    assert FILE_CONTENT not in r

tests/test_config.py

@@ -1,6 +1,8 @@
-from httpie import __version__
-from utils import MockEnvironment, http
-from httpie.context import Environment
+import pytest
+
+from httpie.compat import is_windows
+from httpie.config import Config
+from utils import HTTP_OK, MockEnvironment, http
 
 
 def test_default_options(httpbin):
@@ -8,7 +10,34 @@ def test_default_options(httpbin):
     env.config['default_options'] = ['--form']
     env.config.save()
     r = http(httpbin.url + '/post', 'foo=bar', env=env)
-    assert r.json['form'] == {"foo": "bar"}
+    assert r.json['form'] == {
+        "foo": "bar"
+    }
+
+
+def test_config_file_not_valid(httpbin):
+    env = MockEnvironment()
+    env.create_temp_config_dir()
+    with (env.config_dir / Config.FILENAME).open('w') as f:
+        f.write('{invalid json}')
+    r = http(httpbin + '/get', env=env)
+    assert HTTP_OK in r
+    assert 'http: warning' in r.stderr
+    assert 'invalid config file' in r.stderr
+
+
+@pytest.mark.skipif(is_windows, reason='cannot chmod 000 on Windows')
+def test_config_file_inaccessible(httpbin):
+    env = MockEnvironment()
+    env.create_temp_config_dir()
+    config_path = env.config_dir / Config.FILENAME
+    assert not config_path.exists()
+    config_path.touch(0o000)
+    assert config_path.exists()
+    r = http(httpbin + '/get', env=env)
+    assert HTTP_OK in r
+    assert 'http: warning' in r.stderr
+    assert 'cannot read config file' in r.stderr
 
 
 def test_default_options_overwrite(httpbin):
@@ -16,25 +45,6 @@ def test_default_options_overwrite(httpbin):
     env.config['default_options'] = ['--form']
     env.config.save()
     r = http('--json', httpbin.url + '/post', 'foo=bar', env=env)
-    assert r.json['json'] == {"foo": "bar"}
-
-
-def test_migrate_implicit_content_type():
-    config = MockEnvironment().config
-
-    config['implicit_content_type'] = 'json'
-    config.save()
-    config.load()
-    assert 'implicit_content_type' not in config
-    assert not config['default_options']
-
-    config['implicit_content_type'] = 'form'
-    config.save()
-    config.load()
-    assert 'implicit_content_type' not in config
-    assert config['default_options'] == ['--form']
-
-
-def test_current_version():
-    version = Environment().config['__meta__']['httpie']
-    assert version == __version__
+    assert r.json['json'] == {
+        "foo": "bar"
+    }

tests/test_docs.py

@@ -1,15 +1,23 @@
 import os
-import fnmatch
 import subprocess
+from glob import glob
+from pathlib import Path
 
 import pytest
 
 from utils import TESTS_ROOT
 
 
+SOURCE_DIRECTORIES = [
+    'extras',
+    'httpie',
+    'tests',
+]
+
+
 def has_docutils():
     try:
-        # noinspection PyUnresolvedReferences
+        # noinspection PyUnresolvedReferences,PyPackageRequirements
         import docutils
         return True
     except ImportError:
@@ -17,23 +25,41 @@ def has_docutils():
 
 
 def rst_filenames():
-    for root, dirnames, filenames in os.walk(os.path.dirname(TESTS_ROOT)):
-        if '.tox' not in root:
-            for filename in fnmatch.filter(filenames, '*.rst'):
-                yield os.path.join(root, filename)
+    cwd = os.getcwd()
+    os.chdir(TESTS_ROOT.parent)
+    try:
+        yield from glob('*.rst')
+        for directory in SOURCE_DIRECTORIES:
+            yield from glob(f'{directory}/**/*.rst', recursive=True)
+    finally:
+        os.chdir(cwd)
 
 
-filenames = list(rst_filenames())
+filenames = list(sorted(rst_filenames()))
 assert filenames
 
 
-@pytest.mark.skipif(not has_docutils(), reason='docutils not installed')
+# HACK: hardcoded paths, venv should be irrelevant, etc.
+# TODO: replaces the process with Python code
+VENV_BIN = Path(__file__).parent.parent / 'venv/bin'
+VENV_PYTHON = VENV_BIN / 'python'
+VENV_RST2PSEUDOXML = VENV_BIN / 'rst2pseudoxml.py'
+
+
+@pytest.mark.skipif(not os.path.exists(VENV_RST2PSEUDOXML), reason='docutils not installed')
 @pytest.mark.parametrize('filename', filenames)
 def test_rst_file_syntax(filename):
     p = subprocess.Popen(
-        ['rst2pseudoxml.py', '--report=1', '--exit-status=1', filename],
+        [
+            VENV_PYTHON,
+            VENV_RST2PSEUDOXML,
+            '--report=1',
+            '--exit-status=1',
+            filename,
+        ],
         stderr=subprocess.PIPE,
-        stdout=subprocess.PIPE
+        stdout=subprocess.PIPE,
+        shell=True,
     )
     err = p.communicate()[1]
     assert p.returncode == 0, err.decode('utf8')

tests/test_downloads.py

@@ -1,11 +1,12 @@
 import os
+import tempfile
 import time
+from urllib.request import urlopen
 
 import pytest
 import mock
 from requests.structures import CaseInsensitiveDict
 
-from httpie.compat import urlopen
 from httpie.downloads import (
     parse_content_range, filename_from_content_disposition, filename_from_url,
     get_unique_filename, ContentRangeError, Downloader,
@@ -13,7 +14,7 @@ from httpie.downloads import (
 from utils import http, MockEnvironment
 
 
-class Response(object):
+class Response:
     # noinspection PyDefaultArgument
     def __init__(self, url, headers={}, status_code=200):
         self.url = url
@@ -22,6 +23,7 @@ class Response(object):
 
 
 class TestDownloadUtils:
+
     def test_Content_Range_parsing(self):
         parse = parse_content_range
 
@@ -131,35 +133,59 @@ class TestDownloads:
         assert body == r
 
     def test_download_with_Content_Length(self, httpbin_both):
-        devnull = open(os.devnull, 'w')
-        downloader = Downloader(output_file=devnull, progress_file=devnull)
-        downloader.start(Response(
-            url=httpbin_both.url + '/',
-            headers={'Content-Length': 10}
-        ))
-        time.sleep(1.1)
-        downloader.chunk_downloaded(b'12345')
-        time.sleep(1.1)
-        downloader.chunk_downloaded(b'12345')
-        downloader.finish()
-        assert not downloader.interrupted
+        with open(os.devnull, 'w') as devnull:
+            downloader = Downloader(output_file=devnull, progress_file=devnull)
+            downloader.start(
+                initial_url='/',
+                final_response=Response(
+                    url=httpbin_both.url + '/',
+                    headers={'Content-Length': 10}
+                )
+            )
+            time.sleep(1.1)
+            downloader.chunk_downloaded(b'12345')
+            time.sleep(1.1)
+            downloader.chunk_downloaded(b'12345')
+            downloader.finish()
+            assert not downloader.interrupted
+            downloader._progress_reporter.join()
 
     def test_download_no_Content_Length(self, httpbin_both):
-        devnull = open(os.devnull, 'w')
-        downloader = Downloader(output_file=devnull, progress_file=devnull)
-        downloader.start(Response(url=httpbin_both.url + '/'))
-        time.sleep(1.1)
-        downloader.chunk_downloaded(b'12345')
-        downloader.finish()
-        assert not downloader.interrupted
+        with open(os.devnull, 'w') as devnull:
+            downloader = Downloader(output_file=devnull, progress_file=devnull)
+            downloader.start(
+                final_response=Response(url=httpbin_both.url + '/'),
+                initial_url='/'
+            )
+            time.sleep(1.1)
+            downloader.chunk_downloaded(b'12345')
+            downloader.finish()
+            assert not downloader.interrupted
+            downloader._progress_reporter.join()
 
     def test_download_interrupted(self, httpbin_both):
-        devnull = open(os.devnull, 'w')
-        downloader = Downloader(output_file=devnull, progress_file=devnull)
-        downloader.start(Response(
-            url=httpbin_both.url + '/',
-            headers={'Content-Length': 5}
-        ))
-        downloader.chunk_downloaded(b'1234')
-        downloader.finish()
-        assert downloader.interrupted
+        with open(os.devnull, 'w') as devnull:
+            downloader = Downloader(output_file=devnull, progress_file=devnull)
+            downloader.start(
+                final_response=Response(
+                    url=httpbin_both.url + '/',
+                    headers={'Content-Length': 5}
+                ),
+                initial_url='/'
+            )
+            downloader.chunk_downloaded(b'1234')
+            downloader.finish()
+            assert downloader.interrupted
+            downloader._progress_reporter.join()
+
+    def test_download_with_redirect_original_url_used_for_filename(self, httpbin):
+        # Redirect from `/redirect/1` to `/get`.
+        expected_filename = '1.json'
+        orig_cwd = os.getcwd()
+        os.chdir(tempfile.mkdtemp(prefix='httpie_download_test_'))
+        try:
+            assert os.listdir('.') == []
+            http('--download', httpbin.url + '/redirect/1')
+            assert os.listdir('.') == [expected_filename]
+        finally:
+            os.chdir(orig_cwd)

tests/test_errors.py

@@ -1,49 +1,41 @@
 import mock
 from pytest import raises
-from requests import Request, Timeout
+from requests import Request
 from requests.exceptions import ConnectionError
 
-from httpie import ExitStatus
-from httpie.core import main
-
-error_msg = None
+from httpie.status import ExitStatus
+from utils import HTTP_OK, http
 
 
-@mock.patch('httpie.core.get_response')
-def test_error(get_response):
-    def error(msg, *args, **kwargs):
-        global error_msg
-        error_msg = msg % args
-
+@mock.patch('httpie.core.program')
+def test_error(program):
     exc = ConnectionError('Connection aborted')
     exc.request = Request(method='GET', url='http://www.google.com')
-    get_response.side_effect = exc
-    ret = main(['--ignore-stdin', 'www.google.com'], custom_log_error=error)
-    assert ret == ExitStatus.ERROR
-    assert error_msg == (
+    program.side_effect = exc
+    r = http('www.google.com', tolerate_error_exit_status=True)
+    assert r.exit_status == ExitStatus.ERROR
+    error_msg = (
         'ConnectionError: '
-        'Connection aborted while doing GET request to URL: '
-        'http://www.google.com')
+        'Connection aborted while doing a GET request to URL: '
+        'http://www.google.com'
+    )
+    assert error_msg in r.stderr
 
 
-@mock.patch('httpie.core.get_response')
-def test_error_traceback(get_response):
+@mock.patch('httpie.core.program')
+def test_error_traceback(program):
     exc = ConnectionError('Connection aborted')
     exc.request = Request(method='GET', url='http://www.google.com')
-    get_response.side_effect = exc
+    program.side_effect = exc
     with raises(ConnectionError):
-        main(['--ignore-stdin', '--traceback', 'www.google.com'])
+        http('--traceback', 'www.google.com')
 
 
-@mock.patch('httpie.core.get_response')
-def test_timeout(get_response):
-    def error(msg, *args, **kwargs):
-        global error_msg
-        error_msg = msg % args
+def test_max_headers_limit(httpbin_both):
+    with raises(ConnectionError) as e:
+        http('--max-headers=1', httpbin_both + '/get')
+    assert 'got more than 1 headers' in str(e.value)
 
-    exc = Timeout('Request timed out')
-    exc.request = Request(method='GET', url='http://www.google.com')
-    get_response.side_effect = exc
-    ret = main(['--ignore-stdin', 'www.google.com'], custom_log_error=error)
-    assert ret == ExitStatus.ERROR_TIMEOUT
-    assert error_msg == 'Request timed out (30s).'
+
+def test_max_headers_no_limit(httpbin_both):
+    assert HTTP_OK in http('--max-headers=0', httpbin_both + '/get')

tests/test_exit_status.py

@@ -1,20 +1,20 @@
 import mock
 
-from httpie import ExitStatus
+from httpie.status import ExitStatus
 from utils import MockEnvironment, http, HTTP_OK
 
 
 def test_keyboard_interrupt_during_arg_parsing_exit_status(httpbin):
-    with mock.patch('httpie.cli.parser.parse_args',
+    with mock.patch('httpie.cli.definition.parser.parse_args',
                     side_effect=KeyboardInterrupt()):
-        r = http('GET', httpbin.url + '/get', error_exit_ok=True)
+        r = http('GET', httpbin.url + '/get', tolerate_error_exit_status=True)
         assert r.exit_status == ExitStatus.ERROR_CTRL_C
 
 
 def test_keyboard_interrupt_in_program_exit_status(httpbin):
     with mock.patch('httpie.core.program',
                     side_effect=KeyboardInterrupt()):
-        r = http('GET', httpbin.url + '/get', error_exit_ok=True)
+        r = http('GET', httpbin.url + '/get', tolerate_error_exit_status=True)
         assert r.exit_status == ExitStatus.ERROR_CTRL_C
 
 
@@ -34,7 +34,7 @@ def test_error_response_exits_0_without_check_status(httpbin):
 def test_timeout_exit_status(httpbin):
 
     r = http('--timeout=0.01', 'GET', httpbin.url + '/delay/0.5',
-             error_exit_ok=True)
+             tolerate_error_exit_status=True)
     assert r.exit_status == ExitStatus.ERROR_TIMEOUT
 
 
@@ -43,7 +43,7 @@ def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected(
     env = MockEnvironment(stdout_isatty=False)
     r = http('--check-status', '--headers',
              'GET', httpbin.url + '/status/301',
-             env=env, error_exit_ok=True)
+             env=env, tolerate_error_exit_status=True)
     assert '301 MOVED PERMANENTLY' in r
     assert r.exit_status == ExitStatus.ERROR_HTTP_3XX
     assert '301 moved permanently' in r.stderr.lower()
@@ -52,7 +52,7 @@ def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected(
 def test_3xx_check_status_redirects_allowed_exits_0(httpbin):
     r = http('--check-status', '--follow',
              'GET', httpbin.url + '/status/301',
-             error_exit_ok=True)
+             tolerate_error_exit_status=True)
     # The redirect will be followed so 200 is expected.
     assert HTTP_OK in r
     assert r.exit_status == ExitStatus.SUCCESS
@@ -60,7 +60,7 @@ def test_3xx_check_status_redirects_allowed_exits_0(httpbin):
 
 def test_4xx_check_status_exits_4(httpbin):
     r = http('--check-status', 'GET', httpbin.url + '/status/401',
-             error_exit_ok=True)
+             tolerate_error_exit_status=True)
     assert '401 UNAUTHORIZED' in r
     assert r.exit_status == ExitStatus.ERROR_HTTP_4XX
     # Also stderr should be empty since stdout isn't redirected.
@@ -69,6 +69,6 @@ def test_4xx_check_status_exits_4(httpbin):
 
 def test_5xx_check_status_exits_5(httpbin):
     r = http('--check-status', 'GET', httpbin.url + '/status/500',
-             error_exit_ok=True)
+             tolerate_error_exit_status=True)
     assert '500 INTERNAL SERVER ERROR' in r
     assert r.exit_status == ExitStatus.ERROR_HTTP_5XX

tests/test_httpie.py

@@ -1,30 +1,53 @@
 """High-level tests."""
+import io
+from unittest import mock
+
 import pytest
 
-from httpie.input import ParseError
+import httpie.__main__
+from httpie.context import Environment
+from httpie.status import ExitStatus
+from httpie.cli.exceptions import ParseError
 from utils import MockEnvironment, http, HTTP_OK
 from fixtures import FILE_PATH, FILE_CONTENT
 
 import httpie
 
 
+def test_main_entry_point():
+    # Patch stdin to bypass pytest capture
+    with mock.patch.object(Environment, 'stdin', io.StringIO()):
+        with pytest.raises(SystemExit) as e:
+            httpie.__main__.main()
+        assert e.value.code == ExitStatus.ERROR
+
+
+@mock.patch('httpie.core.main')
+def test_main_entry_point_keyboard_interrupt(main):
+    main.side_effect = KeyboardInterrupt()
+    with mock.patch.object(Environment, 'stdin', io.StringIO()):
+        with pytest.raises(SystemExit) as e:
+            httpie.__main__.main()
+        assert e.value.code == ExitStatus.ERROR_CTRL_C
+
+
 def test_debug():
     r = http('--debug')
-    assert r.exit_status == httpie.ExitStatus.SUCCESS
+    assert r.exit_status == ExitStatus.SUCCESS
     assert 'HTTPie %s' % httpie.__version__ in r.stderr
 
 
 def test_help():
-    r = http('--help', error_exit_ok=True)
-    assert r.exit_status == httpie.ExitStatus.SUCCESS
+    r = http('--help', tolerate_error_exit_status=True)
+    assert r.exit_status == ExitStatus.SUCCESS
     assert 'https://github.com/jakubroztocil/httpie/issues' in r
 
 
 def test_version():
-    r = http('--version', error_exit_ok=True)
-    assert r.exit_status == httpie.ExitStatus.SUCCESS
+    r = http('--version', tolerate_error_exit_status=True)
+    assert r.exit_status == ExitStatus.SUCCESS
     # FIXME: py3 has version in stdout, py2 in stderr
-    assert httpie.__version__ == r.stderr.strip() + r.strip()
+    assert httpie.__version__ == r.strip()
 
 
 def test_GET(httpbin_both):
@@ -69,6 +92,12 @@ def test_POST_stdin(httpbin_both):
     assert FILE_CONTENT in r
 
 
+def test_POST_file(httpbin_both):
+    r = http('--form', 'POST', httpbin_both + '/post', 'file@' + FILE_PATH)
+    assert HTTP_OK in r
+    assert FILE_CONTENT in r
+
+
 def test_headers(httpbin_both):
     r = http('GET', httpbin_both + '/headers', 'Foo:bar')
     assert HTTP_OK in r

tests/test_output.py

@@ -1,11 +1,11 @@
 import os
 from tempfile import gettempdir
+from urllib.request import urlopen
 
 import pytest
 
 from utils import MockEnvironment, http, HTTP_OK, COLOR, CRLF
-from httpie import ExitStatus
-from httpie.compat import urlopen
+from httpie.status import ExitStatus
 from httpie.output.formatters.colors import get_lexer
 
 

tests/test_redirects.py

@@ -1,7 +1,7 @@
 """High-level tests."""
 import pytest
 
-from httpie import ExitStatus
+from httpie.status import ExitStatus
 from utils import http, HTTP_OK
 
 
@@ -28,20 +28,25 @@ def test_follow_all_output_options_used_for_redirects(httpbin):
     assert r.count('GET /') == 3
     assert HTTP_OK not in r
 
-
-def test_follow_redirect_output_options(httpbin):
-    r = http('--check-status',
-             '--follow',
-             '--all',
-             '--print=h',
-             '--history-print=H',
-             httpbin.url + '/redirect/2')
-    assert r.count('GET /') == 2
-    assert 'HTTP/1.1 302 FOUND' not in r
-    assert HTTP_OK in r
+#
+# def test_follow_redirect_output_options(httpbin):
+#     r = http('--check-status',
+#              '--follow',
+#              '--all',
+#              '--print=h',
+#              '--history-print=H',
+#              httpbin.url + '/redirect/2')
+#     assert r.count('GET /') == 2
+#     assert 'HTTP/1.1 302 FOUND' not in r
+#     assert HTTP_OK in r
+#
 
 
 def test_max_redirects(httpbin):
-    r = http('--max-redirects=1', '--follow', httpbin.url + '/redirect/3',
-             error_exit_ok=True)
+    r = http(
+        '--max-redirects=1',
+        '--follow',
+        httpbin.url + '/redirect/3',
+        tolerate_error_exit_status=True,
+    )
     assert r.exit_status == ExitStatus.ERROR_TOO_MANY_REDIRECTS

tests/test_sessions.py

@@ -11,7 +11,7 @@ from utils import MockEnvironment, mk_config_dir, http, HTTP_OK
 from fixtures import UNICODE
 
 
-class SessionTestBase(object):
+class SessionTestBase:
 
     def start_session(self, httpbin):
         """Create and reuse a unique config dir for each test."""
@@ -24,7 +24,7 @@ class SessionTestBase(object):
         """
         Return an environment.
 
-        Each environment created withing a test method
+        Each environment created within a test method
         will share the same config_dir. It is necessary
         for session files being reused.
 
@@ -44,11 +44,16 @@ class TestSessionFlow(SessionTestBase):
         authorization, and response cookies.
 
         """
-        super(TestSessionFlow, self).start_session(httpbin)
-        r1 = http('--follow', '--session=test', '--auth=username:password',
-                  'GET', httpbin.url + '/cookies/set?hello=world',
-                  'Hello:World',
-                  env=self.env())
+        super().start_session(httpbin)
+        r1 = http(
+            '--follow',
+            '--session=test',
+            '--auth=username:password',
+            'GET',
+            httpbin.url + '/cookies/set?hello=world',
+            'Hello:World',
+            env=self.env()
+        )
         assert HTTP_OK in r1
 
     def test_session_created_and_reused(self, httpbin):
@@ -130,20 +135,16 @@ class TestSession(SessionTestBase):
 
     def test_session_by_path(self, httpbin):
         self.start_session(httpbin)
-        session_path = os.path.join(self.config_dir, 'session-by-path.json')
-        r1 = http('--session=' + session_path, 'GET', httpbin.url + '/get',
+        session_path = self.config_dir / 'session-by-path.json'
+        r1 = http('--session', str(session_path), 'GET', httpbin.url + '/get',
                   'Foo:Bar', env=self.env())
         assert HTTP_OK in r1
 
-        r2 = http('--session=' + session_path, 'GET', httpbin.url + '/get',
+        r2 = http('--session', str(session_path), 'GET', httpbin.url + '/get',
                   env=self.env())
         assert HTTP_OK in r2
         assert r2.json['headers']['Foo'] == 'Bar'
 
-    @pytest.mark.skipif(
-        sys.version_info >= (3,),
-        reason="This test fails intermittently on Python 3 - "
-               "see https://github.com/jakubroztocil/httpie/issues/282")
     def test_session_unicode(self, httpbin):
         self.start_session(httpbin)
 

tests/test_ssl.py

@@ -2,17 +2,31 @@ import os
 
 import pytest
 import pytest_httpbin.certs
-from requests.exceptions import SSLError
+import requests.exceptions
 
-from httpie import ExitStatus
-from httpie.input import SSL_VERSION_ARG_MAPPING
-from utils import http, HTTP_OK, TESTS_ROOT
+from httpie.status import ExitStatus
+from httpie.cli.constants import SSL_VERSION_ARG_MAPPING
+from utils import HTTP_OK, TESTS_ROOT, http
+
+
+try:
+    # Handle OpenSSL errors, if installed.
+    # See <https://github.com/jakubroztocil/httpie/issues/729>
+    # noinspection PyUnresolvedReferences
+    import OpenSSL.SSL
+    ssl_errors = (
+        requests.exceptions.SSLError,
+        OpenSSL.SSL.Error,
+    )
+except ImportError:
+    ssl_errors = (
+        requests.exceptions.SSLError,
+    )
 
 
 CLIENT_CERT = os.path.join(TESTS_ROOT, 'client_certs', 'client.crt')
 CLIENT_KEY = os.path.join(TESTS_ROOT, 'client_certs', 'client.key')
 CLIENT_PEM = os.path.join(TESTS_ROOT, 'client_certs', 'client.pem')
-
 # FIXME:
 # We test against a local httpbin instance which uses a self-signed cert.
 # Requests without --verify=<CA_BUNDLE> will fail with a verification error.
@@ -28,10 +42,10 @@ def test_ssl_version(httpbin_secure, ssl_version):
             httpbin_secure + '/get'
         )
         assert HTTP_OK in r
-    except SSLError as e:
+    except ssl_errors as e:
         if ssl_version == 'ssl3':
             # pytest-httpbin doesn't support ssl3
-            assert 'SSLV3_ALERT_HANDSHAKE_FAILURE' in str(e)
+            pass
         else:
             raise
 
@@ -52,17 +66,17 @@ class TestClientCert:
     def test_cert_file_not_found(self, httpbin_secure):
         r = http(httpbin_secure + '/get',
                  '--cert', '/__not_found__',
-                 error_exit_ok=True)
+                 tolerate_error_exit_status=True)
         assert r.exit_status == ExitStatus.ERROR
         assert 'No such file or directory' in r.stderr
 
     def test_cert_file_invalid(self, httpbin_secure):
-        with pytest.raises(SSLError):
+        with pytest.raises(ssl_errors):
             http(httpbin_secure + '/get',
                  '--cert', __file__)
 
     def test_cert_ok_but_missing_key(self, httpbin_secure):
-        with pytest.raises(SSLError):
+        with pytest.raises(ssl_errors):
             http(httpbin_secure + '/get',
                  '--cert', CLIENT_CERT)
 
@@ -79,21 +93,23 @@ class TestServerCert:
         assert HTTP_OK in r
 
     def test_verify_custom_ca_bundle_path(
-            self, httpbin_secure_untrusted):
+        self, httpbin_secure_untrusted
+    ):
         r = http(httpbin_secure_untrusted + '/get', '--verify', CA_BUNDLE)
         assert HTTP_OK in r
 
     def test_self_signed_server_cert_by_default_raises_ssl_error(
-            self,
-            httpbin_secure_untrusted):
-        with pytest.raises(SSLError):
+        self,
+        httpbin_secure_untrusted
+    ):
+        with pytest.raises(ssl_errors):
             http(httpbin_secure_untrusted.url + '/get')
 
     def test_verify_custom_ca_bundle_invalid_path(self, httpbin_secure):
         # since 2.14.0 requests raises IOError
-        with pytest.raises((SSLError, IOError)):
+        with pytest.raises(ssl_errors + (IOError,)):
             http(httpbin_secure.url + '/get', '--verify', '/__not_found__')
 
     def test_verify_custom_ca_bundle_invalid_bundle(self, httpbin_secure):
-        with pytest.raises(SSLError):
+        with pytest.raises(ssl_errors):
             http(httpbin_secure.url + '/get', '--verify', __file__)

tests/test_uploads.py

@@ -2,7 +2,7 @@ import os
 
 import pytest
 
-from httpie.input import ParseError
+from httpie.cli.exceptions import ParseError
 from utils import MockEnvironment, http, HTTP_OK
 from fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT
 
@@ -64,12 +64,17 @@ class TestRequestBodyFromFilePath:
             self, httpbin):
         env = MockEnvironment(stdin_isatty=True)
         r = http('POST', httpbin.url + '/post', 'field-name@' + FILE_PATH_ARG,
-                 env=env, error_exit_ok=True)
+                 env=env, tolerate_error_exit_status=True)
         assert 'perhaps you meant --form?' in r.stderr
 
     def test_request_body_from_file_by_path_no_data_items_allowed(
             self, httpbin):
         env = MockEnvironment(stdin_isatty=False)
-        r = http('POST', httpbin.url + '/post', '@' + FILE_PATH_ARG, 'foo=bar',
-                 env=env, error_exit_ok=True)
+        r = http(
+            'POST',
+            httpbin.url + '/post',
+            '@' + FILE_PATH_ARG, 'foo=bar',
+            env=env,
+            tolerate_error_exit_status=True,
+        )
         assert 'cannot be mixed' in r.stderr

tests/test_windows.py

@@ -27,5 +27,5 @@ class TestFakeWindows:
         )
         r = http('--output', output_file,
                  '--pretty=all', 'GET', httpbin.url + '/get',
-                 env=env, error_exit_ok=True)
+                 env=env, tolerate_error_exit_status=True)
         assert 'Only terminal output can be colorized on Windows' in r.stderr

tests/utils.py

@@ -5,17 +5,20 @@ import sys
 import time
 import json
 import tempfile
+from pathlib import Path
+from typing import Optional, Union
 
-from httpie import ExitStatus, EXIT_STATUS_LABELS
+from httpie.status import ExitStatus
+from httpie.config import Config
 from httpie.context import Environment
 from httpie.core import main
-from httpie.compat import bytes, str
 
 
-TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+TESTS_ROOT = Path(__file__).parent
 CRLF = '\r\n'
 COLOR = '\x1b['
 HTTP_OK = '200 OK'
+# noinspection GrazieInspection
 HTTP_OK_COLOR = (
     'HTTP\x1b[39m\x1b[38;5;245m/\x1b[39m\x1b'
     '[38;5;37m1.1\x1b[39m\x1b[38;5;245m \x1b[39m\x1b[38;5;37m200'
@@ -23,9 +26,9 @@ HTTP_OK_COLOR = (
 )
 
 
-def mk_config_dir():
+def mk_config_dir() -> Path:
     dirname = tempfile.mkdtemp(prefix='httpie_config_')
-    return dirname
+    return Path(dirname)
 
 
 def add_auth(url, auth):
@@ -40,7 +43,7 @@ class MockEnvironment(Environment):
     stdout_isatty = True
     is_windows = False
 
-    def __init__(self, **kwargs):
+    def __init__(self, create_temp_config_dir=True, **kwargs):
         if 'stdout' not in kwargs:
             kwargs['stdout'] = tempfile.TemporaryFile(
                 mode='w+b',
@@ -51,32 +54,41 @@ class MockEnvironment(Environment):
                 mode='w+t',
                 prefix='httpie_stderr'
             )
-        super(MockEnvironment, self).__init__(**kwargs)
+        super().__init__(**kwargs)
+        self._create_temp_config_dir = create_temp_config_dir
         self._delete_config_dir = False
+        self._temp_dir = Path(tempfile.gettempdir())
 
     @property
-    def config(self):
-        if not self.config_dir.startswith(tempfile.gettempdir()):
-            self.config_dir = mk_config_dir()
-            self._delete_config_dir = True
-        return super(MockEnvironment, self).config
+    def config(self) -> Config:
+        if (self._create_temp_config_dir
+                and self._temp_dir not in self.config_dir.parents):
+            self.create_temp_config_dir()
+        return super().config
+
+    def create_temp_config_dir(self):
+        self.config_dir = mk_config_dir()
+        self._delete_config_dir = True
 
     def cleanup(self):
+        self.stdout.close()
+        self.stderr.close()
         if self._delete_config_dir:
-            assert self.config_dir.startswith(tempfile.gettempdir())
+            assert self._temp_dir in self.config_dir.parents
             from shutil import rmtree
-            rmtree(self.config_dir)
+            rmtree(self.config_dir, ignore_errors=True)
 
     def __del__(self):
+        # noinspection PyBroadException
         try:
             self.cleanup()
         except Exception:
             pass
 
 
-class BaseCLIResponse(object):
+class BaseCLIResponse:
     """
-    Represents the result of simulated `$ http' invocation  via `http()`.
+    Represents the result of simulated `$ http' invocation via `http()`.
 
     Holds and provides access to:
 
@@ -85,9 +97,9 @@ class BaseCLIResponse(object):
         - exit_status output: print(self.exit_status)
 
     """
-    stderr = None
-    json = None
-    exit_status = None
+    stderr: str = None
+    json: dict = None
+    exit_status: ExitStatus = None
 
 
 class BytesCLIResponse(bytes, BaseCLIResponse):
@@ -104,10 +116,10 @@ class BytesCLIResponse(bytes, BaseCLIResponse):
 class StrCLIResponse(str, BaseCLIResponse):
 
     @property
-    def json(self):
+    def json(self) -> Optional[dict]:
         """
-        Return deserialized JSON body, if one included in the output
-        and is parseable.
+        Return deserialized the request or response JSON body,
+        if one (and only one) included in the output and is parsable.
 
         """
         if not hasattr(self, '_json'):
@@ -129,6 +141,7 @@ class StrCLIResponse(str, BaseCLIResponse):
                     pass
                 else:
                     try:
+                        # noinspection PyAttributeOutsideInit
                         self._json = json.loads(j)
                     except ValueError:
                         pass
@@ -139,7 +152,12 @@ class ExitStatusError(Exception):
     pass
 
 
-def http(*args, **kwargs):
+def http(
+    *args,
+    program_name='http',
+    tolerate_error_exit_status=False,
+    **kwargs,
+) -> Union[StrCLIResponse, BytesCLIResponse]:
     # noinspection PyUnresolvedReferences
     """
     Run HTTPie and capture stderr/out and exit status.
@@ -159,7 +177,7 @@ def http(*args, **kwargs):
 
     Exceptions are propagated.
 
-    If you pass ``error_exit_ok=True``, then error exit statuses
+    If you pass ``tolerate_error_exit_status=True``, then error exit statuses
     won't result into an exception.
 
     Example:
@@ -171,7 +189,7 @@ def http(*args, **kwargs):
         >>> type(r) == StrCLIResponse
         True
         >>> r.exit_status
-        0
+        <ExitStatus.SUCCESS: 0>
         >>> r.stderr
         ''
         >>> 'HTTP/1.1 200 OK' in r
@@ -180,7 +198,6 @@ def http(*args, **kwargs):
         True
 
     """
-    error_exit_ok = kwargs.pop('error_exit_ok', False)
     env = kwargs.get('env')
     if not env:
         env = kwargs['env'] = MockEnvironment()
@@ -192,11 +209,13 @@ def http(*args, **kwargs):
     args_with_config_defaults = args + env.config.default_options
     add_to_args = []
     if '--debug' not in args_with_config_defaults:
-        if not error_exit_ok and '--traceback' not in args_with_config_defaults:
+        if (not tolerate_error_exit_status
+                and '--traceback' not in args_with_config_defaults):
             add_to_args.append('--traceback')
         if not any('--timeout' in arg for arg in args_with_config_defaults):
             add_to_args.append('--timeout=3')
-    args = add_to_args + args
+
+    complete_args = [program_name, *add_to_args, *args]
 
     def dump_stderr():
         stderr.seek(0)
@@ -204,12 +223,12 @@ def http(*args, **kwargs):
 
     try:
         try:
-            exit_status = main(args=args, **kwargs)
+            exit_status = main(args=complete_args, **kwargs)
             if '--download' in args:
                 # Let the progress reporter thread finish.
                 time.sleep(.5)
         except SystemExit:
-            if error_exit_ok:
+            if tolerate_error_exit_status:
                 exit_status = ExitStatus.ERROR
             else:
                 dump_stderr()
@@ -219,14 +238,12 @@ def http(*args, **kwargs):
             sys.stderr.write(stderr.read())
             raise
         else:
-            if not error_exit_ok and exit_status != ExitStatus.SUCCESS:
+            if (not tolerate_error_exit_status
+                    and exit_status != ExitStatus.SUCCESS):
                 dump_stderr()
                 raise ExitStatusError(
                     'httpie.core.main() unexpectedly returned'
-                    ' a non-zero exit status: {0} ({1})'.format(
-                        exit_status,
-                        EXIT_STATUS_LABELS[exit_status]
-                    )
+                    f' a non-zero exit status: {exit_status}'
                 )
 
         stdout.seek(0)
@@ -235,10 +252,8 @@ def http(*args, **kwargs):
         try:
             output = output.decode('utf8')
         except UnicodeDecodeError:
-            # noinspection PyArgumentList
             r = BytesCLIResponse(output)
         else:
-            # noinspection PyArgumentList
             r = StrCLIResponse(output)
         r.stderr = stderr.read()
         r.exit_status = exit_status

tox.ini

@@ -4,7 +4,7 @@
 
 [tox]
 # pypy3 currently fails because of a Flask issue
-envlist = py27, py37, pypy
+envlist = py37
 
 
 [testenv]
@@ -21,6 +21,3 @@ commands =
         --verbose \
         --doctest-modules \
         {posargs:./httpie ./tests}
-
-[testenv:py27-osx-builtin]
-basepython = /usr/bin/python2.7