forked from extern/httpie-cli
Merge branch 'feature/uploads2020'
This commit is contained in:
Jakub Roztocil
commit
299250b3c3
@ -8,14 +8,17 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
|
||||
|
||||
`2.3.0-dev`_ (unreleased)
|
||||
-------------------------
|
||||
|
||||
* Added support for multipart upload streaming (`#684`_).
|
||||
* Added support for body-from-file upload streaming (``http httpbin.org/post @file``).
|
||||
* Added ``--chunked`` to allow chunked transfer encoding.
|
||||
* Added ``--multipart`` to allow ``multipart/form-data`` encoding for non-file ``--form`` requests as well.
|
||||
* Added ``--boundary`` to allow a custom boundary string for ``multipart/form-data`` requests.
|
||||
* Added support for combining cookies specified on the CLI and in a session file (`#932`_).
|
||||
* Added out of the box SOCKS support with no extra installation (`#904`_).
|
||||
* Added ``--quiet, -q`` flag to enforce silent behaviour.
|
||||
* Added ``--multipart`` to allow ``multipart/form-data`` encoding for non-file ``--form`` requests as well.
|
||||
* Added ``--boundary`` to allow a custom boundary string for ``multipart/form-data`` requests.
|
||||
* Removed Tox testing entirely (`#943`_).
|
||||
* Fixed the handling of invalid ``expires`` dates in ``Set-Cookie`` headers (`#963`_).
|
||||
* Removed Tox testing entirely (`#943`_).
|
||||
|
||||
|
||||
`2.2.0`_ (2020-06-18)
|
||||
|
||||
113
README.rst
113
README.rst
@ -16,7 +16,7 @@ They use simple and natural syntax and provide formatted and colorized output.
|
||||
|
||||
.. class:: no-web no-pdf
|
||||
|
||||
.. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.gif
|
||||
.. image:: https://raw.githubusercontent.com/httpie/httpie/master/httpie.gif
|
||||
:alt: HTTPie in action
|
||||
:width: 100%
|
||||
:align: center
|
||||
@ -35,7 +35,7 @@ where you can select your corresponding HTTPie version as well as run examples d
|
||||
browser using a `termible.io <https://termible.io?utm_source=httpie-readme>`_ embedded terminal.
|
||||
If you are reading this on GitHub, then this text covers the current *development* version.
|
||||
You are invited to submit fixes and improvements to the the docs by editing
|
||||
`README.rst <https://github.com/jakubroztocil/httpie/blob/master/README.rst>`_.
|
||||
`README.rst <https://github.com/httpie/httpie/blob/master/README.rst>`_.
|
||||
|
||||
|
||||
Main features
|
||||
@ -58,7 +58,7 @@ Main features
|
||||
|
||||
.. class:: no-web
|
||||
|
||||
.. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.png
|
||||
.. image:: https://raw.githubusercontent.com/httpie/httpie/master/httpie.png
|
||||
:alt: HTTPie compared to cURL
|
||||
:width: 100%
|
||||
:align: center
|
||||
@ -167,11 +167,11 @@ Otherwise with ``pip``:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ pip install --upgrade https://github.com/jakubroztocil/httpie/archive/master.tar.gz
|
||||
$ pip install --upgrade https://github.com/httpie/httpie/archive/master.tar.gz
|
||||
|
||||
|
||||
Verify that now we have the
|
||||
`current development version identifier <https://github.com/jakubroztocil/httpie/blob/master/httpie/__init__.py#L6>`_
|
||||
`current development version identifier <https://github.com/httpie/httpie/blob/master/httpie/__init__.py#L6>`_
|
||||
with the ``-dev`` suffix, for example:
|
||||
|
||||
.. code-block:: bash
|
||||
@ -234,12 +234,12 @@ Build and print a request without sending it using `offline mode`_:
|
||||
|
||||
|
||||
Use `GitHub API`_ to post a comment on an
|
||||
`issue <https://github.com/jakubroztocil/httpie/issues/83>`_
|
||||
`issue <https://github.com/httpie/httpie/issues/83>`_
|
||||
with `authentication`_:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http -a USERNAME POST https://api.github.com/repos/jakubroztocil/httpie/issues/83/comments body='HTTPie is awesome! :heart:'
|
||||
$ http -a USERNAME POST https://api.github.com/repos/httpie/httpie/issues/83/comments body='HTTPie is awesome! :heart:'
|
||||
|
||||
|
||||
Upload a file using `redirected input`_:
|
||||
@ -471,20 +471,23 @@ their type is distinguished only by the separator used:
|
||||
| ``name==value`` | string parameter to the URL. |
|
||||
| | The ``==`` separator is used. |
|
||||
+-----------------------+-----------------------------------------------------+
|
||||
| Data Fields | Request data fields to be serialized as a JSON |
|
||||
| ``field=value``, | object (default), or to be form-encoded |
|
||||
| ``field=@file.txt`` | (``--form, -f``). |
|
||||
| Data Fields | Request data fields to be serialized as a JSON |
|
||||
| ``field=value``, | object (default), to be form-encoded |
|
||||
| ``field=@file.txt`` | (with ``--form, -f``), or to be serialized as |
|
||||
| | ``multipart/form-data`` (with ``--multipart``). |
|
||||
+-----------------------+-----------------------------------------------------+
|
||||
| Raw JSON fields | Useful when sending JSON and one or |
|
||||
| ``field:=json``, | more fields need to be a ``Boolean``, ``Number``, |
|
||||
| ``field:=@file.json`` | nested ``Object``, or an ``Array``, e.g., |
|
||||
| Raw JSON fields | Useful when sending JSON and one or |
|
||||
| ``field:=json``, | more fields need to be a ``Boolean``, ``Number``, |
|
||||
| ``field:=@file.json`` | nested ``Object``, or an ``Array``, e.g., |
|
||||
| | ``meals:='["ham","spam"]'`` or ``pies:=[1,2,3]`` |
|
||||
| | (note the quotes). |
|
||||
+-----------------------+-----------------------------------------------------+
|
||||
| Form File Fields | Only available with ``--form, -f``. |
|
||||
| ``field@/dir/file`` | For example ``screenshot@~/Pictures/img.png``. |
|
||||
| | The presence of a file field results |
|
||||
| | in a ``multipart/form-data`` request. |
|
||||
| Fields upload fields | Only available with ``--form, -f`` and |
|
||||
| ``field@/dir/file`` | ``--multipart``. |
|
||||
| ``field@file;type`` | For example ``screenshot@~/Pictures/img.png``, or |
|
||||
| | ``'cv@cv.txt;text/markdown'``. |
|
||||
| | With ``--form``, the presence of a file field |
|
||||
| | results in a ``--multipart`` request. |
|
||||
+-----------------------+-----------------------------------------------------+
|
||||
|
||||
|
||||
@ -684,7 +687,7 @@ submitted:
|
||||
<input type="file" name="cv" />
|
||||
</form>
|
||||
|
||||
Note that ``@`` is used to simulate a file upload form field, whereas
|
||||
Please note that ``@`` is used to simulate a file upload form field, whereas
|
||||
``=@`` just embeds the file content as a regular text field value.
|
||||
|
||||
When uploading files, their content type is inferred from the file name. You can manually
|
||||
@ -694,16 +697,12 @@ override the inferred content type:
|
||||
|
||||
$ http -f POST httpbin.org/post name='John Smith' cv@'~/files/data.bin;type=application/pdf'
|
||||
|
||||
Larger multipart uploads (i.e., ``--form`` requests with at least one ``file@path``)
|
||||
are always streamed to avoid memory issues. Additionally, the display of the
|
||||
request body on the terminal is suppressed.
|
||||
|
||||
You can explicitly use ``--multipart`` to enforce ``multipart/form-data`` even
|
||||
for form requests without any files:
|
||||
To perform a ``multipart/form-data`` request even without any files, use
|
||||
``--multipart`` instead of ``--form``:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http --form --multipart --offline example.org hello=world
|
||||
$ http --multipart --offline example.org hello=world
|
||||
|
||||
.. code-block:: http
|
||||
|
||||
@ -718,8 +717,10 @@ for form requests without any files:
|
||||
world
|
||||
--c31279ab254f40aeb06df32b433cbccb--
|
||||
|
||||
By default, HTTPie uses a random unique string as the boundary but you can use
|
||||
``--boundary`` to specify a custom string instead:
|
||||
File uploads are always streamed to avoid memory issues with large files.
|
||||
|
||||
By default, HTTPie uses a random unique string as the multipart boundary
|
||||
but you can use ``--boundary`` to specify a custom string instead:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
@ -739,8 +740,8 @@ By default, HTTPie uses a random unique string as the boundary but you can use
|
||||
--xoxo--
|
||||
|
||||
If you specify a custom ``Content-Type`` header without including the boundary
|
||||
bit, HTTPie will add the boundary value (specified or generated) to the header
|
||||
automatically:
|
||||
bit, HTTPie will add the boundary value (explicitly specified or auto-generated)
|
||||
to the header automatically:
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
@ -1347,9 +1348,14 @@ Redirected Input
|
||||
================
|
||||
|
||||
The universal method for passing request data is through redirected ``stdin``
|
||||
(standard input)—piping. Such data is buffered and then with no further
|
||||
processing used as the request body. There are multiple useful ways to use
|
||||
piping:
|
||||
(standard input)—piping.
|
||||
|
||||
By default, ``stdin`` data is buffered and then with no further processing
|
||||
used as the request body. If you provide ``Content-Length``, then the request
|
||||
body is streamed without buffering. You can also use ``--chunked`` to enable
|
||||
streaming via `chunked transfer encoding`_.
|
||||
|
||||
There are multiple useful ways to use piping:
|
||||
|
||||
Redirect from a file:
|
||||
|
||||
@ -1383,7 +1389,7 @@ You can even pipe web services together using HTTPie:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http GET https://api.github.com/repos/jakubroztocil/httpie | http POST httpbin.org/post
|
||||
$ http GET https://api.github.com/repos/httpie/httpie | http POST httpbin.org/post
|
||||
|
||||
|
||||
You can use ``cat`` to enter multiline data on the terminal:
|
||||
@ -1438,6 +1444,33 @@ verbatim contents of that XML file with ``Content-Type: application/xml``:
|
||||
|
||||
$ http PUT httpbin.org/put @files/data.xml
|
||||
|
||||
File uploads are always streamed to avoid memory issues with large files.
|
||||
|
||||
|
||||
Chunked transfer encoding
|
||||
=========================
|
||||
|
||||
For any request, you can use the ``--chunked`` flag to instruct HTTPie to use
|
||||
``Transfer-Encoding: chunked``:
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http --chunked PUT httpbin.org/put hello=world
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http --chunked --multipart PUT httpbin.org/put hello=world foo@files/data.xml
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http --chunked httpbin.org/post @files/data.xml
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ cat files/data.xml | http --chunked httpbin.org/post
|
||||
|
||||
|
||||
|
||||
Terminal output
|
||||
===============
|
||||
@ -1585,7 +1618,7 @@ is being saved to a file.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http --download https://github.com/jakubroztocil/httpie/archive/master.tar.gz
|
||||
$ http --download https://github.com/httpie/httpie/archive/master.tar.gz
|
||||
|
||||
.. code-block:: http
|
||||
|
||||
@ -1627,7 +1660,7 @@ headers and progress are still shown in the terminal:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ http -d https://github.com/jakubroztocil/httpie/archive/master.tar.gz | tar zxf -
|
||||
$ http -d https://github.com/httpie/httpie/archive/master.tar.gz | tar zxf -
|
||||
|
||||
|
||||
|
||||
@ -2108,27 +2141,27 @@ Alternatives
|
||||
Contributing
|
||||
------------
|
||||
|
||||
See `CONTRIBUTING.rst <https://github.com/jakubroztocil/httpie/blob/master/CONTRIBUTING.rst>`_.
|
||||
See `CONTRIBUTING.rst <https://github.com/httpie/httpie/blob/master/CONTRIBUTING.rst>`_.
|
||||
|
||||
|
||||
Change log
|
||||
----------
|
||||
|
||||
See `CHANGELOG <https://github.com/jakubroztocil/httpie/blob/master/CHANGELOG.rst>`_.
|
||||
See `CHANGELOG <https://github.com/httpie/httpie/blob/master/CHANGELOG.rst>`_.
|
||||
|
||||
|
||||
Artwork
|
||||
-------
|
||||
|
||||
* `Logo <https://github.com/claudiatd/httpie-artwork>`_ by `Cláudia Delgado <https://github.com/claudiatd>`_.
|
||||
* `Animation <https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.gif>`_ by `Allen Smith <https://github.com/loranallensmith>`_ of GitHub.
|
||||
* `Animation <https://raw.githubusercontent.com/httpie/httpie/master/httpie.gif>`_ by `Allen Smith <https://github.com/loranallensmith>`_ of GitHub.
|
||||
|
||||
|
||||
|
||||
Licence
|
||||
-------
|
||||
|
||||
BSD-3-Clause: `LICENSE <https://github.com/jakubroztocil/httpie/blob/master/LICENSE>`_.
|
||||
BSD-3-Clause: `LICENSE <https://github.com/httpie/httpie/blob/master/LICENSE>`_.
|
||||
|
||||
|
||||
|
||||
@ -2141,7 +2174,7 @@ have contributed.
|
||||
|
||||
.. _pip: https://pip.pypa.io/en/stable/installing/
|
||||
.. _GitHub API: https://developer.github.com/v3/issues/comments/#create-a-comment
|
||||
.. _these fine people: https://github.com/jakubroztocil/httpie/contributors
|
||||
.. _these fine people: https://github.com/httpie/httpie/contributors
|
||||
.. _Jakub Roztocil: https://roztocil.co
|
||||
.. _@jakubroztocil: https://twitter.com/jakubroztocil
|
||||
|
||||
|
||||
@ -15,12 +15,11 @@ from httpie.cli.argtypes import (
|
||||
parse_format_options,
|
||||
)
|
||||
from httpie.cli.constants import (
|
||||
DEFAULT_FORMAT_OPTIONS, HTTP_GET, HTTP_POST, OUTPUT_OPTIONS,
|
||||
OUTPUT_OPTIONS_DEFAULT,
|
||||
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED, OUT_RESP_BODY, PRETTY_MAP,
|
||||
PRETTY_STDOUT_TTY_ONLY, SEPARATOR_CREDENTIALS, SEPARATOR_GROUP_ALL_ITEMS,
|
||||
SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
|
||||
OUTPUT_OPTIONS_DEFAULT_OFFLINE,
|
||||
HTTP_GET, HTTP_POST, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
|
||||
OUTPUT_OPTIONS_DEFAULT_OFFLINE, OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED,
|
||||
OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, RequestType,
|
||||
SEPARATOR_CREDENTIALS,
|
||||
SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
|
||||
)
|
||||
from httpie.cli.exceptions import ParseError
|
||||
from httpie.cli.requestitems import RequestItems
|
||||
@ -84,6 +83,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
|
||||
)
|
||||
# Arguments processing and environment setup.
|
||||
self._apply_no_options(no_options)
|
||||
self._process_request_type()
|
||||
self._process_download_options()
|
||||
self._setup_standard_streams()
|
||||
self._process_output_options()
|
||||
@ -95,14 +95,31 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
|
||||
self._body_from_file(self.env.stdin)
|
||||
self._process_url()
|
||||
self._process_auth()
|
||||
|
||||
if self.args.compress:
|
||||
# TODO: allow --compress with --chunked / --multipart
|
||||
if self.args.chunked:
|
||||
self.error('cannot combine --compress and --chunked')
|
||||
if self.args.multipart:
|
||||
self.error('cannot combine --compress and --multipart')
|
||||
|
||||
return self.args
|
||||
|
||||
def _process_request_type(self):
|
||||
request_type = self.args.request_type
|
||||
self.args.json = request_type is RequestType.JSON
|
||||
self.args.multipart = request_type is RequestType.MULTIPART
|
||||
self.args.form = request_type in {
|
||||
RequestType.FORM,
|
||||
RequestType.MULTIPART,
|
||||
}
|
||||
|
||||
def _process_url(self):
|
||||
if not URL_SCHEME_RE.match(self.args.url):
|
||||
if os.path.basename(self.env.program_name) == 'https':
|
||||
scheme = 'https://'
|
||||
else:
|
||||
scheme = self.args.default_scheme + "://"
|
||||
scheme = self.args.default_scheme + '://'
|
||||
|
||||
# See if we're using curl style shorthand for localhost (:3000/foo)
|
||||
shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
|
||||
@ -276,7 +293,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
|
||||
'data (key=value) cannot be mixed. Pass '
|
||||
'--ignore-stdin to let key/value take priority. '
|
||||
'See https://httpie.org/doc#scripting for details.')
|
||||
self.args.data = getattr(fd, 'buffer', fd).read()
|
||||
self.args.data = getattr(fd, 'buffer', fd)
|
||||
|
||||
def _guess_method(self):
|
||||
"""Set `args.method` if not specified to either POST or GET
|
||||
@ -337,6 +354,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
|
||||
self.args.data = request_items.data
|
||||
self.args.files = request_items.files
|
||||
self.args.params = request_items.params
|
||||
self.args.multipart_data = request_items.multipart_data
|
||||
|
||||
if self.args.files and not self.args.form:
|
||||
# `http url @/path/to/file`
|
||||
|
||||
@ -242,9 +242,3 @@ PARSED_DEFAULT_FORMAT_OPTIONS = parse_format_options(
|
||||
s=','.join(DEFAULT_FORMAT_OPTIONS),
|
||||
defaults=None,
|
||||
)
|
||||
|
||||
|
||||
class UnsortedAction(argparse.Action):
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return 1
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
"""Parsing and processing of CLI input (args, auth credentials, files, stdin).
|
||||
|
||||
"""
|
||||
import enum
|
||||
import re
|
||||
|
||||
|
||||
@ -10,6 +11,9 @@ import re
|
||||
|
||||
# ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
|
||||
# <https://tools.ietf.org/html/rfc3986#section-3.1>
|
||||
from enum import Enum
|
||||
|
||||
|
||||
URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)
|
||||
|
||||
HTTP_POST = 'POST'
|
||||
@ -37,6 +41,12 @@ SEPARATOR_GROUP_DATA_ITEMS = frozenset({
|
||||
SEPARATOR_DATA_EMBED_RAW_JSON_FILE
|
||||
})
|
||||
|
||||
SEPARATORS_GROUP_MULTIPART = frozenset({
|
||||
SEPARATOR_DATA_STRING,
|
||||
SEPARATOR_DATA_EMBED_FILE_CONTENTS,
|
||||
SEPARATOR_FILE_UPLOAD,
|
||||
})
|
||||
|
||||
# Separators for items whose value is a filename to be embedded
|
||||
SEPARATOR_GROUP_DATA_EMBED_ITEMS = frozenset({
|
||||
SEPARATOR_DATA_EMBED_FILE_CONTENTS,
|
||||
@ -102,3 +112,9 @@ UNSORTED_FORMAT_OPTIONS_STRING = ','.join(
|
||||
OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
|
||||
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
|
||||
OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY
|
||||
|
||||
|
||||
class RequestType(enum.Enum):
|
||||
FORM = enum.auto()
|
||||
MULTIPART = enum.auto()
|
||||
JSON = enum.auto()
|
||||
|
||||
@ -15,7 +15,8 @@ from httpie.cli.constants import (
|
||||
DEFAULT_FORMAT_OPTIONS, OUTPUT_OPTIONS,
|
||||
OUTPUT_OPTIONS_DEFAULT, OUT_REQ_BODY, OUT_REQ_HEAD,
|
||||
OUT_RESP_BODY, OUT_RESP_HEAD, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY,
|
||||
SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, SORTED_FORMAT_OPTIONS_STRING,
|
||||
RequestType, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY,
|
||||
SORTED_FORMAT_OPTIONS_STRING,
|
||||
UNSORTED_FORMAT_OPTIONS_STRING,
|
||||
)
|
||||
from httpie.output.formatters.colors import (
|
||||
@ -110,7 +111,7 @@ positional.add_argument(
|
||||
|
||||
awesome:=true amount:=42 colors:='["red", "green", "blue"]'
|
||||
|
||||
'@' Form file fields (only with --form, -f):
|
||||
'@' Form file fields (only with --form or --multipart):
|
||||
|
||||
cv@~/Documents/CV.pdf
|
||||
cv@'~/Documents/CV.pdf;type=application/pdf'
|
||||
@ -141,7 +142,9 @@ content_type = parser.add_argument_group(
|
||||
|
||||
content_type.add_argument(
|
||||
'--json', '-j',
|
||||
action='store_true',
|
||||
action='store_const',
|
||||
const=RequestType.JSON,
|
||||
dest='request_type',
|
||||
help='''
|
||||
(default) Data items from the command line are serialized as a JSON object.
|
||||
The Content-Type and Accept headers are set to application/json
|
||||
@ -151,7 +154,9 @@ content_type.add_argument(
|
||||
)
|
||||
content_type.add_argument(
|
||||
'--form', '-f',
|
||||
action='store_true',
|
||||
action='store_const',
|
||||
const=RequestType.FORM,
|
||||
dest='request_type',
|
||||
help='''
|
||||
Data items from the command line are serialized as form fields.
|
||||
|
||||
@ -163,11 +168,12 @@ content_type.add_argument(
|
||||
)
|
||||
content_type.add_argument(
|
||||
'--multipart',
|
||||
default=False,
|
||||
action='store_true',
|
||||
action='store_const',
|
||||
const=RequestType.MULTIPART,
|
||||
dest='request_type',
|
||||
help='''
|
||||
Force the request to be encoded as multipart/form-data even without
|
||||
any file fields. Only has effect only together with --form.
|
||||
Similar to --form, but always sends a multipart/form-data
|
||||
request (i.e., even without files).
|
||||
|
||||
'''
|
||||
)
|
||||
@ -394,7 +400,7 @@ output_options.add_argument(
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='''
|
||||
Always stream the output by line, i.e., behave like `tail -f'.
|
||||
Always stream the response body by line, i.e., behave like `tail -f'.
|
||||
|
||||
Without --stream and with --pretty (either set or implied),
|
||||
HTTPie fetches the whole response before it outputs the processed data.
|
||||
@ -657,6 +663,15 @@ network.add_argument(
|
||||
'''
|
||||
)
|
||||
|
||||
network.add_argument(
|
||||
'--chunked',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help="""
|
||||
|
||||
"""
|
||||
)
|
||||
|
||||
#######################################################################
|
||||
# SSL
|
||||
#######################################################################
|
||||
|
||||
@ -34,20 +34,25 @@ class MultiValueOrderedDict(OrderedDict):
|
||||
super().__setitem__(key, [self[key]])
|
||||
self[key].append(value)
|
||||
|
||||
|
||||
class RequestQueryParamsDict(MultiValueOrderedDict):
|
||||
pass
|
||||
|
||||
|
||||
class RequestDataDict(MultiValueOrderedDict):
|
||||
|
||||
def items(self):
|
||||
for key, values in super(MultiValueOrderedDict, self).items():
|
||||
for key, values in super().items():
|
||||
if not isinstance(values, list):
|
||||
values = [values]
|
||||
for value in values:
|
||||
yield key, value
|
||||
|
||||
|
||||
class RequestQueryParamsDict(MultiValueOrderedDict):
|
||||
pass
|
||||
|
||||
|
||||
class RequestDataDict(MultiValueOrderedDict):
|
||||
pass
|
||||
|
||||
|
||||
class MultipartRequestDataDict(MultiValueOrderedDict):
|
||||
pass
|
||||
|
||||
|
||||
class RequestFilesDict(RequestDataDict):
|
||||
pass
|
||||
|
||||
@ -4,13 +4,15 @@ from typing import Callable, Dict, IO, List, Optional, Tuple, Union
|
||||
|
||||
from httpie.cli.argtypes import KeyValueArg
|
||||
from httpie.cli.constants import (
|
||||
SEPARATOR_DATA_EMBED_FILE_CONTENTS, SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
|
||||
SEPARATORS_GROUP_MULTIPART, SEPARATOR_DATA_EMBED_FILE_CONTENTS,
|
||||
SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
|
||||
SEPARATOR_DATA_RAW_JSON, SEPARATOR_DATA_STRING, SEPARATOR_FILE_UPLOAD,
|
||||
SEPARATOR_FILE_UPLOAD_TYPE, SEPARATOR_HEADER, SEPARATOR_HEADER_EMPTY,
|
||||
SEPARATOR_QUERY_PARAM,
|
||||
)
|
||||
from httpie.cli.dicts import (
|
||||
RequestDataDict, RequestFilesDict, RequestHeadersDict, RequestJSONDataDict,
|
||||
MultipartRequestDataDict, RequestDataDict, RequestFilesDict,
|
||||
RequestHeadersDict, RequestJSONDataDict,
|
||||
RequestQueryParamsDict,
|
||||
)
|
||||
from httpie.cli.exceptions import ParseError
|
||||
@ -24,6 +26,8 @@ class RequestItems:
|
||||
self.data = RequestDataDict() if as_form else RequestJSONDataDict()
|
||||
self.files = RequestFilesDict()
|
||||
self.params = RequestQueryParamsDict()
|
||||
# To preserve the order of fields in file upload multipart requests.
|
||||
self.multipart_data = MultipartRequestDataDict()
|
||||
|
||||
@classmethod
|
||||
def from_args(
|
||||
@ -69,7 +73,11 @@ class RequestItems:
|
||||
|
||||
for arg in request_item_args:
|
||||
processor_func, target_dict = rules[arg.sep]
|
||||
target_dict[arg.key] = processor_func(arg)
|
||||
value = processor_func(arg)
|
||||
target_dict[arg.key] = value
|
||||
|
||||
if arg.sep in SEPARATORS_GROUP_MULTIPART:
|
||||
instance.multipart_data[arg.key] = value
|
||||
|
||||
return instance
|
||||
|
||||
|
||||
@ -2,10 +2,9 @@ import argparse
|
||||
import http.client
|
||||
import json
|
||||
import sys
|
||||
import zlib
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Union
|
||||
from typing import Callable, Iterable, Union
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
import requests
|
||||
@ -16,7 +15,10 @@ from httpie.cli.dicts import RequestHeadersDict
|
||||
from httpie.plugins.registry import plugin_manager
|
||||
from httpie.sessions import get_httpie_session
|
||||
from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter
|
||||
from httpie.uploads import get_multipart_data_and_content_type
|
||||
from httpie.uploads import (
|
||||
compress_request, prepare_request_body,
|
||||
get_multipart_data_and_content_type,
|
||||
)
|
||||
from httpie.utils import get_expired_cookies, repr_dict
|
||||
|
||||
|
||||
@ -31,6 +33,7 @@ DEFAULT_UA = f'HTTPie/{__version__}'
|
||||
def collect_messages(
|
||||
args: argparse.Namespace,
|
||||
config_dir: Path,
|
||||
request_body_read_callback: Callable[[bytes], None] = None,
|
||||
) -> Iterable[Union[requests.PreparedRequest, requests.Response]]:
|
||||
httpie_session = None
|
||||
httpie_session_headers = None
|
||||
@ -46,6 +49,7 @@ def collect_messages(
|
||||
request_kwargs = make_request_kwargs(
|
||||
args=args,
|
||||
base_headers=httpie_session_headers,
|
||||
request_body_read_callback=request_body_read_callback
|
||||
)
|
||||
send_kwargs = make_send_kwargs(args)
|
||||
send_kwargs_mergeable_from_env = make_send_kwargs_mergeable_from_env(args)
|
||||
@ -80,7 +84,10 @@ def collect_messages(
|
||||
prepped_url=prepared_request.url,
|
||||
)
|
||||
if args.compress and prepared_request.body:
|
||||
compress_body(prepared_request, always=args.compress > 1)
|
||||
compress_request(
|
||||
request=prepared_request,
|
||||
always=args.compress > 1,
|
||||
)
|
||||
response_count = 0
|
||||
expired_cookies = []
|
||||
while prepared_request:
|
||||
@ -137,22 +144,6 @@ def max_headers(limit):
|
||||
http.client._MAXHEADERS = orig
|
||||
|
||||
|
||||
def compress_body(request: requests.PreparedRequest, always: bool):
|
||||
deflater = zlib.compressobj()
|
||||
body_bytes = (
|
||||
request.body
|
||||
if isinstance(request.body, bytes)
|
||||
else request.body.encode()
|
||||
)
|
||||
deflated_data = deflater.compress(body_bytes)
|
||||
deflated_data += deflater.flush()
|
||||
is_economical = len(deflated_data) < len(body_bytes)
|
||||
if is_economical or always:
|
||||
request.body = deflated_data
|
||||
request.headers['Content-Encoding'] = 'deflate'
|
||||
request.headers['Content-Length'] = str(len(deflated_data))
|
||||
|
||||
|
||||
def build_requests_session(
|
||||
verify: bool,
|
||||
ssl_version: str = None,
|
||||
@ -251,7 +242,8 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
|
||||
|
||||
def make_request_kwargs(
|
||||
args: argparse.Namespace,
|
||||
base_headers: RequestHeadersDict = None
|
||||
base_headers: RequestHeadersDict = None,
|
||||
request_body_read_callback=lambda chunk: chunk
|
||||
) -> dict:
|
||||
"""
|
||||
Translate our `args` into `requests.Request` keyword arguments.
|
||||
@ -276,23 +268,25 @@ def make_request_kwargs(
|
||||
headers.update(args.headers)
|
||||
headers = finalize_headers(headers)
|
||||
|
||||
if args.form and (files or args.multipart):
|
||||
if (args.form and files) or args.multipart:
|
||||
data, headers['Content-Type'] = get_multipart_data_and_content_type(
|
||||
data=data,
|
||||
files=files,
|
||||
data=args.multipart_data,
|
||||
boundary=args.boundary,
|
||||
content_type=args.headers.get('Content-Type'),
|
||||
)
|
||||
files = None
|
||||
|
||||
kwargs = {
|
||||
'method': args.method.lower(),
|
||||
'url': args.url,
|
||||
'headers': headers,
|
||||
'data': data,
|
||||
'data': prepare_request_body(
|
||||
body=data,
|
||||
body_read_callback=request_body_read_callback,
|
||||
chunked=args.chunked,
|
||||
content_length_header_value=headers.get('Content-Length')
|
||||
),
|
||||
'auth': args.auth,
|
||||
'params': args.params,
|
||||
'files': files,
|
||||
'params': args.params.items(),
|
||||
}
|
||||
|
||||
return kwargs
|
||||
|
||||
@ -2,17 +2,24 @@ import argparse
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
from typing import List, Union
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import requests
|
||||
from pygments import __version__ as pygments_version
|
||||
from requests import __version__ as requests_version
|
||||
|
||||
from httpie import __version__ as httpie_version
|
||||
from httpie.cli.constants import (
|
||||
OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY,
|
||||
OUT_RESP_HEAD,
|
||||
)
|
||||
from httpie.client import collect_messages
|
||||
from httpie.context import Environment
|
||||
from httpie.downloads import Downloader
|
||||
from httpie.output.writer import write_message, write_stream
|
||||
from httpie.output.writer import (
|
||||
write_message,
|
||||
write_stream,
|
||||
)
|
||||
from httpie.plugins.registry import plugin_manager
|
||||
from httpie.status import ExitStatus, http_status_to_exit_status
|
||||
|
||||
@ -111,6 +118,22 @@ def main(
|
||||
return exit_status
|
||||
|
||||
|
||||
def get_output_options(
|
||||
args: argparse.Namespace,
|
||||
message: Union[requests.PreparedRequest, requests.Response]
|
||||
) -> Tuple[bool, bool]:
|
||||
return {
|
||||
requests.PreparedRequest: (
|
||||
OUT_REQ_HEAD in args.output_options,
|
||||
OUT_REQ_BODY in args.output_options,
|
||||
),
|
||||
requests.Response: (
|
||||
OUT_RESP_HEAD in args.output_options,
|
||||
OUT_RESP_BODY in args.output_options,
|
||||
),
|
||||
}[type(message)]
|
||||
|
||||
|
||||
def program(
|
||||
args: argparse.Namespace,
|
||||
env: Environment,
|
||||
@ -132,18 +155,57 @@ def program(
|
||||
)
|
||||
downloader.pre_request(args.headers)
|
||||
|
||||
initial_request = None
|
||||
final_response = None
|
||||
needs_separator = False
|
||||
|
||||
for message in collect_messages(args, env.config.directory):
|
||||
write_message(
|
||||
requests_message=message,
|
||||
env=env,
|
||||
args=args,
|
||||
def maybe_separate():
|
||||
nonlocal needs_separator
|
||||
if env.stdout.isatty() and needs_separator:
|
||||
needs_separator = False
|
||||
getattr(env.stdout, 'buffer', env.stdout).write(b'\n\n')
|
||||
|
||||
initial_request: Optional[requests.PreparedRequest] = None
|
||||
final_response: Optional[requests.Response] = None
|
||||
|
||||
def request_body_read_callback(chunk: bytes):
|
||||
should_pipe_to_stdout = (
|
||||
# Request body output desired
|
||||
OUT_REQ_BODY in args.output_options
|
||||
# & not `.read()` already pre-request (e.g., for compression)
|
||||
and initial_request
|
||||
# & non-EOF chunk
|
||||
and chunk
|
||||
)
|
||||
if isinstance(message, requests.PreparedRequest):
|
||||
if should_pipe_to_stdout:
|
||||
msg = requests.PreparedRequest()
|
||||
msg.is_body_upload_chunk = True
|
||||
msg.body = chunk
|
||||
msg.headers = initial_request.headers
|
||||
write_message(
|
||||
requests_message=msg,
|
||||
env=env,
|
||||
args=args,
|
||||
with_body=True,
|
||||
with_headers=False
|
||||
)
|
||||
|
||||
messages = collect_messages(
|
||||
args=args,
|
||||
config_dir=env.config.directory,
|
||||
request_body_read_callback=request_body_read_callback
|
||||
)
|
||||
for message in messages:
|
||||
maybe_separate()
|
||||
is_request = isinstance(message, requests.PreparedRequest)
|
||||
with_headers, with_body = get_output_options(
|
||||
args=args, message=message)
|
||||
if is_request:
|
||||
if not initial_request:
|
||||
initial_request = message
|
||||
is_streamed_upload = not args.offline and not isinstance(
|
||||
message.body, (str, bytes))
|
||||
if with_body:
|
||||
with_body = not is_streamed_upload
|
||||
needs_separator = is_streamed_upload
|
||||
else:
|
||||
final_response = message
|
||||
if args.check_status or downloader:
|
||||
@ -152,11 +214,20 @@ def program(
|
||||
follow=args.follow
|
||||
)
|
||||
if (not env.stdout_isatty
|
||||
and exit_status != ExitStatus.SUCCESS):
|
||||
and exit_status != ExitStatus.SUCCESS):
|
||||
env.log_error(
|
||||
f'HTTP {message.raw.status} {message.raw.reason}',
|
||||
level='warning'
|
||||
)
|
||||
write_message(
|
||||
requests_message=message,
|
||||
env=env,
|
||||
args=args,
|
||||
with_headers=with_headers,
|
||||
with_body=with_body,
|
||||
)
|
||||
|
||||
maybe_separate()
|
||||
|
||||
if downloader and exit_status == ExitStatus.SUCCESS:
|
||||
# Last response body download.
|
||||
@ -184,7 +255,7 @@ def program(
|
||||
downloader.failed()
|
||||
|
||||
if (not isinstance(args, list) and args.output_file
|
||||
and args.output_file_specified):
|
||||
and args.output_file_specified):
|
||||
args.output_file.close()
|
||||
|
||||
|
||||
|
||||
@ -86,7 +86,7 @@ class ColorFormatter(FormatterPlugin):
|
||||
lexer=lexer,
|
||||
formatter=self.formatter,
|
||||
)
|
||||
return body.strip()
|
||||
return body
|
||||
|
||||
def get_lexer_for_body(
|
||||
self, mime: str,
|
||||
|
||||
@ -4,7 +4,6 @@ from typing import Callable, Iterable, Union
|
||||
from httpie.context import Environment
|
||||
from httpie.models import HTTPMessage
|
||||
from httpie.output.processing import Conversion, Formatting
|
||||
from requests_toolbelt import MultipartEncoder
|
||||
|
||||
|
||||
BINARY_SUPPRESSED_NOTICE = (
|
||||
@ -15,14 +14,6 @@ BINARY_SUPPRESSED_NOTICE = (
|
||||
)
|
||||
|
||||
|
||||
LARGE_UPLOAD_SUPPRESSED_NOTICE = (
|
||||
b'\n'
|
||||
b'+--------------------------------------------------------+\n'
|
||||
b'| NOTE: large form upload data not shown in the terminal |\n'
|
||||
b'+--------------------------------------------------------+'
|
||||
)
|
||||
|
||||
|
||||
class DataSuppressedError(Exception):
|
||||
message = None
|
||||
|
||||
@ -33,13 +24,6 @@ class BinarySuppressedError(DataSuppressedError):
|
||||
message = BINARY_SUPPRESSED_NOTICE
|
||||
|
||||
|
||||
class LargeUploadSuppressedError(DataSuppressedError):
|
||||
"""An error indicating that the body is binary and won't be written,
|
||||
e.g., for terminal output)."""
|
||||
|
||||
message = LARGE_UPLOAD_SUPPRESSED_NOTICE
|
||||
|
||||
|
||||
class BaseStream:
|
||||
"""Base HTTP message output stream class."""
|
||||
|
||||
@ -203,8 +187,6 @@ class BufferedPrettyStream(PrettyStream):
|
||||
body = bytearray()
|
||||
|
||||
for chunk in self.msg.iter_body(self.CHUNK_SIZE):
|
||||
if isinstance(chunk, MultipartEncoder):
|
||||
raise LargeUploadSuppressedError()
|
||||
if not converter and b'\0' in chunk:
|
||||
converter = self.conversion.get_converter(self.mime)
|
||||
if not converter:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import argparse
|
||||
import errno
|
||||
from typing import Union, IO, TextIO, Tuple, Type
|
||||
from typing import IO, TextIO, Tuple, Type, Union
|
||||
|
||||
import requests
|
||||
|
||||
@ -8,12 +8,7 @@ from httpie.context import Environment
|
||||
from httpie.models import HTTPRequest, HTTPResponse
|
||||
from httpie.output.processing import Conversion, Formatting
|
||||
from httpie.output.streams import (
|
||||
RawStream, PrettyStream,
|
||||
BufferedPrettyStream, EncodedStream,
|
||||
BaseStream,
|
||||
)
|
||||
from httpie.cli.constants import (
|
||||
OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD,
|
||||
BaseStream, BufferedPrettyStream, EncodedStream, PrettyStream, RawStream,
|
||||
)
|
||||
|
||||
|
||||
@ -21,26 +16,18 @@ def write_message(
|
||||
requests_message: Union[requests.PreparedRequest, requests.Response],
|
||||
env: Environment,
|
||||
args: argparse.Namespace,
|
||||
with_headers=False,
|
||||
with_body=False,
|
||||
):
|
||||
output_options_by_message_type = {
|
||||
requests.PreparedRequest: {
|
||||
'with_headers': OUT_REQ_HEAD in args.output_options,
|
||||
'with_body': OUT_REQ_BODY in args.output_options,
|
||||
},
|
||||
requests.Response: {
|
||||
'with_headers': OUT_RESP_HEAD in args.output_options,
|
||||
'with_body': OUT_RESP_BODY in args.output_options,
|
||||
},
|
||||
}
|
||||
output_options = output_options_by_message_type[type(requests_message)]
|
||||
if not any(output_options.values()):
|
||||
if not (with_body or with_headers):
|
||||
return
|
||||
write_stream_kwargs = {
|
||||
'stream': build_output_stream_for_message(
|
||||
args=args,
|
||||
env=env,
|
||||
requests_message=requests_message,
|
||||
**output_options,
|
||||
with_body=with_body,
|
||||
with_headers=with_headers,
|
||||
),
|
||||
# NOTE: `env.stdout` will in fact be `stderr` with `--download`
|
||||
'outfile': env.stdout,
|
||||
@ -120,7 +107,8 @@ def build_output_stream_for_message(
|
||||
with_body=with_body,
|
||||
**stream_kwargs,
|
||||
)
|
||||
if env.stdout_isatty and with_body:
|
||||
if (env.stdout_isatty and with_body
|
||||
and not getattr(requests_message, 'is_body_upload_chunk', False)):
|
||||
# Ensure a blank line after the response body.
|
||||
# For terminal output only.
|
||||
yield b'\n\n'
|
||||
|
||||
@ -52,7 +52,7 @@ class HTTPieHTTPSAdapter(HTTPAdapter):
|
||||
verify: bool,
|
||||
ssl_version: str = None,
|
||||
ciphers: str = None,
|
||||
) -> ssl.SSLContext:
|
||||
) -> 'ssl.SSLContext':
|
||||
return create_urllib3_context(
|
||||
ciphers=ciphers,
|
||||
ssl_version=resolve_ssl_version(ssl_version),
|
||||
|
||||
@ -1,32 +1,129 @@
|
||||
from typing import Tuple, Union
|
||||
import zlib
|
||||
from typing import Callable, IO, Iterable, Tuple, Union
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import requests
|
||||
from requests.utils import super_len
|
||||
from requests_toolbelt import MultipartEncoder
|
||||
|
||||
from httpie.cli.dicts import RequestDataDict, RequestFilesDict
|
||||
from httpie.cli.dicts import MultipartRequestDataDict, RequestDataDict
|
||||
|
||||
|
||||
# Multipart uploads smaller than this size gets buffered (otherwise streamed).
|
||||
# NOTE: Unbuffered upload requests cannot be displayed on the terminal.
|
||||
UPLOAD_BUFFER = 1024 * 100
|
||||
class ChunkedUploadStream:
|
||||
def __init__(self, stream: Iterable, callback: Callable):
|
||||
self.callback = callback
|
||||
self.stream = stream
|
||||
|
||||
def __iter__(self) -> Iterable[Union[str, bytes]]:
|
||||
for chunk in self.stream:
|
||||
self.callback(chunk)
|
||||
yield chunk
|
||||
|
||||
|
||||
class ChunkedMultipartUploadStream:
|
||||
chunk_size = 100 * 1024
|
||||
|
||||
def __init__(self, encoder: MultipartEncoder):
|
||||
self.encoder = encoder
|
||||
|
||||
def __iter__(self) -> Iterable[Union[str, bytes]]:
|
||||
while True:
|
||||
chunk = self.encoder.read(self.chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
|
||||
def prepare_request_body(
|
||||
body: Union[str, bytes, IO, MultipartEncoder, RequestDataDict],
|
||||
body_read_callback: Callable[[bytes], bytes],
|
||||
content_length_header_value: int = None,
|
||||
chunked=False,
|
||||
) -> Union[str, bytes, IO, MultipartEncoder, ChunkedUploadStream]:
|
||||
if isinstance(body, RequestDataDict):
|
||||
body = urlencode(body, doseq=True)
|
||||
|
||||
if not hasattr(body, 'read'):
|
||||
if chunked:
|
||||
body = ChunkedUploadStream(
|
||||
# Pass the entire body as one chunk.
|
||||
stream=(chunk.encode() for chunk in [body]),
|
||||
callback=body_read_callback,
|
||||
)
|
||||
else:
|
||||
# File-like object.
|
||||
|
||||
if not super_len(body):
|
||||
# Zero-length -> assume stdin.
|
||||
if content_length_header_value is None and not chunked:
|
||||
#
|
||||
# Read the whole stdin to determine `Content-Length`.
|
||||
#
|
||||
# TODO: Instead of opt-in --chunked, consider making
|
||||
# `Transfer-Encoding: chunked` for STDIN opt-out via
|
||||
# something like --no-chunked.
|
||||
# This would be backwards-incompatible so wait until v3.0.0.
|
||||
#
|
||||
body = body.read()
|
||||
else:
|
||||
orig_read = body.read
|
||||
|
||||
def new_read(*args):
|
||||
chunk = orig_read(*args)
|
||||
body_read_callback(chunk)
|
||||
return chunk
|
||||
|
||||
body.read = new_read
|
||||
|
||||
if chunked:
|
||||
if isinstance(body, MultipartEncoder):
|
||||
body = ChunkedMultipartUploadStream(
|
||||
encoder=body,
|
||||
)
|
||||
else:
|
||||
body = ChunkedUploadStream(
|
||||
stream=body,
|
||||
callback=body_read_callback,
|
||||
)
|
||||
|
||||
return body
|
||||
|
||||
|
||||
def get_multipart_data_and_content_type(
|
||||
data: RequestDataDict,
|
||||
files: RequestFilesDict,
|
||||
data: MultipartRequestDataDict,
|
||||
boundary: str = None,
|
||||
content_type: str = None,
|
||||
) -> Tuple[Union[MultipartEncoder, bytes], str]:
|
||||
fields = list(data.items()) + list(files.items())
|
||||
) -> Tuple[MultipartEncoder, str]:
|
||||
encoder = MultipartEncoder(
|
||||
fields=fields,
|
||||
fields=data.items(),
|
||||
boundary=boundary,
|
||||
)
|
||||
if content_type:
|
||||
content_type = content_type.strip() # maybe auto-strip all headers somewhere
|
||||
content_type = content_type.strip()
|
||||
if 'boundary=' not in content_type:
|
||||
content_type = f'{content_type}; boundary={encoder.boundary_value}'
|
||||
else:
|
||||
content_type = encoder.content_type
|
||||
|
||||
data = encoder.to_string() if encoder.len < UPLOAD_BUFFER else encoder
|
||||
data = encoder
|
||||
return data, content_type
|
||||
|
||||
|
||||
def compress_request(
|
||||
request: requests.PreparedRequest,
|
||||
always: bool,
|
||||
):
|
||||
deflater = zlib.compressobj()
|
||||
if isinstance(request.body, str):
|
||||
body_bytes = request.body.encode()
|
||||
elif hasattr(request.body, 'read'):
|
||||
body_bytes = request.body.read()
|
||||
else:
|
||||
body_bytes = request.body
|
||||
deflated_data = deflater.compress(body_bytes)
|
||||
deflated_data += deflater.flush()
|
||||
is_economical = len(deflated_data) < len(body_bytes)
|
||||
if is_economical or always:
|
||||
request.body = deflated_data
|
||||
request.headers['Content-Encoding'] = 'deflate'
|
||||
request.headers['Content-Length'] = str(len(deflated_data))
|
||||
|
||||
33
tests/fixtures/__init__.py
vendored
33
tests/fixtures/__init__.py
vendored
@ -1,6 +1,5 @@
|
||||
"""Test data"""
|
||||
from os import path
|
||||
import codecs
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def patharg(path):
|
||||
@ -9,32 +8,24 @@ def patharg(path):
|
||||
even in Windows paths.
|
||||
|
||||
"""
|
||||
return path.replace('\\', '\\\\\\')
|
||||
return str(path).replace('\\', '\\\\\\')
|
||||
|
||||
|
||||
FIXTURES_ROOT = path.join(path.abspath(path.dirname(__file__)))
|
||||
FILE_PATH = path.join(FIXTURES_ROOT, 'test.txt')
|
||||
JSON_FILE_PATH = path.join(FIXTURES_ROOT, 'test.json')
|
||||
BIN_FILE_PATH = path.join(FIXTURES_ROOT, 'test.bin')
|
||||
|
||||
FIXTURES_ROOT = Path(__file__).parent
|
||||
FILE_PATH = FIXTURES_ROOT / 'test.txt'
|
||||
JSON_FILE_PATH = FIXTURES_ROOT / 'test.json'
|
||||
BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin'
|
||||
|
||||
FILE_PATH_ARG = patharg(FILE_PATH)
|
||||
BIN_FILE_PATH_ARG = patharg(BIN_FILE_PATH)
|
||||
JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH)
|
||||
|
||||
|
||||
with codecs.open(FILE_PATH, encoding='utf8') as f:
|
||||
# Strip because we don't want new lines in the data so that we can
|
||||
# easily count occurrences also when embedded in JSON (where the new
|
||||
# line would be escaped).
|
||||
FILE_CONTENT = f.read().strip()
|
||||
# Strip because we don't want new lines in the data so that we can
|
||||
# easily count occurrences also when embedded in JSON (where the new
|
||||
# line would be escaped).
|
||||
FILE_CONTENT = FILE_PATH.read_text().strip()
|
||||
|
||||
|
||||
with codecs.open(JSON_FILE_PATH, encoding='utf8') as f:
|
||||
JSON_FILE_CONTENT = f.read()
|
||||
|
||||
|
||||
with open(BIN_FILE_PATH, 'rb') as f:
|
||||
BIN_FILE_CONTENT = f.read()
|
||||
|
||||
JSON_FILE_CONTENT = JSON_FILE_PATH.read_text()
|
||||
BIN_FILE_CONTENT = BIN_FILE_PATH.read_bytes()
|
||||
UNICODE = FILE_CONTENT
|
||||
|
||||
@ -15,7 +15,7 @@ from httpie.cli import constants
|
||||
from httpie.cli.definition import parser
|
||||
from httpie.cli.argtypes import KeyValueArg, KeyValueArgType
|
||||
from httpie.cli.requestitems import RequestItems
|
||||
from utils import HTTP_OK, MockEnvironment, http
|
||||
from utils import HTTP_OK, MockEnvironment, StdinBytesIO, http
|
||||
|
||||
|
||||
class TestItemParsing:
|
||||
@ -312,10 +312,11 @@ class TestNoOptions:
|
||||
class TestStdin:
|
||||
|
||||
def test_ignore_stdin(self, httpbin):
|
||||
with open(FILE_PATH) as f:
|
||||
env = MockEnvironment(stdin=f, stdin_isatty=False)
|
||||
r = http('--ignore-stdin', '--verbose', httpbin.url + '/get',
|
||||
env=env)
|
||||
env = MockEnvironment(
|
||||
stdin=StdinBytesIO(FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
)
|
||||
r = http('--ignore-stdin', '--verbose', httpbin.url + '/get', env=env)
|
||||
assert HTTP_OK in r
|
||||
assert 'GET /get HTTP' in r, "Don't default to POST."
|
||||
assert FILE_CONTENT not in r, "Don't send stdin data."
|
||||
|
||||
@ -12,7 +12,8 @@ import base64
|
||||
import zlib
|
||||
|
||||
from fixtures import FILE_PATH, FILE_CONTENT
|
||||
from utils import http, HTTP_OK, MockEnvironment
|
||||
from httpie.status import ExitStatus
|
||||
from utils import StdinBytesIO, http, HTTP_OK, MockEnvironment
|
||||
|
||||
|
||||
def assert_decompressed_equal(base64_compressed_data, expected_str):
|
||||
@ -27,6 +28,20 @@ def assert_decompressed_equal(base64_compressed_data, expected_str):
|
||||
assert actual_str == expected_str
|
||||
|
||||
|
||||
def test_cannot_combine_compress_with_chunked(httpbin):
|
||||
r = http('--compress', '--chunked', httpbin.url + '/get',
|
||||
tolerate_error_exit_status=True)
|
||||
assert r.exit_status == ExitStatus.ERROR
|
||||
assert 'cannot combine --compress and --chunked' in r.stderr
|
||||
|
||||
|
||||
def test_cannot_combine_compress_with_multipart(httpbin):
|
||||
r = http('--compress', '--multipart', httpbin.url + '/get',
|
||||
tolerate_error_exit_status=True)
|
||||
assert r.exit_status == ExitStatus.ERROR
|
||||
assert 'cannot combine --compress and --multipart' in r.stderr
|
||||
|
||||
|
||||
def test_compress_skip_negative_ratio(httpbin_both):
|
||||
r = http(
|
||||
'--compress',
|
||||
@ -78,15 +93,17 @@ def test_compress_form(httpbin_both):
|
||||
|
||||
|
||||
def test_compress_stdin(httpbin_both):
|
||||
with open(FILE_PATH) as f:
|
||||
env = MockEnvironment(stdin=f, stdin_isatty=False)
|
||||
r = http(
|
||||
'--compress',
|
||||
'--compress',
|
||||
'PATCH',
|
||||
httpbin_both + '/patch',
|
||||
env=env,
|
||||
)
|
||||
env = MockEnvironment(
|
||||
stdin=StdinBytesIO(FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
)
|
||||
r = http(
|
||||
'--compress',
|
||||
'--compress',
|
||||
'PATCH',
|
||||
httpbin_both + '/patch',
|
||||
env=env,
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert r.json['headers']['Content-Encoding'] == 'deflate'
|
||||
assert_decompressed_equal(r.json['data'], FILE_CONTENT.strip())
|
||||
@ -100,7 +117,7 @@ def test_compress_file(httpbin_both):
|
||||
'--compress',
|
||||
'PUT',
|
||||
httpbin_both + '/put',
|
||||
'file@' + FILE_PATH,
|
||||
f'file@{FILE_PATH}',
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert r.json['headers']['Content-Encoding'] == 'deflate'
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
Tests for the provided defaults regarding HTTP method, and --json vs. --form.
|
||||
|
||||
"""
|
||||
from io import BytesIO
|
||||
|
||||
from httpie.client import JSON_ACCEPT
|
||||
from utils import MockEnvironment, http, HTTP_OK
|
||||
from fixtures import FILE_PATH
|
||||
@ -44,9 +46,11 @@ class TestImplicitHTTPMethod:
|
||||
assert r.json['form'] == {'foo': 'bar'}
|
||||
|
||||
def test_implicit_POST_stdin(self, httpbin):
|
||||
with open(FILE_PATH) as f:
|
||||
env = MockEnvironment(stdin_isatty=False, stdin=f)
|
||||
r = http('--form', httpbin.url + '/post', env=env)
|
||||
env = MockEnvironment(
|
||||
stdin_isatty=False,
|
||||
stdin=BytesIO(FILE_PATH.read_bytes())
|
||||
)
|
||||
r = http('--form', httpbin.url + '/post', env=env)
|
||||
assert HTTP_OK in r
|
||||
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
"""High-level tests."""
|
||||
import io
|
||||
import sys
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
@ -8,7 +9,7 @@ import httpie.__main__
|
||||
from httpie.context import Environment
|
||||
from httpie.status import ExitStatus
|
||||
from httpie.cli.exceptions import ParseError
|
||||
from utils import MockEnvironment, http, HTTP_OK
|
||||
from utils import MockEnvironment, StdinBytesIO, http, HTTP_OK
|
||||
from fixtures import FILE_PATH, FILE_CONTENT
|
||||
|
||||
import httpie
|
||||
@ -103,15 +104,17 @@ def test_POST_form_multiple_values(httpbin_both):
|
||||
|
||||
|
||||
def test_POST_stdin(httpbin_both):
|
||||
with open(FILE_PATH) as f:
|
||||
env = MockEnvironment(stdin=f, stdin_isatty=False)
|
||||
r = http('--form', 'POST', httpbin_both + '/post', env=env)
|
||||
env = MockEnvironment(
|
||||
stdin=StdinBytesIO(FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
)
|
||||
r = http('--form', 'POST', httpbin_both + '/post', env=env)
|
||||
assert HTTP_OK in r
|
||||
assert FILE_CONTENT in r
|
||||
|
||||
|
||||
def test_POST_file(httpbin_both):
|
||||
r = http('--form', 'POST', httpbin_both + '/post', 'file@' + FILE_PATH)
|
||||
r = http('--form', 'POST', httpbin_both + '/post', f'file@{FILE_PATH}')
|
||||
assert HTTP_OK in r
|
||||
assert FILE_CONTENT in r
|
||||
|
||||
@ -126,10 +129,10 @@ def test_form_POST_file_redirected_stdin(httpbin):
|
||||
'--form',
|
||||
'POST',
|
||||
httpbin + '/post',
|
||||
'file@' + FILE_PATH,
|
||||
f'file@{FILE_PATH}',
|
||||
tolerate_error_exit_status=True,
|
||||
env=MockEnvironment(
|
||||
stdin=f,
|
||||
stdin=StdinBytesIO(FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
),
|
||||
)
|
||||
|
||||
@ -2,7 +2,7 @@ import pytest
|
||||
|
||||
from httpie.compat import is_windows
|
||||
from httpie.output.streams import BINARY_SUPPRESSED_NOTICE
|
||||
from utils import http, MockEnvironment
|
||||
from utils import StdinBytesIO, http, MockEnvironment
|
||||
from fixtures import BIN_FILE_CONTENT, BIN_FILE_PATH
|
||||
|
||||
|
||||
@ -13,32 +13,37 @@ from fixtures import BIN_FILE_CONTENT, BIN_FILE_PATH
|
||||
reason='Pretty redirect not supported under Windows')
|
||||
def test_pretty_redirected_stream(httpbin):
|
||||
"""Test that --stream works with prettified redirected output."""
|
||||
with open(BIN_FILE_PATH, 'rb') as f:
|
||||
env = MockEnvironment(colors=256, stdin=f,
|
||||
stdin_isatty=False,
|
||||
stdout_isatty=False)
|
||||
r = http('--verbose', '--pretty=all', '--stream', 'GET',
|
||||
httpbin.url + '/get', env=env)
|
||||
env = MockEnvironment(
|
||||
colors=256,
|
||||
stdin=StdinBytesIO(BIN_FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
stdout_isatty=False,
|
||||
)
|
||||
r = http('--verbose', '--pretty=all', '--stream', 'GET',
|
||||
httpbin.url + '/get', env=env)
|
||||
assert BINARY_SUPPRESSED_NOTICE.decode() in r
|
||||
|
||||
|
||||
def test_encoded_stream(httpbin):
|
||||
"""Test that --stream works with non-prettified
|
||||
redirected terminal output."""
|
||||
with open(BIN_FILE_PATH, 'rb') as f:
|
||||
env = MockEnvironment(stdin=f, stdin_isatty=False)
|
||||
r = http('--pretty=none', '--stream', '--verbose', 'GET',
|
||||
httpbin.url + '/get', env=env)
|
||||
env = MockEnvironment(
|
||||
stdin=StdinBytesIO(BIN_FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
)
|
||||
r = http('--pretty=none', '--stream', '--verbose', 'GET',
|
||||
httpbin.url + '/get', env=env)
|
||||
assert BINARY_SUPPRESSED_NOTICE.decode() in r
|
||||
|
||||
|
||||
def test_redirected_stream(httpbin):
|
||||
"""Test that --stream works with non-prettified
|
||||
redirected terminal output."""
|
||||
with open(BIN_FILE_PATH, 'rb') as f:
|
||||
env = MockEnvironment(stdout_isatty=False,
|
||||
stdin_isatty=False,
|
||||
stdin=f)
|
||||
r = http('--pretty=none', '--stream', '--verbose', 'GET',
|
||||
httpbin.url + '/get', env=env)
|
||||
env = MockEnvironment(
|
||||
stdout_isatty=False,
|
||||
stdin_isatty=False,
|
||||
stdin=StdinBytesIO(BIN_FILE_PATH.read_bytes()),
|
||||
)
|
||||
r = http('--pretty=none', '--stream', '--verbose', 'GET',
|
||||
httpbin.url + '/get', env=env)
|
||||
assert BIN_FILE_CONTENT in r
|
||||
|
||||
@ -1,16 +1,57 @@
|
||||
import os
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from httpie.cli.exceptions import ParseError
|
||||
from httpie.client import FORM_CONTENT_TYPE
|
||||
from httpie.output.streams import LARGE_UPLOAD_SUPPRESSED_NOTICE
|
||||
from httpie.status import ExitStatus
|
||||
from utils import MockEnvironment, http, HTTP_OK
|
||||
from utils import (
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT, MockEnvironment, StdinBytesIO, http,
|
||||
HTTP_OK,
|
||||
)
|
||||
from fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT
|
||||
|
||||
|
||||
def test_chunked_json():
|
||||
r = http(
|
||||
'--verbose',
|
||||
'--chunked',
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT + '/post',
|
||||
'hello=world',
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert 'Transfer-Encoding: chunked' in r
|
||||
assert r.count('hello') == 3
|
||||
|
||||
|
||||
def test_chunked_form():
|
||||
r = http(
|
||||
'--verbose',
|
||||
'--chunked',
|
||||
'--form',
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT + '/post',
|
||||
'hello=world',
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert 'Transfer-Encoding: chunked' in r
|
||||
assert r.count('hello') == 2
|
||||
|
||||
|
||||
def test_chunked_stdin():
|
||||
r = http(
|
||||
'--verbose',
|
||||
'--chunked',
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT + '/post',
|
||||
env=MockEnvironment(
|
||||
stdin=StdinBytesIO(FILE_PATH.read_bytes()),
|
||||
stdin_isatty=False,
|
||||
)
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert 'Transfer-Encoding: chunked' in r
|
||||
assert r.count(FILE_CONTENT) == 2
|
||||
|
||||
|
||||
class TestMultipartFormDataFileUpload:
|
||||
|
||||
def test_non_existent_file_raises_parse_error(self, httpbin):
|
||||
@ -55,19 +96,6 @@ class TestMultipartFormDataFileUpload:
|
||||
assert r.count(FILE_CONTENT) == 2
|
||||
assert 'Content-Type: image/vnd.microsoft.icon' in r
|
||||
|
||||
@mock.patch('httpie.uploads.UPLOAD_BUFFER', 0)
|
||||
def test_large_upload_display_suppressed(self, httpbin):
|
||||
r = http(
|
||||
'--form',
|
||||
'--verbose',
|
||||
httpbin.url + '/post',
|
||||
f'test-file@{FILE_PATH_ARG}',
|
||||
'foo=bar',
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert r.count(FILE_CONTENT) == 1
|
||||
assert LARGE_UPLOAD_SUPPRESSED_NOTICE.decode() in r
|
||||
|
||||
def test_form_no_files_urlencoded(self, httpbin):
|
||||
r = http(
|
||||
'--form',
|
||||
@ -79,9 +107,8 @@ class TestMultipartFormDataFileUpload:
|
||||
assert HTTP_OK in r
|
||||
assert FORM_CONTENT_TYPE in r
|
||||
|
||||
def test_form_no_files_multipart(self, httpbin):
|
||||
def test_multipart(self, httpbin):
|
||||
r = http(
|
||||
'--form',
|
||||
'--verbose',
|
||||
'--multipart',
|
||||
httpbin.url + '/post',
|
||||
@ -97,7 +124,6 @@ class TestMultipartFormDataFileUpload:
|
||||
r = http(
|
||||
'--print=HB',
|
||||
'--check-status',
|
||||
'--form',
|
||||
'--multipart',
|
||||
f'--boundary={boundary}',
|
||||
httpbin.url + '/post',
|
||||
@ -112,7 +138,6 @@ class TestMultipartFormDataFileUpload:
|
||||
r = http(
|
||||
'--print=HB',
|
||||
'--check-status',
|
||||
'--form',
|
||||
'--multipart',
|
||||
f'--boundary={boundary}',
|
||||
httpbin.url + '/post',
|
||||
@ -128,7 +153,6 @@ class TestMultipartFormDataFileUpload:
|
||||
boundary_in_header = 'HEADER_BOUNDARY'
|
||||
boundary_in_body = 'BODY_BOUNDARY'
|
||||
r = http(
|
||||
'--form',
|
||||
'--print=HB',
|
||||
'--check-status',
|
||||
'--multipart',
|
||||
@ -141,6 +165,20 @@ class TestMultipartFormDataFileUpload:
|
||||
assert f'multipart/magic; boundary={boundary_in_header}' in r
|
||||
assert r.count(boundary_in_body) == 3
|
||||
|
||||
def test_multipart_chunked(self, httpbin):
|
||||
r = http(
|
||||
'--verbose',
|
||||
'--multipart',
|
||||
'--chunked',
|
||||
# '--offline',
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT + '/post',
|
||||
'AAA=AAA',
|
||||
)
|
||||
assert 'Transfer-Encoding: chunked' in r
|
||||
assert 'multipart/form-data' in r
|
||||
assert 'name="AAA"' in r # in request
|
||||
assert '"AAA": "AAA"', r # in response
|
||||
|
||||
|
||||
class TestRequestBodyFromFilePath:
|
||||
"""
|
||||
@ -149,12 +187,26 @@ class TestRequestBodyFromFilePath:
|
||||
"""
|
||||
|
||||
def test_request_body_from_file_by_path(self, httpbin):
|
||||
r = http('--verbose',
|
||||
'POST', httpbin.url + '/post', '@' + FILE_PATH_ARG)
|
||||
r = http(
|
||||
'--verbose',
|
||||
'POST', httpbin.url + '/post',
|
||||
'@' + FILE_PATH_ARG,
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert FILE_CONTENT in r, r
|
||||
assert r.count(FILE_CONTENT) == 2
|
||||
assert '"Content-Type": "text/plain"' in r
|
||||
|
||||
def test_request_body_from_file_by_path_chunked(self, httpbin):
|
||||
r = http(
|
||||
'--verbose', '--chunked',
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT + '/post',
|
||||
'@' + FILE_PATH_ARG,
|
||||
)
|
||||
assert HTTP_OK in r
|
||||
assert 'Transfer-Encoding: chunked' in r
|
||||
assert '"Content-Type": "text/plain"' in r
|
||||
assert r.count(FILE_CONTENT) == 2
|
||||
|
||||
def test_request_body_from_file_by_path_with_explicit_content_type(
|
||||
self, httpbin):
|
||||
r = http('--verbose',
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
# coding=utf-8
|
||||
"""Utilities for HTTPie test suite."""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
@ -14,6 +14,12 @@ from httpie.context import Environment
|
||||
from httpie.core import main
|
||||
|
||||
|
||||
# pytest-httpbin currently does not support chunked requests:
|
||||
# <https://github.com/kevin1024/pytest-httpbin/issues/33>
|
||||
# <https://github.com/kevin1024/pytest-httpbin/issues/28>
|
||||
HTTPBIN_WITH_CHUNKED_SUPPORT = 'http://httpbin.org'
|
||||
|
||||
|
||||
TESTS_ROOT = Path(__file__).parent
|
||||
CRLF = '\r\n'
|
||||
COLOR = '\x1b['
|
||||
@ -36,6 +42,11 @@ def add_auth(url, auth):
|
||||
return proto + '://' + auth + '@' + rest
|
||||
|
||||
|
||||
class StdinBytesIO(BytesIO):
|
||||
"""To be used for `MockEnvironment.stdin`"""
|
||||
len = 0 # See `prepare_request_body()`
|
||||
|
||||
|
||||
class MockEnvironment(Environment):
|
||||
"""Environment subclass with reasonable defaults for testing."""
|
||||
colors = 0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user