diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 27f862b3..84391811 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,14 +8,17 @@ This project adheres to `Semantic Versioning `_. `2.3.0-dev`_ (unreleased) ------------------------- + * Added support for multipart upload streaming (`#684`_). +* Added support for body-from-file upload streaming (``http httpbin.org/post @file``). +* Added ``--chunked`` to allow chunked transfer encoding. +* Added ``--multipart`` to allow ``multipart/form-data`` encoding for non-file ``--form`` requests as well. +* Added ``--boundary`` to allow a custom boundary string for ``multipart/form-data`` requests. * Added support for combining cookies specified on the CLI and in a session file (`#932`_). * Added out of the box SOCKS support with no extra installation (`#904`_). * Added ``--quiet, -q`` flag to enforce silent behaviour. -* Added ``--multipart`` to allow ``multipart/form-data`` encoding for non-file ``--form`` requests as well. -* Added ``--boundary`` to allow a custom boundary string for ``multipart/form-data`` requests. -* Removed Tox testing entirely (`#943`_). * Fixed the handling of invalid ``expires`` dates in ``Set-Cookie`` headers (`#963`_). +* Removed Tox testing entirely (`#943`_). `2.2.0`_ (2020-06-18) diff --git a/README.rst b/README.rst index ca9a057f..cf9604bf 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ They use simple and natural syntax and provide formatted and colorized output. .. class:: no-web no-pdf - .. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.gif + .. image:: https://raw.githubusercontent.com/httpie/httpie/master/httpie.gif :alt: HTTPie in action :width: 100% :align: center @@ -35,7 +35,7 @@ where you can select your corresponding HTTPie version as well as run examples d browser using a `termible.io `_ embedded terminal. If you are reading this on GitHub, then this text covers the current *development* version. You are invited to submit fixes and improvements to the the docs by editing -`README.rst `_. +`README.rst `_. Main features @@ -58,7 +58,7 @@ Main features .. class:: no-web - .. image:: https://raw.githubusercontent.com/jakubroztocil/httpie/master/httpie.png + .. image:: https://raw.githubusercontent.com/httpie/httpie/master/httpie.png :alt: HTTPie compared to cURL :width: 100% :align: center @@ -167,11 +167,11 @@ Otherwise with ``pip``: .. code-block:: bash - $ pip install --upgrade https://github.com/jakubroztocil/httpie/archive/master.tar.gz + $ pip install --upgrade https://github.com/httpie/httpie/archive/master.tar.gz Verify that now we have the -`current development version identifier `_ +`current development version identifier `_ with the ``-dev`` suffix, for example: .. code-block:: bash @@ -234,12 +234,12 @@ Build and print a request without sending it using `offline mode`_: Use `GitHub API`_ to post a comment on an -`issue `_ +`issue `_ with `authentication`_: .. code-block:: bash - $ http -a USERNAME POST https://api.github.com/repos/jakubroztocil/httpie/issues/83/comments body='HTTPie is awesome! :heart:' + $ http -a USERNAME POST https://api.github.com/repos/httpie/httpie/issues/83/comments body='HTTPie is awesome! :heart:' Upload a file using `redirected input`_: @@ -471,20 +471,23 @@ their type is distinguished only by the separator used: | ``name==value`` | string parameter to the URL. | | | The ``==`` separator is used. | +-----------------------+-----------------------------------------------------+ -| Data Fields | Request data fields to be serialized as a JSON | -| ``field=value``, | object (default), or to be form-encoded | -| ``field=@file.txt`` | (``--form, -f``). | +| Data Fields | Request data fields to be serialized as a JSON | +| ``field=value``, | object (default), to be form-encoded | +| ``field=@file.txt`` | (with ``--form, -f``), or to be serialized as | +| | ``multipart/form-data`` (with ``--multipart``). | +-----------------------+-----------------------------------------------------+ -| Raw JSON fields | Useful when sending JSON and one or | -| ``field:=json``, | more fields need to be a ``Boolean``, ``Number``, | -| ``field:=@file.json`` | nested ``Object``, or an ``Array``, e.g., | +| Raw JSON fields | Useful when sending JSON and one or | +| ``field:=json``, | more fields need to be a ``Boolean``, ``Number``, | +| ``field:=@file.json`` | nested ``Object``, or an ``Array``, e.g., | | | ``meals:='["ham","spam"]'`` or ``pies:=[1,2,3]`` | | | (note the quotes). | +-----------------------+-----------------------------------------------------+ -| Form File Fields | Only available with ``--form, -f``. | -| ``field@/dir/file`` | For example ``screenshot@~/Pictures/img.png``. | -| | The presence of a file field results | -| | in a ``multipart/form-data`` request. | +| Fields upload fields | Only available with ``--form, -f`` and | +| ``field@/dir/file`` | ``--multipart``. | +| ``field@file;type`` | For example ``screenshot@~/Pictures/img.png``, or | +| | ``'cv@cv.txt;text/markdown'``. | +| | With ``--form``, the presence of a file field | +| | results in a ``--multipart`` request. | +-----------------------+-----------------------------------------------------+ @@ -714,12 +717,10 @@ To perform a ``multipart/form-data`` request even without any files, use world --c31279ab254f40aeb06df32b433cbccb-- -Larger multipart uploads are always streamed to avoid memory issues. -Additionally, the display of the request body on the terminal is suppressed -for larger uploads. +File uploads are always streamed to avoid memory issues with large files. -By default, HTTPie uses a random unique string as the boundary but you can use -``--boundary`` to specify a custom string instead: +By default, HTTPie uses a random unique string as the multipart boundary +but you can use ``--boundary`` to specify a custom string instead: .. code-block:: bash @@ -739,8 +740,8 @@ By default, HTTPie uses a random unique string as the boundary but you can use --xoxo-- If you specify a custom ``Content-Type`` header without including the boundary -bit, HTTPie will add the boundary value (specified or generated) to the header -automatically: +bit, HTTPie will add the boundary value (explicitly specified or auto-generated) +to the header automatically: .. code-block:: bash @@ -1347,9 +1348,14 @@ Redirected Input ================ The universal method for passing request data is through redirected ``stdin`` -(standard input)—piping. Such data is buffered and then with no further -processing used as the request body. There are multiple useful ways to use -piping: +(standard input)—piping. + +By default, ``stdin`` data is buffered and then with no further processing +used as the request body. If you provide ``Content-Length``, then the request +body is streamed without buffering. You can also use ``--chunked`` to enable +streaming via `chunked transfer encoding`_. + +There are multiple useful ways to use piping: Redirect from a file: @@ -1383,7 +1389,7 @@ You can even pipe web services together using HTTPie: .. code-block:: bash - $ http GET https://api.github.com/repos/jakubroztocil/httpie | http POST httpbin.org/post + $ http GET https://api.github.com/repos/httpie/httpie | http POST httpbin.org/post You can use ``cat`` to enter multiline data on the terminal: @@ -1438,6 +1444,33 @@ verbatim contents of that XML file with ``Content-Type: application/xml``: $ http PUT httpbin.org/put @files/data.xml +File uploads are always streamed to avoid memory issues with large files. + + +Chunked transfer encoding +========================= + +For any request, you can use the ``--chunked`` flag to instruct HTTPie to use + ``Transfer-Encoding: chunked``: + + +.. code-block:: bash + + $ http --chunked PUT httpbin.org/put hello=world + +.. code-block:: bash + + $ http --chunked --multipart PUT httpbin.org/put hello=world foo@files/data.xml + +.. code-block:: bash + + $ http --chunked httpbin.org/post @files/data.xml + +.. code-block:: bash + + $ cat files/data.xml | http --chunked httpbin.org/post + + Terminal output =============== @@ -1585,7 +1618,7 @@ is being saved to a file. .. code-block:: bash - $ http --download https://github.com/jakubroztocil/httpie/archive/master.tar.gz + $ http --download https://github.com/httpie/httpie/archive/master.tar.gz .. code-block:: http @@ -1627,7 +1660,7 @@ headers and progress are still shown in the terminal: .. code-block:: bash - $ http -d https://github.com/jakubroztocil/httpie/archive/master.tar.gz | tar zxf - + $ http -d https://github.com/httpie/httpie/archive/master.tar.gz | tar zxf - @@ -2108,27 +2141,27 @@ Alternatives Contributing ------------ -See `CONTRIBUTING.rst `_. +See `CONTRIBUTING.rst `_. Change log ---------- -See `CHANGELOG `_. +See `CHANGELOG `_. Artwork ------- * `Logo `_ by `Cláudia Delgado `_. -* `Animation `_ by `Allen Smith `_ of GitHub. +* `Animation `_ by `Allen Smith `_ of GitHub. Licence ------- -BSD-3-Clause: `LICENSE `_. +BSD-3-Clause: `LICENSE `_. @@ -2141,7 +2174,7 @@ have contributed. .. _pip: https://pip.pypa.io/en/stable/installing/ .. _GitHub API: https://developer.github.com/v3/issues/comments/#create-a-comment -.. _these fine people: https://github.com/jakubroztocil/httpie/contributors +.. _these fine people: https://github.com/httpie/httpie/contributors .. _Jakub Roztocil: https://roztocil.co .. _@jakubroztocil: https://twitter.com/jakubroztocil @@ -2154,12 +2187,12 @@ have contributed. :target: https://pypi.python.org/pypi/httpie :alt: Latest version released on PyPi -.. |coverage| image:: https://img.shields.io/codecov/c/github/jakubroztocil/httpie?style=flat-square - :target: https://codecov.io/gh/jakubroztocil/httpie +.. |coverage| image:: https://img.shields.io/codecov/c/github/httpie/httpie?style=flat-square + :target: https://codecov.io/gh/httpie/httpie :alt: Test coverage -.. |build| image:: https://github.com/jakubroztocil/httpie/workflows/Build/badge.svg - :target: https://github.com/jakubroztocil/httpie/actions +.. |build| image:: https://github.com/httpie/httpie/workflows/Build/badge.svg + :target: https://github.com/httpie/httpie/actions :alt: Build status of the master branch on Mac/Linux/Windows .. |gitter| image:: https://img.shields.io/gitter/room/jkbrzt/httpie.svg?style=flat-square diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 3257488f..720e70b1 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -17,7 +17,7 @@ from httpie.cli.argtypes import ( from httpie.cli.constants import ( HTTP_GET, HTTP_POST, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT, OUTPUT_OPTIONS_DEFAULT_OFFLINE, OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED, - OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, RequestContentType, + OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, RequestType, SEPARATOR_CREDENTIALS, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE, ) @@ -83,7 +83,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser): ) # Arguments processing and environment setup. self._apply_no_options(no_options) - self._process_request_content_type() + self._process_request_type() self._process_download_options() self._setup_standard_streams() self._process_output_options() @@ -95,15 +95,23 @@ class HTTPieArgumentParser(argparse.ArgumentParser): self._body_from_file(self.env.stdin) self._process_url() self._process_auth() + + if self.args.compress: + # TODO: allow --compress with --chunked / --multipart + if self.args.chunked: + self.error('cannot combine --compress and --chunked') + if self.args.multipart: + self.error('cannot combine --compress and --multipart') + return self.args - def _process_request_content_type(self): - rct = self.args.request_content_type - self.args.json = rct is RequestContentType.JSON - self.args.multipart = rct is RequestContentType.MULTIPART - self.args.form = rct in { - RequestContentType.FORM, - RequestContentType.MULTIPART, + def _process_request_type(self): + request_type = self.args.request_type + self.args.json = request_type is RequestType.JSON + self.args.multipart = request_type is RequestType.MULTIPART + self.args.form = request_type in { + RequestType.FORM, + RequestType.MULTIPART, } def _process_url(self): @@ -285,7 +293,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser): 'data (key=value) cannot be mixed. Pass ' '--ignore-stdin to let key/value take priority. ' 'See https://httpie.org/doc#scripting for details.') - self.args.data = getattr(fd, 'buffer', fd).read() + self.args.data = getattr(fd, 'buffer', fd) def _guess_method(self): """Set `args.method` if not specified to either POST or GET @@ -346,6 +354,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser): self.args.data = request_items.data self.args.files = request_items.files self.args.params = request_items.params + self.args.multipart_data = request_items.multipart_data if self.args.files and not self.args.form: # `http url @/path/to/file` diff --git a/httpie/cli/argtypes.py b/httpie/cli/argtypes.py index 4f16e7c4..0d4f9eb2 100644 --- a/httpie/cli/argtypes.py +++ b/httpie/cli/argtypes.py @@ -242,9 +242,3 @@ PARSED_DEFAULT_FORMAT_OPTIONS = parse_format_options( s=','.join(DEFAULT_FORMAT_OPTIONS), defaults=None, ) - - -class UnsortedAction(argparse.Action): - - def __call__(self, *args, **kwargs): - return 1 diff --git a/httpie/cli/constants.py b/httpie/cli/constants.py index 7b91d674..5b1d5986 100644 --- a/httpie/cli/constants.py +++ b/httpie/cli/constants.py @@ -41,6 +41,12 @@ SEPARATOR_GROUP_DATA_ITEMS = frozenset({ SEPARATOR_DATA_EMBED_RAW_JSON_FILE }) +SEPARATORS_GROUP_MULTIPART = frozenset({ + SEPARATOR_DATA_STRING, + SEPARATOR_DATA_EMBED_FILE_CONTENTS, + SEPARATOR_FILE_UPLOAD, +}) + # Separators for items whose value is a filename to be embedded SEPARATOR_GROUP_DATA_EMBED_ITEMS = frozenset({ SEPARATOR_DATA_EMBED_FILE_CONTENTS, @@ -108,7 +114,7 @@ OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY -class RequestContentType(enum.Enum): +class RequestType(enum.Enum): FORM = enum.auto() MULTIPART = enum.auto() JSON = enum.auto() diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 837b34a5..38f40aab 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -15,7 +15,7 @@ from httpie.cli.constants import ( DEFAULT_FORMAT_OPTIONS, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT, OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, - RequestContentType, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, + RequestType, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, SORTED_FORMAT_OPTIONS_STRING, UNSORTED_FORMAT_OPTIONS_STRING, ) @@ -111,7 +111,7 @@ positional.add_argument( awesome:=true amount:=42 colors:='["red", "green", "blue"]' - '@' Form file fields (only with --form, -f): + '@' Form file fields (only with --form or --multipart): cv@~/Documents/CV.pdf cv@'~/Documents/CV.pdf;type=application/pdf' @@ -143,8 +143,8 @@ content_type = parser.add_argument_group( content_type.add_argument( '--json', '-j', action='store_const', - const=RequestContentType.JSON, - dest='request_content_type', + const=RequestType.JSON, + dest='request_type', help=''' (default) Data items from the command line are serialized as a JSON object. The Content-Type and Accept headers are set to application/json @@ -155,8 +155,8 @@ content_type.add_argument( content_type.add_argument( '--form', '-f', action='store_const', - const=RequestContentType.FORM, - dest='request_content_type', + const=RequestType.FORM, + dest='request_type', help=''' Data items from the command line are serialized as form fields. @@ -169,8 +169,8 @@ content_type.add_argument( content_type.add_argument( '--multipart', action='store_const', - const=RequestContentType.MULTIPART, - dest='request_content_type', + const=RequestType.MULTIPART, + dest='request_type', help=''' Similar to --form, but always sends a multipart/form-data request (i.e., even without files). @@ -400,7 +400,7 @@ output_options.add_argument( action='store_true', default=False, help=''' - Always stream the output by line, i.e., behave like `tail -f'. + Always stream the response body by line, i.e., behave like `tail -f'. Without --stream and with --pretty (either set or implied), HTTPie fetches the whole response before it outputs the processed data. diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 9ce7475a..84178236 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -34,20 +34,25 @@ class MultiValueOrderedDict(OrderedDict): super().__setitem__(key, [self[key]]) self[key].append(value) - -class RequestQueryParamsDict(MultiValueOrderedDict): - pass - - -class RequestDataDict(MultiValueOrderedDict): - def items(self): - for key, values in super(MultiValueOrderedDict, self).items(): + for key, values in super().items(): if not isinstance(values, list): values = [values] for value in values: yield key, value +class RequestQueryParamsDict(MultiValueOrderedDict): + pass + + +class RequestDataDict(MultiValueOrderedDict): + pass + + +class MultipartRequestDataDict(MultiValueOrderedDict): + pass + + class RequestFilesDict(RequestDataDict): pass diff --git a/httpie/cli/requestitems.py b/httpie/cli/requestitems.py index 46d50903..3168c39c 100644 --- a/httpie/cli/requestitems.py +++ b/httpie/cli/requestitems.py @@ -4,13 +4,15 @@ from typing import Callable, Dict, IO, List, Optional, Tuple, Union from httpie.cli.argtypes import KeyValueArg from httpie.cli.constants import ( - SEPARATOR_DATA_EMBED_FILE_CONTENTS, SEPARATOR_DATA_EMBED_RAW_JSON_FILE, + SEPARATORS_GROUP_MULTIPART, SEPARATOR_DATA_EMBED_FILE_CONTENTS, + SEPARATOR_DATA_EMBED_RAW_JSON_FILE, SEPARATOR_DATA_RAW_JSON, SEPARATOR_DATA_STRING, SEPARATOR_FILE_UPLOAD, SEPARATOR_FILE_UPLOAD_TYPE, SEPARATOR_HEADER, SEPARATOR_HEADER_EMPTY, SEPARATOR_QUERY_PARAM, ) from httpie.cli.dicts import ( - RequestDataDict, RequestFilesDict, RequestHeadersDict, RequestJSONDataDict, + MultipartRequestDataDict, RequestDataDict, RequestFilesDict, + RequestHeadersDict, RequestJSONDataDict, RequestQueryParamsDict, ) from httpie.cli.exceptions import ParseError @@ -24,6 +26,8 @@ class RequestItems: self.data = RequestDataDict() if as_form else RequestJSONDataDict() self.files = RequestFilesDict() self.params = RequestQueryParamsDict() + # To preserve the order of fields in file upload multipart requests. + self.multipart_data = MultipartRequestDataDict() @classmethod def from_args( @@ -69,7 +73,11 @@ class RequestItems: for arg in request_item_args: processor_func, target_dict = rules[arg.sep] - target_dict[arg.key] = processor_func(arg) + value = processor_func(arg) + target_dict[arg.key] = value + + if arg.sep in SEPARATORS_GROUP_MULTIPART: + instance.multipart_data[arg.key] = value return instance diff --git a/httpie/client.py b/httpie/client.py index 99f4127d..9407181d 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -2,7 +2,6 @@ import argparse import http.client import json import sys -import zlib from contextlib import contextmanager from pathlib import Path from typing import Callable, Iterable, Union @@ -17,7 +16,7 @@ from httpie.plugins.registry import plugin_manager from httpie.sessions import get_httpie_session from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter from httpie.uploads import ( - wrap_request_data, + compress_request, prepare_request_body, get_multipart_data_and_content_type, ) from httpie.utils import get_expired_cookies, repr_dict @@ -34,7 +33,7 @@ DEFAULT_UA = f'HTTPie/{__version__}' def collect_messages( args: argparse.Namespace, config_dir: Path, - body_chunk_sent_callback: Callable[[bytes], None]=None, + request_body_read_callback: Callable[[bytes], None] = None, ) -> Iterable[Union[requests.PreparedRequest, requests.Response]]: httpie_session = None httpie_session_headers = None @@ -50,7 +49,7 @@ def collect_messages( request_kwargs = make_request_kwargs( args=args, base_headers=httpie_session_headers, - callback=body_chunk_sent_callback + request_body_read_callback=request_body_read_callback ) send_kwargs = make_send_kwargs(args) send_kwargs_mergeable_from_env = make_send_kwargs_mergeable_from_env(args) @@ -85,7 +84,10 @@ def collect_messages( prepped_url=prepared_request.url, ) if args.compress and prepared_request.body: - compress_body(prepared_request, always=args.compress > 1) + compress_request( + request=prepared_request, + always=args.compress > 1, + ) response_count = 0 expired_cookies = [] while prepared_request: @@ -142,23 +144,6 @@ def max_headers(limit): http.client._MAXHEADERS = orig -def compress_body(request: requests.PreparedRequest, always: bool): - deflater = zlib.compressobj() - if isinstance(request.body, str): - body_bytes = request.body.encode() - elif hasattr(request.body, 'read'): - body_bytes = request.body.read() - else: - body_bytes = request.body - deflated_data = deflater.compress(body_bytes) - deflated_data += deflater.flush() - is_economical = len(deflated_data) < len(body_bytes) - if is_economical or always: - request.body = deflated_data - request.headers['Content-Encoding'] = 'deflate' - request.headers['Content-Length'] = str(len(deflated_data)) - - def build_requests_session( verify: bool, ssl_version: str = None, @@ -258,7 +243,7 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict: def make_request_kwargs( args: argparse.Namespace, base_headers: RequestHeadersDict = None, - callback=lambda chunk: chunk + request_body_read_callback=lambda chunk: chunk ) -> dict: """ Translate our `args` into `requests.Request` keyword arguments. @@ -285,21 +270,23 @@ def make_request_kwargs( if (args.form and files) or args.multipart: data, headers['Content-Type'] = get_multipart_data_and_content_type( - data=data, - files=files, + data=args.multipart_data, boundary=args.boundary, content_type=args.headers.get('Content-Type'), ) - files = None kwargs = { 'method': args.method.lower(), 'url': args.url, 'headers': headers, - 'data': wrap_request_data(data, callback=callback), + 'data': prepare_request_body( + body=data, + body_read_callback=request_body_read_callback, + chunked=args.chunked, + content_length_header_value=headers.get('Content-Length') + ), 'auth': args.auth, - 'params': args.params, - 'files': files, + 'params': args.params.items(), } return kwargs diff --git a/httpie/core.py b/httpie/core.py index e0182135..b2a1b799 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -2,7 +2,7 @@ import argparse import os import platform import sys -from typing import List, Union +from typing import List, Optional, Tuple, Union import requests from pygments import __version__ as pygments_version @@ -16,7 +16,10 @@ from httpie.cli.constants import ( from httpie.client import collect_messages from httpie.context import Environment from httpie.downloads import Downloader -from httpie.output.writer import write_message, write_stream +from httpie.output.writer import ( + write_message, + write_stream, +) from httpie.plugins.registry import plugin_manager from httpie.status import ExitStatus, http_status_to_exit_status @@ -118,16 +121,16 @@ def main( def get_output_options( args: argparse.Namespace, message: Union[requests.PreparedRequest, requests.Response] -) -> dict: +) -> Tuple[bool, bool]: return { - requests.PreparedRequest: { - 'with_headers': OUT_REQ_HEAD in args.output_options, - 'with_body': OUT_REQ_BODY in args.output_options, - }, - requests.Response: { - 'with_headers': OUT_RESP_HEAD in args.output_options, - 'with_body': OUT_RESP_BODY in args.output_options, - }, + requests.PreparedRequest: ( + OUT_REQ_HEAD in args.output_options, + OUT_REQ_BODY in args.output_options, + ), + requests.Response: ( + OUT_RESP_HEAD in args.output_options, + OUT_RESP_BODY in args.output_options, + ), }[type(message)] @@ -152,39 +155,57 @@ def program( ) downloader.pre_request(args.headers) - initial_request = None - final_response = None + needs_separator = False - def upload_callback(chunk): - print('GOT', chunk) + def maybe_separate(): + nonlocal needs_separator + if env.stdout.isatty() and needs_separator: + needs_separator = False + getattr(env.stdout, 'buffer', env.stdout).write(b'\n\n') + + initial_request: Optional[requests.PreparedRequest] = None + final_response: Optional[requests.Response] = None + + def request_body_read_callback(chunk: bytes): + should_pipe_to_stdout = ( + # Request body output desired + OUT_REQ_BODY in args.output_options + # & not `.read()` already pre-request (e.g., for compression) + and initial_request + # & non-EOF chunk + and chunk + ) + if should_pipe_to_stdout: + msg = requests.PreparedRequest() + msg.is_body_upload_chunk = True + msg.body = chunk + msg.headers = initial_request.headers + write_message( + requests_message=msg, + env=env, + args=args, + with_body=True, + with_headers=False + ) messages = collect_messages( args=args, config_dir=env.config.directory, - body_chunk_sent_callback=upload_callback + request_body_read_callback=request_body_read_callback ) for message in messages: + maybe_separate() is_request = isinstance(message, requests.PreparedRequest) - output_options = get_output_options(args=args, message=message) - if not is_request or not output_options['with_body']: - write_message( - requests_message=message, - env=env, - args=args, - **output_options, - ) + with_headers, with_body = get_output_options( + args=args, message=message) if is_request: if not initial_request: initial_request = message - if 0and not args.offline: - output_options['with_body'] = False - write_message( - requests_message=message, - env=env, - args=args, - **output_options, - ) - + is_streamed_upload = not args.offline and not isinstance( + message.body, (str, bytes)) + if with_body: + with_body = not is_streamed_upload + needs_separator = is_streamed_upload else: final_response = message if args.check_status or downloader: @@ -193,11 +214,20 @@ def program( follow=args.follow ) if (not env.stdout_isatty - and exit_status != ExitStatus.SUCCESS): + and exit_status != ExitStatus.SUCCESS): env.log_error( f'HTTP {message.raw.status} {message.raw.reason}', level='warning' ) + write_message( + requests_message=message, + env=env, + args=args, + with_headers=with_headers, + with_body=with_body, + ) + + maybe_separate() if downloader and exit_status == ExitStatus.SUCCESS: # Last response body download. @@ -225,7 +255,7 @@ def program( downloader.failed() if (not isinstance(args, list) and args.output_file - and args.output_file_specified): + and args.output_file_specified): args.output_file.close() diff --git a/httpie/output/formatters/colors.py b/httpie/output/formatters/colors.py index 9416dea5..2506a59e 100644 --- a/httpie/output/formatters/colors.py +++ b/httpie/output/formatters/colors.py @@ -86,7 +86,7 @@ class ColorFormatter(FormatterPlugin): lexer=lexer, formatter=self.formatter, ) - return body.strip() + return body def get_lexer_for_body( self, mime: str, diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 083aa5bb..f104928e 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -4,7 +4,6 @@ from typing import Callable, Iterable, Union from httpie.context import Environment from httpie.models import HTTPMessage from httpie.output.processing import Conversion, Formatting -from requests_toolbelt import MultipartEncoder BINARY_SUPPRESSED_NOTICE = ( @@ -15,21 +14,6 @@ BINARY_SUPPRESSED_NOTICE = ( ) -LARGE_UPLOAD_SUPPRESSED_NOTICE = ( - b'\n' - b'+--------------------------------------------------------+\n' - b'| NOTE: large form upload data not shown in the terminal |\n' - b'+--------------------------------------------------------+' -) - - -def might_suppress(chunk): - if isinstance(chunk, MultipartEncoder): - raise LargeUploadSuppressedError() - # if isinstance(io.IOBase): - # pass - - class DataSuppressedError(Exception): message = None @@ -40,13 +24,6 @@ class BinarySuppressedError(DataSuppressedError): message = BINARY_SUPPRESSED_NOTICE -class LargeUploadSuppressedError(DataSuppressedError): - """An error indicating that the body is binary and won't be written, - e.g., for terminal output).""" - - message = LARGE_UPLOAD_SUPPRESSED_NOTICE - - class BaseStream: """Base HTTP message output stream class.""" @@ -132,8 +109,6 @@ class EncodedStream(BaseStream): def iter_body(self) -> Iterable[bytes]: for line, lf in self.msg.iter_lines(self.CHUNK_SIZE): - if isinstance(line, MultipartEncoder): - raise LargeUploadSuppressedError() if b'\0' in line: raise BinarySuppressedError() yield line.decode(self.msg.encoding) \ @@ -212,8 +187,6 @@ class BufferedPrettyStream(PrettyStream): body = bytearray() for chunk in self.msg.iter_body(self.CHUNK_SIZE): - if isinstance(chunk, MultipartEncoder): - raise LargeUploadSuppressedError() if not converter and b'\0' in chunk: converter = self.conversion.get_converter(self.mime) if not converter: diff --git a/httpie/output/writer.py b/httpie/output/writer.py index e12af59f..d8a34aa3 100644 --- a/httpie/output/writer.py +++ b/httpie/output/writer.py @@ -107,7 +107,8 @@ def build_output_stream_for_message( with_body=with_body, **stream_kwargs, ) - if env.stdout_isatty and with_body: + if (env.stdout_isatty and with_body + and not getattr(requests_message, 'is_body_upload_chunk', False)): # Ensure a blank line after the response body. # For terminal output only. yield b'\n\n' diff --git a/httpie/uploads.py b/httpie/uploads.py index 0eee20e0..f7dc0f5e 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -1,68 +1,129 @@ -from typing import Tuple, Union +import zlib +from typing import Callable, IO, Iterable, Tuple, Union +from urllib.parse import urlencode +import requests from requests.utils import super_len from requests_toolbelt import MultipartEncoder -from httpie.cli.dicts import RequestDataDict, RequestFilesDict +from httpie.cli.dicts import MultipartRequestDataDict, RequestDataDict -# Multipart uploads smaller than this size gets buffered (otherwise streamed). -# NOTE: Unbuffered upload requests cannot be displayed on the terminal. -MULTIPART_UPLOAD_BUFFER = 1024 * 1000 +class ChunkedUploadStream: + def __init__(self, stream: Iterable, callback: Callable): + self.callback = callback + self.stream = stream + + def __iter__(self) -> Iterable[Union[str, bytes]]: + for chunk in self.stream: + self.callback(chunk) + yield chunk + + +class ChunkedMultipartUploadStream: + chunk_size = 100 * 1024 + + def __init__(self, encoder: MultipartEncoder): + self.encoder = encoder + + def __iter__(self) -> Iterable[Union[str, bytes]]: + while True: + chunk = self.encoder.read(self.chunk_size) + if not chunk: + break + yield chunk + + +def prepare_request_body( + body: Union[str, bytes, IO, MultipartEncoder, RequestDataDict], + body_read_callback: Callable[[bytes], bytes], + content_length_header_value: int = None, + chunked=False, +) -> Union[str, bytes, IO, MultipartEncoder, ChunkedUploadStream]: + if isinstance(body, RequestDataDict): + body = urlencode(body, doseq=True) + + if not hasattr(body, 'read'): + if chunked: + body = ChunkedUploadStream( + # Pass the entire body as one chunk. + stream=(chunk.encode() for chunk in [body]), + callback=body_read_callback, + ) + else: + # File-like object. + + if not super_len(body): + # Zero-length -> assume stdin. + if content_length_header_value is None and not chunked: + # + # Read the whole stdin to determine `Content-Length`. + # + # TODO: Instead of opt-in --chunked, consider making + # `Transfer-Encoding: chunked` for STDIN opt-out via + # something like --no-chunked. + # This would be backwards-incompatible so wait until v3.0.0. + # + body = body.read() + else: + orig_read = body.read + + def new_read(*args): + chunk = orig_read(*args) + body_read_callback(chunk) + return chunk + + body.read = new_read + + if chunked: + if isinstance(body, MultipartEncoder): + body = ChunkedMultipartUploadStream( + encoder=body, + ) + else: + body = ChunkedUploadStream( + stream=body, + callback=body_read_callback, + ) + + return body def get_multipart_data_and_content_type( - data: RequestDataDict, - files: RequestFilesDict, + data: MultipartRequestDataDict, boundary: str = None, content_type: str = None, -) -> Tuple[Union[MultipartEncoder, bytes], str]: - fields = list(data.items()) + list(files.items()) +) -> Tuple[MultipartEncoder, str]: encoder = MultipartEncoder( - fields=fields, + fields=data.items(), boundary=boundary, ) if content_type: - content_type = content_type.strip() # maybe auto-strip all headers somewhere + content_type = content_type.strip() if 'boundary=' not in content_type: content_type = f'{content_type}; boundary={encoder.boundary_value}' else: content_type = encoder.content_type - data = encoder.to_string() if 0 and encoder.len < MULTIPART_UPLOAD_BUFFER else encoder + data = encoder return data, content_type -class Stdin: - - def __init__(self, stdin, callback): - self.callback = callback - self.stdin = stdin - - def __iter__(self): - for chunk in self.stdin: - print("__iter__() =>", chunk) - self.callback(chunk) - yield chunk - - @classmethod - def is_stdin(cls, obj): - return super_len(obj) == 0 - - -def wrap_request_data(data, callback=lambda chunk: print('chunk', chunk)): - if hasattr(data, 'read'): - if Stdin.is_stdin(data): - data = Stdin(data, callback=callback) - else: - orig_read = data.read - - def new_read(*args): - val = orig_read(*args) - print('read() =>', val) - callback(callback) - return val - - data.read = new_read - - return data +def compress_request( + request: requests.PreparedRequest, + always: bool, +): + deflater = zlib.compressobj() + if isinstance(request.body, str): + body_bytes = request.body.encode() + elif hasattr(request.body, 'read'): + body_bytes = request.body.read() + else: + body_bytes = request.body + deflated_data = deflater.compress(body_bytes) + deflated_data += deflater.flush() + is_economical = len(deflated_data) < len(body_bytes) + if is_economical or always: + request.body = deflated_data + request.headers['Content-Encoding'] = 'deflate' + request.headers['Content-Length'] = str(len(deflated_data)) diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 294141c1..59faf348 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1,6 +1,5 @@ """Test data""" -from os import path -import codecs +from pathlib import Path def patharg(path): @@ -9,32 +8,24 @@ def patharg(path): even in Windows paths. """ - return path.replace('\\', '\\\\\\') + return str(path).replace('\\', '\\\\\\') -FIXTURES_ROOT = path.join(path.abspath(path.dirname(__file__))) -FILE_PATH = path.join(FIXTURES_ROOT, 'test.txt') -JSON_FILE_PATH = path.join(FIXTURES_ROOT, 'test.json') -BIN_FILE_PATH = path.join(FIXTURES_ROOT, 'test.bin') - +FIXTURES_ROOT = Path(__file__).parent +FILE_PATH = FIXTURES_ROOT / 'test.txt' +JSON_FILE_PATH = FIXTURES_ROOT / 'test.json' +BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin' FILE_PATH_ARG = patharg(FILE_PATH) BIN_FILE_PATH_ARG = patharg(BIN_FILE_PATH) JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH) - -with codecs.open(FILE_PATH, encoding='utf8') as f: - # Strip because we don't want new lines in the data so that we can - # easily count occurrences also when embedded in JSON (where the new - # line would be escaped). - FILE_CONTENT = f.read().strip() +# Strip because we don't want new lines in the data so that we can +# easily count occurrences also when embedded in JSON (where the new +# line would be escaped). +FILE_CONTENT = FILE_PATH.read_text().strip() -with codecs.open(JSON_FILE_PATH, encoding='utf8') as f: - JSON_FILE_CONTENT = f.read() - - -with open(BIN_FILE_PATH, 'rb') as f: - BIN_FILE_CONTENT = f.read() - +JSON_FILE_CONTENT = JSON_FILE_PATH.read_text() +BIN_FILE_CONTENT = BIN_FILE_PATH.read_bytes() UNICODE = FILE_CONTENT diff --git a/tests/test_cli.py b/tests/test_cli.py index 439aad30..f57b2dec 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,7 @@ from httpie.cli import constants from httpie.cli.definition import parser from httpie.cli.argtypes import KeyValueArg, KeyValueArgType from httpie.cli.requestitems import RequestItems -from utils import HTTP_OK, MockEnvironment, http +from utils import HTTP_OK, MockEnvironment, StdinBytesIO, http class TestItemParsing: @@ -312,10 +312,11 @@ class TestNoOptions: class TestStdin: def test_ignore_stdin(self, httpbin): - with open(FILE_PATH) as f: - env = MockEnvironment(stdin=f, stdin_isatty=False) - r = http('--ignore-stdin', '--verbose', httpbin.url + '/get', - env=env) + env = MockEnvironment( + stdin=StdinBytesIO(FILE_PATH.read_bytes()), + stdin_isatty=False, + ) + r = http('--ignore-stdin', '--verbose', httpbin.url + '/get', env=env) assert HTTP_OK in r assert 'GET /get HTTP' in r, "Don't default to POST." assert FILE_CONTENT not in r, "Don't send stdin data." diff --git a/tests/test_compress.py b/tests/test_compress.py index 089ea619..895688bf 100644 --- a/tests/test_compress.py +++ b/tests/test_compress.py @@ -12,7 +12,8 @@ import base64 import zlib from fixtures import FILE_PATH, FILE_CONTENT -from utils import http, HTTP_OK, MockEnvironment +from httpie.status import ExitStatus +from utils import StdinBytesIO, http, HTTP_OK, MockEnvironment def assert_decompressed_equal(base64_compressed_data, expected_str): @@ -27,6 +28,20 @@ def assert_decompressed_equal(base64_compressed_data, expected_str): assert actual_str == expected_str +def test_cannot_combine_compress_with_chunked(httpbin): + r = http('--compress', '--chunked', httpbin.url + '/get', + tolerate_error_exit_status=True) + assert r.exit_status == ExitStatus.ERROR + assert 'cannot combine --compress and --chunked' in r.stderr + + +def test_cannot_combine_compress_with_multipart(httpbin): + r = http('--compress', '--multipart', httpbin.url + '/get', + tolerate_error_exit_status=True) + assert r.exit_status == ExitStatus.ERROR + assert 'cannot combine --compress and --multipart' in r.stderr + + def test_compress_skip_negative_ratio(httpbin_both): r = http( '--compress', @@ -78,15 +93,17 @@ def test_compress_form(httpbin_both): def test_compress_stdin(httpbin_both): - with open(FILE_PATH) as f: - env = MockEnvironment(stdin=f, stdin_isatty=False) - r = http( - '--compress', - '--compress', - 'PATCH', - httpbin_both + '/patch', - env=env, - ) + env = MockEnvironment( + stdin=StdinBytesIO(FILE_PATH.read_bytes()), + stdin_isatty=False, + ) + r = http( + '--compress', + '--compress', + 'PATCH', + httpbin_both + '/patch', + env=env, + ) assert HTTP_OK in r assert r.json['headers']['Content-Encoding'] == 'deflate' assert_decompressed_equal(r.json['data'], FILE_CONTENT.strip()) @@ -100,7 +117,7 @@ def test_compress_file(httpbin_both): '--compress', 'PUT', httpbin_both + '/put', - 'file@' + FILE_PATH, + f'file@{FILE_PATH}', ) assert HTTP_OK in r assert r.json['headers']['Content-Encoding'] == 'deflate' diff --git a/tests/test_defaults.py b/tests/test_defaults.py index 5489c193..73489270 100644 --- a/tests/test_defaults.py +++ b/tests/test_defaults.py @@ -2,6 +2,8 @@ Tests for the provided defaults regarding HTTP method, and --json vs. --form. """ +from io import BytesIO + from httpie.client import JSON_ACCEPT from utils import MockEnvironment, http, HTTP_OK from fixtures import FILE_PATH @@ -44,9 +46,11 @@ class TestImplicitHTTPMethod: assert r.json['form'] == {'foo': 'bar'} def test_implicit_POST_stdin(self, httpbin): - with open(FILE_PATH) as f: - env = MockEnvironment(stdin_isatty=False, stdin=f) - r = http('--form', httpbin.url + '/post', env=env) + env = MockEnvironment( + stdin_isatty=False, + stdin=BytesIO(FILE_PATH.read_bytes()) + ) + r = http('--form', httpbin.url + '/post', env=env) assert HTTP_OK in r diff --git a/tests/test_httpie.py b/tests/test_httpie.py index 613f46ed..f4fe1b37 100644 --- a/tests/test_httpie.py +++ b/tests/test_httpie.py @@ -9,7 +9,7 @@ import httpie.__main__ from httpie.context import Environment from httpie.status import ExitStatus from httpie.cli.exceptions import ParseError -from utils import MockEnvironment, http, HTTP_OK +from utils import MockEnvironment, StdinBytesIO, http, HTTP_OK from fixtures import FILE_PATH, FILE_CONTENT import httpie @@ -104,15 +104,17 @@ def test_POST_form_multiple_values(httpbin_both): def test_POST_stdin(httpbin_both): - with open(FILE_PATH) as f: - env = MockEnvironment(stdin=f, stdin_isatty=False) - r = http('--form', 'POST', httpbin_both + '/post', env=env) + env = MockEnvironment( + stdin=StdinBytesIO(FILE_PATH.read_bytes()), + stdin_isatty=False, + ) + r = http('--form', 'POST', httpbin_both + '/post', env=env) assert HTTP_OK in r assert FILE_CONTENT in r def test_POST_file(httpbin_both): - r = http('--form', 'POST', httpbin_both + '/post', 'file@' + FILE_PATH) + r = http('--form', 'POST', httpbin_both + '/post', f'file@{FILE_PATH}') assert HTTP_OK in r assert FILE_CONTENT in r @@ -127,10 +129,10 @@ def test_form_POST_file_redirected_stdin(httpbin): '--form', 'POST', httpbin + '/post', - 'file@' + FILE_PATH, + f'file@{FILE_PATH}', tolerate_error_exit_status=True, env=MockEnvironment( - stdin=f, + stdin=StdinBytesIO(FILE_PATH.read_bytes()), stdin_isatty=False, ), ) diff --git a/tests/test_stream.py b/tests/test_stream.py index 01ee151d..fa70e93d 100644 --- a/tests/test_stream.py +++ b/tests/test_stream.py @@ -2,7 +2,7 @@ import pytest from httpie.compat import is_windows from httpie.output.streams import BINARY_SUPPRESSED_NOTICE -from utils import http, MockEnvironment +from utils import StdinBytesIO, http, MockEnvironment from fixtures import BIN_FILE_CONTENT, BIN_FILE_PATH @@ -13,32 +13,37 @@ from fixtures import BIN_FILE_CONTENT, BIN_FILE_PATH reason='Pretty redirect not supported under Windows') def test_pretty_redirected_stream(httpbin): """Test that --stream works with prettified redirected output.""" - with open(BIN_FILE_PATH, 'rb') as f: - env = MockEnvironment(colors=256, stdin=f, - stdin_isatty=False, - stdout_isatty=False) - r = http('--verbose', '--pretty=all', '--stream', 'GET', - httpbin.url + '/get', env=env) + env = MockEnvironment( + colors=256, + stdin=StdinBytesIO(BIN_FILE_PATH.read_bytes()), + stdin_isatty=False, + stdout_isatty=False, + ) + r = http('--verbose', '--pretty=all', '--stream', 'GET', + httpbin.url + '/get', env=env) assert BINARY_SUPPRESSED_NOTICE.decode() in r def test_encoded_stream(httpbin): """Test that --stream works with non-prettified redirected terminal output.""" - with open(BIN_FILE_PATH, 'rb') as f: - env = MockEnvironment(stdin=f, stdin_isatty=False) - r = http('--pretty=none', '--stream', '--verbose', 'GET', - httpbin.url + '/get', env=env) + env = MockEnvironment( + stdin=StdinBytesIO(BIN_FILE_PATH.read_bytes()), + stdin_isatty=False, + ) + r = http('--pretty=none', '--stream', '--verbose', 'GET', + httpbin.url + '/get', env=env) assert BINARY_SUPPRESSED_NOTICE.decode() in r def test_redirected_stream(httpbin): """Test that --stream works with non-prettified redirected terminal output.""" - with open(BIN_FILE_PATH, 'rb') as f: - env = MockEnvironment(stdout_isatty=False, - stdin_isatty=False, - stdin=f) - r = http('--pretty=none', '--stream', '--verbose', 'GET', - httpbin.url + '/get', env=env) + env = MockEnvironment( + stdout_isatty=False, + stdin_isatty=False, + stdin=StdinBytesIO(BIN_FILE_PATH.read_bytes()), + ) + r = http('--pretty=none', '--stream', '--verbose', 'GET', + httpbin.url + '/get', env=env) assert BIN_FILE_CONTENT in r diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 9c070dac..7dbf1716 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -1,16 +1,57 @@ import os -from unittest import mock import pytest from httpie.cli.exceptions import ParseError from httpie.client import FORM_CONTENT_TYPE -from httpie.output.streams import LARGE_UPLOAD_SUPPRESSED_NOTICE from httpie.status import ExitStatus -from utils import MockEnvironment, http, HTTP_OK +from utils import ( + HTTPBIN_WITH_CHUNKED_SUPPORT, MockEnvironment, StdinBytesIO, http, + HTTP_OK, +) from fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT +def test_chunked_json(): + r = http( + '--verbose', + '--chunked', + HTTPBIN_WITH_CHUNKED_SUPPORT + '/post', + 'hello=world', + ) + assert HTTP_OK in r + assert 'Transfer-Encoding: chunked' in r + assert r.count('hello') == 3 + + +def test_chunked_form(): + r = http( + '--verbose', + '--chunked', + '--form', + HTTPBIN_WITH_CHUNKED_SUPPORT + '/post', + 'hello=world', + ) + assert HTTP_OK in r + assert 'Transfer-Encoding: chunked' in r + assert r.count('hello') == 2 + + +def test_chunked_stdin(): + r = http( + '--verbose', + '--chunked', + HTTPBIN_WITH_CHUNKED_SUPPORT + '/post', + env=MockEnvironment( + stdin=StdinBytesIO(FILE_PATH.read_bytes()), + stdin_isatty=False, + ) + ) + assert HTTP_OK in r + assert 'Transfer-Encoding: chunked' in r + assert r.count(FILE_CONTENT) == 2 + + class TestMultipartFormDataFileUpload: def test_non_existent_file_raises_parse_error(self, httpbin): @@ -55,19 +96,6 @@ class TestMultipartFormDataFileUpload: assert r.count(FILE_CONTENT) == 2 assert 'Content-Type: image/vnd.microsoft.icon' in r - @mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0) - def test_large_upload_display_suppressed(self, httpbin): - r = http( - '--form', - '--verbose', - httpbin.url + '/post', - f'test-file@{FILE_PATH_ARG}', - 'foo=bar', - ) - assert HTTP_OK in r - assert r.count(FILE_CONTENT) == 1 - assert LARGE_UPLOAD_SUPPRESSED_NOTICE.decode() in r - def test_form_no_files_urlencoded(self, httpbin): r = http( '--form', @@ -91,33 +119,6 @@ class TestMultipartFormDataFileUpload: assert FORM_CONTENT_TYPE not in r assert 'multipart/form-data' in r - def test_multipart_too_large_for_terminal(self, httpbin): - with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0): - r = http( - '--verbose', - '--multipart', - httpbin.url + '/post', - 'AAAA=AAA', - 'BBB=BBB', - ) - assert HTTP_OK in r - assert FORM_CONTENT_TYPE not in r - assert 'multipart/form-data' in r - - def test_multipart_too_large_for_terminal_non_pretty(self, httpbin): - with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0): - r = http( - '--verbose', - '--multipart', - '--pretty=none', - httpbin.url + '/post', - 'AAAA=AAA', - 'BBB=BBB', - ) - assert HTTP_OK in r - assert FORM_CONTENT_TYPE not in r - assert 'multipart/form-data' in r - def test_form_multipart_custom_boundary(self, httpbin): boundary = 'HTTPIE_FTW' r = http( @@ -164,6 +165,20 @@ class TestMultipartFormDataFileUpload: assert f'multipart/magic; boundary={boundary_in_header}' in r assert r.count(boundary_in_body) == 3 + def test_multipart_chunked(self, httpbin): + r = http( + '--verbose', + '--multipart', + '--chunked', + # '--offline', + HTTPBIN_WITH_CHUNKED_SUPPORT + '/post', + 'AAA=AAA', + ) + assert 'Transfer-Encoding: chunked' in r + assert 'multipart/form-data' in r + assert 'name="AAA"' in r # in request + assert '"AAA": "AAA"', r # in response + class TestRequestBodyFromFilePath: """ @@ -172,12 +187,26 @@ class TestRequestBodyFromFilePath: """ def test_request_body_from_file_by_path(self, httpbin): - r = http('--verbose', - 'POST', httpbin.url + '/post', '@' + FILE_PATH_ARG) + r = http( + '--verbose', + 'POST', httpbin.url + '/post', + '@' + FILE_PATH_ARG, + ) assert HTTP_OK in r - assert FILE_CONTENT in r, r + assert r.count(FILE_CONTENT) == 2 assert '"Content-Type": "text/plain"' in r + def test_request_body_from_file_by_path_chunked(self, httpbin): + r = http( + '--verbose', '--chunked', + HTTPBIN_WITH_CHUNKED_SUPPORT + '/post', + '@' + FILE_PATH_ARG, + ) + assert HTTP_OK in r + assert 'Transfer-Encoding: chunked' in r + assert '"Content-Type": "text/plain"' in r + assert r.count(FILE_CONTENT) == 2 + def test_request_body_from_file_by_path_with_explicit_content_type( self, httpbin): r = http('--verbose', diff --git a/tests/utils.py b/tests/utils.py index 829e9f27..b84805e2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,10 +1,10 @@ # coding=utf-8 """Utilities for HTTPie test suite.""" -import os import sys import time import json import tempfile +from io import BytesIO from pathlib import Path from typing import Optional, Union @@ -14,6 +14,12 @@ from httpie.context import Environment from httpie.core import main +# pytest-httpbin currently does not support chunked requests: +# +# +HTTPBIN_WITH_CHUNKED_SUPPORT = 'http://httpbin.org' + + TESTS_ROOT = Path(__file__).parent CRLF = '\r\n' COLOR = '\x1b[' @@ -36,6 +42,11 @@ def add_auth(url, auth): return proto + '://' + auth + '@' + rest +class StdinBytesIO(BytesIO): + """To be used for `MockEnvironment.stdin`""" + len = 0 # See `prepare_request_body()` + + class MockEnvironment(Environment): """Environment subclass with reasonable defaults for testing.""" colors = 0