From 1813cf6156033a4a0301c128c026a0a440790949 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Wed, 19 Aug 2020 10:22:42 +0200 Subject: [PATCH] Add `--multipart` and `--boundary` --- CHANGELOG.rst | 2 ++ README.rst | 60 ++++++++++++++++++++++++++++++++ httpie/cli/argparser.py | 14 ++++---- httpie/cli/definition.py | 19 +++++++++++ httpie/client.py | 11 ++++-- httpie/uploads.py | 22 +++++++++--- tests/test_uploads.py | 74 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 186 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2c279bd..1700ec9a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ This project adheres to `Semantic Versioning `_. * Added support for combining cookies specified on the CLI and in a session file (`#932`_). * Added out of the box SOCKS support with no extra installation (`#904`_). * Added ``--quiet, -q`` flag to enforce silent behaviour. +* Added ``--multipart`` to allow ``multipart/form-data`` encoding for non-file ``--form`` requests as well. +* Added ``--boundary`` to allow a custom boundary string for ``multipart/form-data`` requests. * Removed Tox testing entirely (`#943`_). diff --git a/README.rst b/README.rst index c8e8b497..46b8c760 100644 --- a/README.rst +++ b/README.rst @@ -698,6 +698,66 @@ Larger multipart uploads (i.e., ``--form`` requests with at least one ``file@pat are always streamed to avoid memory issues. Additionally, the display of the request body on the terminal is suppressed. +You can explicitly use ``--multipart`` to enforce ``multipart/form-data`` even +for form requests without any files: + +.. code-block:: bash + + $ http --form --multipart --offline example.org/ hello=world + +.. code-block:: http + + POST / HTTP/1.1 + Content-Length: 129 + Content-Type: multipart/form-data; boundary=c31279ab254f40aeb06df32b433cbccb + Host: example.org + + --c31279ab254f40aeb06df32b433cbccb + Content-Disposition: form-data; name="hello" + + world + --c31279ab254f40aeb06df32b433cbccb-- + +By default, HTTPie uses a random unique string as the boundary. You may use +the ``--boundary`` option to specify a custom boundary string instead: + +.. code-block:: bash + + $ http --form --multipart --boundary=XOXO --offline example.org/ hello=world + +.. code-block:: http + + POST / HTTP/1.1 + Content-Length: 129 + Content-Type: multipart/form-data; boundary=XOXO + Host: example.org + + --XOXO + Content-Disposition: form-data; name="hello" + + world + --XOXO-- + +If you specify a custom ``Content-Type`` header without including the boundary +bit, HTTPie will add the boundary value (specified or generated) to the header +automatically: + +.. code-block:: bash + http --form --multipart --boundary=XOXO --offline example.org/ hello=world Content-Type:love/letter + +.. code-block:: http + + POST / HTTP/1.1 + Content-Length: 129 + Content-Type: love/letter; boundary=XOXO + Host: example.org + + --XOXO + Content-Disposition: form-data; name="hello" + + world + --XOXO-- + HTTP headers ============ diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index ddcfb79c..952af6af 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -75,16 +75,13 @@ class HTTPieArgumentParser(argparse.ArgumentParser): ) -> argparse.Namespace: self.env = env self.args, no_options = super().parse_known_args(args, namespace) - if self.args.debug: self.args.traceback = True - self.has_stdin_data = ( self.env.stdin and not self.args.ignore_stdin and not self.env.stdin_isatty ) - # Arguments processing and environment setup. self._apply_no_options(no_options) self._process_download_options() @@ -94,11 +91,15 @@ class HTTPieArgumentParser(argparse.ArgumentParser): self._process_format_options() self._guess_method() self._parse_items() - if self.has_stdin_data: self._body_from_file(self.env.stdin) + self._process_url() + self._process_auth() + return self.args + + def _process_url(self): if not URL_SCHEME_RE.match(self.args.url): - if os.path.basename(env.program_name) == 'https': + if os.path.basename(self.env.program_name) == 'https': scheme = 'https://' else: scheme = self.args.default_scheme + "://" @@ -114,9 +115,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser): self.args.url += rest else: self.args.url = scheme + self.args.url - self._process_auth() - - return self.args # noinspection PyShadowingBuiltins def _print_message(self, message, file=None): diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 764c6c05..140534fd 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -161,6 +161,25 @@ content_type.add_argument( ''' ) +content_type.add_argument( + '--multipart', + default=False, + action='store_true', + help=''' + Force the request to be encoded as multipart/form-data even without + any file fields. Only has effect only together with --form. + + ''' +) +content_type.add_argument( + '--boundary', + help=''' + Specify a custom boundary string for multipart/form-data requests. + Only has effect only together with --form. + + ''' +) + ####################################################################### # Content processing. diff --git a/httpie/client.py b/httpie/client.py index d696eed6..879a2151 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -16,7 +16,7 @@ from httpie.cli.dicts import RequestHeadersDict from httpie.plugins.registry import plugin_manager from httpie.sessions import get_httpie_session from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter -from httpie.uploads import get_multipart_data +from httpie.uploads import get_multipart_data_and_content_type from httpie.utils import get_expired_cookies, repr_dict @@ -276,8 +276,13 @@ def make_request_kwargs( headers.update(args.headers) headers = finalize_headers(headers) - if args.form and files: - data, headers['Content-Type'] = get_multipart_data(data, files) + if args.form and (files or args.multipart): + data, headers['Content-Type'] = get_multipart_data_and_content_type( + data=data, + files=files, + boundary=args.boundary, + content_type=args.headers.get('Content-Type'), + ) files = None kwargs = { diff --git a/httpie/uploads.py b/httpie/uploads.py index 8ae10b36..d883d746 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -1,20 +1,32 @@ from typing import Tuple, Union -from httpie.cli.dicts import RequestDataDict, RequestFilesDict from requests_toolbelt import MultipartEncoder +from httpie.cli.dicts import RequestDataDict, RequestFilesDict + # Multipart uploads smaller than this size gets buffered (otherwise streamed). # NOTE: Unbuffered upload requests cannot be displayed on the terminal. UPLOAD_BUFFER = 1024 * 100 -def get_multipart_data( +def get_multipart_data_and_content_type( data: RequestDataDict, - files: RequestFilesDict + files: RequestFilesDict, + boundary: str = None, + content_type: str = None, ) -> Tuple[Union[MultipartEncoder, bytes], str]: fields = list(data.items()) + list(files.items()) - encoder = MultipartEncoder(fields=fields) - content_type = encoder.content_type + encoder = MultipartEncoder( + fields=fields, + boundary=boundary, + ) + if content_type: + content_type = content_type.strip() # maybe auto-strip all headers somewhere + if 'boundary=' not in content_type: + content_type = f'{content_type}; boundary={encoder.boundary_value}' + else: + content_type = encoder.content_type + data = encoder.to_string() if encoder.len < UPLOAD_BUFFER else encoder return data, content_type diff --git a/tests/test_uploads.py b/tests/test_uploads.py index e59892a9..2a12aad6 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -4,6 +4,7 @@ from unittest import mock import pytest from httpie.cli.exceptions import ParseError +from httpie.client import FORM_CONTENT_TYPE from httpie.output.streams import LARGE_UPLOAD_SUPPRESSED_NOTICE from httpie.status import ExitStatus from utils import MockEnvironment, http, HTTP_OK @@ -67,6 +68,79 @@ class TestMultipartFormDataFileUpload: assert r.count(FILE_CONTENT) == 1 assert LARGE_UPLOAD_SUPPRESSED_NOTICE.decode() in r + def test_form_no_files_urlencoded(self, httpbin): + r = http( + '--form', + '--verbose', + httpbin.url + '/post', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert HTTP_OK in r + assert FORM_CONTENT_TYPE in r + + def test_form_no_files_multipart(self, httpbin): + r = http( + '--form', + '--verbose', + '--multipart', + httpbin.url + '/post', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert HTTP_OK in r + assert FORM_CONTENT_TYPE not in r + assert 'multipart/form-data' in r + + def test_form_multipart_custom_boundary(self, httpbin): + boundary = 'HTTPIE_FTW' + r = http( + '--print=HB', + '--check-status', + '--form', + '--multipart', + f'--boundary={boundary}', + httpbin.url + '/post', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert f'multipart/form-data; boundary={boundary}' in r + assert r.count(boundary) == 4 + + def test_multipart_custom_content_type_boundary_added(self, httpbin): + boundary = 'HTTPIE_FTW' + r = http( + '--print=HB', + '--check-status', + '--form', + '--multipart', + f'--boundary={boundary}', + httpbin.url + '/post', + 'Content-Type: multipart/magic', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert f'multipart/magic; boundary={boundary}' in r + assert r.count(boundary) == 4 + + def test_multipart_custom_content_type_boundary_preserved(self, httpbin): + # Allow explicit nonsense requests. + boundary_in_header = 'HEADER_BOUNDARY' + boundary_in_body = 'BODY_BOUNDARY' + r = http( + '--form', + '--print=HB', + '--check-status', + '--multipart', + f'--boundary={boundary_in_body}', + httpbin.url + '/post', + f'Content-Type: multipart/magic; boundary={boundary_in_header}', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert f'multipart/magic; boundary={boundary_in_header}' in r + assert r.count(boundary_in_body) == 3 + class TestRequestBodyFromFilePath: """