From e4e40e5b06d5c1d6567019bcc7f0888d8bece842 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 25 Sep 2020 14:44:22 +0200 Subject: [PATCH] Request content type --- README.rst | 16 ++++++++-------- httpie/cli/argparser.py | 15 +++++++++++++-- httpie/cli/constants.py | 10 ++++++++++ httpie/cli/definition.py | 20 +++++++++++++------- httpie/client.py | 13 +++++++------ httpie/output/streams.py | 2 ++ httpie/ssl.py | 2 +- httpie/uploads.py | 4 ++-- tests/test_uploads.py | 35 +++++++++++++++++++++++++++++------ 9 files changed, 85 insertions(+), 32 deletions(-) diff --git a/README.rst b/README.rst index 77a90d03..ca9a057f 100644 --- a/README.rst +++ b/README.rst @@ -684,7 +684,7 @@ submitted: -Note that ``@`` is used to simulate a file upload form field, whereas +Please note that ``@`` is used to simulate a file upload form field, whereas ``=@`` just embeds the file content as a regular text field value. When uploading files, their content type is inferred from the file name. You can manually @@ -694,16 +694,12 @@ override the inferred content type: $ http -f POST httpbin.org/post name='John Smith' cv@'~/files/data.bin;type=application/pdf' -Larger multipart uploads (i.e., ``--form`` requests with at least one ``file@path``) -are always streamed to avoid memory issues. Additionally, the display of the -request body on the terminal is suppressed. - -You can explicitly use ``--multipart`` to enforce ``multipart/form-data`` even -for form requests without any files: +To perform a ``multipart/form-data`` request even without any files, use +``--multipart`` instead of ``--form``: .. code-block:: bash - $ http --form --multipart --offline example.org hello=world + $ http --multipart --offline example.org hello=world .. code-block:: http @@ -718,6 +714,10 @@ for form requests without any files: world --c31279ab254f40aeb06df32b433cbccb-- +Larger multipart uploads are always streamed to avoid memory issues. +Additionally, the display of the request body on the terminal is suppressed +for larger uploads. + By default, HTTPie uses a random unique string as the boundary but you can use ``--boundary`` to specify a custom string instead: diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 952af6af..ddad1bff 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -18,7 +18,8 @@ from httpie.cli.constants import ( DEFAULT_FORMAT_OPTIONS, HTTP_GET, HTTP_POST, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT, OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED, OUT_RESP_BODY, PRETTY_MAP, - PRETTY_STDOUT_TTY_ONLY, SEPARATOR_CREDENTIALS, SEPARATOR_GROUP_ALL_ITEMS, + PRETTY_STDOUT_TTY_ONLY, RequestContentType, + SEPARATOR_CREDENTIALS, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE, OUTPUT_OPTIONS_DEFAULT_OFFLINE, ) @@ -84,6 +85,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser): ) # Arguments processing and environment setup. self._apply_no_options(no_options) + self._process_request_content_type() self._process_download_options() self._setup_standard_streams() self._process_output_options() @@ -97,12 +99,21 @@ class HTTPieArgumentParser(argparse.ArgumentParser): self._process_auth() return self.args + def _process_request_content_type(self): + rct = self.args.request_content_type + self.args.json = rct is RequestContentType.JSON + self.args.multipart = rct is RequestContentType.MULTIPART + self.args.form = rct in { + RequestContentType.FORM, + RequestContentType.MULTIPART, + } + def _process_url(self): if not URL_SCHEME_RE.match(self.args.url): if os.path.basename(self.env.program_name) == 'https': scheme = 'https://' else: - scheme = self.args.default_scheme + "://" + scheme = self.args.default_scheme + '://' # See if we're using curl style shorthand for localhost (:3000/foo) shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url) diff --git a/httpie/cli/constants.py b/httpie/cli/constants.py index a97ac252..7b91d674 100644 --- a/httpie/cli/constants.py +++ b/httpie/cli/constants.py @@ -1,6 +1,7 @@ """Parsing and processing of CLI input (args, auth credentials, files, stdin). """ +import enum import re @@ -10,6 +11,9 @@ import re # ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) # +from enum import Enum + + URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE) HTTP_POST = 'POST' @@ -102,3 +106,9 @@ UNSORTED_FORMAT_OPTIONS_STRING = ','.join( OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY + + +class RequestContentType(enum.Enum): + FORM = enum.auto() + MULTIPART = enum.auto() + JSON = enum.auto() diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 140534fd..d3f17f0f 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -15,7 +15,8 @@ from httpie.cli.constants import ( DEFAULT_FORMAT_OPTIONS, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT, OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, - SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, SORTED_FORMAT_OPTIONS_STRING, + RequestContentType, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, + SORTED_FORMAT_OPTIONS_STRING, UNSORTED_FORMAT_OPTIONS_STRING, ) from httpie.output.formatters.colors import ( @@ -141,7 +142,9 @@ content_type = parser.add_argument_group( content_type.add_argument( '--json', '-j', - action='store_true', + action='store_const', + const=RequestContentType.JSON, + dest='request_content_type', help=''' (default) Data items from the command line are serialized as a JSON object. The Content-Type and Accept headers are set to application/json @@ -151,7 +154,9 @@ content_type.add_argument( ) content_type.add_argument( '--form', '-f', - action='store_true', + action='store_const', + const=RequestContentType.FORM, + dest='request_content_type', help=''' Data items from the command line are serialized as form fields. @@ -163,11 +168,12 @@ content_type.add_argument( ) content_type.add_argument( '--multipart', - default=False, - action='store_true', + action='store_const', + const=RequestContentType.MULTIPART, + dest='request_content_type', help=''' - Force the request to be encoded as multipart/form-data even without - any file fields. Only has effect only together with --form. + Similar to --form, but always sends a multipart/form-data + request (i.e., even without files). ''' ) diff --git a/httpie/client.py b/httpie/client.py index 879a2151..68ac50b8 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -139,11 +139,12 @@ def max_headers(limit): def compress_body(request: requests.PreparedRequest, always: bool): deflater = zlib.compressobj() - body_bytes = ( - request.body - if isinstance(request.body, bytes) - else request.body.encode() - ) + if isinstance(request.body, str): + body_bytes = request.body.encode() + elif hasattr(request.body, 'read'): + body_bytes = request.body.read() + else: + body_bytes = request.body deflated_data = deflater.compress(body_bytes) deflated_data += deflater.flush() is_economical = len(deflated_data) < len(body_bytes) @@ -276,7 +277,7 @@ def make_request_kwargs( headers.update(args.headers) headers = finalize_headers(headers) - if args.form and (files or args.multipart): + if (args.form and files) or args.multipart: data, headers['Content-Type'] = get_multipart_data_and_content_type( data=data, files=files, diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 51dc58cc..8e639208 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -125,6 +125,8 @@ class EncodedStream(BaseStream): def iter_body(self) -> Iterable[bytes]: for line, lf in self.msg.iter_lines(self.CHUNK_SIZE): + if isinstance(line, MultipartEncoder): + raise LargeUploadSuppressedError() if b'\0' in line: raise BinarySuppressedError() yield line.decode(self.msg.encoding) \ diff --git a/httpie/ssl.py b/httpie/ssl.py index 84504380..f41b8020 100644 --- a/httpie/ssl.py +++ b/httpie/ssl.py @@ -52,7 +52,7 @@ class HTTPieHTTPSAdapter(HTTPAdapter): verify: bool, ssl_version: str = None, ciphers: str = None, - ) -> ssl.SSLContext: + ) -> 'ssl.SSLContext': return create_urllib3_context( ciphers=ciphers, ssl_version=resolve_ssl_version(ssl_version), diff --git a/httpie/uploads.py b/httpie/uploads.py index d883d746..f9ca4828 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -7,7 +7,7 @@ from httpie.cli.dicts import RequestDataDict, RequestFilesDict # Multipart uploads smaller than this size gets buffered (otherwise streamed). # NOTE: Unbuffered upload requests cannot be displayed on the terminal. -UPLOAD_BUFFER = 1024 * 100 +MULTIPART_UPLOAD_BUFFER = 1024 * 1000 def get_multipart_data_and_content_type( @@ -28,5 +28,5 @@ def get_multipart_data_and_content_type( else: content_type = encoder.content_type - data = encoder.to_string() if encoder.len < UPLOAD_BUFFER else encoder + data = encoder.to_string() if encoder.len < MULTIPART_UPLOAD_BUFFER else encoder return data, content_type diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 2a12aad6..9c070dac 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -55,7 +55,7 @@ class TestMultipartFormDataFileUpload: assert r.count(FILE_CONTENT) == 2 assert 'Content-Type: image/vnd.microsoft.icon' in r - @mock.patch('httpie.uploads.UPLOAD_BUFFER', 0) + @mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0) def test_large_upload_display_suppressed(self, httpbin): r = http( '--form', @@ -79,9 +79,8 @@ class TestMultipartFormDataFileUpload: assert HTTP_OK in r assert FORM_CONTENT_TYPE in r - def test_form_no_files_multipart(self, httpbin): + def test_multipart(self, httpbin): r = http( - '--form', '--verbose', '--multipart', httpbin.url + '/post', @@ -92,12 +91,38 @@ class TestMultipartFormDataFileUpload: assert FORM_CONTENT_TYPE not in r assert 'multipart/form-data' in r + def test_multipart_too_large_for_terminal(self, httpbin): + with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0): + r = http( + '--verbose', + '--multipart', + httpbin.url + '/post', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert HTTP_OK in r + assert FORM_CONTENT_TYPE not in r + assert 'multipart/form-data' in r + + def test_multipart_too_large_for_terminal_non_pretty(self, httpbin): + with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0): + r = http( + '--verbose', + '--multipart', + '--pretty=none', + httpbin.url + '/post', + 'AAAA=AAA', + 'BBB=BBB', + ) + assert HTTP_OK in r + assert FORM_CONTENT_TYPE not in r + assert 'multipart/form-data' in r + def test_form_multipart_custom_boundary(self, httpbin): boundary = 'HTTPIE_FTW' r = http( '--print=HB', '--check-status', - '--form', '--multipart', f'--boundary={boundary}', httpbin.url + '/post', @@ -112,7 +137,6 @@ class TestMultipartFormDataFileUpload: r = http( '--print=HB', '--check-status', - '--form', '--multipart', f'--boundary={boundary}', httpbin.url + '/post', @@ -128,7 +152,6 @@ class TestMultipartFormDataFileUpload: boundary_in_header = 'HEADER_BOUNDARY' boundary_in_body = 'BODY_BOUNDARY' r = http( - '--form', '--print=HB', '--check-status', '--multipart',