Merge branch 'master' into feature/uploads2020

# Conflicts:
#	httpie/cli/argparser.py
#	httpie/uploads.py

README.rst

@@ -684,7 +684,7 @@ submitted:
         <input type="file" name="cv" />
     </form>
 
-Note that ``@`` is used to simulate a file upload form field, whereas
+Please note that ``@`` is used to simulate a file upload form field, whereas
 ``=@`` just embeds the file content as a regular text field value.
 
 When uploading files, their content type is inferred from the file name. You can manually
@@ -694,16 +694,12 @@ override the inferred content type:
 
    $ http -f POST httpbin.org/post name='John Smith' cv@'~/files/data.bin;type=application/pdf'
 
-Larger multipart uploads (i.e., ``--form`` requests with at least one ``file@path``)
+To perform a ``multipart/form-data`` request even without any files, use
-are always streamed to avoid memory issues. Additionally, the display of the
+``--multipart`` instead of ``--form``:
-request body on the terminal is suppressed.
-
-You can explicitly use ``--multipart`` to enforce ``multipart/form-data`` even
-for form requests without any files:
 
 .. code-block:: bash
 
-    $ http --form --multipart --offline example.org hello=world
+    $ http --multipart --offline example.org hello=world
 
 .. code-block:: http
 
@@ -718,6 +714,10 @@ for form requests without any files:
     world
     --c31279ab254f40aeb06df32b433cbccb--
 
+Larger multipart uploads are always streamed to avoid memory issues.
+Additionally, the display of the request body on the terminal is suppressed
+for larger uploads.
+
 By default, HTTPie uses a random unique string as the boundary but you can use
 ``--boundary`` to specify a custom string instead:
 

httpie/cli/argparser.py

@@ -17,7 +17,8 @@ from httpie.cli.argtypes import (
 from httpie.cli.constants import (
     HTTP_GET, HTTP_POST, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
     OUTPUT_OPTIONS_DEFAULT_OFFLINE, OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED,
-    OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, SEPARATOR_CREDENTIALS,
+    OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, RequestContentType,
+    SEPARATOR_CREDENTIALS,
     SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
 )
 from httpie.cli.exceptions import ParseError
@@ -82,6 +83,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
         )
         # Arguments processing and environment setup.
         self._apply_no_options(no_options)
+        self._process_request_content_type()
         self._process_download_options()
         self._setup_standard_streams()
         self._process_output_options()
@@ -95,12 +97,21 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
         self._process_auth()
         return self.args
 
+    def _process_request_content_type(self):
+        rct = self.args.request_content_type
+        self.args.json = rct is RequestContentType.JSON
+        self.args.multipart = rct is RequestContentType.MULTIPART
+        self.args.form = rct in {
+            RequestContentType.FORM,
+            RequestContentType.MULTIPART,
+        }
+
     def _process_url(self):
         if not URL_SCHEME_RE.match(self.args.url):
             if os.path.basename(self.env.program_name) == 'https':
                 scheme = 'https://'
             else:
-                scheme = self.args.default_scheme + "://"
+                scheme = self.args.default_scheme + '://'
 
             # See if we're using curl style shorthand for localhost (:3000/foo)
             shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
@@ -163,8 +174,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
 
         if self.args.quiet:
             self.env.stderr = self.env.devnull
-            if not (
+            if not (self.args.output_file_specified and not self.args.download):
-                self.args.output_file_specified and not self.args.download):
                 self.env.stdout = self.env.devnull
 
     def _process_auth(self):
@@ -192,8 +202,8 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
             plugin = plugin_manager.get_auth_plugin(self.args.auth_type)()
 
             if (not self.args.ignore_netrc
-                and self.args.auth is None
+                    and self.args.auth is None
-                and plugin.netrc_parse):
+                    and plugin.netrc_parse):
                 # Only host needed, so it’s OK URL not finalized.
                 netrc_credentials = get_netrc_auth(self.args.url)
                 if netrc_credentials:
@@ -221,7 +231,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
                     credentials = parse_auth(self.args.auth)
 
                 if (not credentials.has_password()
-                    and plugin.prompt_password):
+                        and plugin.prompt_password):
                     if self.args.ignore_stdin:
                         # Non-tty stdin read by now
                         self.error(
@@ -275,13 +285,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
                        'data (key=value) cannot be mixed. Pass '
                        '--ignore-stdin to let key/value take priority. '
                        'See https://httpie.org/doc#scripting for details.')
-        buffer = getattr(fd, 'buffer', fd)
+        self.args.data = getattr(fd, 'buffer', fd).read()
-        # if fd is self.env.stdin and not self.args.chunked:
-        #     self.args.data = buffer.read()
-        # else:
-        #     self.args.data = buffer
-        # print(type(fd))
-        self.args.data = buffer
 
     def _guess_method(self):
         """Set `args.method` if not specified to either POST or GET
@@ -317,8 +321,8 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
                 has_data = (
                     self.has_stdin_data
                     or any(
-                    item.sep in SEPARATOR_GROUP_DATA_ITEMS
+                        item.sep in SEPARATOR_GROUP_DATA_ITEMS
-                    for item in self.args.request_items)
+                        for item in self.args.request_items)
                 )
                 self.args.method = HTTP_POST if has_data else HTTP_GET
 
@@ -421,12 +425,11 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
             if self.args.download_resume:
                 self.error('--continue only works with --download')
         if self.args.download_resume and not (
-            self.args.download and self.args.output_file):
+                self.args.download and self.args.output_file):
             self.error('--continue requires --output to be specified')
 
     def _process_format_options(self):
         parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
         for options_group in self.args.format_options or []:
-            parsed_options = parse_format_options(options_group,
+            parsed_options = parse_format_options(options_group, defaults=parsed_options)
-                                                  defaults=parsed_options)
         self.args.format_options = parsed_options

httpie/cli/constants.py

@@ -1,6 +1,7 @@
 """Parsing and processing of CLI input (args, auth credentials, files, stdin).
 
 """
+import enum
 import re
 
 
@@ -10,6 +11,9 @@ import re
 
 # ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
 # <https://tools.ietf.org/html/rfc3986#section-3.1>
+from enum import Enum
+
+
 URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)
 
 HTTP_POST = 'POST'
@@ -102,3 +106,9 @@ UNSORTED_FORMAT_OPTIONS_STRING = ','.join(
 OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
 OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
 OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY
+
+
+class RequestContentType(enum.Enum):
+    FORM = enum.auto()
+    MULTIPART = enum.auto()
+    JSON = enum.auto()

httpie/cli/definition.py

@@ -15,7 +15,8 @@ from httpie.cli.constants import (
     DEFAULT_FORMAT_OPTIONS, OUTPUT_OPTIONS,
     OUTPUT_OPTIONS_DEFAULT, OUT_REQ_BODY, OUT_REQ_HEAD,
     OUT_RESP_BODY, OUT_RESP_HEAD, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY,
-    SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, SORTED_FORMAT_OPTIONS_STRING,
+    RequestContentType, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY,
+    SORTED_FORMAT_OPTIONS_STRING,
     UNSORTED_FORMAT_OPTIONS_STRING,
 )
 from httpie.output.formatters.colors import (
@@ -141,7 +142,9 @@ content_type = parser.add_argument_group(
 
 content_type.add_argument(
     '--json', '-j',
-    action='store_true',
+    action='store_const',
+    const=RequestContentType.JSON,
+    dest='request_content_type',
     help='''
     (default) Data items from the command line are serialized as a JSON object.
     The Content-Type and Accept headers are set to application/json
@@ -151,7 +154,9 @@ content_type.add_argument(
 )
 content_type.add_argument(
     '--form', '-f',
-    action='store_true',
+    action='store_const',
+    const=RequestContentType.FORM,
+    dest='request_content_type',
     help='''
     Data items from the command line are serialized as form fields.
 
@@ -163,11 +168,12 @@ content_type.add_argument(
 )
 content_type.add_argument(
     '--multipart',
-    default=False,
+    action='store_const',
-    action='store_true',
+    const=RequestContentType.MULTIPART,
+    dest='request_content_type',
     help='''
-    Force the request to be encoded as multipart/form-data even without
+    Similar to --form, but always sends a multipart/form-data
-    any file fields. Only has effect only together with --form.
+    request (i.e., even without files).
 
     '''
 )

httpie/client.py

@@ -144,11 +144,12 @@ def max_headers(limit):
 
 def compress_body(request: requests.PreparedRequest, always: bool):
     deflater = zlib.compressobj()
-    body_bytes = (
+    if isinstance(request.body, str):
-        request.body
+        body_bytes = request.body.encode()
-        if isinstance(request.body, bytes)
+    elif hasattr(request.body, 'read'):
-        else request.body.encode()
+        body_bytes = request.body.read()
-    )
+    else:
+        body_bytes = request.body
     deflated_data = deflater.compress(body_bytes)
     deflated_data += deflater.flush()
     is_economical = len(deflated_data) < len(body_bytes)
@@ -282,7 +283,7 @@ def make_request_kwargs(
     headers.update(args.headers)
     headers = finalize_headers(headers)
 
-    if args.form and (files or args.multipart):
+    if (args.form and files) or args.multipart:
         data, headers['Content-Type'] = get_multipart_data_and_content_type(
             data=data,
             files=files,

httpie/output/streams.py

@@ -132,6 +132,8 @@ class EncodedStream(BaseStream):
 
     def iter_body(self) -> Iterable[bytes]:
         for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
+            if isinstance(line, MultipartEncoder):
+                raise LargeUploadSuppressedError()
             if b'\0' in line:
                 raise BinarySuppressedError()
             yield line.decode(self.msg.encoding) \

httpie/ssl.py

@@ -52,7 +52,7 @@ class HTTPieHTTPSAdapter(HTTPAdapter):
         verify: bool,
         ssl_version: str = None,
         ciphers: str = None,
-    ) -> ssl.SSLContext:
+    ) -> 'ssl.SSLContext':
         return create_urllib3_context(
             ciphers=ciphers,
             ssl_version=resolve_ssl_version(ssl_version),

httpie/uploads.py

@@ -8,7 +8,7 @@ from httpie.cli.dicts import RequestDataDict, RequestFilesDict
 
 # Multipart uploads smaller than this size gets buffered (otherwise streamed).
 # NOTE: Unbuffered upload requests cannot be displayed on the terminal.
-UPLOAD_BUFFER = 1024 * 100
+MULTIPART_UPLOAD_BUFFER = 1024 * 1000
 
 
 def get_multipart_data_and_content_type(
@@ -29,7 +29,7 @@ def get_multipart_data_and_content_type(
     else:
         content_type = encoder.content_type
 
-    data = encoder.to_string() if 0 and encoder.len < UPLOAD_BUFFER else encoder
+    data = encoder.to_string() if 0 and encoder.len < MULTIPART_UPLOAD_BUFFER else encoder
     return data, content_type
 
 

tests/test_uploads.py

@@ -55,7 +55,7 @@ class TestMultipartFormDataFileUpload:
         assert r.count(FILE_CONTENT) == 2
         assert 'Content-Type: image/vnd.microsoft.icon' in r
 
-    @mock.patch('httpie.uploads.UPLOAD_BUFFER', 0)
+    @mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0)
     def test_large_upload_display_suppressed(self, httpbin):
         r = http(
             '--form',
@@ -79,9 +79,8 @@ class TestMultipartFormDataFileUpload:
         assert HTTP_OK in r
         assert FORM_CONTENT_TYPE in r
 
-    def test_form_no_files_multipart(self, httpbin):
+    def test_multipart(self, httpbin):
         r = http(
-            '--form',
             '--verbose',
             '--multipart',
             httpbin.url + '/post',
@@ -92,12 +91,38 @@ class TestMultipartFormDataFileUpload:
         assert FORM_CONTENT_TYPE not in r
         assert 'multipart/form-data' in r
 
+    def test_multipart_too_large_for_terminal(self, httpbin):
+        with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0):
+            r = http(
+                '--verbose',
+                '--multipart',
+                httpbin.url + '/post',
+                'AAAA=AAA',
+                'BBB=BBB',
+            )
+        assert HTTP_OK in r
+        assert FORM_CONTENT_TYPE not in r
+        assert 'multipart/form-data' in r
+
+    def test_multipart_too_large_for_terminal_non_pretty(self, httpbin):
+        with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0):
+            r = http(
+                '--verbose',
+                '--multipart',
+                '--pretty=none',
+                httpbin.url + '/post',
+                'AAAA=AAA',
+                'BBB=BBB',
+            )
+        assert HTTP_OK in r
+        assert FORM_CONTENT_TYPE not in r
+        assert 'multipart/form-data' in r
+
     def test_form_multipart_custom_boundary(self, httpbin):
         boundary = 'HTTPIE_FTW'
         r = http(
             '--print=HB',
             '--check-status',
-            '--form',
             '--multipart',
             f'--boundary={boundary}',
             httpbin.url + '/post',
@@ -112,7 +137,6 @@ class TestMultipartFormDataFileUpload:
         r = http(
             '--print=HB',
             '--check-status',
-            '--form',
             '--multipart',
             f'--boundary={boundary}',
             httpbin.url + '/post',
@@ -128,7 +152,6 @@ class TestMultipartFormDataFileUpload:
         boundary_in_header = 'HEADER_BOUNDARY'
         boundary_in_body = 'BODY_BOUNDARY'
         r = http(
-            '--form',
             '--print=HB',
             '--check-status',
             '--multipart',