holm/httpie-cli
1
1
Fork 0
forked from extern/httpie-cli
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/uploads2020

Browse Source 
# Conflicts:
#	httpie/cli/argparser.py
#	httpie/uploads.py
This commit is contained in:
Jakub Roztocil 2020-09-25 14:46:19 +02:00
commit b7754f92ce
9 changed files with 94 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.rst
View File

@ -684,7 +684,7 @@ submitted:
<input type="file" name="cv" />
</form>
Note that ``@`` is used to simulate a file upload form field, whereas
Please note that ``@`` is used to simulate a file upload form field, whereas
``=@`` just embeds the file content as a regular text field value.
When uploading files, their content type is inferred from the file name. You can manually
@ -694,16 +694,12 @@ override the inferred content type:
$ http -f POST httpbin.org/post name='John Smith' cv@'~/files/data.bin;type=application/pdf'
Larger multipart uploads (i.e., ``--form`` requests with at least one ``file@path``)
are always streamed to avoid memory issues. Additionally, the display of the
request body on the terminal is suppressed.
You can explicitly use ``--multipart`` to enforce ``multipart/form-data`` even
for form requests without any files:
To perform a ``multipart/form-data`` request even without any files, use
``--multipart`` instead of ``--form``:
.. code-block:: bash
$ http --form --multipart --offline example.org hello=world
$ http --multipart --offline example.org hello=world
.. code-block:: http
@ -718,6 +714,10 @@ for form requests without any files:
world
--c31279ab254f40aeb06df32b433cbccb--
Larger multipart uploads are always streamed to avoid memory issues.
Additionally, the display of the request body on the terminal is suppressed
for larger uploads.
By default, HTTPie uses a random unique string as the boundary but you can use
``--boundary`` to specify a custom string instead:

41
httpie/cli/argparser.py
View File

@ -17,7 +17,8 @@ from httpie.cli.argtypes import (
from httpie.cli.constants import (
HTTP_GET, HTTP_POST, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
OUTPUT_OPTIONS_DEFAULT_OFFLINE, OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED,
OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, SEPARATOR_CREDENTIALS,
OUT_RESP_BODY, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, RequestContentType,
SEPARATOR_CREDENTIALS,
SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
)
from httpie.cli.exceptions import ParseError
@ -82,6 +83,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
)
# Arguments processing and environment setup.
self._apply_no_options(no_options)
self._process_request_content_type()
self._process_download_options()
self._setup_standard_streams()
self._process_output_options()
@ -95,12 +97,21 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
self._process_auth()
return self.args
def _process_request_content_type(self):
rct = self.args.request_content_type
self.args.json = rct is RequestContentType.JSON
self.args.multipart = rct is RequestContentType.MULTIPART
self.args.form = rct in {
RequestContentType.FORM,
RequestContentType.MULTIPART,
}
def _process_url(self):
if not URL_SCHEME_RE.match(self.args.url):
if os.path.basename(self.env.program_name) == 'https':
scheme = 'https://'
else:
scheme = self.args.default_scheme + "://"
scheme = self.args.default_scheme + '://'
# See if we're using curl style shorthand for localhost (:3000/foo)
shorthand = re.match(r'^:(?!:)(\d*)(/?.*)$', self.args.url)
@ -163,8 +174,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
if self.args.quiet:
self.env.stderr = self.env.devnull
if not (
self.args.output_file_specified and not self.args.download):
if not (self.args.output_file_specified and not self.args.download):
self.env.stdout = self.env.devnull
def _process_auth(self):
@ -192,8 +202,8 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
plugin = plugin_manager.get_auth_plugin(self.args.auth_type)()
if (not self.args.ignore_netrc
and self.args.auth is None
and plugin.netrc_parse):
and self.args.auth is None
and plugin.netrc_parse):
# Only host needed, so it’s OK URL not finalized.
netrc_credentials = get_netrc_auth(self.args.url)
if netrc_credentials:
@ -221,7 +231,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
credentials = parse_auth(self.args.auth)
if (not credentials.has_password()
and plugin.prompt_password):
and plugin.prompt_password):
if self.args.ignore_stdin:
# Non-tty stdin read by now
self.error(
@ -275,13 +285,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
'data (key=value) cannot be mixed. Pass '
'--ignore-stdin to let key/value take priority. '
'See https://httpie.org/doc#scripting for details.')
buffer = getattr(fd, 'buffer', fd)
# if fd is self.env.stdin and not self.args.chunked:
# self.args.data = buffer.read()
# else:
# self.args.data = buffer
# print(type(fd))
self.args.data = buffer
self.args.data = getattr(fd, 'buffer', fd).read()
def _guess_method(self):
"""Set `args.method` if not specified to either POST or GET
@ -317,8 +321,8 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
has_data = (
self.has_stdin_data
or any(
item.sep in SEPARATOR_GROUP_DATA_ITEMS
for item in self.args.request_items)
item.sep in SEPARATOR_GROUP_DATA_ITEMS
for item in self.args.request_items)
)
self.args.method = HTTP_POST if has_data else HTTP_GET
@ -421,12 +425,11 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
if self.args.download_resume:
self.error('--continue only works with --download')
if self.args.download_resume and not (
self.args.download and self.args.output_file):
self.args.download and self.args.output_file):
self.error('--continue requires --output to be specified')
def _process_format_options(self):
parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
for options_group in self.args.format_options or []:
parsed_options = parse_format_options(options_group,
defaults=parsed_options)
parsed_options = parse_format_options(options_group, defaults=parsed_options)
self.args.format_options = parsed_options

10
httpie/cli/constants.py
View File

@ -1,6 +1,7 @@
"""Parsing and processing of CLI input (args, auth credentials, files, stdin).
"""
import enum
import re
@ -10,6 +11,9 @@ import re
# ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
# <https://tools.ietf.org/html/rfc3986#section-3.1>
from enum import Enum
URL_SCHEME_RE = re.compile(r'^[a-z][a-z0-9.+-]*://', re.IGNORECASE)
HTTP_POST = 'POST'
@ -102,3 +106,9 @@ UNSORTED_FORMAT_OPTIONS_STRING = ','.join(
OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY
class RequestContentType(enum.Enum):
FORM = enum.auto()
MULTIPART = enum.auto()
JSON = enum.auto()

20
httpie/cli/definition.py
View File

@ -15,7 +15,8 @@ from httpie.cli.constants import (
DEFAULT_FORMAT_OPTIONS, OUTPUT_OPTIONS,
OUTPUT_OPTIONS_DEFAULT, OUT_REQ_BODY, OUT_REQ_HEAD,
OUT_RESP_BODY, OUT_RESP_HEAD, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY,
SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY, SORTED_FORMAT_OPTIONS_STRING,
RequestContentType, SEPARATOR_GROUP_ALL_ITEMS, SEPARATOR_PROXY,
SORTED_FORMAT_OPTIONS_STRING,
UNSORTED_FORMAT_OPTIONS_STRING,
)
from httpie.output.formatters.colors import (
@ -141,7 +142,9 @@ content_type = parser.add_argument_group(
content_type.add_argument(
'--json', '-j',
action='store_true',
action='store_const',
const=RequestContentType.JSON,
dest='request_content_type',
help='''
(default) Data items from the command line are serialized as a JSON object.
The Content-Type and Accept headers are set to application/json
@ -151,7 +154,9 @@ content_type.add_argument(
)
content_type.add_argument(
'--form', '-f',
action='store_true',
action='store_const',
const=RequestContentType.FORM,
dest='request_content_type',
help='''
Data items from the command line are serialized as form fields.
@ -163,11 +168,12 @@ content_type.add_argument(
)
content_type.add_argument(
'--multipart',
default=False,
action='store_true',
action='store_const',
const=RequestContentType.MULTIPART,
dest='request_content_type',
help='''
Force the request to be encoded as multipart/form-data even without
any file fields. Only has effect only together with --form.
Similar to --form, but always sends a multipart/form-data
request (i.e., even without files).
'''
)

13
httpie/client.py
View File

@ -144,11 +144,12 @@ def max_headers(limit):
def compress_body(request: requests.PreparedRequest, always: bool):
deflater = zlib.compressobj()
body_bytes = (
request.body
if isinstance(request.body, bytes)
else request.body.encode()
)
if isinstance(request.body, str):
body_bytes = request.body.encode()
elif hasattr(request.body, 'read'):
body_bytes = request.body.read()
else:
body_bytes = request.body
deflated_data = deflater.compress(body_bytes)
deflated_data += deflater.flush()
is_economical = len(deflated_data) < len(body_bytes)
@ -282,7 +283,7 @@ def make_request_kwargs(
headers.update(args.headers)
headers = finalize_headers(headers)
if args.form and (files or args.multipart):
if (args.form and files) or args.multipart:
data, headers['Content-Type'] = get_multipart_data_and_content_type(
data=data,
files=files,

2
httpie/output/streams.py
View File

@ -132,6 +132,8 @@ class EncodedStream(BaseStream):
def iter_body(self) -> Iterable[bytes]:
for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
if isinstance(line, MultipartEncoder):
raise LargeUploadSuppressedError()
if b'\0' in line:
raise BinarySuppressedError()
yield line.decode(self.msg.encoding) \

2
httpie/ssl.py
View File

@ -52,7 +52,7 @@ class HTTPieHTTPSAdapter(HTTPAdapter):
verify: bool,
ssl_version: str = None,
ciphers: str = None,
) -> ssl.SSLContext:
) -> 'ssl.SSLContext':
return create_urllib3_context(
ciphers=ciphers,
ssl_version=resolve_ssl_version(ssl_version),

4
httpie/uploads.py
View File

@ -8,7 +8,7 @@ from httpie.cli.dicts import RequestDataDict, RequestFilesDict
# Multipart uploads smaller than this size gets buffered (otherwise streamed).
# NOTE: Unbuffered upload requests cannot be displayed on the terminal.
UPLOAD_BUFFER = 1024 * 100
MULTIPART_UPLOAD_BUFFER = 1024 * 1000
def get_multipart_data_and_content_type(
@ -29,7 +29,7 @@ def get_multipart_data_and_content_type(
else:
content_type = encoder.content_type
data = encoder.to_string() if 0 and encoder.len < UPLOAD_BUFFER else encoder
data = encoder.to_string() if 0 and encoder.len < MULTIPART_UPLOAD_BUFFER else encoder
return data, content_type

35
tests/test_uploads.py
View File

@ -55,7 +55,7 @@ class TestMultipartFormDataFileUpload:
assert r.count(FILE_CONTENT) == 2
assert 'Content-Type: image/vnd.microsoft.icon' in r
@mock.patch('httpie.uploads.UPLOAD_BUFFER', 0)
@mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0)
def test_large_upload_display_suppressed(self, httpbin):
r = http(
'--form',
@ -79,9 +79,8 @@ class TestMultipartFormDataFileUpload:
assert HTTP_OK in r
assert FORM_CONTENT_TYPE in r
def test_form_no_files_multipart(self, httpbin):
def test_multipart(self, httpbin):
r = http(
'--form',
'--verbose',
'--multipart',
httpbin.url + '/post',
@ -92,12 +91,38 @@ class TestMultipartFormDataFileUpload:
assert FORM_CONTENT_TYPE not in r
assert 'multipart/form-data' in r
def test_multipart_too_large_for_terminal(self, httpbin):
with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0):
r = http(
'--verbose',
'--multipart',
httpbin.url + '/post',
'AAAA=AAA',
'BBB=BBB',
)
assert HTTP_OK in r
assert FORM_CONTENT_TYPE not in r
assert 'multipart/form-data' in r
def test_multipart_too_large_for_terminal_non_pretty(self, httpbin):
with mock.patch('httpie.uploads.MULTIPART_UPLOAD_BUFFER', 0):
r = http(
'--verbose',
'--multipart',
'--pretty=none',
httpbin.url + '/post',
'AAAA=AAA',
'BBB=BBB',
)
assert HTTP_OK in r
assert FORM_CONTENT_TYPE not in r
assert 'multipart/form-data' in r
def test_form_multipart_custom_boundary(self, httpbin):
boundary = 'HTTPIE_FTW'
r = http(
'--print=HB',
'--check-status',
'--form',
'--multipart',
f'--boundary={boundary}',
httpbin.url + '/post',
@ -112,7 +137,6 @@ class TestMultipartFormDataFileUpload:
r = http(
'--print=HB',
'--check-status',
'--form',
'--multipart',
f'--boundary={boundary}',
httpbin.url + '/post',
@ -128,7 +152,6 @@ class TestMultipartFormDataFileUpload:
boundary_in_header = 'HEADER_BOUNDARY'
boundary_in_body = 'BODY_BOUNDARY'
r = http(
'--form',
'--print=HB',
'--check-status',
'--multipart',