forked from extern/httpie-cli
parent
d32c8cab12
commit
6cd934d1b8
@ -8,6 +8,7 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
|
|||||||
|
|
||||||
`2.3.0-dev`_ (unreleased)
|
`2.3.0-dev`_ (unreleased)
|
||||||
-------------------------
|
-------------------------
|
||||||
|
* Added support for multipart upload streaming (`#684`_).
|
||||||
* Added support for combining cookies specified on the CLI and in a session file (`#932`_).
|
* Added support for combining cookies specified on the CLI and in a session file (`#932`_).
|
||||||
* Added out of the box SOCKS support with no extra installation (`#904`_).
|
* Added out of the box SOCKS support with no extra installation (`#904`_).
|
||||||
* Added ``--quiet, -q`` flag to enforce silent behaviour.
|
* Added ``--quiet, -q`` flag to enforce silent behaviour.
|
||||||
@ -451,6 +452,7 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
|
|||||||
.. _#128: https://github.com/jakubroztocil/httpie/issues/128
|
.. _#128: https://github.com/jakubroztocil/httpie/issues/128
|
||||||
.. _#488: https://github.com/jakubroztocil/httpie/issues/488
|
.. _#488: https://github.com/jakubroztocil/httpie/issues/488
|
||||||
.. _#668: https://github.com/jakubroztocil/httpie/issues/668
|
.. _#668: https://github.com/jakubroztocil/httpie/issues/668
|
||||||
|
.. _#684: https://github.com/jakubroztocil/httpie/issues/684
|
||||||
.. _#718: https://github.com/jakubroztocil/httpie/issues/718
|
.. _#718: https://github.com/jakubroztocil/httpie/issues/718
|
||||||
.. _#719: https://github.com/jakubroztocil/httpie/issues/719
|
.. _#719: https://github.com/jakubroztocil/httpie/issues/719
|
||||||
.. _#840: https://github.com/jakubroztocil/httpie/issues/840
|
.. _#840: https://github.com/jakubroztocil/httpie/issues/840
|
||||||
|
@ -694,6 +694,10 @@ override the inferred content type:
|
|||||||
|
|
||||||
$ http -f POST httpbin.org/post name='John Smith' cv@'~/files/data.bin;type=application/pdf'
|
$ http -f POST httpbin.org/post name='John Smith' cv@'~/files/data.bin;type=application/pdf'
|
||||||
|
|
||||||
|
Larger multipart uploads (i.e., ``--form`` requests with at least one ``file@path``)
|
||||||
|
are always streamed to avoid memory issues. Additionally, the display of the
|
||||||
|
request body on the terminal is suppressed.
|
||||||
|
|
||||||
|
|
||||||
HTTP headers
|
HTTP headers
|
||||||
============
|
============
|
||||||
|
@ -16,6 +16,7 @@ PACKAGES = [
|
|||||||
'httpie',
|
'httpie',
|
||||||
'Pygments',
|
'Pygments',
|
||||||
'requests',
|
'requests',
|
||||||
|
'requests-toolbelt',
|
||||||
'certifi',
|
'certifi',
|
||||||
'urllib3',
|
'urllib3',
|
||||||
'idna',
|
'idna',
|
||||||
|
@ -99,15 +99,13 @@ def process_file_upload_arg(arg: KeyValueArg) -> Tuple[str, IO, str]:
|
|||||||
parts = arg.value.split(SEPARATOR_FILE_UPLOAD_TYPE)
|
parts = arg.value.split(SEPARATOR_FILE_UPLOAD_TYPE)
|
||||||
filename = parts[0]
|
filename = parts[0]
|
||||||
mime_type = parts[1] if len(parts) > 1 else None
|
mime_type = parts[1] if len(parts) > 1 else None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(os.path.expanduser(filename), 'rb') as f:
|
f = open(os.path.expanduser(filename), 'rb')
|
||||||
contents = f.read()
|
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
raise ParseError('"%s": %s' % (arg.orig, e))
|
raise ParseError('"%s": %s' % (arg.orig, e))
|
||||||
return (
|
return (
|
||||||
os.path.basename(filename),
|
os.path.basename(filename),
|
||||||
BytesIO(contents),
|
f,
|
||||||
mime_type or get_content_type(filename),
|
mime_type or get_content_type(filename),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -11,14 +11,15 @@ from urllib.parse import urlparse, urlunparse
|
|||||||
import requests
|
import requests
|
||||||
# noinspection PyPackageRequirements
|
# noinspection PyPackageRequirements
|
||||||
import urllib3
|
import urllib3
|
||||||
|
|
||||||
from httpie import __version__
|
from httpie import __version__
|
||||||
from httpie.cli.dicts import RequestHeadersDict
|
from httpie.cli.dicts import RequestHeadersDict
|
||||||
from httpie.plugins.registry import plugin_manager
|
from httpie.plugins.registry import plugin_manager
|
||||||
from httpie.sessions import get_httpie_session
|
from httpie.sessions import get_httpie_session
|
||||||
from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter
|
from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter
|
||||||
|
from httpie.uploads import get_multipart_data
|
||||||
from httpie.utils import get_expired_cookies, repr_dict
|
from httpie.utils import get_expired_cookies, repr_dict
|
||||||
|
|
||||||
|
|
||||||
urllib3.disable_warnings()
|
urllib3.disable_warnings()
|
||||||
|
|
||||||
FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8'
|
FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8'
|
||||||
@ -256,6 +257,7 @@ def make_request_kwargs(
|
|||||||
Translate our `args` into `requests.Request` keyword arguments.
|
Translate our `args` into `requests.Request` keyword arguments.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
files = args.files
|
||||||
# Serialize JSON data, if needed.
|
# Serialize JSON data, if needed.
|
||||||
data = args.data
|
data = args.data
|
||||||
auto_json = data and not args.form
|
auto_json = data and not args.form
|
||||||
@ -274,6 +276,10 @@ def make_request_kwargs(
|
|||||||
headers.update(args.headers)
|
headers.update(args.headers)
|
||||||
headers = finalize_headers(headers)
|
headers = finalize_headers(headers)
|
||||||
|
|
||||||
|
if args.form and files:
|
||||||
|
data, headers['Content-Type'] = get_multipart_data(data, files)
|
||||||
|
files = None
|
||||||
|
|
||||||
kwargs = {
|
kwargs = {
|
||||||
'method': args.method.lower(),
|
'method': args.method.lower(),
|
||||||
'url': args.url,
|
'url': args.url,
|
||||||
@ -281,7 +287,7 @@ def make_request_kwargs(
|
|||||||
'data': data,
|
'data': data,
|
||||||
'auth': args.auth,
|
'auth': args.auth,
|
||||||
'params': args.params,
|
'params': args.params,
|
||||||
'files': args.files,
|
'files': files,
|
||||||
}
|
}
|
||||||
|
|
||||||
return kwargs
|
return kwargs
|
||||||
|
@ -4,6 +4,7 @@ from typing import Callable, Iterable, Union
|
|||||||
from httpie.context import Environment
|
from httpie.context import Environment
|
||||||
from httpie.models import HTTPMessage
|
from httpie.models import HTTPMessage
|
||||||
from httpie.output.processing import Conversion, Formatting
|
from httpie.output.processing import Conversion, Formatting
|
||||||
|
from requests_toolbelt import MultipartEncoder
|
||||||
|
|
||||||
|
|
||||||
BINARY_SUPPRESSED_NOTICE = (
|
BINARY_SUPPRESSED_NOTICE = (
|
||||||
@ -14,11 +15,29 @@ BINARY_SUPPRESSED_NOTICE = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class BinarySuppressedError(Exception):
|
LARGE_UPLOAD_SUPPRESSED_NOTICE = (
|
||||||
|
b'\n'
|
||||||
|
b'+--------------------------------------------------------+\n'
|
||||||
|
b'| NOTE: large form upload data not shown in the terminal |\n'
|
||||||
|
b'+--------------------------------------------------------+'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DataSuppressedError(Exception):
|
||||||
|
message = None
|
||||||
|
|
||||||
|
|
||||||
|
class BinarySuppressedError(DataSuppressedError):
|
||||||
|
"""An error indicating that the body is binary and won't be written,
|
||||||
|
e.g., for terminal output)."""
|
||||||
|
message = BINARY_SUPPRESSED_NOTICE
|
||||||
|
|
||||||
|
|
||||||
|
class LargeUploadSuppressedError(DataSuppressedError):
|
||||||
"""An error indicating that the body is binary and won't be written,
|
"""An error indicating that the body is binary and won't be written,
|
||||||
e.g., for terminal output)."""
|
e.g., for terminal output)."""
|
||||||
|
|
||||||
message = BINARY_SUPPRESSED_NOTICE
|
message = LARGE_UPLOAD_SUPPRESSED_NOTICE
|
||||||
|
|
||||||
|
|
||||||
class BaseStream:
|
class BaseStream:
|
||||||
@ -63,7 +82,7 @@ class BaseStream:
|
|||||||
yield chunk
|
yield chunk
|
||||||
if self.on_body_chunk_downloaded:
|
if self.on_body_chunk_downloaded:
|
||||||
self.on_body_chunk_downloaded(chunk)
|
self.on_body_chunk_downloaded(chunk)
|
||||||
except BinarySuppressedError as e:
|
except DataSuppressedError as e:
|
||||||
if self.with_headers:
|
if self.with_headers:
|
||||||
yield b'\n'
|
yield b'\n'
|
||||||
yield e.message
|
yield e.message
|
||||||
@ -184,6 +203,8 @@ class BufferedPrettyStream(PrettyStream):
|
|||||||
body = bytearray()
|
body = bytearray()
|
||||||
|
|
||||||
for chunk in self.msg.iter_body(self.CHUNK_SIZE):
|
for chunk in self.msg.iter_body(self.CHUNK_SIZE):
|
||||||
|
if isinstance(chunk, MultipartEncoder):
|
||||||
|
raise LargeUploadSuppressedError()
|
||||||
if not converter and b'\0' in chunk:
|
if not converter and b'\0' in chunk:
|
||||||
converter = self.conversion.get_converter(self.mime)
|
converter = self.conversion.get_converter(self.mime)
|
||||||
if not converter:
|
if not converter:
|
||||||
|
20
httpie/uploads.py
Normal file
20
httpie/uploads.py
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
from typing import Tuple, Union
|
||||||
|
|
||||||
|
from httpie.cli.dicts import RequestDataDict, RequestFilesDict
|
||||||
|
from requests_toolbelt import MultipartEncoder
|
||||||
|
|
||||||
|
|
||||||
|
# Multipart uploads smaller than this size gets buffered (otherwise streamed).
|
||||||
|
# NOTE: Unbuffered upload requests cannot be displayed on the terminal.
|
||||||
|
UPLOAD_BUFFER = 1024 * 100
|
||||||
|
|
||||||
|
|
||||||
|
def get_multipart_data(
|
||||||
|
data: RequestDataDict,
|
||||||
|
files: RequestFilesDict
|
||||||
|
) -> Tuple[Union[MultipartEncoder, bytes], str]:
|
||||||
|
fields = list(data.items()) + list(files.items())
|
||||||
|
encoder = MultipartEncoder(fields=fields)
|
||||||
|
content_type = encoder.content_type
|
||||||
|
data = encoder.to_string() if encoder.len < UPLOAD_BUFFER else encoder
|
||||||
|
return data, content_type
|
@ -111,3 +111,5 @@ def get_expired_cookies(
|
|||||||
for cookie in cookies
|
for cookie in cookies
|
||||||
if cookie.get('expires', float('Inf')) <= now
|
if cookie.get('expires', float('Inf')) <= now
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
1
setup.py
1
setup.py
@ -38,6 +38,7 @@ tests_require = [
|
|||||||
install_requires = [
|
install_requires = [
|
||||||
'requests[socks]>=2.22.0',
|
'requests[socks]>=2.22.0',
|
||||||
'Pygments>=2.5.2',
|
'Pygments>=2.5.2',
|
||||||
|
'requests-toolbelt>=0.9.1',
|
||||||
]
|
]
|
||||||
install_requires_win_only = [
|
install_requires_win_only = [
|
||||||
'colorama>=0.2.4',
|
'colorama>=0.2.4',
|
||||||
|
@ -1,8 +1,10 @@
|
|||||||
import os
|
import os
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from httpie.cli.exceptions import ParseError
|
from httpie.cli.exceptions import ParseError
|
||||||
|
from httpie.output.streams import LARGE_UPLOAD_SUPPRESSED_NOTICE
|
||||||
from httpie.status import ExitStatus
|
from httpie.status import ExitStatus
|
||||||
from utils import MockEnvironment, http, HTTP_OK
|
from utils import MockEnvironment, http, HTTP_OK
|
||||||
from fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT
|
from fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT
|
||||||
@ -39,15 +41,32 @@ class TestMultipartFormDataFileUpload:
|
|||||||
assert r.count('Content-Type: text/plain') == 2
|
assert r.count('Content-Type: text/plain') == 2
|
||||||
|
|
||||||
def test_upload_custom_content_type(self, httpbin):
|
def test_upload_custom_content_type(self, httpbin):
|
||||||
r = http('--form', '--verbose', 'POST', httpbin.url + '/post',
|
r = http(
|
||||||
f'test-file@{FILE_PATH_ARG};type=image/vnd.microsoft.icon')
|
'--form',
|
||||||
|
'--verbose',
|
||||||
|
httpbin.url + '/post',
|
||||||
|
f'test-file@{FILE_PATH_ARG};type=image/vnd.microsoft.icon'
|
||||||
|
)
|
||||||
assert HTTP_OK in r
|
assert HTTP_OK in r
|
||||||
# Content type is stripped from the filename
|
# Content type is stripped from the filename
|
||||||
assert 'Content-Disposition: form-data; name="test-file";' \
|
assert 'Content-Disposition: form-data; name="test-file";' \
|
||||||
f' filename="{os.path.basename(FILE_PATH)}"' in r
|
f' filename="{os.path.basename(FILE_PATH)}"' in r
|
||||||
assert FILE_CONTENT in r
|
assert r.count(FILE_CONTENT) == 2
|
||||||
assert 'Content-Type: image/vnd.microsoft.icon' in r
|
assert 'Content-Type: image/vnd.microsoft.icon' in r
|
||||||
|
|
||||||
|
@mock.patch('httpie.uploads.UPLOAD_BUFFER', 0)
|
||||||
|
def test_large_upload_display_suppressed(self, httpbin):
|
||||||
|
r = http(
|
||||||
|
'--form',
|
||||||
|
'--verbose',
|
||||||
|
httpbin.url + '/post',
|
||||||
|
f'test-file@{FILE_PATH_ARG}',
|
||||||
|
'foo=bar',
|
||||||
|
)
|
||||||
|
assert HTTP_OK in r
|
||||||
|
assert r.count(FILE_CONTENT) == 1
|
||||||
|
assert LARGE_UPLOAD_SUPPRESSED_NOTICE.decode() in r
|
||||||
|
|
||||||
|
|
||||||
class TestRequestBodyFromFilePath:
|
class TestRequestBodyFromFilePath:
|
||||||
"""
|
"""
|
||||||
|
Loading…
Reference in New Issue
Block a user