Add compressed requests (#739)

* Add optional compression of the request's content

This option allows compression of the files and/or data during uploading,

Examples:

    http --form --compress POST https://localhost/upload csv@./very-big.csv

    http -x -x POST https://localhost/upload foo=bar

    cat /var/log/system.log | http -x POST https://localhost/upload

Signed-off-by: Aleksandr Vinokurov <aleksandr.vin@gmail.com>

* Add tests for compression

Signed-off-by: Aleksandr Vinokurov <aleksandr.vin@gmail.com>

* Fix code style issues

Signed-off-by: Aleksandr Vinokurov <aleksandr.vin@gmail.com>

* Fix zlib compression api missuse in Python3

Signed-off-by: Aleksandr Vinokurov <aleksandr.vin@gmail.com>

* Remove tracing from compression logic

Signed-off-by: Aleksandr Vinokurov <aleksandr.vin@gmail.com>

AUTHORS.rst

@@ -36,5 +36,7 @@ Patches and ideas
 * `Dennis Brakhane <https://github.com/brakhane>`_
 * `Matt Layman <https://github.com/mblayman>`_
 * `Edward Yang <https://github.com/honorabrutroll>`_
+* `Aleksandr Vinokurov <https://github.com/aleksandr-vin>`_
 * `Jeff Byrnes <https://github.com/jeffbyrnes>`_
 
+

httpie/cli.py

@@ -185,6 +185,29 @@ content_type.add_argument(
 )
 
 
+#######################################################################
+# Content processing.
+#######################################################################
+
+content_processing = parser.add_argument_group(
+    title='Content Processing Options',
+    description=None
+)
+
+content_processing.add_argument(
+    '--compress', '-x',
+    action='count',
+    help="""
+    Content compressed (encoded) with Deflate algorithm.
+    The Content-Encoding header is set to deflate.
+
+    Compression is skipped if it appears that compression ratio is
+    negative. Compression can be forced by repeating the argument.
+
+    """
+)
+
+
 #######################################################################
 # Output processing
 #######################################################################

httpie/client.py

@@ -13,6 +13,8 @@ from httpie.input import SSL_VERSION_ARG_MAPPING
 from httpie.plugins import plugin_manager
 from httpie.utils import repr_dict_nice
 
+import zlib
+
 try:
     # https://urllib3.readthedocs.io/en/latest/security.html
     # noinspection PyPackageRequirements
@@ -55,12 +57,37 @@ class HTTPieHTTPAdapter(HTTPAdapter):
         super(HTTPieHTTPAdapter, self).init_poolmanager(*args, **kwargs)
 
 
-def get_requests_session(ssl_version):
+class ContentCompressionHttpAdapter(HTTPAdapter):
+
+    def __init__(self, compress, **kwargs):
+        self.compress = compress
+        super(ContentCompressionHttpAdapter, self).__init__(**kwargs)
+
+    def send(self, request, **kwargs):
+        if request.body and self.compress > 0:
+            deflater = zlib.compressobj()
+            if isinstance(request.body, bytes):
+                deflated_data = deflater.compress(request.body)
+            else:
+                deflated_data = deflater.compress(request.body.encode())
+            deflated_data += deflater.flush()
+            if len(deflated_data) < len(request.body) or self.compress > 1:
+                request.body = deflated_data
+                request.headers['Content-Encoding'] = 'deflate'
+                request.headers['Content-Length'] = str(len(deflated_data))
+        return super(ContentCompressionHttpAdapter, self).send(request, **kwargs)
+
+
+def get_requests_session(ssl_version, compress):
     requests_session = requests.Session()
     requests_session.mount(
         'https://',
         HTTPieHTTPAdapter(ssl_version=ssl_version)
     )
+    if compress:
+        adapter = ContentCompressionHttpAdapter(compress)
+        for prefix in ['http://', 'https://']:
+            requests_session.mount(prefix, adapter)
     for cls in plugin_manager.get_transport_plugins():
         transport_plugin = cls()
         requests_session.mount(prefix=transport_plugin.prefix,
@@ -75,7 +102,7 @@ def get_response(args, config_dir):
     if args.ssl_version:
         ssl_version = SSL_VERSION_ARG_MAPPING[args.ssl_version]
 
-    requests_session = get_requests_session(ssl_version)
+    requests_session = get_requests_session(ssl_version, args.compress)
     requests_session.max_redirects = args.max_redirects
 
     with max_headers(args.max_headers):

tests/test_httpie.py

@@ -49,12 +49,28 @@ def test_POST_JSON_data(httpbin_both):
     assert r.json['json']['foo'] == 'bar'
 
 
+def test_POST_JSON_data_compressed(httpbin_both):
+    r = http('--compress', '--compress', 'POST', httpbin_both + '/post', 'foo=bar')
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert r.json['data'].startswith('data:application/octet-stream;')
+    assert r.json['json'] is None
+
+
 def test_POST_form(httpbin_both):
     r = http('--form', 'POST', httpbin_both + '/post', 'foo=bar')
     assert HTTP_OK in r
     assert '"foo": "bar"' in r
 
 
+def test_POST_form_compressed(httpbin_both):
+    r = http('--form', '--compress', '--compress', 'POST', httpbin_both + '/post', 'foo=bar')
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert r.json['data'] == ""
+    assert '"foo": "bar"' not in r
+
+
 def test_POST_form_multiple_values(httpbin_both):
     r = http('--form', 'POST', httpbin_both + '/post', 'foo=bar', 'foo=baz')
     assert HTTP_OK in r
@@ -69,6 +85,31 @@ def test_POST_stdin(httpbin_both):
     assert FILE_CONTENT in r
 
 
+def test_POST_stdin_compressed(httpbin_both):
+    with open(FILE_PATH) as f:
+        env = MockEnvironment(stdin=f, stdin_isatty=False)
+        r = http('--form', '--compress', '--compress', 'POST', httpbin_both + '/post', env=env)
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert r.json['data'] == ""
+    assert FILE_CONTENT not in r
+
+
+def test_POST_file(httpbin_both):
+    r = http('--form', 'POST', httpbin_both + '/post', 'file@' + FILE_PATH)
+    assert HTTP_OK in r
+    assert FILE_CONTENT in r
+
+
+def test_POST_file_compressed(httpbin_both):
+    r = http('--form', '--compress', '--compress', 'POST', httpbin_both + '/post', 'file@' + FILE_PATH)
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert r.json['headers']['Content-Type'].startswith('multipart/form-data; boundary=')
+    assert r.json['files'] == {}
+    assert FILE_CONTENT not in r
+
+
 def test_headers(httpbin_both):
     r = http('GET', httpbin_both + '/headers', 'Foo:bar')
     assert HTTP_OK in r