Add --raw to allow specifying the raw request body as an alternative to stdin (#1062)

* Add --raw to allow specifying the raw request body without extra processing

As an alternative to `stdin`.

Co-authored-by: Elena Lape <elapinskaite@gmail.com>
Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

* Update README.rst

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

* Update README.rst

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

* Fix default HTTP method on empty data

Co-authored-by: Elena Lape <elapinskaite@gmail.com>
Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

CHANGELOG.rst

@@ -10,6 +10,8 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
 `2.5.0-dev`_ (unreleased)
 -------------------------
 * Fixed ``--continue --download`` with a single byte to be downloaded left. (`#1032`_)
+* Added ``--raw`` to allow specifying the raw request body without extra processing as
+  an alternative to ``stdin``. (`#534`_)
 
 
 `2.4.0`_ (2021-02-06)

README.rst

@@ -500,7 +500,7 @@ their type is distinguished only by the separator used:
 
 
 Note that data fields aren’t the only way to specify request data:
-`Redirected input`_ is a mechanism for passing arbitrary request data.
+The `specifying raw request body`_ section describes mechanisms for passing arbitrary request data.
 
 
 Escaping rules
@@ -1353,8 +1353,20 @@ which you don’t care about. The response headers are downloaded always,
 even if they are not part of the output
 
 
+Specifying raw request body
+===========================
+
+In addition to crafting structured `JSON`_ and `forms`_ requests with the
+`request items`_ syntax, you can provide a raw request body that will be
+sent without further processing. These two approaches for specifying request
+data (i.e., structured and raw) cannot be combined.
+
+There’re three methods for passing raw request data: piping via ``stdin``,
+``--raw='data'``, and ``@/file/path``.
+
+
 Redirected Input
-================
+----------------
 
 The universal method for passing request data is through redirected ``stdin``
 (standard input)—piping.
@@ -1428,7 +1440,6 @@ On OS X, you can send the contents of the clipboard with ``pbpaste``:
 Passing data through ``stdin`` cannot be combined with data fields specified
 on the command line:
 
-
 .. code-block:: bash
 
     $ echo 'data' | http POST example.org more=data   # This is invalid
@@ -1438,6 +1449,22 @@ To prevent HTTPie from reading ``stdin`` data you can use the
 ``--ignore-stdin`` option.
 
 
+Request data via ``--raw``
+--------------------------
+
+In a situation when piping data via ``stdin`` is not convenient (for example,
+when generating API docs examples), you can specify the raw request body via
+the ``--raw`` option.
+
+.. code-block:: bash
+
+    $ http --raw 'Hello, world!' pie.dev/post
+
+.. code-block:: bash
+
+    $ http --raw '{"name": "John"}' pie.dev/post
+
+
 Request data from a filename
 ----------------------------
 

extras/httpie-completion.bash

@@ -18,6 +18,6 @@ _http_complete_options() {
     -v --verbose -h --headers -b --body -S --stream -o --output -d --download
     -c --continue --session --session-read-only -a --auth --auth-type --proxy
     --follow --verify --cert --cert-key --timeout --check-status --ignore-stdin
-    --help --version --traceback --debug"
+    --help --version --traceback --debug --raw"
     COMPREPLY=( $( compgen -W "$options" -- "$cur_word" ) )
 }

extras/httpie-completion.fish

@@ -57,3 +57,4 @@ complete    -c http      -l help         -d 'Show help'
 complete    -c http      -l version      -d 'Show version'
 complete    -c http      -l traceback    -d 'Prints exception traceback should one occur'
 complete    -c http      -l debug        -d 'Show debugging information'
+complete    -c http      -l raw          -d 'Pass raw request data without extra processing'

httpie/cli/argparser.py

@@ -64,6 +64,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
         self.env = None
         self.args = None
         self.has_stdin_data = False
+        self.has_input_data = False
 
     # noinspection PyMethodOverriding
     def parse_args(
@@ -81,6 +82,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
             and not self.args.ignore_stdin
             and not self.env.stdin_isatty
         )
+        self.has_input_data = self.has_stdin_data or self.args.raw is not None
         # Arguments processing and environment setup.
         self._apply_no_options(no_options)
         self._process_request_type()
@@ -91,11 +93,14 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
         self._process_format_options()
         self._guess_method()
         self._parse_items()
-        if self.has_stdin_data:
-            self._body_from_file(self.env.stdin)
         self._process_url()
         self._process_auth()
 
+        if self.args.raw is not None:
+            self._body_from_input(self.args.raw)
+        elif self.has_stdin_data:
+            self._body_from_file(self.env.stdin)
+
         if self.args.compress:
             # TODO: allow --compress with --chunked / --multipart
             if self.args.chunked:
@@ -283,17 +288,31 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
             self.error(msg % ' '.join(invalid))
 
     def _body_from_file(self, fd):
-        """There can only be one source of request data.
+        """Read the data from a file-like object.
 
         Bytes are always read.
 
         """
-        if self.args.data or self.args.files:
+        self._ensure_one_data_source(self.args.data, self.args.files)
-            self.error('Request body (from stdin or a file) and request '
+        self.args.data = getattr(fd, 'buffer', fd)
+
+    def _body_from_input(self, data):
+        """Read the data from the CLI.
+
+        """
+        self._ensure_one_data_source(self.has_stdin_data, self.args.data,
+                                     self.args.files)
+        self.args.data = data.encode('utf-8')
+
+    def _ensure_one_data_source(self, *other_sources):
+        """There can only be one source of input request data.
+
+        """
+        if any(other_sources):
+            self.error('Request body (from stdin, --raw or a file) and request '
                        'data (key=value) cannot be mixed. Pass '
                        '--ignore-stdin to let key/value take priority. '
                        'See https://httpie.org/doc#scripting for details.')
-        self.args.data = getattr(fd, 'buffer', fd)
 
     def _guess_method(self):
         """Set `args.method` if not specified to either POST or GET
@@ -303,7 +322,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
         if self.args.method is None:
             # Invoked as `http URL'.
             assert not self.args.request_items
-            if self.has_stdin_data:
+            if self.has_input_data:
                 self.args.method = HTTP_POST
             else:
                 self.args.method = HTTP_GET
@@ -327,7 +346,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
                 self.args.url = self.args.method
                 # Infer the method
                 has_data = (
-                    self.has_stdin_data
+                    self.has_input_data
                     or any(
                         item.sep in SEPARATOR_GROUP_DATA_ITEMS
                         for item in self.args.request_items)

httpie/cli/definition.py

@@ -185,6 +185,25 @@ content_type.add_argument(
 
     '''
 )
+content_type.add_argument(
+    '--raw',
+    help='''
+    This option allows you to pass raw request data without extra processing
+    (as opposed to the structured request items syntax):
+
+        $ http --raw='data' pie.dev/post
+
+    You can achieve the same by piping the data via stdin:
+
+        $ echo data | http pie.dev/post
+
+    Or have HTTPie load the raw data from a file:
+
+        $ http pie.dev/post @data.txt
+
+
+    '''
+)
 
 
 #######################################################################

tests/test_compress.py

@@ -92,6 +92,19 @@ def test_compress_form(httpbin_both):
     assert '"foo": "bar"' not in r
 
 
+def test_compress_raw(httpbin_both):
+    r = http(
+        '--raw',
+        FILE_CONTENT,
+        '--compress',
+        '--compress',
+        httpbin_both + '/post',
+    )
+    assert HTTP_OK in r
+    assert r.json['headers']['Content-Encoding'] == 'deflate'
+    assert_decompressed_equal(r.json['data'], FILE_CONTENT.strip())
+
+
 def test_compress_stdin(httpbin_both):
     env = MockEnvironment(
         stdin=StdinBytesIO(FILE_PATH.read_bytes()),

tests/test_defaults.py

@@ -45,6 +45,11 @@ class TestImplicitHTTPMethod:
         assert HTTP_OK in r
         assert r.json['form'] == {'foo': 'bar'}
 
+    def test_implicit_POST_raw(self, httpbin):
+        r = http('--raw', 'foo bar', httpbin.url + '/post')
+        assert HTTP_OK in r
+        assert r.json['data'] == 'foo bar'
+
     def test_implicit_POST_stdin(self, httpbin):
         env = MockEnvironment(
             stdin_isatty=False,

tests/test_httpie.py

@@ -103,6 +103,12 @@ def test_POST_form_multiple_values(httpbin_both):
     }
 
 
+def test_POST_raw(httpbin_both):
+    r = http('--raw', 'foo bar', 'POST', httpbin_both + '/post')
+    assert HTTP_OK in r
+    assert '"foo bar"' in r
+
+
 def test_POST_stdin(httpbin_both):
     env = MockEnvironment(
         stdin=StdinBytesIO(FILE_PATH.read_bytes()),
@@ -140,6 +146,35 @@ def test_form_POST_file_redirected_stdin(httpbin):
     assert 'cannot be mixed' in r.stderr
 
 
+def test_raw_POST_key_values_supplied(httpbin):
+    r = http(
+        '--raw',
+        'foo bar',
+        'POST',
+        httpbin + '/post',
+        'foo=bar',
+        tolerate_error_exit_status=True,
+    )
+    assert r.exit_status == ExitStatus.ERROR
+    assert 'cannot be mixed' in r.stderr
+
+
+def test_raw_POST_redirected_stdin(httpbin):
+    r = http(
+        '--raw',
+        'foo bar',
+        'POST',
+        httpbin + '/post',
+        tolerate_error_exit_status=True,
+        env=MockEnvironment(
+            stdin='some=value',
+            stdin_isatty=False,
+        ),
+    )
+    assert r.exit_status == ExitStatus.ERROR
+    assert 'cannot be mixed' in r.stderr
+
+
 def test_headers(httpbin_both):
     r = http('GET', httpbin_both + '/headers', 'Foo:bar')
     assert HTTP_OK in r

tests/test_offline.py

@@ -10,6 +10,27 @@ def test_offline():
     assert 'GET /foo' in r
 
 
+def test_offline_raw():
+    r = http(
+        '--offline',
+        '--raw',
+        'foo bar',
+        'https://this-should.never-resolve/foo',
+    )
+    assert 'POST /foo' in r
+    assert 'foo bar' in r
+
+
+def test_offline_raw_empty_should_use_POST():
+    r = http(
+        '--offline',
+        '--raw',
+        '',
+        'https://this-should.never-resolve/foo',
+    )
+    assert 'POST /foo' in r
+
+
 def test_offline_form():
     r = http(
         '--offline',

tests/test_output.py

@@ -141,6 +141,12 @@ class TestVerboseFlag:
         assert HTTP_OK in r
         assert r.count('__test__') == 2
 
+    def test_verbose_raw(self, httpbin):
+        r = http('--verbose', '--raw', 'foo bar',
+                 'POST', httpbin.url + '/post',)
+        assert HTTP_OK in r
+        assert 'foo bar' in r
+
     def test_verbose_form(self, httpbin):
         # https://github.com/httpie/httpie/issues/53
         r = http('--verbose', '--form', 'POST', httpbin.url + '/post',

tests/test_unicode.py

@@ -20,6 +20,19 @@ def test_unicode_headers_verbose(httpbin):
     assert UNICODE in r
 
 
+def test_unicode_raw(httpbin):
+    r = http('--raw', u'test %s' % UNICODE, 'POST', httpbin.url + '/post')
+    assert HTTP_OK in r
+    assert r.json['data'] == u'test %s' % UNICODE
+
+
+def test_unicode_raw_verbose(httpbin):
+    r = http('--verbose', '--raw', u'test %s' % UNICODE,
+             'POST', httpbin.url + '/post')
+    assert HTTP_OK in r
+    assert UNICODE in r
+
+
 def test_unicode_form_item(httpbin):
     r = http('--form', 'POST', httpbin.url + '/post', u'test=%s' % UNICODE)
     assert HTTP_OK in r