From 098e1d3100a4d33df22e8308e5bccb9358942be6 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Sat, 28 Jul 2012 05:45:44 +0200 Subject: [PATCH] Fixed multipart requests output; binary support. * Bodies of multipart requests are correctly printed (closes #30). * Binary requests and responses should always work (they are also suppressed for terminal output). So things like this work:: http www.google.com/favicon.ico > favicon.ico --- README.rst | 9 ++- httpie/cli.py | 5 +- httpie/core.py | 61 ++++++++------------ httpie/input.py | 20 +++++-- httpie/models.py | 122 ++++++++++++++++++---------------------- httpie/output.py | 77 ++++++++++++++++++++++++- tests/tests.py | 143 +++++++++++++++++++++++++++++++++-------------- 7 files changed, 279 insertions(+), 158 deletions(-) diff --git a/README.rst b/README.rst index e08f2cb5..2027dec0 100644 --- a/README.rst +++ b/README.rst @@ -164,7 +164,11 @@ the second one has via ``stdin``:: Note that when the **output is redirected** (like the examples above), HTTPie applies a different set of defaults than for a console output. Namely, colors aren't used (unless ``--pretty`` is set) and only the response body -is printed (unless ``--print`` options specified). +is printed (unless ``--print`` options specified). It is a convenience +that allows for things like the one above or downloading (smallish) binary +files without having to set any flags:: + + http www.google.com/favicon.ico > favicon.ico An alternative to ``stdin`` is to pass a filename whose content will be used as the request body. It has the advantage that the ``Content-Type`` header @@ -330,6 +334,9 @@ Changelog ========= * `0.2.7dev`_ + * Proper handling of binary requests and responses. + * Fixed printing of ``multipart/form-data`` requests. + * Renamed ``--traceback`` to ``--debug``. * `0.2.6`_ (2012-07-26) * The short option for ``--headers`` is now ``-h`` (``-t`` has been removed, for usage use ``--help``). diff --git a/httpie/cli.py b/httpie/cli.py index edb5b6b6..67b059f7 100644 --- a/httpie/cli.py +++ b/httpie/cli.py @@ -51,9 +51,10 @@ group_type.add_argument( ############################################# parser.add_argument( - '--traceback', action='store_true', default=False, + '--debug', action='store_true', default=False, help=_(''' - Print exception traceback should one occur. + Prints exception traceback should one occur and other + information useful for debugging HTTPie itself. ''') ) diff --git a/httpie/core.py b/httpie/core.py index 52c2d3f2..e18de91f 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -8,7 +8,6 @@ Invocation flow: 4. Write to `stdout` and exit. """ -import os import sys import json @@ -17,9 +16,8 @@ import requests.auth from requests.compat import str from .models import HTTPMessage, Environment -from .output import OutputProcessor -from .input import (PRETTIFY_STDOUT_TTY_ONLY, - OUT_REQ_BODY, OUT_REQ_HEAD, +from .output import OutputProcessor, format +from .input import (OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_HEAD, OUT_RESP_BODY) from .cli import parser @@ -85,7 +83,7 @@ def get_response(args, env): env.stderr.write('\n') sys.exit(1) except Exception as e: - if args.traceback: + if args.debug: raise env.stderr.write(str(e.message) + '\n') sys.exit(1) @@ -93,49 +91,35 @@ def get_response(args, env): def get_output(args, env, request, response): """Format parts of the `request`-`response` exchange - according to `args` and `env` and return a `unicode`. + according to `args` and `env` and return `bytes`. """ - do_prettify = (args.prettify is True - or (args.prettify == PRETTIFY_STDOUT_TTY_ONLY - and env.stdout_isatty)) - do_output_request = (OUT_REQ_HEAD in args.output_options - or OUT_REQ_BODY in args.output_options) + exchange = [] + prettifier = (OutputProcessor(env, pygments_style=args.style) + if args.prettify else None) - do_output_response = (OUT_RESP_HEAD in args.output_options - or OUT_RESP_BODY in args.output_options) - - prettifier = None - if do_prettify: - prettifier = OutputProcessor( - env, pygments_style=args.style) - - buf = [] - - if do_output_request: - req_msg = HTTPMessage.from_request(request) - req = req_msg.format( + if (OUT_REQ_HEAD in args.output_options + or OUT_REQ_BODY in args.output_options): + exchange.append(format( + HTTPMessage.from_request(request), + env=env, prettifier=prettifier, with_headers=OUT_REQ_HEAD in args.output_options, with_body=OUT_REQ_BODY in args.output_options - ) - buf.append(req) - buf.append('\n') - if do_output_response: - buf.append('\n') + )) - if do_output_response: - resp_msg = HTTPMessage.from_response(response) - resp = resp_msg.format( + if (OUT_RESP_HEAD in args.output_options + or OUT_RESP_BODY in args.output_options): + exchange.append(format( + HTTPMessage.from_response(response), + env=env, prettifier=prettifier, with_headers=OUT_RESP_HEAD in args.output_options, - with_body=OUT_RESP_BODY in args.output_options + with_body=OUT_RESP_BODY in args.output_options) ) - buf.append(resp) - buf.append('\n') - return ''.join(buf) + return b''.join(exchange) def get_exist_status(code, allow_redirects=False): @@ -172,8 +156,9 @@ def main(args=sys.argv[1:], env=Environment()): response.raw.status, response.raw.reason) env.stderr.write(err.encode('utf8')) - output = get_output(args, env, response.request, response) - output_bytes = output.encode('utf8') + output_bytes = get_output(args, env, response.request, response) + + # output_bytes = output.encode('utf8') f = getattr(env.stdout, 'buffer', env.stdout) f.write(output_bytes) diff --git a/httpie/input.py b/httpie/input.py index ed31c32d..f837c3c9 100644 --- a/httpie/input.py +++ b/httpie/input.py @@ -103,9 +103,17 @@ class Parser(argparse.ArgumentParser): # Stdin already read (if not a tty) so it's save to prompt. args.auth.prompt_password() + if args.files: + # Will be read multiple times. + for name in args.files: + args.files[name] = args.files[name].read() + + if args.prettify == PRETTIFY_STDOUT_TTY_ONLY: + args.prettify = env.stdout_isatty + return args - def _body_from_file(self, args, f): + def _body_from_file(self, args, data): """Use the content of `f` as the `request.data`. There can only be one source of request data. @@ -114,7 +122,7 @@ class Parser(argparse.ArgumentParser): if args.data: self.error('Request body (from stdin or a file) and request ' 'data (key=value) cannot be mixed.') - args.data = f.read() + args.data = data def _guess_method(self, args, env): """Set `args.method` if not specified to either POST or GET @@ -139,7 +147,7 @@ class Parser(argparse.ArgumentParser): 0, KeyValueArgType(*SEP_GROUP_ITEMS).__call__(args.url)) except argparse.ArgumentTypeError as e: - if args.traceback: + if args.debug: raise self.error(e.message) @@ -169,7 +177,7 @@ class Parser(argparse.ArgumentParser): files=args.files, params=args.params) except ParseError as e: - if args.traceback: + if args.debug: raise self.error(e.message) @@ -406,12 +414,12 @@ def parse_items(items, data=None, headers=None, files=None, params=None): target = params elif item.sep == SEP_FILES: try: - value = open(os.path.expanduser(item.value), 'r') + value = open(os.path.expanduser(value), 'r') except IOError as e: raise ParseError( 'Invalid argument "%s": %s' % (item.orig, e)) if not key: - key = os.path.basename(value.name) + key = os.path.basename(item.value) target = files elif item.sep in [SEP_DATA, SEP_DATA_RAW_JSON]: diff --git a/httpie/models.py b/httpie/models.py index 520df518..2adb0147 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -1,6 +1,6 @@ import os import sys -from requests.compat import urlparse, is_windows +from requests.compat import urlparse, is_windows, bytes, str class Environment(object): @@ -39,39 +39,40 @@ class Environment(object): class HTTPMessage(object): """Model representing an HTTP message.""" - def __init__(self, line, headers, body, content_type=None): - # {Request,Status}-Line - self.line = line + def __init__(self, line, headers, body, encoding=None, content_type=None): + """All args are a `str` except for `body` which is a `bytes`.""" + + assert isinstance(line, str) + assert content_type is None or isinstance(content_type, str) + assert isinstance(body, bytes) + + self.line = line # {Request,Status}-Line self.headers = headers self.body = body + self.encoding = encoding self.content_type = content_type - def format(self, prettifier=None, with_headers=True, with_body=True): - """Return a `unicode` representation of `self`. """ - pretty = prettifier is not None - bits = [] + @classmethod + def from_response(cls, response): + """Make an `HTTPMessage` from `requests.models.Response`.""" + encoding = response.encoding or None + original = response.raw._original_response + response_headers = response.headers + status_line = str('HTTP/{version} {status} {reason}'.format( + version='.'.join(str(original.version)), + status=original.status, + reason=original.reason + )) + body = response.content - if with_headers: - bits.append(self.line) - bits.append(self.headers) - if pretty: - bits = [ - prettifier.process_headers('\n'.join(bits)) - ] - if with_body and self.body: - bits.append('\n') + return cls(line=status_line, + headers=str(original.msg), + body=body, + encoding=encoding, + content_type=str(response_headers.get('Content-Type', ''))) - if with_body and self.body: - if pretty and self.content_type: - bits.append(prettifier.process_body( - self.body, self.content_type)) - else: - bits.append(self.body) - - return '\n'.join(bit.strip() for bit in bits) - - @staticmethod - def from_request(request): + @classmethod + def from_request(cls, request): """Make an `HTTPMessage` from `requests.models.Request`.""" url = urlparse(request.url) @@ -90,54 +91,41 @@ class HTTPMessage(object): qs += type(request)._encode_params(request.params) # Request-Line - request_line = '{method} {path}{query} HTTP/1.1'.format( + request_line = str('{method} {path}{query} HTTP/1.1'.format( method=request.method, path=url.path or '/', query=qs - ) + )) # Headers headers = dict(request.headers) content_type = headers.get('Content-Type') + + if isinstance(content_type, bytes): + # Happens when uploading files. + # TODO: submit a bug report for Requests + content_type = headers['Content-Type'] = content_type.decode('utf8') + if 'Host' not in headers: headers['Host'] = url.netloc - headers = '\n'.join( - str('%s: %s') % (name, value) - for name, value - in headers.items() - ) + headers = '\n'.join('%s: %s' % (name, value) + for name, value in headers.items()) # Body - try: - body = request.data - except AttributeError: - # requests < 0.12.1 - body = request._enc_data - if isinstance(body, dict): - #noinspection PyUnresolvedReferences - body = type(request)._encode_params(body) + if request.files: + body, _ = request._encode_files(request.files) + else: + try: + body = request.data + except AttributeError: + # requests < 0.12.1 + body = request._enc_data + if isinstance(body, dict): + #noinspection PyUnresolvedReferences + body = type(request)._encode_params(body) + body = body.encode('utf8') - return HTTPMessage( - line=request_line, - headers=headers, - body=body, - content_type=content_type - ) - - @classmethod - def from_response(cls, response): - """Make an `HTTPMessage` from `requests.models.Response`.""" - encoding = response.encoding or 'ISO-8859-1' - original = response.raw._original_response - response_headers = response.headers - status_line = 'HTTP/{version} {status} {reason}'.format( - version='.'.join(str(original.version)), - status=original.status, - reason=original.reason - ) - body = response.content.decode(encoding) if response.content else '' - return cls( - line=status_line, - headers=str(original.msg), - body=body, - content_type=response_headers.get('Content-Type')) + return cls(line=request_line, + headers=headers, + body=body, + content_type=content_type) diff --git a/httpie/output.py b/httpie/output.py index 23c630d8..971a13c9 100644 --- a/httpie/output.py +++ b/httpie/output.py @@ -11,13 +11,88 @@ from pygments.lexers import get_lexer_for_mimetype from pygments.formatters.terminal import TerminalFormatter from pygments.formatters.terminal256 import Terminal256Formatter from pygments.util import ClassNotFound -from requests.compat import is_windows +from requests.compat import is_windows, bytes from . import solarized +from .models import Environment DEFAULT_STYLE = 'solarized' AVAILABLE_STYLES = [DEFAULT_STYLE] + list(STYLE_MAP.keys()) +BINARY_SUPPRESSED_NOTICE = ( + '+-----------------------------------------+\n' + '| NOTE: binary data not shown in terminal |\n' + '+-----------------------------------------+' +) + + +def format(msg, prettifier=None, with_headers=True, with_body=True, + env=Environment()): + """Return a UTF8-encoded representation of a `models.HTTPMessage`. + + Sometimes it contains binary data so we always return `bytes`. + + If `prettifier` is set or the output is terminal then a binary + body is not included in the output replaced with notice. + + Generally, when the `stdout` is redirected, the output match the actual + message as match as possible. When we are `--pretty` set (or implied) + or when the output is a terminal, then we prefer readability over + precision. + + """ + chunks = [] + + if with_headers: + headers = '\n'.join([msg.line, msg.headers]).encode('utf8') + + if prettifier: + # Prettifies work on unicode + headers = prettifier.process_headers( + headers.decode('utf8')).encode('utf8') + + chunks.append(headers.strip()) + + if with_body and msg.body or env.stdout_isatty: + chunks.append(b'\n\n') + + if with_body and msg.body: + + body = msg.body + bin_suppressed = False + + if prettifier or env.stdout_isatty: + + # Convert body to UTF8. + try: + body = msg.body.decode(msg.encoding or 'utf8') + except UnicodeDecodeError: + # Assume binary. It could also be that `self.encoding` + # doesn't correspond to the actual encoding. + bin_suppressed = True + body = BINARY_SUPPRESSED_NOTICE.encode('utf8') + if not with_headers: + body = b'\n' + body + + else: + # Convert (possibly back) to UTF8. + body = body.encode('utf8') + + if not bin_suppressed and prettifier and msg.content_type: + # Prettifies work on unicode. + body = (prettifier + .process_body(body.decode('utf8'), + msg.content_type) + .encode('utf8').strip()) + + chunks.append(body) + + if env.stdout_isatty: + chunks.append(b'\n\n') + + formatted = b''.join(chunks) + + return formatted class HTTPLexer(lexer.RegexLexer): diff --git a/tests/tests.py b/tests/tests.py index 1edfb70e..117685be 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -25,9 +25,13 @@ import json import tempfile import unittest import argparse -import requests -from requests.compat import is_py26, is_py3, str +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen +import requests +from requests.compat import is_py26, is_py3, bytes, str ################################################################# # Utils/setup @@ -40,6 +44,7 @@ sys.path.insert(0, os.path.realpath(os.path.join(TESTS_ROOT, '..'))) from httpie import input from httpie.models import Environment from httpie.core import main, get_output +from httpie.output import BINARY_SUPPRESSED_NOTICE HTTPBIN_URL = os.environ.get('HTTPBIN_URL', @@ -55,17 +60,22 @@ def httpbin(path): return HTTPBIN_URL + path -class Response(str): - """ - A unicode subclass holding the output of `main()`, and also - the exit status, the contents of ``stderr``, and de-serialized - JSON response (if possible). +class BytesResponse(bytes): - """ exit_status = None stderr = None json = None + def __eq__(self, other): + return super(BytesResponse, self).__eq__(other) + +class StrResponse(str): + exit_status = None + stderr = None + json = None + def __eq__(self, other): + return super(StrResponse, self).__eq__(other) + def http(*args, **kwargs): """ @@ -85,34 +95,40 @@ def http(*args, **kwargs): stdout = kwargs['env'].stdout = tempfile.TemporaryFile() stderr = kwargs['env'].stderr = tempfile.TemporaryFile() - exit_status = main(args=['--traceback'] + list(args), **kwargs) + exit_status = main(args=['--debug'] + list(args), **kwargs) stdout.seek(0) stderr.seek(0) - r = Response(stdout.read().decode('utf8')) + output = stdout.read() + + try: + r = StrResponse(output.decode('utf8')) + except UnicodeDecodeError: + r = BytesResponse(output) + else: + if TERMINAL_COLOR_PRESENCE_CHECK not in r: + # De-serialize JSON body if possible. + if r.strip().startswith('{'): + #noinspection PyTypeChecker + r.json = json.loads(r) + elif r.count('Content-Type:') == 1 and 'application/json' in r: + try: + j = r.strip()[r.strip().rindex('\n\n'):] + except ValueError: + pass + else: + try: + r.json = json.loads(j) + except ValueError: + pass + r.stderr = stderr.read().decode('utf8') r.exit_status = exit_status stdout.close() stderr.close() - if TERMINAL_COLOR_PRESENCE_CHECK not in r: - # De-serialize JSON body if possible. - if r.strip().startswith('{'): - #noinspection PyTypeChecker - r.json = json.loads(r) - elif r.count('Content-Type:') == 1 and 'application/json' in r: - try: - j = r.strip()[r.strip().rindex('\n\n'):] - except ValueError: - pass - else: - try: - r.json = json.loads(j) - except ValueError: - pass - return r @@ -506,7 +522,7 @@ class VerboseFlagTest(BaseTestCase): class MultipartFormDataFileUploadTest(BaseTestCase): def test_non_existent_file_raises_parse_error(self): - self.assertRaises(input.ParseError, http, + self.assertRaises(SystemExit, http, '--form', '--traceback', 'POST', @@ -517,16 +533,58 @@ class MultipartFormDataFileUploadTest(BaseTestCase): def test_upload_ok(self): r = http( '--form', + '--verbose', 'POST', httpbin('/post'), 'test-file@%s' % TEST_FILE_PATH, 'foo=bar' ) + self.assertIn('HTTP/1.1 200', r) - self.assertIn('"test-file": "%s' % TEST_FILE_CONTENT, r) + self.assertIn('Content-Disposition: form-data; name="foo"', r) + self.assertIn('Content-Disposition: form-data; name="test-file";' + ' filename="test-file"', r) + self.assertEqual(r.count(TEST_FILE_CONTENT), 2) self.assertIn('"foo": "bar"', r) +class TestBinaryResponses(BaseTestCase): + + url = 'http://www.google.com/favicon.ico' + + @property + def bindata(self): + if not hasattr(self, '_bindata'): + self._bindata = urlopen(self.url).read() + return self._bindata + + def test_binary_suppresses_when_terminal(self): + r = http( + 'GET', + self.url + ) + self.assertIn(BINARY_SUPPRESSED_NOTICE, r) + + def test_binary_suppresses_when_not_terminal_but_pretty(self): + r = http( + '--pretty', + 'GET', + self.url, + env=Environment(stdin_isatty=True, + stdout_isatty=False) + ) + self.assertIn(BINARY_SUPPRESSED_NOTICE, r) + + def test_binary_included_and_correct_when_suitable(self): + r = http( + 'GET', + self.url, + env=Environment(stdin_isatty=True, + stdout_isatty=False) + ) + self.assertEqual(r, self.bindata) + + class RequestBodyFromFilePathTest(BaseTestCase): """ `http URL @file' @@ -764,9 +822,9 @@ class ArgumentParserTestCase(unittest.TestCase): self.parser._guess_method(args, Environment()) - self.assertEquals(args.method, 'GET') - self.assertEquals(args.url, 'http://example.com/') - self.assertEquals(args.items, []) + self.assertEqual(args.method, 'GET') + self.assertEqual(args.url, 'http://example.com/') + self.assertEqual(args.items, []) def test_guess_when_method_not_set(self): args = argparse.Namespace() @@ -779,9 +837,9 @@ class ArgumentParserTestCase(unittest.TestCase): stdout_isatty=True, )) - self.assertEquals(args.method, 'GET') - self.assertEquals(args.url, 'http://example.com/') - self.assertEquals(args.items, []) + self.assertEqual(args.method, 'GET') + self.assertEqual(args.url, 'http://example.com/') + self.assertEqual(args.items, []) def test_guess_when_method_set_but_invalid_and_data_field(self): args = argparse.Namespace() @@ -791,9 +849,9 @@ class ArgumentParserTestCase(unittest.TestCase): self.parser._guess_method(args, Environment()) - self.assertEquals(args.method, 'POST') - self.assertEquals(args.url, 'http://example.com/') - self.assertEquals( + self.assertEqual(args.method, 'POST') + self.assertEqual(args.url, 'http://example.com/') + self.assertEqual( args.items, [input.KeyValue( key='data', value='field', sep='=', orig='data=field')]) @@ -809,9 +867,9 @@ class ArgumentParserTestCase(unittest.TestCase): stdout_isatty=True, )) - self.assertEquals(args.method, 'GET') - self.assertEquals(args.url, 'http://example.com/') - self.assertEquals( + self.assertEqual(args.method, 'GET') + self.assertEqual(args.url, 'http://example.com/') + self.assertEqual( args.items, [input.KeyValue( key='test', value='header', sep=':', orig='test:header')]) @@ -827,7 +885,7 @@ class ArgumentParserTestCase(unittest.TestCase): self.parser._guess_method(args, Environment()) - self.assertEquals(args.items, [ + self.assertEqual(args.items, [ input.KeyValue( key='new_item', value='a', sep='=', orig='new_item=a'), input.KeyValue(key @@ -879,8 +937,7 @@ class UnicodeOutputTestCase(BaseTestCase): args.style = 'default' # colorized output contains escape sequences - output = get_output(args, Environment(), response.request, response) - + output = get_output(args, Environment(), response.request, response).decode('utf8') for key, value in response_dict.items(): self.assertIn(key, output) self.assertIn(value, output)