holm/httpie-cli
1
1
Fork 0
forked from extern/httpie-cli
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix incorrect separators and introduce assert_output_matches() (close #1027)

Browse Source
This commit is contained in:
Jakub Roztocil 2021-01-30 22:14:57 +01:00 committed by GitHub
parent 0401d7b31c
commit 0f1e098cc4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 525 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.rst
View File

@ -11,6 +11,7 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
-------------------------
* Fixed upload with ``--session`` (`#1020`_).
* Fixed a missing blank line between request and response (`#1006`_).
`2.3.0`_ (2020-10-25)
@ -485,4 +486,5 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
.. _#934: https://github.com/httpie/httpie/issues/934
.. _#943: https://github.com/httpie/httpie/issues/943
.. _#963: https://github.com/httpie/httpie/issues/963
.. _#1006: https://github.com/httpie/httpie/issues/1006
.. _#1020: https://github.com/httpie/httpie/issues/1020

145
httpie/core.py
View File

@ -9,26 +9,17 @@ from pygments import __version__ as pygments_version
from requests import __version__ as requests_version
from httpie import __version__ as httpie_version
from httpie.cli.constants import (
OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY,
OUT_RESP_HEAD,
)
from httpie.cli.constants import OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD
from httpie.client import collect_messages
from httpie.context import Environment
from httpie.downloads import Downloader
from httpie.output.writer import (
write_message,
write_stream,
)
from httpie.output.writer import write_message, write_stream, MESSAGE_SEPARATOR_BYTES
from httpie.plugins.registry import plugin_manager
from httpie.status import ExitStatus, http_status_to_exit_status
# noinspection PyDefaultArgument
def main(
args: List[Union[str, bytes]] = sys.argv,
env=Environment(),
) -> ExitStatus:
def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitStatus:
"""
The main function.
@ -134,112 +125,82 @@ def get_output_options(
}[type(message)]
def program(
args: argparse.Namespace,
env: Environment,
) -> ExitStatus:
def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
"""
The main program without error handling.
"""
# TODO: Refactor and drastically simplify, especially so that the separator logic is elsewhere.
exit_status = ExitStatus.SUCCESS
downloader = None
initial_request: Optional[requests.PreparedRequest] = None
final_response: Optional[requests.Response] = None
def separate():
getattr(env.stdout, 'buffer', env.stdout).write(MESSAGE_SEPARATOR_BYTES)
def request_body_read_callback(chunk: bytes):
should_pipe_to_stdout = bool(
# Request body output desired
OUT_REQ_BODY in args.output_options
# & not `.read()` already pre-request (e.g., for compression)
and initial_request
# & non-EOF chunk
and chunk
)
if should_pipe_to_stdout:
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = chunk
msg.headers = initial_request.headers
write_message(requests_message=msg, env=env, args=args, with_body=True, with_headers=False)
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(
output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume
)
downloader = Downloader(output_file=args.output_file, progress_file=env.stderr, resume=args.download_resume)
downloader.pre_request(args.headers)
messages = collect_messages(args=args, config_dir=env.config.directory,
request_body_read_callback=request_body_read_callback)
force_separator = False
prev_with_body = False
needs_separator = False
def maybe_separate():
nonlocal needs_separator
if env.stdout_isatty and needs_separator:
needs_separator = False
getattr(env.stdout, 'buffer', env.stdout).write(b'\n\n')
initial_request: Optional[requests.PreparedRequest] = None
final_response: Optional[requests.Response] = None
def request_body_read_callback(chunk: bytes):
should_pipe_to_stdout = (
# Request body output desired
OUT_REQ_BODY in args.output_options
# & not `.read()` already pre-request (e.g., for compression)
and initial_request
# & non-EOF chunk
and chunk
)
if should_pipe_to_stdout:
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = chunk
msg.headers = initial_request.headers
write_message(
requests_message=msg,
env=env,
args=args,
with_body=True,
with_headers=False
)
messages = collect_messages(
args=args,
config_dir=env.config.directory,
request_body_read_callback=request_body_read_callback
)
# Process messages as they’re generated
for message in messages:
maybe_separate()
is_request = isinstance(message, requests.PreparedRequest)
with_headers, with_body = get_output_options(
args=args, message=message)
with_headers, with_body = get_output_options(args=args, message=message)
do_write_body = with_body
if prev_with_body and (with_headers or with_body) and (force_separator or not env.stdout_isatty):
# Separate after a previous message with body, if needed. See test_tokens.py.
separate()
force_separator = False
if is_request:
if not initial_request:
initial_request = message
is_streamed_upload = not isinstance(
message.body, (str, bytes))
is_streamed_upload = not isinstance(message.body, (str, bytes))
if with_body:
with_body = not is_streamed_upload
needs_separator = is_streamed_upload
do_write_body = not is_streamed_upload
force_separator = is_streamed_upload and env.stdout_isatty
else:
final_response = message
if args.check_status or downloader:
exit_status = http_status_to_exit_status(
http_status=message.status_code,
follow=args.follow
)
if (not env.stdout_isatty
and exit_status != ExitStatus.SUCCESS):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning'
)
write_message(
requests_message=message,
env=env,
args=args,
with_headers=with_headers,
with_body=with_body,
)
maybe_separate()
exit_status = http_status_to_exit_status(http_status=message.status_code, follow=args.follow)
if not env.stdout_isatty and exit_status != ExitStatus.SUCCESS:
env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level='warning')
write_message(requests_message=message, env=env, args=args, with_headers=with_headers,
with_body=do_write_body)
prev_with_body = with_body
# Cleanup
if force_separator:
separate()
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(
initial_url=initial_request.url,
final_response=final_response,
)
write_stream(
stream=download_stream,
outfile=download_to,
flush=False,
)
write_stream(stream=download_stream, outfile=download_to, flush=False)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
@ -253,9 +214,7 @@ def program(
finally:
if downloader and not downloader.finished:
downloader.failed()
if (not isinstance(args, list) and args.output_file
and args.output_file_specified):
if not isinstance(args, list) and args.output_file and args.output_file_specified:
args.output_file.close()

6
httpie/output/writer.py
View File

@ -12,6 +12,10 @@ from httpie.output.streams import (
)
MESSAGE_SEPARATOR = '\n\n'
MESSAGE_SEPARATOR_BYTES = MESSAGE_SEPARATOR.encode()
def write_message(
requests_message: Union[requests.PreparedRequest, requests.Response],
env: Environment,
@ -111,7 +115,7 @@ def build_output_stream_for_message(
and not getattr(requests_message, 'is_body_upload_chunk', False)):
# Ensure a blank line after the response body.
# For terminal output only.
yield b'\n\n'
yield MESSAGE_SEPARATOR_BYTES
def get_stream_type_and_kwargs(

2
setup.cfg
View File

@ -4,7 +4,7 @@
[tool:pytest]
# <https://docs.pytest.org/en/latest/customize.html>
norecursedirs = tests/fixtures
addopts = --tb=native
addopts = --tb=native --doctest-modules
[pycodestyle]

9
tests/test_regressions.py
View File

@ -2,6 +2,7 @@
import pytest
from httpie.compat import is_windows
from tests.utils.matching import assert_output_matches, Expect
from utils import HTTP_OK, MockEnvironment, http
@ -27,7 +28,6 @@ def test_output_devnull(httpbin):
http('--output=/dev/null', httpbin + '/get')
@pytest.mark.skip(reason='TODO: fix #1006')
def test_verbose_redirected_stdout_separator(httpbin):
"""
@ -40,3 +40,10 @@ def test_verbose_redirected_stdout_separator(httpbin):
env=MockEnvironment(stdout_isatty=False),
)
assert '}HTTP/' not in r
assert_output_matches(r, [
Expect.REQUEST_HEADERS,
Expect.BODY,
Expect.SEPARATOR,
Expect.RESPONSE_HEADERS,
Expect.BODY,
])

111
tests/test_tokens.py Normal file
View File

@ -0,0 +1,111 @@
"""
The ideas behind these test and the named templates is to ensure consistent output
across all supported different scenarios:
TODO: cover more scenarios
* terminal vs. redirect stdout
* different combinations of `--print=HBhb` (request/response headers/body)
* multipart requests
* streamed uploads
"""
from tests.utils.matching import assert_output_matches, Expect
from utils import http, HTTP_OK, MockEnvironment, HTTPBIN_WITH_CHUNKED_SUPPORT
RAW_REQUEST = [
Expect.REQUEST_HEADERS,
Expect.BODY,
]
RAW_RESPONSE = [
Expect.RESPONSE_HEADERS,
Expect.BODY,
]
RAW_EXCHANGE = [
*RAW_REQUEST,
Expect.SEPARATOR, # Good choice?
*RAW_RESPONSE,
]
RAW_BODY = [
Expect.BODY,
]
TERMINAL_REQUEST = [
*RAW_REQUEST,
Expect.SEPARATOR,
]
TERMINAL_RESPONSE = [
*RAW_RESPONSE,
Expect.SEPARATOR,
]
TERMINAL_EXCHANGE = [
*TERMINAL_REQUEST,
*TERMINAL_RESPONSE,
]
TERMINAL_BODY = [
RAW_BODY,
Expect.SEPARATOR
]
def test_headers():
r = http('--print=H', '--offline', 'pie.dev')
assert_output_matches(r, [Expect.REQUEST_HEADERS])
def test_redirected_headers():
r = http('--print=H', '--offline', 'pie.dev', env=MockEnvironment(stdout_isatty=False))
assert_output_matches(r, [Expect.REQUEST_HEADERS])
def test_terminal_headers_and_body():
r = http('--print=HB', '--offline', 'pie.dev', 'AAA=BBB')
assert_output_matches(r, TERMINAL_REQUEST)
def test_raw_headers_and_body():
r = http(
'--print=HB', '--offline', 'pie.dev', 'AAA=BBB',
env=MockEnvironment(stdout_isatty=False),
)
assert_output_matches(r, RAW_REQUEST)
def test_raw_body():
r = http(
'--print=B', '--offline', 'pie.dev', 'AAA=BBB',
env=MockEnvironment(stdout_isatty=False),
)
assert_output_matches(r, RAW_BODY)
def test_raw_exchange(httpbin):
r = http('--verbose', httpbin + '/post', 'a=b', env=MockEnvironment(stdout_isatty=False))
assert HTTP_OK in r
assert_output_matches(r, RAW_EXCHANGE)
def test_terminal_exchange(httpbin):
r = http('--verbose', httpbin + '/post', 'a=b')
assert HTTP_OK in r
assert_output_matches(r, TERMINAL_EXCHANGE)
def test_headers_multipart_body_separator():
r = http('--print=HB', '--multipart', '--offline', 'pie.dev', 'AAA=BBB')
assert_output_matches(r, TERMINAL_REQUEST)
def test_redirected_headers_multipart_no_separator():
r = http(
'--print=HB', '--multipart', '--offline', 'pie.dev', 'AAA=BBB',
env=MockEnvironment(stdout_isatty=False),
)
assert_output_matches(r, RAW_REQUEST)
def test_verbose_chunked():
r = http('--verbose', '--chunked', HTTPBIN_WITH_CHUNKED_SUPPORT + '/post', 'hello=world')
assert HTTP_OK in r
assert 'Transfer-Encoding: chunked' in r
assert_output_matches(r, TERMINAL_EXCHANGE)

20
tests/utils.py → tests/utils/__init__.py
View File

@ -1,12 +1,14 @@
# coding=utf-8
"""Utilities for HTTPie test suite."""
import re
import shlex
import sys
import time
import json
import tempfile
from io import BytesIO
from pathlib import Path
from typing import Optional, Union
from typing import Optional, Union, List
from httpie.status import ExitStatus
from httpie.config import Config
@ -20,7 +22,7 @@ from httpie.core import main
HTTPBIN_WITH_CHUNKED_SUPPORT = 'http://pie.dev'
TESTS_ROOT = Path(__file__).parent
TESTS_ROOT = Path(__file__).parent.parent
CRLF = '\r\n'
COLOR = '\x1b['
HTTP_OK = '200 OK'
@ -49,7 +51,7 @@ class StdinBytesIO(BytesIO):
class MockEnvironment(Environment):
"""Environment subclass with reasonable defaults for testing."""
colors = 0
colors = 0 # For easier debugging
stdin_isatty = True,
stdout_isatty = True
is_windows = False
@ -113,6 +115,15 @@ class BaseCLIResponse:
devnull: str = None
json: dict = None
exit_status: ExitStatus = None
command: str = None
args: List[str] = []
complete_args: List[str] = []
@property
def command(self):
cmd = ' '.join(shlex.quote(arg) for arg in ['http', *self.args])
# pytest-httpbin to real httpbin.
return re.sub(r'127\.0\.0\.1:\d+', 'httpbin.org', cmd)
class BytesCLIResponse(bytes, BaseCLIResponse):
@ -284,10 +295,13 @@ def http(
r.devnull = devnull_output
r.stderr = stderr.read()
r.exit_status = exit_status
r.args = args
r.complete_args = ' '.join(complete_args)
if r.exit_status != ExitStatus.SUCCESS:
sys.stderr.write(r.stderr)
# print(f'\n\n$ {r.command}\n')
return r
finally:

32
tests/utils/matching/__init__.py Normal file
View File

@ -0,0 +1,32 @@
from typing import Iterable
import pytest
from tests.utils.matching.parsing import OutputMatchingError, expect_tokens, Expect
__all__ = [
'assert_output_matches',
'assert_output_does_not_match',
'Expect',
]
def assert_output_matches(output: str, tokens: Iterable[Expect]):
r"""
Check the command `output` for an exact full sequence of `tokens`.
>>> out = 'GET / HTTP/1.1\r\nAAA:BBB\r\n\r\nCCC\n\n'
>>> assert_output_matches(out, [Expect.REQUEST_HEADERS, Expect.BODY, Expect.SEPARATOR])
"""
# TODO: auto-remove ansi colors to allow for testing of colorized output as well.
expect_tokens(tokens=tokens, s=output)
def assert_output_does_not_match(output: str, tokens: Iterable[Expect]):
r"""
>>> assert_output_does_not_match('\r\n', [Expect.BODY])
"""
with pytest.raises(OutputMatchingError):
assert_output_matches(output=output, tokens=tokens)

107
tests/utils/matching/parsing.py Normal file
View File

@ -0,0 +1,107 @@
import re
from typing import Iterable
from enum import Enum, auto
from httpie.output.writer import MESSAGE_SEPARATOR
from tests.utils import CRLF
class Expect(Enum):
"""
Predefined token types we can expect in the output.
"""
REQUEST_HEADERS = auto()
RESPONSE_HEADERS = auto()
BODY = auto()
SEPARATOR = auto()
SEPARATOR_RE = re.compile(f'^{MESSAGE_SEPARATOR}')
def make_headers_re(message_type: Expect):
assert message_type in {Expect.REQUEST_HEADERS, Expect.RESPONSE_HEADERS}
# language=RegExp
crlf = r'[\r][\n]'
non_crlf = rf'[^{CRLF}]'
# language=RegExp
http_version = r'HTTP/\d+\.\d+'
if message_type is Expect.REQUEST_HEADERS:
# POST /post HTTP/1.1
start_line_re = fr'{non_crlf}*{http_version}{crlf}'
else:
# HTTP/1.1 200 OK
start_line_re = fr'{http_version}{non_crlf}*{crlf}'
return re.compile(
fr'''
^
{start_line_re}
({non_crlf}+:{non_crlf}+{crlf})+
{crlf}
''',
flags=re.VERBOSE
)
BODY_ENDINGS = [
MESSAGE_SEPARATOR,
CRLF, # Not really but useful for testing (just remember not to include it in a body).
]
TOKEN_REGEX_MAP = {
Expect.REQUEST_HEADERS: make_headers_re(Expect.REQUEST_HEADERS),
Expect.RESPONSE_HEADERS: make_headers_re(Expect.RESPONSE_HEADERS),
Expect.SEPARATOR: SEPARATOR_RE,
}
class OutputMatchingError(ValueError):
pass
def expect_tokens(tokens: Iterable[Expect], s: str):
for token in tokens:
s = expect_token(token, s)
if s:
raise OutputMatchingError(f'Unmatched remaining output for {tokens} in {s!r}')
def expect_token(token: Expect, s: str) -> str:
if token is Expect.BODY:
s = expect_body(s)
else:
s = expect_regex(token, s)
return s
def expect_regex(token: Expect, s: str) -> str:
match = TOKEN_REGEX_MAP[token].match(s)
if not match:
raise OutputMatchingError(f'No match for {token} in {s!r}')
return s[match.end():]
def expect_body(s: str) -> str:
"""
We require some text, and continue to read until we find an ending or until the end of the string.
"""
if 'content-disposition:' in s.lower():
# Multipart body heuristic.
final_boundary_re = re.compile('\r\n--[^-]+?--\r\n')
match = final_boundary_re.search(s)
if match:
return s[match.end():]
endings = [s.index(sep) for sep in BODY_ENDINGS if sep in s]
if not endings:
s = '' # Only body
else:
end = min(endings)
if end == 0:
raise OutputMatchingError(f'Empty body: {s!r}')
s = s[end:]
return s

190
tests/utils/matching/test_matching.py Normal file
View File

@ -0,0 +1,190 @@
"""
Here we test our output parsing and matching implementation, not HTTPie itself.
"""
from httpie.output.writer import MESSAGE_SEPARATOR
from tests.utils import CRLF
from tests.utils.matching import assert_output_does_not_match, assert_output_matches, Expect
def test_assert_output_matches_headers_incomplete():
assert_output_does_not_match(f'HTTP/1.1{CRLF}', [Expect.RESPONSE_HEADERS])
def test_assert_output_matches_headers_unterminated():
assert_output_does_not_match(
(
f'HTTP/1.1{CRLF}'
f'AAA:BBB'
f'{CRLF}'
),
[Expect.RESPONSE_HEADERS],
)
def test_assert_output_matches_response_headers():
assert_output_matches(
(
f'HTTP/1.1 200 OK{CRLF}'
f'AAA:BBB{CRLF}'
f'{CRLF}'
),
[Expect.RESPONSE_HEADERS],
)
def test_assert_output_matches_request_headers():
assert_output_matches(
(
f'GET / HTTP/1.1{CRLF}'
f'AAA:BBB{CRLF}'
f'{CRLF}'
),
[Expect.REQUEST_HEADERS],
)
def test_assert_output_matches_headers_and_separator():
assert_output_matches(
(
f'HTTP/1.1{CRLF}'
f'AAA:BBB{CRLF}'
f'{CRLF}'
f'{MESSAGE_SEPARATOR}'
),
[Expect.RESPONSE_HEADERS, Expect.SEPARATOR],
)
def test_assert_output_matches_body_unmatched_crlf():
assert_output_does_not_match(f'AAA{CRLF}', [Expect.BODY])
def test_assert_output_matches_body_unmatched_separator():
assert_output_does_not_match(f'AAA{MESSAGE_SEPARATOR}', [Expect.BODY])
def test_assert_output_matches_body_and_separator():
assert_output_matches(f'AAA{MESSAGE_SEPARATOR}', [Expect.BODY, Expect.SEPARATOR])
def test_assert_output_matches_body_r():
assert_output_matches(f'AAA\r', [Expect.BODY])
def test_assert_output_matches_body_n():
assert_output_matches(f'AAA\n', [Expect.BODY])
def test_assert_output_matches_body_r_body():
assert_output_matches(f'AAA\rBBB', [Expect.BODY])
def test_assert_output_matches_body_n_body():
assert_output_matches(f'AAA\nBBB', [Expect.BODY])
def test_assert_output_matches_headers_and_body():
assert_output_matches(
(
f'HTTP/1.1{CRLF}'
f'AAA:BBB{CRLF}'
f'{CRLF}'
f'CCC'
),
[Expect.RESPONSE_HEADERS, Expect.BODY]
)
def test_assert_output_matches_headers_with_body_and_separator():
assert_output_matches(
(
f'HTTP/1.1 {CRLF}'
f'AAA:BBB{CRLF}{CRLF}'
f'CCC{MESSAGE_SEPARATOR}'
),
[Expect.RESPONSE_HEADERS, Expect.BODY, Expect.SEPARATOR]
)
def test_assert_output_matches_multiple_messages():
assert_output_matches(
(
f'POST / HTTP/1.1{CRLF}'
f'AAA:BBB{CRLF}'
f'{CRLF}'
f'CCC'
f'{MESSAGE_SEPARATOR}'
f'HTTP/1.1 200 OK{CRLF}'
f'EEE:FFF{CRLF}'
f'{CRLF}'
f'GGG'
f'{MESSAGE_SEPARATOR}'
), [
Expect.REQUEST_HEADERS,
Expect.BODY,
Expect.SEPARATOR,
Expect.RESPONSE_HEADERS,
Expect.BODY,
Expect.SEPARATOR,
]
)
def test_assert_output_matches_multipart_body():
output = (
'POST / HTTP/1.1\r\n'
'User-Agent: HTTPie/2.4.0-dev\r\n'
'Accept-Encoding: gzip, deflate\r\n'
'Accept: */*\r\n'
'Connection: keep-alive\r\n'
'Content-Type: multipart/form-data; boundary=1e22169de43e4a2e8d9e41c0a1c93cc5\r\n'
'Content-Length: 212\r\n'
'Host: pie.dev\r\n'
'\r\n'
'--1e22169de43e4a2e8d9e41c0a1c93cc5\r\n'
'Content-Disposition: form-data; name="AAA"\r\n'
'\r\n'
'BBB\r\n'
'--1e22169de43e4a2e8d9e41c0a1c93cc5\r\n'
'Content-Disposition: form-data; name="CCC"\r\n'
'\r\n'
'DDD\r\n'
'--1e22169de43e4a2e8d9e41c0a1c93cc5--\r\n'
)
assert_output_matches(output, [Expect.REQUEST_HEADERS, Expect.BODY])
def test_assert_output_matches_multipart_body_with_separator():
output = (
'POST / HTTP/1.1\r\n'
'User-Agent: HTTPie/2.4.0-dev\r\n'
'Accept-Encoding: gzip, deflate\r\n'
'Accept: */*\r\n'
'Connection: keep-alive\r\n'
'Content-Type: multipart/form-data; boundary=1e22169de43e4a2e8d9e41c0a1c93cc5\r\n'
'Content-Length: 212\r\n'
'Host: pie.dev\r\n'
'\r\n'
'--1e22169de43e4a2e8d9e41c0a1c93cc5\r\n'
'Content-Disposition: form-data; name="AAA"\r\n'
'\r\n'
'BBB\r\n'
'--1e22169de43e4a2e8d9e41c0a1c93cc5\r\n'
'Content-Disposition: form-data; name="CCC"\r\n'
'\r\n'
'DDD\r\n'
'--1e22169de43e4a2e8d9e41c0a1c93cc5--\r\n'
f'{MESSAGE_SEPARATOR}'
)
assert_output_matches(output, [Expect.REQUEST_HEADERS, Expect.BODY, Expect.SEPARATOR])
def test_assert_output_matches_multiple_separators():
assert_output_matches(
MESSAGE_SEPARATOR + MESSAGE_SEPARATOR + 'AAA' + MESSAGE_SEPARATOR + MESSAGE_SEPARATOR,
[Expect.SEPARATOR, Expect.SEPARATOR, Expect.BODY, Expect.SEPARATOR, Expect.SEPARATOR]
)