From c000886546cb6d3a8d49f48054a793d84f4b3eb6 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 25 Nov 2021 02:41:37 +0300 Subject: [PATCH] Preserve individual headers with the same name on responses (#1208) * Preserve individual headers with the same name on responses * Rename RequestHeadersDict to HTTPHeadersDict * Update tests/utils/http_server.py * Update tests/utils/http_server.py * Update httpie/adapters.py Co-authored-by: Jakub Roztocil --- CHANGELOG.md | 1 + httpie/adapters.py | 13 ++++++++++ httpie/cli/dicts.py | 2 +- httpie/cli/requestitems.py | 4 +-- httpie/client.py | 19 +++++++++------ httpie/models.py | 4 ++- httpie/sessions.py | 8 +++--- httpie/ssl.py | 2 +- tests/conftest.py | 1 + tests/test_cli.py | 4 +-- tests/test_httpie.py | 27 ++++++++++++++++++++ tests/utils/http_server.py | 50 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 httpie/adapters.py create mode 100644 tests/utils/http_server.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 74b4956c..13a79a76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [2.7.0.dev0](https://github.com/httpie/httpie/compare/2.6.0...master) (unreleased) - Added support for sending multiple HTTP headers with the same name. ([#130](https://github.com/httpie/httpie/issues/130)) +- Added support for receving multiple HTTP headers with the same name, individually. ([#1207](https://github.com/httpie/httpie/issues/1207)) - Added support for keeping `://` in the URL argument to allow quick conversions of pasted URLs into HTTPie calls just by adding a space after the protocol name (`$ https ://pie.dev` → `https://pie.dev`). ([#1195](https://github.com/httpie/httpie/issues/1195)) ## [2.6.0](https://github.com/httpie/httpie/compare/2.5.0...2.6.0) (2021-10-14) diff --git a/httpie/adapters.py b/httpie/adapters.py new file mode 100644 index 00000000..8e2dd739 --- /dev/null +++ b/httpie/adapters.py @@ -0,0 +1,13 @@ +from httpie.cli.dicts import HTTPHeadersDict +from requests.adapters import HTTPAdapter + + +class HTTPieHTTPAdapter(HTTPAdapter): + + def build_response(self, req, resp): + """Wrap the original headers with the `HTTPHeadersDict` + to preserve multiple headers that have the same name""" + + response = super().build_response(req, resp) + response.headers = HTTPHeadersDict(getattr(resp, 'headers', {})) + return response diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 43a624ee..3d0cab5a 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -9,7 +9,7 @@ class BaseMultiDict(MultiDict): """ -class RequestHeadersDict(CIMultiDict, BaseMultiDict): +class HTTPHeadersDict(CIMultiDict, BaseMultiDict): """ Headers are case-insensitive and multiple values are supported through the `add()` API. diff --git a/httpie/cli/requestitems.py b/httpie/cli/requestitems.py index 73d5e4d0..c1c2568e 100644 --- a/httpie/cli/requestitems.py +++ b/httpie/cli/requestitems.py @@ -11,7 +11,7 @@ from .constants import ( ) from .dicts import ( BaseMultiDict, MultipartRequestDataDict, RequestDataDict, - RequestFilesDict, RequestHeadersDict, RequestJSONDataDict, + RequestFilesDict, HTTPHeadersDict, RequestJSONDataDict, RequestQueryParamsDict, ) from .exceptions import ParseError @@ -21,7 +21,7 @@ from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys class RequestItems: def __init__(self, as_form=False): - self.headers = RequestHeadersDict() + self.headers = HTTPHeadersDict() self.data = RequestDataDict() if as_form else RequestJSONDataDict() self.files = RequestFilesDict() self.params = RequestQueryParamsDict() diff --git a/httpie/client.py b/httpie/client.py index 45b43276..a48527a2 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -11,7 +11,8 @@ import requests # noinspection PyPackageRequirements import urllib3 from . import __version__ -from .cli.dicts import RequestHeadersDict +from .adapters import HTTPieHTTPAdapter +from .cli.dicts import HTTPHeadersDict from .encoding import UTF8 from .plugins.registry import plugin_manager from .sessions import get_httpie_session @@ -153,6 +154,7 @@ def build_requests_session( requests_session = requests.Session() # Install our adapter. + http_adapter = HTTPieHTTPAdapter() https_adapter = HTTPieHTTPSAdapter( ciphers=ciphers, verify=verify, @@ -161,6 +163,7 @@ def build_requests_session( if ssl_version else None ), ) + requests_session.mount('http://', http_adapter) requests_session.mount('https://', https_adapter) # Install adapters from plugins. @@ -179,8 +182,8 @@ def dump_request(kwargs: dict): f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n') -def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict: - final_headers = RequestHeadersDict() +def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict: + final_headers = HTTPHeadersDict() for name, value in headers.items(): if value is not None: # “leading or trailing LWS MAY be removed without @@ -197,13 +200,13 @@ def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict: def apply_missing_repeated_headers( prepared_request: requests.PreparedRequest, - original_headers: RequestHeadersDict + original_headers: HTTPHeadersDict ) -> None: """Update the given `prepared_request`'s headers with the original ones. This allows the requests to be prepared as usual, and then later merged with headers that are specified multiple times.""" - new_headers = RequestHeadersDict(prepared_request.headers) + new_headers = HTTPHeadersDict(prepared_request.headers) for prepared_name, prepared_value in prepared_request.headers.items(): if prepared_name not in original_headers: continue @@ -225,8 +228,8 @@ def apply_missing_repeated_headers( prepared_request.headers = new_headers -def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict: - default_headers = RequestHeadersDict({ +def make_default_headers(args: argparse.Namespace) -> HTTPHeadersDict: + default_headers = HTTPHeadersDict({ 'User-Agent': DEFAULT_UA }) @@ -271,7 +274,7 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict: def make_request_kwargs( args: argparse.Namespace, - base_headers: RequestHeadersDict = None, + base_headers: HTTPHeadersDict = None, request_body_read_callback=lambda chunk: chunk ) -> dict: """ diff --git a/httpie/models.py b/httpie/models.py index 9c72518a..64079d0c 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -72,7 +72,9 @@ class HTTPResponse(HTTPMessage): ) headers.extend( f'Set-Cookie: {cookie}' - for cookie in split_cookies(original.headers.get('Set-Cookie')) + for header, value in original.headers.items() + for cookie in split_cookies(value) + if header == 'Set-Cookie' ) return '\r\n'.join(headers) diff --git a/httpie/sessions.py b/httpie/sessions.py index 0a04fb74..176c03e7 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -13,7 +13,7 @@ from urllib.parse import urlsplit from requests.auth import AuthBase from requests.cookies import RequestsCookieJar, create_cookie -from .cli.dicts import RequestHeadersDict +from .cli.dicts import HTTPHeadersDict from .config import BaseConfigDict, DEFAULT_CONFIG_DIR from .plugins.registry import plugin_manager @@ -65,7 +65,7 @@ class Session(BaseConfigDict): 'password': None } - def update_headers(self, request_headers: RequestHeadersDict): + def update_headers(self, request_headers: HTTPHeadersDict): """ Update the session headers with the request ones while ignoring certain name prefixes. @@ -98,8 +98,8 @@ class Session(BaseConfigDict): self['headers'] = dict(headers) @property - def headers(self) -> RequestHeadersDict: - return RequestHeadersDict(self['headers']) + def headers(self) -> HTTPHeadersDict: + return HTTPHeadersDict(self['headers']) @property def cookies(self) -> RequestsCookieJar: diff --git a/httpie/ssl.py b/httpie/ssl.py index f41b8020..cdec18f8 100644 --- a/httpie/ssl.py +++ b/httpie/ssl.py @@ -1,6 +1,6 @@ import ssl -from requests.adapters import HTTPAdapter +from httpie.adapters import HTTPAdapter # noinspection PyPackageRequirements from urllib3.util.ssl_ import ( DEFAULT_CIPHERS, create_urllib3_context, diff --git a/tests/conftest.py b/tests/conftest.py index a65df162..fe5f2116 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import pytest from pytest_httpbin import certs from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT +from .utils.http_server import http_server # noqa @pytest.fixture(scope='function', autouse=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index a478fcd2..09a39c11 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -39,7 +39,7 @@ class TestItemParsing: # files self.key_value_arg(fr'bar\@baz@{FILE_PATH_ARG}'), ]) - # `RequestHeadersDict` => `dict` + # `HTTPHeadersDict` => `dict` headers = dict(items.headers) assert headers == { @@ -88,7 +88,7 @@ class TestItemParsing: ]) # Parsed headers - # `RequestHeadersDict` => `dict` + # `HTTPHeadersDict` => `dict` headers = dict(items.headers) assert headers == { 'Header': 'value', diff --git a/tests/test_httpie.py b/tests/test_httpie.py index 748e8769..f646d4e0 100644 --- a/tests/test_httpie.py +++ b/tests/test_httpie.py @@ -291,6 +291,33 @@ def test_headers_multiple_headers_representation(httpbin_both, pretty): assert 'c: c' in r +def test_response_headers_multiple(http_server): + r = http('GET', http_server + '/headers', 'Foo:bar', 'Foo:baz') + assert 'Foo: bar' in r + assert 'Foo: baz' in r + + +def test_response_headers_multiple_repeated(http_server): + r = http('GET', http_server + '/headers', 'Foo:bar', 'Foo:baz', + 'Foo:bar') + assert r.count('Foo: bar') == 2 + assert 'Foo: baz' in r + + +@pytest.mark.parametrize('pretty', ['format', 'none']) +def test_response_headers_multiple_representation(http_server, pretty): + r = http('--pretty', pretty, http_server + '/headers', + 'A:A', 'A:B', 'A:C', 'B:A', 'B:B', 'C:C', 'C:c') + + assert 'A: A' in r + assert 'A: B' in r + assert 'A: C' in r + assert 'B: A' in r + assert 'B: B' in r + assert 'C: C' in r + assert 'C: c' in r + + def test_json_input_preserve_order(httpbin_both): r = http('PATCH', httpbin_both + '/patch', 'order:={"map":{"1":"first","2":"second"}}') diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py new file mode 100644 index 00000000..42be768b --- /dev/null +++ b/tests/utils/http_server.py @@ -0,0 +1,50 @@ +import threading + +from collections import defaultdict +from http import HTTPStatus +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse + +import pytest + + +class TestHandler(BaseHTTPRequestHandler): + handlers = defaultdict(dict) + + @classmethod + def handler(cls, method, path): + def inner(func): + cls.handlers[method][path] = func + return func + return inner + + def do_GET(self): + parse_result = urlparse(self.path) + func = self.handlers['GET'].get(parse_result.path) + if func is None: + return self.send_error(HTTPStatus.NOT_FOUND) + + return func(self) + + +@TestHandler.handler('GET', '/headers') +def get_headers(handler): + handler.send_response(200) + for key, value in handler.headers.items(): + handler.send_header(key, value) + handler.send_header('Content-Length', 0) + handler.end_headers() + + +@pytest.fixture(scope="function") +def http_server(): + """A custom HTTP server implementation for our tests, that is + built on top of the http.server module. Handy when we need to + deal with details which httpbin can not capture.""" + + server = HTTPServer(('localhost', 0), TestHandler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + yield '{}:{}'.format(*server.socket.getsockname()) + server.shutdown() + thread.join(timeout=0.5)