Preserve individual headers with the same name on responses (#1208)

* Preserve individual headers with the same name on responses

* Rename RequestHeadersDict to HTTPHeadersDict

* Update tests/utils/http_server.py

* Update tests/utils/http_server.py

* Update httpie/adapters.py

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

CHANGELOG.md

@@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 ## [2.7.0.dev0](https://github.com/httpie/httpie/compare/2.6.0...master) (unreleased)
 
 - Added support for sending multiple HTTP headers with the same name. ([#130](https://github.com/httpie/httpie/issues/130))
+- Added support for receving multiple HTTP headers with the same name, individually. ([#1207](https://github.com/httpie/httpie/issues/1207))
 - Added support for keeping `://` in the URL argument to allow quick conversions of pasted URLs into HTTPie calls just by adding a space after the protocol name (`$ https ://pie.dev` → `https://pie.dev`). ([#1195](https://github.com/httpie/httpie/issues/1195))
 
 ## [2.6.0](https://github.com/httpie/httpie/compare/2.5.0...2.6.0) (2021-10-14)

httpie/adapters.py

@@ -0,0 +1,13 @@
+from httpie.cli.dicts import HTTPHeadersDict
+from requests.adapters import HTTPAdapter
+
+
+class HTTPieHTTPAdapter(HTTPAdapter):
+
+    def build_response(self, req, resp):
+        """Wrap the original headers with the `HTTPHeadersDict`
+        to preserve multiple headers that have the same name"""
+
+        response = super().build_response(req, resp)
+        response.headers = HTTPHeadersDict(getattr(resp, 'headers', {}))
+        return response

httpie/cli/dicts.py

@@ -9,7 +9,7 @@ class BaseMultiDict(MultiDict):
     """
 
 
-class RequestHeadersDict(CIMultiDict, BaseMultiDict):
+class HTTPHeadersDict(CIMultiDict, BaseMultiDict):
     """
     Headers are case-insensitive and multiple values are supported
     through the `add()` API.

httpie/cli/requestitems.py

@@ -11,7 +11,7 @@ from .constants import (
 )
 from .dicts import (
     BaseMultiDict, MultipartRequestDataDict, RequestDataDict,
-    RequestFilesDict, RequestHeadersDict, RequestJSONDataDict,
+    RequestFilesDict, HTTPHeadersDict, RequestJSONDataDict,
     RequestQueryParamsDict,
 )
 from .exceptions import ParseError
@@ -21,7 +21,7 @@ from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys
 class RequestItems:
 
     def __init__(self, as_form=False):
-        self.headers = RequestHeadersDict()
+        self.headers = HTTPHeadersDict()
         self.data = RequestDataDict() if as_form else RequestJSONDataDict()
         self.files = RequestFilesDict()
         self.params = RequestQueryParamsDict()

httpie/client.py

@@ -11,7 +11,8 @@ import requests
 # noinspection PyPackageRequirements
 import urllib3
 from . import __version__
-from .cli.dicts import RequestHeadersDict
+from .adapters import HTTPieHTTPAdapter
+from .cli.dicts import HTTPHeadersDict
 from .encoding import UTF8
 from .plugins.registry import plugin_manager
 from .sessions import get_httpie_session
@@ -153,6 +154,7 @@ def build_requests_session(
     requests_session = requests.Session()
 
     # Install our adapter.
+    http_adapter = HTTPieHTTPAdapter()
     https_adapter = HTTPieHTTPSAdapter(
         ciphers=ciphers,
         verify=verify,
@@ -161,6 +163,7 @@ def build_requests_session(
             if ssl_version else None
         ),
     )
+    requests_session.mount('http://', http_adapter)
     requests_session.mount('https://', https_adapter)
 
     # Install adapters from plugins.
@@ -179,8 +182,8 @@ def dump_request(kwargs: dict):
         f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n')
 
 
-def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
+def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict:
-    final_headers = RequestHeadersDict()
+    final_headers = HTTPHeadersDict()
     for name, value in headers.items():
         if value is not None:
             # “leading or trailing LWS MAY be removed without
@@ -197,13 +200,13 @@ def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
 
 def apply_missing_repeated_headers(
     prepared_request: requests.PreparedRequest,
-    original_headers: RequestHeadersDict
+    original_headers: HTTPHeadersDict
 ) -> None:
     """Update the given `prepared_request`'s headers with the original
     ones. This allows the requests to be prepared as usual, and then later
     merged with headers that are specified multiple times."""
 
-    new_headers = RequestHeadersDict(prepared_request.headers)
+    new_headers = HTTPHeadersDict(prepared_request.headers)
     for prepared_name, prepared_value in prepared_request.headers.items():
         if prepared_name not in original_headers:
             continue
@@ -225,8 +228,8 @@ def apply_missing_repeated_headers(
     prepared_request.headers = new_headers
 
 
-def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict:
+def make_default_headers(args: argparse.Namespace) -> HTTPHeadersDict:
-    default_headers = RequestHeadersDict({
+    default_headers = HTTPHeadersDict({
         'User-Agent': DEFAULT_UA
     })
 
@@ -271,7 +274,7 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
 
 def make_request_kwargs(
     args: argparse.Namespace,
-    base_headers: RequestHeadersDict = None,
+    base_headers: HTTPHeadersDict = None,
     request_body_read_callback=lambda chunk: chunk
 ) -> dict:
     """

httpie/models.py

@@ -72,7 +72,9 @@ class HTTPResponse(HTTPMessage):
         )
         headers.extend(
             f'Set-Cookie: {cookie}'
-            for cookie in split_cookies(original.headers.get('Set-Cookie'))
+            for header, value in original.headers.items()
+            for cookie in split_cookies(value)
+            if header == 'Set-Cookie'
         )
         return '\r\n'.join(headers)
 

httpie/sessions.py

@@ -13,7 +13,7 @@ from urllib.parse import urlsplit
 from requests.auth import AuthBase
 from requests.cookies import RequestsCookieJar, create_cookie
 
-from .cli.dicts import RequestHeadersDict
+from .cli.dicts import HTTPHeadersDict
 from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
 from .plugins.registry import plugin_manager
 
@@ -65,7 +65,7 @@ class Session(BaseConfigDict):
             'password': None
         }
 
-    def update_headers(self, request_headers: RequestHeadersDict):
+    def update_headers(self, request_headers: HTTPHeadersDict):
         """
         Update the session headers with the request ones while ignoring
         certain name prefixes.
@@ -98,8 +98,8 @@ class Session(BaseConfigDict):
         self['headers'] = dict(headers)
 
     @property
-    def headers(self) -> RequestHeadersDict:
+    def headers(self) -> HTTPHeadersDict:
-        return RequestHeadersDict(self['headers'])
+        return HTTPHeadersDict(self['headers'])
 
     @property
     def cookies(self) -> RequestsCookieJar:

httpie/ssl.py

@@ -1,6 +1,6 @@
 import ssl
 
-from requests.adapters import HTTPAdapter
+from httpie.adapters import HTTPAdapter
 # noinspection PyPackageRequirements
 from urllib3.util.ssl_ import (
     DEFAULT_CIPHERS, create_urllib3_context,

tests/conftest.py

@@ -5,6 +5,7 @@ import pytest
 from pytest_httpbin import certs
 
 from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT
+from .utils.http_server import http_server # noqa
 
 
 @pytest.fixture(scope='function', autouse=True)

tests/test_cli.py

@@ -39,7 +39,7 @@ class TestItemParsing:
             # files
             self.key_value_arg(fr'bar\@baz@{FILE_PATH_ARG}'),
         ])
-        # `RequestHeadersDict` => `dict`
+        # `HTTPHeadersDict` => `dict`
         headers = dict(items.headers)
 
         assert headers == {
@@ -88,7 +88,7 @@ class TestItemParsing:
         ])
 
         # Parsed headers
-        # `RequestHeadersDict` => `dict`
+        # `HTTPHeadersDict` => `dict`
         headers = dict(items.headers)
         assert headers == {
             'Header': 'value',

tests/test_httpie.py

@@ -291,6 +291,33 @@ def test_headers_multiple_headers_representation(httpbin_both, pretty):
     assert 'c: c' in r
 
 
+def test_response_headers_multiple(http_server):
+    r = http('GET', http_server + '/headers', 'Foo:bar', 'Foo:baz')
+    assert 'Foo: bar' in r
+    assert 'Foo: baz' in r
+
+
+def test_response_headers_multiple_repeated(http_server):
+    r = http('GET', http_server + '/headers', 'Foo:bar', 'Foo:baz',
+             'Foo:bar')
+    assert r.count('Foo: bar') == 2
+    assert 'Foo: baz' in r
+
+
+@pytest.mark.parametrize('pretty', ['format', 'none'])
+def test_response_headers_multiple_representation(http_server, pretty):
+    r = http('--pretty', pretty, http_server + '/headers',
+             'A:A', 'A:B', 'A:C', 'B:A', 'B:B', 'C:C', 'C:c')
+
+    assert 'A: A' in r
+    assert 'A: B' in r
+    assert 'A: C' in r
+    assert 'B: A' in r
+    assert 'B: B' in r
+    assert 'C: C' in r
+    assert 'C: c' in r
+
+
 def test_json_input_preserve_order(httpbin_both):
     r = http('PATCH', httpbin_both + '/patch',
              'order:={"map":{"1":"first","2":"second"}}')

tests/utils/http_server.py

@@ -0,0 +1,50 @@
+import threading
+
+from collections import defaultdict
+from http import HTTPStatus
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from urllib.parse import urlparse
+
+import pytest
+
+
+class TestHandler(BaseHTTPRequestHandler):
+    handlers = defaultdict(dict)
+
+    @classmethod
+    def handler(cls, method, path):
+        def inner(func):
+            cls.handlers[method][path] = func
+            return func
+        return inner
+
+    def do_GET(self):
+        parse_result = urlparse(self.path)
+        func = self.handlers['GET'].get(parse_result.path)
+        if func is None:
+            return self.send_error(HTTPStatus.NOT_FOUND)
+
+        return func(self)
+
+
+@TestHandler.handler('GET', '/headers')
+def get_headers(handler):
+    handler.send_response(200)
+    for key, value in handler.headers.items():
+        handler.send_header(key, value)
+    handler.send_header('Content-Length', 0)
+    handler.end_headers()
+
+
+@pytest.fixture(scope="function")
+def http_server():
+    """A custom HTTP server implementation for our tests, that is
+    built on top of the http.server module. Handy when we need to
+    deal with details which httpbin can not capture."""
+
+    server = HTTPServer(('localhost', 0), TestHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.start()
+    yield '{}:{}'.format(*server.socket.getsockname())
+    server.shutdown()
+    thread.join(timeout=0.5)