Add support for sending secure cookies over localhost (#1327)

* Add support for sending secure cookies over localhost

* Refactor

* Fix the CI

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

CHANGELOG.md

@@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 - Added support for session persistence of repeated headers with the same name. ([#1335](https://github.com/httpie/httpie/pull/1335))
 - Changed `httpie plugins` to the new `httpie cli` namespace as `httpie cli plugins` (`httpie plugins` continues to work as a hidden alias). ([#1320](https://github.com/httpie/httpie/issues/1320))
 - Fixed redundant creation of `Content-Length` header on `OPTIONS` requests. ([#1310](https://github.com/httpie/httpie/issues/1310))
-
+- Added support for sending `Secure` cookies to the `localhost` (and `.local` suffixed domains). ([#1308](https://github.com/httpie/httpie/issues/1308))
 
 ## [3.1.0](https://github.com/httpie/httpie/compare/3.0.2...3.1.0) (2022-03-08)
 

httpie/compat.py

@@ -1,6 +1,15 @@
 import sys
 from typing import Any, Optional, Iterable
 
+from httpie.cookies import HTTPieCookiePolicy
+from http import cookiejar # noqa
+
+
+# Request does not carry the original policy attached to the
+# cookie jar, so until it is resolved we change the global cookie
+# policy. <https://github.com/psf/requests/issues/5449>
+cookiejar.DefaultCookiePolicy = HTTPieCookiePolicy
+
 
 is_windows = 'win32' in str(sys.platform).lower()
 

httpie/cookies.py

@@ -0,0 +1,25 @@
+from http import cookiejar
+
+
+_LOCALHOST = 'localhost'
+_LOCALHOST_SUFFIX = '.localhost'
+
+
+class HTTPieCookiePolicy(cookiejar.DefaultCookiePolicy):
+    def return_ok_secure(self, cookie, request):
+        """Check whether the given cookie is sent to a secure host."""
+
+        is_secure_protocol = super().return_ok_secure(cookie, request)
+        if is_secure_protocol:
+            return True
+
+        # The original implementation of this method only takes secure protocols
+        # (e.g., https) into account, but the latest developments in modern browsers
+        # (chrome, firefox) assume 'localhost' is also a secure location. So we
+        # override it with our own strategy.
+        return self._is_local_host(cookiejar.request_host(request))
+
+    def _is_local_host(self, hostname):
+        # Implements the static localhost detection algorithm in firefox.
+        # <https://searchfox.org/mozilla-central/rev/d4d7611ee4dd0003b492b865bc5988a4e6afc985/netwerk/dns/DNS.cpp#205-218>
+        return hostname == _LOCALHOST or hostname.endswith(_LOCALHOST_SUFFIX)

httpie/sessions.py

@@ -14,6 +14,7 @@ from requests.auth import AuthBase
 from requests.cookies import RequestsCookieJar, remove_cookie_by_name
 
 from .context import Environment, Levels
+from .cookies import HTTPieCookiePolicy
 from .cli.dicts import HTTPHeadersDict
 from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
 from .utils import url_as_host
@@ -146,7 +147,10 @@ class Session(BaseConfigDict):
         # Runtime state of the Session objects.
         self.env = env
         self._headers = HTTPHeadersDict()
-        self.cookie_jar = RequestsCookieJar()
+        self.cookie_jar = RequestsCookieJar(
+            # See also a temporary workaround for a Requests bug in `compat.py`.
+            policy=HTTPieCookiePolicy(),
+        )
         self.session_id = session_id
         self.bound_host = bound_host
         self.suppress_legacy_warnings = suppress_legacy_warnings

tests/conftest.py

@@ -7,6 +7,7 @@ from pytest_httpbin import certs
 from .utils import ( # noqa
     HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN,
     HTTPBIN_WITH_CHUNKED_SUPPORT,
+    REMOTE_HTTPBIN_DOMAIN,
     mock_env
 )
 from .utils.plugins_cli import ( # noqa
@@ -17,7 +18,7 @@ from .utils.plugins_cli import ( # noqa
     httpie_plugins_success,
     interface,
 )
-from .utils.http_server import http_server # noqa
+from .utils.http_server import http_server, localhost_http_server # noqa
 
 
 @pytest.fixture(scope='function', autouse=True)
@@ -58,6 +59,22 @@ def httpbin_with_chunked_support(_httpbin_with_chunked_support_available):
     pytest.skip(f'{HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN} not resolvable')
 
 
+@pytest.fixture(scope='session')
+def _remote_httpbin_available():
+    try:
+        socket.gethostbyname(REMOTE_HTTPBIN_DOMAIN)
+        return True
+    except OSError:
+        return False
+
+
+@pytest.fixture
+def remote_httpbin(_remote_httpbin_available):
+    if _remote_httpbin_available:
+        return 'http://' + REMOTE_HTTPBIN_DOMAIN
+    pytest.skip(f'{REMOTE_HTTPBIN_DOMAIN} not resolvable')
+
+
 @pytest.fixture(autouse=True, scope='session')
 def pyopenssl_inject():
     """

tests/test_cookie_on_redirects.py

@@ -2,11 +2,6 @@ import pytest
 from .utils import http
 
 
-@pytest.fixture
-def remote_httpbin(httpbin_with_chunked_support):
-    return httpbin_with_chunked_support
-
-
 def _stringify(fixture):
     return fixture + ''
 

tests/test_sessions.py

@@ -800,3 +800,37 @@ def test_session_multiple_headers_with_same_name(basic_session, httpbin):
     )
     assert r.count('Foo: bar') == 2
     assert 'Foo: baz' in r
+
+
+@pytest.mark.parametrize(
+    'server, expected_cookies',
+    [
+        (
+            pytest.lazy_fixture('localhost_http_server'),
+            {'secure_cookie': 'foo', 'insecure_cookie': 'bar'}
+        ),
+        (
+            pytest.lazy_fixture('remote_httpbin'),
+            {'insecure_cookie': 'bar'}
+        )
+    ]
+)
+def test_secure_cookies_on_localhost(mock_env, tmp_path, server, expected_cookies):
+    session_path = tmp_path / 'session.json'
+    http(
+        '--session', str(session_path),
+        server + '/cookies/set',
+        'secure_cookie==foo',
+        'insecure_cookie==bar'
+    )
+
+    with open_session(session_path, mock_env) as session:
+        for cookie in session.cookies:
+            if cookie.name == 'secure_cookie':
+                cookie.secure = True
+
+    r = http(
+        '--session', str(session_path),
+        server + '/cookies'
+    )
+    assert r.json == {'cookies': expected_cookies}

tests/utils/__init__.py

@@ -21,6 +21,8 @@ from httpie.context import Environment
 from httpie.utils import url_as_host
 
 
+REMOTE_HTTPBIN_DOMAIN = 'pie.dev'
+
 # pytest-httpbin currently does not support chunked requests:
 # <https://github.com/kevin1024/pytest-httpbin/issues/33>
 # <https://github.com/kevin1024/pytest-httpbin/issues/28>

tests/utils/http_server.py

@@ -1,9 +1,12 @@
 import threading
+import json
 
 from collections import defaultdict
+from contextlib import contextmanager
 from http import HTTPStatus
+from http.cookies import SimpleCookie
 from http.server import HTTPServer, BaseHTTPRequestHandler
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qs
 
 import pytest
 
@@ -85,6 +88,34 @@ def status_custom_msg(handler):
     handler.end_headers()
 
 
+@TestHandler.handler('GET', '/cookies')
+def get_cookies(handler):
+    cookies = {
+        'cookies': {
+            key: cookie.value
+            for key, cookie in SimpleCookie(handler.headers.get('Cookie')).items()
+        }
+    }
+    payload = json.dumps(cookies)
+
+    handler.send_response(200)
+    handler.send_header('Content-Length', len(payload))
+    handler.send_header('Content-Type', 'application/json')
+    handler.end_headers()
+    handler.wfile.write(payload.encode('utf-8'))
+
+
+@TestHandler.handler('GET', '/cookies/set')
+def set_cookies(handler):
+    options = parse_qs(urlparse(handler.path).query)
+
+    handler.send_response(200)
+    for cookie, [value] in options.items():
+        handler.send_header('Set-Cookie', f'{cookie}={value}')
+
+    handler.end_headers()
+
+
 @TestHandler.handler('GET', '/cookies/set-and-redirect')
 def set_cookie_and_redirect(handler):
     handler.send_response(302)
@@ -98,15 +129,30 @@ def set_cookie_and_redirect(handler):
     handler.end_headers()
 
 
+@contextmanager
+def _http_server():
+    server = HTTPServer(('localhost', 0), TestHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.start()
+    yield server
+    server.shutdown()
+    thread.join()
+
+
 @pytest.fixture(scope="function")
 def http_server():
     """A custom HTTP server implementation for our tests, that is
     built on top of the http.server module. Handy when we need to
     deal with details which httpbin can not capture."""
 
-    server = HTTPServer(('localhost', 0), TestHandler)
+    with _http_server() as server:
-    thread = threading.Thread(target=server.serve_forever)
+        yield '{0}:{1}'.format(*server.socket.getsockname())
-    thread.start()
+
-    yield '{}:{}'.format(*server.socket.getsockname())
+
-    server.shutdown()
+@pytest.fixture(scope="function")
-    thread.join(timeout=0.5)
+def localhost_http_server():
+    """Just like the http_server, but uses the static
+    `localhost` name for the host."""
+
+    with _http_server() as server:
+        yield 'localhost:{1}'.format(*server.socket.getsockname())