Remove expired cookies (#929)

* added a test for expiring cookies

* updated tests

* set up util for extracting expired cookies from response header

* Revert "updated tests"

This reverts commit a4eb5c4498.

* Revert "Revert "updated tests""

This reverts commit d242e21bce.

* added more functionality to get-expired-cookies

* add 'clear expired cookies' from session.json files

* refactored get_expired_cookies

* fixed formatting issues

* ensured key exists in cookie_header dict

* fixed linting errors

* removed unused import

* Added tests for get_expired_cookies util

* Added additional test for get_expired_cookies

* added remove_expired_cookies method directly to sessions class

* extracted logic to clear cookies to sessions.py

* refactored utils

* added tests to check expired cookies being removed from session obj

* added type annotations for methods

* Refactored test_sessions

* Seperated out expiry related tests into own class

* Refactored get_expired_cookies in utils

* Refactored remove cookie methods

* fixed linting errors

* fixed indentation and also pluralized test class name

* removed inheritance from SessionTestbase class

* Moved related test to TestExpiredCookies class

Co-authored-by: kbanc <katherine.bancoft@gmail.com>

httpie/client.py

@@ -11,14 +11,14 @@ from urllib.parse import urlparse, urlunparse
 import requests
 # noinspection PyPackageRequirements
 import urllib3
+from requests.cookies import remove_cookie_by_name
 
 from httpie import __version__
 from httpie.cli.dicts import RequestHeadersDict
 from httpie.plugins.registry import plugin_manager
 from httpie.sessions import get_httpie_session
 from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter
-from httpie.utils import repr_dict
+from httpie.utils import get_expired_cookies, repr_dict
-
 
 urllib3.disable_warnings()
 
@@ -82,6 +82,7 @@ def collect_messages(
     if args.compress and prepared_request.body:
         compress_body(prepared_request, always=args.compress > 1)
     response_count = 0
+    expired_cookies = []
     while prepared_request:
         yield prepared_request
         if not args.offline:
@@ -95,6 +96,10 @@ def collect_messages(
                     **send_kwargs_merged,
                     **send_kwargs,
                 )
+                expired_cookies += get_expired_cookies(
+                    headers=response.raw._original_response.msg._headers
+                )
+
             response_count += 1
             if response.next:
                 if args.max_redirects and response_count == args.max_redirects:
@@ -110,6 +115,9 @@ def collect_messages(
     if httpie_session:
         if httpie_session.is_new() or not args.session_read_only:
             httpie_session.cookies = requests_session.cookies
+            httpie_session.remove_cookies(
+                cookie['name'] for cookie in expired_cookies
+            )
             httpie_session.save()
 
 

httpie/sessions.py

@@ -5,11 +5,11 @@ Persistent, JSON-serialized sessions.
 import os
 import re
 from pathlib import Path
-from typing import Optional, Union
+from typing import Iterable, Optional, Union
 from urllib.parse import urlsplit
 
 from requests.auth import AuthBase
-from requests.cookies import RequestsCookieJar, create_cookie
+from requests.cookies import RequestsCookieJar, create_cookie, remove_cookie_by_name
 
 from httpie.cli.dicts import RequestHeadersDict
 from httpie.config import BaseConfigDict, DEFAULT_CONFIG_DIR
@@ -144,3 +144,8 @@ class Session(BaseConfigDict):
     def auth(self, auth: dict):
         assert {'type', 'raw_auth'} == auth.keys()
         self['auth'] = auth
+
+    def remove_cookies(self, names: Iterable[str]):
+        for name in names:
+            if name in self['cookies']:
+                del self['cookies'][name]

httpie/utils.py

@@ -1,8 +1,12 @@
 from __future__ import division
+
 import json
 import mimetypes
+import time
 from collections import OrderedDict
+from http.cookiejar import parse_ns_headers
 from pprint import pformat
+from typing import List, Tuple
 
 import requests.auth
 
@@ -83,3 +87,27 @@ def get_content_type(filename):
         if encoding:
             content_type = '%s; charset=%s' % (mime, encoding)
         return content_type
+
+
+def get_expired_cookies(headers: List[Tuple[str, str]], curr_timestamp: float = None) -> List[dict]:
+    expired_cookies = []
+    cookie_headers = []
+    curr_timestamp = curr_timestamp or time.time()
+
+    for header_name, content in headers:
+        if header_name == 'Set-Cookie':
+            cookie_headers.append(content)
+
+    extracted_cookies = [
+        dict(cookie, name=cookie[0][0])
+        for cookie in parse_ns_headers(cookie_headers)
+    ]
+
+    for cookie in extracted_cookies:
+        if "expires" in cookie and cookie['expires'] <= curr_timestamp:
+            expired_cookies.append({
+                'name': cookie['name'],
+                'path': cookie.get('path', '/')
+            })
+
+    return expired_cookies

tests/test_sessions.py

@@ -1,14 +1,17 @@
 # coding=utf-8
+import json
 import os
 import shutil
-import sys
+from datetime import datetime
 from tempfile import gettempdir
 
 import pytest
 
-from httpie.plugins.builtin import HTTPBasicAuth
-from utils import MockEnvironment, mk_config_dir, http, HTTP_OK
 from fixtures import UNICODE
+from httpie.plugins.builtin import HTTPBasicAuth
+from httpie.sessions import Session
+from httpie.utils import get_expired_cookies
+from utils import MockEnvironment, mk_config_dir, http, HTTP_OK
 
 
 class SessionTestBase:
@@ -186,3 +189,93 @@ class TestSession(SessionTestBase):
                  httpbin.url + '/get', env=self.env())
         finally:
             os.chdir(cwd)
+
+
+class TestExpiredCookies:
+
+    @pytest.mark.parametrize(
+        argnames=['initial_cookie', 'expired_cookie'],
+        argvalues=[
+            ({'id': {'value': 123}}, 'id'),
+            ({'id': {'value': 123}}, 'token')
+        ]
+    )
+    def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin):
+        config_dir = mk_config_dir()
+        session = Session(config_dir)
+        session['cookies'] = initial_cookie
+        session.remove_cookies([expired_cookie])
+        assert expired_cookie not in session.cookies
+
+        shutil.rmtree(config_dir)
+
+    def test_expired_cookies(self, httpbin):
+        orig_session = {
+            'cookies': {
+                'to_expire': {
+                    'value': 'foo'
+                },
+                'to_stay': {
+                    'value': 'foo'
+                },
+            }
+        }
+        config_dir = mk_config_dir()
+        session_path = config_dir / 'test-session.json'
+        session_path.write_text(json.dumps(orig_session))
+
+        r = http(
+            '--session', str(session_path),
+            '--print=H',
+            httpbin.url + '/cookies/delete?to_expire',
+        )
+        assert 'Cookie: to_expire=foo; to_stay=foo' in r
+
+        updated_session = json.loads(session_path.read_text())
+        assert 'to_stay' in updated_session['cookies']
+        assert 'to_expire' not in updated_session['cookies']
+
+        shutil.rmtree(config_dir)
+
+    @pytest.mark.parametrize(
+        argnames=['raw_header', 'timestamp', 'expected'],
+        argvalues=[
+            (
+                [
+                    ('Set-Cookie', 'hello=world; Path=/; Expires=Thu, 01-Jan-1970 00:00:00 GMT; HttpOnly'),
+                    ('Connection', 'keep-alive')
+                ],
+                None,
+                [
+                    {
+                        'name': 'hello',
+                        'path': '/'
+                    }
+                ]
+            ),
+            (
+                [
+                    ('Set-Cookie', 'hello=world; Path=/; Expires=Thu, 01-Jan-1970 00:00:00 GMT; HttpOnly'),
+                    ('Set-Cookie', 'pea=pod; Path=/ab; Expires=Thu, 01-Jan-1970 00:00:00 GMT; HttpOnly'),
+                    ('Connection', 'keep-alive')
+                ],
+                None,
+                [
+                    {'name': 'hello', 'path': '/'},
+                    {'name': 'pea', 'path': '/ab'}
+                ]
+            ),
+            (
+                [
+                    ('Set-Cookie', 'hello=world; Path=/; Expires=Fri, 12 Jun 2020 12:28:55 GMT; HttpOnly'),
+                    ('Connection', 'keep-alive')
+                ],
+                datetime(2020, 6, 11).timestamp(),
+                [
+
+                ]
+            )
+        ]
+    )
+    def test_get_expired_cookies_manages_multiple_cookie_headers(self, raw_header, timestamp, expected):
+        assert get_expired_cookies(raw_header, curr_timestamp=timestamp) == expected