diff --git a/docs/contributors/README.md b/docs/contributors/README.md new file mode 100644 index 00000000..20b3c278 --- /dev/null +++ b/docs/contributors/README.md @@ -0,0 +1,3 @@ +Here we maintain a database of contributors, from which we generate credits on release blog posts and social medias. + +For the HTTPie blog see: . diff --git a/docs/contributors/fetch.py b/docs/contributors/fetch.py new file mode 100644 index 00000000..7924f464 --- /dev/null +++ b/docs/contributors/fetch.py @@ -0,0 +1,280 @@ +""" +Generate the contributors database. + +FIXME: replace `requests` calls with the HTTPie API, when available. +""" +import json +import os +import re +import sys +from copy import deepcopy +from datetime import datetime +from pathlib import Path +from subprocess import check_output +from time import sleep +from typing import Any, Dict, Optional, Set + +import requests + +FullNames = Set[str] +GitHubLogins = Set[str] +Person = Dict[str, str] +People = Dict[str, Person] +UserInfo = Dict[str, Any] + +CO_AUTHORS = re.compile(r'Co-authored-by: ([^<]+) <').finditer +API_URL = 'https://api.github.com' +REPO = OWNER = 'httpie' +REPO_URL = f'{API_URL}/repos/{REPO}/{OWNER}' + +HERE = Path(__file__).parent +DB_FILE = HERE / 'people.json' + +DEFAULT_PERSON: Person = {'committed': [], 'reported': [], 'github': '', 'twitter': ''} +SKIPPED_LABELS = {'invalid'} + +GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') +assert GITHUB_TOKEN, 'GITHUB_TOKEN envar is missing' + + +class FinishedForNow(Exception): + """Raised when remaining GitHub rate limit is zero.""" + + +def main(previous_release: str, current_release: str) -> int: + since = release_date(previous_release) + until = release_date(current_release) + + contributors = load_awesome_people() + try: + committers = find_committers(since, until) + reporters = find_reporters(since, until) + except Exception as exc: + # We want to save what we fetched so far. So pass. + print(' !! ', exc) + + try: + merge_all_the_people(current_release, contributors, committers, reporters) + fetch_missing_users_details(contributors) + except FinishedForNow: + # We want to save what we fetched so far. So pass. + print(' !! Committers:', committers) + print(' !! Reporters:', reporters) + exit_status = 1 + else: + exit_status = 0 + + save_awesome_people(contributors) + return exit_status + + +def find_committers(since: str, until: str) -> FullNames: + url = f'{REPO_URL}/commits' + page = 1 + per_page = 100 + params = { + 'since': since, + 'until': until, + 'per_page': per_page, + } + committers: FullNames = set() + + while 'there are commits': + params['page'] = page + data = fetch(url, params=params) + + for item in data: + commit = item['commit'] + committers.add(commit['author']['name']) + debug(' >>> Commit', item['html_url']) + for co_author in CO_AUTHORS(commit['message']): + name = co_author.group(1) + committers.add(name) + + if len(data) < per_page: + break + page += 1 + + return committers + + +def find_reporters(since: str, until: str) -> GitHubLogins: + url = f'{API_URL}/search/issues' + page = 1 + per_page = 100 + params = { + 'q': f'repo:{REPO}/{OWNER} is:issue closed:{since}..{until}', + 'per_page': per_page, + } + reporters: GitHubLogins = set() + + while 'there are issues': + params['page'] = page + data = fetch(url, params=params) + + for item in data['items']: + # Filter out unwanted labels. + if any(label['name'] in SKIPPED_LABELS for label in item['labels']): + continue + debug(' >>> Issue', item['html_url']) + reporters.add(item['user']['login']) + + if len(data['items']) < per_page: + break + page += 1 + + return reporters + + +def merge_all_the_people(release: str, contributors: People, committers: FullNames, reporters: GitHubLogins) -> None: + """ + >>> contributors = {'Alice': new_person(github='alice', twitter='alice')} + >>> merge_all_the_people('2.6.0', contributors, {}, {}) + >>> contributors + {'Alice': {'committed': [], 'reported': [], 'github': 'alice', 'twitter': 'alice'}} + + >>> contributors = {'Bob': new_person(github='bob', twitter='bob')} + >>> merge_all_the_people('2.6.0', contributors, {'Bob'}, {'bob'}) + >>> contributors + {'Bob': {'committed': ['2.6.0'], 'reported': ['2.6.0'], 'github': 'bob', 'twitter': 'bob'}} + + >>> contributors = {'Charlotte': new_person(github='charlotte', twitter='charlotte', committed=['2.5.0'], reported=['2.5.0'])} + >>> merge_all_the_people('2.6.0', contributors, {'Charlotte'}, {'charlotte'}) + >>> contributors + {'Charlotte': {'committed': ['2.5.0', '2.6.0'], 'reported': ['2.5.0', '2.6.0'], 'github': 'charlotte', 'twitter': 'charlotte'}} + + """ + # Update known contributors. + for name, details in contributors.items(): + if name in committers: + if release not in details['committed']: + details['committed'].append(release) + committers.remove(name) + if details['github'] in reporters: + if release not in details['reported']: + details['reported'].append(release) + reporters.remove(details['github']) + + # Add new committers. + for name in committers: + user_info = user(fullname=name) + contributors[name] = new_person( + github=user_info['login'], + twitter=user_info['twitter_username'], + committed=[release], + ) + if user_info['login'] in reporters: + contributors[name]['reported'].append(release) + reporters.remove(user_info['login']) + + # Add new reporters. + for github_username in reporters: + user_info = user(github_username=github_username) + contributors[user_info['name'] or user_info['login']] = new_person( + github=github_username, + twitter=user_info['twitter_username'], + reported=[release], + ) + + +def release_date(release: str) -> str: + date = check_output(['git', 'log', '-1', '--format=%ai', release], text=True).strip() + return datetime.strptime(date, '%Y-%m-%d %H:%M:%S %z').isoformat() + + +def load_awesome_people() -> People: + try: + with DB_FILE.open(encoding='utf-8') as fh: + return json.load(fh) + except (FileNotFoundError, ValueError): + return {} + + +def fetch(url: str, params: Optional[Dict[str, str]] = None) -> UserInfo: + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authentication': f'token {GITHUB_TOKEN}' + } + for retry in range(1, 6): + debug(f'[{retry}/5]', f'{url = }', f'{params = }') + with requests.get(url, params=params, headers=headers) as req: + try: + req.raise_for_status() + except requests.exceptions.HTTPError as exc: + if exc.response.status_code == 403: + # 403 Client Error: rate limit exceeded for url: ... + now = int(datetime.utcnow().timestamp()) + xrate_limit_reset = int(exc.response.headers['X-RateLimit-Reset']) + wait = xrate_limit_reset - now + if wait > 20: + raise FinishedForNow() + debug(' !', 'Waiting', wait, 'seconds before another try ...') + sleep(wait) + continue + return req.json() + assert ValueError('Rate limit exceeded') + + +def new_person(**kwargs: str) -> Person: + data = deepcopy(DEFAULT_PERSON) + data.update(**kwargs) + return data + + +def user(fullname: Optional[str] = '', github_username: Optional[str] = '') -> UserInfo: + if github_username: + url = f'{API_URL}/users/{github_username}' + return fetch(url) + + url = f'{API_URL}/search/users' + for query in (f'fullname:{fullname}', f'user:{fullname}'): + params = { + 'q': f'repo:{REPO}/{OWNER} {query}', + 'per_page': 1, + } + user_info = fetch(url, params=params) + if user_info['items']: + user_url = user_info['items'][0]['url'] + return fetch(user_url) + + +def fetch_missing_users_details(people: People) -> None: + for name, details in people.items(): + if details['github'] and details['twitter']: + continue + user_info = user(github_username=details['github'], fullname=name) + if not details['github']: + details['github'] = user_info['login'] + if not details['twitter']: + details['twitter'] = user_info['twitter_username'] + + +def save_awesome_people(people: People) -> None: + with DB_FILE.open(mode='w', encoding='utf-8') as fh: + json.dump(people, fh, indent=4, sort_keys=True) + + +def debug(*args: Any) -> None: + if os.getenv('DEBUG') == '1': + print(*args) + + +if __name__ == '__main__': + ret = 1 + try: + ret = main(*sys.argv[1:]) + except TypeError: + ret = 2 + print(f''' +Fetch contributors to a release. + +Usage: + python {sys.argv[0]} {sys.argv[0]} +Example: + python {sys.argv[0]} 2.4.0 2.5.0 + +Define the DEBUG=1 environment variable to enable verbose output. +''') + except KeyboardInterrupt: + ret = 255 + sys.exit(ret) diff --git a/docs/contributors/generate.py b/docs/contributors/generate.py new file mode 100644 index 00000000..2205d411 --- /dev/null +++ b/docs/contributors/generate.py @@ -0,0 +1,41 @@ +""" +Generate snippets to copy-paste. +""" +import sys + +from jinja2 import Template + +from fetch import HERE, load_awesome_people + +TPL_FILE = HERE / 'snippet.jinja2' +HTTPIE_TEAM = {'jakubroztocil', 'BoboTiG', 'claudiatd'} + + +def generate_snippets(release: str) -> str: + people = load_awesome_people() + contributors = { + name: details + for name, details in people.items() + if details['github'] not in HTTPIE_TEAM + and (release in details['committed'] or release in details['reported']) + } + + template = Template(source=TPL_FILE.read_text(encoding='utf-8')) + output = template.render(contributors=contributors, release=release) + print(output) + return 0 + + +if __name__ == '__main__': + ret = 1 + try: + ret = generate_snippets(sys.argv[1]) + except (IndexError, TypeError): + ret = 2 + print(f''' +Generate snippets for contributors to a release. + +Usage: + python {sys.argv[0]} {sys.argv[0]} +''') + sys.exit(ret) diff --git a/docs/contributors/people.json b/docs/contributors/people.json new file mode 100644 index 00000000..146d9470 --- /dev/null +++ b/docs/contributors/people.json @@ -0,0 +1,240 @@ +{ + "Almad": { + "committed": [ + "2.5.0" + ], + "github": "Almad", + "reported": [], + "twitter": "almadcz" + }, + "Anton Emelyanov": { + "committed": [ + "2.5.0" + ], + "github": "king-menin", + "reported": [], + "twitter": null + }, + "D8ger": { + "committed": [], + "github": "caofanCPU", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Dawid Ferenczy Rogo\u017ean": { + "committed": [], + "github": "ferenczy", + "reported": [ + "2.5.0" + ], + "twitter": "DawidFerenczy" + }, + "Elena Lape": { + "committed": [ + "2.5.0" + ], + "github": "elenalape", + "reported": [], + "twitter": "elena_lape" + }, + "F\u00fash\u0113ng": { + "committed": [], + "github": "lienide", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Giampaolo Rodola": { + "committed": [], + "github": "giampaolo", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Hugh Williams": { + "committed": [], + "github": "hughpv", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Ilya Sukhanov": { + "committed": [ + "2.5.0" + ], + "github": "IlyaSukhanov", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Jakub Roztocil": { + "committed": [ + "2.5.0" + ], + "github": "jakubroztocil", + "reported": [ + "2.5.0" + ], + "twitter": "jakubroztocil" + }, + "Jan Verbeek": { + "committed": [ + "2.5.0" + ], + "github": "blyxxyz", + "reported": [], + "twitter": null + }, + "Jannik Vieten": { + "committed": [ + "2.5.0" + ], + "github": "exploide", + "reported": [], + "twitter": null + }, + "Marcel St\u00f6r": { + "committed": [ + "2.5.0" + ], + "github": "marcelstoer", + "reported": [], + "twitter": "frightanic" + }, + "Mariano Ruiz": { + "committed": [], + "github": "mrsarm", + "reported": [ + "2.5.0" + ], + "twitter": "mrsarm82" + }, + "Micka\u00ebl Schoentgen": { + "committed": [ + "2.5.0" + ], + "github": "BoboTiG", + "reported": [ + "2.5.0" + ], + "twitter": "__tiger222__" + }, + "Miro Hron\u010dok": { + "committed": [ + "2.5.0" + ], + "github": "hroncok", + "reported": [], + "twitter": "hroncok" + }, + "Mohamed Daahir": { + "committed": [], + "github": "ducaale", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Pavel Alexeev aka Pahan-Hubbitus": { + "committed": [], + "github": "Hubbitus", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Samuel Marks": { + "committed": [], + "github": "SamuelMarks", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Sullivan SENECHAL": { + "committed": [], + "github": "soullivaneuh", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Thomas Klinger": { + "committed": [], + "github": "mosesontheweb", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "Yannic Schneider": { + "committed": [], + "github": "cynay", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "a1346054": { + "committed": [ + "2.5.0" + ], + "github": "a1346054", + "reported": [], + "twitter": null + }, + "bl-ue": { + "committed": [ + "2.5.0" + ], + "github": "FiReBlUe45", + "reported": [], + "twitter": null + }, + "henryhu712": { + "committed": [ + "2.5.0" + ], + "github": "henryhu712", + "reported": [], + "twitter": null + }, + "jungle-boogie": { + "committed": [], + "github": "jungle-boogie", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "nixbytes": { + "committed": [ + "2.5.0" + ], + "github": "nixbytes", + "reported": [], + "twitter": "linuxbyte3" + }, + "qiulang": { + "committed": [], + "github": "qiulang", + "reported": [ + "2.5.0" + ], + "twitter": null + }, + "zwx00": { + "committed": [], + "github": "zwx00", + "reported": [ + "2.5.0" + ], + "twitter": null + } +} \ No newline at end of file diff --git a/docs/contributors/snippet.jinja2 b/docs/contributors/snippet.jinja2 new file mode 100644 index 00000000..4fa21b12 --- /dev/null +++ b/docs/contributors/snippet.jinja2 @@ -0,0 +1,13 @@ + + +## Community contributions + +We’d like to thank these amazing people for their contributions to this release: {% for name, details in contributors.items() -%} + [{{ name }}](https://github.com/{{ details.github }}){{ '' if loop.last else ', ' }} +{%- endfor %}. + + + +We’d like to thank these amazing people for their contributions to HTTPie {{ release }}: {% for name, details in contributors.items() if details.twitter -%} + @{{ details.twitter }}{{ '' if loop.last else ', ' }} +{%- endfor %} 🥧