diff --git a/modules/nixos-wiki/default.nix b/modules/nixos-wiki/default.nix index 5a2391c..6884ce8 100644 --- a/modules/nixos-wiki/default.nix +++ b/modules/nixos-wiki/default.nix @@ -56,10 +56,6 @@ in ParserFunctions = null; Cite = null; VisualEditor = null; - AuthManagerOAuth = pkgs.fetchzip { - url = "https://github.com/mohe2015/AuthManagerOAuth/releases/download/v0.3.2/AuthManagerOAuth.zip"; - hash = "sha256-hr/DLyL6IzQs67eA46RdmuVlfCiAbq+eZCRLfjLxUpc="; - }; # Github login ConfirmEdit = null; # Combat SPAM with a simple Captcha DiscussionTools = null; # Adds a new discussion tool to the talk pages Thanks = null; # Adds a "thank" button diff --git a/modules/nixos-wiki/extensions.json b/modules/nixos-wiki/extensions.json index a9a9720..5224f42 100644 --- a/modules/nixos-wiki/extensions.json +++ b/modules/nixos-wiki/extensions.json @@ -3,5 +3,16 @@ "DarkMode": {}, "QuickInstantCommons": {}, "Translate": {}, - "UniversalLanguageSelector": {} + "UniversalLanguageSelector": {}, + "Description2": {}, + "Mermaid": { + "type": "github", + "github_repo": "SemanticMediaWiki/Mermaid", + "github_type": "tag" + }, + "AuthManagerOAuth": { + "type": "github", + "github_repo": "mohe2015/AuthManagerOAuth", + "github_type": "release" + } } diff --git a/modules/nixos-wiki/extensions.nix b/modules/nixos-wiki/extensions.nix index 78412f6..6c8a8eb 100644 --- a/modules/nixos-wiki/extensions.nix +++ b/modules/nixos-wiki/extensions.nix @@ -20,4 +20,16 @@ url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/UniversalLanguageSelector-REL1_43-4ad159f.tar.gz/UniversalLanguageSelector-REL1_43-4ad159f.tar.gz"; hash = "sha256-COB+N6q7lIYM7Vc334W3N8knrnhSlEpLIA/QM6Y6a6o="; }; + "Description2" = fetchzip { + url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/Description2-REL1_43-50e2aef.tar.gz/Description2-REL1_43-50e2aef.tar.gz"; + hash = "sha256-ciUEUcg4tsgpvohuLYztFaGNBowR7p1dIKnNp4ooKtA="; + }; + "Mermaid" = fetchzip { + url = "https://github.com/SemanticMediaWiki/Mermaid/archive/refs/tags/3.1.0.zip"; + hash = "sha256-tLOdAsXsaP/URvKcl5QWQiyhMy70qn8Fi8g3+ecNOWQ="; + }; + "AuthManagerOAuth" = fetchzip { + url = "https://github.com/mohe2015/AuthManagerOAuth/archive/refs/tags/v0.3.2.zip"; + hash = "sha256-0jMxLX7r4w44WMvtDt421vX+s622+j7WjbK6ClS/FKk="; + }; } diff --git a/modules/nixos-wiki/update-extensions.py b/modules/nixos-wiki/update-extensions.py index f5d3f6f..a263ba9 100755 --- a/modules/nixos-wiki/update-extensions.py +++ b/modules/nixos-wiki/update-extensions.py @@ -10,6 +10,7 @@ import urllib.parse from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory +from typing import Dict, Any, List, Optional import requests from bs4 import BeautifulSoup @@ -43,7 +44,7 @@ class Extension: url: str -def download_file(url: str, local_filename: str): +def download_file(url: str, local_filename: str) -> str: with requests.get(url, stream=True) as r: r.raise_for_status() with open(local_filename, "wb") as f: @@ -55,6 +56,63 @@ def download_file(url: str, local_filename: str): return local_filename +def get_github_headers() -> Dict[str, str]: + """Get headers for GitHub API requests, including auth if GITHUB_TOKEN is set""" + headers = {"Accept": "application/vnd.github.v3+json"} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"token {github_token}" + return headers + + +def get_latest_github_release_url(repo: str, extension_type: str) -> str: + """Get latest release or tag from GitHub API""" + headers = get_github_headers() + + if extension_type == "release": + api_url = f"https://api.github.com/repos/{repo}/releases/latest" + response = requests.get(api_url, headers=headers) + response.raise_for_status() + data = response.json() + tag_name = data["tag_name"] + return f"https://github.com/{repo}/archive/refs/tags/{tag_name}.zip" + elif extension_type == "tag": + api_url = f"https://api.github.com/repos/{repo}/tags" + response = requests.get(api_url, headers=headers) + response.raise_for_status() + data = response.json() + if not data: + raise Exception(f"No tags found for {repo}") + latest_tag = data[0]["name"] + return f"https://github.com/{repo}/archive/refs/tags/{latest_tag}.zip" + else: + raise Exception(f"Unknown extension type: {extension_type}") + + +def mirror_custom_extension( + extension_name: str, github_repo: str, extension_type: str +) -> Extension: + """Handle custom GitHub extensions""" + download_url = get_latest_github_release_url(github_repo, extension_type) + print(f"{extension_name}: {download_url}") + + # Use nix-prefetch-url directly for custom extensions + for i in range(30): + try: + data = run( + ["nix", "store", "prefetch-file", "--unpack", download_url, "--json"], + stdout=subprocess.PIPE, + ).stdout.strip() + hash = json.loads(data)["hash"] + except subprocess.CalledProcessError: + print("nix-prefetch-url failed, retrying") + time.sleep(i * 5) + continue + else: + return Extension(name=extension_name, hash=hash, url=download_url) + raise Exception("Failed to fetch extension, see above") + + def mirror_extension(extension_name: str, mediawiki_version: str) -> Extension: download_url = get_latest_url(extension_name, mediawiki_version) base_name = Path(download_url.path).name @@ -84,7 +142,7 @@ def mirror_extension(extension_name: str, mediawiki_version: str) -> Extension: raise Exception("Failed to fetch extension, see above") -def extension_nix_expression(mirrored_extensions: list[Extension]) -> str: +def extension_nix_expression(mirrored_extensions: List[Extension]) -> str: expression = "{ fetchzip }: {\n" for extension in mirrored_extensions: expression += f' "{extension.name}" = fetchzip {{ url = "{extension.url}"; hash = "{extension.hash}"; }};\n' @@ -92,7 +150,7 @@ def extension_nix_expression(mirrored_extensions: list[Extension]) -> str: return expression -def get_mediawiki_version(mediawiki_version: str | None = None) -> str: +def get_mediawiki_version(mediawiki_version: Optional[str] = None) -> str: if mediawiki_version is None: mediawiki_version = run( [ @@ -110,12 +168,12 @@ def get_mediawiki_version(mediawiki_version: str | None = None) -> str: return version_parts[0] + "_" + version_parts[1] -def main(): +def main() -> None: if len(sys.argv) < 2: print("Usage: update-extensions.py extensions.json [mediawiki_version]") sys.exit(1) - extensions = json.loads(Path(sys.argv[1]).read_text()) + extensions: Dict[str, Any] = json.loads(Path(sys.argv[1]).read_text()) mediawiki_version = get_mediawiki_version( sys.argv[2] if len(sys.argv) > 2 else None ) @@ -123,9 +181,16 @@ def main(): # so that gh picks up the correct repository os.chdir(Path(__file__).parent) - mirrored_extensions = [] - for name in extensions.keys(): - extension = mirror_extension(name, mediawiki_version) + mirrored_extensions: List[Extension] = [] + for name, config in extensions.items(): + if config.get("type") == "github": + # Handle GitHub extensions + github_repo: str = config["github_repo"] + github_type: str = config["github_type"] + extension = mirror_custom_extension(name, github_repo, github_type) + else: + # Handle standard MediaWiki extensions + extension = mirror_extension(name, mediawiki_version) mirrored_extensions.append(extension) nix_extensions = Path("extensions.nix")