automatically update mediawiki extensions

.github/workflows/update-extensions.yml

@@ -0,0 +1,22 @@
+name: "Update mediawiki extensions"
+on:
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron:  '51 2 * * *'
+jobs:
+  update-submodules:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v22
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: ./modules/nixos-wiki/update-extensions.py ./modules/nixos-wiki/extensions.json
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          title: Update zsh modules
+          labels: merge-queue

modules/nixos-wiki/default.nix

@@ -46,31 +46,18 @@ in
       uploadsDir = "/var/lib/mediawiki-uploads/";
       passwordFile = cfg.adminPasswordFile;
 
-      extensions.SyntaxHighlight_GeSHi = null; # provides <SyntaxHighlight> tags
-      extensions.ParserFunctions = null;
-      extensions.Cite = null;
-      extensions.VisualEditor = null;
-      extensions.AuthManagerOAuth = pkgs.fetchzip {
-        url = "https://github.com/mohe2015/AuthManagerOAuth/releases/download/v0.3.2/AuthManagerOAuth.zip";
-        hash = "sha256-hr/DLyL6IzQs67eA46RdmuVlfCiAbq+eZCRLfjLxUpc=";
-      }; # Github login
-      extensions.ConfirmEdit = null; # Combat SPAM with a simple Captcha
-      # https://www.mediawiki.org/wiki/Extension:MobileFrontend/
-      extensions.MobileFrontend = pkgs.fetchzip {
-        url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/MobileFrontend-REL1_41-6dbf6c2.tar.gz/MobileFrontend-REL1_41-6dbf6c2.tar.gz";
-        hash = "sha256-LsKPlVId7DzbkS7xc+fSYBiasq4AofCjCfbBaN/eSE8=";
-      }; # Responsive skin
-      extensions.DarkMode = pkgs.fetchzip {
-        url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/DarkMode-REL1_41-d647ab5.tar.gz/DarkMode-REL1_41-d647ab5.tar.gz";
-        hash = "sha256-ERvWGDKfT6nqFbz7q8iqZFe2uhwoSQ2ePUy4tlPDaOE=";
-      };
-
-      #extensions.StopForumSpam = pkgs.fetchzip {
-      #  url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/StopForumSpam-REL1_41-73c94fb/StopForumSpam-REL1_41-861c37b.tar.gz";
-      #  hash = "sha256-/7gfBiKA9CliEPjXjcHrYKp4JMayXwtixlZFvnA5D2E=";
-      #};
-
-
+      extensions = {
+        SyntaxHighlight_GeSHi = null; # provides <SyntaxHighlight> tags
+        ParserFunctions = null;
+        Cite = null;
+        VisualEditor = null;
+        AuthManagerOAuth = pkgs.fetchzip {
+          url = "https://github.com/mohe2015/AuthManagerOAuth/releases/download/v0.3.2/AuthManagerOAuth.zip";
+          hash = "sha256-hr/DLyL6IzQs67eA46RdmuVlfCiAbq+eZCRLfjLxUpc=";
+        }; # Github login
+        ConfirmEdit = null; # Combat SPAM with a simple Captcha
+        # https://www.mediawiki.org/wiki/Extension:MobileFrontend/
+      } // pkgs.callPackages ./extensions.nix {};
       extraConfig = ''
         #$wgDebugLogFile = "/var/log/mediawiki/debug.log";
 

modules/nixos-wiki/extensions.json

@@ -0,0 +1,4 @@
+{
+  "MobileFrontend": {},
+  "DarkMode": {}
+}

modules/nixos-wiki/extensions.nix

@@ -0,0 +1,4 @@
+{ fetchurl }: {
+  "MobileFrontend" = fetchurl { url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/MobileFrontend-REL1_41-7e9d1d4.tar.gz/MobileFrontend-REL1_41-7e9d1d4.tar.gz"; sha256 = "1j9b2f5kxv68yl5652pqqfgchdkcc7p7j0afv2f3i4v6yz9ajr1n"; };
+  "DarkMode" = fetchurl { url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/DarkMode-REL1_41-b2fefd2.tar.gz/DarkMode-REL1_41-b2fefd2.tar.gz"; sha256 = "1inbbqd25qacxmxjxl7rx153rapgk61433cs49cjmwcdx62w4vsj"; };
+}

modules/nixos-wiki/update-extensions.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3 python3Packages.requests python3Packages.beautifulsoup4 gh
+import json
+import os
+import shlex
+import subprocess
+import sys
+import urllib.parse
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import IO
+
+import requests
+from bs4 import BeautifulSoup
+
+
+def get_latest_url(
+    extension_name: str, mediawiki_version: str
+) -> urllib.parse.ParseResult:
+    # <dl><dd><a rel="nofollow" class="external free" href="https://extdist.wmflabs.org/dist/extensions/QuickInstantCommons-REL1_41-2a29b3e.tar.gz">https://extdist.wmflabs.org/dist/extensions/QuickInstantCommons-REL1_41-2a29b3e.tar.gz</a></dd></dl>
+    extension_url = f"https://www.mediawiki.org/wiki/Special:ExtensionDistributor?extdistname={extension_name}&extdistversion=REL{mediawiki_version}"
+    body = requests.get(extension_url)
+    soup = BeautifulSoup(body.text, "html.parser")
+    url = soup.find_all("a", class_="external free")[0].get("href")
+    return urllib.parse.urlparse(url)
+
+
+def run(
+    cmd: list[str],
+    check: bool = True,
+    stdout: int | None = None,
+    stdin: int | None = subprocess.DEVNULL,
+) -> subprocess.CompletedProcess[str]:
+    print("$ " + shlex.join(cmd))
+    return subprocess.run(cmd, check=check, stdout=stdout, stdin=stdin, text=True)
+
+
+@dataclass
+class Extension:
+    name: str
+    hash: str
+    url: str
+
+
+def download_file(url: str, local_filename: str):
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_filename, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                # If you have chunk encoded response uncomment if
+                # and set chunk_size parameter to None.
+                # if chunk:
+                f.write(chunk)
+    return local_filename
+
+
+def mirror_extension(extension_name: str, mediawiki_version: str) -> Extension:
+    download_url = get_latest_url(extension_name, mediawiki_version)
+    base_name = Path(download_url.path).name
+    print(f"{base_name}: {download_url.geturl()}")
+
+    if run(["gh", "release", "view", base_name], check=False).returncode != 0:
+        run(["gh", "release", "create", "--title", base_name, base_name])
+    mirror_url = f"https://github.com/NixOS/nixos-wiki-infra/releases/download/{base_name}/{base_name}"
+    if requests.head(mirror_url).status_code == 404:
+        with TemporaryDirectory() as tmpdir:
+            download_file(download_url.geturl(), f"{tmpdir}/{base_name}")
+            run(["gh", "release", "upload", base_name, f"{tmpdir}/{base_name}"])
+    hash = run(["nix-prefetch-url", mirror_url], stdout=subprocess.PIPE).stdout.strip()
+    return Extension(name=extension_name, hash=hash, url=mirror_url)
+
+
+def write_nix_file(file: IO[str], mirrored_extensions: list[Extension]) -> None:
+    file.write("{ fetchurl }: {\n")
+    for extension in mirrored_extensions:
+        file.write(
+            f'  "{extension.name}" = fetchurl {{ url = "{extension.url}"; sha256 = "{extension.hash}"; }};\n'
+        )
+    file.write("}")
+
+
+def get_mediawiki_version(mediawiki_version: str | None = None) -> str:
+    if mediawiki_version is None:
+        mediawiki_version = run(
+            [
+                "nix",
+                "eval",
+                "--inputs-from",
+                ".#",
+                "--raw",
+                "nixpkgs#mediawiki.version",
+            ],
+            stdout=subprocess.PIPE,
+        ).stdout.strip()
+    mediawiki_version = mediawiki_version.replace(".", "_")
+    version_parts = mediawiki_version.split("_")
+    return version_parts[0] + "_" + version_parts[1]
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: update-extensions.py extensions.json [mediawiki_version]")
+        sys.exit(1)
+
+    extensions = json.loads(Path(sys.argv[1]).read_text())
+    mediawiki_version = get_mediawiki_version(
+        sys.argv[2] if len(sys.argv) > 2 else None
+    )
+
+    # so that gh picks up the correct repository
+    os.chdir(Path(__file__).parent)
+
+    mirrored_extensions = []
+    for name in extensions.keys():
+        extension = mirror_extension(name, mediawiki_version)
+        mirrored_extensions.append(extension)
+
+    with open("extensions.nix", "w") as file:
+        write_nix_file(file, mirrored_extensions)
+
+
+if __name__ == "__main__":
+    main()