From deb9f3b2a0175fa3bfc20d682e6699692f491e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 19 Apr 2024 09:12:58 +0200 Subject: [PATCH] automatically update mediawiki extensions --- .github/workflows/update-extensions.yml | 22 +++++ modules/nixos-wiki/default.nix | 37 +++---- modules/nixos-wiki/extensions.json | 4 + modules/nixos-wiki/extensions.nix | 4 + modules/nixos-wiki/update-extensions.py | 124 ++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/update-extensions.yml create mode 100644 modules/nixos-wiki/extensions.json create mode 100644 modules/nixos-wiki/extensions.nix create mode 100755 modules/nixos-wiki/update-extensions.py diff --git a/.github/workflows/update-extensions.yml b/.github/workflows/update-extensions.yml new file mode 100644 index 0000000..e7c6709 --- /dev/null +++ b/.github/workflows/update-extensions.yml @@ -0,0 +1,22 @@ +name: "Update mediawiki extensions" +on: + repository_dispatch: + workflow_dispatch: + schedule: + - cron: '51 2 * * *' +jobs: + update-submodules: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: cachix/install-nix-action@v22 + with: + nix_path: nixpkgs=channel:nixos-unstable + - run: ./modules/nixos-wiki/update-extensions.py ./modules/nixos-wiki/extensions.json + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6 + with: + title: Update zsh modules + labels: merge-queue diff --git a/modules/nixos-wiki/default.nix b/modules/nixos-wiki/default.nix index 9340262..efe6a43 100644 --- a/modules/nixos-wiki/default.nix +++ b/modules/nixos-wiki/default.nix @@ -46,31 +46,18 @@ in uploadsDir = "/var/lib/mediawiki-uploads/"; passwordFile = cfg.adminPasswordFile; - extensions.SyntaxHighlight_GeSHi = null; # provides tags - extensions.ParserFunctions = null; - extensions.Cite = null; - extensions.VisualEditor = null; - extensions.AuthManagerOAuth = pkgs.fetchzip { - url = "https://github.com/mohe2015/AuthManagerOAuth/releases/download/v0.3.2/AuthManagerOAuth.zip"; - hash = "sha256-hr/DLyL6IzQs67eA46RdmuVlfCiAbq+eZCRLfjLxUpc="; - }; # Github login - extensions.ConfirmEdit = null; # Combat SPAM with a simple Captcha - # https://www.mediawiki.org/wiki/Extension:MobileFrontend/ - extensions.MobileFrontend = pkgs.fetchzip { - url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/MobileFrontend-REL1_41-6dbf6c2.tar.gz/MobileFrontend-REL1_41-6dbf6c2.tar.gz"; - hash = "sha256-LsKPlVId7DzbkS7xc+fSYBiasq4AofCjCfbBaN/eSE8="; - }; # Responsive skin - extensions.DarkMode = pkgs.fetchzip { - url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/DarkMode-REL1_41-d647ab5.tar.gz/DarkMode-REL1_41-d647ab5.tar.gz"; - hash = "sha256-ERvWGDKfT6nqFbz7q8iqZFe2uhwoSQ2ePUy4tlPDaOE="; - }; - - #extensions.StopForumSpam = pkgs.fetchzip { - # url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/StopForumSpam-REL1_41-73c94fb/StopForumSpam-REL1_41-861c37b.tar.gz"; - # hash = "sha256-/7gfBiKA9CliEPjXjcHrYKp4JMayXwtixlZFvnA5D2E="; - #}; - - + extensions = { + SyntaxHighlight_GeSHi = null; # provides tags + ParserFunctions = null; + Cite = null; + VisualEditor = null; + AuthManagerOAuth = pkgs.fetchzip { + url = "https://github.com/mohe2015/AuthManagerOAuth/releases/download/v0.3.2/AuthManagerOAuth.zip"; + hash = "sha256-hr/DLyL6IzQs67eA46RdmuVlfCiAbq+eZCRLfjLxUpc="; + }; # Github login + ConfirmEdit = null; # Combat SPAM with a simple Captcha + # https://www.mediawiki.org/wiki/Extension:MobileFrontend/ + } // pkgs.callPackages ./extensions.nix {}; extraConfig = '' #$wgDebugLogFile = "/var/log/mediawiki/debug.log"; diff --git a/modules/nixos-wiki/extensions.json b/modules/nixos-wiki/extensions.json new file mode 100644 index 0000000..d25eb16 --- /dev/null +++ b/modules/nixos-wiki/extensions.json @@ -0,0 +1,4 @@ +{ + "MobileFrontend": {}, + "DarkMode": {} +} diff --git a/modules/nixos-wiki/extensions.nix b/modules/nixos-wiki/extensions.nix new file mode 100644 index 0000000..e7b6702 --- /dev/null +++ b/modules/nixos-wiki/extensions.nix @@ -0,0 +1,4 @@ +{ fetchurl }: { + "MobileFrontend" = fetchurl { url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/MobileFrontend-REL1_41-7e9d1d4.tar.gz/MobileFrontend-REL1_41-7e9d1d4.tar.gz"; sha256 = "1j9b2f5kxv68yl5652pqqfgchdkcc7p7j0afv2f3i4v6yz9ajr1n"; }; + "DarkMode" = fetchurl { url = "https://github.com/NixOS/nixos-wiki-infra/releases/download/DarkMode-REL1_41-b2fefd2.tar.gz/DarkMode-REL1_41-b2fefd2.tar.gz"; sha256 = "1inbbqd25qacxmxjxl7rx153rapgk61433cs49cjmwcdx62w4vsj"; }; +} \ No newline at end of file diff --git a/modules/nixos-wiki/update-extensions.py b/modules/nixos-wiki/update-extensions.py new file mode 100755 index 0000000..4012174 --- /dev/null +++ b/modules/nixos-wiki/update-extensions.py @@ -0,0 +1,124 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 python3Packages.requests python3Packages.beautifulsoup4 gh +import json +import os +import shlex +import subprocess +import sys +import urllib.parse +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import IO + +import requests +from bs4 import BeautifulSoup + + +def get_latest_url( + extension_name: str, mediawiki_version: str +) -> urllib.parse.ParseResult: + #
https://extdist.wmflabs.org/dist/extensions/QuickInstantCommons-REL1_41-2a29b3e.tar.gz
+ extension_url = f"https://www.mediawiki.org/wiki/Special:ExtensionDistributor?extdistname={extension_name}&extdistversion=REL{mediawiki_version}" + body = requests.get(extension_url) + soup = BeautifulSoup(body.text, "html.parser") + url = soup.find_all("a", class_="external free")[0].get("href") + return urllib.parse.urlparse(url) + + +def run( + cmd: list[str], + check: bool = True, + stdout: int | None = None, + stdin: int | None = subprocess.DEVNULL, +) -> subprocess.CompletedProcess[str]: + print("$ " + shlex.join(cmd)) + return subprocess.run(cmd, check=check, stdout=stdout, stdin=stdin, text=True) + + +@dataclass +class Extension: + name: str + hash: str + url: str + + +def download_file(url: str, local_filename: str): + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(local_filename, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + # If you have chunk encoded response uncomment if + # and set chunk_size parameter to None. + # if chunk: + f.write(chunk) + return local_filename + + +def mirror_extension(extension_name: str, mediawiki_version: str) -> Extension: + download_url = get_latest_url(extension_name, mediawiki_version) + base_name = Path(download_url.path).name + print(f"{base_name}: {download_url.geturl()}") + + if run(["gh", "release", "view", base_name], check=False).returncode != 0: + run(["gh", "release", "create", "--title", base_name, base_name]) + mirror_url = f"https://github.com/NixOS/nixos-wiki-infra/releases/download/{base_name}/{base_name}" + if requests.head(mirror_url).status_code == 404: + with TemporaryDirectory() as tmpdir: + download_file(download_url.geturl(), f"{tmpdir}/{base_name}") + run(["gh", "release", "upload", base_name, f"{tmpdir}/{base_name}"]) + hash = run(["nix-prefetch-url", mirror_url], stdout=subprocess.PIPE).stdout.strip() + return Extension(name=extension_name, hash=hash, url=mirror_url) + + +def write_nix_file(file: IO[str], mirrored_extensions: list[Extension]) -> None: + file.write("{ fetchurl }: {\n") + for extension in mirrored_extensions: + file.write( + f' "{extension.name}" = fetchurl {{ url = "{extension.url}"; sha256 = "{extension.hash}"; }};\n' + ) + file.write("}") + + +def get_mediawiki_version(mediawiki_version: str | None = None) -> str: + if mediawiki_version is None: + mediawiki_version = run( + [ + "nix", + "eval", + "--inputs-from", + ".#", + "--raw", + "nixpkgs#mediawiki.version", + ], + stdout=subprocess.PIPE, + ).stdout.strip() + mediawiki_version = mediawiki_version.replace(".", "_") + version_parts = mediawiki_version.split("_") + return version_parts[0] + "_" + version_parts[1] + + +def main(): + if len(sys.argv) < 2: + print("Usage: update-extensions.py extensions.json [mediawiki_version]") + sys.exit(1) + + extensions = json.loads(Path(sys.argv[1]).read_text()) + mediawiki_version = get_mediawiki_version( + sys.argv[2] if len(sys.argv) > 2 else None + ) + + # so that gh picks up the correct repository + os.chdir(Path(__file__).parent) + + mirrored_extensions = [] + for name in extensions.keys(): + extension = mirror_extension(name, mediawiki_version) + mirrored_extensions.append(extension) + + with open("extensions.nix", "w") as file: + write_nix_file(file, mirrored_extensions) + + +if __name__ == "__main__": + main()