extern/nixos-wiki-infra
1
0
Fork 1
mirror of https://github.com/Mic92/nixos-wiki-infra.git synced 2024-11-24 09:13:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

linkcheck: download xmldump conservatively

Browse Source 
Signed-off-by: phanirithvij <phanirithvij2000@gmail.com>
This commit is contained in:
phanirithvij 2024-09-18 11:12:16 +05:30
parent 31e229cefc
commit 7128152b64
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
checks/linkcheck/lychee.sh
View File

@ -8,7 +8,8 @@ workdir="$SCRIPT_DIR/workdir"
mkdir -p "$workdir" mkdir -p "$workdir"
pushd "$workdir" || exit pushd "$workdir" || exit
curl "https://wiki.nixos.org/wikidump.xml.zst" | zstd -d >wikidump.xml curl -o wikidump.xml.zst "https://wiki.nixos.org/wikidump.xml.zst" -z wikidump.xml.zst
<wikidump.xml.zst zstd -d >wikidump.xml
# filter unimportant pages like User:* Talk:* # filter unimportant pages like User:* Talk:*
python3 ../main.py filter wikidump.xml wikidump-filtered.xml python3 ../main.py filter wikidump.xml wikidump-filtered.xml
@ -17,7 +18,7 @@ python3 ../main.py filter wikidump.xml wikidump-filtered.xml
python3 ../main.py badlinks ../allowed.links exclude-args python3 ../main.py badlinks ../allowed.links exclude-args
extrargs=( extrargs=(
# exlude sending requests to the wiki # exclude sending requests to the wiki
"--exclude" "wiki.nixos.org/wiki" "--exclude" "wiki.nixos.org/wiki"
# default is too high # default is too high
"--max-concurrency" "16" "--max-concurrency" "16"