From caf2e728fce8db0a8ca46d13282a5a75f0a323d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 18 May 2024 14:05:08 +0200
Subject: [PATCH] add robots.txt

fixes https://github.com/NixOS/nixos-wiki-infra/issues/77
---
 modules/nixos-wiki/default.nix |  1 +
 modules/nixos-wiki/robots.txt  | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100644 modules/nixos-wiki/robots.txt

diff --git a/modules/nixos-wiki/default.nix b/modules/nixos-wiki/default.nix
index e34c93d..c5e2e26 100644
--- a/modules/nixos-wiki/default.nix
+++ b/modules/nixos-wiki/default.nix
@@ -161,6 +161,7 @@ in
       forceSSL = lib.mkDefault true;
       locations."=/nixos.png".alias = ./nixos.png;
       locations."=/favicon.ico".alias = ./favicon.ico;
+      locations."=/robots.txt".alias = ./robots.txt;
     };
   };
 }
diff --git a/modules/nixos-wiki/robots.txt b/modules/nixos-wiki/robots.txt
new file mode 100644
index 0000000..3c6530b
--- /dev/null
+++ b/modules/nixos-wiki/robots.txt
@@ -0,0 +1,20 @@
+User-agent: *
+Disallow: /index.php?diff=
+Disallow: /index.php?oldid=
+Disallow: /index.php?title=Help
+Disallow: /index.php?title=Image
+Disallow: /index.php?title=MediaWiki
+Disallow: /index.php?title=Special:
+Disallow: /index.php?title=Template
+Disallow: /wiki/Special:
+Disallow: /api.php
+Disallow: /index.php?returnto=
+Disallow: /index.php?*userExpLevel=
+
+# Allows robots to find all the pages containing content
+Allow: /wiki/Special:AllPages
+
+# Allow the Internet Archiver to index action=raw and thereby store the raw wikitext of pages
+User-agent: ia_archiver
+Allow: /*&action=raw
+Allow: /*&action=edit