From aa698c86f0227666a775ae2297ee6c5486894f40 Mon Sep 17 00:00:00 2001
From: Jet Pham <jetthomaspham@gmail.com>
Date: Mon, 16 Feb 2026 14:41:09 -0800
Subject: [PATCH] Update robots.txt to disallow bots

---
 roles/mediawiki/files/robots.txt | 117 +++++++++++++++++++++++++++++--
 1 file changed, 113 insertions(+), 4 deletions(-)

diff --git a/roles/mediawiki/files/robots.txt b/roles/mediawiki/files/robots.txt
index 54e49a0d..a16f5c63 100644
--- a/roles/mediawiki/files/robots.txt
+++ b/roles/mediawiki/files/robots.txt
@@ -1,4 +1,11 @@
 User-agent: *
+Disallow: /
+
+User-agent: Googlebot
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
 Disallow: /wiki/86
 Disallow: /86
 Disallow: /index.php?page=86
@@ -6,8 +13,110 @@ Noindex: /wiki/86
 Noindex: /86
 Noindex: /index.php?page=86
 
-User-agent: ClaudeBot
-Disallow: /
+User-agent: Bingbot
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
 
-User-agent: Amazonbot
-Disallow: /
+User-agent: Slurp
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: DuckDuckBot
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: Baiduspider
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: YandexBot
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: ChatGPT-User
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: OAI-SearchBot
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: Claude-Web
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86
+
+User-agent: PerplexityBot
+Allow: /
+Disallow: /wiki/Special:
+Disallow: /index.php?
+Disallow: /api.php
+Disallow: /wiki/86
+Disallow: /86
+Disallow: /index.php?page=86
+Noindex: /wiki/86
+Noindex: /86
+Noindex: /index.php?page=86