From 036d8be30d980c7ce8c798201defb83b4e306650 Mon Sep 17 00:00:00 2001
From: limepotato <limepot@protonmail.ch>
Date: Sat, 6 Jul 2024 15:51:22 -0600
Subject: [PATCH] Fill robots.txt

---
 packages/backend/assets/robots.txt | 37 ++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/packages/backend/assets/robots.txt b/packages/backend/assets/robots.txt
index dc17e04e3f..f82ad0eda0 100644
--- a/packages/backend/assets/robots.txt
+++ b/packages/backend/assets/robots.txt
@@ -1,4 +1,37 @@
 user-agent: *
-allow: /
+disallow: /
 
-# todo: sitemap
+# explicit disallows because some bots are assholes that need that
+
+User-Agent: Googlebot
+Disallow: /
+
+User-Agent: Storebot-Google
+Disallow: /
+
+User-Agent: GoogleOther
+Disallow: /
+
+User-Agent: Google-Extended
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-Agent: FacebookBot
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
-- 
GitLab