mirror of
https://activitypub.software/limepotato/sh-jormungandr-patches.git
synced 2024-11-09 12:51:29 -07:00
56 lines
1.1 KiB
Diff
56 lines
1.1 KiB
Diff
From 036d8be30d980c7ce8c798201defb83b4e306650 Mon Sep 17 00:00:00 2001
|
|
From: limepotato <limepot@protonmail.ch>
|
|
Date: Sat, 6 Jul 2024 15:51:22 -0600
|
|
Subject: [PATCH] Fill robots.txt
|
|
|
|
---
|
|
packages/backend/assets/robots.txt | 37 ++++++++++++++++++++++++++++--
|
|
1 file changed, 35 insertions(+), 2 deletions(-)
|
|
|
|
diff --git a/packages/backend/assets/robots.txt b/packages/backend/assets/robots.txt
|
|
index dc17e04e3f..f82ad0eda0 100644
|
|
--- a/packages/backend/assets/robots.txt
|
|
+++ b/packages/backend/assets/robots.txt
|
|
@@ -1,4 +1,37 @@
|
|
user-agent: *
|
|
-allow: /
|
|
+disallow: /
|
|
|
|
-# todo: sitemap
|
|
+# explicit disallows because some bots are assholes that need that
|
|
+
|
|
+User-Agent: Googlebot
|
|
+Disallow: /
|
|
+
|
|
+User-Agent: Storebot-Google
|
|
+Disallow: /
|
|
+
|
|
+User-Agent: GoogleOther
|
|
+Disallow: /
|
|
+
|
|
+User-Agent: Google-Extended
|
|
+Disallow: /
|
|
+
|
|
+User-agent: CCBot
|
|
+Disallow: /
|
|
+
|
|
+User-agent: ChatGPT-User
|
|
+Disallow: /
|
|
+
|
|
+User-agent: GPTBot
|
|
+Disallow: /
|
|
+
|
|
+User-agent: Google-Extended
|
|
+Disallow: /
|
|
+
|
|
+User-agent: Omgilibot
|
|
+Disallow: /
|
|
+
|
|
+User-Agent: FacebookBot
|
|
+Disallow: /
|
|
+
|
|
+User-agent: Amazonbot
|
|
+Disallow: /
|
|
--
|
|
GitLab
|
|
|