From abb59b22f97d5c3987808667f95f3a8339328a7b Mon Sep 17 00:00:00 2001 From: limepotato Date: Sun, 30 Jun 2024 14:25:35 -0600 Subject: [PATCH] robots.txt --- public/robots.txt | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 public/robots.txt diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..f82ad0e --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,37 @@ +user-agent: * +disallow: / + +# explicit disallows because some bots are assholes that need that + +User-Agent: Googlebot +Disallow: / + +User-Agent: Storebot-Google +Disallow: / + +User-Agent: GoogleOther +Disallow: / + +User-Agent: Google-Extended +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: GPTBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: Omgilibot +Disallow: / + +User-Agent: FacebookBot +Disallow: / + +User-agent: Amazonbot +Disallow: /