diff --git a/packages/backend/assets/robots.txt b/packages/backend/assets/robots.txt index dc17e04e3..f82ad0eda 100644 --- a/packages/backend/assets/robots.txt +++ b/packages/backend/assets/robots.txt @@ -1,4 +1,37 @@ user-agent: * -allow: / +disallow: / -# todo: sitemap +# explicit disallows because some bots are assholes that need that + +User-Agent: Googlebot +Disallow: / + +User-Agent: Storebot-Google +Disallow: / + +User-Agent: GoogleOther +Disallow: / + +User-Agent: Google-Extended +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: GPTBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: Omgilibot +Disallow: / + +User-Agent: FacebookBot +Disallow: / + +User-agent: Amazonbot +Disallow: /