From ef363df08dd3cd5fae2ee9bd710cb21d84badafc Mon Sep 17 00:00:00 2001 From: limepotato Date: Wed, 30 Oct 2024 16:12:25 -0600 Subject: [PATCH] add more to robots.txt --- frontend/public/robots.txt | 51 +++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/frontend/public/robots.txt b/frontend/public/robots.txt index 7004e58..0c1614f 100644 --- a/frontend/public/robots.txt +++ b/frontend/public/robots.txt @@ -1,6 +1,9 @@ user-agent: * disallow: / +Crawl-delay: 120 +Disallow: / + # explicit disallows because some bots are assholes that need that User-Agent: Googlebot @@ -24,9 +27,6 @@ Disallow: / User-agent: GPTBot Disallow: / -User-agent: Google-Extended -Disallow: / - User-agent: Omgilibot Disallow: / @@ -46,4 +46,49 @@ User-agent: Bytespider Disallow: / User-agent: Applebot-Extended +Disallow: / + +User-agent: googlebot-image +Disallow: / + +User-agent: googlebot-mobile +Disallow: / + +User-agent: MSNBot +Disallow: / + +User-agent: Slurp +Disallow: / + +User-agent: Teoma +Disallow: / + +User-agent: Gigabot +Disallow: / + +User-agent: Robozilla +Disallow: / + +User-agent: Nutch +Disallow: / + +User-agent: ia_archiver +Disallow: / + +User-agent: baiduspider +Disallow: / + +User-agent: naverbot +Disallow: / + +User-agent: yeti +Disallow: / + +User-agent: yahoo-mmcrawler +Disallow: / + +User-agent: psbot +Disallow: / + +User-agent: yahoo-blogs/v3.9 Disallow: / \ No newline at end of file