frontend/public/robots.txt

@@ -1,6 +1,9 @@
 user-agent: *
 disallow: /
 
+Crawl-delay: 120
+Disallow: /
+
 # explicit disallows because some bots are assholes that need that
 
 User-Agent: Googlebot
@@ -24,9 +27,6 @@ Disallow: /
 User-agent: GPTBot
 Disallow: /
 
-User-agent: Google-Extended
-Disallow: /
-
 User-agent: Omgilibot
 Disallow: /
 
@@ -47,3 +47,48 @@ Disallow: /
 
 User-agent: Applebot-Extended
 Disallow: /
+
+User-agent: googlebot-image
+Disallow: /
+
+User-agent: googlebot-mobile
+Disallow: /
+
+User-agent: MSNBot
+Disallow: /
+
+User-agent: Slurp
+Disallow: /
+
+User-agent: Teoma
+Disallow: /
+
+User-agent: Gigabot
+Disallow: /
+
+User-agent: Robozilla
+Disallow: /
+
+User-agent: Nutch
+Disallow: /
+
+User-agent: ia_archiver
+Disallow: /
+
+User-agent: baiduspider
+Disallow: /
+
+User-agent: naverbot
+Disallow: /
+
+User-agent: yeti
+Disallow: /
+
+User-agent: yahoo-mmcrawler
+Disallow: /
+
+User-agent: psbot
+Disallow: /
+
+User-agent: yahoo-blogs/v3.9
+Disallow: /