sitemap and dynamically generated robots.txt

add more to robots.txt
remove todo
2024-10-30 16:32:44 -06:00 · 2024-10-30 16:12:25 -06:00 · 2024-10-30 16:07:04 -06:00
7 changed files with 258 additions and 325 deletions
--- a/frontend/astro.config.mjs
+++ b/frontend/astro.config.mjs
@ -1,15 +1,13 @@
 import { defineConfig } from "astro/config";
-import compressor from "astro-compressor";

+import compressor from "astro-compressor";
 import mdx from "@astrojs/mdx";
+import sitemap from "@astrojs/sitemap";

 // https://astro.build/config
 export default defineConfig({
  site: "https://nelle.observer",
  outDir: './build',
-  integrations: [
-    mdx(),
-    compressor(),
-  ],
+  integrations: [mdx(), compressor(), sitemap()]
 });
-// min and compressor always last!
+// min and compressor always last!
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@ -11,10 +11,14 @@
        "@astrojs/check": "^0.9.1",
        "@astrojs/mdx": "^3.1.3",
        "@astrojs/rss": "^4.0.7",
+        "@astrojs/sitemap": "^3.2.1",
        "astro": "^4.13.1",
        "astro-compressor": "^0.4.1",
-        "astro-font": "^0.0.81",
-        "astro-min": "^1.3.1"
+        "astro-font": "^0.0.81"
+      },
+      "devDependencies": {
+        "prettier": "^3.3.3",
+        "prettier-plugin-astro": "^0.14.1"
      }
    },
    "node_modules/@ampproject/remapping": {
@ -178,6 +182,17 @@
        "kleur": "^4.1.5"
      }
    },
+    "node_modules/@astrojs/sitemap": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.2.1.tgz",
+      "integrity": "sha512-uxMfO8f7pALq0ADL6Lk68UV6dNYjJ2xGUzyjjVj60JLBs5a6smtlkBYv3tQ0DzoqwS7c9n4FUx5lgv0yPo/fgA==",
+      "license": "MIT",
+      "dependencies": {
+        "sitemap": "^8.0.0",
+        "stream-replace-string": "^2.0.0",
+        "zod": "^3.23.8"
+      }
+    },
    "node_modules/@astrojs/telemetry": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.1.0.tgz",
@ -1478,42 +1493,6 @@
        "url": "https://opencollective.com/unified"
      }
    },
-    "node_modules/@minify-html/node-darwin-x64": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@minify-html/node-darwin-x64/-/node-darwin-x64-0.15.0.tgz",
-      "integrity": "sha512-D9M9UDku/8I5VEMS0gTLFFQK1DFXK8io+QZvR5cbya4u8NmdDQix/t3EyCR4Wgv/Gfk86gwIS+zfMSvuKcpb5A==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@minify-html/node-linux-x64": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@minify-html/node-linux-x64/-/node-linux-x64-0.15.0.tgz",
-      "integrity": "sha512-cO893EV6O9ZHUFX+2Yge546OCo/eCiatjzJDmUmrPP56fQ7pzTRquHs4ko3t8Rg6tMKG7RT49mBuF09JWPnrgg==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@minify-html/node-win32-x64": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@minify-html/node-win32-x64/-/node-win32-x64-0.15.0.tgz",
-      "integrity": "sha512-n92IFdtntchlUtyrq13pRI8TT3sOddbzuo4EPTSeocuTJMXaR77v0JYDu0fIjxXNawgGq6nBEeicxAcH4CbvUQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@ -1549,23 +1528,6 @@
        "node": ">= 8"
      }
    },
-    "node_modules/@redwerkz/html-min": {
-      "version": "0.15.1",
-      "resolved": "https://registry.npmjs.org/@redwerkz/html-min/-/html-min-0.15.1.tgz",
-      "integrity": "sha512-Kwi6DSoOKI6q2oyt0kVTfWOBcQHBychE1BvOEyVF/j+nJjjRxEWvA0b0wx+p9ovM46P8CHD/8Y3lptYiAteZrQ==",
-      "license": "MIT",
-      "bin": {
-        "minify-html": "cli.js"
-      },
-      "engines": {
-        "node": ">= 18.17.1"
-      },
-      "optionalDependencies": {
-        "@minify-html/node-darwin-x64": "0.15.0",
-        "@minify-html/node-linux-x64": "0.15.0",
-        "@minify-html/node-win32-x64": "0.15.0"
-      }
-    },
    "node_modules/@rollup/rollup-android-arm-eabi": {
      "version": "4.19.2",
      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.19.2.tgz",
@ -1783,15 +1745,6 @@
        "@types/hast": "^3.0.4"
      }
    },
-    "node_modules/@trysound/sax": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz",
-      "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==",
-      "license": "ISC",
-      "engines": {
-        "node": ">=10.13.0"
-      }
-    },
    "node_modules/@types/acorn": {
      "version": "4.0.6",
      "resolved": "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz",
@ -1911,6 +1864,24 @@
        "@types/unist": "*"
      }
    },
+    "node_modules/@types/node": {
+      "version": "22.8.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.4.tgz",
+      "integrity": "sha512-SpNNxkftTJOPk0oN+y2bIqurEXHTA2AOZ3EJDDKeJ5VzkvvORSvmQXGQarcOzWV1ac7DCaPBEdMDxBsM+d8jWw==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~6.19.8"
+      }
+    },
+    "node_modules/@types/sax": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz",
+      "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/unist": {
      "version": "3.0.2",
      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz",
@ -2138,6 +2109,12 @@
        "node": ">= 8"
      }
    },
+    "node_modules/arg": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
+      "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==",
+      "license": "MIT"
+    },
    "node_modules/argparse": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
@ -2264,19 +2241,6 @@
      "integrity": "sha512-VmK4kqzNpOAt2LASVbw2cbbe9sD6K30E7jOjqzHxHieHMTWInkdOljtzHmNGvKr3P0MeF4Wy+Z7j6CdETFecDA==",
      "license": "MIT"
    },
-    "node_modules/astro-min": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/astro-min/-/astro-min-1.3.1.tgz",
-      "integrity": "sha512-TvHT24cGygf+h47WVrSxaF/6vUwJ8s+TlmwkKlNHNDqRmW5FNiUHspZnEwLsaHTtVGZWZN4NMSVPoD5oLyyrmQ==",
-      "license": "ISC",
-      "dependencies": {
-        "@redwerkz/html-min": "^0.15.1",
-        "svgo": "^3.2.0"
-      },
-      "peerDependencies": {
-        "astro": "*"
-      }
-    },
    "node_modules/axobject-query": {
      "version": "4.1.0",
      "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz",
@ -2314,12 +2278,6 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
-    "node_modules/boolbase": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
-      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
-      "license": "ISC"
-    },
    "node_modules/boxen": {
      "version": "7.1.1",
      "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.1.1.tgz",
@ -2789,15 +2747,6 @@
        "url": "https://github.com/sponsors/wooorm"
      }
    },
-    "node_modules/commander": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
-      "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 10"
-      }
-    },
    "node_modules/common-ancestor-path": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz",
@ -2833,47 +2782,6 @@
        "node": ">= 8"
      }
    },
-    "node_modules/css-select": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
-      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
-      "license": "BSD-2-Clause",
-      "dependencies": {
-        "boolbase": "^1.0.0",
-        "css-what": "^6.1.0",
-        "domhandler": "^5.0.2",
-        "domutils": "^3.0.1",
-        "nth-check": "^2.0.1"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
-      }
-    },
-    "node_modules/css-tree": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz",
-      "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==",
-      "license": "MIT",
-      "dependencies": {
-        "mdn-data": "2.0.30",
-        "source-map-js": "^1.0.1"
-      },
-      "engines": {
-        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
-      }
-    },
-    "node_modules/css-what": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
-      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
-      "license": "BSD-2-Clause",
-      "engines": {
-        "node": ">= 6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
-      }
-    },
    "node_modules/cssesc": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
@ -2886,39 +2794,6 @@
        "node": ">=4"
      }
    },
-    "node_modules/csso": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz",
-      "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==",
-      "license": "MIT",
-      "dependencies": {
-        "css-tree": "~2.2.0"
-      },
-      "engines": {
-        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0",
-        "npm": ">=7.0.0"
-      }
-    },
-    "node_modules/csso/node_modules/css-tree": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz",
-      "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==",
-      "license": "MIT",
-      "dependencies": {
-        "mdn-data": "2.0.28",
-        "source-map-js": "^1.0.1"
-      },
-      "engines": {
-        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0",
-        "npm": ">=7.0.0"
-      }
-    },
-    "node_modules/csso/node_modules/mdn-data": {
-      "version": "2.0.28",
-      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz",
-      "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==",
-      "license": "CC0-1.0"
-    },
    "node_modules/debug": {
      "version": "4.3.6",
      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz",
@ -3014,61 +2889,6 @@
      "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==",
      "license": "MIT"
    },
-    "node_modules/dom-serializer": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
-      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
-      "license": "MIT",
-      "dependencies": {
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.2",
-        "entities": "^4.2.0"
-      },
-      "funding": {
-        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
-      }
-    },
-    "node_modules/domelementtype": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
-      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/fb55"
-        }
-      ],
-      "license": "BSD-2-Clause"
-    },
-    "node_modules/domhandler": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
-      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
-      "license": "BSD-2-Clause",
-      "dependencies": {
-        "domelementtype": "^2.3.0"
-      },
-      "engines": {
-        "node": ">= 4"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/domhandler?sponsor=1"
-      }
-    },
-    "node_modules/domutils": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
-      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
-      "license": "BSD-2-Clause",
-      "dependencies": {
-        "dom-serializer": "^2.0.0",
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.3"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/domutils?sponsor=1"
-      }
-    },
    "node_modules/dset": {
      "version": "3.1.3",
      "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.3.tgz",
@ -4597,12 +4417,6 @@
        "url": "https://opencollective.com/unified"
      }
    },
-    "node_modules/mdn-data": {
-      "version": "2.0.30",
-      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
-      "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==",
-      "license": "CC0-1.0"
-    },
    "node_modules/merge-stream": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@ -5454,18 +5268,6 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
-    "node_modules/nth-check": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
-      "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
-      "license": "BSD-2-Clause",
-      "dependencies": {
-        "boolbase": "^1.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/nth-check?sponsor=1"
-      }
-    },
    "node_modules/onetime": {
      "version": "6.0.0",
      "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz",
@ -5773,6 +5575,37 @@
        "node": ">=18.12"
      }
    },
+    "node_modules/prettier": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz",
+      "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==",
+      "devOptional": true,
+      "license": "MIT",
+      "bin": {
+        "prettier": "bin/prettier.cjs"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/prettier/prettier?sponsor=1"
+      }
+    },
+    "node_modules/prettier-plugin-astro": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/prettier-plugin-astro/-/prettier-plugin-astro-0.14.1.tgz",
+      "integrity": "sha512-RiBETaaP9veVstE4vUwSIcdATj6dKmXljouXc/DDNwBSPTp8FRkLGDSGFClKsAFeeg+13SB0Z1JZvbD76bigJw==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "@astrojs/compiler": "^2.9.1",
+        "prettier": "^3.0.0",
+        "sass-formatter": "^0.7.6"
+      },
+      "engines": {
+        "node": "^14.15.0 || >=16.0.0"
+      }
+    },
    "node_modules/prismjs": {
      "version": "1.29.0",
      "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",
@ -6192,6 +6025,29 @@
        "queue-microtask": "^1.2.2"
      }
    },
+    "node_modules/s.color": {
+      "version": "0.0.15",
+      "resolved": "https://registry.npmjs.org/s.color/-/s.color-0.0.15.tgz",
+      "integrity": "sha512-AUNrbEUHeKY8XsYr/DYpl+qk5+aM+DChopnWOPEzn8YKzOhv4l2zH6LzZms3tOZP3wwdOyc0RmTciyi46HLIuA==",
+      "devOptional": true,
+      "license": "MIT"
+    },
+    "node_modules/sass-formatter": {
+      "version": "0.7.9",
+      "resolved": "https://registry.npmjs.org/sass-formatter/-/sass-formatter-0.7.9.tgz",
+      "integrity": "sha512-CWZ8XiSim+fJVG0cFLStwDvft1VI7uvXdCNJYXhDvowiv+DsbD1nXLiQ4zrE5UBvj5DWZJ93cwN0NX5PMsr1Pw==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "suf-log": "^2.5.3"
+      }
+    },
+    "node_modules/sax": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz",
+      "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==",
+      "license": "ISC"
+    },
    "node_modules/section-matter": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",
@ -6317,6 +6173,31 @@
      "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
      "license": "MIT"
    },
+    "node_modules/sitemap": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-8.0.0.tgz",
+      "integrity": "sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "^17.0.5",
+        "@types/sax": "^1.2.1",
+        "arg": "^5.0.0",
+        "sax": "^1.2.4"
+      },
+      "bin": {
+        "sitemap": "dist/cli.js"
+      },
+      "engines": {
+        "node": ">=14.0.0",
+        "npm": ">=6.0.0"
+      }
+    },
+    "node_modules/sitemap/node_modules/@types/node": {
+      "version": "17.0.45",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz",
+      "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==",
+      "license": "MIT"
+    },
    "node_modules/source-map": {
      "version": "0.7.4",
      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz",
@ -6363,6 +6244,12 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/stream-replace-string": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/stream-replace-string/-/stream-replace-string-2.0.0.tgz",
+      "integrity": "sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==",
+      "license": "MIT"
+    },
    "node_modules/string-width": {
      "version": "7.2.0",
      "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
@ -6454,6 +6341,16 @@
        "inline-style-parser": "0.1.1"
      }
    },
+    "node_modules/suf-log": {
+      "version": "2.5.3",
+      "resolved": "https://registry.npmjs.org/suf-log/-/suf-log-2.5.3.tgz",
+      "integrity": "sha512-KvC8OPjzdNOe+xQ4XWJV2whQA0aM1kGVczMQ8+dStAO6KfEB140JEVQ9dE76ONZ0/Ylf67ni4tILPJB41U0eow==",
+      "devOptional": true,
+      "license": "MIT",
+      "dependencies": {
+        "s.color": "0.0.15"
+      }
+    },
    "node_modules/supports-color": {
      "version": "5.5.0",
      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
@ -6466,31 +6363,6 @@
        "node": ">=4"
      }
    },
-    "node_modules/svgo": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.2.tgz",
-      "integrity": "sha512-OoohrmuUlBs8B8o6MB2Aevn+pRIH9zDALSR+6hhqVfa6fRwG/Qw9VUMSMW9VNg2CFc/MTIfabtdOVl9ODIJjpw==",
-      "license": "MIT",
-      "dependencies": {
-        "@trysound/sax": "0.2.0",
-        "commander": "^7.2.0",
-        "css-select": "^5.1.0",
-        "css-tree": "^2.3.1",
-        "css-what": "^6.1.0",
-        "csso": "^5.0.5",
-        "picocolors": "^1.0.0"
-      },
-      "bin": {
-        "svgo": "bin/svgo"
-      },
-      "engines": {
-        "node": ">=14.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/svgo"
-      }
-    },
    "node_modules/to-fast-properties": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
@ -6600,6 +6472,12 @@
        "semver": "^7.3.8"
      }
    },
+    "node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "license": "MIT"
+    },
    "node_modules/unified": {
      "version": "11.0.5",
      "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz",
--- a/frontend/package.json
+++ b/frontend/package.json
@ -13,6 +13,7 @@
    "@astrojs/check": "^0.9.1",
    "@astrojs/mdx": "^3.1.3",
    "@astrojs/rss": "^4.0.7",
+    "@astrojs/sitemap": "^3.2.1",
    "astro": "^4.13.1",
    "astro-compressor": "^0.4.1",
    "astro-font": "^0.0.81"
--- a/frontend/public/robots.txt
+++ b/frontend/public/robots.txt
@ -1,49 +0,0 @@
-user-agent: *
-disallow: /
-
-# explicit disallows because some bots are assholes that need that
-
-User-Agent: Googlebot
-Disallow: /
-
-User-Agent: Storebot-Google
-Disallow: /
-
-User-Agent: GoogleOther
-Disallow: /
-
-User-Agent: Google-Extended
-Disallow: /
-
-User-agent: CCBot
-Disallow: /
-
-User-agent: ChatGPT-User
-Disallow: /
-
-User-agent: GPTBot
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
-
-User-agent: Omgilibot
-Disallow: /
-
-User-agent: Omgili
-Disallow: /
-
-User-Agent: FacebookBot
-Disallow: /
-
-User-agent: Amazonbot
-Disallow: /
-
-User-agent: anthropic-ai
-Disallow: /
-
-User-agent: Bytespider
-Disallow: /
-
-User-agent: Applebot-Extended
-Disallow: /
--- a/frontend/public/scripts/main.js
+++ b/frontend/public/scripts/main.js
@ -124,8 +124,7 @@ function showNtfyInfo() {
  }
 }

-//// color scheme changer
-///TODO: make default gray-scale?
+//// color scheme change
 const headTag = document.getElementsByTagName('head')[0];
 const styleTag = document.createElement("style");

--- a/frontend/src/components/Head.astro
+++ b/frontend/src/components/Head.astro
@ -27,9 +27,6 @@ const { title } = Astro.props;

 <meta name="robots" content={"noindex, nofollow"} />

-<!-- Canonical URL --><!--
-<link rel="canonical" href={canonicalURL} />-->
-
 <!-- Primary Meta Tags -->
 <title>{title}</title>
 <meta name="title" content={title} />
@ -54,4 +51,7 @@ const { title } = Astro.props;
 <link href="https://ouroboros.gay/@limepot" rel="me" />

 <!-- fuck dark reader -->
-<meta name="darkreader-lock"/>
+<meta name="darkreader-lock"/>
+
+<!-- sitemap -->
+<link rel="sitemap" href="/sitemap-index.xml" />
--- a/frontend/src/pages/robots.txt.ts
+++ b/frontend/src/pages/robots.txt.ts
@ -0,0 +1,106 @@
+import type { APIRoute } from 'astro';
+
+const getRobotsTxt = (sitemapURL: URL) => `
+
+user-agent: *
+disallow: /
+
+Crawl-delay: 120
+Disallow: /
+
+Sitemap: ${sitemapURL.href}
+
+# explicit disallows because some bots are assholes that need that
+
+User-Agent: Googlebot
+Disallow: /
+
+User-Agent: Storebot-Google
+Disallow: /
+
+User-Agent: GoogleOther
+Disallow: /
+
+User-Agent: Google-Extended
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: Omgili
+Disallow: /
+
+User-Agent: FacebookBot
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: googlebot-image
+Disallow: /
+
+User-agent: googlebot-mobile
+Disallow: /
+
+User-agent: MSNBot
+Disallow: /
+
+User-agent: Slurp
+Disallow: /
+
+User-agent: Teoma
+Disallow: /
+
+User-agent: Gigabot
+Disallow: /
+
+User-agent: Robozilla
+Disallow: /
+
+User-agent: Nutch
+Disallow: /
+
+User-agent: ia_archiver
+Disallow: /
+
+User-agent: baiduspider
+Disallow: /
+
+User-agent: naverbot
+Disallow: /
+
+User-agent: yeti
+Disallow: /
+
+User-agent: yahoo-mmcrawler
+Disallow: /
+
+User-agent: psbot
+Disallow: /
+
+User-agent: yahoo-blogs/v3.9
+Disallow: /
+`;
+
+export const GET: APIRoute = ({ site }) => {
+    const sitemapURL = new URL('sitemap-index.xml', site);
+    return new Response(getRobotsTxt(sitemapURL));
+};
Author	SHA1	Message	Date
limepotato	4a4bc244e4	sitemap and dynamically generated robots.txt	2024-10-30 16:32:44 -06:00
limepotato	ef363df08d	add more to robots.txt	2024-10-30 16:12:25 -06:00
limepotato	73ccde566f	remove todo	2024-10-30 16:07:04 -06:00