From 4a4bc244e4111311dd92fc96a5efae1168b7df57 Mon Sep 17 00:00:00 2001 From: limepotato Date: Wed, 30 Oct 2024 16:32:44 -0600 Subject: [PATCH] sitemap and dynamically generated robots.txt --- frontend/astro.config.mjs | 10 +- frontend/package-lock.json | 406 ++++++------------ frontend/package.json | 1 + frontend/src/components/Head.astro | 8 +- .../robots.txt => src/pages/robots.txt.ts} | 14 +- 5 files changed, 164 insertions(+), 275 deletions(-) rename frontend/{public/robots.txt => src/pages/robots.txt.ts} (80%) diff --git a/frontend/astro.config.mjs b/frontend/astro.config.mjs index 33b73c3..70f7829 100644 --- a/frontend/astro.config.mjs +++ b/frontend/astro.config.mjs @@ -1,15 +1,13 @@ import { defineConfig } from "astro/config"; -import compressor from "astro-compressor"; +import compressor from "astro-compressor"; import mdx from "@astrojs/mdx"; +import sitemap from "@astrojs/sitemap"; // https://astro.build/config export default defineConfig({ site: "https://nelle.observer", outDir: './build', - integrations: [ - mdx(), - compressor(), - ], + integrations: [mdx(), compressor(), sitemap()] }); -// min and compressor always last! +// min and compressor always last! \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6f7b9f7..85eb916 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -11,10 +11,14 @@ "@astrojs/check": "^0.9.1", "@astrojs/mdx": "^3.1.3", "@astrojs/rss": "^4.0.7", + "@astrojs/sitemap": "^3.2.1", "astro": "^4.13.1", "astro-compressor": "^0.4.1", - "astro-font": "^0.0.81", - "astro-min": "^1.3.1" + "astro-font": "^0.0.81" + }, + "devDependencies": { + "prettier": "^3.3.3", + "prettier-plugin-astro": "^0.14.1" } }, "node_modules/@ampproject/remapping": { @@ -178,6 +182,17 @@ "kleur": "^4.1.5" } }, + "node_modules/@astrojs/sitemap": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.2.1.tgz", + "integrity": "sha512-uxMfO8f7pALq0ADL6Lk68UV6dNYjJ2xGUzyjjVj60JLBs5a6smtlkBYv3tQ0DzoqwS7c9n4FUx5lgv0yPo/fgA==", + "license": "MIT", + "dependencies": { + "sitemap": "^8.0.0", + "stream-replace-string": "^2.0.0", + "zod": "^3.23.8" + } + }, "node_modules/@astrojs/telemetry": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.1.0.tgz", @@ -1478,42 +1493,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/@minify-html/node-darwin-x64": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@minify-html/node-darwin-x64/-/node-darwin-x64-0.15.0.tgz", - "integrity": "sha512-D9M9UDku/8I5VEMS0gTLFFQK1DFXK8io+QZvR5cbya4u8NmdDQix/t3EyCR4Wgv/Gfk86gwIS+zfMSvuKcpb5A==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@minify-html/node-linux-x64": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@minify-html/node-linux-x64/-/node-linux-x64-0.15.0.tgz", - "integrity": "sha512-cO893EV6O9ZHUFX+2Yge546OCo/eCiatjzJDmUmrPP56fQ7pzTRquHs4ko3t8Rg6tMKG7RT49mBuF09JWPnrgg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@minify-html/node-win32-x64": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@minify-html/node-win32-x64/-/node-win32-x64-0.15.0.tgz", - "integrity": "sha512-n92IFdtntchlUtyrq13pRI8TT3sOddbzuo4EPTSeocuTJMXaR77v0JYDu0fIjxXNawgGq6nBEeicxAcH4CbvUQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1549,23 +1528,6 @@ "node": ">= 8" } }, - "node_modules/@redwerkz/html-min": { - "version": "0.15.1", - "resolved": "https://registry.npmjs.org/@redwerkz/html-min/-/html-min-0.15.1.tgz", - "integrity": "sha512-Kwi6DSoOKI6q2oyt0kVTfWOBcQHBychE1BvOEyVF/j+nJjjRxEWvA0b0wx+p9ovM46P8CHD/8Y3lptYiAteZrQ==", - "license": "MIT", - "bin": { - "minify-html": "cli.js" - }, - "engines": { - "node": ">= 18.17.1" - }, - "optionalDependencies": { - "@minify-html/node-darwin-x64": "0.15.0", - "@minify-html/node-linux-x64": "0.15.0", - "@minify-html/node-win32-x64": "0.15.0" - } - }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.19.2.tgz", @@ -1783,15 +1745,6 @@ "@types/hast": "^3.0.4" } }, - "node_modules/@trysound/sax": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "license": "ISC", - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/@types/acorn": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz", @@ -1911,6 +1864,24 @@ "@types/unist": "*" } }, + "node_modules/@types/node": { + "version": "22.8.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.4.tgz", + "integrity": "sha512-SpNNxkftTJOPk0oN+y2bIqurEXHTA2AOZ3EJDDKeJ5VzkvvORSvmQXGQarcOzWV1ac7DCaPBEdMDxBsM+d8jWw==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.8" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/unist": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz", @@ -2138,6 +2109,12 @@ "node": ">= 8" } }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -2264,19 +2241,6 @@ "integrity": "sha512-VmK4kqzNpOAt2LASVbw2cbbe9sD6K30E7jOjqzHxHieHMTWInkdOljtzHmNGvKr3P0MeF4Wy+Z7j6CdETFecDA==", "license": "MIT" }, - "node_modules/astro-min": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/astro-min/-/astro-min-1.3.1.tgz", - "integrity": "sha512-TvHT24cGygf+h47WVrSxaF/6vUwJ8s+TlmwkKlNHNDqRmW5FNiUHspZnEwLsaHTtVGZWZN4NMSVPoD5oLyyrmQ==", - "license": "ISC", - "dependencies": { - "@redwerkz/html-min": "^0.15.1", - "svgo": "^3.2.0" - }, - "peerDependencies": { - "astro": "*" - } - }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -2314,12 +2278,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "license": "ISC" - }, "node_modules/boxen": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.1.1.tgz", @@ -2789,15 +2747,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/common-ancestor-path": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz", @@ -2833,47 +2782,6 @@ "node": ">= 8" } }, - "node_modules/css-select": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", - "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-tree": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz", - "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.0.30", - "source-map-js": "^1.0.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" - } - }, - "node_modules/css-what": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", - "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -2886,39 +2794,6 @@ "node": ">=4" } }, - "node_modules/csso": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", - "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", - "license": "MIT", - "dependencies": { - "css-tree": "~2.2.0" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", - "npm": ">=7.0.0" - } - }, - "node_modules/csso/node_modules/css-tree": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", - "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.0.28", - "source-map-js": "^1.0.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", - "npm": ">=7.0.0" - } - }, - "node_modules/csso/node_modules/mdn-data": { - "version": "2.0.28", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", - "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==", - "license": "CC0-1.0" - }, "node_modules/debug": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", @@ -3014,61 +2889,6 @@ "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", "license": "MIT" }, - "node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/domelementtype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "BSD-2-Clause" - }, - "node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "license": "BSD-2-Clause", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/domutils": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", - "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", - "license": "BSD-2-Clause", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, "node_modules/dset": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.3.tgz", @@ -4597,12 +4417,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/mdn-data": { - "version": "2.0.30", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz", - "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", - "license": "CC0-1.0" - }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -5454,18 +5268,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, "node_modules/onetime": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", @@ -5773,6 +5575,37 @@ "node": ">=18.12" } }, + "node_modules/prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "devOptional": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-plugin-astro": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/prettier-plugin-astro/-/prettier-plugin-astro-0.14.1.tgz", + "integrity": "sha512-RiBETaaP9veVstE4vUwSIcdATj6dKmXljouXc/DDNwBSPTp8FRkLGDSGFClKsAFeeg+13SB0Z1JZvbD76bigJw==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "@astrojs/compiler": "^2.9.1", + "prettier": "^3.0.0", + "sass-formatter": "^0.7.6" + }, + "engines": { + "node": "^14.15.0 || >=16.0.0" + } + }, "node_modules/prismjs": { "version": "1.29.0", "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", @@ -6192,6 +6025,29 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/s.color": { + "version": "0.0.15", + "resolved": "https://registry.npmjs.org/s.color/-/s.color-0.0.15.tgz", + "integrity": "sha512-AUNrbEUHeKY8XsYr/DYpl+qk5+aM+DChopnWOPEzn8YKzOhv4l2zH6LzZms3tOZP3wwdOyc0RmTciyi46HLIuA==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/sass-formatter": { + "version": "0.7.9", + "resolved": "https://registry.npmjs.org/sass-formatter/-/sass-formatter-0.7.9.tgz", + "integrity": "sha512-CWZ8XiSim+fJVG0cFLStwDvft1VI7uvXdCNJYXhDvowiv+DsbD1nXLiQ4zrE5UBvj5DWZJ93cwN0NX5PMsr1Pw==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "suf-log": "^2.5.3" + } + }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "license": "ISC" + }, "node_modules/section-matter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", @@ -6317,6 +6173,31 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "license": "MIT" }, + "node_modules/sitemap": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-8.0.0.tgz", + "integrity": "sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==", + "license": "MIT", + "dependencies": { + "@types/node": "^17.0.5", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.2.4" + }, + "bin": { + "sitemap": "dist/cli.js" + }, + "engines": { + "node": ">=14.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/sitemap/node_modules/@types/node": { + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", + "license": "MIT" + }, "node_modules/source-map": { "version": "0.7.4", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", @@ -6363,6 +6244,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stream-replace-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/stream-replace-string/-/stream-replace-string-2.0.0.tgz", + "integrity": "sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==", + "license": "MIT" + }, "node_modules/string-width": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", @@ -6454,6 +6341,16 @@ "inline-style-parser": "0.1.1" } }, + "node_modules/suf-log": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/suf-log/-/suf-log-2.5.3.tgz", + "integrity": "sha512-KvC8OPjzdNOe+xQ4XWJV2whQA0aM1kGVczMQ8+dStAO6KfEB140JEVQ9dE76ONZ0/Ylf67ni4tILPJB41U0eow==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "s.color": "0.0.15" + } + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -6466,31 +6363,6 @@ "node": ">=4" } }, - "node_modules/svgo": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.2.tgz", - "integrity": "sha512-OoohrmuUlBs8B8o6MB2Aevn+pRIH9zDALSR+6hhqVfa6fRwG/Qw9VUMSMW9VNg2CFc/MTIfabtdOVl9ODIJjpw==", - "license": "MIT", - "dependencies": { - "@trysound/sax": "0.2.0", - "commander": "^7.2.0", - "css-select": "^5.1.0", - "css-tree": "^2.3.1", - "css-what": "^6.1.0", - "csso": "^5.0.5", - "picocolors": "^1.0.0" - }, - "bin": { - "svgo": "bin/svgo" - }, - "engines": { - "node": ">=14.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/svgo" - } - }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -6600,6 +6472,12 @@ "semver": "^7.3.8" } }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" + }, "node_modules/unified": { "version": "11.0.5", "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", diff --git a/frontend/package.json b/frontend/package.json index 80b4032..2d7bfdd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,6 +13,7 @@ "@astrojs/check": "^0.9.1", "@astrojs/mdx": "^3.1.3", "@astrojs/rss": "^4.0.7", + "@astrojs/sitemap": "^3.2.1", "astro": "^4.13.1", "astro-compressor": "^0.4.1", "astro-font": "^0.0.81" diff --git a/frontend/src/components/Head.astro b/frontend/src/components/Head.astro index 3c83563..39bca92 100755 --- a/frontend/src/components/Head.astro +++ b/frontend/src/components/Head.astro @@ -27,9 +27,6 @@ const { title } = Astro.props; - - {title} @@ -54,4 +51,7 @@ const { title } = Astro.props; - \ No newline at end of file + + + + \ No newline at end of file diff --git a/frontend/public/robots.txt b/frontend/src/pages/robots.txt.ts similarity index 80% rename from frontend/public/robots.txt rename to frontend/src/pages/robots.txt.ts index 0c1614f..5943d1c 100644 --- a/frontend/public/robots.txt +++ b/frontend/src/pages/robots.txt.ts @@ -1,9 +1,15 @@ +import type { APIRoute } from 'astro'; + +const getRobotsTxt = (sitemapURL: URL) => ` + user-agent: * disallow: / Crawl-delay: 120 Disallow: / +Sitemap: ${sitemapURL.href} + # explicit disallows because some bots are assholes that need that User-Agent: Googlebot @@ -91,4 +97,10 @@ User-agent: psbot Disallow: / User-agent: yahoo-blogs/v3.9 -Disallow: / \ No newline at end of file +Disallow: / +`; + +export const GET: APIRoute = ({ site }) => { + const sitemapURL = new URL('sitemap-index.xml', site); + return new Response(getRobotsTxt(sitemapURL)); +}; \ No newline at end of file