fix robots file

src/layouts/Layout.astro

@@ -25,6 +25,7 @@ const { title, description, lang } = Astro.props;
             type="application/rss+xml"
             href={new URL("rss.xml", Astro.site)}
         />
+        <link rel="sitemap" href="/sitemap-index.xml" />
         <meta name="generator" content={Astro.generator} />
         <title>{title}</title>
     </head>

src/pages/robots.txt.ts

@@ -0,0 +1,16 @@
+import type { APIRoute } from "astro";
+
+const robotsTxt = `
+User-agent: *
+Allow: /
+
+Sitemap: ${new URL("sitemap-index.xml", import.meta.env.SITE).href}
+`.trim();
+
+export const GET: APIRoute = () => {
+  return new Response(robotsTxt, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+    },
+  });
+};