chore(seo): implement sitemap logic

next.config.js

@@ -82,6 +82,10 @@ const moduleExports = async () => {
           source: '/robots.txt',
           destination: '/api/robots',
         },
+        {
+          source: '/sitemap/:reference.xml',
+          destination: '/api/sitemap/:reference',
+        },
       ];
     },
     images: {

package.json

@@ -150,6 +150,7 @@
     "schema-dts": "^1.1.2",
     "sharp": "^0.32.1",
     "simple-git": "^3.22.0",
+    "sitemap": "^7.1.1",
     "superjson": "^1.13.3",
     "tiktoken": "^1.0.10",
     "ts-custom-error": "^3.3.1",

src/pages/api/robots.ts

@@ -1,18 +1,30 @@
 import { NextApiRequest, NextApiResponse } from 'next';
 import getConfig from 'next/config';
 
-import devRobotsFile from '@etabli/src/pages/assets/public/dev/robots.txt';
-import prodRobotsFile from '@etabli/src/pages/assets/public/prod/robots.txt';
 import { apiHandlerWrapper } from '@etabli/src/utils/api';
+import { linkRegistry } from '@etabli/src/utils/routes/registry';
 
 const { publicRuntimeConfig } = getConfig();
 
 export function handler(req: NextApiRequest, res: NextApiResponse) {
   // Only allow indexing in production
   if (publicRuntimeConfig.appMode === 'prod') {
-    res.send(prodRobotsFile);
+    // Note: sitemap URLs need to be absolute (ref: https://stackoverflow.com/a/14218476/3608410)
+    res.send(
+      `
+User-agent: *
+Allow: /
+Sitemap: ${linkRegistry.get('sitemapIndex', undefined, { absolute: true })}
+`.trim()
+    );
   } else {
-    res.send(devRobotsFile);
+    res.send(
+      `
+User-agent: *
+Disallow: /
+Allow: /.well-known/
+`.trim()
+    );
   }
 }
 

src/pages/api/sitemap/[sitemap].ts

@@ -0,0 +1,84 @@
+import { NextApiRequest, NextApiResponse } from 'next';
+import { SitemapIndexStream, SitemapItemLoose, SitemapStream } from 'sitemap';
+import { createGzip } from 'zlib';
+import { z } from 'zod';
+
+import { prisma } from '@etabli/src/prisma/client';
+import { apiHandlerWrapper } from '@etabli/src/utils/api';
+import { linkRegistry } from '@etabli/src/utils/routes/registry';
+import { getBaseUrl } from '@etabli/src/utils/url';
+
+const PathSchema = z.literal('index').or(z.coerce.number().positive());
+
+const chunkSize = 40_000; // The maximum allowed by Google is 50k URLs or 50MB
+
+export async function handler(req: NextApiRequest, res: NextApiResponse) {
+  // Either to return all sitemaps index, or a specific sitemap
+  const value = PathSchema.parse(req.query.sitemap);
+
+  // Listing static routes first
+  const routes: string[] = [
+    linkRegistry.get('assistant', undefined),
+    linkRegistry.get('explore', undefined),
+    linkRegistry.get('home', undefined),
+    linkRegistry.get('initiatives', undefined),
+  ];
+
+  res.setHeader('Content-Type', 'application/xml');
+  res.setHeader('Content-Encoding', 'gzip');
+
+  if (value === 'index') {
+    const initiativesCount = await prisma.initiative.count({});
+    const sitemapsCount = Math.ceil((routes.length + initiativesCount) / chunkSize);
+
+    const stream = new SitemapIndexStream({});
+    const pipeline = stream.pipe(createGzip());
+
+    for (let i = 1; i <= sitemapsCount; i++) {
+      // URLs must be absolute to be indexed
+      stream.write({ url: linkRegistry.get('sitemap', { sitemapId: i }, { absolute: true }) });
+    }
+
+    stream.end();
+    pipeline.pipe(res).on('error', (error) => {
+      throw error;
+    });
+  } else {
+    const stream = new SitemapStream({ hostname: getBaseUrl() });
+    const pipeline = stream.pipe(createGzip());
+
+    const page = value;
+
+    const initiatives = await prisma.initiative.findMany({
+      select: {
+        id: true,
+        updatedAt: true,
+      },
+      orderBy: {
+        createdAt: 'asc',
+      },
+      skip: (page - 1) * chunkSize, // The static routes are limited so we are fine if the first chunk will be for example 40_013 length
+      take: chunkSize,
+    });
+
+    for (const staticRoute of routes) {
+      stream.write({ url: staticRoute, changefreq: 'weekly', priority: 0.8 } as SitemapItemLoose);
+    }
+
+    for (const initiative of initiatives) {
+      stream.write({
+        url: linkRegistry.get('initiative', { initiativeId: initiative.id }),
+        lastmod: initiative.updatedAt.toISOString(),
+        changefreq: 'monthly',
+        priority: 0.5,
+      } as SitemapItemLoose);
+    }
+
+    stream.end();
+    pipeline.pipe(res).on('error', (error) => {
+      throw error;
+    });
+  }
+}
+
+export default apiHandlerWrapper(handler);

src/utils/routes/list.ts

@@ -35,6 +35,18 @@ export const localizedRoutes = {
       en: (p) => `/initiatives`,
     }
   ),
+  sitemap: defineLocalizedRoute(
+    { sitemapId: param.path.number },
+    {
+      en: (p) => `/sitemap/${p.sitemapId}.xml`,
+    }
+  ),
+  sitemapIndex: defineLocalizedRoute(
+    {},
+    {
+      en: (p) => `/sitemap/index.xml`,
+    }
+  ),
 };
 
 // function createLocalizedRouter(lang: Lang, localeRoutes: typeof localizedRoutes) {
@@ -75,5 +87,7 @@ export const routes = {
     home: defineRoute(localizedRoutes.home.params, localizedRoutes.home.paths.en),
     initiative: defineRoute(localizedRoutes.initiative.params, localizedRoutes.initiative.paths.en),
     initiatives: defineRoute(localizedRoutes.initiatives.params, localizedRoutes.initiatives.paths.en),
+    sitemap: defineRoute(localizedRoutes.sitemap.params, localizedRoutes.sitemap.paths.en),
+    sitemapIndex: defineRoute(localizedRoutes.sitemapIndex.params, localizedRoutes.sitemapIndex.paths.en),
   }).routes,
 };