// The site answer (GET /site, SEO plugin v0.2.0) and the root-file // mapping built on it. The answer is cached for SITE_CACHE_TTL_MS; the site // cache purge clears it. Three states: // // ok the backend answered: its root_files list decides which // one-segment files are proxied // absent the backend answered 404 (no SEO plugin, or one before // v0.2.0): the layer keeps its v0.1.1 behaviour (verification // files looked up by name, robots.txt and the sitemaps left to // the site's proxy) // unknown the backend did not answer, or answered something other than // a 200 object or a 404: the last definite answer (ok or absent) // seen since the server started or the last purge is then used; // with none, robots.txt, sitemap.xml and llms.txt still go to the // backend (a failure is a 503 with Retry-After) and every other // path behaves as with absent; nothing is cached // // Nitro auto-imports ($fetch, useRuntimeConfig) are used inside the // functions only, so `node --test` can import this file. import type { SeoSiteAnswer } from '../../app/utils/types.ts' import type { SeoServerConfig } from './config.ts' import { createLru } from './lru.ts' export const SITE_CACHE_TTL_MS = 300_000 export type SeoSiteState = { state: 'ok'; site: SeoSiteAnswer } | { state: 'absent' } | { state: 'unknown' } // The clock is read at call time, so a test can move Date.now. const cache = createLru({ max: 1, ttlMs: SITE_CACHE_TTL_MS, now: () => Date.now() }) /** The last definite answer (ok or absent), used while the backend is unknown. */ let lastKnown: SeoSiteState | null = null /** Empties the site cache (the purge calls it through clearSeoCaches). */ export function clearSiteCache(): void { cache.clear() lastKnown = null } export function siteCacheSize(): number { return cache.size } /** The site state from a fetch result: status and parsed body. */ export function siteState(status: number | null, body: unknown): SeoSiteState { if (status === 200 && body !== null && typeof body === 'object' && !Array.isArray(body)) return { state: 'ok', site: body as SeoSiteAnswer } if (status === 404) return { state: 'absent' } return { state: 'unknown' } } /** The cached site state for this server (cfg.enabled is the caller's check). */ export async function siteAnswer(cfg: SeoServerConfig, headers: Record): Promise { const got = await cache.load( 'site', async () => { try { const res = await $fetch.raw('/site', { baseURL: cfg.apiBase, headers, timeout: cfg.timeoutMs, ignoreResponseError: true }) return siteState(res.status, res._data) } catch { return { state: 'unknown' } as SeoSiteState } }, (s) => s.state !== 'unknown', ) if (got.state !== 'unknown') lastKnown = got else if (lastKnown) return lastKnown return got } /** A one-segment file name the site answer may list: [A-Za-z0-9._-]+ ending .html, .txt or .xml. */ const ROOT_FILE = /^\/([A-Za-z0-9][A-Za-z0-9._-]*\.(?:html|txt|xml))$/ /** The IndexNow key file name: 8 to 128 of [A-Za-z0-9-], then .txt. */ const INDEXNOW_KEY_FILE = /^[A-Za-z0-9-]{8,128}\.txt$/ /** Sitemap files of the index: sitemap--.xml, sitemap-news.xml. */ const SITEMAP_FILE = /^\/(sitemap-[a-z0-9-]+\.xml)$/ const FIXED = new Set(['/robots.txt', '/sitemap.xml', '/llms.txt']) export interface RootTarget { kind: 'root' | 'sitemap' | 'verification' | 'indexnow' /** Absolute backend URL built from the server-only bases and an allow-listed name only. */ url: string } /** * Where a root file request goes on the backend, in the order to try (the * first answer that is not 404 is served), or null when the layer does not * serve the path. With a site answer (state ok, possibly the remembered * one while the backend is down): robots.txt, sitemap.xml, llms.txt and sitemap-.xml always, * any other one-segment .html, .txt or .xml file when root_files lists it * (a verification file, then the IndexNow key file). With a backend never * reached since start (unknown, no last answer): robots.txt, sitemap.xml * and llms.txt only, so the failing fetch answers 503 with Retry-After and * a crawler never reads the app's 404 as an allow-all robots.txt (a staging * site's Disallow lives on the backend, L4 WR-08); every other path passes * through. With an older backend or one without the SEO plugin (absent, * the site answer is 404, possibly the remembered one while the backend is * down) every path passes through, as in v0.1.1. Unknown here means the * backend has not answered since start or the last purge. Nothing from the request but the * matched name reaches the URL. */ export function rootFileTargets(path: string, site: SeoSiteState, backendBase: string, apiBase: string): RootTarget[] | null { if (site.state === 'absent') return null const pathname = path.split('?')[0] ?? '' if (FIXED.has(pathname)) return [{ kind: 'root', url: `${backendBase}${pathname}` }] if (site.state !== 'ok') return null const sitemap = SITEMAP_FILE.exec(pathname) if (sitemap) return [{ kind: 'sitemap', url: `${backendBase}/_sitemap/files/${sitemap[1]}` }] const file = ROOT_FILE.exec(pathname) if (!file || !file[1]) return null const name = file[1] const listed = Array.isArray(site.site.root_files) ? site.site.root_files : [] if (!listed.includes(name)) return null const targets: RootTarget[] = [{ kind: 'verification', url: `${apiBase}/verification/${encodeURIComponent(name)}` }] if (INDEXNOW_KEY_FILE.test(name)) targets.push({ kind: 'indexnow', url: `${backendBase}/_sitemap/indexnow/${encodeURIComponent(name)}` }) return targets }