// Serves the site-root files from the CMS backend: /robots.txt, // /sitemap.xml, /sitemap-.xml (the Go sitemap's files under // /_sitemap/files/), /llms.txt, and each verification file and the IndexNow // key file the site answer lists in root_files. The backend's status, // content type and X-Robots-Tag are passed on; a backend that fails (no // answer, 5xx) gives 503 with Retry-After: 60, so a crawler retries instead // of reading a missing robots.txt as "everything allowed". Any other path // passes through. // // Only once the backend has given its site answer (SEO v0.2.0; the last // one seen is kept while the backend is down). With a backend before v0.2.0 // (the site answer is 404), or one not reached since the server started, // nothing is proxied here, as in v0.1.1: robots.txt and the sitemaps stay // with the site's proxy (or fall through to the app) and verification // files keep the v0.1.1 lookup (seo-verification.ts). Off with // the module option seo.rootFiles false or the layer switched off. The // backend URLs come from the server-only bases and an allow-listed name only // (rootFileTargets), never from the request. import { seoServerConfig } from '../lib/config.ts' import { seoForwardedFor } from '../lib/resolve.ts' import { rootFileTargets, siteAnswer } from '../lib/site.ts' const RETRY_AFTER = '60' export default defineEventHandler(async (event) => { if (event.method !== 'GET' && event.method !== 'HEAD') return const pathname = event.path.split('?')[0] ?? '' // Cheap test before any call: only one-segment .html, .txt and .xml paths. if (!/^\/[A-Za-z0-9][A-Za-z0-9._-]*\.(?:html|txt|xml)$/.test(pathname)) return const cfg = seoServerConfig(useRuntimeConfig(event)) if (!cfg.enabled || !cfg.rootFiles) return const headers = seoForwardedFor(event) const targets = rootFileTargets(pathname, await siteAnswer(cfg, headers), cfg.backendBase, cfg.apiBase) if (!targets) return let res: Awaited> | null = null let kind = targets[0]?.kind for (const target of targets) { res = await $fetch .raw(target.url, { responseType: 'text', ignoreResponseError: true, redirect: 'manual', headers, timeout: cfg.timeoutMs }) .catch(() => null) kind = target.kind if (!res || res.status !== 404) break } if (!res || res.status >= 500) { setResponseStatus(event, 503, 'Service Unavailable') setResponseHeaders(event, { 'retry-after': RETRY_AFTER, 'cache-control': 'no-store', 'content-type': 'text/plain; charset=utf-8' }) return event.method === 'HEAD' ? '' : 'Service Unavailable' } setResponseStatus(event, res.status) const out: Record = { 'content-type': res.headers.get('content-type') ?? 'text/plain; charset=utf-8', 'x-content-type-options': 'nosniff', } const robots = res.headers.get('x-robots-tag') if (robots) out['x-robots-tag'] = robots if (kind === 'verification' && res.status === 200) { // The v0.1.1 headers of a verification file. out['content-security-policy'] = res.headers.get('content-security-policy') ?? "default-src 'none'; sandbox" out['x-robots-tag'] = robots ?? 'noindex' } const cacheControl = res.headers.get('cache-control') if (cacheControl) out['cache-control'] = cacheControl setResponseHeaders(event, out) return event.method === 'HEAD' ? '' : ((res._data as string | undefined) ?? '') })