3868f3d74d
An SEO-less site's robots.txt, sitemap.xml and llms.txt keep reaching the app during an outage longer than the cache TTL; a backend that never answered still gets the 503. quick-261104-xae, review WR-04.
75 lines
3.7 KiB
TypeScript
75 lines
3.7 KiB
TypeScript
// Serves the site-root files from the CMS backend: /robots.txt,
|
|
// /sitemap.xml, /sitemap-<name>.xml (the Go sitemap's files under
|
|
// /_sitemap/files/), /llms.txt, and each verification file and the IndexNow
|
|
// key file the site answer lists in root_files. The backend's status,
|
|
// content type and X-Robots-Tag are passed on; a backend that fails (no
|
|
// answer, 5xx) gives 503 with Retry-After: 60, so a crawler retries instead
|
|
// of reading a missing robots.txt as "everything allowed". Any other path
|
|
// passes through.
|
|
//
|
|
// Once the backend has given its site answer (SEO v0.2.0) every file above
|
|
// is proxied. The last answer, ok or absent, is kept while the backend is
|
|
// down, so on a backend without the SEO plugin the files keep reaching the
|
|
// app during an outage. Only while the backend has not answered since the
|
|
// server started or the last purge are robots.txt, sitemap.xml and
|
|
// llms.txt proxied by their fixed names: the failing fetch answers 503
|
|
// with Retry-After, so a staging site never exposes the app's 404 as an
|
|
// allow-all robots.txt (L4 WR-08). With a backend before v0.2.0 (the site
|
|
// answer is 404) nothing is proxied here, as in v0.1.1: robots.txt and the
|
|
// sitemaps stay with the site's proxy (or fall through to the app); there
|
|
// and while the backend is unknown, verification files keep the v0.1.1
|
|
// lookup (seo-verification.ts). Off with
|
|
// the module option seo.rootFiles false or the layer switched off. The
|
|
// backend URLs come from the server-only bases and an allow-listed name only
|
|
// (rootFileTargets), never from the request.
|
|
|
|
import { seoServerConfig } from '../lib/config.ts'
|
|
import { seoForwardedFor } from '../lib/resolve.ts'
|
|
import { rootFileTargets, siteAnswer } from '../lib/site.ts'
|
|
|
|
const RETRY_AFTER = '60'
|
|
|
|
export default defineEventHandler(async (event) => {
|
|
if (event.method !== 'GET' && event.method !== 'HEAD') return
|
|
const pathname = event.path.split('?')[0] ?? ''
|
|
// Cheap test before any call: only one-segment .html, .txt and .xml paths.
|
|
if (!/^\/[A-Za-z0-9][A-Za-z0-9._-]*\.(?:html|txt|xml)$/.test(pathname)) return
|
|
const cfg = seoServerConfig(useRuntimeConfig(event))
|
|
if (!cfg.enabled || !cfg.rootFiles) return
|
|
const headers = seoForwardedFor(event)
|
|
const targets = rootFileTargets(pathname, await siteAnswer(cfg, headers), cfg.backendBase, cfg.apiBase)
|
|
if (!targets) return
|
|
|
|
let res: Awaited<ReturnType<typeof $fetch.raw>> | null = null
|
|
let kind = targets[0]?.kind
|
|
for (const target of targets) {
|
|
res = await $fetch
|
|
.raw(target.url, { responseType: 'text', ignoreResponseError: true, redirect: 'manual', headers, timeout: cfg.timeoutMs })
|
|
.catch(() => null)
|
|
kind = target.kind
|
|
if (!res || res.status !== 404) break
|
|
}
|
|
if (!res || res.status >= 500) {
|
|
setResponseStatus(event, 503, 'Service Unavailable')
|
|
setResponseHeaders(event, { 'retry-after': RETRY_AFTER, 'cache-control': 'no-store', 'content-type': 'text/plain; charset=utf-8' })
|
|
return event.method === 'HEAD' ? '' : 'Service Unavailable'
|
|
}
|
|
|
|
setResponseStatus(event, res.status)
|
|
const out: Record<string, string> = {
|
|
'content-type': res.headers.get('content-type') ?? 'text/plain; charset=utf-8',
|
|
'x-content-type-options': 'nosniff',
|
|
}
|
|
const robots = res.headers.get('x-robots-tag')
|
|
if (robots) out['x-robots-tag'] = robots
|
|
if (kind === 'verification' && res.status === 200) {
|
|
// The v0.1.1 headers of a verification file.
|
|
out['content-security-policy'] = res.headers.get('content-security-policy') ?? "default-src 'none'; sandbox"
|
|
out['x-robots-tag'] = robots ?? 'noindex'
|
|
}
|
|
const cacheControl = res.headers.get('cache-control')
|
|
if (cacheControl) out['cache-control'] = cacheControl
|
|
setResponseHeaders(event, out)
|
|
return event.method === 'HEAD' ? '' : ((res._data as string | undefined) ?? '')
|
|
})
|