From d1629439c7333e671cd826d071fd1f8f266598ff Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 09:26:00 +0100 Subject: [PATCH] feat: root files from the CMS and the 404 report - server/middleware/seo-root.ts proxies /robots.txt, /sitemap.xml, /sitemap-.xml, /llms.txt and the verification and IndexNow key files the site answer lists in root_files, with the backend's status, content type and X-Robots-Tag; a backend failure is 503 Retry-After 60 - server/lib/site.ts caches GET /_seo/api/v1/site (300 s, cleared by the purge); a backend without it (404) keeps the v0.1.1 behaviour: no root proxies, the verification lookup by name - the 404 report and the verification lookup honour seo.cms and rootFiles - smoke runs for the layer off (seo.cms and the v0.1.1 enabled switch), the CMS down (fail open) and no 404 token; npm run purge:check --- package.json | 3 +- server/lib/resolve.ts | 8 +- server/lib/site.ts | 100 ++++++++++++++++++ server/middleware/seo-root.ts | 65 ++++++++++++ server/middleware/seo-verification.ts | 20 +++- server/plugins/seo-notfound.ts | 16 +-- testing/expect-seo-down.txt | 9 ++ testing/expect-seo-notoken.txt | 3 + testing/expect-seo-off.txt | 13 +++ testing/expect-seo.txt | 24 +++++ .../a1b2c3d4e5f60718293a4b5c6d7e8f90.txt | 1 + testing/fixtures/seo/root/llms.txt | 3 + testing/fixtures/seo/root/robots.txt | 3 + testing/fixtures/seo/root/sitemap-posts-1.xml | 2 + testing/fixtures/seo/root/sitemap.xml | 2 + testing/fixtures/seo/site.json | 13 +++ .../fixtures/seo/verification/google1234.html | 1 + tests/rootFiles.test.ts | 72 +++++++++++++ 18 files changed, 342 insertions(+), 16 deletions(-) create mode 100644 server/lib/site.ts create mode 100644 server/middleware/seo-root.ts create mode 100644 testing/expect-seo-down.txt create mode 100644 testing/expect-seo-notoken.txt create mode 100644 testing/expect-seo-off.txt create mode 100644 testing/fixtures/seo/indexnow/a1b2c3d4e5f60718293a4b5c6d7e8f90.txt create mode 100644 testing/fixtures/seo/root/llms.txt create mode 100644 testing/fixtures/seo/root/robots.txt create mode 100644 testing/fixtures/seo/root/sitemap-posts-1.xml create mode 100644 testing/fixtures/seo/root/sitemap.xml create mode 100644 testing/fixtures/seo/site.json create mode 100644 testing/fixtures/seo/verification/google1234.html create mode 100644 tests/rootFiles.test.ts diff --git a/package.json b/package.json index c1bfa0e..eea3025 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "build:playground": "nuxt build .playground", "test": "node --test --test-reporter=tap tests/*.test.ts", "dev:prepare": "nuxt prepare .playground", - "smoke:ssr": "node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect.txt --no-site && node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect-seo.txt" + "smoke:ssr": "node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect.txt --no-site && node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect-seo.txt && node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect-seo-off.txt --env NUXT_PUBLIC_SEO_CMS=false && node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect-seo-off.txt --env NUXT_PUBLIC_SEO_ENABLED=false && node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect-seo-down.txt --env NUXT_SEO_API_BASE=http://127.0.0.1:1/_seo/api/v1 && node testing/smoke-ssr.mjs --output .playground/.output --fixtures testing/fixtures --expect testing/expect-seo-notoken.txt --env NUXT_SEO_NOTFOUND_TOKEN=", + "purge:check": "node testing/purge-check.mjs --output .playground/.output --fixtures testing/fixtures --path /blog/lwia-grzywa && node testing/purge-token-check.mjs --output .playground/.output --fixtures testing/fixtures --path /blog/lwia-grzywa" }, "engines": { "node": ">=22.18" diff --git a/server/lib/resolve.ts b/server/lib/resolve.ts index a51c530..a31d9fa 100644 --- a/server/lib/resolve.ts +++ b/server/lib/resolve.ts @@ -16,6 +16,7 @@ import { seoAnswer } from '../../app/utils/seo.ts' import type { SeoResolveAnswer } from '../../app/utils/types.ts' import { seoServerConfig, type SeoServerConfig } from './config.ts' import { createLru, type Lru } from './lru.ts' +import { clearSiteCache, siteCacheSize } from './site.ts' /** Most resolve answers kept. */ export const RESOLVE_CACHE_MAX = 1000 @@ -31,14 +32,15 @@ function resolveCache(ttlSeconds: number): Lru { return cache } -/** Empties the layer's SEO caches (the resolve answers); the site cache purge calls it. */ +/** Empties the layer's SEO caches (resolve answers and the site answer); the site cache purge calls it. */ export function clearSeoCaches(): void { cache?.clear() + clearSiteCache() } /** How many answers the layer's caches hold (tests and diagnostics). */ -export function seoCacheSizes(): { resolve: number } { - return { resolve: cache?.size ?? 0 } +export function seoCacheSizes(): { resolve: number; site: number } { + return { resolve: cache?.size ?? 0, site: siteCacheSize() } } /** The x-forwarded-for value the SEO calls carry for this request. */ diff --git a/server/lib/site.ts b/server/lib/site.ts new file mode 100644 index 0000000..b6e2bd4 --- /dev/null +++ b/server/lib/site.ts @@ -0,0 +1,100 @@ +// The site answer (GET /site, SEO plugin v0.2.0) and the root-file +// mapping built on it. The answer is cached for SITE_CACHE_TTL_MS; the site +// cache purge clears it. Three states: +// +// ok the backend answered: its root_files list decides which +// one-segment files are proxied +// absent the backend answered 404 (no SEO plugin, or one before +// v0.2.0): the layer keeps its v0.1.1 behaviour (verification +// files looked up by name, robots.txt and the sitemaps left to +// the site's proxy) +// unknown the backend did not answer: the last ok answer is used when +// there is one; nothing is cached +// +// Nitro auto-imports ($fetch, useRuntimeConfig) are used inside the +// functions only, so `node --test` can import this file. + +import type { SeoSiteAnswer } from '../../app/utils/types.ts' +import type { SeoServerConfig } from './config.ts' +import { createLru } from './lru.ts' + +export const SITE_CACHE_TTL_MS = 300_000 + +export type SeoSiteState = { state: 'ok'; site: SeoSiteAnswer } | { state: 'absent' } | { state: 'unknown' } + +const cache = createLru({ max: 1, ttlMs: SITE_CACHE_TTL_MS }) +let lastOk: SeoSiteState | null = null + +/** Empties the site cache (the purge calls it through clearSeoCaches). */ +export function clearSiteCache(): void { + cache.clear() + lastOk = null +} + +export function siteCacheSize(): number { + return cache.size +} + +/** The site state from a fetch result: status and parsed body. */ +export function siteState(status: number | null, body: unknown): SeoSiteState { + if (status === 200 && body !== null && typeof body === 'object' && !Array.isArray(body)) return { state: 'ok', site: body as SeoSiteAnswer } + if (status === 404) return { state: 'absent' } + return { state: 'unknown' } +} + +/** The cached site state for this server (cfg.enabled is the caller's check). */ +export async function siteAnswer(cfg: SeoServerConfig, headers: Record): Promise { + const got = await cache.load( + 'site', + async () => { + try { + const res = await $fetch.raw('/site', { baseURL: cfg.apiBase, headers, timeout: cfg.timeoutMs, ignoreResponseError: true }) + return siteState(res.status, res._data) + } catch { + return { state: 'unknown' } as SeoSiteState + } + }, + (s) => s.state !== 'unknown', + ) + if (got.state === 'ok') lastOk = got + if (got.state === 'unknown' && lastOk) return lastOk + return got +} + +/** A one-segment file name the site answer may list: [A-Za-z0-9._-]+ ending .html, .txt or .xml. */ +const ROOT_FILE = /^\/([A-Za-z0-9][A-Za-z0-9._-]*\.(?:html|txt|xml))$/ +/** The IndexNow key file name: 8 to 128 of [A-Za-z0-9-], then .txt. */ +const INDEXNOW_KEY_FILE = /^[A-Za-z0-9-]{8,128}\.txt$/ +/** Sitemap files of the index: sitemap--.xml, sitemap-news.xml. */ +const SITEMAP_FILE = /^\/(sitemap-[a-z0-9-]+\.xml)$/ +const FIXED = new Set(['/robots.txt', '/sitemap.xml', '/llms.txt']) + +export interface RootTarget { + kind: 'root' | 'sitemap' | 'verification' | 'indexnow' + /** Absolute backend URL built from the server-only bases and an allow-listed name only. */ + url: string +} + +/** + * Where a root file request goes on the backend, in the order to try (the + * first answer that is not 404 is served), or null when the layer does not + * serve the path. robots.txt, sitemap.xml, llms.txt and sitemap-.xml + * are proxied unless the site state is absent; any other one-segment + * .html, .txt or .xml file only when the site answer lists it in + * root_files (a verification file, then the IndexNow key file). Nothing + * from the request but the matched name reaches the URL. + */ +export function rootFileTargets(path: string, site: SeoSiteState, backendBase: string, apiBase: string): RootTarget[] | null { + const pathname = path.split('?')[0] ?? '' + if (FIXED.has(pathname)) return site.state === 'absent' ? null : [{ kind: 'root', url: `${backendBase}${pathname}` }] + const sitemap = SITEMAP_FILE.exec(pathname) + if (sitemap) return site.state === 'absent' ? null : [{ kind: 'sitemap', url: `${backendBase}/_sitemap/files/${sitemap[1]}` }] + const file = ROOT_FILE.exec(pathname) + if (!file || !file[1] || site.state !== 'ok') return null + const name = file[1] + const listed = Array.isArray(site.site.root_files) ? site.site.root_files : [] + if (!listed.includes(name)) return null + const targets: RootTarget[] = [{ kind: 'verification', url: `${apiBase}/verification/${encodeURIComponent(name)}` }] + if (INDEXNOW_KEY_FILE.test(name)) targets.push({ kind: 'indexnow', url: `${backendBase}/_sitemap/indexnow/${encodeURIComponent(name)}` }) + return targets +} diff --git a/server/middleware/seo-root.ts b/server/middleware/seo-root.ts new file mode 100644 index 0000000..c363caa --- /dev/null +++ b/server/middleware/seo-root.ts @@ -0,0 +1,65 @@ +// Serves the site-root files from the CMS backend: /robots.txt, +// /sitemap.xml, /sitemap-.xml (the Go sitemap's files under +// /_sitemap/files/), /llms.txt, and each verification file and the IndexNow +// key file the site answer lists in root_files. The backend's status, +// content type and X-Robots-Tag are passed on; a backend that fails (no +// answer, 5xx) gives 503 with Retry-After: 60, so a crawler retries instead +// of reading a missing robots.txt as "everything allowed". Any other path +// passes through. +// +// With a backend before SEO v0.2.0 (the site answer is 404) nothing is +// proxied here: robots.txt and the sitemaps stay with the site's proxy and +// verification files keep the v0.1.1 lookup (seo-verification.ts). Off with +// the module option seo.rootFiles false or the layer switched off. The +// backend URLs come from the server-only bases and an allow-listed name only +// (rootFileTargets), never from the request. + +import { seoServerConfig } from '../lib/config.ts' +import { seoForwardedFor } from '../lib/resolve.ts' +import { rootFileTargets, siteAnswer } from '../lib/site.ts' + +const RETRY_AFTER = '60' + +export default defineEventHandler(async (event) => { + if (event.method !== 'GET' && event.method !== 'HEAD') return + const pathname = event.path.split('?')[0] ?? '' + // Cheap test before any call: only one-segment .html, .txt and .xml paths. + if (!/^\/[A-Za-z0-9][A-Za-z0-9._-]*\.(?:html|txt|xml)$/.test(pathname)) return + const cfg = seoServerConfig(useRuntimeConfig(event)) + if (!cfg.enabled || !cfg.rootFiles) return + const headers = seoForwardedFor(event) + const targets = rootFileTargets(pathname, await siteAnswer(cfg, headers), cfg.backendBase, cfg.apiBase) + if (!targets) return + + let res: Awaited> | null = null + let kind = targets[0]?.kind + for (const target of targets) { + res = await $fetch + .raw(target.url, { responseType: 'text', ignoreResponseError: true, redirect: 'manual', headers, timeout: cfg.timeoutMs }) + .catch(() => null) + kind = target.kind + if (!res || res.status !== 404) break + } + if (!res || res.status >= 500) { + setResponseStatus(event, 503, 'Service Unavailable') + setResponseHeaders(event, { 'retry-after': RETRY_AFTER, 'cache-control': 'no-store', 'content-type': 'text/plain; charset=utf-8' }) + return event.method === 'HEAD' ? '' : 'Service Unavailable' + } + + setResponseStatus(event, res.status) + const out: Record = { + 'content-type': res.headers.get('content-type') ?? 'text/plain; charset=utf-8', + 'x-content-type-options': 'nosniff', + } + const robots = res.headers.get('x-robots-tag') + if (robots) out['x-robots-tag'] = robots + if (kind === 'verification' && res.status === 200) { + // The v0.1.1 headers of a verification file. + out['content-security-policy'] = res.headers.get('content-security-policy') ?? "default-src 'none'; sandbox" + out['x-robots-tag'] = robots ?? 'noindex' + } + const cacheControl = res.headers.get('cache-control') + if (cacheControl) out['cache-control'] = cacheControl + setResponseHeaders(event, out) + return event.method === 'HEAD' ? '' : ((res._data as string | undefined) ?? '') +}) diff --git a/server/middleware/seo-verification.ts b/server/middleware/seo-verification.ts index 7c034fe..c269413 100644 --- a/server/middleware/seo-verification.ts +++ b/server/middleware/seo-verification.ts @@ -4,8 +4,16 @@ // them. Only a single-segment .html, .txt or .xml path is looked up, never a // name the site serves itself; anything the backend does not answer 200 // falls through to the normal routes. +// +// This is the v0.1.1 lookup, for a backend without the site answer (SEO +// plugin before v0.2.0, /site answers 404) or one that does not answer it +// now. With a site answer, seo-root.ts serves exactly the files its +// root_files lists and every other name passes through untouched. import { seoVerificationName } from '../../app/utils/seo.ts' +import { seoServerConfig } from '../lib/config.ts' +import { seoForwardedFor } from '../lib/resolve.ts' +import { siteAnswer } from '../lib/site.ts' const RESERVED = ['robots.txt', 'sitemap.xml', 'sitemap.xsl', 'rss.xml', 'feed.xml', 'atom.xml', 'llms.txt'] @@ -13,15 +21,17 @@ export default defineEventHandler(async (event) => { if (event.method !== 'GET' && event.method !== 'HEAD') return const name = seoVerificationName(event.path, RESERVED) if (!name) return - const config = useRuntimeConfig(event) - if ((config.public.seo as { enabled?: boolean } | undefined)?.enabled === false) return - const seo = config.seo as { apiBase: string; timeoutMs?: number } + const cfg = seoServerConfig(useRuntimeConfig(event)) + if (!cfg.enabled || !cfg.rootFiles) return + const headers = seoForwardedFor(event) + if ((await siteAnswer(cfg, headers)).state === 'ok') return const res = await $fetch .raw(`/verification/${encodeURIComponent(name)}`, { - baseURL: seo.apiBase, + baseURL: cfg.apiBase, responseType: 'text', ignoreResponseError: true, - timeout: Number(seo.timeoutMs) || 1500, + headers, + timeout: cfg.timeoutMs, }) .catch(() => null) if (!res || res.status !== 200) return diff --git a/server/plugins/seo-notfound.ts b/server/plugins/seo-notfound.ts index 7f065f6..4eb9cb3 100644 --- a/server/plugins/seo-notfound.ts +++ b/server/plugins/seo-notfound.ts @@ -4,11 +4,14 @@ // skips afterResponse, so both the `error` hook and `afterResponse` report, // once per request. Only page views count: GET requests that accept // text/html, never Nuxt's own paths (/_nuxt, /__nuxt_error, …) or the API -// routes, and nothing without the token configured. A failed report is -// logged once and never touches the response. +// routes, and nothing without the token configured or with the layer off +// (public.seo.enabled or seo.cms false). The report never delays the +// response: it is sent after it (event.waitUntil keeps it alive). A failed +// report is logged once and never touches the response. import type { H3Event } from 'h3' import { seoResolvableUrl } from '../../app/utils/seo.ts' +import { seoServerConfig } from '../lib/config.ts' let warned = false @@ -16,10 +19,9 @@ function report(event: H3Event) { if (event.context.seoNotFoundReported) return if (event.method !== 'GET') return if (!(getRequestHeader(event, 'accept') ?? '').includes('text/html')) return - const config = useRuntimeConfig(event) - if ((config.public.seo as { enabled?: boolean } | undefined)?.enabled === false) return - const seo = config.seo as { apiBase: string; notfoundToken?: string; timeoutMs?: number } - const token = seo.notfoundToken?.trim() + const seo = seoServerConfig(useRuntimeConfig(event)) + if (!seo.enabled) return + const token = seo.notfoundToken if (!token) return const url = seoResolvableUrl(event.path) if (!url) return @@ -32,7 +34,7 @@ function report(event: H3Event) { method: 'POST', body: { url: url.slice(0, 2048), referer: referer.slice(0, 1024) }, headers: { authorization: `Bearer ${token}`, ...(forwarded ? { 'x-forwarded-for': forwarded } : {}) }, - timeout: Number(seo.timeoutMs) || 1500, + timeout: seo.timeoutMs, }).catch((err: unknown) => { if (!warned) { warned = true diff --git a/testing/expect-seo-down.txt b/testing/expect-seo-down.txt new file mode 100644 index 0000000..a3d5fc3 --- /dev/null +++ b/testing/expect-seo-down.txt @@ -0,0 +1,9 @@ +# The CMS unreachable (NUXT_SEO_API_BASE on a closed port): pages render +# normally with their own head (fail open); a root file is 503 with +# Retry-After, so crawlers retry; verification lookups fall through. +200 /about contains=Own about | Playground notcontains=About us | Acme noheader=X-Robots-Tag count=application/ld+json:0 +200 /plain contains=Plain page | Playground +503 /robots.txt header=Retry-After: 60 +503 /sitemap-posts-1.xml header=Retry-After: 60 +404 /google0123.html +apicalls 0 diff --git a/testing/expect-seo-notoken.txt b/testing/expect-seo-notoken.txt new file mode 100644 index 0000000..112510e --- /dev/null +++ b/testing/expect-seo-notoken.txt @@ -0,0 +1,3 @@ +# No 404 token (NUXT_SEO_NOTFOUND_TOKEN empty): a 404 page sends no report. +404 /kontakt +noreport /kontakt diff --git a/testing/expect-seo-off.txt b/testing/expect-seo-off.txt new file mode 100644 index 0000000..9745a68 --- /dev/null +++ b/testing/expect-seo-off.txt @@ -0,0 +1,13 @@ +# The layer switched off: seo.cms false (NUXT_PUBLIC_SEO_CMS=false) or the +# v0.1.1 switch public.seo.enabled false (NUXT_PUBLIC_SEO_ENABLED=false). +# No resolve, no CMS head, no redirect, no root files, no 404 report: the +# pages keep their own head and the API sees no request at all. +200 /about contains=Own about | Playground contains=href="https://own-canonical.test/about" noheader=X-Robots-Tag +200 /plain?page=abc contains=Plain page | Playground +404 /stary noheader=Location +404 /robots.txt +404 /google1234.html +404 /google0123.html +404 /kontakt +apicalls 0 +noreport /kontakt diff --git a/testing/expect-seo.txt b/testing/expect-seo.txt index 92a0ad6..8892571 100644 --- a/testing/expect-seo.txt +++ b/testing/expect-seo.txt @@ -39,3 +39,27 @@ resolves /stary 1 200 /api/__seo__/resolve?url=/blog/lwia-grzywa contains="title":"Lwia grzywa | Example" header=X-Robots-Tag: noindex resolves /blog/lwia-grzywa 1 422 /api/__seo__/resolve?url=https://evil.test/ + +# Root files from the backend, with its status, content type and +# X-Robots-Tag; a backend failure is 503 with Retry-After. +200 /robots.txt contains=Disallow: /konto type=text/plain header=X-Content-Type-Options: nosniff +200 /sitemap.xml contains= Fixture llms.txt diff --git a/testing/fixtures/seo/root/robots.txt b/testing/fixtures/seo/root/robots.txt new file mode 100644 index 0000000..82a34d8 --- /dev/null +++ b/testing/fixtures/seo/root/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /konto +Sitemap: https://example.test/sitemap.xml diff --git a/testing/fixtures/seo/root/sitemap-posts-1.xml b/testing/fixtures/seo/root/sitemap-posts-1.xml new file mode 100644 index 0000000..7e9b339 --- /dev/null +++ b/testing/fixtures/seo/root/sitemap-posts-1.xml @@ -0,0 +1,2 @@ + +https://example.test/blog/lwia-grzywa diff --git a/testing/fixtures/seo/root/sitemap.xml b/testing/fixtures/seo/root/sitemap.xml new file mode 100644 index 0000000..d82bb35 --- /dev/null +++ b/testing/fixtures/seo/root/sitemap.xml @@ -0,0 +1,2 @@ + +https://example.test/sitemap-posts-1.xml diff --git a/testing/fixtures/seo/site.json b/testing/fixtures/seo/site.json new file mode 100644 index 0000000..17316fd --- /dev/null +++ b/testing/fixtures/seo/site.json @@ -0,0 +1,13 @@ +{ + "site_url": "https://example.test", + "default_locale": "pl", + "locales": ["pl", "en"], + "unprefixed_locale": "pl", + "links": { "new_tab": true }, + "html_sitemap": { "enabled": true, "paths": { "pl": "/mapa-strony", "en": "/en/sitemap" }, "page_size": 100 }, + "routes": [{ "type": "post", "pattern": "/blog/:slug" }], + "root_files": ["google1234.html", "a1b2c3d4e5f60718293a4b5c6d7e8f90.txt"], + "llms": true, + "pagination": { "form": "query", "param": "page", "segments": {} }, + "home_crumb": { "name": "Start", "url": "https://example.test/" } +} diff --git a/testing/fixtures/seo/verification/google1234.html b/testing/fixtures/seo/verification/google1234.html new file mode 100644 index 0000000..d864c3b --- /dev/null +++ b/testing/fixtures/seo/verification/google1234.html @@ -0,0 +1 @@ +google-site-verification: google1234.html \ No newline at end of file diff --git a/tests/rootFiles.test.ts b/tests/rootFiles.test.ts new file mode 100644 index 0000000..55e18ef --- /dev/null +++ b/tests/rootFiles.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { rootFileTargets, siteState, type SeoSiteState } from '../server/lib/site.ts' +import { seoBackendBase, seoServerConfig } from '../server/lib/config.ts' + +const B = 'http://127.0.0.1:8096' +const A = 'http://127.0.0.1:8096/_seo/api/v1' +const KEY = 'a1b2c3d4e5f60718293a4b5c6d7e8f90.txt' +const ok: SeoSiteState = { state: 'ok', site: { root_files: ['google1234.html', KEY, 'BingSiteAuth.xml'] } } + +test('rootFileTargets: robots.txt, sitemap.xml, llms.txt and sitemap files go to the backend', () => { + assert.deepEqual(rootFileTargets('/robots.txt', ok, B, A), [{ kind: 'root', url: `${B}/robots.txt` }]) + assert.deepEqual(rootFileTargets('/sitemap.xml?x=1', ok, B, A), [{ kind: 'root', url: `${B}/sitemap.xml` }]) + assert.deepEqual(rootFileTargets('/llms.txt', { state: 'unknown' }, B, A), [{ kind: 'root', url: `${B}/llms.txt` }]) + assert.deepEqual(rootFileTargets('/sitemap-posts-1.xml', ok, B, A), [{ kind: 'sitemap', url: `${B}/_sitemap/files/sitemap-posts-1.xml` }]) +}) + +test('rootFileTargets: an absent site answer (older backend) serves none of them', () => { + for (const p of ['/robots.txt', '/sitemap.xml', '/llms.txt', '/sitemap-posts-1.xml', '/google1234.html']) { + assert.equal(rootFileTargets(p, { state: 'absent' }, B, A), null, p) + } +}) + +test('rootFileTargets: listed root files only, verification first then the IndexNow key', () => { + assert.deepEqual(rootFileTargets('/google1234.html', ok, B, A), [{ kind: 'verification', url: `${A}/verification/google1234.html` }]) + assert.deepEqual(rootFileTargets(`/${KEY}`, ok, B, A), [ + { kind: 'verification', url: `${A}/verification/${KEY}` }, + { kind: 'indexnow', url: `${B}/_sitemap/indexnow/${KEY}` }, + ]) + for (const p of ['/random.txt', '/kontakt', '/google1234.html/x', '/a/google1234.html', '/GOOGLE1234.HTML']) { + assert.equal(rootFileTargets(p, ok, B, A), null, p) + } + assert.equal(rootFileTargets('/google1234.html', { state: 'unknown' }, B, A), null) +}) + +test('rootFileTargets: no request text beyond an allow-listed name reaches the URL', () => { + for (const p of ['/sitemap-../../x.xml', '/sitemap-%2e%2e.xml', '/robots.txt/../admin', '//evil.test/robots.txt', '/sitemap-A.xml']) { + assert.equal(rootFileTargets(p, ok, B, A), null, p) + } +}) + +test('siteState: 200 object ok, 404 absent, anything else unknown', () => { + assert.equal(siteState(200, { root_files: [] }).state, 'ok') + assert.equal(siteState(404, { error: 'not_found' }).state, 'absent') + for (const [s, b] of [[500, {}], [200, 'x'], [200, null], [null, null], [200, []]] as const) assert.equal(siteState(s, b).state, 'unknown') +}) + +test('seoBackendBase: explicit, derived from apiBase, or its origin', () => { + assert.equal(seoBackendBase('http://go:9000/', A), 'http://go:9000') + assert.equal(seoBackendBase('', A), B) + assert.equal(seoBackendBase('', 'http://go:9000/_seo/api/v1/'), 'http://go:9000') + assert.equal(seoBackendBase('', 'http://go:9000/seo'), 'http://go:9000') + assert.equal(seoBackendBase('', 'not a url'), '') +}) + +test('seoServerConfig: defaults, aliases and tokens', () => { + const d = seoServerConfig({}) + assert.equal(d.enabled, true) + assert.equal(d.apiBase, A) + assert.equal(d.backendBase, B) + assert.equal(d.timeoutMs, 1500) + assert.equal(d.resolveCacheTtl, 60) + assert.equal(d.rootFiles, true) + assert.deepEqual(d.resolveFiles, ['pdf']) + assert.equal(seoServerConfig({ public: { seo: { enabled: false } } }).enabled, false) + assert.equal(seoServerConfig({ public: { seo: { cms: 'false' } } }).enabled, false) + assert.equal(seoServerConfig({ public: { seo: { rootFiles: false } } }).rootFiles, false) + assert.equal(seoServerConfig({ seo: { resolveCacheTtl: '0' } }).resolveCacheTtl, 0) + assert.equal(seoServerConfig({ seo: { timeoutMs: '2500' } }).timeoutMs, 2500) + assert.equal(seoServerConfig({ seo: { notfoundToken: 't' } }).routeReportToken, 't') + assert.equal(seoServerConfig({ seo: { notfoundToken: 't', routeReportToken: 'r' } }).routeReportToken, 'r') +})