fix: root files follow the last site answer, a 404 included, while the backend is down

An SEO-less site's robots.txt, sitemap.xml and llms.txt keep reaching
the app during an outage longer than the cache TTL; a backend that never
answered still gets the 503.

quick-261104-xae, review WR-04.

server/lib/site.ts

@@ -8,10 +8,12 @@
 //            v0.2.0): the layer keeps its v0.1.1 behaviour (verification
 //            files looked up by name, robots.txt and the sitemaps left to
 //            the site's proxy)
-//   unknown  the backend did not answer: the last ok answer is used when
-//            there is one; else robots.txt, sitemap.xml and llms.txt still
-//            go to the backend (whose failure is a 503 with Retry-After) and
-//            every other path behaves as with absent; nothing is cached
+//   unknown  the backend did not answer, or answered something other than
+//            a 200 object or a 404: the last definite answer (ok or absent)
+//            seen since the server started or the last purge is then used;
+//            with none, robots.txt, sitemap.xml and llms.txt still go to the
+//            backend (a failure is a 503 with Retry-After) and every other
+//            path behaves as with absent; nothing is cached
 //
 // Nitro auto-imports ($fetch, useRuntimeConfig) are used inside the
 // functions only, so `node --test` can import this file.
@@ -24,13 +26,15 @@ export const SITE_CACHE_TTL_MS = 300_000
 
 export type SeoSiteState = { state: 'ok'; site: SeoSiteAnswer } | { state: 'absent' } | { state: 'unknown' }
 
-const cache = createLru<SeoSiteState>({ max: 1, ttlMs: SITE_CACHE_TTL_MS })
-let lastOk: SeoSiteState | null = null
+// The clock is read at call time, so a test can move Date.now.
+const cache = createLru<SeoSiteState>({ max: 1, ttlMs: SITE_CACHE_TTL_MS, now: () => Date.now() })
+/** The last definite answer (ok or absent), used while the backend is unknown. */
+let lastKnown: SeoSiteState | null = null
 
 /** Empties the site cache (the purge calls it through clearSeoCaches). */
 export function clearSiteCache(): void {
   cache.clear()
-  lastOk = null
+  lastKnown = null
 }
 
 export function siteCacheSize(): number {
@@ -58,8 +62,8 @@ export async function siteAnswer(cfg: SeoServerConfig, headers: Record<string, s
     },
     (s) => s.state !== 'unknown',
   )
-  if (got.state === 'ok') lastOk = got
-  if (got.state === 'unknown' && lastOk) return lastOk
+  if (got.state !== 'unknown') lastKnown = got
+  else if (lastKnown) return lastKnown
   return got
 }
 
@@ -80,16 +84,18 @@ export interface RootTarget {
 /**
  * Where a root file request goes on the backend, in the order to try (the
  * first answer that is not 404 is served), or null when the layer does not
- * serve the path. With a site answer (state ok, possibly the last one
- * seen): robots.txt, sitemap.xml, llms.txt and sitemap-<name>.xml always,
+ * serve the path. With a site answer (state ok, possibly the remembered
+ * one while the backend is down): robots.txt, sitemap.xml, llms.txt and sitemap-<name>.xml always,
  * any other one-segment .html, .txt or .xml file when root_files lists it
  * (a verification file, then the IndexNow key file). With a backend never
  * reached since start (unknown, no last answer): robots.txt, sitemap.xml
  * and llms.txt only, so the failing fetch answers 503 with Retry-After and
  * a crawler never reads the app's 404 as an allow-all robots.txt (a staging
  * site's Disallow lives on the backend, L4 WR-08); every other path passes
- * through. With an older backend (absent, the site answer is 404) every
- * path passes through, as in v0.1.1. Nothing from the request but the
+ * through. With an older backend or one without the SEO plugin (absent,
+ * the site answer is 404, possibly the remembered one while the backend is
+ * down) every path passes through, as in v0.1.1. Unknown here means the
+ * backend has not answered since start or the last purge. Nothing from the request but the
  * matched name reaches the URL.
  */
 export function rootFileTargets(path: string, site: SeoSiteState, backendBase: string, apiBase: string): RootTarget[] | null {

server/middleware/seo-root.ts

@@ -7,10 +7,12 @@
 // of reading a missing robots.txt as "everything allowed". Any other path
 // passes through.
 //
-// Once the backend has given its site answer (SEO v0.2.0; the last one
-// seen is kept while the backend is down) every file above is proxied.
-// While the backend has not answered since the server started, only
-// robots.txt, sitemap.xml and llms.txt are: the failing fetch answers 503
+// Once the backend has given its site answer (SEO v0.2.0) every file above
+// is proxied. The last answer, ok or absent, is kept while the backend is
+// down, so on a backend without the SEO plugin the files keep reaching the
+// app during an outage. Only while the backend has not answered since the
+// server started or the last purge are robots.txt, sitemap.xml and
+// llms.txt proxied by their fixed names: the failing fetch answers 503
 // with Retry-After, so a staging site never exposes the app's 404 as an
 // allow-all robots.txt (L4 WR-08). With a backend before v0.2.0 (the site
 // answer is 404) nothing is proxied here, as in v0.1.1: robots.txt and the