fix: root files follow the last site answer, a 404 included, while the backend is down

An SEO-less site's robots.txt, sitemap.xml and llms.txt keep reaching
the app during an outage longer than the cache TTL; a backend that never
answered still gets the 503.

quick-261104-xae, review WR-04.

tests/rootFiles.test.ts

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict'
-import test from 'node:test'
-import { rootFileTargets, siteState, type SeoSiteState } from '../server/lib/site.ts'
+import test, { type TestContext } from 'node:test'
+import { clearSiteCache, rootFileTargets, siteAnswer, siteState, SITE_CACHE_TTL_MS, type SeoSiteState } from '../server/lib/site.ts'
 import { seoBackendBase, seoServerConfig } from '../server/lib/config.ts'
 
 const B = 'http://127.0.0.1:8096'
@@ -84,3 +84,100 @@ test('seoServerConfig: defaults, aliases and tokens', () => {
   assert.equal(seoServerConfig({ seo: { notfoundToken: 't' } }).routeReportToken, 't')
   assert.equal(seoServerConfig({ seo: { notfoundToken: 't', routeReportToken: 'r' } }).routeReportToken, 'r')
 })
+
+// siteAnswer with a stubbed Nitro $fetch and a moved clock. Each answer is
+// a {status, _data} pair or an Error the fetch throws.
+type Reply = { status: number; _data: unknown } | Error
+
+function stubSite(t: TestContext) {
+  const g = globalThis as Record<string, unknown>
+  const prev = g.$fetch
+  const replies: Reply[] = []
+  const state = { calls: 0, clock: 1_000_000 }
+  g.$fetch = {
+    raw: async () => {
+      state.calls++
+      const r = replies.shift()
+      if (!r) throw new Error('no reply queued')
+      if (r instanceof Error) throw r
+      return r
+    },
+  }
+  t.mock.method(Date, 'now', () => state.clock)
+  clearSiteCache()
+  const ask = async (r: Reply): Promise<SeoSiteState> => {
+    replies.push(r)
+    return siteAnswer(seoServerConfig({}), {})
+  }
+  const expire = () => {
+    state.clock += SITE_CACHE_TTL_MS + 1
+  }
+  const restore = () => {
+    clearSiteCache()
+    g.$fetch = prev
+  }
+  return { state, ask, expire, restore }
+}
+
+const down = () => new Error('connect ECONNREFUSED')
+
+test('siteAnswer: a remembered absent outlives the cache while the backend is down (WR-04)', async (t) => {
+  const s = stubSite(t)
+  try {
+    assert.equal((await s.ask({ status: 404, _data: { error: 'not_found' } })).state, 'absent')
+    s.expire()
+    for (const r of [down(), { status: 500, _data: {} }, { status: 200, _data: '<html>' }, { status: 401, _data: {} }] as Reply[]) {
+      const before = s.state.calls
+      const got = await s.ask(r)
+      assert.equal(got.state, 'absent', String(r instanceof Error ? r.message : r.status))
+      assert.equal(s.state.calls, before + 1, 'each call fetches again')
+      for (const p of ['/robots.txt', '/sitemap.xml', '/llms.txt']) assert.equal(rootFileTargets(p, got, B, A), null, p)
+    }
+  } finally {
+    s.restore()
+  }
+})
+
+test('siteAnswer: the latest definite answer wins (WR-04)', async (t) => {
+  const s = stubSite(t)
+  try {
+    assert.equal((await s.ask({ status: 200, _data: { root_files: [] } })).state, 'ok')
+    s.expire()
+    assert.equal((await s.ask({ status: 404, _data: {} })).state, 'absent')
+    s.expire()
+    assert.equal((await s.ask(down())).state, 'absent')
+    s.expire()
+    assert.equal((await s.ask({ status: 200, _data: { root_files: [] } })).state, 'ok')
+    s.expire()
+    const got = await s.ask(down())
+    assert.equal(got.state, 'ok')
+    assert.deepEqual(rootFileTargets('/robots.txt', got, B, A), [{ kind: 'root', url: `${B}/robots.txt` }])
+  } finally {
+    s.restore()
+  }
+})
+
+test('siteAnswer: a backend that never answered stays unknown and owns the fixed root files', async (t) => {
+  const s = stubSite(t)
+  try {
+    const got = await s.ask(down())
+    assert.equal(got.state, 'unknown')
+    assert.deepEqual(rootFileTargets('/robots.txt', got, B, A), [{ kind: 'root', url: `${B}/robots.txt` }])
+    const before = s.state.calls
+    assert.equal((await s.ask(down())).state, 'unknown')
+    assert.equal(s.state.calls, before + 1, 'unknown is never cached')
+  } finally {
+    s.restore()
+  }
+})
+
+test('siteAnswer: a purge forgets the remembered answer', async (t) => {
+  const s = stubSite(t)
+  try {
+    assert.equal((await s.ask({ status: 404, _data: {} })).state, 'absent')
+    clearSiteCache()
+    assert.equal((await s.ask(down())).state, 'unknown')
+  } finally {
+    s.restore()
+  }
+})