From 6636cde70a6d866e6d7511c24f5d3b70258455f9 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 09:31:39 +0100 Subject: [PATCH] fix: root files fall through until the backend has given a site answer With the SEO backend unreachable since start, /robots.txt and /sitemap.xml answered 503 where v0.1.1 left them to the app (a consumer's smoke expects the app's 404). The root proxies now need a site answer; the last one seen is kept while the backend is down, so a failure after it is still 503. --- README.md | 2 +- server/lib/site.ts | 22 +++++++++++++--------- server/middleware/seo-root.ts | 9 ++++++--- testing/expect-seo-down.txt | 11 ++++++----- tests/rootFiles.test.ts | 10 ++++++---- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index daf3ded..432649b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The layer's only module is its own local one (`lib/seo-module.ts`); it pulls in - **One JSON-LD graph (v0.2.0).** The graph holds the CMS nodes (the answer's `jsonld`) plus the nodes pages and layers add with `useSchemaNode(node)`, merged by `@id`: a node whose `@id` the CMS already has fills in only the keys the CMS node lacks; a new `@id` (or none) is appended. No node at all, no script. unhead escapes `<` inside the JSON, so a value cannot end the script element. - **Robots and staging.** A `staging` answer (the backend outside `SUMMER_ENV=production`) makes every page `noindex, nofollow`, known or not. The `X-Robots-Tag` response header follows the same value, or the page's `x_robots_tag`; a redirect carries none. - **404 log.** Each page view answered 404 (a GET that accepts `text/html`, from an unknown route or a page that throws a 404) is reported once with `POST /notfound`, `{"url", "referer"}` and `Authorization: Bearer `, after the response, so it never delays it. Without a token nothing is reported. -- **Root files.** With a backend that answers `GET /site` (v0.2.0), `/robots.txt`, `/sitemap.xml`, `/llms.txt` and `/sitemap-.xml` (served from the backend's `/_sitemap/files/`) are proxied to `backendBase`, and so is each one-segment `.html`, `.txt` or `.xml` file the site answer lists in `root_files`: a verification file (`/verification/`) or the IndexNow key file (`/_sitemap/indexnow/.txt`). The backend's status, content type and `X-Robots-Tag` are passed on, with `X-Content-Type-Options: nosniff`; a backend that does not answer or answers 5xx gives 503 with `Retry-After: 60`, so crawlers retry instead of reading a missing robots.txt as "everything allowed". Any other path passes through to the app. The site answer is cached for 300 seconds and cleared by the purge. With a backend whose `/site` answers 404 (before v0.2.0, or no SEO plugin), robots.txt and the sitemaps are not proxied (route them to the backend in the site's proxy) and verification files keep the v0.1.1 lookup: a request for a single-segment `.html`, `.txt` or `.xml` file (`/google0123.html`, `/BingSiteAuth.xml`) is looked up under `GET /verification/` and served with `X-Content-Type-Options: nosniff`, a sandboxing CSP and `X-Robots-Tag: noindex` when the plugin has it; otherwise it falls through to the app's routes. `robots.txt`, `sitemap.xml`, `sitemap.xsl`, `rss.xml`, `feed.xml`, `atom.xml` and `llms.txt` are never looked up that way. `rootFiles: false` turns both off. +- **Root files.** With a backend that answers `GET /site` (v0.2.0), `/robots.txt`, `/sitemap.xml`, `/llms.txt` and `/sitemap-.xml` (served from the backend's `/_sitemap/files/`) are proxied to `backendBase`, and so is each one-segment `.html`, `.txt` or `.xml` file the site answer lists in `root_files`: a verification file (`/verification/`) or the IndexNow key file (`/_sitemap/indexnow/.txt`). The backend's status, content type and `X-Robots-Tag` are passed on, with `X-Content-Type-Options: nosniff`; a backend that does not answer or answers 5xx gives 503 with `Retry-After: 60`, so crawlers retry instead of reading a missing robots.txt as "everything allowed". Any other path passes through to the app. The site answer is cached for 300 seconds and cleared by the purge; while the backend is down the last one seen is used. With a backend whose `/site` answers 404 (before v0.2.0, or no SEO plugin), or one not reached since the server started, robots.txt and the sitemaps are not proxied (route them to the backend in the site's proxy) and verification files keep the v0.1.1 lookup: a request for a single-segment `.html`, `.txt` or `.xml` file (`/google0123.html`, `/BingSiteAuth.xml`) is looked up under `GET /verification/` and served with `X-Content-Type-Options: nosniff`, a sandboxing CSP and `X-Robots-Tag: noindex` when the plugin has it; otherwise it falls through to the app's routes. `robots.txt`, `sitemap.xml`, `sitemap.xsl`, `rss.xml`, `feed.xml`, `atom.xml` and `llms.txt` are never looked up that way. `rootFiles: false` turns both off. - **Site cache purge (v0.2.0).** `POST /api/__site__/cache/purge` (moved here from vue-journal-layer, vue-menu-layer and vue-faq-layer, same contract) removes every Nitro route-cache entry and empties the layer's resolve and site caches, so an admin save shows on the next page view. The backend calls it on loopback after the save commits, with `Authorization: Bearer ` and `{"kind","slug"}`; the answer is `{"purged": }`. It answers 404 without a token of at least 32 characters and for any proxied request (X-Forwarded-For, X-Real-IP or Forwarded) unless `cachePurge.allowProxied` is true, and 401 for a missing or wrong bearer. `/api/__site__/**` is never route-cached. While another layer still ships its own copy of the route and that copy answers, `server/plugins/seo-cache-purge.ts` empties the SEO caches after any 200 purge (also after the journal's deprecated `/api/__journal__/cache/purge`). - **Visitor address.** The server calls forward the visitor's `X-Forwarded-For` plus the address of the peer that reached Nuxt, also on route-cached (`swr`) renders, so the plugin's per-IP throttles (`seo-resolve`, `seo-notfound`, 600 a minute) keep one bucket per visitor. List the Nuxt server's address in the backend's `http.trusted_proxies`. diff --git a/server/lib/site.ts b/server/lib/site.ts index b6e2bd4..9d0655d 100644 --- a/server/lib/site.ts +++ b/server/lib/site.ts @@ -9,7 +9,8 @@ // files looked up by name, robots.txt and the sitemaps left to // the site's proxy) // unknown the backend did not answer: the last ok answer is used when -// there is one; nothing is cached +// there is one, else the layer behaves as with absent (as +// v0.1.1 did with the backend down); nothing is cached // // Nitro auto-imports ($fetch, useRuntimeConfig) are used inside the // functions only, so `node --test` can import this file. @@ -78,19 +79,22 @@ export interface RootTarget { /** * Where a root file request goes on the backend, in the order to try (the * first answer that is not 404 is served), or null when the layer does not - * serve the path. robots.txt, sitemap.xml, llms.txt and sitemap-.xml - * are proxied unless the site state is absent; any other one-segment - * .html, .txt or .xml file only when the site answer lists it in - * root_files (a verification file, then the IndexNow key file). Nothing - * from the request but the matched name reaches the URL. + * serve the path. Only with a site answer (state ok, possibly the last one + * seen): robots.txt, sitemap.xml, llms.txt and sitemap-.xml always, + * any other one-segment .html, .txt or .xml file when root_files lists it + * (a verification file, then the IndexNow key file). Without one (an older + * backend, or a backend never reached since start) every path passes + * through, as in v0.1.1. Nothing from the request but the matched name + * reaches the URL. */ export function rootFileTargets(path: string, site: SeoSiteState, backendBase: string, apiBase: string): RootTarget[] | null { + if (site.state !== 'ok') return null const pathname = path.split('?')[0] ?? '' - if (FIXED.has(pathname)) return site.state === 'absent' ? null : [{ kind: 'root', url: `${backendBase}${pathname}` }] + if (FIXED.has(pathname)) return [{ kind: 'root', url: `${backendBase}${pathname}` }] const sitemap = SITEMAP_FILE.exec(pathname) - if (sitemap) return site.state === 'absent' ? null : [{ kind: 'sitemap', url: `${backendBase}/_sitemap/files/${sitemap[1]}` }] + if (sitemap) return [{ kind: 'sitemap', url: `${backendBase}/_sitemap/files/${sitemap[1]}` }] const file = ROOT_FILE.exec(pathname) - if (!file || !file[1] || site.state !== 'ok') return null + if (!file || !file[1]) return null const name = file[1] const listed = Array.isArray(site.site.root_files) ? site.site.root_files : [] if (!listed.includes(name)) return null diff --git a/server/middleware/seo-root.ts b/server/middleware/seo-root.ts index c363caa..55370b5 100644 --- a/server/middleware/seo-root.ts +++ b/server/middleware/seo-root.ts @@ -7,9 +7,12 @@ // of reading a missing robots.txt as "everything allowed". Any other path // passes through. // -// With a backend before SEO v0.2.0 (the site answer is 404) nothing is -// proxied here: robots.txt and the sitemaps stay with the site's proxy and -// verification files keep the v0.1.1 lookup (seo-verification.ts). Off with +// Only once the backend has given its site answer (SEO v0.2.0; the last +// one seen is kept while the backend is down). With a backend before v0.2.0 +// (the site answer is 404), or one not reached since the server started, +// nothing is proxied here, as in v0.1.1: robots.txt and the sitemaps stay +// with the site's proxy (or fall through to the app) and verification +// files keep the v0.1.1 lookup (seo-verification.ts). Off with // the module option seo.rootFiles false or the layer switched off. The // backend URLs come from the server-only bases and an allow-listed name only // (rootFileTargets), never from the request. diff --git a/testing/expect-seo-down.txt b/testing/expect-seo-down.txt index a3d5fc3..6c850da 100644 --- a/testing/expect-seo-down.txt +++ b/testing/expect-seo-down.txt @@ -1,9 +1,10 @@ -# The CMS unreachable (NUXT_SEO_API_BASE on a closed port): pages render -# normally with their own head (fail open); a root file is 503 with -# Retry-After, so crawlers retry; verification lookups fall through. +# The CMS unreachable since the server started (NUXT_SEO_API_BASE on a +# closed port): pages render normally with their own head (fail open) and, +# with no site answer ever seen, root files fall through to the app as in +# v0.1.1 (a backend failure after a site answer is 503, expect-seo.txt). 200 /about contains=Own about | Playground notcontains=About us | Acme noheader=X-Robots-Tag count=application/ld+json:0 200 /plain contains=Plain page | Playground -503 /robots.txt header=Retry-After: 60 -503 /sitemap-posts-1.xml header=Retry-After: 60 +404 /robots.txt noheader=Retry-After +404 /sitemap-posts-1.xml 404 /google0123.html apicalls 0 diff --git a/tests/rootFiles.test.ts b/tests/rootFiles.test.ts index 55e18ef..3d60c36 100644 --- a/tests/rootFiles.test.ts +++ b/tests/rootFiles.test.ts @@ -11,13 +11,15 @@ const ok: SeoSiteState = { state: 'ok', site: { root_files: ['google1234.html', test('rootFileTargets: robots.txt, sitemap.xml, llms.txt and sitemap files go to the backend', () => { assert.deepEqual(rootFileTargets('/robots.txt', ok, B, A), [{ kind: 'root', url: `${B}/robots.txt` }]) assert.deepEqual(rootFileTargets('/sitemap.xml?x=1', ok, B, A), [{ kind: 'root', url: `${B}/sitemap.xml` }]) - assert.deepEqual(rootFileTargets('/llms.txt', { state: 'unknown' }, B, A), [{ kind: 'root', url: `${B}/llms.txt` }]) + assert.deepEqual(rootFileTargets('/llms.txt', ok, B, A), [{ kind: 'root', url: `${B}/llms.txt` }]) assert.deepEqual(rootFileTargets('/sitemap-posts-1.xml', ok, B, A), [{ kind: 'sitemap', url: `${B}/_sitemap/files/sitemap-posts-1.xml` }]) }) -test('rootFileTargets: an absent site answer (older backend) serves none of them', () => { - for (const p of ['/robots.txt', '/sitemap.xml', '/llms.txt', '/sitemap-posts-1.xml', '/google1234.html']) { - assert.equal(rootFileTargets(p, { state: 'absent' }, B, A), null, p) +test('rootFileTargets: without a site answer (older backend, or never reached) nothing is served, as in v0.1.1', () => { + for (const state of [{ state: 'absent' }, { state: 'unknown' }] as SeoSiteState[]) { + for (const p of ['/robots.txt', '/sitemap.xml', '/llms.txt', '/sitemap-posts-1.xml', '/google1234.html']) { + assert.equal(rootFileTargets(p, state, B, A), null, `${state.state} ${p}`) + } } })