From 7a26692001db42b910a30ace18c046c555a2eacf Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 22:40:38 +0100 Subject: [PATCH] feat: the route report sends the noindex static paths (tracer) - noindexStaticRoutes: requiresAuth, noindex/none seo.robots and static pages under seo.noindexRoutes, i18n copies included, never patterns or the layer's HTML sitemap pages (G1, gap-2 contract) - #seo-routes exports noindexPaths; routeReportBody(paths, noindex) answers {routes, noindex} with the same limits and keeps the JSON body within the older plugin's 4,113,024 bytes, dropping noindex entries first - smoke grammar noindex= and lacksnoindex=; README carries the contract --- README.md | 16 ++++-- lib/routes.ts | 25 ++++++++- lib/seo-module.ts | 12 +++-- server/lib/routeReport.ts | 61 ++++++++++++++++++---- server/plugins/seo-route-report.ts | 29 ++++++----- server/seo-routes.d.ts | 2 + testing/expect-seo.txt | 6 ++- testing/smoke-ssr.mjs | 16 ++++-- tests/routes.test.ts | 82 ++++++++++++++++++++++++++++-- 9 files changed, 206 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 4ee5021..fea769b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,17 @@ The layer's only module is its own local one (`lib/seo-module.ts`); it pulls in definePageMeta({ seo: { breadcrumbs: false } }) // no visible breadcrumbs here ``` -- **Route report (v0.2.0, D-33).** At server start the layer posts its static routes (pages without parameters, without the noindex ones above, without `noindexRoutes`, and with `@nuxtjs/i18n`'s `prefix_except_default` or `prefix_and_default` without the copies of a page in the other locales, such as `/pl/contact` next to `/contact`: the CMS knows those as the page's translations, which an editor gives the page in SEO › Pages) to `POST /registry/routes` as `{"routes": [...]}` (at most 2000 paths) with `Authorization: Bearer `, once per process, after the start; a failure is tried once more after 60 s, a 404 (a backend without the registry) or a refused token is logged and not retried. Nothing is sent without a token, with `routeReport: false`, with the layer off or while prerendering. The route list is written at build time into the server template `#seo-routes`; no network call happens at build time. +- **Route report (v0.2.0, D-33).** At server start the layer posts its static routes (pages without parameters, without the noindex ones above, without `noindexRoutes`, and with `@nuxtjs/i18n`'s `prefix_except_default` or `prefix_and_default` without the copies of a page in the other locales, such as `/pl/contact` next to `/contact`: the CMS knows those as the page's translations, which an editor gives the page in SEO › Pages) to `POST /registry/routes` with `Authorization: Bearer `, once per process, after the start. The same request carries `noindex`, the static paths the build serves noindex (v0.2.0, G1): pages whose meta has `requiresAuth: true` or a `seo.robots` with `noindex` or `none` (client-only pages included), static pages under a `noindexRoutes` pattern, and the `@nuxtjs/i18n` copies of those pages as their own locale-prefixed paths (`/pl/konto`); never a pattern, never the layer's HTML sitemap pages, never a path in `routes`. The plugin leaves those paths out of its XML and HTML sitemaps: + + ``` + POST /_seo/api/v1/registry/routes + Authorization: Bearer + Content-Type: application/json + + {"routes": ["/", "/blog", "/kontakt"], "noindex": ["/konto", "/pl/konto", "/szkic"]} + ``` + + Each list holds at most 2000 paths of at most 2048 bytes. `noindex` is always sent (also `[]`), so the plugin's set follows each build; a plugin older than the key ignores it. The JSON body stays at or under 4,113,024 bytes, the most an older plugin reads: `noindex` entries are dropped from the end first, then `routes` entries. A failure is tried once more after 60 s, a 404 (a backend without the registry) or a refused token is logged and not retried. Nothing is sent without a token, with `routeReport: false`, with the layer off or while prerendering. The route list is written at build time into the server template `#seo-routes`; no network call happens at build time. - **Drift check (v0.2.0, D-20c).** At server start the layer compares the site answer's content-type URL patterns (`routes`), pagination form, parameter and segments and HTML sitemap paths with its build-time routes and the `pagination` and `htmlSitemap` options, and logs one `[vue-seo-layer] drift: …` warning per mismatch. A CMS pattern counts as served by a route of the same shape or a catch-all under a static prefix; a catch-all at the root does not. The check never blocks or fails the start, and a backend without a site answer is skipped. - **Breadcrumbs, pagination and the HTML sitemap (v0.2.0).** See Components. - **Visitor address.** The server calls forward the visitor's `X-Forwarded-For` plus the address of the peer that reached Nuxt, also on route-cached (`swr`) renders, so the plugin's per-IP throttles (`seo-resolve`, `seo-notfound`, 600 a minute) keep one bucket per visitor. List the Nuxt server's address in the backend's `http.trusted_proxies`. @@ -58,7 +68,7 @@ Module options under the `seo` key of the app's `nuxt.config.ts` (v0.2.0). They | `resolveFiles` | `['pdf']` | File extensions the Nitro middleware still resolves. | | `noindexRoutes` | `[]` | Route-rule patterns the app declares noindex: served with `X-Robots-Tag: noindex, nofollow` and a robots meta tag, and allowed to render in the browser only (build guard). | | `indexable` | `true` | `false` for a site that is never indexed: the build guard checks nothing. | -| `routeReport` | `true` | Report the static routes to the plugin's page registry at server start (needs `seo.routeReportToken` or `seo.notfoundToken`). | +| `routeReport` | `true` | Report the static routes to the plugin's page registry at server start, with the static paths served noindex in the same request's `noindex` list (needs `seo.routeReportToken` or `seo.notfoundToken`). | | `pagination` | `{ form: 'query', param: 'page', segments: { pl: 'strona', en: 'page' } }` | The pagination URL form the site is built for: `query` (`/blog?page=2`) or `path` (`/blog/strona/2`, the segment per locale). The resolve and site answers win at run time; the drift check logs a difference. | | `htmlSitemap` | `{ paths: { pl: '/mapa-strony', en: '/sitemap' } }` | The HTML sitemap pages by locale (full paths, overview at the path, groups at `/`); `false` registers none. A path an app page already has stays the app's. An app without pages of its own (only `app.vue`) sets `false`: the sitemap pages would be its only routes, which turns Nuxt's router on and makes every other path answer 404. | @@ -129,4 +139,4 @@ pnpm purge:check # testing/purge-check.mjs and purge-token-check.mjs pnpm guard:check # testing/guard-check.mjs: builds the fixture apps under testing/guard ``` -`testing/fixture-api.mjs` stands in for the plugin's public API and the Go binary's root files: resolve answers from `testing/fixtures/seo/resolve.json` and `testing/fixtures/resolve.json`, the site answer from `testing/fixtures/seo/site.json` (or another file with `--site-file`), the 404 report and the route report with `FIXTURE_NOTFOUND_TOKEN`, the HTML sitemap from `testing/fixtures/seo/html.json`, verification files, root files and the IndexNow key under `testing/fixtures/seo/`. `testing/smoke-ssr.mjs` checks status, body, content type and headers per line (`contains=`, `notcontains=`, `count=TEXT:N`, `type=`, `header=Name: value`, `noheader=`, `reqheader=`, each repeatable), `report` and `noreport` lines for the 404 reports, `routereport N [has=PATH] [lacks=PATH]` for the route report, `log TEXT:N` for the server's output, and `resolves PATH N` and `apicalls N` lines for the calls the API saw; `--no-site` makes the API answer `/site` with 404, as an older backend does. `pnpm smoke:ssr` runs `testing/expect.txt` (the v0.1.1 behaviour, against a backend without `/site`), `testing/expect-seo.txt`, `testing/expect-seo-off.txt` (with `NUXT_PUBLIC_SEO_CMS=false` and with `NUXT_PUBLIC_SEO_ENABLED=false`), `testing/expect-seo-down.txt` (the backend unreachable), `testing/expect-seo-notoken.txt` and `testing/expect-seo-drift.txt` (a site answer that differs from the build, with `NUXT_PUBLIC_SEO_ROUTE_REPORT=false`). `testing/guard-check.mjs` builds four fixture apps that extend the layer and expects the build guard to fail `indexable-ssr-false` (a Nuxt and a Nitro route rule), `client-page` (a `.client.vue` page) and `global-ssr-false`, and to pass `noindex-ok`, whose server then answers the declared route with `X-Robots-Tag: noindex, nofollow`. `testing/purge-check.mjs` proves on the route-cached playground (`/blog/**` swr) that a purge shows a changed CMS answer at once and that the route refuses unauthenticated, wrong-token and proxied requests; `testing/purge-token-check.mjs` that the purge token never reaches a page or the log. +`testing/fixture-api.mjs` stands in for the plugin's public API and the Go binary's root files: resolve answers from `testing/fixtures/seo/resolve.json` and `testing/fixtures/resolve.json`, the site answer from `testing/fixtures/seo/site.json` (or another file with `--site-file`), the 404 report and the route report with `FIXTURE_NOTFOUND_TOKEN`, the HTML sitemap from `testing/fixtures/seo/html.json`, verification files, root files and the IndexNow key under `testing/fixtures/seo/`. `testing/smoke-ssr.mjs` checks status, body, content type and headers per line (`contains=`, `notcontains=`, `count=TEXT:N`, `type=`, `header=Name: value`, `noheader=`, `reqheader=`, each repeatable), `report` and `noreport` lines for the 404 reports, `routereport N [has=PATH] [lacks=PATH] [noindex=PATH] [lacksnoindex=PATH]` for the route report, `log TEXT:N` for the server's output, and `resolves PATH N` and `apicalls N` lines for the calls the API saw; `--no-site` makes the API answer `/site` with 404, as an older backend does. `pnpm smoke:ssr` runs `testing/expect.txt` (the v0.1.1 behaviour, against a backend without `/site`), `testing/expect-seo.txt`, `testing/expect-seo-off.txt` (with `NUXT_PUBLIC_SEO_CMS=false` and with `NUXT_PUBLIC_SEO_ENABLED=false`), `testing/expect-seo-down.txt` (the backend unreachable), `testing/expect-seo-notoken.txt` and `testing/expect-seo-drift.txt` (a site answer that differs from the build, with `NUXT_PUBLIC_SEO_ROUTE_REPORT=false`). `testing/guard-check.mjs` builds four fixture apps that extend the layer and expects the build guard to fail `indexable-ssr-false` (a Nuxt and a Nitro route rule), `client-page` (a `.client.vue` page) and `global-ssr-false`, and to pass `noindex-ok`, whose server then answers the declared route with `X-Robots-Tag: noindex, nofollow`. `testing/purge-check.mjs` proves on the route-cached playground (`/blog/**` swr) that a purge shows a changed CMS answer at once and that the route refuses unauthenticated, wrong-token and proxied requests; `testing/purge-token-check.mjs` that the purge token never reaches a page or the log. diff --git a/lib/routes.ts b/lib/routes.ts index 97ca3f4..fdd6ff3 100644 --- a/lib/routes.ts +++ b/lib/routes.ts @@ -83,6 +83,28 @@ export function staticRoutes(pages: readonly RoutePage[], noindexRoutes: readonl return [...out].sort() } +/** + * The paths without parameters the build serves noindex, sorted (G1, the + * route report's `noindex` list): pages whose meta keeps them out of the + * index (requiresAuth, a noindex or none seo.robots) and static pages under + * seo.noindexRoutes. Client-only pages count (they are noindex by the + * guard's rule) and so do the @nuxtjs/i18n copies in other locales, which + * are listed as their own locale-prefixed paths ('/pl/konto'). Never a + * pattern, never a path with a parameter or a catch-all, never one of the + * layer's HTML sitemap pages, and never a path staticRoutes lists. + */ +export function noindexStaticRoutes(pages: readonly RoutePage[], noindexRoutes: readonly string[]): string[] { + const out = new Set() + for (const page of flattenPages(pages) as Array) { + if (Array.isArray(page.children) && page.children.some((c) => c.path === '')) continue + const path = page.fullPath + if (path.includes(':') || path.includes('*')) continue + if (isLayerPage(page.name)) continue + if (robotsNoindex(pageMetaRobots(page.meta)) || isNoindex(path, noindexRoutes)) out.add(path) + } + return [...out].sort() +} + /** Every page's route pattern ('/posts/:slug', '/**' for a catch-all), sorted. */ export function routePatterns(pages: readonly RoutePage[]): RoutePattern[] { const seen = new Map() @@ -94,11 +116,12 @@ export function routePatterns(pages: readonly RoutePage[]): RoutePattern[] { } /** The body of the server template #seo-routes. */ -export function routesTemplate(staticPaths: readonly string[], patterns: readonly RoutePattern[]): string { +export function routesTemplate(staticPaths: readonly string[], patterns: readonly RoutePattern[], noindexPaths: readonly string[] = []): string { return [ '// Generated by vue-seo-layer (lib/routes.ts) from the resolved pages.', `export const staticRoutes = ${JSON.stringify(staticPaths)}`, `export const routePatterns = ${JSON.stringify(patterns)}`, + `export const noindexPaths = ${JSON.stringify(noindexPaths)}`, '', ].join('\n') } diff --git a/lib/seo-module.ts b/lib/seo-module.ts index 335aac6..ff9eee1 100644 --- a/lib/seo-module.ts +++ b/lib/seo-module.ts @@ -12,7 +12,7 @@ import { fileURLToPath } from 'node:url' import { addServerTemplate, defineNuxtModule, extendPages, useLogger } from 'nuxt/kit' import { findOffenders, guardMessage, noindexRule, type GuardPage } from './guard.ts' import { htmlSitemapOption, paginationOption, type SeoHtmlSitemapOption, type SeoPaginationOption } from './options.ts' -import { localeCopySuffixes, routePatterns, routesTemplate, staticRoutes, type RoutePage } from './routes.ts' +import { localeCopySuffixes, noindexStaticRoutes, routePatterns, routesTemplate, staticRoutes, type RoutePage } from './routes.ts' export interface SeoModuleOptions { /** @@ -175,15 +175,19 @@ export default defineNuxtModule({ }) } - // The route artefact (#seo-routes) for the route report and the drift - // check: written from the resolved pages when Nitro bundles the + // The route artefact (#seo-routes) for the route report (indexable and + // noindex static paths) and the drift check: written from the resolved pages when Nitro bundles the // server, so no network call happens at build time. addServerTemplate({ filename: '#seo-routes', getContents: () => { const resolved = (pages ?? []) as RoutePage[] const i18n = (nuxt.options as { i18n?: unknown }).i18n - return routesTemplate(staticRoutes(resolved, noindexRoutes, localeCopySuffixes(i18n)), routePatterns(resolved)) + return routesTemplate( + staticRoutes(resolved, noindexRoutes, localeCopySuffixes(i18n)), + routePatterns(resolved), + noindexStaticRoutes(resolved, noindexRoutes), + ) }, }) nuxt.hook('nitro:build:before', (nitro) => { diff --git a/server/lib/routeReport.ts b/server/lib/routeReport.ts index 7059924..c334c19 100644 --- a/server/lib/routeReport.ts +++ b/server/lib/routeReport.ts @@ -1,22 +1,63 @@ // The route report (D-33): at server start the layer posts its static -// routes (#seo-routes, built from the pages) to the SEO plugin's -// POST /_seo/api/v1/registry/routes with the route-report token, so the -// CMS's page registry knows every frontend page. Pure helpers for the +// routes and the static paths it serves noindex (#seo-routes, built from +// the pages) to the SEO plugin's POST /_seo/api/v1/registry/routes with the +// route-report token, so the CMS's page registry knows every frontend page +// and its sitemaps leave the noindex ones out (G1). Pure helpers for the // Nitro plugin server/plugins/seo-route-report.ts, so `node --test` drives // them. -/** The plugin's limits: at most 2000 paths of at most 2048 bytes each. */ +/** The plugin's limits: at most 2000 paths of at most 2048 bytes each, per list. */ export const ROUTE_REPORT_MAX_ROUTES = 2000 export const ROUTE_REPORT_MAX_BYTES = 2048 +/** + * The most body a plugin built before the `noindex` key reads + * (sm-seo-plugin's maxRouteReportBody: MaxReportedRoutes*(MaxRouteBytes+8)+1024, + * 4,113,024 bytes); the serialised JSON body never exceeds it. + */ +export const ROUTE_REPORT_MAX_BODY = ROUTE_REPORT_MAX_ROUTES * (ROUTE_REPORT_MAX_BYTES + 8) + 1024 /** The one retry after a failed report waits this long. */ export const ROUTE_REPORT_RETRY_MS = 60_000 -/** The request body: the paths within the plugin's limits. */ -export function routeReportBody(paths: readonly string[]): { routes: string[] } { - const routes = paths - .filter((p) => typeof p === 'string' && p.startsWith('/') && Buffer.byteLength(p, 'utf8') <= ROUTE_REPORT_MAX_BYTES) - .slice(0, ROUTE_REPORT_MAX_ROUTES) - return { routes } +export interface RouteReportBody { + routes: string[] + noindex: string[] +} + +const withinLimits = (paths: readonly string[]) => + paths.filter((p) => typeof p === 'string' && p.startsWith('/') && Buffer.byteLength(p, 'utf8') <= ROUTE_REPORT_MAX_BYTES).slice(0, ROUTE_REPORT_MAX_ROUTES) + +/** The longest prefix of `paths` whose JSON array entries fit into `room` bytes (commas included). */ +function fitting(paths: readonly string[], room: number): string[] { + let used = 0 + let n = 0 + for (const p of paths) { + const size = Buffer.byteLength(JSON.stringify(p), 'utf8') + (n > 0 ? 1 : 0) + if (used + size > room) break + used += size + n++ + } + return paths.slice(0, n) +} + +/** + * The request body: both lists filtered to paths starting with / of at most + * 2048 bytes and cut to 2000 entries; a path in `routes` is never repeated in + * `noindex`. `noindex` is always sent (also []), so the plugin's set follows + * each build. The serialised body stays at or under ROUTE_REPORT_MAX_BODY: + * `noindex` entries are dropped from the end first, then `routes` entries. + */ +export function routeReportBody(paths: readonly string[], noindexPaths: readonly string[] = []): RouteReportBody { + const routes = withinLimits(paths) + const listed = new Set(routes) + let noindex = withinLimits(noindexPaths.filter((p) => !listed.has(p))) + const routesSize = Buffer.byteLength(JSON.stringify({ routes, noindex: [] }), 'utf8') + if (routesSize > ROUTE_REPORT_MAX_BODY) { + // Every noindex entry is gone and the body is still too large: cut routes. + const empty = Buffer.byteLength(JSON.stringify({ routes: [], noindex: [] }), 'utf8') + return { routes: fitting(routes, ROUTE_REPORT_MAX_BODY - empty), noindex: [] } + } + noindex = fitting(noindex, ROUTE_REPORT_MAX_BODY - routesSize) + return { routes, noindex } } /** diff --git a/server/plugins/seo-route-report.ts b/server/plugins/seo-route-report.ts index 5714c6c..5999500 100644 --- a/server/plugins/seo-route-report.ts +++ b/server/plugins/seo-route-report.ts @@ -1,18 +1,21 @@ // Reports the site's static routes to the CMS's page registry once per // server process (D-33): POST /registry/routes with -// `Authorization: Bearer ` and {routes: [path, ...]}. -// The paths come from the build (#seo-routes): pages without parameters, -// without the ones whose meta keeps them out of the index (requiresAuth, a -// noindex seo.robots) and without seo.noindexRoutes. It never delays the -// start: the report goes out after the plugin returns, a failure (no -// answer, a 5xx) is tried once more after 60 s, and a 404 (a backend without -// the registry) or a refused token is logged once and not retried. Nothing -// is sent with the layer off (seo.cms, public.seo.enabled), with -// seo.routeReport false, without a token (NUXT_SEO_ROUTE_REPORT_TOKEN, else -// NUXT_SEO_NOTFOUND_TOKEN) or while prerendering. The token stays in the +// `Authorization: Bearer ` and +// {routes: [path, ...], noindex: [path, ...]}. The paths come from the build +// (#seo-routes): `routes` are the pages without parameters that may be +// indexed; `noindex` the static paths the build serves noindex (requiresAuth, +// a noindex or none seo.robots, a static page under seo.noindexRoutes, their +// i18n copies), so the CMS leaves them out of its sitemaps (G1). `noindex` +// is always sent, also empty; a plugin older than the key ignores it. It +// never delays the start: the report goes out after the plugin returns, a +// failure (no answer, a 5xx) is tried once more after 60 s, and a 404 (a +// backend without the registry) or a refused token is logged once and not +// retried. Nothing is sent with the layer off (seo.cms, public.seo.enabled), +// with seo.routeReport false, without a token (NUXT_SEO_ROUTE_REPORT_TOKEN, +// else NUXT_SEO_NOTFOUND_TOKEN) or while prerendering. The token stays in the // server-only runtime config and goes to the configured apiBase only. -import { staticRoutes } from '#seo-routes' +import { noindexPaths, staticRoutes } from '#seo-routes' import { seoServerConfig } from '../lib/config.ts' import { ROUTE_REPORT_RETRY_MS, routeReportBody, routeReportOutcome } from '../lib/routeReport.ts' @@ -20,7 +23,7 @@ export default defineNitroPlugin(() => { if (import.meta.prerender) return const cfg = seoServerConfig(useRuntimeConfig()) if (!cfg.enabled || !cfg.routeReport || !cfg.routeReportToken) return - const body = routeReportBody(staticRoutes) + const body = routeReportBody(staticRoutes, noindexPaths) const send = async (attempt: number): Promise => { let status: number | null = null @@ -39,7 +42,7 @@ export default defineNitroPlugin(() => { reason = (err as Error)?.message ?? String(err) } const outcome = routeReportOutcome(status) - if (outcome === 'sent') console.info(`[vue-seo-layer] route report: ${body.routes.length} routes sent`) + if (outcome === 'sent') console.info(`[vue-seo-layer] route report: ${body.routes.length} routes and ${body.noindex.length} noindex paths sent`) else if (outcome === 'absent') console.info('[vue-seo-layer] route report: the CMS has no page registry (404), nothing reported') else if (outcome === 'refused') console.warn(`[vue-seo-layer] route report: the CMS refused the token (${status}); set NUXT_SEO_ROUTE_REPORT_TOKEN to golem15.seo.route_report_token`) else if (attempt === 1) { diff --git a/server/seo-routes.d.ts b/server/seo-routes.d.ts index 25985f8..24d573a 100644 --- a/server/seo-routes.d.ts +++ b/server/seo-routes.d.ts @@ -5,4 +5,6 @@ declare module '#seo-routes' { export const staticRoutes: string[] /** Every page's route pattern (the drift check). */ export const routePatterns: Array<{ pattern: string; catchAll: boolean }> + /** Static paths the build serves noindex (the route report's noindex list, G1). */ + export const noindexPaths: string[] } diff --git a/testing/expect-seo.txt b/testing/expect-seo.txt index 678eb09..77915f3 100644 --- a/testing/expect-seo.txt +++ b/testing/expect-seo.txt @@ -81,8 +81,10 @@ noreport /stary # The route report at server start: the static paths with the token, not # the signed-in page, the page with a noindex seo.robots or the declared -# noindex route. The build matches the site answer: no drift warning. -routereport 1 has=/ has=/blog has=/plain has=/about lacks=/konto lacks=/szkic lacks=/prywatne lacks=/mapa-strony +# noindex route; those three go in its noindex list (G1), never an +# indexable path or the layer's HTML sitemap page. The build matches the +# site answer: no drift warning. +routereport 1 has=/ has=/blog has=/plain has=/about lacks=/konto lacks=/szkic lacks=/prywatne lacks=/mapa-strony noindex=/konto noindex=/szkic noindex=/prywatne lacksnoindex=/ lacksnoindex=/mapa-strony log [vue-seo-layer] drift::0 # L1 breadcrumbs: a layout instance (app.vue) and a page instance render diff --git a/testing/smoke-ssr.mjs b/testing/smoke-ssr.mjs index b9669ee..447f03f 100644 --- a/testing/smoke-ssr.mjs +++ b/testing/smoke-ssr.mjs @@ -12,10 +12,12 @@ // resolves PATH N the API has seen exactly N resolve calls for PATH so far // (PATH * counts every resolve call) // apicalls N the API has seen exactly N requests so far -// routereport N [has=PATH]... [lacks=PATH]... +// routereport N [has=PATH]... [lacks=PATH]... [noindex=PATH]... [lacksnoindex=PATH]... // exactly N route reports (POST /registry/routes) reached // the API by the end of the run, each with the token; -// every has= path is in the last one, no lacks= path is +// every has= path is in the last one's routes, no lacks= +// path is; every noindex= path is in its noindex list, no +// lacksnoindex= path is // log TEXT:N the server's output holds exactly N occurrences of TEXT // (waits up to 3 s for them; N after the last colon) // A value runs until the next ` contains=`, ` notcontains=`, ` count=`, @@ -235,9 +237,9 @@ async function main() { } const routePosts = () => api.requests.filter((r) => r.method === 'POST' && r.url.endsWith('/registry/routes')) for (const line of routeReports) { - const m = /^routereport\s+(\d+)((?:\s+(?:has|lacks)=\S+)*)$/.exec(line) + const m = /^routereport\s+(\d+)((?:\s+(?:has|lacks|noindex|lacksnoindex)=\S+)*)$/.exec(line) if (!m) { - failures.push(`${line}: expected "routereport N [has=PATH]... [lacks=PATH]..."`) + failures.push(`${line}: expected "routereport N [has=PATH]... [lacks=PATH]... [noindex=PATH]... [lacksnoindex=PATH]..."`) continue } const want = Number(m[1]) @@ -246,11 +248,15 @@ async function main() { const posts = routePosts() if (posts.length !== want) failures.push(`${line}: the API saw ${posts.length} route reports`) if (posts.some((r) => r.authorization !== `Bearer ${FIXTURE_NOTFOUND_TOKEN}`)) failures.push(`${line}: a route report carried the wrong token`) - const last = /** @type {any} */ (posts[posts.length - 1]?.body)?.routes ?? [] + const lastBody = /** @type {any} */ (posts[posts.length - 1]?.body) + const last = lastBody?.routes ?? [] + const lastNoindex = lastBody?.noindex ?? [] for (const opt of (m[2] ?? '').trim().split(/\s+/).filter(Boolean)) { const [key, path] = [opt.slice(0, opt.indexOf('=')), opt.slice(opt.indexOf('=') + 1)] if (key === 'has' && !last.includes(path)) failures.push(`${line}: ${path} is not in the report ${JSON.stringify(last)}`) if (key === 'lacks' && last.includes(path)) failures.push(`${line}: ${path} is in the report ${JSON.stringify(last)}`) + if (key === 'noindex' && !lastNoindex.includes(path)) failures.push(`${line}: ${path} is not in the report's noindex list ${JSON.stringify(lastNoindex)}`) + if (key === 'lacksnoindex' && lastNoindex.includes(path)) failures.push(`${line}: ${path} is in the report's noindex list ${JSON.stringify(lastNoindex)}`) } } if (api.requests.some((r) => r.url.endsWith('/notfound') && r.authorization !== `Bearer ${FIXTURE_NOTFOUND_TOKEN}`)) { diff --git a/tests/routes.test.ts b/tests/routes.test.ts index 64fe3c8..b3f1211 100644 --- a/tests/routes.test.ts +++ b/tests/routes.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert/strict' import test from 'node:test' import { htmlSitemapOption, paginationOption } from '../lib/options.ts' -import { localeCopySuffixes, pageMetaRobots, robotsNoindex, routePatterns, routesTemplate, staticRoutes } from '../lib/routes.ts' +import { localeCopySuffixes, noindexStaticRoutes, pageMetaRobots, robotsNoindex, routePatterns, routesTemplate, staticRoutes } from '../lib/routes.ts' import { DRIFT_PREFIX, driftWarnings, routeServes } from '../server/lib/drift.ts' -import { ROUTE_REPORT_MAX_ROUTES, routeReportBody, routeReportOutcome } from '../server/lib/routeReport.ts' +import { ROUTE_REPORT_MAX_BODY, ROUTE_REPORT_MAX_ROUTES, routeReportBody, routeReportOutcome } from '../server/lib/routeReport.ts' const pages = [ { path: '/', name: 'index' }, @@ -46,15 +46,87 @@ test('routePatterns: page paths as patterns, catch-alls marked', () => { assert.equal(new Set(out.map((p) => p.pattern)).size, out.length) }) -test('routesTemplate: an ES module with both lists', () => { - const src = routesTemplate(['/'], [{ pattern: '/', catchAll: false }]) +test('routesTemplate: an ES module with the three lists', () => { + const src = routesTemplate(['/'], [{ pattern: '/', catchAll: false }], ['/konto']) assert.match(src, /export const staticRoutes = \["\/"\]/) assert.match(src, /export const routePatterns = \[\{"pattern":"\/","catchAll":false\}\]/) + assert.match(src, /export const noindexPaths = \["\/konto"\]/) + assert.match(routesTemplate([], []), /export const noindexPaths = \[\]/) }) -test('routeReportBody: within the plugin limits', () => { +test('noindexStaticRoutes: the static paths the build serves noindex (G1, gap-2 contract)', () => { + const out = noindexStaticRoutes(pages, ['/prywatne/**']) + // requiresAuth, a noindex seo.robots and a static page under a noindex pattern + assert.deepEqual(out, ['/konto', '/prywatne', '/szkic']) + // never an ordinary page, a page with parameters, a catch-all or the layer's sitemap page + for (const p of ['/', '/blog', '/otwarty', '/posts/:slug()', '/:slug(.*)*', '/mapa-strony']) assert.ok(!out.includes(p), p) + // never a path the route report lists as indexable + const indexable = staticRoutes(pages, ['/prywatne/**']) + assert.deepEqual(out.filter((p) => indexable.includes(p)), []) +}) + +test('noindexStaticRoutes: client-only noindex pages kept, patterns never sent', () => { + const list = [ + { path: '/panel', name: 'panel', mode: 'client', meta: { requiresAuth: true } }, + { path: '/p/:id', name: 'p-id', meta: { requiresAuth: true } }, + { path: '/p/**', name: 'p-all' }, + { path: '/none', name: 'none', meta: { seo: { robots: 'none' } } }, + ] + assert.deepEqual(noindexStaticRoutes(list, ['/p/**']), ['/none', '/panel']) +}) + +test('noindexStaticRoutes: the i18n copies of a noindex page are listed with their prefix, sorted', () => { + const localized = [ + { path: '/', name: 'index___en' }, + { path: '/pl', name: 'index___pl' }, + { path: '/konto', name: 'konto___en', meta: { requiresAuth: true } }, + { path: '/pl/konto', name: 'konto___pl', meta: { requiresAuth: true } }, + { path: '/szkic', name: 'szkic___en', meta: { seo: { robots: 'noindex, follow' } } }, + { path: '/pl/szkic', name: 'szkic___pl', meta: { seo: { robots: 'noindex, follow' } } }, + { path: '/sitemap', name: 'seo-sitemap-en___en' }, + ] + assert.deepEqual(noindexStaticRoutes(localized, []), ['/konto', '/pl/konto', '/pl/szkic', '/szkic']) +}) + +test('routeReportBody: within the plugin limits, both lists', () => { assert.deepEqual(routeReportBody(['/', '/blog', 'nope', '/' + 'a'.repeat(2048)]).routes, ['/', '/blog']) assert.equal(routeReportBody(Array.from({ length: 2500 }, (_, i) => `/p${i}`)).routes.length, ROUTE_REPORT_MAX_ROUTES) + assert.deepEqual(routeReportBody(['/', '/blog'], ['/konto', 'x', '/' + 'b'.repeat(2048), '/szkic']), { routes: ['/', '/blog'], noindex: ['/konto', '/szkic'] }) + assert.equal(routeReportBody([], Array.from({ length: 2500 }, (_, i) => `/n${i}`)).noindex.length, ROUTE_REPORT_MAX_ROUTES) + // always sent, also empty + assert.deepEqual(routeReportBody(['/']), { routes: ['/'], noindex: [] }) + // a path is never in both lists + assert.deepEqual(routeReportBody(['/', '/konto'], ['/konto', '/szkic']).noindex, ['/szkic']) +}) + +const bodyBytes = (b: unknown) => Buffer.byteLength(JSON.stringify(b), 'utf8') + +test('routeReportBody: the JSON body stays within the older plugin limit, noindex dropped first', () => { + assert.equal(ROUTE_REPORT_MAX_BODY, 2000 * (2048 + 8) + 1024) + assert.equal(ROUTE_REPORT_MAX_BODY, 4_113_024) + const routes = Array.from({ length: 2000 }, (_, i) => '/' + String(i).padStart(4, '0') + 'r'.repeat(2043)) + const noindex = Array.from({ length: 100 }, (_, i) => '/' + String(i).padStart(3, '0') + 'n'.repeat(196)) + assert.ok(routes.every((p) => Buffer.byteLength(p) === 2048)) + assert.ok(noindex.every((p) => Buffer.byteLength(p) === 200)) + const body = routeReportBody(routes, noindex) + assert.equal(body.routes.length, 2000) + assert.ok(body.noindex.length > 0 && body.noindex.length < 100, String(body.noindex.length)) + assert.deepEqual(body.noindex, noindex.slice(0, body.noindex.length)) + assert.ok(bodyBytes(body) <= ROUTE_REPORT_MAX_BODY) + // the next noindex entry would not have fitted + assert.ok(bodyBytes({ routes: body.routes, noindex: noindex.slice(0, body.noindex.length + 1) }) > ROUTE_REPORT_MAX_BODY) +}) + +test('routeReportBody: small lists come back whole; routes alone over the limit are cut from the end', () => { + assert.deepEqual(routeReportBody(['/a', '/b'], ['/c']), { routes: ['/a', '/b'], noindex: ['/c'] }) + // 2048 bytes of characters JSON escapes: each path serialises to about twice its size + const heavy = Array.from({ length: 2000 }, (_, i) => '/' + String(i).padStart(4, '0') + '"'.repeat(2043)) + const body = routeReportBody(heavy, ['/konto']) + assert.deepEqual(body.noindex, []) + assert.ok(body.routes.length > 0 && body.routes.length < 2000, String(body.routes.length)) + assert.deepEqual(body.routes, heavy.slice(0, body.routes.length)) + assert.ok(bodyBytes(body) <= ROUTE_REPORT_MAX_BODY) + assert.ok(bodyBytes({ routes: heavy.slice(0, body.routes.length + 1), noindex: [] }) > ROUTE_REPORT_MAX_BODY) }) test('routeReportOutcome: sent, absent, refused or one retry', () => {