package seo_test import ( "encoding/json" "net/http" "strings" "testing" sitemapclasses "git.golem15.com/golem15/sm-sitemap-plugin/classes" ) // rebuildSitemap runs the sitemap plugin's rebuild on the test app, as its // job and command do. func (e *testEnv) rebuildSitemap(t *testing.T) { t.Helper() if _, err := sitemapclasses.Rebuild(t.Context(), e.app); err != nil { t.Fatalf("sitemap rebuild: %v", err) } } // sitemapFile fetches one sitemap file by name; a missing file is "". func (e *testEnv) sitemapFile(t *testing.T, name string) string { t.Helper() rec := e.call(t, http.MethodGet, "/_sitemap/files/"+name, "", nil) if rec.Code == http.StatusNotFound { return "" } if rec.Code != http.StatusOK { t.Fatalf("GET %s = %d: %s", name, rec.Code, rec.Body.String()) } return rec.Body.String() } // sitemapReason is the excluded reason the sitemap stored for a record's // address in a locale ("" while it is listed). func (e *testEnv) sitemapReason(t *testing.T, typeKey, id, locale string) string { t.Helper() var rows []string if err := e.db.Raw(`SELECT excluded_reason FROM golem15_sitemap_urls WHERE type_key = ? AND record_id = ? AND locale = ?`, typeKey, id, locale).Scan(&rows).Error; err != nil { t.Fatal(err) } if len(rows) == 0 { t.Fatalf("no sitemap row for %s %s %s", typeKey, id, locale) } return rows[0] } // TestSitemapFilterTracer is the plan's tracer: an editor sets a page to // noindex in the SEO tab, the sitemap rebuild asks the SEO plugin's // published filter, and the page leaves the sitemap file with the reason // noindex. Staging lists the same set as production (the non-production // lock never excludes an address). func TestSitemapFilterTracer(t *testing.T) { for _, envName := range []string{"production", "staging"} { t.Run(envName, func(t *testing.T) { env := bootEnv(t, "SUMMER_ENV="+envName) token := env.admin(t, true) first := env.createPage(t, token, map[string]any{"title": "First", "slug": "first", "published": true}) second := env.createPage(t, token, map[string]any{"title": "Second", "slug": "second", "published": true}) env.rebuildSitemap(t) file := env.sitemapFile(t, "sitemap-pages-1.xml") if !strings.Contains(file, ".test/pages/first") || !strings.Contains(file, ".test/pages/second") { t.Fatalf("before noindex the file lists both pages:\n%s", file) } env.expect(t, http.StatusOK, http.MethodPut, adminAPI("/acme/pages/pages/"+first), token, map[string]any{ "title": "First", "slug": "first", "published": true, "seo_robots": map[string]any{"en": "noindex"}, }) env.rebuildSitemap(t) file = env.sitemapFile(t, "sitemap-pages-1.xml") if strings.Contains(file, ".test/pages/first") { t.Fatalf("the noindex page is still listed:\n%s", file) } if !strings.Contains(file, ".test/pages/second") { t.Fatalf("the indexable page left the file:\n%s", file) } if got := env.sitemapReason(t, "acme-page", first, "en"); got != "noindex" { t.Fatalf("excluded reason = %q, want noindex", got) } if got := env.sitemapReason(t, "acme-page", second, "en"); got != "" { t.Fatalf("second page reason = %q, want listed", got) } // The pl address has no noindex of its own and stays listed. if got := env.sitemapReason(t, "acme-page", first, "pl"); got != "" { t.Fatalf("pl address reason = %q, want listed", got) } }) } } // seoStatus fetches the SEO tab's P1 of a fake page. func (e *testEnv) seoStatus(t *testing.T, token, id string) string { t.Helper() return e.expect(t, http.StatusOK, http.MethodGet, adminAPI("/acme/pages/pages/partials/seo_status?id="+id), token, nil).Body.String() } // switchTypeOff turns a content type off in the sitemap settings. func (e *testEnv) switchTypeOff(t *testing.T, typeKey string) { t.Helper() if err := e.db.Exec(`INSERT INTO golem15_sitemap_types (type_key, include, slug, titles, updated_at) VALUES (?, FALSE, '', '{}', NOW()) ON CONFLICT (type_key) DO UPDATE SET include = FALSE`, typeKey).Error; err != nil { t.Fatal(err) } } // TestFilterReasons covers the other reasons (D-95, D-24): a canonical // pointing elsewhere, a redirected address, an address the URL rules set // to noindex; an additional address of the sitemap settings is judged like // any address; and P1 of the SEO tab shows status.excluded_title with the // reason's text, also for a type switched off in the sitemap settings. func TestFilterReasons(t *testing.T) { env := bootEnv(t) token := env.admin(t, true) canonical := env.createPage(t, token, map[string]any{"title": "Copy", "slug": "copy", "published": true}) moved := env.createPage(t, token, map[string]any{"title": "Moved", "slug": "moved", "published": true}) ruled := env.createPage(t, token, map[string]any{"title": "Ruled", "slug": "ruled", "published": true}) plain := env.createPage(t, token, map[string]any{"title": "Plain", "slug": "plain", "published": true}) env.expect(t, http.StatusOK, http.MethodPut, adminAPI("/acme/pages/pages/"+canonical), token, map[string]any{ "title": "Copy", "slug": "copy", "published": true, "seo_canonical": map[string]any{"en": "/pages/plain"}, }) env.expect(t, http.StatusCreated, http.MethodPost, "/_seo/api/v1/redirects", token, map[string]any{"source": "/pages/moved", "target": "/pages/plain"}) env.expect(t, http.StatusCreated, http.MethodPost, "/_seo/api/v1/rules", token, map[string]any{"url": "/pages/ruled", "match": "exact", "robots": "noindex,follow"}) env.rebuildSitemap(t) for id, want := range map[string]string{canonical: "canonical", moved: "redirect", ruled: "noindex", plain: ""} { if got := env.sitemapReason(t, "acme-page", id, "en"); got != want { t.Errorf("page %s reason = %q, want %q", id, got, want) } } file := env.sitemapFile(t, "sitemap-pages-1.xml") if !strings.Contains(file, ".test/pages/plain") || strings.Contains(file, ".test/pages/copy") || strings.Contains(file, ".test/pages/moved") { t.Fatalf("pages file:\n%s", file) } for id, want := range map[string]string{ canonical: "The canonical URL points to “/pages/plain”, so the sitemaps list that address instead.", moved: "This address redirects to “/pages/plain”, so the sitemaps leave it out.", ruled: "This page is set to noindex, so the sitemaps leave it out.", } { status := env.seoStatus(t, token, id) if !strings.Contains(status, "Not in the sitemaps") || !strings.Contains(status, want) { t.Errorf("P1 of %s = %s, want %q", id, status, want) } if !strings.Contains(status, `"attrs":{"class":"summer-callout","role":"status"}`) { t.Errorf("P1 of %s: the sitemap callout is not plain: %s", id, status) } } if status := env.seoStatus(t, token, plain); strings.Contains(status, "Not in the sitemaps") { t.Errorf("P1 of a listed page = %s", status) } env.switchTypeOff(t, "acme-page") if status := env.seoStatus(t, token, plain); !strings.Contains(status, "Its content type is turned off in the sitemap settings.") { t.Errorf("P1 with the type off = %s", status) } } // TestNoindexPublishedConflict covers D-12 with the sitemap (D-24): a batch // setting noindex on an address a sitemap file lists reports // noindex_published; on a published address the sitemaps leave out (its // type switched off) it does not. func TestNoindexPublishedConflict(t *testing.T) { env := bootEnv(t) token := env.admin(t, true) env.createPage(t, token, map[string]any{"title": "Listed", "slug": "listed", "published": true}) env.rebuildSitemap(t) agent := env.principal(t, env.admin(t, false, "golem15.seo.manage")) batch := func() map[string]bool { dry := env.mcpExpect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/batch", agent, map[string]any{"dry_run": true, "changes": []any{ map[string]any{"op": "set_seo", "url": "/pages/listed", "values": map[string]any{"robots": "noindex"}}, }}) return conflictKinds(dry) } if kinds := batch(); !kinds["noindex_published"] { t.Fatalf("conflicts = %v, want noindex_published for a listed address", kinds) } env.switchTypeOff(t, "acme-page") env.rebuildSitemap(t) if kinds := batch(); kinds["noindex_published"] { t.Fatalf("conflicts = %v, want none for an address no sitemap lists", kinds) } } // TestSitemapBlockedIssue covers the sitemap_blocked CMS issue (D-24): a // listed address the production robots.txt keeps Googlebot from is an // error pointing to seo_update_robots, with the deciding line. func TestSitemapBlockedIssue(t *testing.T) { env := bootEnv(t) token := env.admin(t, true) env.createPage(t, token, map[string]any{"title": "Blocked", "slug": "x", "published": true}) env.createPage(t, token, map[string]any{"title": "Open", "slug": "y", "published": true}) env.rebuildSitemap(t) env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/robots", token, map[string]any{"content": "User-agent: *\nDisallow: /pages/x\n"}) var body struct { Data []struct { Kind string `json:"kind"` URL string `json:"url"` Tool string `json:"tool"` Severity string `json:"severity"` Details map[string]any `json:"details"` } `json:"data"` } raw := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=sitemap_blocked", token, nil).Body.Bytes() if err := json.Unmarshal(raw, &body); err != nil { t.Fatal(err) } if len(body.Data) != 1 { t.Fatalf("sitemap_blocked issues = %s, want one", raw) } got := body.Data[0] if got.URL != "/pages/x" || got.Tool != "seo_update_robots" || got.Severity != "error" || got.Details["line"] != float64(2) || got.Details["rule"] != "Disallow: /pages/x" { t.Fatalf("issue = %+v", got) } } // routeReportBody posts a route report body as it is and decodes the 200 // answer. func (e *testEnv) routeReportBody(t *testing.T, token string, body map[string]any) map[string]any { t.Helper() rec := e.expect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/registry/routes", token, body) var out map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { t.Fatal(err) } return out } // pendingRebuild reports whether the sitemap's debounced rebuild is // requested, and clears the request as the rebuild job does. func (e *testEnv) pendingRebuild(t *testing.T) bool { t.Helper() var pending *string if err := e.db.Raw(`SELECT rebuild_pending_until::text FROM golem15_sitemap_settings WHERE id = 1`).Scan(&pending).Error; err != nil { t.Fatal(err) } if err := e.db.Exec(`UPDATE golem15_sitemap_settings SET rebuild_pending_until = NULL WHERE id = 1`).Error; err != nil { t.Fatal(err) } return pending != nil } // TestNoindexPathsTracer is plan 33-15's tracer (G1, D-24, D-33, the gap-2 // contract): the frontend's route report names /subscriptions noindex, the // sitemap is asked for a rebuild, and the rebuild leaves that Additional // address out of the XML and HTML sitemaps with the reason noindex. A // report without the key keeps the set; an empty list clears it; the same // set again asks for no rebuild. func TestNoindexPathsTracer(t *testing.T) { env := bootEnv(t, "SUMMER_GOLEM15__SEO__ROUTE_REPORT_TOKEN=report-token") if err := env.db.Exec(`UPDATE golem15_sitemap_settings SET extra_urls = '[{"url":"/subscriptions"},{"url":"/kontakt"}]', html_enabled = TRUE WHERE id = 1`).Error; err != nil { t.Fatal(err) } env.rebuildSitemap(t) listed := func() bool { t.Helper() return strings.Contains(env.sitemapFile(t, "sitemap-pages-1.xml"), ".test/subscriptions") } inHTML := func() bool { t.Helper() rec := env.call(t, http.MethodGet, "/_sitemap/api/v1/html/pages", "", nil) if rec.Code != http.StatusOK { t.Fatalf("HTML group = %d: %s", rec.Code, rec.Body.String()) } return strings.Contains(rec.Body.String(), "/subscriptions") } extraReason := func() string { t.Helper() var rows []string if err := env.db.Raw(`SELECT excluded_reason FROM golem15_sitemap_urls WHERE type_key = 'extra' AND url LIKE '%/subscriptions'`).Scan(&rows).Error; err != nil { t.Fatal(err) } if len(rows) != 1 { t.Fatalf("extra rows for /subscriptions = %v", rows) } return rows[0] } if !listed() || !inHTML() { t.Fatal("before the report the Additional address is listed") } env.pendingRebuild(t) out := env.routeReportBody(t, "report-token", map[string]any{"routes": []string{"/"}, "noindex": []string{"/subscriptions"}}) if out["noindex"] != float64(1) { t.Fatalf("answer = %v, want noindex 1", out) } if !env.pendingRebuild(t) { t.Fatal("a changed noindex set requested no sitemap rebuild") } env.rebuildSitemap(t) if listed() || inHTML() { t.Fatal("the reported noindex address is still in the sitemaps") } if got := extraReason(); got != "noindex" { t.Fatalf("excluded reason = %q, want noindex", got) } if !strings.Contains(env.sitemapFile(t, "sitemap-pages-1.xml"), ".test/kontakt") { t.Fatal("an indexable Additional address left the file") } // The same set again: nothing changed, no rebuild requested. if out := env.routeReportBody(t, "report-token", map[string]any{"routes": []string{"/"}, "noindex": []string{"/subscriptions"}}); out["noindex"] != float64(1) { t.Fatalf("same set again = %v", out) } if env.pendingRebuild(t) { t.Fatal("an unchanged noindex set requested a rebuild") } // A report without the key keeps the set. if out := env.routeReportBody(t, "report-token", map[string]any{"routes": []string{"/"}}); out["noindex"] != float64(1) { t.Fatalf("report without noindex = %v, want the stored 1", out) } env.rebuildSitemap(t) if listed() { t.Fatal("a report without the noindex key cleared the set") } // An empty list clears it, and the next rebuild lists the address again. if out := env.routeReportBody(t, "report-token", map[string]any{"routes": []string{"/"}, "noindex": []string{}}); out["noindex"] != float64(0) { t.Fatalf("empty noindex = %v", out) } if !env.pendingRebuild(t) { t.Fatal("clearing the set requested no rebuild") } env.rebuildSitemap(t) if !listed() || !inHTML() || extraReason() != "" { t.Fatal("after the empty list the Additional address is not listed again") } }