sm-seo-plugin/sitemap_filter_test.go

package seo_test

import (
	"encoding/json"
	"net/http"
	"strings"
	"testing"

	sitemapclasses "git.golem15.com/golem15/sm-sitemap-plugin/classes"
)

// rebuildSitemap runs the sitemap plugin's rebuild on the test app, as its
// job and command do.
func (e *testEnv) rebuildSitemap(t *testing.T) {
	t.Helper()
	if _, err := sitemapclasses.Rebuild(t.Context(), e.app); err != nil {
		t.Fatalf("sitemap rebuild: %v", err)
	}
}

// sitemapFile fetches one sitemap file by name; a missing file is "".
func (e *testEnv) sitemapFile(t *testing.T, name string) string {
	t.Helper()
	rec := e.call(t, http.MethodGet, "/_sitemap/files/"+name, "", nil)
	if rec.Code == http.StatusNotFound {
		return ""
	}
	if rec.Code != http.StatusOK {
		t.Fatalf("GET %s = %d: %s", name, rec.Code, rec.Body.String())
	}
	return rec.Body.String()
}

// sitemapReason is the excluded reason the sitemap stored for a record's
// address in a locale ("" while it is listed).
func (e *testEnv) sitemapReason(t *testing.T, typeKey, id, locale string) string {
	t.Helper()
	var rows []string
	if err := e.db.Raw(`SELECT excluded_reason FROM golem15_sitemap_urls WHERE type_key = ? AND record_id = ? AND locale = ?`, typeKey, id, locale).Scan(&rows).Error; err != nil {
		t.Fatal(err)
	}
	if len(rows) == 0 {
		t.Fatalf("no sitemap row for %s %s %s", typeKey, id, locale)
	}
	return rows[0]
}

// TestSitemapFilterTracer is the plan's tracer: an editor sets a page to
// noindex in the SEO tab, the sitemap rebuild asks the SEO plugin's
// published filter, and the page leaves the sitemap file with the reason
// noindex. Staging lists the same set as production (the non-production
// lock never excludes an address).
func TestSitemapFilterTracer(t *testing.T) {
	for _, envName := range []string{"production", "staging"} {
		t.Run(envName, func(t *testing.T) {
			env := bootEnv(t, "SUMMER_ENV="+envName)
			token := env.admin(t, true)
			first := env.createPage(t, token, map[string]any{"title": "First", "slug": "first", "published": true})
			second := env.createPage(t, token, map[string]any{"title": "Second", "slug": "second", "published": true})

			env.rebuildSitemap(t)
			file := env.sitemapFile(t, "sitemap-pages-1.xml")
			if !strings.Contains(file, ".test/pages/first</loc>") || !strings.Contains(file, ".test/pages/second</loc>") {
				t.Fatalf("before noindex the file lists both pages:\n%s", file)
			}

			env.expect(t, http.StatusOK, http.MethodPut, adminAPI("/acme/pages/pages/"+first), token, map[string]any{
				"title": "First", "slug": "first", "published": true,
				"seo_robots": map[string]any{"en": "noindex"},
			})
			env.rebuildSitemap(t)
			file = env.sitemapFile(t, "sitemap-pages-1.xml")
			if strings.Contains(file, ".test/pages/first</loc>") {
				t.Fatalf("the noindex page is still listed:\n%s", file)
			}
			if !strings.Contains(file, ".test/pages/second</loc>") {
				t.Fatalf("the indexable page left the file:\n%s", file)
			}
			if got := env.sitemapReason(t, "acme-page", first, "en"); got != "noindex" {
				t.Fatalf("excluded reason = %q, want noindex", got)
			}
			if got := env.sitemapReason(t, "acme-page", second, "en"); got != "" {
				t.Fatalf("second page reason = %q, want listed", got)
			}
			// The pl address has no noindex of its own and stays listed.
			if got := env.sitemapReason(t, "acme-page", first, "pl"); got != "" {
				t.Fatalf("pl address reason = %q, want listed", got)
			}
		})
	}
}

// seoStatus fetches the SEO tab's P1 of a fake page.
func (e *testEnv) seoStatus(t *testing.T, token, id string) string {
	t.Helper()
	return e.expect(t, http.StatusOK, http.MethodGet, adminAPI("/acme/pages/pages/partials/seo_status?id="+id), token, nil).Body.String()
}

// switchTypeOff turns a content type off in the sitemap settings.
func (e *testEnv) switchTypeOff(t *testing.T, typeKey string) {
	t.Helper()
	if err := e.db.Exec(`INSERT INTO golem15_sitemap_types (type_key, include, slug, titles, updated_at) VALUES (?, FALSE, '', '{}', NOW())
ON CONFLICT (type_key) DO UPDATE SET include = FALSE`, typeKey).Error; err != nil {
		t.Fatal(err)
	}
}

// TestFilterReasons covers the other reasons (D-95, D-24): a canonical
// pointing elsewhere, a redirected address, an address the URL rules set
// to noindex; an additional address of the sitemap settings is judged like
// any address; and P1 of the SEO tab shows status.excluded_title with the
// reason's text, also for a type switched off in the sitemap settings.
func TestFilterReasons(t *testing.T) {
	env := bootEnv(t)
	token := env.admin(t, true)
	canonical := env.createPage(t, token, map[string]any{"title": "Copy", "slug": "copy", "published": true})
	moved := env.createPage(t, token, map[string]any{"title": "Moved", "slug": "moved", "published": true})
	ruled := env.createPage(t, token, map[string]any{"title": "Ruled", "slug": "ruled", "published": true})
	plain := env.createPage(t, token, map[string]any{"title": "Plain", "slug": "plain", "published": true})
	env.expect(t, http.StatusOK, http.MethodPut, adminAPI("/acme/pages/pages/"+canonical), token, map[string]any{
		"title": "Copy", "slug": "copy", "published": true, "seo_canonical": map[string]any{"en": "/pages/plain"},
	})
	env.expect(t, http.StatusCreated, http.MethodPost, "/_seo/api/v1/redirects", token, map[string]any{"source": "/pages/moved", "target": "/pages/plain"})
	env.expect(t, http.StatusCreated, http.MethodPost, "/_seo/api/v1/rules", token, map[string]any{"url": "/pages/ruled", "match": "exact", "robots": "noindex,follow"})
	env.rebuildSitemap(t)

	for id, want := range map[string]string{canonical: "canonical", moved: "redirect", ruled: "noindex", plain: ""} {
		if got := env.sitemapReason(t, "acme-page", id, "en"); got != want {
			t.Errorf("page %s reason = %q, want %q", id, got, want)
		}
	}
	file := env.sitemapFile(t, "sitemap-pages-1.xml")
	if !strings.Contains(file, ".test/pages/plain</loc>") || strings.Contains(file, ".test/pages/copy</loc>") || strings.Contains(file, ".test/pages/moved</loc>") {
		t.Fatalf("pages file:\n%s", file)
	}

	for id, want := range map[string]string{
		canonical: "The canonical URL points to “/pages/plain”, so the sitemaps list that address instead.",
		moved:     "This address redirects to “/pages/plain”, so the sitemaps leave it out.",
		ruled:     "This page is set to noindex, so the sitemaps leave it out.",
	} {
		status := env.seoStatus(t, token, id)
		if !strings.Contains(status, "Not in the sitemaps") || !strings.Contains(status, want) {
			t.Errorf("P1 of %s = %s, want %q", id, status, want)
		}
		if !strings.Contains(status, `"attrs":{"class":"summer-callout","role":"status"}`) {
			t.Errorf("P1 of %s: the sitemap callout is not plain: %s", id, status)
		}
	}
	if status := env.seoStatus(t, token, plain); strings.Contains(status, "Not in the sitemaps") {
		t.Errorf("P1 of a listed page = %s", status)
	}
	env.switchTypeOff(t, "acme-page")
	if status := env.seoStatus(t, token, plain); !strings.Contains(status, "Its content type is turned off in the sitemap settings.") {
		t.Errorf("P1 with the type off = %s", status)
	}
}

// TestNoindexPublishedConflict covers D-12 with the sitemap (D-24): a batch
// setting noindex on an address a sitemap file lists reports
// noindex_published; on a published address the sitemaps leave out (its
// type switched off) it does not.
func TestNoindexPublishedConflict(t *testing.T) {
	env := bootEnv(t)
	token := env.admin(t, true)
	env.createPage(t, token, map[string]any{"title": "Listed", "slug": "listed", "published": true})
	env.rebuildSitemap(t)
	agent := env.principal(t, env.admin(t, false, "golem15.seo.manage"))
	batch := func() map[string]bool {
		dry := env.mcpExpect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/batch", agent, map[string]any{"dry_run": true, "changes": []any{
			map[string]any{"op": "set_seo", "url": "/pages/listed", "values": map[string]any{"robots": "noindex"}},
		}})
		return conflictKinds(dry)
	}
	if kinds := batch(); !kinds["noindex_published"] {
		t.Fatalf("conflicts = %v, want noindex_published for a listed address", kinds)
	}
	env.switchTypeOff(t, "acme-page")
	env.rebuildSitemap(t)
	if kinds := batch(); kinds["noindex_published"] {
		t.Fatalf("conflicts = %v, want none for an address no sitemap lists", kinds)
	}
}

// TestSitemapBlockedIssue covers the sitemap_blocked CMS issue (D-24): a
// listed address the production robots.txt keeps Googlebot from is an
// error pointing to seo_update_robots, with the deciding line.
func TestSitemapBlockedIssue(t *testing.T) {
	env := bootEnv(t)
	token := env.admin(t, true)
	env.createPage(t, token, map[string]any{"title": "Blocked", "slug": "x", "published": true})
	env.createPage(t, token, map[string]any{"title": "Open", "slug": "y", "published": true})
	env.rebuildSitemap(t)
	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/robots", token, map[string]any{"content": "User-agent: *\nDisallow: /pages/x\n"})

	var body struct {
		Data []struct {
			Kind     string         `json:"kind"`
			URL      string         `json:"url"`
			Tool     string         `json:"tool"`
			Severity string         `json:"severity"`
			Details  map[string]any `json:"details"`
		} `json:"data"`
	}
	raw := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=sitemap_blocked", token, nil).Body.Bytes()
	if err := json.Unmarshal(raw, &body); err != nil {
		t.Fatal(err)
	}
	if len(body.Data) != 1 {
		t.Fatalf("sitemap_blocked issues = %s, want one", raw)
	}
	got := body.Data[0]
	if got.URL != "/pages/x" || got.Tool != "seo_update_robots" || got.Severity != "error" || got.Details["line"] != float64(2) || got.Details["rule"] != "Disallow: /pages/x" {
		t.Fatalf("issue = %+v", got)
	}
}