sm-seo-plugin/robots_test.go

package seo_test

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/models"
)

// robotsTxt fetches /robots.txt.
func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder {
	t.Helper()
	req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
	req.RemoteAddr = "198.51.100.9:4000"
	rec := httptest.NewRecorder()
	e.h.ServeHTTP(rec, req)
	if rec.Code != http.StatusOK {
		t.Fatalf("/robots.txt = %d %s, want 200", rec.Code, rec.Body.String())
	}
	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
		t.Fatalf("/robots.txt content type = %q, want text/plain", ct)
	}
	return rec
}

// sitemapIndexURL is the sitemap plugin's index robots.txt names: the
// fixture sets no sitemap site address, so it is on app.url.
const sitemapIndexURL = "https://app.example.test/sitemap.xml"

// TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19,
// T-32-20): the stored rules, a Disallow group per switched-off AI bot and
// the Sitemap line in production; the non-production lock whatever is
// stored; and a 200 with the last good or the allow-all file when the
// store cannot be read.
func TestRobotsServed(t *testing.T) {
	env := bootEnv(t)
	if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+sitemapIndexURL+"\n" {
		t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body)
	}
	if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`,
		"User-agent: *\nDisallow: /admin\n", `{"GPTBot": false, "CCBot": false, "ClaudeBot": true}`).Error; err != nil {
		t.Fatal(err)
	}
	rec := env.robotsTxt(t)
	want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + sitemapIndexURL + "\n"
	if body := rec.Body.String(); body != want {
		t.Fatalf("robots.txt = %q, want %q", body, want)
	}
	if rec.Header().Get("X-Robots-Tag") != "" {
		t.Fatalf("production robots.txt carries X-Robots-Tag %q", rec.Header().Get("X-Robots-Tag"))
	}

	// The store fails: the last good body, still 200.
	if err := env.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil {
		t.Fatal(err)
	}
	if body := env.robotsTxt(t).Body.String(); body != want {
		t.Fatalf("robots.txt with the store gone = %q, want the last good body", body)
	}
	fresh := bootEnv(t)
	if err := fresh.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil {
		t.Fatal(err)
	}
	if body := fresh.robotsTxt(t).Body.String(); body != "User-agent: *\nDisallow:\n" {
		t.Fatalf("robots.txt with the store gone and nothing cached = %q, want allow-all", body)
	}

	// Outside production: the lock, whatever is stored.
	staging := bootEnv(t, "SUMMER_ENV=staging")
	if err := staging.db.Exec(`UPDATE golem15_seo_robots SET content = ? WHERE id = 1`, "User-agent: *\nAllow: /\n").Error; err != nil {
		t.Fatal(err)
	}
	rec = staging.robotsTxt(t)
	if rec.Body.String() != "User-agent: *\nDisallow: /\n" || rec.Header().Get("X-Robots-Tag") != "noindex, nofollow" {
		t.Fatalf("staging robots.txt = %q %v, want the lock with X-Robots-Tag", rec.Body.String(), rec.Header())
	}
}

// TestRobotsTester checks the tester against Google's documented examples
// (D-22, D-99): the longest match wins, Allow wins a tie, $ anchors the
// end, the matching line is reported, and a switched-off bot is blocked by
// its switch.
func TestRobotsTester(t *testing.T) {
	env := bootEnv(t)
	svc := env.seoPlugin(t).Service()
	none := models.BotSwitches{}
	cases := []struct {
		content, agent, path string
		allowed              bool
		line                 int
		rule                 string
	}{
		{"User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, 2, "Allow: /p"},
		{"User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, 2, "Allow: /folder"},
		{"User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, 3, "Disallow: /*.htm"},
		{"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, 2, "Allow: /$"},
		{"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, 3, "Disallow: /"},
		{"User-agent: *\nDisallow: /konto/\n", "Bingbot", "/blog", true, 0, ""},
		{"User-agent: Googlebot\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n", "Googlebot", "/public", true, 0, ""},
	}
	for _, c := range cases {
		got := svc.TestRobots(c.content, none, c.agent, c.path)
		if got.Allowed != c.allowed || got.Line != c.line || got.Rule != c.rule || got.Bot {
			t.Fatalf("TestRobots(%q, %s, %s) = %+v, want allowed=%v line=%d rule=%q", c.content, c.agent, c.path, got, c.allowed, c.line, c.rule)
		}
	}
	off := models.BotSwitches{"GPTBot": false}
	if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "GPTBot", "/blog"); got.Allowed || !got.Bot || got.Rule != "Disallow: /" {
		t.Fatalf("switched-off bot = %+v, want blocked by its switch", got)
	}
	if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "Googlebot", "/blog"); !got.Allowed || got.Bot {
		t.Fatalf("Googlebot with GPTBot off = %+v, want allowed", got)
	}
	if got := svc.TestRobots("", off, "ClaudeBot", "/"); !got.Allowed {
		t.Fatalf("a bot that is on = %+v, want allowed", got)
	}
}

// TestRobotsScreen smoke-tests the robots.txt screen (UI-SPEC C10, D-22,
// D-24, D-25): the form layout, a save of the rules and a bot switch with
// history, the noindex-blocked notice, the size refusal, the tester action
// on unsaved values and the served-file partial.
func TestRobotsScreen(t *testing.T) {
	env := bootEnv(t)
	token := env.admin(t, true)
	api := adminAPI("/golem15/seo/robots")

	fields := env.formSchema(t, token, "golem15/seo/robots")
	var names []string
	for _, f := range fields {
		names = append(names, f.Name)
	}
	if strings.Join(names[:4], ",") != "staging,content,tester,served" || len(names) != 4+11+2 || names[4] != "bot_oai_searchbot" || names[8] != "bot_claude_user" || strings.Join(names[15:], ",") != "llms_enabled,llms_intro" {
		t.Fatalf("robots form fields = %v, want staging, content, tester, served, eleven bot switches, then the llms.txt switch and introduction", names)
	}

	env.expect(t, http.StatusCreated, http.MethodPost, adminAPI("/golem15/seo/rules"), token, map[string]any{"url": "/private/x", "match": "exact", "robots": "noindex,follow"})
	rec := env.expect(t, http.StatusOK, http.MethodPut, api+"/1", token, map[string]any{
		"content": "User-agent: *\r\nDisallow: /private\r\n", "bot_gptbot": false,
	})
	var notice bool
	for _, n := range saveNotices(t, rec.Body.Bytes()) {
		if n.Field == "content" && strings.Contains(n.Message, "1 noindex address is blocked") && strings.Contains(n.Message, "/private/x") {
			notice = true
		}
	}
	if !notice {
		t.Fatalf("save = %s, want robots.blocks_noindex on content", rec.Body.String())
	}
	var row struct {
		Content string
		Bots    string
	}
	if err := env.db.Raw(`SELECT content, bots::text AS bots FROM golem15_seo_robots WHERE id = 1`).Scan(&row).Error; err != nil {
		t.Fatal(err)
	}
	if row.Content != "User-agent: *\nDisallow: /private\n" || !strings.Contains(row.Bots, `"GPTBot": false`) {
		t.Fatalf("stored robots = %+v, want the rules with Unix line ends and GPTBot off", row)
	}
	if rows := env.revisions(t, "robots"); len(rows) != 1 || rows[0].Via != "admin" {
		t.Fatalf("robots history = %+v, want one admin revision", rows)
	}
	show := env.expect(t, http.StatusOK, http.MethodGet, api+"/1", token, nil).Body.String()
	if !strings.Contains(show, `"bot_gptbot":false`) || !strings.Contains(show, `"bot_ccbot":true`) {
		t.Fatalf("show = %s, want the switches", show)
	}
	served := env.expect(t, http.StatusOK, http.MethodGet, api+"/partials/robots_served?id=1", token, nil).Body.String()
	if !strings.Contains(served, "User-agent: GPTBot") || !strings.Contains(served, "Sitemap: "+sitemapIndexURL) {
		t.Fatalf("served partial = %s, want the composed file", served)
	}

	rec = env.call(t, http.MethodPut, api+"/1", token, map[string]any{"content": strings.Repeat("#", 501*1024)})
	if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "at most 500 KB") {
		t.Fatalf("oversized robots.txt = %d, want 422 robots.too_large", rec.Code)
	}

	rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/widgets/tester", token, map[string]any{
		"record_id": 1, "values": map[string]any{"content": "User-agent: *\nDisallow: /a\n", "bot_gptbot": true},
		"payload": map[string]any{"op": "test", "url": "/a/b", "agent": "Googlebot"},
	})
	if body := rec.Body.String(); !strings.Contains(body, `"allowed":false`) || !strings.Contains(body, `"line":2`) || !strings.Contains(body, `"rule":"Disallow: /a"`) {
		t.Fatalf("tester = %s, want blocked by line 2 of the unsaved text", body)
	}
	rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/widgets/tester", token, map[string]any{
		"record_id": 1, "values": map[string]any{"content": "", "bot_claude_user": false},
		"payload": map[string]any{"op": "test", "url": "/", "agent": "Claude-User"},
	})
	if body := rec.Body.String(); !strings.Contains(body, `"bot":true`) {
		t.Fatalf("tester with Claude-User off = %s, want blocked by its switch", body)
	}
	rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/widgets/tester", token, map[string]any{"record_id": 1, "payload": map[string]any{"op": "context"}})
	if body := rec.Body.String(); !strings.Contains(body, `"Claude-User"`) || !strings.Contains(body, `"*"`) || !strings.Contains(body, `"Googlebot-Image"`) {
		t.Fatalf("tester context = %s, want the crawler list", body)
	}
}

// get runs an anonymous GET.
func (e *testEnv) get(t *testing.T, path string) *httptest.ResponseRecorder {
	t.Helper()
	req := httptest.NewRequest(http.MethodGet, path, nil)
	req.RemoteAddr = "198.51.100.11:4000"
	rec := httptest.NewRecorder()
	e.h.ServeHTTP(rec, req)
	return rec
}

// TestVerificationFile checks the verification-file API (D-05, D-103,
// T-32-21): a stored file is served with its content type through
// /_seo/api/v1/verification/{name}; an unknown or malformed name is 404,
// and nothing is registered at the site root.
func TestVerificationFile(t *testing.T) {
	env := bootEnv(t)
	if err := env.db.Exec(`UPDATE golem15_seo_settings SET verify_files = ? WHERE id = 1`,
		`[{"filename":"google1234.html","body":"google-site-verification: google1234.html"},{"filename":"BingSiteAuth.xml","body":"<users><user>ABC</user></users>"}]`).Error; err != nil {
		t.Fatal(err)
	}
	rec := env.get(t, "/_seo/api/v1/verification/google1234.html")
	if rec.Code != http.StatusOK || rec.Body.String() != "google-site-verification: google1234.html" {
		t.Fatalf("verification file = %d %q, want the stored body", rec.Code, rec.Body.String())
	}
	if h := rec.Header(); h.Get("Content-Type") != "text/html; charset=utf-8" || h.Get("X-Content-Type-Options") != "nosniff" || h.Get("X-Robots-Tag") != "noindex" || !strings.Contains(h.Get("Content-Security-Policy"), "sandbox") {
		t.Fatalf("verification headers = %v", h)
	}
	if rec := env.get(t, "/_seo/api/v1/verification/BingSiteAuth.xml"); rec.Code != http.StatusOK || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/xml") {
		t.Fatalf("xml verification file = %d %v", rec.Code, rec.Header())
	}
	for _, path := range []string{
		"/_seo/api/v1/verification/missing.html",
		"/_seo/api/v1/verification/google1234.php",
		"/_seo/api/v1/verification/..%2Fgoogle1234.html",
		"/google1234.html",
	} {
		if rec := env.get(t, path); rec.Code != http.StatusNotFound {
			t.Fatalf("GET %s = %d %s, want 404", path, rec.Code, rec.Body.String())
		}
	}
}