sm-seo-plugin/robots_test.go

package seo_test

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/models"
)

// robotsTxt fetches /robots.txt.
func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder {
	t.Helper()
	req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
	req.RemoteAddr = "198.51.100.9:4000"
	rec := httptest.NewRecorder()
	e.h.ServeHTTP(rec, req)
	if rec.Code != http.StatusOK {
		t.Fatalf("/robots.txt = %d %s, want 200", rec.Code, rec.Body.String())
	}
	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
		t.Fatalf("/robots.txt content type = %q, want text/plain", ct)
	}
	return rec
}

// TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19,
// T-32-20): the stored rules, a Disallow group per switched-off AI bot and
// the Sitemap line in production; the non-production lock whatever is
// stored; and a 200 with the last good or the allow-all file when the
// store cannot be read.
func TestRobotsServed(t *testing.T) {
	env := bootEnv(t)
	if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+siteURL+"/sitemap.xml\n" {
		t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body)
	}
	if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`,
		"User-agent: *\nDisallow: /admin\n", `{"GPTBot": false, "CCBot": false, "ClaudeBot": true}`).Error; err != nil {
		t.Fatal(err)
	}
	rec := env.robotsTxt(t)
	want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + siteURL + "/sitemap.xml\n"
	if body := rec.Body.String(); body != want {
		t.Fatalf("robots.txt = %q, want %q", body, want)
	}
	if rec.Header().Get("X-Robots-Tag") != "" {
		t.Fatalf("production robots.txt carries X-Robots-Tag %q", rec.Header().Get("X-Robots-Tag"))
	}

	// The store fails: the last good body, still 200.
	if err := env.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil {
		t.Fatal(err)
	}
	if body := env.robotsTxt(t).Body.String(); body != want {
		t.Fatalf("robots.txt with the store gone = %q, want the last good body", body)
	}
	fresh := bootEnv(t)
	if err := fresh.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil {
		t.Fatal(err)
	}
	if body := fresh.robotsTxt(t).Body.String(); body != "User-agent: *\nDisallow:\n" {
		t.Fatalf("robots.txt with the store gone and nothing cached = %q, want allow-all", body)
	}

	// Outside production: the lock, whatever is stored.
	staging := bootEnv(t, "SUMMER_ENV=staging")
	if err := staging.db.Exec(`UPDATE golem15_seo_robots SET content = ? WHERE id = 1`, "User-agent: *\nAllow: /\n").Error; err != nil {
		t.Fatal(err)
	}
	rec = staging.robotsTxt(t)
	if rec.Body.String() != "User-agent: *\nDisallow: /\n" || rec.Header().Get("X-Robots-Tag") != "noindex, nofollow" {
		t.Fatalf("staging robots.txt = %q %v, want the lock with X-Robots-Tag", rec.Body.String(), rec.Header())
	}
}

// TestRobotsTester checks the tester against Google's documented examples
// (D-22, D-99): the longest match wins, Allow wins a tie, $ anchors the
// end, the matching line is reported, and a switched-off bot is blocked by
// its switch.
func TestRobotsTester(t *testing.T) {
	env := bootEnv(t)
	svc := env.seoPlugin(t).Service()
	none := models.BotSwitches{}
	cases := []struct {
		content, agent, path string
		allowed              bool
		line                 int
		rule                 string
	}{
		{"User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, 2, "Allow: /p"},
		{"User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, 2, "Allow: /folder"},
		{"User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, 3, "Disallow: /*.htm"},
		{"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, 2, "Allow: /$"},
		{"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, 3, "Disallow: /"},
		{"User-agent: *\nDisallow: /konto/\n", "Bingbot", "/blog", true, 0, ""},
		{"User-agent: Googlebot\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n", "Googlebot", "/public", true, 0, ""},
	}
	for _, c := range cases {
		got := svc.TestRobots(c.content, none, c.agent, c.path)
		if got.Allowed != c.allowed || got.Line != c.line || got.Rule != c.rule || got.Bot {
			t.Fatalf("TestRobots(%q, %s, %s) = %+v, want allowed=%v line=%d rule=%q", c.content, c.agent, c.path, got, c.allowed, c.line, c.rule)
		}
	}
	off := models.BotSwitches{"GPTBot": false}
	if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "GPTBot", "/blog"); got.Allowed || !got.Bot || got.Rule != "Disallow: /" {
		t.Fatalf("switched-off bot = %+v, want blocked by its switch", got)
	}
	if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "Googlebot", "/blog"); !got.Allowed || got.Bot {
		t.Fatalf("Googlebot with GPTBot off = %+v, want allowed", got)
	}
	if got := svc.TestRobots("", off, "ClaudeBot", "/"); !got.Allowed {
		t.Fatalf("a bot that is on = %+v, want allowed", got)
	}
}