package seo_test import ( "net/http" "net/http/httptest" "strings" "testing" "git.golem15.com/golem15/sm-seo-plugin/models" ) // robotsTxt fetches /robots.txt. func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil) req.RemoteAddr = "198.51.100.9:4000" rec := httptest.NewRecorder() e.h.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("/robots.txt = %d %s, want 200", rec.Code, rec.Body.String()) } if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { t.Fatalf("/robots.txt content type = %q, want text/plain", ct) } return rec } // TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19, // T-32-20): the stored rules, a Disallow group per switched-off AI bot and // the Sitemap line in production; the non-production lock whatever is // stored; and a 200 with the last good or the allow-all file when the // store cannot be read. func TestRobotsServed(t *testing.T) { env := bootEnv(t) if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+siteURL+"/sitemap.xml\n" { t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body) } if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`, "User-agent: *\nDisallow: /admin\n", `{"GPTBot": false, "CCBot": false, "ClaudeBot": true}`).Error; err != nil { t.Fatal(err) } rec := env.robotsTxt(t) want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + siteURL + "/sitemap.xml\n" if body := rec.Body.String(); body != want { t.Fatalf("robots.txt = %q, want %q", body, want) } if rec.Header().Get("X-Robots-Tag") != "" { t.Fatalf("production robots.txt carries X-Robots-Tag %q", rec.Header().Get("X-Robots-Tag")) } // The store fails: the last good body, still 200. if err := env.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil { t.Fatal(err) } if body := env.robotsTxt(t).Body.String(); body != want { t.Fatalf("robots.txt with the store gone = %q, want the last good body", body) } fresh := bootEnv(t) if err := fresh.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil { t.Fatal(err) } if body := fresh.robotsTxt(t).Body.String(); body != "User-agent: *\nDisallow:\n" { t.Fatalf("robots.txt with the store gone and nothing cached = %q, want allow-all", body) } // Outside production: the lock, whatever is stored. staging := bootEnv(t, "SUMMER_ENV=staging") if err := staging.db.Exec(`UPDATE golem15_seo_robots SET content = ? WHERE id = 1`, "User-agent: *\nAllow: /\n").Error; err != nil { t.Fatal(err) } rec = staging.robotsTxt(t) if rec.Body.String() != "User-agent: *\nDisallow: /\n" || rec.Header().Get("X-Robots-Tag") != "noindex, nofollow" { t.Fatalf("staging robots.txt = %q %v, want the lock with X-Robots-Tag", rec.Body.String(), rec.Header()) } } // TestRobotsTester checks the tester against Google's documented examples // (D-22, D-99): the longest match wins, Allow wins a tie, $ anchors the // end, the matching line is reported, and a switched-off bot is blocked by // its switch. func TestRobotsTester(t *testing.T) { env := bootEnv(t) svc := env.seoPlugin(t).Service() none := models.BotSwitches{} cases := []struct { content, agent, path string allowed bool line int rule string }{ {"User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, 2, "Allow: /p"}, {"User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, 2, "Allow: /folder"}, {"User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, 3, "Disallow: /*.htm"}, {"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, 2, "Allow: /$"}, {"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, 3, "Disallow: /"}, {"User-agent: *\nDisallow: /konto/\n", "Bingbot", "/blog", true, 0, ""}, {"User-agent: Googlebot\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n", "Googlebot", "/public", true, 0, ""}, } for _, c := range cases { got := svc.TestRobots(c.content, none, c.agent, c.path) if got.Allowed != c.allowed || got.Line != c.line || got.Rule != c.rule || got.Bot { t.Fatalf("TestRobots(%q, %s, %s) = %+v, want allowed=%v line=%d rule=%q", c.content, c.agent, c.path, got, c.allowed, c.line, c.rule) } } off := models.BotSwitches{"GPTBot": false} if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "GPTBot", "/blog"); got.Allowed || !got.Bot || got.Rule != "Disallow: /" { t.Fatalf("switched-off bot = %+v, want blocked by its switch", got) } if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "Googlebot", "/blog"); !got.Allowed || got.Bot { t.Fatalf("Googlebot with GPTBot off = %+v, want allowed", got) } if got := svc.TestRobots("", off, "ClaudeBot", "/"); !got.Allowed { t.Fatalf("a bot that is on = %+v, want allowed", got) } }