package seo_test import ( "net/http" "net/http/httptest" "strings" "testing" "git.golem15.com/golem15/sm-seo-plugin/models" ) // robotsTxt fetches /robots.txt. func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil) req.RemoteAddr = "198.51.100.9:4000" rec := httptest.NewRecorder() e.h.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("/robots.txt = %d %s, want 200", rec.Code, rec.Body.String()) } if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { t.Fatalf("/robots.txt content type = %q, want text/plain", ct) } return rec } // TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19, // T-32-20): the stored rules, a Disallow group per switched-off AI bot and // the Sitemap line in production; the non-production lock whatever is // stored; and a 200 with the last good or the allow-all file when the // store cannot be read. func TestRobotsServed(t *testing.T) { env := bootEnv(t) if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+siteURL+"/sitemap.xml\n" { t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body) } if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`, "User-agent: *\nDisallow: /admin\n", `{"GPTBot": false, "CCBot": false, "ClaudeBot": true}`).Error; err != nil { t.Fatal(err) } rec := env.robotsTxt(t) want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + siteURL + "/sitemap.xml\n" if body := rec.Body.String(); body != want { t.Fatalf("robots.txt = %q, want %q", body, want) } if rec.Header().Get("X-Robots-Tag") != "" { t.Fatalf("production robots.txt carries X-Robots-Tag %q", rec.Header().Get("X-Robots-Tag")) } // The store fails: the last good body, still 200. if err := env.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil { t.Fatal(err) } if body := env.robotsTxt(t).Body.String(); body != want { t.Fatalf("robots.txt with the store gone = %q, want the last good body", body) } fresh := bootEnv(t) if err := fresh.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil { t.Fatal(err) } if body := fresh.robotsTxt(t).Body.String(); body != "User-agent: *\nDisallow:\n" { t.Fatalf("robots.txt with the store gone and nothing cached = %q, want allow-all", body) } // Outside production: the lock, whatever is stored. staging := bootEnv(t, "SUMMER_ENV=staging") if err := staging.db.Exec(`UPDATE golem15_seo_robots SET content = ? WHERE id = 1`, "User-agent: *\nAllow: /\n").Error; err != nil { t.Fatal(err) } rec = staging.robotsTxt(t) if rec.Body.String() != "User-agent: *\nDisallow: /\n" || rec.Header().Get("X-Robots-Tag") != "noindex, nofollow" { t.Fatalf("staging robots.txt = %q %v, want the lock with X-Robots-Tag", rec.Body.String(), rec.Header()) } } // TestRobotsTester checks the tester against Google's documented examples // (D-22, D-99): the longest match wins, Allow wins a tie, $ anchors the // end, the matching line is reported, and a switched-off bot is blocked by // its switch. func TestRobotsTester(t *testing.T) { env := bootEnv(t) svc := env.seoPlugin(t).Service() none := models.BotSwitches{} cases := []struct { content, agent, path string allowed bool line int rule string }{ {"User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, 2, "Allow: /p"}, {"User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, 2, "Allow: /folder"}, {"User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, 3, "Disallow: /*.htm"}, {"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, 2, "Allow: /$"}, {"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, 3, "Disallow: /"}, {"User-agent: *\nDisallow: /konto/\n", "Bingbot", "/blog", true, 0, ""}, {"User-agent: Googlebot\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n", "Googlebot", "/public", true, 0, ""}, } for _, c := range cases { got := svc.TestRobots(c.content, none, c.agent, c.path) if got.Allowed != c.allowed || got.Line != c.line || got.Rule != c.rule || got.Bot { t.Fatalf("TestRobots(%q, %s, %s) = %+v, want allowed=%v line=%d rule=%q", c.content, c.agent, c.path, got, c.allowed, c.line, c.rule) } } off := models.BotSwitches{"GPTBot": false} if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "GPTBot", "/blog"); got.Allowed || !got.Bot || got.Rule != "Disallow: /" { t.Fatalf("switched-off bot = %+v, want blocked by its switch", got) } if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "Googlebot", "/blog"); !got.Allowed || got.Bot { t.Fatalf("Googlebot with GPTBot off = %+v, want allowed", got) } if got := svc.TestRobots("", off, "ClaudeBot", "/"); !got.Allowed { t.Fatalf("a bot that is on = %+v, want allowed", got) } } // TestRobotsScreen smoke-tests the robots.txt screen (UI-SPEC C10, D-22, // D-24, D-25): the form layout, a save of the rules and a bot switch with // history, the noindex-blocked notice, the size refusal, the tester action // on unsaved values and the served-file partial. func TestRobotsScreen(t *testing.T) { env := bootEnv(t) token := env.admin(t, true) api := adminAPI("/golem15/seo/robots") fields := env.formSchema(t, token, "golem15/seo/robots") var names []string for _, f := range fields { names = append(names, f.Name) } if strings.Join(names[:4], ",") != "staging,content,tester,served" || len(names) != 4+11 || names[4] != "bot_oai_searchbot" || names[8] != "bot_claude_user" { t.Fatalf("robots form fields = %v, want staging, content, tester, served, then eleven bot switches", names) } env.expect(t, http.StatusCreated, http.MethodPost, adminAPI("/golem15/seo/rules"), token, map[string]any{"url": "/private/x", "match": "exact", "robots": "noindex,follow"}) rec := env.expect(t, http.StatusOK, http.MethodPut, api+"/1", token, map[string]any{ "content": "User-agent: *\r\nDisallow: /private\r\n", "bot_gptbot": false, }) var notice bool for _, n := range saveNotices(t, rec.Body.Bytes()) { if n.Field == "content" && strings.Contains(n.Message, "1 noindex address is blocked") && strings.Contains(n.Message, "/private/x") { notice = true } } if !notice { t.Fatalf("save = %s, want robots.blocks_noindex on content", rec.Body.String()) } var row struct { Content string Bots string } if err := env.db.Raw(`SELECT content, bots::text AS bots FROM golem15_seo_robots WHERE id = 1`).Scan(&row).Error; err != nil { t.Fatal(err) } if row.Content != "User-agent: *\nDisallow: /private\n" || !strings.Contains(row.Bots, `"GPTBot": false`) { t.Fatalf("stored robots = %+v, want the rules with Unix line ends and GPTBot off", row) } if rows := env.revisions(t, "robots"); len(rows) != 1 || rows[0].Via != "admin" { t.Fatalf("robots history = %+v, want one admin revision", rows) } show := env.expect(t, http.StatusOK, http.MethodGet, api+"/1", token, nil).Body.String() if !strings.Contains(show, `"bot_gptbot":false`) || !strings.Contains(show, `"bot_ccbot":true`) { t.Fatalf("show = %s, want the switches", show) } served := env.expect(t, http.StatusOK, http.MethodGet, api+"/partials/robots_served?id=1", token, nil).Body.String() if !strings.Contains(served, "User-agent: GPTBot") || !strings.Contains(served, "Sitemap: "+siteURL+"/sitemap.xml") { t.Fatalf("served partial = %s, want the composed file", served) } rec = env.call(t, http.MethodPut, api+"/1", token, map[string]any{"content": strings.Repeat("#", 501*1024)}) if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "at most 500 KB") { t.Fatalf("oversized robots.txt = %d, want 422 robots.too_large", rec.Code) } rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/widgets/tester", token, map[string]any{ "record_id": 1, "values": map[string]any{"content": "User-agent: *\nDisallow: /a\n", "bot_gptbot": true}, "payload": map[string]any{"op": "test", "url": "/a/b", "agent": "Googlebot"}, }) if body := rec.Body.String(); !strings.Contains(body, `"allowed":false`) || !strings.Contains(body, `"line":2`) || !strings.Contains(body, `"rule":"Disallow: /a"`) { t.Fatalf("tester = %s, want blocked by line 2 of the unsaved text", body) } rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/widgets/tester", token, map[string]any{ "record_id": 1, "values": map[string]any{"content": "", "bot_claude_user": false}, "payload": map[string]any{"op": "test", "url": "/", "agent": "Claude-User"}, }) if body := rec.Body.String(); !strings.Contains(body, `"bot":true`) { t.Fatalf("tester with Claude-User off = %s, want blocked by its switch", body) } rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/widgets/tester", token, map[string]any{"record_id": 1, "payload": map[string]any{"op": "context"}}) if body := rec.Body.String(); !strings.Contains(body, `"Claude-User"`) || !strings.Contains(body, `"*"`) || !strings.Contains(body, `"Googlebot-Image"`) { t.Fatalf("tester context = %s, want the crawler list", body) } } // get runs an anonymous GET. func (e *testEnv) get(t *testing.T, path string) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(http.MethodGet, path, nil) req.RemoteAddr = "198.51.100.11:4000" rec := httptest.NewRecorder() e.h.ServeHTTP(rec, req) return rec } // TestVerificationFile checks the verification-file API (D-05, D-103, // T-32-21): a stored file is served with its content type through // /_seo/api/v1/verification/{name}; an unknown or malformed name is 404, // and nothing is registered at the site root. func TestVerificationFile(t *testing.T) { env := bootEnv(t) if err := env.db.Exec(`UPDATE golem15_seo_settings SET verify_files = ? WHERE id = 1`, `[{"filename":"google1234.html","body":"google-site-verification: google1234.html"},{"filename":"BingSiteAuth.xml","body":"ABC"}]`).Error; err != nil { t.Fatal(err) } rec := env.get(t, "/_seo/api/v1/verification/google1234.html") if rec.Code != http.StatusOK || rec.Body.String() != "google-site-verification: google1234.html" { t.Fatalf("verification file = %d %q, want the stored body", rec.Code, rec.Body.String()) } if h := rec.Header(); h.Get("Content-Type") != "text/html; charset=utf-8" || h.Get("X-Content-Type-Options") != "nosniff" || h.Get("X-Robots-Tag") != "noindex" || !strings.Contains(h.Get("Content-Security-Policy"), "sandbox") { t.Fatalf("verification headers = %v", h) } if rec := env.get(t, "/_seo/api/v1/verification/BingSiteAuth.xml"); rec.Code != http.StatusOK || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/xml") { t.Fatalf("xml verification file = %d %v", rec.Code, rec.Header()) } for _, path := range []string{ "/_seo/api/v1/verification/missing.html", "/_seo/api/v1/verification/google1234.php", "/_seo/api/v1/verification/..%2Fgoogle1234.html", "/google1234.html", } { if rec := env.get(t, path); rec.Code != http.StatusNotFound { t.Fatalf("GET %s = %d %s, want 404", path, rec.Code, rec.Body.String()) } } }