package classes import ( "strings" "testing" "git.golem15.com/golem15/sm-seo-plugin/models" ) // TestRobotsMatcherGoogleExamples runs the tester on the examples of // Google's robots.txt documentation: the longest match wins, Allow wins a // tie, * matches any text, $ anchors the end, paths are case-sensitive, // and a group naming the agent replaces the * group. func TestRobotsMatcherGoogleExamples(t *testing.T) { svc := testService(t, "production", "https://example.test") for _, tc := range []struct { name, file, agent, path string allowed bool rule string }{ {"longest match: allow /p beats disallow /", "User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, "Allow: /p"}, {"tie: allow wins", "User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, "Allow: /folder"}, {"longer wildcard disallow beats allow", "User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, "Disallow: /*.htm"}, {"$ allows the root only", "User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, "Allow: /$"}, {"$ does not allow below the root", "User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, "Disallow: /"}, {"* and $: a .gif file", "User-agent: *\nDisallow: /*.gif$\n", "Googlebot", "/files/a.gif", false, "Disallow: /*.gif$"}, {"* and $: a .gif with a query is not the end", "User-agent: *\nDisallow: /*.gif$\n", "Googlebot", "/files/a.gif?x=1", true, ""}, {"a trailing * changes nothing", "User-agent: *\nDisallow: /fish*\n", "Googlebot", "/fish.html", false, "Disallow: /fish*"}, {"paths are case-sensitive", "User-agent: *\nDisallow: /fish\n", "Googlebot", "/Fish.asp", true, ""}, {"a prefix, not a word", "User-agent: *\nDisallow: /fish\n", "Googlebot", "/catfish", true, ""}, {"a prefix covers longer paths", "User-agent: *\nDisallow: /fish\n", "Googlebot", "/fishheads/yummy.html", false, "Disallow: /fish"}, {"a directory rule", "User-agent: *\nDisallow: /fish/\n", "Googlebot", "/fish", true, ""}, {"an empty Disallow allows everything", "User-agent: *\nDisallow:\n", "Googlebot", "/anything", true, "Disallow:"}, {"the agent's group replaces *", "User-agent: *\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /\n", "Googlebot", "/x", true, "Allow: /"}, {"other agents keep the * group", "User-agent: *\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /\n", "Bingbot", "/x", false, "Disallow: /"}, {"agent names ignore case", "User-agent: googlebot\nDisallow: /private\n", "Googlebot", "/private/a", false, "Disallow: /private"}, {"no rule matches", "User-agent: *\nDisallow: /admin\n", "Googlebot", "/blog", true, ""}, } { t.Run(tc.name, func(t *testing.T) { got := svc.TestRobots(tc.file, nil, tc.agent, tc.path) if got.Allowed != tc.allowed || got.Rule != tc.rule || got.Bot || got.Agent != tc.agent || got.Staging { t.Fatalf("TestRobots(%q, %s) = %+v; want allowed %v by %q", tc.path, tc.agent, got, tc.allowed, tc.rule) } if tc.rule != "" && strings.Split(tc.file, "\n")[got.Line-1] != tc.rule { t.Fatalf("line %d is not %q", got.Line, tc.rule) } if tc.rule == "" && got.Line != 0 { t.Fatalf("line = %d without a matching rule", got.Line) } }) } } func TestRobotsBotSwitches(t *testing.T) { svc := testService(t, "production", "https://example.test") content := "User-agent: *\nDisallow: /admin\n" off := models.BotSwitches{"GPTBot": false, "ClaudeBot": true} got := svc.TestRobots(content, off, "GPTBot", "/blog") if got.Allowed || !got.Bot || got.Rule != "Disallow: /" || got.Line < generatedStart(content) { t.Fatalf("a switched-off bot = %+v", got) } if got := svc.TestRobots(content, off, "ClaudeBot", "/blog"); !got.Allowed || got.Bot { t.Fatalf("a switched-on bot = %+v", got) } if got := svc.TestRobots(content, off, "ClaudeBot", "/admin/x"); got.Allowed || got.Bot || got.Rule != "Disallow: /admin" { t.Fatalf("the editor's rule = %+v", got) } if got := testService(t, "staging", "https://example.test").TestRobots(content, nil, "Googlebot", "/blog"); !got.Staging || !got.Allowed { t.Fatalf("outside production the tester tests the production file and says so: %+v", got) } if got := testService(t, "production", "").TestRobots(content, nil, "Googlebot", "/admin"); got.Allowed { t.Fatalf("without a site URL = %+v", got) } } func TestRobotsCompose(t *testing.T) { sitemaps := []string{"https://example.test/sitemap.xml"} for _, tc := range []struct { name, content string bots models.BotSwitches sitemaps []string want string }{ {"rules, a bot and a sitemap", "User-agent: *\r\nDisallow: /admin\r\n\r\n", models.BotSwitches{"CCBot": false}, sitemaps, "User-agent: *\nDisallow: /admin\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: https://example.test/sitemap.xml\n"}, {"bots in list order", "", models.BotSwitches{"CCBot": false, "GPTBot": false, "OAI-SearchBot": true}, nil, "User-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n"}, {"sitemaps only", "", nil, []string{"https://example.test/a.xml", "https://example.test/b.xml"}, "Sitemap: https://example.test/a.xml\nSitemap: https://example.test/b.xml\n"}, {"rules only", "User-agent: *\nDisallow:", nil, nil, "User-agent: *\nDisallow:\n"}, {"nothing", " \n", nil, nil, ""}, } { t.Run(tc.name, func(t *testing.T) { if got := ComposeRobots(tc.content, tc.bots, tc.sitemaps); got != tc.want { t.Fatalf("ComposeRobots = %q\nwant %q", got, tc.want) } }) } for content, want := range map[string]int{"": 1, "a": 2, "a\nb\n\n": 3, "a\r\nb": 3} { if got := generatedStart(content); got != want { t.Errorf("generatedStart(%q) = %d, want %d", content, got, want) } } for _, bot := range Bots { if field := BotField(bot.Token); strings.ContainsAny(field, "-ABCDEFGHIJKLMNOPQRSTUVWXYZ") || !strings.HasPrefix(field, "bot_") { t.Errorf("BotField(%s) = %s", bot.Token, field) } if got, ok := BotByToken(strings.ToLower(bot.Token)); !ok || got != bot { t.Errorf("BotByToken(%s) = %+v %v", bot.Token, got, ok) } } if _, ok := BotByToken("Googlebot"); ok { t.Error("Googlebot is not an AI bot of the list") } } func TestRobotsSitemapsConfig(t *testing.T) { if got := testService(t, "production", "https://example.test").RobotsSitemaps(); len(got) != 1 || got[0] != "https://example.test/sitemap.xml" { t.Fatalf("default = %v", got) } got := testService(t, "production", "https://example.test", "SUMMER_GOLEM15__SEO__ROBOTS_SITEMAPS=/a.xml, https://cdn.example.test/b.xml").RobotsSitemaps() if strings.Join(got, " ") != "https://example.test/a.xml https://cdn.example.test/b.xml" { t.Fatalf("configured = %v", got) } }