Jakub Zych
9e45ddc848
Pure tests of the classes package (no database): the resolver's value chains with every layer present and absent, robots per directive, the staging lock and drafts; the redirect graph (chains, flattened and back-and-forth graphs, loops by exact, pattern and regex probes, the 10-hop cap); URL normalisation across every address rule; the robots grammar over every directive combination; Google's robots.txt matcher examples; templates and unknown placeholders; CSV export and import; owner resolution; and every kind of the issue report. go.mod: gorm.io/driver/postgres becomes a direct requirement (the tests' dry-run database).
127 lines
6.6 KiB
Go
127 lines
6.6 KiB
Go
package classes
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
)
|
|
|
|
// TestRobotsMatcherGoogleExamples runs the tester on the examples of
|
|
// Google's robots.txt documentation: the longest match wins, Allow wins a
|
|
// tie, * matches any text, $ anchors the end, paths are case-sensitive,
|
|
// and a group naming the agent replaces the * group.
|
|
func TestRobotsMatcherGoogleExamples(t *testing.T) {
|
|
svc := testService(t, "production", "https://example.test")
|
|
for _, tc := range []struct {
|
|
name, file, agent, path string
|
|
allowed bool
|
|
rule string
|
|
}{
|
|
{"longest match: allow /p beats disallow /", "User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, "Allow: /p"},
|
|
{"tie: allow wins", "User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, "Allow: /folder"},
|
|
{"longer wildcard disallow beats allow", "User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, "Disallow: /*.htm"},
|
|
{"$ allows the root only", "User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, "Allow: /$"},
|
|
{"$ does not allow below the root", "User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, "Disallow: /"},
|
|
{"* and $: a .gif file", "User-agent: *\nDisallow: /*.gif$\n", "Googlebot", "/files/a.gif", false, "Disallow: /*.gif$"},
|
|
{"* and $: a .gif with a query is not the end", "User-agent: *\nDisallow: /*.gif$\n", "Googlebot", "/files/a.gif?x=1", true, ""},
|
|
{"a trailing * changes nothing", "User-agent: *\nDisallow: /fish*\n", "Googlebot", "/fish.html", false, "Disallow: /fish*"},
|
|
{"paths are case-sensitive", "User-agent: *\nDisallow: /fish\n", "Googlebot", "/Fish.asp", true, ""},
|
|
{"a prefix, not a word", "User-agent: *\nDisallow: /fish\n", "Googlebot", "/catfish", true, ""},
|
|
{"a prefix covers longer paths", "User-agent: *\nDisallow: /fish\n", "Googlebot", "/fishheads/yummy.html", false, "Disallow: /fish"},
|
|
{"a directory rule", "User-agent: *\nDisallow: /fish/\n", "Googlebot", "/fish", true, ""},
|
|
{"an empty Disallow allows everything", "User-agent: *\nDisallow:\n", "Googlebot", "/anything", true, "Disallow:"},
|
|
{"the agent's group replaces *", "User-agent: *\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /\n", "Googlebot", "/x", true, "Allow: /"},
|
|
{"other agents keep the * group", "User-agent: *\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /\n", "Bingbot", "/x", false, "Disallow: /"},
|
|
{"agent names ignore case", "User-agent: googlebot\nDisallow: /private\n", "Googlebot", "/private/a", false, "Disallow: /private"},
|
|
{"no rule matches", "User-agent: *\nDisallow: /admin\n", "Googlebot", "/blog", true, ""},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := svc.TestRobots(tc.file, nil, tc.agent, tc.path)
|
|
if got.Allowed != tc.allowed || got.Rule != tc.rule || got.Bot || got.Agent != tc.agent || got.Staging {
|
|
t.Fatalf("TestRobots(%q, %s) = %+v; want allowed %v by %q", tc.path, tc.agent, got, tc.allowed, tc.rule)
|
|
}
|
|
if tc.rule != "" && strings.Split(tc.file, "\n")[got.Line-1] != tc.rule {
|
|
t.Fatalf("line %d is not %q", got.Line, tc.rule)
|
|
}
|
|
if tc.rule == "" && got.Line != 0 {
|
|
t.Fatalf("line = %d without a matching rule", got.Line)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRobotsBotSwitches(t *testing.T) {
|
|
svc := testService(t, "production", "https://example.test")
|
|
content := "User-agent: *\nDisallow: /admin\n"
|
|
off := models.BotSwitches{"GPTBot": false, "ClaudeBot": true}
|
|
|
|
got := svc.TestRobots(content, off, "GPTBot", "/blog")
|
|
if got.Allowed || !got.Bot || got.Rule != "Disallow: /" || got.Line < generatedStart(content) {
|
|
t.Fatalf("a switched-off bot = %+v", got)
|
|
}
|
|
if got := svc.TestRobots(content, off, "ClaudeBot", "/blog"); !got.Allowed || got.Bot {
|
|
t.Fatalf("a switched-on bot = %+v", got)
|
|
}
|
|
if got := svc.TestRobots(content, off, "ClaudeBot", "/admin/x"); got.Allowed || got.Bot || got.Rule != "Disallow: /admin" {
|
|
t.Fatalf("the editor's rule = %+v", got)
|
|
}
|
|
if got := testService(t, "staging", "https://example.test").TestRobots(content, nil, "Googlebot", "/blog"); !got.Staging || !got.Allowed {
|
|
t.Fatalf("outside production the tester tests the production file and says so: %+v", got)
|
|
}
|
|
if got := testService(t, "production", "").TestRobots(content, nil, "Googlebot", "/admin"); got.Allowed {
|
|
t.Fatalf("without a site URL = %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestRobotsCompose(t *testing.T) {
|
|
sitemaps := []string{"https://example.test/sitemap.xml"}
|
|
for _, tc := range []struct {
|
|
name, content string
|
|
bots models.BotSwitches
|
|
sitemaps []string
|
|
want string
|
|
}{
|
|
{"rules, a bot and a sitemap", "User-agent: *\r\nDisallow: /admin\r\n\r\n", models.BotSwitches{"CCBot": false}, sitemaps,
|
|
"User-agent: *\nDisallow: /admin\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: https://example.test/sitemap.xml\n"},
|
|
{"bots in list order", "", models.BotSwitches{"CCBot": false, "GPTBot": false, "OAI-SearchBot": true}, nil,
|
|
"User-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n"},
|
|
{"sitemaps only", "", nil, []string{"https://example.test/a.xml", "https://example.test/b.xml"},
|
|
"Sitemap: https://example.test/a.xml\nSitemap: https://example.test/b.xml\n"},
|
|
{"rules only", "User-agent: *\nDisallow:", nil, nil, "User-agent: *\nDisallow:\n"},
|
|
{"nothing", " \n", nil, nil, ""},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := ComposeRobots(tc.content, tc.bots, tc.sitemaps); got != tc.want {
|
|
t.Fatalf("ComposeRobots = %q\nwant %q", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
for content, want := range map[string]int{"": 1, "a": 2, "a\nb\n\n": 3, "a\r\nb": 3} {
|
|
if got := generatedStart(content); got != want {
|
|
t.Errorf("generatedStart(%q) = %d, want %d", content, got, want)
|
|
}
|
|
}
|
|
for _, bot := range Bots {
|
|
if field := BotField(bot.Token); strings.ContainsAny(field, "-ABCDEFGHIJKLMNOPQRSTUVWXYZ") || !strings.HasPrefix(field, "bot_") {
|
|
t.Errorf("BotField(%s) = %s", bot.Token, field)
|
|
}
|
|
if got, ok := BotByToken(strings.ToLower(bot.Token)); !ok || got != bot {
|
|
t.Errorf("BotByToken(%s) = %+v %v", bot.Token, got, ok)
|
|
}
|
|
}
|
|
if _, ok := BotByToken("Googlebot"); ok {
|
|
t.Error("Googlebot is not an AI bot of the list")
|
|
}
|
|
}
|
|
|
|
func TestRobotsSitemapsConfig(t *testing.T) {
|
|
if got := testService(t, "production", "https://example.test").RobotsSitemaps(); len(got) != 1 || got[0] != "https://example.test/sitemap.xml" {
|
|
t.Fatalf("default = %v", got)
|
|
}
|
|
got := testService(t, "production", "https://example.test", "SUMMER_GOLEM15__SEO__ROBOTS_SITEMAPS=/a.xml, https://cdn.example.test/b.xml").RobotsSitemaps()
|
|
if strings.Join(got, " ") != "https://example.test/a.xml https://cdn.example.test/b.xml" {
|
|
t.Fatalf("configured = %v", got)
|
|
}
|
|
}
|