package seo_test import ( "bytes" "encoding/json" "fmt" "io/fs" "net/http" "os" "strings" "testing" "time" seo "git.golem15.com/golem15/sm-seo-plugin" "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" "git.golem15.com/golem15/summercms/modules/bonfire" "github.com/goccy/go-yaml" ) var ( issuesAPI = adminAPI("/golem15/seo/issues") crawlsAPI = adminAPI("/golem15/seo/crawls") ) // adminList is a list answer of the admin. type adminList struct { Data []map[string]any `json:"data"` Meta struct { Total int `json:"total"` RowStates map[string][]string `json:"row_states"` } `json:"meta"` } func (e *testEnv) adminList(t *testing.T, path, token string) adminList { t.Helper() var out adminList if err := json.Unmarshal(e.expect(t, http.StatusOK, http.MethodGet, path, token, nil).Body.Bytes(), &out); err != nil { t.Fatal(err) } return out } // TestCrawlAdmin walks Site audit and Crawls (UI-SPEC C23, C24): the // toolbar start and its refusals, the issue list (errors first, labels, // fix column, ignored rows badged, the status filter), P7, check again, // ignore and stop ignoring, the crawl list and P6 with its status callout // and the rendered comparison line, the crawl API's render answer, cancel // and the refused delete of a running crawl. func TestCrawlAdmin(t *testing.T) { env := bootCrawlEnv(t) token := env.admin(t, true) site := newFakeSite() site.sitemap("/", "/p/two-h1", "/p/no-desc", "/p/lonely") site.plant( doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/p/no-desc", "/p/gone"}}, doc{path: "/p/two-h1", title: "Two", h1: []string{"Two", "Again"}}, doc{path: "/p/no-desc", title: "No description", noDesc: true}, doc{path: "/p/lonely", title: "Lonely"}, ) site.set("/p/gone", 404, "gone") ctx := siteCtx(t, site) // Start crawl from the toolbar, then a second start refuses. rec := env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/toolbar/start_crawl", token, map[string]any{}) if !strings.Contains(rec.Body.String(), "Crawl started") { t.Fatalf("start = %s", rec.Body.String()) } rec = env.call(t, http.MethodPost, crawlsAPI+"/toolbar/start_crawl", token, map[string]any{}) if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "A crawl is already running") { t.Fatalf("second start = %d %s", rec.Code, rec.Body.String()) } if rec := env.call(t, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{}); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "crawls.already_running") { t.Fatalf("API second start = %d %s", rec.Code, rec.Body.String()) } // The queued crawl: P6 queued, render skipped line; deleting it refuses. crawls := env.adminList(t, crawlsAPI, token) if len(crawls.Data) != 1 { t.Fatalf("crawls = %v", crawls.Data) } queuedID := fmt.Sprint(crawls.Data[0]["id"]) p6 := env.expect(t, http.StatusOK, http.MethodGet, crawlsAPI+"/partials/crawl_summary?id="+queuedID, token, nil).Body.String() if !strings.Contains(p6, "Waiting for a free worker") { t.Fatalf("queued P6 = %s", p6) } if rec := env.call(t, http.MethodDelete, crawlsAPI+"/"+queuedID, token, nil); rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "A running crawl cannot be deleted") { t.Fatalf("delete queued = %d %s", rec.Code, rec.Body.String()) } if rec := env.call(t, http.MethodDelete, "/_seo/api/v1/crawls?ids="+queuedID, token, nil); rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "crawls.delete_running") { t.Fatalf("API delete queued = %d %s", rec.Code, rec.Body.String()) } var id uint fmt.Sscan(queuedID, &id) env.runCrawl(t, ctx, id) // Site audit: errors first, labels, fix column. issues := env.adminList(t, issuesAPI, token) if len(issues.Data) == 0 { t.Fatal("no issues in the Site audit list") } if first := issues.Data[0]; first["severity"] != "Error" { t.Fatalf("first row = %v, want an error", first) } var lastRank int for _, row := range issues.Data { rank := map[string]int{"Error": 1, "Warning": 2, "Notice": 3}[fmt.Sprint(row["severity"])] if rank < lastRank { t.Fatalf("rows not errors first: %v", issues.Data) } lastRank = rank } var h1ID, descID string for _, row := range issues.Data { switch { case row["kind_cell"] == "More than one H1": h1ID = fmt.Sprint(row["id"]) if row["fix_cell"] != "Site templates (code)" || row["summary"] != "2 H1 headings" || row["source"] != "Crawl" { t.Fatalf("h1 row = %v", row) } case row["kind_cell"] == "Missing meta description": descID = fmt.Sprint(row["id"]) if row["fix_cell"] != "MCP: seo_set_url_seo" { t.Fatalf("description row = %v", row) } } } if h1ID == "" || descID == "" { t.Fatalf("rows = %v", issues.Data) } // P7 of the broken link and of the orphan page. var goneID, orphanID string for _, row := range issues.Data { if row["url"] == "/p/gone" { goneID = fmt.Sprint(row["id"]) } if row["url"] == "/p/lonely" && row["kind_cell"] == "Orphan page (no internal links)" { orphanID = fmt.Sprint(row["id"]) } } if orphanID == "" { t.Fatalf("no orphan row: %v", issues.Data) } p7 := env.expect(t, http.StatusOK, http.MethodGet, issuesAPI+"/partials/issue_details?id="+goneID, token, nil).Body.String() for _, want := range []string{"What was found", "Linked from", "How to fix", "seo_create_redirect", "Open the page", "summer-callout"} { if !strings.Contains(p7, want) { t.Fatalf("P7 broken link lacks %q: %s", want, p7) } } if p7 := env.expect(t, http.StatusOK, http.MethodGet, issuesAPI+"/partials/issue_details?id="+orphanID, token, nil).Body.String(); !strings.Contains(p7, "No suggestions") { t.Fatalf("P7 orphan = %s", p7) } // Ignore, row state, status filter, stop ignoring. env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/"+h1ID+"/actions/ignore", token, map[string]any{}) issues = env.adminList(t, issuesAPI, token) if st := issues.Meta.RowStates[h1ID]; len(st) != 1 || st[0] != "disabled" { t.Fatalf("ignored row states = %v", issues.Meta.RowStates) } open := env.adminList(t, issuesAPI+"?filter[status]=open", token) for _, row := range open.Data { if fmt.Sprint(row["id"]) == h1ID { t.Fatal("an ignored issue in the open filter") } } if rec := env.call(t, http.MethodPost, issuesAPI+"/"+h1ID+"/actions/ignore", token, map[string]any{}); rec.Code != http.StatusConflict { t.Fatalf("ignore twice = %d", rec.Code) } env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/"+h1ID+"/actions/unignore", token, map[string]any{}) rec = env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/bulk/ignore", token, map[string]any{"ids": []string{h1ID, descID}}) if !strings.Contains(rec.Body.String(), `"affected":2`) { t.Fatalf("bulk ignore = %s", rec.Body.String()) } env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/"+descID+"/actions/unignore", token, map[string]any{}) // Check again: the description is fixed on the site. site.plant(doc{path: "/p/no-desc", title: "No description"}) rec = env.callCtx(t, ctx, http.MethodPost, issuesAPI+"/"+descID+"/actions/recheck", token, map[string]any{}) if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "no issues remain") { t.Fatalf("recheck = %d %s", rec.Code, rec.Body.String()) } rec = env.callCtx(t, ctx, http.MethodPost, "/_seo/api/v1/issues/"+goneID+"/recheck", token, map[string]any{}) if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"remaining":1`) || !strings.Contains(rec.Body.String(), "1 issue remains") { t.Fatalf("API recheck of a still broken link = %d %s", rec.Code, rec.Body.String()) } // Crawls: the finished crawl's P6 and the API's render answer. crawls = env.adminList(t, crawlsAPI, token) row := crawls.Data[0] if row["status"] != "Finished" || !strings.HasPrefix(fmt.Sprint(row["trigger_cell"]), "Manual: ") || row["pages_cell"] == nil { t.Fatalf("crawl row = %v", row) } p6 = env.expect(t, http.StatusOK, http.MethodGet, crawlsAPI+"/partials/crawl_summary?id="+queuedID, token, nil).Body.String() for _, want := range []string{"summer-stats", "Pages fetched", "More than one H1", "filter%5Bkind%5D=h1_multiple", "comparison with the JavaScript-rendered pages was skipped", "golem15.seo.crawl_chrome_path"} { if !strings.Contains(p6, want) { t.Fatalf("P6 lacks %q: %s", want, p6) } } got := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/crawls/"+queuedID, token, nil).Body.String() if !jsonContains(got, `"render":{"status":"skipped","pages":0,"message":"The comparison with the JavaScript-rendered pages was skipped`) || !jsonContains(got, `"kinds":[`) { t.Fatalf("GET crawl = %s", got) } // Cancel a queued crawl: it keeps its row, P6 says who cancelled it. env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/toolbar/start_crawl", token, map[string]any{}) crawls = env.adminList(t, crawlsAPI, token) nextID := fmt.Sprint(crawls.Data[0]["id"]) if nextID == queuedID { t.Fatalf("crawls not newest first: %v", crawls.Data) } env.expect(t, http.StatusOK, http.MethodPost, crawlsAPI+"/"+nextID+"/actions/cancel", token, map[string]any{}) p6 = env.expect(t, http.StatusOK, http.MethodGet, crawlsAPI+"/partials/crawl_summary?id="+nextID, token, nil).Body.String() if !strings.Contains(p6, "Cancelled by") { t.Fatalf("cancelled P6 = %s", p6) } var next uint fmt.Sscan(nextID, &next) env.runCrawl(t, ctx, next) if c := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/crawls/"+nextID, token, nil).Body.String(); !jsonContains(c, `"status":"cancelled"`) { t.Fatalf("cancelled crawl after its job = %s", c) } // The cancelled crawl is the newest finished one; the audit shows its // (empty) result, and the earlier crawl can be deleted. env.expect(t, http.StatusOK, http.MethodDelete, "/_seo/api/v1/crawls?ids="+queuedID, token, nil) if list := env.adminList(t, crawlsAPI, token); len(list.Data) != 1 { t.Fatalf("crawls after delete = %v", list.Data) } } // TestCrawlTools checks the MCP parity of the audit (D-10): the nine // tools exist, bind to their routes and methods, the reads are GET, the // delete is destructive, and MCP can start, follow and read a crawl and // ignore an issue. func TestCrawlTools(t *testing.T) { p := &seo.Plugin{} want := map[string]string{ "start_crawl": "POST /_seo/api/v1/crawls", "cancel_crawl": "POST /_seo/api/v1/crawls/{id}/cancel", "list_crawls": "GET /_seo/api/v1/crawls", "get_crawl": "GET /_seo/api/v1/crawls/{id}", "delete_crawls": "DELETE /_seo/api/v1/crawls", "list_issues": "GET /_seo/api/v1/issues", "recheck_url": "POST /_seo/api/v1/issues/{id}/recheck", "ignore_issues": "POST /_seo/api/v1/issues/ignore", "unignore_issues": "POST /_seo/api/v1/issues/unignore", } found := 0 for _, tool := range p.MCP().Tools { route, ok := want[tool.Name] if !ok { continue } found++ if got := tool.Method + " " + tool.Path; got != route { t.Errorf("tool %s binds %s, want %s", tool.Name, got, route) } if tool.Name == "delete_crawls" && (tool.Destructive == nil || !*tool.Destructive) { t.Error("delete_crawls is not destructive") } } if found != len(want) { t.Fatalf("found %d of the %d audit tools", found, len(want)) } if testing.Short() { return } env := bootCrawlEnv(t) agent := env.principal(t, env.admin(t, false, "golem15.seo.manage")) site := newFakeSite() site.sitemap("/") site.plant(doc{path: "/", title: "Home", h1: []string{"Home", "Twice"}}) started := env.mcpExpect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", agent, map[string]any{}) data, _ := started["data"].(map[string]any) if data["trigger"] != "mcp" { t.Fatalf("MCP start = %v", started) } var id uint fmt.Sscan(fmt.Sprint(data["id"]), &id) env.runCrawl(t, siteCtx(t, site), id) list := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/crawls", agent, nil) if rows, _ := list["data"].([]any); len(rows) != 1 { t.Fatalf("list_crawls = %v", list) } issues := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=h1_multiple", agent, nil) rows, _ := issues["data"].([]any) if len(rows) != 1 { t.Fatalf("list_issues = %v", issues) } issueID := rows[0].(map[string]any)["id"] env.mcpExpect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/issues/ignore", agent, map[string]any{"ids": []any{issueID}}) if open := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=h1_multiple", agent, nil); len(open["data"].([]any)) != 0 { t.Fatalf("an ignored issue is listed as open: %v", open) } if ignored := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=h1_multiple&status=ignored", agent, nil); len(ignored["data"].([]any)) != 1 { t.Fatalf("status=ignored = %v", ignored) } env.mcpExpect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/issues/unignore", agent, map[string]any{"ids": []any{issueID}}) if rec := env.mcp(t, http.MethodPost, "/_seo/api/v1/crawls/"+fmt.Sprint(id)+"/cancel", agent, map[string]any{}); rec.Code != http.StatusConflict { t.Fatalf("cancel a finished crawl = %d", rec.Code) } } // TestIssueKindLabels checks that every kind the audit stores has its // issues.kind_ label and issues.sum_ Details cell in both // phrasebooks (UI-SPEC C23). func TestIssueKindLabels(t *testing.T) { for _, locale := range []string{"en", "pl"} { raw, err := fs.ReadFile(os.DirFS("lang/"+locale), "lang.yaml") if err != nil { t.Fatal(err) } var book struct { Issues map[string]any `yaml:"issues"` } if err := yaml.Unmarshal(raw, &book); err != nil { t.Fatal(err) } for _, kind := range crawl.Kinds() { for _, key := range []string{"kind_" + kind, "sum_" + kind} { if v, ok := book.Issues[key].(string); !ok || v == "" { t.Errorf("lang/%s: issues.%s is missing", locale, key) } } } } } // stubInput is a console input with bare flags. type stubInput struct{ flags map[string]string } func (stubInput) Args() []string { return nil } func (stubInput) Argument(string) (string, bool) { return "", false } func (s stubInput) Flag(n string) (string, bool) { v, ok := s.flags[n]; return v, ok } func (stubInput) Flags(string) []string { return nil } // runCommand runs one of the SEO plugin's console commands. func (e *testEnv) runCommand(t *testing.T, name string, flags map[string]string) (string, error) { t.Helper() for _, c := range e.seoPlugin(t).Commands() { if c.Name == name { var buf bytes.Buffer err := c.Run(t.Context(), stubInput{flags: flags}, bonfire.NewOutput(strings.NewReader(""), &buf, &buf)) return buf.String(), err } } t.Fatalf("no command %s", name) return "", nil } // TestCrawlSchedule checks the Site audit settings and the nightly crawl // (UI-SPEC C20, P33-40): the tab's ranges refuse with validation.between, // generic_anchors starts with the en and pl lists and drives the // generic_anchor check, seo:crawl --scheduled does nothing while the // schedule is off and queues a crawl with trigger schedule when daily, // weekly runs only on Mondays, and crawls beyond crawl_keep are deleted // with their pages and issues. func TestCrawlSchedule(t *testing.T) { monday := time.Date(2026, 11, 2, 2, 30, 0, 0, time.UTC) for _, tc := range []struct { schedule string day time.Time want bool }{ {"off", monday, false}, {"daily", monday, true}, {"daily", monday.AddDate(0, 0, 1), true}, {"weekly", monday, true}, {"weekly", monday.AddDate(0, 0, 1), false}, {"weekly", monday.AddDate(0, 0, 7), true}, } { if got := crawl.ScheduledDue(tc.schedule, tc.day); got != tc.want { t.Errorf("ScheduledDue(%s, %s) = %v, want %v", tc.schedule, tc.day.Weekday(), got, tc.want) } } if testing.Short() { return } env := bootCrawlEnv(t) token := env.admin(t, true) settings := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/settings", token, nil).Body.String() for _, want := range []string{`"crawl_schedule":"off"`, `"crawl_max_pages":5000`, `"crawl_delay":250`, `"crawl_keep":10`, `kliknij tutaj`, `read more`} { if !jsonContains(settings, want) { t.Fatalf("settings lack %s: %s", want, settings) } } tab := "" for _, f := range env.formSchema(t, token, "golem15/seo/settings") { if f.Name == "crawl_delay" { tab = f.Tab } } if tab != "Site audit" { t.Fatalf("crawl_delay tab = %q", tab) } for field, value := range map[string]any{"crawl_delay": 50, "crawl_max_pages": 60000, "crawl_keep": 0} { rec := env.call(t, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{field: value}) if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "must be between") { t.Fatalf("%s=%v = %d %s", field, value, rec.Code, rec.Body.String()) } } env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_delay": 100, "crawl_keep": 1, "generic_anchors": "Zobacz\nclick here"}) out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}) if err != nil || !strings.Contains(out, "No crawl is due") { t.Fatalf("scheduled while off: %q %v", out, err) } var crawls int64 env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_crawls`).Scan(&crawls) if crawls != 0 { t.Fatalf("crawls while off = %d", crawls) } env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_schedule": "daily"}) site := newFakeSite() site.sitemap("/") site.plant(doc{path: "/", title: "Home", body: `

Zobacz kliknij tutaj

`}) site.plant(doc{path: "/a", title: "A"}, doc{path: "/b", title: "B"}) out, err = env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}) if err != nil || !strings.Contains(out, "Queued crawl #") { t.Fatalf("scheduled while daily: %q %v", out, err) } var first struct { ID uint Trigger string } env.db.Raw(`SELECT id, trigger FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&first) if first.Trigger != "schedule" { t.Fatalf("scheduled crawl trigger = %q", first.Trigger) } if out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}); err != nil || !strings.Contains(out, "still running") { t.Fatalf("scheduled while one is queued: %q %v", out, err) } ctx := siteCtx(t, site) env.runCrawl(t, ctx, first.ID) generic := findIssue(t, env.issues(t, token, ""), "generic_anchor", "/") if generic.Details["count"] != float64(1) || generic.Details["text"] != "Zobacz" { t.Fatalf("generic_anchor with the setting = %+v", generic) } // crawl_keep 1: the second crawl deletes the first with its pages // and issues. second := env.crawlOnce(t, token, ctx) var left []uint env.db.Raw(`SELECT id FROM golem15_seo_crawls ORDER BY id`).Scan(&left) if len(left) != 1 || left[0] != second { t.Fatalf("crawls kept = %v, want only %d", left, second) } var orphanRows int64 env.db.Raw(`SELECT (SELECT COUNT(*) FROM golem15_seo_crawl_pages WHERE crawl_id = ?) + (SELECT COUNT(*) FROM golem15_seo_issues WHERE crawl_id = ?)`, first.ID, first.ID).Scan(&orphanRows) if orphanRows != 0 { t.Fatalf("pages and issues of the deleted crawl = %d", orphanRows) } }