feat: site audit settings and the nightly crawl

- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages
  (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and
  generic_anchors seeded with the en and pl lists; ranges refuse with
  lagoon::validation.between on the screen and through the API
- the crawler reads its limits and generic anchors from the settings;
  crawls beyond crawl_keep are deleted with their pages and issues
- seo:crawl [--scheduled] and the nightly schedule entry at 02:30
  (P33-40): off does nothing, weekly runs on Mondays in app.timezone
- TestCrawlSchedule
This commit is contained in:
Jakub Zych
2026-11-04 15:41:54 +01:00
parent c4f0b4f94c
commit fc53d93b9d
13 changed files with 513 additions and 20 deletions

View File

@@ -1,6 +1,7 @@
package seo_test
import (
"bytes"
"encoding/json"
"fmt"
"io/fs"
@@ -8,9 +9,11 @@ import (
"os"
"strings"
"testing"
"time"
seo "git.golem15.com/golem15/sm-seo-plugin"
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
"git.golem15.com/golem15/summercms/modules/bonfire"
"github.com/goccy/go-yaml"
)
@@ -323,3 +326,127 @@ func TestIssueKindLabels(t *testing.T) {
}
}
}
// stubInput is a console input with bare flags.
type stubInput struct{ flags map[string]string }
func (stubInput) Args() []string { return nil }
func (stubInput) Argument(string) (string, bool) { return "", false }
func (s stubInput) Flag(n string) (string, bool) { v, ok := s.flags[n]; return v, ok }
func (stubInput) Flags(string) []string { return nil }
// runCommand runs one of the SEO plugin's console commands.
func (e *testEnv) runCommand(t *testing.T, name string, flags map[string]string) (string, error) {
t.Helper()
for _, c := range e.seoPlugin(t).Commands() {
if c.Name == name {
var buf bytes.Buffer
err := c.Run(t.Context(), stubInput{flags: flags}, bonfire.NewOutput(strings.NewReader(""), &buf, &buf))
return buf.String(), err
}
}
t.Fatalf("no command %s", name)
return "", nil
}
// TestCrawlSchedule checks the Site audit settings and the nightly crawl
// (UI-SPEC C20, P33-40): the tab's ranges refuse with validation.between,
// generic_anchors starts with the en and pl lists and drives the
// generic_anchor check, seo:crawl --scheduled does nothing while the
// schedule is off and queues a crawl with trigger schedule when daily,
// weekly runs only on Mondays, and crawls beyond crawl_keep are deleted
// with their pages and issues.
func TestCrawlSchedule(t *testing.T) {
monday := time.Date(2026, 11, 2, 2, 30, 0, 0, time.UTC)
for _, tc := range []struct {
schedule string
day time.Time
want bool
}{
{"off", monday, false}, {"daily", monday, true}, {"daily", monday.AddDate(0, 0, 1), true},
{"weekly", monday, true}, {"weekly", monday.AddDate(0, 0, 1), false}, {"weekly", monday.AddDate(0, 0, 7), true},
} {
if got := crawl.ScheduledDue(tc.schedule, tc.day); got != tc.want {
t.Errorf("ScheduledDue(%s, %s) = %v, want %v", tc.schedule, tc.day.Weekday(), got, tc.want)
}
}
if testing.Short() {
return
}
env := bootCrawlEnv(t)
token := env.admin(t, true)
settings := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/settings", token, nil).Body.String()
for _, want := range []string{`"crawl_schedule":"off"`, `"crawl_max_pages":5000`, `"crawl_delay":250`, `"crawl_keep":10`, `kliknij tutaj`, `read more`} {
if !jsonContains(settings, want) {
t.Fatalf("settings lack %s: %s", want, settings)
}
}
tab := ""
for _, f := range env.formSchema(t, token, "golem15/seo/settings") {
if f.Name == "crawl_delay" {
tab = f.Tab
}
}
if tab != "Site audit" {
t.Fatalf("crawl_delay tab = %q", tab)
}
for field, value := range map[string]any{"crawl_delay": 50, "crawl_max_pages": 60000, "crawl_keep": 0} {
rec := env.call(t, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{field: value})
if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "must be between") {
t.Fatalf("%s=%v = %d %s", field, value, rec.Code, rec.Body.String())
}
}
env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_delay": 100, "crawl_keep": 1, "generic_anchors": "Zobacz\nclick here"})
out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"})
if err != nil || !strings.Contains(out, "No crawl is due") {
t.Fatalf("scheduled while off: %q %v", out, err)
}
var crawls int64
env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_crawls`).Scan(&crawls)
if crawls != 0 {
t.Fatalf("crawls while off = %d", crawls)
}
env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_schedule": "daily"})
site := newFakeSite()
site.sitemap("/")
site.plant(doc{path: "/", title: "Home", body: `<p><a href="/a">Zobacz</a> <a href="/b">kliknij tutaj</a></p>`})
site.plant(doc{path: "/a", title: "A"}, doc{path: "/b", title: "B"})
out, err = env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"})
if err != nil || !strings.Contains(out, "Queued crawl #") {
t.Fatalf("scheduled while daily: %q %v", out, err)
}
var first struct {
ID uint
Trigger string
}
env.db.Raw(`SELECT id, trigger FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&first)
if first.Trigger != "schedule" {
t.Fatalf("scheduled crawl trigger = %q", first.Trigger)
}
if out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}); err != nil || !strings.Contains(out, "still running") {
t.Fatalf("scheduled while one is queued: %q %v", out, err)
}
ctx := siteCtx(t, site)
env.runCrawl(t, ctx, first.ID)
generic := findIssue(t, env.issues(t, token, ""), "generic_anchor", "/")
if generic.Details["count"] != float64(1) || generic.Details["text"] != "Zobacz" {
t.Fatalf("generic_anchor with the setting = %+v", generic)
}
// crawl_keep 1: the second crawl deletes the first with its pages
// and issues.
second := env.crawlOnce(t, token, ctx)
var left []uint
env.db.Raw(`SELECT id FROM golem15_seo_crawls ORDER BY id`).Scan(&left)
if len(left) != 1 || left[0] != second {
t.Fatalf("crawls kept = %v, want only %d", left, second)
}
var orphanRows int64
env.db.Raw(`SELECT (SELECT COUNT(*) FROM golem15_seo_crawl_pages WHERE crawl_id = ?) + (SELECT COUNT(*) FROM golem15_seo_issues WHERE crawl_id = ?)`, first.ID, first.ID).Scan(&orphanRows)
if orphanRows != 0 {
t.Fatalf("pages and issues of the deleted crawl = %d", orphanRows)
}
}