feat: site audit settings and the nightly crawl

- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages
  (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and
  generic_anchors seeded with the en and pl lists; ranges refuse with
  lagoon::validation.between on the screen and through the API
- the crawler reads its limits and generic anchors from the settings;
  crawls beyond crawl_keep are deleted with their pages and issues
- seo:crawl [--scheduled] and the nightly schedule entry at 02:30
  (P33-40): off does nothing, weekly runs on Mondays in app.timezone
- TestCrawlSchedule

classes/crawl/job.go

@@ -115,9 +115,25 @@ type Limits struct {
 // DefaultKeep is the number of crawls kept.
 const DefaultKeep = 10
 
-// LoadLimits reads the crawl limits.
+// LoadLimits reads the crawl limits from the Site audit settings; a value
+// outside its range (written around the settings screen) falls back to the
+// default.
 func LoadLimits(db *gorm.DB) (Limits, error) {
-	return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: DefaultGenericAnchors}, nil
+	st, err := classes.LoadSettings(db)
+	if err != nil {
+		return Limits{}, err
+	}
+	out := Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: ParseAnchors(st.GenericAnchors)}
+	if st.CrawlMaxPages >= models.MinCrawlMaxPages && st.CrawlMaxPages <= models.MaxCrawlMaxPages {
+		out.MaxPages = st.CrawlMaxPages
+	}
+	if st.CrawlDelay >= models.MinCrawlDelay && st.CrawlDelay <= models.MaxCrawlDelay {
+		out.Delay = time.Duration(st.CrawlDelay) * time.Millisecond
+	}
+	if st.CrawlKeep >= models.MinCrawlKeep && st.CrawlKeep <= models.MaxCrawlKeep {
+		out.Keep = st.CrawlKeep
+	}
+	return out, nil
 }
 
 // LoadCrawl reads one crawl.
@@ -393,3 +409,16 @@ var ErrNotRunning = errCancelledCrawl
 
 // pluginLog is the crawler's logger (component seo).
 var pluginLog = diary.For("seo")
+
+// ScheduledDue reports whether the nightly entry starts a crawl on day
+// (in the app's location) for the crawl_schedule setting: daily every
+// night, weekly on Mondays, never when off (P33-40).
+func ScheduledDue(schedule string, day time.Time) bool {
+	switch schedule {
+	case models.CrawlDaily:
+		return true
+	case models.CrawlWeekly:
+		return day.Weekday() == time.Monday
+	}
+	return false
+}

classes/settings_write.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
@@ -46,6 +47,8 @@ var SettingsColumns = []string{
 	"local_postal_code", "local_city", "local_region", "local_country",
 	"local_latitude", "local_longitude", "local_hours", "page_form",
 	"page_segment", "page_title_suffix", "page_description_suffix",
+	"crawl_schedule", "crawl_max_pages", "crawl_delay", "crawl_keep",
+	"generic_anchors",
 }
 
 // settingsText maps the plain text settings columns to their fields.
@@ -88,6 +91,17 @@ func settingsText(st *models.Settings) map[string]*string {
 		"page_segment":            &st.PageSegment,
 		"page_title_suffix":       &st.PageTitleSuffix,
 		"page_description_suffix": &st.PageDescriptionSuffix,
+		"crawl_schedule":          &st.CrawlSchedule,
+		"generic_anchors":         &st.GenericAnchors,
+	}
+}
+
+// settingsInts maps the number settings columns to their fields.
+func settingsInts(st *models.Settings) map[string]*int {
+	return map[string]*int{
+		"crawl_max_pages": &st.CrawlMaxPages,
+		"crawl_delay":     &st.CrawlDelay,
+		"crawl_keep":      &st.CrawlKeep,
 	}
 }
 
@@ -139,6 +153,11 @@ func SettingsValues(st models.Settings) map[string]any {
 		"page_segment":            st.PageSegment,
 		"page_title_suffix":       st.PageTitleSuffix,
 		"page_description_suffix": st.PageDescriptionSuffix,
+		"crawl_schedule":          st.CrawlSchedule,
+		"crawl_max_pages":         st.CrawlMaxPages,
+		"crawl_delay":             st.CrawlDelay,
+		"crawl_keep":              st.CrawlKeep,
+		"generic_anchors":         st.GenericAnchors,
 	}
 }
 
@@ -170,6 +189,26 @@ func SetSettingsValue(st *models.Settings, column string, v any) error {
 		*field, err = text()
 		return err
 	}
+	if field, ok := settingsInts(st)[column]; ok {
+		switch n := v.(type) {
+		case float64:
+			if n != float64(int(n)) {
+				return fmt.Errorf("golem15.seo: settings %s is not a whole number", column)
+			}
+			*field = int(n)
+		case int:
+			*field = n
+		case string:
+			i, err := strconv.Atoi(strings.TrimSpace(n))
+			if err != nil {
+				return fmt.Errorf("golem15.seo: settings %s is not a number", column)
+			}
+			*field = i
+		default:
+			return fmt.Errorf("golem15.seo: settings %s is not a number", column)
+		}
+		return nil
+	}
 	var err error
 	switch column {
 	case "url_lowercase":
@@ -282,6 +321,11 @@ func (s *Service) SaveSettings(ctx context.Context, tx *gorm.DB, in SettingsWrit
 	for field, list := range row.Problems() {
 		problems[field] = append(problems[field], list...)
 	}
+	for field, list := range row.AuditProblems() {
+		if _, sent := in.Values[field]; sent {
+			problems[field] = append(problems[field], list...)
+		}
+	}
 	types, err := LoadTypeSettings(tx)
 	if err != nil {
 		return models.Settings{}, err

commands.go

@@ -2,19 +2,24 @@ package seo
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/bonfire"
 	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"git.golem15.com/golem15/summercms/modules/pact"
 	"gorm.io/gorm"
 )
 
-// Commands registers seo:generate-entries.
+// Commands registers seo:generate-entries and seo:crawl.
 func (p *Plugin) Commands() []bonfire.Command {
-	return []bonfire.Command{{
+	return []bonfire.Command{p.crawlCommand(), {
 		Name:        "seo:generate-entries",
 		Description: "Queue the job that creates missing SEO entries",
 		Long: `Queues the entry generation job on the background queue. The job walks the
@@ -59,6 +64,94 @@ changed. A worker (summer serve or queue:work) runs the job.`,
 	}}
 }
 
+// crawlCommand is seo:crawl [--scheduled]: it queues a crawl of the site
+// (D-14) for the worker; --scheduled is the nightly entry (P33-40), which
+// queues one only when the Site audit setting crawl_schedule is daily, or
+// weekly on a Monday in app.timezone, and skips while a crawl runs.
+func (p *Plugin) crawlCommand() bonfire.Command {
+	return bonfire.Command{
+		Name:        "seo:crawl",
+		Description: "Queue a crawl of the site for the site audit",
+		Long: `Queues a crawl job on the background queue. The crawler fetches the site's own
+pages over HTTP like a search engine (from the home page, the sitemap index
+and its files, and same-host links; two requests at a time, the Site audit
+settings' pause and page limit) and stores what it finds in SEO › Site audit.
+One crawl runs at a time. With --scheduled, as the nightly schedule runs it,
+the command queues a crawl only when the Site audit setting "Automatic crawl"
+is daily, or weekly and it is Monday in app.timezone, and does nothing while
+a crawl is queued or running. A worker (summer serve or queue:work) runs the
+job.`,
+		Flags: []bonfire.Flag{
+			{Name: "scheduled", Description: "Run as the nightly schedule: follow the Automatic crawl setting", Bare: true},
+		},
+		Examples: []bonfire.Example{
+			{Command: "seo:crawl", Description: "Queue a crawl now"},
+			{Command: "seo:crawl --scheduled", Description: "What the nightly schedule runs"},
+		},
+		Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
+			svc := p.Service()
+			if svc == nil {
+				return classes.ErrNotBooted
+			}
+			db, done, err := p.commandDB(ctx)
+			if err != nil {
+				return err
+			}
+			defer done()
+			scheduled := bonfire.FlagBool(in, "scheduled")
+			trigger, name := models.TriggerAdmin, "console"
+			if scheduled {
+				st, err := classes.LoadSettings(db)
+				if err != nil {
+					return err
+				}
+				if !crawl.ScheduledDue(st.CrawlSchedule, time.Now().In(appLocation(p.app))) {
+					out.Info("No crawl is due (Automatic crawl: " + st.CrawlSchedule + ").")
+					return nil
+				}
+				trigger, name = models.TriggerSchedule, ""
+			}
+			var row models.Crawl
+			err = lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error {
+				var err error
+				row, err = crawl.Start(ctx, svc, tx, trigger, name)
+				return err
+			})
+			var running *crawl.AlreadyRunningError
+			switch {
+			case errors.As(err, &running) && scheduled:
+				out.Info(fmt.Sprintf("Crawl #%d is still running; no new crawl was queued.", running.Crawl.ID))
+				return nil
+			case errors.As(err, &running):
+				return fmt.Errorf("Crawl #%d is already running. Wait for it, or cancel it in SEO › Crawls.", running.Crawl.ID)
+			case errors.Is(err, crawl.ErrNoSiteURL):
+				return fmt.Errorf("The site address for crawls is not configured (golem15.seo.site_url).")
+			case err != nil:
+				return err
+			}
+			job := uint(0)
+			if row.JobID != nil {
+				job = *row.JobID
+			}
+			out.Success(fmt.Sprintf("Queued crawl #%d of %s (job #%d).", row.ID, row.Site, job))
+			return nil
+		},
+	}
+}
+
+// appLocation is the app.timezone location, UTC when unset or unknown.
+func appLocation(app *backpack.App) *time.Location {
+	if app == nil || app.Config == nil {
+		return time.UTC
+	}
+	if name := strings.TrimSpace(app.Config.String("app.timezone")); name != "" {
+		if loc, err := time.LoadLocation(name); err == nil {
+			return loc
+		}
+	}
+	return time.UTC
+}
+
 // commandDB returns the app database, opening database.dsn when the binary
 // published none for console commands. done closes a pool it opened.
 func (p *Plugin) commandDB(ctx context.Context) (*gorm.DB, func(), error) {

controllers/api/settings.go

@@ -56,6 +56,11 @@ type UpdateSettingsInput struct {
 	PageSegment           *string                             `json:"page_segment,omitempty" jsonschema:"the path form's segment in the default locale, lowercase letters, digits and hyphens (default strona for pl, page for en)"`
 	PageTitleSuffix       *string                             `json:"page_title_suffix,omitempty" jsonschema:"added after the page title on page 2 and later (default locale); placeholders %page% and %pages%"`
 	PageDescriptionSuffix *string                             `json:"page_description_suffix,omitempty" jsonschema:"added at the end of the meta description on page 2 and later (default locale); placeholders %page% and %pages%"`
+	CrawlSchedule         *string                             `json:"crawl_schedule,omitempty" jsonschema:"the automatic crawl at night: off, daily or weekly (Mondays)"`
+	CrawlMaxPages         *int                                `json:"crawl_max_pages,omitempty" jsonschema:"pages per crawl, 10 to 50000 (default 5000)"`
+	CrawlDelay            *int                                `json:"crawl_delay,omitempty" jsonschema:"the pause between two crawler requests in milliseconds, 100 to 5000 (default 250); at most two requests run at a time"`
+	CrawlKeep             *int                                `json:"crawl_keep,omitempty" jsonschema:"crawls kept, 1 to 50 (default 10); older crawls are deleted with their pages and issues"`
+	GenericAnchors        *string                             `json:"generic_anchors,omitempty" jsonschema:"generic link texts, one per line; the crawl reports links whose whole text is one of them, ignoring letter case"`
 	LocalHours            *[]models.OpeningHours              `json:"local_hours,omitempty" jsonschema:"opening hours rows: day (mon … sun), opens and closes in 24-hour HH:MM; two rows split a day, a day without rows is closed"`
 	Types                 map[string]classes.TypeSettingWrite `json:"types,omitempty" jsonschema:"per content type key: inject (show the SEO tab), title_template, description_template, robots"`
 }

controllers/settings.go

@@ -56,14 +56,19 @@ func (settingsAdminController) SingletonRecord(_ context.Context, db *gorm.DB) (
 	return &rows[0], nil
 }
 
-// FormBeforeUpdate refuses templates with unknown placeholders and
-// malformed or duplicate verification file names.
+// FormBeforeUpdate refuses templates with unknown placeholders, malformed
+// or duplicate verification file names and Site audit values outside their
+// ranges.
 func (c settingsAdminController) FormBeforeUpdate(ctx context.Context, model any) error {
 	s, ok := model.(*models.Settings)
 	if !ok || s == nil {
 		return nil
 	}
-	if err := problemsError(ctx, translator(c.app), s.Problems()); err != nil {
+	problems := s.Problems()
+	for field, list := range s.AuditProblems() {
+		problems[field] = append(problems[field], list...)
+	}
+	if err := problemsError(ctx, translator(c.app), problems); err != nil {
 		return err
 	}
 	s.NofollowDomains = models.NormalizeDomains(s.NofollowDomains)

crawl_admin_test.go

@@ -1,6 +1,7 @@
 package seo_test
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/fs"
@@ -8,9 +9,11 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	seo "git.golem15.com/golem15/sm-seo-plugin"
 	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
+	"git.golem15.com/golem15/summercms/modules/bonfire"
 	"github.com/goccy/go-yaml"
 )
 
@@ -323,3 +326,127 @@ func TestIssueKindLabels(t *testing.T) {
 		}
 	}
 }
+
+// stubInput is a console input with bare flags.
+type stubInput struct{ flags map[string]string }
+
+func (stubInput) Args() []string                 { return nil }
+func (stubInput) Argument(string) (string, bool) { return "", false }
+func (s stubInput) Flag(n string) (string, bool) { v, ok := s.flags[n]; return v, ok }
+func (stubInput) Flags(string) []string          { return nil }
+
+// runCommand runs one of the SEO plugin's console commands.
+func (e *testEnv) runCommand(t *testing.T, name string, flags map[string]string) (string, error) {
+	t.Helper()
+	for _, c := range e.seoPlugin(t).Commands() {
+		if c.Name == name {
+			var buf bytes.Buffer
+			err := c.Run(t.Context(), stubInput{flags: flags}, bonfire.NewOutput(strings.NewReader(""), &buf, &buf))
+			return buf.String(), err
+		}
+	}
+	t.Fatalf("no command %s", name)
+	return "", nil
+}
+
+// TestCrawlSchedule checks the Site audit settings and the nightly crawl
+// (UI-SPEC C20, P33-40): the tab's ranges refuse with validation.between,
+// generic_anchors starts with the en and pl lists and drives the
+// generic_anchor check, seo:crawl --scheduled does nothing while the
+// schedule is off and queues a crawl with trigger schedule when daily,
+// weekly runs only on Mondays, and crawls beyond crawl_keep are deleted
+// with their pages and issues.
+func TestCrawlSchedule(t *testing.T) {
+	monday := time.Date(2026, 11, 2, 2, 30, 0, 0, time.UTC)
+	for _, tc := range []struct {
+		schedule string
+		day      time.Time
+		want     bool
+	}{
+		{"off", monday, false}, {"daily", monday, true}, {"daily", monday.AddDate(0, 0, 1), true},
+		{"weekly", monday, true}, {"weekly", monday.AddDate(0, 0, 1), false}, {"weekly", monday.AddDate(0, 0, 7), true},
+	} {
+		if got := crawl.ScheduledDue(tc.schedule, tc.day); got != tc.want {
+			t.Errorf("ScheduledDue(%s, %s) = %v, want %v", tc.schedule, tc.day.Weekday(), got, tc.want)
+		}
+	}
+	if testing.Short() {
+		return
+	}
+
+	env := bootCrawlEnv(t)
+	token := env.admin(t, true)
+	settings := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/settings", token, nil).Body.String()
+	for _, want := range []string{`"crawl_schedule":"off"`, `"crawl_max_pages":5000`, `"crawl_delay":250`, `"crawl_keep":10`, `kliknij tutaj`, `read more`} {
+		if !jsonContains(settings, want) {
+			t.Fatalf("settings lack %s: %s", want, settings)
+		}
+	}
+	tab := ""
+	for _, f := range env.formSchema(t, token, "golem15/seo/settings") {
+		if f.Name == "crawl_delay" {
+			tab = f.Tab
+		}
+	}
+	if tab != "Site audit" {
+		t.Fatalf("crawl_delay tab = %q", tab)
+	}
+	for field, value := range map[string]any{"crawl_delay": 50, "crawl_max_pages": 60000, "crawl_keep": 0} {
+		rec := env.call(t, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{field: value})
+		if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "must be between") {
+			t.Fatalf("%s=%v = %d %s", field, value, rec.Code, rec.Body.String())
+		}
+	}
+	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_delay": 100, "crawl_keep": 1, "generic_anchors": "Zobacz\nclick here"})
+
+	out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"})
+	if err != nil || !strings.Contains(out, "No crawl is due") {
+		t.Fatalf("scheduled while off: %q %v", out, err)
+	}
+	var crawls int64
+	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_crawls`).Scan(&crawls)
+	if crawls != 0 {
+		t.Fatalf("crawls while off = %d", crawls)
+	}
+
+	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_schedule": "daily"})
+	site := newFakeSite()
+	site.sitemap("/")
+	site.plant(doc{path: "/", title: "Home", body: `<p><a href="/a">Zobacz</a> <a href="/b">kliknij tutaj</a></p>`})
+	site.plant(doc{path: "/a", title: "A"}, doc{path: "/b", title: "B"})
+	out, err = env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"})
+	if err != nil || !strings.Contains(out, "Queued crawl #") {
+		t.Fatalf("scheduled while daily: %q %v", out, err)
+	}
+	var first struct {
+		ID      uint
+		Trigger string
+	}
+	env.db.Raw(`SELECT id, trigger FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&first)
+	if first.Trigger != "schedule" {
+		t.Fatalf("scheduled crawl trigger = %q", first.Trigger)
+	}
+	if out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}); err != nil || !strings.Contains(out, "still running") {
+		t.Fatalf("scheduled while one is queued: %q %v", out, err)
+	}
+	ctx := siteCtx(t, site)
+	env.runCrawl(t, ctx, first.ID)
+	generic := findIssue(t, env.issues(t, token, ""), "generic_anchor", "/")
+	if generic.Details["count"] != float64(1) || generic.Details["text"] != "Zobacz" {
+		t.Fatalf("generic_anchor with the setting = %+v", generic)
+	}
+
+	// crawl_keep 1: the second crawl deletes the first with its pages
+	// and issues.
+	second := env.crawlOnce(t, token, ctx)
+	var left []uint
+	env.db.Raw(`SELECT id FROM golem15_seo_crawls ORDER BY id`).Scan(&left)
+	if len(left) != 1 || left[0] != second {
+		t.Fatalf("crawls kept = %v, want only %d", left, second)
+	}
+	var orphanRows int64
+	env.db.Raw(`SELECT (SELECT COUNT(*) FROM golem15_seo_crawl_pages WHERE crawl_id = ?) + (SELECT COUNT(*) FROM golem15_seo_issues WHERE crawl_id = ?)`, first.ID, first.ID).Scan(&orphanRows)
+	if orphanRows != 0 {
+		t.Fatalf("pages and issues of the deleted crawl = %d", orphanRows)
+	}
+}

lang/en/lang.yaml

@@ -83,6 +83,20 @@ status:
     staging: On this environment every page is served with noindex and robots.txt blocks everything (non-production lock).
     primary_gone: "The primary category “:name” is no longer assigned, so the first category is used. Choose another one below."
 settings:
+    tab_audit: "Site audit"
+    crawl_schedule: "Automatic crawl"
+    crawl_schedule_off: "Off"
+    crawl_schedule_daily: "Daily"
+    crawl_schedule_weekly: "Weekly"
+    crawl_schedule_comment: "Runs at night. You can always start a crawl by hand in SEO › Crawls."
+    crawl_max_pages: "Pages per crawl"
+    crawl_max_pages_comment: "The crawl stops after this many pages. Default 5000."
+    crawl_delay: "Pause between requests (ms)"
+    crawl_delay_comment: "At most two requests run at a time. A longer pause spares a small server."
+    crawl_keep: "Crawls kept"
+    crawl_keep_comment: "Older crawls are deleted with their pages and issues."
+    generic_anchors: "Generic link texts"
+    generic_anchors_comment: "One per line. The crawl reports links whose whole text is one of these, ignoring letter case."
     placeholders: Placeholders
     placeholders_code: Placeholder
     placeholders_meaning: Replaced with

lang/pl/lang.yaml

@@ -85,6 +85,20 @@ status:
     staging: W tym środowisku każda strona ma noindex, a robots.txt blokuje wszystko (blokada środowiska nieprodukcyjnego).
     primary_gone: "Kategoria główna „:name” nie jest już przypisana, więc używana jest pierwsza kategoria. Wybierz inną poniżej."
 settings:
+    tab_audit: "Audyt witryny"
+    crawl_schedule: "Automatyczne skanowanie"
+    crawl_schedule_off: "Wyłączone"
+    crawl_schedule_daily: "Codziennie"
+    crawl_schedule_weekly: "Co tydzień"
+    crawl_schedule_comment: "Uruchamiane w nocy. Skanowanie zawsze możesz uruchomić ręcznie w SEO › Skanowania."
+    crawl_max_pages: "Stron na skanowanie"
+    crawl_max_pages_comment: "Skanowanie kończy się po tylu stronach. Domyślnie 5000."
+    crawl_delay: "Przerwa między żądaniami (ms)"
+    crawl_delay_comment: "Jednocześnie trwają najwyżej dwa żądania. Dłuższa przerwa oszczędza mały serwer."
+    crawl_keep: "Przechowywane skanowania"
+    crawl_keep_comment: "Starsze skanowania są usuwane razem ze stronami i problemami."
+    generic_anchors: "Ogólnikowe teksty linków"
+    generic_anchors_comment: "Jeden w wierszu. Skanowanie zgłasza linki, których cały tekst jest jednym z nich, bez względu na wielkość liter."
     placeholders: Znaczniki
     placeholders_code: Znacznik
     placeholders_meaning: Zastępowany przez

models/settings.go

@@ -211,6 +211,15 @@ type Settings struct {
 	LocalLatitude   string                          `gorm:"column:local_latitude"`
 	LocalLongitude  string                          `gorm:"column:local_longitude"`
 	LocalHours      lagoon.Jsonable[[]OpeningHours] `gorm:"column:local_hours"`
+	// The Site audit tab (D-14, UI-SPEC C20): the automatic crawl (off,
+	// daily or weekly at night), pages per crawl (10 to 50 000), the pause
+	// between requests in milliseconds (100 to 5000), the crawls kept (1 to
+	// 50) and the generic link texts, one per line.
+	CrawlSchedule  string `gorm:"column:crawl_schedule"`
+	CrawlMaxPages  int    `gorm:"column:crawl_max_pages"`
+	CrawlDelay     int    `gorm:"column:crawl_delay"`
+	CrawlKeep      int    `gorm:"column:crawl_keep"`
+	GenericAnchors string `gorm:"column:generic_anchors"`
 	// RoutesReportedAt is the time of the newest route report (D-33),
 	// written by the report only.
 	RoutesReportedAt *time.Time `gorm:"column:routes_reported_at"`
@@ -237,7 +246,8 @@ func (Settings) Fillable() []string {
 		"local_price_range", "local_street", "local_postal_code", "local_city",
 		"local_region", "local_country", "local_latitude", "local_longitude",
 		"local_hours", "page_form", "page_segment", "page_title_suffix",
-		"page_description_suffix",
+		"page_description_suffix", "crawl_schedule", "crawl_max_pages",
+		"crawl_delay", "crawl_keep", "generic_anchors",
 	}
 }
 
@@ -247,19 +257,72 @@ func (Settings) Rules() map[string]string { return map[string]string{} }
 // DefaultSettings is what the resolver reads when the row is missing.
 func DefaultSettings() Settings {
 	return Settings{
-		ID:            1,
-		TitleTemplate: DefaultTitleTemplate,
-		HomeTemplate:  DefaultHomeTemplate,
-		QueryParams:   lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true},
-		URLHost:       HostKeep,
-		URLSlash:      SlashKeep,
-		LinksNewTab:   true,
-		PageForm:      PageFormQuery,
-		PublisherType: PublisherOrganization,
-		LocalType:     DefaultLocalType,
+		ID:             1,
+		TitleTemplate:  DefaultTitleTemplate,
+		HomeTemplate:   DefaultHomeTemplate,
+		QueryParams:    lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true},
+		URLHost:        HostKeep,
+		URLSlash:       SlashKeep,
+		LinksNewTab:    true,
+		PageForm:       PageFormQuery,
+		PublisherType:  PublisherOrganization,
+		LocalType:      DefaultLocalType,
+		CrawlSchedule:  CrawlOff,
+		CrawlMaxPages:  DefaultCrawlMaxPages,
+		CrawlDelay:     DefaultCrawlDelay,
+		CrawlKeep:      DefaultCrawlKeep,
+		GenericAnchors: DefaultGenericAnchors,
 	}
 }
 
+// The automatic crawl settings (crawl_schedule).
+const (
+	CrawlOff    = "off"
+	CrawlDaily  = "daily"
+	CrawlWeekly = "weekly"
+)
+
+// The Site audit defaults and ranges (UI-SPEC C20).
+const (
+	DefaultCrawlMaxPages = 5000
+	DefaultCrawlDelay    = 250
+	DefaultCrawlKeep     = 10
+	MinCrawlMaxPages     = 10
+	MaxCrawlMaxPages     = 50_000
+	MinCrawlDelay        = 100
+	MaxCrawlDelay        = 5000
+	MinCrawlKeep         = 1
+	MaxCrawlKeep         = 50
+)
+
+// DefaultGenericAnchors is the seed of the generic link texts: the en and
+// pl lists of UI-SPEC C27, one per line.
+const DefaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link"
+
+// AuditProblems refuses Site audit values outside their ranges
+// (lagoon::validation.between.numeric, UI-SPEC C20) and an unknown
+// schedule. The settings screen and the API check it on save; Problems
+// leaves it out, so a partial Settings value checks its templates only.
+func (s Settings) AuditProblems() map[string][]Problem {
+	out := map[string][]Problem{}
+	between := func(field string, v, min, max int) {
+		if v < min || v > max {
+			out[field] = append(out[field], Problem{Key: "lagoon::validation.between.numeric", Params: map[string]string{
+				"attribute": strings.ReplaceAll(field, "_", " "), "min": strconv.Itoa(min), "max": strconv.Itoa(max),
+			}})
+		}
+	}
+	between("crawl_max_pages", s.CrawlMaxPages, MinCrawlMaxPages, MaxCrawlMaxPages)
+	between("crawl_delay", s.CrawlDelay, MinCrawlDelay, MaxCrawlDelay)
+	between("crawl_keep", s.CrawlKeep, MinCrawlKeep, MaxCrawlKeep)
+	switch s.CrawlSchedule {
+	case CrawlOff, CrawlDaily, CrawlWeekly:
+	default:
+		out["crawl_schedule"] = append(out["crawl_schedule"], Problem{Key: "golem15.seo::lang.api.value_invalid"})
+	}
+	return out
+}
+
 // Hours returns the local business's opening hours.
 func (s Settings) Hours() []OpeningHours { return s.LocalHours.Get() }
 

models/settings/fields.yaml

@@ -411,6 +411,50 @@ fields:
                     type: text
                     span: right
 
+    crawl_schedule:
+        label: golem15.seo::lang.settings.crawl_schedule
+        comment: golem15.seo::lang.settings.crawl_schedule_comment
+        type: dropdown
+        span: left
+        default: "off"
+        options:
+            "off": golem15.seo::lang.settings.crawl_schedule_off
+            daily: golem15.seo::lang.settings.crawl_schedule_daily
+            weekly: golem15.seo::lang.settings.crawl_schedule_weekly
+        tab: golem15.seo::lang.settings.tab_audit
+
+    crawl_max_pages:
+        label: golem15.seo::lang.settings.crawl_max_pages
+        comment: golem15.seo::lang.settings.crawl_max_pages_comment
+        type: number
+        span: right
+        default: 5000
+        tab: golem15.seo::lang.settings.tab_audit
+
+    crawl_delay:
+        label: golem15.seo::lang.settings.crawl_delay
+        comment: golem15.seo::lang.settings.crawl_delay_comment
+        type: number
+        span: left
+        default: 250
+        tab: golem15.seo::lang.settings.tab_audit
+
+    crawl_keep:
+        label: golem15.seo::lang.settings.crawl_keep
+        comment: golem15.seo::lang.settings.crawl_keep_comment
+        type: number
+        span: right
+        default: 10
+        tab: golem15.seo::lang.settings.tab_audit
+
+    generic_anchors:
+        label: golem15.seo::lang.settings.generic_anchors
+        comment: golem15.seo::lang.settings.generic_anchors_comment
+        type: textarea
+        size: small
+        span: full
+        tab: golem15.seo::lang.settings.tab_audit
+
     verify_google:
         label: golem15.seo::lang.settings.verify_google
         comment: golem15.seo::lang.settings.verify_code_comment

schedule.go

@@ -0,0 +1,14 @@
+package seo
+
+import "git.golem15.com/golem15/summercms/modules/pact"
+
+// Schedule runs seo:crawl --scheduled every night at 02:30 app time
+// (P33-40): the command does nothing while the Site audit setting
+// crawl_schedule is off, and runs a weekly schedule on Mondays only.
+func (p *Plugin) Schedule() []pact.ScheduledCommand {
+	return []pact.ScheduledCommand{
+		{Command: "seo:crawl", Args: []string{"--scheduled"}, Cadence: pact.DailyAt(2, 30)},
+	}
+}
+
+var _ pact.HasSchedule = (*Plugin)(nil)

settings_test.go

@@ -31,7 +31,7 @@ func TestSettingsTypeTabs(t *testing.T) {
 		env := bootEnv(t)
 		token := env.admin(t, true)
 		fields := env.formSchema(t, token, "golem15/seo/settings")
-		if got, want := strings.Join(tabsOf(fields), " | "), "General | Page | Tags | Notes | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Verification"; got != want {
+		if got, want := strings.Join(tabsOf(fields), " | "), "General | Page | Tags | Notes | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Site audit | Verification"; got != want {
 			t.Fatalf("settings tabs = %q, want %q", got, want)
 		}
 		findField(t, fields, "type_acme_page_inject")
@@ -81,7 +81,7 @@ func TestSettingsTypeTabs(t *testing.T) {
 	t.Run("no other content types", func(t *testing.T) {
 		env := bootPlugins(t, []string{"golem15.seo"})
 		token := env.admin(t, true)
-		if got, want := strings.Join(tabsOf(env.formSchema(t, token, "golem15/seo/settings")), " | "), "General | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Verification"; got != want {
+		if got, want := strings.Join(tabsOf(env.formSchema(t, token, "golem15/seo/settings")), " | "), "General | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Site audit | Verification"; got != want {
 			t.Fatalf("settings tabs = %q, want %q", got, want)
 		}
 	})

updates/202610280006_add_golem15_seo_audit_settings.go

@@ -0,0 +1,41 @@
+package updates
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// defaultGenericAnchors is the seed of the generic link texts: the en and
+// pl lists of UI-SPEC C27, one per line.
+const defaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link"
+
+// addAuditSettings adds the SEO settings' Site audit tab (UI-SPEC C20,
+// D-14): the automatic crawl (off, daily or weekly), pages per crawl, the
+// pause between requests in milliseconds, the crawls kept and the generic
+// link texts, seeded with the en and pl lists.
+var addAuditSettings = []*gormigrate.Migration{
+	{
+		ID: "202610280006_add_golem15_seo_audit_settings",
+		Migrate: func(tx *gorm.DB) error {
+			if err := tx.Exec(`ALTER TABLE golem15_seo_settings
+	ADD COLUMN crawl_schedule TEXT NOT NULL DEFAULT 'off',
+	ADD COLUMN crawl_max_pages INTEGER NOT NULL DEFAULT 5000,
+	ADD COLUMN crawl_delay INTEGER NOT NULL DEFAULT 250,
+	ADD COLUMN crawl_keep INTEGER NOT NULL DEFAULT 10,
+	ADD COLUMN generic_anchors TEXT NOT NULL DEFAULT ''`).Error; err != nil {
+				return err
+			}
+			return tx.Exec(`UPDATE golem15_seo_settings SET generic_anchors = ? WHERE generic_anchors = ''`, defaultGenericAnchors).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return tx.Exec(`ALTER TABLE golem15_seo_settings
+	DROP COLUMN IF EXISTS crawl_schedule, DROP COLUMN IF EXISTS crawl_max_pages,
+	DROP COLUMN IF EXISTS crawl_delay, DROP COLUMN IF EXISTS crawl_keep,
+	DROP COLUMN IF EXISTS generic_anchors`).Error
+		},
+	},
+}
+
+func init() {
+	Register(addAuditSettings...)
+}