feat: site audit settings and the nightly crawl

- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages
  (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and
  generic_anchors seeded with the en and pl lists; ranges refuse with
  lagoon::validation.between on the screen and through the API
- the crawler reads its limits and generic anchors from the settings;
  crawls beyond crawl_keep are deleted with their pages and issues
- seo:crawl [--scheduled] and the nightly schedule entry at 02:30
  (P33-40): off does nothing, weekly runs on Mondays in app.timezone
- TestCrawlSchedule

classes/crawl/job.go

@@ -115,9 +115,25 @@ type Limits struct {
 // DefaultKeep is the number of crawls kept.
 const DefaultKeep = 10
 
-// LoadLimits reads the crawl limits.
+// LoadLimits reads the crawl limits from the Site audit settings; a value
+// outside its range (written around the settings screen) falls back to the
+// default.
 func LoadLimits(db *gorm.DB) (Limits, error) {
-	return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: DefaultGenericAnchors}, nil
+	st, err := classes.LoadSettings(db)
+	if err != nil {
+		return Limits{}, err
+	}
+	out := Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: ParseAnchors(st.GenericAnchors)}
+	if st.CrawlMaxPages >= models.MinCrawlMaxPages && st.CrawlMaxPages <= models.MaxCrawlMaxPages {
+		out.MaxPages = st.CrawlMaxPages
+	}
+	if st.CrawlDelay >= models.MinCrawlDelay && st.CrawlDelay <= models.MaxCrawlDelay {
+		out.Delay = time.Duration(st.CrawlDelay) * time.Millisecond
+	}
+	if st.CrawlKeep >= models.MinCrawlKeep && st.CrawlKeep <= models.MaxCrawlKeep {
+		out.Keep = st.CrawlKeep
+	}
+	return out, nil
 }
 
 // LoadCrawl reads one crawl.
@@ -393,3 +409,16 @@ var ErrNotRunning = errCancelledCrawl
 
 // pluginLog is the crawler's logger (component seo).
 var pluginLog = diary.For("seo")
+
+// ScheduledDue reports whether the nightly entry starts a crawl on day
+// (in the app's location) for the crawl_schedule setting: daily every
+// night, weekly on Mondays, never when off (P33-40).
+func ScheduledDue(schedule string, day time.Time) bool {
+	switch schedule {
+	case models.CrawlDaily:
+		return true
+	case models.CrawlWeekly:
+		return day.Weekday() == time.Monday
+	}
+	return false
+}

classes/settings_write.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
@@ -46,6 +47,8 @@ var SettingsColumns = []string{
 	"local_postal_code", "local_city", "local_region", "local_country",
 	"local_latitude", "local_longitude", "local_hours", "page_form",
 	"page_segment", "page_title_suffix", "page_description_suffix",
+	"crawl_schedule", "crawl_max_pages", "crawl_delay", "crawl_keep",
+	"generic_anchors",
 }
 
 // settingsText maps the plain text settings columns to their fields.
@@ -88,6 +91,17 @@ func settingsText(st *models.Settings) map[string]*string {
 		"page_segment":            &st.PageSegment,
 		"page_title_suffix":       &st.PageTitleSuffix,
 		"page_description_suffix": &st.PageDescriptionSuffix,
+		"crawl_schedule":          &st.CrawlSchedule,
+		"generic_anchors":         &st.GenericAnchors,
+	}
+}
+
+// settingsInts maps the number settings columns to their fields.
+func settingsInts(st *models.Settings) map[string]*int {
+	return map[string]*int{
+		"crawl_max_pages": &st.CrawlMaxPages,
+		"crawl_delay":     &st.CrawlDelay,
+		"crawl_keep":      &st.CrawlKeep,
 	}
 }
 
@@ -139,6 +153,11 @@ func SettingsValues(st models.Settings) map[string]any {
 		"page_segment":            st.PageSegment,
 		"page_title_suffix":       st.PageTitleSuffix,
 		"page_description_suffix": st.PageDescriptionSuffix,
+		"crawl_schedule":          st.CrawlSchedule,
+		"crawl_max_pages":         st.CrawlMaxPages,
+		"crawl_delay":             st.CrawlDelay,
+		"crawl_keep":              st.CrawlKeep,
+		"generic_anchors":         st.GenericAnchors,
 	}
 }
 
@@ -170,6 +189,26 @@ func SetSettingsValue(st *models.Settings, column string, v any) error {
 		*field, err = text()
 		return err
 	}
+	if field, ok := settingsInts(st)[column]; ok {
+		switch n := v.(type) {
+		case float64:
+			if n != float64(int(n)) {
+				return fmt.Errorf("golem15.seo: settings %s is not a whole number", column)
+			}
+			*field = int(n)
+		case int:
+			*field = n
+		case string:
+			i, err := strconv.Atoi(strings.TrimSpace(n))
+			if err != nil {
+				return fmt.Errorf("golem15.seo: settings %s is not a number", column)
+			}
+			*field = i
+		default:
+			return fmt.Errorf("golem15.seo: settings %s is not a number", column)
+		}
+		return nil
+	}
 	var err error
 	switch column {
 	case "url_lowercase":
@@ -282,6 +321,11 @@ func (s *Service) SaveSettings(ctx context.Context, tx *gorm.DB, in SettingsWrit
 	for field, list := range row.Problems() {
 		problems[field] = append(problems[field], list...)
 	}
+	for field, list := range row.AuditProblems() {
+		if _, sent := in.Values[field]; sent {
+			problems[field] = append(problems[field], list...)
+		}
+	}
 	types, err := LoadTypeSettings(tx)
 	if err != nil {
 		return models.Settings{}, err