feat: site audit settings and the nightly crawl

- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages
  (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and
  generic_anchors seeded with the en and pl lists; ranges refuse with
  lagoon::validation.between on the screen and through the API
- the crawler reads its limits and generic anchors from the settings;
  crawls beyond crawl_keep are deleted with their pages and issues
- seo:crawl [--scheduled] and the nightly schedule entry at 02:30
  (P33-40): off does nothing, weekly runs on Mondays in app.timezone
- TestCrawlSchedule
This commit is contained in:
Jakub Zych
2026-11-04 15:41:54 +01:00
parent c4f0b4f94c
commit fc53d93b9d
13 changed files with 513 additions and 20 deletions

View File

@@ -56,6 +56,11 @@ type UpdateSettingsInput struct {
PageSegment *string `json:"page_segment,omitempty" jsonschema:"the path form's segment in the default locale, lowercase letters, digits and hyphens (default strona for pl, page for en)"`
PageTitleSuffix *string `json:"page_title_suffix,omitempty" jsonschema:"added after the page title on page 2 and later (default locale); placeholders %page% and %pages%"`
PageDescriptionSuffix *string `json:"page_description_suffix,omitempty" jsonschema:"added at the end of the meta description on page 2 and later (default locale); placeholders %page% and %pages%"`
CrawlSchedule *string `json:"crawl_schedule,omitempty" jsonschema:"the automatic crawl at night: off, daily or weekly (Mondays)"`
CrawlMaxPages *int `json:"crawl_max_pages,omitempty" jsonschema:"pages per crawl, 10 to 50000 (default 5000)"`
CrawlDelay *int `json:"crawl_delay,omitempty" jsonschema:"the pause between two crawler requests in milliseconds, 100 to 5000 (default 250); at most two requests run at a time"`
CrawlKeep *int `json:"crawl_keep,omitempty" jsonschema:"crawls kept, 1 to 50 (default 10); older crawls are deleted with their pages and issues"`
GenericAnchors *string `json:"generic_anchors,omitempty" jsonschema:"generic link texts, one per line; the crawl reports links whose whole text is one of them, ignoring letter case"`
LocalHours *[]models.OpeningHours `json:"local_hours,omitempty" jsonschema:"opening hours rows: day (mon … sun), opens and closes in 24-hour HH:MM; two rows split a day, a day without rows is closed"`
Types map[string]classes.TypeSettingWrite `json:"types,omitempty" jsonschema:"per content type key: inject (show the SEO tab), title_template, description_template, robots"`
}