feat: site audit settings and the nightly crawl

- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages
  (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and
  generic_anchors seeded with the en and pl lists; ranges refuse with
  lagoon::validation.between on the screen and through the API
- the crawler reads its limits and generic anchors from the settings;
  crawls beyond crawl_keep are deleted with their pages and issues
- seo:crawl [--scheduled] and the nightly schedule entry at 02:30
  (P33-40): off does nothing, weekly runs on Mondays in app.timezone
- TestCrawlSchedule
This commit is contained in:
Jakub Zych
2026-11-04 15:41:54 +01:00
parent c4f0b4f94c
commit fc53d93b9d
13 changed files with 513 additions and 20 deletions

View File

@@ -56,14 +56,19 @@ func (settingsAdminController) SingletonRecord(_ context.Context, db *gorm.DB) (
return &rows[0], nil
}
// FormBeforeUpdate refuses templates with unknown placeholders and
// malformed or duplicate verification file names.
// FormBeforeUpdate refuses templates with unknown placeholders, malformed
// or duplicate verification file names and Site audit values outside their
// ranges.
func (c settingsAdminController) FormBeforeUpdate(ctx context.Context, model any) error {
s, ok := model.(*models.Settings)
if !ok || s == nil {
return nil
}
if err := problemsError(ctx, translator(c.app), s.Problems()); err != nil {
problems := s.Problems()
for field, list := range s.AuditProblems() {
problems[field] = append(problems[field], list...)
}
if err := problemsError(ctx, translator(c.app), problems); err != nil {
return err
}
s.NofollowDomains = models.NormalizeDomains(s.NofollowDomains)