feat: site audit settings and the nightly crawl

- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages
  (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and
  generic_anchors seeded with the en and pl lists; ranges refuse with
  lagoon::validation.between on the screen and through the API
- the crawler reads its limits and generic anchors from the settings;
  crawls beyond crawl_keep are deleted with their pages and issues
- seo:crawl [--scheduled] and the nightly schedule entry at 02:30
  (P33-40): off does nothing, weekly runs on Mondays in app.timezone
- TestCrawlSchedule
This commit is contained in:
Jakub Zych
2026-11-04 15:41:54 +01:00
parent c4f0b4f94c
commit fc53d93b9d
13 changed files with 513 additions and 20 deletions

View File

@@ -0,0 +1,41 @@
package updates
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)
// defaultGenericAnchors is the seed of the generic link texts: the en and
// pl lists of UI-SPEC C27, one per line.
const defaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link"
// addAuditSettings adds the SEO settings' Site audit tab (UI-SPEC C20,
// D-14): the automatic crawl (off, daily or weekly), pages per crawl, the
// pause between requests in milliseconds, the crawls kept and the generic
// link texts, seeded with the en and pl lists.
var addAuditSettings = []*gormigrate.Migration{
{
ID: "202610280006_add_golem15_seo_audit_settings",
Migrate: func(tx *gorm.DB) error {
if err := tx.Exec(`ALTER TABLE golem15_seo_settings
ADD COLUMN crawl_schedule TEXT NOT NULL DEFAULT 'off',
ADD COLUMN crawl_max_pages INTEGER NOT NULL DEFAULT 5000,
ADD COLUMN crawl_delay INTEGER NOT NULL DEFAULT 250,
ADD COLUMN crawl_keep INTEGER NOT NULL DEFAULT 10,
ADD COLUMN generic_anchors TEXT NOT NULL DEFAULT ''`).Error; err != nil {
return err
}
return tx.Exec(`UPDATE golem15_seo_settings SET generic_anchors = ? WHERE generic_anchors = ''`, defaultGenericAnchors).Error
},
Rollback: func(tx *gorm.DB) error {
return tx.Exec(`ALTER TABLE golem15_seo_settings
DROP COLUMN IF EXISTS crawl_schedule, DROP COLUMN IF EXISTS crawl_max_pages,
DROP COLUMN IF EXISTS crawl_delay, DROP COLUMN IF EXISTS crawl_keep,
DROP COLUMN IF EXISTS generic_anchors`).Error
},
},
}
func init() {
Register(addAuditSettings...)
}