feat: site audit settings and the nightly crawl
- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and generic_anchors seeded with the en and pl lists; ranges refuse with lagoon::validation.between on the screen and through the API - the crawler reads its limits and generic anchors from the settings; crawls beyond crawl_keep are deleted with their pages and issues - seo:crawl [--scheduled] and the nightly schedule entry at 02:30 (P33-40): off does nothing, weekly runs on Mondays in app.timezone - TestCrawlSchedule
This commit is contained in:
41
updates/202610280006_add_golem15_seo_audit_settings.go
Normal file
41
updates/202610280006_add_golem15_seo_audit_settings.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// defaultGenericAnchors is the seed of the generic link texts: the en and
|
||||
// pl lists of UI-SPEC C27, one per line.
|
||||
const defaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link"
|
||||
|
||||
// addAuditSettings adds the SEO settings' Site audit tab (UI-SPEC C20,
|
||||
// D-14): the automatic crawl (off, daily or weekly), pages per crawl, the
|
||||
// pause between requests in milliseconds, the crawls kept and the generic
|
||||
// link texts, seeded with the en and pl lists.
|
||||
var addAuditSettings = []*gormigrate.Migration{
|
||||
{
|
||||
ID: "202610280006_add_golem15_seo_audit_settings",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
if err := tx.Exec(`ALTER TABLE golem15_seo_settings
|
||||
ADD COLUMN crawl_schedule TEXT NOT NULL DEFAULT 'off',
|
||||
ADD COLUMN crawl_max_pages INTEGER NOT NULL DEFAULT 5000,
|
||||
ADD COLUMN crawl_delay INTEGER NOT NULL DEFAULT 250,
|
||||
ADD COLUMN crawl_keep INTEGER NOT NULL DEFAULT 10,
|
||||
ADD COLUMN generic_anchors TEXT NOT NULL DEFAULT ''`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Exec(`UPDATE golem15_seo_settings SET generic_anchors = ? WHERE generic_anchors = ''`, defaultGenericAnchors).Error
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Exec(`ALTER TABLE golem15_seo_settings
|
||||
DROP COLUMN IF EXISTS crawl_schedule, DROP COLUMN IF EXISTS crawl_max_pages,
|
||||
DROP COLUMN IF EXISTS crawl_delay, DROP COLUMN IF EXISTS crawl_keep,
|
||||
DROP COLUMN IF EXISTS generic_anchors`).Error
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(addAuditSettings...)
|
||||
}
|
||||
Reference in New Issue
Block a user