feat: site audit settings and the nightly crawl
- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and generic_anchors seeded with the en and pl lists; ranges refuse with lagoon::validation.between on the screen and through the API - the crawler reads its limits and generic anchors from the settings; crawls beyond crawl_keep are deleted with their pages and issues - seo:crawl [--scheduled] and the nightly schedule entry at 02:30 (P33-40): off does nothing, weekly runs on Mondays in app.timezone - TestCrawlSchedule
This commit is contained in:
Jakub Zych
@@ -211,6 +211,15 @@ type Settings struct {
|
||||
LocalLatitude string `gorm:"column:local_latitude"`
|
||||
LocalLongitude string `gorm:"column:local_longitude"`
|
||||
LocalHours lagoon.Jsonable[[]OpeningHours] `gorm:"column:local_hours"`
|
||||
// The Site audit tab (D-14, UI-SPEC C20): the automatic crawl (off,
|
||||
// daily or weekly at night), pages per crawl (10 to 50 000), the pause
|
||||
// between requests in milliseconds (100 to 5000), the crawls kept (1 to
|
||||
// 50) and the generic link texts, one per line.
|
||||
CrawlSchedule string `gorm:"column:crawl_schedule"`
|
||||
CrawlMaxPages int `gorm:"column:crawl_max_pages"`
|
||||
CrawlDelay int `gorm:"column:crawl_delay"`
|
||||
CrawlKeep int `gorm:"column:crawl_keep"`
|
||||
GenericAnchors string `gorm:"column:generic_anchors"`
|
||||
// RoutesReportedAt is the time of the newest route report (D-33),
|
||||
// written by the report only.
|
||||
RoutesReportedAt *time.Time `gorm:"column:routes_reported_at"`
|
||||
@@ -237,7 +246,8 @@ func (Settings) Fillable() []string {
|
||||
"local_price_range", "local_street", "local_postal_code", "local_city",
|
||||
"local_region", "local_country", "local_latitude", "local_longitude",
|
||||
"local_hours", "page_form", "page_segment", "page_title_suffix",
|
||||
"page_description_suffix",
|
||||
"page_description_suffix", "crawl_schedule", "crawl_max_pages",
|
||||
"crawl_delay", "crawl_keep", "generic_anchors",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,19 +257,72 @@ func (Settings) Rules() map[string]string { return map[string]string{} }
|
||||
// DefaultSettings is what the resolver reads when the row is missing.
|
||||
func DefaultSettings() Settings {
|
||||
return Settings{
|
||||
ID: 1,
|
||||
TitleTemplate: DefaultTitleTemplate,
|
||||
HomeTemplate: DefaultHomeTemplate,
|
||||
QueryParams: lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true},
|
||||
URLHost: HostKeep,
|
||||
URLSlash: SlashKeep,
|
||||
LinksNewTab: true,
|
||||
PageForm: PageFormQuery,
|
||||
PublisherType: PublisherOrganization,
|
||||
LocalType: DefaultLocalType,
|
||||
ID: 1,
|
||||
TitleTemplate: DefaultTitleTemplate,
|
||||
HomeTemplate: DefaultHomeTemplate,
|
||||
QueryParams: lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true},
|
||||
URLHost: HostKeep,
|
||||
URLSlash: SlashKeep,
|
||||
LinksNewTab: true,
|
||||
PageForm: PageFormQuery,
|
||||
PublisherType: PublisherOrganization,
|
||||
LocalType: DefaultLocalType,
|
||||
CrawlSchedule: CrawlOff,
|
||||
CrawlMaxPages: DefaultCrawlMaxPages,
|
||||
CrawlDelay: DefaultCrawlDelay,
|
||||
CrawlKeep: DefaultCrawlKeep,
|
||||
GenericAnchors: DefaultGenericAnchors,
|
||||
}
|
||||
}
|
||||
|
||||
// The automatic crawl settings (crawl_schedule).
|
||||
const (
|
||||
CrawlOff = "off"
|
||||
CrawlDaily = "daily"
|
||||
CrawlWeekly = "weekly"
|
||||
)
|
||||
|
||||
// The Site audit defaults and ranges (UI-SPEC C20).
|
||||
const (
|
||||
DefaultCrawlMaxPages = 5000
|
||||
DefaultCrawlDelay = 250
|
||||
DefaultCrawlKeep = 10
|
||||
MinCrawlMaxPages = 10
|
||||
MaxCrawlMaxPages = 50_000
|
||||
MinCrawlDelay = 100
|
||||
MaxCrawlDelay = 5000
|
||||
MinCrawlKeep = 1
|
||||
MaxCrawlKeep = 50
|
||||
)
|
||||
|
||||
// DefaultGenericAnchors is the seed of the generic link texts: the en and
|
||||
// pl lists of UI-SPEC C27, one per line.
|
||||
const DefaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link"
|
||||
|
||||
// AuditProblems refuses Site audit values outside their ranges
|
||||
// (lagoon::validation.between.numeric, UI-SPEC C20) and an unknown
|
||||
// schedule. The settings screen and the API check it on save; Problems
|
||||
// leaves it out, so a partial Settings value checks its templates only.
|
||||
func (s Settings) AuditProblems() map[string][]Problem {
|
||||
out := map[string][]Problem{}
|
||||
between := func(field string, v, min, max int) {
|
||||
if v < min || v > max {
|
||||
out[field] = append(out[field], Problem{Key: "lagoon::validation.between.numeric", Params: map[string]string{
|
||||
"attribute": strings.ReplaceAll(field, "_", " "), "min": strconv.Itoa(min), "max": strconv.Itoa(max),
|
||||
}})
|
||||
}
|
||||
}
|
||||
between("crawl_max_pages", s.CrawlMaxPages, MinCrawlMaxPages, MaxCrawlMaxPages)
|
||||
between("crawl_delay", s.CrawlDelay, MinCrawlDelay, MaxCrawlDelay)
|
||||
between("crawl_keep", s.CrawlKeep, MinCrawlKeep, MaxCrawlKeep)
|
||||
switch s.CrawlSchedule {
|
||||
case CrawlOff, CrawlDaily, CrawlWeekly:
|
||||
default:
|
||||
out["crawl_schedule"] = append(out["crawl_schedule"], Problem{Key: "golem15.seo::lang.api.value_invalid"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Hours returns the local business's opening hours.
|
||||
func (s Settings) Hours() []OpeningHours { return s.LocalHours.Get() }
|
||||
|
||||
|
||||
@@ -411,6 +411,50 @@ fields:
|
||||
type: text
|
||||
span: right
|
||||
|
||||
crawl_schedule:
|
||||
label: golem15.seo::lang.settings.crawl_schedule
|
||||
comment: golem15.seo::lang.settings.crawl_schedule_comment
|
||||
type: dropdown
|
||||
span: left
|
||||
default: "off"
|
||||
options:
|
||||
"off": golem15.seo::lang.settings.crawl_schedule_off
|
||||
daily: golem15.seo::lang.settings.crawl_schedule_daily
|
||||
weekly: golem15.seo::lang.settings.crawl_schedule_weekly
|
||||
tab: golem15.seo::lang.settings.tab_audit
|
||||
|
||||
crawl_max_pages:
|
||||
label: golem15.seo::lang.settings.crawl_max_pages
|
||||
comment: golem15.seo::lang.settings.crawl_max_pages_comment
|
||||
type: number
|
||||
span: right
|
||||
default: 5000
|
||||
tab: golem15.seo::lang.settings.tab_audit
|
||||
|
||||
crawl_delay:
|
||||
label: golem15.seo::lang.settings.crawl_delay
|
||||
comment: golem15.seo::lang.settings.crawl_delay_comment
|
||||
type: number
|
||||
span: left
|
||||
default: 250
|
||||
tab: golem15.seo::lang.settings.tab_audit
|
||||
|
||||
crawl_keep:
|
||||
label: golem15.seo::lang.settings.crawl_keep
|
||||
comment: golem15.seo::lang.settings.crawl_keep_comment
|
||||
type: number
|
||||
span: right
|
||||
default: 10
|
||||
tab: golem15.seo::lang.settings.tab_audit
|
||||
|
||||
generic_anchors:
|
||||
label: golem15.seo::lang.settings.generic_anchors
|
||||
comment: golem15.seo::lang.settings.generic_anchors_comment
|
||||
type: textarea
|
||||
size: small
|
||||
span: full
|
||||
tab: golem15.seo::lang.settings.tab_audit
|
||||
|
||||
verify_google:
|
||||
label: golem15.seo::lang.settings.verify_google
|
||||
comment: golem15.seo::lang.settings.verify_code_comment
|
||||
|
||||
Reference in New Issue
Block a user