diff --git a/classes/crawl/job.go b/classes/crawl/job.go index 3ca7af2..7126dd3 100644 --- a/classes/crawl/job.go +++ b/classes/crawl/job.go @@ -115,9 +115,25 @@ type Limits struct { // DefaultKeep is the number of crawls kept. const DefaultKeep = 10 -// LoadLimits reads the crawl limits. +// LoadLimits reads the crawl limits from the Site audit settings; a value +// outside its range (written around the settings screen) falls back to the +// default. func LoadLimits(db *gorm.DB) (Limits, error) { - return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: DefaultGenericAnchors}, nil + st, err := classes.LoadSettings(db) + if err != nil { + return Limits{}, err + } + out := Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: ParseAnchors(st.GenericAnchors)} + if st.CrawlMaxPages >= models.MinCrawlMaxPages && st.CrawlMaxPages <= models.MaxCrawlMaxPages { + out.MaxPages = st.CrawlMaxPages + } + if st.CrawlDelay >= models.MinCrawlDelay && st.CrawlDelay <= models.MaxCrawlDelay { + out.Delay = time.Duration(st.CrawlDelay) * time.Millisecond + } + if st.CrawlKeep >= models.MinCrawlKeep && st.CrawlKeep <= models.MaxCrawlKeep { + out.Keep = st.CrawlKeep + } + return out, nil } // LoadCrawl reads one crawl. @@ -393,3 +409,16 @@ var ErrNotRunning = errCancelledCrawl // pluginLog is the crawler's logger (component seo). var pluginLog = diary.For("seo") + +// ScheduledDue reports whether the nightly entry starts a crawl on day +// (in the app's location) for the crawl_schedule setting: daily every +// night, weekly on Mondays, never when off (P33-40). +func ScheduledDue(schedule string, day time.Time) bool { + switch schedule { + case models.CrawlDaily: + return true + case models.CrawlWeekly: + return day.Weekday() == time.Monday + } + return false +} diff --git a/classes/settings_write.go b/classes/settings_write.go index 85fdd86..a713dec 100644 --- a/classes/settings_write.go +++ b/classes/settings_write.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" "time" @@ -46,6 +47,8 @@ var SettingsColumns = []string{ "local_postal_code", "local_city", "local_region", "local_country", "local_latitude", "local_longitude", "local_hours", "page_form", "page_segment", "page_title_suffix", "page_description_suffix", + "crawl_schedule", "crawl_max_pages", "crawl_delay", "crawl_keep", + "generic_anchors", } // settingsText maps the plain text settings columns to their fields. @@ -88,6 +91,17 @@ func settingsText(st *models.Settings) map[string]*string { "page_segment": &st.PageSegment, "page_title_suffix": &st.PageTitleSuffix, "page_description_suffix": &st.PageDescriptionSuffix, + "crawl_schedule": &st.CrawlSchedule, + "generic_anchors": &st.GenericAnchors, + } +} + +// settingsInts maps the number settings columns to their fields. +func settingsInts(st *models.Settings) map[string]*int { + return map[string]*int{ + "crawl_max_pages": &st.CrawlMaxPages, + "crawl_delay": &st.CrawlDelay, + "crawl_keep": &st.CrawlKeep, } } @@ -139,6 +153,11 @@ func SettingsValues(st models.Settings) map[string]any { "page_segment": st.PageSegment, "page_title_suffix": st.PageTitleSuffix, "page_description_suffix": st.PageDescriptionSuffix, + "crawl_schedule": st.CrawlSchedule, + "crawl_max_pages": st.CrawlMaxPages, + "crawl_delay": st.CrawlDelay, + "crawl_keep": st.CrawlKeep, + "generic_anchors": st.GenericAnchors, } } @@ -170,6 +189,26 @@ func SetSettingsValue(st *models.Settings, column string, v any) error { *field, err = text() return err } + if field, ok := settingsInts(st)[column]; ok { + switch n := v.(type) { + case float64: + if n != float64(int(n)) { + return fmt.Errorf("golem15.seo: settings %s is not a whole number", column) + } + *field = int(n) + case int: + *field = n + case string: + i, err := strconv.Atoi(strings.TrimSpace(n)) + if err != nil { + return fmt.Errorf("golem15.seo: settings %s is not a number", column) + } + *field = i + default: + return fmt.Errorf("golem15.seo: settings %s is not a number", column) + } + return nil + } var err error switch column { case "url_lowercase": @@ -282,6 +321,11 @@ func (s *Service) SaveSettings(ctx context.Context, tx *gorm.DB, in SettingsWrit for field, list := range row.Problems() { problems[field] = append(problems[field], list...) } + for field, list := range row.AuditProblems() { + if _, sent := in.Values[field]; sent { + problems[field] = append(problems[field], list...) + } + } types, err := LoadTypeSettings(tx) if err != nil { return models.Settings{}, err diff --git a/commands.go b/commands.go index fa9c487..619e0d4 100644 --- a/commands.go +++ b/commands.go @@ -2,19 +2,24 @@ package seo import ( "context" + "errors" "fmt" "strings" + "time" "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" ) -// Commands registers seo:generate-entries. +// Commands registers seo:generate-entries and seo:crawl. func (p *Plugin) Commands() []bonfire.Command { - return []bonfire.Command{{ + return []bonfire.Command{p.crawlCommand(), { Name: "seo:generate-entries", Description: "Queue the job that creates missing SEO entries", Long: `Queues the entry generation job on the background queue. The job walks the @@ -59,6 +64,94 @@ changed. A worker (summer serve or queue:work) runs the job.`, }} } +// crawlCommand is seo:crawl [--scheduled]: it queues a crawl of the site +// (D-14) for the worker; --scheduled is the nightly entry (P33-40), which +// queues one only when the Site audit setting crawl_schedule is daily, or +// weekly on a Monday in app.timezone, and skips while a crawl runs. +func (p *Plugin) crawlCommand() bonfire.Command { + return bonfire.Command{ + Name: "seo:crawl", + Description: "Queue a crawl of the site for the site audit", + Long: `Queues a crawl job on the background queue. The crawler fetches the site's own +pages over HTTP like a search engine (from the home page, the sitemap index +and its files, and same-host links; two requests at a time, the Site audit +settings' pause and page limit) and stores what it finds in SEO › Site audit. +One crawl runs at a time. With --scheduled, as the nightly schedule runs it, +the command queues a crawl only when the Site audit setting "Automatic crawl" +is daily, or weekly and it is Monday in app.timezone, and does nothing while +a crawl is queued or running. A worker (summer serve or queue:work) runs the +job.`, + Flags: []bonfire.Flag{ + {Name: "scheduled", Description: "Run as the nightly schedule: follow the Automatic crawl setting", Bare: true}, + }, + Examples: []bonfire.Example{ + {Command: "seo:crawl", Description: "Queue a crawl now"}, + {Command: "seo:crawl --scheduled", Description: "What the nightly schedule runs"}, + }, + Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { + svc := p.Service() + if svc == nil { + return classes.ErrNotBooted + } + db, done, err := p.commandDB(ctx) + if err != nil { + return err + } + defer done() + scheduled := bonfire.FlagBool(in, "scheduled") + trigger, name := models.TriggerAdmin, "console" + if scheduled { + st, err := classes.LoadSettings(db) + if err != nil { + return err + } + if !crawl.ScheduledDue(st.CrawlSchedule, time.Now().In(appLocation(p.app))) { + out.Info("No crawl is due (Automatic crawl: " + st.CrawlSchedule + ").") + return nil + } + trigger, name = models.TriggerSchedule, "" + } + var row models.Crawl + err = lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error { + var err error + row, err = crawl.Start(ctx, svc, tx, trigger, name) + return err + }) + var running *crawl.AlreadyRunningError + switch { + case errors.As(err, &running) && scheduled: + out.Info(fmt.Sprintf("Crawl #%d is still running; no new crawl was queued.", running.Crawl.ID)) + return nil + case errors.As(err, &running): + return fmt.Errorf("Crawl #%d is already running. Wait for it, or cancel it in SEO › Crawls.", running.Crawl.ID) + case errors.Is(err, crawl.ErrNoSiteURL): + return fmt.Errorf("The site address for crawls is not configured (golem15.seo.site_url).") + case err != nil: + return err + } + job := uint(0) + if row.JobID != nil { + job = *row.JobID + } + out.Success(fmt.Sprintf("Queued crawl #%d of %s (job #%d).", row.ID, row.Site, job)) + return nil + }, + } +} + +// appLocation is the app.timezone location, UTC when unset or unknown. +func appLocation(app *backpack.App) *time.Location { + if app == nil || app.Config == nil { + return time.UTC + } + if name := strings.TrimSpace(app.Config.String("app.timezone")); name != "" { + if loc, err := time.LoadLocation(name); err == nil { + return loc + } + } + return time.UTC +} + // commandDB returns the app database, opening database.dsn when the binary // published none for console commands. done closes a pool it opened. func (p *Plugin) commandDB(ctx context.Context) (*gorm.DB, func(), error) { diff --git a/controllers/api/settings.go b/controllers/api/settings.go index c75acae..4b18656 100644 --- a/controllers/api/settings.go +++ b/controllers/api/settings.go @@ -56,6 +56,11 @@ type UpdateSettingsInput struct { PageSegment *string `json:"page_segment,omitempty" jsonschema:"the path form's segment in the default locale, lowercase letters, digits and hyphens (default strona for pl, page for en)"` PageTitleSuffix *string `json:"page_title_suffix,omitempty" jsonschema:"added after the page title on page 2 and later (default locale); placeholders %page% and %pages%"` PageDescriptionSuffix *string `json:"page_description_suffix,omitempty" jsonschema:"added at the end of the meta description on page 2 and later (default locale); placeholders %page% and %pages%"` + CrawlSchedule *string `json:"crawl_schedule,omitempty" jsonschema:"the automatic crawl at night: off, daily or weekly (Mondays)"` + CrawlMaxPages *int `json:"crawl_max_pages,omitempty" jsonschema:"pages per crawl, 10 to 50000 (default 5000)"` + CrawlDelay *int `json:"crawl_delay,omitempty" jsonschema:"the pause between two crawler requests in milliseconds, 100 to 5000 (default 250); at most two requests run at a time"` + CrawlKeep *int `json:"crawl_keep,omitempty" jsonschema:"crawls kept, 1 to 50 (default 10); older crawls are deleted with their pages and issues"` + GenericAnchors *string `json:"generic_anchors,omitempty" jsonschema:"generic link texts, one per line; the crawl reports links whose whole text is one of them, ignoring letter case"` LocalHours *[]models.OpeningHours `json:"local_hours,omitempty" jsonschema:"opening hours rows: day (mon … sun), opens and closes in 24-hour HH:MM; two rows split a day, a day without rows is closed"` Types map[string]classes.TypeSettingWrite `json:"types,omitempty" jsonschema:"per content type key: inject (show the SEO tab), title_template, description_template, robots"` } diff --git a/controllers/settings.go b/controllers/settings.go index b875426..69b8bab 100644 --- a/controllers/settings.go +++ b/controllers/settings.go @@ -56,14 +56,19 @@ func (settingsAdminController) SingletonRecord(_ context.Context, db *gorm.DB) ( return &rows[0], nil } -// FormBeforeUpdate refuses templates with unknown placeholders and -// malformed or duplicate verification file names. +// FormBeforeUpdate refuses templates with unknown placeholders, malformed +// or duplicate verification file names and Site audit values outside their +// ranges. func (c settingsAdminController) FormBeforeUpdate(ctx context.Context, model any) error { s, ok := model.(*models.Settings) if !ok || s == nil { return nil } - if err := problemsError(ctx, translator(c.app), s.Problems()); err != nil { + problems := s.Problems() + for field, list := range s.AuditProblems() { + problems[field] = append(problems[field], list...) + } + if err := problemsError(ctx, translator(c.app), problems); err != nil { return err } s.NofollowDomains = models.NormalizeDomains(s.NofollowDomains) diff --git a/crawl_admin_test.go b/crawl_admin_test.go index ffda2f7..a3b5280 100644 --- a/crawl_admin_test.go +++ b/crawl_admin_test.go @@ -1,6 +1,7 @@ package seo_test import ( + "bytes" "encoding/json" "fmt" "io/fs" @@ -8,9 +9,11 @@ import ( "os" "strings" "testing" + "time" seo "git.golem15.com/golem15/sm-seo-plugin" "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" + "git.golem15.com/golem15/summercms/modules/bonfire" "github.com/goccy/go-yaml" ) @@ -323,3 +326,127 @@ func TestIssueKindLabels(t *testing.T) { } } } + +// stubInput is a console input with bare flags. +type stubInput struct{ flags map[string]string } + +func (stubInput) Args() []string { return nil } +func (stubInput) Argument(string) (string, bool) { return "", false } +func (s stubInput) Flag(n string) (string, bool) { v, ok := s.flags[n]; return v, ok } +func (stubInput) Flags(string) []string { return nil } + +// runCommand runs one of the SEO plugin's console commands. +func (e *testEnv) runCommand(t *testing.T, name string, flags map[string]string) (string, error) { + t.Helper() + for _, c := range e.seoPlugin(t).Commands() { + if c.Name == name { + var buf bytes.Buffer + err := c.Run(t.Context(), stubInput{flags: flags}, bonfire.NewOutput(strings.NewReader(""), &buf, &buf)) + return buf.String(), err + } + } + t.Fatalf("no command %s", name) + return "", nil +} + +// TestCrawlSchedule checks the Site audit settings and the nightly crawl +// (UI-SPEC C20, P33-40): the tab's ranges refuse with validation.between, +// generic_anchors starts with the en and pl lists and drives the +// generic_anchor check, seo:crawl --scheduled does nothing while the +// schedule is off and queues a crawl with trigger schedule when daily, +// weekly runs only on Mondays, and crawls beyond crawl_keep are deleted +// with their pages and issues. +func TestCrawlSchedule(t *testing.T) { + monday := time.Date(2026, 11, 2, 2, 30, 0, 0, time.UTC) + for _, tc := range []struct { + schedule string + day time.Time + want bool + }{ + {"off", monday, false}, {"daily", monday, true}, {"daily", monday.AddDate(0, 0, 1), true}, + {"weekly", monday, true}, {"weekly", monday.AddDate(0, 0, 1), false}, {"weekly", monday.AddDate(0, 0, 7), true}, + } { + if got := crawl.ScheduledDue(tc.schedule, tc.day); got != tc.want { + t.Errorf("ScheduledDue(%s, %s) = %v, want %v", tc.schedule, tc.day.Weekday(), got, tc.want) + } + } + if testing.Short() { + return + } + + env := bootCrawlEnv(t) + token := env.admin(t, true) + settings := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/settings", token, nil).Body.String() + for _, want := range []string{`"crawl_schedule":"off"`, `"crawl_max_pages":5000`, `"crawl_delay":250`, `"crawl_keep":10`, `kliknij tutaj`, `read more`} { + if !jsonContains(settings, want) { + t.Fatalf("settings lack %s: %s", want, settings) + } + } + tab := "" + for _, f := range env.formSchema(t, token, "golem15/seo/settings") { + if f.Name == "crawl_delay" { + tab = f.Tab + } + } + if tab != "Site audit" { + t.Fatalf("crawl_delay tab = %q", tab) + } + for field, value := range map[string]any{"crawl_delay": 50, "crawl_max_pages": 60000, "crawl_keep": 0} { + rec := env.call(t, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{field: value}) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "must be between") { + t.Fatalf("%s=%v = %d %s", field, value, rec.Code, rec.Body.String()) + } + } + env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_delay": 100, "crawl_keep": 1, "generic_anchors": "Zobacz\nclick here"}) + + out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}) + if err != nil || !strings.Contains(out, "No crawl is due") { + t.Fatalf("scheduled while off: %q %v", out, err) + } + var crawls int64 + env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_crawls`).Scan(&crawls) + if crawls != 0 { + t.Fatalf("crawls while off = %d", crawls) + } + + env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_schedule": "daily"}) + site := newFakeSite() + site.sitemap("/") + site.plant(doc{path: "/", title: "Home", body: `

Zobacz kliknij tutaj

`}) + site.plant(doc{path: "/a", title: "A"}, doc{path: "/b", title: "B"}) + out, err = env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}) + if err != nil || !strings.Contains(out, "Queued crawl #") { + t.Fatalf("scheduled while daily: %q %v", out, err) + } + var first struct { + ID uint + Trigger string + } + env.db.Raw(`SELECT id, trigger FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&first) + if first.Trigger != "schedule" { + t.Fatalf("scheduled crawl trigger = %q", first.Trigger) + } + if out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}); err != nil || !strings.Contains(out, "still running") { + t.Fatalf("scheduled while one is queued: %q %v", out, err) + } + ctx := siteCtx(t, site) + env.runCrawl(t, ctx, first.ID) + generic := findIssue(t, env.issues(t, token, ""), "generic_anchor", "/") + if generic.Details["count"] != float64(1) || generic.Details["text"] != "Zobacz" { + t.Fatalf("generic_anchor with the setting = %+v", generic) + } + + // crawl_keep 1: the second crawl deletes the first with its pages + // and issues. + second := env.crawlOnce(t, token, ctx) + var left []uint + env.db.Raw(`SELECT id FROM golem15_seo_crawls ORDER BY id`).Scan(&left) + if len(left) != 1 || left[0] != second { + t.Fatalf("crawls kept = %v, want only %d", left, second) + } + var orphanRows int64 + env.db.Raw(`SELECT (SELECT COUNT(*) FROM golem15_seo_crawl_pages WHERE crawl_id = ?) + (SELECT COUNT(*) FROM golem15_seo_issues WHERE crawl_id = ?)`, first.ID, first.ID).Scan(&orphanRows) + if orphanRows != 0 { + t.Fatalf("pages and issues of the deleted crawl = %d", orphanRows) + } +} diff --git a/lang/en/lang.yaml b/lang/en/lang.yaml index d824fe8..f7044e3 100644 --- a/lang/en/lang.yaml +++ b/lang/en/lang.yaml @@ -83,6 +83,20 @@ status: staging: On this environment every page is served with noindex and robots.txt blocks everything (non-production lock). primary_gone: "The primary category “:name” is no longer assigned, so the first category is used. Choose another one below." settings: + tab_audit: "Site audit" + crawl_schedule: "Automatic crawl" + crawl_schedule_off: "Off" + crawl_schedule_daily: "Daily" + crawl_schedule_weekly: "Weekly" + crawl_schedule_comment: "Runs at night. You can always start a crawl by hand in SEO › Crawls." + crawl_max_pages: "Pages per crawl" + crawl_max_pages_comment: "The crawl stops after this many pages. Default 5000." + crawl_delay: "Pause between requests (ms)" + crawl_delay_comment: "At most two requests run at a time. A longer pause spares a small server." + crawl_keep: "Crawls kept" + crawl_keep_comment: "Older crawls are deleted with their pages and issues." + generic_anchors: "Generic link texts" + generic_anchors_comment: "One per line. The crawl reports links whose whole text is one of these, ignoring letter case." placeholders: Placeholders placeholders_code: Placeholder placeholders_meaning: Replaced with diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index 19c793a..c9c3eb0 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -85,6 +85,20 @@ status: staging: W tym środowisku każda strona ma noindex, a robots.txt blokuje wszystko (blokada środowiska nieprodukcyjnego). primary_gone: "Kategoria główna „:name” nie jest już przypisana, więc używana jest pierwsza kategoria. Wybierz inną poniżej." settings: + tab_audit: "Audyt witryny" + crawl_schedule: "Automatyczne skanowanie" + crawl_schedule_off: "Wyłączone" + crawl_schedule_daily: "Codziennie" + crawl_schedule_weekly: "Co tydzień" + crawl_schedule_comment: "Uruchamiane w nocy. Skanowanie zawsze możesz uruchomić ręcznie w SEO › Skanowania." + crawl_max_pages: "Stron na skanowanie" + crawl_max_pages_comment: "Skanowanie kończy się po tylu stronach. Domyślnie 5000." + crawl_delay: "Przerwa między żądaniami (ms)" + crawl_delay_comment: "Jednocześnie trwają najwyżej dwa żądania. Dłuższa przerwa oszczędza mały serwer." + crawl_keep: "Przechowywane skanowania" + crawl_keep_comment: "Starsze skanowania są usuwane razem ze stronami i problemami." + generic_anchors: "Ogólnikowe teksty linków" + generic_anchors_comment: "Jeden w wierszu. Skanowanie zgłasza linki, których cały tekst jest jednym z nich, bez względu na wielkość liter." placeholders: Znaczniki placeholders_code: Znacznik placeholders_meaning: Zastępowany przez diff --git a/models/settings.go b/models/settings.go index c446c8d..cfa9756 100644 --- a/models/settings.go +++ b/models/settings.go @@ -211,6 +211,15 @@ type Settings struct { LocalLatitude string `gorm:"column:local_latitude"` LocalLongitude string `gorm:"column:local_longitude"` LocalHours lagoon.Jsonable[[]OpeningHours] `gorm:"column:local_hours"` + // The Site audit tab (D-14, UI-SPEC C20): the automatic crawl (off, + // daily or weekly at night), pages per crawl (10 to 50 000), the pause + // between requests in milliseconds (100 to 5000), the crawls kept (1 to + // 50) and the generic link texts, one per line. + CrawlSchedule string `gorm:"column:crawl_schedule"` + CrawlMaxPages int `gorm:"column:crawl_max_pages"` + CrawlDelay int `gorm:"column:crawl_delay"` + CrawlKeep int `gorm:"column:crawl_keep"` + GenericAnchors string `gorm:"column:generic_anchors"` // RoutesReportedAt is the time of the newest route report (D-33), // written by the report only. RoutesReportedAt *time.Time `gorm:"column:routes_reported_at"` @@ -237,7 +246,8 @@ func (Settings) Fillable() []string { "local_price_range", "local_street", "local_postal_code", "local_city", "local_region", "local_country", "local_latitude", "local_longitude", "local_hours", "page_form", "page_segment", "page_title_suffix", - "page_description_suffix", + "page_description_suffix", "crawl_schedule", "crawl_max_pages", + "crawl_delay", "crawl_keep", "generic_anchors", } } @@ -247,19 +257,72 @@ func (Settings) Rules() map[string]string { return map[string]string{} } // DefaultSettings is what the resolver reads when the row is missing. func DefaultSettings() Settings { return Settings{ - ID: 1, - TitleTemplate: DefaultTitleTemplate, - HomeTemplate: DefaultHomeTemplate, - QueryParams: lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true}, - URLHost: HostKeep, - URLSlash: SlashKeep, - LinksNewTab: true, - PageForm: PageFormQuery, - PublisherType: PublisherOrganization, - LocalType: DefaultLocalType, + ID: 1, + TitleTemplate: DefaultTitleTemplate, + HomeTemplate: DefaultHomeTemplate, + QueryParams: lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true}, + URLHost: HostKeep, + URLSlash: SlashKeep, + LinksNewTab: true, + PageForm: PageFormQuery, + PublisherType: PublisherOrganization, + LocalType: DefaultLocalType, + CrawlSchedule: CrawlOff, + CrawlMaxPages: DefaultCrawlMaxPages, + CrawlDelay: DefaultCrawlDelay, + CrawlKeep: DefaultCrawlKeep, + GenericAnchors: DefaultGenericAnchors, } } +// The automatic crawl settings (crawl_schedule). +const ( + CrawlOff = "off" + CrawlDaily = "daily" + CrawlWeekly = "weekly" +) + +// The Site audit defaults and ranges (UI-SPEC C20). +const ( + DefaultCrawlMaxPages = 5000 + DefaultCrawlDelay = 250 + DefaultCrawlKeep = 10 + MinCrawlMaxPages = 10 + MaxCrawlMaxPages = 50_000 + MinCrawlDelay = 100 + MaxCrawlDelay = 5000 + MinCrawlKeep = 1 + MaxCrawlKeep = 50 +) + +// DefaultGenericAnchors is the seed of the generic link texts: the en and +// pl lists of UI-SPEC C27, one per line. +const DefaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link" + +// AuditProblems refuses Site audit values outside their ranges +// (lagoon::validation.between.numeric, UI-SPEC C20) and an unknown +// schedule. The settings screen and the API check it on save; Problems +// leaves it out, so a partial Settings value checks its templates only. +func (s Settings) AuditProblems() map[string][]Problem { + out := map[string][]Problem{} + between := func(field string, v, min, max int) { + if v < min || v > max { + out[field] = append(out[field], Problem{Key: "lagoon::validation.between.numeric", Params: map[string]string{ + "attribute": strings.ReplaceAll(field, "_", " "), "min": strconv.Itoa(min), "max": strconv.Itoa(max), + }}) + } + } + between("crawl_max_pages", s.CrawlMaxPages, MinCrawlMaxPages, MaxCrawlMaxPages) + between("crawl_delay", s.CrawlDelay, MinCrawlDelay, MaxCrawlDelay) + between("crawl_keep", s.CrawlKeep, MinCrawlKeep, MaxCrawlKeep) + switch s.CrawlSchedule { + case CrawlOff, CrawlDaily, CrawlWeekly: + default: + out["crawl_schedule"] = append(out["crawl_schedule"], Problem{Key: "golem15.seo::lang.api.value_invalid"}) + } + return out +} + // Hours returns the local business's opening hours. func (s Settings) Hours() []OpeningHours { return s.LocalHours.Get() } diff --git a/models/settings/fields.yaml b/models/settings/fields.yaml index e50eb14..191be22 100644 --- a/models/settings/fields.yaml +++ b/models/settings/fields.yaml @@ -411,6 +411,50 @@ fields: type: text span: right + crawl_schedule: + label: golem15.seo::lang.settings.crawl_schedule + comment: golem15.seo::lang.settings.crawl_schedule_comment + type: dropdown + span: left + default: "off" + options: + "off": golem15.seo::lang.settings.crawl_schedule_off + daily: golem15.seo::lang.settings.crawl_schedule_daily + weekly: golem15.seo::lang.settings.crawl_schedule_weekly + tab: golem15.seo::lang.settings.tab_audit + + crawl_max_pages: + label: golem15.seo::lang.settings.crawl_max_pages + comment: golem15.seo::lang.settings.crawl_max_pages_comment + type: number + span: right + default: 5000 + tab: golem15.seo::lang.settings.tab_audit + + crawl_delay: + label: golem15.seo::lang.settings.crawl_delay + comment: golem15.seo::lang.settings.crawl_delay_comment + type: number + span: left + default: 250 + tab: golem15.seo::lang.settings.tab_audit + + crawl_keep: + label: golem15.seo::lang.settings.crawl_keep + comment: golem15.seo::lang.settings.crawl_keep_comment + type: number + span: right + default: 10 + tab: golem15.seo::lang.settings.tab_audit + + generic_anchors: + label: golem15.seo::lang.settings.generic_anchors + comment: golem15.seo::lang.settings.generic_anchors_comment + type: textarea + size: small + span: full + tab: golem15.seo::lang.settings.tab_audit + verify_google: label: golem15.seo::lang.settings.verify_google comment: golem15.seo::lang.settings.verify_code_comment diff --git a/schedule.go b/schedule.go new file mode 100644 index 0000000..b09a0d3 --- /dev/null +++ b/schedule.go @@ -0,0 +1,14 @@ +package seo + +import "git.golem15.com/golem15/summercms/modules/pact" + +// Schedule runs seo:crawl --scheduled every night at 02:30 app time +// (P33-40): the command does nothing while the Site audit setting +// crawl_schedule is off, and runs a weekly schedule on Mondays only. +func (p *Plugin) Schedule() []pact.ScheduledCommand { + return []pact.ScheduledCommand{ + {Command: "seo:crawl", Args: []string{"--scheduled"}, Cadence: pact.DailyAt(2, 30)}, + } +} + +var _ pact.HasSchedule = (*Plugin)(nil) diff --git a/settings_test.go b/settings_test.go index e22cd8e..0de57b3 100644 --- a/settings_test.go +++ b/settings_test.go @@ -31,7 +31,7 @@ func TestSettingsTypeTabs(t *testing.T) { env := bootEnv(t) token := env.admin(t, true) fields := env.formSchema(t, token, "golem15/seo/settings") - if got, want := strings.Join(tabsOf(fields), " | "), "General | Page | Tags | Notes | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Verification"; got != want { + if got, want := strings.Join(tabsOf(fields), " | "), "General | Page | Tags | Notes | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Site audit | Verification"; got != want { t.Fatalf("settings tabs = %q, want %q", got, want) } findField(t, fields, "type_acme_page_inject") @@ -81,7 +81,7 @@ func TestSettingsTypeTabs(t *testing.T) { t.Run("no other content types", func(t *testing.T) { env := bootPlugins(t, []string{"golem15.seo"}) token := env.admin(t, true) - if got, want := strings.Join(tabsOf(env.formSchema(t, token, "golem15/seo/settings")), " | "), "General | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Verification"; got != want { + if got, want := strings.Join(tabsOf(env.formSchema(t, token, "golem15/seo/settings")), " | "), "General | Site page | Indexing | Addresses | Pagination | Links | Publisher | Local business | Site audit | Verification"; got != want { t.Fatalf("settings tabs = %q, want %q", got, want) } }) diff --git a/updates/202610280006_add_golem15_seo_audit_settings.go b/updates/202610280006_add_golem15_seo_audit_settings.go new file mode 100644 index 0000000..8ace09e --- /dev/null +++ b/updates/202610280006_add_golem15_seo_audit_settings.go @@ -0,0 +1,41 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// defaultGenericAnchors is the seed of the generic link texts: the en and +// pl lists of UI-SPEC C27, one per line. +const defaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link" + +// addAuditSettings adds the SEO settings' Site audit tab (UI-SPEC C20, +// D-14): the automatic crawl (off, daily or weekly), pages per crawl, the +// pause between requests in milliseconds, the crawls kept and the generic +// link texts, seeded with the en and pl lists. +var addAuditSettings = []*gormigrate.Migration{ + { + ID: "202610280006_add_golem15_seo_audit_settings", + Migrate: func(tx *gorm.DB) error { + if err := tx.Exec(`ALTER TABLE golem15_seo_settings + ADD COLUMN crawl_schedule TEXT NOT NULL DEFAULT 'off', + ADD COLUMN crawl_max_pages INTEGER NOT NULL DEFAULT 5000, + ADD COLUMN crawl_delay INTEGER NOT NULL DEFAULT 250, + ADD COLUMN crawl_keep INTEGER NOT NULL DEFAULT 10, + ADD COLUMN generic_anchors TEXT NOT NULL DEFAULT ''`).Error; err != nil { + return err + } + return tx.Exec(`UPDATE golem15_seo_settings SET generic_anchors = ? WHERE generic_anchors = ''`, defaultGenericAnchors).Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec(`ALTER TABLE golem15_seo_settings + DROP COLUMN IF EXISTS crawl_schedule, DROP COLUMN IF EXISTS crawl_max_pages, + DROP COLUMN IF EXISTS crawl_delay, DROP COLUMN IF EXISTS crawl_keep, + DROP COLUMN IF EXISTS generic_anchors`).Error + }, + }, +} + +func init() { + Register(addAuditSettings...) +}