feat: site audit settings and the nightly crawl
- Site audit tab (UI-SPEC C20): crawl_schedule, crawl_max_pages (10-50 000), crawl_delay (100-5000 ms), crawl_keep (1-50) and generic_anchors seeded with the en and pl lists; ranges refuse with lagoon::validation.between on the screen and through the API - the crawler reads its limits and generic anchors from the settings; crawls beyond crawl_keep are deleted with their pages and issues - seo:crawl [--scheduled] and the nightly schedule entry at 02:30 (P33-40): off does nothing, weekly runs on Mondays in app.timezone - TestCrawlSchedule
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package seo_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -8,9 +9,11 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seo "git.golem15.com/golem15/sm-seo-plugin"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
||||
"git.golem15.com/golem15/summercms/modules/bonfire"
|
||||
"github.com/goccy/go-yaml"
|
||||
)
|
||||
|
||||
@@ -323,3 +326,127 @@ func TestIssueKindLabels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stubInput is a console input with bare flags.
|
||||
type stubInput struct{ flags map[string]string }
|
||||
|
||||
func (stubInput) Args() []string { return nil }
|
||||
func (stubInput) Argument(string) (string, bool) { return "", false }
|
||||
func (s stubInput) Flag(n string) (string, bool) { v, ok := s.flags[n]; return v, ok }
|
||||
func (stubInput) Flags(string) []string { return nil }
|
||||
|
||||
// runCommand runs one of the SEO plugin's console commands.
|
||||
func (e *testEnv) runCommand(t *testing.T, name string, flags map[string]string) (string, error) {
|
||||
t.Helper()
|
||||
for _, c := range e.seoPlugin(t).Commands() {
|
||||
if c.Name == name {
|
||||
var buf bytes.Buffer
|
||||
err := c.Run(t.Context(), stubInput{flags: flags}, bonfire.NewOutput(strings.NewReader(""), &buf, &buf))
|
||||
return buf.String(), err
|
||||
}
|
||||
}
|
||||
t.Fatalf("no command %s", name)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// TestCrawlSchedule checks the Site audit settings and the nightly crawl
|
||||
// (UI-SPEC C20, P33-40): the tab's ranges refuse with validation.between,
|
||||
// generic_anchors starts with the en and pl lists and drives the
|
||||
// generic_anchor check, seo:crawl --scheduled does nothing while the
|
||||
// schedule is off and queues a crawl with trigger schedule when daily,
|
||||
// weekly runs only on Mondays, and crawls beyond crawl_keep are deleted
|
||||
// with their pages and issues.
|
||||
func TestCrawlSchedule(t *testing.T) {
|
||||
monday := time.Date(2026, 11, 2, 2, 30, 0, 0, time.UTC)
|
||||
for _, tc := range []struct {
|
||||
schedule string
|
||||
day time.Time
|
||||
want bool
|
||||
}{
|
||||
{"off", monday, false}, {"daily", monday, true}, {"daily", monday.AddDate(0, 0, 1), true},
|
||||
{"weekly", monday, true}, {"weekly", monday.AddDate(0, 0, 1), false}, {"weekly", monday.AddDate(0, 0, 7), true},
|
||||
} {
|
||||
if got := crawl.ScheduledDue(tc.schedule, tc.day); got != tc.want {
|
||||
t.Errorf("ScheduledDue(%s, %s) = %v, want %v", tc.schedule, tc.day.Weekday(), got, tc.want)
|
||||
}
|
||||
}
|
||||
if testing.Short() {
|
||||
return
|
||||
}
|
||||
|
||||
env := bootCrawlEnv(t)
|
||||
token := env.admin(t, true)
|
||||
settings := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/settings", token, nil).Body.String()
|
||||
for _, want := range []string{`"crawl_schedule":"off"`, `"crawl_max_pages":5000`, `"crawl_delay":250`, `"crawl_keep":10`, `kliknij tutaj`, `read more`} {
|
||||
if !jsonContains(settings, want) {
|
||||
t.Fatalf("settings lack %s: %s", want, settings)
|
||||
}
|
||||
}
|
||||
tab := ""
|
||||
for _, f := range env.formSchema(t, token, "golem15/seo/settings") {
|
||||
if f.Name == "crawl_delay" {
|
||||
tab = f.Tab
|
||||
}
|
||||
}
|
||||
if tab != "Site audit" {
|
||||
t.Fatalf("crawl_delay tab = %q", tab)
|
||||
}
|
||||
for field, value := range map[string]any{"crawl_delay": 50, "crawl_max_pages": 60000, "crawl_keep": 0} {
|
||||
rec := env.call(t, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{field: value})
|
||||
if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "must be between") {
|
||||
t.Fatalf("%s=%v = %d %s", field, value, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_delay": 100, "crawl_keep": 1, "generic_anchors": "Zobacz\nclick here"})
|
||||
|
||||
out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"})
|
||||
if err != nil || !strings.Contains(out, "No crawl is due") {
|
||||
t.Fatalf("scheduled while off: %q %v", out, err)
|
||||
}
|
||||
var crawls int64
|
||||
env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_crawls`).Scan(&crawls)
|
||||
if crawls != 0 {
|
||||
t.Fatalf("crawls while off = %d", crawls)
|
||||
}
|
||||
|
||||
env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"crawl_schedule": "daily"})
|
||||
site := newFakeSite()
|
||||
site.sitemap("/")
|
||||
site.plant(doc{path: "/", title: "Home", body: `<p><a href="/a">Zobacz</a> <a href="/b">kliknij tutaj</a></p>`})
|
||||
site.plant(doc{path: "/a", title: "A"}, doc{path: "/b", title: "B"})
|
||||
out, err = env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"})
|
||||
if err != nil || !strings.Contains(out, "Queued crawl #") {
|
||||
t.Fatalf("scheduled while daily: %q %v", out, err)
|
||||
}
|
||||
var first struct {
|
||||
ID uint
|
||||
Trigger string
|
||||
}
|
||||
env.db.Raw(`SELECT id, trigger FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&first)
|
||||
if first.Trigger != "schedule" {
|
||||
t.Fatalf("scheduled crawl trigger = %q", first.Trigger)
|
||||
}
|
||||
if out, err := env.runCommand(t, "seo:crawl", map[string]string{"scheduled": "true"}); err != nil || !strings.Contains(out, "still running") {
|
||||
t.Fatalf("scheduled while one is queued: %q %v", out, err)
|
||||
}
|
||||
ctx := siteCtx(t, site)
|
||||
env.runCrawl(t, ctx, first.ID)
|
||||
generic := findIssue(t, env.issues(t, token, ""), "generic_anchor", "/")
|
||||
if generic.Details["count"] != float64(1) || generic.Details["text"] != "Zobacz" {
|
||||
t.Fatalf("generic_anchor with the setting = %+v", generic)
|
||||
}
|
||||
|
||||
// crawl_keep 1: the second crawl deletes the first with its pages
|
||||
// and issues.
|
||||
second := env.crawlOnce(t, token, ctx)
|
||||
var left []uint
|
||||
env.db.Raw(`SELECT id FROM golem15_seo_crawls ORDER BY id`).Scan(&left)
|
||||
if len(left) != 1 || left[0] != second {
|
||||
t.Fatalf("crawls kept = %v, want only %d", left, second)
|
||||
}
|
||||
var orphanRows int64
|
||||
env.db.Raw(`SELECT (SELECT COUNT(*) FROM golem15_seo_crawl_pages WHERE crawl_id = ?) + (SELECT COUNT(*) FROM golem15_seo_issues WHERE crawl_id = ?)`, first.ID, first.ID).Scan(&orphanRows)
|
||||
if orphanRows != 0 {
|
||||
t.Fatalf("pages and issues of the deleted crawl = %d", orphanRows)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user