84cb27a810
BREAKING CHANGE: /sitemap.xsl is gone (404), the definitions screen and the Phase 17 settings tile are replaced by the settings singleton, and priority, change frequency, priority tiers and the update manifest are no longer written or read. The config keys golem15.sitemap.cache_seconds and golem15.sitemap.priority_tiers are removed; site_url, lastmod_ignore and rebuild_delay_seconds are new. - sitemap:import and sitemap:import-winter write the additional addresses (and site_url, default_locale) instead of the definition - the golem15_sitemap_definitions table stays in the database, unread
257 lines
12 KiB
Go
257 lines
12 KiB
Go
package sitemap_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
|
"git.golem15.com/golem15/sm-sitemap-plugin/console"
|
|
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
|
)
|
|
|
|
// storedExtra reads the additional addresses of settings row 1.
|
|
func (e *sitemapEnv) storedExtra(t *testing.T) []models.ExtraURL {
|
|
t.Helper()
|
|
s, err := classes.LoadSettings(e.gdb)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s.Extra()
|
|
}
|
|
|
|
// extraURLs lists the addresses of rows.
|
|
func extraURLs(rows []models.ExtraURL) string {
|
|
out := make([]string, 0, len(rows))
|
|
for _, r := range rows {
|
|
out = append(out, r.URL)
|
|
}
|
|
return strings.Join(out, " ")
|
|
}
|
|
|
|
// TestImportWinter pins sitemap:import-winter on Postgres (D-32): the url
|
|
// items of a WinterCMS definition become the additional addresses (static
|
|
// and CMS page items through --page-url, types renamed by --map first),
|
|
// every other item is dropped and listed, --theme picks a definition,
|
|
// --dry-run writes nothing, stored addresses are replaced only with
|
|
// --force, and frontend_url and frontend_default_locale become site_url and
|
|
// default_locale while the PHP-parity keys are reported as ignored; missing,
|
|
// empty and malformed exports and an unsafe frontend_url are refused.
|
|
func TestImportWinter(t *testing.T) {
|
|
e := bootIndex(t)
|
|
ctx := context.Background()
|
|
dir := t.TempDir()
|
|
row := func(id int, theme string, items string) string {
|
|
quoted, _ := json.Marshal(items)
|
|
return `{"id": ` + string(rune('0'+id)) + `, "theme": "` + theme + `", "data": ` + string(quoted) + "}\n"
|
|
}
|
|
items := `[{"type":"url","url":"/","changefreq":"always","priority":"1.0"},` +
|
|
`{"type":"blog-posts","reference":null,"nesting":"1","cmsPage":"blog/post"},` +
|
|
`{"type":"acme-post","reference":"7"},` +
|
|
`{"type":"static-page","reference":"about"},` +
|
|
`{"type":"cms-page","reference":"news"},` +
|
|
`{"type":"remove-me"},` +
|
|
`{"type":"url","url":"javascript://%0aalert(1)%0a//x"},` +
|
|
`{"type":"url","url":"/"}]`
|
|
defs := writeFile(t, dir, "definitions.jsonl", row(1, "main", items)+"\n"+row(2, "other", `[{"type":"url","url":"/other"}]`))
|
|
run := func(opts console.WinterOptions) (console.WinterReport, string, error) {
|
|
out, buf := testOutput()
|
|
report, err := console.ImportWinterFiles(ctx, e.gdb, opts, out)
|
|
return report, buf.String(), err
|
|
}
|
|
|
|
t.Run("refusals", func(t *testing.T) {
|
|
for name, tc := range map[string]struct {
|
|
opts console.WinterOptions
|
|
want string
|
|
}{
|
|
"no file": {console.WinterOptions{}, "give --definitions, --settings or both"},
|
|
"missing file": {console.WinterOptions{DefinitionsFile: dir + "/missing.jsonl"}, "File not found"},
|
|
"empty file": {console.WinterOptions{DefinitionsFile: writeFile(t, dir, "empty.jsonl", "\n\n")}, "holds no definition"},
|
|
"bad line": {console.WinterOptions{DefinitionsFile: writeFile(t, dir, "bad.jsonl", "{\n")}, "line 1"},
|
|
"several rows": {console.WinterOptions{DefinitionsFile: defs}, "holds 2 definitions (themes 'main', 'other'); pick one with --theme"},
|
|
"unknown theme": {console.WinterOptions{DefinitionsFile: defs, Theme: "nope"}, "no definition of theme 'nope'"},
|
|
"data not items": {console.WinterOptions{DefinitionsFile: writeFile(t, dir, "scalar.jsonl", `{"id":1,"theme":"t","data":"5"}`)}, "not a list of items"},
|
|
"bad settings": {console.WinterOptions{SettingsFile: writeFile(t, dir, "settings-bad.json", `[1]`)}, "must be a JSON object"},
|
|
} {
|
|
_, out, err := run(tc.opts)
|
|
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
|
t.Fatalf("%s: err = %v, want %q\n%s", name, err, tc.want, out)
|
|
}
|
|
}
|
|
if got := e.storedExtra(t); len(got) != 0 {
|
|
t.Fatalf("a refused import wrote %+v", got)
|
|
}
|
|
})
|
|
|
|
opts := console.WinterOptions{DefinitionsFile: defs, Theme: "main", Maps: map[string]string{"remove-me": ""}, PageURLs: map[string]string{"news": "/news"}}
|
|
wantLines := []string{
|
|
"item 1 (type 'blog-posts'): cmsPage 'blog/post' dropped (CMS pages are not supported)",
|
|
"item 1 (type 'blog-posts'): dropped (the content types list their records themselves)",
|
|
"item 2 (type 'acme-post', reference '7'): dropped (the content types list their records themselves)",
|
|
"item 3 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)",
|
|
"item 4 (type 'cms-page', reference 'news'): copied as an additional address '/news'",
|
|
"item 5 (type 'remove-me'): type 'remove-me' mapped to ''",
|
|
"item 5 (type 'remove-me'): dropped (mapped to nothing)",
|
|
"item 6 (type 'url', url 'javascript://%0aalert(1)%0a//x'): dropped (the url is not a path or an http(s) URL)",
|
|
"item 7 (type 'url', url '/'): dropped (address '/' is already listed)",
|
|
}
|
|
|
|
t.Run("dry run writes nothing", func(t *testing.T) {
|
|
dry := opts
|
|
dry.DryRun = true
|
|
report, out, err := run(dry)
|
|
if err != nil {
|
|
t.Fatalf("dry run: %v\n%s", err, out)
|
|
}
|
|
if report.Kept != 2 || report.Dropped != 6 || report.Remapped != 2 {
|
|
t.Fatalf("report = %+v\n%s", report, out)
|
|
}
|
|
for _, line := range wantLines {
|
|
if !strings.Contains(out, line) {
|
|
t.Errorf("output lacks %q:\n%s", line, out)
|
|
}
|
|
}
|
|
if !strings.Contains(out, "Dry run: 2 items kept, 2 remapped, 6 dropped.") || !strings.Contains(out, "Dry run complete (no changes made).") {
|
|
t.Fatalf("dry run output:\n%s", out)
|
|
}
|
|
if got := e.storedExtra(t); len(got) != 0 {
|
|
t.Fatalf("the dry run wrote %+v", got)
|
|
}
|
|
})
|
|
|
|
t.Run("copies, refuses a second copy, replaces with --force", func(t *testing.T) {
|
|
report, out, err := run(opts)
|
|
if err != nil {
|
|
t.Fatalf("import: %v\n%s", err, out)
|
|
}
|
|
if got := extraURLs(e.storedExtra(t)); report.Kept != 2 || got != "/ /news" {
|
|
t.Fatalf("copied = %q (report %+v)", got, report)
|
|
}
|
|
if !strings.Contains(out, "Definition copied: 2 items kept, 2 remapped, 6 dropped.") || !strings.Contains(out, "theme 'main'") {
|
|
t.Fatalf("output:\n%s", out)
|
|
}
|
|
other := console.WinterOptions{DefinitionsFile: defs, Theme: "other"}
|
|
if _, out, err := run(other); err == nil || !strings.Contains(err.Error(), "already has 2 additional addresses; rerun with --force") {
|
|
t.Fatalf("second copy: %v\n%s", err, out)
|
|
}
|
|
dry := other
|
|
dry.DryRun = true
|
|
if _, out, err := run(dry); err != nil || !strings.Contains(out, "already has 2 additional addresses") {
|
|
t.Fatalf("dry run over addresses: %v\n%s", err, out)
|
|
}
|
|
if got := extraURLs(e.storedExtra(t)); got != "/ /news" {
|
|
t.Fatalf("a refused copy changed the addresses: %q", got)
|
|
}
|
|
other.Force = true
|
|
if _, out, err := run(other); err != nil {
|
|
t.Fatalf("--force: %v\n%s", err, out)
|
|
}
|
|
if got := extraURLs(e.storedExtra(t)); got != "/other" {
|
|
t.Fatalf("after --force = %q", got)
|
|
}
|
|
})
|
|
|
|
t.Run("data held as JSON or null", func(t *testing.T) {
|
|
raw := writeFile(t, dir, "raw.jsonl", `{"id":5,"theme":"t","data":[{"type":"url","url":"/raw"}]}`)
|
|
if _, out, err := run(console.WinterOptions{DefinitionsFile: raw, Force: true}); err != nil {
|
|
t.Fatalf("raw JSON data: %v\n%s", err, out)
|
|
}
|
|
if got := extraURLs(e.storedExtra(t)); got != "/raw" {
|
|
t.Fatalf("raw data = %q", got)
|
|
}
|
|
null := writeFile(t, dir, "null.jsonl", `{"id":6,"theme":"t","data":null}`)
|
|
if report, out, err := run(console.WinterOptions{DefinitionsFile: null, Force: true}); err != nil || report.Kept != 0 {
|
|
t.Fatalf("null data: %+v %v\n%s", report, err, out)
|
|
}
|
|
if got := e.storedExtra(t); len(got) != 0 {
|
|
t.Fatalf("null data stored %+v", got)
|
|
}
|
|
})
|
|
|
|
t.Run("settings", func(t *testing.T) {
|
|
first := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test","frontend_default_locale":"pl","enable_priority_tiers":"1","priority_blog":0.7,"unknown":"x"}`)
|
|
report, out, err := run(console.WinterOptions{SettingsFile: first})
|
|
if err != nil || !report.Settings || !strings.Contains(out, "setting 'unknown' ignored (not a sitemap setting of this port)") ||
|
|
!strings.Contains(out, "setting 'priority_blog' ignored (the per-type sitemaps do not use it)") || !strings.Contains(out, "Settings copied.") {
|
|
t.Fatalf("settings: %+v %v\n%s", report, err, out)
|
|
}
|
|
s, err := classes.LoadSettings(e.gdb)
|
|
if err != nil || models.Text(s.SiteURL) != "https://www.example.test" || models.Text(s.DefaultLocale) != "pl" {
|
|
t.Fatalf("copied settings = %+v %v", s, err)
|
|
}
|
|
second := writeFile(t, dir, "settings2.json", `"{\"frontend_url\":\"https://other.example.test\"}"`)
|
|
if _, out, err := run(console.WinterOptions{SettingsFile: second}); err != nil {
|
|
t.Fatalf("second settings: %v\n%s", err, out)
|
|
}
|
|
if s, _ = classes.LoadSettings(e.gdb); models.Text(s.SiteURL) != "https://other.example.test" || s.DefaultLocale != nil {
|
|
t.Fatalf("second settings = %+v", s)
|
|
}
|
|
dry := writeFile(t, dir, "settings3.json", `{"frontend_url":"https://dry.example.test"}`)
|
|
if _, out, err := run(console.WinterOptions{SettingsFile: dry, DryRun: true}); err != nil || !strings.Contains(out, "Dry run: the settings would be copied.") {
|
|
t.Fatalf("settings dry run: %v\n%s", err, out)
|
|
}
|
|
// T-17-15: an unsafe frontend_url stops the import, also as a dry
|
|
// run, and the row keeps its value.
|
|
quoted, _ := json.Marshal(map[string]string{"frontend_url": `https://x"/><y a="`})
|
|
unsafe := writeFile(t, dir, "settings4.json", string(quoted))
|
|
for _, dryRun := range []bool{false, true} {
|
|
report, out, err := run(console.WinterOptions{SettingsFile: unsafe, DryRun: dryRun})
|
|
if err == nil || !strings.Contains(err.Error(), "frontend_url") || report.Settings {
|
|
t.Fatalf("unsafe frontend_url (dry run %v): %+v %v\n%s", dryRun, report, err, out)
|
|
}
|
|
}
|
|
if s, _ = classes.LoadSettings(e.gdb); models.Text(s.SiteURL) != "https://other.example.test" {
|
|
t.Fatalf("a refused or dry import changed the settings: %+v", s)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestImportFile pins sitemap:import: a JSON list of addresses replaces the
|
|
// additional addresses (or is appended, skipping listed ones), lastmod is
|
|
// kept, changefreq and priority are ignored, and a missing url, an unsafe
|
|
// url or a bad lastmod stops the import before anything is written.
|
|
func TestImportFile(t *testing.T) {
|
|
e := bootIndex(t)
|
|
ctx := context.Background()
|
|
dir := t.TempDir()
|
|
manifest := writeFile(t, dir, "urls.json", `[{"url":"/x","changefreq":"daily","lastmod":"2026-10-01"},{"url":"/y","priority":"0.9"}]`)
|
|
out, buf := testOutput()
|
|
if err := console.ImportFile(ctx, e.gdb, manifest, "", false, out); err != nil {
|
|
t.Fatalf("sitemap:import: %v", err)
|
|
}
|
|
got := e.storedExtra(t)
|
|
if extraURLs(got) != "/x /y" || got[0].LastMod != "2026-10-01" || got[1].LastMod != "" {
|
|
t.Fatalf("imported = %+v", got)
|
|
}
|
|
if !strings.Contains(buf.String(), "Imported 2 addresses into the sitemap's additional addresses.") {
|
|
t.Fatalf("output:\n%s", buf.String())
|
|
}
|
|
more := writeFile(t, dir, "more.json", `[{"url":"/y"},{"url":"https://www.example.test/z"}]`)
|
|
out, buf = testOutput()
|
|
if err := console.ImportFile(ctx, e.gdb, more, "main", true, out); err != nil {
|
|
t.Fatalf("--append: %v", err)
|
|
}
|
|
if got := extraURLs(e.storedExtra(t)); got != "/x /y https://www.example.test/z" {
|
|
t.Fatalf("appended = %q", got)
|
|
}
|
|
if !strings.Contains(buf.String(), "Imported 1 addresses") || !strings.Contains(buf.String(), "Kept 2 existing addresses.") || !strings.Contains(buf.String(), "Theme 'main' was given") {
|
|
t.Fatalf("--append output:\n%s", buf.String())
|
|
}
|
|
for name, c := range map[string]struct{ body, want string }{
|
|
"missing url": {`[{"url":"/a"},{"changefreq":"daily"}]`, "Entry 1 is missing a 'url' field"},
|
|
"unsafe url": {`[{"url":"javascript:alert(1)"}]`, "Entry 0 has a url that is not a path or an http(s) URL"},
|
|
"bad lastmod": {`[{"url":"/a","lastmod":"soon"}]`, "Entry 0 has a lastmod that is not a date"},
|
|
"not a list": {`5`, "Invalid JSON"},
|
|
} {
|
|
file := writeFile(t, dir, strings.ReplaceAll(name, " ", "-")+".json", c.body)
|
|
if err := console.ImportFile(ctx, e.gdb, file, "", false, out); err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Fatalf("%s: %v", name, err)
|
|
}
|
|
}
|
|
if got := extraURLs(e.storedExtra(t)); got != "/x /y https://www.example.test/z" {
|
|
t.Fatalf("a refused import changed the addresses: %q", got)
|
|
}
|
|
}
|