From 4e55939a1f42a96a8d68315c69669148192308cb Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Mon, 12 Oct 2026 19:38:04 +0200 Subject: [PATCH] test(17-06): sitemap parity unit tests - TestURLTo (every Url::to golden case, BackendURL, URLPath), TestXMLEscaping (PHP XmlEscapingTest as behaviour), TestTiers (each default rule, first match, off, bad pattern fails Boot), TestRewrite, TestHooks, TestURLCap, TestManifest (fail-safe reads, key normalization, dates), TestFillable (PHP MassAssignmentTest), TestDefinitionItemJSON, TestBackendURL (trusted forwarded proto) - TestImportWinter on Postgres: maps and drops with report, --theme, --dry-run, --force, settings upsert, refusals --- classes/generator_test.go | 151 ++++++++++++++++++++++++++++++ classes/manifest_test.go | 81 ++++++++++++++++ classes/rewrite_test.go | 52 +++++++++++ classes/serialize_test.go | 95 +++++++++++++++++++ classes/tiers_test.go | 115 +++++++++++++++++++++++ classes/urlto_test.go | 57 +++++++++++ import_test.go | 192 ++++++++++++++++++++++++++++++++++++++ models/definition_test.go | 145 ++++++++++++++++++++++++++++ routes_test.go | 69 ++++++++++++++ 9 files changed, 957 insertions(+) create mode 100644 classes/generator_test.go create mode 100644 classes/manifest_test.go create mode 100644 classes/rewrite_test.go create mode 100644 classes/serialize_test.go create mode 100644 classes/tiers_test.go create mode 100644 classes/urlto_test.go create mode 100644 import_test.go create mode 100644 models/definition_test.go create mode 100644 routes_test.go diff --git a/classes/generator_test.go b/classes/generator_test.go new file mode 100644 index 0000000..d04abe5 --- /dev/null +++ b/classes/generator_test.go @@ -0,0 +1,151 @@ +package classes_test + +import ( + "context" + "errors" + "strconv" + "strings" + "testing" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/festival" +) + +func urlItems(paths ...string) []models.DefinitionItem { + out := make([]models.DefinitionItem, 0, len(paths)) + for _, p := range paths { + out = append(out, models.DefinitionItem{Type: models.ItemTypeURL, URL: text(p), Changefreq: text("daily"), Priority: text("0.5")}) + } + return out +} + +// TestHooks pins the typed successors of the PHP events (D-14): a +// BeforeAddItem listener that sets Skip drops the URL before it is counted, +// the first skipping listener stops the rest, AddItem listeners see the +// written loc and their Extra is written before the url closing tag in +// order, and a listener error stops the generation. +func TestHooks(t *testing.T) { + bus := festival.New() + var seenBefore, seenAdd, afterSkip []string + bus.Listen("acme.first", func(_ context.Context, e *classes.BeforeAddItem) error { + seenBefore = append(seenBefore, e.Info.URL) + if strings.HasSuffix(e.Info.URL, "/skip") { + e.Skip = true + } + return nil + }) + bus.Listen("acme.second", func(_ context.Context, e *classes.BeforeAddItem) error { + afterSkip = append(afterSkip, e.Info.URL) + return nil + }) + bus.Listen("acme.extra", func(_ context.Context, e *classes.AddItem) error { + seenAdd = append(seenAdd, e.Info.URL) + e.Extra = append(e.Extra, ""+classes.EscapeText(e.Info.URL+"?a&b")+"") + return nil + }) + bus.Listen("acme.more", func(_ context.Context, e *classes.AddItem) error { + e.Extra = append(e.Extra, "") + return nil + }) + gen := &classes.Generator{Root: "https://h.test", Events: bus} + xml, err := gen.Generate(context.Background(), urlItems("/a", "/skip", "/b")) + if err != nil { + t.Fatal(err) + } + if strings.Join(seenBefore, ",") != "https://h.test/a,https://h.test/skip,https://h.test/b" { + // The url item's URL is filled through Url::to before the event. + t.Fatalf("BeforeAddItem saw %v", seenBefore) + } + if strings.Join(afterSkip, ",") != "https://h.test/a,https://h.test/b" { + t.Fatalf("a listener after the skipping one ran for the skipped URL: %v", afterSkip) + } + if strings.Contains(xml, "/skip") { + t.Fatalf("the skipped URL is written:\n%s", xml) + } + if strings.Join(seenAdd, ",") != "https://h.test/a,https://h.test/b" { + t.Fatalf("AddItem saw %v", seenAdd) + } + want := "https://h.test/adaily0.5https://h.test/a?a&b" + if !strings.Contains(xml, want) { + t.Fatalf("extra not written before the closing tag:\n%s", xml) + } + + // A skip on the exact URL the event carries. + bus2 := festival.New() + var second []string + bus2.Listen("acme.skip", func(_ context.Context, e *classes.BeforeAddItem) error { + e.Skip = strings.HasSuffix(e.Info.URL, "/skip") + return nil + }) + bus2.Listen("acme.later", func(_ context.Context, e *classes.BeforeAddItem) error { + second = append(second, e.Info.URL) + return nil + }) + gen = &classes.Generator{Root: "https://h.test", Events: bus2} + xml, err = gen.Generate(context.Background(), urlItems("/a", "/skip")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(xml, "/skip<") || strings.Join(second, ",") != "https://h.test/a" { + t.Fatalf("UntilHandled did not stop at the skipping listener: %v\n%s", second, xml) + } + + // Without listeners or a bus nothing changes. + gen = &classes.Generator{Root: "https://h.test"} + if xml, err := gen.Generate(context.Background(), urlItems("/a")); err != nil || !strings.Contains(xml, "https://h.test/a") { + t.Fatalf("no bus: %v\n%s", err, xml) + } + + // A listener error stops the generation. + failing := festival.New() + failing.Listen("acme.fail", func(_ context.Context, e *classes.AddItem) error { return errors.New("listener failed") }) + gen = &classes.Generator{Root: "https://h.test", Events: failing} + if _, err := gen.Generate(context.Background(), urlItems("/a")); err == nil || !strings.Contains(err.Error(), "listener failed") { + t.Fatalf("listener error = %v", err) + } +} + +// TestURLCap pins the protocol limit: of 50,001 URLs the 50,000th is +// written and the 50,001st is not; skipped and empty URLs are not counted; +// no items is an empty document. +func TestURLCap(t *testing.T) { + paths := make([]string, classes.MaxURLs+1) + for i := range paths { + paths[i] = "/u/" + strconv.Itoa(i+1) + } + gen := &classes.Generator{Root: "https://api.example.test"} + xml, err := gen.Generate(context.Background(), urlItems(paths...)) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(xml, ""); n != classes.MaxURLs { + t.Fatalf("%d url elements, want %d", n, classes.MaxURLs) + } + if !strings.Contains(xml, "https://api.example.test/u/50000") || strings.Contains(xml, "https://api.example.test/u/50001") { + t.Fatal("the 50000th URL must be written and the 50001st must not") + } + + // Skipped URLs do not count toward the cap. + bus := festival.New() + bus.Listen("acme.skip", func(_ context.Context, e *classes.BeforeAddItem) error { + e.Skip = strings.HasSuffix(e.Info.URL, "/u/1") + return nil + }) + gen = &classes.Generator{Root: "https://api.example.test", Events: bus} + xml, err = gen.Generate(context.Background(), urlItems(paths...)) + if err != nil { + t.Fatal(err) + } + if strings.Count(xml, "") != classes.MaxURLs || !strings.Contains(xml, "/u/50001") { + t.Fatal("a skipped URL took a place under the cap") + } + + // A generator is reusable: a second run starts its count again. + if again, err := gen.Generate(context.Background(), urlItems("/x")); err != nil || strings.Count(again, "") != 1 { + t.Fatalf("second run = %v, %d urls", err, strings.Count(again, "")) + } + if empty, err := (&classes.Generator{Root: "https://h"}).Generate(context.Background(), nil); err != nil || empty != "" { + t.Fatalf("no items = %q %v", empty, err) + } +} diff --git a/classes/manifest_test.go b/classes/manifest_test.go new file mode 100644 index 0000000..f6c66eb --- /dev/null +++ b/classes/manifest_test.go @@ -0,0 +1,81 @@ +package classes_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" +) + +// TestManifest pins the updates manifest reader: it fails safe (a missing, +// empty-path, directory, oversized, non-object or invalid file is no +// manifest, so lastmod is omitted), normalizes keys, ignores non-string +// values, and parses the ISO dates strtotime reads, a date without a zone +// as UTC. +func TestManifest(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path + } + for name, path := range map[string]string{ + "empty path": "", + "spaces": " ", + "missing": filepath.Join(dir, "missing.json"), + "directory": dir, + "list": write("list.json", `["/a"]`), + "string": write("string.json", `"x"`), + "invalid": write("invalid.json", `{"/a": `), + "empty file": write("empty.json", ""), + "oversized": write("big.json", `{"/a":"2026-01-01","pad":"`+strings.Repeat("x", classes.ManifestMaxBytes)+`"}`), + "null": write("null.json", "null"), + "leading space": write("space.json", " \n"), + } { + if m := classes.LoadManifest(path); m != nil { + t.Errorf("%s: manifest = %v, want none", name, m) + } + } + if fifo := filepath.Join(dir, "link.json"); os.Symlink(dir, fifo) == nil { + if m := classes.LoadManifest(fifo); m != nil { + t.Error("a symlink to a directory is not a manifest") + } + } + + path := write("updates.json", ` {"blog/a/":"2026-01-02T03:04:05+02:00","/b":"2026-02-03","c":"2026-03-04 05:06:07","/":"2026-04-05T06:07","/num":12,"/bad":"yesterday","/epoch":"1970-01-01T00:00:00Z","/nested":{"x":1}}`) + m := classes.LoadManifest(path) + if m == nil { + t.Fatal("valid manifest not read") + } + if _, ok := m["/num"]; ok { + t.Fatal("a non-string value was kept") + } + for url, want := range map[string]string{ + "https://h/blog/a": "2026-01-02T01:04:05Z", + "https://h/blog/a/": "2026-01-02T01:04:05Z", + "https://h/b?x=1": "2026-02-03T00:00:00Z", + "/c": "2026-03-04T05:06:07Z", + "https://h": "2026-04-05T06:07:00Z", + "https://h/": "2026-04-05T06:07:00Z", + } { + got, ok := m.Lastmod(url) + if !ok || got.UTC().Format(time.RFC3339) != want { + t.Errorf("Lastmod(%s) = %v %v, want %s", url, got, ok, want) + } + } + for _, url := range []string{"https://h/bad", "https://h/epoch", "https://h/unlisted", "https://h/num"} { + if got, ok := m.Lastmod(url); ok { + t.Errorf("Lastmod(%s) = %v, want none", url, got) + } + } + var none classes.Manifest + if _, ok := none.Lastmod("https://h/b"); ok { + t.Fatal("a nil manifest answered") + } +} diff --git a/classes/rewrite_test.go b/classes/rewrite_test.go new file mode 100644 index 0000000..b896399 --- /dev/null +++ b/classes/rewrite_test.go @@ -0,0 +1,52 @@ +package classes_test + +import ( + "testing" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" +) + +// TestRewrite pins the PHP routes.php frontend_url rewrite: the backend URL +// is replaced inside loc, inside xhtml:link href and in the exact stylesheet +// href, nowhere else; slashes are trimmed on both; an empty or "0" frontend +// URL and equal URLs leave the document unchanged. +func TestRewrite(t *testing.T) { + const backend = "https://api.ex.pl" + doc := "\n" + + `` + "\n" + + `https://api.ex.pl/a2026https://api.ex.pl1` + + `` + + `https://api.ex.pl/xhttps://other.pl/https://api.ex.plcp` + "\n" + want := "\n" + + `` + "\n" + + `https://www.ex.pl/a2026https://api.ex.pl1` + + `` + + `https://api.ex.pl/xhttps://other.pl/https://www.ex.plcp` + "\n" + for _, frontend := range []string{"https://www.ex.pl", "https://www.ex.pl/", "https://www.ex.pl///"} { + for _, back := range []string{backend, backend + "/"} { + if got := classes.Rewrite(doc, back, frontend); got != want { + t.Fatalf("Rewrite(%q, %q):\n%s\nwant:\n%s", back, frontend, got, want) + } + } + } + for _, frontend := range []string{"", "0"} { + if got := classes.Rewrite(doc, backend, frontend); got != doc { + t.Fatalf("frontend %q changed the document", frontend) + } + } + if got := classes.Rewrite(doc, backend, backend); got != doc { + t.Fatal("equal URLs changed the document") + } + if got := classes.Rewrite(doc, backend+"/", backend); got != doc { + t.Fatal("equal URLs up to a slash changed the document") + } + // An empty backend URL replaces nothing. + if got := classes.Rewrite(doc, "", "https://www.ex.pl"); got != doc { + t.Fatal("an empty backend URL changed the document") + } + // A stylesheet on another path is not the PHP pattern. + other := `` + if got := classes.Rewrite(other, backend, "https://www.ex.pl"); got != other { + t.Fatalf("other stylesheet = %q", got) + } +} diff --git a/classes/serialize_test.go b/classes/serialize_test.go new file mode 100644 index 0000000..4980471 --- /dev/null +++ b/classes/serialize_test.go @@ -0,0 +1,95 @@ +package classes_test + +import ( + "context" + "strings" + "testing" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/sm-sitemap-plugin/models" +) + +// TestXMLEscaping ports the PHP XmlEscapingTest (SITE-002) as behaviour: +// every value the sitemap writes is escaped exactly as libxml escapes it, +// text nodes (loc, lastmod, changefreq, priority) and attribute values +// (hreflang, href), and no raw value reaches the output. +func TestXMLEscaping(t *testing.T) { + const raw = "a&bd\"e'f\rg\th\ni" + if got := classes.EscapeText(raw); got != "a&b<c>d\"e'f g\th\ni" { + t.Fatalf("EscapeText = %q", got) + } + if got := classes.EscapeAttr(raw); got != "a&b<c>d"e'f g h i" { + t.Fatalf("EscapeAttr = %q", got) + } + if got := classes.EscapeText("zażółć — &"); got != "zażółć — &amp;" { + t.Fatalf("EscapeText keeps UTF-8 and escapes an existing entity again: %q", got) + } + + xml := classes.Serialize("https://h/sitemap.xsl", []classes.URLEntry{{ + Loc: "https://h/a?x=1&y=<2>", + Lastmod: "2026-01-02T03:04:05+00:00", + Changefreq: "weekly&<>", + Priority: "0.5\r", + Alternates: []classes.Alternate{ + {Hreflang: "pl\"x", Href: "https://h/pl?a=1&b=\"2\"\t\n"}, + {Hreflang: "en", Href: "https://h/en'x"}, + }, + }}) + want := "\n" + + "\n" + + `` + + "https://h/a?x=1&y=<2>2026-01-02T03:04:05+00:00weekly&<>0.5 " + + `` + + `` + + "\n" + if xml != want { + t.Fatalf("serialized:\n%q\nwant:\n%q", xml, want) + } + + // One alternate is not written; an empty urlset is self-closed. + single := classes.Serialize("s", []classes.URLEntry{{Loc: "l", Changefreq: "c", Priority: "p", Alternates: []classes.Alternate{{Hreflang: "pl", Href: "x"}}}}) + if strings.Contains(single, "xhtml:link") || strings.Contains(single, "") { + t.Fatalf("single alternate or empty lastmod written: %q", single) + } + if empty := classes.Serialize("s", nil); !strings.HasSuffix(empty, `sitemap.xsd"/>`+"\n") { + t.Fatalf("empty urlset = %q", empty) + } + + // Through the generator: stored changefreq and priority, a resolved + // alternate's locale and a url item with every special character. + url := "/q?a=1&b=&c=\"y\"" + changefreq := "daily&" + priority := "0.9\"'" + gen := &classes.Generator{Root: "https://api.example.test", Settings: models.Settings{}, Resolver: fakeResolver{t: t, types: map[string]map[string]goldenResolved{ + "acme-post": {"1": {URL: "/p?x=1&y=2", Alternates: [][2]string{{"pl&<", "/p?x=1&y=2"}, {"en\"", "/en/p?z=<1>"}}}}, + }}} + out, err := gen.Generate(context.Background(), []models.DefinitionItem{ + {Type: models.ItemTypeURL, URL: &url, Changefreq: &changefreq, Priority: &priority}, + {Type: "acme-post", Reference: text("1"), Changefreq: &changefreq, Priority: &priority}, + }) + if err != nil { + t.Fatal(err) + } + body := out[strings.Index(out, ""):] + for _, bad := range []string{"", "&b", "&y", `"y"`, "", "daily&<", `hreflang="pl&<"`, `en""`, "<1>"} { + if strings.Contains(body, bad) { + t.Fatalf("raw value %q reached the output:\n%s", bad, body) + } + } + for _, good := range []string{ + "daily&<weekly>", + "0.9\"'", + `hreflang="pl&<"`, + `hreflang="en""`, + "y=2", + } { + if !strings.Contains(body, good) { + t.Fatalf("output lacks %q:\n%s", good, body) + } + } + if strings.Count(body, "") != 2 { + t.Fatalf("urls = %d:\n%s", strings.Count(body, ""), body) + } +} + +func text(s string) *string { return &s } diff --git a/classes/tiers_test.go b/classes/tiers_test.go new file mode 100644 index 0000000..05c44d8 --- /dev/null +++ b/classes/tiers_test.go @@ -0,0 +1,115 @@ +package classes_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + sitemap "git.golem15.com/golem15/sm-sitemap-plugin" + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/compass" +) + +// TestTiers pins the priority tiers (D-13): each default rule of the PHP +// getPriorityForUrl with paths from its list, trailing slashes, the first +// match winning, the settings' values, tiers turned off, a host list that +// replaces the defaults, and a bad rule failing the plugin's Boot. +func TestTiers(t *testing.T) { + tiers := pluginTiers(t) + if len(tiers) != 5 { + t.Fatalf("default rules = %d", len(tiers)) + } + s := models.DefaultSettings() + for url, want := range map[string]string{ + "https://h": "1.0", + "https://h/": "1.0", + "https://h/en": "1.0", + "https://h/en/": "1.0", + "https://h/uslugi/strony-www": "0.8", + "https://h/en/services/web": "0.8", + "https://h/realizacje/projekt/": "0.8", + "https://h/en/work/x": "0.8", + "https://h/blog/kategoria/wpis": "0.6", + "https://h/en/blog/category/post/": "0.6", + "https://h/blog": "0.4", + "https://h/blog/kategoria": "0.4", + "https://h/uslugi": "0.4", + "https://h/en/work": "0.4", + "https://h/polityka-prywatnosci": "0.2", + "https://h/en/terms/": "0.2", + "https://h/laboratorium-ai/zabawa-animacja": "0.2", + "https://h/en/ai-lab/blob-playground": "0.2", + "https://h/kontakt?x=1": "0.2", + } { + if got, ok := tiers.Priority(url, s); !ok || got != want { + t.Errorf("Priority(%s) = %q %v, want %s", url, got, ok, want) + } + } + for _, url := range []string{"https://h/other", "https://h/blog/a/b/c", "https://h/uslugi/a/b", "https://h/en/blogs/x", "https://h/kontakt/more"} { + if got, ok := tiers.Priority(url, s); ok { + t.Errorf("Priority(%s) = %q, want no rule", url, got) + } + } + custom := s + custom.PriorityBlog = text("0.65") + if got, _ := tiers.Priority("https://h/blog/a/b", custom); got != "0.65" { + t.Fatalf("custom blog priority = %q", got) + } + off := s + off.EnablePriorityTiers = false + if _, ok := tiers.Priority("https://h/", off); ok { + t.Fatal("tiers apply while turned off") + } + + // First match wins: an exact rule listed before a pattern that also matches. + ordered, err := classes.CompileTiers([]classes.TierConfig{ + {Setting: models.PriorityLegal, Exact: []string{"/blog"}}, + {Setting: models.PriorityCategories, Pattern: "^/blog$"}, + }) + if err != nil { + t.Fatal(err) + } + if got, _ := ordered.Priority("https://h/blog/", s); got != "0.2" { + t.Fatalf("first match = %q", got) + } + if empty, err := classes.CompileTiers(nil); err != nil || len(empty) != 0 { + t.Fatalf("no rules = %v %v", empty, err) + } + if none, err := classes.LoadTiers(nil); err != nil || len(none) != 0 { + t.Fatalf("LoadTiers(nil) = %v %v", none, err) + } + + // Every refused rule names its index; a bad RE2 pattern too. + for name, rule := range map[string]classes.TierConfig{ + "unknown setting": {Setting: "priority_other", Exact: []string{""}}, + "neither": {Setting: models.PriorityHomepage}, + "both": {Setting: models.PriorityHomepage, Exact: []string{""}, Pattern: "^/$"}, + "lookahead": {Setting: models.PriorityBlog, Pattern: "^/(?=blog)"}, + "unbalanced": {Setting: models.PriorityBlog, Pattern: "^/(blog"}, + } { + _, err := classes.CompileTiers([]classes.TierConfig{{Setting: models.PriorityHomepage, Exact: []string{""}}, rule}) + if err == nil || !strings.Contains(err.Error(), classes.TiersConfigKey+"[1]") { + t.Errorf("%s: err = %v", name, err) + } + } + + // A bad host rule fails the plugin's Boot. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte("sitemap:\n priority_tiers:\n - setting: priority_blog\n pattern: '^/(blog'\n"), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) + if err != nil { + t.Fatal(err) + } + p := &sitemap.Plugin{} + if err := cfg.MergePlugin("golem15.sitemap", p.ConfigFS()); err != nil { + t.Fatal(err) + } + if err := p.Boot(backpack.New(cfg)); err == nil || !strings.Contains(err.Error(), "priority_tiers[0]") { + t.Fatalf("Boot with a bad pattern = %v", err) + } +} diff --git a/classes/urlto_test.go b/classes/urlto_test.go new file mode 100644 index 0000000..f2e6d0c --- /dev/null +++ b/classes/urlto_test.go @@ -0,0 +1,57 @@ +package classes_test + +import ( + "encoding/json" + "strings" + "testing" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" +) + +// TestURLTo runs every case of testdata/urlto.json through URLTo and +// compares it with the line PHP's Url::to wrote to urlto.golden (SC-3), then +// pins the helpers built on it: BackendURL and URLPath. +func TestURLTo(t *testing.T) { + var in struct { + Root string `json:"root"` + URLs []*string `json:"urls"` + } + if err := json.Unmarshal(readTestdata(t, "urlto.json"), &in); err != nil { + t.Fatal(err) + } + want := strings.Split(strings.TrimSuffix(string(readTestdata(t, "urlto.golden")), "\n"), "\n") + if len(in.URLs) < 50 || len(want) != len(in.URLs) { + t.Fatalf("%d inputs, %d golden lines", len(in.URLs), len(want)) + } + for i, u := range in.URLs { + path := "" + input := "" + if u != nil { + path, input = *u, *u + } + if got := classes.URLTo(in.Root, input); got != want[i] { + t.Errorf("case %d URLTo(%q) = %q, PHP %q", i, path, got, want[i]) + } + } + + for _, tc := range []struct{ root, want string }{ + {"https://api.ex.pl", "https://api.ex.pl"}, + {"https://api.ex.pl/", "https://api.ex.pl"}, + {"http://h:8080/base", "http://h:8080/base"}, + } { + if got := classes.BackendURL(tc.root); got != tc.want { + t.Errorf("BackendURL(%q) = %q, want %q", tc.root, got, tc.want) + } + } + for raw, want := range map[string]string{ + "https://h/a/b?x=1#f": "/a/b", + "https://h": "", + "https://h/": "/", + "/rel/path": "/rel/path", + "": "", + } { + if got := classes.URLPath(raw); got != want { + t.Errorf("URLPath(%q) = %q, want %q", raw, got, want) + } + } +} diff --git a/import_test.go b/import_test.go new file mode 100644 index 0000000..1350a7d --- /dev/null +++ b/import_test.go @@ -0,0 +1,192 @@ +package sitemap_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.golem15.com/golem15/sm-sitemap-plugin/console" + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/signpost" +) + +// TestImportWinter pins sitemap:import-winter on Postgres (D-12): type maps +// and drops with their report lines, the reference check, --theme with +// several exported definitions, --dry-run writing nothing, the refusal of a +// second copy without --force and the replacement with it, the settings +// upsert onto row 1 with PHP defaults for missing keys, and the refusals for +// missing, empty and malformed exports. +func TestImportWinter(t *testing.T) { + e := bootSitemap(t) + ctx := context.Background() + reg, ok := e.app.Lookup[*signpost.Registry]() + if !ok { + t.Fatal("signpost registry is not published") + } + dir := t.TempDir() + row := func(id int, theme string, items string) string { + quoted, _ := json.Marshal(items) + return `{"id": ` + string(rune('0'+id)) + `, "theme": "` + theme + `", "data": ` + string(quoted) + "}\n" + } + items := `[{"type":"url","url":"/"},` + + `{"type":"old-posts","reference":null,"nesting":"1"},` + + `{"type":"acme-post","reference":"7"},` + + `{"type":"acme-post","reference":"8"},` + + `{"type":"gone-type","reference":"1"},` + + `{"type":"static-page","reference":"about"},` + + `{"type":"remove-me"}]` + defs := writeFile(t, dir, "definitions.jsonl", row(1, "main", items)+"\n"+row(2, "other", `[{"type":"url","url":"/other"}]`)) + stored := func(t *testing.T) []models.DefinitionItem { + t.Helper() + var rows []models.Definition + if err := e.gdb.Where("theme = ?", models.DefinitionTheme).Find(&rows).Error; err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + return nil + } + if len(rows) != 1 { + t.Fatalf("%d definitions", len(rows)) + } + return rows[0].Items() + } + run := func(opts console.WinterOptions) (console.WinterReport, string, error) { + out, buf := testOutput() + report, err := console.ImportWinterFiles(ctx, e.gdb, reg, opts, out) + return report, buf.String(), err + } + + t.Run("refusals", func(t *testing.T) { + for name, tc := range map[string]struct { + opts console.WinterOptions + want string + }{ + "no file": {console.WinterOptions{}, "give --definitions, --settings or both"}, + "missing file": {console.WinterOptions{DefinitionsFile: dir + "/missing.jsonl"}, "File not found"}, + "empty file": {console.WinterOptions{DefinitionsFile: writeFile(t, dir, "empty.jsonl", "\n\n")}, "holds no definition"}, + "bad line": {console.WinterOptions{DefinitionsFile: writeFile(t, dir, "bad.jsonl", "{\n")}, "line 1"}, + "several rows": {console.WinterOptions{DefinitionsFile: defs}, "holds 2 definitions (themes 'main', 'other'); pick one with --theme"}, + "unknown theme": {console.WinterOptions{DefinitionsFile: defs, Theme: "nope"}, "no definition of theme 'nope'"}, + "data not items": {console.WinterOptions{DefinitionsFile: writeFile(t, dir, "scalar.jsonl", `{"id":1,"theme":"t","data":"5"}`)}, "not a list of items"}, + "bad settings": {console.WinterOptions{SettingsFile: writeFile(t, dir, "settings-bad.json", `[1]`)}, "must be a JSON object"}, + } { + _, out, err := run(tc.opts) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("%s: err = %v, want %q\n%s", name, err, tc.want, out) + } + } + if got := stored(t); got != nil { + t.Fatalf("a refused import wrote %+v", got) + } + }) + + opts := console.WinterOptions{DefinitionsFile: defs, Theme: "main", Maps: map[string]string{"old-posts": "acme-list", "remove-me": ""}} + wantLines := []string{ + "item 1 (type 'old-posts'): type 'old-posts' mapped to 'acme-list'", + "item 3 (type 'acme-post', reference '8'): dropped (reference '8' is not among the current references of 'acme-post')", + "item 4 (type 'gone-type', reference '1'): dropped (type 'gone-type' is not registered; map it with --map gone-type=)", + "item 5 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", + "item 6 (type 'remove-me'): dropped (mapped to nothing)", + } + + t.Run("dry run writes nothing", func(t *testing.T) { + dry := opts + dry.DryRun = true + report, out, err := run(dry) + if err != nil { + t.Fatalf("dry run: %v\n%s", err, out) + } + if report.Kept != 3 || report.Dropped != 4 || report.Remapped != 2 { + t.Fatalf("report = %+v\n%s", report, out) + } + for _, line := range wantLines { + if !strings.Contains(out, line) { + t.Errorf("output lacks %q:\n%s", line, out) + } + } + if !strings.Contains(out, "Dry run: 3 items kept, 2 remapped, 4 dropped.") || !strings.Contains(out, "Dry run complete (no changes made).") { + t.Fatalf("dry run output:\n%s", out) + } + if got := stored(t); got != nil { + t.Fatalf("the dry run wrote %+v", got) + } + }) + + t.Run("copies, refuses a second copy, replaces with --force", func(t *testing.T) { + report, out, err := run(opts) + if err != nil { + t.Fatalf("import: %v\n%s", err, out) + } + got := stored(t) + if report.Kept != 3 || len(got) != 3 || got[0].Type != "url" || got[1].Type != "acme-list" || !got[1].Nesting || got[2].Type != "acme-post" || models.Text(got[2].Reference) != "7" { + t.Fatalf("copied = %+v (report %+v)", got, report) + } + if !strings.Contains(out, "Definition copied: 3 items kept, 2 remapped, 4 dropped.") || !strings.Contains(out, "theme 'main'") { + t.Fatalf("output:\n%s", out) + } + other := console.WinterOptions{DefinitionsFile: defs, Theme: "other"} + if _, out, err := run(other); err == nil || !strings.Contains(err.Error(), "already has 3 items; rerun with --force") { + t.Fatalf("second copy: %v\n%s", err, out) + } + // A dry run over existing items warns instead of failing. + dry := other + dry.DryRun = true + if _, out, err := run(dry); err != nil || !strings.Contains(out, "already has 3 items") { + t.Fatalf("dry run over items: %v\n%s", err, out) + } + if len(stored(t)) != 3 { + t.Fatal("a refused copy changed the definition") + } + other.Force = true + if _, out, err := run(other); err != nil { + t.Fatalf("--force: %v\n%s", err, out) + } + if got := stored(t); len(got) != 1 || models.Text(got[0].URL) != "/other" { + t.Fatalf("after --force = %+v", got) + } + }) + + t.Run("data held as JSON or null", func(t *testing.T) { + raw := writeFile(t, dir, "raw.jsonl", `{"id":5,"theme":"t","data":[{"type":"url","url":"/raw"}]}`) + if _, out, err := run(console.WinterOptions{DefinitionsFile: raw, Force: true}); err != nil { + t.Fatalf("raw JSON data: %v\n%s", err, out) + } + if got := stored(t); len(got) != 1 || models.Text(got[0].URL) != "/raw" { + t.Fatalf("raw data = %+v", got) + } + null := writeFile(t, dir, "null.jsonl", `{"id":6,"theme":"t","data":null}`) + if report, out, err := run(console.WinterOptions{DefinitionsFile: null, Force: true}); err != nil || report.Kept != 0 { + t.Fatalf("null data: %+v %v\n%s", report, err, out) + } + if got := stored(t); len(got) != 0 { + t.Fatalf("null data stored %+v", got) + } + }) + + t.Run("settings upsert", func(t *testing.T) { + first := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test","enable_priority_tiers":"1","priority_blog":0.7,"unknown":"x"}`) + report, out, err := run(console.WinterOptions{SettingsFile: first}) + if err != nil || !report.Settings || !strings.Contains(out, "setting 'unknown' ignored") || !strings.Contains(out, "Settings copied.") { + t.Fatalf("settings: %+v %v\n%s", report, err, out) + } + second := writeFile(t, dir, "settings2.json", `"{\"frontend_url\":\"https://other.example.test\",\"enable_priority_tiers\":\"0\"}"`) + if _, out, err := run(console.WinterOptions{SettingsFile: second}); err != nil { + t.Fatalf("second settings: %v\n%s", err, out) + } + var rows []models.Settings + if err := e.gdb.Find(&rows).Error; err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].ID != 1 || models.Text(rows[0].FrontendURL) != "https://other.example.test" || rows[0].EnablePriorityTiers || rows[0].Priority(models.PriorityBlog) != "0.6" { + t.Fatalf("settings rows = %+v", rows) + } + dry := writeFile(t, dir, "settings3.json", `{"frontend_url":"https://dry.example.test"}`) + if _, out, err := run(console.WinterOptions{SettingsFile: dry, DryRun: true}); err != nil || !strings.Contains(out, "Dry run: the settings would be copied.") { + t.Fatalf("settings dry run: %v\n%s", err, out) + } + if err := e.gdb.Find(&rows).Error; err != nil || models.Text(rows[0].FrontendURL) != "https://other.example.test" { + t.Fatalf("the dry run changed the settings: %+v %v", rows, err) + } + }) +} diff --git a/models/definition_test.go b/models/definition_test.go new file mode 100644 index 0000000..1c9eb77 --- /dev/null +++ b/models/definition_test.go @@ -0,0 +1,145 @@ +package models_test + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/lagoon" +) + +// TestFillable ports the PHP MassAssignmentTest (SITE-001) as behaviour: +// Definition.Fillable is exactly theme and data, and a fill with extra keys +// (id, timestamps, unknown keys) writes only those two. +func TestFillable(t *testing.T) { + if got := (models.Definition{}).Fillable(); !slices.Equal(got, []string{"theme", "data"}) { + t.Fatalf("Fillable = %v, want [theme data]", got) + } + def := models.Definition{ID: 4} + requested := map[string]any{ + "id": 99, + "theme": "site", + "data": `[{"type":"url","url":"/a"}]`, + "created_at": "2020-01-01 00:00:00", + "updated_at": "2020-01-01 00:00:00", + "is_admin": true, + } + for _, production := range []bool{true, false} { + d := def + if err := lagoon.Fill(&d, d.Fillable(), requested, production); err != nil { + t.Fatalf("Fill: %v", err) + } + if d.ID != 4 || d.CreatedAt != nil || d.UpdatedAt != nil || models.Text(d.Theme) != "site" { + t.Fatalf("filled definition = %+v", d) + } + if items := d.Items(); len(items) != 1 || models.Text(items[0].URL) != "/a" { + t.Fatalf("filled data = %+v", items) + } + } + if (models.Definition{}).TableName() != "golem15_sitemap_definitions" || (models.Definition{}).MorphName() != `Golem15\Sitemap\Models\Definition` { + t.Fatal("table or morph name changed") + } + if rules := (models.Definition{}).Rules(); len(rules) != 0 { + t.Fatalf("Rules = %v", rules) + } + if got := (models.Settings{}).Fillable(); slices.Contains(got, "id") || len(got) != 9 { + t.Fatalf("Settings.Fillable = %v", got) + } +} + +// TestDefinitionItemJSON pins the tolerant decoding of PHP-stored items +// (strings, numbers, booleans and null for text fields, PHP truthiness for +// nesting, unknown keys ignored, a missing type is url, a null item a +// default url item, an object-keyed list in key order) and the PHP toArray +// encoding (key set and order, cmsPage always null, null text kept). +func TestDefinitionItemJSON(t *testing.T) { + for _, tc := range []struct { + raw string + want string + }{ + {`{"type":"url","url":"/a","priority":"0.5","changefreq":"daily","nesting":"0","reference":null,"urlPattern":null,"cmsPage":"x","code":"c","exists":false,"fillable":["a"]}`, + `type=url url=/a ref= pattern= nesting=false priority=0.5 changefreq=daily`}, + {`{"url":"/b"}`, `type=url url=/b ref= pattern= nesting=false priority= changefreq=`}, + {`{"type":null}`, `type= url= ref= pattern= nesting=false priority= changefreq=`}, + {`{"type":"journal-post","reference":7,"priority":0.5,"changefreq":true,"nesting":1}`, + `type=journal-post url= ref=7 pattern= nesting=true priority=0.5 changefreq=1`}, + {`{"type":"x","priority":1.0,"reference":12.50,"changefreq":false,"nesting":"1"}`, + `type=x url= ref=12.5 pattern= nesting=true priority=1 changefreq=`}, + {`{"type":"x","nesting":"false"}`, `type=x url= ref= pattern= nesting=true priority= changefreq=`}, + {`{"type":"x","nesting":""}`, `type=x url= ref= pattern= nesting=false priority= changefreq=`}, + {`{"type":"x","nesting":null,"urlPattern":"/p/:slug"}`, `type=x url= ref= pattern=/p/:slug nesting=false priority= changefreq=`}, + {`{"type":"x","reference":12345678901234}`, `type=x url= ref=12345678901234 pattern= nesting=false priority= changefreq=`}, + {`null`, `type=url url= ref= pattern= nesting=false priority= changefreq=`}, + } { + var item models.DefinitionItem + if err := json.Unmarshal([]byte(tc.raw), &item); err != nil { + t.Fatalf("%s: %v", tc.raw, err) + } + if got := describe(item); got != tc.want { + t.Errorf("%s\n got %s\nwant %s", tc.raw, got, tc.want) + } + } + for _, raw := range []string{`{"type":["x"]}`, `{"url":{"a":1}}`, `{"nesting":[1]}`, `[1]`, `"x"`} { + var item models.DefinitionItem + if err := json.Unmarshal([]byte(raw), &item); err == nil { + t.Errorf("%s decoded without an error", raw) + } + } + + // A list, an object keyed like a PHP array with gaps, null and empty. + var list models.DefinitionItems + if err := json.Unmarshal([]byte(`{"3":{"url":"/c"},"1":{"url":"/a"},"x":{"url":"/x"}}`), &list); err != nil || len(list) != 3 || + models.Text(list[0].URL) != "/c" || models.Text(list[1].URL) != "/a" || models.Text(list[2].URL) != "/x" { + t.Fatalf("object list = %+v %v", list, err) + } + for raw, n := range map[string]int{`[]`: 0, `null`: 0, `[{"url":"/a"},null]`: 2} { + var l models.DefinitionItems + if err := json.Unmarshal([]byte(raw), &l); err != nil || len(l) != n { + t.Fatalf("%s = %+v %v", raw, l, err) + } + } + var bad models.DefinitionItems + if err := json.Unmarshal([]byte(`"items"`), &bad); err == nil { + t.Fatal("a string decoded as an item list") + } + + // Encoding: the PHP toArray key set and order. + pattern := "/p/:slug" + item := models.DefinitionItem{Type: "journal-post", Reference: str("7"), URLPattern: &pattern, Nesting: true, Priority: str("0.5")} + raw, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + if want := `{"changefreq":null,"priority":"0.5","nesting":true,"type":"journal-post","url":null,"reference":"7","cmsPage":null,"urlPattern":"/p/:slug"}`; string(raw) != want { + t.Fatalf("Marshal = %s\nwant %s", raw, want) + } + if raw, _ := json.Marshal(models.DefinitionItems(nil)); string(raw) != "[]" { + t.Fatalf("nil list = %s", raw) + } + var back models.DefinitionItem + if err := json.Unmarshal(raw, &back); err != nil || describe(back) != describe(item) { + t.Fatalf("round trip = %s, %v", describe(back), err) + } + if models.Text(nil) != "" || models.Text(str("x")) != "x" { + t.Fatal("Text") + } + if s, err := models.DecodeText(json.RawMessage(`1e3`)); err != nil || *s != "1000" { + t.Fatalf("DecodeText(1e3) = %v %v", s, err) + } +} + +func str(s string) *string { return &s } + +func describe(d models.DefinitionItem) string { + p := func(s *string) string { + if s == nil { + return "" + } + return *s + } + return strings.TrimSpace(fmt.Sprintf("type=%s url=%s ref=%s pattern=%s nesting=%v priority=%s changefreq=%s", + d.Type, p(d.URL), p(d.Reference), p(d.URLPattern), d.Nesting, p(d.Priority), p(d.Changefreq))) +} diff --git a/routes_test.go b/routes_test.go new file mode 100644 index 0000000..63a83f1 --- /dev/null +++ b/routes_test.go @@ -0,0 +1,69 @@ +package sitemap + +import ( + "crypto/tls" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/compass" +) + +// TestBackendURL pins the root URL the sitemap's absolute URLs and the +// frontend_url rewrite start from: app.url when set; else the request's +// Host with https for TLS, or for the last X-Forwarded-Proto hop of a peer +// in http.trusted_proxies, while a client that sends the header directly +// gets http. +func TestBackendURL(t *testing.T) { + app := func(t *testing.T, files map[string]string) *backpack.App { + t.Helper() + dir := t.TempDir() + for name, body := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) + if err != nil { + t.Fatal(err) + } + return backpack.New(cfg) + } + trusted := app(t, map[string]string{"http.yaml": "trusted_proxies:\n - 10.0.0.0/8\n"}) + for _, tc := range []struct { + name string + app *backpack.App + remote string + proto []string + tls bool + want string + }{ + {"app.url wins", app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n"}), "203.0.113.9:1234", []string{"http"}, false, "https://api.example.test"}, + {"plain request", trusted, "203.0.113.9:1234", nil, false, "http://h.test"}, + {"TLS", trusted, "203.0.113.9:1234", nil, true, "https://h.test"}, + {"trusted proxy says https", trusted, "10.1.2.3:5000", []string{"https"}, false, "https://h.test"}, + {"trusted proxy, last hop wins", trusted, "10.1.2.3:5000", []string{"https, http"}, false, "http://h.test"}, + {"trusted proxy, last header wins", trusted, "10.1.2.3:5000", []string{"http", "HTTPS"}, false, "https://h.test"}, + {"untrusted client says https", trusted, "203.0.113.9:1234", []string{"https"}, false, "http://h.test"}, + {"no trusted proxies configured", app(t, nil), "10.1.2.3:5000", []string{"https"}, false, "http://h.test"}, + {"nil app", nil, "10.1.2.3:5000", []string{"https"}, false, "http://h.test"}, + {"IPv4-mapped trusted peer", trusted, "[::ffff:10.1.2.3]:5000", []string{"https"}, false, "https://h.test"}, + {"unparsable peer", trusted, "not-an-address", []string{"https"}, false, "http://h.test"}, + } { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest("GET", "http://h.test/sitemap.xml", nil) + r.RemoteAddr = tc.remote + for _, v := range tc.proto { + r.Header.Add("X-Forwarded-Proto", v) + } + if tc.tls { + r.TLS = &tls.ConnectionState{} + } + if got := backendURL(tc.app, r); got != tc.want { + t.Fatalf("backendURL = %q, want %q", got, tc.want) + } + }) + } +}