From 84cb27a810f3924f75f76e64cdfac0bb3c318fd7 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 11:12:51 +0100 Subject: [PATCH] feat!: remove the PHP-parity sitemap, tiers, definitions screen and XSL BREAKING CHANGE: /sitemap.xsl is gone (404), the definitions screen and the Phase 17 settings tile are replaced by the settings singleton, and priority, change frequency, priority tiers and the update manifest are no longer written or read. The config keys golem15.sitemap.cache_seconds and golem15.sitemap.priority_tiers are removed; site_url, lastmod_ignore and rebuild_delay_seconds are new. - sitemap:import and sitemap:import-winter write the additional addresses (and site_url, default_locale) instead of the definition - the golem15_sitemap_definitions table stays in the database, unread --- assets/sitemap.xsl | 261 --------- classes/events.go | 56 -- classes/generator.go | 268 --------- classes/generator_test.go | 183 ------ classes/golden_test.go | 343 ----------- classes/manifest.go | 112 ---- classes/manifest_test.go | 81 --- classes/rewrite.go | 69 --- classes/rewrite_test.go | 131 ----- classes/serialize.go | 105 ---- classes/serialize_test.go | 95 ---- classes/testdata/full.golden | 58 -- classes/testdata/full.json | 693 ----------------------- classes/testdata/tracer.golden | 3 - classes/testdata/tracer.json | 18 - classes/testdata/urlto.golden | 58 -- classes/testdata/urlto.json | 63 --- classes/tiers.go | 98 ---- classes/tiers_test.go | 94 --- classes/urlto.go | 566 ------------------ classes/urlto_test.go | 84 --- config/config.yaml | 56 +- console/helpers.go | 39 +- console/import.go | 197 +++---- console/import_winter.go | 311 +++++----- console/import_winter_test.go | 5 +- controllers/admin_registry.go | 3 +- controllers/definitions.go | 162 ------ controllers/definitions/config_form.yaml | 24 - controllers/definitions_test.go | 39 -- controllers/settings.go | 12 +- import_test.go | 229 ++++---- lang/en/lang.yaml | 41 +- lang/pl/lang.yaml | 41 +- models/definition.go | 48 -- models/definition/fields.yaml | 69 --- models/definition_item.go | 255 --------- models/definition_test.go | 145 ----- models/legacy.go | 177 ++++++ models/settings.go | 103 +--- sitemap_smoke_test.go | 93 +-- tests/php/gen-goldens.php | 465 --------------- 42 files changed, 582 insertions(+), 5371 deletions(-) delete mode 100644 assets/sitemap.xsl delete mode 100644 classes/events.go delete mode 100644 classes/generator.go delete mode 100644 classes/generator_test.go delete mode 100644 classes/golden_test.go delete mode 100644 classes/manifest.go delete mode 100644 classes/manifest_test.go delete mode 100644 classes/rewrite.go delete mode 100644 classes/rewrite_test.go delete mode 100644 classes/serialize.go delete mode 100644 classes/serialize_test.go delete mode 100644 classes/testdata/full.golden delete mode 100644 classes/testdata/full.json delete mode 100644 classes/testdata/tracer.golden delete mode 100644 classes/testdata/tracer.json delete mode 100644 classes/testdata/urlto.golden delete mode 100644 classes/testdata/urlto.json delete mode 100644 classes/tiers.go delete mode 100644 classes/tiers_test.go delete mode 100644 classes/urlto.go delete mode 100644 classes/urlto_test.go delete mode 100644 controllers/definitions.go delete mode 100644 controllers/definitions/config_form.yaml delete mode 100644 controllers/definitions_test.go delete mode 100644 models/definition.go delete mode 100644 models/definition/fields.yaml delete mode 100644 models/definition_item.go delete mode 100644 models/definition_test.go create mode 100644 models/legacy.go delete mode 100644 tests/php/gen-goldens.php diff --git a/assets/sitemap.xsl b/assets/sitemap.xsl deleted file mode 100644 index c6b6bbb..0000000 --- a/assets/sitemap.xsl +++ /dev/null @@ -1,261 +0,0 @@ - - - - - - - - - - Sitemap - <xsl:if test="sitemap:sitemapindex">Index</xsl:if> - - - - -
-
-

Sitemap

- - Index - - - Images - - - Video - - - Xhtml - -
-

- - - This index contains - - sitemaps. - - - This index contains - - URLs. - - -

-

- This is an XML sitemap, meant for consumption by search engines. - You can find more information about XML sitemaps on sitemaps.org. -

-
- - - -
-
- - - - - - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
URLLast Modified
- - - - - - - -
-
-
- - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
URLChange Freq.PriorityLast Modified
- - -

- - - -

- - - - 		- -
-
-
- - - - - - - - - - - - - - - -

- - - - - - - - - - - - - - - - - - - - - -

- - - - - - - -

- Image: - - - - - - - - - -

- - - - - - - - - - - - - - - - -

- Video: - - - - - - - - - - - - thumb - - - - - - - -

- - - diff --git a/classes/events.go b/classes/events.go deleted file mode 100644 index 15d41cd..0000000 --- a/classes/events.go +++ /dev/null @@ -1,56 +0,0 @@ -package classes - -import ( - "time" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// ItemInfo is what the sitemap knows about one URL it is about to write, the -// PHP $itemInfo array: URL (before the final Url::to in BeforeAddItem, the -// written loc in AddItem), MTime, the record's modification time when the -// item has one, Lastmod, the written lastmod ("" when omitted), and the -// hreflang Alternates. -type ItemInfo struct { - URL string - MTime *time.Time - Lastmod string - Alternates []Alternate -} - -// BeforeAddItem is the typed successor of the PHP event -// golem15.sitemap.beforeAddItem. The generator fires it with -// festival.Bus.UntilHandled for every URL before it is counted and written; -// a listener that sets Skip drops the URL (PHP: a listener returned false). -// -// app.Events.Listen("acme.seo", func(ctx context.Context, e *classes.BeforeAddItem) error { -// if strings.HasSuffix(e.Info.URL, "/draft") { -// e.Skip = true -// } -// return nil -// }) -type BeforeAddItem struct { - Item models.DefinitionItem - Info ItemInfo - Skip bool -} - -// IsHandled reports Skip, so UntilHandled stops at the first listener that -// drops the URL (festival.Handleable). -func (e *BeforeAddItem) IsHandled() bool { return e != nil && e.Skip } - -// AddItem is the typed successor of the PHP event golem15.sitemap.addItem. -// The generator fires it with festival.Bus.Fire after a URL's elements are -// built; listeners may append child XML to Extra, which is written as it is, -// before the url closing tag, so it must be escaped already (EscapeText, -// EscapeAttr). -// -// app.Events.Listen("acme.images", func(ctx context.Context, e *classes.AddItem) error { -// e.Extra = append(e.Extra, ""+classes.EscapeText(img)+"") -// return nil -// }) -type AddItem struct { - Item models.DefinitionItem - Info ItemInfo - Extra []string -} diff --git a/classes/generator.go b/classes/generator.go deleted file mode 100644 index 8960c9a..0000000 --- a/classes/generator.go +++ /dev/null @@ -1,268 +0,0 @@ -// Package classes generates the sitemap: it walks the definition items, -// resolves registry items through the framework's menu item type registry -// and writes the PHP-identical XML. -package classes - -import ( - "context" - "time" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/festival" - "git.golem15.com/golem15/summercms/modules/pact" -) - -// MaxURLs is the sitemap protocol's URL limit (PHP Definition::MAX_URLS): -// the 50000th URL is written, the 50001st is not. -const MaxURLs = 50000 - -// lastmodLayout is PHP date('c') in UTC: "+00:00", never "Z". -const lastmodLayout = "2006-01-02T15:04:05-07:00" - -// Resolver resolves a registry item; *signpost.Registry implements it. -type Resolver interface { - Resolve(ctx context.Context, key, reference, itemPattern string, nesting bool, unprefixedLocale string) (*pact.ResolvedMenuItem, error) -} - -// Generator writes the sitemap of a definition, following PHP -// Definition::generateSitemap and addItemToSet. -// -// Root is the backend root URL (scheme and host without a trailing slash) -// that Url::to builds absolute URLs from. Resolver resolves registry items; -// nil skips them. Settings are the sitemap settings (tiers, priorities, -// frontend default locale, manifest path), Tiers the compiled tier rules, -// and Events the bus BeforeAddItem and AddItem are fired on (nil fires -// nothing). Manifest, when set, is used instead of reading the settings' -// updates_manifest_path. -type Generator struct { - Root string - Resolver Resolver - Settings models.Settings - Tiers Tiers - Events *festival.Bus - Manifest Manifest - - manifestLoaded bool - entries []URLEntry - count int -} - -// itemInfo is PHP's $itemInfo: the URL (set marks it present), the record's -// modification time and the hreflang alternates. -type itemInfo struct { - url string - set bool - mtime *time.Time - alternates []Alternate -} - -// Generate returns the sitemap XML of items, or "" when there are none (PHP -// generateSitemap). A url item is added with its URL. A registry item is -// resolved with its URL pattern, its nesting and the settings' frontend -// default locale as the unprefixed locale; its URL is added when it has one, -// then its sub-items: when the item has no URL of its own (a list type), -// each sub-item is added once per alternate, with that alternate's URL as -// loc and every alternate as hreflang, as the PHP list types returned them; -// otherwise each sub-item is added once with its own URL. A type the -// registry does not know, or a Resolve that finds nothing, adds nothing. -// -// Generate does not apply the frontend_url rewrite; the route does -// (Rewrite). -func (g *Generator) Generate(ctx context.Context, items []models.DefinitionItem) (string, error) { - if len(items) == 0 { - return "", nil - } - if ctx == nil { - ctx = context.Background() - } - g.entries, g.count = nil, 0 - for i := range items { - item := &items[i] - if item.Type == models.ItemTypeURL { - if err := g.addItem(ctx, item, itemInfo{}); err != nil { - return "", err - } - continue - } - resolved, err := g.resolve(ctx, item) - if err != nil { - return "", err - } - if resolved == nil { - continue - } - if resolved.URL != "" { - if err := g.addItem(ctx, item, infoOf(*resolved)); err != nil { - return "", err - } - if err := g.addChildren(ctx, item, resolved.Items); err != nil { - return "", err - } - continue - } - if err := g.addPerLocale(ctx, item, resolved.Items); err != nil { - return "", err - } - } - return Serialize(g.Stylesheet(), g.entries), nil -} - -// Stylesheet is the stylesheet href PHP built: -// Url::buildUrl(Url::current(), ['path' => 'sitemap.xsl']). -func (g *Generator) Stylesheet() string { - return replacePath(URLTo(g.Root, "/sitemap.xml"), "sitemap.xsl") -} - -func (g *Generator) resolve(ctx context.Context, item *models.DefinitionItem) (*pact.ResolvedMenuItem, error) { - if g.Resolver == nil || item.Type == "" { - return nil, nil - } - if reg, ok := g.Resolver.(interface { - Type(string) (pact.MenuItemType, bool) - }); ok { - if _, known := reg.Type(item.Type); !known { - return nil, nil - } - } - return g.Resolver.Resolve(ctx, item.Type, models.Text(item.Reference), models.Text(item.URLPattern), item.Nesting, models.Text(g.Settings.FrontendDefaultLocale)) -} - -// addChildren adds every child with a URL, then its own children. -func (g *Generator) addChildren(ctx context.Context, item *models.DefinitionItem, children []pact.ResolvedMenuItem) error { - for _, child := range children { - if child.URL != "" { - if err := g.addItem(ctx, item, infoOf(child)); err != nil { - return err - } - } - if err := g.addChildren(ctx, item, child.Items); err != nil { - return err - } - } - return nil -} - -// addPerLocale adds the children of a list type: each once per alternate -// (once with its URL when it has no alternates), then its own children. -func (g *Generator) addPerLocale(ctx context.Context, item *models.DefinitionItem, children []pact.ResolvedMenuItem) error { - for _, child := range children { - info := infoOf(child) - if len(child.Alternates) == 0 { - if child.URL != "" { - if err := g.addItem(ctx, item, info); err != nil { - return err - } - } - if err := g.addChildren(ctx, item, child.Items); err != nil { - return err - } - continue - } - for _, alt := range child.Alternates { - perLocale := info - perLocale.url = alt.URL - if perLocale.url != "" { - if err := g.addItem(ctx, item, perLocale); err != nil { - return err - } - } - if err := g.addChildren(ctx, item, child.Items); err != nil { - return err - } - } - } - return nil -} - -func infoOf(r pact.ResolvedMenuItem) itemInfo { - info := itemInfo{url: r.URL, set: true, mtime: r.MTime} - for _, a := range r.Alternates { - info.alternates = append(info.alternates, Alternate{Hreflang: a.Locale, Href: a.URL}) - } - return info -} - -func (g *Generator) manifest() Manifest { - if !g.manifestLoaded { - if g.Manifest == nil { - g.Manifest = LoadManifest(models.Text(g.Settings.UpdatesManifestPath)) - } - g.manifestLoaded = true - } - return g.Manifest -} - -// addItem is PHP Definition::addItemToSet, in its order: the URL cap, a -// missing URL filled through Url::to, lastmod from the record or the -// manifest, BeforeAddItem, an empty URL dropped and the URL passed through -// Url::to again, the count, the tier or item priority, the elements and -// AddItem. Unlike PHP, a URL or alternate that does not come out as an -// http or https URL is dropped. -func (g *Generator) addItem(ctx context.Context, item *models.DefinitionItem, info itemInfo) error { - if g.count >= MaxURLs { - return nil - } - if !info.set { - info.url = URLTo(g.Root, models.Text(item.URL)) - } - var lastmod time.Time - hasLastmod := false - if info.mtime != nil { - lastmod, hasLastmod = *info.mtime, true - } else if t, ok := g.manifest().Lastmod(info.url); ok { - lastmod, hasLastmod = t, true - } - lastmodText := "" - if hasLastmod && lastmod.Unix() != 0 { - lastmodText = lastmod.UTC().Format(lastmodLayout) - } - if g.Events != nil { - before := &BeforeAddItem{Item: *item, Info: ItemInfo{URL: info.url, MTime: info.mtime, Lastmod: lastmodText, Alternates: append([]Alternate(nil), info.alternates...)}} - skipped, err := g.Events.UntilHandled(ctx, before) - if err != nil { - return err - } - if skipped { - return nil - } - } - if info.url == "" || info.url == "0" { - return nil - } - loc := URLTo(g.Root, info.url) - if !httpURL(loc) { - // Only http and https URLs are written: a javascript: or other - // scheme that an import, a registry type or a pattern let through - // would be a clickable link in sitemap.xsl. - return nil - } - g.count++ - priority, ok := g.Tiers.Priority(loc, g.Settings) - if !ok { - priority = models.Text(item.Priority) - } - entry := URLEntry{ - Loc: loc, - Lastmod: lastmodText, - Changefreq: models.Text(item.Changefreq), - Priority: priority, - } - if len(info.alternates) > 1 { - for _, a := range info.alternates { - href := URLTo(g.Root, a.Href) - if !httpURL(href) { - continue - } - entry.Alternates = append(entry.Alternates, Alternate{Hreflang: a.Hreflang, Href: href}) - } - } - if g.Events != nil { - added := &AddItem{Item: *item, Info: ItemInfo{URL: loc, MTime: info.mtime, Lastmod: lastmodText, Alternates: append([]Alternate(nil), entry.Alternates...)}} - if err := g.Events.Fire(ctx, added); err != nil { - return err - } - entry.Extra = added.Extra - } - g.entries = append(g.entries, entry) - return nil -} diff --git a/classes/generator_test.go b/classes/generator_test.go deleted file mode 100644 index 2d28c45..0000000 --- a/classes/generator_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package classes_test - -import ( - "context" - "errors" - "strconv" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/festival" -) - -func urlItems(paths ...string) []models.DefinitionItem { - out := make([]models.DefinitionItem, 0, len(paths)) - for _, p := range paths { - out = append(out, models.DefinitionItem{Type: models.ItemTypeURL, URL: text(p), Changefreq: text("daily"), Priority: text("0.5")}) - } - return out -} - -// TestHooks pins the typed successors of the PHP events (D-14): a -// BeforeAddItem listener that sets Skip drops the URL before it is counted, -// the first skipping listener stops the rest, AddItem listeners see the -// written loc and their Extra is written before the url closing tag in -// order, and a listener error stops the generation. -func TestHooks(t *testing.T) { - bus := festival.New() - var seenBefore, seenAdd, afterSkip []string - bus.Listen("acme.first", func(_ context.Context, e *classes.BeforeAddItem) error { - seenBefore = append(seenBefore, e.Info.URL) - if strings.HasSuffix(e.Info.URL, "/skip") { - e.Skip = true - } - return nil - }) - bus.Listen("acme.second", func(_ context.Context, e *classes.BeforeAddItem) error { - afterSkip = append(afterSkip, e.Info.URL) - return nil - }) - bus.Listen("acme.extra", func(_ context.Context, e *classes.AddItem) error { - seenAdd = append(seenAdd, e.Info.URL) - e.Extra = append(e.Extra, ""+classes.EscapeText(e.Info.URL+"?a&b")+"") - return nil - }) - bus.Listen("acme.more", func(_ context.Context, e *classes.AddItem) error { - e.Extra = append(e.Extra, "") - return nil - }) - gen := &classes.Generator{Root: "https://h.test", Events: bus} - xml, err := gen.Generate(context.Background(), urlItems("/a", "/skip", "/b")) - if err != nil { - t.Fatal(err) - } - if strings.Join(seenBefore, ",") != "https://h.test/a,https://h.test/skip,https://h.test/b" { - // The url item's URL is filled through Url::to before the event. - t.Fatalf("BeforeAddItem saw %v", seenBefore) - } - if strings.Join(afterSkip, ",") != "https://h.test/a,https://h.test/b" { - t.Fatalf("a listener after the skipping one ran for the skipped URL: %v", afterSkip) - } - if strings.Contains(xml, "/skip") { - t.Fatalf("the skipped URL is written:\n%s", xml) - } - if strings.Join(seenAdd, ",") != "https://h.test/a,https://h.test/b" { - t.Fatalf("AddItem saw %v", seenAdd) - } - want := "https://h.test/adaily0.5https://h.test/a?a&b" - if !strings.Contains(xml, want) { - t.Fatalf("extra not written before the closing tag:\n%s", xml) - } - - // A skip on the exact URL the event carries. - bus2 := festival.New() - var second []string - bus2.Listen("acme.skip", func(_ context.Context, e *classes.BeforeAddItem) error { - e.Skip = strings.HasSuffix(e.Info.URL, "/skip") - return nil - }) - bus2.Listen("acme.later", func(_ context.Context, e *classes.BeforeAddItem) error { - second = append(second, e.Info.URL) - return nil - }) - gen = &classes.Generator{Root: "https://h.test", Events: bus2} - xml, err = gen.Generate(context.Background(), urlItems("/a", "/skip")) - if err != nil { - t.Fatal(err) - } - if strings.Contains(xml, "/skip<") || strings.Join(second, ",") != "https://h.test/a" { - t.Fatalf("UntilHandled did not stop at the skipping listener: %v\n%s", second, xml) - } - - // Without listeners or a bus nothing changes. - gen = &classes.Generator{Root: "https://h.test"} - if xml, err := gen.Generate(context.Background(), urlItems("/a")); err != nil || !strings.Contains(xml, "https://h.test/a") { - t.Fatalf("no bus: %v\n%s", err, xml) - } - - // A listener error stops the generation. - failing := festival.New() - failing.Listen("acme.fail", func(_ context.Context, e *classes.AddItem) error { return errors.New("listener failed") }) - gen = &classes.Generator{Root: "https://h.test", Events: failing} - if _, err := gen.Generate(context.Background(), urlItems("/a")); err == nil || !strings.Contains(err.Error(), "listener failed") { - t.Fatalf("listener error = %v", err) - } -} - -// TestURLCap pins the protocol limit: of 50,001 URLs the 50,000th is -// written and the 50,001st is not; skipped and empty URLs are not counted; -// no items is an empty document. -func TestURLCap(t *testing.T) { - paths := make([]string, classes.MaxURLs+1) - for i := range paths { - paths[i] = "/u/" + strconv.Itoa(i+1) - } - gen := &classes.Generator{Root: "https://api.example.test"} - xml, err := gen.Generate(context.Background(), urlItems(paths...)) - if err != nil { - t.Fatal(err) - } - if n := strings.Count(xml, ""); n != classes.MaxURLs { - t.Fatalf("%d url elements, want %d", n, classes.MaxURLs) - } - if !strings.Contains(xml, "https://api.example.test/u/50000") || strings.Contains(xml, "https://api.example.test/u/50001") { - t.Fatal("the 50000th URL must be written and the 50001st must not") - } - - // Skipped URLs do not count toward the cap. - bus := festival.New() - bus.Listen("acme.skip", func(_ context.Context, e *classes.BeforeAddItem) error { - e.Skip = strings.HasSuffix(e.Info.URL, "/u/1") - return nil - }) - gen = &classes.Generator{Root: "https://api.example.test", Events: bus} - xml, err = gen.Generate(context.Background(), urlItems(paths...)) - if err != nil { - t.Fatal(err) - } - if strings.Count(xml, "") != classes.MaxURLs || !strings.Contains(xml, "/u/50001") { - t.Fatal("a skipped URL took a place under the cap") - } - - // A generator is reusable: a second run starts its count again. - if again, err := gen.Generate(context.Background(), urlItems("/x")); err != nil || strings.Count(again, "") != 1 { - t.Fatalf("second run = %v, %d urls", err, strings.Count(again, "")) - } - if empty, err := (&classes.Generator{Root: "https://h"}).Generate(context.Background(), nil); err != nil || empty != "" { - t.Fatalf("no items = %q %v", empty, err) - } -} - -// TestOnlyHTTPLocs pins the CR-01 output filter: a URL that Url::to keeps -// but that is not http or https (a javascript: URL FILTER_VALIDATE_URL -// accepts, mailto:, a bare fragment) is not written and not counted. A -// protocol-relative URL comes out of Url::to on the root host. -func TestOnlyHTTPLocs(t *testing.T) { - gen := &classes.Generator{Root: "https://api.example.test"} - xml, err := gen.Generate(context.Background(), urlItems( - "javascript://%0aalert(1)%0a//x", - "JavaScript://example.test/%0aalert(1)", - "//evil.test/x", - "mailto:a@example.test", - "#top", - "/ok", - "https://other.example.test/page", - )) - if err != nil { - t.Fatal(err) - } - if n := strings.Count(xml, ""); n != 3 { - t.Fatalf("%d url elements, want 3:\n%s", n, xml) - } - if !strings.Contains(xml, "https://api.example.test/evil.test/x") { - t.Fatalf("the protocol-relative URL did not stay on the root host:\n%s", xml) - } - if strings.Contains(strings.ToLower(xml), "javascript") || strings.Contains(xml, "mailto:") || strings.Contains(xml, "#top") { - t.Fatalf("a non-http URL was written:\n%s", xml) - } - if !strings.Contains(xml, "https://api.example.test/ok") || !strings.Contains(xml, "https://other.example.test/page") { - t.Fatalf("an http(s) URL is missing:\n%s", xml) - } -} diff --git a/classes/golden_test.go b/classes/golden_test.go deleted file mode 100644 index d4d3452..0000000 --- a/classes/golden_test.go +++ /dev/null @@ -1,343 +0,0 @@ -package classes_test - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "testing" - "time" - - sitemap "git.golem15.com/golem15/sm-sitemap-plugin" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/compass" - "git.golem15.com/golem15/summercms/modules/festival" - "git.golem15.com/golem15/summercms/modules/pact" -) - -// The goldens in testdata are written by tests/php/gen-goldens.php from the -// same JSON inputs with the PHP plugin's generation code and the vendored -// Winter URL generator; these tests feed the inputs through the Go port and -// compare bytes. - -// goldenResolved is a registry item's resolve result in a golden input. -type goldenResolved struct { - URL string `json:"url"` - MTime *string `json:"mtime"` - Alternates [][2]string `json:"alternates"` - Items []goldenResolved `json:"items"` -} - -func (g goldenResolved) toPact(t *testing.T) pact.ResolvedMenuItem { - t.Helper() - out := pact.ResolvedMenuItem{URL: g.URL} - if g.MTime != nil { - at, err := time.Parse(time.RFC3339, *g.MTime) - if err != nil { - t.Fatalf("mtime %q: %v", *g.MTime, err) - } - out.MTime = &at - } - for _, a := range g.Alternates { - out.Alternates = append(out.Alternates, pact.MenuItemAlternate{Locale: a[0], URL: a[1]}) - } - for _, child := range g.Items { - out.Items = append(out.Items, child.toPact(t)) - } - return out -} - -// goldenCase is one sitemap case of a golden input: the stored items, the -// registry types' resolve results, the settings (PHP keys), the updates -// manifest, the URLs a BeforeAddItem listener skips and the text of the -// extra element an AddItem listener appends per URL. -type goldenCase struct { - Name string `json:"name"` - Root string `json:"root"` - Items models.DefinitionItems `json:"items"` - Types map[string]map[string]goldenResolved `json:"types"` - Settings map[string]json.RawMessage `json:"settings"` - Manifest map[string]any `json:"manifest"` - Skip []string `json:"skip"` - Extra map[string]string `json:"extra"` -} - -// fakeResolver answers Resolve from a golden case's types. -type fakeResolver struct { - t *testing.T - types map[string]map[string]goldenResolved -} - -func (f fakeResolver) Type(key string) (pact.MenuItemType, bool) { - _, ok := f.types[key] - return pact.MenuItemType{Key: key}, ok -} - -func (f fakeResolver) Resolve(_ context.Context, key, reference, _ string, _ bool, _ string) (*pact.ResolvedMenuItem, error) { - r, ok := f.types[key][reference] - if !ok { - return nil, nil - } - out := r.toPact(f.t) - return &out, nil -} - -// pluginTiers compiles the tier rules the plugin's own config/config.yaml -// ships. -func pluginTiers(t *testing.T) classes.Tiers { - t.Helper() - cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Environ: []string{}}) - if err != nil { - t.Fatal(err) - } - if err := cfg.MergePlugin("golem15.sitemap", (&sitemap.Plugin{}).ConfigFS()); err != nil { - t.Fatal(err) - } - tiers, err := classes.LoadTiers(cfg) - if err != nil { - t.Fatal(err) - } - return tiers -} - -// caseSettings builds the settings of a case from its PHP keys. -func caseSettings(t *testing.T, c goldenCase) models.Settings { - t.Helper() - s := models.DefaultSettings() - texts := map[string]**string{ - "frontend_url": &s.FrontendURL, - "frontend_default_locale": &s.FrontendDefaultLocale, - "priority_homepage": &s.PriorityHomepage, - "priority_services": &s.PriorityServices, - "priority_blog": &s.PriorityBlog, - "priority_categories": &s.PriorityCategories, - "priority_legal": &s.PriorityLegal, - } - for key, raw := range c.Settings { - if key == "enable_priority_tiers" { - if err := json.Unmarshal(raw, &s.EnablePriorityTiers); err != nil { - t.Fatal(err) - } - continue - } - dest, ok := texts[key] - if !ok { - t.Fatalf("case %s: unknown setting %s", c.Name, key) - } - v, err := models.DecodeText(raw) - if err != nil { - t.Fatal(err) - } - *dest = v - } - if c.Manifest != nil { - raw, err := json.Marshal(c.Manifest) - if err != nil { - t.Fatal(err) - } - path := filepath.Join(t.TempDir(), "updates.json") - if err := os.WriteFile(path, raw, 0o644); err != nil { - t.Fatal(err) - } - s.UpdatesManifestPath = &path - } - return s -} - -// renderCase runs a case through the generator and the route's rewrite. -func renderCase(t *testing.T, c goldenCase) string { - t.Helper() - bus := festival.New() - bus.Listen("test.skip", func(_ context.Context, e *classes.BeforeAddItem) error { - if slices.Contains(c.Skip, e.Info.URL) { - e.Skip = true - } - return nil - }) - bus.Listen("test.extra", func(_ context.Context, e *classes.AddItem) error { - if text, ok := c.Extra[e.Info.URL]; ok { - e.Extra = append(e.Extra, ""+classes.EscapeText(text)+"") - } - return nil - }) - settings := caseSettings(t, c) - gen := &classes.Generator{ - Root: c.Root, - Resolver: fakeResolver{t: t, types: c.Types}, - Settings: settings, - Tiers: pluginTiers(t), - Events: bus, - } - xml, err := gen.Generate(context.Background(), c.Items) - if err != nil { - t.Fatalf("generate %s: %v", c.Name, err) - } - return classes.Rewrite(xml, classes.BackendURL(c.Root), models.Text(settings.FrontendURL)) -} - -func readTestdata(t *testing.T, name string) []byte { - t.Helper() - raw, err := os.ReadFile(filepath.Join("testdata", name)) - if err != nil { - t.Fatal(err) - } - return raw -} - -// TestSitemapGolden compares the Go sitemap with the PHP goldens. -func TestSitemapGolden(t *testing.T) { - t.Run("urlto", func(t *testing.T) { - var in struct { - Root string `json:"root"` - URLs []*string `json:"urls"` - } - if err := json.Unmarshal(readTestdata(t, "urlto.json"), &in); err != nil { - t.Fatal(err) - } - want := strings.Split(strings.TrimSuffix(string(readTestdata(t, "urlto.golden")), "\n"), "\n") - if len(want) != len(in.URLs) { - t.Fatalf("urlto.golden has %d lines for %d inputs", len(want), len(in.URLs)) - } - var got strings.Builder - for i, u := range in.URLs { - path := "" - if u != nil { - path = *u - } - out := classes.URLTo(in.Root, path) - if out != want[i] { - t.Errorf("URLTo(%q) = %q, PHP %q", path, out, want[i]) - } - got.WriteString(out + "\n") - } - if got.String() != string(readTestdata(t, "urlto.golden")) { - t.Error("urlto output differs from the golden file") - } - }) - - t.Run("tracer", func(t *testing.T) { - var in struct { - Case goldenCase `json:"case"` - } - if err := json.Unmarshal(readTestdata(t, "tracer.json"), &in); err != nil { - t.Fatal(err) - } - got := renderCase(t, in.Case) - if want := string(readTestdata(t, "tracer.golden")); got != want { - t.Fatalf("tracer sitemap differs from the PHP golden\n got: %q\nwant: %q", got, want) - } - }) - - t.Run("full", func(t *testing.T) { - var in struct { - Cases []goldenCase `json:"cases"` - } - if err := json.Unmarshal(readTestdata(t, "full.json"), &in); err != nil { - t.Fatal(err) - } - golden := string(readTestdata(t, "full.golden")) - var all strings.Builder - for i, c := range in.Cases { - got := renderCase(t, c) - all.WriteString("=== " + c.Name + "\n" + got) - header := "=== " + c.Name + "\n" - start := strings.Index(golden, header) - if start < 0 { - t.Errorf("case %s is missing from full.golden", c.Name) - continue - } - want := golden[start+len(header):] - if i+1 < len(in.Cases) { - if end := strings.Index(want, "=== "+in.Cases[i+1].Name+"\n"); end >= 0 { - want = want[:end] - } - } - if got != want { - t.Errorf("case %s differs from the PHP golden\n got: %q\nwant: %q", c.Name, got, want) - } - } - if all.String() != golden { - t.Error("full output differs from full.golden") - } - }) - - t.Run("cap", func(t *testing.T) { - items := make([]models.DefinitionItem, classes.MaxURLs+1) - for i := range items { - u := "/u/" + strconv.Itoa(i+1) - items[i] = models.DefinitionItem{Type: models.ItemTypeURL, URL: &u} - } - gen := &classes.Generator{Root: "https://api.example.test"} - xml, err := gen.Generate(context.Background(), items) - if err != nil { - t.Fatal(err) - } - if n := strings.Count(xml, ""); n != classes.MaxURLs { - t.Fatalf("%d urls written, want %d", n, classes.MaxURLs) - } - if !strings.Contains(xml, "https://api.example.test/u/50000") || strings.Contains(xml, "/u/50001<") { - t.Fatal("the 50000th URL must be written and the 50001st must not") - } - }) -} - -// TestSitemapTiersConfig checks the boot-time tier rule checks, a host's -// replacement of the default list and the manifest's fail-safe reads. -func TestSitemapTiersConfig(t *testing.T) { - for name, rules := range map[string][]classes.TierConfig{ - "unknown setting": {{Setting: "priority_other", Exact: []string{""}}}, - "neither": {{Setting: models.PriorityHomepage}}, - "both": {{Setting: models.PriorityHomepage, Exact: []string{""}, Pattern: "^/$"}}, - "bad pattern": {{Setting: models.PriorityBlog, Pattern: "^/(blog"}}, - } { - if _, err := classes.CompileTiers(rules); err == nil { - t.Errorf("%s: compiled, want an error", name) - } - } - - dir := t.TempDir() - host := "sitemap:\n priority_tiers:\n - setting: priority_legal\n pattern: '^/legal(/.*)?$'\n" - if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte(host), 0o644); err != nil { - t.Fatal(err) - } - cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) - if err != nil { - t.Fatal(err) - } - if err := cfg.MergePlugin("golem15.sitemap", (&sitemap.Plugin{}).ConfigFS()); err != nil { - t.Fatal(err) - } - tiers, err := classes.LoadTiers(cfg) - if err != nil || len(tiers) != 1 { - t.Fatalf("host tiers = %+v, %v", tiers, err) - } - s := models.DefaultSettings() - if p, ok := tiers.Priority("https://h/legal/terms/", s); !ok || p != "0.2" { - t.Fatalf("host rule priority = %q %v", p, ok) - } - if _, ok := tiers.Priority("https://h/", s); ok { - t.Fatal("the host list must replace the default homepage rule") - } - - if m := classes.LoadManifest(dir); m != nil { - t.Fatal("a directory is not a manifest") - } - list := filepath.Join(dir, "list.json") - if err := os.WriteFile(list, []byte(`["/a"]`), 0o644); err != nil { - t.Fatal(err) - } - if m := classes.LoadManifest(list); m != nil { - t.Fatal("a JSON list is not a manifest") - } - big := filepath.Join(dir, "big.json") - if err := os.WriteFile(big, append([]byte(`{"/a":"2026-01-01","x":"`), append(make([]byte, classes.ManifestMaxBytes), '"', '}')...), 0o644); err != nil { - t.Fatal(err) - } - if m := classes.LoadManifest(big); m != nil { - t.Fatal("a manifest over the size cap must be ignored") - } -} diff --git a/classes/manifest.go b/classes/manifest.go deleted file mode 100644 index 7cb7f4f..0000000 --- a/classes/manifest.go +++ /dev/null @@ -1,112 +0,0 @@ -package classes - -import ( - "bytes" - "encoding/json" - "io" - "os" - "strings" - "time" -) - -// ManifestMaxBytes is the largest updates manifest the sitemap reads. -const ManifestMaxBytes = 5 << 20 - -// Manifest maps a normalized URL path to its last-modified date text, the -// updates manifest a frontend build writes. -type Manifest map[string]string - -// LoadManifest reads the updates manifest at path: a regular file of at -// most ManifestMaxBytes holding a JSON object of URL path to date. It fails -// safe like PHP: an empty path, a missing, unreadable, oversized or invalid -// file is no manifest (nil), logged without the file's content. Keys are -// normalized like PHP normalizeManifestPath; values that are not strings are -// ignored. -func LoadManifest(path string) Manifest { - path = strings.TrimSpace(path) - if path == "" { - return nil - } - info, err := os.Stat(path) - if err != nil || !info.Mode().IsRegular() { - return nil - } - if info.Size() > ManifestMaxBytes { - pluginLog.Warn("golem15.sitemap: updates manifest is too large; lastmod omitted", "path", path, "max_bytes", ManifestMaxBytes) - return nil - } - f, err := os.Open(path) - if err != nil { - return nil - } - defer f.Close() - raw, err := io.ReadAll(io.LimitReader(f, ManifestMaxBytes+1)) - if err != nil || len(raw) > ManifestMaxBytes { - return nil - } - raw = bytes.TrimSpace(raw) - var decoded map[string]any - if len(raw) == 0 || raw[0] != '{' || json.Unmarshal(raw, &decoded) != nil { - pluginLog.Warn("golem15.sitemap: updates manifest is not a JSON object; lastmod omitted", "path", path) - return nil - } - m := Manifest{} - for key, value := range decoded { - if s, ok := value.(string); ok { - m[normalizeManifestPath(key)] = s - } - } - return m -} - -// normalizeManifestPath is PHP normalizeManifestPath: a leading slash, no -// trailing slash except for the root. -func normalizeManifestPath(path string) string { - path = "/" + strings.TrimLeft(path, "/") - if path == "/" { - return path - } - return strings.TrimRight(path, "/") -} - -// Lastmod is PHP lookupManifestLastmod: the date listed for the URL's path, -// parsed like the ISO 8601 forms strtotime reads (a date without a zone is -// UTC). ok is false when the path is not listed or its date does not parse. -func (m Manifest) Lastmod(url string) (time.Time, bool) { - if len(m) == 0 { - return time.Time{}, false - } - path := URLPath(url) - if path == "" { - path = "/" - } - text, ok := m[normalizeManifestPath(path)] - if !ok { - return time.Time{}, false - } - t, ok := parseManifestDate(text) - if !ok || t.Unix() == 0 { - return time.Time{}, false - } - return t, true -} - -var manifestLayouts = []string{ - time.RFC3339Nano, - "2006-01-02T15:04:05.999999999Z0700", - "2006-01-02T15:04:05", - "2006-01-02T15:04", - "2006-01-02 15:04:05Z07:00", - "2006-01-02 15:04:05", - "2006-01-02", -} - -func parseManifestDate(s string) (time.Time, bool) { - s = strings.TrimSpace(s) - for _, layout := range manifestLayouts { - if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil { - return t, true - } - } - return time.Time{}, false -} diff --git a/classes/manifest_test.go b/classes/manifest_test.go deleted file mode 100644 index f6c66eb..0000000 --- a/classes/manifest_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package classes_test - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" -) - -// TestManifest pins the updates manifest reader: it fails safe (a missing, -// empty-path, directory, oversized, non-object or invalid file is no -// manifest, so lastmod is omitted), normalizes keys, ignores non-string -// values, and parses the ISO dates strtotime reads, a date without a zone -// as UTC. -func TestManifest(t *testing.T) { - dir := t.TempDir() - write := func(name, body string) string { - t.Helper() - path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - return path - } - for name, path := range map[string]string{ - "empty path": "", - "spaces": " ", - "missing": filepath.Join(dir, "missing.json"), - "directory": dir, - "list": write("list.json", `["/a"]`), - "string": write("string.json", `"x"`), - "invalid": write("invalid.json", `{"/a": `), - "empty file": write("empty.json", ""), - "oversized": write("big.json", `{"/a":"2026-01-01","pad":"`+strings.Repeat("x", classes.ManifestMaxBytes)+`"}`), - "null": write("null.json", "null"), - "leading space": write("space.json", " \n"), - } { - if m := classes.LoadManifest(path); m != nil { - t.Errorf("%s: manifest = %v, want none", name, m) - } - } - if fifo := filepath.Join(dir, "link.json"); os.Symlink(dir, fifo) == nil { - if m := classes.LoadManifest(fifo); m != nil { - t.Error("a symlink to a directory is not a manifest") - } - } - - path := write("updates.json", ` {"blog/a/":"2026-01-02T03:04:05+02:00","/b":"2026-02-03","c":"2026-03-04 05:06:07","/":"2026-04-05T06:07","/num":12,"/bad":"yesterday","/epoch":"1970-01-01T00:00:00Z","/nested":{"x":1}}`) - m := classes.LoadManifest(path) - if m == nil { - t.Fatal("valid manifest not read") - } - if _, ok := m["/num"]; ok { - t.Fatal("a non-string value was kept") - } - for url, want := range map[string]string{ - "https://h/blog/a": "2026-01-02T01:04:05Z", - "https://h/blog/a/": "2026-01-02T01:04:05Z", - "https://h/b?x=1": "2026-02-03T00:00:00Z", - "/c": "2026-03-04T05:06:07Z", - "https://h": "2026-04-05T06:07:00Z", - "https://h/": "2026-04-05T06:07:00Z", - } { - got, ok := m.Lastmod(url) - if !ok || got.UTC().Format(time.RFC3339) != want { - t.Errorf("Lastmod(%s) = %v %v, want %s", url, got, ok, want) - } - } - for _, url := range []string{"https://h/bad", "https://h/epoch", "https://h/unlisted", "https://h/num"} { - if got, ok := m.Lastmod(url); ok { - t.Errorf("Lastmod(%s) = %v, want none", url, got) - } - } - var none classes.Manifest - if _, ok := none.Lastmod("https://h/b"); ok { - t.Fatal("a nil manifest answered") - } -} diff --git a/classes/rewrite.go b/classes/rewrite.go deleted file mode 100644 index 4125911..0000000 --- a/classes/rewrite.go +++ /dev/null @@ -1,69 +0,0 @@ -package classes - -import ( - "regexp" - "strings" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// The PHP routes.php patterns of the frontend rewrite. -var ( - locElement = regexp.MustCompile(`(.*?)`) - xhtmlLinkRef = regexp.MustCompile(`(]*href=")([^"]*)(")`) -) - -// Rewrite is the frontend_url rewrite of PHP routes.php: with frontendURL -// set, backendURL is replaced literally (str_replace) inside every loc -// element, inside every xhtml:link href and in the exact stylesheet href. -// Both URLs are right-trimmed of slashes first. An empty frontendURL (or -// "0", which PHP reads as false) leaves xml unchanged. -// -// PHP inserted the stored text as it was. Three rules keep that text out of -// the document unless it is a plain http(s) URL (T-17-15): -// -// - a frontendURL that models.AllowedFrontendURL refuses leaves xml -// unchanged, whatever is stored in the settings row; -// - the frontend URL is inserted through EscapeText in loc and through -// EscapeAttr in the hrefs. Both are the identity for every accepted -// value, which keeps the PHP bytes; -// - a loc or xhtml:link href whose rewritten value does not start with -// http:// or https:// keeps its value. The generator applies the same -// filter before the rewrite. -func Rewrite(xml, backendURL, frontendURL string) string { - if frontendURL == "" || frontendURL == "0" { - return xml - } - if !models.AllowedFrontendURL(frontendURL) { - return xml - } - frontend := strings.TrimRight(frontendURL, "/") - backend := strings.TrimRight(backendURL, "/") - if backend == "" { - return xml - } - frontendText, frontendAttr := EscapeText(frontend), EscapeAttr(frontend) - xml = locElement.ReplaceAllStringFunc(xml, func(m string) string { - inner := m[len("") : len(m)-len("")] - rewritten := strings.ReplaceAll(inner, backend, frontendText) - if !httpURL(rewritten) { - return m - } - return "" + rewritten + "" - }) - xml = xhtmlLinkRef.ReplaceAllStringFunc(xml, func(m string) string { - sub := xhtmlLinkRef.FindStringSubmatch(m) - rewritten := strings.ReplaceAll(sub[2], backend, frontendAttr) - if !httpURL(rewritten) { - return m - } - return sub[1] + rewritten + sub[3] - }) - return strings.ReplaceAll(xml, `href="`+backend+`/sitemap.xsl"`, `href="`+frontendAttr+`/sitemap.xsl"`) -} - -// BackendURL is PHP rtrim(url('/'), '/') for a request whose root URL is -// root: the URL the rewrite replaces. -func BackendURL(root string) string { - return strings.TrimRight(URLTo(root, "/"), "/") -} diff --git a/classes/rewrite_test.go b/classes/rewrite_test.go deleted file mode 100644 index 124ad05..0000000 --- a/classes/rewrite_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package classes_test - -import ( - "context" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" -) - -// rewriteBackend and rewriteDoc are the backend URL and the document of the -// rewrite tests: a stylesheet instruction, a loc, hreflang links and the -// backend URL in places the rewrite leaves alone. -const rewriteBackend = "https://api.ex.pl" - -var rewriteDoc = "\n" + - `` + "\n" + - `https://api.ex.pl/a2026https://api.ex.pl1` + - `` + - `https://api.ex.pl/xhttps://other.pl/https://api.ex.plcp` + "\n" - -// TestRewrite pins the PHP routes.php frontend_url rewrite: the backend URL -// is replaced inside loc, inside xhtml:link href and in the exact stylesheet -// href, nowhere else; slashes are trimmed on both; an empty or "0" frontend -// URL and equal URLs leave the document unchanged. -func TestRewrite(t *testing.T) { - const backend = rewriteBackend - doc := rewriteDoc - want := "\n" + - `` + "\n" + - `https://www.ex.pl/a2026https://api.ex.pl1` + - `` + - `https://api.ex.pl/xhttps://other.pl/https://www.ex.plcp` + "\n" - for _, frontend := range []string{"https://www.ex.pl", "https://www.ex.pl/", "https://www.ex.pl///"} { - for _, back := range []string{backend, backend + "/"} { - if got := classes.Rewrite(doc, back, frontend); got != want { - t.Fatalf("Rewrite(%q, %q):\n%s\nwant:\n%s", back, frontend, got, want) - } - } - } - for _, frontend := range []string{"", "0"} { - if got := classes.Rewrite(doc, backend, frontend); got != doc { - t.Fatalf("frontend %q changed the document", frontend) - } - } - if got := classes.Rewrite(doc, backend, backend); got != doc { - t.Fatal("equal URLs changed the document") - } - if got := classes.Rewrite(doc, backend+"/", backend); got != doc { - t.Fatal("equal URLs up to a slash changed the document") - } - // An empty backend URL replaces nothing. - if got := classes.Rewrite(doc, "", "https://www.ex.pl"); got != doc { - t.Fatal("an empty backend URL changed the document") - } - // A stylesheet on another path is not the PHP pattern. - other := `` - if got := classes.Rewrite(other, backend, "https://www.ex.pl"); got != other { - t.Fatalf("other stylesheet = %q", got) - } -} - -// The two frontend_url values of the T-17-15 reproduction: a script URL the -// stylesheet would render as a link, and a value that closes the attribute -// and the element it is written into. -var frontendAttacks = []string{ - "javascript:alert(document.domain)//", - `https://x"/>", - " https://www.ex.pl", - "https://www.ex.pl\n", - "//www.ex.pl", - "www.ex.pl", - "https:///x", - "data:text/html,x", - }, frontendAttacks...) - for _, frontend := range refused { - if got := classes.Rewrite(rewriteDoc, rewriteBackend, frontend); got != rewriteDoc { - t.Errorf("frontend %q changed the document:\n%s", frontend, got) - } - } - - // The reproduction through the generator. - const root = "https://api.example.test" - xml, err := (&classes.Generator{Root: root}).Generate(context.Background(), urlItems("/blog/x")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(xml, `href="https://api.example.test/sitemap.xsl"`) { - t.Fatalf("the generated document lacks the stylesheet instruction:\n%s", xml) - } - for _, frontend := range frontendAttacks { - got := classes.Rewrite(xml, classes.BackendURL(root), frontend) - if got != xml { - t.Errorf("frontend %q changed the generated sitemap:\n%s", frontend, got) - } - if !strings.Contains(got, "https://api.example.test/blog/x") { - t.Errorf("frontend %q: the loc is not on the backend URL:\n%s", frontend, got) - } - if lower := strings.ToLower(got); strings.Contains(lower, "javascript") || strings.Contains(lower, "https://www.example.test/blog/x") || !strings.Contains(got, `href="https://www.example.test/sitemap.xsl"`) || strings.Contains(got, "api.example.test") { - t.Errorf("a valid frontend URL did not rewrite the sitemap:\n%s", got) - } - if got := classes.Rewrite(xml, classes.BackendURL(root), "HTTPS://WWW.EXAMPLE.TEST"); !strings.Contains(got, "HTTPS://WWW.EXAMPLE.TEST/blog/x") { - t.Errorf("an upper-case frontend URL did not rewrite the loc:\n%s", got) - } - - // The http(s) output filter after the rewrite: replacing "s://api.ex.pl" - // would write httphttps://www.ex.pl/a, so loc and href keep their values. - entry := `https://api.ex.pl/a` - if got := classes.Rewrite(entry, "s://api.ex.pl", "https://www.ex.pl"); got != entry { - t.Errorf("a rewrite to a non-http(s) URL was written:\n%s", got) - } -} diff --git a/classes/serialize.go b/classes/serialize.go deleted file mode 100644 index 5de12e1..0000000 --- a/classes/serialize.go +++ /dev/null @@ -1,105 +0,0 @@ -package classes - -import "strings" - -// The sitemap document is written by hand in the exact format PHP's -// DOMDocument::saveXML (libxml2) produced for the PHP plugin: the XML -// declaration and the stylesheet processing instruction each followed by a -// newline, everything else on one line, an empty urlset self-closed, -// xhtml:link elements self-closed without a space. The standard library's -// XML encoder escapes and closes elements differently, so the writer is a -// strings.Builder. - -// The urlset opening tag's attributes, in the order PHP set them. -const urlsetAttrs = ` xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"` + - ` xmlns:xhtml="http://www.w3.org/1999/xhtml"` + - ` xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"` + - ` xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"` - -// textEscaper is libxml's escaping of text nodes: &, <, > and carriage -// return; quotes, tabs, newlines and UTF-8 stay literal. -var textEscaper = strings.NewReplacer("&", "&", "<", "<", ">", ">", "\r", " ") - -// attrEscaper is libxml's escaping of attribute values: also the double -// quote, tab and newline; the single quote stays literal. -var attrEscaper = strings.NewReplacer( - "&", "&", "<", "<", ">", ">", `"`, """, - "\r", " ", "\t", " ", "\n", " ", -) - -// EscapeText escapes s as libxml writes a text node. -func EscapeText(s string) string { return textEscaper.Replace(s) } - -// EscapeAttr escapes s as libxml writes an attribute value. -func EscapeAttr(s string) string { return attrEscaper.Replace(s) } - -// Alternate is one hreflang link of a sitemap URL. -type Alternate struct { - Hreflang string - Href string -} - -// URLEntry is one url element: Loc, an optional Lastmod, Changefreq and -// Priority as they are written, the alternates (written only when there are -// more than one) and Extra, pre-escaped child XML that AddItem listeners -// appended, written before the closing tag. -type URLEntry struct { - Loc string - Lastmod string - Changefreq string - Priority string - Alternates []Alternate - Extra []string -} - -// writeURL appends one url element. -func writeURL(b *strings.Builder, e URLEntry) { - b.WriteString("") - b.WriteString(EscapeText(e.Loc)) - b.WriteString("") - if e.Lastmod != "" { - b.WriteString("") - b.WriteString(EscapeText(e.Lastmod)) - b.WriteString("") - } - b.WriteString("") - b.WriteString(EscapeText(e.Changefreq)) - b.WriteString("") - b.WriteString(EscapeText(e.Priority)) - b.WriteString("") - if len(e.Alternates) > 1 { - for _, a := range e.Alternates { - b.WriteString(``) - } - } - for _, x := range e.Extra { - b.WriteString(x) - } - b.WriteString("") -} - -// Serialize writes the sitemap document: the XML declaration, the -// stylesheet processing instruction pointing at stylesheet (written as it -// is, like libxml writes PI data) and the urlset with entries. -func Serialize(stylesheet string, entries []URLEntry) string { - var b strings.Builder - b.WriteString("\n") - b.WriteString(`\n\n") - return b.String() - } - b.WriteString(">") - for _, e := range entries { - writeURL(&b, e) - } - b.WriteString("\n") - return b.String() -} diff --git a/classes/serialize_test.go b/classes/serialize_test.go deleted file mode 100644 index 4980471..0000000 --- a/classes/serialize_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package classes_test - -import ( - "context" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// TestXMLEscaping ports the PHP XmlEscapingTest (SITE-002) as behaviour: -// every value the sitemap writes is escaped exactly as libxml escapes it, -// text nodes (loc, lastmod, changefreq, priority) and attribute values -// (hreflang, href), and no raw value reaches the output. -func TestXMLEscaping(t *testing.T) { - const raw = "a&bd\"e'f\rg\th\ni" - if got := classes.EscapeText(raw); got != "a&b<c>d\"e'f g\th\ni" { - t.Fatalf("EscapeText = %q", got) - } - if got := classes.EscapeAttr(raw); got != "a&b<c>d"e'f g h i" { - t.Fatalf("EscapeAttr = %q", got) - } - if got := classes.EscapeText("zażółć — &"); got != "zażółć — &amp;" { - t.Fatalf("EscapeText keeps UTF-8 and escapes an existing entity again: %q", got) - } - - xml := classes.Serialize("https://h/sitemap.xsl", []classes.URLEntry{{ - Loc: "https://h/a?x=1&y=<2>", - Lastmod: "2026-01-02T03:04:05+00:00", - Changefreq: "weekly&<>", - Priority: "0.5\r", - Alternates: []classes.Alternate{ - {Hreflang: "pl\"x", Href: "https://h/pl?a=1&b=\"2\"\t\n"}, - {Hreflang: "en", Href: "https://h/en'x"}, - }, - }}) - want := "\n" + - "\n" + - `` + - "https://h/a?x=1&y=<2>2026-01-02T03:04:05+00:00weekly&<>0.5 " + - `` + - `` + - "\n" - if xml != want { - t.Fatalf("serialized:\n%q\nwant:\n%q", xml, want) - } - - // One alternate is not written; an empty urlset is self-closed. - single := classes.Serialize("s", []classes.URLEntry{{Loc: "l", Changefreq: "c", Priority: "p", Alternates: []classes.Alternate{{Hreflang: "pl", Href: "x"}}}}) - if strings.Contains(single, "xhtml:link") || strings.Contains(single, "") { - t.Fatalf("single alternate or empty lastmod written: %q", single) - } - if empty := classes.Serialize("s", nil); !strings.HasSuffix(empty, `sitemap.xsd"/>`+"\n") { - t.Fatalf("empty urlset = %q", empty) - } - - // Through the generator: stored changefreq and priority, a resolved - // alternate's locale and a url item with every special character. - url := "/q?a=1&b=&c=\"y\"" - changefreq := "daily&" - priority := "0.9\"'" - gen := &classes.Generator{Root: "https://api.example.test", Settings: models.Settings{}, Resolver: fakeResolver{t: t, types: map[string]map[string]goldenResolved{ - "acme-post": {"1": {URL: "/p?x=1&y=2", Alternates: [][2]string{{"pl&<", "/p?x=1&y=2"}, {"en\"", "/en/p?z=<1>"}}}}, - }}} - out, err := gen.Generate(context.Background(), []models.DefinitionItem{ - {Type: models.ItemTypeURL, URL: &url, Changefreq: &changefreq, Priority: &priority}, - {Type: "acme-post", Reference: text("1"), Changefreq: &changefreq, Priority: &priority}, - }) - if err != nil { - t.Fatal(err) - } - body := out[strings.Index(out, ""):] - for _, bad := range []string{"", "&b", "&y", `"y"`, "", "daily&<", `hreflang="pl&<"`, `en""`, "<1>"} { - if strings.Contains(body, bad) { - t.Fatalf("raw value %q reached the output:\n%s", bad, body) - } - } - for _, good := range []string{ - "daily&<weekly>", - "0.9\"'", - `hreflang="pl&<"`, - `hreflang="en""`, - "y=2", - } { - if !strings.Contains(body, good) { - t.Fatalf("output lacks %q:\n%s", good, body) - } - } - if strings.Count(body, "") != 2 { - t.Fatalf("urls = %d:\n%s", strings.Count(body, ""), body) - } -} - -func text(s string) *string { return &s } diff --git a/classes/testdata/full.golden b/classes/testdata/full.golden deleted file mode 100644 index 4553e1e..0000000 --- a/classes/testdata/full.golden +++ /dev/null @@ -1,58 +0,0 @@ -=== escaping - - -https://a&b<c>.pl/xa&b<c>"d'e f g -h0.5 & <1> "q" 'a'https://api.example.test/za%C5%BC%C3%B3%C5%82%C4%87%20%26%20co2026-01-02T03:04:05+00:00x>y<0.9> -=== empty-urlset - - - -=== no-items -=== hreflang-list - - -https://api.example.test/pl/a2026-02-01T00:00:00+00:00monthly0.4https://api.example.test/en/a2026-02-01T00:00:00+00:00monthly0.4https://api.example.test/pl/bmonthly0.4https://api.example.test/en/bmonthly0.4https://api.example.test/de/bmonthly0.4https://api.example.test/solomonthly0.4 -=== nested - - -https://api.example.test/kategoria/a2026-03-01T12:00:00+00:00daily0.7https://api.example.test/kategoria/b2026-03-02T12:00:00+00:00daily0.7https://api.example.test/kategoria/cdaily0.7 -=== tiers-default - - -https://api.example.test/weekly1.0https://api.example.test/enweekly1.0https://api.example.test/enweekly1.0https://api.example.test/uslugi/xweekly0.8https://api.example.test/uslugi/x/yweekly0.3https://api.example.test/realizacje/rweekly0.8https://api.example.test/en/work/wweekly0.8https://api.example.test/blog/a/bweekly0.6https://api.example.test/en/blog/a/bweekly0.6https://api.example.test/blog/aweekly0.4https://api.example.test/blogweekly0.4https://api.example.test/en/blogweekly0.4https://api.example.test/uslugiweekly0.4https://api.example.test/kontaktweekly0.2https://api.example.test/kontaktweekly0.2https://api.example.test/en/ai-lab/blob-playgroundweekly0.2https://api.example.test/otherweekly0.3https://elsewhere.pl/uslugi/zweekly0.8https://api.example.test/o-nas?x=1weekly0.2https://api.example.test/blog/a/b/cweekly0.3 -=== tiers-settings - - -https://api.example.test/weekly0.95https://api.example.test/blog/a/bweeklyhttps://api.example.test/regulaminweekly0.15https://api.example.test/xweekly0.3 -=== tiers-off - - -https://api.example.test/weekly0.1https://api.example.test/blog/a/bweekly0.2 -=== rewrite - - -https://www.example.test/weekly0.5https://www.example.test/aweekly0.5https://www.example.test/b?x=https%3A%2F%2Fapi.example.test%2Fcweekly0.5https://other.example.test/api.example.testweekly0.5https://www.example.test/pdaily0.7https://www.example.test/en/pdaily0.7 -=== rewrite-same - - -https://api.example.test/aweekly0.5 -=== manifest - - -https://api.example.test/2026-05-01T08:00:00+00:00weekly0.5https://api.example.test/about2026-05-02T00:00:00+00:00weekly0.5https://api.example.test/about2026-05-02T00:00:00+00:00weekly0.5https://api.example.test/blog/xweekly0.5https://api.example.test/noneweekly0.5https://api.example.test/rel2026-05-03T01:02:03+00:00daily0.7https://api.example.test/about2026-06-01T00:00:00+00:00daily0.7 -=== skip-extra - - -https://api.example.test/extraweekly0.5a&b <c>https://api.example.test/keepweekly0.5 -=== precision - - -https://api.example.test/p1daily0.5https://api.example.test/p2daily1https://api.example.test/p371https://api.example.test/p4daily1.0https://api.example.test/p5daily0.50https://api.example.test/p6daily0.6 -=== url-null - - -https://api.example.test/https://api.example.test/0daily0.5 -=== port-root - - -http://localhost:8096/weekly0.5http://localhost:8096/a%20bweekly0.5 diff --git a/classes/testdata/full.json b/classes/testdata/full.json deleted file mode 100644 index d2c4efe..0000000 --- a/classes/testdata/full.json +++ /dev/null @@ -1,693 +0,0 @@ -{ - "cases": [ - { - "name": "escaping", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "https://a&b.pl/x", - "changefreq": "a&b\"d'e\rf\tg\nh", - "priority": "0.5 & <1> \"q\" 'a'" - }, - { - "type": "acme-post", - "reference": "1", - "urlPattern": "", - "changefreq": "x>y", - "priority": "<0.9>", - "nesting": "1" - } - ], - "types": { - "acme-post": { - "1": { - "url": "/zażółć & co", - "mtime": "2026-01-02T03:04:05+00:00", - "alternates": [ - [ - "x\"y&z<\t\n\r'>", - "https://q&r.pl/p" - ], - [ - "en", - "/en/zażółć" - ] - ] - } - } - } - }, - { - "name": "empty-urlset", - "root": "https://api.example.test", - "items": [ - { - "type": "acme-post", - "reference": "404", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - }, - { - "type": "unknown-type", - "reference": "1", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-post": {} - } - }, - { - "name": "no-items", - "root": "https://api.example.test", - "items": [] - }, - { - "name": "hreflang-list", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "acme-list", - "reference": "", - "urlPattern": "", - "changefreq": "monthly", - "priority": "0.4", - "nesting": "1" - } - ], - "types": { - "acme-list": { - "": { - "url": "", - "items": [ - { - "url": "/pl/a", - "mtime": "2026-02-01T00:00:00+00:00", - "alternates": [ - [ - "pl", - "/pl/a" - ], - [ - "en", - "/en/a" - ] - ] - }, - { - "url": "/pl/b", - "mtime": null, - "alternates": [ - [ - "pl", - "/pl/b" - ], - [ - "en", - "/en/b" - ], - [ - "de", - "/de/b" - ] - ] - }, - { - "url": "/solo", - "alternates": [] - } - ] - } - } - } - }, - { - "name": "nested", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "acme-cat", - "reference": "5", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-cat": { - "5": { - "url": "/kategoria/a", - "mtime": "2026-03-01T12:00:00+00:00", - "alternates": [ - [ - "pl", - "/kategoria/a" - ], - [ - "en", - "/en/category/a" - ] - ], - "items": [ - { - "url": "/kategoria/b", - "mtime": "2026-03-02T12:00:00+00:00", - "alternates": [ - [ - "pl", - "/kategoria/b" - ], - [ - "en", - "/en/category/b" - ] - ], - "items": [ - { - "url": "/kategoria/c", - "alternates": [ - [ - "x-default", - "/kategoria/c" - ] - ] - } - ] - } - ] - } - } - } - }, - { - "name": "tiers-default", - "root": "https://api.example.test", - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/uslugi/x", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/uslugi/x/y", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/realizacje/r", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/work/w", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a/b", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/blog/a/b", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/blog/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/uslugi", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/kontakt", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/kontakt/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/ai-lab/blob-playground", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/other", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "https://elsewhere.pl/uslugi/z", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/o-nas?x=1", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a/b/c", - "changefreq": "weekly", - "priority": "0.3" - } - ] - }, - { - "name": "tiers-settings", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": true, - "priority_homepage": "0.95", - "priority_blog": "", - "priority_legal": "0.15" - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a/b", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/regulamin", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/x", - "changefreq": "weekly", - "priority": "0.3" - } - ] - }, - { - "name": "tiers-off", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.1" - }, - { - "type": "url", - "url": "/blog/a/b", - "changefreq": "weekly", - "priority": "0.2" - } - ] - }, - { - "name": "rewrite", - "root": "https://api.example.test", - "settings": { - "frontend_url": "https://www.example.test/", - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/a", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "https://api.example.test/b?x=https://api.example.test/c", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "https://other.example.test/api.example.test", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "acme-list", - "reference": "", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-list": { - "": { - "url": "", - "items": [ - { - "url": "/p", - "alternates": [ - [ - "pl", - "/p" - ], - [ - "en", - "https://api.example.test/en/p" - ] - ] - } - ] - } - } - } - }, - { - "name": "rewrite-same", - "root": "https://api.example.test", - "settings": { - "frontend_url": "https://api.example.test", - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/a", - "changefreq": "weekly", - "priority": "0.5" - } - ] - }, - { - "name": "manifest", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "manifest": { - "/": "2026-05-01T10:00:00+02:00", - "about/": "2026-05-02", - "/blog/x": "not a date", - "/rel": "2026-05-03T01:02:03Z" - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/about", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/about/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/blog/x", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/none", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "acme-post", - "reference": "2", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - }, - { - "type": "acme-post", - "reference": "3", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-post": { - "2": { - "url": "/rel", - "mtime": null, - "alternates": [ - [ - "pl", - "/rel" - ] - ] - }, - "3": { - "url": "/about", - "mtime": "2026-06-01T00:00:00+00:00", - "alternates": [ - [ - "pl", - "/about" - ] - ] - } - } - } - }, - { - "name": "skip-extra", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "skip": [ - "https://api.example.test/skip-me", - "/rel-skip" - ], - "extra": { - "https://api.example.test/extra": "a&b " - }, - "items": [ - { - "type": "url", - "url": "/skip-me", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/extra", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/keep", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "acme-post", - "reference": "9", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-post": { - "9": { - "url": "/rel-skip", - "alternates": [ - [ - "pl", - "/rel-skip" - ] - ] - } - } - } - }, - { - "name": "precision", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/p1", - "changefreq": "daily", - "priority": 0.5 - }, - { - "type": "url", - "url": "/p2", - "changefreq": "daily", - "priority": 1.0 - }, - { - "type": "url", - "url": "/p3", - "changefreq": 7, - "priority": 1 - }, - { - "type": "url", - "url": "/p4", - "changefreq": "daily", - "priority": "1.0" - }, - { - "type": "url", - "url": "/p5", - "changefreq": "daily", - "priority": "0.50" - }, - { - "url": "/p6", - "changefreq": "daily", - "priority": "0.6", - "cmsPage": "x", - "exists": true, - "fillable": [ - "a" - ] - } - ] - }, - { - "name": "url-null", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": null, - "changefreq": null, - "priority": null - }, - { - "type": "url", - "url": "0", - "changefreq": "daily", - "priority": "0.5" - } - ] - }, - { - "name": "port-root", - "root": "http://localhost:8096", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/a b", - "changefreq": "weekly", - "priority": "0.5" - } - ] - } - ] -} \ No newline at end of file diff --git a/classes/testdata/tracer.golden b/classes/testdata/tracer.golden deleted file mode 100644 index 4a6dc3d..0000000 --- a/classes/testdata/tracer.golden +++ /dev/null @@ -1,3 +0,0 @@ - - -https://api.example.test/always1.0https://api.example.test/news/hello2026-03-04T10:00:00+00:00weekly0.5 diff --git a/classes/testdata/tracer.json b/classes/testdata/tracer.json deleted file mode 100644 index ff5db2b..0000000 --- a/classes/testdata/tracer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "case": { - "root": "https://api.example.test", - "items": [ - {"type": "url", "url": "/", "changefreq": "always", "priority": "1.0", "nesting": "0"}, - {"type": "acme-post", "reference": "7", "urlPattern": "", "changefreq": "weekly", "priority": "0.5", "nesting": "1"} - ], - "types": { - "acme-post": { - "7": { - "url": "/news/hello", - "mtime": "2026-03-04T10:00:00+00:00", - "alternates": [["pl", "/news/hello"], ["en", "/en/news/hello"]] - } - } - } - } -} diff --git a/classes/testdata/urlto.golden b/classes/testdata/urlto.golden deleted file mode 100644 index 3deb178..0000000 --- a/classes/testdata/urlto.golden +++ /dev/null @@ -1,58 +0,0 @@ -https://api.ex.pl/ -https://api.ex.pl/ -https://api.ex.pl/ -https://api.ex.pl/blog -https://api.ex.pl/blog -https://api.ex.pl/blog -https://api.ex.pl/blog/za%C5%BC%C3%B3%C5%82%C4%87%20g%C4%99%C5%9Bl%C4%85 -https://api.ex.pl/en/blog/%3Aslug -https://x.pl/q/ -http://x.pl/ -HTTP://X.pl:443/a -https://h:8080/a -https://h/a -http://h/ -https://api.ex.pl/a?b=1&c=2 -https://api.ex.pl/a?x%5B%5D=1&x%5B%5D=2&0=y -https://api.ex.pl/a?q=a&0=b%20c -https://api.ex.pl/a -https://api.ex.pl/a?b=&0=c -https://api.ex.pl/a?x&y -https://api.ex.pl/a?a=2 -https://api.ex.pl/a?a%5Bb%5D=1 -https://api.ex.pl/a#b%20c -https://api.ex.pl/a -#frag -cdn.x/a -mailto:a%40b.pl -tel:%2B48%20123 -sms:%2B48123 -https://api.ex.pl/a/c -https://api.ex.pl/a//b -https://api.ex.pl/a/%20b/%25zz -https://api.ex.pl/a%2Fb -https://api.ex.pl/%C5%9Bcie%C5%BCka/%C4%85%C4%99?x=%C5%BC#%C5%82 -https://api.ex.pl/ftp%3A//x.pl/a%20b -ftp://x.pl/a -news:///comp.x -https://api.ex.pl/a%3Ab -https://api.ex.pl/a%3A1 -https://api.ex.pl/user%40host -https://user:pass@h/x -https://api.ex.pl/%20%20/lead -https://api.ex.pl/trail -https://api.ex.pl/tab_here -https://api.ex.pl/new_line -https://api.ex.pl/ -https://api.ex.pl/javascript%3Aalert%281%29 -https://a&b.pl/x -https://h/ -https://api.ex.pl/en -https://api.ex.pl/en -https://api.ex.pl/a?x=%2B -https://api.ex.pl/a?k=v%20w -https://api.ex.pl/a%20b?x=1#y%20z -https://h/a?b=1#c -https://api.ex.pl/~tilde/_under.dot-dash -https://api.ex.pl/a%27b%22c -https://api.ex.pl/%3Cx%3E%26y diff --git a/classes/testdata/urlto.json b/classes/testdata/urlto.json deleted file mode 100644 index 3e0ca04..0000000 --- a/classes/testdata/urlto.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "root": "https://api.ex.pl", - "urls": [ - "/", - "", - null, - "/blog/", - "/blog", - "blog", - "/blog/zażółć gęślą", - "/en/blog/:slug", - "https://x.pl/q/", - "http://x.pl", - "HTTP://X.pl:443/a", - "https://h:8080/a", - "https://h:443/a", - "http://h:80/", - "/a?b=1&c=2", - "/a?x[]=1&x[]=2&y", - "/a?q=a%26b+c", - "/a?", - "/a?b=&c", - "/a?0=x&1=y", - "/a?a=1&a=2", - "/a?a[b]=1", - "/a#b c", - "/a#", - "#frag", - "//cdn.x/a", - "mailto:a@b.pl", - "tel:+48 123", - "sms:+48123", - "/../a/./b/../c", - "/a//b", - "/a/%20b/%zz", - "/a%2Fb", - "/ścieżka/ąę?x=ż#ł", - "ftp://x.pl/a b", - "ftp://x.pl/a", - "news:comp.x", - "/a:b", - "a:1", - "/user@host", - "https://user:pass@h/x", - " /lead", - "/trail ", - "/tab\there", - "/new\nline", - "/a/b/../../..", - "javascript:alert(1)", - "https://a&b.pl/x", - "https://h/./", - "/en", - "/en/", - "/a?x=%2B", - "/a?k=v w", - "/a b?x=1#y z", - "https://h/a?b=1#c", - "/~tilde/_under.dot-dash", - "/a'b\"c", - "/&y" - ] -} \ No newline at end of file diff --git a/classes/tiers.go b/classes/tiers.go deleted file mode 100644 index 2e8ca06..0000000 --- a/classes/tiers.go +++ /dev/null @@ -1,98 +0,0 @@ -package classes - -import ( - "fmt" - "regexp" - "slices" - "strings" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/compass" -) - -// TiersConfigKey is the config key of the priority tier rules. The plugin's -// config/config.yaml ships the rules of the PHP getPriorityForUrl; a host -// replaces the whole list in its own config. -const TiersConfigKey = "golem15.sitemap.priority_tiers" - -// TierConfig is one configured tier rule: Setting names the priority setting -// whose value applies (priority_homepage, priority_services, priority_blog, -// priority_categories or priority_legal), and exactly one of Exact (paths -// that match as they are) and Pattern (a regular expression, RE2 syntax) -// selects the URLs. -type TierConfig struct { - Setting string `koanf:"setting"` - Exact []string `koanf:"exact"` - Pattern string `koanf:"pattern"` -} - -// TierRule is a compiled tier rule. -type TierRule struct { - Setting string - Exact []string - Pattern *regexp.Regexp -} - -// Tiers is the ordered tier rule list; the first matching rule wins. -type Tiers []TierRule - -// CompileTiers checks and compiles tier rules. Every error names the rule: -// an unknown setting, neither or both of exact and pattern, or a pattern -// that does not compile. -func CompileTiers(rules []TierConfig) (Tiers, error) { - out := make(Tiers, 0, len(rules)) - for i, r := range rules { - if _, ok := models.PriorityDefaults[r.Setting]; !ok { - return nil, fmt.Errorf("%s[%d]: setting %q is not one of priority_homepage, priority_services, priority_blog, priority_categories, priority_legal", TiersConfigKey, i, r.Setting) - } - hasPattern := strings.TrimSpace(r.Pattern) != "" - if hasPattern == (len(r.Exact) > 0) { - return nil, fmt.Errorf("%s[%d]: set exactly one of exact and pattern", TiersConfigKey, i) - } - rule := TierRule{Setting: r.Setting, Exact: append([]string(nil), r.Exact...)} - if hasPattern { - re, err := regexp.Compile(r.Pattern) - if err != nil { - return nil, fmt.Errorf("%s[%d]: pattern: %w", TiersConfigKey, i, err) - } - rule.Pattern = re - } - out = append(out, rule) - } - return out, nil -} - -// LoadTiers reads and compiles the rules at TiersConfigKey; an absent key -// is no rules. -func LoadTiers(cfg *compass.Config) (Tiers, error) { - if cfg == nil || !cfg.Has(TiersConfigKey) { - return Tiers{}, nil - } - var rules []TierConfig - if err := cfg.LoadSection(TiersConfigKey, &rules); err != nil { - return nil, fmt.Errorf("%s must be a list of {setting, exact} or {setting, pattern} rules: %w", TiersConfigKey, err) - } - return CompileTiers(rules) -} - -// Priority is PHP getPriorityForUrl with configured rules: when the -// settings enable tiers, the URL's path (parse_url's, "/" when it has none) -// right-trimmed of slashes is matched against the rules in order and the -// first match's setting value is returned. ok is false when tiers are off or -// no rule matches, so the item's own priority applies. -func (t Tiers) Priority(url string, s models.Settings) (string, bool) { - if !s.EnablePriorityTiers { - return "", false - } - path := URLPath(url) - if path == "" { - path = "/" - } - path = strings.TrimRight(path, "/") - for _, r := range t { - if r.Pattern != nil && r.Pattern.MatchString(path) || r.Pattern == nil && slices.Contains(r.Exact, path) { - return s.Priority(r.Setting), true - } - } - return "", false -} diff --git a/classes/tiers_test.go b/classes/tiers_test.go deleted file mode 100644 index ab550bb..0000000 --- a/classes/tiers_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package classes_test - -import ( - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// TestTiers pins the priority tiers (D-13): each default rule of the PHP -// getPriorityForUrl with paths from its list, trailing slashes, the first -// match winning, the settings' values, tiers turned off, a host list that -// replaces the defaults, and a bad rule failing the plugin's Boot. -func TestTiers(t *testing.T) { - tiers := pluginTiers(t) - if len(tiers) != 5 { - t.Fatalf("default rules = %d", len(tiers)) - } - s := models.DefaultSettings() - for url, want := range map[string]string{ - "https://h": "1.0", - "https://h/": "1.0", - "https://h/en": "1.0", - "https://h/en/": "1.0", - "https://h/uslugi/strony-www": "0.8", - "https://h/en/services/web": "0.8", - "https://h/realizacje/projekt/": "0.8", - "https://h/en/work/x": "0.8", - "https://h/blog/kategoria/wpis": "0.6", - "https://h/en/blog/category/post/": "0.6", - "https://h/blog": "0.4", - "https://h/blog/kategoria": "0.4", - "https://h/uslugi": "0.4", - "https://h/en/work": "0.4", - "https://h/polityka-prywatnosci": "0.2", - "https://h/en/terms/": "0.2", - "https://h/laboratorium-ai/zabawa-animacja": "0.2", - "https://h/en/ai-lab/blob-playground": "0.2", - "https://h/kontakt?x=1": "0.2", - } { - if got, ok := tiers.Priority(url, s); !ok || got != want { - t.Errorf("Priority(%s) = %q %v, want %s", url, got, ok, want) - } - } - for _, url := range []string{"https://h/other", "https://h/blog/a/b/c", "https://h/uslugi/a/b", "https://h/en/blogs/x", "https://h/kontakt/more"} { - if got, ok := tiers.Priority(url, s); ok { - t.Errorf("Priority(%s) = %q, want no rule", url, got) - } - } - custom := s - custom.PriorityBlog = text("0.65") - if got, _ := tiers.Priority("https://h/blog/a/b", custom); got != "0.65" { - t.Fatalf("custom blog priority = %q", got) - } - off := s - off.EnablePriorityTiers = false - if _, ok := tiers.Priority("https://h/", off); ok { - t.Fatal("tiers apply while turned off") - } - - // First match wins: an exact rule listed before a pattern that also matches. - ordered, err := classes.CompileTiers([]classes.TierConfig{ - {Setting: models.PriorityLegal, Exact: []string{"/blog"}}, - {Setting: models.PriorityCategories, Pattern: "^/blog$"}, - }) - if err != nil { - t.Fatal(err) - } - if got, _ := ordered.Priority("https://h/blog/", s); got != "0.2" { - t.Fatalf("first match = %q", got) - } - if empty, err := classes.CompileTiers(nil); err != nil || len(empty) != 0 { - t.Fatalf("no rules = %v %v", empty, err) - } - if none, err := classes.LoadTiers(nil); err != nil || len(none) != 0 { - t.Fatalf("LoadTiers(nil) = %v %v", none, err) - } - - // Every refused rule names its index; a bad RE2 pattern too. - for name, rule := range map[string]classes.TierConfig{ - "unknown setting": {Setting: "priority_other", Exact: []string{""}}, - "neither": {Setting: models.PriorityHomepage}, - "both": {Setting: models.PriorityHomepage, Exact: []string{""}, Pattern: "^/$"}, - "lookahead": {Setting: models.PriorityBlog, Pattern: "^/(?=blog)"}, - "unbalanced": {Setting: models.PriorityBlog, Pattern: "^/(blog"}, - } { - _, err := classes.CompileTiers([]classes.TierConfig{{Setting: models.PriorityHomepage, Exact: []string{""}}, rule}) - if err == nil || !strings.Contains(err.Error(), classes.TiersConfigKey+"[1]") { - t.Errorf("%s: err = %v", name, err) - } - } - -} diff --git a/classes/urlto.go b/classes/urlto.go deleted file mode 100644 index c14d022..0000000 --- a/classes/urlto.go +++ /dev/null @@ -1,566 +0,0 @@ -package classes - -import ( - "regexp" - "strconv" - "strings" -) - -// This file ports the parts of WinterCMS's URL generator the PHP sitemap -// called, so the Go sitemap writes the same bytes: Laravel's Url::to plus -// Winter's UrlGenerator::buildUrl, PHP's parse_url, rawurlencode, -// rawurldecode and urldecode, and FILTER_VALIDATE_URL. It is deliberately -// hand written: net/url escapes differently. The golden table -// testdata/urlto.golden, generated from the vendored Winter router, pins -// it. - -// validURLPrefix is Laravel UrlGenerator::isValidUrl's fast path. -var validURLPrefix = regexp.MustCompile(`^(#|//|https?://|(mailto|tel|sms):)`) - -// URLTo is Winter's Url::to(path) for a request whose root URL is root -// (scheme and host, plus a base path if any, without a trailing slash). A -// valid URL is kept; anything else becomes root plus "/" plus the path -// trimmed of slashes, the query string appended as it is. The result then -// goes through BuildURL, which re-encodes every path segment, so "/" is the -// root with its slash, "/blog/" loses its trailing slash and ":" becomes -// "%3A". -func URLTo(root, path string) string { - if isValidURL(path) { - return BuildURL(path) - } - p, query := path, "" - if i := strings.IndexByte(path, '?'); i >= 0 { - p, query = path[:i], path[i:] - } - inner := strings.Trim(p+"/", "/") - return BuildURL(strings.Trim(root+"/"+inner, "/") + query) -} - -// isValidURL is Laravel UrlGenerator::isValidUrl. -func isValidURL(path string) bool { - if validURLPrefix.MatchString(path) { - return true - } - return filterValidateURL(path) -} - -// filterURLChars is the character set PHP's FILTER_VALIDATE_URL accepts; -// a value with any other byte fails. -const filterURLChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789$-_.+!*'(),{}|\\^~[]`<>#%\";/?:@&=" - -// filterValidateURL approximates PHP filter_var(FILTER_VALIDATE_URL): only -// URL characters, a scheme, a host unless the scheme is mailto, news or -// file, and a valid host name for http and https. -func filterValidateURL(s string) bool { - if s == "" { - return false - } - for i := 0; i < len(s); i++ { - if !strings.ContainsRune(filterURLChars, rune(s[i])) { - return false - } - } - u := parseURL(s) - if !u.has(partScheme) || u.scheme == "" { - return false - } - scheme := strings.ToLower(u.scheme) - if !u.has(partHost) || u.host == "" { - return scheme == "mailto" || scheme == "news" || scheme == "file" - } - if scheme == "http" || scheme == "https" { - return validHostName(u.host) - } - return true -} - -// validHostName reports whether host is dot-separated labels of letters, -// digits and inner hyphens. -func validHostName(host string) bool { - host = strings.TrimSuffix(host, ".") - if host == "" || len(host) > 253 { - return false - } - for _, label := range strings.Split(host, ".") { - if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { - return false - } - for i := 0; i < len(label); i++ { - c := label[i] - if !(isAlnum(c) || c == '-') { - return false - } - } - } - return true -} - -// The parts a parsed URL may hold, as PHP parse_url's array keys. -const ( - partScheme = 1 << iota - partHost - partUser - partPass - partPort - partPath - partQuery - partFragment -) - -// phpURL is the array PHP parse_url returns; set records which keys exist. -type phpURL struct { - set int - scheme, host, user, pass, port, path, query, frag string -} - -func (u phpURL) has(part int) bool { return u.set&part != 0 } - -// parseURL ports PHP parse_url for the URL shapes a sitemap meets: an -// optional scheme, an authority after "//", a path, a query and a fragment. -// Control characters in a part are replaced by "_", as PHP does. -func parseURL(s string) phpURL { - var u phpURL - rest := s - if i := strings.IndexByte(rest, ':'); i > 0 && validScheme(rest[:i]) { - after := rest[i+1:] - if !strings.HasPrefix(after, "//") && portOnly(after) { - // "host:port/path" without a scheme. - } else { - u.scheme, u.set = rest[:i], u.set|partScheme - rest = after - } - } - if strings.HasPrefix(rest, "//") { - rest = rest[2:] - end := strings.IndexAny(rest, "/?#") - authority := rest - if end >= 0 { - authority, rest = rest[:end], rest[end:] - } else { - rest = "" - } - u.parseAuthority(authority) - } - if i := strings.IndexByte(rest, '#'); i >= 0 { - u.frag, u.set = rest[i+1:], u.set|partFragment - rest = rest[:i] - } - if i := strings.IndexByte(rest, '?'); i >= 0 { - u.query, u.set = rest[i+1:], u.set|partQuery - rest = rest[:i] - } - if rest != "" { - u.path, u.set = rest, u.set|partPath - } - u.scheme = replaceControl(u.scheme) - u.host = replaceControl(u.host) - u.user = replaceControl(u.user) - u.pass = replaceControl(u.pass) - u.path = replaceControl(u.path) - u.query = replaceControl(u.query) - u.frag = replaceControl(u.frag) - return u -} - -func (u *phpURL) parseAuthority(a string) { - if i := strings.LastIndexByte(a, '@'); i >= 0 { - userinfo := a[:i] - a = a[i+1:] - if j := strings.IndexByte(userinfo, ':'); j >= 0 { - u.user, u.pass = userinfo[:j], userinfo[j+1:] - u.set |= partUser | partPass - } else { - u.user = userinfo - u.set |= partUser - } - } - if i := strings.LastIndexByte(a, ':'); i >= 0 && !strings.HasSuffix(a, "]") { - port := a[i+1:] - if port != "" && allDigits(port) { - u.port, u.set = port, u.set|partPort - } - a = a[:i] - } - if a != "" { - u.host, u.set = a, u.set|partHost - } -} - -func validScheme(s string) bool { - for i := 0; i < len(s); i++ { - c := s[i] - if !(isAlnum(c) || c == '+' || c == '-' || c == '.') { - return false - } - } - return s != "" -} - -// portOnly reports whether s is a port number up to the first "/", the -// case PHP parse_url reads as host:port rather than scheme:path. -func portOnly(s string) bool { - end := strings.IndexByte(s, '/') - if end < 0 { - end = len(s) - } - return end > 0 && end <= 5 && allDigits(s[:end]) -} - -func allDigits(s string) bool { - for i := 0; i < len(s); i++ { - if s[i] < '0' || s[i] > '9' { - return false - } - } - return s != "" -} - -func isAlnum(c byte) bool { - return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' -} - -func replaceControl(s string) string { - if strings.IndexFunc(s, func(r rune) bool { return r < 0x20 || r == 0x7f }) < 0 { - return s - } - b := []byte(s) - for i, c := range b { - if c < 0x20 || c == 0x7f { - b[i] = '_' - } - } - return string(b) -} - -// phpTrim is PHP trim() with its default character list. -func phpTrim(s string) string { - return strings.Trim(s, " \t\n\r\x00\x0b") -} - -// singleColonSchemes are written "scheme:" instead of "scheme://". -var singleColonSchemes = map[string]bool{"mailto": true, "tel": true, "sms": true} - -// defaultPorts are Winter UrlGenerator::isDefaultPort's known ports. -var defaultPorts = map[string]int{ - "ftp": 21, "ssh": 22, "sftp": 22, "telnet": 23, "smtp": 25, "dns": 53, - "gopher": 70, "http": 80, "pop3": 110, "imap": 143, "ldap": 389, - "https": 443, "smtps": 465, "ldaps": 636, "ftps": 990, "imaps": 993, - "pop3s": 995, -} - -// BuildURL is Winter UrlGenerator::buildUrl($url) with no replacements: the -// query is URL-decoded, the URL parsed, every path segment re-encoded with -// rawurlencode(rawurldecode()) after "." and ".." are resolved, the query -// rebuilt with buildStr and the fragment re-encoded. -func BuildURL(raw string) string { - return buildParts(parseURL(decodeQueryParams(raw))) -} - -// replacePath is buildUrl($url, ['path' => path]). -func replacePath(raw, path string) string { - u := parseURL(decodeQueryParams(raw)) - u.path, u.set = path, u.set|partPath - return buildParts(u) -} - -// decodeQueryParams URL-decodes everything after the first "?". -func decodeQueryParams(s string) string { - if i := strings.IndexByte(s, '?'); i >= 0 { - return s[:i] + "?" + urldecode(s[i+1:]) - } - return s -} - -func buildParts(u phpURL) string { - u.scheme, u.host, u.user, u.pass = phpTrim(u.scheme), phpTrim(u.host), phpTrim(u.user), phpTrim(u.pass) - u.path, u.query, u.frag = phpTrim(u.path), phpTrim(u.query), phpTrim(u.frag) - port := 0 - if u.has(partPort) { - if n, err := strconv.Atoi(phpTrim(u.port)); err == nil && n >= 1 && n <= 65535 { - port = n - } - } - var b strings.Builder - hasScheme := u.has(partScheme) && u.scheme != "" - if hasScheme { - b.WriteString(u.scheme) - if singleColonSchemes[u.scheme] { - b.WriteString(":") - } else { - b.WriteString("://") - } - } - if u.has(partUser) && u.user != "" { - b.WriteString(u.user) - if u.has(partPass) && u.pass != "" { - b.WriteString(":" + u.pass) - } - b.WriteString("@") - } - if u.has(partHost) && u.host != "" { - b.WriteString(u.host) - } - if port != 0 { - if def, ok := defaultPorts[strings.ToLower(u.scheme)]; !ok || def != port { - b.WriteString(":" + strconv.Itoa(port)) - } - } - path := "" - if u.has(partPath) && u.path != "" { - var kept []string - for _, seg := range strings.Split(u.path, "/") { - switch { - case seg == ".": - case seg == ".." && len(kept) > 0: - kept = kept[:len(kept)-1] - default: - kept = append(kept, seg) - } - } - for i, seg := range kept { - kept[i] = rawurlencode(rawurldecode(seg)) - } - path = strings.Join(kept, "/") - } - if hasScheme && !singleColonSchemes[u.scheme] { - if !strings.HasPrefix(path, "/") { - b.WriteString("/") - } - b.WriteString(path) - } else { - b.WriteString(path) - } - if u.has(partQuery) && u.query != "" { - b.WriteString("?" + buildStr(parseQueryPairs(u.query), "")) - } - if u.has(partFragment) && u.frag != "" { - b.WriteString("#" + rawurlencode(rawurldecode(u.frag))) - } - return b.String() -} - -// phpKey is a PHP array key: an int or a string. -type phpKey struct { - isInt bool - i int64 - s string -} - -func (k phpKey) String() string { - if k.isInt { - return strconv.FormatInt(k.i, 10) - } - return k.s -} - -// keyOf is PHP's array key cast of a string: a canonical decimal integer -// becomes an int key. -func keyOf(s string) phpKey { - if n, err := strconv.ParseInt(s, 10, 64); err == nil && strconv.FormatInt(n, 10) == s { - return phpKey{isInt: true, i: n} - } - return phpKey{s: s} -} - -// phpArray is an ordered PHP array of strings, nested arrays or nulls. -type phpArray struct { - keys []phpKey - vals []any - next int64 -} - -func (a *phpArray) index(k phpKey) int { - for i, have := range a.keys { - if have == k { - return i - } - } - return -1 -} - -func (a *phpArray) set(k phpKey, v any) { - if i := a.index(k); i >= 0 { - a.vals[i] = v - return - } - a.keys = append(a.keys, k) - a.vals = append(a.vals, v) - if k.isInt && k.i >= a.next { - a.next = k.i + 1 - } -} - -func (a *phpArray) push(v any) { - a.set(phpKey{isInt: true, i: a.next}, v) -} - -// parseQueryPairs is buildUrl's split of a string query into -// $queryParams. -func parseQueryPairs(query string) *phpArray { - params := &phpArray{} - for _, pair := range strings.Split(query, "&") { - key, value := pair, pair - if i := strings.IndexByte(pair, '='); i >= 0 { - key, value = pair[:i], pair[i+1:] - } - if !strings.Contains(pair, "=") { - params.push(key) - continue - } - if strings.Contains(urldecode(pair), "[") { - k := keyOf(key) - var inner *phpArray - if i := params.index(k); i >= 0 { - inner, _ = params.vals[i].(*phpArray) - } - if inner == nil { - inner = &phpArray{} - } - inner.push(value) - params.set(k, inner) - continue - } - params.set(keyOf(key), value) - } - return params -} - -// buildStr is Winter UrlGenerator::buildStr. -func buildStr(query *phpArray, prefix string) string { - var out []string - for i, k := range query.keys { - v := query.vals[i] - var key string - if k.isInt && k.i == int64(i) { - if prefix != "" { - if strings.HasSuffix(prefix, "]") { - key = prefix - } else { - key = prefix + "[]" - } - } else { - key, _ = v.(string) - v = nil - } - } else if prefix != "" { - key = prefix + "[" + k.String() + "]" - } else { - key = k.String() - } - if nested, ok := v.(*phpArray); ok { - out = append(out, buildStr(nested, key)) - continue - } - part := rawurlencode(rawurldecode(key)) - if s, ok := v.(string); ok { - part += "=" + rawurlencode(rawurldecode(s)) - } - out = append(out, part) - } - return strings.Join(out, "&") -} - -// rawurlencode is PHP rawurlencode: every byte except A-Z a-z 0-9 - _ . ~ -// becomes %XX with upper-case hex. -func rawurlencode(s string) string { - const hex = "0123456789ABCDEF" - var b strings.Builder - for i := 0; i < len(s); i++ { - c := s[i] - if isAlnum(c) || c == '-' || c == '_' || c == '.' || c == '~' { - b.WriteByte(c) - continue - } - b.WriteByte('%') - b.WriteByte(hex[c>>4]) - b.WriteByte(hex[c&15]) - } - return b.String() -} - -// rawurldecode is PHP rawurldecode: %XX becomes its byte, anything else, -// including a malformed escape, stays. -func rawurldecode(s string) string { return percentDecode(s, false) } - -// urldecode is PHP urldecode: rawurldecode that also turns "+" into a -// space. -func urldecode(s string) string { return percentDecode(s, true) } - -func percentDecode(s string, plus bool) string { - if !strings.ContainsAny(s, "%+") { - return s - } - var b strings.Builder - for i := 0; i < len(s); i++ { - c := s[i] - if c == '+' && plus { - b.WriteByte(' ') - continue - } - if c == '%' && i+2 < len(s) && isHex(s[i+1]) && isHex(s[i+2]) { - b.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2])) - i += 2 - continue - } - b.WriteByte(c) - } - return b.String() -} - -func isHex(c byte) bool { - return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' -} - -func unhex(c byte) byte { - switch { - case '0' <= c && c <= '9': - return c - '0' - case 'a' <= c && c <= 'f': - return c - 'a' + 10 - default: - return c - 'A' + 10 - } -} - -// URLPath is the path PHP parse_url($url, PHP_URL_PATH) returns, "" when -// the URL has none. -func URLPath(raw string) string { - u := parseURL(raw) - if !u.has(partPath) { - return "" - } - return u.path -} - -// schemePrefix matches a URL that starts with a scheme and its colon. -var schemePrefix = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`) - -// AllowedItemURL reports whether a url item's address may be stored: empty, -// a path (relative or starting with a single "/"), or an absolute http or -// https URL. Any other scheme (javascript:, data:, mailto: and the like), a -// protocol-relative "//host" or "/\host", a bare fragment, surrounding -// whitespace and control characters are refused. Url::to keeps any URL -// FILTER_VALIDATE_URL accepts, so without this a javascript: URL reached -// , which sitemap.xsl renders as a link. -func AllowedItemURL(u string) bool { - if u == "" { - return true - } - if strings.TrimSpace(u) != u || strings.ContainsFunc(u, func(r rune) bool { return r < 0x20 || r == 0x7f }) { - return false - } - if httpURL(u) { - return true - } - if schemePrefix.MatchString(u) || u[0] == '#' || u[0] == '\\' { - return false - } - return !strings.HasPrefix(u, "//") && !strings.HasPrefix(u, `/\`) -} - -// httpURL reports whether u starts with http:// or https:// -// (case-insensitive). -func httpURL(u string) bool { - lower := strings.ToLower(u) - return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") -} diff --git a/classes/urlto_test.go b/classes/urlto_test.go deleted file mode 100644 index bbf424e..0000000 --- a/classes/urlto_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package classes_test - -import ( - "encoding/json" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" -) - -// TestURLTo runs every case of testdata/urlto.json through URLTo and -// compares it with the line PHP's Url::to wrote to urlto.golden (SC-3), then -// pins the helpers built on it: BackendURL and URLPath. -func TestURLTo(t *testing.T) { - var in struct { - Root string `json:"root"` - URLs []*string `json:"urls"` - } - if err := json.Unmarshal(readTestdata(t, "urlto.json"), &in); err != nil { - t.Fatal(err) - } - want := strings.Split(strings.TrimSuffix(string(readTestdata(t, "urlto.golden")), "\n"), "\n") - if len(in.URLs) < 50 || len(want) != len(in.URLs) { - t.Fatalf("%d inputs, %d golden lines", len(in.URLs), len(want)) - } - for i, u := range in.URLs { - path := "" - input := "" - if u != nil { - path, input = *u, *u - } - if got := classes.URLTo(in.Root, input); got != want[i] { - t.Errorf("case %d URLTo(%q) = %q, PHP %q", i, path, got, want[i]) - } - } - - for _, tc := range []struct{ root, want string }{ - {"https://api.ex.pl", "https://api.ex.pl"}, - {"https://api.ex.pl/", "https://api.ex.pl"}, - {"http://h:8080/base", "http://h:8080/base"}, - } { - if got := classes.BackendURL(tc.root); got != tc.want { - t.Errorf("BackendURL(%q) = %q, want %q", tc.root, got, tc.want) - } - } - for raw, want := range map[string]string{ - "https://h/a/b?x=1#f": "/a/b", - "https://h": "", - "https://h/": "/", - "/rel/path": "/rel/path", - "": "", - } { - if got := classes.URLPath(raw); got != want { - t.Errorf("URLPath(%q) = %q, want %q", raw, got, want) - } - } -} - -// TestAllowedItemURL pins the save rule of a url item's address (CR-01). -func TestAllowedItemURL(t *testing.T) { - for u, want := range map[string]bool{ - "": true, - "/": true, - "/about": true, - "about/team?x=1": true, - "https://example.test/a": true, - "HTTP://example.test/a": true, - "javascript://%0aalert(1)%0a//x": false, - "javascript:alert(1)": false, - "data:text/html,x": false, - "mailto:a@example.test": false, - "//evil.test/x": false, - `/\evil.test/x`: false, - `\\evil.test`: false, - "#top": false, - " /about": false, - "/about\n": false, - "java\tscript:alert(1)": false, - } { - if got := classes.AllowedItemURL(u); got != want { - t.Errorf("AllowedItemURL(%q) = %v, want %v", u, got, want) - } - } -} diff --git a/config/config.yaml b/config/config.yaml index 6d346a7..89018c6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,44 +1,20 @@ # golem15.sitemap defaults, read as golem15.sitemap.. # -# cache_seconds is how long a generated /sitemap.xml is served from memory. -# A save of the definition or the settings shows at once; registry content -# (a new or changed post) shows once the entry expires. 0 turns the cache -# off. -cache_seconds: 60 +# site_url is the public site address every of the sitemaps starts +# with when the Site address setting is empty; without both, app.url is +# used. The request's Host is never used. Set it per environment, e.g.: # -# The priority tier rules assign a URL the value of one of the five -# priority settings when Enable priority tiers is on. A URL's path, without -# trailing slashes ("" for the homepage), is matched against the rules in -# order and the first match wins; a URL no rule matches keeps its item's -# priority. Each rule names a setting (priority_homepage, priority_services, -# priority_blog, priority_categories or priority_legal) and either exact -# (a list of paths) or pattern (a regular expression in RE2 syntax, checked -# at boot). +# site_url: https://www.example.com # -# The defaults below are the rules the WinterCMS plugin had in code -# (getPriorityForUrl), so a site configured the same way produces the same -# priorities. A host replaces the whole list in its own config, for example -# in config/golem15.yaml under the same key below sitemap:. -priority_tiers: - - setting: priority_homepage - exact: ["", "/en"] - - setting: priority_services - pattern: '^/(uslugi|en/services|realizacje|en/work)/[^/]+$' - - setting: priority_blog - pattern: '^/(blog|en/blog)/[^/]+/[^/]+$' - - setting: priority_categories - pattern: '^/(uslugi|en/services|realizacje|en/work|blog|en/blog)(/[^/]+)?$' - - setting: priority_legal - exact: - - /polityka-prywatnosci - - /en/privacy-policy - - /regulamin - - /en/terms - - /o-nas - - /en/about - - /kontakt - - /en/contact - - /laboratorium-ai - - /en/ai-lab - - /laboratorium-ai/zabawa-animacja - - /en/ai-lab/blob-playground +# lastmod_ignore lists the record attributes whose change does not move an +# address's lastmod: a save that changes only these (or nothing) leaves it +# as it was. A host replaces the whole list in its own config. +lastmod_ignore: + - updated_at + - created_at + - views + - sort_order +# +# rebuild_delay_seconds is how long after a record change the queued +# rebuild runs; further changes in that time join the same rebuild. +rebuild_delay_seconds: 60 diff --git a/console/helpers.go b/console/helpers.go index 4912bcb..7d35fe3 100644 --- a/console/helpers.go +++ b/console/helpers.go @@ -1,6 +1,7 @@ -// Package console holds the sitemap plugin's commands: sitemap:import (the -// PHP ImportCommand) and sitemap:import-winter (a copy of a WinterCMS -// site's definition and settings). +// Package console holds the sitemap plugin's commands: sitemap:rebuild, +// sitemap:import (a JSON list of additional addresses) and +// sitemap:import-winter (a copy of a WinterCMS site's url items and +// settings). package console import ( @@ -56,20 +57,24 @@ func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) return gdb, func() { _ = sqlDB.Close() }, nil } -// singletonDefinition loads the definition with the fixed theme key, -// creating it when there is none (PHP Definition::firstOrCreate). -func singletonDefinition(tx *gorm.DB) (models.Definition, error) { - var defs []models.Definition - if err := tx.Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { - return models.Definition{}, err +// loadSettingsForUpdate reads settings row 1 inside tx, locked, creating +// it when the migration's row is gone. +func loadSettingsForUpdate(tx *gorm.DB) (models.Settings, error) { + if err := tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error; err != nil { + return models.Settings{}, err } - if len(defs) > 0 { - return defs[0], nil + var s models.Settings + if err := tx.Raw(`SELECT * FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Scan(&s).Error; err != nil { + return models.Settings{}, err } - theme := models.DefinitionTheme - def := models.Definition{Theme: &theme} - if err := tx.Create(&def).Error; err != nil { - return models.Definition{}, err - } - return def, nil + return s, nil +} + +// saveExtra stores the additional addresses on settings row 1. +func saveExtra(tx *gorm.DB, rows []models.ExtraURL) error { + if rows == nil { + rows = []models.ExtraURL{} + } + value := lagoon.Jsonable[[]models.ExtraURL]{Data: rows, Valid: true} + return tx.Exec(`UPDATE golem15_sitemap_settings SET extra_urls = ? WHERE id = 1`, value).Error } diff --git a/console/import.go b/console/import.go index 3d47adb..f238fa2 100644 --- a/console/import.go +++ b/console/import.go @@ -1,55 +1,55 @@ package console import ( - "bytes" "context" "encoding/json" "errors" "fmt" "os" + "strings" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "gorm.io/gorm" ) -// Import is sitemap:import FILE [--theme=] [--append], the PHP -// ImportCommand: it reads a JSON array of URL entries ({"url": ..., -// "changefreq": ..., "priority": ...}; changefreq defaults to monthly, -// priority to 0.5) into the sitemap definition. Without --append the -// definition keeps its non-url items first and the imported URLs replace -// its url items; with --append the URLs are added after every existing item. -// The definition is always the one /sitemap.xml serves (theme default); -// --theme is accepted for PHP compatibility and a theme other than default -// is reported. An entry whose url is not a path or an http(s) URL -// (classes.AllowedItemURL) is refused, as the admin form refuses it. +// Import is sitemap:import FILE [--append]: it reads a JSON array of URL +// entries ({"url": "/path", "lastmod": "2026-10-28"}; lastmod optional) into +// the settings' additional addresses, in one transaction. Without --append +// the imported addresses replace the additional addresses; with --append +// they are added after them, an address already listed is skipped. An +// entry whose url is not a path or an http(s) URL (models.AllowedAddress), +// or whose lastmod is not a date, stops the import before anything is +// written. The Phase 17 keys changefreq and priority are ignored, and +// --theme is accepted for the PHP command's sake. The files follow at the +// next rebuild (sitemap:rebuild). func Import(app *backpack.App) bonfire.Command { return bonfire.Command{ Name: "sitemap:import", - Description: "Import sitemap URLs from a JSON manifest file", - Long: `Reads a JSON array of URL entries ({"url": "/path", "changefreq": -"monthly", "priority": "0.5"}; changefreq and priority are optional) and puts -them into the sitemap definition that /sitemap.xml serves, in one -transaction. Every entry needs a url that is a path or an http(s) URL; one -bad entry stops the import before anything is written. + Description: "Import additional sitemap addresses from a JSON manifest file", + Long: `Reads a JSON array of URL entries ({"url": "/path", "lastmod": +"2026-10-28"}; lastmod is optional) and puts them into the sitemap's +additional addresses (Settings > Sitemaps > Additional addresses), in one +transaction. Every entry needs a url that is a path or an http(s) URL; one bad +entry stops the import before anything is written. The old changefreq and +priority keys are ignored: the sitemaps no longer write them. -By default the import replaces the definition's url items and keeps its -dynamic items (pages, posts and other generated types). --append adds the -URLs and keeps every existing item. --theme is accepted for the PHP command's -sake: only the default definition is served, so the URLs always go there. -It ends with "Imported N URLs into sitemap definition for theme 'default'."`, +By default the import replaces the additional addresses. --append adds the +addresses after them and skips those already listed. --theme is accepted for +the PHP command's sake and does nothing. The files change at the next rebuild +(sitemap:rebuild, a settings save or the nightly run). It ends with +"Imported N addresses into the sitemap's additional addresses."`, Args: []bonfire.Arg{ {Name: "file", Description: "Path to the JSON manifest (relative to the working directory or absolute)", Required: true}, }, Flags: []bonfire.Flag{ - {Name: "theme", Description: "Theme code of the PHP plugin; /sitemap.xml serves only the default definition"}, - {Name: "append", Description: "Add URLs without removing existing url-type items", Bare: true}, + {Name: "theme", Description: "Accepted for the PHP command; ignored"}, + {Name: "append", Description: "Add the addresses instead of replacing the additional addresses", Bare: true}, }, Examples: []bonfire.Example{ - {Command: "sitemap:import urls.json", Description: "Replace the url items with the manifest's URLs"}, - {Command: "sitemap:import urls.json --append", Description: "Add the URLs and keep every existing item"}, + {Command: "sitemap:import urls.json", Description: "Replace the additional addresses with the manifest's addresses"}, + {Command: "sitemap:import urls.json --append", Description: "Add the addresses and keep the existing ones"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { file, _ := in.Argument("file") @@ -72,136 +72,67 @@ func ImportFile(ctx context.Context, db *gorm.DB, file, theme string, appendMode } return fmt.Errorf("Cannot read %s: %w", file, err) } - entries, labels, ok := orderedEntries(raw) + entries, labels, ok := models.LegacyEntries(raw) if !ok { return errors.New("Invalid JSON: expected an array of URL entries") } - newItems := make([]models.DefinitionItem, 0, len(entries)) + rows := make([]models.ExtraURL, 0, len(entries)) for i, entry := range entries { var fields map[string]json.RawMessage _ = json.Unmarshal(entry, &fields) url, err := models.DecodeText(fields["url"]) - if fields["url"] == nil || err != nil || phpEmpty(fields["url"]) { + if fields["url"] == nil || err != nil || strings.TrimSpace(models.Text(url)) == "" { return fmt.Errorf("Entry %s is missing a 'url' field", labels[i]) } - if !classes.AllowedItemURL(models.Text(url)) { + u := strings.TrimSpace(models.Text(url)) + if !models.AllowedAddress(u) { return fmt.Errorf("Entry %s has a url that is not a path or an http(s) URL", labels[i]) } - item := models.DefinitionItem{Type: models.ItemTypeURL, URL: url} - item.Changefreq = textOr(fields["changefreq"], "monthly") - item.Priority = textOr(fields["priority"], "0.5") - newItems = append(newItems, item) + row := models.ExtraURL{URL: u} + if raw, ok := fields["lastmod"]; ok { + mod, err := models.DecodeText(raw) + if err != nil || (strings.TrimSpace(models.Text(mod)) != "" && !models.ValidDate(models.Text(mod))) { + return fmt.Errorf("Entry %s has a lastmod that is not a date (YYYY-MM-DD)", labels[i]) + } + row.LastMod = strings.TrimSpace(models.Text(mod)) + } + rows = append(rows, row) } - var kept int + var kept, added int err = db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - def, err := singletonDefinition(tx) + s, err := loadSettingsForUpdate(tx) if err != nil { return err } - existing := def.Items() - if !appendMode { - var dynamic []models.DefinitionItem - for _, item := range existing { - if item.Type != models.ItemTypeURL { - dynamic = append(dynamic, item) - } + next := []models.ExtraURL{} + seen := map[string]bool{} + if appendMode { + for _, e := range s.Extra() { + next = append(next, e) + seen[e.URL] = true } - existing = dynamic + kept = len(next) } - kept = len(existing) - def.SetItems(append(append([]models.DefinitionItem{}, existing...), newItems...)) - return tx.Save(&def).Error + for _, r := range rows { + if seen[r.URL] { + continue + } + seen[r.URL] = true + next = append(next, r) + added++ + } + return saveExtra(tx, next) }) if err != nil { return err } - out.Success(fmt.Sprintf("Imported %d URLs into sitemap definition for theme '%s'.", len(newItems), models.DefinitionTheme)) + out.Success(fmt.Sprintf("Imported %d addresses into the sitemap's additional addresses.", added)) if kept > 0 { - label := "existing non-URL (dynamic)" - if appendMode { - label = "existing" - } - out.Info(fmt.Sprintf("Kept %d %s items.", kept, label)) + out.Info(fmt.Sprintf("Kept %d existing addresses.", kept)) } - if theme != "" && theme != models.DefinitionTheme { - out.Warning(fmt.Sprintf("Theme '%s' was given, but /sitemap.xml serves only the '%s' definition, so the URLs were imported there.", theme, models.DefinitionTheme)) + if theme != "" { + out.Warning(fmt.Sprintf("Theme '%s' was given; the sitemap has one set of additional addresses, so the addresses were imported there.", theme)) } + out.Info("Run sitemap:rebuild (or save the sitemap settings) to update the files now.") return nil } - -// orderedEntries splits a JSON array, or a JSON object in key order (a PHP -// array with keys), into its values with their PHP labels (index or key). -func orderedEntries(raw []byte) ([]json.RawMessage, []string, bool) { - raw = bytes.TrimSpace(raw) - if len(raw) == 0 { - return nil, nil, false - } - switch raw[0] { - case '[': - var list []json.RawMessage - if err := json.Unmarshal(raw, &list); err != nil { - return nil, nil, false - } - labels := make([]string, len(list)) - for i := range list { - labels[i] = fmt.Sprint(i) - } - return list, labels, true - case '{': - dec := json.NewDecoder(bytes.NewReader(raw)) - if _, err := dec.Token(); err != nil { - return nil, nil, false - } - var list []json.RawMessage - var labels []string - for dec.More() { - tok, err := dec.Token() - if err != nil { - return nil, nil, false - } - key, _ := tok.(string) - var v json.RawMessage - if err := dec.Decode(&v); err != nil { - return nil, nil, false - } - list = append(list, v) - labels = append(labels, key) - } - return list, labels, true - } - return nil, nil, false -} - -// phpEmpty is PHP empty() of a decoded JSON value. -func phpEmpty(raw json.RawMessage) bool { - var v any - if err := json.Unmarshal(raw, &v); err != nil { - return true - } - switch t := v.(type) { - case nil: - return true - case string: - return t == "" || t == "0" - case bool: - return !t - case float64: - return t == 0 - case []any: - return len(t) == 0 - case map[string]any: - return len(t) == 0 - } - return false -} - -// textOr is PHP `$entry[key] ?? fallback` as text: a missing or null value -// is the fallback. -func textOr(raw json.RawMessage, fallback string) *string { - if raw != nil { - if s, err := models.DecodeText(raw); err == nil && s != nil { - return s - } - } - return &fallback -} diff --git a/console/import_winter.go b/console/import_winter.go index 066a1b7..b7700d9 100644 --- a/console/import_winter.go +++ b/console/import_winter.go @@ -8,15 +8,12 @@ import ( "errors" "fmt" "os" - "slices" "sort" "strings" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" - "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) @@ -34,11 +31,12 @@ var droppedTypes = map[string]bool{ // of scripts/winter-export-settings.sql (the JSON object of the // golem15_sitemap_settings system setting). Theme picks a definition when // the export holds several. Maps rename item types (old to new) before they -// are checked. PageURLs turns a static or CMS page item into a url item: -// its key is the item's reference (the PHP page file name, for example -// blog), its value the page's URL (/blog). Force replaces a definition that -// has items; DryRun reports without writing. A frontend_url that -// models.AllowedFrontendURL refuses stops the import and nothing is written. +// are checked. PageURLs turns a static or CMS page item into an additional +// address: its key is the item's reference (the PHP page file name, for +// example blog), its value the page's URL (/blog). Force replaces +// additional addresses that are already stored; DryRun reports without +// writing. A frontend_url that models.AllowedFrontendURL refuses stops the +// import and nothing is written. type WinterOptions struct { DefinitionsFile string SettingsFile string @@ -49,7 +47,7 @@ type WinterOptions struct { DryRun bool } -// WinterReport is what sitemap:import-winter did: the items kept, the +// WinterReport is what sitemap:import-winter did: the addresses kept, the // report lines of every dropped or remapped item, and whether the settings // were copied. type WinterReport struct { @@ -60,50 +58,59 @@ type WinterReport struct { Settings bool } +// winterSettings are the WinterCMS settings the per-type sitemap reads. +type winterSettings struct { + SiteURL string + DefaultLocale string +} + // ImportWinter is sitemap:import-winter: it copies a WinterCMS site's -// sitemap definition and settings into the definition /sitemap.xml serves -// (theme default) and the settings row. Item types are mapped (url kept, -// static and CMS page types dropped unless --page-url reference=/path names -// their URL, --map old=new renames applied first), -// an item of a registry type that takes references is dropped when its -// reference is not among the type's current references, an item of an -// unknown type is dropped, a url item whose URL is not a path or an http(s) -// URL (classes.AllowedItemURL) is dropped, a URL pattern that is not a path -// (signpost.ValidItemPattern) is cleared, and every dropped, remapped or -// cleared item is printed. A definition that already has items is refused -// without --force. Settings keys missing from the export take the PHP -// defaults. A frontend_url that models.AllowedFrontendURL refuses stops the -// import and nothing is written. +// sitemap definition into the settings' additional addresses and its +// frontend_url and frontend_default_locale into the site address and the +// default locale (D-32). url items are kept (static and CMS page items +// become addresses when --page-url reference=/path names their URL, +// --map old=new renames applied first); every other item is dropped and +// listed, because the content types list their records themselves. A url +// item whose URL is not a path or an http(s) URL (models.AllowedAddress) +// is dropped. Stored additional addresses are replaced only with --force. +// Settings keys of the PHP-parity format (priorities, tiers, the update +// manifest) are reported as ignored. A frontend_url that +// models.AllowedFrontendURL refuses stops the import and nothing is +// written. func ImportWinter(app *backpack.App) bonfire.Command { return bonfire.Command{ Name: "sitemap:import-winter", Description: "Copy a WinterCMS sitemap definition and settings export into this application", Long: `Copies the sitemap definition of a WinterCMS site (the JSON lines that scripts/winter-export-definitions.sql prints) and its settings (the JSON -object from scripts/winter-export-settings.sql) in one transaction. Item -types this application knows are kept; --map renames a type (old=new) and ---page-url turns a static or CMS page item into a url item with the given -path. Items that still have no known type are dropped and listed. When the -export holds several themes, --theme picks the one to copy. +object from scripts/winter-export-settings.sql) in one transaction. The +definition's url items become the sitemap's additional addresses; --page-url +turns a static or CMS page item into an address with the given path, and +--map renames a type first (old=new). Every other item is dropped and listed: +the content types (posts, categories and so on) list their records +themselves. The settings' frontend_url becomes the site address and +frontend_default_locale the default locale; priorities, tiers and the update +manifest are no longer used. When the export holds several themes, --theme +picks the one to copy. Run it once when an application moves from WinterCMS, --dry-run first: ---dry-run reports kept, remapped and dropped items and writes nothing. A -definition that already has items is replaced only with --force. It ends -with "Definition copied: N items kept, N remapped, N dropped." and "Settings -copied." for the parts the export holds.`, +--dry-run reports kept, remapped and dropped items and writes nothing. Stored +additional addresses are replaced only with --force. It ends with "Definition +copied: N items kept, N remapped, N dropped." and "Settings copied." for the +parts the export holds.`, Flags: []bonfire.Flag{ {Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"}, {Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"}, {Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"}, {Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true}, - {Name: "page-url", Description: "Copy a static or CMS page item as a url item, reference=/path (repeatable)", Repeatable: true}, - {Name: "force", Description: "Replace a definition that already has items", Bare: true}, + {Name: "page-url", Description: "Copy a static or CMS page item as an address, reference=/path (repeatable)", Repeatable: true}, + {Name: "force", Description: "Replace additional addresses that are already stored", Bare: true}, {Name: "dry-run", Description: "Report what would be copied without writing", Bare: true}, }, Examples: []bonfire.Example{ {Command: "sitemap:import-winter --definitions definitions.jsonl --settings settings.json --dry-run", Description: "Show what would be kept, remapped and dropped, without writing"}, - {Command: "sitemap:import-winter --definitions definitions.jsonl --theme main --map blog-post=journal-post", Description: "Copy the main theme's definition, renaming an item type"}, - {Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the definition, turning the about page into a url item"}, + {Command: "sitemap:import-winter --definitions definitions.jsonl --theme main", Description: "Copy the url items of the main theme's definition"}, + {Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the additional addresses, turning the about page into an address"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { maps, err := parseMaps(in.Flags("map")) @@ -128,11 +135,7 @@ copied." for the parts the export holds.`, return err } defer done() - var reg *signpost.Registry - if app != nil { - reg, _ = app.Lookup[*signpost.Registry]() - } - _, err = ImportWinterFiles(ctx, db, reg, opts, out) + _, err = ImportWinterFiles(ctx, db, opts, out) return err }, } @@ -178,15 +181,14 @@ type winterDefinition struct { Data json.RawMessage `json:"data"` } -// ImportWinterFiles runs sitemap:import-winter against db; reg is the menu -// item type registry the item types are checked against (nil knows none). -func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) { +// ImportWinterFiles runs sitemap:import-winter against db. +func ImportWinterFiles(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) { var report WinterReport if opts.DefinitionsFile == "" && opts.SettingsFile == "" { return report, errors.New("give --definitions, --settings or both") } var ( - items []models.DefinitionItem + rows []models.ExtraURL haveDef bool ) if opts.DefinitionsFile != "" { @@ -194,14 +196,14 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, if err != nil { return report, err } - out.Info(fmt.Sprintf("Copying definition %s (theme '%s') into the '%s' definition.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme), models.DefinitionTheme)) - items, err = mapItems(ctx, reg, row.Data, opts.Maps, opts.PageURLs, &report) + out.Info(fmt.Sprintf("Copying the url items of definition %s (theme '%s') into the additional addresses.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme))) + rows, err = mapItems(row.Data, opts.Maps, opts.PageURLs, &report) if err != nil { return report, err } haveDef = true } - var settings models.Settings + var settings winterSettings if opts.SettingsFile != "" { var err error settings, err = readWinterSettings(opts.SettingsFile, &report) @@ -214,12 +216,12 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, out.Warning(line) } if haveDef { - var existing []models.Definition - if err := db.WithContext(ctx).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&existing).Error; err != nil { + var existing models.Settings + if err := db.WithContext(ctx).Where("id = ?", 1).Limit(1).Find(&existing).Error; err != nil { return report, err } - if len(existing) > 0 && len(existing[0].Items()) > 0 && !opts.Force { - msg := fmt.Sprintf("The sitemap definition already has %d items; rerun with --force to replace them.", len(existing[0].Items())) + if n := len(existing.Extra()); n > 0 && !opts.Force { + msg := fmt.Sprintf("The sitemap already has %d additional addresses; rerun with --force to replace them.", n) if !opts.DryRun { return report, errors.New(msg) } @@ -238,19 +240,17 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, return report, nil } err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if _, err := loadSettingsForUpdate(tx); err != nil { + return err + } if haveDef { - def, err := singletonDefinition(tx) - if err != nil { - return err - } - def.SetItems(items) - if err := tx.Save(&def).Error; err != nil { + if err := saveExtra(tx, rows); err != nil { return err } } if report.Settings { - settings.ID = 1 - if err := tx.Save(&settings).Error; err != nil { + if err := tx.Exec(`UPDATE golem15_sitemap_settings SET site_url = ?, default_locale = ? WHERE id = 1`, + nullable(settings.SiteURL), nullable(settings.DefaultLocale)).Error; err != nil { return err } } @@ -268,6 +268,14 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, return report, nil } +// nullable is nil for "", else s. +func nullable(s string) any { + if s == "" { + return nil + } + return s +} + // pickDefinition reads the definitions export and returns the row of theme, // or the only row when theme is empty. func pickDefinition(file, theme string) (winterDefinition, error) { @@ -316,150 +324,100 @@ func pickDefinition(file, theme string) (winterDefinition, error) { return rows[0], nil } -// exportedJSON is a JSON value that the export may hold as text (a TEXT -// column selected as it is) or as JSON: a string is decoded once more. -func exportedJSON(raw json.RawMessage) json.RawMessage { - raw = bytes.TrimSpace(raw) - if len(raw) > 0 && raw[0] == '"' { - var text string - if json.Unmarshal(raw, &text) == nil { - return json.RawMessage(strings.TrimSpace(text)) +// mapItems decodes the exported data and keeps its url items as +// additional addresses. +func mapItems(data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.ExtraURL, error) { + items, labels, err := models.ParseLegacyItems(data) + if err != nil { + return nil, err + } + out := []models.ExtraURL{} + seen := map[string]bool{} + keep := func(label, u string) { + if seen[u] { + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (address '%s' is already listed)", label, u)) + report.Dropped++ + return } + seen[u] = true + out = append(out, models.ExtraURL{URL: u}) + report.Kept++ } - return raw -} - -// mapItems decodes the exported data and maps its items. -func mapItems(ctx context.Context, reg *signpost.Registry, data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.DefinitionItem, error) { - data = exportedJSON(data) - if len(data) == 0 || bytes.Equal(data, []byte("null")) { - return []models.DefinitionItem{}, nil - } - rawItems, labels, ok := orderedEntries(data) - if !ok { - return nil, errors.New("the definition data is not a list of items") - } - out := []models.DefinitionItem{} - refs := map[string][]string{} - for i, raw := range rawItems { - var item models.DefinitionItem - if err := json.Unmarshal(raw, &item); err != nil { - return nil, fmt.Errorf("definition item %s: %w", labels[i], err) - } - var fields map[string]any - _ = json.Unmarshal(raw, &fields) + for i, item := range items { label := describeItem(labels[i], item) - if page, ok := fields["cmsPage"].(string); ok && page != "" { - report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, page)) + if item.CMSPage != "" { + report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, item.CMSPage)) } - if p := models.Text(item.URLPattern); p != "" && !signpost.ValidItemPattern(p) { - report.Lines = append(report.Lines, fmt.Sprintf("%s: urlPattern '%s' cleared (not a path starting with a single /)", label, p)) - item.URLPattern = nil - } - if url, ok := pageURLs[models.Text(item.Reference)]; ok && droppedTypes[item.Type] { - if !classes.AllowedItemURL(url) { + if url, ok := pageURLs[item.Reference]; ok && droppedTypes[item.Type] { + if !models.AllowedAddress(url) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (--page-url '%s' is not a path or an http(s) URL)", label, url)) report.Dropped++ continue } - report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as type 'url' with url '%s'", label, url)) + report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as an additional address '%s'", label, url)) report.Remapped++ - item.Type = models.ItemTypeURL - item.URL = &url - item.Reference = nil - out = append(out, item) - report.Kept++ + keep(label, url) continue } if to, ok := maps[item.Type]; ok && to != item.Type { report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to)) report.Remapped++ item.Type = to - if to == "" { - report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) - report.Dropped++ - continue - } } - if item.Type == models.ItemTypeURL { - if !classes.AllowedItemURL(models.Text(item.URL)) { + switch { + case item.Type == models.LegacyItemURL: + if !models.AllowedAddress(item.URL) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the url is not a path or an http(s) URL)", label)) report.Dropped++ continue } - out = append(out, item) - report.Kept++ - continue - } - if droppedTypes[item.Type] { + keep(label, item.URL) + case item.Type == "": + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) + report.Dropped++ + case droppedTypes[item.Type]: report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label)) report.Dropped++ - continue - } - if _, known := reg.Type(item.Type); !known { - report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (type '%s' is not registered; map it with --map %s=)", label, item.Type, item.Type)) + default: + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the content types list their records themselves)", label)) report.Dropped++ - continue } - values, cached := refs[item.Type] - if !cached { - set, err := signpost.ReferenceOptions(ctx, reg, item.Type) - if err != nil { - return nil, err - } - if set.Applies { - values = []string{} - for _, o := range set.Options { - values = append(values, o.Value) - } - } - refs[item.Type] = values - } - if values != nil && !slices.Contains(values, models.Text(item.Reference)) { - report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (reference '%s' is not among the current references of '%s')", label, models.Text(item.Reference), item.Type)) - report.Dropped++ - continue - } - out = append(out, item) - report.Kept++ } return out, nil } -func describeItem(label string, item models.DefinitionItem) string { +func describeItem(label string, item models.LegacyItem) string { s := fmt.Sprintf("item %s (type '%s'", label, item.Type) - if item.URL != nil && *item.URL != "" { - s += fmt.Sprintf(", url '%s'", *item.URL) + if item.URL != "" { + s += fmt.Sprintf(", url '%s'", item.URL) } - if item.Reference != nil && *item.Reference != "" { - s += fmt.Sprintf(", reference '%s'", *item.Reference) + if item.Reference != "" { + s += fmt.Sprintf(", reference '%s'", item.Reference) } return s + ")" } -// readWinterSettings reads the exported settings object onto the PHP -// defaults. A frontend_url that models.AllowedFrontendURL refuses is an -// error; the value is quoted so its control characters are not printed raw. -func readWinterSettings(file string, report *WinterReport) (models.Settings, error) { +// ignoredWinterSettings are the PHP settings of the PHP-parity format the +// per-type sitemap no longer reads. +var ignoredWinterSettings = map[string]bool{ + "updates_manifest_path": true, "enable_priority_tiers": true, + "priority_homepage": true, "priority_services": true, "priority_blog": true, + "priority_categories": true, "priority_legal": true, +} + +// readWinterSettings reads the exported settings object. A frontend_url +// that models.AllowedFrontendURL refuses is an error; the value is quoted so +// its control characters are not printed raw. +func readWinterSettings(file string, report *WinterReport) (winterSettings, error) { raw, err := os.ReadFile(file) if err != nil { - return models.Settings{}, fmt.Errorf("File not found: %s", file) + return winterSettings{}, fmt.Errorf("File not found: %s", file) } var fields map[string]json.RawMessage - if err := json.Unmarshal(exportedJSON(raw), &fields); err != nil { - return models.Settings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) - } - s := models.DefaultSettings() - texts := map[string]**string{ - "frontend_url": &s.FrontendURL, - "frontend_default_locale": &s.FrontendDefaultLocale, - "updates_manifest_path": &s.UpdatesManifestPath, - models.PriorityHomepage: &s.PriorityHomepage, - models.PriorityServices: &s.PriorityServices, - models.PriorityBlog: &s.PriorityBlog, - models.PriorityCategories: &s.PriorityCategories, - models.PriorityLegal: &s.PriorityLegal, + if err := json.Unmarshal(models.LegacyJSON(raw), &fields); err != nil { + return winterSettings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) } + var s winterSettings keys := make([]string, 0, len(fields)) for key := range fields { keys = append(keys, key) @@ -467,22 +425,25 @@ func readWinterSettings(file string, report *WinterReport) (models.Settings, err sort.Strings(keys) for _, key := range keys { value := fields[key] - if dest, ok := texts[key]; ok { + switch { + case key == "frontend_url" || key == "frontend_default_locale": text, err := models.DecodeText(value) if err != nil { - return models.Settings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) + return winterSettings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) } - *dest = text - continue + if key == "frontend_url" { + s.SiteURL = models.Text(text) + } else { + s.DefaultLocale = strings.TrimSpace(models.Text(text)) + } + case ignoredWinterSettings[key]: + report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (the per-type sitemaps do not use it)", key)) + default: + report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) } - if key == "enable_priority_tiers" { - s.EnablePriorityTiers = !phpEmpty(value) - continue - } - report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) } - if url := models.Text(s.FrontendURL); !models.AllowedFrontendURL(url) { - return models.Settings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, url) + if !models.AllowedFrontendURL(s.SiteURL) { + return winterSettings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, s.SiteURL) } return s, nil } diff --git a/console/import_winter_test.go b/console/import_winter_test.go index 97dcae8..8172fec 100644 --- a/console/import_winter_test.go +++ b/console/import_winter_test.go @@ -9,7 +9,6 @@ import ( "strings" "testing" - "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/bonfire" ) @@ -50,7 +49,7 @@ func TestWinterSettingsFrontendURL(t *testing.T) { for _, dry := range []bool{false, true} { var buf bytes.Buffer out := bonfire.NewOutput(strings.NewReader(""), &buf, &buf) - report, err := ImportWinterFiles(context.Background(), nil, nil, WinterOptions{SettingsFile: file, DryRun: dry}, out) + report, err := ImportWinterFiles(context.Background(), nil, WinterOptions{SettingsFile: file, DryRun: dry}, out) if err == nil || !strings.Contains(err.Error(), "setting frontend_url") { t.Errorf("ImportWinterFiles(%q, dry run %v) = %v, want an error naming setting frontend_url", value, dry, err) } @@ -85,7 +84,7 @@ func TestWinterSettingsFrontendURL(t *testing.T) { t.Errorf("%s: readWinterSettings = %v", name, err) continue } - if got := models.Text(s.FrontendURL); got != tc.want { + if got := s.SiteURL; got != tc.want { t.Errorf("%s: frontend_url = %q, want %q", name, got, tc.want) } } diff --git a/controllers/admin_registry.go b/controllers/admin_registry.go index 5e72d50..bd8944b 100644 --- a/controllers/admin_registry.go +++ b/controllers/admin_registry.go @@ -1,3 +1,5 @@ +// Package controllers holds the sitemap plugin's admin controllers: the +// settings singleton. package controllers import ( @@ -10,7 +12,6 @@ import ( // a database handle or registry of its own. func AdminControllers(app func() *backpack.App) []pact.AdminController { return []pact.AdminController{ - definitionsAdminController{app: app}, settingsAdminController{app: app}, } } diff --git a/controllers/definitions.go b/controllers/definitions.go deleted file mode 100644 index 07e87f1..0000000 --- a/controllers/definitions.go +++ /dev/null @@ -1,162 +0,0 @@ -// Package controllers holds the sitemap plugin's admin controllers. -package controllers - -import ( - "context" - "errors" - "fmt" - "strings" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/backpack" - "git.golem15.com/golem15/summercms/modules/cabana" - "git.golem15.com/golem15/summercms/modules/pact" - "git.golem15.com/golem15/summercms/modules/signpost" - "gorm.io/gorm" -) - -const ( - // DefinitionsControllerID is the cabana controller id of the sitemap - // definition form. - DefinitionsControllerID = "golem15.sitemap.definitions" - // PermissionAccessDefinitions is Definitions::$requiredPermissions. - PermissionAccessDefinitions = "golem15.sitemap.access_definitions" -) - -// typeURLLabel is the label of the built-in url item type. -const typeURLLabel = "golem15.sitemap::lang.item.url" - -// The phrase keys of the item row refusals. -const ( - errURLScheme = "golem15.sitemap::lang.errors.url_scheme" - errURLPattern = "golem15.sitemap::lang.errors.url_pattern" - errReference = "golem15.sitemap::lang.item.reference_required" -) - -// definitionsAdminController edits the one sitemap definition of the -// application (theme default): a singleton form whose items are a repeater -// with a type dropdown (url plus the registry types) and a reference -// dropdown that depends on it. -type definitionsAdminController struct { - app func() *backpack.App -} - -func (definitionsAdminController) ID() string { return DefinitionsControllerID } - -func (definitionsAdminController) ModelName() string { return `Golem15\Sitemap\Models\Definition` } - -func (definitionsAdminController) ConfigDir() string { return "controllers/definitions" } - -func (definitionsAdminController) RequiredPermissions() []string { - return []string{PermissionAccessDefinitions} -} - -func (definitionsAdminController) NewRecord() any { return &models.Definition{} } - -// The singleton record is the definition with the fixed theme key, created -// with no items when there is none (PHP index(): firstOrCreate, then the -// update form). -func (definitionsAdminController) SingletonRecord(_ context.Context, db *gorm.DB) (any, error) { - if db == nil { - return nil, errors.New("golem15.sitemap: database is not configured") - } - var defs []models.Definition - if err := db.Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { - return nil, err - } - if len(defs) > 0 { - return &defs[0], nil - } - theme := models.DefinitionTheme - def := &models.Definition{Theme: &theme} - def.SetItems(nil) - if err := db.Create(def).Error; err != nil { - return nil, err - } - return def, nil -} - -func (c definitionsAdminController) registry() *signpost.Registry { - if c.app == nil { - return nil - } - app := c.app() - if app == nil { - return nil - } - reg, _ := app.Lookup[*signpost.Registry]() - return reg -} - -// DropdownOptions serves the item type dropdown: url, then every registry -// type in registry order with its label (PHP getTypeOptions). -func (c definitionsAdminController) DropdownOptions(string) []pact.Option { - out := []pact.Option{{Value: models.ItemTypeURL, Label: typeURLLabel}} - for _, t := range c.registry().Types() { - out = append(out, pact.Option{Value: t.Key, Label: t.Label}) - } - return out -} - -// DependentOptions serves the reference dropdown of an item row: the -// references of the row's type through signpost.ReferenceOptions; url and a -// type without references answer Applies false. -func (c definitionsAdminController) DependentOptions(ctx context.Context, _ string, values map[string]any) (pact.DependentOptionSet, error) { - typ, _ := values["type"].(string) - return signpost.ReferenceOptions(ctx, c.registry(), typ) -} - -// FormBeforeCreate checks the item rows (see checkItems). -func (c definitionsAdminController) FormBeforeCreate(_ context.Context, model any) error { - return c.checkItems(model) -} - -// FormBeforeUpdate checks the item rows (see checkItems). -func (c definitionsAdminController) FormBeforeUpdate(_ context.Context, model any) error { - return c.checkItems(model) -} - -// checkItems refuses, per row, a url outside classes.AllowedItemURL -// (data..url), a URL pattern that is not a path -// (signpost.ValidItemPattern, data..urlPattern) and a missing reference -// for a registry type that takes one (data..reference). The PHP plugin -// checked none of them: a javascript: URL would reach , which -// sitemap.xsl renders as a link, and a row without its reference resolves to -// nothing and is left out of the sitemap without a word. -func (c definitionsAdminController) checkItems(model any) error { - def, ok := model.(*models.Definition) - if !ok || def == nil { - return nil - } - reg := c.registry() - details := map[string]any{} - for i, item := range def.Items() { - if item.Type != models.ItemTypeURL { - if t, known := reg.Type(item.Type); known && t.References != nil && strings.TrimSpace(models.Text(item.Reference)) == "" { - details[fmt.Sprintf("data.%d.reference", i)] = []string{errReference} - } - } - if !classes.AllowedItemURL(models.Text(item.URL)) { - details[fmt.Sprintf("data.%d.url", i)] = []string{errURLScheme} - } - if p := models.Text(item.URLPattern); p != "" && !signpost.ValidItemPattern(p) { - details[fmt.Sprintf("data.%d.urlPattern", i)] = []string{errURLPattern} - } - } - if len(details) > 0 { - return &cabana.ValidationError{Details: details} - } - return nil -} - -var ( - _ pact.AdminController = definitionsAdminController{} - _ pact.AdminPermissioned = definitionsAdminController{} - _ pact.AdminRecordSource = definitionsAdminController{} - _ pact.AdminSingleton = definitionsAdminController{} - _ pact.DropdownOptionsProvider = definitionsAdminController{} - _ pact.DependentOptionsProvider = definitionsAdminController{} - _ pact.FormBeforeCreate = definitionsAdminController{} - _ pact.FormBeforeUpdate = definitionsAdminController{} -) diff --git a/controllers/definitions/config_form.yaml b/controllers/definitions/config_form.yaml deleted file mode 100644 index 89f044c..0000000 --- a/controllers/definitions/config_form.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# =================================== -# Form Behavior Config -# =================================== - -# Record name -name: golem15.sitemap::lang.definition.title - -# Model Form Field configuration -form: ~/plugins/golem15/sitemap/models/definition/fields.yaml - -# Model Class name -modelClass: Golem15\Sitemap\Models\Definition - -# Default redirect location -defaultRedirect: system/settings - -# Update page -update: - redirect: golem15/sitemap/definitions - redirectClose: system/settings - -messages: - update: golem15.sitemap::lang.definition.title - saved: backend::lang.messages.form.saved diff --git a/controllers/definitions_test.go b/controllers/definitions_test.go deleted file mode 100644 index 1ae0677..0000000 --- a/controllers/definitions_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package controllers - -import ( - "errors" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/cabana" -) - -func ptr(s string) *string { return &s } - -// TestCheckItems pins the CR-01 save check: per row, a url outside -// classes.AllowedItemURL and a URL pattern that is not a path are refused -// under data..url and data..urlPattern. -func TestCheckItems(t *testing.T) { - c := definitionsAdminController{} - def := &models.Definition{} - def.SetItems([]models.DefinitionItem{ - {Type: models.ItemTypeURL, URL: ptr("/about")}, - {Type: models.ItemTypeURL, URL: ptr("javascript://%0aalert(1)%0a//x")}, - {Type: "acme-post", URLPattern: ptr("//evil.test/:slug")}, - {Type: "acme-post", URLPattern: ptr("/blog/:slug")}, - }) - err := c.checkItems(def) - var verr *cabana.ValidationError - if !errors.As(err, &verr) { - t.Fatalf("checkItems = %v, want a validation error", err) - } - if len(verr.Details) != 2 || verr.Details["data.1.url"] == nil || verr.Details["data.2.urlPattern"] == nil { - t.Fatalf("details = %v", verr.Details) - } - - ok := &models.Definition{} - ok.SetItems([]models.DefinitionItem{{Type: models.ItemTypeURL, URL: ptr("https://example.test/")}}) - if err := c.checkItems(ok); err != nil { - t.Fatalf("a valid definition was refused: %v", err) - } -} diff --git a/controllers/settings.go b/controllers/settings.go index 940a03b..6575482 100644 --- a/controllers/settings.go +++ b/controllers/settings.go @@ -15,9 +15,15 @@ import ( "gorm.io/gorm" ) -// SettingsControllerID is the cabana controller id of the sitemap settings -// singleton (UI-SPEC C26), linked from the sitemap settings tile. -const SettingsControllerID = "golem15.sitemap.settings" +const ( + // SettingsControllerID is the cabana controller id of the sitemap + // settings singleton (UI-SPEC C26), linked from the sitemap settings + // tile. + SettingsControllerID = "golem15.sitemap.settings" + // PermissionAccessDefinitions guards the sitemap settings. The code is + // the Phase 17 (and WinterCMS) one, so existing roles keep their access. + PermissionAccessDefinitions = "golem15.sitemap.access_definitions" +) // The reasons the files partial counts as "left out", in display order // (P33-11). A deleted or unpublished address is a hole, not left out. diff --git a/import_test.go b/import_test.go index 5fa3297..25602b4 100644 --- a/import_test.go +++ b/import_test.go @@ -6,54 +6,58 @@ import ( "strings" "testing" + "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/console" "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/signpost" ) -// TestImportWinter pins sitemap:import-winter on Postgres (D-12): type maps -// and drops with their report lines, the reference check, --theme with -// several exported definitions, --dry-run writing nothing, the refusal of a -// second copy without --force and the replacement with it, the settings -// upsert onto row 1 with PHP defaults for missing keys, and the refusals for -// missing, empty and malformed exports. -func TestImportWinter(t *testing.T) { - e := bootSitemap(t) - ctx := context.Background() - reg, ok := e.app.Lookup[*signpost.Registry]() - if !ok { - t.Fatal("signpost registry is not published") +// storedExtra reads the additional addresses of settings row 1. +func (e *sitemapEnv) storedExtra(t *testing.T) []models.ExtraURL { + t.Helper() + s, err := classes.LoadSettings(e.gdb) + if err != nil { + t.Fatal(err) } + return s.Extra() +} + +// extraURLs lists the addresses of rows. +func extraURLs(rows []models.ExtraURL) string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.URL) + } + return strings.Join(out, " ") +} + +// TestImportWinter pins sitemap:import-winter on Postgres (D-32): the url +// items of a WinterCMS definition become the additional addresses (static +// and CMS page items through --page-url, types renamed by --map first), +// every other item is dropped and listed, --theme picks a definition, +// --dry-run writes nothing, stored addresses are replaced only with +// --force, and frontend_url and frontend_default_locale become site_url and +// default_locale while the PHP-parity keys are reported as ignored; missing, +// empty and malformed exports and an unsafe frontend_url are refused. +func TestImportWinter(t *testing.T) { + e := bootIndex(t) + ctx := context.Background() dir := t.TempDir() row := func(id int, theme string, items string) string { quoted, _ := json.Marshal(items) return `{"id": ` + string(rune('0'+id)) + `, "theme": "` + theme + `", "data": ` + string(quoted) + "}\n" } - items := `[{"type":"url","url":"/"},` + - `{"type":"old-posts","reference":null,"nesting":"1"},` + + items := `[{"type":"url","url":"/","changefreq":"always","priority":"1.0"},` + + `{"type":"blog-posts","reference":null,"nesting":"1","cmsPage":"blog/post"},` + `{"type":"acme-post","reference":"7"},` + - `{"type":"acme-post","reference":"8"},` + - `{"type":"gone-type","reference":"1"},` + `{"type":"static-page","reference":"about"},` + - `{"type":"remove-me"}]` + `{"type":"cms-page","reference":"news"},` + + `{"type":"remove-me"},` + + `{"type":"url","url":"javascript://%0aalert(1)%0a//x"},` + + `{"type":"url","url":"/"}]` defs := writeFile(t, dir, "definitions.jsonl", row(1, "main", items)+"\n"+row(2, "other", `[{"type":"url","url":"/other"}]`)) - stored := func(t *testing.T) []models.DefinitionItem { - t.Helper() - var rows []models.Definition - if err := e.gdb.Where("theme = ?", models.DefinitionTheme).Find(&rows).Error; err != nil { - t.Fatal(err) - } - if len(rows) == 0 { - return nil - } - if len(rows) != 1 { - t.Fatalf("%d definitions", len(rows)) - } - return rows[0].Items() - } run := func(opts console.WinterOptions) (console.WinterReport, string, error) { out, buf := testOutput() - report, err := console.ImportWinterFiles(ctx, e.gdb, reg, opts, out) + report, err := console.ImportWinterFiles(ctx, e.gdb, opts, out) return report, buf.String(), err } @@ -76,18 +80,22 @@ func TestImportWinter(t *testing.T) { t.Fatalf("%s: err = %v, want %q\n%s", name, err, tc.want, out) } } - if got := stored(t); got != nil { + if got := e.storedExtra(t); len(got) != 0 { t.Fatalf("a refused import wrote %+v", got) } }) - opts := console.WinterOptions{DefinitionsFile: defs, Theme: "main", Maps: map[string]string{"old-posts": "acme-list", "remove-me": ""}} + opts := console.WinterOptions{DefinitionsFile: defs, Theme: "main", Maps: map[string]string{"remove-me": ""}, PageURLs: map[string]string{"news": "/news"}} wantLines := []string{ - "item 1 (type 'old-posts'): type 'old-posts' mapped to 'acme-list'", - "item 3 (type 'acme-post', reference '8'): dropped (reference '8' is not among the current references of 'acme-post')", - "item 4 (type 'gone-type', reference '1'): dropped (type 'gone-type' is not registered; map it with --map gone-type=)", - "item 5 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", - "item 6 (type 'remove-me'): dropped (mapped to nothing)", + "item 1 (type 'blog-posts'): cmsPage 'blog/post' dropped (CMS pages are not supported)", + "item 1 (type 'blog-posts'): dropped (the content types list their records themselves)", + "item 2 (type 'acme-post', reference '7'): dropped (the content types list their records themselves)", + "item 3 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", + "item 4 (type 'cms-page', reference 'news'): copied as an additional address '/news'", + "item 5 (type 'remove-me'): type 'remove-me' mapped to ''", + "item 5 (type 'remove-me'): dropped (mapped to nothing)", + "item 6 (type 'url', url 'javascript://%0aalert(1)%0a//x'): dropped (the url is not a path or an http(s) URL)", + "item 7 (type 'url', url '/'): dropped (address '/' is already listed)", } t.Run("dry run writes nothing", func(t *testing.T) { @@ -97,7 +105,7 @@ func TestImportWinter(t *testing.T) { if err != nil { t.Fatalf("dry run: %v\n%s", err, out) } - if report.Kept != 3 || report.Dropped != 4 || report.Remapped != 2 { + if report.Kept != 2 || report.Dropped != 6 || report.Remapped != 2 { t.Fatalf("report = %+v\n%s", report, out) } for _, line := range wantLines { @@ -105,10 +113,10 @@ func TestImportWinter(t *testing.T) { t.Errorf("output lacks %q:\n%s", line, out) } } - if !strings.Contains(out, "Dry run: 3 items kept, 2 remapped, 4 dropped.") || !strings.Contains(out, "Dry run complete (no changes made).") { + if !strings.Contains(out, "Dry run: 2 items kept, 2 remapped, 6 dropped.") || !strings.Contains(out, "Dry run complete (no changes made).") { t.Fatalf("dry run output:\n%s", out) } - if got := stored(t); got != nil { + if got := e.storedExtra(t); len(got) != 0 { t.Fatalf("the dry run wrote %+v", got) } }) @@ -118,32 +126,30 @@ func TestImportWinter(t *testing.T) { if err != nil { t.Fatalf("import: %v\n%s", err, out) } - got := stored(t) - if report.Kept != 3 || len(got) != 3 || got[0].Type != "url" || got[1].Type != "acme-list" || !got[1].Nesting || got[2].Type != "acme-post" || models.Text(got[2].Reference) != "7" { - t.Fatalf("copied = %+v (report %+v)", got, report) + if got := extraURLs(e.storedExtra(t)); report.Kept != 2 || got != "/ /news" { + t.Fatalf("copied = %q (report %+v)", got, report) } - if !strings.Contains(out, "Definition copied: 3 items kept, 2 remapped, 4 dropped.") || !strings.Contains(out, "theme 'main'") { + if !strings.Contains(out, "Definition copied: 2 items kept, 2 remapped, 6 dropped.") || !strings.Contains(out, "theme 'main'") { t.Fatalf("output:\n%s", out) } other := console.WinterOptions{DefinitionsFile: defs, Theme: "other"} - if _, out, err := run(other); err == nil || !strings.Contains(err.Error(), "already has 3 items; rerun with --force") { + if _, out, err := run(other); err == nil || !strings.Contains(err.Error(), "already has 2 additional addresses; rerun with --force") { t.Fatalf("second copy: %v\n%s", err, out) } - // A dry run over existing items warns instead of failing. dry := other dry.DryRun = true - if _, out, err := run(dry); err != nil || !strings.Contains(out, "already has 3 items") { - t.Fatalf("dry run over items: %v\n%s", err, out) + if _, out, err := run(dry); err != nil || !strings.Contains(out, "already has 2 additional addresses") { + t.Fatalf("dry run over addresses: %v\n%s", err, out) } - if len(stored(t)) != 3 { - t.Fatal("a refused copy changed the definition") + if got := extraURLs(e.storedExtra(t)); got != "/ /news" { + t.Fatalf("a refused copy changed the addresses: %q", got) } other.Force = true if _, out, err := run(other); err != nil { t.Fatalf("--force: %v\n%s", err, out) } - if got := stored(t); len(got) != 1 || models.Text(got[0].URL) != "/other" { - t.Fatalf("after --force = %+v", got) + if got := extraURLs(e.storedExtra(t)); got != "/other" { + t.Fatalf("after --force = %q", got) } }) @@ -152,70 +158,42 @@ func TestImportWinter(t *testing.T) { if _, out, err := run(console.WinterOptions{DefinitionsFile: raw, Force: true}); err != nil { t.Fatalf("raw JSON data: %v\n%s", err, out) } - if got := stored(t); len(got) != 1 || models.Text(got[0].URL) != "/raw" { - t.Fatalf("raw data = %+v", got) + if got := extraURLs(e.storedExtra(t)); got != "/raw" { + t.Fatalf("raw data = %q", got) } null := writeFile(t, dir, "null.jsonl", `{"id":6,"theme":"t","data":null}`) if report, out, err := run(console.WinterOptions{DefinitionsFile: null, Force: true}); err != nil || report.Kept != 0 { t.Fatalf("null data: %+v %v\n%s", report, err, out) } - if got := stored(t); len(got) != 0 { + if got := e.storedExtra(t); len(got) != 0 { t.Fatalf("null data stored %+v", got) } }) - t.Run("url and pattern checks", func(t *testing.T) { - bad := writeFile(t, dir, "bad-urls.jsonl", row(7, "t", `[{"type":"url","url":"javascript://%0aalert(1)%0a//x"},`+ - `{"type":"url","url":"/ok"},`+ - `{"type":"acme-post","reference":"7","urlPattern":"//evil.test/:slug"}]`)) - report, out, err := run(console.WinterOptions{DefinitionsFile: bad, Force: true}) - if err != nil { - t.Fatalf("import: %v\n%s", err, out) - } - if report.Kept != 2 || report.Dropped != 1 { - t.Fatalf("report = %+v\n%s", report, out) - } - for _, line := range []string{ - "item 0 (type 'url', url 'javascript://%0aalert(1)%0a//x'): dropped (the url is not a path or an http(s) URL)", - "item 2 (type 'acme-post', reference '7'): urlPattern '//evil.test/:slug' cleared (not a path starting with a single /)", - } { - if !strings.Contains(out, line) { - t.Errorf("output lacks %q:\n%s", line, out) - } - } - got := stored(t) - if len(got) != 2 || models.Text(got[0].URL) != "/ok" || got[1].URLPattern != nil { - t.Fatalf("stored = %+v", got) - } - }) - - t.Run("settings upsert", func(t *testing.T) { - first := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test","enable_priority_tiers":"1","priority_blog":0.7,"unknown":"x"}`) + t.Run("settings", func(t *testing.T) { + first := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test","frontend_default_locale":"pl","enable_priority_tiers":"1","priority_blog":0.7,"unknown":"x"}`) report, out, err := run(console.WinterOptions{SettingsFile: first}) - if err != nil || !report.Settings || !strings.Contains(out, "setting 'unknown' ignored") || !strings.Contains(out, "Settings copied.") { + if err != nil || !report.Settings || !strings.Contains(out, "setting 'unknown' ignored (not a sitemap setting of this port)") || + !strings.Contains(out, "setting 'priority_blog' ignored (the per-type sitemaps do not use it)") || !strings.Contains(out, "Settings copied.") { t.Fatalf("settings: %+v %v\n%s", report, err, out) } - second := writeFile(t, dir, "settings2.json", `"{\"frontend_url\":\"https://other.example.test\",\"enable_priority_tiers\":\"0\"}"`) + s, err := classes.LoadSettings(e.gdb) + if err != nil || models.Text(s.SiteURL) != "https://www.example.test" || models.Text(s.DefaultLocale) != "pl" { + t.Fatalf("copied settings = %+v %v", s, err) + } + second := writeFile(t, dir, "settings2.json", `"{\"frontend_url\":\"https://other.example.test\"}"`) if _, out, err := run(console.WinterOptions{SettingsFile: second}); err != nil { t.Fatalf("second settings: %v\n%s", err, out) } - var rows []models.Settings - if err := e.gdb.Find(&rows).Error; err != nil { - t.Fatal(err) - } - if len(rows) != 1 || rows[0].ID != 1 || models.Text(rows[0].FrontendURL) != "https://other.example.test" || rows[0].EnablePriorityTiers || rows[0].Priority(models.PriorityBlog) != "0.6" { - t.Fatalf("settings rows = %+v", rows) + if s, _ = classes.LoadSettings(e.gdb); models.Text(s.SiteURL) != "https://other.example.test" || s.DefaultLocale != nil { + t.Fatalf("second settings = %+v", s) } dry := writeFile(t, dir, "settings3.json", `{"frontend_url":"https://dry.example.test"}`) if _, out, err := run(console.WinterOptions{SettingsFile: dry, DryRun: true}); err != nil || !strings.Contains(out, "Dry run: the settings would be copied.") { t.Fatalf("settings dry run: %v\n%s", err, out) } - if err := e.gdb.Find(&rows).Error; err != nil || models.Text(rows[0].FrontendURL) != "https://other.example.test" { - t.Fatalf("the dry run changed the settings: %+v %v", rows, err) - } - - // T-17-15: a frontend_url that is not a plain http(s) URL stops the - // import, also as a dry run, and the row keeps its value. + // T-17-15: an unsafe frontend_url stops the import, also as a dry + // run, and the row keeps its value. quoted, _ := json.Marshal(map[string]string{"frontend_url": `https://x"/> &. Leave it empty to keep the backend URL.' file_size: Enter a whole number from 1 to 50 000. slug: Use lowercase letters, digits and hyphens, e.g. posts. @@ -47,10 +12,6 @@ errors: html_path: Start the address with / and use a path only, e.g. /mapa-strony. page_size: Enter a whole number from 50 to 500. lastmod: 'Enter a date as YYYY-MM-DD, e.g. 2026-10-28, or leave it empty.' -definition: - not_found: No sitemap definition was found. Try creating one first. - title: Sitemap definition - view: View sitemap.xml settings: label: Sitemaps description: XML sitemaps per content type, images, News, IndexNow and the HTML sitemap. diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index 2ee3f53..a97c42a 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -2,44 +2,9 @@ plugin: name: Mapy witryny description: Mapy XML według typów treści, ze stałymi plikami, prawdziwym lastmod i hreflang. permissions: - access_settings: Zarządzaj ustawieniami wtyczki - access_definitions: Zarządzaj definicją mapy strony -item: - location: 'Lokacja:' - priority: Priorytet - changefreq: Częstotliwość zmian - always: cały czas - hourly: co godzinę - daily: codziennie - weekly: co tydzień - monthly: co miesiąc - yearly: co rok - never: nigdy - editor_title: Edytuj pozycję mapy - type: Typ - allow_nested_items: Pozwól na zagnieżdżanie pozycji - allow_nested_items_comment: Zagnieżdżone pozycje mogą zostać wygenerowane dynamicznie przez statyczne strony i niektóre rodzaje typów pozycji. - url: URL - reference: Odnośnik - title_required: Tytuł jest wymagany - unknown_type: Nieznany typ pozycji - unnamed: Pozycja bez nazwy - add_item: Dodaj pozycję - new_item: Nowa pozycja - reference_required: Odnośnik dla pozycji jest wymagany. - url_required: Adres URL jest wymagany - url_pattern: Wzorzec URL - url_pattern_comment: Użyj składni :placeholder, np. /blog/:slug - url_pattern_required: Podaj wzorzec URL - page: Strona - check: Sprawdź - definition: Definicja - save_definition: Zapisywanie definicji... - load_indicator: Czyszczenie definicji... - empty_confirm: Wyczyścić tę definicję? + access_definitions: Zarządzaj ustawieniami map witryny errors: url_scheme: Podaj ścieżkę, np. /o-nas, albo link zaczynający się od http:// lub https://. - url_pattern: Podaj ścieżkę zaczynającą się od jednego /, np. /blog/:slug. frontend_url: 'Podaj pełny adres zaczynający się od http:// lub https://, bez spacji, cudzysłowów i znaków < > &. Zostaw puste, aby zachować adres backendu.' file_size: Wpisz liczbę całkowitą od 1 do 50 000. slug: Użyj małych liter, cyfr i łączników, np. posts. @@ -47,10 +12,6 @@ errors: html_path: Zacznij adres od / i użyj samej ścieżki, np. /mapa-strony. page_size: Wpisz liczbę całkowitą od 50 do 500. lastmod: 'Wpisz datę w formacie RRRR-MM-DD, np. 2026-10-28, albo zostaw puste.' -definition: - not_found: Nie znaleziono definicji mapy strony. Musisz ją utworzyć. - title: Definicja mapy strony - view: Zobacz sitemap.xml settings: label: Mapy witryny description: Mapy XML według typów treści, obrazy, News, IndexNow i mapa HTML. diff --git a/models/definition.go b/models/definition.go deleted file mode 100644 index d0a1cf0..0000000 --- a/models/definition.go +++ /dev/null @@ -1,48 +0,0 @@ -package models - -import ( - "time" - - "git.golem15.com/golem15/summercms/modules/lagoon" -) - -// DefinitionTheme is the fixed theme key of the one sitemap definition of an -// application (D-11). The theme column stays so the table matches PHP; the -// import commands write this key whatever the PHP theme was. -const DefinitionTheme = "default" - -// Definition is a sitemap definition stored on golem15_sitemap_definitions, -// the PHP Golem15\Sitemap\Models\Definition. Data holds the items as JSON -// text (PHP $jsonable data). -type Definition struct { - ID uint `gorm:"column:id;primaryKey"` - Theme *string `gorm:"column:theme"` - Data lagoon.Jsonable[DefinitionItems] `gorm:"column:data"` - CreatedAt *time.Time `gorm:"column:created_at"` - UpdatedAt *time.Time `gorm:"column:updated_at"` -} - -func (Definition) TableName() string { return "golem15_sitemap_definitions" } - -// MorphName is the PHP class string of the model. -func (Definition) MorphName() string { return `Golem15\Sitemap\Models\Definition` } - -// Fillable is exactly theme and data, the PHP $fillable with $guarded = ['*'] -// (MassAssignmentTest). -func (Definition) Fillable() []string { return []string{"theme", "data"} } - -// Rules is empty: the PHP model validated nothing. The admin form checks -// each item row's reference against its type. -func (Definition) Rules() map[string]string { return map[string]string{} } - -// Items returns the stored items. -func (d Definition) Items() []DefinitionItem { return d.Data.Get() } - -// SetItems stores items as the definition's data. -func (d *Definition) SetItems(items []DefinitionItem) { - d.Data = lagoon.Jsonable[DefinitionItems]{Data: DefinitionItems(items), Valid: true} -} - -func init() { - Register(Definition{}) -} diff --git a/models/definition/fields.yaml b/models/definition/fields.yaml deleted file mode 100644 index e90e6e9..0000000 --- a/models/definition/fields.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# =================================== -# Form Field Definitions -# =================================== -# -# The WinterCMS form was one tab holding the SitemapItems widget. Here the -# items are a repeater on data; each row is one DefinitionItem. The CMS page -# field is not ported: items are URLs or registry types. - -fields: - data: - label: golem15.sitemap::lang.item.definition - type: repeater - prompt: golem15.sitemap::lang.item.add_item - titleFrom: url - form: - fields: - type: - label: golem15.sitemap::lang.item.type - type: dropdown - options: getTypeOptions - span: left - reference: - label: golem15.sitemap::lang.item.reference - type: dropdown - dependsOn: type - emptyOption: backend::lang.form.none - span: right - url: - label: golem15.sitemap::lang.item.url - type: text - span: left - urlPattern: - label: golem15.sitemap::lang.item.url_pattern - comment: golem15.sitemap::lang.item.url_pattern_comment - type: text - span: right - priority: - label: golem15.sitemap::lang.item.priority - type: dropdown - span: left - options: - '0.1': '0.1' - '0.2': '0.2' - '0.3': '0.3' - '0.4': '0.4' - '0.5': '0.5' - '0.6': '0.6' - '0.7': '0.7' - '0.8': '0.8' - '0.9': '0.9' - '1.0': '1.0' - changefreq: - label: golem15.sitemap::lang.item.changefreq - type: dropdown - span: right - options: - always: golem15.sitemap::lang.item.always - hourly: golem15.sitemap::lang.item.hourly - daily: golem15.sitemap::lang.item.daily - weekly: golem15.sitemap::lang.item.weekly - monthly: golem15.sitemap::lang.item.monthly - yearly: golem15.sitemap::lang.item.yearly - never: golem15.sitemap::lang.item.never - nesting: - label: golem15.sitemap::lang.item.allow_nested_items - comment: golem15.sitemap::lang.item.allow_nested_items_comment - type: checkbox - default: true - span: full diff --git a/models/definition_item.go b/models/definition_item.go deleted file mode 100644 index a22c058..0000000 --- a/models/definition_item.go +++ /dev/null @@ -1,255 +0,0 @@ -package models - -import ( - "bytes" - "encoding/json" - "fmt" - "strconv" - "strings" -) - -// ItemTypeURL is the built-in item type of an explicit URL. Every other -// item type is a menu item type of the framework's registry (signpost). -const ItemTypeURL = "url" - -// DefinitionItem is one sitemap definition item, the PHP -// Golem15\Sitemap\Classes\DefinitionItem. Type is "url" or a registry menu -// item type key; URL is the address of a url item; Reference is the record a -// registry item points at; URLPattern overrides the type's URL pattern; -// Nesting asks a registry item for its sub-items; Priority and Changefreq -// are written to the sitemap as they are stored. -// -// Text fields are pointers so a stored JSON null stays null on the next -// save. Decoding is tolerant of what PHP stored: numbers and booleans are -// accepted for text fields, strings and numbers for Nesting, and unknown keys -// (cmsPage, code, exists, fillable and any other) are ignored. Encoding -// writes the PHP toArray key set: changefreq, priority, nesting, type, url, -// reference, cmsPage (always null, CMS pages are not supported) and -// urlPattern. -type DefinitionItem struct { - Type string - URL *string - Reference *string - URLPattern *string - Nesting bool - Priority *string - Changefreq *string -} - -// Text returns the value of a text field, "" for null. -func Text(p *string) string { - if p == nil { - return "" - } - return *p -} - -// UnmarshalJSON decodes one stored item tolerantly. A missing type is the -// PHP default "url"; a null type is "". -func (d *DefinitionItem) UnmarshalJSON(raw []byte) error { - if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { - // PHP initFromArray kept a default item for a null entry. - *d = DefinitionItem{Type: ItemTypeURL} - return nil - } - var fields map[string]json.RawMessage - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - if err := dec.Decode(&fields); err != nil { - return fmt.Errorf("sitemap definition item: %w", err) - } - item := DefinitionItem{Type: ItemTypeURL} - if v, ok := fields["type"]; ok { - s, err := DecodeText(v) - if err != nil { - return fmt.Errorf("sitemap definition item type: %w", err) - } - item.Type = Text(s) - } - for key, dest := range map[string]**string{ - "url": &item.URL, - "reference": &item.Reference, - "urlPattern": &item.URLPattern, - "priority": &item.Priority, - "changefreq": &item.Changefreq, - } { - v, ok := fields[key] - if !ok { - continue - } - s, err := DecodeText(v) - if err != nil { - return fmt.Errorf("sitemap definition item %s: %w", key, err) - } - *dest = s - } - if v, ok := fields["nesting"]; ok { - b, err := decodeTruthy(v) - if err != nil { - return fmt.Errorf("sitemap definition item nesting: %w", err) - } - item.Nesting = b - } - *d = item - return nil -} - -// MarshalJSON writes the PHP toArray key set in its order. -func (d DefinitionItem) MarshalJSON() ([]byte, error) { - var b bytes.Buffer - b.WriteByte('{') - write := func(key string, v any, last bool) error { - kb, _ := json.Marshal(key) - b.Write(kb) - b.WriteByte(':') - vb, err := json.Marshal(v) - if err != nil { - return err - } - b.Write(vb) - if !last { - b.WriteByte(',') - } - return nil - } - for _, f := range []struct { - key string - v any - }{ - {"changefreq", d.Changefreq}, - {"priority", d.Priority}, - {"nesting", d.Nesting}, - {"type", d.Type}, - {"url", d.URL}, - {"reference", d.Reference}, - {"cmsPage", nil}, - } { - if err := write(f.key, f.v, false); err != nil { - return nil, err - } - } - if err := write("urlPattern", d.URLPattern, true); err != nil { - return nil, err - } - b.WriteByte('}') - return b.Bytes(), nil -} - -// DefinitionItems is the stored item list of a definition. It decodes a -// JSON array, and a JSON object (a PHP array whose keys were not sequential) -// in key order. -type DefinitionItems []DefinitionItem - -// UnmarshalJSON accepts a JSON array or object of items; null is no items. -func (l *DefinitionItems) UnmarshalJSON(raw []byte) error { - raw = bytes.TrimSpace(raw) - if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { - *l = nil - return nil - } - if raw[0] == '[' { - var items []DefinitionItem - if err := json.Unmarshal(raw, &items); err != nil { - return err - } - *l = items - return nil - } - if raw[0] != '{' { - return fmt.Errorf("sitemap definition data must be a list of items") - } - dec := json.NewDecoder(bytes.NewReader(raw)) - if _, err := dec.Token(); err != nil { - return err - } - var items []DefinitionItem - for dec.More() { - if _, err := dec.Token(); err != nil { - return err - } - var item DefinitionItem - if err := dec.Decode(&item); err != nil { - return err - } - items = append(items, item) - } - *l = items - return nil -} - -// MarshalJSON writes a JSON array; no items is []. -func (l DefinitionItems) MarshalJSON() ([]byte, error) { - if l == nil { - return []byte("[]"), nil - } - return json.Marshal([]DefinitionItem(l)) -} - -// DecodeText decodes a stored text value: a string as it is, a number in -// its PHP string form, a boolean as PHP casts it ("1" or ""), null as nil. -func DecodeText(raw json.RawMessage) (*string, error) { - var v any - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - if err := dec.Decode(&v); err != nil { - return nil, err - } - var s string - switch t := v.(type) { - case nil: - return nil, nil - case string: - s = t - case json.Number: - s = phpNumberString(t) - case bool: - if t { - s = "1" - } - default: - return nil, fmt.Errorf("must be text, got %T", v) - } - return &s, nil -} - -// decodeTruthy decodes a stored flag with PHP truthiness: false, 0, "", -// "0" and null are false. -func decodeTruthy(raw json.RawMessage) (bool, error) { - var v any - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - if err := dec.Decode(&v); err != nil { - return false, err - } - switch t := v.(type) { - case nil: - return false, nil - case bool: - return t, nil - case string: - return t != "" && t != "0", nil - case json.Number: - f, err := t.Float64() - if err != nil { - return false, err - } - return f != 0, nil - } - return false, fmt.Errorf("must be a flag, got %T", v) -} - -// phpNumberString is PHP's string cast of a json_decode'd number: an -// integer as written, a float in its shortest round-trip form (1.0 is "1"). -func phpNumberString(n json.Number) string { - text := n.String() - if !strings.ContainsAny(text, ".eE") { - if i, err := strconv.ParseInt(text, 10, 64); err == nil { - return strconv.FormatInt(i, 10) - } - } - f, err := n.Float64() - if err != nil { - return text - } - return strconv.FormatFloat(f, 'f', -1, 64) -} diff --git a/models/definition_test.go b/models/definition_test.go deleted file mode 100644 index dd5f2b7..0000000 --- a/models/definition_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package models_test - -import ( - "encoding/json" - "fmt" - "slices" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/lagoon" -) - -// TestFillable ports the PHP MassAssignmentTest (SITE-001) as behaviour: -// Definition.Fillable is exactly theme and data, and a fill with extra keys -// (id, timestamps, unknown keys) writes only those two. -func TestFillable(t *testing.T) { - if got := (models.Definition{}).Fillable(); !slices.Equal(got, []string{"theme", "data"}) { - t.Fatalf("Fillable = %v, want [theme data]", got) - } - def := models.Definition{ID: 4} - requested := map[string]any{ - "id": 99, - "theme": "site", - "data": `[{"type":"url","url":"/a"}]`, - "created_at": "2020-01-01 00:00:00", - "updated_at": "2020-01-01 00:00:00", - "is_admin": true, - } - for _, production := range []bool{true, false} { - d := def - if err := lagoon.Fill(&d, d.Fillable(), requested, production); err != nil { - t.Fatalf("Fill: %v", err) - } - if d.ID != 4 || d.CreatedAt != nil || d.UpdatedAt != nil || models.Text(d.Theme) != "site" { - t.Fatalf("filled definition = %+v", d) - } - if items := d.Items(); len(items) != 1 || models.Text(items[0].URL) != "/a" { - t.Fatalf("filled data = %+v", items) - } - } - if (models.Definition{}).TableName() != "golem15_sitemap_definitions" || (models.Definition{}).MorphName() != `Golem15\Sitemap\Models\Definition` { - t.Fatal("table or morph name changed") - } - if rules := (models.Definition{}).Rules(); len(rules) != 0 { - t.Fatalf("Rules = %v", rules) - } - if got := (models.Settings{}).Fillable(); slices.Contains(got, "id") || len(got) != 3 { - t.Fatalf("Settings.Fillable = %v", got) - } -} - -// TestDefinitionItemJSON pins the tolerant decoding of PHP-stored items -// (strings, numbers, booleans and null for text fields, PHP truthiness for -// nesting, unknown keys ignored, a missing type is url, a null item a -// default url item, an object-keyed list in key order) and the PHP toArray -// encoding (key set and order, cmsPage always null, null text kept). -func TestDefinitionItemJSON(t *testing.T) { - for _, tc := range []struct { - raw string - want string - }{ - {`{"type":"url","url":"/a","priority":"0.5","changefreq":"daily","nesting":"0","reference":null,"urlPattern":null,"cmsPage":"x","code":"c","exists":false,"fillable":["a"]}`, - `type=url url=/a ref= pattern= nesting=false priority=0.5 changefreq=daily`}, - {`{"url":"/b"}`, `type=url url=/b ref= pattern= nesting=false priority= changefreq=`}, - {`{"type":null}`, `type= url= ref= pattern= nesting=false priority= changefreq=`}, - {`{"type":"journal-post","reference":7,"priority":0.5,"changefreq":true,"nesting":1}`, - `type=journal-post url= ref=7 pattern= nesting=true priority=0.5 changefreq=1`}, - {`{"type":"x","priority":1.0,"reference":12.50,"changefreq":false,"nesting":"1"}`, - `type=x url= ref=12.5 pattern= nesting=true priority=1 changefreq=`}, - {`{"type":"x","nesting":"false"}`, `type=x url= ref= pattern= nesting=true priority= changefreq=`}, - {`{"type":"x","nesting":""}`, `type=x url= ref= pattern= nesting=false priority= changefreq=`}, - {`{"type":"x","nesting":null,"urlPattern":"/p/:slug"}`, `type=x url= ref= pattern=/p/:slug nesting=false priority= changefreq=`}, - {`{"type":"x","reference":12345678901234}`, `type=x url= ref=12345678901234 pattern= nesting=false priority= changefreq=`}, - {`null`, `type=url url= ref= pattern= nesting=false priority= changefreq=`}, - } { - var item models.DefinitionItem - if err := json.Unmarshal([]byte(tc.raw), &item); err != nil { - t.Fatalf("%s: %v", tc.raw, err) - } - if got := describe(item); got != tc.want { - t.Errorf("%s\n got %s\nwant %s", tc.raw, got, tc.want) - } - } - for _, raw := range []string{`{"type":["x"]}`, `{"url":{"a":1}}`, `{"nesting":[1]}`, `[1]`, `"x"`} { - var item models.DefinitionItem - if err := json.Unmarshal([]byte(raw), &item); err == nil { - t.Errorf("%s decoded without an error", raw) - } - } - - // A list, an object keyed like a PHP array with gaps, null and empty. - var list models.DefinitionItems - if err := json.Unmarshal([]byte(`{"3":{"url":"/c"},"1":{"url":"/a"},"x":{"url":"/x"}}`), &list); err != nil || len(list) != 3 || - models.Text(list[0].URL) != "/c" || models.Text(list[1].URL) != "/a" || models.Text(list[2].URL) != "/x" { - t.Fatalf("object list = %+v %v", list, err) - } - for raw, n := range map[string]int{`[]`: 0, `null`: 0, `[{"url":"/a"},null]`: 2} { - var l models.DefinitionItems - if err := json.Unmarshal([]byte(raw), &l); err != nil || len(l) != n { - t.Fatalf("%s = %+v %v", raw, l, err) - } - } - var bad models.DefinitionItems - if err := json.Unmarshal([]byte(`"items"`), &bad); err == nil { - t.Fatal("a string decoded as an item list") - } - - // Encoding: the PHP toArray key set and order. - pattern := "/p/:slug" - item := models.DefinitionItem{Type: "journal-post", Reference: str("7"), URLPattern: &pattern, Nesting: true, Priority: str("0.5")} - raw, err := json.Marshal(item) - if err != nil { - t.Fatal(err) - } - if want := `{"changefreq":null,"priority":"0.5","nesting":true,"type":"journal-post","url":null,"reference":"7","cmsPage":null,"urlPattern":"/p/:slug"}`; string(raw) != want { - t.Fatalf("Marshal = %s\nwant %s", raw, want) - } - if raw, _ := json.Marshal(models.DefinitionItems(nil)); string(raw) != "[]" { - t.Fatalf("nil list = %s", raw) - } - var back models.DefinitionItem - if err := json.Unmarshal(raw, &back); err != nil || describe(back) != describe(item) { - t.Fatalf("round trip = %s, %v", describe(back), err) - } - if models.Text(nil) != "" || models.Text(str("x")) != "x" { - t.Fatal("Text") - } - if s, err := models.DecodeText(json.RawMessage(`1e3`)); err != nil || *s != "1000" { - t.Fatalf("DecodeText(1e3) = %v %v", s, err) - } -} - -func str(s string) *string { return &s } - -func describe(d models.DefinitionItem) string { - p := func(s *string) string { - if s == nil { - return "" - } - return *s - } - return strings.TrimSpace(fmt.Sprintf("type=%s url=%s ref=%s pattern=%s nesting=%v priority=%s changefreq=%s", - d.Type, p(d.URL), p(d.Reference), p(d.URLPattern), d.Nesting, p(d.Priority), p(d.Changefreq))) -} diff --git a/models/legacy.go b/models/legacy.go new file mode 100644 index 0000000..19e1336 --- /dev/null +++ b/models/legacy.go @@ -0,0 +1,177 @@ +package models + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// Text returns *p, "" for nil. +func Text(p *string) string { + if p == nil { + return "" + } + return *p +} + +// LegacyItemURL is the type of a Phase 17 (and WinterCMS) sitemap +// definition item that is an explicit address. +const LegacyItemURL = "url" + +// LegacyItem is one item of a Phase 17 or WinterCMS sitemap definition, as +// golem15_sitemap_definitions.data and the WinterCMS export store it. Only +// the fields the per-type format still reads are kept: the type (a missing +// type is url, as in PHP), the address of a url item and the reference of a +// registry item. +type LegacyItem struct { + Type string + URL string + Reference string + CMSPage string +} + +// LegacyEntries splits a JSON array, or a JSON object in key order (a PHP +// array with keys), into its values with their PHP labels (index or key). +// ok is false for any other value. +func LegacyEntries(raw []byte) ([]json.RawMessage, []string, bool) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return nil, nil, false + } + switch raw[0] { + case '[': + var list []json.RawMessage + if err := json.Unmarshal(raw, &list); err != nil { + return nil, nil, false + } + labels := make([]string, len(list)) + for i := range list { + labels[i] = strconv.Itoa(i) + } + return list, labels, true + case '{': + dec := json.NewDecoder(bytes.NewReader(raw)) + if _, err := dec.Token(); err != nil { + return nil, nil, false + } + var list []json.RawMessage + var labels []string + for dec.More() { + tok, err := dec.Token() + if err != nil { + return nil, nil, false + } + key, _ := tok.(string) + var v json.RawMessage + if err := dec.Decode(&v); err != nil { + return nil, nil, false + } + list = append(list, v) + labels = append(labels, key) + } + return list, labels, true + } + return nil, nil, false +} + +// ParseLegacyItem decodes one stored item tolerantly: text fields may be +// strings, numbers, booleans or null; a missing type is url; unknown keys +// are ignored. +func ParseLegacyItem(raw json.RawMessage) (LegacyItem, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return LegacyItem{}, fmt.Errorf("an item must be an object: %w", err) + } + item := LegacyItem{Type: LegacyItemURL} + for key, dest := range map[string]*string{"type": &item.Type, "url": &item.URL, "reference": &item.Reference, "cmsPage": &item.CMSPage} { + v, ok := fields[key] + if !ok { + continue + } + text, err := DecodeText(v) + if err != nil { + return LegacyItem{}, fmt.Errorf("%s %w", key, err) + } + *dest = strings.TrimSpace(Text(text)) + } + return item, nil +} + +// ParseLegacyItems decodes the data of a definition: a JSON array or a PHP +// keyed object of items, itself possibly stored as a JSON string. Empty or +// null data has no items. +func ParseLegacyItems(data []byte) ([]LegacyItem, []string, error) { + data = LegacyJSON(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + return nil, nil, nil + } + raws, labels, ok := LegacyEntries(data) + if !ok { + return nil, nil, fmt.Errorf("the definition data is not a list of items") + } + items := make([]LegacyItem, 0, len(raws)) + for i, raw := range raws { + item, err := ParseLegacyItem(raw) + if err != nil { + return nil, nil, fmt.Errorf("definition item %s: %w", labels[i], err) + } + items = append(items, item) + } + return items, labels, nil +} + +// LegacyJSON is a JSON value that may be held as text (a TEXT column +// selected as it is) or as JSON: a string is decoded once more. +func LegacyJSON(raw []byte) []byte { + raw = bytes.TrimSpace(raw) + if len(raw) > 0 && raw[0] == '"' { + var text string + if json.Unmarshal(raw, &text) == nil { + return []byte(strings.TrimSpace(text)) + } + } + return raw +} + +// DecodeText decodes a stored text value with PHP's string conversion: +// null is nil, a number its PHP text, true "1" and false "". +func DecodeText(raw json.RawMessage) (*string, error) { + var v any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&v); err != nil { + return nil, err + } + var s string + switch t := v.(type) { + case nil: + return nil, nil + case string: + s = t + case json.Number: + s = phpNumberString(t) + case bool: + if t { + s = "1" + } + default: + return nil, fmt.Errorf("must be text, got %T", v) + } + return &s, nil +} + +func phpNumberString(n json.Number) string { + text := n.String() + if !strings.ContainsAny(text, ".eE") { + if i, err := strconv.ParseInt(text, 10, 64); err == nil { + return strconv.FormatInt(i, 10) + } + } + f, err := n.Float64() + if err != nil { + return text + } + return strconv.FormatFloat(f, 'f', -1, 64) +} diff --git a/models/settings.go b/models/settings.go index 8615fe1..7e38ad5 100644 --- a/models/settings.go +++ b/models/settings.go @@ -12,50 +12,23 @@ import ( "git.golem15.com/golem15/summercms/modules/lagoon" ) -// errFrontendURL is the phrase key of the frontend_url refusal. +// errFrontendURL is the phrase key of the site address refusal (the +// Phase 17 frontend_url phrase, unchanged). const errFrontendURL = "golem15.sitemap::lang.errors.frontend_url" -// The settings keys of the five priority tiers, in the order PHP checked -// them. A tier rule names one of them. -const ( - PriorityHomepage = "priority_homepage" - PriorityServices = "priority_services" - PriorityBlog = "priority_blog" - PriorityCategories = "priority_categories" - PriorityLegal = "priority_legal" -) - -// PriorityDefaults are the PHP defaults of the priority settings. -var PriorityDefaults = map[string]string{ - PriorityHomepage: "1.0", - PriorityServices: "0.8", - PriorityBlog: "0.6", - PriorityCategories: "0.4", - PriorityLegal: "0.2", -} - // Settings is the sitemap settings singleton (row id 1) on -// golem15_sitemap_settings, the PHP Golem15\Sitemap\Models\Settings that -// WinterCMS kept in system_settings under golem15_sitemap_settings. -// FrontendURL replaces the backend URL in the generated XML; -// FrontendDefaultLocale is the locale whose URLs carry no prefix; -// UpdatesManifestPath names a JSON file of URL path to last-modified date; -// EnablePriorityTiers turns the tier priorities on, and the priority fields -// hold each tier's value as text. A null priority reads as its PHP default. -// The PHP settings form validated nothing; the port checks FrontendURL in -// BeforeValidate. +// golem15_sitemap_settings (UI-SPEC C26). SiteURL is the public site +// address every starts with (empty: golem15.sitemap.site_url, then +// app.url), FileSize the addresses per file, DefaultLocale the site's +// default locale (the unprefixed one, and the target of x-default) and +// ExtraURLs the additional addresses. RebuildPendingUntil is the debounce +// time of the next queued rebuild, NumberedSize the file size the files are +// numbered for, BuiltAt the end of the last rebuild and RebuildRequestedAt +// the last rebuild asked for from the admin. The Phase 17 columns +// (frontend_url, priorities, tiers, the update manifest) stay in the table +// and are no longer read (D-32). type Settings struct { - ID uint `gorm:"column:id;primaryKey"` - FrontendURL *string `gorm:"column:frontend_url"` - FrontendDefaultLocale *string `gorm:"column:frontend_default_locale"` - UpdatesManifestPath *string `gorm:"column:updates_manifest_path"` - EnablePriorityTiers bool `gorm:"column:enable_priority_tiers"` - PriorityHomepage *string `gorm:"column:priority_homepage"` - PriorityServices *string `gorm:"column:priority_services"` - PriorityBlog *string `gorm:"column:priority_blog"` - PriorityCategories *string `gorm:"column:priority_categories"` - PriorityLegal *string `gorm:"column:priority_legal"` - + ID uint `gorm:"column:id;primaryKey"` SiteURL *string `gorm:"column:site_url"` FileSize int `gorm:"column:file_size"` DefaultLocale *string `gorm:"column:default_locale"` @@ -100,8 +73,7 @@ func (Settings) Fillable() []string { return []string{"site_url", "file_size", "extra_urls"} } -// Rules is empty: the PHP settings form validated nothing. The one check of -// the port, on frontend_url, is in BeforeValidate. +// Rules is empty: the settings controller runs Validate before a save. func (Settings) Rules() map[string]string { return map[string]string{} } // The phrase keys of the other settings refusals. @@ -127,8 +99,7 @@ func (s Settings) Validate() error { } // Problems lists the settings' refusals by field (UI-SPEC C26): a site -// address that AllowedFrontendURL refuses (site_url and the Phase 17 -// frontend_url), a file size outside 1..MaxFileSize and an additional +// address that AllowedFrontendURL refuses, a file size outside 1..MaxFileSize and an additional // address that is not a path or an http(s) URL (extra_urls..url) or // whose lastmod is not a date (extra_urls..lastmod). func (s Settings) Problems() map[string][]string { @@ -136,9 +107,6 @@ func (s Settings) Problems() map[string][]string { if !AllowedFrontendURL(Text(s.SiteURL)) { out["site_url"] = []string{errFrontendURL} } - if !AllowedFrontendURL(Text(s.FrontendURL)) { - out["frontend_url"] = []string{errFrontendURL} - } if s.FileSize < 1 || s.FileSize > MaxFileSize { out["file_size"] = []string{errFileSize} } @@ -153,14 +121,12 @@ func (s Settings) Problems() map[string][]string { return out } -// AllowedFrontendURL reports whether u may be the frontend_url setting: the -// empty string, or an absolute http or https URL with a host and without -// whitespace, control characters, a double quote, <, > or &. The PHP plugin -// stored any text. The value is written into every loc, every hreflang link -// and the stylesheet instruction of the sitemap, and sitemap.xsl renders -// each loc as a link on the public origin, so a value outside this form -// (a javascript: URL, or text that closes the attribute it is written into) -// would reach the readers of /sitemap.xml (T-17-15). +// AllowedFrontendURL reports whether u may be the site address: the empty +// string, or an absolute http or https URL with a host and without +// whitespace, control characters, a double quote, <, > or &. The value +// starts every and hreflang link of the sitemaps, so a value outside +// this form (a javascript: URL, or text that closes the attribute it is +// written into) would reach their readers (T-17-15, T-33-12). func AllowedFrontendURL(u string) bool { if u == "" { return true @@ -183,32 +149,9 @@ func AllowedFrontendURL(u string) bool { return parsed.Opaque == "" && parsed.Hostname() != "" } -// DefaultSettings is the settings PHP read when nothing was saved: tiers on -// with the default priorities. +// DefaultSettings is the settings read when no row is stored. func DefaultSettings() Settings { - return Settings{ID: 1, EnablePriorityTiers: true, FileSize: DefaultFileSize} -} - -// Priority returns the value of a priority setting: the stored text, or the -// PHP default when it is null. -func (s Settings) Priority(key string) string { - var p *string - switch key { - case PriorityHomepage: - p = s.PriorityHomepage - case PriorityServices: - p = s.PriorityServices - case PriorityBlog: - p = s.PriorityBlog - case PriorityCategories: - p = s.PriorityCategories - case PriorityLegal: - p = s.PriorityLegal - } - if p == nil { - return PriorityDefaults[key] - } - return *p + return Settings{ID: 1, FileSize: DefaultFileSize} } func init() { diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go index bf85e6d..99075e0 100644 --- a/sitemap_smoke_test.go +++ b/sitemap_smoke_test.go @@ -2,7 +2,6 @@ package sitemap_test import ( "bytes" - "context" "encoding/json" "fmt" "io" @@ -16,87 +15,19 @@ import ( "time" _ "git.golem15.com/golem15/sm-sitemap-plugin" - "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/bouncer" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/compass" "git.golem15.com/golem15/summercms/modules/lagoon" - "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) -// testTypesPluginID is the fake content plugin that registers menu item -// types for the sitemap smoke tests. The real Journal and sitemap -// combination is proven by the host application's tests. -const testTypesPluginID = "acme.sitemaptypes" - -// acmeTypes is a content plugin with one menu item type, acme-post, whose -// reference "7" resolves to a fixed post with two alternates. -type acmeTypes struct{} - -func (acmeTypes) ID() string { return testTypesPluginID } -func (acmeTypes) Requires() []string { return nil } -func (acmeTypes) Register(*backpack.App) error { return nil } -func (acmeTypes) Boot(*backpack.App) error { return nil } -func (acmeTypes) MenuItemTypes() []pact.MenuItemType { - mtime := time.Date(2026, 3, 4, 10, 0, 0, 0, time.UTC) - return []pact.MenuItemType{acmeList(), { - Key: "acme-post", - Label: "Acme post", - DefaultPattern: "/news/:slug", - References: func(context.Context) ([]pact.MenuItemReference, error) { - return []pact.MenuItemReference{{Value: "7", Label: "Hello"}}, nil - }, - Resolve: func(_ context.Context, in pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { - if in.Reference != "7" { - return nil, nil - } - return &pact.ResolvedMenuItem{ - Title: "Hello", - URL: "/news/hello", - MTime: &mtime, - Alternates: []pact.MenuItemAlternate{ - {Locale: "pl", URL: "/news/hello"}, - {Locale: "en", URL: "/en/news/hello"}, - }, - }, nil - }, - }} -} - -// acmeList is a list type of the fake plugin: no URL of its own, one post -// with two alternates as its item. -func acmeList() pact.MenuItemType { - return pact.MenuItemType{ - Key: "acme-list", - Label: "Acme posts", - DefaultPattern: "/news/:slug", - PatternFrom: "acme-post", - DynamicItems: true, - Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { - return &pact.ResolvedMenuItem{Items: []pact.ResolvedMenuItem{{ - Title: "A", - URL: "/news/a", - Alternates: []pact.MenuItemAlternate{ - {Locale: "pl", URL: "/news/a"}, - {Locale: "en", URL: "/en/news/a"}, - }, - }}}, nil - }, - } -} - -var _ pact.HasMenuItemTypes = acmeTypes{} - -func init() { - party.Register(acmeTypes{}) -} - -// sitemapEnv is a booted application with the sitemap and the fake types. +// sitemapEnv is a booted application with the sitemap and the fake +// content plugin. type sitemapEnv struct { h http.Handler app *backpack.App @@ -104,13 +35,6 @@ type sitemapEnv struct { plugins []party.Plugin } -// bootSitemap boots the application with the sitemap and the fake types; -// overrides set further config keys. -func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { - t.Helper() - return bootPlugins(t, []string{testTypesPluginID, "golem15.sitemap"}, overrides...) -} - // bootPlugins boots the application with the plugins ids on a fresh // database; overrides set further config keys. func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitemapEnv { @@ -174,19 +98,6 @@ func (e *sitemapEnv) get(t *testing.T, path string) *httptest.ResponseRecorder { return rec } -func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem) models.Definition { - t.Helper() - theme := models.DefinitionTheme - def := models.Definition{Theme: &theme} - def.SetItems(items) - if err := e.gdb.Create(&def).Error; err != nil { - t.Fatal(err) - } - return def -} - -func text(s string) *string { return &s } - func writeFile(t *testing.T, dir, name, body string) string { t.Helper() path := filepath.Join(dir, name) diff --git a/tests/php/gen-goldens.php b/tests/php/gen-goldens.php deleted file mode 100644 index c9c0c91..0000000 --- a/tests/php/gen-goldens.php +++ /dev/null @@ -1,465 +0,0 @@ - reference => - * resolved item {url, mtime, alternates: [[locale, url], ...], items}), - * settings (Sitemap settings values; absent keys take the PHP defaults), - * manifest (the updates manifest object), skip (URLs a beforeAddItem - * listener drops) and extra (URL => text of an element an addItem - * listener appends). - * - * A registry type's resolved item is given in the Go shape (one item per - * record with ordered alternates); this script turns it into what the PHP - * headless resolvers returned: a list type (no url) yields one entry per - * alternate for each child (a child without alternates once), any other type - * its children with their default URL. - */ - -$vendor = getenv('WINTER_VENDOR'); -if (!$vendor || !is_file($vendor . '/autoload.php')) { - fwrite(STDERR, "set WINTER_VENDOR to the vendor directory of a WinterCMS application\n"); - exit(2); -} -require $vendor . '/autoload.php'; - -use Illuminate\Http\Request; -use Illuminate\Routing\RouteCollection; -use Winter\Storm\Router\UrlGenerator; - -date_default_timezone_set('UTC'); - -if ($argc !== 3) { - fwrite(STDERR, "usage: php gen-goldens.php INPUT.json OUTPUT.golden\n"); - exit(2); -} - -$input = json_decode(file_get_contents($argv[1]), true); -if (!is_array($input)) { - fwrite(STDERR, "invalid JSON input\n"); - exit(1); -} - -function urlGenerator(string $root): UrlGenerator -{ - return new UrlGenerator(new RouteCollection, Request::create(rtrim($root, '/') . '/sitemap.xml')); -} - -if (isset($input['urls'])) { - $url = urlGenerator($input['root']); - $out = ''; - foreach ($input['urls'] as $path) { - $out .= $url->to($path) . "\n"; - } - file_put_contents($argv[2], $out); - exit(0); -} - -if (isset($input['case'])) { - file_put_contents($argv[2], (new GoldenSitemap($input['case']))->render()); - exit(0); -} - -$out = ''; -foreach ($input['cases'] as $case) { - $out .= '=== ' . $case['name'] . "\n" . (new GoldenSitemap($case))->render(); -} -file_put_contents($argv[2], $out); - -/** - * Golem15\Sitemap\Classes\DefinitionItem, unchanged properties. - */ -class GoldenDefinitionItem -{ - public $changefreq; - public $priority; - public $nesting = false; - public $type = 'url'; - public $url = null; - public $code = null; - public $reference = null; - public $cmsPage = null; - public $urlPattern = null; - public $exists = false; - - public static function initFromArray(array $items): array - { - $result = []; - foreach ($items as $itemData) { - $obj = new self; - foreach ($itemData as $name => $value) { - if (property_exists($obj, $name)) { - $obj->$name = $value; - } - } - $result[] = $obj; - } - return $result; - } -} - -/** - * The generation code of Golem15\Sitemap\Models\Definition and routes.php. - */ -class GoldenSitemap -{ - const MAX_URLS = 50000; - - protected $case; - protected $url; - protected $items; - protected $updatesManifest = null; - protected $urlCount = 0; - protected $urlSet; - protected $xmlObject; - - public function __construct(array $case) - { - $this->case = $case; - $this->url = urlGenerator($case['root']); - $this->items = GoldenDefinitionItem::initFromArray($case['items'] ?? []); - } - - protected function setting(string $key, $default) - { - $settings = $this->case['settings'] ?? []; - return array_key_exists($key, $settings) ? $settings[$key] : $default; - } - - /** routes.php */ - public function render(): string - { - $xml = $this->generateSitemap(); - - $frontendUrl = $this->setting('frontend_url', null); - if ($frontendUrl) { - $frontendUrl = rtrim($frontendUrl, '/'); - $backendUrl = rtrim($this->url->to('/'), '/'); - $xml = preg_replace_callback('/(.*?)<\/loc>/', function ($m) use ($backendUrl, $frontendUrl) { - return '' . str_replace($backendUrl, $frontendUrl, $m[1]) . ''; - }, $xml); - $xml = preg_replace_callback('/(]*href=")([^"]*)(")/', function ($m) use ($backendUrl, $frontendUrl) { - return $m[1] . str_replace($backendUrl, $frontendUrl, $m[2]) . $m[3]; - }, $xml); - $xml = str_replace( - 'href="' . $backendUrl . '/sitemap.xsl"', - 'href="' . $frontendUrl . '/sitemap.xsl"', - $xml - ); - } - - return $xml; - } - - /** - * What the PHP headless resolvers returned for a resolved item in the - * Go shape. - */ - protected function phpResolved(array $r, bool $listChildren): array - { - $info = []; - if (!empty($r['url'])) { - $info['url'] = $r['url']; - } - if (array_key_exists('mtime', $r)) { - $info['mtime'] = $r['mtime'] === null ? null : new DateTime($r['mtime']); - } - $links = []; - foreach ($r['alternates'] ?? [] as [$locale, $href]) { - $links[$locale] = $href; - } - $info['alternateLinks'] = $links; - if (isset($r['items'])) { - $info['items'] = []; - foreach ($r['items'] as $child) { - $childInfo = $this->phpResolved($child, false); - if ($listChildren && $childInfo['alternateLinks']) { - foreach ($childInfo['alternateLinks'] as $locale => $href) { - $perLocale = $childInfo; - $perLocale['url'] = $href; - $info['items'][] = $perLocale; - } - } else { - $info['items'][] = $childInfo; - } - } - } - return $info; - } - - protected function resolveItem(GoldenDefinitionItem $item) - { - $types = $this->case['types'] ?? []; - $ref = (string) $item->reference; - if (!isset($types[$item->type][$ref])) { - return null; - } - $r = $types[$item->type][$ref]; - return $this->phpResolved($r, empty($r['url'])); - } - - public function generateSitemap(): string - { - if (!$this->items) { - return ''; - } - - foreach ($this->items as $item) { - if ($item->type === 'url') { - $this->addItemToSet($item); - } else { - $menuItemInfo = $this->resolveItem($item); - if (!is_array($menuItemInfo)) { - continue; - } - - if (isset($menuItemInfo['url'])) { - $this->addItemToSet($item, $menuItemInfo); - } - - if (isset($menuItemInfo['items'])) { - $menuItemIterator = function ($menuItems) use (&$menuItemIterator, $item) { - foreach ($menuItems as $menuItem) { - if (isset($menuItem['url'])) { - $this->addItemToSet($item, $menuItem); - } - - if (isset($menuItem['items'])) { - $menuItemIterator($menuItem['items']); - } - } - }; - - $menuItemIterator($menuItemInfo['items']); - } - } - } - - $urlSet = $this->getUrlSet(); - $xml = $this->getXmlObject(); - $xml->appendChild($urlSet); - - return $xml->saveXML(); - } - - protected function getXmlObject(): DOMDocument - { - if ($this->xmlObject !== null) { - return $this->xmlObject; - } - - $xml = new DOMDocument; - $xml->encoding = 'UTF-8'; - $xss = $xml->createProcessingInstruction('xml-stylesheet', - 'type="text/xsl" href="' . UrlGenerator::buildUrl($this->url->current(), ['path' => 'sitemap.xsl']) . '"' - ); - $xml->appendChild($xss); - - return $this->xmlObject = $xml; - } - - protected function getUrlSet(): DOMElement - { - if ($this->urlSet !== null) { - return $this->urlSet; - } - - $xml = $this->getXmlObject(); - $urlSet = $xml->createElement('urlset'); - $urlSet->setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9'); - $urlSet->setAttribute('xmlns:xhtml', 'http://www.w3.org/1999/xhtml'); - $urlSet->setAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance'); - $urlSet->setAttribute('xsi:schemaLocation', 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd'); - - return $this->urlSet = $urlSet; - } - - protected function addItemToSet(GoldenDefinitionItem $item, ?array $itemInfo = null): void - { - $xml = $this->getXmlObject(); - $urlSet = $this->getUrlSet(); - - if ($this->urlCount >= self::MAX_URLS) { - return; - } - - if (!isset($itemInfo['url'])) { - $itemInfo['url'] = $this->url->to($item->url); - } - - $lastModified = $itemInfo['mtime'] ?? null; - - if ($lastModified === null) { - $lastModified = $this->lookupManifestLastmod($itemInfo['url']); - } - - if ($lastModified instanceof \DateTime) { - $lastModified = $lastModified->getTimestamp(); - } - $itemInfo['lastModified'] = $lastModified ? date('c', $lastModified) : null; - - // golem15.sitemap.beforeAddItem: a listener that drops the case's - // skip URLs. - if (in_array($itemInfo['url'], $this->case['skip'] ?? [], true)) { - return; - } - - if (empty($itemInfo['url'])) { - return; - } else { - $itemInfo['url'] = $this->url->to($itemInfo['url']); - } - - $this->urlCount++; - - $tierPriority = $this->getPriorityForUrl($itemInfo['url']); - $effectivePriority = $tierPriority ?? $item->priority; - - $urlElement = $xml->createElement('url'); - - $loc = $xml->createElement('loc'); - $loc->appendChild($xml->createTextNode($itemInfo['url'])); - $urlElement->appendChild($loc); - - if (!empty($itemInfo['lastModified'])) { - $lastmod = $xml->createElement('lastmod'); - $lastmod->appendChild($xml->createTextNode((string) $itemInfo['lastModified'])); - $urlElement->appendChild($lastmod); - } - - $changefreq = $xml->createElement('changefreq'); - $changefreq->appendChild($xml->createTextNode((string) $item->changefreq)); - $urlElement->appendChild($changefreq); - - $priority = $xml->createElement('priority'); - $priority->appendChild($xml->createTextNode((string) $effectivePriority)); - $urlElement->appendChild($priority); - - if (!empty($itemInfo['alternateLinks']) && count($itemInfo['alternateLinks']) > 1) { - foreach ($itemInfo['alternateLinks'] as $hreflang => $href) { - $link = $xml->createElement('xhtml:link'); - $link->setAttribute('rel', 'alternate'); - $link->setAttribute('hreflang', $hreflang); - $link->setAttribute('href', $this->url->to($href)); - $urlElement->appendChild($link); - } - } - - $urlSet->appendChild($urlElement); - - // golem15.sitemap.addItem: a listener that appends the case's extra - // element to the URL. - $extra = $this->case['extra'] ?? []; - if (isset($extra[$itemInfo['url']])) { - $element = $xml->createElement('extra'); - $element->appendChild($xml->createTextNode($extra[$itemInfo['url']])); - $urlElement->appendChild($element); - } - } - - protected function lookupManifestLastmod(string $url): ?int - { - $manifest = $this->getUpdatesManifest(); - if (!$manifest) { - return null; - } - - $path = parse_url($url, PHP_URL_PATH) ?: '/'; - $path = $this->normalizeManifestPath($path); - - if (!isset($manifest[$path])) { - return null; - } - - $timestamp = strtotime((string) $manifest[$path]); - - return $timestamp ?: null; - } - - protected function getUpdatesManifest(): array - { - if ($this->updatesManifest !== null) { - return $this->updatesManifest; - } - - $decoded = $this->case['manifest'] ?? null; - if (!is_array($decoded)) { - return $this->updatesManifest = []; - } - - $normalized = []; - foreach ($decoded as $key => $value) { - $normalized[$this->normalizeManifestPath((string) $key)] = $value; - } - - return $this->updatesManifest = $normalized; - } - - protected function normalizeManifestPath(string $path): string - { - $path = '/' . ltrim($path, '/'); - - return $path === '/' ? '/' : rtrim($path, '/'); - } - - protected function getPriorityForUrl(string $url): ?string - { - if (!$this->setting('enable_priority_tiers', true)) { - return null; - } - - $path = parse_url($url, PHP_URL_PATH) ?: '/'; - $path = rtrim($path, '/'); - - if ($path === '' || $path === '/en') { - return $this->setting('priority_homepage', '1.0'); - } - - if (preg_match('#^/(uslugi|en/services|realizacje|en/work)/[^/]+$#', $path)) { - return $this->setting('priority_services', '0.8'); - } - - if (preg_match('#^/(blog|en/blog)/[^/]+/[^/]+$#', $path)) { - return $this->setting('priority_blog', '0.6'); - } - - if (preg_match('#^/(uslugi|en/services|realizacje|en/work|blog|en/blog)(/[^/]+)?$#', $path)) { - return $this->setting('priority_categories', '0.4'); - } - - $legalPaths = [ - '/polityka-prywatnosci', '/en/privacy-policy', - '/regulamin', '/en/terms', - '/o-nas', '/en/about', - '/kontakt', '/en/contact', - '/laboratorium-ai', '/en/ai-lab', - '/laboratorium-ai/zabawa-animacja', '/en/ai-lab/blob-playground', - ]; - if (in_array($path, $legalPaths)) { - return $this->setting('priority_legal', '0.2'); - } - - return null; - } -}