diff --git a/assets/sitemap.xsl b/assets/sitemap.xsl new file mode 100644 index 0000000..c6b6bbb --- /dev/null +++ b/assets/sitemap.xsl @@ -0,0 +1,261 @@ + + + + + + + + + + Sitemap + <xsl:if test="sitemap:sitemapindex">Index</xsl:if> + + + + +
+
+

Sitemap

+ + Index + + + Images + + + Video + + + Xhtml + +
+

+ + + This index contains + + sitemaps. + + + This index contains + + URLs. + + +

+

+ This is an XML sitemap, meant for consumption by search engines. + You can find more information about XML sitemaps on sitemaps.org. +

+
+ + + +
+
+ + + +
+ + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
URLLast Modified
+ + + + + + + +
+
+
+
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
URLChange Freq.PriorityLast Modified
+ + +

+ + + +

+ + + +
+ +
+
+
+
+ + + + + + + + + + + + + + +

+ + + + + + + + + + + + + + + + + + + + + +

+ +
+ + + + + +

+ Image: + + + + + + + + + +

+
+ + + + + + + + + + + + + + + +

+ Video: + + + + + + + + + + + + thumb + + + + + + + +

+
+ +
diff --git a/classes/events.go b/classes/events.go new file mode 100644 index 0000000..15d41cd --- /dev/null +++ b/classes/events.go @@ -0,0 +1,56 @@ +package classes + +import ( + "time" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" +) + +// ItemInfo is what the sitemap knows about one URL it is about to write, the +// PHP $itemInfo array: URL (before the final Url::to in BeforeAddItem, the +// written loc in AddItem), MTime, the record's modification time when the +// item has one, Lastmod, the written lastmod ("" when omitted), and the +// hreflang Alternates. +type ItemInfo struct { + URL string + MTime *time.Time + Lastmod string + Alternates []Alternate +} + +// BeforeAddItem is the typed successor of the PHP event +// golem15.sitemap.beforeAddItem. The generator fires it with +// festival.Bus.UntilHandled for every URL before it is counted and written; +// a listener that sets Skip drops the URL (PHP: a listener returned false). +// +// app.Events.Listen("acme.seo", func(ctx context.Context, e *classes.BeforeAddItem) error { +// if strings.HasSuffix(e.Info.URL, "/draft") { +// e.Skip = true +// } +// return nil +// }) +type BeforeAddItem struct { + Item models.DefinitionItem + Info ItemInfo + Skip bool +} + +// IsHandled reports Skip, so UntilHandled stops at the first listener that +// drops the URL (festival.Handleable). +func (e *BeforeAddItem) IsHandled() bool { return e != nil && e.Skip } + +// AddItem is the typed successor of the PHP event golem15.sitemap.addItem. +// The generator fires it with festival.Bus.Fire after a URL's elements are +// built; listeners may append child XML to Extra, which is written as it is, +// before the url closing tag, so it must be escaped already (EscapeText, +// EscapeAttr). +// +// app.Events.Listen("acme.images", func(ctx context.Context, e *classes.AddItem) error { +// e.Extra = append(e.Extra, ""+classes.EscapeText(img)+"") +// return nil +// }) +type AddItem struct { + Item models.DefinitionItem + Info ItemInfo + Extra []string +} diff --git a/classes/generator.go b/classes/generator.go index a4f5729..c57192a 100644 --- a/classes/generator.go +++ b/classes/generator.go @@ -8,11 +8,12 @@ import ( "time" "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/festival" "git.golem15.com/golem15/summercms/modules/pact" ) // MaxURLs is the sitemap protocol's URL limit (PHP Definition::MAX_URLS): -// the 50,000th URL is written, the 50,001st is not. +// the 50000th URL is written, the 50001st is not. const MaxURLs = 50000 // lastmodLayout is PHP date('c') in UTC: "+00:00", never "Z". @@ -23,18 +24,27 @@ type Resolver interface { Resolve(ctx context.Context, key, reference, itemPattern string, nesting bool, unprefixedLocale string) (*pact.ResolvedMenuItem, error) } -// Generator writes the sitemap of a definition. Root is the backend root URL -// (scheme and host without a trailing slash) that Url::to builds absolute -// URLs from. Resolver resolves registry items; nil skips them. -// UnprefixedLocale is handed to every Resolve as the locale without a URL -// prefix ("" leaves it to the type). +// Generator writes the sitemap of a definition, following PHP +// Definition::generateSitemap and addItemToSet. +// +// Root is the backend root URL (scheme and host without a trailing slash) +// that Url::to builds absolute URLs from. Resolver resolves registry items; +// nil skips them. Settings are the sitemap settings (tiers, priorities, +// frontend default locale, manifest path), Tiers the compiled tier rules, +// and Events the bus BeforeAddItem and AddItem are fired on (nil fires +// nothing). Manifest, when set, is used instead of reading the settings' +// updates_manifest_path. type Generator struct { - Root string - Resolver Resolver - UnprefixedLocale string + Root string + Resolver Resolver + Settings models.Settings + Tiers Tiers + Events *festival.Bus + Manifest Manifest - entries []URLEntry - count int + manifestLoaded bool + entries []URLEntry + count int } // itemInfo is PHP's $itemInfo: the URL (set marks it present), the record's @@ -47,19 +57,31 @@ type itemInfo struct { } // Generate returns the sitemap XML of items, or "" when there are none (PHP -// generateSitemap). A url item is added with its URL; a registry item is -// resolved with its URL pattern and nesting, its URL added when it has one -// and its sub-items added recursively. A type the registry does not know, -// or a Resolve that finds nothing, adds nothing. +// generateSitemap). A url item is added with its URL. A registry item is +// resolved with its URL pattern, its nesting and the settings' frontend +// default locale as the unprefixed locale; its URL is added when it has one, +// then its sub-items: when the item has no URL of its own (a list type), +// each sub-item is added once per alternate, with that alternate's URL as +// loc and every alternate as hreflang, as the PHP list types returned them; +// otherwise each sub-item is added once with its own URL. A type the +// registry does not know, or a Resolve that finds nothing, adds nothing. +// +// Generate does not apply the frontend_url rewrite; the route does +// (Rewrite). func (g *Generator) Generate(ctx context.Context, items []models.DefinitionItem) (string, error) { if len(items) == 0 { return "", nil } + if ctx == nil { + ctx = context.Background() + } g.entries, g.count = nil, 0 for i := range items { item := &items[i] if item.Type == models.ItemTypeURL { - g.addItem(item, itemInfo{}) + if err := g.addItem(ctx, item, itemInfo{}); err != nil { + return "", err + } continue } resolved, err := g.resolve(ctx, item) @@ -70,9 +92,17 @@ func (g *Generator) Generate(ctx context.Context, items []models.DefinitionItem) continue } if resolved.URL != "" { - g.addItem(item, infoOf(*resolved)) + if err := g.addItem(ctx, item, infoOf(*resolved)); err != nil { + return "", err + } + if err := g.addChildren(ctx, item, resolved.Items); err != nil { + return "", err + } + continue + } + if err := g.addPerLocale(ctx, item, resolved.Items); err != nil { + return "", err } - g.addChildren(item, resolved.Items) } return Serialize(g.Stylesheet(), g.entries), nil } @@ -94,17 +124,54 @@ func (g *Generator) resolve(ctx context.Context, item *models.DefinitionItem) (* return nil, nil } } - return g.Resolver.Resolve(ctx, item.Type, models.Text(item.Reference), models.Text(item.URLPattern), item.Nesting, g.UnprefixedLocale) + return g.Resolver.Resolve(ctx, item.Type, models.Text(item.Reference), models.Text(item.URLPattern), item.Nesting, models.Text(g.Settings.FrontendDefaultLocale)) } // addChildren adds every child with a URL, then its own children. -func (g *Generator) addChildren(item *models.DefinitionItem, children []pact.ResolvedMenuItem) { +func (g *Generator) addChildren(ctx context.Context, item *models.DefinitionItem, children []pact.ResolvedMenuItem) error { for _, child := range children { if child.URL != "" { - g.addItem(item, infoOf(child)) + if err := g.addItem(ctx, item, infoOf(child)); err != nil { + return err + } + } + if err := g.addChildren(ctx, item, child.Items); err != nil { + return err } - g.addChildren(item, child.Items) } + return nil +} + +// addPerLocale adds the children of a list type: each once per alternate +// (once with its URL when it has no alternates), then its own children. +func (g *Generator) addPerLocale(ctx context.Context, item *models.DefinitionItem, children []pact.ResolvedMenuItem) error { + for _, child := range children { + info := infoOf(child) + if len(child.Alternates) == 0 { + if child.URL != "" { + if err := g.addItem(ctx, item, info); err != nil { + return err + } + } + if err := g.addChildren(ctx, item, child.Items); err != nil { + return err + } + continue + } + for _, alt := range child.Alternates { + perLocale := info + perLocale.url = alt.URL + if perLocale.url != "" { + if err := g.addItem(ctx, item, perLocale); err != nil { + return err + } + } + if err := g.addChildren(ctx, item, child.Items); err != nil { + return err + } + } + } + return nil } func infoOf(r pact.ResolvedMenuItem) itemInfo { @@ -115,33 +182,76 @@ func infoOf(r pact.ResolvedMenuItem) itemInfo { return info } -// addItem is PHP Definition::addItemToSet. -func (g *Generator) addItem(item *models.DefinitionItem, info itemInfo) { +func (g *Generator) manifest() Manifest { + if !g.manifestLoaded { + if g.Manifest == nil { + g.Manifest = LoadManifest(models.Text(g.Settings.UpdatesManifestPath)) + } + g.manifestLoaded = true + } + return g.Manifest +} + +// addItem is PHP Definition::addItemToSet, in its order: the URL cap, a +// missing URL filled through Url::to, lastmod from the record or the +// manifest, BeforeAddItem, an empty URL dropped and the URL passed through +// Url::to again, the count, the tier or item priority, the elements and +// AddItem. +func (g *Generator) addItem(ctx context.Context, item *models.DefinitionItem, info itemInfo) error { if g.count >= MaxURLs { - return + return nil } if !info.set { info.url = URLTo(g.Root, models.Text(item.URL)) } - lastmod := "" - if info.mtime != nil && info.mtime.Unix() != 0 { - lastmod = info.mtime.UTC().Format(lastmodLayout) + var lastmod time.Time + hasLastmod := false + if info.mtime != nil { + lastmod, hasLastmod = *info.mtime, true + } else if t, ok := g.manifest().Lastmod(info.url); ok { + lastmod, hasLastmod = t, true + } + lastmodText := "" + if hasLastmod && lastmod.Unix() != 0 { + lastmodText = lastmod.UTC().Format(lastmodLayout) + } + if g.Events != nil { + before := &BeforeAddItem{Item: *item, Info: ItemInfo{URL: info.url, MTime: info.mtime, Lastmod: lastmodText, Alternates: append([]Alternate(nil), info.alternates...)}} + skipped, err := g.Events.UntilHandled(ctx, before) + if err != nil { + return err + } + if skipped { + return nil + } } if info.url == "" || info.url == "0" { - return + return nil } loc := URLTo(g.Root, info.url) g.count++ + priority, ok := g.Tiers.Priority(loc, g.Settings) + if !ok { + priority = models.Text(item.Priority) + } entry := URLEntry{ Loc: loc, - Lastmod: lastmod, + Lastmod: lastmodText, Changefreq: models.Text(item.Changefreq), - Priority: models.Text(item.Priority), + Priority: priority, } if len(info.alternates) > 1 { for _, a := range info.alternates { entry.Alternates = append(entry.Alternates, Alternate{Hreflang: a.Hreflang, Href: URLTo(g.Root, a.Href)}) } } + if g.Events != nil { + added := &AddItem{Item: *item, Info: ItemInfo{URL: loc, MTime: info.mtime, Lastmod: lastmodText, Alternates: append([]Alternate(nil), entry.Alternates...)}} + if err := g.Events.Fire(ctx, added); err != nil { + return err + } + entry.Extra = added.Extra + } g.entries = append(g.entries, entry) + return nil } diff --git a/classes/golden_test.go b/classes/golden_test.go index 58a1682..d4d3452 100644 --- a/classes/golden_test.go +++ b/classes/golden_test.go @@ -5,11 +5,17 @@ import ( "encoding/json" "os" "path/filepath" + "slices" + "strconv" + "strings" "testing" "time" + sitemap "git.golem15.com/golem15/sm-sitemap-plugin" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/compass" + "git.golem15.com/golem15/summercms/modules/festival" "git.golem15.com/golem15/summercms/modules/pact" ) @@ -45,12 +51,19 @@ func (g goldenResolved) toPact(t *testing.T) pact.ResolvedMenuItem { return out } -// goldenCase is one sitemap case of a golden input. +// goldenCase is one sitemap case of a golden input: the stored items, the +// registry types' resolve results, the settings (PHP keys), the updates +// manifest, the URLs a BeforeAddItem listener skips and the text of the +// extra element an AddItem listener appends per URL. type goldenCase struct { - Name string `json:"name"` - Root string `json:"root"` - Items models.DefinitionItems `json:"items"` - Types map[string]map[string]goldenResolved `json:"types"` + Name string `json:"name"` + Root string `json:"root"` + Items models.DefinitionItems `json:"items"` + Types map[string]map[string]goldenResolved `json:"types"` + Settings map[string]json.RawMessage `json:"settings"` + Manifest map[string]any `json:"manifest"` + Skip []string `json:"skip"` + Extra map[string]string `json:"extra"` } // fakeResolver answers Resolve from a golden case's types. @@ -73,14 +86,97 @@ func (f fakeResolver) Resolve(_ context.Context, key, reference, _ string, _ boo return &out, nil } +// pluginTiers compiles the tier rules the plugin's own config/config.yaml +// ships. +func pluginTiers(t *testing.T) classes.Tiers { + t.Helper() + cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Environ: []string{}}) + if err != nil { + t.Fatal(err) + } + if err := cfg.MergePlugin("golem15.sitemap", (&sitemap.Plugin{}).ConfigFS()); err != nil { + t.Fatal(err) + } + tiers, err := classes.LoadTiers(cfg) + if err != nil { + t.Fatal(err) + } + return tiers +} + +// caseSettings builds the settings of a case from its PHP keys. +func caseSettings(t *testing.T, c goldenCase) models.Settings { + t.Helper() + s := models.DefaultSettings() + texts := map[string]**string{ + "frontend_url": &s.FrontendURL, + "frontend_default_locale": &s.FrontendDefaultLocale, + "priority_homepage": &s.PriorityHomepage, + "priority_services": &s.PriorityServices, + "priority_blog": &s.PriorityBlog, + "priority_categories": &s.PriorityCategories, + "priority_legal": &s.PriorityLegal, + } + for key, raw := range c.Settings { + if key == "enable_priority_tiers" { + if err := json.Unmarshal(raw, &s.EnablePriorityTiers); err != nil { + t.Fatal(err) + } + continue + } + dest, ok := texts[key] + if !ok { + t.Fatalf("case %s: unknown setting %s", c.Name, key) + } + v, err := models.DecodeText(raw) + if err != nil { + t.Fatal(err) + } + *dest = v + } + if c.Manifest != nil { + raw, err := json.Marshal(c.Manifest) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "updates.json") + if err := os.WriteFile(path, raw, 0o644); err != nil { + t.Fatal(err) + } + s.UpdatesManifestPath = &path + } + return s +} + +// renderCase runs a case through the generator and the route's rewrite. func renderCase(t *testing.T, c goldenCase) string { t.Helper() - gen := &classes.Generator{Root: c.Root, Resolver: fakeResolver{t: t, types: c.Types}} + bus := festival.New() + bus.Listen("test.skip", func(_ context.Context, e *classes.BeforeAddItem) error { + if slices.Contains(c.Skip, e.Info.URL) { + e.Skip = true + } + return nil + }) + bus.Listen("test.extra", func(_ context.Context, e *classes.AddItem) error { + if text, ok := c.Extra[e.Info.URL]; ok { + e.Extra = append(e.Extra, ""+classes.EscapeText(text)+"") + } + return nil + }) + settings := caseSettings(t, c) + gen := &classes.Generator{ + Root: c.Root, + Resolver: fakeResolver{t: t, types: c.Types}, + Settings: settings, + Tiers: pluginTiers(t), + Events: bus, + } xml, err := gen.Generate(context.Background(), c.Items) if err != nil { t.Fatalf("generate %s: %v", c.Name, err) } - return xml + return classes.Rewrite(xml, classes.BackendURL(c.Root), models.Text(settings.FrontendURL)) } func readTestdata(t *testing.T, name string) []byte { @@ -94,6 +190,35 @@ func readTestdata(t *testing.T, name string) []byte { // TestSitemapGolden compares the Go sitemap with the PHP goldens. func TestSitemapGolden(t *testing.T) { + t.Run("urlto", func(t *testing.T) { + var in struct { + Root string `json:"root"` + URLs []*string `json:"urls"` + } + if err := json.Unmarshal(readTestdata(t, "urlto.json"), &in); err != nil { + t.Fatal(err) + } + want := strings.Split(strings.TrimSuffix(string(readTestdata(t, "urlto.golden")), "\n"), "\n") + if len(want) != len(in.URLs) { + t.Fatalf("urlto.golden has %d lines for %d inputs", len(want), len(in.URLs)) + } + var got strings.Builder + for i, u := range in.URLs { + path := "" + if u != nil { + path = *u + } + out := classes.URLTo(in.Root, path) + if out != want[i] { + t.Errorf("URLTo(%q) = %q, PHP %q", path, out, want[i]) + } + got.WriteString(out + "\n") + } + if got.String() != string(readTestdata(t, "urlto.golden")) { + t.Error("urlto output differs from the golden file") + } + }) + t.Run("tracer", func(t *testing.T) { var in struct { Case goldenCase `json:"case"` @@ -106,4 +231,113 @@ func TestSitemapGolden(t *testing.T) { t.Fatalf("tracer sitemap differs from the PHP golden\n got: %q\nwant: %q", got, want) } }) + + t.Run("full", func(t *testing.T) { + var in struct { + Cases []goldenCase `json:"cases"` + } + if err := json.Unmarshal(readTestdata(t, "full.json"), &in); err != nil { + t.Fatal(err) + } + golden := string(readTestdata(t, "full.golden")) + var all strings.Builder + for i, c := range in.Cases { + got := renderCase(t, c) + all.WriteString("=== " + c.Name + "\n" + got) + header := "=== " + c.Name + "\n" + start := strings.Index(golden, header) + if start < 0 { + t.Errorf("case %s is missing from full.golden", c.Name) + continue + } + want := golden[start+len(header):] + if i+1 < len(in.Cases) { + if end := strings.Index(want, "=== "+in.Cases[i+1].Name+"\n"); end >= 0 { + want = want[:end] + } + } + if got != want { + t.Errorf("case %s differs from the PHP golden\n got: %q\nwant: %q", c.Name, got, want) + } + } + if all.String() != golden { + t.Error("full output differs from full.golden") + } + }) + + t.Run("cap", func(t *testing.T) { + items := make([]models.DefinitionItem, classes.MaxURLs+1) + for i := range items { + u := "/u/" + strconv.Itoa(i+1) + items[i] = models.DefinitionItem{Type: models.ItemTypeURL, URL: &u} + } + gen := &classes.Generator{Root: "https://api.example.test"} + xml, err := gen.Generate(context.Background(), items) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(xml, ""); n != classes.MaxURLs { + t.Fatalf("%d urls written, want %d", n, classes.MaxURLs) + } + if !strings.Contains(xml, "https://api.example.test/u/50000") || strings.Contains(xml, "/u/50001<") { + t.Fatal("the 50000th URL must be written and the 50001st must not") + } + }) +} + +// TestSitemapTiersConfig checks the boot-time tier rule checks, a host's +// replacement of the default list and the manifest's fail-safe reads. +func TestSitemapTiersConfig(t *testing.T) { + for name, rules := range map[string][]classes.TierConfig{ + "unknown setting": {{Setting: "priority_other", Exact: []string{""}}}, + "neither": {{Setting: models.PriorityHomepage}}, + "both": {{Setting: models.PriorityHomepage, Exact: []string{""}, Pattern: "^/$"}}, + "bad pattern": {{Setting: models.PriorityBlog, Pattern: "^/(blog"}}, + } { + if _, err := classes.CompileTiers(rules); err == nil { + t.Errorf("%s: compiled, want an error", name) + } + } + + dir := t.TempDir() + host := "sitemap:\n priority_tiers:\n - setting: priority_legal\n pattern: '^/legal(/.*)?$'\n" + if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte(host), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) + if err != nil { + t.Fatal(err) + } + if err := cfg.MergePlugin("golem15.sitemap", (&sitemap.Plugin{}).ConfigFS()); err != nil { + t.Fatal(err) + } + tiers, err := classes.LoadTiers(cfg) + if err != nil || len(tiers) != 1 { + t.Fatalf("host tiers = %+v, %v", tiers, err) + } + s := models.DefaultSettings() + if p, ok := tiers.Priority("https://h/legal/terms/", s); !ok || p != "0.2" { + t.Fatalf("host rule priority = %q %v", p, ok) + } + if _, ok := tiers.Priority("https://h/", s); ok { + t.Fatal("the host list must replace the default homepage rule") + } + + if m := classes.LoadManifest(dir); m != nil { + t.Fatal("a directory is not a manifest") + } + list := filepath.Join(dir, "list.json") + if err := os.WriteFile(list, []byte(`["/a"]`), 0o644); err != nil { + t.Fatal(err) + } + if m := classes.LoadManifest(list); m != nil { + t.Fatal("a JSON list is not a manifest") + } + big := filepath.Join(dir, "big.json") + if err := os.WriteFile(big, append([]byte(`{"/a":"2026-01-01","x":"`), append(make([]byte, classes.ManifestMaxBytes), '"', '}')...), 0o644); err != nil { + t.Fatal(err) + } + if m := classes.LoadManifest(big); m != nil { + t.Fatal("a manifest over the size cap must be ignored") + } } diff --git a/classes/manifest.go b/classes/manifest.go new file mode 100644 index 0000000..b6d224c --- /dev/null +++ b/classes/manifest.go @@ -0,0 +1,113 @@ +package classes + +import ( + "bytes" + "encoding/json" + "io" + "log/slog" + "os" + "strings" + "time" +) + +// ManifestMaxBytes is the largest updates manifest the sitemap reads. +const ManifestMaxBytes = 5 << 20 + +// Manifest maps a normalized URL path to its last-modified date text, the +// updates manifest a frontend build writes. +type Manifest map[string]string + +// LoadManifest reads the updates manifest at path: a regular file of at +// most ManifestMaxBytes holding a JSON object of URL path to date. It fails +// safe like PHP: an empty path, a missing, unreadable, oversized or invalid +// file is no manifest (nil), logged without the file's content. Keys are +// normalized like PHP normalizeManifestPath; values that are not strings are +// ignored. +func LoadManifest(path string) Manifest { + path = strings.TrimSpace(path) + if path == "" { + return nil + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() { + return nil + } + if info.Size() > ManifestMaxBytes { + slog.Default().Warn("golem15.sitemap: updates manifest is too large; lastmod omitted", "path", path, "max_bytes", ManifestMaxBytes) + return nil + } + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + raw, err := io.ReadAll(io.LimitReader(f, ManifestMaxBytes+1)) + if err != nil || len(raw) > ManifestMaxBytes { + return nil + } + raw = bytes.TrimSpace(raw) + var decoded map[string]any + if len(raw) == 0 || raw[0] != '{' || json.Unmarshal(raw, &decoded) != nil { + slog.Default().Warn("golem15.sitemap: updates manifest is not a JSON object; lastmod omitted", "path", path) + return nil + } + m := Manifest{} + for key, value := range decoded { + if s, ok := value.(string); ok { + m[normalizeManifestPath(key)] = s + } + } + return m +} + +// normalizeManifestPath is PHP normalizeManifestPath: a leading slash, no +// trailing slash except for the root. +func normalizeManifestPath(path string) string { + path = "/" + strings.TrimLeft(path, "/") + if path == "/" { + return path + } + return strings.TrimRight(path, "/") +} + +// Lastmod is PHP lookupManifestLastmod: the date listed for the URL's path, +// parsed like the ISO 8601 forms strtotime reads (a date without a zone is +// UTC). ok is false when the path is not listed or its date does not parse. +func (m Manifest) Lastmod(url string) (time.Time, bool) { + if len(m) == 0 { + return time.Time{}, false + } + path := URLPath(url) + if path == "" { + path = "/" + } + text, ok := m[normalizeManifestPath(path)] + if !ok { + return time.Time{}, false + } + t, ok := parseManifestDate(text) + if !ok || t.Unix() == 0 { + return time.Time{}, false + } + return t, true +} + +var manifestLayouts = []string{ + time.RFC3339Nano, + "2006-01-02T15:04:05.999999999Z0700", + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "2006-01-02 15:04:05Z07:00", + "2006-01-02 15:04:05", + "2006-01-02", +} + +func parseManifestDate(s string) (time.Time, bool) { + s = strings.TrimSpace(s) + for _, layout := range manifestLayouts { + if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/classes/rewrite.go b/classes/rewrite.go new file mode 100644 index 0000000..f2f7e72 --- /dev/null +++ b/classes/rewrite.go @@ -0,0 +1,49 @@ +package classes + +import ( + "regexp" + "strings" +) + +// The PHP routes.php patterns of the frontend rewrite. +var ( + locElement = regexp.MustCompile(`(.*?)`) + xhtmlLinkRef = regexp.MustCompile(`(]*href=")([^"]*)(")`) +) + +// Rewrite is the frontend_url rewrite of PHP routes.php: with frontendURL +// set, backendURL is replaced literally (str_replace) inside every loc +// element, inside every xhtml:link href and in the exact stylesheet href. +// Both URLs are right-trimmed of slashes first. An empty frontendURL (or +// "0", which PHP reads as false) leaves xml unchanged. +func Rewrite(xml, backendURL, frontendURL string) string { + if frontendURL == "" || frontendURL == "0" { + return xml + } + frontend := strings.TrimRight(frontendURL, "/") + backend := strings.TrimRight(backendURL, "/") + replace := func(s string) string { + if backend == "" { + return s + } + return strings.ReplaceAll(s, backend, frontend) + } + xml = locElement.ReplaceAllStringFunc(xml, func(m string) string { + inner := m[len("") : len(m)-len("")] + return "" + replace(inner) + "" + }) + xml = xhtmlLinkRef.ReplaceAllStringFunc(xml, func(m string) string { + sub := xhtmlLinkRef.FindStringSubmatch(m) + return sub[1] + replace(sub[2]) + sub[3] + }) + if backend != "" { + xml = strings.ReplaceAll(xml, `href="`+backend+`/sitemap.xsl"`, `href="`+frontend+`/sitemap.xsl"`) + } + return xml +} + +// BackendURL is PHP rtrim(url('/'), '/') for a request whose root URL is +// root: the URL the rewrite replaces. +func BackendURL(root string) string { + return strings.TrimRight(URLTo(root, "/"), "/") +} diff --git a/classes/testdata/full.golden b/classes/testdata/full.golden new file mode 100644 index 0000000..4553e1e --- /dev/null +++ b/classes/testdata/full.golden @@ -0,0 +1,58 @@ +=== escaping + + +https://a&b<c>.pl/xa&b<c>"d'e f g +h0.5 & <1> "q" 'a'https://api.example.test/za%C5%BC%C3%B3%C5%82%C4%87%20%26%20co2026-01-02T03:04:05+00:00x>y<0.9> +=== empty-urlset + + + +=== no-items +=== hreflang-list + + +https://api.example.test/pl/a2026-02-01T00:00:00+00:00monthly0.4https://api.example.test/en/a2026-02-01T00:00:00+00:00monthly0.4https://api.example.test/pl/bmonthly0.4https://api.example.test/en/bmonthly0.4https://api.example.test/de/bmonthly0.4https://api.example.test/solomonthly0.4 +=== nested + + +https://api.example.test/kategoria/a2026-03-01T12:00:00+00:00daily0.7https://api.example.test/kategoria/b2026-03-02T12:00:00+00:00daily0.7https://api.example.test/kategoria/cdaily0.7 +=== tiers-default + + +https://api.example.test/weekly1.0https://api.example.test/enweekly1.0https://api.example.test/enweekly1.0https://api.example.test/uslugi/xweekly0.8https://api.example.test/uslugi/x/yweekly0.3https://api.example.test/realizacje/rweekly0.8https://api.example.test/en/work/wweekly0.8https://api.example.test/blog/a/bweekly0.6https://api.example.test/en/blog/a/bweekly0.6https://api.example.test/blog/aweekly0.4https://api.example.test/blogweekly0.4https://api.example.test/en/blogweekly0.4https://api.example.test/uslugiweekly0.4https://api.example.test/kontaktweekly0.2https://api.example.test/kontaktweekly0.2https://api.example.test/en/ai-lab/blob-playgroundweekly0.2https://api.example.test/otherweekly0.3https://elsewhere.pl/uslugi/zweekly0.8https://api.example.test/o-nas?x=1weekly0.2https://api.example.test/blog/a/b/cweekly0.3 +=== tiers-settings + + +https://api.example.test/weekly0.95https://api.example.test/blog/a/bweeklyhttps://api.example.test/regulaminweekly0.15https://api.example.test/xweekly0.3 +=== tiers-off + + +https://api.example.test/weekly0.1https://api.example.test/blog/a/bweekly0.2 +=== rewrite + + +https://www.example.test/weekly0.5https://www.example.test/aweekly0.5https://www.example.test/b?x=https%3A%2F%2Fapi.example.test%2Fcweekly0.5https://other.example.test/api.example.testweekly0.5https://www.example.test/pdaily0.7https://www.example.test/en/pdaily0.7 +=== rewrite-same + + +https://api.example.test/aweekly0.5 +=== manifest + + +https://api.example.test/2026-05-01T08:00:00+00:00weekly0.5https://api.example.test/about2026-05-02T00:00:00+00:00weekly0.5https://api.example.test/about2026-05-02T00:00:00+00:00weekly0.5https://api.example.test/blog/xweekly0.5https://api.example.test/noneweekly0.5https://api.example.test/rel2026-05-03T01:02:03+00:00daily0.7https://api.example.test/about2026-06-01T00:00:00+00:00daily0.7 +=== skip-extra + + +https://api.example.test/extraweekly0.5a&b <c>https://api.example.test/keepweekly0.5 +=== precision + + +https://api.example.test/p1daily0.5https://api.example.test/p2daily1https://api.example.test/p371https://api.example.test/p4daily1.0https://api.example.test/p5daily0.50https://api.example.test/p6daily0.6 +=== url-null + + +https://api.example.test/https://api.example.test/0daily0.5 +=== port-root + + +http://localhost:8096/weekly0.5http://localhost:8096/a%20bweekly0.5 diff --git a/classes/testdata/full.json b/classes/testdata/full.json new file mode 100644 index 0000000..d2c4efe --- /dev/null +++ b/classes/testdata/full.json @@ -0,0 +1,693 @@ +{ + "cases": [ + { + "name": "escaping", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": "https://a&b.pl/x", + "changefreq": "a&b\"d'e\rf\tg\nh", + "priority": "0.5 & <1> \"q\" 'a'" + }, + { + "type": "acme-post", + "reference": "1", + "urlPattern": "", + "changefreq": "x>y", + "priority": "<0.9>", + "nesting": "1" + } + ], + "types": { + "acme-post": { + "1": { + "url": "/zażółć & co", + "mtime": "2026-01-02T03:04:05+00:00", + "alternates": [ + [ + "x\"y&z<\t\n\r'>", + "https://q&r.pl/p" + ], + [ + "en", + "/en/zażółć" + ] + ] + } + } + } + }, + { + "name": "empty-urlset", + "root": "https://api.example.test", + "items": [ + { + "type": "acme-post", + "reference": "404", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + }, + { + "type": "unknown-type", + "reference": "1", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + } + ], + "types": { + "acme-post": {} + } + }, + { + "name": "no-items", + "root": "https://api.example.test", + "items": [] + }, + { + "name": "hreflang-list", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "acme-list", + "reference": "", + "urlPattern": "", + "changefreq": "monthly", + "priority": "0.4", + "nesting": "1" + } + ], + "types": { + "acme-list": { + "": { + "url": "", + "items": [ + { + "url": "/pl/a", + "mtime": "2026-02-01T00:00:00+00:00", + "alternates": [ + [ + "pl", + "/pl/a" + ], + [ + "en", + "/en/a" + ] + ] + }, + { + "url": "/pl/b", + "mtime": null, + "alternates": [ + [ + "pl", + "/pl/b" + ], + [ + "en", + "/en/b" + ], + [ + "de", + "/de/b" + ] + ] + }, + { + "url": "/solo", + "alternates": [] + } + ] + } + } + } + }, + { + "name": "nested", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "acme-cat", + "reference": "5", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + } + ], + "types": { + "acme-cat": { + "5": { + "url": "/kategoria/a", + "mtime": "2026-03-01T12:00:00+00:00", + "alternates": [ + [ + "pl", + "/kategoria/a" + ], + [ + "en", + "/en/category/a" + ] + ], + "items": [ + { + "url": "/kategoria/b", + "mtime": "2026-03-02T12:00:00+00:00", + "alternates": [ + [ + "pl", + "/kategoria/b" + ], + [ + "en", + "/en/category/b" + ] + ], + "items": [ + { + "url": "/kategoria/c", + "alternates": [ + [ + "x-default", + "/kategoria/c" + ] + ] + } + ] + } + ] + } + } + } + }, + { + "name": "tiers-default", + "root": "https://api.example.test", + "items": [ + { + "type": "url", + "url": "/", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/en", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/en/", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/uslugi/x", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/uslugi/x/y", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/realizacje/r", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/en/work/w", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/blog/a/b", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/en/blog/a/b", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/blog/a", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/blog", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/en/blog/", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/uslugi", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/kontakt", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/kontakt/", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/en/ai-lab/blob-playground", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/other", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "https://elsewhere.pl/uslugi/z", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/o-nas?x=1", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/blog/a/b/c", + "changefreq": "weekly", + "priority": "0.3" + } + ] + }, + { + "name": "tiers-settings", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": true, + "priority_homepage": "0.95", + "priority_blog": "", + "priority_legal": "0.15" + }, + "items": [ + { + "type": "url", + "url": "/", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/blog/a/b", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/regulamin", + "changefreq": "weekly", + "priority": "0.3" + }, + { + "type": "url", + "url": "/x", + "changefreq": "weekly", + "priority": "0.3" + } + ] + }, + { + "name": "tiers-off", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": "/", + "changefreq": "weekly", + "priority": "0.1" + }, + { + "type": "url", + "url": "/blog/a/b", + "changefreq": "weekly", + "priority": "0.2" + } + ] + }, + { + "name": "rewrite", + "root": "https://api.example.test", + "settings": { + "frontend_url": "https://www.example.test/", + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": "/", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/a", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "https://api.example.test/b?x=https://api.example.test/c", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "https://other.example.test/api.example.test", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "acme-list", + "reference": "", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + } + ], + "types": { + "acme-list": { + "": { + "url": "", + "items": [ + { + "url": "/p", + "alternates": [ + [ + "pl", + "/p" + ], + [ + "en", + "https://api.example.test/en/p" + ] + ] + } + ] + } + } + } + }, + { + "name": "rewrite-same", + "root": "https://api.example.test", + "settings": { + "frontend_url": "https://api.example.test", + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": "/a", + "changefreq": "weekly", + "priority": "0.5" + } + ] + }, + { + "name": "manifest", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "manifest": { + "/": "2026-05-01T10:00:00+02:00", + "about/": "2026-05-02", + "/blog/x": "not a date", + "/rel": "2026-05-03T01:02:03Z" + }, + "items": [ + { + "type": "url", + "url": "/", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/about", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/about/", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/blog/x", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/none", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "acme-post", + "reference": "2", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + }, + { + "type": "acme-post", + "reference": "3", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + } + ], + "types": { + "acme-post": { + "2": { + "url": "/rel", + "mtime": null, + "alternates": [ + [ + "pl", + "/rel" + ] + ] + }, + "3": { + "url": "/about", + "mtime": "2026-06-01T00:00:00+00:00", + "alternates": [ + [ + "pl", + "/about" + ] + ] + } + } + } + }, + { + "name": "skip-extra", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "skip": [ + "https://api.example.test/skip-me", + "/rel-skip" + ], + "extra": { + "https://api.example.test/extra": "a&b " + }, + "items": [ + { + "type": "url", + "url": "/skip-me", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/extra", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/keep", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "acme-post", + "reference": "9", + "urlPattern": "", + "changefreq": "daily", + "priority": "0.7", + "nesting": "1" + } + ], + "types": { + "acme-post": { + "9": { + "url": "/rel-skip", + "alternates": [ + [ + "pl", + "/rel-skip" + ] + ] + } + } + } + }, + { + "name": "precision", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": "/p1", + "changefreq": "daily", + "priority": 0.5 + }, + { + "type": "url", + "url": "/p2", + "changefreq": "daily", + "priority": 1.0 + }, + { + "type": "url", + "url": "/p3", + "changefreq": 7, + "priority": 1 + }, + { + "type": "url", + "url": "/p4", + "changefreq": "daily", + "priority": "1.0" + }, + { + "type": "url", + "url": "/p5", + "changefreq": "daily", + "priority": "0.50" + }, + { + "url": "/p6", + "changefreq": "daily", + "priority": "0.6", + "cmsPage": "x", + "exists": true, + "fillable": [ + "a" + ] + } + ] + }, + { + "name": "url-null", + "root": "https://api.example.test", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": null, + "changefreq": null, + "priority": null + }, + { + "type": "url", + "url": "0", + "changefreq": "daily", + "priority": "0.5" + } + ] + }, + { + "name": "port-root", + "root": "http://localhost:8096", + "settings": { + "enable_priority_tiers": false + }, + "items": [ + { + "type": "url", + "url": "/", + "changefreq": "weekly", + "priority": "0.5" + }, + { + "type": "url", + "url": "/a b", + "changefreq": "weekly", + "priority": "0.5" + } + ] + } + ] +} \ No newline at end of file diff --git a/classes/testdata/urlto.golden b/classes/testdata/urlto.golden new file mode 100644 index 0000000..3deb178 --- /dev/null +++ b/classes/testdata/urlto.golden @@ -0,0 +1,58 @@ +https://api.ex.pl/ +https://api.ex.pl/ +https://api.ex.pl/ +https://api.ex.pl/blog +https://api.ex.pl/blog +https://api.ex.pl/blog +https://api.ex.pl/blog/za%C5%BC%C3%B3%C5%82%C4%87%20g%C4%99%C5%9Bl%C4%85 +https://api.ex.pl/en/blog/%3Aslug +https://x.pl/q/ +http://x.pl/ +HTTP://X.pl:443/a +https://h:8080/a +https://h/a +http://h/ +https://api.ex.pl/a?b=1&c=2 +https://api.ex.pl/a?x%5B%5D=1&x%5B%5D=2&0=y +https://api.ex.pl/a?q=a&0=b%20c +https://api.ex.pl/a +https://api.ex.pl/a?b=&0=c +https://api.ex.pl/a?x&y +https://api.ex.pl/a?a=2 +https://api.ex.pl/a?a%5Bb%5D=1 +https://api.ex.pl/a#b%20c +https://api.ex.pl/a +#frag +cdn.x/a +mailto:a%40b.pl +tel:%2B48%20123 +sms:%2B48123 +https://api.ex.pl/a/c +https://api.ex.pl/a//b +https://api.ex.pl/a/%20b/%25zz +https://api.ex.pl/a%2Fb +https://api.ex.pl/%C5%9Bcie%C5%BCka/%C4%85%C4%99?x=%C5%BC#%C5%82 +https://api.ex.pl/ftp%3A//x.pl/a%20b +ftp://x.pl/a +news:///comp.x +https://api.ex.pl/a%3Ab +https://api.ex.pl/a%3A1 +https://api.ex.pl/user%40host +https://user:pass@h/x +https://api.ex.pl/%20%20/lead +https://api.ex.pl/trail +https://api.ex.pl/tab_here +https://api.ex.pl/new_line +https://api.ex.pl/ +https://api.ex.pl/javascript%3Aalert%281%29 +https://a&b.pl/x +https://h/ +https://api.ex.pl/en +https://api.ex.pl/en +https://api.ex.pl/a?x=%2B +https://api.ex.pl/a?k=v%20w +https://api.ex.pl/a%20b?x=1#y%20z +https://h/a?b=1#c +https://api.ex.pl/~tilde/_under.dot-dash +https://api.ex.pl/a%27b%22c +https://api.ex.pl/%3Cx%3E%26y diff --git a/classes/testdata/urlto.json b/classes/testdata/urlto.json new file mode 100644 index 0000000..3e0ca04 --- /dev/null +++ b/classes/testdata/urlto.json @@ -0,0 +1,63 @@ +{ + "root": "https://api.ex.pl", + "urls": [ + "/", + "", + null, + "/blog/", + "/blog", + "blog", + "/blog/zażółć gęślą", + "/en/blog/:slug", + "https://x.pl/q/", + "http://x.pl", + "HTTP://X.pl:443/a", + "https://h:8080/a", + "https://h:443/a", + "http://h:80/", + "/a?b=1&c=2", + "/a?x[]=1&x[]=2&y", + "/a?q=a%26b+c", + "/a?", + "/a?b=&c", + "/a?0=x&1=y", + "/a?a=1&a=2", + "/a?a[b]=1", + "/a#b c", + "/a#", + "#frag", + "//cdn.x/a", + "mailto:a@b.pl", + "tel:+48 123", + "sms:+48123", + "/../a/./b/../c", + "/a//b", + "/a/%20b/%zz", + "/a%2Fb", + "/ścieżka/ąę?x=ż#ł", + "ftp://x.pl/a b", + "ftp://x.pl/a", + "news:comp.x", + "/a:b", + "a:1", + "/user@host", + "https://user:pass@h/x", + " /lead", + "/trail ", + "/tab\there", + "/new\nline", + "/a/b/../../..", + "javascript:alert(1)", + "https://a&b.pl/x", + "https://h/./", + "/en", + "/en/", + "/a?x=%2B", + "/a?k=v w", + "/a b?x=1#y z", + "https://h/a?b=1#c", + "/~tilde/_under.dot-dash", + "/a'b\"c", + "/&y" + ] +} \ No newline at end of file diff --git a/classes/tiers.go b/classes/tiers.go new file mode 100644 index 0000000..2e8ca06 --- /dev/null +++ b/classes/tiers.go @@ -0,0 +1,98 @@ +package classes + +import ( + "fmt" + "regexp" + "slices" + "strings" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/compass" +) + +// TiersConfigKey is the config key of the priority tier rules. The plugin's +// config/config.yaml ships the rules of the PHP getPriorityForUrl; a host +// replaces the whole list in its own config. +const TiersConfigKey = "golem15.sitemap.priority_tiers" + +// TierConfig is one configured tier rule: Setting names the priority setting +// whose value applies (priority_homepage, priority_services, priority_blog, +// priority_categories or priority_legal), and exactly one of Exact (paths +// that match as they are) and Pattern (a regular expression, RE2 syntax) +// selects the URLs. +type TierConfig struct { + Setting string `koanf:"setting"` + Exact []string `koanf:"exact"` + Pattern string `koanf:"pattern"` +} + +// TierRule is a compiled tier rule. +type TierRule struct { + Setting string + Exact []string + Pattern *regexp.Regexp +} + +// Tiers is the ordered tier rule list; the first matching rule wins. +type Tiers []TierRule + +// CompileTiers checks and compiles tier rules. Every error names the rule: +// an unknown setting, neither or both of exact and pattern, or a pattern +// that does not compile. +func CompileTiers(rules []TierConfig) (Tiers, error) { + out := make(Tiers, 0, len(rules)) + for i, r := range rules { + if _, ok := models.PriorityDefaults[r.Setting]; !ok { + return nil, fmt.Errorf("%s[%d]: setting %q is not one of priority_homepage, priority_services, priority_blog, priority_categories, priority_legal", TiersConfigKey, i, r.Setting) + } + hasPattern := strings.TrimSpace(r.Pattern) != "" + if hasPattern == (len(r.Exact) > 0) { + return nil, fmt.Errorf("%s[%d]: set exactly one of exact and pattern", TiersConfigKey, i) + } + rule := TierRule{Setting: r.Setting, Exact: append([]string(nil), r.Exact...)} + if hasPattern { + re, err := regexp.Compile(r.Pattern) + if err != nil { + return nil, fmt.Errorf("%s[%d]: pattern: %w", TiersConfigKey, i, err) + } + rule.Pattern = re + } + out = append(out, rule) + } + return out, nil +} + +// LoadTiers reads and compiles the rules at TiersConfigKey; an absent key +// is no rules. +func LoadTiers(cfg *compass.Config) (Tiers, error) { + if cfg == nil || !cfg.Has(TiersConfigKey) { + return Tiers{}, nil + } + var rules []TierConfig + if err := cfg.LoadSection(TiersConfigKey, &rules); err != nil { + return nil, fmt.Errorf("%s must be a list of {setting, exact} or {setting, pattern} rules: %w", TiersConfigKey, err) + } + return CompileTiers(rules) +} + +// Priority is PHP getPriorityForUrl with configured rules: when the +// settings enable tiers, the URL's path (parse_url's, "/" when it has none) +// right-trimmed of slashes is matched against the rules in order and the +// first match's setting value is returned. ok is false when tiers are off or +// no rule matches, so the item's own priority applies. +func (t Tiers) Priority(url string, s models.Settings) (string, bool) { + if !s.EnablePriorityTiers { + return "", false + } + path := URLPath(url) + if path == "" { + path = "/" + } + path = strings.TrimRight(path, "/") + for _, r := range t { + if r.Pattern != nil && r.Pattern.MatchString(path) || r.Pattern == nil && slices.Contains(r.Exact, path) { + return s.Priority(r.Setting), true + } + } + return "", false +} diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..354b890 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,38 @@ +# golem15.sitemap defaults, read as golem15.sitemap.. +# +# The priority tier rules assign a URL the value of one of the five +# priority settings when Enable priority tiers is on. A URL's path, without +# trailing slashes ("" for the homepage), is matched against the rules in +# order and the first match wins; a URL no rule matches keeps its item's +# priority. Each rule names a setting (priority_homepage, priority_services, +# priority_blog, priority_categories or priority_legal) and either exact +# (a list of paths) or pattern (a regular expression in RE2 syntax, checked +# at boot). +# +# The defaults below are the rules the WinterCMS plugin had in code +# (getPriorityForUrl), so a site configured the same way produces the same +# priorities. A host replaces the whole list in its own config, for example +# in config/golem15.yaml under the same key below sitemap:. +priority_tiers: + - setting: priority_homepage + exact: ["", "/en"] + - setting: priority_services + pattern: '^/(uslugi|en/services|realizacje|en/work)/[^/]+$' + - setting: priority_blog + pattern: '^/(blog|en/blog)/[^/]+/[^/]+$' + - setting: priority_categories + pattern: '^/(uslugi|en/services|realizacje|en/work|blog|en/blog)(/[^/]+)?$' + - setting: priority_legal + exact: + - /polityka-prywatnosci + - /en/privacy-policy + - /regulamin + - /en/terms + - /o-nas + - /en/about + - /kontakt + - /en/contact + - /laboratorium-ai + - /en/ai-lab + - /laboratorium-ai/zabawa-animacja + - /en/ai-lab/blob-playground diff --git a/console/helpers.go b/console/helpers.go new file mode 100644 index 0000000..4912bcb --- /dev/null +++ b/console/helpers.go @@ -0,0 +1,75 @@ +// Package console holds the sitemap plugin's commands: sitemap:import (the +// PHP ImportCommand) and sitemap:import-winter (a copy of a WinterCMS +// site's definition and settings). +package console + +import ( + "context" + "fmt" + "strings" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" + "git.golem15.com/golem15/summercms/modules/lagoon" + "gorm.io/gorm" +) + +func flagValue(in bonfire.Input, name string) string { + v, ok := in.Flag(name) + if !ok { + return "" + } + return strings.TrimSpace(v) +} + +func flagOn(in bonfire.Input, name string) bool { + switch strings.ToLower(flagValue(in, name)) { + case "true", "1", "yes": + return true + } + return false +} + +// lookupDB returns the app database. The generated main.go publishes no +// database for plugin commands, so when none is published it opens +// database.dsn and publishes it the way lagoon's migrate does. The returned +// func closes a pool this call opened; it is a no-op otherwise. +func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) { + if app == nil { + return nil, nil, fmt.Errorf("golem15.sitemap: app is nil") + } + if db, ok := app.Lookup[*gorm.DB](); ok && db != nil { + return db, func() {}, nil + } + if lagoon.DSN(app.Config) == "" { + return nil, nil, fmt.Errorf("golem15.sitemap: database is not configured (set SUMMER_DATABASE__DSN)") + } + sqlDB, gdb, err := lagoon.OpenFromApp(ctx, app) + if err != nil { + return nil, nil, err + } + if err := lagoon.Publish(app, sqlDB, gdb); err != nil { + _ = sqlDB.Close() + return nil, nil, err + } + return gdb, func() { _ = sqlDB.Close() }, nil +} + +// singletonDefinition loads the definition with the fixed theme key, +// creating it when there is none (PHP Definition::firstOrCreate). +func singletonDefinition(tx *gorm.DB) (models.Definition, error) { + var defs []models.Definition + if err := tx.Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { + return models.Definition{}, err + } + if len(defs) > 0 { + return defs[0], nil + } + theme := models.DefinitionTheme + def := models.Definition{Theme: &theme} + if err := tx.Create(&def).Error; err != nil { + return models.Definition{}, err + } + return def, nil +} diff --git a/console/import.go b/console/import.go new file mode 100644 index 0000000..3fe6c45 --- /dev/null +++ b/console/import.go @@ -0,0 +1,187 @@ +package console + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" + "gorm.io/gorm" +) + +// Import is sitemap:import FILE [--theme=] [--append], the PHP +// ImportCommand: it reads a JSON array of URL entries ({"url": ..., +// "changefreq": ..., "priority": ...}; changefreq defaults to monthly, +// priority to 0.5) into the sitemap definition. Without --append the +// definition keeps its non-url items first and the imported URLs replace +// its url items; with --append the URLs are added after every existing item. +// The definition is always the one /sitemap.xml serves (theme default); +// --theme is accepted for PHP compatibility and a theme other than default +// is reported. +func Import(app *backpack.App) bonfire.Command { + return bonfire.Command{ + Name: "sitemap:import", + Description: "Import sitemap URLs from a JSON manifest file", + Args: []bonfire.Arg{ + {Name: "file", Description: "Path to the JSON manifest (relative to the working directory or absolute)", Required: true}, + }, + Flags: []bonfire.Flag{ + {Name: "theme", Description: "Theme code of the PHP plugin; /sitemap.xml serves only the default definition"}, + {Name: "append", Description: "Add URLs without removing existing url-type items", Bare: true}, + }, + Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { + file, _ := in.Argument("file") + db, done, err := lookupDB(ctx, app) + if err != nil { + return err + } + defer done() + return ImportFile(ctx, db, file, flagValue(in, "theme"), flagOn(in, "append"), out) + }, + } +} + +// ImportFile runs sitemap:import against db. +func ImportFile(ctx context.Context, db *gorm.DB, file, theme string, appendMode bool, out bonfire.Output) error { + raw, err := os.ReadFile(file) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("File not found: %s", file) + } + return fmt.Errorf("Cannot read %s: %w", file, err) + } + entries, labels, ok := orderedEntries(raw) + if !ok { + return errors.New("Invalid JSON: expected an array of URL entries") + } + newItems := make([]models.DefinitionItem, 0, len(entries)) + for i, entry := range entries { + var fields map[string]json.RawMessage + _ = json.Unmarshal(entry, &fields) + url, err := models.DecodeText(fields["url"]) + if fields["url"] == nil || err != nil || phpEmpty(fields["url"]) { + return fmt.Errorf("Entry %s is missing a 'url' field", labels[i]) + } + item := models.DefinitionItem{Type: models.ItemTypeURL, URL: url} + item.Changefreq = textOr(fields["changefreq"], "monthly") + item.Priority = textOr(fields["priority"], "0.5") + newItems = append(newItems, item) + } + var kept int + err = db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + def, err := singletonDefinition(tx) + if err != nil { + return err + } + existing := def.Items() + if !appendMode { + var dynamic []models.DefinitionItem + for _, item := range existing { + if item.Type != models.ItemTypeURL { + dynamic = append(dynamic, item) + } + } + existing = dynamic + } + kept = len(existing) + def.SetItems(append(append([]models.DefinitionItem{}, existing...), newItems...)) + return tx.Save(&def).Error + }) + if err != nil { + return err + } + out.Info(fmt.Sprintf("Imported %d URLs into sitemap definition for theme '%s'.", len(newItems), models.DefinitionTheme)) + if kept > 0 { + label := "existing non-URL (dynamic)" + if appendMode { + label = "existing" + } + out.Info(fmt.Sprintf("Kept %d %s items.", kept, label)) + } + if theme != "" && theme != models.DefinitionTheme { + out.Warning(fmt.Sprintf("Theme '%s' was given, but /sitemap.xml serves only the '%s' definition, so the URLs were imported there.", theme, models.DefinitionTheme)) + } + return nil +} + +// orderedEntries splits a JSON array, or a JSON object in key order (a PHP +// array with keys), into its values with their PHP labels (index or key). +func orderedEntries(raw []byte) ([]json.RawMessage, []string, bool) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return nil, nil, false + } + switch raw[0] { + case '[': + var list []json.RawMessage + if err := json.Unmarshal(raw, &list); err != nil { + return nil, nil, false + } + labels := make([]string, len(list)) + for i := range list { + labels[i] = fmt.Sprint(i) + } + return list, labels, true + case '{': + dec := json.NewDecoder(bytes.NewReader(raw)) + if _, err := dec.Token(); err != nil { + return nil, nil, false + } + var list []json.RawMessage + var labels []string + for dec.More() { + tok, err := dec.Token() + if err != nil { + return nil, nil, false + } + key, _ := tok.(string) + var v json.RawMessage + if err := dec.Decode(&v); err != nil { + return nil, nil, false + } + list = append(list, v) + labels = append(labels, key) + } + return list, labels, true + } + return nil, nil, false +} + +// phpEmpty is PHP empty() of a decoded JSON value. +func phpEmpty(raw json.RawMessage) bool { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return true + } + switch t := v.(type) { + case nil: + return true + case string: + return t == "" || t == "0" + case bool: + return !t + case float64: + return t == 0 + case []any: + return len(t) == 0 + case map[string]any: + return len(t) == 0 + } + return false +} + +// textOr is PHP `$entry[key] ?? fallback` as text: a missing or null value +// is the fallback. +func textOr(raw json.RawMessage, fallback string) *string { + if raw != nil { + if s, err := models.DecodeText(raw); err == nil && s != nil { + return s + } + } + return &fallback +} diff --git a/console/import_winter.go b/console/import_winter.go new file mode 100644 index 0000000..f84165e --- /dev/null +++ b/console/import_winter.go @@ -0,0 +1,407 @@ +package console + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "slices" + "sort" + "strings" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" + "git.golem15.com/golem15/summercms/modules/signpost" + "gorm.io/gorm" +) + +// droppedTypes are the WinterCMS item types the port does not support: +// Winter.Pages static pages and CMS pages. +var droppedTypes = map[string]bool{ + "static-page": true, + "all-static-pages": true, + "cms-page": true, +} + +// WinterOptions are the inputs of sitemap:import-winter. DefinitionsFile is +// the output of scripts/winter-export-definitions.sql (one JSON object per +// line: id, theme, data, created_at, updated_at); SettingsFile the output +// of scripts/winter-export-settings.sql (the JSON object of the +// golem15_sitemap_settings system setting). Theme picks a definition when +// the export holds several. Maps rename item types (old to new) before they +// are checked. Force replaces a definition that has items; DryRun reports +// without writing. +type WinterOptions struct { + DefinitionsFile string + SettingsFile string + Theme string + Maps map[string]string + Force bool + DryRun bool +} + +// WinterReport is what sitemap:import-winter did: the items kept, the +// report lines of every dropped or remapped item, and whether the settings +// were copied. +type WinterReport struct { + Kept int + Dropped int + Remapped int + Lines []string + Settings bool +} + +// ImportWinter is sitemap:import-winter: it copies a WinterCMS site's +// sitemap definition and settings into the definition /sitemap.xml serves +// (theme default) and the settings row. Item types are mapped (url kept, +// static and CMS page types dropped, --map old=new renames applied first), +// an item of a registry type that takes references is dropped when its +// reference is not among the type's current references, an item of an +// unknown type is dropped, and every dropped or remapped item is printed. A +// definition that already has items is refused without --force. Settings +// keys missing from the export take the PHP defaults. +func ImportWinter(app *backpack.App) bonfire.Command { + return bonfire.Command{ + Name: "sitemap:import-winter", + Description: "Copy a WinterCMS sitemap definition and settings export into this application", + Flags: []bonfire.Flag{ + {Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"}, + {Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"}, + {Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"}, + {Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true}, + {Name: "force", Description: "Replace a definition that already has items", Bare: true}, + {Name: "dry-run", Description: "Report what would be copied without writing", Bare: true}, + }, + Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { + maps, err := parseMaps(in.Flags("map")) + if err != nil { + return err + } + opts := WinterOptions{ + DefinitionsFile: flagValue(in, "definitions"), + SettingsFile: flagValue(in, "settings"), + Theme: flagValue(in, "theme"), + Maps: maps, + Force: flagOn(in, "force"), + DryRun: flagOn(in, "dry-run"), + } + db, done, err := lookupDB(ctx, app) + if err != nil { + return err + } + defer done() + var reg *signpost.Registry + if app != nil { + reg, _ = app.Lookup[*signpost.Registry]() + } + _, err = ImportWinterFiles(ctx, db, reg, opts, out) + return err + }, + } +} + +// parseMaps reads --map old=new values. +func parseMaps(values []string) (map[string]string, error) { + maps := map[string]string{} + for _, v := range values { + from, to, ok := strings.Cut(v, "=") + from, to = strings.TrimSpace(from), strings.TrimSpace(to) + if !ok || from == "" { + return nil, fmt.Errorf("--map %q: use old=new", v) + } + maps[from] = to + } + return maps, nil +} + +// winterDefinition is one exported golem15_sitemap_definitions row. +type winterDefinition struct { + ID json.RawMessage `json:"id"` + Theme *string `json:"theme"` + Data json.RawMessage `json:"data"` +} + +// ImportWinterFiles runs sitemap:import-winter against db; reg is the menu +// item type registry the item types are checked against (nil knows none). +func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) { + var report WinterReport + if opts.DefinitionsFile == "" && opts.SettingsFile == "" { + return report, errors.New("give --definitions, --settings or both") + } + var ( + items []models.DefinitionItem + haveDef bool + ) + if opts.DefinitionsFile != "" { + row, err := pickDefinition(opts.DefinitionsFile, opts.Theme) + if err != nil { + return report, err + } + out.Info(fmt.Sprintf("Copying definition %s (theme '%s') into the '%s' definition.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme), models.DefinitionTheme)) + items, err = mapItems(ctx, reg, row.Data, opts.Maps, &report) + if err != nil { + return report, err + } + haveDef = true + } + var settings models.Settings + if opts.SettingsFile != "" { + var err error + settings, err = readWinterSettings(opts.SettingsFile, &report) + if err != nil { + return report, err + } + report.Settings = true + } + for _, line := range report.Lines { + out.Warning(line) + } + if haveDef { + var existing []models.Definition + if err := db.WithContext(ctx).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&existing).Error; err != nil { + return report, err + } + if len(existing) > 0 && len(existing[0].Items()) > 0 && !opts.Force { + msg := fmt.Sprintf("The sitemap definition already has %d items; rerun with --force to replace them.", len(existing[0].Items())) + if !opts.DryRun { + return report, errors.New(msg) + } + out.Warning(msg) + } + } + summary := fmt.Sprintf("%d items kept, %d remapped, %d dropped.", report.Kept, report.Remapped, report.Dropped) + if opts.DryRun { + if haveDef { + out.Info("Dry run: " + summary) + } + if report.Settings { + out.Info("Dry run: the settings would be copied.") + } + out.Info("Dry run complete (no changes made).") + return report, nil + } + err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if haveDef { + def, err := singletonDefinition(tx) + if err != nil { + return err + } + def.SetItems(items) + if err := tx.Save(&def).Error; err != nil { + return err + } + } + if report.Settings { + settings.ID = 1 + if err := tx.Save(&settings).Error; err != nil { + return err + } + } + return nil + }) + if err != nil { + return report, err + } + if haveDef { + out.Info("Definition copied: " + summary) + } + if report.Settings { + out.Info("Settings copied.") + } + return report, nil +} + +// pickDefinition reads the definitions export and returns the row of theme, +// or the only row when theme is empty. +func pickDefinition(file, theme string) (winterDefinition, error) { + f, err := os.Open(file) + if err != nil { + return winterDefinition{}, fmt.Errorf("File not found: %s", file) + } + defer f.Close() + var rows []winterDefinition + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 1<<20), 64<<20) + n := 0 + for scanner.Scan() { + n++ + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + var row winterDefinition + if err := json.Unmarshal(line, &row); err != nil { + return winterDefinition{}, fmt.Errorf("%s line %d: %w", file, n, err) + } + rows = append(rows, row) + } + if err := scanner.Err(); err != nil { + return winterDefinition{}, err + } + if len(rows) == 0 { + return winterDefinition{}, fmt.Errorf("%s holds no definition", file) + } + if theme != "" { + for _, row := range rows { + if models.Text(row.Theme) == theme { + return row, nil + } + } + return winterDefinition{}, fmt.Errorf("%s holds no definition of theme '%s'", file, theme) + } + if len(rows) > 1 { + themes := make([]string, 0, len(rows)) + for _, row := range rows { + themes = append(themes, "'"+models.Text(row.Theme)+"'") + } + return winterDefinition{}, fmt.Errorf("%s holds %d definitions (themes %s); pick one with --theme", file, len(rows), strings.Join(themes, ", ")) + } + return rows[0], nil +} + +// exportedJSON is a JSON value that the export may hold as text (a TEXT +// column selected as it is) or as JSON: a string is decoded once more. +func exportedJSON(raw json.RawMessage) json.RawMessage { + raw = bytes.TrimSpace(raw) + if len(raw) > 0 && raw[0] == '"' { + var text string + if json.Unmarshal(raw, &text) == nil { + return json.RawMessage(strings.TrimSpace(text)) + } + } + return raw +} + +// mapItems decodes the exported data and maps its items. +func mapItems(ctx context.Context, reg *signpost.Registry, data json.RawMessage, maps map[string]string, report *WinterReport) ([]models.DefinitionItem, error) { + data = exportedJSON(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + return []models.DefinitionItem{}, nil + } + rawItems, labels, ok := orderedEntries(data) + if !ok { + return nil, errors.New("the definition data is not a list of items") + } + out := []models.DefinitionItem{} + refs := map[string][]string{} + for i, raw := range rawItems { + var item models.DefinitionItem + if err := json.Unmarshal(raw, &item); err != nil { + return nil, fmt.Errorf("definition item %s: %w", labels[i], err) + } + var fields map[string]any + _ = json.Unmarshal(raw, &fields) + label := describeItem(labels[i], item) + if page, ok := fields["cmsPage"].(string); ok && page != "" { + report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, page)) + } + if to, ok := maps[item.Type]; ok && to != item.Type { + report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to)) + report.Remapped++ + item.Type = to + if to == "" { + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) + report.Dropped++ + continue + } + } + if item.Type == models.ItemTypeURL { + out = append(out, item) + report.Kept++ + continue + } + if droppedTypes[item.Type] { + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label)) + report.Dropped++ + continue + } + if _, known := reg.Type(item.Type); !known { + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (type '%s' is not registered; map it with --map %s=)", label, item.Type, item.Type)) + report.Dropped++ + continue + } + values, cached := refs[item.Type] + if !cached { + set, err := signpost.ReferenceOptions(ctx, reg, item.Type) + if err != nil { + return nil, err + } + if set.Applies { + values = []string{} + for _, o := range set.Options { + values = append(values, o.Value) + } + } + refs[item.Type] = values + } + if values != nil && !slices.Contains(values, models.Text(item.Reference)) { + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (reference '%s' is not among the current references of '%s')", label, models.Text(item.Reference), item.Type)) + report.Dropped++ + continue + } + out = append(out, item) + report.Kept++ + } + return out, nil +} + +func describeItem(label string, item models.DefinitionItem) string { + s := fmt.Sprintf("item %s (type '%s'", label, item.Type) + if item.URL != nil && *item.URL != "" { + s += fmt.Sprintf(", url '%s'", *item.URL) + } + if item.Reference != nil && *item.Reference != "" { + s += fmt.Sprintf(", reference '%s'", *item.Reference) + } + return s + ")" +} + +// readWinterSettings reads the exported settings object onto the PHP +// defaults. +func readWinterSettings(file string, report *WinterReport) (models.Settings, error) { + raw, err := os.ReadFile(file) + if err != nil { + return models.Settings{}, fmt.Errorf("File not found: %s", file) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(exportedJSON(raw), &fields); err != nil { + return models.Settings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) + } + s := models.DefaultSettings() + texts := map[string]**string{ + "frontend_url": &s.FrontendURL, + "frontend_default_locale": &s.FrontendDefaultLocale, + "updates_manifest_path": &s.UpdatesManifestPath, + models.PriorityHomepage: &s.PriorityHomepage, + models.PriorityServices: &s.PriorityServices, + models.PriorityBlog: &s.PriorityBlog, + models.PriorityCategories: &s.PriorityCategories, + models.PriorityLegal: &s.PriorityLegal, + } + keys := make([]string, 0, len(fields)) + for key := range fields { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := fields[key] + if dest, ok := texts[key]; ok { + text, err := models.DecodeText(value) + if err != nil { + return models.Settings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) + } + *dest = text + continue + } + if key == "enable_priority_tiers" { + s.EnablePriorityTiers = !phpEmpty(value) + continue + } + report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) + } + return s, nil +} diff --git a/models/definition_item.go b/models/definition_item.go index 24a0d42..a22c058 100644 --- a/models/definition_item.go +++ b/models/definition_item.go @@ -60,7 +60,7 @@ func (d *DefinitionItem) UnmarshalJSON(raw []byte) error { } item := DefinitionItem{Type: ItemTypeURL} if v, ok := fields["type"]; ok { - s, err := decodeText(v) + s, err := DecodeText(v) if err != nil { return fmt.Errorf("sitemap definition item type: %w", err) } @@ -77,7 +77,7 @@ func (d *DefinitionItem) UnmarshalJSON(raw []byte) error { if !ok { continue } - s, err := decodeText(v) + s, err := DecodeText(v) if err != nil { return fmt.Errorf("sitemap definition item %s: %w", key, err) } @@ -185,9 +185,9 @@ func (l DefinitionItems) MarshalJSON() ([]byte, error) { return json.Marshal([]DefinitionItem(l)) } -// decodeText decodes a stored text value: a string as it is, a number in +// DecodeText decodes a stored text value: a string as it is, a number in // its PHP string form, a boolean as PHP casts it ("1" or ""), null as nil. -func decodeText(raw json.RawMessage) (*string, error) { +func DecodeText(raw json.RawMessage) (*string, error) { var v any dec := json.NewDecoder(bytes.NewReader(raw)) dec.UseNumber() diff --git a/models/settings.go b/models/settings.go new file mode 100644 index 0000000..d834a92 --- /dev/null +++ b/models/settings.go @@ -0,0 +1,87 @@ +package models + +// The settings keys of the five priority tiers, in the order PHP checked +// them. A tier rule names one of them. +const ( + PriorityHomepage = "priority_homepage" + PriorityServices = "priority_services" + PriorityBlog = "priority_blog" + PriorityCategories = "priority_categories" + PriorityLegal = "priority_legal" +) + +// PriorityDefaults are the PHP defaults of the priority settings. +var PriorityDefaults = map[string]string{ + PriorityHomepage: "1.0", + PriorityServices: "0.8", + PriorityBlog: "0.6", + PriorityCategories: "0.4", + PriorityLegal: "0.2", +} + +// Settings is the sitemap settings singleton (row id 1) on +// golem15_sitemap_settings, the PHP Golem15\Sitemap\Models\Settings that +// WinterCMS kept in system_settings under golem15_sitemap_settings. +// FrontendURL replaces the backend URL in the generated XML; +// FrontendDefaultLocale is the locale whose URLs carry no prefix; +// UpdatesManifestPath names a JSON file of URL path to last-modified date; +// EnablePriorityTiers turns the tier priorities on, and the priority fields +// hold each tier's value as text. A null priority reads as its PHP default. +type Settings struct { + ID uint `gorm:"column:id;primaryKey"` + FrontendURL *string `gorm:"column:frontend_url"` + FrontendDefaultLocale *string `gorm:"column:frontend_default_locale"` + UpdatesManifestPath *string `gorm:"column:updates_manifest_path"` + EnablePriorityTiers bool `gorm:"column:enable_priority_tiers"` + PriorityHomepage *string `gorm:"column:priority_homepage"` + PriorityServices *string `gorm:"column:priority_services"` + PriorityBlog *string `gorm:"column:priority_blog"` + PriorityCategories *string `gorm:"column:priority_categories"` + PriorityLegal *string `gorm:"column:priority_legal"` +} + +func (Settings) TableName() string { return "golem15_sitemap_settings" } + +// Fillable lists only the settings columns. +func (Settings) Fillable() []string { + return []string{ + "frontend_url", "frontend_default_locale", "updates_manifest_path", + "enable_priority_tiers", PriorityHomepage, PriorityServices, + PriorityBlog, PriorityCategories, PriorityLegal, + } +} + +// Rules is empty: the PHP settings form validated nothing. +func (Settings) Rules() map[string]string { return map[string]string{} } + +// DefaultSettings is the settings PHP read when nothing was saved: tiers on +// with the default priorities. +func DefaultSettings() Settings { + return Settings{ID: 1, EnablePriorityTiers: true} +} + +// Priority returns the value of a priority setting: the stored text, or the +// PHP default when it is null. +func (s Settings) Priority(key string) string { + var p *string + switch key { + case PriorityHomepage: + p = s.PriorityHomepage + case PriorityServices: + p = s.PriorityServices + case PriorityBlog: + p = s.PriorityBlog + case PriorityCategories: + p = s.PriorityCategories + case PriorityLegal: + p = s.PriorityLegal + } + if p == nil { + return PriorityDefaults[key] + } + return *p +} + +func init() { + Register(Settings{}) +} diff --git a/plugin.go b/plugin.go index 0b4934e..fc942bd 100644 --- a/plugin.go +++ b/plugin.go @@ -5,9 +5,16 @@ package sitemap import ( + "embed" + "fmt" + "io/fs" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/sm-sitemap-plugin/console" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/sm-sitemap-plugin/updates" "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" "github.com/go-gormigrate/gormigrate/v2" @@ -15,14 +22,25 @@ import ( var ( _ party.Plugin = (*Plugin)(nil) + _ pact.HasConfig = (*Plugin)(nil) _ pact.HasMigrations = (*Plugin)(nil) _ pact.HasModels = (*Plugin)(nil) _ pact.HasRoutes = (*Plugin)(nil) + _ pact.HasCommands = (*Plugin)(nil) ) +//go:embed config +var configFS embed.FS + +// sitemapXSL is the PHP plugin's assets/sitemap.xsl, byte for byte. +// +//go:embed assets/sitemap.xsl +var sitemapXSL []byte + // Plugin is the golem15.sitemap plugin. type Plugin struct { - app *backpack.App + app *backpack.App + tiers classes.Tiers } func (p *Plugin) ID() string { return "golem15.sitemap" } @@ -36,11 +54,33 @@ func (p *Plugin) Register(app *backpack.App) error { return nil } +// Boot compiles the priority tier rules (classes.TiersConfigKey); a rule +// with an unknown setting or a pattern that does not compile fails the boot. func (p *Plugin) Boot(app *backpack.App) error { p.app = app + var tiers classes.Tiers + var err error + if app != nil { + tiers, err = classes.LoadTiers(app.Config) + } + if err != nil { + return fmt.Errorf("golem15.sitemap: %w", err) + } + p.tiers = tiers return nil } +// ConfigFS ships config/config.yaml: the default priority tier rules. +func (p *Plugin) ConfigFS() fs.FS { return configFS } + +// Commands registers sitemap:import and sitemap:import-winter. +func (p *Plugin) Commands() []bonfire.Command { + return []bonfire.Command{ + console.Import(p.app), + console.ImportWinter(p.app), + } +} + func (p *Plugin) Migrations() []*gormigrate.Migration { return updates.All() } func (p *Plugin) Models() []any { return models.All() } diff --git a/routes.go b/routes.go index 728cd86..f9ac4eb 100644 --- a/routes.go +++ b/routes.go @@ -20,20 +20,30 @@ import ( // The response headers of the PHP routes (live responses). const ( contentTypeXML = "application/xml" + contentTypeXSL = "text/xsl; charset=UTF-8" cacheControl = "no-cache, private" ) -// Routes mounts GET /sitemap.xml at the application root, outside any -// throttle group and without the backend middleware: search engines fetch it -// anonymously. +// Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application +// root, outside any throttle group and without the backend middleware: +// search engines and browsers fetch them anonymously. func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapXML) + r.Get("/sitemap.xsl", p.sitemapXSL) return nil } -// sitemapXML serves the definition with the fixed theme key (D-11): 404 -// when there is none, an empty application/xml body when it has no items, -// else the generated XML. +// sitemapXSL serves the embedded stylesheet, the PHP file byte for byte. +func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeXSL) + w.Header().Set("Cache-Control", cacheControl) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(sitemapXSL) +} + +// sitemapXML serves the definition with the fixed theme key: 404 when there +// is none, an empty application/xml body when it has no items, else the +// generated XML with the settings' frontend_url rewrite applied. func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) { app := p.app db, err := database(app) @@ -52,7 +62,14 @@ func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - gen := &classes.Generator{Root: backendURL(app, r)} + settings, err := loadSettings(db.WithContext(r.Context())) + if err != nil { + slog.Default().Error("golem15.sitemap: load settings", "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + root := backendURL(app, r) + gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events} if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil { gen.Resolver = reg } @@ -62,12 +79,25 @@ func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } + xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL)) w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(xml)) } +// loadSettings reads the settings row; without one the PHP defaults apply. +func loadSettings(db *gorm.DB) (models.Settings, error) { + var rows []models.Settings + if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { + return models.Settings{}, err + } + if len(rows) == 0 { + return models.DefaultSettings(), nil + } + return rows[0], nil +} + // database returns the app's published *gorm.DB. func database(app *backpack.App) (*gorm.DB, error) { if app == nil { diff --git a/scripts/winter-export-definitions.sql b/scripts/winter-export-definitions.sql new file mode 100644 index 0000000..6014361 --- /dev/null +++ b/scripts/winter-export-definitions.sql @@ -0,0 +1,27 @@ +-- Sitemap definitions of a WinterCMS site with the Golem15.Sitemap plugin, +-- one JSON object per line: the --definitions input of +-- sitemap:import-winter. +-- +-- mysql -N -B --raw < winter-export-definitions.sql > sitemap-definitions.jsonl +-- +-- Read-only: it selects from the definitions table and changes nothing. +-- JSON_OBJECT needs MySQL 5.7.8+ or MariaDB 10.2.3+. +-- +-- Check the table name on the server before running: +-- SHOW TABLES LIKE '%sitemap_definitions'; +-- This file assumes golem15_sitemap_definitions. A site still on the +-- Winter.Sitemap or RainLab.Sitemap plugin uses winter_sitemap_definitions +-- or rainlab_sitemap_definitions; change the FROM line to match. +-- +-- data is the stored item list as text; the command decodes it. A site with +-- several themes exports one row per theme; pass --theme to the command to +-- pick the one to copy. +SELECT JSON_OBJECT( + 'id', d.id, + 'theme', d.theme, + 'data', d.data, + 'created_at', d.created_at, + 'updated_at', d.updated_at +) +FROM golem15_sitemap_definitions d +ORDER BY d.id; diff --git a/scripts/winter-export-settings.sql b/scripts/winter-export-settings.sql new file mode 100644 index 0000000..f5f0a17 --- /dev/null +++ b/scripts/winter-export-settings.sql @@ -0,0 +1,16 @@ +-- Sitemap settings of a WinterCMS site with the Golem15.Sitemap plugin: the +-- JSON object the plugin keeps in system_settings, the --settings input of +-- sitemap:import-winter. +-- +-- mysql -N -B --raw < winter-export-settings.sql > sitemap-settings.json +-- +-- Read-only: it selects one system_settings value and changes nothing. +-- +-- Check that the row exists before running: +-- SELECT item FROM system_settings WHERE item LIKE '%sitemap%'; +-- An empty result means the settings were never saved; skip --settings and +-- the command keeps the defaults. +SELECT s.value +FROM system_settings s +WHERE s.item = 'golem15_sitemap_settings' +LIMIT 1; diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go index 2ea4bb4..79884d5 100644 --- a/sitemap_smoke_test.go +++ b/sitemap_smoke_test.go @@ -1,21 +1,28 @@ package sitemap_test import ( + "bytes" "context" + "encoding/json" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "testing" "time" _ "git.golem15.com/golem15/sm-sitemap-plugin" + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/sm-sitemap-plugin/console" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/compass" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" + "git.golem15.com/golem15/summercms/modules/signpost" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) @@ -35,7 +42,7 @@ func (acmeTypes) Register(*backpack.App) error { return nil } func (acmeTypes) Boot(*backpack.App) error { return nil } func (acmeTypes) MenuItemTypes() []pact.MenuItemType { mtime := time.Date(2026, 3, 4, 10, 0, 0, 0, time.UTC) - return []pact.MenuItemType{{ + return []pact.MenuItemType{acmeList(), { Key: "acme-post", Label: "Acme post", DefaultPattern: "/news/:slug", @@ -59,6 +66,28 @@ func (acmeTypes) MenuItemTypes() []pact.MenuItemType { }} } +// acmeList is a list type of the fake plugin: no URL of its own, one post +// with two alternates as its item. +func acmeList() pact.MenuItemType { + return pact.MenuItemType{ + Key: "acme-list", + Label: "Acme posts", + DefaultPattern: "/news/:slug", + PatternFrom: "acme-post", + DynamicItems: true, + Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { + return &pact.ResolvedMenuItem{Items: []pact.ResolvedMenuItem{{ + Title: "A", + URL: "/news/a", + Alternates: []pact.MenuItemAlternate{ + {Locale: "pl", URL: "/news/a"}, + {Locale: "en", URL: "/en/news/a"}, + }, + }}}, nil + }, + } +} + var _ pact.HasMenuItemTypes = acmeTypes{} func init() { @@ -187,3 +216,146 @@ func TestSitemapTracer(t *testing.T) { t.Fatalf("stored data = %s", raw) } } + +func writeFile(t *testing.T, dir, name, body string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func testOutput() (bonfire.Output, *bytes.Buffer) { + var buf bytes.Buffer + return bonfire.NewOutput(strings.NewReader(""), &buf, &buf), &buf +} + +// TestSitemapImportSmoke copies a Winter export through sitemap:import-winter, +// then checks the frontend_url rewrite, the stylesheet route, a +// BeforeAddItem listener and sitemap:import. +func TestSitemapImportSmoke(t *testing.T) { + e := bootSitemap(t) + ctx := context.Background() + reg, ok := e.app.Lookup[*signpost.Registry]() + if !ok { + t.Fatal("signpost registry is not published") + } + dir := t.TempDir() + data := `[{"changefreq":"always","priority":"1.0","nesting":"0","type":"url","url":"/","code":null,"reference":null,"cmsPage":null,"urlPattern":null,"exists":false,"fillable":["changefreq"]},` + + `{"changefreq":"monthly","priority":"0.5","nesting":"0","type":"static-page","url":null,"reference":"about","cmsPage":null,"urlPattern":null},` + + `{"changefreq":"weekly","priority":"0.4","nesting":"1","type":"blog-posts","url":null,"reference":null,"cmsPage":"blog/post","urlPattern":"/news/:slug"},` + + `{"changefreq":"weekly","priority":"0.4","nesting":"0","type":"acme-post","url":null,"reference":"99","cmsPage":null,"urlPattern":null}]` + quotedRaw, _ := json.Marshal(data) + quoted := string(quotedRaw) + defs := writeFile(t, dir, "definitions.jsonl", `{"id": 3, "theme": "site-theme", "data": `+quoted+`, "created_at": "2026-01-01 10:00:00", "updated_at": "2026-01-02 10:00:00"}`+"\n") + settings := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test/","frontend_default_locale":"pl","enable_priority_tiers":"0","priority_homepage":"0.9","legacy_key":"x"}`) + + out, buf := testOutput() + report, err := console.ImportWinterFiles(ctx, e.gdb, reg, console.WinterOptions{ + DefinitionsFile: defs, SettingsFile: settings, Maps: map[string]string{"blog-posts": "acme-list"}, + }, out) + if err != nil { + t.Fatalf("import-winter: %v\n%s", err, buf.String()) + } + if report.Kept != 2 || report.Dropped != 2 || report.Remapped != 1 || !report.Settings { + t.Fatalf("report = %+v\n%s", report, buf.String()) + } + for _, want := range []string{ + "item 1 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", + "item 2 (type 'blog-posts'): type 'blog-posts' mapped to 'acme-list'", + "item 2 (type 'blog-posts'): cmsPage 'blog/post' dropped", + "item 3 (type 'acme-post', reference '99'): dropped (reference '99' is not among the current references of 'acme-post')", + "setting 'legacy_key' ignored", + "theme 'site-theme'", + } { + if !strings.Contains(buf.String(), want) { + t.Errorf("report lacks %q:\n%s", want, buf.String()) + } + } + var def models.Definition + if err := e.gdb.Where("theme = ?", models.DefinitionTheme).First(&def).Error; err != nil { + t.Fatal(err) + } + if items := def.Items(); len(items) != 2 || items[0].Type != "url" || items[1].Type != "acme-list" || models.Text(items[1].URLPattern) != "/news/:slug" { + t.Fatalf("copied items = %+v", items) + } + var st models.Settings + if err := e.gdb.First(&st, 1).Error; err != nil { + t.Fatal(err) + } + if models.Text(st.FrontendURL) != "https://www.example.test/" || st.EnablePriorityTiers || st.Priority(models.PriorityHomepage) != "0.9" || st.Priority(models.PriorityBlog) != "0.6" { + t.Fatalf("copied settings = %+v", st) + } + + // A second copy without --force is refused. + out, buf = testOutput() + if _, err := console.ImportWinterFiles(ctx, e.gdb, reg, console.WinterOptions{DefinitionsFile: defs}, out); err == nil || !strings.Contains(err.Error(), "--force") { + t.Fatalf("second import: %v\n%s", err, buf.String()) + } + + // The frontend URL replaces the backend URL in loc, hreflang and the + // stylesheet href; the list type expands per alternate. + rec := e.get(t, "/sitemap.xml") + body := rec.Body.String() + if rec.Code != http.StatusOK || rec.Header().Get("Cache-Control") != "no-cache, private" { + t.Fatalf("GET /sitemap.xml: %d %v", rec.Code, rec.Header()) + } + for _, want := range []string{ + `href="https://www.example.test/sitemap.xsl"`, + `https://www.example.test/`, + `https://www.example.test/news/a`, + `https://www.example.test/en/news/a`, + `hreflang="en" href="https://www.example.test/en/news/a"/>`, + } { + if !strings.Contains(body, want) { + t.Errorf("sitemap lacks %s:\n%s", want, body) + } + } + if strings.Contains(body, "api.example.test") { + t.Errorf("the backend URL is left in the sitemap:\n%s", body) + } + + // /sitemap.xsl answers the PHP file. + rec = e.get(t, "/sitemap.xsl") + want, err := os.ReadFile(filepath.Join("assets", "sitemap.xsl")) + if err != nil { + t.Fatal(err) + } + if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "text/xsl; charset=UTF-8" || !bytes.Equal(rec.Body.Bytes(), want) { + t.Fatalf("GET /sitemap.xsl: %d %q (%d bytes)", rec.Code, rec.Header().Get("Content-Type"), rec.Body.Len()) + } + + // A BeforeAddItem listener that sets Skip drops its URL. + e.app.Events.Listen("acme.skip", func(_ context.Context, ev *classes.BeforeAddItem) error { + if ev.Info.URL == "/en/news/a" { + ev.Skip = true + } + return nil + }) + body = e.get(t, "/sitemap.xml").Body.String() + if strings.Contains(body, "https://www.example.test/en/news/a") || !strings.Contains(body, "https://www.example.test/news/a") { + t.Fatalf("the skipped URL is still written:\n%s", body) + } + + // sitemap:import replaces the url items and keeps the dynamic ones first. + manifest := writeFile(t, dir, "urls.json", `[{"url":"/x","changefreq":"daily"},{"url":"/y","priority":"0.9"}]`) + out, buf = testOutput() + if err := console.ImportFile(ctx, e.gdb, manifest, "", false, out); err != nil { + t.Fatalf("sitemap:import: %v", err) + } + if err := e.gdb.Where("theme = ?", models.DefinitionTheme).First(&def).Error; err != nil { + t.Fatal(err) + } + items := def.Items() + if len(items) != 3 || items[0].Type != "acme-list" || models.Text(items[1].URL) != "/x" || models.Text(items[1].Priority) != "0.5" || models.Text(items[2].Changefreq) != "monthly" { + t.Fatalf("imported items = %+v", items) + } + if !strings.Contains(buf.String(), "Imported 2 URLs into sitemap definition for theme 'default'.") || !strings.Contains(buf.String(), "Kept 1 existing non-URL (dynamic) items.") { + t.Fatalf("sitemap:import output:\n%s", buf.String()) + } + bad := writeFile(t, dir, "bad.json", `[{"url":"/x"},{"changefreq":"daily"}]`) + if err := console.ImportFile(ctx, e.gdb, bad, "", true, out); err == nil || err.Error() != "Entry 1 is missing a 'url' field" { + t.Fatalf("missing url: %v", err) + } +} diff --git a/tests/php/gen-goldens.php b/tests/php/gen-goldens.php index 67d73b1..c9c0c91 100644 --- a/tests/php/gen-goldens.php +++ b/tests/php/gen-goldens.php @@ -33,8 +33,8 @@ * A registry type's resolved item is given in the Go shape (one item per * record with ordered alternates); this script turns it into what the PHP * headless resolvers returned: a list type (no url) yields one entry per - * alternate for each child, any other type its children with their default - * URL. + * alternate for each child (a child without alternates once), any other type + * its children with their default URL. */ $vendor = getenv('WINTER_VENDOR'); @@ -194,7 +194,7 @@ class GoldenSitemap $info['items'] = []; foreach ($r['items'] as $child) { $childInfo = $this->phpResolved($child, false); - if ($listChildren) { + if ($listChildren && $childInfo['alternateLinks']) { foreach ($childInfo['alternateLinks'] as $locale => $href) { $perLocale = $childInfo; $perLocale['url'] = $href; diff --git a/updates/202610110002_create_golem15_sitemap_settings.go b/updates/202610110002_create_golem15_sitemap_settings.go new file mode 100644 index 0000000..f207bd6 --- /dev/null +++ b/updates/202610110002_create_golem15_sitemap_settings.go @@ -0,0 +1,44 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// createSettings creates the settings singleton golem15_sitemap_settings +// (row id 1). WinterCMS kept these values in system_settings under +// golem15_sitemap_settings; sitemap:import-winter copies them here. +var createSettings = []*gormigrate.Migration{ + { + ID: "202610110002_create_golem15_sitemap_settings", + Migrate: func(tx *gorm.DB) error { + for _, stmt := range []string{ + `CREATE TABLE golem15_sitemap_settings ( + id INTEGER PRIMARY KEY, + frontend_url TEXT NULL, + frontend_default_locale TEXT NULL, + updates_manifest_path TEXT NULL, + enable_priority_tiers BOOLEAN NOT NULL DEFAULT TRUE, + priority_homepage TEXT NULL DEFAULT '1.0', + priority_services TEXT NULL DEFAULT '0.8', + priority_blog TEXT NULL DEFAULT '0.6', + priority_categories TEXT NULL DEFAULT '0.4', + priority_legal TEXT NULL DEFAULT '0.2' +)`, + `INSERT INTO golem15_sitemap_settings (id) VALUES (1)`, + } { + if err := tx.Exec(stmt).Error; err != nil { + return err + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec(`DROP TABLE IF EXISTS golem15_sitemap_settings`).Error + }, + }, +} + +func init() { + Register(createSettings...) +}