diff --git a/assets/sitemap.xsl b/assets/sitemap.xsl deleted file mode 100644 index c6b6bbb..0000000 --- a/assets/sitemap.xsl +++ /dev/null @@ -1,261 +0,0 @@ - - - - - - - - - - Sitemap - <xsl:if test="sitemap:sitemapindex">Index</xsl:if> - - - - -
-
-

Sitemap

- - Index - - - Images - - - Video - - - Xhtml - -
-

- - - This index contains - - sitemaps. - - - This index contains - - URLs. - - -

-

- This is an XML sitemap, meant for consumption by search engines. - You can find more information about XML sitemaps on sitemaps.org. -

-
- - - -
-
- - - - - - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
URLLast Modified
- - - - - - - -
-
-
- - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
URLChange Freq.PriorityLast Modified
- - -

- - - -

- - - - 		- -
-
-
- - - - - - - - - - - - - - - -

- - - - - - - - - - - - - - - - - - - - - -

- - - - - - - -

- Image: - - - - - - - - - -

- - - - - - - - - - - - - - - - -

- Video: - - - - - - - - - - - - thumb - - - - - - - -

- - - diff --git a/classes/events.go b/classes/events.go deleted file mode 100644 index 15d41cd..0000000 --- a/classes/events.go +++ /dev/null @@ -1,56 +0,0 @@ -package classes - -import ( - "time" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// ItemInfo is what the sitemap knows about one URL it is about to write, the -// PHP $itemInfo array: URL (before the final Url::to in BeforeAddItem, the -// written loc in AddItem), MTime, the record's modification time when the -// item has one, Lastmod, the written lastmod ("" when omitted), and the -// hreflang Alternates. -type ItemInfo struct { - URL string - MTime *time.Time - Lastmod string - Alternates []Alternate -} - -// BeforeAddItem is the typed successor of the PHP event -// golem15.sitemap.beforeAddItem. The generator fires it with -// festival.Bus.UntilHandled for every URL before it is counted and written; -// a listener that sets Skip drops the URL (PHP: a listener returned false). -// -// app.Events.Listen("acme.seo", func(ctx context.Context, e *classes.BeforeAddItem) error { -// if strings.HasSuffix(e.Info.URL, "/draft") { -// e.Skip = true -// } -// return nil -// }) -type BeforeAddItem struct { - Item models.DefinitionItem - Info ItemInfo - Skip bool -} - -// IsHandled reports Skip, so UntilHandled stops at the first listener that -// drops the URL (festival.Handleable). -func (e *BeforeAddItem) IsHandled() bool { return e != nil && e.Skip } - -// AddItem is the typed successor of the PHP event golem15.sitemap.addItem. -// The generator fires it with festival.Bus.Fire after a URL's elements are -// built; listeners may append child XML to Extra, which is written as it is, -// before the url closing tag, so it must be escaped already (EscapeText, -// EscapeAttr). -// -// app.Events.Listen("acme.images", func(ctx context.Context, e *classes.AddItem) error { -// e.Extra = append(e.Extra, ""+classes.EscapeText(img)+"") -// return nil -// }) -type AddItem struct { - Item models.DefinitionItem - Info ItemInfo - Extra []string -} diff --git a/classes/generator.go b/classes/generator.go deleted file mode 100644 index 8960c9a..0000000 --- a/classes/generator.go +++ /dev/null @@ -1,268 +0,0 @@ -// Package classes generates the sitemap: it walks the definition items, -// resolves registry items through the framework's menu item type registry -// and writes the PHP-identical XML. -package classes - -import ( - "context" - "time" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/festival" - "git.golem15.com/golem15/summercms/modules/pact" -) - -// MaxURLs is the sitemap protocol's URL limit (PHP Definition::MAX_URLS): -// the 50000th URL is written, the 50001st is not. -const MaxURLs = 50000 - -// lastmodLayout is PHP date('c') in UTC: "+00:00", never "Z". -const lastmodLayout = "2006-01-02T15:04:05-07:00" - -// Resolver resolves a registry item; *signpost.Registry implements it. -type Resolver interface { - Resolve(ctx context.Context, key, reference, itemPattern string, nesting bool, unprefixedLocale string) (*pact.ResolvedMenuItem, error) -} - -// Generator writes the sitemap of a definition, following PHP -// Definition::generateSitemap and addItemToSet. -// -// Root is the backend root URL (scheme and host without a trailing slash) -// that Url::to builds absolute URLs from. Resolver resolves registry items; -// nil skips them. Settings are the sitemap settings (tiers, priorities, -// frontend default locale, manifest path), Tiers the compiled tier rules, -// and Events the bus BeforeAddItem and AddItem are fired on (nil fires -// nothing). Manifest, when set, is used instead of reading the settings' -// updates_manifest_path. -type Generator struct { - Root string - Resolver Resolver - Settings models.Settings - Tiers Tiers - Events *festival.Bus - Manifest Manifest - - manifestLoaded bool - entries []URLEntry - count int -} - -// itemInfo is PHP's $itemInfo: the URL (set marks it present), the record's -// modification time and the hreflang alternates. -type itemInfo struct { - url string - set bool - mtime *time.Time - alternates []Alternate -} - -// Generate returns the sitemap XML of items, or "" when there are none (PHP -// generateSitemap). A url item is added with its URL. A registry item is -// resolved with its URL pattern, its nesting and the settings' frontend -// default locale as the unprefixed locale; its URL is added when it has one, -// then its sub-items: when the item has no URL of its own (a list type), -// each sub-item is added once per alternate, with that alternate's URL as -// loc and every alternate as hreflang, as the PHP list types returned them; -// otherwise each sub-item is added once with its own URL. A type the -// registry does not know, or a Resolve that finds nothing, adds nothing. -// -// Generate does not apply the frontend_url rewrite; the route does -// (Rewrite). -func (g *Generator) Generate(ctx context.Context, items []models.DefinitionItem) (string, error) { - if len(items) == 0 { - return "", nil - } - if ctx == nil { - ctx = context.Background() - } - g.entries, g.count = nil, 0 - for i := range items { - item := &items[i] - if item.Type == models.ItemTypeURL { - if err := g.addItem(ctx, item, itemInfo{}); err != nil { - return "", err - } - continue - } - resolved, err := g.resolve(ctx, item) - if err != nil { - return "", err - } - if resolved == nil { - continue - } - if resolved.URL != "" { - if err := g.addItem(ctx, item, infoOf(*resolved)); err != nil { - return "", err - } - if err := g.addChildren(ctx, item, resolved.Items); err != nil { - return "", err - } - continue - } - if err := g.addPerLocale(ctx, item, resolved.Items); err != nil { - return "", err - } - } - return Serialize(g.Stylesheet(), g.entries), nil -} - -// Stylesheet is the stylesheet href PHP built: -// Url::buildUrl(Url::current(), ['path' => 'sitemap.xsl']). -func (g *Generator) Stylesheet() string { - return replacePath(URLTo(g.Root, "/sitemap.xml"), "sitemap.xsl") -} - -func (g *Generator) resolve(ctx context.Context, item *models.DefinitionItem) (*pact.ResolvedMenuItem, error) { - if g.Resolver == nil || item.Type == "" { - return nil, nil - } - if reg, ok := g.Resolver.(interface { - Type(string) (pact.MenuItemType, bool) - }); ok { - if _, known := reg.Type(item.Type); !known { - return nil, nil - } - } - return g.Resolver.Resolve(ctx, item.Type, models.Text(item.Reference), models.Text(item.URLPattern), item.Nesting, models.Text(g.Settings.FrontendDefaultLocale)) -} - -// addChildren adds every child with a URL, then its own children. -func (g *Generator) addChildren(ctx context.Context, item *models.DefinitionItem, children []pact.ResolvedMenuItem) error { - for _, child := range children { - if child.URL != "" { - if err := g.addItem(ctx, item, infoOf(child)); err != nil { - return err - } - } - if err := g.addChildren(ctx, item, child.Items); err != nil { - return err - } - } - return nil -} - -// addPerLocale adds the children of a list type: each once per alternate -// (once with its URL when it has no alternates), then its own children. -func (g *Generator) addPerLocale(ctx context.Context, item *models.DefinitionItem, children []pact.ResolvedMenuItem) error { - for _, child := range children { - info := infoOf(child) - if len(child.Alternates) == 0 { - if child.URL != "" { - if err := g.addItem(ctx, item, info); err != nil { - return err - } - } - if err := g.addChildren(ctx, item, child.Items); err != nil { - return err - } - continue - } - for _, alt := range child.Alternates { - perLocale := info - perLocale.url = alt.URL - if perLocale.url != "" { - if err := g.addItem(ctx, item, perLocale); err != nil { - return err - } - } - if err := g.addChildren(ctx, item, child.Items); err != nil { - return err - } - } - } - return nil -} - -func infoOf(r pact.ResolvedMenuItem) itemInfo { - info := itemInfo{url: r.URL, set: true, mtime: r.MTime} - for _, a := range r.Alternates { - info.alternates = append(info.alternates, Alternate{Hreflang: a.Locale, Href: a.URL}) - } - return info -} - -func (g *Generator) manifest() Manifest { - if !g.manifestLoaded { - if g.Manifest == nil { - g.Manifest = LoadManifest(models.Text(g.Settings.UpdatesManifestPath)) - } - g.manifestLoaded = true - } - return g.Manifest -} - -// addItem is PHP Definition::addItemToSet, in its order: the URL cap, a -// missing URL filled through Url::to, lastmod from the record or the -// manifest, BeforeAddItem, an empty URL dropped and the URL passed through -// Url::to again, the count, the tier or item priority, the elements and -// AddItem. Unlike PHP, a URL or alternate that does not come out as an -// http or https URL is dropped. -func (g *Generator) addItem(ctx context.Context, item *models.DefinitionItem, info itemInfo) error { - if g.count >= MaxURLs { - return nil - } - if !info.set { - info.url = URLTo(g.Root, models.Text(item.URL)) - } - var lastmod time.Time - hasLastmod := false - if info.mtime != nil { - lastmod, hasLastmod = *info.mtime, true - } else if t, ok := g.manifest().Lastmod(info.url); ok { - lastmod, hasLastmod = t, true - } - lastmodText := "" - if hasLastmod && lastmod.Unix() != 0 { - lastmodText = lastmod.UTC().Format(lastmodLayout) - } - if g.Events != nil { - before := &BeforeAddItem{Item: *item, Info: ItemInfo{URL: info.url, MTime: info.mtime, Lastmod: lastmodText, Alternates: append([]Alternate(nil), info.alternates...)}} - skipped, err := g.Events.UntilHandled(ctx, before) - if err != nil { - return err - } - if skipped { - return nil - } - } - if info.url == "" || info.url == "0" { - return nil - } - loc := URLTo(g.Root, info.url) - if !httpURL(loc) { - // Only http and https URLs are written: a javascript: or other - // scheme that an import, a registry type or a pattern let through - // would be a clickable link in sitemap.xsl. - return nil - } - g.count++ - priority, ok := g.Tiers.Priority(loc, g.Settings) - if !ok { - priority = models.Text(item.Priority) - } - entry := URLEntry{ - Loc: loc, - Lastmod: lastmodText, - Changefreq: models.Text(item.Changefreq), - Priority: priority, - } - if len(info.alternates) > 1 { - for _, a := range info.alternates { - href := URLTo(g.Root, a.Href) - if !httpURL(href) { - continue - } - entry.Alternates = append(entry.Alternates, Alternate{Hreflang: a.Hreflang, Href: href}) - } - } - if g.Events != nil { - added := &AddItem{Item: *item, Info: ItemInfo{URL: loc, MTime: info.mtime, Lastmod: lastmodText, Alternates: append([]Alternate(nil), entry.Alternates...)}} - if err := g.Events.Fire(ctx, added); err != nil { - return err - } - entry.Extra = added.Extra - } - g.entries = append(g.entries, entry) - return nil -} diff --git a/classes/generator_test.go b/classes/generator_test.go deleted file mode 100644 index 2d28c45..0000000 --- a/classes/generator_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package classes_test - -import ( - "context" - "errors" - "strconv" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/festival" -) - -func urlItems(paths ...string) []models.DefinitionItem { - out := make([]models.DefinitionItem, 0, len(paths)) - for _, p := range paths { - out = append(out, models.DefinitionItem{Type: models.ItemTypeURL, URL: text(p), Changefreq: text("daily"), Priority: text("0.5")}) - } - return out -} - -// TestHooks pins the typed successors of the PHP events (D-14): a -// BeforeAddItem listener that sets Skip drops the URL before it is counted, -// the first skipping listener stops the rest, AddItem listeners see the -// written loc and their Extra is written before the url closing tag in -// order, and a listener error stops the generation. -func TestHooks(t *testing.T) { - bus := festival.New() - var seenBefore, seenAdd, afterSkip []string - bus.Listen("acme.first", func(_ context.Context, e *classes.BeforeAddItem) error { - seenBefore = append(seenBefore, e.Info.URL) - if strings.HasSuffix(e.Info.URL, "/skip") { - e.Skip = true - } - return nil - }) - bus.Listen("acme.second", func(_ context.Context, e *classes.BeforeAddItem) error { - afterSkip = append(afterSkip, e.Info.URL) - return nil - }) - bus.Listen("acme.extra", func(_ context.Context, e *classes.AddItem) error { - seenAdd = append(seenAdd, e.Info.URL) - e.Extra = append(e.Extra, ""+classes.EscapeText(e.Info.URL+"?a&b")+"") - return nil - }) - bus.Listen("acme.more", func(_ context.Context, e *classes.AddItem) error { - e.Extra = append(e.Extra, "") - return nil - }) - gen := &classes.Generator{Root: "https://h.test", Events: bus} - xml, err := gen.Generate(context.Background(), urlItems("/a", "/skip", "/b")) - if err != nil { - t.Fatal(err) - } - if strings.Join(seenBefore, ",") != "https://h.test/a,https://h.test/skip,https://h.test/b" { - // The url item's URL is filled through Url::to before the event. - t.Fatalf("BeforeAddItem saw %v", seenBefore) - } - if strings.Join(afterSkip, ",") != "https://h.test/a,https://h.test/b" { - t.Fatalf("a listener after the skipping one ran for the skipped URL: %v", afterSkip) - } - if strings.Contains(xml, "/skip") { - t.Fatalf("the skipped URL is written:\n%s", xml) - } - if strings.Join(seenAdd, ",") != "https://h.test/a,https://h.test/b" { - t.Fatalf("AddItem saw %v", seenAdd) - } - want := "https://h.test/adaily0.5https://h.test/a?a&b" - if !strings.Contains(xml, want) { - t.Fatalf("extra not written before the closing tag:\n%s", xml) - } - - // A skip on the exact URL the event carries. - bus2 := festival.New() - var second []string - bus2.Listen("acme.skip", func(_ context.Context, e *classes.BeforeAddItem) error { - e.Skip = strings.HasSuffix(e.Info.URL, "/skip") - return nil - }) - bus2.Listen("acme.later", func(_ context.Context, e *classes.BeforeAddItem) error { - second = append(second, e.Info.URL) - return nil - }) - gen = &classes.Generator{Root: "https://h.test", Events: bus2} - xml, err = gen.Generate(context.Background(), urlItems("/a", "/skip")) - if err != nil { - t.Fatal(err) - } - if strings.Contains(xml, "/skip<") || strings.Join(second, ",") != "https://h.test/a" { - t.Fatalf("UntilHandled did not stop at the skipping listener: %v\n%s", second, xml) - } - - // Without listeners or a bus nothing changes. - gen = &classes.Generator{Root: "https://h.test"} - if xml, err := gen.Generate(context.Background(), urlItems("/a")); err != nil || !strings.Contains(xml, "https://h.test/a") { - t.Fatalf("no bus: %v\n%s", err, xml) - } - - // A listener error stops the generation. - failing := festival.New() - failing.Listen("acme.fail", func(_ context.Context, e *classes.AddItem) error { return errors.New("listener failed") }) - gen = &classes.Generator{Root: "https://h.test", Events: failing} - if _, err := gen.Generate(context.Background(), urlItems("/a")); err == nil || !strings.Contains(err.Error(), "listener failed") { - t.Fatalf("listener error = %v", err) - } -} - -// TestURLCap pins the protocol limit: of 50,001 URLs the 50,000th is -// written and the 50,001st is not; skipped and empty URLs are not counted; -// no items is an empty document. -func TestURLCap(t *testing.T) { - paths := make([]string, classes.MaxURLs+1) - for i := range paths { - paths[i] = "/u/" + strconv.Itoa(i+1) - } - gen := &classes.Generator{Root: "https://api.example.test"} - xml, err := gen.Generate(context.Background(), urlItems(paths...)) - if err != nil { - t.Fatal(err) - } - if n := strings.Count(xml, ""); n != classes.MaxURLs { - t.Fatalf("%d url elements, want %d", n, classes.MaxURLs) - } - if !strings.Contains(xml, "https://api.example.test/u/50000") || strings.Contains(xml, "https://api.example.test/u/50001") { - t.Fatal("the 50000th URL must be written and the 50001st must not") - } - - // Skipped URLs do not count toward the cap. - bus := festival.New() - bus.Listen("acme.skip", func(_ context.Context, e *classes.BeforeAddItem) error { - e.Skip = strings.HasSuffix(e.Info.URL, "/u/1") - return nil - }) - gen = &classes.Generator{Root: "https://api.example.test", Events: bus} - xml, err = gen.Generate(context.Background(), urlItems(paths...)) - if err != nil { - t.Fatal(err) - } - if strings.Count(xml, "") != classes.MaxURLs || !strings.Contains(xml, "/u/50001") { - t.Fatal("a skipped URL took a place under the cap") - } - - // A generator is reusable: a second run starts its count again. - if again, err := gen.Generate(context.Background(), urlItems("/x")); err != nil || strings.Count(again, "") != 1 { - t.Fatalf("second run = %v, %d urls", err, strings.Count(again, "")) - } - if empty, err := (&classes.Generator{Root: "https://h"}).Generate(context.Background(), nil); err != nil || empty != "" { - t.Fatalf("no items = %q %v", empty, err) - } -} - -// TestOnlyHTTPLocs pins the CR-01 output filter: a URL that Url::to keeps -// but that is not http or https (a javascript: URL FILTER_VALIDATE_URL -// accepts, mailto:, a bare fragment) is not written and not counted. A -// protocol-relative URL comes out of Url::to on the root host. -func TestOnlyHTTPLocs(t *testing.T) { - gen := &classes.Generator{Root: "https://api.example.test"} - xml, err := gen.Generate(context.Background(), urlItems( - "javascript://%0aalert(1)%0a//x", - "JavaScript://example.test/%0aalert(1)", - "//evil.test/x", - "mailto:a@example.test", - "#top", - "/ok", - "https://other.example.test/page", - )) - if err != nil { - t.Fatal(err) - } - if n := strings.Count(xml, ""); n != 3 { - t.Fatalf("%d url elements, want 3:\n%s", n, xml) - } - if !strings.Contains(xml, "https://api.example.test/evil.test/x") { - t.Fatalf("the protocol-relative URL did not stay on the root host:\n%s", xml) - } - if strings.Contains(strings.ToLower(xml), "javascript") || strings.Contains(xml, "mailto:") || strings.Contains(xml, "#top") { - t.Fatalf("a non-http URL was written:\n%s", xml) - } - if !strings.Contains(xml, "https://api.example.test/ok") || !strings.Contains(xml, "https://other.example.test/page") { - t.Fatalf("an http(s) URL is missing:\n%s", xml) - } -} diff --git a/classes/golden_test.go b/classes/golden_test.go deleted file mode 100644 index d4d3452..0000000 --- a/classes/golden_test.go +++ /dev/null @@ -1,343 +0,0 @@ -package classes_test - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "testing" - "time" - - sitemap "git.golem15.com/golem15/sm-sitemap-plugin" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/compass" - "git.golem15.com/golem15/summercms/modules/festival" - "git.golem15.com/golem15/summercms/modules/pact" -) - -// The goldens in testdata are written by tests/php/gen-goldens.php from the -// same JSON inputs with the PHP plugin's generation code and the vendored -// Winter URL generator; these tests feed the inputs through the Go port and -// compare bytes. - -// goldenResolved is a registry item's resolve result in a golden input. -type goldenResolved struct { - URL string `json:"url"` - MTime *string `json:"mtime"` - Alternates [][2]string `json:"alternates"` - Items []goldenResolved `json:"items"` -} - -func (g goldenResolved) toPact(t *testing.T) pact.ResolvedMenuItem { - t.Helper() - out := pact.ResolvedMenuItem{URL: g.URL} - if g.MTime != nil { - at, err := time.Parse(time.RFC3339, *g.MTime) - if err != nil { - t.Fatalf("mtime %q: %v", *g.MTime, err) - } - out.MTime = &at - } - for _, a := range g.Alternates { - out.Alternates = append(out.Alternates, pact.MenuItemAlternate{Locale: a[0], URL: a[1]}) - } - for _, child := range g.Items { - out.Items = append(out.Items, child.toPact(t)) - } - return out -} - -// goldenCase is one sitemap case of a golden input: the stored items, the -// registry types' resolve results, the settings (PHP keys), the updates -// manifest, the URLs a BeforeAddItem listener skips and the text of the -// extra element an AddItem listener appends per URL. -type goldenCase struct { - Name string `json:"name"` - Root string `json:"root"` - Items models.DefinitionItems `json:"items"` - Types map[string]map[string]goldenResolved `json:"types"` - Settings map[string]json.RawMessage `json:"settings"` - Manifest map[string]any `json:"manifest"` - Skip []string `json:"skip"` - Extra map[string]string `json:"extra"` -} - -// fakeResolver answers Resolve from a golden case's types. -type fakeResolver struct { - t *testing.T - types map[string]map[string]goldenResolved -} - -func (f fakeResolver) Type(key string) (pact.MenuItemType, bool) { - _, ok := f.types[key] - return pact.MenuItemType{Key: key}, ok -} - -func (f fakeResolver) Resolve(_ context.Context, key, reference, _ string, _ bool, _ string) (*pact.ResolvedMenuItem, error) { - r, ok := f.types[key][reference] - if !ok { - return nil, nil - } - out := r.toPact(f.t) - return &out, nil -} - -// pluginTiers compiles the tier rules the plugin's own config/config.yaml -// ships. -func pluginTiers(t *testing.T) classes.Tiers { - t.Helper() - cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Environ: []string{}}) - if err != nil { - t.Fatal(err) - } - if err := cfg.MergePlugin("golem15.sitemap", (&sitemap.Plugin{}).ConfigFS()); err != nil { - t.Fatal(err) - } - tiers, err := classes.LoadTiers(cfg) - if err != nil { - t.Fatal(err) - } - return tiers -} - -// caseSettings builds the settings of a case from its PHP keys. -func caseSettings(t *testing.T, c goldenCase) models.Settings { - t.Helper() - s := models.DefaultSettings() - texts := map[string]**string{ - "frontend_url": &s.FrontendURL, - "frontend_default_locale": &s.FrontendDefaultLocale, - "priority_homepage": &s.PriorityHomepage, - "priority_services": &s.PriorityServices, - "priority_blog": &s.PriorityBlog, - "priority_categories": &s.PriorityCategories, - "priority_legal": &s.PriorityLegal, - } - for key, raw := range c.Settings { - if key == "enable_priority_tiers" { - if err := json.Unmarshal(raw, &s.EnablePriorityTiers); err != nil { - t.Fatal(err) - } - continue - } - dest, ok := texts[key] - if !ok { - t.Fatalf("case %s: unknown setting %s", c.Name, key) - } - v, err := models.DecodeText(raw) - if err != nil { - t.Fatal(err) - } - *dest = v - } - if c.Manifest != nil { - raw, err := json.Marshal(c.Manifest) - if err != nil { - t.Fatal(err) - } - path := filepath.Join(t.TempDir(), "updates.json") - if err := os.WriteFile(path, raw, 0o644); err != nil { - t.Fatal(err) - } - s.UpdatesManifestPath = &path - } - return s -} - -// renderCase runs a case through the generator and the route's rewrite. -func renderCase(t *testing.T, c goldenCase) string { - t.Helper() - bus := festival.New() - bus.Listen("test.skip", func(_ context.Context, e *classes.BeforeAddItem) error { - if slices.Contains(c.Skip, e.Info.URL) { - e.Skip = true - } - return nil - }) - bus.Listen("test.extra", func(_ context.Context, e *classes.AddItem) error { - if text, ok := c.Extra[e.Info.URL]; ok { - e.Extra = append(e.Extra, ""+classes.EscapeText(text)+"") - } - return nil - }) - settings := caseSettings(t, c) - gen := &classes.Generator{ - Root: c.Root, - Resolver: fakeResolver{t: t, types: c.Types}, - Settings: settings, - Tiers: pluginTiers(t), - Events: bus, - } - xml, err := gen.Generate(context.Background(), c.Items) - if err != nil { - t.Fatalf("generate %s: %v", c.Name, err) - } - return classes.Rewrite(xml, classes.BackendURL(c.Root), models.Text(settings.FrontendURL)) -} - -func readTestdata(t *testing.T, name string) []byte { - t.Helper() - raw, err := os.ReadFile(filepath.Join("testdata", name)) - if err != nil { - t.Fatal(err) - } - return raw -} - -// TestSitemapGolden compares the Go sitemap with the PHP goldens. -func TestSitemapGolden(t *testing.T) { - t.Run("urlto", func(t *testing.T) { - var in struct { - Root string `json:"root"` - URLs []*string `json:"urls"` - } - if err := json.Unmarshal(readTestdata(t, "urlto.json"), &in); err != nil { - t.Fatal(err) - } - want := strings.Split(strings.TrimSuffix(string(readTestdata(t, "urlto.golden")), "\n"), "\n") - if len(want) != len(in.URLs) { - t.Fatalf("urlto.golden has %d lines for %d inputs", len(want), len(in.URLs)) - } - var got strings.Builder - for i, u := range in.URLs { - path := "" - if u != nil { - path = *u - } - out := classes.URLTo(in.Root, path) - if out != want[i] { - t.Errorf("URLTo(%q) = %q, PHP %q", path, out, want[i]) - } - got.WriteString(out + "\n") - } - if got.String() != string(readTestdata(t, "urlto.golden")) { - t.Error("urlto output differs from the golden file") - } - }) - - t.Run("tracer", func(t *testing.T) { - var in struct { - Case goldenCase `json:"case"` - } - if err := json.Unmarshal(readTestdata(t, "tracer.json"), &in); err != nil { - t.Fatal(err) - } - got := renderCase(t, in.Case) - if want := string(readTestdata(t, "tracer.golden")); got != want { - t.Fatalf("tracer sitemap differs from the PHP golden\n got: %q\nwant: %q", got, want) - } - }) - - t.Run("full", func(t *testing.T) { - var in struct { - Cases []goldenCase `json:"cases"` - } - if err := json.Unmarshal(readTestdata(t, "full.json"), &in); err != nil { - t.Fatal(err) - } - golden := string(readTestdata(t, "full.golden")) - var all strings.Builder - for i, c := range in.Cases { - got := renderCase(t, c) - all.WriteString("=== " + c.Name + "\n" + got) - header := "=== " + c.Name + "\n" - start := strings.Index(golden, header) - if start < 0 { - t.Errorf("case %s is missing from full.golden", c.Name) - continue - } - want := golden[start+len(header):] - if i+1 < len(in.Cases) { - if end := strings.Index(want, "=== "+in.Cases[i+1].Name+"\n"); end >= 0 { - want = want[:end] - } - } - if got != want { - t.Errorf("case %s differs from the PHP golden\n got: %q\nwant: %q", c.Name, got, want) - } - } - if all.String() != golden { - t.Error("full output differs from full.golden") - } - }) - - t.Run("cap", func(t *testing.T) { - items := make([]models.DefinitionItem, classes.MaxURLs+1) - for i := range items { - u := "/u/" + strconv.Itoa(i+1) - items[i] = models.DefinitionItem{Type: models.ItemTypeURL, URL: &u} - } - gen := &classes.Generator{Root: "https://api.example.test"} - xml, err := gen.Generate(context.Background(), items) - if err != nil { - t.Fatal(err) - } - if n := strings.Count(xml, ""); n != classes.MaxURLs { - t.Fatalf("%d urls written, want %d", n, classes.MaxURLs) - } - if !strings.Contains(xml, "https://api.example.test/u/50000") || strings.Contains(xml, "/u/50001<") { - t.Fatal("the 50000th URL must be written and the 50001st must not") - } - }) -} - -// TestSitemapTiersConfig checks the boot-time tier rule checks, a host's -// replacement of the default list and the manifest's fail-safe reads. -func TestSitemapTiersConfig(t *testing.T) { - for name, rules := range map[string][]classes.TierConfig{ - "unknown setting": {{Setting: "priority_other", Exact: []string{""}}}, - "neither": {{Setting: models.PriorityHomepage}}, - "both": {{Setting: models.PriorityHomepage, Exact: []string{""}, Pattern: "^/$"}}, - "bad pattern": {{Setting: models.PriorityBlog, Pattern: "^/(blog"}}, - } { - if _, err := classes.CompileTiers(rules); err == nil { - t.Errorf("%s: compiled, want an error", name) - } - } - - dir := t.TempDir() - host := "sitemap:\n priority_tiers:\n - setting: priority_legal\n pattern: '^/legal(/.*)?$'\n" - if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte(host), 0o644); err != nil { - t.Fatal(err) - } - cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) - if err != nil { - t.Fatal(err) - } - if err := cfg.MergePlugin("golem15.sitemap", (&sitemap.Plugin{}).ConfigFS()); err != nil { - t.Fatal(err) - } - tiers, err := classes.LoadTiers(cfg) - if err != nil || len(tiers) != 1 { - t.Fatalf("host tiers = %+v, %v", tiers, err) - } - s := models.DefaultSettings() - if p, ok := tiers.Priority("https://h/legal/terms/", s); !ok || p != "0.2" { - t.Fatalf("host rule priority = %q %v", p, ok) - } - if _, ok := tiers.Priority("https://h/", s); ok { - t.Fatal("the host list must replace the default homepage rule") - } - - if m := classes.LoadManifest(dir); m != nil { - t.Fatal("a directory is not a manifest") - } - list := filepath.Join(dir, "list.json") - if err := os.WriteFile(list, []byte(`["/a"]`), 0o644); err != nil { - t.Fatal(err) - } - if m := classes.LoadManifest(list); m != nil { - t.Fatal("a JSON list is not a manifest") - } - big := filepath.Join(dir, "big.json") - if err := os.WriteFile(big, append([]byte(`{"/a":"2026-01-01","x":"`), append(make([]byte, classes.ManifestMaxBytes), '"', '}')...), 0o644); err != nil { - t.Fatal(err) - } - if m := classes.LoadManifest(big); m != nil { - t.Fatal("a manifest over the size cap must be ignored") - } -} diff --git a/classes/manifest.go b/classes/manifest.go deleted file mode 100644 index 7cb7f4f..0000000 --- a/classes/manifest.go +++ /dev/null @@ -1,112 +0,0 @@ -package classes - -import ( - "bytes" - "encoding/json" - "io" - "os" - "strings" - "time" -) - -// ManifestMaxBytes is the largest updates manifest the sitemap reads. -const ManifestMaxBytes = 5 << 20 - -// Manifest maps a normalized URL path to its last-modified date text, the -// updates manifest a frontend build writes. -type Manifest map[string]string - -// LoadManifest reads the updates manifest at path: a regular file of at -// most ManifestMaxBytes holding a JSON object of URL path to date. It fails -// safe like PHP: an empty path, a missing, unreadable, oversized or invalid -// file is no manifest (nil), logged without the file's content. Keys are -// normalized like PHP normalizeManifestPath; values that are not strings are -// ignored. -func LoadManifest(path string) Manifest { - path = strings.TrimSpace(path) - if path == "" { - return nil - } - info, err := os.Stat(path) - if err != nil || !info.Mode().IsRegular() { - return nil - } - if info.Size() > ManifestMaxBytes { - pluginLog.Warn("golem15.sitemap: updates manifest is too large; lastmod omitted", "path", path, "max_bytes", ManifestMaxBytes) - return nil - } - f, err := os.Open(path) - if err != nil { - return nil - } - defer f.Close() - raw, err := io.ReadAll(io.LimitReader(f, ManifestMaxBytes+1)) - if err != nil || len(raw) > ManifestMaxBytes { - return nil - } - raw = bytes.TrimSpace(raw) - var decoded map[string]any - if len(raw) == 0 || raw[0] != '{' || json.Unmarshal(raw, &decoded) != nil { - pluginLog.Warn("golem15.sitemap: updates manifest is not a JSON object; lastmod omitted", "path", path) - return nil - } - m := Manifest{} - for key, value := range decoded { - if s, ok := value.(string); ok { - m[normalizeManifestPath(key)] = s - } - } - return m -} - -// normalizeManifestPath is PHP normalizeManifestPath: a leading slash, no -// trailing slash except for the root. -func normalizeManifestPath(path string) string { - path = "/" + strings.TrimLeft(path, "/") - if path == "/" { - return path - } - return strings.TrimRight(path, "/") -} - -// Lastmod is PHP lookupManifestLastmod: the date listed for the URL's path, -// parsed like the ISO 8601 forms strtotime reads (a date without a zone is -// UTC). ok is false when the path is not listed or its date does not parse. -func (m Manifest) Lastmod(url string) (time.Time, bool) { - if len(m) == 0 { - return time.Time{}, false - } - path := URLPath(url) - if path == "" { - path = "/" - } - text, ok := m[normalizeManifestPath(path)] - if !ok { - return time.Time{}, false - } - t, ok := parseManifestDate(text) - if !ok || t.Unix() == 0 { - return time.Time{}, false - } - return t, true -} - -var manifestLayouts = []string{ - time.RFC3339Nano, - "2006-01-02T15:04:05.999999999Z0700", - "2006-01-02T15:04:05", - "2006-01-02T15:04", - "2006-01-02 15:04:05Z07:00", - "2006-01-02 15:04:05", - "2006-01-02", -} - -func parseManifestDate(s string) (time.Time, bool) { - s = strings.TrimSpace(s) - for _, layout := range manifestLayouts { - if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil { - return t, true - } - } - return time.Time{}, false -} diff --git a/classes/manifest_test.go b/classes/manifest_test.go deleted file mode 100644 index f6c66eb..0000000 --- a/classes/manifest_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package classes_test - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" -) - -// TestManifest pins the updates manifest reader: it fails safe (a missing, -// empty-path, directory, oversized, non-object or invalid file is no -// manifest, so lastmod is omitted), normalizes keys, ignores non-string -// values, and parses the ISO dates strtotime reads, a date without a zone -// as UTC. -func TestManifest(t *testing.T) { - dir := t.TempDir() - write := func(name, body string) string { - t.Helper() - path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - return path - } - for name, path := range map[string]string{ - "empty path": "", - "spaces": " ", - "missing": filepath.Join(dir, "missing.json"), - "directory": dir, - "list": write("list.json", `["/a"]`), - "string": write("string.json", `"x"`), - "invalid": write("invalid.json", `{"/a": `), - "empty file": write("empty.json", ""), - "oversized": write("big.json", `{"/a":"2026-01-01","pad":"`+strings.Repeat("x", classes.ManifestMaxBytes)+`"}`), - "null": write("null.json", "null"), - "leading space": write("space.json", " \n"), - } { - if m := classes.LoadManifest(path); m != nil { - t.Errorf("%s: manifest = %v, want none", name, m) - } - } - if fifo := filepath.Join(dir, "link.json"); os.Symlink(dir, fifo) == nil { - if m := classes.LoadManifest(fifo); m != nil { - t.Error("a symlink to a directory is not a manifest") - } - } - - path := write("updates.json", ` {"blog/a/":"2026-01-02T03:04:05+02:00","/b":"2026-02-03","c":"2026-03-04 05:06:07","/":"2026-04-05T06:07","/num":12,"/bad":"yesterday","/epoch":"1970-01-01T00:00:00Z","/nested":{"x":1}}`) - m := classes.LoadManifest(path) - if m == nil { - t.Fatal("valid manifest not read") - } - if _, ok := m["/num"]; ok { - t.Fatal("a non-string value was kept") - } - for url, want := range map[string]string{ - "https://h/blog/a": "2026-01-02T01:04:05Z", - "https://h/blog/a/": "2026-01-02T01:04:05Z", - "https://h/b?x=1": "2026-02-03T00:00:00Z", - "/c": "2026-03-04T05:06:07Z", - "https://h": "2026-04-05T06:07:00Z", - "https://h/": "2026-04-05T06:07:00Z", - } { - got, ok := m.Lastmod(url) - if !ok || got.UTC().Format(time.RFC3339) != want { - t.Errorf("Lastmod(%s) = %v %v, want %s", url, got, ok, want) - } - } - for _, url := range []string{"https://h/bad", "https://h/epoch", "https://h/unlisted", "https://h/num"} { - if got, ok := m.Lastmod(url); ok { - t.Errorf("Lastmod(%s) = %v, want none", url, got) - } - } - var none classes.Manifest - if _, ok := none.Lastmod("https://h/b"); ok { - t.Fatal("a nil manifest answered") - } -} diff --git a/classes/rewrite.go b/classes/rewrite.go deleted file mode 100644 index 4125911..0000000 --- a/classes/rewrite.go +++ /dev/null @@ -1,69 +0,0 @@ -package classes - -import ( - "regexp" - "strings" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// The PHP routes.php patterns of the frontend rewrite. -var ( - locElement = regexp.MustCompile(`(.*?)`) - xhtmlLinkRef = regexp.MustCompile(`(]*href=")([^"]*)(")`) -) - -// Rewrite is the frontend_url rewrite of PHP routes.php: with frontendURL -// set, backendURL is replaced literally (str_replace) inside every loc -// element, inside every xhtml:link href and in the exact stylesheet href. -// Both URLs are right-trimmed of slashes first. An empty frontendURL (or -// "0", which PHP reads as false) leaves xml unchanged. -// -// PHP inserted the stored text as it was. Three rules keep that text out of -// the document unless it is a plain http(s) URL (T-17-15): -// -// - a frontendURL that models.AllowedFrontendURL refuses leaves xml -// unchanged, whatever is stored in the settings row; -// - the frontend URL is inserted through EscapeText in loc and through -// EscapeAttr in the hrefs. Both are the identity for every accepted -// value, which keeps the PHP bytes; -// - a loc or xhtml:link href whose rewritten value does not start with -// http:// or https:// keeps its value. The generator applies the same -// filter before the rewrite. -func Rewrite(xml, backendURL, frontendURL string) string { - if frontendURL == "" || frontendURL == "0" { - return xml - } - if !models.AllowedFrontendURL(frontendURL) { - return xml - } - frontend := strings.TrimRight(frontendURL, "/") - backend := strings.TrimRight(backendURL, "/") - if backend == "" { - return xml - } - frontendText, frontendAttr := EscapeText(frontend), EscapeAttr(frontend) - xml = locElement.ReplaceAllStringFunc(xml, func(m string) string { - inner := m[len("") : len(m)-len("")] - rewritten := strings.ReplaceAll(inner, backend, frontendText) - if !httpURL(rewritten) { - return m - } - return "" + rewritten + "" - }) - xml = xhtmlLinkRef.ReplaceAllStringFunc(xml, func(m string) string { - sub := xhtmlLinkRef.FindStringSubmatch(m) - rewritten := strings.ReplaceAll(sub[2], backend, frontendAttr) - if !httpURL(rewritten) { - return m - } - return sub[1] + rewritten + sub[3] - }) - return strings.ReplaceAll(xml, `href="`+backend+`/sitemap.xsl"`, `href="`+frontendAttr+`/sitemap.xsl"`) -} - -// BackendURL is PHP rtrim(url('/'), '/') for a request whose root URL is -// root: the URL the rewrite replaces. -func BackendURL(root string) string { - return strings.TrimRight(URLTo(root, "/"), "/") -} diff --git a/classes/rewrite_test.go b/classes/rewrite_test.go deleted file mode 100644 index 124ad05..0000000 --- a/classes/rewrite_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package classes_test - -import ( - "context" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" -) - -// rewriteBackend and rewriteDoc are the backend URL and the document of the -// rewrite tests: a stylesheet instruction, a loc, hreflang links and the -// backend URL in places the rewrite leaves alone. -const rewriteBackend = "https://api.ex.pl" - -var rewriteDoc = "\n" + - `` + "\n" + - `https://api.ex.pl/a2026https://api.ex.pl1` + - `` + - `https://api.ex.pl/xhttps://other.pl/https://api.ex.plcp` + "\n" - -// TestRewrite pins the PHP routes.php frontend_url rewrite: the backend URL -// is replaced inside loc, inside xhtml:link href and in the exact stylesheet -// href, nowhere else; slashes are trimmed on both; an empty or "0" frontend -// URL and equal URLs leave the document unchanged. -func TestRewrite(t *testing.T) { - const backend = rewriteBackend - doc := rewriteDoc - want := "\n" + - `` + "\n" + - `https://www.ex.pl/a2026https://api.ex.pl1` + - `` + - `https://api.ex.pl/xhttps://other.pl/https://www.ex.plcp` + "\n" - for _, frontend := range []string{"https://www.ex.pl", "https://www.ex.pl/", "https://www.ex.pl///"} { - for _, back := range []string{backend, backend + "/"} { - if got := classes.Rewrite(doc, back, frontend); got != want { - t.Fatalf("Rewrite(%q, %q):\n%s\nwant:\n%s", back, frontend, got, want) - } - } - } - for _, frontend := range []string{"", "0"} { - if got := classes.Rewrite(doc, backend, frontend); got != doc { - t.Fatalf("frontend %q changed the document", frontend) - } - } - if got := classes.Rewrite(doc, backend, backend); got != doc { - t.Fatal("equal URLs changed the document") - } - if got := classes.Rewrite(doc, backend+"/", backend); got != doc { - t.Fatal("equal URLs up to a slash changed the document") - } - // An empty backend URL replaces nothing. - if got := classes.Rewrite(doc, "", "https://www.ex.pl"); got != doc { - t.Fatal("an empty backend URL changed the document") - } - // A stylesheet on another path is not the PHP pattern. - other := `` - if got := classes.Rewrite(other, backend, "https://www.ex.pl"); got != other { - t.Fatalf("other stylesheet = %q", got) - } -} - -// The two frontend_url values of the T-17-15 reproduction: a script URL the -// stylesheet would render as a link, and a value that closes the attribute -// and the element it is written into. -var frontendAttacks = []string{ - "javascript:alert(document.domain)//", - `https://x"/>", - " https://www.ex.pl", - "https://www.ex.pl\n", - "//www.ex.pl", - "www.ex.pl", - "https:///x", - "data:text/html,x", - }, frontendAttacks...) - for _, frontend := range refused { - if got := classes.Rewrite(rewriteDoc, rewriteBackend, frontend); got != rewriteDoc { - t.Errorf("frontend %q changed the document:\n%s", frontend, got) - } - } - - // The reproduction through the generator. - const root = "https://api.example.test" - xml, err := (&classes.Generator{Root: root}).Generate(context.Background(), urlItems("/blog/x")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(xml, `href="https://api.example.test/sitemap.xsl"`) { - t.Fatalf("the generated document lacks the stylesheet instruction:\n%s", xml) - } - for _, frontend := range frontendAttacks { - got := classes.Rewrite(xml, classes.BackendURL(root), frontend) - if got != xml { - t.Errorf("frontend %q changed the generated sitemap:\n%s", frontend, got) - } - if !strings.Contains(got, "https://api.example.test/blog/x") { - t.Errorf("frontend %q: the loc is not on the backend URL:\n%s", frontend, got) - } - if lower := strings.ToLower(got); strings.Contains(lower, "javascript") || strings.Contains(lower, "https://www.example.test/blog/x") || !strings.Contains(got, `href="https://www.example.test/sitemap.xsl"`) || strings.Contains(got, "api.example.test") { - t.Errorf("a valid frontend URL did not rewrite the sitemap:\n%s", got) - } - if got := classes.Rewrite(xml, classes.BackendURL(root), "HTTPS://WWW.EXAMPLE.TEST"); !strings.Contains(got, "HTTPS://WWW.EXAMPLE.TEST/blog/x") { - t.Errorf("an upper-case frontend URL did not rewrite the loc:\n%s", got) - } - - // The http(s) output filter after the rewrite: replacing "s://api.ex.pl" - // would write httphttps://www.ex.pl/a, so loc and href keep their values. - entry := `https://api.ex.pl/a` - if got := classes.Rewrite(entry, "s://api.ex.pl", "https://www.ex.pl"); got != entry { - t.Errorf("a rewrite to a non-http(s) URL was written:\n%s", got) - } -} diff --git a/classes/serialize.go b/classes/serialize.go deleted file mode 100644 index 5de12e1..0000000 --- a/classes/serialize.go +++ /dev/null @@ -1,105 +0,0 @@ -package classes - -import "strings" - -// The sitemap document is written by hand in the exact format PHP's -// DOMDocument::saveXML (libxml2) produced for the PHP plugin: the XML -// declaration and the stylesheet processing instruction each followed by a -// newline, everything else on one line, an empty urlset self-closed, -// xhtml:link elements self-closed without a space. The standard library's -// XML encoder escapes and closes elements differently, so the writer is a -// strings.Builder. - -// The urlset opening tag's attributes, in the order PHP set them. -const urlsetAttrs = ` xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"` + - ` xmlns:xhtml="http://www.w3.org/1999/xhtml"` + - ` xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"` + - ` xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"` - -// textEscaper is libxml's escaping of text nodes: &, <, > and carriage -// return; quotes, tabs, newlines and UTF-8 stay literal. -var textEscaper = strings.NewReplacer("&", "&", "<", "<", ">", ">", "\r", " ") - -// attrEscaper is libxml's escaping of attribute values: also the double -// quote, tab and newline; the single quote stays literal. -var attrEscaper = strings.NewReplacer( - "&", "&", "<", "<", ">", ">", `"`, """, - "\r", " ", "\t", " ", "\n", " ", -) - -// EscapeText escapes s as libxml writes a text node. -func EscapeText(s string) string { return textEscaper.Replace(s) } - -// EscapeAttr escapes s as libxml writes an attribute value. -func EscapeAttr(s string) string { return attrEscaper.Replace(s) } - -// Alternate is one hreflang link of a sitemap URL. -type Alternate struct { - Hreflang string - Href string -} - -// URLEntry is one url element: Loc, an optional Lastmod, Changefreq and -// Priority as they are written, the alternates (written only when there are -// more than one) and Extra, pre-escaped child XML that AddItem listeners -// appended, written before the closing tag. -type URLEntry struct { - Loc string - Lastmod string - Changefreq string - Priority string - Alternates []Alternate - Extra []string -} - -// writeURL appends one url element. -func writeURL(b *strings.Builder, e URLEntry) { - b.WriteString("") - b.WriteString(EscapeText(e.Loc)) - b.WriteString("") - if e.Lastmod != "" { - b.WriteString("") - b.WriteString(EscapeText(e.Lastmod)) - b.WriteString("") - } - b.WriteString("") - b.WriteString(EscapeText(e.Changefreq)) - b.WriteString("") - b.WriteString(EscapeText(e.Priority)) - b.WriteString("") - if len(e.Alternates) > 1 { - for _, a := range e.Alternates { - b.WriteString(``) - } - } - for _, x := range e.Extra { - b.WriteString(x) - } - b.WriteString("") -} - -// Serialize writes the sitemap document: the XML declaration, the -// stylesheet processing instruction pointing at stylesheet (written as it -// is, like libxml writes PI data) and the urlset with entries. -func Serialize(stylesheet string, entries []URLEntry) string { - var b strings.Builder - b.WriteString("\n") - b.WriteString(`\n\n") - return b.String() - } - b.WriteString(">") - for _, e := range entries { - writeURL(&b, e) - } - b.WriteString("\n") - return b.String() -} diff --git a/classes/serialize_test.go b/classes/serialize_test.go deleted file mode 100644 index 4980471..0000000 --- a/classes/serialize_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package classes_test - -import ( - "context" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// TestXMLEscaping ports the PHP XmlEscapingTest (SITE-002) as behaviour: -// every value the sitemap writes is escaped exactly as libxml escapes it, -// text nodes (loc, lastmod, changefreq, priority) and attribute values -// (hreflang, href), and no raw value reaches the output. -func TestXMLEscaping(t *testing.T) { - const raw = "a&bd\"e'f\rg\th\ni" - if got := classes.EscapeText(raw); got != "a&b<c>d\"e'f g\th\ni" { - t.Fatalf("EscapeText = %q", got) - } - if got := classes.EscapeAttr(raw); got != "a&b<c>d"e'f g h i" { - t.Fatalf("EscapeAttr = %q", got) - } - if got := classes.EscapeText("zażółć — &"); got != "zażółć — &amp;" { - t.Fatalf("EscapeText keeps UTF-8 and escapes an existing entity again: %q", got) - } - - xml := classes.Serialize("https://h/sitemap.xsl", []classes.URLEntry{{ - Loc: "https://h/a?x=1&y=<2>", - Lastmod: "2026-01-02T03:04:05+00:00", - Changefreq: "weekly&<>", - Priority: "0.5\r", - Alternates: []classes.Alternate{ - {Hreflang: "pl\"x", Href: "https://h/pl?a=1&b=\"2\"\t\n"}, - {Hreflang: "en", Href: "https://h/en'x"}, - }, - }}) - want := "\n" + - "\n" + - `` + - "https://h/a?x=1&y=<2>2026-01-02T03:04:05+00:00weekly&<>0.5 " + - `` + - `` + - "\n" - if xml != want { - t.Fatalf("serialized:\n%q\nwant:\n%q", xml, want) - } - - // One alternate is not written; an empty urlset is self-closed. - single := classes.Serialize("s", []classes.URLEntry{{Loc: "l", Changefreq: "c", Priority: "p", Alternates: []classes.Alternate{{Hreflang: "pl", Href: "x"}}}}) - if strings.Contains(single, "xhtml:link") || strings.Contains(single, "") { - t.Fatalf("single alternate or empty lastmod written: %q", single) - } - if empty := classes.Serialize("s", nil); !strings.HasSuffix(empty, `sitemap.xsd"/>`+"\n") { - t.Fatalf("empty urlset = %q", empty) - } - - // Through the generator: stored changefreq and priority, a resolved - // alternate's locale and a url item with every special character. - url := "/q?a=1&b=&c=\"y\"" - changefreq := "daily&" - priority := "0.9\"'" - gen := &classes.Generator{Root: "https://api.example.test", Settings: models.Settings{}, Resolver: fakeResolver{t: t, types: map[string]map[string]goldenResolved{ - "acme-post": {"1": {URL: "/p?x=1&y=2", Alternates: [][2]string{{"pl&<", "/p?x=1&y=2"}, {"en\"", "/en/p?z=<1>"}}}}, - }}} - out, err := gen.Generate(context.Background(), []models.DefinitionItem{ - {Type: models.ItemTypeURL, URL: &url, Changefreq: &changefreq, Priority: &priority}, - {Type: "acme-post", Reference: text("1"), Changefreq: &changefreq, Priority: &priority}, - }) - if err != nil { - t.Fatal(err) - } - body := out[strings.Index(out, ""):] - for _, bad := range []string{"", "&b", "&y", `"y"`, "", "daily&<", `hreflang="pl&<"`, `en""`, "<1>"} { - if strings.Contains(body, bad) { - t.Fatalf("raw value %q reached the output:\n%s", bad, body) - } - } - for _, good := range []string{ - "daily&<weekly>", - "0.9\"'", - `hreflang="pl&<"`, - `hreflang="en""`, - "y=2", - } { - if !strings.Contains(body, good) { - t.Fatalf("output lacks %q:\n%s", good, body) - } - } - if strings.Count(body, "") != 2 { - t.Fatalf("urls = %d:\n%s", strings.Count(body, ""), body) - } -} - -func text(s string) *string { return &s } diff --git a/classes/testdata/full.golden b/classes/testdata/full.golden deleted file mode 100644 index 4553e1e..0000000 --- a/classes/testdata/full.golden +++ /dev/null @@ -1,58 +0,0 @@ -=== escaping - - -https://a&b<c>.pl/xa&b<c>"d'e f g -h0.5 & <1> "q" 'a'https://api.example.test/za%C5%BC%C3%B3%C5%82%C4%87%20%26%20co2026-01-02T03:04:05+00:00x>y<0.9> -=== empty-urlset - - - -=== no-items -=== hreflang-list - - -https://api.example.test/pl/a2026-02-01T00:00:00+00:00monthly0.4https://api.example.test/en/a2026-02-01T00:00:00+00:00monthly0.4https://api.example.test/pl/bmonthly0.4https://api.example.test/en/bmonthly0.4https://api.example.test/de/bmonthly0.4https://api.example.test/solomonthly0.4 -=== nested - - -https://api.example.test/kategoria/a2026-03-01T12:00:00+00:00daily0.7https://api.example.test/kategoria/b2026-03-02T12:00:00+00:00daily0.7https://api.example.test/kategoria/cdaily0.7 -=== tiers-default - - -https://api.example.test/weekly1.0https://api.example.test/enweekly1.0https://api.example.test/enweekly1.0https://api.example.test/uslugi/xweekly0.8https://api.example.test/uslugi/x/yweekly0.3https://api.example.test/realizacje/rweekly0.8https://api.example.test/en/work/wweekly0.8https://api.example.test/blog/a/bweekly0.6https://api.example.test/en/blog/a/bweekly0.6https://api.example.test/blog/aweekly0.4https://api.example.test/blogweekly0.4https://api.example.test/en/blogweekly0.4https://api.example.test/uslugiweekly0.4https://api.example.test/kontaktweekly0.2https://api.example.test/kontaktweekly0.2https://api.example.test/en/ai-lab/blob-playgroundweekly0.2https://api.example.test/otherweekly0.3https://elsewhere.pl/uslugi/zweekly0.8https://api.example.test/o-nas?x=1weekly0.2https://api.example.test/blog/a/b/cweekly0.3 -=== tiers-settings - - -https://api.example.test/weekly0.95https://api.example.test/blog/a/bweeklyhttps://api.example.test/regulaminweekly0.15https://api.example.test/xweekly0.3 -=== tiers-off - - -https://api.example.test/weekly0.1https://api.example.test/blog/a/bweekly0.2 -=== rewrite - - -https://www.example.test/weekly0.5https://www.example.test/aweekly0.5https://www.example.test/b?x=https%3A%2F%2Fapi.example.test%2Fcweekly0.5https://other.example.test/api.example.testweekly0.5https://www.example.test/pdaily0.7https://www.example.test/en/pdaily0.7 -=== rewrite-same - - -https://api.example.test/aweekly0.5 -=== manifest - - -https://api.example.test/2026-05-01T08:00:00+00:00weekly0.5https://api.example.test/about2026-05-02T00:00:00+00:00weekly0.5https://api.example.test/about2026-05-02T00:00:00+00:00weekly0.5https://api.example.test/blog/xweekly0.5https://api.example.test/noneweekly0.5https://api.example.test/rel2026-05-03T01:02:03+00:00daily0.7https://api.example.test/about2026-06-01T00:00:00+00:00daily0.7 -=== skip-extra - - -https://api.example.test/extraweekly0.5a&b <c>https://api.example.test/keepweekly0.5 -=== precision - - -https://api.example.test/p1daily0.5https://api.example.test/p2daily1https://api.example.test/p371https://api.example.test/p4daily1.0https://api.example.test/p5daily0.50https://api.example.test/p6daily0.6 -=== url-null - - -https://api.example.test/https://api.example.test/0daily0.5 -=== port-root - - -http://localhost:8096/weekly0.5http://localhost:8096/a%20bweekly0.5 diff --git a/classes/testdata/full.json b/classes/testdata/full.json deleted file mode 100644 index d2c4efe..0000000 --- a/classes/testdata/full.json +++ /dev/null @@ -1,693 +0,0 @@ -{ - "cases": [ - { - "name": "escaping", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "https://a&b.pl/x", - "changefreq": "a&b\"d'e\rf\tg\nh", - "priority": "0.5 & <1> \"q\" 'a'" - }, - { - "type": "acme-post", - "reference": "1", - "urlPattern": "", - "changefreq": "x>y", - "priority": "<0.9>", - "nesting": "1" - } - ], - "types": { - "acme-post": { - "1": { - "url": "/zażółć & co", - "mtime": "2026-01-02T03:04:05+00:00", - "alternates": [ - [ - "x\"y&z<\t\n\r'>", - "https://q&r.pl/p" - ], - [ - "en", - "/en/zażółć" - ] - ] - } - } - } - }, - { - "name": "empty-urlset", - "root": "https://api.example.test", - "items": [ - { - "type": "acme-post", - "reference": "404", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - }, - { - "type": "unknown-type", - "reference": "1", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-post": {} - } - }, - { - "name": "no-items", - "root": "https://api.example.test", - "items": [] - }, - { - "name": "hreflang-list", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "acme-list", - "reference": "", - "urlPattern": "", - "changefreq": "monthly", - "priority": "0.4", - "nesting": "1" - } - ], - "types": { - "acme-list": { - "": { - "url": "", - "items": [ - { - "url": "/pl/a", - "mtime": "2026-02-01T00:00:00+00:00", - "alternates": [ - [ - "pl", - "/pl/a" - ], - [ - "en", - "/en/a" - ] - ] - }, - { - "url": "/pl/b", - "mtime": null, - "alternates": [ - [ - "pl", - "/pl/b" - ], - [ - "en", - "/en/b" - ], - [ - "de", - "/de/b" - ] - ] - }, - { - "url": "/solo", - "alternates": [] - } - ] - } - } - } - }, - { - "name": "nested", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "acme-cat", - "reference": "5", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-cat": { - "5": { - "url": "/kategoria/a", - "mtime": "2026-03-01T12:00:00+00:00", - "alternates": [ - [ - "pl", - "/kategoria/a" - ], - [ - "en", - "/en/category/a" - ] - ], - "items": [ - { - "url": "/kategoria/b", - "mtime": "2026-03-02T12:00:00+00:00", - "alternates": [ - [ - "pl", - "/kategoria/b" - ], - [ - "en", - "/en/category/b" - ] - ], - "items": [ - { - "url": "/kategoria/c", - "alternates": [ - [ - "x-default", - "/kategoria/c" - ] - ] - } - ] - } - ] - } - } - } - }, - { - "name": "tiers-default", - "root": "https://api.example.test", - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/uslugi/x", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/uslugi/x/y", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/realizacje/r", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/work/w", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a/b", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/blog/a/b", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/blog/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/uslugi", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/kontakt", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/kontakt/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/en/ai-lab/blob-playground", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/other", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "https://elsewhere.pl/uslugi/z", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/o-nas?x=1", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a/b/c", - "changefreq": "weekly", - "priority": "0.3" - } - ] - }, - { - "name": "tiers-settings", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": true, - "priority_homepage": "0.95", - "priority_blog": "", - "priority_legal": "0.15" - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/blog/a/b", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/regulamin", - "changefreq": "weekly", - "priority": "0.3" - }, - { - "type": "url", - "url": "/x", - "changefreq": "weekly", - "priority": "0.3" - } - ] - }, - { - "name": "tiers-off", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.1" - }, - { - "type": "url", - "url": "/blog/a/b", - "changefreq": "weekly", - "priority": "0.2" - } - ] - }, - { - "name": "rewrite", - "root": "https://api.example.test", - "settings": { - "frontend_url": "https://www.example.test/", - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/a", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "https://api.example.test/b?x=https://api.example.test/c", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "https://other.example.test/api.example.test", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "acme-list", - "reference": "", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-list": { - "": { - "url": "", - "items": [ - { - "url": "/p", - "alternates": [ - [ - "pl", - "/p" - ], - [ - "en", - "https://api.example.test/en/p" - ] - ] - } - ] - } - } - } - }, - { - "name": "rewrite-same", - "root": "https://api.example.test", - "settings": { - "frontend_url": "https://api.example.test", - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/a", - "changefreq": "weekly", - "priority": "0.5" - } - ] - }, - { - "name": "manifest", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "manifest": { - "/": "2026-05-01T10:00:00+02:00", - "about/": "2026-05-02", - "/blog/x": "not a date", - "/rel": "2026-05-03T01:02:03Z" - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/about", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/about/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/blog/x", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/none", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "acme-post", - "reference": "2", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - }, - { - "type": "acme-post", - "reference": "3", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-post": { - "2": { - "url": "/rel", - "mtime": null, - "alternates": [ - [ - "pl", - "/rel" - ] - ] - }, - "3": { - "url": "/about", - "mtime": "2026-06-01T00:00:00+00:00", - "alternates": [ - [ - "pl", - "/about" - ] - ] - } - } - } - }, - { - "name": "skip-extra", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "skip": [ - "https://api.example.test/skip-me", - "/rel-skip" - ], - "extra": { - "https://api.example.test/extra": "a&b " - }, - "items": [ - { - "type": "url", - "url": "/skip-me", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/extra", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/keep", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "acme-post", - "reference": "9", - "urlPattern": "", - "changefreq": "daily", - "priority": "0.7", - "nesting": "1" - } - ], - "types": { - "acme-post": { - "9": { - "url": "/rel-skip", - "alternates": [ - [ - "pl", - "/rel-skip" - ] - ] - } - } - } - }, - { - "name": "precision", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/p1", - "changefreq": "daily", - "priority": 0.5 - }, - { - "type": "url", - "url": "/p2", - "changefreq": "daily", - "priority": 1.0 - }, - { - "type": "url", - "url": "/p3", - "changefreq": 7, - "priority": 1 - }, - { - "type": "url", - "url": "/p4", - "changefreq": "daily", - "priority": "1.0" - }, - { - "type": "url", - "url": "/p5", - "changefreq": "daily", - "priority": "0.50" - }, - { - "url": "/p6", - "changefreq": "daily", - "priority": "0.6", - "cmsPage": "x", - "exists": true, - "fillable": [ - "a" - ] - } - ] - }, - { - "name": "url-null", - "root": "https://api.example.test", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": null, - "changefreq": null, - "priority": null - }, - { - "type": "url", - "url": "0", - "changefreq": "daily", - "priority": "0.5" - } - ] - }, - { - "name": "port-root", - "root": "http://localhost:8096", - "settings": { - "enable_priority_tiers": false - }, - "items": [ - { - "type": "url", - "url": "/", - "changefreq": "weekly", - "priority": "0.5" - }, - { - "type": "url", - "url": "/a b", - "changefreq": "weekly", - "priority": "0.5" - } - ] - } - ] -} \ No newline at end of file diff --git a/classes/testdata/tracer.golden b/classes/testdata/tracer.golden deleted file mode 100644 index 4a6dc3d..0000000 --- a/classes/testdata/tracer.golden +++ /dev/null @@ -1,3 +0,0 @@ - - -https://api.example.test/always1.0https://api.example.test/news/hello2026-03-04T10:00:00+00:00weekly0.5 diff --git a/classes/testdata/tracer.json b/classes/testdata/tracer.json deleted file mode 100644 index ff5db2b..0000000 --- a/classes/testdata/tracer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "case": { - "root": "https://api.example.test", - "items": [ - {"type": "url", "url": "/", "changefreq": "always", "priority": "1.0", "nesting": "0"}, - {"type": "acme-post", "reference": "7", "urlPattern": "", "changefreq": "weekly", "priority": "0.5", "nesting": "1"} - ], - "types": { - "acme-post": { - "7": { - "url": "/news/hello", - "mtime": "2026-03-04T10:00:00+00:00", - "alternates": [["pl", "/news/hello"], ["en", "/en/news/hello"]] - } - } - } - } -} diff --git a/classes/testdata/urlto.golden b/classes/testdata/urlto.golden deleted file mode 100644 index 3deb178..0000000 --- a/classes/testdata/urlto.golden +++ /dev/null @@ -1,58 +0,0 @@ -https://api.ex.pl/ -https://api.ex.pl/ -https://api.ex.pl/ -https://api.ex.pl/blog -https://api.ex.pl/blog -https://api.ex.pl/blog -https://api.ex.pl/blog/za%C5%BC%C3%B3%C5%82%C4%87%20g%C4%99%C5%9Bl%C4%85 -https://api.ex.pl/en/blog/%3Aslug -https://x.pl/q/ -http://x.pl/ -HTTP://X.pl:443/a -https://h:8080/a -https://h/a -http://h/ -https://api.ex.pl/a?b=1&c=2 -https://api.ex.pl/a?x%5B%5D=1&x%5B%5D=2&0=y -https://api.ex.pl/a?q=a&0=b%20c -https://api.ex.pl/a -https://api.ex.pl/a?b=&0=c -https://api.ex.pl/a?x&y -https://api.ex.pl/a?a=2 -https://api.ex.pl/a?a%5Bb%5D=1 -https://api.ex.pl/a#b%20c -https://api.ex.pl/a -#frag -cdn.x/a -mailto:a%40b.pl -tel:%2B48%20123 -sms:%2B48123 -https://api.ex.pl/a/c -https://api.ex.pl/a//b -https://api.ex.pl/a/%20b/%25zz -https://api.ex.pl/a%2Fb -https://api.ex.pl/%C5%9Bcie%C5%BCka/%C4%85%C4%99?x=%C5%BC#%C5%82 -https://api.ex.pl/ftp%3A//x.pl/a%20b -ftp://x.pl/a -news:///comp.x -https://api.ex.pl/a%3Ab -https://api.ex.pl/a%3A1 -https://api.ex.pl/user%40host -https://user:pass@h/x -https://api.ex.pl/%20%20/lead -https://api.ex.pl/trail -https://api.ex.pl/tab_here -https://api.ex.pl/new_line -https://api.ex.pl/ -https://api.ex.pl/javascript%3Aalert%281%29 -https://a&b.pl/x -https://h/ -https://api.ex.pl/en -https://api.ex.pl/en -https://api.ex.pl/a?x=%2B -https://api.ex.pl/a?k=v%20w -https://api.ex.pl/a%20b?x=1#y%20z -https://h/a?b=1#c -https://api.ex.pl/~tilde/_under.dot-dash -https://api.ex.pl/a%27b%22c -https://api.ex.pl/%3Cx%3E%26y diff --git a/classes/testdata/urlto.json b/classes/testdata/urlto.json deleted file mode 100644 index 3e0ca04..0000000 --- a/classes/testdata/urlto.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "root": "https://api.ex.pl", - "urls": [ - "/", - "", - null, - "/blog/", - "/blog", - "blog", - "/blog/zażółć gęślą", - "/en/blog/:slug", - "https://x.pl/q/", - "http://x.pl", - "HTTP://X.pl:443/a", - "https://h:8080/a", - "https://h:443/a", - "http://h:80/", - "/a?b=1&c=2", - "/a?x[]=1&x[]=2&y", - "/a?q=a%26b+c", - "/a?", - "/a?b=&c", - "/a?0=x&1=y", - "/a?a=1&a=2", - "/a?a[b]=1", - "/a#b c", - "/a#", - "#frag", - "//cdn.x/a", - "mailto:a@b.pl", - "tel:+48 123", - "sms:+48123", - "/../a/./b/../c", - "/a//b", - "/a/%20b/%zz", - "/a%2Fb", - "/ścieżka/ąę?x=ż#ł", - "ftp://x.pl/a b", - "ftp://x.pl/a", - "news:comp.x", - "/a:b", - "a:1", - "/user@host", - "https://user:pass@h/x", - " /lead", - "/trail ", - "/tab\there", - "/new\nline", - "/a/b/../../..", - "javascript:alert(1)", - "https://a&b.pl/x", - "https://h/./", - "/en", - "/en/", - "/a?x=%2B", - "/a?k=v w", - "/a b?x=1#y z", - "https://h/a?b=1#c", - "/~tilde/_under.dot-dash", - "/a'b\"c", - "/&y" - ] -} \ No newline at end of file diff --git a/classes/tiers.go b/classes/tiers.go deleted file mode 100644 index 2e8ca06..0000000 --- a/classes/tiers.go +++ /dev/null @@ -1,98 +0,0 @@ -package classes - -import ( - "fmt" - "regexp" - "slices" - "strings" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/compass" -) - -// TiersConfigKey is the config key of the priority tier rules. The plugin's -// config/config.yaml ships the rules of the PHP getPriorityForUrl; a host -// replaces the whole list in its own config. -const TiersConfigKey = "golem15.sitemap.priority_tiers" - -// TierConfig is one configured tier rule: Setting names the priority setting -// whose value applies (priority_homepage, priority_services, priority_blog, -// priority_categories or priority_legal), and exactly one of Exact (paths -// that match as they are) and Pattern (a regular expression, RE2 syntax) -// selects the URLs. -type TierConfig struct { - Setting string `koanf:"setting"` - Exact []string `koanf:"exact"` - Pattern string `koanf:"pattern"` -} - -// TierRule is a compiled tier rule. -type TierRule struct { - Setting string - Exact []string - Pattern *regexp.Regexp -} - -// Tiers is the ordered tier rule list; the first matching rule wins. -type Tiers []TierRule - -// CompileTiers checks and compiles tier rules. Every error names the rule: -// an unknown setting, neither or both of exact and pattern, or a pattern -// that does not compile. -func CompileTiers(rules []TierConfig) (Tiers, error) { - out := make(Tiers, 0, len(rules)) - for i, r := range rules { - if _, ok := models.PriorityDefaults[r.Setting]; !ok { - return nil, fmt.Errorf("%s[%d]: setting %q is not one of priority_homepage, priority_services, priority_blog, priority_categories, priority_legal", TiersConfigKey, i, r.Setting) - } - hasPattern := strings.TrimSpace(r.Pattern) != "" - if hasPattern == (len(r.Exact) > 0) { - return nil, fmt.Errorf("%s[%d]: set exactly one of exact and pattern", TiersConfigKey, i) - } - rule := TierRule{Setting: r.Setting, Exact: append([]string(nil), r.Exact...)} - if hasPattern { - re, err := regexp.Compile(r.Pattern) - if err != nil { - return nil, fmt.Errorf("%s[%d]: pattern: %w", TiersConfigKey, i, err) - } - rule.Pattern = re - } - out = append(out, rule) - } - return out, nil -} - -// LoadTiers reads and compiles the rules at TiersConfigKey; an absent key -// is no rules. -func LoadTiers(cfg *compass.Config) (Tiers, error) { - if cfg == nil || !cfg.Has(TiersConfigKey) { - return Tiers{}, nil - } - var rules []TierConfig - if err := cfg.LoadSection(TiersConfigKey, &rules); err != nil { - return nil, fmt.Errorf("%s must be a list of {setting, exact} or {setting, pattern} rules: %w", TiersConfigKey, err) - } - return CompileTiers(rules) -} - -// Priority is PHP getPriorityForUrl with configured rules: when the -// settings enable tiers, the URL's path (parse_url's, "/" when it has none) -// right-trimmed of slashes is matched against the rules in order and the -// first match's setting value is returned. ok is false when tiers are off or -// no rule matches, so the item's own priority applies. -func (t Tiers) Priority(url string, s models.Settings) (string, bool) { - if !s.EnablePriorityTiers { - return "", false - } - path := URLPath(url) - if path == "" { - path = "/" - } - path = strings.TrimRight(path, "/") - for _, r := range t { - if r.Pattern != nil && r.Pattern.MatchString(path) || r.Pattern == nil && slices.Contains(r.Exact, path) { - return s.Priority(r.Setting), true - } - } - return "", false -} diff --git a/classes/tiers_test.go b/classes/tiers_test.go deleted file mode 100644 index ab550bb..0000000 --- a/classes/tiers_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package classes_test - -import ( - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" -) - -// TestTiers pins the priority tiers (D-13): each default rule of the PHP -// getPriorityForUrl with paths from its list, trailing slashes, the first -// match winning, the settings' values, tiers turned off, a host list that -// replaces the defaults, and a bad rule failing the plugin's Boot. -func TestTiers(t *testing.T) { - tiers := pluginTiers(t) - if len(tiers) != 5 { - t.Fatalf("default rules = %d", len(tiers)) - } - s := models.DefaultSettings() - for url, want := range map[string]string{ - "https://h": "1.0", - "https://h/": "1.0", - "https://h/en": "1.0", - "https://h/en/": "1.0", - "https://h/uslugi/strony-www": "0.8", - "https://h/en/services/web": "0.8", - "https://h/realizacje/projekt/": "0.8", - "https://h/en/work/x": "0.8", - "https://h/blog/kategoria/wpis": "0.6", - "https://h/en/blog/category/post/": "0.6", - "https://h/blog": "0.4", - "https://h/blog/kategoria": "0.4", - "https://h/uslugi": "0.4", - "https://h/en/work": "0.4", - "https://h/polityka-prywatnosci": "0.2", - "https://h/en/terms/": "0.2", - "https://h/laboratorium-ai/zabawa-animacja": "0.2", - "https://h/en/ai-lab/blob-playground": "0.2", - "https://h/kontakt?x=1": "0.2", - } { - if got, ok := tiers.Priority(url, s); !ok || got != want { - t.Errorf("Priority(%s) = %q %v, want %s", url, got, ok, want) - } - } - for _, url := range []string{"https://h/other", "https://h/blog/a/b/c", "https://h/uslugi/a/b", "https://h/en/blogs/x", "https://h/kontakt/more"} { - if got, ok := tiers.Priority(url, s); ok { - t.Errorf("Priority(%s) = %q, want no rule", url, got) - } - } - custom := s - custom.PriorityBlog = text("0.65") - if got, _ := tiers.Priority("https://h/blog/a/b", custom); got != "0.65" { - t.Fatalf("custom blog priority = %q", got) - } - off := s - off.EnablePriorityTiers = false - if _, ok := tiers.Priority("https://h/", off); ok { - t.Fatal("tiers apply while turned off") - } - - // First match wins: an exact rule listed before a pattern that also matches. - ordered, err := classes.CompileTiers([]classes.TierConfig{ - {Setting: models.PriorityLegal, Exact: []string{"/blog"}}, - {Setting: models.PriorityCategories, Pattern: "^/blog$"}, - }) - if err != nil { - t.Fatal(err) - } - if got, _ := ordered.Priority("https://h/blog/", s); got != "0.2" { - t.Fatalf("first match = %q", got) - } - if empty, err := classes.CompileTiers(nil); err != nil || len(empty) != 0 { - t.Fatalf("no rules = %v %v", empty, err) - } - if none, err := classes.LoadTiers(nil); err != nil || len(none) != 0 { - t.Fatalf("LoadTiers(nil) = %v %v", none, err) - } - - // Every refused rule names its index; a bad RE2 pattern too. - for name, rule := range map[string]classes.TierConfig{ - "unknown setting": {Setting: "priority_other", Exact: []string{""}}, - "neither": {Setting: models.PriorityHomepage}, - "both": {Setting: models.PriorityHomepage, Exact: []string{""}, Pattern: "^/$"}, - "lookahead": {Setting: models.PriorityBlog, Pattern: "^/(?=blog)"}, - "unbalanced": {Setting: models.PriorityBlog, Pattern: "^/(blog"}, - } { - _, err := classes.CompileTiers([]classes.TierConfig{{Setting: models.PriorityHomepage, Exact: []string{""}}, rule}) - if err == nil || !strings.Contains(err.Error(), classes.TiersConfigKey+"[1]") { - t.Errorf("%s: err = %v", name, err) - } - } - -} diff --git a/classes/urlto.go b/classes/urlto.go deleted file mode 100644 index c14d022..0000000 --- a/classes/urlto.go +++ /dev/null @@ -1,566 +0,0 @@ -package classes - -import ( - "regexp" - "strconv" - "strings" -) - -// This file ports the parts of WinterCMS's URL generator the PHP sitemap -// called, so the Go sitemap writes the same bytes: Laravel's Url::to plus -// Winter's UrlGenerator::buildUrl, PHP's parse_url, rawurlencode, -// rawurldecode and urldecode, and FILTER_VALIDATE_URL. It is deliberately -// hand written: net/url escapes differently. The golden table -// testdata/urlto.golden, generated from the vendored Winter router, pins -// it. - -// validURLPrefix is Laravel UrlGenerator::isValidUrl's fast path. -var validURLPrefix = regexp.MustCompile(`^(#|//|https?://|(mailto|tel|sms):)`) - -// URLTo is Winter's Url::to(path) for a request whose root URL is root -// (scheme and host, plus a base path if any, without a trailing slash). A -// valid URL is kept; anything else becomes root plus "/" plus the path -// trimmed of slashes, the query string appended as it is. The result then -// goes through BuildURL, which re-encodes every path segment, so "/" is the -// root with its slash, "/blog/" loses its trailing slash and ":" becomes -// "%3A". -func URLTo(root, path string) string { - if isValidURL(path) { - return BuildURL(path) - } - p, query := path, "" - if i := strings.IndexByte(path, '?'); i >= 0 { - p, query = path[:i], path[i:] - } - inner := strings.Trim(p+"/", "/") - return BuildURL(strings.Trim(root+"/"+inner, "/") + query) -} - -// isValidURL is Laravel UrlGenerator::isValidUrl. -func isValidURL(path string) bool { - if validURLPrefix.MatchString(path) { - return true - } - return filterValidateURL(path) -} - -// filterURLChars is the character set PHP's FILTER_VALIDATE_URL accepts; -// a value with any other byte fails. -const filterURLChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789$-_.+!*'(),{}|\\^~[]`<>#%\";/?:@&=" - -// filterValidateURL approximates PHP filter_var(FILTER_VALIDATE_URL): only -// URL characters, a scheme, a host unless the scheme is mailto, news or -// file, and a valid host name for http and https. -func filterValidateURL(s string) bool { - if s == "" { - return false - } - for i := 0; i < len(s); i++ { - if !strings.ContainsRune(filterURLChars, rune(s[i])) { - return false - } - } - u := parseURL(s) - if !u.has(partScheme) || u.scheme == "" { - return false - } - scheme := strings.ToLower(u.scheme) - if !u.has(partHost) || u.host == "" { - return scheme == "mailto" || scheme == "news" || scheme == "file" - } - if scheme == "http" || scheme == "https" { - return validHostName(u.host) - } - return true -} - -// validHostName reports whether host is dot-separated labels of letters, -// digits and inner hyphens. -func validHostName(host string) bool { - host = strings.TrimSuffix(host, ".") - if host == "" || len(host) > 253 { - return false - } - for _, label := range strings.Split(host, ".") { - if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { - return false - } - for i := 0; i < len(label); i++ { - c := label[i] - if !(isAlnum(c) || c == '-') { - return false - } - } - } - return true -} - -// The parts a parsed URL may hold, as PHP parse_url's array keys. -const ( - partScheme = 1 << iota - partHost - partUser - partPass - partPort - partPath - partQuery - partFragment -) - -// phpURL is the array PHP parse_url returns; set records which keys exist. -type phpURL struct { - set int - scheme, host, user, pass, port, path, query, frag string -} - -func (u phpURL) has(part int) bool { return u.set&part != 0 } - -// parseURL ports PHP parse_url for the URL shapes a sitemap meets: an -// optional scheme, an authority after "//", a path, a query and a fragment. -// Control characters in a part are replaced by "_", as PHP does. -func parseURL(s string) phpURL { - var u phpURL - rest := s - if i := strings.IndexByte(rest, ':'); i > 0 && validScheme(rest[:i]) { - after := rest[i+1:] - if !strings.HasPrefix(after, "//") && portOnly(after) { - // "host:port/path" without a scheme. - } else { - u.scheme, u.set = rest[:i], u.set|partScheme - rest = after - } - } - if strings.HasPrefix(rest, "//") { - rest = rest[2:] - end := strings.IndexAny(rest, "/?#") - authority := rest - if end >= 0 { - authority, rest = rest[:end], rest[end:] - } else { - rest = "" - } - u.parseAuthority(authority) - } - if i := strings.IndexByte(rest, '#'); i >= 0 { - u.frag, u.set = rest[i+1:], u.set|partFragment - rest = rest[:i] - } - if i := strings.IndexByte(rest, '?'); i >= 0 { - u.query, u.set = rest[i+1:], u.set|partQuery - rest = rest[:i] - } - if rest != "" { - u.path, u.set = rest, u.set|partPath - } - u.scheme = replaceControl(u.scheme) - u.host = replaceControl(u.host) - u.user = replaceControl(u.user) - u.pass = replaceControl(u.pass) - u.path = replaceControl(u.path) - u.query = replaceControl(u.query) - u.frag = replaceControl(u.frag) - return u -} - -func (u *phpURL) parseAuthority(a string) { - if i := strings.LastIndexByte(a, '@'); i >= 0 { - userinfo := a[:i] - a = a[i+1:] - if j := strings.IndexByte(userinfo, ':'); j >= 0 { - u.user, u.pass = userinfo[:j], userinfo[j+1:] - u.set |= partUser | partPass - } else { - u.user = userinfo - u.set |= partUser - } - } - if i := strings.LastIndexByte(a, ':'); i >= 0 && !strings.HasSuffix(a, "]") { - port := a[i+1:] - if port != "" && allDigits(port) { - u.port, u.set = port, u.set|partPort - } - a = a[:i] - } - if a != "" { - u.host, u.set = a, u.set|partHost - } -} - -func validScheme(s string) bool { - for i := 0; i < len(s); i++ { - c := s[i] - if !(isAlnum(c) || c == '+' || c == '-' || c == '.') { - return false - } - } - return s != "" -} - -// portOnly reports whether s is a port number up to the first "/", the -// case PHP parse_url reads as host:port rather than scheme:path. -func portOnly(s string) bool { - end := strings.IndexByte(s, '/') - if end < 0 { - end = len(s) - } - return end > 0 && end <= 5 && allDigits(s[:end]) -} - -func allDigits(s string) bool { - for i := 0; i < len(s); i++ { - if s[i] < '0' || s[i] > '9' { - return false - } - } - return s != "" -} - -func isAlnum(c byte) bool { - return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' -} - -func replaceControl(s string) string { - if strings.IndexFunc(s, func(r rune) bool { return r < 0x20 || r == 0x7f }) < 0 { - return s - } - b := []byte(s) - for i, c := range b { - if c < 0x20 || c == 0x7f { - b[i] = '_' - } - } - return string(b) -} - -// phpTrim is PHP trim() with its default character list. -func phpTrim(s string) string { - return strings.Trim(s, " \t\n\r\x00\x0b") -} - -// singleColonSchemes are written "scheme:" instead of "scheme://". -var singleColonSchemes = map[string]bool{"mailto": true, "tel": true, "sms": true} - -// defaultPorts are Winter UrlGenerator::isDefaultPort's known ports. -var defaultPorts = map[string]int{ - "ftp": 21, "ssh": 22, "sftp": 22, "telnet": 23, "smtp": 25, "dns": 53, - "gopher": 70, "http": 80, "pop3": 110, "imap": 143, "ldap": 389, - "https": 443, "smtps": 465, "ldaps": 636, "ftps": 990, "imaps": 993, - "pop3s": 995, -} - -// BuildURL is Winter UrlGenerator::buildUrl($url) with no replacements: the -// query is URL-decoded, the URL parsed, every path segment re-encoded with -// rawurlencode(rawurldecode()) after "." and ".." are resolved, the query -// rebuilt with buildStr and the fragment re-encoded. -func BuildURL(raw string) string { - return buildParts(parseURL(decodeQueryParams(raw))) -} - -// replacePath is buildUrl($url, ['path' => path]). -func replacePath(raw, path string) string { - u := parseURL(decodeQueryParams(raw)) - u.path, u.set = path, u.set|partPath - return buildParts(u) -} - -// decodeQueryParams URL-decodes everything after the first "?". -func decodeQueryParams(s string) string { - if i := strings.IndexByte(s, '?'); i >= 0 { - return s[:i] + "?" + urldecode(s[i+1:]) - } - return s -} - -func buildParts(u phpURL) string { - u.scheme, u.host, u.user, u.pass = phpTrim(u.scheme), phpTrim(u.host), phpTrim(u.user), phpTrim(u.pass) - u.path, u.query, u.frag = phpTrim(u.path), phpTrim(u.query), phpTrim(u.frag) - port := 0 - if u.has(partPort) { - if n, err := strconv.Atoi(phpTrim(u.port)); err == nil && n >= 1 && n <= 65535 { - port = n - } - } - var b strings.Builder - hasScheme := u.has(partScheme) && u.scheme != "" - if hasScheme { - b.WriteString(u.scheme) - if singleColonSchemes[u.scheme] { - b.WriteString(":") - } else { - b.WriteString("://") - } - } - if u.has(partUser) && u.user != "" { - b.WriteString(u.user) - if u.has(partPass) && u.pass != "" { - b.WriteString(":" + u.pass) - } - b.WriteString("@") - } - if u.has(partHost) && u.host != "" { - b.WriteString(u.host) - } - if port != 0 { - if def, ok := defaultPorts[strings.ToLower(u.scheme)]; !ok || def != port { - b.WriteString(":" + strconv.Itoa(port)) - } - } - path := "" - if u.has(partPath) && u.path != "" { - var kept []string - for _, seg := range strings.Split(u.path, "/") { - switch { - case seg == ".": - case seg == ".." && len(kept) > 0: - kept = kept[:len(kept)-1] - default: - kept = append(kept, seg) - } - } - for i, seg := range kept { - kept[i] = rawurlencode(rawurldecode(seg)) - } - path = strings.Join(kept, "/") - } - if hasScheme && !singleColonSchemes[u.scheme] { - if !strings.HasPrefix(path, "/") { - b.WriteString("/") - } - b.WriteString(path) - } else { - b.WriteString(path) - } - if u.has(partQuery) && u.query != "" { - b.WriteString("?" + buildStr(parseQueryPairs(u.query), "")) - } - if u.has(partFragment) && u.frag != "" { - b.WriteString("#" + rawurlencode(rawurldecode(u.frag))) - } - return b.String() -} - -// phpKey is a PHP array key: an int or a string. -type phpKey struct { - isInt bool - i int64 - s string -} - -func (k phpKey) String() string { - if k.isInt { - return strconv.FormatInt(k.i, 10) - } - return k.s -} - -// keyOf is PHP's array key cast of a string: a canonical decimal integer -// becomes an int key. -func keyOf(s string) phpKey { - if n, err := strconv.ParseInt(s, 10, 64); err == nil && strconv.FormatInt(n, 10) == s { - return phpKey{isInt: true, i: n} - } - return phpKey{s: s} -} - -// phpArray is an ordered PHP array of strings, nested arrays or nulls. -type phpArray struct { - keys []phpKey - vals []any - next int64 -} - -func (a *phpArray) index(k phpKey) int { - for i, have := range a.keys { - if have == k { - return i - } - } - return -1 -} - -func (a *phpArray) set(k phpKey, v any) { - if i := a.index(k); i >= 0 { - a.vals[i] = v - return - } - a.keys = append(a.keys, k) - a.vals = append(a.vals, v) - if k.isInt && k.i >= a.next { - a.next = k.i + 1 - } -} - -func (a *phpArray) push(v any) { - a.set(phpKey{isInt: true, i: a.next}, v) -} - -// parseQueryPairs is buildUrl's split of a string query into -// $queryParams. -func parseQueryPairs(query string) *phpArray { - params := &phpArray{} - for _, pair := range strings.Split(query, "&") { - key, value := pair, pair - if i := strings.IndexByte(pair, '='); i >= 0 { - key, value = pair[:i], pair[i+1:] - } - if !strings.Contains(pair, "=") { - params.push(key) - continue - } - if strings.Contains(urldecode(pair), "[") { - k := keyOf(key) - var inner *phpArray - if i := params.index(k); i >= 0 { - inner, _ = params.vals[i].(*phpArray) - } - if inner == nil { - inner = &phpArray{} - } - inner.push(value) - params.set(k, inner) - continue - } - params.set(keyOf(key), value) - } - return params -} - -// buildStr is Winter UrlGenerator::buildStr. -func buildStr(query *phpArray, prefix string) string { - var out []string - for i, k := range query.keys { - v := query.vals[i] - var key string - if k.isInt && k.i == int64(i) { - if prefix != "" { - if strings.HasSuffix(prefix, "]") { - key = prefix - } else { - key = prefix + "[]" - } - } else { - key, _ = v.(string) - v = nil - } - } else if prefix != "" { - key = prefix + "[" + k.String() + "]" - } else { - key = k.String() - } - if nested, ok := v.(*phpArray); ok { - out = append(out, buildStr(nested, key)) - continue - } - part := rawurlencode(rawurldecode(key)) - if s, ok := v.(string); ok { - part += "=" + rawurlencode(rawurldecode(s)) - } - out = append(out, part) - } - return strings.Join(out, "&") -} - -// rawurlencode is PHP rawurlencode: every byte except A-Z a-z 0-9 - _ . ~ -// becomes %XX with upper-case hex. -func rawurlencode(s string) string { - const hex = "0123456789ABCDEF" - var b strings.Builder - for i := 0; i < len(s); i++ { - c := s[i] - if isAlnum(c) || c == '-' || c == '_' || c == '.' || c == '~' { - b.WriteByte(c) - continue - } - b.WriteByte('%') - b.WriteByte(hex[c>>4]) - b.WriteByte(hex[c&15]) - } - return b.String() -} - -// rawurldecode is PHP rawurldecode: %XX becomes its byte, anything else, -// including a malformed escape, stays. -func rawurldecode(s string) string { return percentDecode(s, false) } - -// urldecode is PHP urldecode: rawurldecode that also turns "+" into a -// space. -func urldecode(s string) string { return percentDecode(s, true) } - -func percentDecode(s string, plus bool) string { - if !strings.ContainsAny(s, "%+") { - return s - } - var b strings.Builder - for i := 0; i < len(s); i++ { - c := s[i] - if c == '+' && plus { - b.WriteByte(' ') - continue - } - if c == '%' && i+2 < len(s) && isHex(s[i+1]) && isHex(s[i+2]) { - b.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2])) - i += 2 - continue - } - b.WriteByte(c) - } - return b.String() -} - -func isHex(c byte) bool { - return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' -} - -func unhex(c byte) byte { - switch { - case '0' <= c && c <= '9': - return c - '0' - case 'a' <= c && c <= 'f': - return c - 'a' + 10 - default: - return c - 'A' + 10 - } -} - -// URLPath is the path PHP parse_url($url, PHP_URL_PATH) returns, "" when -// the URL has none. -func URLPath(raw string) string { - u := parseURL(raw) - if !u.has(partPath) { - return "" - } - return u.path -} - -// schemePrefix matches a URL that starts with a scheme and its colon. -var schemePrefix = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`) - -// AllowedItemURL reports whether a url item's address may be stored: empty, -// a path (relative or starting with a single "/"), or an absolute http or -// https URL. Any other scheme (javascript:, data:, mailto: and the like), a -// protocol-relative "//host" or "/\host", a bare fragment, surrounding -// whitespace and control characters are refused. Url::to keeps any URL -// FILTER_VALIDATE_URL accepts, so without this a javascript: URL reached -// , which sitemap.xsl renders as a link. -func AllowedItemURL(u string) bool { - if u == "" { - return true - } - if strings.TrimSpace(u) != u || strings.ContainsFunc(u, func(r rune) bool { return r < 0x20 || r == 0x7f }) { - return false - } - if httpURL(u) { - return true - } - if schemePrefix.MatchString(u) || u[0] == '#' || u[0] == '\\' { - return false - } - return !strings.HasPrefix(u, "//") && !strings.HasPrefix(u, `/\`) -} - -// httpURL reports whether u starts with http:// or https:// -// (case-insensitive). -func httpURL(u string) bool { - lower := strings.ToLower(u) - return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") -} diff --git a/classes/urlto_test.go b/classes/urlto_test.go deleted file mode 100644 index bbf424e..0000000 --- a/classes/urlto_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package classes_test - -import ( - "encoding/json" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" -) - -// TestURLTo runs every case of testdata/urlto.json through URLTo and -// compares it with the line PHP's Url::to wrote to urlto.golden (SC-3), then -// pins the helpers built on it: BackendURL and URLPath. -func TestURLTo(t *testing.T) { - var in struct { - Root string `json:"root"` - URLs []*string `json:"urls"` - } - if err := json.Unmarshal(readTestdata(t, "urlto.json"), &in); err != nil { - t.Fatal(err) - } - want := strings.Split(strings.TrimSuffix(string(readTestdata(t, "urlto.golden")), "\n"), "\n") - if len(in.URLs) < 50 || len(want) != len(in.URLs) { - t.Fatalf("%d inputs, %d golden lines", len(in.URLs), len(want)) - } - for i, u := range in.URLs { - path := "" - input := "" - if u != nil { - path, input = *u, *u - } - if got := classes.URLTo(in.Root, input); got != want[i] { - t.Errorf("case %d URLTo(%q) = %q, PHP %q", i, path, got, want[i]) - } - } - - for _, tc := range []struct{ root, want string }{ - {"https://api.ex.pl", "https://api.ex.pl"}, - {"https://api.ex.pl/", "https://api.ex.pl"}, - {"http://h:8080/base", "http://h:8080/base"}, - } { - if got := classes.BackendURL(tc.root); got != tc.want { - t.Errorf("BackendURL(%q) = %q, want %q", tc.root, got, tc.want) - } - } - for raw, want := range map[string]string{ - "https://h/a/b?x=1#f": "/a/b", - "https://h": "", - "https://h/": "/", - "/rel/path": "/rel/path", - "": "", - } { - if got := classes.URLPath(raw); got != want { - t.Errorf("URLPath(%q) = %q, want %q", raw, got, want) - } - } -} - -// TestAllowedItemURL pins the save rule of a url item's address (CR-01). -func TestAllowedItemURL(t *testing.T) { - for u, want := range map[string]bool{ - "": true, - "/": true, - "/about": true, - "about/team?x=1": true, - "https://example.test/a": true, - "HTTP://example.test/a": true, - "javascript://%0aalert(1)%0a//x": false, - "javascript:alert(1)": false, - "data:text/html,x": false, - "mailto:a@example.test": false, - "//evil.test/x": false, - `/\evil.test/x`: false, - `\\evil.test`: false, - "#top": false, - " /about": false, - "/about\n": false, - "java\tscript:alert(1)": false, - } { - if got := classes.AllowedItemURL(u); got != want { - t.Errorf("AllowedItemURL(%q) = %v, want %v", u, got, want) - } - } -} diff --git a/config/config.yaml b/config/config.yaml index 6d346a7..89018c6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,44 +1,20 @@ # golem15.sitemap defaults, read as golem15.sitemap.. # -# cache_seconds is how long a generated /sitemap.xml is served from memory. -# A save of the definition or the settings shows at once; registry content -# (a new or changed post) shows once the entry expires. 0 turns the cache -# off. -cache_seconds: 60 +# site_url is the public site address every of the sitemaps starts +# with when the Site address setting is empty; without both, app.url is +# used. The request's Host is never used. Set it per environment, e.g.: # -# The priority tier rules assign a URL the value of one of the five -# priority settings when Enable priority tiers is on. A URL's path, without -# trailing slashes ("" for the homepage), is matched against the rules in -# order and the first match wins; a URL no rule matches keeps its item's -# priority. Each rule names a setting (priority_homepage, priority_services, -# priority_blog, priority_categories or priority_legal) and either exact -# (a list of paths) or pattern (a regular expression in RE2 syntax, checked -# at boot). +# site_url: https://www.example.com # -# The defaults below are the rules the WinterCMS plugin had in code -# (getPriorityForUrl), so a site configured the same way produces the same -# priorities. A host replaces the whole list in its own config, for example -# in config/golem15.yaml under the same key below sitemap:. -priority_tiers: - - setting: priority_homepage - exact: ["", "/en"] - - setting: priority_services - pattern: '^/(uslugi|en/services|realizacje|en/work)/[^/]+$' - - setting: priority_blog - pattern: '^/(blog|en/blog)/[^/]+/[^/]+$' - - setting: priority_categories - pattern: '^/(uslugi|en/services|realizacje|en/work|blog|en/blog)(/[^/]+)?$' - - setting: priority_legal - exact: - - /polityka-prywatnosci - - /en/privacy-policy - - /regulamin - - /en/terms - - /o-nas - - /en/about - - /kontakt - - /en/contact - - /laboratorium-ai - - /en/ai-lab - - /laboratorium-ai/zabawa-animacja - - /en/ai-lab/blob-playground +# lastmod_ignore lists the record attributes whose change does not move an +# address's lastmod: a save that changes only these (or nothing) leaves it +# as it was. A host replaces the whole list in its own config. +lastmod_ignore: + - updated_at + - created_at + - views + - sort_order +# +# rebuild_delay_seconds is how long after a record change the queued +# rebuild runs; further changes in that time join the same rebuild. +rebuild_delay_seconds: 60 diff --git a/console/helpers.go b/console/helpers.go index 4912bcb..7d35fe3 100644 --- a/console/helpers.go +++ b/console/helpers.go @@ -1,6 +1,7 @@ -// Package console holds the sitemap plugin's commands: sitemap:import (the -// PHP ImportCommand) and sitemap:import-winter (a copy of a WinterCMS -// site's definition and settings). +// Package console holds the sitemap plugin's commands: sitemap:rebuild, +// sitemap:import (a JSON list of additional addresses) and +// sitemap:import-winter (a copy of a WinterCMS site's url items and +// settings). package console import ( @@ -56,20 +57,24 @@ func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) return gdb, func() { _ = sqlDB.Close() }, nil } -// singletonDefinition loads the definition with the fixed theme key, -// creating it when there is none (PHP Definition::firstOrCreate). -func singletonDefinition(tx *gorm.DB) (models.Definition, error) { - var defs []models.Definition - if err := tx.Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { - return models.Definition{}, err +// loadSettingsForUpdate reads settings row 1 inside tx, locked, creating +// it when the migration's row is gone. +func loadSettingsForUpdate(tx *gorm.DB) (models.Settings, error) { + if err := tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error; err != nil { + return models.Settings{}, err } - if len(defs) > 0 { - return defs[0], nil + var s models.Settings + if err := tx.Raw(`SELECT * FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Scan(&s).Error; err != nil { + return models.Settings{}, err } - theme := models.DefinitionTheme - def := models.Definition{Theme: &theme} - if err := tx.Create(&def).Error; err != nil { - return models.Definition{}, err - } - return def, nil + return s, nil +} + +// saveExtra stores the additional addresses on settings row 1. +func saveExtra(tx *gorm.DB, rows []models.ExtraURL) error { + if rows == nil { + rows = []models.ExtraURL{} + } + value := lagoon.Jsonable[[]models.ExtraURL]{Data: rows, Valid: true} + return tx.Exec(`UPDATE golem15_sitemap_settings SET extra_urls = ? WHERE id = 1`, value).Error } diff --git a/console/import.go b/console/import.go index 3d47adb..f238fa2 100644 --- a/console/import.go +++ b/console/import.go @@ -1,55 +1,55 @@ package console import ( - "bytes" "context" "encoding/json" "errors" "fmt" "os" + "strings" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "gorm.io/gorm" ) -// Import is sitemap:import FILE [--theme=] [--append], the PHP -// ImportCommand: it reads a JSON array of URL entries ({"url": ..., -// "changefreq": ..., "priority": ...}; changefreq defaults to monthly, -// priority to 0.5) into the sitemap definition. Without --append the -// definition keeps its non-url items first and the imported URLs replace -// its url items; with --append the URLs are added after every existing item. -// The definition is always the one /sitemap.xml serves (theme default); -// --theme is accepted for PHP compatibility and a theme other than default -// is reported. An entry whose url is not a path or an http(s) URL -// (classes.AllowedItemURL) is refused, as the admin form refuses it. +// Import is sitemap:import FILE [--append]: it reads a JSON array of URL +// entries ({"url": "/path", "lastmod": "2026-10-28"}; lastmod optional) into +// the settings' additional addresses, in one transaction. Without --append +// the imported addresses replace the additional addresses; with --append +// they are added after them, an address already listed is skipped. An +// entry whose url is not a path or an http(s) URL (models.AllowedAddress), +// or whose lastmod is not a date, stops the import before anything is +// written. The Phase 17 keys changefreq and priority are ignored, and +// --theme is accepted for the PHP command's sake. The files follow at the +// next rebuild (sitemap:rebuild). func Import(app *backpack.App) bonfire.Command { return bonfire.Command{ Name: "sitemap:import", - Description: "Import sitemap URLs from a JSON manifest file", - Long: `Reads a JSON array of URL entries ({"url": "/path", "changefreq": -"monthly", "priority": "0.5"}; changefreq and priority are optional) and puts -them into the sitemap definition that /sitemap.xml serves, in one -transaction. Every entry needs a url that is a path or an http(s) URL; one -bad entry stops the import before anything is written. + Description: "Import additional sitemap addresses from a JSON manifest file", + Long: `Reads a JSON array of URL entries ({"url": "/path", "lastmod": +"2026-10-28"}; lastmod is optional) and puts them into the sitemap's +additional addresses (Settings > Sitemaps > Additional addresses), in one +transaction. Every entry needs a url that is a path or an http(s) URL; one bad +entry stops the import before anything is written. The old changefreq and +priority keys are ignored: the sitemaps no longer write them. -By default the import replaces the definition's url items and keeps its -dynamic items (pages, posts and other generated types). --append adds the -URLs and keeps every existing item. --theme is accepted for the PHP command's -sake: only the default definition is served, so the URLs always go there. -It ends with "Imported N URLs into sitemap definition for theme 'default'."`, +By default the import replaces the additional addresses. --append adds the +addresses after them and skips those already listed. --theme is accepted for +the PHP command's sake and does nothing. The files change at the next rebuild +(sitemap:rebuild, a settings save or the nightly run). It ends with +"Imported N addresses into the sitemap's additional addresses."`, Args: []bonfire.Arg{ {Name: "file", Description: "Path to the JSON manifest (relative to the working directory or absolute)", Required: true}, }, Flags: []bonfire.Flag{ - {Name: "theme", Description: "Theme code of the PHP plugin; /sitemap.xml serves only the default definition"}, - {Name: "append", Description: "Add URLs without removing existing url-type items", Bare: true}, + {Name: "theme", Description: "Accepted for the PHP command; ignored"}, + {Name: "append", Description: "Add the addresses instead of replacing the additional addresses", Bare: true}, }, Examples: []bonfire.Example{ - {Command: "sitemap:import urls.json", Description: "Replace the url items with the manifest's URLs"}, - {Command: "sitemap:import urls.json --append", Description: "Add the URLs and keep every existing item"}, + {Command: "sitemap:import urls.json", Description: "Replace the additional addresses with the manifest's addresses"}, + {Command: "sitemap:import urls.json --append", Description: "Add the addresses and keep the existing ones"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { file, _ := in.Argument("file") @@ -72,136 +72,67 @@ func ImportFile(ctx context.Context, db *gorm.DB, file, theme string, appendMode } return fmt.Errorf("Cannot read %s: %w", file, err) } - entries, labels, ok := orderedEntries(raw) + entries, labels, ok := models.LegacyEntries(raw) if !ok { return errors.New("Invalid JSON: expected an array of URL entries") } - newItems := make([]models.DefinitionItem, 0, len(entries)) + rows := make([]models.ExtraURL, 0, len(entries)) for i, entry := range entries { var fields map[string]json.RawMessage _ = json.Unmarshal(entry, &fields) url, err := models.DecodeText(fields["url"]) - if fields["url"] == nil || err != nil || phpEmpty(fields["url"]) { + if fields["url"] == nil || err != nil || strings.TrimSpace(models.Text(url)) == "" { return fmt.Errorf("Entry %s is missing a 'url' field", labels[i]) } - if !classes.AllowedItemURL(models.Text(url)) { + u := strings.TrimSpace(models.Text(url)) + if !models.AllowedAddress(u) { return fmt.Errorf("Entry %s has a url that is not a path or an http(s) URL", labels[i]) } - item := models.DefinitionItem{Type: models.ItemTypeURL, URL: url} - item.Changefreq = textOr(fields["changefreq"], "monthly") - item.Priority = textOr(fields["priority"], "0.5") - newItems = append(newItems, item) + row := models.ExtraURL{URL: u} + if raw, ok := fields["lastmod"]; ok { + mod, err := models.DecodeText(raw) + if err != nil || (strings.TrimSpace(models.Text(mod)) != "" && !models.ValidDate(models.Text(mod))) { + return fmt.Errorf("Entry %s has a lastmod that is not a date (YYYY-MM-DD)", labels[i]) + } + row.LastMod = strings.TrimSpace(models.Text(mod)) + } + rows = append(rows, row) } - var kept int + var kept, added int err = db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - def, err := singletonDefinition(tx) + s, err := loadSettingsForUpdate(tx) if err != nil { return err } - existing := def.Items() - if !appendMode { - var dynamic []models.DefinitionItem - for _, item := range existing { - if item.Type != models.ItemTypeURL { - dynamic = append(dynamic, item) - } + next := []models.ExtraURL{} + seen := map[string]bool{} + if appendMode { + for _, e := range s.Extra() { + next = append(next, e) + seen[e.URL] = true } - existing = dynamic + kept = len(next) } - kept = len(existing) - def.SetItems(append(append([]models.DefinitionItem{}, existing...), newItems...)) - return tx.Save(&def).Error + for _, r := range rows { + if seen[r.URL] { + continue + } + seen[r.URL] = true + next = append(next, r) + added++ + } + return saveExtra(tx, next) }) if err != nil { return err } - out.Success(fmt.Sprintf("Imported %d URLs into sitemap definition for theme '%s'.", len(newItems), models.DefinitionTheme)) + out.Success(fmt.Sprintf("Imported %d addresses into the sitemap's additional addresses.", added)) if kept > 0 { - label := "existing non-URL (dynamic)" - if appendMode { - label = "existing" - } - out.Info(fmt.Sprintf("Kept %d %s items.", kept, label)) + out.Info(fmt.Sprintf("Kept %d existing addresses.", kept)) } - if theme != "" && theme != models.DefinitionTheme { - out.Warning(fmt.Sprintf("Theme '%s' was given, but /sitemap.xml serves only the '%s' definition, so the URLs were imported there.", theme, models.DefinitionTheme)) + if theme != "" { + out.Warning(fmt.Sprintf("Theme '%s' was given; the sitemap has one set of additional addresses, so the addresses were imported there.", theme)) } + out.Info("Run sitemap:rebuild (or save the sitemap settings) to update the files now.") return nil } - -// orderedEntries splits a JSON array, or a JSON object in key order (a PHP -// array with keys), into its values with their PHP labels (index or key). -func orderedEntries(raw []byte) ([]json.RawMessage, []string, bool) { - raw = bytes.TrimSpace(raw) - if len(raw) == 0 { - return nil, nil, false - } - switch raw[0] { - case '[': - var list []json.RawMessage - if err := json.Unmarshal(raw, &list); err != nil { - return nil, nil, false - } - labels := make([]string, len(list)) - for i := range list { - labels[i] = fmt.Sprint(i) - } - return list, labels, true - case '{': - dec := json.NewDecoder(bytes.NewReader(raw)) - if _, err := dec.Token(); err != nil { - return nil, nil, false - } - var list []json.RawMessage - var labels []string - for dec.More() { - tok, err := dec.Token() - if err != nil { - return nil, nil, false - } - key, _ := tok.(string) - var v json.RawMessage - if err := dec.Decode(&v); err != nil { - return nil, nil, false - } - list = append(list, v) - labels = append(labels, key) - } - return list, labels, true - } - return nil, nil, false -} - -// phpEmpty is PHP empty() of a decoded JSON value. -func phpEmpty(raw json.RawMessage) bool { - var v any - if err := json.Unmarshal(raw, &v); err != nil { - return true - } - switch t := v.(type) { - case nil: - return true - case string: - return t == "" || t == "0" - case bool: - return !t - case float64: - return t == 0 - case []any: - return len(t) == 0 - case map[string]any: - return len(t) == 0 - } - return false -} - -// textOr is PHP `$entry[key] ?? fallback` as text: a missing or null value -// is the fallback. -func textOr(raw json.RawMessage, fallback string) *string { - if raw != nil { - if s, err := models.DecodeText(raw); err == nil && s != nil { - return s - } - } - return &fallback -} diff --git a/console/import_winter.go b/console/import_winter.go index 066a1b7..b7700d9 100644 --- a/console/import_winter.go +++ b/console/import_winter.go @@ -8,15 +8,12 @@ import ( "errors" "fmt" "os" - "slices" "sort" "strings" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" - "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) @@ -34,11 +31,12 @@ var droppedTypes = map[string]bool{ // of scripts/winter-export-settings.sql (the JSON object of the // golem15_sitemap_settings system setting). Theme picks a definition when // the export holds several. Maps rename item types (old to new) before they -// are checked. PageURLs turns a static or CMS page item into a url item: -// its key is the item's reference (the PHP page file name, for example -// blog), its value the page's URL (/blog). Force replaces a definition that -// has items; DryRun reports without writing. A frontend_url that -// models.AllowedFrontendURL refuses stops the import and nothing is written. +// are checked. PageURLs turns a static or CMS page item into an additional +// address: its key is the item's reference (the PHP page file name, for +// example blog), its value the page's URL (/blog). Force replaces +// additional addresses that are already stored; DryRun reports without +// writing. A frontend_url that models.AllowedFrontendURL refuses stops the +// import and nothing is written. type WinterOptions struct { DefinitionsFile string SettingsFile string @@ -49,7 +47,7 @@ type WinterOptions struct { DryRun bool } -// WinterReport is what sitemap:import-winter did: the items kept, the +// WinterReport is what sitemap:import-winter did: the addresses kept, the // report lines of every dropped or remapped item, and whether the settings // were copied. type WinterReport struct { @@ -60,50 +58,59 @@ type WinterReport struct { Settings bool } +// winterSettings are the WinterCMS settings the per-type sitemap reads. +type winterSettings struct { + SiteURL string + DefaultLocale string +} + // ImportWinter is sitemap:import-winter: it copies a WinterCMS site's -// sitemap definition and settings into the definition /sitemap.xml serves -// (theme default) and the settings row. Item types are mapped (url kept, -// static and CMS page types dropped unless --page-url reference=/path names -// their URL, --map old=new renames applied first), -// an item of a registry type that takes references is dropped when its -// reference is not among the type's current references, an item of an -// unknown type is dropped, a url item whose URL is not a path or an http(s) -// URL (classes.AllowedItemURL) is dropped, a URL pattern that is not a path -// (signpost.ValidItemPattern) is cleared, and every dropped, remapped or -// cleared item is printed. A definition that already has items is refused -// without --force. Settings keys missing from the export take the PHP -// defaults. A frontend_url that models.AllowedFrontendURL refuses stops the -// import and nothing is written. +// sitemap definition into the settings' additional addresses and its +// frontend_url and frontend_default_locale into the site address and the +// default locale (D-32). url items are kept (static and CMS page items +// become addresses when --page-url reference=/path names their URL, +// --map old=new renames applied first); every other item is dropped and +// listed, because the content types list their records themselves. A url +// item whose URL is not a path or an http(s) URL (models.AllowedAddress) +// is dropped. Stored additional addresses are replaced only with --force. +// Settings keys of the PHP-parity format (priorities, tiers, the update +// manifest) are reported as ignored. A frontend_url that +// models.AllowedFrontendURL refuses stops the import and nothing is +// written. func ImportWinter(app *backpack.App) bonfire.Command { return bonfire.Command{ Name: "sitemap:import-winter", Description: "Copy a WinterCMS sitemap definition and settings export into this application", Long: `Copies the sitemap definition of a WinterCMS site (the JSON lines that scripts/winter-export-definitions.sql prints) and its settings (the JSON -object from scripts/winter-export-settings.sql) in one transaction. Item -types this application knows are kept; --map renames a type (old=new) and ---page-url turns a static or CMS page item into a url item with the given -path. Items that still have no known type are dropped and listed. When the -export holds several themes, --theme picks the one to copy. +object from scripts/winter-export-settings.sql) in one transaction. The +definition's url items become the sitemap's additional addresses; --page-url +turns a static or CMS page item into an address with the given path, and +--map renames a type first (old=new). Every other item is dropped and listed: +the content types (posts, categories and so on) list their records +themselves. The settings' frontend_url becomes the site address and +frontend_default_locale the default locale; priorities, tiers and the update +manifest are no longer used. When the export holds several themes, --theme +picks the one to copy. Run it once when an application moves from WinterCMS, --dry-run first: ---dry-run reports kept, remapped and dropped items and writes nothing. A -definition that already has items is replaced only with --force. It ends -with "Definition copied: N items kept, N remapped, N dropped." and "Settings -copied." for the parts the export holds.`, +--dry-run reports kept, remapped and dropped items and writes nothing. Stored +additional addresses are replaced only with --force. It ends with "Definition +copied: N items kept, N remapped, N dropped." and "Settings copied." for the +parts the export holds.`, Flags: []bonfire.Flag{ {Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"}, {Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"}, {Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"}, {Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true}, - {Name: "page-url", Description: "Copy a static or CMS page item as a url item, reference=/path (repeatable)", Repeatable: true}, - {Name: "force", Description: "Replace a definition that already has items", Bare: true}, + {Name: "page-url", Description: "Copy a static or CMS page item as an address, reference=/path (repeatable)", Repeatable: true}, + {Name: "force", Description: "Replace additional addresses that are already stored", Bare: true}, {Name: "dry-run", Description: "Report what would be copied without writing", Bare: true}, }, Examples: []bonfire.Example{ {Command: "sitemap:import-winter --definitions definitions.jsonl --settings settings.json --dry-run", Description: "Show what would be kept, remapped and dropped, without writing"}, - {Command: "sitemap:import-winter --definitions definitions.jsonl --theme main --map blog-post=journal-post", Description: "Copy the main theme's definition, renaming an item type"}, - {Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the definition, turning the about page into a url item"}, + {Command: "sitemap:import-winter --definitions definitions.jsonl --theme main", Description: "Copy the url items of the main theme's definition"}, + {Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the additional addresses, turning the about page into an address"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { maps, err := parseMaps(in.Flags("map")) @@ -128,11 +135,7 @@ copied." for the parts the export holds.`, return err } defer done() - var reg *signpost.Registry - if app != nil { - reg, _ = app.Lookup[*signpost.Registry]() - } - _, err = ImportWinterFiles(ctx, db, reg, opts, out) + _, err = ImportWinterFiles(ctx, db, opts, out) return err }, } @@ -178,15 +181,14 @@ type winterDefinition struct { Data json.RawMessage `json:"data"` } -// ImportWinterFiles runs sitemap:import-winter against db; reg is the menu -// item type registry the item types are checked against (nil knows none). -func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) { +// ImportWinterFiles runs sitemap:import-winter against db. +func ImportWinterFiles(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) { var report WinterReport if opts.DefinitionsFile == "" && opts.SettingsFile == "" { return report, errors.New("give --definitions, --settings or both") } var ( - items []models.DefinitionItem + rows []models.ExtraURL haveDef bool ) if opts.DefinitionsFile != "" { @@ -194,14 +196,14 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, if err != nil { return report, err } - out.Info(fmt.Sprintf("Copying definition %s (theme '%s') into the '%s' definition.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme), models.DefinitionTheme)) - items, err = mapItems(ctx, reg, row.Data, opts.Maps, opts.PageURLs, &report) + out.Info(fmt.Sprintf("Copying the url items of definition %s (theme '%s') into the additional addresses.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme))) + rows, err = mapItems(row.Data, opts.Maps, opts.PageURLs, &report) if err != nil { return report, err } haveDef = true } - var settings models.Settings + var settings winterSettings if opts.SettingsFile != "" { var err error settings, err = readWinterSettings(opts.SettingsFile, &report) @@ -214,12 +216,12 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, out.Warning(line) } if haveDef { - var existing []models.Definition - if err := db.WithContext(ctx).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&existing).Error; err != nil { + var existing models.Settings + if err := db.WithContext(ctx).Where("id = ?", 1).Limit(1).Find(&existing).Error; err != nil { return report, err } - if len(existing) > 0 && len(existing[0].Items()) > 0 && !opts.Force { - msg := fmt.Sprintf("The sitemap definition already has %d items; rerun with --force to replace them.", len(existing[0].Items())) + if n := len(existing.Extra()); n > 0 && !opts.Force { + msg := fmt.Sprintf("The sitemap already has %d additional addresses; rerun with --force to replace them.", n) if !opts.DryRun { return report, errors.New(msg) } @@ -238,19 +240,17 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, return report, nil } err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if _, err := loadSettingsForUpdate(tx); err != nil { + return err + } if haveDef { - def, err := singletonDefinition(tx) - if err != nil { - return err - } - def.SetItems(items) - if err := tx.Save(&def).Error; err != nil { + if err := saveExtra(tx, rows); err != nil { return err } } if report.Settings { - settings.ID = 1 - if err := tx.Save(&settings).Error; err != nil { + if err := tx.Exec(`UPDATE golem15_sitemap_settings SET site_url = ?, default_locale = ? WHERE id = 1`, + nullable(settings.SiteURL), nullable(settings.DefaultLocale)).Error; err != nil { return err } } @@ -268,6 +268,14 @@ func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, return report, nil } +// nullable is nil for "", else s. +func nullable(s string) any { + if s == "" { + return nil + } + return s +} + // pickDefinition reads the definitions export and returns the row of theme, // or the only row when theme is empty. func pickDefinition(file, theme string) (winterDefinition, error) { @@ -316,150 +324,100 @@ func pickDefinition(file, theme string) (winterDefinition, error) { return rows[0], nil } -// exportedJSON is a JSON value that the export may hold as text (a TEXT -// column selected as it is) or as JSON: a string is decoded once more. -func exportedJSON(raw json.RawMessage) json.RawMessage { - raw = bytes.TrimSpace(raw) - if len(raw) > 0 && raw[0] == '"' { - var text string - if json.Unmarshal(raw, &text) == nil { - return json.RawMessage(strings.TrimSpace(text)) +// mapItems decodes the exported data and keeps its url items as +// additional addresses. +func mapItems(data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.ExtraURL, error) { + items, labels, err := models.ParseLegacyItems(data) + if err != nil { + return nil, err + } + out := []models.ExtraURL{} + seen := map[string]bool{} + keep := func(label, u string) { + if seen[u] { + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (address '%s' is already listed)", label, u)) + report.Dropped++ + return } + seen[u] = true + out = append(out, models.ExtraURL{URL: u}) + report.Kept++ } - return raw -} - -// mapItems decodes the exported data and maps its items. -func mapItems(ctx context.Context, reg *signpost.Registry, data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.DefinitionItem, error) { - data = exportedJSON(data) - if len(data) == 0 || bytes.Equal(data, []byte("null")) { - return []models.DefinitionItem{}, nil - } - rawItems, labels, ok := orderedEntries(data) - if !ok { - return nil, errors.New("the definition data is not a list of items") - } - out := []models.DefinitionItem{} - refs := map[string][]string{} - for i, raw := range rawItems { - var item models.DefinitionItem - if err := json.Unmarshal(raw, &item); err != nil { - return nil, fmt.Errorf("definition item %s: %w", labels[i], err) - } - var fields map[string]any - _ = json.Unmarshal(raw, &fields) + for i, item := range items { label := describeItem(labels[i], item) - if page, ok := fields["cmsPage"].(string); ok && page != "" { - report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, page)) + if item.CMSPage != "" { + report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, item.CMSPage)) } - if p := models.Text(item.URLPattern); p != "" && !signpost.ValidItemPattern(p) { - report.Lines = append(report.Lines, fmt.Sprintf("%s: urlPattern '%s' cleared (not a path starting with a single /)", label, p)) - item.URLPattern = nil - } - if url, ok := pageURLs[models.Text(item.Reference)]; ok && droppedTypes[item.Type] { - if !classes.AllowedItemURL(url) { + if url, ok := pageURLs[item.Reference]; ok && droppedTypes[item.Type] { + if !models.AllowedAddress(url) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (--page-url '%s' is not a path or an http(s) URL)", label, url)) report.Dropped++ continue } - report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as type 'url' with url '%s'", label, url)) + report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as an additional address '%s'", label, url)) report.Remapped++ - item.Type = models.ItemTypeURL - item.URL = &url - item.Reference = nil - out = append(out, item) - report.Kept++ + keep(label, url) continue } if to, ok := maps[item.Type]; ok && to != item.Type { report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to)) report.Remapped++ item.Type = to - if to == "" { - report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) - report.Dropped++ - continue - } } - if item.Type == models.ItemTypeURL { - if !classes.AllowedItemURL(models.Text(item.URL)) { + switch { + case item.Type == models.LegacyItemURL: + if !models.AllowedAddress(item.URL) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the url is not a path or an http(s) URL)", label)) report.Dropped++ continue } - out = append(out, item) - report.Kept++ - continue - } - if droppedTypes[item.Type] { + keep(label, item.URL) + case item.Type == "": + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) + report.Dropped++ + case droppedTypes[item.Type]: report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label)) report.Dropped++ - continue - } - if _, known := reg.Type(item.Type); !known { - report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (type '%s' is not registered; map it with --map %s=)", label, item.Type, item.Type)) + default: + report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the content types list their records themselves)", label)) report.Dropped++ - continue } - values, cached := refs[item.Type] - if !cached { - set, err := signpost.ReferenceOptions(ctx, reg, item.Type) - if err != nil { - return nil, err - } - if set.Applies { - values = []string{} - for _, o := range set.Options { - values = append(values, o.Value) - } - } - refs[item.Type] = values - } - if values != nil && !slices.Contains(values, models.Text(item.Reference)) { - report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (reference '%s' is not among the current references of '%s')", label, models.Text(item.Reference), item.Type)) - report.Dropped++ - continue - } - out = append(out, item) - report.Kept++ } return out, nil } -func describeItem(label string, item models.DefinitionItem) string { +func describeItem(label string, item models.LegacyItem) string { s := fmt.Sprintf("item %s (type '%s'", label, item.Type) - if item.URL != nil && *item.URL != "" { - s += fmt.Sprintf(", url '%s'", *item.URL) + if item.URL != "" { + s += fmt.Sprintf(", url '%s'", item.URL) } - if item.Reference != nil && *item.Reference != "" { - s += fmt.Sprintf(", reference '%s'", *item.Reference) + if item.Reference != "" { + s += fmt.Sprintf(", reference '%s'", item.Reference) } return s + ")" } -// readWinterSettings reads the exported settings object onto the PHP -// defaults. A frontend_url that models.AllowedFrontendURL refuses is an -// error; the value is quoted so its control characters are not printed raw. -func readWinterSettings(file string, report *WinterReport) (models.Settings, error) { +// ignoredWinterSettings are the PHP settings of the PHP-parity format the +// per-type sitemap no longer reads. +var ignoredWinterSettings = map[string]bool{ + "updates_manifest_path": true, "enable_priority_tiers": true, + "priority_homepage": true, "priority_services": true, "priority_blog": true, + "priority_categories": true, "priority_legal": true, +} + +// readWinterSettings reads the exported settings object. A frontend_url +// that models.AllowedFrontendURL refuses is an error; the value is quoted so +// its control characters are not printed raw. +func readWinterSettings(file string, report *WinterReport) (winterSettings, error) { raw, err := os.ReadFile(file) if err != nil { - return models.Settings{}, fmt.Errorf("File not found: %s", file) + return winterSettings{}, fmt.Errorf("File not found: %s", file) } var fields map[string]json.RawMessage - if err := json.Unmarshal(exportedJSON(raw), &fields); err != nil { - return models.Settings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) - } - s := models.DefaultSettings() - texts := map[string]**string{ - "frontend_url": &s.FrontendURL, - "frontend_default_locale": &s.FrontendDefaultLocale, - "updates_manifest_path": &s.UpdatesManifestPath, - models.PriorityHomepage: &s.PriorityHomepage, - models.PriorityServices: &s.PriorityServices, - models.PriorityBlog: &s.PriorityBlog, - models.PriorityCategories: &s.PriorityCategories, - models.PriorityLegal: &s.PriorityLegal, + if err := json.Unmarshal(models.LegacyJSON(raw), &fields); err != nil { + return winterSettings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) } + var s winterSettings keys := make([]string, 0, len(fields)) for key := range fields { keys = append(keys, key) @@ -467,22 +425,25 @@ func readWinterSettings(file string, report *WinterReport) (models.Settings, err sort.Strings(keys) for _, key := range keys { value := fields[key] - if dest, ok := texts[key]; ok { + switch { + case key == "frontend_url" || key == "frontend_default_locale": text, err := models.DecodeText(value) if err != nil { - return models.Settings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) + return winterSettings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) } - *dest = text - continue + if key == "frontend_url" { + s.SiteURL = models.Text(text) + } else { + s.DefaultLocale = strings.TrimSpace(models.Text(text)) + } + case ignoredWinterSettings[key]: + report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (the per-type sitemaps do not use it)", key)) + default: + report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) } - if key == "enable_priority_tiers" { - s.EnablePriorityTiers = !phpEmpty(value) - continue - } - report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) } - if url := models.Text(s.FrontendURL); !models.AllowedFrontendURL(url) { - return models.Settings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, url) + if !models.AllowedFrontendURL(s.SiteURL) { + return winterSettings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, s.SiteURL) } return s, nil } diff --git a/console/import_winter_test.go b/console/import_winter_test.go index 97dcae8..8172fec 100644 --- a/console/import_winter_test.go +++ b/console/import_winter_test.go @@ -9,7 +9,6 @@ import ( "strings" "testing" - "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/bonfire" ) @@ -50,7 +49,7 @@ func TestWinterSettingsFrontendURL(t *testing.T) { for _, dry := range []bool{false, true} { var buf bytes.Buffer out := bonfire.NewOutput(strings.NewReader(""), &buf, &buf) - report, err := ImportWinterFiles(context.Background(), nil, nil, WinterOptions{SettingsFile: file, DryRun: dry}, out) + report, err := ImportWinterFiles(context.Background(), nil, WinterOptions{SettingsFile: file, DryRun: dry}, out) if err == nil || !strings.Contains(err.Error(), "setting frontend_url") { t.Errorf("ImportWinterFiles(%q, dry run %v) = %v, want an error naming setting frontend_url", value, dry, err) } @@ -85,7 +84,7 @@ func TestWinterSettingsFrontendURL(t *testing.T) { t.Errorf("%s: readWinterSettings = %v", name, err) continue } - if got := models.Text(s.FrontendURL); got != tc.want { + if got := s.SiteURL; got != tc.want { t.Errorf("%s: frontend_url = %q, want %q", name, got, tc.want) } } diff --git a/controllers/admin_registry.go b/controllers/admin_registry.go index 5e72d50..bd8944b 100644 --- a/controllers/admin_registry.go +++ b/controllers/admin_registry.go @@ -1,3 +1,5 @@ +// Package controllers holds the sitemap plugin's admin controllers: the +// settings singleton. package controllers import ( @@ -10,7 +12,6 @@ import ( // a database handle or registry of its own. func AdminControllers(app func() *backpack.App) []pact.AdminController { return []pact.AdminController{ - definitionsAdminController{app: app}, settingsAdminController{app: app}, } } diff --git a/controllers/definitions.go b/controllers/definitions.go deleted file mode 100644 index 07e87f1..0000000 --- a/controllers/definitions.go +++ /dev/null @@ -1,162 +0,0 @@ -// Package controllers holds the sitemap plugin's admin controllers. -package controllers - -import ( - "context" - "errors" - "fmt" - "strings" - - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/backpack" - "git.golem15.com/golem15/summercms/modules/cabana" - "git.golem15.com/golem15/summercms/modules/pact" - "git.golem15.com/golem15/summercms/modules/signpost" - "gorm.io/gorm" -) - -const ( - // DefinitionsControllerID is the cabana controller id of the sitemap - // definition form. - DefinitionsControllerID = "golem15.sitemap.definitions" - // PermissionAccessDefinitions is Definitions::$requiredPermissions. - PermissionAccessDefinitions = "golem15.sitemap.access_definitions" -) - -// typeURLLabel is the label of the built-in url item type. -const typeURLLabel = "golem15.sitemap::lang.item.url" - -// The phrase keys of the item row refusals. -const ( - errURLScheme = "golem15.sitemap::lang.errors.url_scheme" - errURLPattern = "golem15.sitemap::lang.errors.url_pattern" - errReference = "golem15.sitemap::lang.item.reference_required" -) - -// definitionsAdminController edits the one sitemap definition of the -// application (theme default): a singleton form whose items are a repeater -// with a type dropdown (url plus the registry types) and a reference -// dropdown that depends on it. -type definitionsAdminController struct { - app func() *backpack.App -} - -func (definitionsAdminController) ID() string { return DefinitionsControllerID } - -func (definitionsAdminController) ModelName() string { return `Golem15\Sitemap\Models\Definition` } - -func (definitionsAdminController) ConfigDir() string { return "controllers/definitions" } - -func (definitionsAdminController) RequiredPermissions() []string { - return []string{PermissionAccessDefinitions} -} - -func (definitionsAdminController) NewRecord() any { return &models.Definition{} } - -// The singleton record is the definition with the fixed theme key, created -// with no items when there is none (PHP index(): firstOrCreate, then the -// update form). -func (definitionsAdminController) SingletonRecord(_ context.Context, db *gorm.DB) (any, error) { - if db == nil { - return nil, errors.New("golem15.sitemap: database is not configured") - } - var defs []models.Definition - if err := db.Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { - return nil, err - } - if len(defs) > 0 { - return &defs[0], nil - } - theme := models.DefinitionTheme - def := &models.Definition{Theme: &theme} - def.SetItems(nil) - if err := db.Create(def).Error; err != nil { - return nil, err - } - return def, nil -} - -func (c definitionsAdminController) registry() *signpost.Registry { - if c.app == nil { - return nil - } - app := c.app() - if app == nil { - return nil - } - reg, _ := app.Lookup[*signpost.Registry]() - return reg -} - -// DropdownOptions serves the item type dropdown: url, then every registry -// type in registry order with its label (PHP getTypeOptions). -func (c definitionsAdminController) DropdownOptions(string) []pact.Option { - out := []pact.Option{{Value: models.ItemTypeURL, Label: typeURLLabel}} - for _, t := range c.registry().Types() { - out = append(out, pact.Option{Value: t.Key, Label: t.Label}) - } - return out -} - -// DependentOptions serves the reference dropdown of an item row: the -// references of the row's type through signpost.ReferenceOptions; url and a -// type without references answer Applies false. -func (c definitionsAdminController) DependentOptions(ctx context.Context, _ string, values map[string]any) (pact.DependentOptionSet, error) { - typ, _ := values["type"].(string) - return signpost.ReferenceOptions(ctx, c.registry(), typ) -} - -// FormBeforeCreate checks the item rows (see checkItems). -func (c definitionsAdminController) FormBeforeCreate(_ context.Context, model any) error { - return c.checkItems(model) -} - -// FormBeforeUpdate checks the item rows (see checkItems). -func (c definitionsAdminController) FormBeforeUpdate(_ context.Context, model any) error { - return c.checkItems(model) -} - -// checkItems refuses, per row, a url outside classes.AllowedItemURL -// (data..url), a URL pattern that is not a path -// (signpost.ValidItemPattern, data..urlPattern) and a missing reference -// for a registry type that takes one (data..reference). The PHP plugin -// checked none of them: a javascript: URL would reach , which -// sitemap.xsl renders as a link, and a row without its reference resolves to -// nothing and is left out of the sitemap without a word. -func (c definitionsAdminController) checkItems(model any) error { - def, ok := model.(*models.Definition) - if !ok || def == nil { - return nil - } - reg := c.registry() - details := map[string]any{} - for i, item := range def.Items() { - if item.Type != models.ItemTypeURL { - if t, known := reg.Type(item.Type); known && t.References != nil && strings.TrimSpace(models.Text(item.Reference)) == "" { - details[fmt.Sprintf("data.%d.reference", i)] = []string{errReference} - } - } - if !classes.AllowedItemURL(models.Text(item.URL)) { - details[fmt.Sprintf("data.%d.url", i)] = []string{errURLScheme} - } - if p := models.Text(item.URLPattern); p != "" && !signpost.ValidItemPattern(p) { - details[fmt.Sprintf("data.%d.urlPattern", i)] = []string{errURLPattern} - } - } - if len(details) > 0 { - return &cabana.ValidationError{Details: details} - } - return nil -} - -var ( - _ pact.AdminController = definitionsAdminController{} - _ pact.AdminPermissioned = definitionsAdminController{} - _ pact.AdminRecordSource = definitionsAdminController{} - _ pact.AdminSingleton = definitionsAdminController{} - _ pact.DropdownOptionsProvider = definitionsAdminController{} - _ pact.DependentOptionsProvider = definitionsAdminController{} - _ pact.FormBeforeCreate = definitionsAdminController{} - _ pact.FormBeforeUpdate = definitionsAdminController{} -) diff --git a/controllers/definitions/config_form.yaml b/controllers/definitions/config_form.yaml deleted file mode 100644 index 89f044c..0000000 --- a/controllers/definitions/config_form.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# =================================== -# Form Behavior Config -# =================================== - -# Record name -name: golem15.sitemap::lang.definition.title - -# Model Form Field configuration -form: ~/plugins/golem15/sitemap/models/definition/fields.yaml - -# Model Class name -modelClass: Golem15\Sitemap\Models\Definition - -# Default redirect location -defaultRedirect: system/settings - -# Update page -update: - redirect: golem15/sitemap/definitions - redirectClose: system/settings - -messages: - update: golem15.sitemap::lang.definition.title - saved: backend::lang.messages.form.saved diff --git a/controllers/definitions_test.go b/controllers/definitions_test.go deleted file mode 100644 index 1ae0677..0000000 --- a/controllers/definitions_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package controllers - -import ( - "errors" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/cabana" -) - -func ptr(s string) *string { return &s } - -// TestCheckItems pins the CR-01 save check: per row, a url outside -// classes.AllowedItemURL and a URL pattern that is not a path are refused -// under data..url and data..urlPattern. -func TestCheckItems(t *testing.T) { - c := definitionsAdminController{} - def := &models.Definition{} - def.SetItems([]models.DefinitionItem{ - {Type: models.ItemTypeURL, URL: ptr("/about")}, - {Type: models.ItemTypeURL, URL: ptr("javascript://%0aalert(1)%0a//x")}, - {Type: "acme-post", URLPattern: ptr("//evil.test/:slug")}, - {Type: "acme-post", URLPattern: ptr("/blog/:slug")}, - }) - err := c.checkItems(def) - var verr *cabana.ValidationError - if !errors.As(err, &verr) { - t.Fatalf("checkItems = %v, want a validation error", err) - } - if len(verr.Details) != 2 || verr.Details["data.1.url"] == nil || verr.Details["data.2.urlPattern"] == nil { - t.Fatalf("details = %v", verr.Details) - } - - ok := &models.Definition{} - ok.SetItems([]models.DefinitionItem{{Type: models.ItemTypeURL, URL: ptr("https://example.test/")}}) - if err := c.checkItems(ok); err != nil { - t.Fatalf("a valid definition was refused: %v", err) - } -} diff --git a/controllers/settings.go b/controllers/settings.go index 940a03b..6575482 100644 --- a/controllers/settings.go +++ b/controllers/settings.go @@ -15,9 +15,15 @@ import ( "gorm.io/gorm" ) -// SettingsControllerID is the cabana controller id of the sitemap settings -// singleton (UI-SPEC C26), linked from the sitemap settings tile. -const SettingsControllerID = "golem15.sitemap.settings" +const ( + // SettingsControllerID is the cabana controller id of the sitemap + // settings singleton (UI-SPEC C26), linked from the sitemap settings + // tile. + SettingsControllerID = "golem15.sitemap.settings" + // PermissionAccessDefinitions guards the sitemap settings. The code is + // the Phase 17 (and WinterCMS) one, so existing roles keep their access. + PermissionAccessDefinitions = "golem15.sitemap.access_definitions" +) // The reasons the files partial counts as "left out", in display order // (P33-11). A deleted or unpublished address is a hole, not left out. diff --git a/import_test.go b/import_test.go index 5fa3297..25602b4 100644 --- a/import_test.go +++ b/import_test.go @@ -6,54 +6,58 @@ import ( "strings" "testing" + "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/console" "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/signpost" ) -// TestImportWinter pins sitemap:import-winter on Postgres (D-12): type maps -// and drops with their report lines, the reference check, --theme with -// several exported definitions, --dry-run writing nothing, the refusal of a -// second copy without --force and the replacement with it, the settings -// upsert onto row 1 with PHP defaults for missing keys, and the refusals for -// missing, empty and malformed exports. -func TestImportWinter(t *testing.T) { - e := bootSitemap(t) - ctx := context.Background() - reg, ok := e.app.Lookup[*signpost.Registry]() - if !ok { - t.Fatal("signpost registry is not published") +// storedExtra reads the additional addresses of settings row 1. +func (e *sitemapEnv) storedExtra(t *testing.T) []models.ExtraURL { + t.Helper() + s, err := classes.LoadSettings(e.gdb) + if err != nil { + t.Fatal(err) } + return s.Extra() +} + +// extraURLs lists the addresses of rows. +func extraURLs(rows []models.ExtraURL) string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.URL) + } + return strings.Join(out, " ") +} + +// TestImportWinter pins sitemap:import-winter on Postgres (D-32): the url +// items of a WinterCMS definition become the additional addresses (static +// and CMS page items through --page-url, types renamed by --map first), +// every other item is dropped and listed, --theme picks a definition, +// --dry-run writes nothing, stored addresses are replaced only with +// --force, and frontend_url and frontend_default_locale become site_url and +// default_locale while the PHP-parity keys are reported as ignored; missing, +// empty and malformed exports and an unsafe frontend_url are refused. +func TestImportWinter(t *testing.T) { + e := bootIndex(t) + ctx := context.Background() dir := t.TempDir() row := func(id int, theme string, items string) string { quoted, _ := json.Marshal(items) return `{"id": ` + string(rune('0'+id)) + `, "theme": "` + theme + `", "data": ` + string(quoted) + "}\n" } - items := `[{"type":"url","url":"/"},` + - `{"type":"old-posts","reference":null,"nesting":"1"},` + + items := `[{"type":"url","url":"/","changefreq":"always","priority":"1.0"},` + + `{"type":"blog-posts","reference":null,"nesting":"1","cmsPage":"blog/post"},` + `{"type":"acme-post","reference":"7"},` + - `{"type":"acme-post","reference":"8"},` + - `{"type":"gone-type","reference":"1"},` + `{"type":"static-page","reference":"about"},` + - `{"type":"remove-me"}]` + `{"type":"cms-page","reference":"news"},` + + `{"type":"remove-me"},` + + `{"type":"url","url":"javascript://%0aalert(1)%0a//x"},` + + `{"type":"url","url":"/"}]` defs := writeFile(t, dir, "definitions.jsonl", row(1, "main", items)+"\n"+row(2, "other", `[{"type":"url","url":"/other"}]`)) - stored := func(t *testing.T) []models.DefinitionItem { - t.Helper() - var rows []models.Definition - if err := e.gdb.Where("theme = ?", models.DefinitionTheme).Find(&rows).Error; err != nil { - t.Fatal(err) - } - if len(rows) == 0 { - return nil - } - if len(rows) != 1 { - t.Fatalf("%d definitions", len(rows)) - } - return rows[0].Items() - } run := func(opts console.WinterOptions) (console.WinterReport, string, error) { out, buf := testOutput() - report, err := console.ImportWinterFiles(ctx, e.gdb, reg, opts, out) + report, err := console.ImportWinterFiles(ctx, e.gdb, opts, out) return report, buf.String(), err } @@ -76,18 +80,22 @@ func TestImportWinter(t *testing.T) { t.Fatalf("%s: err = %v, want %q\n%s", name, err, tc.want, out) } } - if got := stored(t); got != nil { + if got := e.storedExtra(t); len(got) != 0 { t.Fatalf("a refused import wrote %+v", got) } }) - opts := console.WinterOptions{DefinitionsFile: defs, Theme: "main", Maps: map[string]string{"old-posts": "acme-list", "remove-me": ""}} + opts := console.WinterOptions{DefinitionsFile: defs, Theme: "main", Maps: map[string]string{"remove-me": ""}, PageURLs: map[string]string{"news": "/news"}} wantLines := []string{ - "item 1 (type 'old-posts'): type 'old-posts' mapped to 'acme-list'", - "item 3 (type 'acme-post', reference '8'): dropped (reference '8' is not among the current references of 'acme-post')", - "item 4 (type 'gone-type', reference '1'): dropped (type 'gone-type' is not registered; map it with --map gone-type=)", - "item 5 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", - "item 6 (type 'remove-me'): dropped (mapped to nothing)", + "item 1 (type 'blog-posts'): cmsPage 'blog/post' dropped (CMS pages are not supported)", + "item 1 (type 'blog-posts'): dropped (the content types list their records themselves)", + "item 2 (type 'acme-post', reference '7'): dropped (the content types list their records themselves)", + "item 3 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", + "item 4 (type 'cms-page', reference 'news'): copied as an additional address '/news'", + "item 5 (type 'remove-me'): type 'remove-me' mapped to ''", + "item 5 (type 'remove-me'): dropped (mapped to nothing)", + "item 6 (type 'url', url 'javascript://%0aalert(1)%0a//x'): dropped (the url is not a path or an http(s) URL)", + "item 7 (type 'url', url '/'): dropped (address '/' is already listed)", } t.Run("dry run writes nothing", func(t *testing.T) { @@ -97,7 +105,7 @@ func TestImportWinter(t *testing.T) { if err != nil { t.Fatalf("dry run: %v\n%s", err, out) } - if report.Kept != 3 || report.Dropped != 4 || report.Remapped != 2 { + if report.Kept != 2 || report.Dropped != 6 || report.Remapped != 2 { t.Fatalf("report = %+v\n%s", report, out) } for _, line := range wantLines { @@ -105,10 +113,10 @@ func TestImportWinter(t *testing.T) { t.Errorf("output lacks %q:\n%s", line, out) } } - if !strings.Contains(out, "Dry run: 3 items kept, 2 remapped, 4 dropped.") || !strings.Contains(out, "Dry run complete (no changes made).") { + if !strings.Contains(out, "Dry run: 2 items kept, 2 remapped, 6 dropped.") || !strings.Contains(out, "Dry run complete (no changes made).") { t.Fatalf("dry run output:\n%s", out) } - if got := stored(t); got != nil { + if got := e.storedExtra(t); len(got) != 0 { t.Fatalf("the dry run wrote %+v", got) } }) @@ -118,32 +126,30 @@ func TestImportWinter(t *testing.T) { if err != nil { t.Fatalf("import: %v\n%s", err, out) } - got := stored(t) - if report.Kept != 3 || len(got) != 3 || got[0].Type != "url" || got[1].Type != "acme-list" || !got[1].Nesting || got[2].Type != "acme-post" || models.Text(got[2].Reference) != "7" { - t.Fatalf("copied = %+v (report %+v)", got, report) + if got := extraURLs(e.storedExtra(t)); report.Kept != 2 || got != "/ /news" { + t.Fatalf("copied = %q (report %+v)", got, report) } - if !strings.Contains(out, "Definition copied: 3 items kept, 2 remapped, 4 dropped.") || !strings.Contains(out, "theme 'main'") { + if !strings.Contains(out, "Definition copied: 2 items kept, 2 remapped, 6 dropped.") || !strings.Contains(out, "theme 'main'") { t.Fatalf("output:\n%s", out) } other := console.WinterOptions{DefinitionsFile: defs, Theme: "other"} - if _, out, err := run(other); err == nil || !strings.Contains(err.Error(), "already has 3 items; rerun with --force") { + if _, out, err := run(other); err == nil || !strings.Contains(err.Error(), "already has 2 additional addresses; rerun with --force") { t.Fatalf("second copy: %v\n%s", err, out) } - // A dry run over existing items warns instead of failing. dry := other dry.DryRun = true - if _, out, err := run(dry); err != nil || !strings.Contains(out, "already has 3 items") { - t.Fatalf("dry run over items: %v\n%s", err, out) + if _, out, err := run(dry); err != nil || !strings.Contains(out, "already has 2 additional addresses") { + t.Fatalf("dry run over addresses: %v\n%s", err, out) } - if len(stored(t)) != 3 { - t.Fatal("a refused copy changed the definition") + if got := extraURLs(e.storedExtra(t)); got != "/ /news" { + t.Fatalf("a refused copy changed the addresses: %q", got) } other.Force = true if _, out, err := run(other); err != nil { t.Fatalf("--force: %v\n%s", err, out) } - if got := stored(t); len(got) != 1 || models.Text(got[0].URL) != "/other" { - t.Fatalf("after --force = %+v", got) + if got := extraURLs(e.storedExtra(t)); got != "/other" { + t.Fatalf("after --force = %q", got) } }) @@ -152,70 +158,42 @@ func TestImportWinter(t *testing.T) { if _, out, err := run(console.WinterOptions{DefinitionsFile: raw, Force: true}); err != nil { t.Fatalf("raw JSON data: %v\n%s", err, out) } - if got := stored(t); len(got) != 1 || models.Text(got[0].URL) != "/raw" { - t.Fatalf("raw data = %+v", got) + if got := extraURLs(e.storedExtra(t)); got != "/raw" { + t.Fatalf("raw data = %q", got) } null := writeFile(t, dir, "null.jsonl", `{"id":6,"theme":"t","data":null}`) if report, out, err := run(console.WinterOptions{DefinitionsFile: null, Force: true}); err != nil || report.Kept != 0 { t.Fatalf("null data: %+v %v\n%s", report, err, out) } - if got := stored(t); len(got) != 0 { + if got := e.storedExtra(t); len(got) != 0 { t.Fatalf("null data stored %+v", got) } }) - t.Run("url and pattern checks", func(t *testing.T) { - bad := writeFile(t, dir, "bad-urls.jsonl", row(7, "t", `[{"type":"url","url":"javascript://%0aalert(1)%0a//x"},`+ - `{"type":"url","url":"/ok"},`+ - `{"type":"acme-post","reference":"7","urlPattern":"//evil.test/:slug"}]`)) - report, out, err := run(console.WinterOptions{DefinitionsFile: bad, Force: true}) - if err != nil { - t.Fatalf("import: %v\n%s", err, out) - } - if report.Kept != 2 || report.Dropped != 1 { - t.Fatalf("report = %+v\n%s", report, out) - } - for _, line := range []string{ - "item 0 (type 'url', url 'javascript://%0aalert(1)%0a//x'): dropped (the url is not a path or an http(s) URL)", - "item 2 (type 'acme-post', reference '7'): urlPattern '//evil.test/:slug' cleared (not a path starting with a single /)", - } { - if !strings.Contains(out, line) { - t.Errorf("output lacks %q:\n%s", line, out) - } - } - got := stored(t) - if len(got) != 2 || models.Text(got[0].URL) != "/ok" || got[1].URLPattern != nil { - t.Fatalf("stored = %+v", got) - } - }) - - t.Run("settings upsert", func(t *testing.T) { - first := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test","enable_priority_tiers":"1","priority_blog":0.7,"unknown":"x"}`) + t.Run("settings", func(t *testing.T) { + first := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test","frontend_default_locale":"pl","enable_priority_tiers":"1","priority_blog":0.7,"unknown":"x"}`) report, out, err := run(console.WinterOptions{SettingsFile: first}) - if err != nil || !report.Settings || !strings.Contains(out, "setting 'unknown' ignored") || !strings.Contains(out, "Settings copied.") { + if err != nil || !report.Settings || !strings.Contains(out, "setting 'unknown' ignored (not a sitemap setting of this port)") || + !strings.Contains(out, "setting 'priority_blog' ignored (the per-type sitemaps do not use it)") || !strings.Contains(out, "Settings copied.") { t.Fatalf("settings: %+v %v\n%s", report, err, out) } - second := writeFile(t, dir, "settings2.json", `"{\"frontend_url\":\"https://other.example.test\",\"enable_priority_tiers\":\"0\"}"`) + s, err := classes.LoadSettings(e.gdb) + if err != nil || models.Text(s.SiteURL) != "https://www.example.test" || models.Text(s.DefaultLocale) != "pl" { + t.Fatalf("copied settings = %+v %v", s, err) + } + second := writeFile(t, dir, "settings2.json", `"{\"frontend_url\":\"https://other.example.test\"}"`) if _, out, err := run(console.WinterOptions{SettingsFile: second}); err != nil { t.Fatalf("second settings: %v\n%s", err, out) } - var rows []models.Settings - if err := e.gdb.Find(&rows).Error; err != nil { - t.Fatal(err) - } - if len(rows) != 1 || rows[0].ID != 1 || models.Text(rows[0].FrontendURL) != "https://other.example.test" || rows[0].EnablePriorityTiers || rows[0].Priority(models.PriorityBlog) != "0.6" { - t.Fatalf("settings rows = %+v", rows) + if s, _ = classes.LoadSettings(e.gdb); models.Text(s.SiteURL) != "https://other.example.test" || s.DefaultLocale != nil { + t.Fatalf("second settings = %+v", s) } dry := writeFile(t, dir, "settings3.json", `{"frontend_url":"https://dry.example.test"}`) if _, out, err := run(console.WinterOptions{SettingsFile: dry, DryRun: true}); err != nil || !strings.Contains(out, "Dry run: the settings would be copied.") { t.Fatalf("settings dry run: %v\n%s", err, out) } - if err := e.gdb.Find(&rows).Error; err != nil || models.Text(rows[0].FrontendURL) != "https://other.example.test" { - t.Fatalf("the dry run changed the settings: %+v %v", rows, err) - } - - // T-17-15: a frontend_url that is not a plain http(s) URL stops the - // import, also as a dry run, and the row keeps its value. + // T-17-15: an unsafe frontend_url stops the import, also as a dry + // run, and the row keeps its value. quoted, _ := json.Marshal(map[string]string{"frontend_url": `https://x"/> &. Leave it empty to keep the backend URL.' file_size: Enter a whole number from 1 to 50 000. slug: Use lowercase letters, digits and hyphens, e.g. posts. @@ -47,10 +12,6 @@ errors: html_path: Start the address with / and use a path only, e.g. /mapa-strony. page_size: Enter a whole number from 50 to 500. lastmod: 'Enter a date as YYYY-MM-DD, e.g. 2026-10-28, or leave it empty.' -definition: - not_found: No sitemap definition was found. Try creating one first. - title: Sitemap definition - view: View sitemap.xml settings: label: Sitemaps description: XML sitemaps per content type, images, News, IndexNow and the HTML sitemap. diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index 2ee3f53..a97c42a 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -2,44 +2,9 @@ plugin: name: Mapy witryny description: Mapy XML według typów treści, ze stałymi plikami, prawdziwym lastmod i hreflang. permissions: - access_settings: Zarządzaj ustawieniami wtyczki - access_definitions: Zarządzaj definicją mapy strony -item: - location: 'Lokacja:' - priority: Priorytet - changefreq: Częstotliwość zmian - always: cały czas - hourly: co godzinę - daily: codziennie - weekly: co tydzień - monthly: co miesiąc - yearly: co rok - never: nigdy - editor_title: Edytuj pozycję mapy - type: Typ - allow_nested_items: Pozwól na zagnieżdżanie pozycji - allow_nested_items_comment: Zagnieżdżone pozycje mogą zostać wygenerowane dynamicznie przez statyczne strony i niektóre rodzaje typów pozycji. - url: URL - reference: Odnośnik - title_required: Tytuł jest wymagany - unknown_type: Nieznany typ pozycji - unnamed: Pozycja bez nazwy - add_item: Dodaj pozycję - new_item: Nowa pozycja - reference_required: Odnośnik dla pozycji jest wymagany. - url_required: Adres URL jest wymagany - url_pattern: Wzorzec URL - url_pattern_comment: Użyj składni :placeholder, np. /blog/:slug - url_pattern_required: Podaj wzorzec URL - page: Strona - check: Sprawdź - definition: Definicja - save_definition: Zapisywanie definicji... - load_indicator: Czyszczenie definicji... - empty_confirm: Wyczyścić tę definicję? + access_definitions: Zarządzaj ustawieniami map witryny errors: url_scheme: Podaj ścieżkę, np. /o-nas, albo link zaczynający się od http:// lub https://. - url_pattern: Podaj ścieżkę zaczynającą się od jednego /, np. /blog/:slug. frontend_url: 'Podaj pełny adres zaczynający się od http:// lub https://, bez spacji, cudzysłowów i znaków < > &. Zostaw puste, aby zachować adres backendu.' file_size: Wpisz liczbę całkowitą od 1 do 50 000. slug: Użyj małych liter, cyfr i łączników, np. posts. @@ -47,10 +12,6 @@ errors: html_path: Zacznij adres od / i użyj samej ścieżki, np. /mapa-strony. page_size: Wpisz liczbę całkowitą od 50 do 500. lastmod: 'Wpisz datę w formacie RRRR-MM-DD, np. 2026-10-28, albo zostaw puste.' -definition: - not_found: Nie znaleziono definicji mapy strony. Musisz ją utworzyć. - title: Definicja mapy strony - view: Zobacz sitemap.xml settings: label: Mapy witryny description: Mapy XML według typów treści, obrazy, News, IndexNow i mapa HTML. diff --git a/models/definition.go b/models/definition.go deleted file mode 100644 index d0a1cf0..0000000 --- a/models/definition.go +++ /dev/null @@ -1,48 +0,0 @@ -package models - -import ( - "time" - - "git.golem15.com/golem15/summercms/modules/lagoon" -) - -// DefinitionTheme is the fixed theme key of the one sitemap definition of an -// application (D-11). The theme column stays so the table matches PHP; the -// import commands write this key whatever the PHP theme was. -const DefinitionTheme = "default" - -// Definition is a sitemap definition stored on golem15_sitemap_definitions, -// the PHP Golem15\Sitemap\Models\Definition. Data holds the items as JSON -// text (PHP $jsonable data). -type Definition struct { - ID uint `gorm:"column:id;primaryKey"` - Theme *string `gorm:"column:theme"` - Data lagoon.Jsonable[DefinitionItems] `gorm:"column:data"` - CreatedAt *time.Time `gorm:"column:created_at"` - UpdatedAt *time.Time `gorm:"column:updated_at"` -} - -func (Definition) TableName() string { return "golem15_sitemap_definitions" } - -// MorphName is the PHP class string of the model. -func (Definition) MorphName() string { return `Golem15\Sitemap\Models\Definition` } - -// Fillable is exactly theme and data, the PHP $fillable with $guarded = ['*'] -// (MassAssignmentTest). -func (Definition) Fillable() []string { return []string{"theme", "data"} } - -// Rules is empty: the PHP model validated nothing. The admin form checks -// each item row's reference against its type. -func (Definition) Rules() map[string]string { return map[string]string{} } - -// Items returns the stored items. -func (d Definition) Items() []DefinitionItem { return d.Data.Get() } - -// SetItems stores items as the definition's data. -func (d *Definition) SetItems(items []DefinitionItem) { - d.Data = lagoon.Jsonable[DefinitionItems]{Data: DefinitionItems(items), Valid: true} -} - -func init() { - Register(Definition{}) -} diff --git a/models/definition/fields.yaml b/models/definition/fields.yaml deleted file mode 100644 index e90e6e9..0000000 --- a/models/definition/fields.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# =================================== -# Form Field Definitions -# =================================== -# -# The WinterCMS form was one tab holding the SitemapItems widget. Here the -# items are a repeater on data; each row is one DefinitionItem. The CMS page -# field is not ported: items are URLs or registry types. - -fields: - data: - label: golem15.sitemap::lang.item.definition - type: repeater - prompt: golem15.sitemap::lang.item.add_item - titleFrom: url - form: - fields: - type: - label: golem15.sitemap::lang.item.type - type: dropdown - options: getTypeOptions - span: left - reference: - label: golem15.sitemap::lang.item.reference - type: dropdown - dependsOn: type - emptyOption: backend::lang.form.none - span: right - url: - label: golem15.sitemap::lang.item.url - type: text - span: left - urlPattern: - label: golem15.sitemap::lang.item.url_pattern - comment: golem15.sitemap::lang.item.url_pattern_comment - type: text - span: right - priority: - label: golem15.sitemap::lang.item.priority - type: dropdown - span: left - options: - '0.1': '0.1' - '0.2': '0.2' - '0.3': '0.3' - '0.4': '0.4' - '0.5': '0.5' - '0.6': '0.6' - '0.7': '0.7' - '0.8': '0.8' - '0.9': '0.9' - '1.0': '1.0' - changefreq: - label: golem15.sitemap::lang.item.changefreq - type: dropdown - span: right - options: - always: golem15.sitemap::lang.item.always - hourly: golem15.sitemap::lang.item.hourly - daily: golem15.sitemap::lang.item.daily - weekly: golem15.sitemap::lang.item.weekly - monthly: golem15.sitemap::lang.item.monthly - yearly: golem15.sitemap::lang.item.yearly - never: golem15.sitemap::lang.item.never - nesting: - label: golem15.sitemap::lang.item.allow_nested_items - comment: golem15.sitemap::lang.item.allow_nested_items_comment - type: checkbox - default: true - span: full diff --git a/models/definition_item.go b/models/definition_item.go deleted file mode 100644 index a22c058..0000000 --- a/models/definition_item.go +++ /dev/null @@ -1,255 +0,0 @@ -package models - -import ( - "bytes" - "encoding/json" - "fmt" - "strconv" - "strings" -) - -// ItemTypeURL is the built-in item type of an explicit URL. Every other -// item type is a menu item type of the framework's registry (signpost). -const ItemTypeURL = "url" - -// DefinitionItem is one sitemap definition item, the PHP -// Golem15\Sitemap\Classes\DefinitionItem. Type is "url" or a registry menu -// item type key; URL is the address of a url item; Reference is the record a -// registry item points at; URLPattern overrides the type's URL pattern; -// Nesting asks a registry item for its sub-items; Priority and Changefreq -// are written to the sitemap as they are stored. -// -// Text fields are pointers so a stored JSON null stays null on the next -// save. Decoding is tolerant of what PHP stored: numbers and booleans are -// accepted for text fields, strings and numbers for Nesting, and unknown keys -// (cmsPage, code, exists, fillable and any other) are ignored. Encoding -// writes the PHP toArray key set: changefreq, priority, nesting, type, url, -// reference, cmsPage (always null, CMS pages are not supported) and -// urlPattern. -type DefinitionItem struct { - Type string - URL *string - Reference *string - URLPattern *string - Nesting bool - Priority *string - Changefreq *string -} - -// Text returns the value of a text field, "" for null. -func Text(p *string) string { - if p == nil { - return "" - } - return *p -} - -// UnmarshalJSON decodes one stored item tolerantly. A missing type is the -// PHP default "url"; a null type is "". -func (d *DefinitionItem) UnmarshalJSON(raw []byte) error { - if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { - // PHP initFromArray kept a default item for a null entry. - *d = DefinitionItem{Type: ItemTypeURL} - return nil - } - var fields map[string]json.RawMessage - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - if err := dec.Decode(&fields); err != nil { - return fmt.Errorf("sitemap definition item: %w", err) - } - item := DefinitionItem{Type: ItemTypeURL} - if v, ok := fields["type"]; ok { - s, err := DecodeText(v) - if err != nil { - return fmt.Errorf("sitemap definition item type: %w", err) - } - item.Type = Text(s) - } - for key, dest := range map[string]**string{ - "url": &item.URL, - "reference": &item.Reference, - "urlPattern": &item.URLPattern, - "priority": &item.Priority, - "changefreq": &item.Changefreq, - } { - v, ok := fields[key] - if !ok { - continue - } - s, err := DecodeText(v) - if err != nil { - return fmt.Errorf("sitemap definition item %s: %w", key, err) - } - *dest = s - } - if v, ok := fields["nesting"]; ok { - b, err := decodeTruthy(v) - if err != nil { - return fmt.Errorf("sitemap definition item nesting: %w", err) - } - item.Nesting = b - } - *d = item - return nil -} - -// MarshalJSON writes the PHP toArray key set in its order. -func (d DefinitionItem) MarshalJSON() ([]byte, error) { - var b bytes.Buffer - b.WriteByte('{') - write := func(key string, v any, last bool) error { - kb, _ := json.Marshal(key) - b.Write(kb) - b.WriteByte(':') - vb, err := json.Marshal(v) - if err != nil { - return err - } - b.Write(vb) - if !last { - b.WriteByte(',') - } - return nil - } - for _, f := range []struct { - key string - v any - }{ - {"changefreq", d.Changefreq}, - {"priority", d.Priority}, - {"nesting", d.Nesting}, - {"type", d.Type}, - {"url", d.URL}, - {"reference", d.Reference}, - {"cmsPage", nil}, - } { - if err := write(f.key, f.v, false); err != nil { - return nil, err - } - } - if err := write("urlPattern", d.URLPattern, true); err != nil { - return nil, err - } - b.WriteByte('}') - return b.Bytes(), nil -} - -// DefinitionItems is the stored item list of a definition. It decodes a -// JSON array, and a JSON object (a PHP array whose keys were not sequential) -// in key order. -type DefinitionItems []DefinitionItem - -// UnmarshalJSON accepts a JSON array or object of items; null is no items. -func (l *DefinitionItems) UnmarshalJSON(raw []byte) error { - raw = bytes.TrimSpace(raw) - if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { - *l = nil - return nil - } - if raw[0] == '[' { - var items []DefinitionItem - if err := json.Unmarshal(raw, &items); err != nil { - return err - } - *l = items - return nil - } - if raw[0] != '{' { - return fmt.Errorf("sitemap definition data must be a list of items") - } - dec := json.NewDecoder(bytes.NewReader(raw)) - if _, err := dec.Token(); err != nil { - return err - } - var items []DefinitionItem - for dec.More() { - if _, err := dec.Token(); err != nil { - return err - } - var item DefinitionItem - if err := dec.Decode(&item); err != nil { - return err - } - items = append(items, item) - } - *l = items - return nil -} - -// MarshalJSON writes a JSON array; no items is []. -func (l DefinitionItems) MarshalJSON() ([]byte, error) { - if l == nil { - return []byte("[]"), nil - } - return json.Marshal([]DefinitionItem(l)) -} - -// DecodeText decodes a stored text value: a string as it is, a number in -// its PHP string form, a boolean as PHP casts it ("1" or ""), null as nil. -func DecodeText(raw json.RawMessage) (*string, error) { - var v any - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - if err := dec.Decode(&v); err != nil { - return nil, err - } - var s string - switch t := v.(type) { - case nil: - return nil, nil - case string: - s = t - case json.Number: - s = phpNumberString(t) - case bool: - if t { - s = "1" - } - default: - return nil, fmt.Errorf("must be text, got %T", v) - } - return &s, nil -} - -// decodeTruthy decodes a stored flag with PHP truthiness: false, 0, "", -// "0" and null are false. -func decodeTruthy(raw json.RawMessage) (bool, error) { - var v any - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - if err := dec.Decode(&v); err != nil { - return false, err - } - switch t := v.(type) { - case nil: - return false, nil - case bool: - return t, nil - case string: - return t != "" && t != "0", nil - case json.Number: - f, err := t.Float64() - if err != nil { - return false, err - } - return f != 0, nil - } - return false, fmt.Errorf("must be a flag, got %T", v) -} - -// phpNumberString is PHP's string cast of a json_decode'd number: an -// integer as written, a float in its shortest round-trip form (1.0 is "1"). -func phpNumberString(n json.Number) string { - text := n.String() - if !strings.ContainsAny(text, ".eE") { - if i, err := strconv.ParseInt(text, 10, 64); err == nil { - return strconv.FormatInt(i, 10) - } - } - f, err := n.Float64() - if err != nil { - return text - } - return strconv.FormatFloat(f, 'f', -1, 64) -} diff --git a/models/definition_test.go b/models/definition_test.go deleted file mode 100644 index dd5f2b7..0000000 --- a/models/definition_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package models_test - -import ( - "encoding/json" - "fmt" - "slices" - "strings" - "testing" - - "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/lagoon" -) - -// TestFillable ports the PHP MassAssignmentTest (SITE-001) as behaviour: -// Definition.Fillable is exactly theme and data, and a fill with extra keys -// (id, timestamps, unknown keys) writes only those two. -func TestFillable(t *testing.T) { - if got := (models.Definition{}).Fillable(); !slices.Equal(got, []string{"theme", "data"}) { - t.Fatalf("Fillable = %v, want [theme data]", got) - } - def := models.Definition{ID: 4} - requested := map[string]any{ - "id": 99, - "theme": "site", - "data": `[{"type":"url","url":"/a"}]`, - "created_at": "2020-01-01 00:00:00", - "updated_at": "2020-01-01 00:00:00", - "is_admin": true, - } - for _, production := range []bool{true, false} { - d := def - if err := lagoon.Fill(&d, d.Fillable(), requested, production); err != nil { - t.Fatalf("Fill: %v", err) - } - if d.ID != 4 || d.CreatedAt != nil || d.UpdatedAt != nil || models.Text(d.Theme) != "site" { - t.Fatalf("filled definition = %+v", d) - } - if items := d.Items(); len(items) != 1 || models.Text(items[0].URL) != "/a" { - t.Fatalf("filled data = %+v", items) - } - } - if (models.Definition{}).TableName() != "golem15_sitemap_definitions" || (models.Definition{}).MorphName() != `Golem15\Sitemap\Models\Definition` { - t.Fatal("table or morph name changed") - } - if rules := (models.Definition{}).Rules(); len(rules) != 0 { - t.Fatalf("Rules = %v", rules) - } - if got := (models.Settings{}).Fillable(); slices.Contains(got, "id") || len(got) != 3 { - t.Fatalf("Settings.Fillable = %v", got) - } -} - -// TestDefinitionItemJSON pins the tolerant decoding of PHP-stored items -// (strings, numbers, booleans and null for text fields, PHP truthiness for -// nesting, unknown keys ignored, a missing type is url, a null item a -// default url item, an object-keyed list in key order) and the PHP toArray -// encoding (key set and order, cmsPage always null, null text kept). -func TestDefinitionItemJSON(t *testing.T) { - for _, tc := range []struct { - raw string - want string - }{ - {`{"type":"url","url":"/a","priority":"0.5","changefreq":"daily","nesting":"0","reference":null,"urlPattern":null,"cmsPage":"x","code":"c","exists":false,"fillable":["a"]}`, - `type=url url=/a ref= pattern= nesting=false priority=0.5 changefreq=daily`}, - {`{"url":"/b"}`, `type=url url=/b ref= pattern= nesting=false priority= changefreq=`}, - {`{"type":null}`, `type= url= ref= pattern= nesting=false priority= changefreq=`}, - {`{"type":"journal-post","reference":7,"priority":0.5,"changefreq":true,"nesting":1}`, - `type=journal-post url= ref=7 pattern= nesting=true priority=0.5 changefreq=1`}, - {`{"type":"x","priority":1.0,"reference":12.50,"changefreq":false,"nesting":"1"}`, - `type=x url= ref=12.5 pattern= nesting=true priority=1 changefreq=`}, - {`{"type":"x","nesting":"false"}`, `type=x url= ref= pattern= nesting=true priority= changefreq=`}, - {`{"type":"x","nesting":""}`, `type=x url= ref= pattern= nesting=false priority= changefreq=`}, - {`{"type":"x","nesting":null,"urlPattern":"/p/:slug"}`, `type=x url= ref= pattern=/p/:slug nesting=false priority= changefreq=`}, - {`{"type":"x","reference":12345678901234}`, `type=x url= ref=12345678901234 pattern= nesting=false priority= changefreq=`}, - {`null`, `type=url url= ref= pattern= nesting=false priority= changefreq=`}, - } { - var item models.DefinitionItem - if err := json.Unmarshal([]byte(tc.raw), &item); err != nil { - t.Fatalf("%s: %v", tc.raw, err) - } - if got := describe(item); got != tc.want { - t.Errorf("%s\n got %s\nwant %s", tc.raw, got, tc.want) - } - } - for _, raw := range []string{`{"type":["x"]}`, `{"url":{"a":1}}`, `{"nesting":[1]}`, `[1]`, `"x"`} { - var item models.DefinitionItem - if err := json.Unmarshal([]byte(raw), &item); err == nil { - t.Errorf("%s decoded without an error", raw) - } - } - - // A list, an object keyed like a PHP array with gaps, null and empty. - var list models.DefinitionItems - if err := json.Unmarshal([]byte(`{"3":{"url":"/c"},"1":{"url":"/a"},"x":{"url":"/x"}}`), &list); err != nil || len(list) != 3 || - models.Text(list[0].URL) != "/c" || models.Text(list[1].URL) != "/a" || models.Text(list[2].URL) != "/x" { - t.Fatalf("object list = %+v %v", list, err) - } - for raw, n := range map[string]int{`[]`: 0, `null`: 0, `[{"url":"/a"},null]`: 2} { - var l models.DefinitionItems - if err := json.Unmarshal([]byte(raw), &l); err != nil || len(l) != n { - t.Fatalf("%s = %+v %v", raw, l, err) - } - } - var bad models.DefinitionItems - if err := json.Unmarshal([]byte(`"items"`), &bad); err == nil { - t.Fatal("a string decoded as an item list") - } - - // Encoding: the PHP toArray key set and order. - pattern := "/p/:slug" - item := models.DefinitionItem{Type: "journal-post", Reference: str("7"), URLPattern: &pattern, Nesting: true, Priority: str("0.5")} - raw, err := json.Marshal(item) - if err != nil { - t.Fatal(err) - } - if want := `{"changefreq":null,"priority":"0.5","nesting":true,"type":"journal-post","url":null,"reference":"7","cmsPage":null,"urlPattern":"/p/:slug"}`; string(raw) != want { - t.Fatalf("Marshal = %s\nwant %s", raw, want) - } - if raw, _ := json.Marshal(models.DefinitionItems(nil)); string(raw) != "[]" { - t.Fatalf("nil list = %s", raw) - } - var back models.DefinitionItem - if err := json.Unmarshal(raw, &back); err != nil || describe(back) != describe(item) { - t.Fatalf("round trip = %s, %v", describe(back), err) - } - if models.Text(nil) != "" || models.Text(str("x")) != "x" { - t.Fatal("Text") - } - if s, err := models.DecodeText(json.RawMessage(`1e3`)); err != nil || *s != "1000" { - t.Fatalf("DecodeText(1e3) = %v %v", s, err) - } -} - -func str(s string) *string { return &s } - -func describe(d models.DefinitionItem) string { - p := func(s *string) string { - if s == nil { - return "" - } - return *s - } - return strings.TrimSpace(fmt.Sprintf("type=%s url=%s ref=%s pattern=%s nesting=%v priority=%s changefreq=%s", - d.Type, p(d.URL), p(d.Reference), p(d.URLPattern), d.Nesting, p(d.Priority), p(d.Changefreq))) -} diff --git a/models/legacy.go b/models/legacy.go new file mode 100644 index 0000000..19e1336 --- /dev/null +++ b/models/legacy.go @@ -0,0 +1,177 @@ +package models + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// Text returns *p, "" for nil. +func Text(p *string) string { + if p == nil { + return "" + } + return *p +} + +// LegacyItemURL is the type of a Phase 17 (and WinterCMS) sitemap +// definition item that is an explicit address. +const LegacyItemURL = "url" + +// LegacyItem is one item of a Phase 17 or WinterCMS sitemap definition, as +// golem15_sitemap_definitions.data and the WinterCMS export store it. Only +// the fields the per-type format still reads are kept: the type (a missing +// type is url, as in PHP), the address of a url item and the reference of a +// registry item. +type LegacyItem struct { + Type string + URL string + Reference string + CMSPage string +} + +// LegacyEntries splits a JSON array, or a JSON object in key order (a PHP +// array with keys), into its values with their PHP labels (index or key). +// ok is false for any other value. +func LegacyEntries(raw []byte) ([]json.RawMessage, []string, bool) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return nil, nil, false + } + switch raw[0] { + case '[': + var list []json.RawMessage + if err := json.Unmarshal(raw, &list); err != nil { + return nil, nil, false + } + labels := make([]string, len(list)) + for i := range list { + labels[i] = strconv.Itoa(i) + } + return list, labels, true + case '{': + dec := json.NewDecoder(bytes.NewReader(raw)) + if _, err := dec.Token(); err != nil { + return nil, nil, false + } + var list []json.RawMessage + var labels []string + for dec.More() { + tok, err := dec.Token() + if err != nil { + return nil, nil, false + } + key, _ := tok.(string) + var v json.RawMessage + if err := dec.Decode(&v); err != nil { + return nil, nil, false + } + list = append(list, v) + labels = append(labels, key) + } + return list, labels, true + } + return nil, nil, false +} + +// ParseLegacyItem decodes one stored item tolerantly: text fields may be +// strings, numbers, booleans or null; a missing type is url; unknown keys +// are ignored. +func ParseLegacyItem(raw json.RawMessage) (LegacyItem, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return LegacyItem{}, fmt.Errorf("an item must be an object: %w", err) + } + item := LegacyItem{Type: LegacyItemURL} + for key, dest := range map[string]*string{"type": &item.Type, "url": &item.URL, "reference": &item.Reference, "cmsPage": &item.CMSPage} { + v, ok := fields[key] + if !ok { + continue + } + text, err := DecodeText(v) + if err != nil { + return LegacyItem{}, fmt.Errorf("%s %w", key, err) + } + *dest = strings.TrimSpace(Text(text)) + } + return item, nil +} + +// ParseLegacyItems decodes the data of a definition: a JSON array or a PHP +// keyed object of items, itself possibly stored as a JSON string. Empty or +// null data has no items. +func ParseLegacyItems(data []byte) ([]LegacyItem, []string, error) { + data = LegacyJSON(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + return nil, nil, nil + } + raws, labels, ok := LegacyEntries(data) + if !ok { + return nil, nil, fmt.Errorf("the definition data is not a list of items") + } + items := make([]LegacyItem, 0, len(raws)) + for i, raw := range raws { + item, err := ParseLegacyItem(raw) + if err != nil { + return nil, nil, fmt.Errorf("definition item %s: %w", labels[i], err) + } + items = append(items, item) + } + return items, labels, nil +} + +// LegacyJSON is a JSON value that may be held as text (a TEXT column +// selected as it is) or as JSON: a string is decoded once more. +func LegacyJSON(raw []byte) []byte { + raw = bytes.TrimSpace(raw) + if len(raw) > 0 && raw[0] == '"' { + var text string + if json.Unmarshal(raw, &text) == nil { + return []byte(strings.TrimSpace(text)) + } + } + return raw +} + +// DecodeText decodes a stored text value with PHP's string conversion: +// null is nil, a number its PHP text, true "1" and false "". +func DecodeText(raw json.RawMessage) (*string, error) { + var v any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&v); err != nil { + return nil, err + } + var s string + switch t := v.(type) { + case nil: + return nil, nil + case string: + s = t + case json.Number: + s = phpNumberString(t) + case bool: + if t { + s = "1" + } + default: + return nil, fmt.Errorf("must be text, got %T", v) + } + return &s, nil +} + +func phpNumberString(n json.Number) string { + text := n.String() + if !strings.ContainsAny(text, ".eE") { + if i, err := strconv.ParseInt(text, 10, 64); err == nil { + return strconv.FormatInt(i, 10) + } + } + f, err := n.Float64() + if err != nil { + return text + } + return strconv.FormatFloat(f, 'f', -1, 64) +} diff --git a/models/settings.go b/models/settings.go index 8615fe1..7e38ad5 100644 --- a/models/settings.go +++ b/models/settings.go @@ -12,50 +12,23 @@ import ( "git.golem15.com/golem15/summercms/modules/lagoon" ) -// errFrontendURL is the phrase key of the frontend_url refusal. +// errFrontendURL is the phrase key of the site address refusal (the +// Phase 17 frontend_url phrase, unchanged). const errFrontendURL = "golem15.sitemap::lang.errors.frontend_url" -// The settings keys of the five priority tiers, in the order PHP checked -// them. A tier rule names one of them. -const ( - PriorityHomepage = "priority_homepage" - PriorityServices = "priority_services" - PriorityBlog = "priority_blog" - PriorityCategories = "priority_categories" - PriorityLegal = "priority_legal" -) - -// PriorityDefaults are the PHP defaults of the priority settings. -var PriorityDefaults = map[string]string{ - PriorityHomepage: "1.0", - PriorityServices: "0.8", - PriorityBlog: "0.6", - PriorityCategories: "0.4", - PriorityLegal: "0.2", -} - // Settings is the sitemap settings singleton (row id 1) on -// golem15_sitemap_settings, the PHP Golem15\Sitemap\Models\Settings that -// WinterCMS kept in system_settings under golem15_sitemap_settings. -// FrontendURL replaces the backend URL in the generated XML; -// FrontendDefaultLocale is the locale whose URLs carry no prefix; -// UpdatesManifestPath names a JSON file of URL path to last-modified date; -// EnablePriorityTiers turns the tier priorities on, and the priority fields -// hold each tier's value as text. A null priority reads as its PHP default. -// The PHP settings form validated nothing; the port checks FrontendURL in -// BeforeValidate. +// golem15_sitemap_settings (UI-SPEC C26). SiteURL is the public site +// address every starts with (empty: golem15.sitemap.site_url, then +// app.url), FileSize the addresses per file, DefaultLocale the site's +// default locale (the unprefixed one, and the target of x-default) and +// ExtraURLs the additional addresses. RebuildPendingUntil is the debounce +// time of the next queued rebuild, NumberedSize the file size the files are +// numbered for, BuiltAt the end of the last rebuild and RebuildRequestedAt +// the last rebuild asked for from the admin. The Phase 17 columns +// (frontend_url, priorities, tiers, the update manifest) stay in the table +// and are no longer read (D-32). type Settings struct { - ID uint `gorm:"column:id;primaryKey"` - FrontendURL *string `gorm:"column:frontend_url"` - FrontendDefaultLocale *string `gorm:"column:frontend_default_locale"` - UpdatesManifestPath *string `gorm:"column:updates_manifest_path"` - EnablePriorityTiers bool `gorm:"column:enable_priority_tiers"` - PriorityHomepage *string `gorm:"column:priority_homepage"` - PriorityServices *string `gorm:"column:priority_services"` - PriorityBlog *string `gorm:"column:priority_blog"` - PriorityCategories *string `gorm:"column:priority_categories"` - PriorityLegal *string `gorm:"column:priority_legal"` - + ID uint `gorm:"column:id;primaryKey"` SiteURL *string `gorm:"column:site_url"` FileSize int `gorm:"column:file_size"` DefaultLocale *string `gorm:"column:default_locale"` @@ -100,8 +73,7 @@ func (Settings) Fillable() []string { return []string{"site_url", "file_size", "extra_urls"} } -// Rules is empty: the PHP settings form validated nothing. The one check of -// the port, on frontend_url, is in BeforeValidate. +// Rules is empty: the settings controller runs Validate before a save. func (Settings) Rules() map[string]string { return map[string]string{} } // The phrase keys of the other settings refusals. @@ -127,8 +99,7 @@ func (s Settings) Validate() error { } // Problems lists the settings' refusals by field (UI-SPEC C26): a site -// address that AllowedFrontendURL refuses (site_url and the Phase 17 -// frontend_url), a file size outside 1..MaxFileSize and an additional +// address that AllowedFrontendURL refuses, a file size outside 1..MaxFileSize and an additional // address that is not a path or an http(s) URL (extra_urls..url) or // whose lastmod is not a date (extra_urls..lastmod). func (s Settings) Problems() map[string][]string { @@ -136,9 +107,6 @@ func (s Settings) Problems() map[string][]string { if !AllowedFrontendURL(Text(s.SiteURL)) { out["site_url"] = []string{errFrontendURL} } - if !AllowedFrontendURL(Text(s.FrontendURL)) { - out["frontend_url"] = []string{errFrontendURL} - } if s.FileSize < 1 || s.FileSize > MaxFileSize { out["file_size"] = []string{errFileSize} } @@ -153,14 +121,12 @@ func (s Settings) Problems() map[string][]string { return out } -// AllowedFrontendURL reports whether u may be the frontend_url setting: the -// empty string, or an absolute http or https URL with a host and without -// whitespace, control characters, a double quote, <, > or &. The PHP plugin -// stored any text. The value is written into every loc, every hreflang link -// and the stylesheet instruction of the sitemap, and sitemap.xsl renders -// each loc as a link on the public origin, so a value outside this form -// (a javascript: URL, or text that closes the attribute it is written into) -// would reach the readers of /sitemap.xml (T-17-15). +// AllowedFrontendURL reports whether u may be the site address: the empty +// string, or an absolute http or https URL with a host and without +// whitespace, control characters, a double quote, <, > or &. The value +// starts every and hreflang link of the sitemaps, so a value outside +// this form (a javascript: URL, or text that closes the attribute it is +// written into) would reach their readers (T-17-15, T-33-12). func AllowedFrontendURL(u string) bool { if u == "" { return true @@ -183,32 +149,9 @@ func AllowedFrontendURL(u string) bool { return parsed.Opaque == "" && parsed.Hostname() != "" } -// DefaultSettings is the settings PHP read when nothing was saved: tiers on -// with the default priorities. +// DefaultSettings is the settings read when no row is stored. func DefaultSettings() Settings { - return Settings{ID: 1, EnablePriorityTiers: true, FileSize: DefaultFileSize} -} - -// Priority returns the value of a priority setting: the stored text, or the -// PHP default when it is null. -func (s Settings) Priority(key string) string { - var p *string - switch key { - case PriorityHomepage: - p = s.PriorityHomepage - case PriorityServices: - p = s.PriorityServices - case PriorityBlog: - p = s.PriorityBlog - case PriorityCategories: - p = s.PriorityCategories - case PriorityLegal: - p = s.PriorityLegal - } - if p == nil { - return PriorityDefaults[key] - } - return *p + return Settings{ID: 1, FileSize: DefaultFileSize} } func init() { diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go index bf85e6d..99075e0 100644 --- a/sitemap_smoke_test.go +++ b/sitemap_smoke_test.go @@ -2,7 +2,6 @@ package sitemap_test import ( "bytes" - "context" "encoding/json" "fmt" "io" @@ -16,87 +15,19 @@ import ( "time" _ "git.golem15.com/golem15/sm-sitemap-plugin" - "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/bouncer" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/compass" "git.golem15.com/golem15/summercms/modules/lagoon" - "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) -// testTypesPluginID is the fake content plugin that registers menu item -// types for the sitemap smoke tests. The real Journal and sitemap -// combination is proven by the host application's tests. -const testTypesPluginID = "acme.sitemaptypes" - -// acmeTypes is a content plugin with one menu item type, acme-post, whose -// reference "7" resolves to a fixed post with two alternates. -type acmeTypes struct{} - -func (acmeTypes) ID() string { return testTypesPluginID } -func (acmeTypes) Requires() []string { return nil } -func (acmeTypes) Register(*backpack.App) error { return nil } -func (acmeTypes) Boot(*backpack.App) error { return nil } -func (acmeTypes) MenuItemTypes() []pact.MenuItemType { - mtime := time.Date(2026, 3, 4, 10, 0, 0, 0, time.UTC) - return []pact.MenuItemType{acmeList(), { - Key: "acme-post", - Label: "Acme post", - DefaultPattern: "/news/:slug", - References: func(context.Context) ([]pact.MenuItemReference, error) { - return []pact.MenuItemReference{{Value: "7", Label: "Hello"}}, nil - }, - Resolve: func(_ context.Context, in pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { - if in.Reference != "7" { - return nil, nil - } - return &pact.ResolvedMenuItem{ - Title: "Hello", - URL: "/news/hello", - MTime: &mtime, - Alternates: []pact.MenuItemAlternate{ - {Locale: "pl", URL: "/news/hello"}, - {Locale: "en", URL: "/en/news/hello"}, - }, - }, nil - }, - }} -} - -// acmeList is a list type of the fake plugin: no URL of its own, one post -// with two alternates as its item. -func acmeList() pact.MenuItemType { - return pact.MenuItemType{ - Key: "acme-list", - Label: "Acme posts", - DefaultPattern: "/news/:slug", - PatternFrom: "acme-post", - DynamicItems: true, - Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { - return &pact.ResolvedMenuItem{Items: []pact.ResolvedMenuItem{{ - Title: "A", - URL: "/news/a", - Alternates: []pact.MenuItemAlternate{ - {Locale: "pl", URL: "/news/a"}, - {Locale: "en", URL: "/en/news/a"}, - }, - }}}, nil - }, - } -} - -var _ pact.HasMenuItemTypes = acmeTypes{} - -func init() { - party.Register(acmeTypes{}) -} - -// sitemapEnv is a booted application with the sitemap and the fake types. +// sitemapEnv is a booted application with the sitemap and the fake +// content plugin. type sitemapEnv struct { h http.Handler app *backpack.App @@ -104,13 +35,6 @@ type sitemapEnv struct { plugins []party.Plugin } -// bootSitemap boots the application with the sitemap and the fake types; -// overrides set further config keys. -func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { - t.Helper() - return bootPlugins(t, []string{testTypesPluginID, "golem15.sitemap"}, overrides...) -} - // bootPlugins boots the application with the plugins ids on a fresh // database; overrides set further config keys. func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitemapEnv { @@ -174,19 +98,6 @@ func (e *sitemapEnv) get(t *testing.T, path string) *httptest.ResponseRecorder { return rec } -func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem) models.Definition { - t.Helper() - theme := models.DefinitionTheme - def := models.Definition{Theme: &theme} - def.SetItems(items) - if err := e.gdb.Create(&def).Error; err != nil { - t.Fatal(err) - } - return def -} - -func text(s string) *string { return &s } - func writeFile(t *testing.T, dir, name, body string) string { t.Helper() path := filepath.Join(dir, name) diff --git a/tests/php/gen-goldens.php b/tests/php/gen-goldens.php deleted file mode 100644 index c9c0c91..0000000 --- a/tests/php/gen-goldens.php +++ /dev/null @@ -1,465 +0,0 @@ - reference => - * resolved item {url, mtime, alternates: [[locale, url], ...], items}), - * settings (Sitemap settings values; absent keys take the PHP defaults), - * manifest (the updates manifest object), skip (URLs a beforeAddItem - * listener drops) and extra (URL => text of an element an addItem - * listener appends). - * - * A registry type's resolved item is given in the Go shape (one item per - * record with ordered alternates); this script turns it into what the PHP - * headless resolvers returned: a list type (no url) yields one entry per - * alternate for each child (a child without alternates once), any other type - * its children with their default URL. - */ - -$vendor = getenv('WINTER_VENDOR'); -if (!$vendor || !is_file($vendor . '/autoload.php')) { - fwrite(STDERR, "set WINTER_VENDOR to the vendor directory of a WinterCMS application\n"); - exit(2); -} -require $vendor . '/autoload.php'; - -use Illuminate\Http\Request; -use Illuminate\Routing\RouteCollection; -use Winter\Storm\Router\UrlGenerator; - -date_default_timezone_set('UTC'); - -if ($argc !== 3) { - fwrite(STDERR, "usage: php gen-goldens.php INPUT.json OUTPUT.golden\n"); - exit(2); -} - -$input = json_decode(file_get_contents($argv[1]), true); -if (!is_array($input)) { - fwrite(STDERR, "invalid JSON input\n"); - exit(1); -} - -function urlGenerator(string $root): UrlGenerator -{ - return new UrlGenerator(new RouteCollection, Request::create(rtrim($root, '/') . '/sitemap.xml')); -} - -if (isset($input['urls'])) { - $url = urlGenerator($input['root']); - $out = ''; - foreach ($input['urls'] as $path) { - $out .= $url->to($path) . "\n"; - } - file_put_contents($argv[2], $out); - exit(0); -} - -if (isset($input['case'])) { - file_put_contents($argv[2], (new GoldenSitemap($input['case']))->render()); - exit(0); -} - -$out = ''; -foreach ($input['cases'] as $case) { - $out .= '=== ' . $case['name'] . "\n" . (new GoldenSitemap($case))->render(); -} -file_put_contents($argv[2], $out); - -/** - * Golem15\Sitemap\Classes\DefinitionItem, unchanged properties. - */ -class GoldenDefinitionItem -{ - public $changefreq; - public $priority; - public $nesting = false; - public $type = 'url'; - public $url = null; - public $code = null; - public $reference = null; - public $cmsPage = null; - public $urlPattern = null; - public $exists = false; - - public static function initFromArray(array $items): array - { - $result = []; - foreach ($items as $itemData) { - $obj = new self; - foreach ($itemData as $name => $value) { - if (property_exists($obj, $name)) { - $obj->$name = $value; - } - } - $result[] = $obj; - } - return $result; - } -} - -/** - * The generation code of Golem15\Sitemap\Models\Definition and routes.php. - */ -class GoldenSitemap -{ - const MAX_URLS = 50000; - - protected $case; - protected $url; - protected $items; - protected $updatesManifest = null; - protected $urlCount = 0; - protected $urlSet; - protected $xmlObject; - - public function __construct(array $case) - { - $this->case = $case; - $this->url = urlGenerator($case['root']); - $this->items = GoldenDefinitionItem::initFromArray($case['items'] ?? []); - } - - protected function setting(string $key, $default) - { - $settings = $this->case['settings'] ?? []; - return array_key_exists($key, $settings) ? $settings[$key] : $default; - } - - /** routes.php */ - public function render(): string - { - $xml = $this->generateSitemap(); - - $frontendUrl = $this->setting('frontend_url', null); - if ($frontendUrl) { - $frontendUrl = rtrim($frontendUrl, '/'); - $backendUrl = rtrim($this->url->to('/'), '/'); - $xml = preg_replace_callback('/(.*?)<\/loc>/', function ($m) use ($backendUrl, $frontendUrl) { - return '' . str_replace($backendUrl, $frontendUrl, $m[1]) . ''; - }, $xml); - $xml = preg_replace_callback('/(]*href=")([^"]*)(")/', function ($m) use ($backendUrl, $frontendUrl) { - return $m[1] . str_replace($backendUrl, $frontendUrl, $m[2]) . $m[3]; - }, $xml); - $xml = str_replace( - 'href="' . $backendUrl . '/sitemap.xsl"', - 'href="' . $frontendUrl . '/sitemap.xsl"', - $xml - ); - } - - return $xml; - } - - /** - * What the PHP headless resolvers returned for a resolved item in the - * Go shape. - */ - protected function phpResolved(array $r, bool $listChildren): array - { - $info = []; - if (!empty($r['url'])) { - $info['url'] = $r['url']; - } - if (array_key_exists('mtime', $r)) { - $info['mtime'] = $r['mtime'] === null ? null : new DateTime($r['mtime']); - } - $links = []; - foreach ($r['alternates'] ?? [] as [$locale, $href]) { - $links[$locale] = $href; - } - $info['alternateLinks'] = $links; - if (isset($r['items'])) { - $info['items'] = []; - foreach ($r['items'] as $child) { - $childInfo = $this->phpResolved($child, false); - if ($listChildren && $childInfo['alternateLinks']) { - foreach ($childInfo['alternateLinks'] as $locale => $href) { - $perLocale = $childInfo; - $perLocale['url'] = $href; - $info['items'][] = $perLocale; - } - } else { - $info['items'][] = $childInfo; - } - } - } - return $info; - } - - protected function resolveItem(GoldenDefinitionItem $item) - { - $types = $this->case['types'] ?? []; - $ref = (string) $item->reference; - if (!isset($types[$item->type][$ref])) { - return null; - } - $r = $types[$item->type][$ref]; - return $this->phpResolved($r, empty($r['url'])); - } - - public function generateSitemap(): string - { - if (!$this->items) { - return ''; - } - - foreach ($this->items as $item) { - if ($item->type === 'url') { - $this->addItemToSet($item); - } else { - $menuItemInfo = $this->resolveItem($item); - if (!is_array($menuItemInfo)) { - continue; - } - - if (isset($menuItemInfo['url'])) { - $this->addItemToSet($item, $menuItemInfo); - } - - if (isset($menuItemInfo['items'])) { - $menuItemIterator = function ($menuItems) use (&$menuItemIterator, $item) { - foreach ($menuItems as $menuItem) { - if (isset($menuItem['url'])) { - $this->addItemToSet($item, $menuItem); - } - - if (isset($menuItem['items'])) { - $menuItemIterator($menuItem['items']); - } - } - }; - - $menuItemIterator($menuItemInfo['items']); - } - } - } - - $urlSet = $this->getUrlSet(); - $xml = $this->getXmlObject(); - $xml->appendChild($urlSet); - - return $xml->saveXML(); - } - - protected function getXmlObject(): DOMDocument - { - if ($this->xmlObject !== null) { - return $this->xmlObject; - } - - $xml = new DOMDocument; - $xml->encoding = 'UTF-8'; - $xss = $xml->createProcessingInstruction('xml-stylesheet', - 'type="text/xsl" href="' . UrlGenerator::buildUrl($this->url->current(), ['path' => 'sitemap.xsl']) . '"' - ); - $xml->appendChild($xss); - - return $this->xmlObject = $xml; - } - - protected function getUrlSet(): DOMElement - { - if ($this->urlSet !== null) { - return $this->urlSet; - } - - $xml = $this->getXmlObject(); - $urlSet = $xml->createElement('urlset'); - $urlSet->setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9'); - $urlSet->setAttribute('xmlns:xhtml', 'http://www.w3.org/1999/xhtml'); - $urlSet->setAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance'); - $urlSet->setAttribute('xsi:schemaLocation', 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd'); - - return $this->urlSet = $urlSet; - } - - protected function addItemToSet(GoldenDefinitionItem $item, ?array $itemInfo = null): void - { - $xml = $this->getXmlObject(); - $urlSet = $this->getUrlSet(); - - if ($this->urlCount >= self::MAX_URLS) { - return; - } - - if (!isset($itemInfo['url'])) { - $itemInfo['url'] = $this->url->to($item->url); - } - - $lastModified = $itemInfo['mtime'] ?? null; - - if ($lastModified === null) { - $lastModified = $this->lookupManifestLastmod($itemInfo['url']); - } - - if ($lastModified instanceof \DateTime) { - $lastModified = $lastModified->getTimestamp(); - } - $itemInfo['lastModified'] = $lastModified ? date('c', $lastModified) : null; - - // golem15.sitemap.beforeAddItem: a listener that drops the case's - // skip URLs. - if (in_array($itemInfo['url'], $this->case['skip'] ?? [], true)) { - return; - } - - if (empty($itemInfo['url'])) { - return; - } else { - $itemInfo['url'] = $this->url->to($itemInfo['url']); - } - - $this->urlCount++; - - $tierPriority = $this->getPriorityForUrl($itemInfo['url']); - $effectivePriority = $tierPriority ?? $item->priority; - - $urlElement = $xml->createElement('url'); - - $loc = $xml->createElement('loc'); - $loc->appendChild($xml->createTextNode($itemInfo['url'])); - $urlElement->appendChild($loc); - - if (!empty($itemInfo['lastModified'])) { - $lastmod = $xml->createElement('lastmod'); - $lastmod->appendChild($xml->createTextNode((string) $itemInfo['lastModified'])); - $urlElement->appendChild($lastmod); - } - - $changefreq = $xml->createElement('changefreq'); - $changefreq->appendChild($xml->createTextNode((string) $item->changefreq)); - $urlElement->appendChild($changefreq); - - $priority = $xml->createElement('priority'); - $priority->appendChild($xml->createTextNode((string) $effectivePriority)); - $urlElement->appendChild($priority); - - if (!empty($itemInfo['alternateLinks']) && count($itemInfo['alternateLinks']) > 1) { - foreach ($itemInfo['alternateLinks'] as $hreflang => $href) { - $link = $xml->createElement('xhtml:link'); - $link->setAttribute('rel', 'alternate'); - $link->setAttribute('hreflang', $hreflang); - $link->setAttribute('href', $this->url->to($href)); - $urlElement->appendChild($link); - } - } - - $urlSet->appendChild($urlElement); - - // golem15.sitemap.addItem: a listener that appends the case's extra - // element to the URL. - $extra = $this->case['extra'] ?? []; - if (isset($extra[$itemInfo['url']])) { - $element = $xml->createElement('extra'); - $element->appendChild($xml->createTextNode($extra[$itemInfo['url']])); - $urlElement->appendChild($element); - } - } - - protected function lookupManifestLastmod(string $url): ?int - { - $manifest = $this->getUpdatesManifest(); - if (!$manifest) { - return null; - } - - $path = parse_url($url, PHP_URL_PATH) ?: '/'; - $path = $this->normalizeManifestPath($path); - - if (!isset($manifest[$path])) { - return null; - } - - $timestamp = strtotime((string) $manifest[$path]); - - return $timestamp ?: null; - } - - protected function getUpdatesManifest(): array - { - if ($this->updatesManifest !== null) { - return $this->updatesManifest; - } - - $decoded = $this->case['manifest'] ?? null; - if (!is_array($decoded)) { - return $this->updatesManifest = []; - } - - $normalized = []; - foreach ($decoded as $key => $value) { - $normalized[$this->normalizeManifestPath((string) $key)] = $value; - } - - return $this->updatesManifest = $normalized; - } - - protected function normalizeManifestPath(string $path): string - { - $path = '/' . ltrim($path, '/'); - - return $path === '/' ? '/' : rtrim($path, '/'); - } - - protected function getPriorityForUrl(string $url): ?string - { - if (!$this->setting('enable_priority_tiers', true)) { - return null; - } - - $path = parse_url($url, PHP_URL_PATH) ?: '/'; - $path = rtrim($path, '/'); - - if ($path === '' || $path === '/en') { - return $this->setting('priority_homepage', '1.0'); - } - - if (preg_match('#^/(uslugi|en/services|realizacje|en/work)/[^/]+$#', $path)) { - return $this->setting('priority_services', '0.8'); - } - - if (preg_match('#^/(blog|en/blog)/[^/]+/[^/]+$#', $path)) { - return $this->setting('priority_blog', '0.6'); - } - - if (preg_match('#^/(uslugi|en/services|realizacje|en/work|blog|en/blog)(/[^/]+)?$#', $path)) { - return $this->setting('priority_categories', '0.4'); - } - - $legalPaths = [ - '/polityka-prywatnosci', '/en/privacy-policy', - '/regulamin', '/en/terms', - '/o-nas', '/en/about', - '/kontakt', '/en/contact', - '/laboratorium-ai', '/en/ai-lab', - '/laboratorium-ai/zabawa-animacja', '/en/ai-lab/blob-playground', - ]; - if (in_array($path, $legalPaths)) { - return $this->setting('priority_legal', '0.2'); - } - - return null; - } -}