golem15/sm-sitemap-plugin
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: image sitemap files (tracer)

Browse Source 
- migration 202610280004: image_file_no, kind, published_at and parent_refs
  on the slot table; the image, News, IndexNow and HTML sitemap settings
- the rebuild stores each page's featured and body images (absolute on the
  site, deduplicated, at most 1000) and numbers the rows with images into
  sitemap-images-<n>.xml with the per-type files' stable rules
- the index lists the image files while images_enabled is on; the files
  route serves them with image:image and image:loc only
This commit is contained in:
Jakub Zych
2026-11-04 12:19:05 +01:00
parent 64453d2a91
commit 5bf56fde57
9 changed files with 587 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
classes/images.go Normal file
View File

@@ -0,0 +1,50 @@
package classes
import (
"strings"
"git.golem15.com/golem15/sm-sitemap-plugin/models"
"git.golem15.com/golem15/summercms/modules/pact"
)
// MaxImagesPerURL is the most images one <url> of an image sitemap lists
// (Google's limit).
const MaxImagesPerURL = 1000
// ImagesGroup is the file group of the image sitemaps:
// sitemap-images-<n>.xml. No content type may use it (ReservedSlugs).
const ImagesGroup = "images"
// PageImages lists the images of one locale of a record for the image
// sitemap (D-31): the featured Image first, then Images in document order,
// each a path on the site or an absolute http(s) URL (a protocol-relative
// //host/... address is read as https). Another value (a data: or
// javascript: URL, a relative path) is skipped, and an address is listed
// once, compared as absolute on site. At most MaxImagesPerURL are kept.
func PageImages(l pact.ContentLocale, site string) []string {
out := []string{}
seen := map[string]bool{}
add := func(u string) {
u = strings.TrimSpace(u)
if strings.HasPrefix(u, "//") {
u = "https:" + u
}
if u == "" || !models.AllowedAddress(u) {
return
}
key := u
if abs := Absolute(site, u); abs != "" {
key = abs
}
if seen[key] || len(out) >= MaxImagesPerURL {
return
}
seen[key] = true
out = append(out, u)
}
add(l.Image)
for _, img := range l.Images {
add(img.URL)
}
return out
}

86
classes/rebuild.go
View File

@@ -3,6 +3,7 @@ package classes
import (
"context"
"crypto/sha256"
"database/sql/driver"
"encoding/hex"
"errors"
"fmt"
@@ -86,7 +87,9 @@ type slotKey struct {
// wanted is one address a rebuild found: its group, its row key, its path
// and title, its first lastmod (for a new row) and the reason it is left
// out ("" when listed). An additional address carries its own lastmod,
// which the editor sets, so it is written on every rebuild.
// which the editor sets, so it is written on every rebuild. images, kind,
// published and parents are the record's facts for the image and News
// sitemaps and the HTML sitemap.
type wanted struct {
key slotKey
grp string
@@ -97,6 +100,10 @@ type wanted struct {
extra bool
extraMod *time.Time
alternates []models.Alternate
images []string
kind string
published *time.Time
parents []models.Ref
}
// collected is what a rebuild read from the content types and settings.
@@ -131,11 +138,11 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
}
reg, _ := app.Lookup[*signpost.Registry]()
filter, _ := app.Lookup[contract.Filter]()
found, err := collect(ctx, reg, filter, settings, types)
site := SiteURL(app, settings)
found, err := collect(ctx, reg, filter, settings, types, site)
if err != nil {
return res, err
}
site := SiteURL(app, settings)
err = db.Transaction(func(tx *gorm.DB) error {
if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
return err
@@ -158,8 +165,10 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
// one (D-95): an excluded address keeps its row with the Filter's reason, a
// Decision.Group moves it to that file group, and Decision.LastMod is the
// first lastmod of a new row. The hreflang alternates of a record are its
// listed addresses (D-55 to D-57).
func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter, settings models.Settings, types map[string]models.TypeSetting) (collected, error) {
// listed addresses (D-55 to D-57). Each address keeps its page's images
// (PageImages, on site), the type's kind, the record's first publication
// and its parents.
func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter, settings models.Settings, types map[string]models.TypeSetting, site string) (collected, error) {
out := collected{unpublished: map[slotKey]bool{}}
seen := map[slotKey]bool{}
defaultLocale := models.Text(settings.DefaultLocale)
@@ -183,6 +192,10 @@ func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter
if reg != nil {
for _, typ := range reg.ContentTypes() {
cfg := EffectiveType(typ, types)
kind := ""
if typ.Content != nil {
kind = typ.Content.Kind
}
after := ""
for {
page, err := reg.Records(ctx, typ.Key, after, recordsPage, defaultLocale)
@@ -195,16 +208,26 @@ func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter
out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true
continue
}
parents := make([]models.Ref, 0, len(rec.Parents))
for _, p := range rec.Parents {
if p.Type != "" && p.ID != "" {
parents = append(parents, models.Ref{Type: p.Type, ID: p.ID})
}
}
for _, l := range rec.Locales {
if strings.TrimSpace(l.URL) == "" {
continue
}
w := wanted{
key: slotKey{typ.Key, rec.ID, l.Locale},
grp: cfg.Slug,
url: l.URL,
title: l.Title,
firstMod: rec.LastMod,
key: slotKey{typ.Key, rec.ID, l.Locale},
grp: cfg.Slug,
url: l.URL,
title: l.Title,
firstMod: rec.LastMod,
images: PageImages(l, site),
kind: kind,
published: rec.PublishedAt,
parents: parents,
}
if !cfg.Include {
w.reason = contract.ReasonType
@@ -368,7 +391,10 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
FirstSeen: now,
LastMod: utcPtr(mod),
Alternates: lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true},
Images: lagoon.Jsonable[[]string]{Data: []string{}, Valid: true},
Images: lagoon.Jsonable[[]string]{Data: nonNil(w.images), Valid: true},
Kind: w.kind,
PublishedAt: utcPtr(w.published),
ParentRefs: lagoon.Jsonable[[]models.Ref]{Data: nonNil(w.parents), Valid: true},
ExcludedReason: w.reason,
UpdatedAt: now,
}
@@ -388,6 +414,10 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
r.Title = w.title
r.ExcludedReason = w.reason
r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}
r.Images = lagoon.Jsonable[[]string]{Data: nonNil(w.images), Valid: true}
r.Kind = w.kind
r.PublishedAt = utcPtr(w.published)
r.ParentRefs = lagoon.Jsonable[[]models.Ref]{Data: nonNil(w.parents), Valid: true}
if w.extra {
r.LastMod = utcPtr(w.extraMod)
}
@@ -456,6 +486,21 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
}
}
}
// The image files: the listed rows with images, numbered like the
// per-group files (stable, holes kept, renumbered with them).
imageSlots := make([]*Slot, 0, len(slots))
for _, s := range slots {
r := rowsByID[s.ID]
imageSlots = append(imageSlots, &Slot{ID: r.ID, Grp: ImagesGroup, FileNo: r.ImageFileNo, FirstSeen: r.FirstSeen, Listed: s.Listed && len(r.Images.Get()) > 0})
}
AssignSlots(imageSlots, size, renumber)
for _, s := range imageSlots {
r := rowsByID[s.ID]
if !sameInt(r.ImageFileNo, s.FileNo) {
r.ImageFileNo = s.FileNo
dirty[r.ID] = true
}
}
res.Files = len(files)
for id := range dirty {
r := rowsByID[id]
@@ -467,6 +512,11 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
"file_no": r.FileNo,
"lastmod": r.LastMod,
"alternates": r.Alternates,
"images": r.Images,
"image_file_no": r.ImageFileNo,
"kind": r.Kind,
"published_at": r.PublishedAt,
"parent_refs": r.ParentRefs,
"excluded_reason": r.ExcludedReason,
"updated_at": r.UpdatedAt,
}).Error; err != nil {
@@ -481,12 +531,18 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
// rowChanged reports whether a rebuild changed a stored row.
func rowChanged(a, b models.URL) bool {
if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) {
if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || a.Kind != b.Kind ||
!sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) || !sameTime(a.PublishedAt, b.PublishedAt) {
return true
}
av, _ := a.Alternates.Value()
bv, _ := b.Alternates.Value()
return av != bv
return jsonDiffers(a.Alternates, b.Alternates) || jsonDiffers(a.Images, b.Images) || jsonDiffers(a.ParentRefs, b.ParentRefs)
}
// jsonDiffers reports whether two stored JSON values differ.
func jsonDiffers(a, b driver.Valuer) bool {
av, _ := a.Value()
bv, _ := b.Value()
return fmt.Sprint(av) != fmt.Sprint(bv)
}
func sameInt(a, b *int) bool {

33
classes/writer.go
View File

@@ -14,6 +14,7 @@ import (
const (
NSSitemap = "http://www.sitemaps.org/schemas/sitemap/0.9"
NSXHTML = "http://www.w3.org/1999/xhtml"
NSImage = "http://www.google.com/schemas/sitemap-image/1.1"
)
// IndexFile is one <sitemap> of the index: the file's absolute URL and the
@@ -124,6 +125,38 @@ func WriteURLSet(w io.Writer, entries []FileEntry) error {
return bw.Flush()
}
// ImageEntry is one <url> of an image sitemap: the page's absolute address
// and its images' absolute addresses (at most MaxImagesPerURL).
type ImageEntry struct {
Loc string
Images []string
}
// WriteImageURLSet writes one image sitemap file: a <url> per page with its
// <loc> and one <image:image><image:loc> per image. Only image:image and
// image:loc are written, the tags Google still reads (D-31).
func WriteImageURLSet(w io.Writer, entries []ImageEntry) error {
bw := bufio.NewWriter(w)
bw.WriteString(xml.Header)
bw.WriteString(`<urlset xmlns="` + NSSitemap + `" xmlns:image="` + NSImage + `">`)
for _, e := range entries {
bw.WriteString("\n<url><loc>")
escape(bw, e.Loc)
bw.WriteString("</loc>")
for i, img := range e.Images {
if i >= MaxImagesPerURL {
break
}
bw.WriteString("<image:image><image:loc>")
escape(bw, img)
bw.WriteString("</image:loc></image:image>")
}
bw.WriteString("</url>")
}
bw.WriteString("\n</urlset>\n")
return bw.Flush()
}
// validSlug reports whether s may name a file group.
func validSlug(s string) bool {
return s != "" && len(s) <= 64 && fileNamePattern.MatchString(FileName(s, 1)) && !strings.HasSuffix(s, "-")