diff --git a/classes/images.go b/classes/images.go new file mode 100644 index 0000000..1820850 --- /dev/null +++ b/classes/images.go @@ -0,0 +1,50 @@ +package classes + +import ( + "strings" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/pact" +) + +// MaxImagesPerURL is the most images one of an image sitemap lists +// (Google's limit). +const MaxImagesPerURL = 1000 + +// ImagesGroup is the file group of the image sitemaps: +// sitemap-images-.xml. No content type may use it (ReservedSlugs). +const ImagesGroup = "images" + +// PageImages lists the images of one locale of a record for the image +// sitemap (D-31): the featured Image first, then Images in document order, +// each a path on the site or an absolute http(s) URL (a protocol-relative +// //host/... address is read as https). Another value (a data: or +// javascript: URL, a relative path) is skipped, and an address is listed +// once, compared as absolute on site. At most MaxImagesPerURL are kept. +func PageImages(l pact.ContentLocale, site string) []string { + out := []string{} + seen := map[string]bool{} + add := func(u string) { + u = strings.TrimSpace(u) + if strings.HasPrefix(u, "//") { + u = "https:" + u + } + if u == "" || !models.AllowedAddress(u) { + return + } + key := u + if abs := Absolute(site, u); abs != "" { + key = abs + } + if seen[key] || len(out) >= MaxImagesPerURL { + return + } + seen[key] = true + out = append(out, u) + } + add(l.Image) + for _, img := range l.Images { + add(img.URL) + } + return out +} diff --git a/classes/rebuild.go b/classes/rebuild.go index 3cf5827..601f764 100644 --- a/classes/rebuild.go +++ b/classes/rebuild.go @@ -3,6 +3,7 @@ package classes import ( "context" "crypto/sha256" + "database/sql/driver" "encoding/hex" "errors" "fmt" @@ -86,7 +87,9 @@ type slotKey struct { // wanted is one address a rebuild found: its group, its row key, its path // and title, its first lastmod (for a new row) and the reason it is left // out ("" when listed). An additional address carries its own lastmod, -// which the editor sets, so it is written on every rebuild. +// which the editor sets, so it is written on every rebuild. images, kind, +// published and parents are the record's facts for the image and News +// sitemaps and the HTML sitemap. type wanted struct { key slotKey grp string @@ -97,6 +100,10 @@ type wanted struct { extra bool extraMod *time.Time alternates []models.Alternate + images []string + kind string + published *time.Time + parents []models.Ref } // collected is what a rebuild read from the content types and settings. @@ -131,11 +138,11 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { } reg, _ := app.Lookup[*signpost.Registry]() filter, _ := app.Lookup[contract.Filter]() - found, err := collect(ctx, reg, filter, settings, types) + site := SiteURL(app, settings) + found, err := collect(ctx, reg, filter, settings, types, site) if err != nil { return res, err } - site := SiteURL(app, settings) err = db.Transaction(func(tx *gorm.DB) error { if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil { return err @@ -158,8 +165,10 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { // one (D-95): an excluded address keeps its row with the Filter's reason, a // Decision.Group moves it to that file group, and Decision.LastMod is the // first lastmod of a new row. The hreflang alternates of a record are its -// listed addresses (D-55 to D-57). -func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter, settings models.Settings, types map[string]models.TypeSetting) (collected, error) { +// listed addresses (D-55 to D-57). Each address keeps its page's images +// (PageImages, on site), the type's kind, the record's first publication +// and its parents. +func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter, settings models.Settings, types map[string]models.TypeSetting, site string) (collected, error) { out := collected{unpublished: map[slotKey]bool{}} seen := map[slotKey]bool{} defaultLocale := models.Text(settings.DefaultLocale) @@ -183,6 +192,10 @@ func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter if reg != nil { for _, typ := range reg.ContentTypes() { cfg := EffectiveType(typ, types) + kind := "" + if typ.Content != nil { + kind = typ.Content.Kind + } after := "" for { page, err := reg.Records(ctx, typ.Key, after, recordsPage, defaultLocale) @@ -195,16 +208,26 @@ func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true continue } + parents := make([]models.Ref, 0, len(rec.Parents)) + for _, p := range rec.Parents { + if p.Type != "" && p.ID != "" { + parents = append(parents, models.Ref{Type: p.Type, ID: p.ID}) + } + } for _, l := range rec.Locales { if strings.TrimSpace(l.URL) == "" { continue } w := wanted{ - key: slotKey{typ.Key, rec.ID, l.Locale}, - grp: cfg.Slug, - url: l.URL, - title: l.Title, - firstMod: rec.LastMod, + key: slotKey{typ.Key, rec.ID, l.Locale}, + grp: cfg.Slug, + url: l.URL, + title: l.Title, + firstMod: rec.LastMod, + images: PageImages(l, site), + kind: kind, + published: rec.PublishedAt, + parents: parents, } if !cfg.Include { w.reason = contract.ReasonType @@ -368,7 +391,10 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) FirstSeen: now, LastMod: utcPtr(mod), Alternates: lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}, - Images: lagoon.Jsonable[[]string]{Data: []string{}, Valid: true}, + Images: lagoon.Jsonable[[]string]{Data: nonNil(w.images), Valid: true}, + Kind: w.kind, + PublishedAt: utcPtr(w.published), + ParentRefs: lagoon.Jsonable[[]models.Ref]{Data: nonNil(w.parents), Valid: true}, ExcludedReason: w.reason, UpdatedAt: now, } @@ -388,6 +414,10 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) r.Title = w.title r.ExcludedReason = w.reason r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true} + r.Images = lagoon.Jsonable[[]string]{Data: nonNil(w.images), Valid: true} + r.Kind = w.kind + r.PublishedAt = utcPtr(w.published) + r.ParentRefs = lagoon.Jsonable[[]models.Ref]{Data: nonNil(w.parents), Valid: true} if w.extra { r.LastMod = utcPtr(w.extraMod) } @@ -456,6 +486,21 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) } } } + // The image files: the listed rows with images, numbered like the + // per-group files (stable, holes kept, renumbered with them). + imageSlots := make([]*Slot, 0, len(slots)) + for _, s := range slots { + r := rowsByID[s.ID] + imageSlots = append(imageSlots, &Slot{ID: r.ID, Grp: ImagesGroup, FileNo: r.ImageFileNo, FirstSeen: r.FirstSeen, Listed: s.Listed && len(r.Images.Get()) > 0}) + } + AssignSlots(imageSlots, size, renumber) + for _, s := range imageSlots { + r := rowsByID[s.ID] + if !sameInt(r.ImageFileNo, s.FileNo) { + r.ImageFileNo = s.FileNo + dirty[r.ID] = true + } + } res.Files = len(files) for id := range dirty { r := rowsByID[id] @@ -467,6 +512,11 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) "file_no": r.FileNo, "lastmod": r.LastMod, "alternates": r.Alternates, + "images": r.Images, + "image_file_no": r.ImageFileNo, + "kind": r.Kind, + "published_at": r.PublishedAt, + "parent_refs": r.ParentRefs, "excluded_reason": r.ExcludedReason, "updated_at": r.UpdatedAt, }).Error; err != nil { @@ -481,12 +531,18 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) // rowChanged reports whether a rebuild changed a stored row. func rowChanged(a, b models.URL) bool { - if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) { + if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || a.Kind != b.Kind || + !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) || !sameTime(a.PublishedAt, b.PublishedAt) { return true } - av, _ := a.Alternates.Value() - bv, _ := b.Alternates.Value() - return av != bv + return jsonDiffers(a.Alternates, b.Alternates) || jsonDiffers(a.Images, b.Images) || jsonDiffers(a.ParentRefs, b.ParentRefs) +} + +// jsonDiffers reports whether two stored JSON values differ. +func jsonDiffers(a, b driver.Valuer) bool { + av, _ := a.Value() + bv, _ := b.Value() + return fmt.Sprint(av) != fmt.Sprint(bv) } func sameInt(a, b *int) bool { diff --git a/classes/writer.go b/classes/writer.go index 0862a7d..425ba4b 100644 --- a/classes/writer.go +++ b/classes/writer.go @@ -14,6 +14,7 @@ import ( const ( NSSitemap = "http://www.sitemaps.org/schemas/sitemap/0.9" NSXHTML = "http://www.w3.org/1999/xhtml" + NSImage = "http://www.google.com/schemas/sitemap-image/1.1" ) // IndexFile is one of the index: the file's absolute URL and the @@ -124,6 +125,38 @@ func WriteURLSet(w io.Writer, entries []FileEntry) error { return bw.Flush() } +// ImageEntry is one of an image sitemap: the page's absolute address +// and its images' absolute addresses (at most MaxImagesPerURL). +type ImageEntry struct { + Loc string + Images []string +} + +// WriteImageURLSet writes one image sitemap file: a per page with its +// and one per image. Only image:image and +// image:loc are written, the tags Google still reads (D-31). +func WriteImageURLSet(w io.Writer, entries []ImageEntry) error { + bw := bufio.NewWriter(w) + bw.WriteString(xml.Header) + bw.WriteString(``) + for _, e := range entries { + bw.WriteString("\n") + escape(bw, e.Loc) + bw.WriteString("") + for i, img := range e.Images { + if i >= MaxImagesPerURL { + break + } + bw.WriteString("") + escape(bw, img) + bw.WriteString("") + } + bw.WriteString("") + } + bw.WriteString("\n\n") + return bw.Flush() +} + // validSlug reports whether s may name a file group. func validSlug(s string) bool { return s != "" && len(s) <= 64 && fileNamePattern.MatchString(FileName(s, 1)) && !strings.HasSuffix(s, "-") diff --git a/content_fixture_test.go b/content_fixture_test.go index 3f7b934..14f81b0 100644 --- a/content_fixture_test.go +++ b/content_fixture_test.go @@ -28,12 +28,16 @@ const ( offerModel = "acme_offers" ) -// fakeRecord is one record of a fake content type. +// fakeRecord is one record of a fake content type. PublishedAt and Parents +// are the optional descriptor facts (the News sitemap, the HTML sitemap's +// tree). type fakeRecord struct { - ID int - Published bool - LastMod time.Time - Locales []pact.ContentLocale + ID int + Published bool + LastMod time.Time + Locales []pact.ContentLocale + PublishedAt *time.Time + Parents []pact.ContentRef } // store holds the fake records per content type key. @@ -100,10 +104,12 @@ func (s *store) page(typ string, in pact.ContentList) pact.ContentPage { } mod := r.LastMod out.Records = append(out.Records, pact.ContentRecord{ - ID: strconv.Itoa(r.ID), - Published: r.Published, - LastMod: &mod, - Locales: slices.Clone(r.Locales), + ID: strconv.Itoa(r.ID), + Published: r.Published, + LastMod: &mod, + Locales: slices.Clone(r.Locales), + PublishedAt: r.PublishedAt, + Parents: slices.Clone(r.Parents), }) after = r.ID } diff --git a/models/settings.go b/models/settings.go index 7e38ad5..1e013b8 100644 --- a/models/settings.go +++ b/models/settings.go @@ -27,6 +27,15 @@ const errFrontendURL = "golem15.sitemap::lang.errors.frontend_url" // the last rebuild asked for from the admin. The Phase 17 columns // (frontend_url, priorities, tiers, the update manifest) stay in the table // and are no longer read (D-32). +// +// The image and News sitemaps, IndexNow and the HTML sitemap (D-31, D-34): +// ImagesEnabled lists the sitemap-images files, NewsEnabled sitemap-news.xml +// with NewsPublication (empty: app.name) and NewsSources (type: and +// category:: values; empty: every article type). IndexNowEnabled +// notifies api.indexnow.org with IndexNowKey after each rebuild, and the +// last answer is IndexNowLastAt, IndexNowLastCount and IndexNowLastStatus. +// HTMLEnabled serves the HTML sitemap API, HTMLPaths is the site's HTML +// sitemap address per locale and HTMLPageSize the links per page. type Settings struct { ID uint `gorm:"column:id;primaryKey"` SiteURL *string `gorm:"column:site_url"` @@ -37,6 +46,19 @@ type Settings struct { NumberedSize *int `gorm:"column:numbered_size"` BuiltAt *time.Time `gorm:"column:built_at"` RebuildRequestedAt *time.Time `gorm:"column:rebuild_requested_at"` + + ImagesEnabled bool `gorm:"column:images_enabled"` + NewsEnabled bool `gorm:"column:news_enabled"` + NewsPublication *string `gorm:"column:news_publication"` + NewsSources lagoon.Jsonable[[]string] `gorm:"column:news_sources"` + IndexNowEnabled bool `gorm:"column:indexnow_enabled"` + IndexNowKey *string `gorm:"column:indexnow_key"` + IndexNowLastAt *time.Time `gorm:"column:indexnow_last_at"` + IndexNowLastCount *int `gorm:"column:indexnow_last_count"` + IndexNowLastStatus *int `gorm:"column:indexnow_last_status"` + HTMLEnabled bool `gorm:"column:html_enabled"` + HTMLPaths lagoon.Jsonable[map[string]string] `gorm:"column:html_paths"` + HTMLPageSize int `gorm:"column:html_page_size"` } // DefaultFileSize is the number of addresses per sitemap file when the @@ -151,7 +173,55 @@ func AllowedFrontendURL(u string) bool { // DefaultSettings is the settings read when no row is stored. func DefaultSettings() Settings { - return Settings{ID: 1, FileSize: DefaultFileSize} + return Settings{ID: 1, FileSize: DefaultFileSize, ImagesEnabled: true, HTMLEnabled: true, HTMLPageSize: DefaultHTMLPageSize} +} + +// The HTML sitemap's links per page: DefaultHTMLPageSize when not stored, +// MinHTMLPageSize to MaxHTMLPageSize when set (UI-SPEC C26). +const ( + DefaultHTMLPageSize = 150 + MinHTMLPageSize = 50 + MaxHTMLPageSize = 500 +) + +// DefaultHTMLPaths are the HTML sitemap's addresses when none is stored. +func DefaultHTMLPaths() map[string]string { + return map[string]string{"pl": "/mapa-strony", "en": "/sitemap"} +} + +// EffectiveHTMLPageSize is the stored page size, or DefaultHTMLPageSize +// when it is outside MinHTMLPageSize..MaxHTMLPageSize. +func (s Settings) EffectiveHTMLPageSize() int { + if s.HTMLPageSize < MinHTMLPageSize || s.HTMLPageSize > MaxHTMLPageSize { + return DefaultHTMLPageSize + } + return s.HTMLPageSize +} + +// EffectiveHTMLPaths are the stored HTML sitemap addresses without blank +// ones, or DefaultHTMLPaths when none is stored. +func (s Settings) EffectiveHTMLPaths() map[string]string { + out := map[string]string{} + for locale, path := range s.HTMLPaths.Get() { + if p := strings.TrimSpace(path); p != "" && strings.TrimSpace(locale) != "" { + out[locale] = p + } + } + if len(out) == 0 { + return DefaultHTMLPaths() + } + return out +} + +// Sources returns the News sources without blank values. +func (s Settings) Sources() []string { + var out []string + for _, v := range s.NewsSources.Get() { + if v = strings.TrimSpace(v); v != "" { + out = append(out, v) + } + } + return out } func init() { diff --git a/models/url.go b/models/url.go index 8d020ea..fa50f00 100644 --- a/models/url.go +++ b/models/url.go @@ -32,11 +32,23 @@ type Alternate struct { Href string `json:"href"` } +// Ref points at a record: Type is its content type key and ID the record's +// ID within it (pact.ContentRef as stored in parent_refs). +type Ref struct { + Type string `json:"type"` + ID string `json:"id"` +} + // URL is one row of the slot table golem15_sitemap_urls (D-30): an address // of a content type's record in one locale, or an additional address. Grp // and FileNo place it in the file sitemap--.xml; FileNo is // nil until the address is numbered. ExcludedReason is "" for a listed // address; a row left out keeps its FileNo, so its file keeps a hole. +// Images are the page's image addresses (D-31), ImageFileNo the number of +// its sitemap-images file (nil while it has none). Kind is the content +// type's kind (pact.ContentKindArticle and the others), PublishedAt the +// record's first publication and ParentRefs its parents, for the News +// sitemap and the HTML sitemap. type URL struct { ID uint64 `gorm:"column:id;primaryKey"` Grp string `gorm:"column:grp"` @@ -50,6 +62,10 @@ type URL struct { LastMod *time.Time `gorm:"column:lastmod"` Alternates lagoon.Jsonable[[]Alternate] `gorm:"column:alternates"` Images lagoon.Jsonable[[]string] `gorm:"column:images"` + ImageFileNo *int `gorm:"column:image_file_no"` + Kind string `gorm:"column:kind"` + PublishedAt *time.Time `gorm:"column:published_at"` + ParentRefs lagoon.Jsonable[[]Ref] `gorm:"column:parent_refs"` ExcludedReason string `gorm:"column:excluded_reason"` UpdatedAt time.Time `gorm:"column:updated_at"` } diff --git a/routes.go b/routes.go index e05cbb2..5a8bae0 100644 --- a/routes.go +++ b/routes.go @@ -97,6 +97,20 @@ GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } + if settings.ImagesEnabled { + var images []indexRow + if err := db.Raw(`SELECT image_file_no AS file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls +WHERE excluded_reason = '' AND image_file_no IS NOT NULL AND jsonb_array_length(images) > 0 +GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil { + pluginLog.Error("golem15.sitemap: image index", "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + for i := range images { + images[i].Grp = classes.ImagesGroup + } + rows = append(rows, images...) + } files := make([]classes.IndexFile, 0, len(rows)) for _, row := range rows { files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod}) @@ -111,6 +125,7 @@ GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil { // sitemapFile serves one file: its listed addresses in the order they were // first seen. An unknown file, or one without a listed address, is a 404. +// sitemap-images-.xml is an image sitemap file (imageFile). func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { grp, n, ok := classes.ParseFileName(r.PathValue("file")) if !ok { @@ -130,6 +145,10 @@ func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { return } site := classes.SiteURL(p.app, settings) + if grp == classes.ImagesGroup { + p.imageFile(w, r, db, settings, site, n) + return + } var rows []models.URL if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err) @@ -162,6 +181,48 @@ func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { writeXML(w, buf.Bytes(), true) } +// imageFile serves image sitemap file n: each listed page with images +// numbered into it, in the order first seen, with its images (D-31). It is +// a 404 while the image sitemap is off, and for a file without such a page. +func (p *Plugin) imageFile(w http.ResponseWriter, r *http.Request, db *gorm.DB, settings models.Settings, site string, n int) { + if !settings.ImagesEnabled { + http.NotFound(w, r) + return + } + var rows []models.URL + if err := db.Where("image_file_no = ? AND excluded_reason = ''", n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { + pluginLog.Error("golem15.sitemap: image file", "file", n, "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + entries := make([]classes.ImageEntry, 0, len(rows)) + for _, row := range rows { + loc := classes.Absolute(site, row.URL) + if loc == "" { + continue + } + entry := classes.ImageEntry{Loc: loc} + for _, img := range row.Images.Get() { + if abs := classes.Absolute(site, img); abs != "" { + entry.Images = append(entry.Images, abs) + } + } + if len(entry.Images) > 0 { + entries = append(entries, entry) + } + } + if len(entries) == 0 { + http.NotFound(w, r) + return + } + var buf bytes.Buffer + if err := classes.WriteImageURLSet(&buf, entries); err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + writeXML(w, buf.Bytes(), true) +} + // writeXML answers 200 with an XML body. A file carries X-Robots-Tag // noindex, so search engines read it without indexing it as a page. func writeXML(w http.ResponseWriter, body []byte, noindex bool) { diff --git a/sitemap_extras_test.go b/sitemap_extras_test.go new file mode 100644 index 0000000..e05b6f0 --- /dev/null +++ b/sitemap_extras_test.go @@ -0,0 +1,183 @@ +package sitemap_test + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" + "testing" + "time" + + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/pact" + "git.golem15.com/golem15/summercms/modules/party" +) + +// journalPluginID is a fake blog plugin with an article type (posts at +// /blog/) and a collection type (categories at /kategoria/) +// whose records live in contentStore. It knows nothing about the sitemap. +const journalPluginID = "acme.journal" + +// The fake blog content types. +const ( + postType = "acme-post" + categoryType = "acme-category" +) + +// journalPlugin is acme.journal. +type journalPlugin struct{} + +func (journalPlugin) ID() string { return journalPluginID } +func (journalPlugin) Requires() []string { return nil } +func (journalPlugin) Register(*backpack.App) error { return nil } +func (journalPlugin) Boot(*backpack.App) error { return nil } + +func (journalPlugin) MenuItemTypes() []pact.MenuItemType { + typ := func(key, label, kind, pattern string) pact.MenuItemType { + return pact.MenuItemType{ + Key: key, + Label: label, + DefaultPattern: pattern, + Placeholders: []string{"slug"}, + Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { + return nil, nil + }, + Content: &pact.ContentDescriptor{ + Model: key, + Kind: kind, + Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) { + return contentStore.page(key, in), nil + }, + Describe: func(context.Context, pact.ContentDescribe) (*pact.ContentRecord, error) { + return nil, nil + }, + }, + } + } + return []pact.MenuItemType{ + typ(postType, "Posts", pact.ContentKindArticle, "/blog/:slug"), + typ(categoryType, "Categories", pact.ContentKindCollection, "/kategoria/:slug"), + } +} + +var _ pact.HasMenuItemTypes = journalPlugin{} + +func init() { + party.Register(journalPlugin{}) +} + +// bootJournal boots the fake blog plugin and the sitemap on a fresh +// database with an empty content store; overrides set config keys. +func bootJournal(t *testing.T, overrides ...map[string]any) *sitemapEnv { + t.Helper() + contentStore.reset(t) + return bootPlugins(t, []string{journalPluginID, "golem15.sitemap"}, overrides...) +} + +// post is a published post in one locale at /blog/, first published +// at published, with images. +func post(id int, slug, locale string, published time.Time, images ...string) fakeRecord { + l := pact.ContentLocale{Locale: locale, URL: "/blog/" + slug, Title: "Post " + slug} + for i, img := range images { + if i == 0 { + l.Image = img + } + l.Images = append(l.Images, pact.ContentImage{URL: img}) + } + return fakeRecord{ID: id, Published: true, LastMod: published, PublishedAt: &published, Locales: []pact.ContentLocale{l}} +} + +// exec runs one SQL statement on the test database. +func (e *sitemapEnv) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if err := e.gdb.Exec(sql, args...).Error; err != nil { + t.Fatal(err) + } +} + +// imageLocs lists the image:loc values of an image sitemap body. +func imageLocs(body string) []string { + var out []string + for _, m := range regexp.MustCompile(`([^<]*)`).FindAllStringSubmatch(body, -1) { + out = append(out, m[1]) + } + return out +} + +// TestImageSitemap is the tracer of D-31's image sitemap: a post's +// featured and body images appear, after a rebuild, in sitemap-images-1.xml +// linked from the index: the page's and one image:image/image:loc per +// image (featured first, repeated and unsafe addresses dropped, paths made +// absolute), only those two tags, at most 1000 per page; a page without +// images is not listed; images_enabled off takes the files out of the index +// and answers 404. +func TestImageSitemap(t *testing.T) { + e := bootJournal(t) + at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC) + contentStore.put(postType, post(1, "a", "en", at, "/media/a.jpg", "/media/a.jpg", "https://cdn.example.test/b.png", "javascript:alert(1)", "/media/c&d.webp")) + contentStore.put(postType, post(2, "b", "en", at.Add(time.Hour))) + many := make([]string, 1200) + for i := range many { + many[i] = fmt.Sprintf("/media/many-%d.jpg", i) + } + contentStore.put(postType, post(3, "c", "en", at.Add(2*time.Hour), many...)) + e.rebuild(t) + + index := e.get(t, "/sitemap.xml").Body.String() + if got := locs(index); strings.Join(got, " ") != "https://api.example.test/sitemap-posts-1.xml https://api.example.test/sitemap-images-1.xml" { + t.Fatalf("index locs = %v\n%s", got, index) + } + rec := e.get(t, "/_sitemap/files/sitemap-images-1.xml") + body := rec.Body.String() + if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" || rec.Header().Get("X-Robots-Tag") != "noindex" { + t.Fatalf("image file: %d %v %s", rec.Code, rec.Header(), body) + } + if !strings.Contains(body, ``) { + t.Fatalf("image file head:\n%s", body[:min(len(body), 300)]) + } + if got := locs(body); strings.Join(got, " ") != "https://api.example.test/blog/a https://api.example.test/blog/c" { + t.Fatalf("image file pages = %v", got) + } + first := body[strings.Index(body, "https://api.example.test/blog/a"):] + first = first[:strings.Index(first, "")] + if got := imageLocs(first); strings.Join(got, " ") != "https://api.example.test/media/a.jpg https://cdn.example.test/b.png https://api.example.test/media/c&d.webp" { + t.Fatalf("post a images = %v\n%s", got, first) + } + if n := len(imageLocs(body)); n != 3+1000 { + t.Fatalf("image file lists %d images, want 1003", n) + } + for _, banned := range []string{"image:caption", "image:title", "javascript:", "", "xhtml"} { + if strings.Contains(body, banned) { + t.Errorf("image file contains %q", banned) + } + } + if rec := e.get(t, "/_sitemap/files/sitemap-images-2.xml"); rec.Code != http.StatusNotFound { + t.Errorf("image file 2: %d", rec.Code) + } + + // Stable numbering: with a file size of 1 the second page with images + // is renumbered into its own file once, and a page that loses its images + // leaves a hole instead of moving the later file. + e.setFileSize(t, 1) + e.rebuild(t) + if got := locs(e.get(t, "/_sitemap/files/sitemap-images-2.xml").Body.String()); strings.Join(got, " ") != "https://api.example.test/blog/c" { + t.Fatalf("image file 2 after the size change = %v", got) + } + contentStore.put(postType, post(1, "a", "en", at)) + e.rebuild(t) + if rec := e.get(t, "/_sitemap/files/sitemap-images-1.xml"); rec.Code != http.StatusNotFound { + t.Errorf("image file 1 without images: %d %s", rec.Code, rec.Body.String()) + } + if got := locs(e.get(t, "/_sitemap/files/sitemap-images-2.xml").Body.String()); strings.Join(got, " ") != "https://api.example.test/blog/c" { + t.Fatalf("image file 2 moved = %v", got) + } + + e.exec(t, `UPDATE golem15_sitemap_settings SET images_enabled = FALSE WHERE id = 1`) + if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-images") { + t.Fatalf("index with the image sitemap off:\n%s", index) + } + if rec := e.get(t, "/_sitemap/files/sitemap-images-2.xml"); rec.Code != http.StatusNotFound { + t.Errorf("image file with the image sitemap off: %d", rec.Code) + } +} diff --git a/updates/202610280004_add_golem15_sitemap_media_news_indexnow.go b/updates/202610280004_add_golem15_sitemap_media_news_indexnow.go new file mode 100644 index 0000000..810f106 --- /dev/null +++ b/updates/202610280004_add_golem15_sitemap_media_news_indexnow.go @@ -0,0 +1,87 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// mediaURLColumns are the slot table columns of the image and News +// sitemaps and the HTML sitemap (P33-13): the image file number places the +// rows with images in the sitemap-images files (stable, like file_no), +// published_at and kind feed the News sitemap, parent_refs (the record's +// parents as [{type, id}]) the News sources and the HTML sitemap's +// category tree. +var mediaURLColumns = []string{ + "image_file_no INTEGER NULL", + "published_at TIMESTAMPTZ NULL", + "kind TEXT NOT NULL DEFAULT ''", + "parent_refs JSONB NOT NULL DEFAULT '[]'", +} + +// mediaSettingsColumns are the settings of the image and News sitemaps, +// IndexNow and the HTML sitemap (UI-SPEC C26). html_paths is the HTML +// sitemap's address per locale. The JSONB columns take NULL, like +// extra_urls: a lagoon.Jsonable that is not valid writes NULL. +var mediaSettingsColumns = []string{ + "images_enabled BOOLEAN NOT NULL DEFAULT TRUE", + "news_enabled BOOLEAN NOT NULL DEFAULT FALSE", + "news_publication TEXT NULL", + "news_sources JSONB NULL DEFAULT '[]'", + "indexnow_enabled BOOLEAN NOT NULL DEFAULT FALSE", + "indexnow_key TEXT NULL", + "indexnow_last_at TIMESTAMPTZ NULL", + "indexnow_last_count INTEGER NULL", + "indexnow_last_status INTEGER NULL", + "html_enabled BOOLEAN NOT NULL DEFAULT TRUE", + `html_paths JSONB NULL DEFAULT '{"pl":"/mapa-strony","en":"/sitemap"}'`, + "html_page_size INTEGER NOT NULL DEFAULT 150", +} + +// columnName is the name part of a column definition. +func columnName(def string) string { + for i, r := range def { + if r == ' ' { + return def[:i] + } + } + return def +} + +var addMedia = []*gormigrate.Migration{ + { + ID: "202610280004_add_golem15_sitemap_media_news_indexnow", + Migrate: func(tx *gorm.DB) error { + for _, column := range mediaURLColumns { + if err := tx.Exec(`ALTER TABLE golem15_sitemap_urls ADD COLUMN ` + column).Error; err != nil { + return err + } + } + for _, column := range mediaSettingsColumns { + if err := tx.Exec(`ALTER TABLE golem15_sitemap_settings ADD COLUMN ` + column).Error; err != nil { + return err + } + } + return tx.Exec(`CREATE INDEX golem15_sitemap_urls_image_file_index ON golem15_sitemap_urls (` + columnName(mediaURLColumns[0]) + `)`).Error + }, + Rollback: func(tx *gorm.DB) error { + if err := tx.Exec(`DROP INDEX IF EXISTS golem15_sitemap_urls_image_file_index`).Error; err != nil { + return err + } + for _, column := range mediaURLColumns { + if err := tx.Exec(`ALTER TABLE golem15_sitemap_urls DROP COLUMN IF EXISTS ` + columnName(column)).Error; err != nil { + return err + } + } + for _, column := range mediaSettingsColumns { + if err := tx.Exec(`ALTER TABLE golem15_sitemap_settings DROP COLUMN IF EXISTS ` + columnName(column)).Error; err != nil { + return err + } + } + return nil + }, + }, +} + +func init() { + Register(addMedia...) +}