feat: sitemap index and per-type files from a slot table (tracer)
- contract package: Filter, Listing and their types (D-95)
- golem15_sitemap_urls slot table and the new settings columns
- Rebuild over signpost Records, stable slot assignment, rebuild job
- GET /sitemap.xml index and GET /_sitemap/files/{file} from the table
This commit is contained in:
Jakub Zych
406
classes/rebuild.go
Normal file
406
classes/rebuild.go
Normal file
@@ -0,0 +1,406 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// RebuildKind is the River kind of the rebuild job.
|
||||
const RebuildKind = "golem15.sitemap.rebuild"
|
||||
|
||||
// RebuildArgs are the arguments of the rebuild job: it has none, a rebuild
|
||||
// always covers every content type and the additional addresses.
|
||||
type RebuildArgs struct{}
|
||||
|
||||
// Kind is RebuildKind.
|
||||
func (RebuildArgs) Kind() string { return RebuildKind }
|
||||
|
||||
// recordsPage is the page size the rebuild asks the content types for.
|
||||
const recordsPage = 500
|
||||
|
||||
// ErrNoDatabase is returned when the application publishes no database.
|
||||
var ErrNoDatabase = errors.New("golem15.sitemap: database is not configured")
|
||||
|
||||
// RebuildResult is what one rebuild changed, as absolute URLs: Added are
|
||||
// newly listed addresses, Changed listed addresses whose URL changed and
|
||||
// Removed addresses that left the files. Listed counts the listed
|
||||
// addresses and Files the files they fill.
|
||||
type RebuildResult struct {
|
||||
Added []string
|
||||
Changed []string
|
||||
Removed []string
|
||||
Listed int
|
||||
Files int
|
||||
}
|
||||
|
||||
// DefaultSlug is the file name part of a content type's group: the last
|
||||
// segment of its key in plural form (journal-post: posts, journal-category:
|
||||
// categories, seo-page: pages).
|
||||
func DefaultSlug(key string) string {
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
if i := strings.LastIndexAny(key, "-_."); i >= 0 {
|
||||
key = key[i+1:]
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range key {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
s := b.String()
|
||||
switch {
|
||||
case s == "":
|
||||
return "pages"
|
||||
case strings.HasSuffix(s, "y") && len(s) > 1 && !strings.ContainsRune("aeiou", rune(s[len(s)-2])):
|
||||
return s[:len(s)-1] + "ies"
|
||||
case strings.HasSuffix(s, "s"):
|
||||
return s
|
||||
}
|
||||
return s + "s"
|
||||
}
|
||||
|
||||
// ExtraRecordID is the record_id of an additional address: the hex SHA-256
|
||||
// of the address.
|
||||
func ExtraRecordID(u string) string {
|
||||
sum := sha256.Sum256([]byte(u))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// slotKey identifies a row of the slot table.
|
||||
type slotKey struct {
|
||||
typeKey, recordID, locale string
|
||||
}
|
||||
|
||||
// wanted is one address a rebuild found: its group, its row key, its path
|
||||
// and title, its first lastmod (for a new row) and the reason it is left
|
||||
// out ("" when listed). An additional address carries its own lastmod,
|
||||
// which the editor sets, so it is written on every rebuild.
|
||||
type wanted struct {
|
||||
key slotKey
|
||||
grp string
|
||||
url string
|
||||
title string
|
||||
firstMod *time.Time
|
||||
reason string
|
||||
extra bool
|
||||
extraMod *time.Time
|
||||
alternates []models.Alternate
|
||||
}
|
||||
|
||||
// collected is what a rebuild read from the content types and settings.
|
||||
type collected struct {
|
||||
rows []wanted
|
||||
unpublished map[slotKey]bool
|
||||
}
|
||||
|
||||
// Rebuild reads every content type's published records (signpost Records,
|
||||
// D-93) and the additional addresses, writes them into the slot table and
|
||||
// numbers the files, in one transaction that holds the settings row, so
|
||||
// two rebuilds never interleave. A record that is gone or unpublished
|
||||
// leaves its rows in the table with a reason, as a hole; only a changed
|
||||
// file size renumbers the files (D-30).
|
||||
func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
|
||||
var res RebuildResult
|
||||
if app == nil {
|
||||
return res, ErrNoDatabase
|
||||
}
|
||||
db, ok := app.Lookup[*gorm.DB]()
|
||||
if !ok || db == nil {
|
||||
return res, ErrNoDatabase
|
||||
}
|
||||
db = db.WithContext(ctx)
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
reg, _ := app.Lookup[*signpost.Registry]()
|
||||
found, err := collect(ctx, reg, settings)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
site := SiteURL(app, settings)
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
locked, err := LoadSettings(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err = apply(tx, found, locked, site)
|
||||
return err
|
||||
})
|
||||
return res, err
|
||||
}
|
||||
|
||||
// collect walks the content types page by page and adds the additional
|
||||
// addresses.
|
||||
func collect(ctx context.Context, reg *signpost.Registry, settings models.Settings) (collected, error) {
|
||||
out := collected{unpublished: map[slotKey]bool{}}
|
||||
seen := map[slotKey]bool{}
|
||||
add := func(w wanted) {
|
||||
if seen[w.key] {
|
||||
return
|
||||
}
|
||||
seen[w.key] = true
|
||||
out.rows = append(out.rows, w)
|
||||
}
|
||||
unprefixed := models.Text(settings.DefaultLocale)
|
||||
if reg != nil {
|
||||
for _, typ := range reg.ContentTypes() {
|
||||
slug := DefaultSlug(typ.Key)
|
||||
after := ""
|
||||
for {
|
||||
page, err := reg.Records(ctx, typ.Key, after, recordsPage, unprefixed)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err)
|
||||
}
|
||||
for _, rec := range page.Records {
|
||||
if !rec.Published {
|
||||
out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true
|
||||
continue
|
||||
}
|
||||
for _, l := range rec.Locales {
|
||||
if strings.TrimSpace(l.URL) == "" {
|
||||
continue
|
||||
}
|
||||
add(wanted{
|
||||
key: slotKey{typ.Key, rec.ID, l.Locale},
|
||||
grp: slug,
|
||||
url: l.URL,
|
||||
title: l.Title,
|
||||
firstMod: rec.LastMod,
|
||||
})
|
||||
}
|
||||
}
|
||||
if page.Next == "" || page.Next == after {
|
||||
break
|
||||
}
|
||||
after = page.Next
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, e := range settings.Extra() {
|
||||
u := strings.TrimSpace(e.URL)
|
||||
if !models.AllowedAddress(u) {
|
||||
continue
|
||||
}
|
||||
mod := ParseDate(e.LastMod)
|
||||
add(wanted{
|
||||
key: slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""},
|
||||
grp: models.ExtraGroup,
|
||||
url: u,
|
||||
extra: true,
|
||||
extraMod: mod,
|
||||
firstMod: mod,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ParseDate reads a date as the admin's date picker or an import sends it
|
||||
// (RFC 3339, "2006-01-02 15:04:05" or "2006-01-02", UTC); nil when empty or
|
||||
// unreadable.
|
||||
func ParseDate(s string) *time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
t = t.UTC()
|
||||
return &t
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// apply writes the found addresses into the slot table and numbers the
|
||||
// files, inside the rebuild's transaction.
|
||||
func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) {
|
||||
var res RebuildResult
|
||||
var existing []models.URL
|
||||
if err := tx.Order("id ASC").Find(&existing).Error; err != nil {
|
||||
return res, err
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
byKey := make(map[slotKey]*models.URL, len(existing))
|
||||
for i := range existing {
|
||||
r := &existing[i]
|
||||
byKey[slotKey{r.TypeKey, r.RecordID, r.Locale}] = r
|
||||
}
|
||||
dirty := map[uint64]bool{}
|
||||
var created []*models.URL
|
||||
matched := map[slotKey]bool{}
|
||||
for _, w := range found.rows {
|
||||
matched[w.key] = true
|
||||
r, ok := byKey[w.key]
|
||||
if !ok {
|
||||
mod := w.firstMod
|
||||
if mod == nil && !w.extra {
|
||||
mod = &now
|
||||
}
|
||||
row := &models.URL{
|
||||
Grp: w.grp,
|
||||
TypeKey: w.key.typeKey,
|
||||
RecordID: w.key.recordID,
|
||||
Locale: w.key.locale,
|
||||
URL: w.url,
|
||||
Title: w.title,
|
||||
FirstSeen: now,
|
||||
LastMod: utcPtr(mod),
|
||||
Alternates: lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true},
|
||||
Images: lagoon.Jsonable[[]string]{Data: []string{}, Valid: true},
|
||||
ExcludedReason: w.reason,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
created = append(created, row)
|
||||
if w.reason == "" {
|
||||
res.Added = append(res.Added, Absolute(site, w.url))
|
||||
}
|
||||
continue
|
||||
}
|
||||
wasListed := r.ExcludedReason == ""
|
||||
before := *r
|
||||
if r.Grp != w.grp {
|
||||
r.Grp = w.grp
|
||||
r.FileNo = nil
|
||||
}
|
||||
r.URL = w.url
|
||||
r.Title = w.title
|
||||
r.ExcludedReason = w.reason
|
||||
r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}
|
||||
if w.extra {
|
||||
r.LastMod = utcPtr(w.extraMod)
|
||||
}
|
||||
listed := w.reason == ""
|
||||
switch {
|
||||
case listed && !wasListed:
|
||||
res.Added = append(res.Added, Absolute(site, r.URL))
|
||||
case !listed && wasListed:
|
||||
res.Removed = append(res.Removed, Absolute(site, before.URL))
|
||||
case listed && before.URL != r.URL:
|
||||
res.Changed = append(res.Changed, Absolute(site, r.URL))
|
||||
}
|
||||
if rowChanged(before, *r) {
|
||||
dirty[r.ID] = true
|
||||
}
|
||||
}
|
||||
for i := range existing {
|
||||
r := &existing[i]
|
||||
key := slotKey{r.TypeKey, r.RecordID, r.Locale}
|
||||
if matched[key] {
|
||||
continue
|
||||
}
|
||||
reason := models.ReasonDeleted
|
||||
if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] {
|
||||
reason = models.ReasonUnpublished
|
||||
}
|
||||
if r.ExcludedReason == reason {
|
||||
continue
|
||||
}
|
||||
if r.ExcludedReason == "" {
|
||||
res.Removed = append(res.Removed, Absolute(site, r.URL))
|
||||
}
|
||||
r.ExcludedReason = reason
|
||||
dirty[r.ID] = true
|
||||
}
|
||||
if len(created) > 0 {
|
||||
if err := tx.CreateInBatches(created, 500).Error; err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
size := settings.EffectiveFileSize()
|
||||
renumber := settings.NumberedSize == nil || *settings.NumberedSize != size
|
||||
slots := make([]*Slot, 0, len(existing)+len(created))
|
||||
rowsByID := make(map[uint64]*models.URL, len(existing)+len(created))
|
||||
for i := range existing {
|
||||
r := &existing[i]
|
||||
rowsByID[r.ID] = r
|
||||
slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
|
||||
}
|
||||
for _, r := range created {
|
||||
rowsByID[r.ID] = r
|
||||
slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
|
||||
}
|
||||
AssignSlots(slots, size, renumber)
|
||||
files := map[string]bool{}
|
||||
for _, s := range slots {
|
||||
r := rowsByID[s.ID]
|
||||
if !sameInt(r.FileNo, s.FileNo) {
|
||||
r.FileNo = s.FileNo
|
||||
dirty[r.ID] = true
|
||||
}
|
||||
if s.Listed {
|
||||
res.Listed++
|
||||
if s.FileNo != nil {
|
||||
files[FileName(s.Grp, *s.FileNo)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
res.Files = len(files)
|
||||
for id := range dirty {
|
||||
r := rowsByID[id]
|
||||
r.UpdatedAt = now
|
||||
if err := tx.Model(&models.URL{}).Where("id = ?", id).Updates(map[string]any{
|
||||
"grp": r.Grp,
|
||||
"url": r.URL,
|
||||
"title": r.Title,
|
||||
"file_no": r.FileNo,
|
||||
"lastmod": r.LastMod,
|
||||
"alternates": r.Alternates,
|
||||
"excluded_reason": r.ExcludedReason,
|
||||
"updated_at": r.UpdatedAt,
|
||||
}).Error; err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// rowChanged reports whether a rebuild changed a stored row.
|
||||
func rowChanged(a, b models.URL) bool {
|
||||
if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) {
|
||||
return true
|
||||
}
|
||||
av, _ := a.Alternates.Value()
|
||||
bv, _ := b.Alternates.Value()
|
||||
return av != bv
|
||||
}
|
||||
|
||||
func sameInt(a, b *int) bool {
|
||||
return (a == nil && b == nil) || (a != nil && b != nil && *a == *b)
|
||||
}
|
||||
|
||||
func sameTime(a, b *time.Time) bool {
|
||||
return (a == nil && b == nil) || (a != nil && b != nil && a.Equal(*b))
|
||||
}
|
||||
|
||||
func utcPtr(t *time.Time) *time.Time {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
u := t.UTC().Truncate(time.Microsecond)
|
||||
return &u
|
||||
}
|
||||
|
||||
func nonNil[T any](s []T) []T {
|
||||
if s == nil {
|
||||
return []T{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
61
classes/settings.go
Normal file
61
classes/settings.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// SiteURLConfigKey is the config key of the site address the sitemaps use
|
||||
// when the setting is empty.
|
||||
const SiteURLConfigKey = "golem15.sitemap.site_url"
|
||||
|
||||
// LoadSettings reads the settings row; without one the defaults apply.
|
||||
func LoadSettings(db *gorm.DB) (models.Settings, error) {
|
||||
var rows []models.Settings
|
||||
if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
|
||||
return models.Settings{}, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return models.DefaultSettings(), nil
|
||||
}
|
||||
return rows[0], nil
|
||||
}
|
||||
|
||||
// SiteURL is the absolute site address every <loc> starts with, without a
|
||||
// trailing slash: the site_url setting, else the config key
|
||||
// golem15.sitemap.site_url, else app.url (CONTEXT 2026-10-28). A value that
|
||||
// is not an absolute http(s) URL (models.AllowedFrontendURL) is skipped.
|
||||
// The request's Host header is never used; "" means no site address is
|
||||
// configured.
|
||||
func SiteURL(app *backpack.App, s models.Settings) string {
|
||||
candidates := []string{models.Text(s.SiteURL)}
|
||||
if app != nil && app.Config != nil {
|
||||
candidates = append(candidates, app.Config.String(SiteURLConfigKey), app.Config.String("app.url"))
|
||||
}
|
||||
for _, c := range candidates {
|
||||
c = strings.TrimRight(strings.TrimSpace(c), "/")
|
||||
if c != "" && models.AllowedFrontendURL(c) {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Absolute turns a stored address into the absolute URL of a <loc>: a path
|
||||
// starting with a single / is put on site; an absolute http(s) URL is kept.
|
||||
// Anything else, or a path without a site address, is "".
|
||||
func Absolute(site, u string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//"):
|
||||
if site == "" {
|
||||
return ""
|
||||
}
|
||||
return site + u
|
||||
case models.AllowedFrontendURL(u):
|
||||
return u
|
||||
}
|
||||
return ""
|
||||
}
|
||||
95
classes/slots.go
Normal file
95
classes/slots.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Slot is what file numbering needs of one row of the slot table: its
|
||||
// group, its file number (nil while unnumbered), when it was first seen and
|
||||
// whether it is listed. A row left out keeps its number: its file keeps the
|
||||
// hole instead of the later files shifting (D-30, RESEARCH Pitfall 5).
|
||||
type Slot struct {
|
||||
ID uint64
|
||||
Grp string
|
||||
FileNo *int
|
||||
FirstSeen time.Time
|
||||
Listed bool
|
||||
}
|
||||
|
||||
// AssignSlots numbers the rows for files of size addresses and reports
|
||||
// whether any number changed. Without renumber, a numbered row keeps its
|
||||
// number and every listed row without one is appended to the last file of
|
||||
// its group while that file has room (counting the holes of left-out rows),
|
||||
// then to the next file, oldest first (first seen, then id). With renumber
|
||||
// (the file size changed), every group's listed rows are numbered again
|
||||
// from file 1, oldest first, and the left-out rows lose their numbers.
|
||||
func AssignSlots(rows []*Slot, size int, renumber bool) bool {
|
||||
if size < 1 {
|
||||
size = 1
|
||||
}
|
||||
groups := map[string][]*Slot{}
|
||||
var order []string
|
||||
for _, r := range rows {
|
||||
if _, ok := groups[r.Grp]; !ok {
|
||||
order = append(order, r.Grp)
|
||||
}
|
||||
groups[r.Grp] = append(groups[r.Grp], r)
|
||||
}
|
||||
changed := false
|
||||
set := func(r *Slot, n *int) {
|
||||
if (r.FileNo == nil) != (n == nil) || (r.FileNo != nil && *r.FileNo != *n) {
|
||||
changed = true
|
||||
}
|
||||
r.FileNo = n
|
||||
}
|
||||
for _, grp := range order {
|
||||
members := groups[grp]
|
||||
slices.SortStableFunc(members, func(a, b *Slot) int {
|
||||
if c := a.FirstSeen.Compare(b.FirstSeen); c != 0 {
|
||||
return c
|
||||
}
|
||||
switch {
|
||||
case a.ID < b.ID:
|
||||
return -1
|
||||
case a.ID > b.ID:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
if renumber {
|
||||
i := 0
|
||||
for _, r := range members {
|
||||
if !r.Listed {
|
||||
set(r, nil)
|
||||
continue
|
||||
}
|
||||
n := i/size + 1
|
||||
set(r, &n)
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
counts := map[int]int{}
|
||||
last := 0
|
||||
for _, r := range members {
|
||||
if r.FileNo == nil {
|
||||
continue
|
||||
}
|
||||
counts[*r.FileNo]++
|
||||
last = max(last, *r.FileNo)
|
||||
}
|
||||
for _, r := range members {
|
||||
if r.FileNo != nil || !r.Listed {
|
||||
continue
|
||||
}
|
||||
if last == 0 || counts[last] >= size {
|
||||
last++
|
||||
}
|
||||
n := last
|
||||
set(r, &n)
|
||||
counts[last]++
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
@@ -1,16 +1,11 @@
|
||||
package classes_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sitemap "git.golem15.com/golem15/sm-sitemap-plugin"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
)
|
||||
|
||||
// TestTiers pins the priority tiers (D-13): each default rule of the PHP
|
||||
@@ -96,20 +91,4 @@ func TestTiers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A bad host rule fails the plugin's Boot.
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte("sitemap:\n priority_tiers:\n - setting: priority_blog\n pattern: '^/(blog'\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &sitemap.Plugin{}
|
||||
if err := cfg.MergePlugin("golem15.sitemap", p.ConfigFS()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := p.Boot(backpack.New(cfg)); err == nil || !strings.Contains(err.Error(), "priority_tiers[0]") {
|
||||
t.Fatalf("Boot with a bad pattern = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
135
classes/writer.go
Normal file
135
classes/writer.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The XML namespaces of the sitemap files.
|
||||
const (
|
||||
NSSitemap = "http://www.sitemaps.org/schemas/sitemap/0.9"
|
||||
NSXHTML = "http://www.w3.org/1999/xhtml"
|
||||
)
|
||||
|
||||
// IndexFile is one <sitemap> of the index: the file's absolute URL and the
|
||||
// newest lastmod of its addresses (nil when none has one).
|
||||
type IndexFile struct {
|
||||
Loc string
|
||||
LastMod *time.Time
|
||||
}
|
||||
|
||||
// FileAlternate is one xhtml:link alternate of a <url>, with an absolute
|
||||
// href.
|
||||
type FileAlternate struct {
|
||||
Hreflang string
|
||||
Href string
|
||||
}
|
||||
|
||||
// FileEntry is one <url> of a file: the absolute address, its lastmod and
|
||||
// its hreflang alternates.
|
||||
type FileEntry struct {
|
||||
Loc string
|
||||
LastMod *time.Time
|
||||
Alternates []FileAlternate
|
||||
}
|
||||
|
||||
// fileNamePattern is the name of a per-group file.
|
||||
var fileNamePattern = regexp.MustCompile(`^sitemap-([a-z0-9][a-z0-9-]*)-([1-9][0-9]{0,8})\.xml$`)
|
||||
|
||||
// FileName is the name of file n of group grp: sitemap-<grp>-<n>.xml.
|
||||
func FileName(grp string, n int) string {
|
||||
return "sitemap-" + grp + "-" + strconv.Itoa(n) + ".xml"
|
||||
}
|
||||
|
||||
// ParseFileName splits a file name made by FileName; ok is false for any
|
||||
// other name.
|
||||
func ParseFileName(name string) (grp string, n int, ok bool) {
|
||||
m := fileNamePattern.FindStringSubmatch(name)
|
||||
if m == nil {
|
||||
return "", 0, false
|
||||
}
|
||||
n, err := strconv.Atoi(m[2])
|
||||
if err != nil || n < 1 {
|
||||
return "", 0, false
|
||||
}
|
||||
return m[1], n, true
|
||||
}
|
||||
|
||||
// lastmod formats a lastmod in RFC 3339, UTC.
|
||||
func lastmod(t time.Time) string { return t.UTC().Format(time.RFC3339) }
|
||||
|
||||
// escape writes s with encoding/xml's escaping, which covers &, <, >, both
|
||||
// quotes and control whitespace, so it is safe in text and in attribute
|
||||
// values.
|
||||
func escape(w *bufio.Writer, s string) {
|
||||
_ = xml.EscapeText(w, []byte(s))
|
||||
}
|
||||
|
||||
// WriteIndex writes the sitemap index: one <sitemap> per file with its
|
||||
// <loc> and, when known, its <lastmod>.
|
||||
func WriteIndex(w io.Writer, files []IndexFile) error {
|
||||
bw := bufio.NewWriter(w)
|
||||
bw.WriteString(xml.Header)
|
||||
bw.WriteString(`<sitemapindex xmlns="` + NSSitemap + `">`)
|
||||
for _, f := range files {
|
||||
bw.WriteString("\n<sitemap><loc>")
|
||||
escape(bw, f.Loc)
|
||||
bw.WriteString("</loc>")
|
||||
if f.LastMod != nil {
|
||||
bw.WriteString("<lastmod>" + lastmod(*f.LastMod) + "</lastmod>")
|
||||
}
|
||||
bw.WriteString("</sitemap>")
|
||||
}
|
||||
bw.WriteString("\n</sitemapindex>\n")
|
||||
return bw.Flush()
|
||||
}
|
||||
|
||||
// WriteURLSet writes one sitemap file: a <url> per entry with its <loc>, its
|
||||
// <lastmod> when known and its xhtml:link alternates. The xhtml namespace
|
||||
// is declared only when an entry has alternates. No <priority> and no
|
||||
// <changefreq> are written (D-30).
|
||||
func WriteURLSet(w io.Writer, entries []FileEntry) error {
|
||||
bw := bufio.NewWriter(w)
|
||||
bw.WriteString(xml.Header)
|
||||
bw.WriteString(`<urlset xmlns="` + NSSitemap + `"`)
|
||||
for _, e := range entries {
|
||||
if len(e.Alternates) > 0 {
|
||||
bw.WriteString(` xmlns:xhtml="` + NSXHTML + `"`)
|
||||
break
|
||||
}
|
||||
}
|
||||
bw.WriteString(">")
|
||||
for _, e := range entries {
|
||||
bw.WriteString("\n<url><loc>")
|
||||
escape(bw, e.Loc)
|
||||
bw.WriteString("</loc>")
|
||||
if e.LastMod != nil {
|
||||
bw.WriteString("<lastmod>" + lastmod(*e.LastMod) + "</lastmod>")
|
||||
}
|
||||
for _, a := range e.Alternates {
|
||||
bw.WriteString(`<xhtml:link rel="alternate" hreflang="`)
|
||||
escape(bw, a.Hreflang)
|
||||
bw.WriteString(`" href="`)
|
||||
escape(bw, a.Href)
|
||||
bw.WriteString(`"/>`)
|
||||
}
|
||||
bw.WriteString("</url>")
|
||||
}
|
||||
bw.WriteString("\n</urlset>\n")
|
||||
return bw.Flush()
|
||||
}
|
||||
|
||||
// validSlug reports whether s may name a file group.
|
||||
func validSlug(s string) bool {
|
||||
return s != "" && len(s) <= 64 && fileNamePattern.MatchString(FileName(s, 1)) && !strings.HasSuffix(s, "-")
|
||||
}
|
||||
|
||||
// ValidSlug reports whether s may name a file group: lowercase letters,
|
||||
// digits and hyphens, starting with a letter or digit, at most 64
|
||||
// characters.
|
||||
func ValidSlug(s string) bool { return validSlug(s) }
|
||||
Reference in New Issue
Block a user