feat: sitemap index and per-type files from a slot table (tracer)

- contract package: Filter, Listing and their types (D-95)
- golem15_sitemap_urls slot table and the new settings columns
- Rebuild over signpost Records, stable slot assignment, rebuild job
- GET /sitemap.xml index and GET /_sitemap/files/{file} from the table
This commit is contained in:
Jakub Zych
2026-11-04 10:58:05 +01:00
parent 049f3ee270
commit 3359f1a196
17 changed files with 1564 additions and 577 deletions

View File

@@ -3,10 +3,12 @@ package models
import (
"net/url"
"strings"
"time"
"unicode"
"unicode/utf8"
"git.golem15.com/golem15/summercms/modules/cabana"
"git.golem15.com/golem15/summercms/modules/lagoon"
"gorm.io/gorm"
)
@@ -53,8 +55,44 @@ type Settings struct {
PriorityBlog *string `gorm:"column:priority_blog"`
PriorityCategories *string `gorm:"column:priority_categories"`
PriorityLegal *string `gorm:"column:priority_legal"`
SiteURL *string `gorm:"column:site_url"`
FileSize int `gorm:"column:file_size"`
DefaultLocale *string `gorm:"column:default_locale"`
ExtraURLs lagoon.Jsonable[[]ExtraURL] `gorm:"column:extra_urls"`
RebuildPendingUntil *time.Time `gorm:"column:rebuild_pending_until"`
NumberedSize *int `gorm:"column:numbered_size"`
BuiltAt *time.Time `gorm:"column:built_at"`
RebuildRequestedAt *time.Time `gorm:"column:rebuild_requested_at"`
}
// DefaultFileSize is the number of addresses per sitemap file when the
// setting is not stored (D-30); MaxFileSize is the protocol's limit.
const (
DefaultFileSize = 1000
MaxFileSize = 50000
)
// ExtraURL is one additional address of the settings: URL is a path (or an
// absolute http(s) URL on the site), LastMod its optional last change as
// the admin's date picker sends it.
type ExtraURL struct {
URL string `json:"url"`
LastMod string `json:"lastmod"`
}
// EffectiveFileSize is the stored file size, or DefaultFileSize when it is
// outside 1..MaxFileSize.
func (s Settings) EffectiveFileSize() int {
if s.FileSize < 1 || s.FileSize > MaxFileSize {
return DefaultFileSize
}
return s.FileSize
}
// Extra returns the additional addresses.
func (s Settings) Extra() []ExtraURL { return s.ExtraURLs.Get() }
func (Settings) TableName() string { return "golem15_sitemap_settings" }
// Fillable lists only the settings columns.
@@ -114,7 +152,7 @@ func AllowedFrontendURL(u string) bool {
// DefaultSettings is the settings PHP read when nothing was saved: tiers on
// with the default priorities.
func DefaultSettings() Settings {
return Settings{ID: 1, EnablePriorityTiers: true}
return Settings{ID: 1, EnablePriorityTiers: true, FileSize: DefaultFileSize}
}
// Priority returns the value of a priority setting: the stored text, or the

76
models/url.go Normal file
View File

@@ -0,0 +1,76 @@
package models
import (
"strings"
"time"
"unicode"
"unicode/utf8"
"git.golem15.com/golem15/summercms/modules/lagoon"
)
// ExtraTypeKey is the type_key of the settings' additional addresses in
// golem15_sitemap_urls; their record_id is the hex SHA-256 of the address.
const ExtraTypeKey = "extra"
// ExtraGroup is the file group of the additional addresses.
const ExtraGroup = "pages"
// The excluded reasons the sitemap sets itself. A Filter's reasons
// (contract.ReasonNoindex and the others) are stored as it gives them.
const (
// ReasonDeleted is an address whose record or locale is gone.
ReasonDeleted = "deleted"
// ReasonUnpublished is an address whose record is not published.
ReasonUnpublished = "unpublished"
)
// Alternate is one hreflang alternate of an address: Hreflang the locale
// code or x-default, Href the alternate's path or absolute URL.
type Alternate struct {
Hreflang string `json:"hreflang"`
Href string `json:"href"`
}
// URL is one row of the slot table golem15_sitemap_urls (D-30): an address
// of a content type's record in one locale, or an additional address. Grp
// and FileNo place it in the file sitemap-<grp>-<file_no>.xml; FileNo is
// nil until the address is numbered. ExcludedReason is "" for a listed
// address; a row left out keeps its FileNo, so its file keeps a hole.
type URL struct {
ID uint64 `gorm:"column:id;primaryKey"`
Grp string `gorm:"column:grp"`
TypeKey string `gorm:"column:type_key"`
RecordID string `gorm:"column:record_id"`
Locale string `gorm:"column:locale"`
URL string `gorm:"column:url"`
Title string `gorm:"column:title"`
FileNo *int `gorm:"column:file_no"`
FirstSeen time.Time `gorm:"column:first_seen"`
LastMod *time.Time `gorm:"column:lastmod"`
Alternates lagoon.Jsonable[[]Alternate] `gorm:"column:alternates"`
Images lagoon.Jsonable[[]string] `gorm:"column:images"`
ExcludedReason string `gorm:"column:excluded_reason"`
UpdatedAt time.Time `gorm:"column:updated_at"`
}
func (URL) TableName() string { return "golem15_sitemap_urls" }
func init() {
Register(URL{})
}
// AllowedAddress reports whether u may be a stored address: a path that
// starts with a single / (no whitespace, control characters, quotes, < or
// >), or an absolute http(s) URL that AllowedFrontendURL accepts.
func AllowedAddress(u string) bool {
if u == "" || !utf8.ValidString(u) {
return false
}
if strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//") {
return !strings.ContainsFunc(u, func(r rune) bool {
return unicode.IsControl(r) || unicode.IsSpace(r) || r == '"' || r == '<' || r == '>' || r == '\\'
})
}
return AllowedFrontendURL(u)
}