feat: sitemap index and per-type files from a slot table (tracer)
- contract package: Filter, Listing and their types (D-95)
- golem15_sitemap_urls slot table and the new settings columns
- Rebuild over signpost Records, stable slot assignment, rebuild job
- GET /sitemap.xml index and GET /_sitemap/files/{file} from the table
This commit is contained in:
@@ -3,10 +3,12 @@ package models
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -53,8 +55,44 @@ type Settings struct {
|
||||
PriorityBlog *string `gorm:"column:priority_blog"`
|
||||
PriorityCategories *string `gorm:"column:priority_categories"`
|
||||
PriorityLegal *string `gorm:"column:priority_legal"`
|
||||
|
||||
SiteURL *string `gorm:"column:site_url"`
|
||||
FileSize int `gorm:"column:file_size"`
|
||||
DefaultLocale *string `gorm:"column:default_locale"`
|
||||
ExtraURLs lagoon.Jsonable[[]ExtraURL] `gorm:"column:extra_urls"`
|
||||
RebuildPendingUntil *time.Time `gorm:"column:rebuild_pending_until"`
|
||||
NumberedSize *int `gorm:"column:numbered_size"`
|
||||
BuiltAt *time.Time `gorm:"column:built_at"`
|
||||
RebuildRequestedAt *time.Time `gorm:"column:rebuild_requested_at"`
|
||||
}
|
||||
|
||||
// DefaultFileSize is the number of addresses per sitemap file when the
|
||||
// setting is not stored (D-30); MaxFileSize is the protocol's limit.
|
||||
const (
|
||||
DefaultFileSize = 1000
|
||||
MaxFileSize = 50000
|
||||
)
|
||||
|
||||
// ExtraURL is one additional address of the settings: URL is a path (or an
|
||||
// absolute http(s) URL on the site), LastMod its optional last change as
|
||||
// the admin's date picker sends it.
|
||||
type ExtraURL struct {
|
||||
URL string `json:"url"`
|
||||
LastMod string `json:"lastmod"`
|
||||
}
|
||||
|
||||
// EffectiveFileSize is the stored file size, or DefaultFileSize when it is
|
||||
// outside 1..MaxFileSize.
|
||||
func (s Settings) EffectiveFileSize() int {
|
||||
if s.FileSize < 1 || s.FileSize > MaxFileSize {
|
||||
return DefaultFileSize
|
||||
}
|
||||
return s.FileSize
|
||||
}
|
||||
|
||||
// Extra returns the additional addresses.
|
||||
func (s Settings) Extra() []ExtraURL { return s.ExtraURLs.Get() }
|
||||
|
||||
func (Settings) TableName() string { return "golem15_sitemap_settings" }
|
||||
|
||||
// Fillable lists only the settings columns.
|
||||
@@ -114,7 +152,7 @@ func AllowedFrontendURL(u string) bool {
|
||||
// DefaultSettings is the settings PHP read when nothing was saved: tiers on
|
||||
// with the default priorities.
|
||||
func DefaultSettings() Settings {
|
||||
return Settings{ID: 1, EnablePriorityTiers: true}
|
||||
return Settings{ID: 1, EnablePriorityTiers: true, FileSize: DefaultFileSize}
|
||||
}
|
||||
|
||||
// Priority returns the value of a priority setting: the stored text, or the
|
||||
|
||||
76
models/url.go
Normal file
76
models/url.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
)
|
||||
|
||||
// ExtraTypeKey is the type_key of the settings' additional addresses in
|
||||
// golem15_sitemap_urls; their record_id is the hex SHA-256 of the address.
|
||||
const ExtraTypeKey = "extra"
|
||||
|
||||
// ExtraGroup is the file group of the additional addresses.
|
||||
const ExtraGroup = "pages"
|
||||
|
||||
// The excluded reasons the sitemap sets itself. A Filter's reasons
|
||||
// (contract.ReasonNoindex and the others) are stored as it gives them.
|
||||
const (
|
||||
// ReasonDeleted is an address whose record or locale is gone.
|
||||
ReasonDeleted = "deleted"
|
||||
// ReasonUnpublished is an address whose record is not published.
|
||||
ReasonUnpublished = "unpublished"
|
||||
)
|
||||
|
||||
// Alternate is one hreflang alternate of an address: Hreflang the locale
|
||||
// code or x-default, Href the alternate's path or absolute URL.
|
||||
type Alternate struct {
|
||||
Hreflang string `json:"hreflang"`
|
||||
Href string `json:"href"`
|
||||
}
|
||||
|
||||
// URL is one row of the slot table golem15_sitemap_urls (D-30): an address
|
||||
// of a content type's record in one locale, or an additional address. Grp
|
||||
// and FileNo place it in the file sitemap-<grp>-<file_no>.xml; FileNo is
|
||||
// nil until the address is numbered. ExcludedReason is "" for a listed
|
||||
// address; a row left out keeps its FileNo, so its file keeps a hole.
|
||||
type URL struct {
|
||||
ID uint64 `gorm:"column:id;primaryKey"`
|
||||
Grp string `gorm:"column:grp"`
|
||||
TypeKey string `gorm:"column:type_key"`
|
||||
RecordID string `gorm:"column:record_id"`
|
||||
Locale string `gorm:"column:locale"`
|
||||
URL string `gorm:"column:url"`
|
||||
Title string `gorm:"column:title"`
|
||||
FileNo *int `gorm:"column:file_no"`
|
||||
FirstSeen time.Time `gorm:"column:first_seen"`
|
||||
LastMod *time.Time `gorm:"column:lastmod"`
|
||||
Alternates lagoon.Jsonable[[]Alternate] `gorm:"column:alternates"`
|
||||
Images lagoon.Jsonable[[]string] `gorm:"column:images"`
|
||||
ExcludedReason string `gorm:"column:excluded_reason"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
}
|
||||
|
||||
func (URL) TableName() string { return "golem15_sitemap_urls" }
|
||||
|
||||
func init() {
|
||||
Register(URL{})
|
||||
}
|
||||
|
||||
// AllowedAddress reports whether u may be a stored address: a path that
|
||||
// starts with a single / (no whitespace, control characters, quotes, < or
|
||||
// >), or an absolute http(s) URL that AllowedFrontendURL accepts.
|
||||
func AllowedAddress(u string) bool {
|
||||
if u == "" || !utf8.ValidString(u) {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//") {
|
||||
return !strings.ContainsFunc(u, func(r rune) bool {
|
||||
return unicode.IsControl(r) || unicode.IsSpace(r) || r == '"' || r == '<' || r == '>' || r == '\\'
|
||||
})
|
||||
}
|
||||
return AllowedFrontendURL(u)
|
||||
}
|
||||
Reference in New Issue
Block a user