feat: HTML sitemap groups API
- GET /_sitemap/api/v1/html?locale= answers the overview: the listed
addresses of the XML sitemaps by group, in the order pages, offers,
services, categories (as a tree), posts (newest first), tags, authors,
other content types, each with its title in the locale, total, the
first html_page_size links and more
- GET /_sitemap/api/v1/html/{group}?locale=&page= answers one page of a
group; past the last page, an unknown group or the HTML sitemap off
answer 404; every answer is noindex; per-IP bucket sitemap-html
- the HTML sitemap tab: html_enabled, the address per locale (html_paths)
and html_page_size 50 to 500, with errors.html_path and errors.page_size
- contract.Listing.Settings answers HTMLEnabled, HTMLPaths, HTMLPageSize
and KeyFile
This commit is contained in:
Jakub Zych
456
classes/htmlgroups.go
Normal file
456
classes/htmlgroups.go
Normal file
@@ -0,0 +1,456 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/contract"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"git.golem15.com/golem15/summercms/modules/phrasebook"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"git.golem15.com/golem15/summercms/modules/towel"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// HTMLEntry is one link of the HTML sitemap: the address's title (its
|
||||
// path when it has none) and absolute URL; Children nest a category tree.
|
||||
type HTMLEntry struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Children []HTMLEntry `json:"children,omitempty"`
|
||||
}
|
||||
|
||||
// HTMLGroup is one group of the HTML sitemap overview: Key and Slug name
|
||||
// the file group (the group page is addressed by Slug), Title is its
|
||||
// heading in the requested locale, Total its addresses, Entries the first
|
||||
// page of them and More is true when the group has more than one page.
|
||||
type HTMLGroup struct {
|
||||
Key string `json:"key"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Total int `json:"total"`
|
||||
More bool `json:"more"`
|
||||
Entries []HTMLEntry `json:"entries"`
|
||||
}
|
||||
|
||||
// HTMLOverview is the answer of GET /_sitemap/api/v1/html: the locale and
|
||||
// the non-empty groups in the HTML sitemap order.
|
||||
type HTMLOverview struct {
|
||||
Locale string `json:"locale"`
|
||||
PageSize int `json:"page_size"`
|
||||
Groups []HTMLGroup `json:"groups"`
|
||||
}
|
||||
|
||||
// HTMLGroupPage is the answer of GET /_sitemap/api/v1/html/{group}: one
|
||||
// page (from 1) of a group's links out of Pages.
|
||||
type HTMLGroupPage struct {
|
||||
Group string `json:"group"`
|
||||
Key string `json:"key"`
|
||||
Title string `json:"title"`
|
||||
Locale string `json:"locale"`
|
||||
Page int `json:"page"`
|
||||
Pages int `json:"pages"`
|
||||
Total int `json:"total"`
|
||||
Entries []HTMLEntry `json:"entries"`
|
||||
}
|
||||
|
||||
// htmlGroupOrder is the order of the groups the HTML sitemap knows by
|
||||
// name (UI-SPEC L3): pages, offers and services, categories, posts, tags,
|
||||
// authors. The groups of other content types follow in registration order,
|
||||
// then any other group (a Filter's) alphabetically.
|
||||
var htmlGroupOrder = []string{"pages", "offers", "services", "categories", "posts", "tags", "authors"}
|
||||
|
||||
// maxTreeRows bounds the rows a category tree is built from; a larger
|
||||
// group is listed alphabetically.
|
||||
const maxTreeRows = 10000
|
||||
|
||||
// The orders of a group's links.
|
||||
const (
|
||||
orderTitle = "title" // alphabetical by title
|
||||
orderNewest = "newest" // newest first (articles)
|
||||
orderTree = "tree" // tree order (collections with parents)
|
||||
)
|
||||
|
||||
// htmlRow is what the HTML sitemap reads of a slot table row.
|
||||
type htmlRow struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
Grp string `gorm:"column:grp"`
|
||||
TypeKey string `gorm:"column:type_key"`
|
||||
RecordID string `gorm:"column:record_id"`
|
||||
URL string `gorm:"column:url"`
|
||||
Title string `gorm:"column:title"`
|
||||
PublishedAt *time.Time `gorm:"column:published_at"`
|
||||
ParentRefs string `gorm:"column:parent_refs"`
|
||||
}
|
||||
|
||||
// htmlGroupInfo is one group of the HTML sitemap before its links are
|
||||
// read: its slug, title, total and order.
|
||||
type htmlGroupInfo struct {
|
||||
slug string
|
||||
title string
|
||||
total int
|
||||
order string
|
||||
}
|
||||
|
||||
// HTMLSitemap reads the HTML sitemap (D-34): the listed addresses of the
|
||||
// XML sitemaps (never an excluded one, D-24) of locale and those without
|
||||
// one, by group. locale "" means the default_locale setting (no locale
|
||||
// filter when that is empty too).
|
||||
type HTMLSitemap struct {
|
||||
App *backpack.App
|
||||
DB *gorm.DB
|
||||
}
|
||||
|
||||
// scope is the listed rows of locale.
|
||||
func (h HTMLSitemap) scope(locale string) *gorm.DB {
|
||||
q := h.DB.Table("golem15_sitemap_urls").Where("excluded_reason = ''")
|
||||
if locale != "" {
|
||||
q = q.Where("locale = ? OR locale = ''", locale)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// locale is the locale the HTML sitemap filters by.
|
||||
func (h HTMLSitemap) locale(s models.Settings, locale string) string {
|
||||
if locale = strings.TrimSpace(locale); locale != "" {
|
||||
return locale
|
||||
}
|
||||
return models.Text(s.DefaultLocale)
|
||||
}
|
||||
|
||||
// groups lists the groups with listed rows of locale in the HTML sitemap
|
||||
// order, with their titles in locale and their orders.
|
||||
func (h HTMLSitemap) groups(ctx context.Context, s models.Settings, locale string) ([]htmlGroupInfo, error) {
|
||||
var counts []struct {
|
||||
Grp string `gorm:"column:grp"`
|
||||
N int `gorm:"column:n"`
|
||||
Articles int `gorm:"column:articles"`
|
||||
Nested int `gorm:"column:nested"`
|
||||
}
|
||||
if err := h.scope(locale).Select(`grp, COUNT(*) AS n,
|
||||
COUNT(*) FILTER (WHERE kind = ?) AS articles,
|
||||
COUNT(*) FILTER (WHERE kind = ? AND jsonb_array_length(parent_refs) > 0) AS nested`, pact.ContentKindArticle, pact.ContentKindCollection).
|
||||
Group("grp").Scan(&counts).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
titles, labels, typeOrder, err := groupTitles(ctx, h.App, h.DB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
def := models.Text(s.DefaultLocale)
|
||||
out := make([]htmlGroupInfo, 0, len(counts))
|
||||
for _, c := range counts {
|
||||
if c.N == 0 {
|
||||
continue
|
||||
}
|
||||
g := htmlGroupInfo{slug: c.Grp, total: c.N, order: orderTitle}
|
||||
switch {
|
||||
case c.Articles == c.N:
|
||||
g.order = orderNewest
|
||||
case c.Nested > 0 && c.N <= maxTreeRows:
|
||||
g.order = orderTree
|
||||
}
|
||||
g.title = pickTitle(titles[c.Grp], locale, def)
|
||||
if g.title == "" {
|
||||
if label := labels[c.Grp]; label != "" {
|
||||
g.title = translateIn(ctx, h.App, locale, label)
|
||||
}
|
||||
}
|
||||
if g.title == "" {
|
||||
g.title = c.Grp
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
rank := func(slug string) (int, int) {
|
||||
if i := slices.Index(htmlGroupOrder, slug); i >= 0 {
|
||||
return 0, i
|
||||
}
|
||||
if i, ok := typeOrder[slug]; ok {
|
||||
return 1, i
|
||||
}
|
||||
return 2, 0
|
||||
}
|
||||
slices.SortStableFunc(out, func(a, b htmlGroupInfo) int {
|
||||
ta, ia := rank(a.slug)
|
||||
tb, ib := rank(b.slug)
|
||||
if ta != tb {
|
||||
return ta - tb
|
||||
}
|
||||
if ia != ib {
|
||||
return ia - ib
|
||||
}
|
||||
return strings.Compare(a.slug, b.slug)
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pickTitle is the title of locale, else of the default locale, else "".
|
||||
func pickTitle(titles map[string]string, locale, def string) string {
|
||||
if t := strings.TrimSpace(titles[locale]); t != "" {
|
||||
return t
|
||||
}
|
||||
if t := strings.TrimSpace(titles[def]); t != "" {
|
||||
return t
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// translateIn translates a phrase key (a content type label) in locale;
|
||||
// without a translator, or for a plain label, it is returned as is.
|
||||
func translateIn(ctx context.Context, app *backpack.App, locale, key string) string {
|
||||
if app == nil {
|
||||
return key
|
||||
}
|
||||
tr, ok := app.Lookup[*phrasebook.Translator]()
|
||||
if !ok || tr == nil {
|
||||
return key
|
||||
}
|
||||
if locale != "" {
|
||||
ctx = towel.WithLocale(ctx, locale)
|
||||
}
|
||||
return tr.Get(ctx, key, nil)
|
||||
}
|
||||
|
||||
// groupTitles collects the group headings (by group slug, per locale):
|
||||
// the published Filter's group titles, overridden by the content types'
|
||||
// saved HTML sitemap headings; the content types' labels by slug; and the
|
||||
// content types' registration order by slug.
|
||||
func groupTitles(ctx context.Context, app *backpack.App, db *gorm.DB) (titles map[string]map[string]string, labels map[string]string, order map[string]int, err error) {
|
||||
titles, labels, order = map[string]map[string]string{}, map[string]string{}, map[string]int{}
|
||||
if app == nil {
|
||||
return titles, labels, order, nil
|
||||
}
|
||||
if filter, ok := app.Lookup[contract.Filter](); ok && filter != nil {
|
||||
groups, err := filter.Groups(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
for _, g := range groups {
|
||||
if len(g.Titles) > 0 {
|
||||
titles[g.Key] = g.Titles
|
||||
}
|
||||
}
|
||||
}
|
||||
if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
|
||||
rows, err := LoadTypeSettings(db)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
for i, typ := range reg.ContentTypes() {
|
||||
cfg := EffectiveType(typ, rows)
|
||||
if len(cfg.Titles) > 0 {
|
||||
titles[cfg.Slug] = cfg.Titles
|
||||
}
|
||||
if _, seen := labels[cfg.Slug]; !seen {
|
||||
labels[cfg.Slug] = typ.Label
|
||||
order[cfg.Slug] = i
|
||||
}
|
||||
}
|
||||
}
|
||||
return titles, labels, order, nil
|
||||
}
|
||||
|
||||
// entry is one link with what ordering and the tree need.
|
||||
type entry struct {
|
||||
HTMLEntry
|
||||
node models.Ref
|
||||
parent *models.Ref
|
||||
}
|
||||
|
||||
// rows reads the rows of one group in its order: newest first for
|
||||
// articles, alphabetical by title otherwise (a tree is ordered later).
|
||||
// offset and limit page the read; limit < 0 reads every row.
|
||||
func (h HTMLSitemap) rows(g htmlGroupInfo, locale string, offset, limit int) ([]htmlRow, error) {
|
||||
q := h.scope(locale).Select("id, grp, type_key, record_id, url, title, published_at, parent_refs::text AS parent_refs").Where("grp = ?", g.slug)
|
||||
switch g.order {
|
||||
case orderNewest:
|
||||
q = q.Order("published_at DESC NULLS LAST, first_seen DESC, id DESC")
|
||||
default:
|
||||
q = q.Order("lower(COALESCE(NULLIF(title, ''), url)) ASC, id ASC")
|
||||
}
|
||||
if limit >= 0 {
|
||||
q = q.Offset(offset).Limit(limit)
|
||||
}
|
||||
var out []htmlRow
|
||||
return out, q.Scan(&out).Error
|
||||
}
|
||||
|
||||
// toEntries turns rows into links on site; a row whose address cannot be
|
||||
// made absolute is skipped.
|
||||
func toEntries(rows []htmlRow, site string) []entry {
|
||||
out := make([]entry, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
u := Absolute(site, r.URL)
|
||||
if u == "" {
|
||||
continue
|
||||
}
|
||||
title := strings.TrimSpace(r.Title)
|
||||
if title == "" {
|
||||
title = r.URL
|
||||
}
|
||||
e := entry{HTMLEntry: HTMLEntry{Title: title, URL: u}, node: models.Ref{Type: r.TypeKey, ID: r.RecordID}}
|
||||
var parents []models.Ref
|
||||
if r.ParentRefs != "" {
|
||||
_ = json.Unmarshal([]byte(r.ParentRefs), &parents)
|
||||
}
|
||||
if len(parents) > 0 {
|
||||
e.parent = &parents[0]
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// treeOrder orders entries as a tree (alphabetical siblings, depth first)
|
||||
// and returns them flat with each entry's parent index in the result (-1
|
||||
// for a root). An entry whose first parent is not in the group, or that
|
||||
// would close a cycle, is a root.
|
||||
func treeOrder(entries []entry) ([]entry, []int) {
|
||||
index := make(map[models.Ref]int, len(entries))
|
||||
for i, e := range entries {
|
||||
index[e.node] = i
|
||||
}
|
||||
children := map[int][]int{}
|
||||
var roots []int
|
||||
for i, e := range entries {
|
||||
if e.parent != nil {
|
||||
if p, ok := index[*e.parent]; ok && p != i {
|
||||
children[p] = append(children[p], i)
|
||||
continue
|
||||
}
|
||||
}
|
||||
roots = append(roots, i)
|
||||
}
|
||||
flat := make([]entry, 0, len(entries))
|
||||
parents := make([]int, 0, len(entries))
|
||||
visited := make([]bool, len(entries))
|
||||
var walk func(i, parent int)
|
||||
walk = func(i, parent int) {
|
||||
if visited[i] {
|
||||
return
|
||||
}
|
||||
visited[i] = true
|
||||
at := len(flat)
|
||||
flat = append(flat, entries[i])
|
||||
parents = append(parents, parent)
|
||||
for _, c := range children[i] {
|
||||
walk(c, at)
|
||||
}
|
||||
}
|
||||
for _, r := range roots {
|
||||
walk(r, -1)
|
||||
}
|
||||
// Entries only reachable through a cycle are roots.
|
||||
for i := range entries {
|
||||
walk(i, -1)
|
||||
}
|
||||
return flat, parents
|
||||
}
|
||||
|
||||
// nest builds the links of flat[start:end] (tree order, parents indices
|
||||
// into flat): an entry whose parent is on the same page is its child,
|
||||
// every other is a top-level link.
|
||||
func nest(flat []entry, parents []int, start, end int) []HTMLEntry {
|
||||
nodes := make([]*HTMLEntry, end-start)
|
||||
for i := start; i < end; i++ {
|
||||
e := flat[i].HTMLEntry
|
||||
e.Children = nil
|
||||
nodes[i-start] = &e
|
||||
}
|
||||
// Children are attached deepest first so a parent copies its finished
|
||||
// children.
|
||||
for i := end - 1; i >= start; i-- {
|
||||
p := parents[i]
|
||||
if p >= start && p < end {
|
||||
parent := nodes[p-start]
|
||||
parent.Children = append([]HTMLEntry{*nodes[i-start]}, parent.Children...)
|
||||
nodes[i-start] = nil
|
||||
}
|
||||
}
|
||||
out := []HTMLEntry{}
|
||||
for _, n := range nodes {
|
||||
if n != nil {
|
||||
out = append(out, *n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// page reads page n (from 1) of group g: its links and the number of
|
||||
// pages.
|
||||
func (h HTMLSitemap) page(g htmlGroupInfo, locale, site string, size, n int) ([]HTMLEntry, int, error) {
|
||||
pages := (g.total + size - 1) / size
|
||||
if g.order == orderTree {
|
||||
rows, err := h.rows(g, locale, 0, -1)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
flat, parents := treeOrder(toEntries(rows, site))
|
||||
pages = (len(flat) + size - 1) / size
|
||||
start := (n - 1) * size
|
||||
if start >= len(flat) {
|
||||
return []HTMLEntry{}, pages, nil
|
||||
}
|
||||
return nest(flat, parents, start, min(start+size, len(flat))), pages, nil
|
||||
}
|
||||
rows, err := h.rows(g, locale, (n-1)*size, size)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
entries := toEntries(rows, site)
|
||||
out := make([]HTMLEntry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
out = append(out, e.HTMLEntry)
|
||||
}
|
||||
return out, pages, nil
|
||||
}
|
||||
|
||||
// Overview answers the HTML sitemap overview of locale: every non-empty
|
||||
// group in order with its first html_page_size links and more set when it
|
||||
// has more.
|
||||
func (h HTMLSitemap) Overview(ctx context.Context, s models.Settings, locale string) (HTMLOverview, error) {
|
||||
locale = h.locale(s, locale)
|
||||
size := s.EffectiveHTMLPageSize()
|
||||
out := HTMLOverview{Locale: locale, PageSize: size, Groups: []HTMLGroup{}}
|
||||
groups, err := h.groups(ctx, s, locale)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
site := SiteURL(h.App, s)
|
||||
for _, g := range groups {
|
||||
entries, pages, err := h.page(g, locale, site, size, 1)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
out.Groups = append(out.Groups, HTMLGroup{Key: g.slug, Slug: g.slug, Title: g.title, Total: g.total, More: pages > 1, Entries: entries})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Group answers page n of the group addressed by slug in locale; ok is
|
||||
// false for an unknown or empty group and a page past the last.
|
||||
func (h HTMLSitemap) Group(ctx context.Context, s models.Settings, locale, slug string, n int) (HTMLGroupPage, bool, error) {
|
||||
locale = h.locale(s, locale)
|
||||
groups, err := h.groups(ctx, s, locale)
|
||||
if err != nil || n < 1 {
|
||||
return HTMLGroupPage{}, false, err
|
||||
}
|
||||
i := slices.IndexFunc(groups, func(g htmlGroupInfo) bool { return g.slug == slug })
|
||||
if i < 0 {
|
||||
return HTMLGroupPage{}, false, nil
|
||||
}
|
||||
g := groups[i]
|
||||
entries, pages, err := h.page(g, locale, SiteURL(h.App, s), s.EffectiveHTMLPageSize(), n)
|
||||
if err != nil || n > pages || len(entries) == 0 {
|
||||
return HTMLGroupPage{}, false, err
|
||||
}
|
||||
return HTMLGroupPage{Group: g.slug, Key: g.slug, Title: g.title, Locale: locale, Page: n, Pages: pages, Total: g.total, Entries: entries}, true, nil
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/contract"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -40,9 +39,10 @@ func (l Listing) db(ctx context.Context) (*gorm.DB, error) {
|
||||
}
|
||||
|
||||
// Settings answers the site address, the index address, the default
|
||||
// locale and the groups with listed addresses (by key), with their titles:
|
||||
// a content type's saved HTML sitemap headings, else the titles of the
|
||||
// published Filter's group of that key.
|
||||
// locale and the groups with listed addresses (by key), with their titles
|
||||
// (a content type's saved HTML sitemap headings, else the titles of the
|
||||
// published Filter's group of that key), the HTML sitemap settings and the
|
||||
// IndexNow key file name (while IndexNow is on with a key).
|
||||
func (l Listing) Settings(ctx context.Context) (contract.ListingSettings, error) {
|
||||
var out contract.ListingSettings
|
||||
db, err := l.db(ctx)
|
||||
@@ -65,34 +65,18 @@ func (l Listing) Settings(ctx context.Context) (contract.ListingSettings, error)
|
||||
if err := db.Raw(`SELECT grp, COUNT(*) AS n FROM golem15_sitemap_urls WHERE excluded_reason = '' GROUP BY grp ORDER BY grp`).Scan(&counts).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
titles := map[string]map[string]string{}
|
||||
if filter, ok := l.App.Lookup[contract.Filter](); ok && filter != nil {
|
||||
groups, err := filter.Groups(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, g := range groups {
|
||||
if len(g.Titles) > 0 {
|
||||
titles[g.Key] = g.Titles
|
||||
}
|
||||
}
|
||||
}
|
||||
if reg, ok := l.App.Lookup[*signpost.Registry](); ok && reg != nil {
|
||||
rows, err := LoadTypeSettings(db)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, typ := range reg.ContentTypes() {
|
||||
cfg := EffectiveType(typ, rows)
|
||||
if len(cfg.Titles) > 0 {
|
||||
titles[cfg.Slug] = cfg.Titles
|
||||
}
|
||||
}
|
||||
titles, _, _, err := groupTitles(ctx, l.App, db)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Groups = make([]contract.GroupInfo, 0, len(counts))
|
||||
for _, c := range counts {
|
||||
out.Groups = append(out.Groups, contract.GroupInfo{Key: c.Grp, Slug: c.Grp, Titles: titles[c.Grp], Count: c.N})
|
||||
}
|
||||
out.HTMLEnabled = settings.HTMLEnabled
|
||||
out.HTMLPaths = settings.EffectiveHTMLPaths()
|
||||
out.HTMLPageSize = settings.EffectiveHTMLPageSize()
|
||||
out.KeyFile = KeyFileName(ActiveKey(settings))
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
@@ -59,3 +60,39 @@ func Absolute(site, u string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MergeHTMLPaths merges submitted HTML sitemap addresses (locale to path)
|
||||
// into the stored ones: a sent locale replaces its address, a blank one
|
||||
// removes it, the others keep theirs. problem is true when a sent address
|
||||
// is not a models.ValidHTMLPath; nothing is merged then.
|
||||
func MergeHTMLPaths(db *gorm.DB, sent map[string]string) (map[string]string, bool) {
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return nil, true
|
||||
}
|
||||
merged := settings.EffectiveHTMLPaths()
|
||||
for locale, path := range sent {
|
||||
locale, path = strings.TrimSpace(locale), strings.TrimSpace(path)
|
||||
if locale == "" {
|
||||
continue
|
||||
}
|
||||
if path == "" {
|
||||
delete(merged, locale)
|
||||
continue
|
||||
}
|
||||
if !models.ValidHTMLPath(path) {
|
||||
return nil, true
|
||||
}
|
||||
merged[locale] = path
|
||||
}
|
||||
return merged, false
|
||||
}
|
||||
|
||||
// StoreHTMLPaths writes the HTML sitemap addresses to settings row 1.
|
||||
func StoreHTMLPaths(db *gorm.DB, paths map[string]string) error {
|
||||
raw, err := json.Marshal(paths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Exec(`UPDATE golem15_sitemap_settings SET html_paths = ?::jsonb WHERE id = 1`, string(raw)).Error
|
||||
}
|
||||
|
||||
@@ -92,8 +92,10 @@ type Listing interface {
|
||||
// none is configured), IndexURL the absolute address of the sitemap index,
|
||||
// DefaultLocale the site's default locale ("" when not set) and Groups the
|
||||
// file groups with listed addresses. HTMLEnabled, HTMLPaths (locale to
|
||||
// path), HTMLPageSize and KeyFile (the IndexNow key file name) describe the
|
||||
// HTML sitemap and IndexNow; they stay zero until those settings exist.
|
||||
// path) and HTMLPageSize are the HTML sitemap's settings (the sitemap
|
||||
// serves its data at /_sitemap/api/v1/html); KeyFile is the IndexNow key
|
||||
// file name (<key>.txt, served at /_sitemap/indexnow/<key>.txt for the
|
||||
// site root), "" while IndexNow is off.
|
||||
type ListingSettings struct {
|
||||
SiteURL string
|
||||
IndexURL string
|
||||
|
||||
116
controllers/api/html.go
Normal file
116
controllers/api/html.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/wire"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// GroupPattern is the {group} constraint of the HTML sitemap group route:
|
||||
// a file group slug.
|
||||
const GroupPattern = `[a-z0-9][a-z0-9-]{0,63}`
|
||||
|
||||
// localePattern is the form a locale query value must have.
|
||||
var localePattern = regexp.MustCompile(`^[a-z]{2,3}([-_][A-Za-z0-9]{2,8})*$`)
|
||||
|
||||
// pagePattern is the form of the page query value: 1 to 6 digits.
|
||||
var pagePattern = regexp.MustCompile(`^[0-9]{1,6}$`)
|
||||
|
||||
// htmlRequest checks the shared parts of an HTML sitemap request and
|
||||
// returns the settings and database, or answers 404 (the HTML sitemap is
|
||||
// off, a malformed locale) or 500 itself.
|
||||
func htmlRequest(app func() *backpack.App, w http.ResponseWriter, r *http.Request) (classes.HTMLSitemap, bool) {
|
||||
w.Header().Set("X-Robots-Tag", "noindex")
|
||||
w.Header().Set("Cache-Control", "no-cache, private")
|
||||
a := app()
|
||||
if a == nil {
|
||||
http.NotFound(w, r)
|
||||
return classes.HTMLSitemap{}, false
|
||||
}
|
||||
if loc := r.URL.Query().Get("locale"); loc != "" && !localePattern.MatchString(loc) {
|
||||
http.NotFound(w, r)
|
||||
return classes.HTMLSitemap{}, false
|
||||
}
|
||||
db, ok := a.Lookup[*gorm.DB]()
|
||||
if !ok || db == nil {
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return classes.HTMLSitemap{}, false
|
||||
}
|
||||
return classes.HTMLSitemap{App: a, DB: db.WithContext(r.Context())}, true
|
||||
}
|
||||
|
||||
// HTMLOverview serves GET /_sitemap/api/v1/html?locale=: the HTML sitemap
|
||||
// overview (D-34; classes.HTMLSitemap.Overview). It lists exactly the
|
||||
// addresses of the XML sitemaps and answers 404 while the HTML sitemap is
|
||||
// off. Every answer carries X-Robots-Tag: noindex.
|
||||
func HTMLOverview(app func() *backpack.App) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
h, ok := htmlRequest(app, w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
settings, err := classes.LoadSettings(h.DB)
|
||||
if err != nil {
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !settings.HTMLEnabled {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
out, err := h.Overview(r.Context(), settings, r.URL.Query().Get("locale"))
|
||||
if err != nil {
|
||||
log.Error("golem15.sitemap: html overview", "error", err)
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
wire.WriteJSON(w, http.StatusOK, out)
|
||||
}
|
||||
}
|
||||
|
||||
// HTMLGroup serves GET /_sitemap/api/v1/html/{group}?locale=&page=: one
|
||||
// page (from 1, the default) of a group (classes.HTMLSitemap.Group). An
|
||||
// unknown group, a page past the last, a malformed page or the HTML
|
||||
// sitemap off answer 404.
|
||||
func HTMLGroup(app func() *backpack.App) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
h, ok := htmlRequest(app, w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page := 1
|
||||
if raw := r.URL.Query().Get("page"); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if !pagePattern.MatchString(raw) || err != nil || n < 1 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
page = n
|
||||
}
|
||||
settings, err := classes.LoadSettings(h.DB)
|
||||
if err != nil {
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !settings.HTMLEnabled {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
out, found, err := h.Group(r.Context(), settings, r.URL.Query().Get("locale"), r.PathValue("group"), page)
|
||||
if err != nil {
|
||||
log.Error("golem15.sitemap: html group", "error", err)
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
wire.WriteJSON(w, http.StatusOK, out)
|
||||
}
|
||||
}
|
||||
7
controllers/api/logger.go
Normal file
7
controllers/api/logger.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package api
|
||||
|
||||
import "git.golem15.com/golem15/summercms/modules/diary"
|
||||
|
||||
// log is the handlers' logger: its records carry component=sitemap, like
|
||||
// the rest of the plugin (log.components.sitemap).
|
||||
var log = diary.For("sitemap")
|
||||
@@ -20,12 +20,19 @@ import (
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// The settings form's tab keys: the Content types tab follows Files.
|
||||
// The settings form's tab keys: the Content types tab follows Files; the
|
||||
// HTML sitemap tab's address field is added after html_enabled.
|
||||
const (
|
||||
TabFiles = "golem15.sitemap::lang.settings.tab_files"
|
||||
TabTypes = "golem15.sitemap::lang.settings.tab_types"
|
||||
TabHTML = "golem15.sitemap::lang.settings.tab_html"
|
||||
)
|
||||
|
||||
// htmlPathField is the HTML sitemap address field: translatable, stored
|
||||
// as a locale map in html_paths by the plugin's form extension (the
|
||||
// settings row has no translation writer of its own).
|
||||
const htmlPathField = "html_path"
|
||||
|
||||
// The phrase keys of the Content types refusals (UI-SPEC C26).
|
||||
const (
|
||||
errSlug = "golem15.sitemap::lang.errors.slug"
|
||||
@@ -40,17 +47,13 @@ func typeFieldKey(key string) string {
|
||||
return typeFieldUnsafe.ReplaceAllString(strings.ToLower(key), "_")
|
||||
}
|
||||
|
||||
// AdminActionExtensions adds the Content types tab to the plugin's own
|
||||
// settings singleton: one block per signpost content type, in registration
|
||||
// order. Nothing is added when no content type is registered.
|
||||
// AdminActionExtensions extends the plugin's own settings singleton: the
|
||||
// Content types tab (one block per signpost content type, in registration
|
||||
// order; none without content types) and the HTML sitemap address.
|
||||
func (p *Plugin) AdminActionExtensions() []pact.AdminActionExtension {
|
||||
reg := p.registry()
|
||||
if reg == nil {
|
||||
return nil
|
||||
}
|
||||
types := reg.ContentTypes()
|
||||
if len(types) == 0 {
|
||||
return nil
|
||||
var types []pact.MenuItemType
|
||||
if reg := p.registry(); reg != nil {
|
||||
types = reg.ContentTypes()
|
||||
}
|
||||
form := p.typeSettingsTab(types)
|
||||
if form == nil {
|
||||
@@ -67,9 +70,10 @@ func (p *Plugin) registry() *signpost.Registry {
|
||||
return reg
|
||||
}
|
||||
|
||||
// typeSettingsDocument is the fields document of the Content types tab
|
||||
// (UI-SPEC C26): per type, the include switch (labelled with the type's
|
||||
// label), the file name and the HTML sitemap heading.
|
||||
// typeSettingsDocument is the fields document of the extension (UI-SPEC
|
||||
// C26): per type, the include switch (labelled with the type's label), the
|
||||
// file name and the HTML sitemap heading on the Content types tab, then
|
||||
// the HTML sitemap address after html_enabled.
|
||||
func typeSettingsDocument(types []pact.MenuItemType) ([]byte, error) {
|
||||
fields := yaml.MapSlice{}
|
||||
for _, typ := range types {
|
||||
@@ -101,15 +105,24 @@ func typeSettingsDocument(types []pact.MenuItemType) ([]byte, error) {
|
||||
}},
|
||||
)
|
||||
}
|
||||
return yaml.Marshal(yaml.MapSlice{
|
||||
{Key: "tabs", Value: []yaml.MapSlice{{{Key: "name", Value: TabTypes}, {Key: "after", Value: TabFiles}}}},
|
||||
{Key: "fields", Value: fields},
|
||||
})
|
||||
fields = append(fields, yaml.MapItem{Key: htmlPathField, Value: yaml.MapSlice{
|
||||
{Key: "label", Value: "golem15.sitemap::lang.settings.html_path"},
|
||||
{Key: "comment", Value: "golem15.sitemap::lang.settings.html_path_comment"},
|
||||
{Key: "type", Value: "mltext"},
|
||||
{Key: "span", Value: "left"},
|
||||
{Key: "tab", Value: TabHTML},
|
||||
{Key: "after", Value: "html_enabled"},
|
||||
}})
|
||||
doc := yaml.MapSlice{}
|
||||
if len(types) > 0 {
|
||||
doc = append(doc, yaml.MapItem{Key: "tabs", Value: []yaml.MapSlice{{{Key: "name", Value: TabTypes}, {Key: "after", Value: TabFiles}}}})
|
||||
}
|
||||
return yaml.Marshal(append(doc, yaml.MapItem{Key: "fields", Value: fields}))
|
||||
}
|
||||
|
||||
// typeSettingsTab is the plugin's form extension on its own settings
|
||||
// controller: the Content types tab, loaded from and saved to
|
||||
// golem15_sitemap_types.
|
||||
// golem15_sitemap_types, and the HTML sitemap address (html_paths).
|
||||
func (p *Plugin) typeSettingsTab(types []pact.MenuItemType) *pact.AdminFormExtension {
|
||||
doc, err := typeSettingsDocument(types)
|
||||
if err != nil {
|
||||
@@ -138,10 +151,18 @@ func (p *Plugin) typeSettingsTab(types []pact.MenuItemType) *pact.AdminFormExten
|
||||
}
|
||||
out[base+"title"] = titles
|
||||
}
|
||||
settings, err := classes.LoadSettings(in.DB.WithContext(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[htmlPathField] = settings.EffectiveHTMLPaths()
|
||||
return out, nil
|
||||
},
|
||||
Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) {
|
||||
return pact.AdminFormSaveResult{}, saveTypeSettings(ctx, in, types)
|
||||
if err := saveTypeSettings(ctx, in, types); err != nil {
|
||||
return pact.AdminFormSaveResult{}, err
|
||||
}
|
||||
return pact.AdminFormSaveResult{}, saveHTMLPaths(in)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -212,6 +233,32 @@ func saveTypeSettings(ctx context.Context, in pact.AdminFormSave, types []pact.M
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveHTMLPaths stores a submitted HTML sitemap address: the locales sent
|
||||
// replace theirs in html_paths (a blank one removes it), the others keep
|
||||
// theirs. An address that models.ValidHTMLPath refuses is refused with
|
||||
// errors.html_path.
|
||||
func saveHTMLPaths(in pact.AdminFormSave) error {
|
||||
raw, sent := in.Values[htmlPathField]
|
||||
if !sent {
|
||||
return nil
|
||||
}
|
||||
paths := map[string]string{}
|
||||
switch v := raw.(type) {
|
||||
case map[string]string:
|
||||
paths = v
|
||||
case map[string]any:
|
||||
for k, s := range v {
|
||||
text, _ := s.(string)
|
||||
paths[k] = text
|
||||
}
|
||||
}
|
||||
merged, problem := classes.MergeHTMLPaths(in.DB, paths)
|
||||
if problem {
|
||||
return pact.AdminFormErrors{htmlPathField: {models.HTMLPathProblem}}
|
||||
}
|
||||
return classes.StoreHTMLPaths(in.DB, merged)
|
||||
}
|
||||
|
||||
// titlesOf reads a submitted translatable value (a map of locale to text);
|
||||
// blank texts are dropped.
|
||||
func titlesOf(v any) map[string]string {
|
||||
|
||||
@@ -92,7 +92,7 @@ func (Settings) TableName() string { return "golem15_sitemap_settings" }
|
||||
|
||||
// Fillable lists the columns the settings form edits.
|
||||
func (Settings) Fillable() []string {
|
||||
return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources", "indexnow_enabled"}
|
||||
return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources", "indexnow_enabled", "html_enabled", "html_page_size"}
|
||||
}
|
||||
|
||||
// Rules is empty: the settings controller runs Validate before a save.
|
||||
@@ -105,6 +105,8 @@ const (
|
||||
errLastmod = "golem15.sitemap::lang.errors.lastmod"
|
||||
errNewsName = "golem15.sitemap::lang.errors.news_publication"
|
||||
errNewsSrc = "golem15.sitemap::lang.errors.news_source"
|
||||
errHTMLPath = "golem15.sitemap::lang.errors.html_path"
|
||||
errPageSize = "golem15.sitemap::lang.errors.page_size"
|
||||
)
|
||||
|
||||
// MaxNewsPublication is the longest publication name.
|
||||
@@ -128,7 +130,9 @@ func (s Settings) Validate() error {
|
||||
// Problems lists the settings' refusals by field (UI-SPEC C26): a site
|
||||
// address that AllowedFrontendURL refuses, a file size outside 1..MaxFileSize and an additional
|
||||
// address that is not a path or an http(s) URL (extra_urls.<i>.url) or
|
||||
// whose lastmod is not a date (extra_urls.<i>.lastmod).
|
||||
// whose lastmod is not a date (extra_urls.<i>.lastmod), a publication name
|
||||
// over 255 characters, a malformed News source, an HTML sitemap address
|
||||
// that ValidHTMLPath refuses (html_path) and a page size outside 50..500.
|
||||
func (s Settings) Problems() map[string][]string {
|
||||
out := map[string][]string{}
|
||||
if !AllowedFrontendURL(Text(s.SiteURL)) {
|
||||
@@ -154,9 +158,42 @@ func (s Settings) Problems() map[string][]string {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, p := range s.HTMLPaths.Get() {
|
||||
if p = strings.TrimSpace(p); p != "" && !ValidHTMLPath(p) {
|
||||
out["html_path"] = []string{errHTMLPath}
|
||||
break
|
||||
}
|
||||
}
|
||||
if s.HTMLPageSize < MinHTMLPageSize || s.HTMLPageSize > MaxHTMLPageSize {
|
||||
out["html_page_size"] = []string{errPageSize}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HTMLPathProblem is the refusal of an HTML sitemap address (errors.html_path).
|
||||
const HTMLPathProblem = errHTMLPath
|
||||
|
||||
// ValidHTMLPath reports whether p may be the HTML sitemap's address: a
|
||||
// path that starts with a single /, at most 255 characters of letters,
|
||||
// digits and - _ . ~ / (no query, fragment, "//" or ".." segment), and not
|
||||
// the site root.
|
||||
func ValidHTMLPath(p string) bool {
|
||||
if len(p) < 2 || len(p) > 255 || p[0] != '/' || strings.Contains(p, "//") {
|
||||
return false
|
||||
}
|
||||
for _, seg := range strings.Split(p[1:], "/") {
|
||||
if seg == "." || seg == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, r := range p {
|
||||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("-_.~/", r)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// The prefixes of the news_sources values (P33-15): type:<type key> and
|
||||
// category:<type key>:<id>.
|
||||
const (
|
||||
|
||||
@@ -127,3 +127,21 @@ fields:
|
||||
action: hide
|
||||
field: indexnow_enabled
|
||||
condition: unchecked
|
||||
|
||||
# The address per locale (html_path, an mltext field) is added after
|
||||
# this switch by the plugin's form extension, stored in html_paths.
|
||||
html_enabled:
|
||||
label: golem15.sitemap::lang.settings.html_enabled
|
||||
comment: golem15.sitemap::lang.settings.html_enabled_comment
|
||||
type: switch
|
||||
default: true
|
||||
span: full
|
||||
tab: golem15.sitemap::lang.settings.tab_html
|
||||
|
||||
html_page_size:
|
||||
label: golem15.sitemap::lang.settings.html_page_size
|
||||
comment: golem15.sitemap::lang.settings.html_page_size_comment
|
||||
type: number
|
||||
default: 150
|
||||
span: right
|
||||
tab: golem15.sitemap::lang.settings.tab_html
|
||||
|
||||
@@ -75,6 +75,9 @@ func TestSettingsValidate(t *testing.T) {
|
||||
if s.FileSize == 0 {
|
||||
s.FileSize = 1000
|
||||
}
|
||||
if s.HTMLPageSize == 0 {
|
||||
s.HTMLPageSize = models.DefaultHTMLPageSize
|
||||
}
|
||||
return s
|
||||
}
|
||||
extra := func(urls ...string) lagoon.Jsonable[[]models.ExtraURL] {
|
||||
@@ -88,8 +91,8 @@ func TestSettingsValidate(t *testing.T) {
|
||||
"null site_url": ok(models.Settings{}),
|
||||
"empty site_url": ok(models.Settings{SiteURL: text("")}),
|
||||
"site_url": ok(models.Settings{SiteURL: text("https://www.ex.pl/")}),
|
||||
"file size 1": {FileSize: 1},
|
||||
"file size 50 000": {FileSize: 50000},
|
||||
"file size 1": ok(models.Settings{FileSize: 1}),
|
||||
"file size 50 000": ok(models.Settings{FileSize: 50000}),
|
||||
"extra addresses": ok(models.Settings{ExtraURLs: extra("/", "/o-nas?x=1&y=2", "https://www.ex.pl/kontakt")}),
|
||||
} {
|
||||
if err := s.Validate(); err != nil {
|
||||
@@ -103,8 +106,8 @@ func TestSettingsValidate(t *testing.T) {
|
||||
"javascript site_url": {ok(models.Settings{SiteURL: text("javascript:alert(document.domain)//")}), map[string]any{"site_url": []string{"golem15.sitemap::lang.errors.frontend_url"}}},
|
||||
"attribute site_url": {ok(models.Settings{SiteURL: text(`https://x"/><y a="`)}), map[string]any{"site_url": []string{"golem15.sitemap::lang.errors.frontend_url"}}},
|
||||
"path site_url": {ok(models.Settings{SiteURL: text("/x")}), map[string]any{"site_url": []string{"golem15.sitemap::lang.errors.frontend_url"}}},
|
||||
"file size 0": {models.Settings{}, map[string]any{"file_size": []string{"golem15.sitemap::lang.errors.file_size"}}},
|
||||
"file size 50 001": {models.Settings{FileSize: 50001}, map[string]any{"file_size": []string{"golem15.sitemap::lang.errors.file_size"}}},
|
||||
"file size 0": {models.Settings{HTMLPageSize: 150}, map[string]any{"file_size": []string{"golem15.sitemap::lang.errors.file_size"}}},
|
||||
"file size 50 001": {models.Settings{FileSize: 50001, HTMLPageSize: 150}, map[string]any{"file_size": []string{"golem15.sitemap::lang.errors.file_size"}}},
|
||||
"javascript address": {ok(models.Settings{ExtraURLs: extra("/", "javascript:alert(1)")}), map[string]any{"extra_urls.1.url": []string{"golem15.sitemap::lang.errors.url_scheme"}}},
|
||||
"protocol-relative": {ok(models.Settings{ExtraURLs: extra("//evil.test/x")}), map[string]any{"extra_urls.0.url": []string{"golem15.sitemap::lang.errors.url_scheme"}}},
|
||||
"address with a space": {ok(models.Settings{ExtraURLs: extra("/a b")}), map[string]any{"extra_urls.0.url": []string{"golem15.sitemap::lang.errors.url_scheme"}}},
|
||||
|
||||
41
routes.go
41
routes.go
@@ -21,8 +21,12 @@ const (
|
||||
cacheControl = "no-cache, private"
|
||||
)
|
||||
|
||||
// BucketSitemap is the throttle bucket of the sitemap routes.
|
||||
const BucketSitemap = "sitemap-xml"
|
||||
// The throttle buckets: BucketSitemap of the sitemap files and the key
|
||||
// file, BucketHTML of the HTML sitemap API.
|
||||
const (
|
||||
BucketSitemap = "sitemap-xml"
|
||||
BucketHTML = "sitemap-html"
|
||||
)
|
||||
|
||||
// FilesPath is the route of the per-group files. The index lists them at
|
||||
// the site root (sitemap-<slug>-<n>.xml); the site's SEO layer or proxy
|
||||
@@ -33,36 +37,49 @@ const FilesPath = "/_sitemap/files/"
|
||||
// (one file) and GET /_sitemap/indexnow/{file} (the IndexNow key file)
|
||||
// without the backend middleware: search engines fetch them anonymously.
|
||||
// The sitemaps are served from the slot table, never generated per
|
||||
// request, and every route is throttled per client IP (BucketSitemap).
|
||||
// request, and every route is throttled per client IP (BucketSitemap). GET
|
||||
// /_sitemap/api/v1/html and /_sitemap/api/v1/html/{group} answer the HTML
|
||||
// sitemap's data (BucketHTML).
|
||||
func (p *Plugin) Routes(r pact.Router) error {
|
||||
r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap)
|
||||
r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap)
|
||||
r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`)
|
||||
r.Get("/_sitemap/indexnow/{file}", apicontrollers.KeyFile(p.booted), "throttle:"+BucketSitemap)
|
||||
r.Where("file", apicontrollers.KeyFilePattern)
|
||||
// The HTML sitemap's data (D-34), read by the site's server when it
|
||||
// renders the HTML sitemap page; public, noindex, per-IP bucket
|
||||
// sitemap-html.
|
||||
r.Get("/_sitemap/api/v1/html", apicontrollers.HTMLOverview(p.booted), "throttle:"+BucketHTML)
|
||||
r.Get("/_sitemap/api/v1/html/{group}", apicontrollers.HTMLGroup(p.booted), "throttle:"+BucketHTML)
|
||||
r.Where("group", apicontrollers.GroupPattern)
|
||||
return nil
|
||||
}
|
||||
|
||||
// booted returns the booted application.
|
||||
func (p *Plugin) booted() *backpack.App { return p.app }
|
||||
|
||||
// Buckets registers sitemap-xml: 60 requests a minute per client IP (behind
|
||||
// the configured trusted proxies). 429 bodies are surf's
|
||||
// {"message":"Too Many Attempts."}.
|
||||
// Buckets registers sitemap-xml (60 requests a minute) and sitemap-html
|
||||
// (300 a minute: the site's server renders every HTML sitemap page from
|
||||
// it) per client IP, behind the configured trusted proxies. 429 bodies are
|
||||
// surf's {"message":"Too Many Attempts."}.
|
||||
func (p *Plugin) Buckets() map[string]surf.Bucket {
|
||||
trusted := surf.TrustedProxies(nil)
|
||||
if p.app != nil {
|
||||
trusted = surf.TrustedProxies(p.app.Config)
|
||||
}
|
||||
return map[string]surf.Bucket{
|
||||
BucketSitemap: {
|
||||
Name: BucketSitemap,
|
||||
Max: 60,
|
||||
bucket := func(name string, max int) surf.Bucket {
|
||||
return surf.Bucket{
|
||||
Name: name,
|
||||
Max: max,
|
||||
Decay: time.Minute,
|
||||
Key: func(r *http.Request) string {
|
||||
return "sitemap-xml|" + surf.ClientIP(r, trusted)
|
||||
return name + "|" + surf.ClientIP(r, trusted)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return map[string]surf.Bucket{
|
||||
BucketSitemap: bucket(BucketSitemap, 60),
|
||||
BucketHTML: bucket(BucketHTML, 300),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/contract"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
||||
@@ -536,3 +537,192 @@ func TestIndexNowStaging(t *testing.T) {
|
||||
t.Fatalf("key file outside production: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// htmlJSON fetches an HTML sitemap API path and decodes its answer.
|
||||
func (e *sitemapEnv) htmlJSON(t *testing.T, path string, status int) map[string]any {
|
||||
t.Helper()
|
||||
rec := e.get(t, path)
|
||||
if rec.Code != status {
|
||||
t.Fatalf("GET %s: %d, want %d: %s", path, rec.Code, status, rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("X-Robots-Tag") != "noindex" {
|
||||
t.Errorf("GET %s lacks X-Robots-Tag noindex: %v", path, rec.Header())
|
||||
}
|
||||
out := map[string]any{}
|
||||
if status == http.StatusOK {
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode %s: %v", rec.Body.String(), err)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// titlesOfEntries lists the titles of decoded entries, children in
|
||||
// brackets.
|
||||
func titlesOfEntries(v any) string {
|
||||
list, _ := v.([]any)
|
||||
var parts []string
|
||||
for _, item := range list {
|
||||
e, _ := item.(map[string]any)
|
||||
s, _ := e["title"].(string)
|
||||
if kids, ok := e["children"]; ok {
|
||||
s += "[" + titlesOfEntries(kids) + "]"
|
||||
}
|
||||
parts = append(parts, s)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// TestHTMLGroups pins D-34's HTML sitemap API: the overview of a locale
|
||||
// lists the groups in the L3 order (pages, offers, categories, posts) with
|
||||
// titles in the locale, categories nested by parent, posts newest first,
|
||||
// others alphabetical, the first html_page_size links and more when there
|
||||
// are more; a group page pages the rest; past the last page, an unknown
|
||||
// group and the HTML sitemap off are 404; excluded addresses never appear;
|
||||
// every answer is noindex. The settings tab validates its values and
|
||||
// contract.Listing answers the HTML sitemap and key file settings.
|
||||
func TestHTMLGroups(t *testing.T) {
|
||||
contentStore.reset(t)
|
||||
e := bootPlugins(t, []string{contentPluginID, journalPluginID, filterPluginID, "golem15.sitemap"})
|
||||
at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
|
||||
pl := func(id int, url, title string) fakeRecord {
|
||||
return fakeRecord{ID: id, Published: true, LastMod: at, Locales: []pact.ContentLocale{{Locale: "pl", URL: url, Title: title}}}
|
||||
}
|
||||
contentStore.put(pageType, pl(1, "/pages/o-nas", "O nas"))
|
||||
contentStore.put(pageType, pl(2, "/pages/kontakt", "Kontakt"))
|
||||
contentStore.put(pageType, pl(3, "/pages/noindex-me", "Ukryta"))
|
||||
contentStore.put(pageType, fakeRecord{ID: 4, Published: true, LastMod: at, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/about", Title: "About"}}})
|
||||
contentStore.put(offerType, pl(1, "/offers/zestaw", "Zestaw"))
|
||||
cat := func(id int, slug, title string, parent int) fakeRecord {
|
||||
r := pl(id, "/kategoria/"+slug, title)
|
||||
if parent > 0 {
|
||||
r.Parents = []pact.ContentRef{{Type: categoryType, ID: fmt.Sprint(parent)}}
|
||||
}
|
||||
return r
|
||||
}
|
||||
contentStore.put(categoryType, cat(1, "grzyby", "Grzyby", 0))
|
||||
contentStore.put(categoryType, cat(2, "lecznicze", "Lecznicze", 1))
|
||||
contentStore.put(categoryType, cat(3, "jadalne", "Jadalne", 1))
|
||||
contentStore.put(categoryType, cat(4, "inne", "Inne", 0))
|
||||
for i := 1; i <= 160; i++ {
|
||||
contentStore.put(postType, post(i, fmt.Sprintf("p%03d", i), "pl", at.Add(time.Duration(i)*time.Hour)))
|
||||
}
|
||||
e.exec(t, `INSERT INTO golem15_sitemap_types (type_key, include, slug, titles, updated_at) VALUES (?, TRUE, '', '{"pl":"Wpisy","en":"Posts"}', NOW())`, postType)
|
||||
e.rebuild(t)
|
||||
|
||||
ov := e.htmlJSON(t, "/_sitemap/api/v1/html?locale=pl", http.StatusOK)
|
||||
groups, _ := ov["groups"].([]any)
|
||||
var order []string
|
||||
byKey := map[string]map[string]any{}
|
||||
for _, g := range groups {
|
||||
m := g.(map[string]any)
|
||||
order = append(order, m["slug"].(string))
|
||||
byKey[m["slug"].(string)] = m
|
||||
}
|
||||
if strings.Join(order, " ") != "pages offers categories posts" {
|
||||
t.Fatalf("group order = %v\n%v", order, ov)
|
||||
}
|
||||
if got := titlesOfEntries(byKey["pages"]["entries"]); got != "Kontakt, O nas" {
|
||||
t.Errorf("pages = %s", got)
|
||||
}
|
||||
if got := titlesOfEntries(byKey["categories"]["entries"]); got != "Grzyby[Jadalne, Lecznicze], Inne" {
|
||||
t.Errorf("categories tree = %s", got)
|
||||
}
|
||||
posts := byKey["posts"]
|
||||
entries, _ := posts["entries"].([]any)
|
||||
if posts["title"] != "Wpisy" || posts["total"] != float64(160) || posts["more"] != true || len(entries) != 150 {
|
||||
t.Fatalf("posts group = title %v total %v more %v entries %d", posts["title"], posts["total"], posts["more"], len(entries))
|
||||
}
|
||||
if first := entries[0].(map[string]any); first["title"] != "Post p160" || first["url"] != "https://api.example.test/blog/p160" {
|
||||
t.Errorf("newest post first: %v", first)
|
||||
}
|
||||
if byKey["categories"]["title"] != "Categories" || byKey["pages"]["more"] != false {
|
||||
t.Errorf("categories title %v, pages more %v", byKey["categories"]["title"], byKey["pages"]["more"])
|
||||
}
|
||||
all := fmt.Sprint(ov)
|
||||
for _, banned := range []string{"Ukryta", "noindex-me", "About"} {
|
||||
if strings.Contains(all, banned) {
|
||||
t.Errorf("the pl overview lists %q", banned)
|
||||
}
|
||||
}
|
||||
if en := fmt.Sprint(e.htmlJSON(t, "/_sitemap/api/v1/html?locale=en", http.StatusOK)); !strings.Contains(en, "About") || strings.Contains(en, "Kontakt") {
|
||||
t.Errorf("the en overview: %s", en)
|
||||
}
|
||||
|
||||
page2 := e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl&page=2", http.StatusOK)
|
||||
if p2, _ := page2["entries"].([]any); page2["page"] != float64(2) || page2["pages"] != float64(2) || page2["group"] != "posts" || page2["title"] != "Wpisy" || len(p2) != 10 {
|
||||
t.Fatalf("posts page 2 = %v", page2)
|
||||
}
|
||||
if got := titlesOfEntries(page2["entries"]); !strings.HasPrefix(got, "Post p010, Post p009") || !strings.HasSuffix(got, "Post p001") {
|
||||
t.Errorf("posts page 2 = %s", got)
|
||||
}
|
||||
if p1 := e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl", http.StatusOK); p1["page"] != float64(1) {
|
||||
t.Errorf("posts without page = %v", p1["page"])
|
||||
}
|
||||
if got := titlesOfEntries(e.htmlJSON(t, "/_sitemap/api/v1/html/categories?locale=pl", http.StatusOK)["entries"]); got != "Grzyby[Jadalne, Lecznicze], Inne" {
|
||||
t.Errorf("categories page = %s", got)
|
||||
}
|
||||
for _, path := range []string{
|
||||
"/_sitemap/api/v1/html/posts?locale=pl&page=3",
|
||||
"/_sitemap/api/v1/html/posts?locale=pl&page=0",
|
||||
"/_sitemap/api/v1/html/posts?locale=pl&page=x",
|
||||
"/_sitemap/api/v1/html/unknown?locale=pl",
|
||||
"/_sitemap/api/v1/html/offers?locale=en",
|
||||
"/_sitemap/api/v1/html?locale=%3Cx%3E",
|
||||
} {
|
||||
e.htmlJSON(t, path, http.StatusNotFound)
|
||||
}
|
||||
|
||||
// The settings tab.
|
||||
token := e.admin(t, permSitemap)
|
||||
const base = "/golem15/sitemap/settings"
|
||||
schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String()
|
||||
for _, want := range []string{`"html_enabled"`, `"html_path"`, `"html_page_size"`, "HTML sitemap"} {
|
||||
if !strings.Contains(schema, want) {
|
||||
t.Errorf("form schema lacks %s", want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
body map[string]any
|
||||
field string
|
||||
}{
|
||||
{map[string]any{"html_page_size": 49}, "html_page_size"},
|
||||
{map[string]any{"html_page_size": 501}, "html_page_size"},
|
||||
{map[string]any{"html_path": "mapa"}, "html_path"},
|
||||
{map[string]any{"html_path": "/mapa?x=1"}, "html_path"},
|
||||
{map[string]any{"html_path": "//evil.test"}, "html_path"},
|
||||
} {
|
||||
if rec := e.call(t, http.MethodPut, base+"/1", token, c.body); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"`+c.field+`"`) {
|
||||
t.Errorf("%v: %d %s", c.body, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
listing, ok := e.app.Lookup[contract.Listing]()
|
||||
if !ok {
|
||||
t.Fatal("no contract.Listing")
|
||||
}
|
||||
ls, err := listing.Settings(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ls.HTMLEnabled || ls.HTMLPageSize != 150 || ls.HTMLPaths["pl"] != "/mapa-strony" || ls.HTMLPaths["en"] != "/sitemap" || ls.KeyFile != "" {
|
||||
t.Fatalf("listing settings = %+v", ls)
|
||||
}
|
||||
if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"html_path": "/site-map", "html_page_size": 100}); rec.Code != http.StatusOK {
|
||||
t.Fatalf("save the HTML tab: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`)
|
||||
ls, _ = listing.Settings(t.Context())
|
||||
if ls.HTMLPageSize != 100 || ls.HTMLPaths["en"] != "/site-map" || ls.HTMLPaths["pl"] != "/mapa-strony" || ls.KeyFile != "0123456789abcdef0123456789abcdef.txt" {
|
||||
t.Fatalf("listing settings after the save = %+v", ls)
|
||||
}
|
||||
if p := e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl&page=2", http.StatusOK); p["pages"] != float64(2) || len(p["entries"].([]any)) != 60 {
|
||||
t.Errorf("posts page 2 at size 100 = pages %v", p["pages"])
|
||||
}
|
||||
|
||||
e.call(t, http.MethodPut, base+"/1", token, map[string]any{"html_enabled": false})
|
||||
e.htmlJSON(t, "/_sitemap/api/v1/html?locale=pl", http.StatusNotFound)
|
||||
e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl", http.StatusNotFound)
|
||||
if ls, _ := listing.Settings(t.Context()); ls.HTMLEnabled {
|
||||
t.Error("listing says the HTML sitemap is on")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user