feat: sitemap management API and MCP tools
- GET and PUT /_sitemap/api/v1/settings and POST /_sitemap/api/v1/rebuild resolve the administrator from a backend token, a personal API token or the MCP dispatch and require the sitemap permission (401/403 otherwise); bucket sitemap-api - PUT changes the fields sent with the settings screen's checks (422 validation_failed, nothing stored), merges the HTML sitemap addresses, writes content types, generates the IndexNow key and queues a rebuild - GET answers the settings, the content types, the index files with their counts, the left-out counts, the IndexNow state and the News source values - MCP tools sitemap_get_settings, sitemap_update_settings and sitemap_rebuild bound to those routes, listed with the permission - the content type settings write and the file listing move to classes, shared by the form, S1, the index and the API; S1 and the index now list the image and News files
This commit is contained in:
82
classes/files.go
Normal file
82
classes/files.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// SitemapFile is one file of the sitemap index: its name, the number of
|
||||
// addresses (pages for an image file, articles for the News file) and the
|
||||
// newest lastmod of its addresses.
|
||||
type SitemapFile struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
LastMod *time.Time `json:"lastmod"`
|
||||
}
|
||||
|
||||
// SitemapFiles lists the files the index links, in its order: the
|
||||
// per-group files, the image files while images_enabled is on, then
|
||||
// sitemap-news.xml while news_enabled is on and it lists an article.
|
||||
func SitemapFiles(db *gorm.DB, app *backpack.App, s models.Settings, now time.Time) ([]SitemapFile, error) {
|
||||
type fileRow struct {
|
||||
Grp string `gorm:"column:grp"`
|
||||
FileNo int `gorm:"column:file_no"`
|
||||
N int `gorm:"column:n"`
|
||||
LastMod *time.Time `gorm:"column:lastmod"`
|
||||
}
|
||||
var rows []fileRow
|
||||
if err := db.Raw(`SELECT grp, file_no, COUNT(*) AS n, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason = '' AND file_no IS NOT NULL
|
||||
GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.ImagesEnabled {
|
||||
var images []fileRow
|
||||
if err := db.Raw(`SELECT image_file_no AS file_no, COUNT(*) AS n, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason = '' AND image_file_no IS NOT NULL AND jsonb_array_length(images) > 0
|
||||
GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range images {
|
||||
images[i].Grp = ImagesGroup
|
||||
}
|
||||
rows = append(rows, images...)
|
||||
}
|
||||
out := make([]SitemapFile, 0, len(rows)+1)
|
||||
for _, r := range rows {
|
||||
out = append(out, SitemapFile{Name: FileName(r.Grp, r.FileNo), Count: r.N, LastMod: r.LastMod})
|
||||
}
|
||||
if s.NewsEnabled {
|
||||
news, err := NewsEntries(db, app, s, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(news) > 0 {
|
||||
newest := news[0].Published
|
||||
out = append(out, SitemapFile{Name: NewsFile, Count: len(news), LastMod: &newest})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// LeftOut counts the addresses left out of the sitemaps by reason (a
|
||||
// Filter's or the type switch's; deleted and unpublished rows are holes,
|
||||
// not left out).
|
||||
func LeftOut(db *gorm.DB) (map[string]int, error) {
|
||||
var rows []struct {
|
||||
Reason string `gorm:"column:excluded_reason"`
|
||||
N int `gorm:"column:n"`
|
||||
}
|
||||
if err := db.Raw(`SELECT excluded_reason, COUNT(*) AS n FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason NOT IN ('', ?, ?) GROUP BY excluded_reason`, models.ReasonDeleted, models.ReasonUnpublished).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]int, len(rows))
|
||||
for _, r := range rows {
|
||||
out[r.Reason] = r.N
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
145
classes/typesettings.go
Normal file
145
classes/typesettings.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// The phrase keys of the content type refusals (UI-SPEC C26).
|
||||
const (
|
||||
errSlug = "golem15.sitemap::lang.errors.slug"
|
||||
errSlugTaken = "golem15.sitemap::lang.errors.slug_taken"
|
||||
)
|
||||
|
||||
var typeFieldUnsafe = regexp.MustCompile(`[^a-z0-9_]`)
|
||||
|
||||
// TypeFieldBase is the prefix of a content type's settings fields:
|
||||
// type_<key>_, where every character of the key outside [a-z0-9_] becomes
|
||||
// "_".
|
||||
func TypeFieldBase(key string) string {
|
||||
return "type_" + typeFieldUnsafe.ReplaceAllString(strings.ToLower(key), "_") + "_"
|
||||
}
|
||||
|
||||
// SaveTypeSettings stores submitted content type settings: values holds
|
||||
// TypeFieldBase(key)+"include" (a switch), +"slug" and +"title" (a locale
|
||||
// map) per type; a type or value not sent keeps what is stored. A slug
|
||||
// outside [a-z0-9-] is refused with errors.slug, one that another type (or
|
||||
// the image and News sitemaps) uses with errors.slug_taken; an empty slug
|
||||
// is the type's default. Refusals are by field name; nothing is written
|
||||
// while there are any.
|
||||
func SaveTypeSettings(db *gorm.DB, types []pact.MenuItemType, values map[string]any) (map[string][]string, error) {
|
||||
rows, err := LoadTypeSettings(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refusals := map[string][]string{}
|
||||
next := make([]models.TypeSetting, 0, len(types))
|
||||
slugOwner := map[string]string{}
|
||||
for _, typ := range types {
|
||||
base := TypeFieldBase(typ.Key)
|
||||
cfg := EffectiveType(typ, rows)
|
||||
row, ok := rows[typ.Key]
|
||||
if !ok {
|
||||
row = models.TypeSetting{TypeKey: typ.Key, Include: true}
|
||||
}
|
||||
if raw, sent := values[base+"include"]; sent {
|
||||
row.Include = truthy(raw)
|
||||
}
|
||||
slug := cfg.Slug
|
||||
if raw, sent := values[base+"slug"]; sent {
|
||||
slug = strings.TrimSpace(stringOf(raw))
|
||||
if slug == "" {
|
||||
slug = DefaultSlug(typ.Key)
|
||||
} else if !ValidSlug(slug) {
|
||||
refusals[base+"slug"] = append(refusals[base+"slug"], errSlug)
|
||||
}
|
||||
}
|
||||
if ReservedSlugs[slug] {
|
||||
refusals[base+"slug"] = append(refusals[base+"slug"], errSlugTaken)
|
||||
} else if owner, taken := slugOwner[slug]; taken && owner != typ.Key {
|
||||
refusals[base+"slug"] = append(refusals[base+"slug"], errSlugTaken)
|
||||
}
|
||||
slugOwner[slug] = typ.Key
|
||||
if slug == DefaultSlug(typ.Key) {
|
||||
row.Slug = ""
|
||||
} else {
|
||||
row.Slug = slug
|
||||
}
|
||||
if raw, sent := values[base+"title"]; sent {
|
||||
row.Titles = lagoon.Jsonable[map[string]string]{Data: titlesOf(raw), Valid: true}
|
||||
}
|
||||
next = append(next, row)
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return refusals, nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, row := range next {
|
||||
row.UpdatedAt = now
|
||||
if !row.Titles.Valid {
|
||||
row.Titles = lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true}
|
||||
}
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "type_key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"include", "slug", "titles", "updated_at"}),
|
||||
}).Create(&row).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// titlesOf reads a submitted translatable value (a map of locale to text);
|
||||
// blank texts are dropped.
|
||||
func titlesOf(v any) map[string]string {
|
||||
out := map[string]string{}
|
||||
switch m := v.(type) {
|
||||
case map[string]string:
|
||||
for k, s := range m {
|
||||
if strings.TrimSpace(s) != "" {
|
||||
out[k] = strings.TrimSpace(s)
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
for k, s := range m {
|
||||
if text, ok := s.(string); ok && strings.TrimSpace(text) != "" {
|
||||
out[k] = strings.TrimSpace(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// truthy reads a submitted switch value.
|
||||
func truthy(v any) bool {
|
||||
switch b := v.(type) {
|
||||
case bool:
|
||||
return b
|
||||
case string:
|
||||
return b == "1" || strings.EqualFold(b, "true") || strings.EqualFold(b, "on")
|
||||
case float64:
|
||||
return b != 0
|
||||
case fmt.Stringer:
|
||||
return b.String() == "1"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// stringOf reads a submitted text value.
|
||||
func stringOf(v any) string {
|
||||
switch s := v.(type) {
|
||||
case string:
|
||||
return s
|
||||
case nil:
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
129
controllers/api/auth.go
Normal file
129
controllers/api/auth.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/controllers"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/phrasebook"
|
||||
"git.golem15.com/golem15/summercms/modules/wire"
|
||||
)
|
||||
|
||||
// PermissionManage is the permission every management route requires: the
|
||||
// sitemap settings permission of the admin screen.
|
||||
const PermissionManage = controllers.PermissionAccessDefinitions
|
||||
|
||||
// tokenCover is the scope a personal API token must cover.
|
||||
const tokenCover = "golem15.sitemap.*"
|
||||
|
||||
// maxBody caps a management request body.
|
||||
const maxBody = 1 << 20
|
||||
|
||||
// The answers of the management API.
|
||||
const (
|
||||
errAuthenticationRequired = "Authentication required"
|
||||
errInsufficientPerms = "Insufficient permissions"
|
||||
errInternal = "Internal Server Error"
|
||||
errInvalidBody = "The request body is invalid."
|
||||
)
|
||||
|
||||
// ProblemsError is a refused write: phrase keys by field.
|
||||
type ProblemsError struct {
|
||||
Problems map[string][]string
|
||||
}
|
||||
|
||||
func (e *ProblemsError) Error() string { return "golem15.sitemap: validation failed" }
|
||||
|
||||
// apiError is an error a handler answers with its own status and body.
|
||||
type apiError struct {
|
||||
Status int
|
||||
Body any
|
||||
}
|
||||
|
||||
func (e *apiError) Error() string { return http.StatusText(e.Status) }
|
||||
|
||||
// handler is a management endpoint: it returns the status and the body to
|
||||
// answer, or an error the wrapper maps.
|
||||
type handler func(ctx context.Context, app *backpack.App, r *http.Request) (int, any, error)
|
||||
|
||||
// manage wraps a management endpoint (D-10): it resolves the backend
|
||||
// administrator from a backend JWT, a personal API token (covering
|
||||
// golem15.sitemap.*) or the in-process MCP dispatch marker
|
||||
// (cabana.ResolveBackend, so an MCP call runs as the linked
|
||||
// administrator), refuses 401 without one and 403 without the sitemap
|
||||
// permission, and answers a refused write 422
|
||||
// {"error":"validation_failed","errors":{field:[message]}} in the request
|
||||
// locale. Every answer carries X-Robots-Tag: noindex.
|
||||
func manage(app func() *backpack.App, h handler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Robots-Tag", "noindex")
|
||||
a := app()
|
||||
if a == nil {
|
||||
wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "The sitemap is not available"})
|
||||
return
|
||||
}
|
||||
out, principal, err := cabana.ResolveBackend(a, r, tokenCover)
|
||||
if err != nil || principal == nil {
|
||||
if errors.Is(err, cabana.ErrTokenScope) {
|
||||
wire.WriteJSON(w, http.StatusForbidden, map[string]string{"error": errInsufficientPerms})
|
||||
return
|
||||
}
|
||||
wire.WriteJSON(w, http.StatusUnauthorized, map[string]string{"error": errAuthenticationRequired})
|
||||
return
|
||||
}
|
||||
if !cabana.Allows(principal, []string{PermissionManage}) {
|
||||
wire.WriteJSON(w, http.StatusForbidden, map[string]string{"error": errInsufficientPerms})
|
||||
return
|
||||
}
|
||||
status, body, err := h(out.Context(), a, out)
|
||||
if err != nil {
|
||||
writeError(w, out.Context(), a, err)
|
||||
return
|
||||
}
|
||||
wire.WriteJSON(w, status, body)
|
||||
}
|
||||
}
|
||||
|
||||
// writeError maps a handler error to its answer.
|
||||
func writeError(w http.ResponseWriter, ctx context.Context, app *backpack.App, err error) {
|
||||
var api *apiError
|
||||
var problems *ProblemsError
|
||||
switch {
|
||||
case errors.As(err, &api):
|
||||
wire.WriteJSON(w, api.Status, api.Body)
|
||||
case errors.As(err, &problems):
|
||||
tr, _ := app.Lookup[*phrasebook.Translator]()
|
||||
messages := map[string][]string{}
|
||||
for field, keys := range problems.Problems {
|
||||
for _, key := range keys {
|
||||
messages[field] = append(messages[field], controllers.Phrase(ctx, tr, key, nil))
|
||||
}
|
||||
}
|
||||
wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]any{"error": "validation_failed", "errors": messages})
|
||||
default:
|
||||
log.Error("golem15.sitemap: management API", "error", err)
|
||||
wire.WriteJSON(w, http.StatusInternalServerError, map[string]any{"error": errInternal})
|
||||
}
|
||||
}
|
||||
|
||||
// decode reads a JSON body of at most maxBody bytes into v; an empty body
|
||||
// leaves v as it is.
|
||||
func decode(r *http.Request, v any) error {
|
||||
if r.Body == nil {
|
||||
return nil
|
||||
}
|
||||
dec := json.NewDecoder(io.LimitReader(r.Body, maxBody+1))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(v); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{"error": errInvalidBody}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
350
controllers/api/settings.go
Normal file
350
controllers/api/settings.go
Normal file
@@ -0,0 +1,350 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/golem15/summercms/modules/phrasebook"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// errUnknownType is the refusal of a types entry for a key no content type
|
||||
// has.
|
||||
const errUnknownType = "golem15.sitemap::lang.errors.type_unknown"
|
||||
|
||||
// TypeInput is the settings of one content type in UpdateSettingsInput:
|
||||
// only the fields sent change.
|
||||
type TypeInput struct {
|
||||
Include *bool `json:"include,omitempty" jsonschema:"list the type's records in the sitemaps"`
|
||||
Slug *string `json:"slug,omitempty" jsonschema:"the file name part of the type's files (sitemap-<slug>-<n>.xml): lowercase letters, digits and hyphens; empty for the default"`
|
||||
Titles map[string]string `json:"titles,omitempty" jsonschema:"the type's heading in the HTML sitemap per locale"`
|
||||
}
|
||||
|
||||
// UpdateSettingsInput is the body of PUT /_sitemap/api/v1/settings: only
|
||||
// the fields sent change, with the checks of the settings screen.
|
||||
type UpdateSettingsInput struct {
|
||||
SiteURL *string `json:"site_url,omitempty" jsonschema:"the public site address every sitemap address starts with, e.g. https://www.example.com; empty for the config value"`
|
||||
FileSize *int `json:"file_size,omitempty" jsonschema:"addresses per sitemap file, 1 to 50000; a change renumbers every file"`
|
||||
ExtraURLs *[]models.ExtraURL `json:"extra_urls,omitempty" jsonschema:"the additional addresses (url: a path or an absolute URL on the site; lastmod: optional YYYY-MM-DD); replaces the list"`
|
||||
ImagesEnabled *bool `json:"images_enabled,omitempty" jsonschema:"list the pages' images in the sitemap-images files"`
|
||||
NewsEnabled *bool `json:"news_enabled,omitempty" jsonschema:"serve the Google News sitemap sitemap-news.xml (only for sites accepted in Google News)"`
|
||||
NewsPublication *string `json:"news_publication,omitempty" jsonschema:"the publication name as registered in Google News; empty for the site name"`
|
||||
NewsSources *[]string `json:"news_sources,omitempty" jsonschema:"the article types and categories of the News sitemap, values of news_source_options (type:<key> or category:<key>:<id>); empty for every article type"`
|
||||
IndexNowEnabled *bool `json:"indexnow_enabled,omitempty" jsonschema:"notify IndexNow engines of new, changed and removed addresses after each rebuild (production only); turning it on generates the key"`
|
||||
IndexNowNewKey *bool `json:"indexnow_new_key,omitempty" jsonschema:"generate a new IndexNow key; the old key file stops working at once"`
|
||||
HTMLEnabled *bool `json:"html_enabled,omitempty" jsonschema:"serve the HTML sitemap's data to the site"`
|
||||
HTMLPaths map[string]string `json:"html_paths,omitempty" jsonschema:"the site's HTML sitemap address per locale, e.g. {\"pl\":\"/mapa-strony\"}; a blank value removes the locale"`
|
||||
HTMLPageSize *int `json:"html_page_size,omitempty" jsonschema:"links per HTML sitemap page, 50 to 500"`
|
||||
Types map[string]TypeInput `json:"types,omitempty" jsonschema:"content type settings by type key"`
|
||||
}
|
||||
|
||||
// settingsValues is the settings part of an answer.
|
||||
func settingsValues(s models.Settings) map[string]any {
|
||||
return map[string]any{
|
||||
"site_url": models.Text(s.SiteURL),
|
||||
"file_size": s.EffectiveFileSize(),
|
||||
"default_locale": models.Text(s.DefaultLocale),
|
||||
"extra_urls": nonNil(s.Extra()),
|
||||
"images_enabled": s.ImagesEnabled,
|
||||
"news_enabled": s.NewsEnabled,
|
||||
"news_publication": models.Text(s.NewsPublication),
|
||||
"news_sources": nonNil(s.Sources()),
|
||||
"indexnow_enabled": s.IndexNowEnabled,
|
||||
"html_enabled": s.HTMLEnabled,
|
||||
"html_paths": s.EffectiveHTMLPaths(),
|
||||
"html_page_size": s.EffectiveHTMLPageSize(),
|
||||
}
|
||||
}
|
||||
|
||||
func nonNil[T any](s []T) []T {
|
||||
if s == nil {
|
||||
return []T{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fileJSON is one file in an answer, with its public address.
|
||||
type fileJSON struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Count int `json:"count"`
|
||||
LastMod *time.Time `json:"lastmod"`
|
||||
}
|
||||
|
||||
// settingsBody is the answer of the settings routes: the settings, the
|
||||
// content types (label and settings by key), the files of the index, the
|
||||
// addresses left out by reason, the rebuild state, the IndexNow state and
|
||||
// the values news_sources takes.
|
||||
func settingsBody(ctx context.Context, app *backpack.App, db *gorm.DB) (map[string]any, error) {
|
||||
s, err := classes.LoadSettings(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
site := classes.SiteURL(app, s)
|
||||
reg, _ := app.Lookup[*signpost.Registry]()
|
||||
tr, _ := app.Lookup[*phrasebook.Translator]()
|
||||
types := map[string]any{}
|
||||
if reg != nil {
|
||||
rows, err := classes.LoadTypeSettings(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, typ := range reg.ContentTypes() {
|
||||
cfg := classes.EffectiveType(typ, rows)
|
||||
label := typ.Label
|
||||
if tr != nil {
|
||||
label = tr.Get(ctx, typ.Label, nil)
|
||||
}
|
||||
titles := cfg.Titles
|
||||
if titles == nil {
|
||||
titles = map[string]string{}
|
||||
}
|
||||
types[typ.Key] = map[string]any{"label": label, "include": cfg.Include, "slug": cfg.Slug, "titles": titles}
|
||||
}
|
||||
}
|
||||
list, err := classes.SitemapFiles(db, app, s, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := make([]fileJSON, 0, len(list))
|
||||
for _, f := range list {
|
||||
files = append(files, fileJSON{Name: f.Name, URL: site + "/" + f.Name, Count: f.Count, LastMod: f.LastMod})
|
||||
}
|
||||
leftOut, err := classes.LeftOut(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sources, err := classes.NewsSources(db, reg, models.Text(s.DefaultLocale))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
index := ""
|
||||
if site != "" {
|
||||
index = site + "/sitemap.xml"
|
||||
}
|
||||
return map[string]any{
|
||||
"settings": settingsValues(s),
|
||||
"types": types,
|
||||
"index_url": index,
|
||||
"files": files,
|
||||
"left_out": leftOut,
|
||||
"built_at": s.BuiltAt,
|
||||
"building": classes.Building(s.RebuildRequestedAt, s.BuiltAt, time.Now()),
|
||||
"indexnow": classes.LoadIndexNowStatus(db, app, s),
|
||||
"news_source_options": sources,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// database returns the application's database bound to ctx.
|
||||
func database(ctx context.Context, app *backpack.App) (*gorm.DB, error) {
|
||||
db, ok := app.Lookup[*gorm.DB]()
|
||||
if !ok || db == nil {
|
||||
return nil, classes.ErrNoDatabase
|
||||
}
|
||||
return db.WithContext(ctx), nil
|
||||
}
|
||||
|
||||
// ShowSettings serves GET /_sitemap/api/v1/settings: the sitemap settings,
|
||||
// the content types, the files with their address counts, the left-out
|
||||
// counts, the IndexNow key and last notification (settingsBody).
|
||||
func ShowSettings(app func() *backpack.App) http.HandlerFunc {
|
||||
return manage(app, func(ctx context.Context, a *backpack.App, _ *http.Request) (int, any, error) {
|
||||
db, err := database(ctx, a)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
body, err := settingsBody(ctx, a, db)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return http.StatusOK, map[string]any{"data": body}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateSettings serves PUT /_sitemap/api/v1/settings: the fields sent
|
||||
// change with the checks of the settings screen (422 validation_failed
|
||||
// names the refused fields, types.<key>.<field> for a content type), the
|
||||
// IndexNow key is generated as on the screen and a rebuild is queued once
|
||||
// the change commits. It answers like ShowSettings.
|
||||
func UpdateSettings(app func() *backpack.App) http.HandlerFunc {
|
||||
return manage(app, func(ctx context.Context, a *backpack.App, r *http.Request) (int, any, error) {
|
||||
var in UpdateSettingsInput
|
||||
if err := decode(r, &in); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
gdb, err := database(ctx, a)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
err = lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error {
|
||||
return applySettings(ctx, a, tx, in)
|
||||
})
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
body, err := settingsBody(ctx, a, gdb)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return http.StatusOK, map[string]any{"data": body}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// applySettings writes in inside tx: the settings columns sent, the HTML
|
||||
// sitemap addresses merged per locale, the content types, the IndexNow
|
||||
// key, then queues a rebuild after the commit.
|
||||
func applySettings(ctx context.Context, app *backpack.App, tx *gorm.DB, in UpdateSettingsInput) error {
|
||||
if err := tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := classes.LoadSettings(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
columns := map[string]any{}
|
||||
if in.SiteURL != nil {
|
||||
v := strings.TrimSpace(*in.SiteURL)
|
||||
s.SiteURL = &v
|
||||
columns["site_url"] = v
|
||||
}
|
||||
if in.FileSize != nil {
|
||||
s.FileSize = *in.FileSize
|
||||
columns["file_size"] = *in.FileSize
|
||||
}
|
||||
if in.ExtraURLs != nil {
|
||||
s.ExtraURLs = lagoon.Jsonable[[]models.ExtraURL]{Data: nonNil(*in.ExtraURLs), Valid: true}
|
||||
columns["extra_urls"] = s.ExtraURLs
|
||||
}
|
||||
if in.ImagesEnabled != nil {
|
||||
s.ImagesEnabled = *in.ImagesEnabled
|
||||
columns["images_enabled"] = *in.ImagesEnabled
|
||||
}
|
||||
if in.NewsEnabled != nil {
|
||||
s.NewsEnabled = *in.NewsEnabled
|
||||
columns["news_enabled"] = *in.NewsEnabled
|
||||
}
|
||||
if in.NewsPublication != nil {
|
||||
v := strings.TrimSpace(*in.NewsPublication)
|
||||
s.NewsPublication = &v
|
||||
columns["news_publication"] = v
|
||||
}
|
||||
if in.NewsSources != nil {
|
||||
list := []string{}
|
||||
for _, v := range *in.NewsSources {
|
||||
if v = strings.TrimSpace(v); v != "" {
|
||||
list = append(list, v)
|
||||
}
|
||||
}
|
||||
s.NewsSources = lagoon.Jsonable[[]string]{Data: list, Valid: true}
|
||||
columns["news_sources"] = s.NewsSources
|
||||
}
|
||||
if in.IndexNowEnabled != nil {
|
||||
s.IndexNowEnabled = *in.IndexNowEnabled
|
||||
columns["indexnow_enabled"] = *in.IndexNowEnabled
|
||||
}
|
||||
if in.HTMLEnabled != nil {
|
||||
s.HTMLEnabled = *in.HTMLEnabled
|
||||
columns["html_enabled"] = *in.HTMLEnabled
|
||||
}
|
||||
if in.HTMLPageSize != nil {
|
||||
s.HTMLPageSize = *in.HTMLPageSize
|
||||
columns["html_page_size"] = *in.HTMLPageSize
|
||||
}
|
||||
problems := s.Problems()
|
||||
delete(problems, "html_path")
|
||||
var paths map[string]string
|
||||
if in.HTMLPaths != nil {
|
||||
merged, bad := classes.MergeHTMLPaths(tx, in.HTMLPaths)
|
||||
if bad {
|
||||
problems["html_paths"] = []string{models.HTMLPathProblem}
|
||||
}
|
||||
paths = merged
|
||||
}
|
||||
var reg *signpost.Registry
|
||||
if app != nil {
|
||||
reg, _ = app.Lookup[*signpost.Registry]()
|
||||
}
|
||||
values, names := map[string]any{}, map[string]string{}
|
||||
for key, t := range in.Types {
|
||||
if reg == nil {
|
||||
problems["types."+key] = []string{errUnknownType}
|
||||
continue
|
||||
}
|
||||
if _, ok := reg.ContentType(key); !ok {
|
||||
problems["types."+key] = []string{errUnknownType}
|
||||
continue
|
||||
}
|
||||
base := classes.TypeFieldBase(key)
|
||||
names[base+"include"], names[base+"slug"], names[base+"title"] = "types."+key+".include", "types."+key+".slug", "types."+key+".titles"
|
||||
if t.Include != nil {
|
||||
values[base+"include"] = *t.Include
|
||||
}
|
||||
if t.Slug != nil {
|
||||
values[base+"slug"] = *t.Slug
|
||||
}
|
||||
if t.Titles != nil {
|
||||
values[base+"title"] = t.Titles
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return &ProblemsError{Problems: problems}
|
||||
}
|
||||
if len(columns) > 0 {
|
||||
if err := tx.Model(&models.Settings{}).Where("id = ?", 1).Updates(columns).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if paths != nil {
|
||||
if err := classes.StoreHTMLPaths(tx, paths); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if reg != nil && len(values) > 0 {
|
||||
refusals, err := classes.SaveTypeSettings(tx, reg.ContentTypes(), values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
out := map[string][]string{}
|
||||
for field, keys := range refusals {
|
||||
if name, ok := names[field]; ok {
|
||||
field = name
|
||||
}
|
||||
out[field] = keys
|
||||
}
|
||||
return &ProblemsError{Problems: out}
|
||||
}
|
||||
}
|
||||
if _, err := classes.EnsureKey(tx, s.IndexNowEnabled, in.IndexNowNewKey != nil && *in.IndexNowNewKey); err != nil {
|
||||
return err
|
||||
}
|
||||
return classes.RequestRebuild(ctx, app, tx)
|
||||
}
|
||||
|
||||
// Rebuild serves POST /_sitemap/api/v1/rebuild: it queues a rebuild of the
|
||||
// sitemap files now (the settings screen then shows files.building) and
|
||||
// answers 202 {"data":{"queued":true}}.
|
||||
func Rebuild(app func() *backpack.App) http.HandlerFunc {
|
||||
return manage(app, func(ctx context.Context, a *backpack.App, _ *http.Request) (int, any, error) {
|
||||
gdb, err := database(ctx, a)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if err := lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error {
|
||||
return classes.RequestRebuild(ctx, a, tx)
|
||||
}); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return http.StatusAccepted, map[string]any{"data": map[string]any{"queued": true}}, nil
|
||||
})
|
||||
}
|
||||
@@ -292,15 +292,8 @@ func (c settingsAdminController) filesData(ctx context.Context) (any, error) {
|
||||
}
|
||||
site := classes.SiteURL(app, settings)
|
||||
view := filesView{Building: classes.Building(settings.RebuildRequestedAt, settings.BuiltAt, time.Now())}
|
||||
var files []struct {
|
||||
Grp string `gorm:"column:grp"`
|
||||
FileNo int `gorm:"column:file_no"`
|
||||
N int `gorm:"column:n"`
|
||||
LastMod *time.Time `gorm:"column:lastmod"`
|
||||
}
|
||||
if err := db.Raw(`SELECT grp, file_no, COUNT(*) AS n, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason = '' AND file_no IS NOT NULL
|
||||
GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&files).Error; err != nil {
|
||||
files, err := classes.SitemapFiles(db, app, settings, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The partial kit links only root-relative addresses, so a file links
|
||||
@@ -317,28 +310,23 @@ GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&files).Error; err != nil {
|
||||
index := fileRow{Name: "sitemap.xml", URL: link("sitemap.xml")}
|
||||
var newest *time.Time
|
||||
for _, f := range files {
|
||||
index.Count += f.N
|
||||
// The index counts addresses; the image and News files list
|
||||
// some of them again.
|
||||
if !strings.HasPrefix(f.Name, "sitemap-"+classes.ImagesGroup+"-") && f.Name != classes.NewsFile {
|
||||
index.Count += f.Count
|
||||
}
|
||||
if f.LastMod != nil && (newest == nil || f.LastMod.After(*newest)) {
|
||||
newest = f.LastMod
|
||||
}
|
||||
name := classes.FileName(f.Grp, f.FileNo)
|
||||
view.Files = append(view.Files, fileRow{Name: name, URL: link(name), Count: f.N, LastMod: formatTime(f.LastMod)})
|
||||
view.Files = append(view.Files, fileRow{Name: f.Name, URL: link(f.Name), Count: f.Count, LastMod: formatTime(f.LastMod)})
|
||||
}
|
||||
index.LastMod = formatTime(newest)
|
||||
view.Files = append([]fileRow{index}, view.Files...)
|
||||
}
|
||||
var counts []struct {
|
||||
Reason string `gorm:"column:excluded_reason"`
|
||||
N int `gorm:"column:n"`
|
||||
}
|
||||
if err := db.Raw(`SELECT excluded_reason, COUNT(*) AS n FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason NOT IN ('', ?, ?) GROUP BY excluded_reason`, models.ReasonDeleted, models.ReasonUnpublished).Scan(&counts).Error; err != nil {
|
||||
byReason, err := classes.LeftOut(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byReason := map[string]int{}
|
||||
for _, row := range counts {
|
||||
byReason[row.Reason] = row.N
|
||||
}
|
||||
tr := translator(c.app)
|
||||
for _, reason := range leftOutReasons {
|
||||
if n := byReason[reason]; n > 0 {
|
||||
|
||||
@@ -2,22 +2,17 @@ package sitemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/controllers"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"git.golem15.com/golem15/summercms/modules/phrasebook"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"github.com/goccy/go-yaml"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// The settings form's tab keys: the Content types tab follows Files; the
|
||||
@@ -33,20 +28,6 @@ const (
|
||||
// settings row has no translation writer of its own).
|
||||
const htmlPathField = "html_path"
|
||||
|
||||
// The phrase keys of the Content types refusals (UI-SPEC C26).
|
||||
const (
|
||||
errSlug = "golem15.sitemap::lang.errors.slug"
|
||||
errSlugTaken = "golem15.sitemap::lang.errors.slug_taken"
|
||||
)
|
||||
|
||||
var typeFieldUnsafe = regexp.MustCompile(`[^a-z0-9_]`)
|
||||
|
||||
// typeFieldKey turns a content type key into the part of a field name:
|
||||
// every character outside [a-z0-9_] becomes "_".
|
||||
func typeFieldKey(key string) string {
|
||||
return typeFieldUnsafe.ReplaceAllString(strings.ToLower(key), "_")
|
||||
}
|
||||
|
||||
// AdminActionExtensions extends the plugin's own settings singleton: the
|
||||
// Content types tab (one block per signpost content type, in registration
|
||||
// order; none without content types) and the HTML sitemap address.
|
||||
@@ -77,7 +58,7 @@ func (p *Plugin) registry() *signpost.Registry {
|
||||
func typeSettingsDocument(types []pact.MenuItemType) ([]byte, error) {
|
||||
fields := yaml.MapSlice{}
|
||||
for _, typ := range types {
|
||||
base := "type_" + typeFieldKey(typ.Key) + "_"
|
||||
base := classes.TypeFieldBase(typ.Key)
|
||||
label := typ.Label
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = typ.Key
|
||||
@@ -142,7 +123,7 @@ func (p *Plugin) typeSettingsTab(types []pact.MenuItemType) *pact.AdminFormExten
|
||||
out := map[string]any{}
|
||||
for _, typ := range types {
|
||||
cfg := classes.EffectiveType(typ, rows)
|
||||
base := "type_" + typeFieldKey(typ.Key) + "_"
|
||||
base := classes.TypeFieldBase(typ.Key)
|
||||
out[base+"include"] = cfg.Include
|
||||
out[base+"slug"] = cfg.Slug
|
||||
titles := cfg.Titles
|
||||
@@ -167,68 +148,15 @@ func (p *Plugin) typeSettingsTab(types []pact.MenuItemType) *pact.AdminFormExten
|
||||
}
|
||||
}
|
||||
|
||||
// saveTypeSettings stores the submitted Content types values. A slug
|
||||
// outside [a-z0-9-] is refused with errors.slug, one that another type (or
|
||||
// the image and News sitemaps) uses with errors.slug_taken; an empty slug
|
||||
// is the type's default.
|
||||
func saveTypeSettings(ctx context.Context, in pact.AdminFormSave, types []pact.MenuItemType) error {
|
||||
rows, err := classes.LoadTypeSettings(in.DB)
|
||||
// saveTypeSettings stores the submitted Content types values
|
||||
// (classes.SaveTypeSettings).
|
||||
func saveTypeSettings(_ context.Context, in pact.AdminFormSave, types []pact.MenuItemType) error {
|
||||
refusals, err := classes.SaveTypeSettings(in.DB, types, in.Values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
refusals := pact.AdminFormErrors{}
|
||||
next := make([]models.TypeSetting, 0, len(types))
|
||||
slugOwner := map[string]string{}
|
||||
for _, typ := range types {
|
||||
base := "type_" + typeFieldKey(typ.Key) + "_"
|
||||
cfg := classes.EffectiveType(typ, rows)
|
||||
row, ok := rows[typ.Key]
|
||||
if !ok {
|
||||
row = models.TypeSetting{TypeKey: typ.Key, Include: true}
|
||||
}
|
||||
if raw, sent := in.Values[base+"include"]; sent {
|
||||
row.Include = truthy(raw)
|
||||
}
|
||||
slug := cfg.Slug
|
||||
if raw, sent := in.Values[base+"slug"]; sent {
|
||||
slug = strings.TrimSpace(stringOf(raw))
|
||||
if slug == "" {
|
||||
slug = classes.DefaultSlug(typ.Key)
|
||||
} else if !classes.ValidSlug(slug) {
|
||||
refusals[base+"slug"] = append(refusals[base+"slug"], errSlug)
|
||||
}
|
||||
}
|
||||
if classes.ReservedSlugs[slug] {
|
||||
refusals[base+"slug"] = append(refusals[base+"slug"], errSlugTaken)
|
||||
} else if owner, taken := slugOwner[slug]; taken && owner != typ.Key {
|
||||
refusals[base+"slug"] = append(refusals[base+"slug"], errSlugTaken)
|
||||
}
|
||||
slugOwner[slug] = typ.Key
|
||||
if slug == classes.DefaultSlug(typ.Key) {
|
||||
row.Slug = ""
|
||||
} else {
|
||||
row.Slug = slug
|
||||
}
|
||||
if raw, sent := in.Values[base+"title"]; sent {
|
||||
row.Titles = lagoon.Jsonable[map[string]string]{Data: titlesOf(raw), Valid: true}
|
||||
}
|
||||
next = append(next, row)
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return refusals
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, row := range next {
|
||||
row.UpdatedAt = now
|
||||
if !row.Titles.Valid {
|
||||
row.Titles = lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true}
|
||||
}
|
||||
if err := in.DB.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "type_key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"include", "slug", "titles", "updated_at"}),
|
||||
}).Create(&row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return pact.AdminFormErrors(refusals)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -259,51 +187,6 @@ func saveHTMLPaths(in pact.AdminFormSave) error {
|
||||
return classes.StoreHTMLPaths(in.DB, merged)
|
||||
}
|
||||
|
||||
// titlesOf reads a submitted translatable value (a map of locale to text);
|
||||
// blank texts are dropped.
|
||||
func titlesOf(v any) map[string]string {
|
||||
out := map[string]string{}
|
||||
switch m := v.(type) {
|
||||
case map[string]string:
|
||||
for k, s := range m {
|
||||
if strings.TrimSpace(s) != "" {
|
||||
out[k] = strings.TrimSpace(s)
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
for k, s := range m {
|
||||
if text, ok := s.(string); ok && strings.TrimSpace(text) != "" {
|
||||
out[k] = strings.TrimSpace(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// truthy reads a submitted switch value.
|
||||
func truthy(v any) bool {
|
||||
switch b := v.(type) {
|
||||
case bool:
|
||||
return b
|
||||
case string:
|
||||
return b == "1" || strings.EqualFold(b, "true") || strings.EqualFold(b, "on")
|
||||
case fmt.Stringer:
|
||||
return b.String() == "1"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// stringOf reads a submitted text value.
|
||||
func stringOf(v any) string {
|
||||
switch s := v.(type) {
|
||||
case string:
|
||||
return s
|
||||
case nil:
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
|
||||
// translator returns the application's translator, nil when none.
|
||||
func (p *Plugin) translator() *phrasebook.Translator {
|
||||
if p == nil || p.app == nil {
|
||||
|
||||
@@ -14,6 +14,7 @@ errors:
|
||||
lastmod: 'Enter a date as YYYY-MM-DD, e.g. 2026-10-28, or leave it empty.'
|
||||
news_publication: Enter at most 255 characters, without line breaks.
|
||||
news_source: 'Use the values listed under the field: type:<type key> or category:<type key>:<id>.'
|
||||
type_unknown: There is no content type with this key.
|
||||
settings:
|
||||
label: Sitemaps
|
||||
description: XML sitemaps per content type, images, News, IndexNow and the HTML sitemap.
|
||||
|
||||
@@ -14,6 +14,7 @@ errors:
|
||||
lastmod: 'Wpisz datę w formacie RRRR-MM-DD, np. 2026-10-28, albo zostaw puste.'
|
||||
news_publication: Wpisz najwyżej 255 znaków, bez podziału wiersza.
|
||||
news_source: 'Użyj wartości z listy pod polem: type:<klucz typu> albo category:<klucz typu>:<id>.'
|
||||
type_unknown: Nie ma typu treści o tym kluczu.
|
||||
settings:
|
||||
label: Mapy witryny
|
||||
description: Mapy XML według typów treści, obrazy, News, IndexNow i mapa HTML.
|
||||
|
||||
55
mcp.go
Normal file
55
mcp.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
apicontrollers "git.golem15.com/golem15/sm-sitemap-plugin/controllers/api"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
var _ pact.HasMCP = (*Plugin)(nil)
|
||||
|
||||
// mcpGuard is the guard every sitemap tool runs as: the backend account
|
||||
// linked to the MCP user. The routes resolve that administrator themselves.
|
||||
const mcpGuard = "backend"
|
||||
|
||||
// MCP declares the sitemap tools (D-10, UI-SPEC MCP parity): reading and
|
||||
// changing the settings screen's values (with the file list, the IndexNow
|
||||
// key and status) and queueing a rebuild, each bound to a /_sitemap/api/v1
|
||||
// management route, run as the backend account linked to the MCP user and
|
||||
// listed only when that account has the sitemap permission. pier prefixes
|
||||
// them with sitemap_.
|
||||
func (p *Plugin) MCP() pact.MCP {
|
||||
return pact.MCP{Tools: []pact.MCPTool{
|
||||
{
|
||||
Name: "get_settings",
|
||||
Title: "Show the sitemap settings",
|
||||
Description: "Shows the sitemap settings (site address, file size, additional addresses, image and News sitemaps, IndexNow, HTML sitemap), the content types with their file names and headings, the files of the index with their address counts, the addresses left out by reason, the IndexNow key and last notification, and the values the News sources take.",
|
||||
Guard: mcpGuard,
|
||||
Method: http.MethodGet,
|
||||
Path: "/_sitemap/api/v1/settings",
|
||||
Input: struct{}{},
|
||||
Permissions: []string{apicontrollers.PermissionManage},
|
||||
},
|
||||
{
|
||||
Name: "update_settings",
|
||||
Title: "Update the sitemap settings",
|
||||
Description: "Changes the sitemap settings sent and, under types, the settings of content types by key, with the checks of the settings screen. Turning IndexNow on generates its key; indexnow_new_key replaces it. The sitemaps are rebuilt afterwards.",
|
||||
Guard: mcpGuard,
|
||||
Method: http.MethodPut,
|
||||
Path: "/_sitemap/api/v1/settings",
|
||||
Input: apicontrollers.UpdateSettingsInput{},
|
||||
Permissions: []string{apicontrollers.PermissionManage},
|
||||
},
|
||||
{
|
||||
Name: "rebuild",
|
||||
Title: "Rebuild the sitemaps",
|
||||
Description: "Queues a rebuild of the sitemap files now, as a settings save does; get_settings shows building until it has run.",
|
||||
Guard: mcpGuard,
|
||||
Method: http.MethodPost,
|
||||
Path: "/_sitemap/api/v1/rebuild",
|
||||
Input: struct{}{},
|
||||
Permissions: []string{apicontrollers.PermissionManage},
|
||||
},
|
||||
}}
|
||||
}
|
||||
70
routes.go
70
routes.go
@@ -22,10 +22,12 @@ const (
|
||||
)
|
||||
|
||||
// The throttle buckets: BucketSitemap of the sitemap files and the key
|
||||
// file, BucketHTML of the HTML sitemap API.
|
||||
// file, BucketHTML of the HTML sitemap API, BucketAPI of the management
|
||||
// API.
|
||||
const (
|
||||
BucketSitemap = "sitemap-xml"
|
||||
BucketHTML = "sitemap-html"
|
||||
BucketAPI = "sitemap-api"
|
||||
)
|
||||
|
||||
// FilesPath is the route of the per-group files. The index lists them at
|
||||
@@ -52,15 +54,25 @@ func (p *Plugin) Routes(r pact.Router) error {
|
||||
r.Get("/_sitemap/api/v1/html", apicontrollers.HTMLOverview(p.booted), "throttle:"+BucketHTML)
|
||||
r.Get("/_sitemap/api/v1/html/{group}", apicontrollers.HTMLGroup(p.booted), "throttle:"+BucketHTML)
|
||||
r.Where("group", apicontrollers.GroupPattern)
|
||||
// The management API (D-10): each route resolves the administrator
|
||||
// from a backend token, a personal API token or the MCP dispatch and
|
||||
// requires the sitemap permission; each one is an MCP tool (mcp.go).
|
||||
r.Group("/_sitemap/api/v1", nil, func(g pact.Router) {
|
||||
api := "throttle:" + BucketAPI
|
||||
g.Get("/settings", apicontrollers.ShowSettings(p.booted), api)
|
||||
g.Put("/settings", apicontrollers.UpdateSettings(p.booted), api)
|
||||
g.Post("/rebuild", apicontrollers.Rebuild(p.booted), api)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// booted returns the booted application.
|
||||
func (p *Plugin) booted() *backpack.App { return p.app }
|
||||
|
||||
// Buckets registers sitemap-xml (60 requests a minute) and sitemap-html
|
||||
// (300 a minute: the site's server renders every HTML sitemap page from
|
||||
// it) per client IP, behind the configured trusted proxies. 429 bodies are
|
||||
// Buckets registers sitemap-xml (60 requests a minute), sitemap-html (300
|
||||
// a minute: the site's server renders every HTML sitemap page from it) and
|
||||
// sitemap-api (300 a minute) per client IP, behind the configured trusted
|
||||
// proxies. 429 bodies are
|
||||
// surf's {"message":"Too Many Attempts."}.
|
||||
func (p *Plugin) Buckets() map[string]surf.Bucket {
|
||||
trusted := surf.TrustedProxies(nil)
|
||||
@@ -80,18 +92,14 @@ func (p *Plugin) Buckets() map[string]surf.Bucket {
|
||||
return map[string]surf.Bucket{
|
||||
BucketSitemap: bucket(BucketSitemap, 60),
|
||||
BucketHTML: bucket(BucketHTML, 300),
|
||||
BucketAPI: bucket(BucketAPI, 300),
|
||||
}
|
||||
}
|
||||
|
||||
// indexRow is one file of the index query.
|
||||
type indexRow struct {
|
||||
Grp string `gorm:"column:grp"`
|
||||
FileNo int `gorm:"column:file_no"`
|
||||
LastMod *time.Time `gorm:"column:lastmod"`
|
||||
}
|
||||
|
||||
// sitemapIndex serves the sitemap index: one <sitemap> per file that lists
|
||||
// at least one address, with the newest lastmod of its addresses. Without a
|
||||
// at least one address (classes.SitemapFiles: the per-group files, the
|
||||
// image files and the News file), with the newest lastmod of its
|
||||
// addresses. Without a
|
||||
// site address (the setting, golem15.sitemap.site_url or app.url) it is a
|
||||
// 404: a <loc> is never built from the request's Host.
|
||||
func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -113,43 +121,15 @@ func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
var rows []indexRow
|
||||
if err := db.Raw(`SELECT grp, file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason = '' AND file_no IS NOT NULL
|
||||
GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil {
|
||||
list, err := classes.SitemapFiles(db, p.app, settings, time.Now().UTC())
|
||||
if err != nil {
|
||||
pluginLog.Error("golem15.sitemap: index", "error", err)
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if settings.ImagesEnabled {
|
||||
var images []indexRow
|
||||
if err := db.Raw(`SELECT image_file_no AS file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
|
||||
WHERE excluded_reason = '' AND image_file_no IS NOT NULL AND jsonb_array_length(images) > 0
|
||||
GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil {
|
||||
pluginLog.Error("golem15.sitemap: image index", "error", err)
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
for i := range images {
|
||||
images[i].Grp = classes.ImagesGroup
|
||||
}
|
||||
rows = append(rows, images...)
|
||||
}
|
||||
files := make([]classes.IndexFile, 0, len(rows)+1)
|
||||
for _, row := range rows {
|
||||
files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod})
|
||||
}
|
||||
if settings.NewsEnabled {
|
||||
news, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC())
|
||||
if err != nil {
|
||||
pluginLog.Error("golem15.sitemap: news index", "error", err)
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(news) > 0 {
|
||||
newest := news[0].Published
|
||||
files = append(files, classes.IndexFile{Loc: site + "/" + classes.NewsFile, LastMod: &newest})
|
||||
}
|
||||
files := make([]classes.IndexFile, 0, len(list))
|
||||
for _, f := range list {
|
||||
files = append(files, classes.IndexFile{Loc: site + "/" + f.Name, LastMod: f.LastMod})
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := classes.WriteIndex(&buf, files); err != nil {
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package sitemap_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -15,10 +17,14 @@ import (
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/contract"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/bouncer"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"git.golem15.com/golem15/summercms/modules/party"
|
||||
"git.golem15.com/golem15/summercms/modules/pier"
|
||||
"git.golem15.com/golem15/summercms/modules/surf"
|
||||
)
|
||||
|
||||
// journalPluginID is a fake blog plugin with an article type (posts at
|
||||
@@ -726,3 +732,232 @@ func TestHTMLGroups(t *testing.T) {
|
||||
t.Error("listing says the HTML sitemap is on")
|
||||
}
|
||||
}
|
||||
|
||||
// api sends a management API request with a bearer token (none when "")
|
||||
// and a JSON body.
|
||||
func (e *sitemapEnv) api(t *testing.T, method, path, token string, body any) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader = bytes.NewReader(raw)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, reader)
|
||||
req.Host = "api.example.test"
|
||||
req.RemoteAddr = fmt.Sprintf("198.51.100.%d:4000", adminSeq.Add(1)%250+1)
|
||||
req.Header.Set("Accept-Language", "en")
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
e.h.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
// apiJSON sends a management API request, fails unless it answers status
|
||||
// and decodes the answer.
|
||||
func (e *sitemapEnv) apiJSON(t *testing.T, status int, method, path, token string, body any) map[string]any {
|
||||
t.Helper()
|
||||
rec := e.api(t, method, path, token, body)
|
||||
if rec.Code != status {
|
||||
t.Fatalf("%s %s = %d, want %d: %s", method, path, rec.Code, status, rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("X-Robots-Tag") != "noindex" {
|
||||
t.Errorf("%s %s lacks X-Robots-Tag noindex", method, path)
|
||||
}
|
||||
out := map[string]any{}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode %s: %v", rec.Body.String(), err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestSitemapTools pins D-10 for the sitemap: GET and PUT
|
||||
// /_sitemap/api/v1/settings and POST /_sitemap/api/v1/rebuild need a
|
||||
// backend administrator with the sitemap permission (401 without one, 403
|
||||
// without the permission); PUT checks like the form (422, nothing
|
||||
// stored), merges the HTML sitemap addresses, writes content types,
|
||||
// generates the IndexNow key and queues a rebuild; an MCP dispatch runs as
|
||||
// the linked administrator; the MCP catalog declares get_settings,
|
||||
// update_settings and rebuild bound to those routes behind the permission.
|
||||
func TestSitemapTools(t *testing.T) {
|
||||
e := bootJournal(t)
|
||||
contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC), "/media/a.jpg"))
|
||||
e.rebuild(t)
|
||||
routes := []struct{ method, path string }{
|
||||
{http.MethodGet, "/_sitemap/api/v1/settings"},
|
||||
{http.MethodPut, "/_sitemap/api/v1/settings"},
|
||||
{http.MethodPost, "/_sitemap/api/v1/rebuild"},
|
||||
}
|
||||
other := e.admin(t)
|
||||
for _, r := range routes {
|
||||
if rec := e.api(t, r.method, r.path, "", map[string]any{}); rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s %s without a token: %d %s", r.method, r.path, rec.Code, rec.Body.String())
|
||||
}
|
||||
if rec := e.api(t, r.method, r.path, other, map[string]any{}); rec.Code != http.StatusForbidden {
|
||||
t.Errorf("%s %s without the permission: %d %s", r.method, r.path, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
token := e.admin(t, permSitemap)
|
||||
data := e.apiJSON(t, http.StatusOK, http.MethodGet, "/_sitemap/api/v1/settings", token, nil)["data"].(map[string]any)
|
||||
got := fmt.Sprint(data)
|
||||
for _, want := range []string{"sitemap-posts-1.xml", "sitemap-images-1.xml", "https://api.example.test/sitemap.xml", "type:acme-post", "html_page_size:150", "indexnow:map["} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("GET settings lacks %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if types := data["types"].(map[string]any); types[postType] == nil || types[postType].(map[string]any)["slug"] != "posts" {
|
||||
t.Errorf("types = %v", data["types"])
|
||||
}
|
||||
|
||||
// Refusals name the fields and store nothing.
|
||||
for _, c := range []struct {
|
||||
body map[string]any
|
||||
field string
|
||||
}{
|
||||
{map[string]any{"site_url": "https://other.test", "file_size": 0}, "file_size"},
|
||||
{map[string]any{"html_page_size": 49}, "html_page_size"},
|
||||
{map[string]any{"html_paths": map[string]string{"pl": "mapa"}}, "html_paths"},
|
||||
{map[string]any{"news_sources": []string{"posts"}}, "news_sources"},
|
||||
{map[string]any{"types": map[string]any{postType: map[string]any{"slug": "Bad Slug"}}}, "types." + postType + ".slug"},
|
||||
{map[string]any{"types": map[string]any{"nope": map[string]any{"include": false}}}, "types.nope"},
|
||||
} {
|
||||
body := e.apiJSON(t, http.StatusUnprocessableEntity, http.MethodPut, "/_sitemap/api/v1/settings", token, c.body)
|
||||
errs, _ := body["errors"].(map[string]any)
|
||||
if body["error"] != "validation_failed" || errs[c.field] == nil {
|
||||
t.Errorf("%v: %v", c.body, body)
|
||||
}
|
||||
}
|
||||
if rec := e.api(t, http.MethodPut, "/_sitemap/api/v1/settings", token, map[string]any{"unknown_key": 1}); rec.Code != http.StatusUnprocessableEntity {
|
||||
t.Errorf("unknown key: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var site *string
|
||||
e.gdb.Raw(`SELECT site_url FROM golem15_sitemap_settings WHERE id = 1`).Scan(&site)
|
||||
if site != nil && *site != "" {
|
||||
t.Fatalf("a refused PUT stored site_url %q", *site)
|
||||
}
|
||||
|
||||
jobsBefore := e.riverJobs(t, classes.RebuildKind)
|
||||
data = e.apiJSON(t, http.StatusOK, http.MethodPut, "/_sitemap/api/v1/settings", token, map[string]any{
|
||||
"site_url": "https://www.example.test",
|
||||
"news_enabled": true,
|
||||
"news_sources": []string{"type:acme-post"},
|
||||
"indexnow_enabled": true,
|
||||
"html_paths": map[string]string{"en": "/site-map"},
|
||||
"html_page_size": 200,
|
||||
"types": map[string]any{postType: map[string]any{"slug": "blog", "titles": map[string]string{"en": "Blog"}}},
|
||||
})["data"].(map[string]any)
|
||||
settings := data["settings"].(map[string]any)
|
||||
paths := settings["html_paths"].(map[string]any)
|
||||
if settings["site_url"] != "https://www.example.test" || settings["news_enabled"] != true || settings["html_page_size"] != float64(200) || paths["en"] != "/site-map" || paths["pl"] != "/mapa-strony" {
|
||||
t.Fatalf("settings after PUT = %v", settings)
|
||||
}
|
||||
key := data["indexnow"].(map[string]any)["key"].(string)
|
||||
if len(key) != 32 || data["indexnow"].(map[string]any)["key_url"] != "https://www.example.test/"+key+".txt" {
|
||||
t.Fatalf("indexnow after PUT = %v", data["indexnow"])
|
||||
}
|
||||
if pt := data["types"].(map[string]any)[postType].(map[string]any); pt["slug"] != "blog" || pt["titles"].(map[string]any)["en"] != "Blog" {
|
||||
t.Fatalf("post type after PUT = %v", pt)
|
||||
}
|
||||
if n := e.riverJobs(t, classes.RebuildKind); n != jobsBefore+1 {
|
||||
t.Fatalf("PUT queued %d rebuilds", n-jobsBefore)
|
||||
}
|
||||
data = e.apiJSON(t, http.StatusOK, http.MethodPut, "/_sitemap/api/v1/settings", token, map[string]any{"indexnow_new_key": true})["data"].(map[string]any)
|
||||
if fresh := data["indexnow"].(map[string]any)["key"].(string); fresh == key || len(fresh) != 32 {
|
||||
t.Fatalf("new key = %q (old %q)", fresh, key)
|
||||
}
|
||||
|
||||
if body := e.apiJSON(t, http.StatusAccepted, http.MethodPost, "/_sitemap/api/v1/rebuild", token, nil); fmt.Sprint(body["data"]) != "map[queued:true]" {
|
||||
t.Fatalf("rebuild = %v", body)
|
||||
}
|
||||
if n := e.riverJobs(t, classes.RebuildKind); n != jobsBefore+3 {
|
||||
t.Fatalf("%d rebuilds queued, want %d", n, jobsBefore+3)
|
||||
}
|
||||
|
||||
// An MCP call: in-process, without a token, as the linked administrator.
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
_, principal, err := cabana.ResolveBackend(e.app, req)
|
||||
if err != nil || principal == nil {
|
||||
t.Fatalf("principal: %v", err)
|
||||
}
|
||||
mcpReq := httptest.NewRequest(http.MethodGet, "/_sitemap/api/v1/settings", nil)
|
||||
mcpReq.RemoteAddr = "198.51.100.7:4000"
|
||||
mcpReq = mcpReq.WithContext(bouncer.WithDispatch(mcpReq.Context(), &bouncer.Dispatch{Guards: []string{"backend"}, Principal: principal}))
|
||||
rec := httptest.NewRecorder()
|
||||
e.h.ServeHTTP(rec, mcpReq)
|
||||
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"settings"`) {
|
||||
t.Fatalf("MCP get_settings: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// The catalog.
|
||||
var tools []pact.MCPTool
|
||||
for _, p := range e.plugins {
|
||||
if m, ok := p.(pact.HasMCP); ok && p.ID() == "golem15.sitemap" {
|
||||
tools = m.MCP().Tools
|
||||
}
|
||||
}
|
||||
want := map[string]string{"get_settings": "GET /_sitemap/api/v1/settings", "update_settings": "PUT /_sitemap/api/v1/settings", "rebuild": "POST /_sitemap/api/v1/rebuild"}
|
||||
if len(tools) != len(want) {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
for _, tool := range tools {
|
||||
if want[tool.Name] != tool.Method+" "+tool.Path {
|
||||
t.Errorf("tool %s is bound to %s %s", tool.Name, tool.Method, tool.Path)
|
||||
}
|
||||
if tool.Guard != "backend" || len(tool.Permissions) != 1 || tool.Permissions[0] != permSitemap {
|
||||
t.Errorf("tool %s: guard %q permissions %v", tool.Name, tool.Guard, tool.Permissions)
|
||||
}
|
||||
}
|
||||
|
||||
// pier compiles the declarations against the registered routes.
|
||||
router, err := surf.BuildRouter(e.app, e.plugins)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.app.Config.Set(pier.EnabledKey, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pier.Mount(mountHost{r: router}, e.app, e.plugins); err != nil {
|
||||
t.Fatalf("pier.Mount: %v", err)
|
||||
}
|
||||
pr, ok := e.app.Lookup[*pier.Pier]()
|
||||
if !ok {
|
||||
t.Fatal("pier was not mounted")
|
||||
}
|
||||
if got := pr.ToolNames(); strings.Join(got, " ") != "sitemap_get_settings sitemap_rebuild sitemap_update_settings" {
|
||||
t.Fatalf("compiled tools %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// mountHost adapts a built surf router to pier.Host for a compile-only
|
||||
// mount.
|
||||
type mountHost struct{ r *surf.Router }
|
||||
|
||||
func (h mountHost) Routes() []pier.RouteInfo {
|
||||
var out []pier.RouteInfo
|
||||
for _, ri := range h.r.Routes() {
|
||||
out = append(out, pier.RouteInfo{Method: ri.Method, Pattern: ri.Pattern, PluginID: ri.PluginID, Middleware: ri.Middleware})
|
||||
}
|
||||
return out
|
||||
}
|
||||
func (h mountHost) Handler(method, pattern string) (http.Handler, bool) {
|
||||
return h.r.Handler(method, pattern)
|
||||
}
|
||||
func (mountHost) MountStream(string, string, http.Handler, ...string) error { return nil }
|
||||
func (mountHost) MountJSON(string, string, http.Handler) error { return nil }
|
||||
func (h mountHost) Owner(method, path string) (string, bool) {
|
||||
for _, ri := range h.r.Routes() {
|
||||
if ri.Method == method && ri.Pattern == path {
|
||||
return ri.PluginID, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user