sm-sitemap-plugin/routes.go

package sitemap

import (
	"bytes"
	"errors"
	"net/http"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
	apicontrollers "git.golem15.com/golem15/sm-sitemap-plugin/controllers/api"
	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/pact"
	"git.golem15.com/golem15/summercms/modules/surf"
	"gorm.io/gorm"
)

// The response headers of the sitemap routes.
const (
	contentTypeXML = "application/xml"
	cacheControl   = "no-cache, private"
)

// The throttle buckets: BucketSitemap of the sitemap files and the key
// file, BucketHTML of the HTML sitemap API.
const (
	BucketSitemap = "sitemap-xml"
	BucketHTML    = "sitemap-html"
)

// FilesPath is the route of the per-group files. The index lists them at
// the site root (sitemap-<slug>-<n>.xml); the site's SEO layer or proxy
// maps the root names to this route.
const FilesPath = "/_sitemap/files/"

// Routes mounts GET /sitemap.xml (the index), GET /_sitemap/files/{file}
// (one file) and GET /_sitemap/indexnow/{file} (the IndexNow key file)
// without the backend middleware: search engines fetch them anonymously.
// The sitemaps are served from the slot table, never generated per
// request, and every route is throttled per client IP (BucketSitemap). GET
// /_sitemap/api/v1/html and /_sitemap/api/v1/html/{group} answer the HTML
// sitemap's data (BucketHTML).
func (p *Plugin) Routes(r pact.Router) error {
	r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap)
	r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap)
	r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`)
	r.Get("/_sitemap/indexnow/{file}", apicontrollers.KeyFile(p.booted), "throttle:"+BucketSitemap)
	r.Where("file", apicontrollers.KeyFilePattern)
	// The HTML sitemap's data (D-34), read by the site's server when it
	// renders the HTML sitemap page; public, noindex, per-IP bucket
	// sitemap-html.
	r.Get("/_sitemap/api/v1/html", apicontrollers.HTMLOverview(p.booted), "throttle:"+BucketHTML)
	r.Get("/_sitemap/api/v1/html/{group}", apicontrollers.HTMLGroup(p.booted), "throttle:"+BucketHTML)
	r.Where("group", apicontrollers.GroupPattern)
	return nil
}

// booted returns the booted application.
func (p *Plugin) booted() *backpack.App { return p.app }

// Buckets registers sitemap-xml (60 requests a minute) and sitemap-html
// (300 a minute: the site's server renders every HTML sitemap page from
// it) per client IP, behind the configured trusted proxies. 429 bodies are
// surf's {"message":"Too Many Attempts."}.
func (p *Plugin) Buckets() map[string]surf.Bucket {
	trusted := surf.TrustedProxies(nil)
	if p.app != nil {
		trusted = surf.TrustedProxies(p.app.Config)
	}
	bucket := func(name string, max int) surf.Bucket {
		return surf.Bucket{
			Name:  name,
			Max:   max,
			Decay: time.Minute,
			Key: func(r *http.Request) string {
				return name + "|" + surf.ClientIP(r, trusted)
			},
		}
	}
	return map[string]surf.Bucket{
		BucketSitemap: bucket(BucketSitemap, 60),
		BucketHTML:    bucket(BucketHTML, 300),
	}
}

// indexRow is one file of the index query.
type indexRow struct {
	Grp     string     `gorm:"column:grp"`
	FileNo  int        `gorm:"column:file_no"`
	LastMod *time.Time `gorm:"column:lastmod"`
}

// sitemapIndex serves the sitemap index: one <sitemap> per file that lists
// at least one address, with the newest lastmod of its addresses. Without a
// site address (the setting, golem15.sitemap.site_url or app.url) it is a
// 404: a <loc> is never built from the request's Host.
func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) {
	db, err := database(p.app)
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	db = db.WithContext(r.Context())
	settings, err := classes.LoadSettings(db)
	if err != nil {
		pluginLog.Error("golem15.sitemap: load settings", "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	site := classes.SiteURL(p.app, settings)
	if site == "" {
		pluginLog.Warn("golem15.sitemap: no site address; set the sitemap setting, golem15.sitemap.site_url or app.url")
		http.NotFound(w, r)
		return
	}
	var rows []indexRow
	if err := db.Raw(`SELECT grp, file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
WHERE excluded_reason = '' AND file_no IS NOT NULL
GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil {
		pluginLog.Error("golem15.sitemap: index", "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	if settings.ImagesEnabled {
		var images []indexRow
		if err := db.Raw(`SELECT image_file_no AS file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
WHERE excluded_reason = '' AND image_file_no IS NOT NULL AND jsonb_array_length(images) > 0
GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil {
			pluginLog.Error("golem15.sitemap: image index", "error", err)
			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
			return
		}
		for i := range images {
			images[i].Grp = classes.ImagesGroup
		}
		rows = append(rows, images...)
	}
	files := make([]classes.IndexFile, 0, len(rows)+1)
	for _, row := range rows {
		files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod})
	}
	if settings.NewsEnabled {
		news, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC())
		if err != nil {
			pluginLog.Error("golem15.sitemap: news index", "error", err)
			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
			return
		}
		if len(news) > 0 {
			newest := news[0].Published
			files = append(files, classes.IndexFile{Loc: site + "/" + classes.NewsFile, LastMod: &newest})
		}
	}
	var buf bytes.Buffer
	if err := classes.WriteIndex(&buf, files); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	writeXML(w, buf.Bytes(), false)
}

// sitemapFile serves one file: its listed addresses in the order they were
// first seen. An unknown file, or one without a listed address, is a 404.
// sitemap-images-<n>.xml is an image sitemap file (imageFile).
func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) {
	if r.PathValue("file") == classes.NewsFile {
		p.newsFile(w, r)
		return
	}
	grp, n, ok := classes.ParseFileName(r.PathValue("file"))
	if !ok {
		http.NotFound(w, r)
		return
	}
	db, err := database(p.app)
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	db = db.WithContext(r.Context())
	settings, err := classes.LoadSettings(db)
	if err != nil {
		pluginLog.Error("golem15.sitemap: load settings", "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	site := classes.SiteURL(p.app, settings)
	if grp == classes.ImagesGroup {
		p.imageFile(w, r, db, settings, site, n)
		return
	}
	var rows []models.URL
	if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil {
		pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	entries := make([]classes.FileEntry, 0, len(rows))
	for _, row := range rows {
		loc := classes.Absolute(site, row.URL)
		if loc == "" {
			continue
		}
		entry := classes.FileEntry{Loc: loc, LastMod: row.LastMod}
		for _, a := range row.Alternates.Get() {
			if href := classes.Absolute(site, a.Href); href != "" {
				entry.Alternates = append(entry.Alternates, classes.FileAlternate{Hreflang: a.Hreflang, Href: href})
			}
		}
		entries = append(entries, entry)
	}
	if len(entries) == 0 {
		http.NotFound(w, r)
		return
	}
	var buf bytes.Buffer
	if err := classes.WriteURLSet(&buf, entries); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	writeXML(w, buf.Bytes(), true)
}

// imageFile serves image sitemap file n: each listed page with images
// numbered into it, in the order first seen, with its images (D-31). It is
// a 404 while the image sitemap is off, and for a file without such a page.
func (p *Plugin) imageFile(w http.ResponseWriter, r *http.Request, db *gorm.DB, settings models.Settings, site string, n int) {
	if !settings.ImagesEnabled {
		http.NotFound(w, r)
		return
	}
	var rows []models.URL
	if err := db.Where("image_file_no = ? AND excluded_reason = ''", n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil {
		pluginLog.Error("golem15.sitemap: image file", "file", n, "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	entries := make([]classes.ImageEntry, 0, len(rows))
	for _, row := range rows {
		loc := classes.Absolute(site, row.URL)
		if loc == "" {
			continue
		}
		entry := classes.ImageEntry{Loc: loc}
		for _, img := range row.Images.Get() {
			if abs := classes.Absolute(site, img); abs != "" {
				entry.Images = append(entry.Images, abs)
			}
		}
		if len(entry.Images) > 0 {
			entries = append(entries, entry)
		}
	}
	if len(entries) == 0 {
		http.NotFound(w, r)
		return
	}
	var buf bytes.Buffer
	if err := classes.WriteImageURLSet(&buf, entries); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	writeXML(w, buf.Bytes(), true)
}

// newsFile serves sitemap-news.xml (D-31): the articles of the last two
// days (classes.NewsEntries). It is a 404 while the News sitemap is off.
func (p *Plugin) newsFile(w http.ResponseWriter, r *http.Request) {
	db, err := database(p.app)
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	db = db.WithContext(r.Context())
	settings, err := classes.LoadSettings(db)
	if err != nil {
		pluginLog.Error("golem15.sitemap: load settings", "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	if !settings.NewsEnabled || classes.SiteURL(p.app, settings) == "" {
		http.NotFound(w, r)
		return
	}
	entries, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC())
	if err != nil {
		pluginLog.Error("golem15.sitemap: news file", "error", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	var buf bytes.Buffer
	if err := classes.WriteNewsURLSet(&buf, entries); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	writeXML(w, buf.Bytes(), true)
}

// writeXML answers 200 with an XML body. A file carries X-Robots-Tag
// noindex, so search engines read it without indexing it as a page.
func writeXML(w http.ResponseWriter, body []byte, noindex bool) {
	w.Header().Set("Content-Type", contentTypeXML)
	w.Header().Set("Cache-Control", cacheControl)
	if noindex {
		w.Header().Set("X-Robots-Tag", "noindex")
	}
	w.WriteHeader(http.StatusOK)
	_, _ = w.Write(body)
}

// database returns the app's published *gorm.DB.
func database(app *backpack.App) (*gorm.DB, error) {
	if app == nil {
		return nil, errors.New("golem15.sitemap: app is not booted")
	}
	db, ok := app.Lookup[*gorm.DB]()
	if !ok || db == nil {
		return nil, errors.New("golem15.sitemap: database is not configured")
	}
	return db, nil
}