package sitemap import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "log/slog" "net" "net/http" "net/netip" "strings" "sync" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/signpost" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) // The response headers of the PHP routes (live responses). const ( contentTypeXML = "application/xml" contentTypeXSL = "text/xsl; charset=UTF-8" cacheControl = "no-cache, private" ) // BucketSitemap is the throttle bucket of GET /sitemap.xml. const BucketSitemap = "sitemap-xml" // CacheConfigKey is the config key of the number of seconds a generated // sitemap is served from memory; 0 turns the cache off. const CacheConfigKey = "golem15.sitemap.cache_seconds" // cacheMaxEntries bounds the cache: one entry per root URL (app.url, or the // request's scheme and Host when app.url is not set) and stored definition // and settings. A full cache is emptied. const cacheMaxEntries = 16 // Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application // root without the backend middleware: search engines and browsers fetch // them anonymously. /sitemap.xml resolves every registry item, so it is // throttled per client IP (BucketSitemap) and its output is cached. func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapXML, "throttle:"+BucketSitemap) r.Get("/sitemap.xsl", p.sitemapXSL) return nil } // Buckets registers sitemap-xml: 60 requests a minute per client IP (behind // the configured trusted proxies). 429 bodies are surf's // {"message":"Too Many Attempts."}. func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { trusted = surf.TrustedProxies(p.app.Config) } return map[string]surf.Bucket{ BucketSitemap: { Name: BucketSitemap, Max: 60, Decay: time.Minute, Key: func(r *http.Request) string { return "sitemap-xml|" + surf.ClientIP(r, trusted) }, }, } } // sitemapCache holds generated sitemaps for the configured time. The key // covers the root URL and the stored definition and settings, so a save of // either is served at once; registry content (a new post) shows after the // entry expires. gen lets one generation run at a time, so concurrent misses // wait for it instead of each resolving every registry item. type sitemapCache struct { ttl time.Duration gen sync.Mutex mu sync.Mutex entries map[string]cachedSitemap } type cachedSitemap struct { xml string expires time.Time } func newSitemapCache(ttl time.Duration) *sitemapCache { return &sitemapCache{ttl: ttl, entries: map[string]cachedSitemap{}} } func (c *sitemapCache) get(key string) (string, bool) { if c == nil || c.ttl <= 0 || key == "" { return "", false } c.mu.Lock() defer c.mu.Unlock() e, ok := c.entries[key] if !ok || time.Now().After(e.expires) { return "", false } return e.xml, true } func (c *sitemapCache) put(key, xml string) { if c == nil || c.ttl <= 0 || key == "" { return } c.mu.Lock() defer c.mu.Unlock() if len(c.entries) >= cacheMaxEntries { clear(c.entries) } c.entries[key] = cachedSitemap{xml: xml, expires: time.Now().Add(c.ttl)} } // cacheKey is a digest of the root URL, the definition's items and the // settings; "" (no caching) when either does not encode. func cacheKey(root string, def models.Definition, settings models.Settings) string { items, err := json.Marshal(def.Items()) if err != nil { return "" } set, err := json.Marshal(settings) if err != nil { return "" } h := sha256.New() for _, part := range [][]byte{[]byte(root), items, set} { h.Write(part) h.Write([]byte{0}) } return hex.EncodeToString(h.Sum(nil)) } // sitemapXSL serves the embedded stylesheet, the PHP file byte for byte. func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", contentTypeXSL) w.Header().Set("Cache-Control", cacheControl) w.WriteHeader(http.StatusOK) _, _ = w.Write(sitemapXSL) } // sitemapXML serves the definition with the fixed theme key: 404 when there // is none, an empty application/xml body when it has no items, else the // generated XML with the settings' frontend_url rewrite applied. func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) { app := p.app db, err := database(app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } var defs []models.Definition if err := db.WithContext(r.Context()).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { slog.Default().Error("golem15.sitemap: load definition", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } if len(defs) == 0 { slog.Default().Info("golem15.sitemap: No sitemap definition was found. Try creating one first.") http.NotFound(w, r) return } settings, err := loadSettings(db.WithContext(r.Context())) if err != nil { slog.Default().Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } root := backendURL(app, r) xml, err := p.generate(r, root, defs[0], settings) if err != nil { slog.Default().Error("golem15.sitemap: generate", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(xml)) } // generate returns the sitemap of def with the frontend_url rewrite // applied, from the cache when it holds it. func (p *Plugin) generate(r *http.Request, root string, def models.Definition, settings models.Settings) (string, error) { cache := p.cache key := "" if cache != nil && cache.ttl > 0 { key = cacheKey(root, def, settings) if xml, ok := cache.get(key); ok { return xml, nil } cache.gen.Lock() defer cache.gen.Unlock() if xml, ok := cache.get(key); ok { return xml, nil } } app := p.app gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events} if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil { gen.Resolver = reg } xml, err := gen.Generate(r.Context(), def.Items()) if err != nil { return "", err } xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL)) cache.put(key, xml) return xml, nil } // loadSettings reads the settings row; without one the PHP defaults apply. func loadSettings(db *gorm.DB) (models.Settings, error) { var rows []models.Settings if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { return models.Settings{}, err } if len(rows) == 0 { return models.DefaultSettings(), nil } return rows[0], nil } // database returns the app's published *gorm.DB. func database(app *backpack.App) (*gorm.DB, error) { if app == nil { return nil, errors.New("golem15.sitemap: app is not booted") } db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return nil, errors.New("golem15.sitemap: database is not configured") } return db, nil } // backendURL is the root URL the sitemap's absolute URLs start with, PHP // url('/') right-trimmed of slashes: app.url when it is set, else the // request's scheme and Host. The scheme is https for a TLS connection, or // when the immediate peer is in http.trusted_proxies and its last // X-Forwarded-Proto value says https; a client that sends the header // directly is ignored. func backendURL(app *backpack.App, r *http.Request) string { if app != nil && app.Config != nil { if u := strings.TrimRight(strings.TrimSpace(app.Config.String("app.url")), "/"); u != "" { return u } } scheme := "http" if isHTTPS(app, r) { scheme = "https" } return strings.TrimRight(scheme+"://"+r.Host, "/") } func isHTTPS(app *backpack.App, r *http.Request) bool { if r.TLS != nil { return true } if app == nil || app.Config == nil { return false } trusted := surf.TrustedProxies(app.Config) if len(trusted) == 0 || !peerTrusted(r.RemoteAddr, trusted) { return false } values := r.Header.Values("X-Forwarded-Proto") if len(values) == 0 { return false } hops := strings.Split(values[len(values)-1], ",") return strings.EqualFold(strings.TrimSpace(hops[len(hops)-1]), "https") } func peerTrusted(remoteAddr string, trusted []netip.Prefix) bool { host, _, err := net.SplitHostPort(remoteAddr) if err != nil { host = remoteAddr } addr, err := netip.ParseAddr(strings.Trim(host, "[]")) if err != nil { return false } addr = addr.Unmap() for _, p := range trusted { if p.Contains(addr) { return true } } return false }