package sitemap import ( "bytes" "errors" "net/http" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" apicontrollers "git.golem15.com/golem15/sm-sitemap-plugin/controllers/api" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) // The response headers of the sitemap routes. const ( contentTypeXML = "application/xml" cacheControl = "no-cache, private" ) // The throttle buckets: BucketSitemap of the sitemap files and the key // file, BucketHTML of the HTML sitemap API, BucketAPI of the management // API. const ( BucketSitemap = "sitemap-xml" BucketHTML = "sitemap-html" BucketAPI = "sitemap-api" ) // FilesPath is the route of the per-group files. The index lists them at // the site root (sitemap--.xml); the site's SEO layer or proxy // maps the root names to this route. const FilesPath = "/_sitemap/files/" // Routes mounts GET /sitemap.xml (the index), GET /_sitemap/files/{file} // (one file) and GET /_sitemap/indexnow/{file} (the IndexNow key file) // without the backend middleware: search engines fetch them anonymously. // The sitemaps are served from the slot table, never generated per // request, and every route is throttled per client IP (BucketSitemap). GET // /_sitemap/api/v1/html and /_sitemap/api/v1/html/{group} answer the HTML // sitemap's data (BucketHTML). func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap) r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap) r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`) r.Get("/_sitemap/indexnow/{file}", apicontrollers.KeyFile(p.booted), "throttle:"+BucketSitemap) r.Where("file", apicontrollers.KeyFilePattern) // The HTML sitemap's data (D-34), read by the site's server when it // renders the HTML sitemap page; public, noindex, per-IP bucket // sitemap-html. r.Get("/_sitemap/api/v1/html", apicontrollers.HTMLOverview(p.booted), "throttle:"+BucketHTML) r.Get("/_sitemap/api/v1/html/{group}", apicontrollers.HTMLGroup(p.booted), "throttle:"+BucketHTML) r.Where("group", apicontrollers.GroupPattern) // The management API (D-10): each route resolves the administrator // from a backend token, a personal API token or the MCP dispatch and // requires the sitemap permission; each one is an MCP tool (mcp.go). r.Group("/_sitemap/api/v1", nil, func(g pact.Router) { api := "throttle:" + BucketAPI g.Get("/settings", apicontrollers.ShowSettings(p.booted), api) g.Put("/settings", apicontrollers.UpdateSettings(p.booted), api) g.Post("/rebuild", apicontrollers.Rebuild(p.booted), api) }) return nil } // booted returns the booted application. func (p *Plugin) booted() *backpack.App { return p.app } // Buckets registers sitemap-xml (60 requests a minute), sitemap-html (300 // a minute: the site's server renders every HTML sitemap page from it) and // sitemap-api (300 a minute) per client IP, behind the configured trusted // proxies. 429 bodies are // surf's {"message":"Too Many Attempts."}. func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { trusted = surf.TrustedProxies(p.app.Config) } bucket := func(name string, max int) surf.Bucket { return surf.Bucket{ Name: name, Max: max, Decay: time.Minute, Key: func(r *http.Request) string { return name + "|" + surf.ClientIP(r, trusted) }, } } return map[string]surf.Bucket{ BucketSitemap: bucket(BucketSitemap, 60), BucketHTML: bucket(BucketHTML, 300), BucketAPI: bucket(BucketAPI, 300), } } // sitemapIndex serves the sitemap index: one per file that lists // at least one address (classes.SitemapFiles: the per-group files, the // image files and the News file), with the newest lastmod of its // addresses. Without a // site address (the setting, golem15.sitemap.site_url or app.url) it is a // 404: a is never built from the request's Host. func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) { db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } site := classes.SiteURL(p.app, settings) if site == "" { pluginLog.Warn("golem15.sitemap: no site address; set the sitemap setting, golem15.sitemap.site_url or app.url") http.NotFound(w, r) return } list, err := classes.SitemapFiles(db, p.app, settings, time.Now().UTC()) if err != nil { pluginLog.Error("golem15.sitemap: index", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } files := make([]classes.IndexFile, 0, len(list)) for _, f := range list { files = append(files, classes.IndexFile{Loc: site + "/" + f.Name, LastMod: f.LastMod}) } var buf bytes.Buffer if err := classes.WriteIndex(&buf, files); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), false) } // sitemapFile serves one file: its listed addresses in the order they were // first seen. An unknown file, or one without a listed address, is a 404. // sitemap-images-.xml is an image sitemap file (imageFile). func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { if r.PathValue("file") == classes.NewsFile { p.newsFile(w, r) return } grp, n, ok := classes.ParseFileName(r.PathValue("file")) if !ok { http.NotFound(w, r) return } db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } site := classes.SiteURL(p.app, settings) if grp == classes.ImagesGroup { p.imageFile(w, r, db, settings, site, n) return } var rows []models.URL if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } entries := make([]classes.FileEntry, 0, len(rows)) for _, row := range rows { loc := classes.Absolute(site, row.URL) if loc == "" { continue } entry := classes.FileEntry{Loc: loc, LastMod: row.LastMod} for _, a := range row.Alternates.Get() { if href := classes.Absolute(site, a.Href); href != "" { entry.Alternates = append(entry.Alternates, classes.FileAlternate{Hreflang: a.Hreflang, Href: href}) } } entries = append(entries, entry) } if len(entries) == 0 { http.NotFound(w, r) return } var buf bytes.Buffer if err := classes.WriteURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // imageFile serves image sitemap file n: each listed page with images // numbered into it, in the order first seen, with its images (D-31). It is // a 404 while the image sitemap is off, and for a file without such a page. func (p *Plugin) imageFile(w http.ResponseWriter, r *http.Request, db *gorm.DB, settings models.Settings, site string, n int) { if !settings.ImagesEnabled { http.NotFound(w, r) return } var rows []models.URL if err := db.Where("image_file_no = ? AND excluded_reason = ''", n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: image file", "file", n, "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } entries := make([]classes.ImageEntry, 0, len(rows)) for _, row := range rows { loc := classes.Absolute(site, row.URL) if loc == "" { continue } entry := classes.ImageEntry{Loc: loc} for _, img := range row.Images.Get() { if abs := classes.Absolute(site, img); abs != "" { entry.Images = append(entry.Images, abs) } } if len(entry.Images) > 0 { entries = append(entries, entry) } } if len(entries) == 0 { http.NotFound(w, r) return } var buf bytes.Buffer if err := classes.WriteImageURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // newsFile serves sitemap-news.xml (D-31): the articles of the last two // days (classes.NewsEntries). It is a 404 while the News sitemap is off. func (p *Plugin) newsFile(w http.ResponseWriter, r *http.Request) { db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } if !settings.NewsEnabled || classes.SiteURL(p.app, settings) == "" { http.NotFound(w, r) return } entries, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC()) if err != nil { pluginLog.Error("golem15.sitemap: news file", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } var buf bytes.Buffer if err := classes.WriteNewsURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // writeXML answers 200 with an XML body. A file carries X-Robots-Tag // noindex, so search engines read it without indexing it as a page. func writeXML(w http.ResponseWriter, body []byte, noindex bool) { w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) if noindex { w.Header().Set("X-Robots-Tag", "noindex") } w.WriteHeader(http.StatusOK) _, _ = w.Write(body) } // database returns the app's published *gorm.DB. func database(app *backpack.App) (*gorm.DB, error) { if app == nil { return nil, errors.New("golem15.sitemap: app is not booted") } db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return nil, errors.New("golem15.sitemap: database is not configured") } return db, nil }