package sitemap import ( "bytes" "errors" "net/http" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" apicontrollers "git.golem15.com/golem15/sm-sitemap-plugin/controllers/api" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) // The response headers of the sitemap routes. const ( contentTypeXML = "application/xml" cacheControl = "no-cache, private" ) // The throttle buckets: BucketSitemap of the sitemap files and the key // file, BucketHTML of the HTML sitemap API. const ( BucketSitemap = "sitemap-xml" BucketHTML = "sitemap-html" ) // FilesPath is the route of the per-group files. The index lists them at // the site root (sitemap--.xml); the site's SEO layer or proxy // maps the root names to this route. const FilesPath = "/_sitemap/files/" // Routes mounts GET /sitemap.xml (the index), GET /_sitemap/files/{file} // (one file) and GET /_sitemap/indexnow/{file} (the IndexNow key file) // without the backend middleware: search engines fetch them anonymously. // The sitemaps are served from the slot table, never generated per // request, and every route is throttled per client IP (BucketSitemap). GET // /_sitemap/api/v1/html and /_sitemap/api/v1/html/{group} answer the HTML // sitemap's data (BucketHTML). func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap) r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap) r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`) r.Get("/_sitemap/indexnow/{file}", apicontrollers.KeyFile(p.booted), "throttle:"+BucketSitemap) r.Where("file", apicontrollers.KeyFilePattern) // The HTML sitemap's data (D-34), read by the site's server when it // renders the HTML sitemap page; public, noindex, per-IP bucket // sitemap-html. r.Get("/_sitemap/api/v1/html", apicontrollers.HTMLOverview(p.booted), "throttle:"+BucketHTML) r.Get("/_sitemap/api/v1/html/{group}", apicontrollers.HTMLGroup(p.booted), "throttle:"+BucketHTML) r.Where("group", apicontrollers.GroupPattern) return nil } // booted returns the booted application. func (p *Plugin) booted() *backpack.App { return p.app } // Buckets registers sitemap-xml (60 requests a minute) and sitemap-html // (300 a minute: the site's server renders every HTML sitemap page from // it) per client IP, behind the configured trusted proxies. 429 bodies are // surf's {"message":"Too Many Attempts."}. func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { trusted = surf.TrustedProxies(p.app.Config) } bucket := func(name string, max int) surf.Bucket { return surf.Bucket{ Name: name, Max: max, Decay: time.Minute, Key: func(r *http.Request) string { return name + "|" + surf.ClientIP(r, trusted) }, } } return map[string]surf.Bucket{ BucketSitemap: bucket(BucketSitemap, 60), BucketHTML: bucket(BucketHTML, 300), } } // indexRow is one file of the index query. type indexRow struct { Grp string `gorm:"column:grp"` FileNo int `gorm:"column:file_no"` LastMod *time.Time `gorm:"column:lastmod"` } // sitemapIndex serves the sitemap index: one per file that lists // at least one address, with the newest lastmod of its addresses. Without a // site address (the setting, golem15.sitemap.site_url or app.url) it is a // 404: a is never built from the request's Host. func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) { db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } site := classes.SiteURL(p.app, settings) if site == "" { pluginLog.Warn("golem15.sitemap: no site address; set the sitemap setting, golem15.sitemap.site_url or app.url") http.NotFound(w, r) return } var rows []indexRow if err := db.Raw(`SELECT grp, file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls WHERE excluded_reason = '' AND file_no IS NOT NULL GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: index", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } if settings.ImagesEnabled { var images []indexRow if err := db.Raw(`SELECT image_file_no AS file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls WHERE excluded_reason = '' AND image_file_no IS NOT NULL AND jsonb_array_length(images) > 0 GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil { pluginLog.Error("golem15.sitemap: image index", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } for i := range images { images[i].Grp = classes.ImagesGroup } rows = append(rows, images...) } files := make([]classes.IndexFile, 0, len(rows)+1) for _, row := range rows { files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod}) } if settings.NewsEnabled { news, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC()) if err != nil { pluginLog.Error("golem15.sitemap: news index", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } if len(news) > 0 { newest := news[0].Published files = append(files, classes.IndexFile{Loc: site + "/" + classes.NewsFile, LastMod: &newest}) } } var buf bytes.Buffer if err := classes.WriteIndex(&buf, files); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), false) } // sitemapFile serves one file: its listed addresses in the order they were // first seen. An unknown file, or one without a listed address, is a 404. // sitemap-images-.xml is an image sitemap file (imageFile). func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { if r.PathValue("file") == classes.NewsFile { p.newsFile(w, r) return } grp, n, ok := classes.ParseFileName(r.PathValue("file")) if !ok { http.NotFound(w, r) return } db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } site := classes.SiteURL(p.app, settings) if grp == classes.ImagesGroup { p.imageFile(w, r, db, settings, site, n) return } var rows []models.URL if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } entries := make([]classes.FileEntry, 0, len(rows)) for _, row := range rows { loc := classes.Absolute(site, row.URL) if loc == "" { continue } entry := classes.FileEntry{Loc: loc, LastMod: row.LastMod} for _, a := range row.Alternates.Get() { if href := classes.Absolute(site, a.Href); href != "" { entry.Alternates = append(entry.Alternates, classes.FileAlternate{Hreflang: a.Hreflang, Href: href}) } } entries = append(entries, entry) } if len(entries) == 0 { http.NotFound(w, r) return } var buf bytes.Buffer if err := classes.WriteURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // imageFile serves image sitemap file n: each listed page with images // numbered into it, in the order first seen, with its images (D-31). It is // a 404 while the image sitemap is off, and for a file without such a page. func (p *Plugin) imageFile(w http.ResponseWriter, r *http.Request, db *gorm.DB, settings models.Settings, site string, n int) { if !settings.ImagesEnabled { http.NotFound(w, r) return } var rows []models.URL if err := db.Where("image_file_no = ? AND excluded_reason = ''", n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: image file", "file", n, "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } entries := make([]classes.ImageEntry, 0, len(rows)) for _, row := range rows { loc := classes.Absolute(site, row.URL) if loc == "" { continue } entry := classes.ImageEntry{Loc: loc} for _, img := range row.Images.Get() { if abs := classes.Absolute(site, img); abs != "" { entry.Images = append(entry.Images, abs) } } if len(entry.Images) > 0 { entries = append(entries, entry) } } if len(entries) == 0 { http.NotFound(w, r) return } var buf bytes.Buffer if err := classes.WriteImageURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // newsFile serves sitemap-news.xml (D-31): the articles of the last two // days (classes.NewsEntries). It is a 404 while the News sitemap is off. func (p *Plugin) newsFile(w http.ResponseWriter, r *http.Request) { db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } if !settings.NewsEnabled || classes.SiteURL(p.app, settings) == "" { http.NotFound(w, r) return } entries, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC()) if err != nil { pluginLog.Error("golem15.sitemap: news file", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } var buf bytes.Buffer if err := classes.WriteNewsURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // writeXML answers 200 with an XML body. A file carries X-Robots-Tag // noindex, so search engines read it without indexing it as a page. func writeXML(w http.ResponseWriter, body []byte, noindex bool) { w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) if noindex { w.Header().Set("X-Robots-Tag", "noindex") } w.WriteHeader(http.StatusOK) _, _ = w.Write(body) } // database returns the app's published *gorm.DB. func database(app *backpack.App) (*gorm.DB, error) { if app == nil { return nil, errors.New("golem15.sitemap: app is not booted") } db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return nil, errors.New("golem15.sitemap: database is not configured") } return db, nil }