package sitemap import ( "bytes" "errors" "net/http" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) // The response headers of the sitemap routes. const ( contentTypeXML = "application/xml" cacheControl = "no-cache, private" ) // BucketSitemap is the throttle bucket of the sitemap routes. const BucketSitemap = "sitemap-xml" // FilesPath is the route of the per-group files. The index lists them at // the site root (sitemap--.xml); the site's SEO layer or proxy // maps the root names to this route. const FilesPath = "/_sitemap/files/" // Routes mounts GET /sitemap.xml (the index) and GET // /_sitemap/files/{file} (one file) without the backend middleware: search // engines fetch them anonymously. Both are served from the slot table, never // generated per request, and throttled per client IP (BucketSitemap). func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap) r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap) r.Where("file", `sitemap-[a-z0-9-]+-[0-9]+\.xml`) return nil } // Buckets registers sitemap-xml: 60 requests a minute per client IP (behind // the configured trusted proxies). 429 bodies are surf's // {"message":"Too Many Attempts."}. func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { trusted = surf.TrustedProxies(p.app.Config) } return map[string]surf.Bucket{ BucketSitemap: { Name: BucketSitemap, Max: 60, Decay: time.Minute, Key: func(r *http.Request) string { return "sitemap-xml|" + surf.ClientIP(r, trusted) }, }, } } // indexRow is one file of the index query. type indexRow struct { Grp string `gorm:"column:grp"` FileNo int `gorm:"column:file_no"` LastMod *time.Time `gorm:"column:lastmod"` } // sitemapIndex serves the sitemap index: one per file that lists // at least one address, with the newest lastmod of its addresses. Without a // site address (the setting, golem15.sitemap.site_url or app.url) it is a // 404: a is never built from the request's Host. func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) { db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } site := classes.SiteURL(p.app, settings) if site == "" { pluginLog.Warn("golem15.sitemap: no site address; set the sitemap setting, golem15.sitemap.site_url or app.url") http.NotFound(w, r) return } var rows []indexRow if err := db.Raw(`SELECT grp, file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls WHERE excluded_reason = '' AND file_no IS NOT NULL GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: index", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } files := make([]classes.IndexFile, 0, len(rows)) for _, row := range rows { files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod}) } var buf bytes.Buffer if err := classes.WriteIndex(&buf, files); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), false) } // sitemapFile serves one file: its listed addresses in the order they were // first seen. An unknown file, or one without a listed address, is a 404. func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { grp, n, ok := classes.ParseFileName(r.PathValue("file")) if !ok { http.NotFound(w, r) return } db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } db = db.WithContext(r.Context()) settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } site := classes.SiteURL(p.app, settings) var rows []models.URL if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } entries := make([]classes.FileEntry, 0, len(rows)) for _, row := range rows { loc := classes.Absolute(site, row.URL) if loc == "" { continue } entry := classes.FileEntry{Loc: loc, LastMod: row.LastMod} for _, a := range row.Alternates.Get() { if href := classes.Absolute(site, a.Href); href != "" { entry.Alternates = append(entry.Alternates, classes.FileAlternate{Hreflang: a.Hreflang, Href: href}) } } entries = append(entries, entry) } if len(entries) == 0 { http.NotFound(w, r) return } var buf bytes.Buffer if err := classes.WriteURLSet(&buf, entries); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } writeXML(w, buf.Bytes(), true) } // writeXML answers 200 with an XML body. A file carries X-Robots-Tag // noindex, so search engines read it without indexing it as a page. func writeXML(w http.ResponseWriter, body []byte, noindex bool) { w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) if noindex { w.Header().Set("X-Robots-Tag", "noindex") } w.WriteHeader(http.StatusOK) _, _ = w.Write(body) } // database returns the app's published *gorm.DB. func database(app *backpack.App) (*gorm.DB, error) { if app == nil { return nil, errors.New("golem15.sitemap: app is not booted") } db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return nil, errors.New("golem15.sitemap: database is not configured") } return db, nil }