package seo import ( "net/http" "time" "git.golem15.com/golem15/sm-seo-plugin/classes" apicontrollers "git.golem15.com/golem15/sm-seo-plugin/controllers/api" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/surf" ) // The throttle buckets of the public endpoints. const ( // BucketResolve is the resolve endpoint's bucket. BucketResolve = "seo-resolve" // BucketNotFound is the 404 report endpoint's bucket. BucketNotFound = "seo-notfound" // BucketRobots is the /robots.txt bucket. BucketRobots = "seo-robots" // BucketAPI is the management API's bucket (P-24). BucketAPI = "seo-api" ) // Routes mounts GET /robots.txt and GET /llms.txt at the site root, the public /_seo/api/v1 // endpoints and the management API. The public ones carry no backend // middleware: crawlers and the frontend server call them anonymously (the // 404 report with its own token). The management routes (bucket seo-api) // resolve the administrator themselves from a backend token or the MCP // dispatch and require golem15.seo.manage (D-10, D-15); each one is an MCP // tool (mcp.go). Every /_seo/api/v1 answer carries X-Robots-Tag: noindex. func (p *Plugin) Routes(r pact.Router) error { r.Get("/robots.txt", apicontrollers.Robots(p.Service), "throttle:"+BucketRobots) r.Get("/llms.txt", apicontrollers.LLMS(p.Service), "throttle:"+BucketRobots) r.Group("/_seo/api/v1", nil, func(g pact.Router) { g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve) g.Get("/site", noindex(apicontrollers.Site(p.Service)), "throttle:"+BucketResolve) g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound) // The frontend server's route report (D-33), token-authenticated // like the 404 report. g.Post("/registry/routes", noindex(apicontrollers.RouteReport(p.Service)), "throttle:"+BucketNotFound) // Verification files are served here only; the frontend maps them // at the site root (D-103). A root route would capture every // one-segment page of a site served at GET /. g.Get("/verification/{name}", noindex(apicontrollers.Verification(p.Service)), "throttle:"+BucketResolve) g.Where("name", classes.VerificationNamePattern) }) r.Group("/_seo/api/v1", nil, func(g pact.Router) { api := "throttle:" + BucketAPI g.Get("/seo", apicontrollers.GetSEO(p.Service), api) g.Put("/seo", apicontrollers.SetSEO(p.Service), api) g.Get("/graph", apicontrollers.Graph(p.Service), api) g.Get("/links", apicontrollers.Links(p.Service), api) g.Get("/rules", apicontrollers.ListRules(p.Service), api) g.Post("/rules", apicontrollers.CreateRule(p.Service), api) g.Get("/rules/{id}", apicontrollers.ShowRule(p.Service), api) g.Put("/rules/{id}", apicontrollers.UpdateRule(p.Service), api) g.Delete("/rules/{id}", apicontrollers.DeleteRule(p.Service), api) g.Get("/redirects", apicontrollers.ListRedirects(p.Service), api) g.Post("/redirects", apicontrollers.CreateRedirect(p.Service), api) g.Get("/redirects/{id}", apicontrollers.ShowRedirect(p.Service), api) g.Put("/redirects/{id}", apicontrollers.UpdateRedirect(p.Service), api) g.Delete("/redirects/{id}", apicontrollers.DeleteRedirect(p.Service), api) g.Get("/pages", apicontrollers.ListPages(p.Service), api) g.Post("/pages", apicontrollers.CreatePage(p.Service), api) g.Put("/pages/{id}", apicontrollers.UpdatePage(p.Service), api) g.Delete("/pages/{id}", apicontrollers.DeletePage(p.Service), api) g.Get("/autolinks", apicontrollers.ListAutolinks(p.Service), api) g.Post("/autolinks", apicontrollers.CreateAutolink(p.Service), api) g.Put("/autolinks/{id}", apicontrollers.UpdateAutolink(p.Service), api) g.Delete("/autolinks", apicontrollers.DeleteAutolinks(p.Service), api) g.Post("/autolinks/from-keyphrases", apicontrollers.AutolinksFromKeyphrases(p.Service), api) g.Get("/notfound", apicontrollers.ListNotFound(p.Service), api) g.Post("/notfound/{id}/redirect", apicontrollers.RedirectNotFound(p.Service), api) g.Post("/notfound/{id}/gone", apicontrollers.GoneNotFound(p.Service), api) g.Delete("/notfound/{id}", apicontrollers.DeleteNotFound(p.Service), api) g.Get("/robots", apicontrollers.ShowRobots(p.Service), api) g.Put("/robots", apicontrollers.UpdateRobots(p.Service), api) g.Get("/robots/test", apicontrollers.TestRobotsURL(p.Service), api) g.Get("/settings", apicontrollers.ShowSettings(p.Service), api) g.Put("/settings", apicontrollers.UpdateSettings(p.Service), api) g.Get("/history", apicontrollers.ListHistory(p.Service), api) g.Get("/history/{id}", apicontrollers.ShowHistory(p.Service), api) g.Post("/history/{id}/undo", apicontrollers.UndoChange(p.Service), api) g.Post("/history/batches/{batch}/undo", apicontrollers.UndoBatch(p.Service), api) g.Get("/export", apicontrollers.ExportCSV(p.Service), api) g.Post("/import", apicontrollers.ImportCSV(p.Service), api) g.Post("/batch", apicontrollers.ApplyBatch(p.Service), api) g.Get("/issues", apicontrollers.ListIssues(p.Service), api) g.Post("/issues/ignore", apicontrollers.IgnoreIssues(p.Service), api) g.Post("/issues/unignore", apicontrollers.UnignoreIssues(p.Service), api) g.Post("/issues/{id}/recheck", apicontrollers.RecheckIssue(p.Service), api) g.Get("/crawls", apicontrollers.ListCrawls(p.Service), api) g.Post("/crawls", apicontrollers.StartCrawl(p.Service), api) g.Delete("/crawls", apicontrollers.DeleteCrawls(p.Service), api) g.Get("/crawls/{id}", apicontrollers.ShowCrawl(p.Service), api) g.Post("/crawls/{id}/cancel", apicontrollers.CancelCrawl(p.Service), api) }) return nil } // noindex adds X-Robots-Tag: noindex to every answer of h. func noindex(h http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Robots-Tag", "noindex") h(w, r) } } // Buckets registers the per-client-IP buckets behind the configured // trusted proxies: seo-resolve 600 a minute (the frontend server resolves // every page it renders), seo-notfound 600 a minute (one report per 404 // page), seo-robots 120 a minute and seo-api 300 a minute (an MCP agent // working through an audit). The resolve calls and 404 reports come from // the frontend server, so that server must be a trusted proxy forwarding // X-Forwarded-For, or the first two buckets cap the whole site (README, // "Behind the frontend server"). func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { trusted = surf.TrustedProxies(p.app.Config) } bucket := func(name string, max int) surf.Bucket { return surf.Bucket{ Name: name, Max: max, Decay: time.Minute, Key: func(r *http.Request) string { return name + "|" + surf.ClientIP(r, trusted) }, } } return map[string]surf.Bucket{ BucketResolve: bucket(BucketResolve, 600), BucketNotFound: bucket(BucketNotFound, 600), BucketRobots: bucket(BucketRobots, 120), BucketAPI: bucket(BucketAPI, 300), } }