c4f0b4f94c
- golem15.seo.issues (Site audit): current issues errors first, labels, summary and fix columns, filters, P7 issue_details, Check this address again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged - golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one status callout, staging and rendered-comparison lines, stats strip and per-kind table; Cancel; a running crawl refuses delete - Start crawl toolbar action on both, refusing already_running and no_site_url; navigation Site audit (gauge) and Crawls (globe) - routes and MCP tools start/cancel/list/get/delete crawls, list_issues with filters (open by default), recheck_url, ignore and unignore - AdminOperations maps both controllers; toolbar actions get their own operation kind - TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels
134 lines
6.4 KiB
Go
134 lines
6.4 KiB
Go
package seo
|
|
|
|
import (
|
|
"net/http"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
|
apicontrollers "git.golem15.com/golem15/sm-seo-plugin/controllers/api"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/surf"
|
|
)
|
|
|
|
// The throttle buckets of the public endpoints.
|
|
const (
|
|
// BucketResolve is the resolve endpoint's bucket.
|
|
BucketResolve = "seo-resolve"
|
|
// BucketNotFound is the 404 report endpoint's bucket.
|
|
BucketNotFound = "seo-notfound"
|
|
// BucketRobots is the /robots.txt bucket.
|
|
BucketRobots = "seo-robots"
|
|
// BucketAPI is the management API's bucket (P-24).
|
|
BucketAPI = "seo-api"
|
|
)
|
|
|
|
// Routes mounts GET /robots.txt and GET /llms.txt at the site root, the public /_seo/api/v1
|
|
// endpoints and the management API. The public ones carry no backend
|
|
// middleware: crawlers and the frontend server call them anonymously (the
|
|
// 404 report with its own token). The management routes (bucket seo-api)
|
|
// resolve the administrator themselves from a backend token or the MCP
|
|
// dispatch and require golem15.seo.manage (D-10, D-15); each one is an MCP
|
|
// tool (mcp.go). Every /_seo/api/v1 answer carries X-Robots-Tag: noindex.
|
|
func (p *Plugin) Routes(r pact.Router) error {
|
|
r.Get("/robots.txt", apicontrollers.Robots(p.Service), "throttle:"+BucketRobots)
|
|
r.Get("/llms.txt", apicontrollers.LLMS(p.Service), "throttle:"+BucketRobots)
|
|
r.Group("/_seo/api/v1", nil, func(g pact.Router) {
|
|
g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve)
|
|
g.Get("/site", noindex(apicontrollers.Site(p.Service)), "throttle:"+BucketResolve)
|
|
g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound)
|
|
// The frontend server's route report (D-33), token-authenticated
|
|
// like the 404 report.
|
|
g.Post("/registry/routes", noindex(apicontrollers.RouteReport(p.Service)), "throttle:"+BucketNotFound)
|
|
// Verification files are served here only; the frontend maps them
|
|
// at the site root (D-103). A root route would capture every
|
|
// one-segment page of a site served at GET /.
|
|
g.Get("/verification/{name}", noindex(apicontrollers.Verification(p.Service)), "throttle:"+BucketResolve)
|
|
g.Where("name", classes.VerificationNamePattern)
|
|
})
|
|
r.Group("/_seo/api/v1", nil, func(g pact.Router) {
|
|
api := "throttle:" + BucketAPI
|
|
g.Get("/seo", apicontrollers.GetSEO(p.Service), api)
|
|
g.Put("/seo", apicontrollers.SetSEO(p.Service), api)
|
|
g.Get("/graph", apicontrollers.Graph(p.Service), api)
|
|
g.Get("/rules", apicontrollers.ListRules(p.Service), api)
|
|
g.Post("/rules", apicontrollers.CreateRule(p.Service), api)
|
|
g.Get("/rules/{id}", apicontrollers.ShowRule(p.Service), api)
|
|
g.Put("/rules/{id}", apicontrollers.UpdateRule(p.Service), api)
|
|
g.Delete("/rules/{id}", apicontrollers.DeleteRule(p.Service), api)
|
|
g.Get("/redirects", apicontrollers.ListRedirects(p.Service), api)
|
|
g.Post("/redirects", apicontrollers.CreateRedirect(p.Service), api)
|
|
g.Get("/redirects/{id}", apicontrollers.ShowRedirect(p.Service), api)
|
|
g.Put("/redirects/{id}", apicontrollers.UpdateRedirect(p.Service), api)
|
|
g.Delete("/redirects/{id}", apicontrollers.DeleteRedirect(p.Service), api)
|
|
g.Get("/pages", apicontrollers.ListPages(p.Service), api)
|
|
g.Post("/pages", apicontrollers.CreatePage(p.Service), api)
|
|
g.Put("/pages/{id}", apicontrollers.UpdatePage(p.Service), api)
|
|
g.Delete("/pages/{id}", apicontrollers.DeletePage(p.Service), api)
|
|
g.Get("/notfound", apicontrollers.ListNotFound(p.Service), api)
|
|
g.Post("/notfound/{id}/redirect", apicontrollers.RedirectNotFound(p.Service), api)
|
|
g.Post("/notfound/{id}/gone", apicontrollers.GoneNotFound(p.Service), api)
|
|
g.Delete("/notfound/{id}", apicontrollers.DeleteNotFound(p.Service), api)
|
|
g.Get("/robots", apicontrollers.ShowRobots(p.Service), api)
|
|
g.Put("/robots", apicontrollers.UpdateRobots(p.Service), api)
|
|
g.Get("/robots/test", apicontrollers.TestRobotsURL(p.Service), api)
|
|
g.Get("/settings", apicontrollers.ShowSettings(p.Service), api)
|
|
g.Put("/settings", apicontrollers.UpdateSettings(p.Service), api)
|
|
g.Get("/history", apicontrollers.ListHistory(p.Service), api)
|
|
g.Get("/history/{id}", apicontrollers.ShowHistory(p.Service), api)
|
|
g.Post("/history/{id}/undo", apicontrollers.UndoChange(p.Service), api)
|
|
g.Post("/history/batches/{batch}/undo", apicontrollers.UndoBatch(p.Service), api)
|
|
g.Get("/export", apicontrollers.ExportCSV(p.Service), api)
|
|
g.Post("/import", apicontrollers.ImportCSV(p.Service), api)
|
|
g.Post("/batch", apicontrollers.ApplyBatch(p.Service), api)
|
|
g.Get("/issues", apicontrollers.ListIssues(p.Service), api)
|
|
g.Post("/issues/ignore", apicontrollers.IgnoreIssues(p.Service), api)
|
|
g.Post("/issues/unignore", apicontrollers.UnignoreIssues(p.Service), api)
|
|
g.Post("/issues/{id}/recheck", apicontrollers.RecheckIssue(p.Service), api)
|
|
g.Get("/crawls", apicontrollers.ListCrawls(p.Service), api)
|
|
g.Post("/crawls", apicontrollers.StartCrawl(p.Service), api)
|
|
g.Delete("/crawls", apicontrollers.DeleteCrawls(p.Service), api)
|
|
g.Get("/crawls/{id}", apicontrollers.ShowCrawl(p.Service), api)
|
|
g.Post("/crawls/{id}/cancel", apicontrollers.CancelCrawl(p.Service), api)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// noindex adds X-Robots-Tag: noindex to every answer of h.
|
|
func noindex(h http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("X-Robots-Tag", "noindex")
|
|
h(w, r)
|
|
}
|
|
}
|
|
|
|
// Buckets registers the per-client-IP buckets behind the configured
|
|
// trusted proxies: seo-resolve 600 a minute (the frontend server resolves
|
|
// every page it renders), seo-notfound 600 a minute (one report per 404
|
|
// page), seo-robots 120 a minute and seo-api 300 a minute (an MCP agent
|
|
// working through an audit). The resolve calls and 404 reports come from
|
|
// the frontend server, so that server must be a trusted proxy forwarding
|
|
// X-Forwarded-For, or the first two buckets cap the whole site (README,
|
|
// "Behind the frontend server").
|
|
func (p *Plugin) Buckets() map[string]surf.Bucket {
|
|
trusted := surf.TrustedProxies(nil)
|
|
if p.app != nil {
|
|
trusted = surf.TrustedProxies(p.app.Config)
|
|
}
|
|
bucket := func(name string, max int) surf.Bucket {
|
|
return surf.Bucket{
|
|
Name: name,
|
|
Max: max,
|
|
Decay: time.Minute,
|
|
Key: func(r *http.Request) string {
|
|
return name + "|" + surf.ClientIP(r, trusted)
|
|
},
|
|
}
|
|
}
|
|
return map[string]surf.Bucket{
|
|
BucketResolve: bucket(BucketResolve, 600),
|
|
BucketNotFound: bucket(BucketNotFound, 600),
|
|
BucketRobots: bucket(BucketRobots, 120),
|
|
BucketAPI: bucket(BucketAPI, 300),
|
|
}
|
|
}
|