Jakub Zych
3dd936202f
- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
68 lines
2.2 KiB
Go
68 lines
2.2 KiB
Go
package seo
|
|
|
|
import (
|
|
"net/http"
|
|
"time"
|
|
|
|
apicontrollers "git.golem15.com/golem15/sm-seo-plugin/controllers/api"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/surf"
|
|
)
|
|
|
|
// The throttle buckets of the public endpoints.
|
|
const (
|
|
// BucketResolve is the resolve endpoint's bucket.
|
|
BucketResolve = "seo-resolve"
|
|
// BucketNotFound is the 404 report endpoint's bucket.
|
|
BucketNotFound = "seo-notfound"
|
|
// BucketRobots is the /robots.txt bucket.
|
|
BucketRobots = "seo-robots"
|
|
)
|
|
|
|
// Routes mounts GET /robots.txt at the site root and the /_seo/api/v1
|
|
// endpoints. None carries backend middleware: crawlers and the frontend
|
|
// server call them anonymously (the 404 report with its own token). Every
|
|
// /_seo/api/v1 answer carries X-Robots-Tag: noindex.
|
|
func (p *Plugin) Routes(r pact.Router) error {
|
|
r.Get("/robots.txt", apicontrollers.Robots(p.Service), "throttle:"+BucketRobots)
|
|
r.Group("/_seo/api/v1", nil, func(g pact.Router) {
|
|
g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve)
|
|
g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// noindex adds X-Robots-Tag: noindex to every answer of h.
|
|
func noindex(h http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("X-Robots-Tag", "noindex")
|
|
h(w, r)
|
|
}
|
|
}
|
|
|
|
// Buckets registers the per-client-IP buckets behind the configured
|
|
// trusted proxies: seo-resolve 600 a minute (the frontend server resolves
|
|
// every page it renders), seo-notfound 600 a minute (one report per 404
|
|
// page) and seo-robots 120 a minute.
|
|
func (p *Plugin) Buckets() map[string]surf.Bucket {
|
|
trusted := surf.TrustedProxies(nil)
|
|
if p.app != nil {
|
|
trusted = surf.TrustedProxies(p.app.Config)
|
|
}
|
|
bucket := func(name string, max int) surf.Bucket {
|
|
return surf.Bucket{
|
|
Name: name,
|
|
Max: max,
|
|
Decay: time.Minute,
|
|
Key: func(r *http.Request) string {
|
|
return name + "|" + surf.ClientIP(r, trusted)
|
|
},
|
|
}
|
|
}
|
|
return map[string]surf.Bucket{
|
|
BucketResolve: bucket(BucketResolve, 600),
|
|
BucketNotFound: bucket(BucketNotFound, 600),
|
|
BucketRobots: bucket(BucketRobots, 120),
|
|
}
|
|
}
|