Jakub Zych
3dd936202f
- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
44 lines
1.4 KiB
Go
44 lines
1.4 KiB
Go
package api
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
|
)
|
|
|
|
// Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In
|
|
// production it is the stored rules, a Disallow group for every
|
|
// switched-off AI bot and the Sitemap lines; outside production it is
|
|
// "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow,
|
|
// whatever is stored, and no admin setting can change that. It never
|
|
// answers a 5xx: when the stored file cannot be read it serves the last
|
|
// body it served, else an allow-all file, and logs the error.
|
|
func Robots(service func() *classes.Service) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
svc := service()
|
|
body := classes.FallbackRobots
|
|
staging := false
|
|
if svc == nil {
|
|
log.Error("golem15.seo: robots.txt served before boot")
|
|
} else {
|
|
staging = svc.Staging()
|
|
served, err := svc.ServedFile(r.Context(), !staging)
|
|
switch {
|
|
case err == nil:
|
|
body = served
|
|
case svc.LastServedRobots() != "":
|
|
log.Error("golem15.seo: robots.txt, serving the last good file", "error", err)
|
|
body = svc.LastServedRobots()
|
|
default:
|
|
log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err)
|
|
}
|
|
}
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
if staging {
|
|
w.Header().Set("X-Robots-Tag", "noindex, nofollow")
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(body))
|
|
}
|
|
}
|