package api import ( "net/http" "git.golem15.com/golem15/sm-seo-plugin/classes" ) // Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In // production it is the stored rules, a Disallow group for every // switched-off AI bot and the Sitemap lines; outside production it is // "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow, // whatever is stored, and no admin setting can change that. It never // answers a 5xx: when the stored file cannot be read it serves the last // body it served, else an allow-all file, and logs the error. func Robots(service func() *classes.Service) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { svc := service() body := classes.FallbackRobots staging := false if svc == nil { log.Error("golem15.seo: robots.txt served before boot") } else { staging = svc.Staging() served, err := svc.ServedFile(r.Context(), !staging) switch { case err == nil: body = served case svc.LastServedRobots() != "": log.Error("golem15.seo: robots.txt, serving the last good file", "error", err) body = svc.LastServedRobots() default: log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err) } } w.Header().Set("Content-Type", "text/plain; charset=utf-8") if staging { w.Header().Set("X-Robots-Tag", "noindex, nofollow") } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(body)) } }