package api import ( "context" "net/http" "strings" "git.golem15.com/golem15/sm-seo-plugin/classes" "git.golem15.com/golem15/sm-seo-plugin/models" "gorm.io/gorm" ) // Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In // production it is the stored rules, a Disallow group for every // switched-off AI bot and the Sitemap lines; outside production it is // "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow, // whatever is stored, and no admin setting can change that. It never // answers a 5xx: when the stored file cannot be read it serves the last // body it served, else an allow-all file, and logs the error. func Robots(service func() *classes.Service) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { svc := service() body := classes.FallbackRobots staging := false if svc == nil { log.Error("golem15.seo: robots.txt served before boot") } else { staging = svc.Staging() served, err := svc.ServedFile(r.Context(), !staging) switch { case err == nil: body = served case svc.LastServedRobots() != "": log.Error("golem15.seo: robots.txt, serving the last good file", "error", err) body = svc.LastServedRobots() default: log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err) } } w.Header().Set("Content-Type", "text/plain; charset=utf-8") if staging { w.Header().Set("X-Robots-Tag", "noindex, nofollow") } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(body)) } } // UpdateRobotsInput is the body of PUT /_seo/api/v1/robots. type UpdateRobotsInput struct { Content *string `json:"content,omitempty" jsonschema:"the robots.txt rules; Sitemap lines and the AI bot rules are added when the file is served"` Bots map[string]bool `json:"bots,omitempty" jsonschema:"AI bot switches by user-agent token, for example {\"GPTBot\": false}; a bot switched off is disallowed"` // LlmsEnabled and LlmsIntro are the llms.txt switch and introduction. LlmsEnabled *bool `json:"llms_enabled,omitempty" jsonschema:"serve /llms.txt: the site's indexable pages from the sitemaps, for AI tools (off by default)"` LlmsIntro *string `json:"llms_intro,omitempty" jsonschema:"a short description of the site at the top of llms.txt; empty uses the default meta description"` } // TestRobotsInput is the query of GET /_seo/api/v1/robots/test. type TestRobotsInput struct { URL string `json:"url" query:"url" jsonschema:"the address to test, a path starting with /"` Agent string `json:"agent,omitempty" query:"agent" jsonschema:"the crawler's user agent token (Googlebot by default)"` } // robotsJSON is the robots.txt screen in an answer: the rules, every AI // bot with its switch, the file crawlers get in production and whether // this site runs outside production (then crawlers get the lock). func robotsJSON(ctx context.Context, svc *classes.Service, row models.Robots) (map[string]any, error) { bots := make([]map[string]any, 0, len(classes.Bots)) for _, b := range classes.Bots { bots = append(bots, map[string]any{"token": b.Token, "operator": b.Operator, "kind": b.Kind, "allowed": row.Bots.Allowed(b.Token)}) } production := classes.ComposeRobots(row.Content, row.Bots, svc.SitemapLines(ctx)) return map[string]any{ "content": row.Content, "bots": bots, "llms_enabled": row.LlmsEnabled, "llms_intro": row.LlmsIntro, "production": production, "staging": svc.Staging(), }, nil } // ShowRobots serves GET /_seo/api/v1/robots. func ShowRobots(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { db, err := read(ctx, svc) if err != nil { return 0, nil, err } row, err := classes.LoadRobots(db) if err != nil { return 0, nil, err } body, err := robotsJSON(ctx, svc, row) if err != nil { return 0, nil, err } return http.StatusOK, map[string]any{"data": body}, nil }) } // UpdateRobots serves PUT /_seo/api/v1/robots: the rules and the bot // switches sent change (D-22, D-25). The answer lists the noindex // addresses the new rules keep Googlebot from (robots.blocks_noindex). func UpdateRobots(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { var in UpdateRobotsInput if err := decode(r, classes.MaxRobotsBytes+(64<<10), &in); err != nil { return 0, nil, err } var row models.Robots var blocked []string err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { var err error row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots, LlmsEnabled: in.LlmsEnabled, LlmsIntro: in.LlmsIntro}) return err }) if err != nil { return 0, nil, err } body, err := robotsJSON(ctx, svc, row) if err != nil { return 0, nil, err } out := map[string]any{"data": body} if len(blocked) > 0 { out["warnings"] = []map[string]any{{"kind": "blocks_noindex", "urls": blocked}} } return http.StatusOK, out, nil }) } // TestRobotsURL serves GET /_seo/api/v1/robots/test?url=&agent=: whether // the crawler may fetch the address under the saved production file, // through Google's matcher, with the deciding line (D-99). func TestRobotsURL(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { path := strings.TrimSpace(r.URL.Query().Get("url")) if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes { return 0, nil, &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{ "error": "validation_failed", "errors": map[string][]string{"url": {svc.PhraseCtx(ctx, "golem15.seo::lang.robots.url_required", nil)}}, }} } agent := strings.TrimSpace(r.URL.Query().Get("agent")) if agent == "" { agent = "Googlebot" } db, err := read(ctx, svc) if err != nil { return 0, nil, err } row, err := classes.LoadRobots(db) if err != nil { return 0, nil, err } return http.StatusOK, map[string]any{"url": path, "result": svc.TestRobots(row.Content, row.Bots, agent, path)}, nil }) }