feat: site endpoint, llms.txt and the sitemap index in robots.txt

- GET /_seo/api/v1/site (public, noindex): site_url, default_locale,
  locales, unprefixed_locale, links.new_tab, html_sitemap (from the
  sitemap plugin's Listing), routes of every content type, root_files
  (verification files and the IndexNow key file) and llms
- GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by
  default) and in production, the indexable pages from the sitemaps per
  group under the site name and llms_intro (else the default
  description); otherwise 404; seo_update_robots takes both fields
- robots.txt names the sitemap plugin's index unless
  golem15.seo.robots_sitemaps is set; the config default is now a comment
This commit is contained in:
Jakub Zych
2026-11-04 13:32:12 +01:00
parent 49d3879954
commit 742a5b930b
16 changed files with 645 additions and 29 deletions

View File

@@ -50,6 +50,9 @@ func Robots(service func() *classes.Service) http.HandlerFunc {
type UpdateRobotsInput struct {
Content *string `json:"content,omitempty" jsonschema:"the robots.txt rules; Sitemap lines and the AI bot rules are added when the file is served"`
Bots map[string]bool `json:"bots,omitempty" jsonschema:"AI bot switches by user-agent token, for example {\"GPTBot\": false}; a bot switched off is disallowed"`
// LlmsEnabled and LlmsIntro are the llms.txt switch and introduction.
LlmsEnabled *bool `json:"llms_enabled,omitempty" jsonschema:"serve /llms.txt: the site's indexable pages from the sitemaps, for AI tools (off by default)"`
LlmsIntro *string `json:"llms_intro,omitempty" jsonschema:"a short description of the site at the top of llms.txt; empty uses the default meta description"`
}
// TestRobotsInput is the query of GET /_seo/api/v1/robots/test.
@@ -66,12 +69,14 @@ func robotsJSON(ctx context.Context, svc *classes.Service, row models.Robots) (m
for _, b := range classes.Bots {
bots = append(bots, map[string]any{"token": b.Token, "operator": b.Operator, "kind": b.Kind, "allowed": row.Bots.Allowed(b.Token)})
}
production := classes.ComposeRobots(row.Content, row.Bots, svc.RobotsSitemaps())
production := classes.ComposeRobots(row.Content, row.Bots, svc.SitemapLines(ctx))
return map[string]any{
"content": row.Content,
"bots": bots,
"production": production,
"staging": svc.Staging(),
"content": row.Content,
"bots": bots,
"llms_enabled": row.LlmsEnabled,
"llms_intro": row.LlmsIntro,
"production": production,
"staging": svc.Staging(),
}, nil
}
@@ -107,7 +112,7 @@ func UpdateRobots(service func() *classes.Service) http.HandlerFunc {
var blocked []string
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
var err error
row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots})
row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots, LlmsEnabled: in.LlmsEnabled, LlmsIntro: in.LlmsIntro})
return err
})
if err != nil {