feat: site endpoint, llms.txt and the sitemap index in robots.txt
- GET /_seo/api/v1/site (public, noindex): site_url, default_locale, locales, unprefixed_locale, links.new_tab, html_sitemap (from the sitemap plugin's Listing), routes of every content type, root_files (verification files and the IndexNow key file) and llms - GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by default) and in production, the indexable pages from the sitemaps per group under the site name and llms_intro (else the default description); otherwise 404; seo_update_robots takes both fields - robots.txt names the sitemap plugin's index unless golem15.seo.robots_sitemaps is set; the config default is now a comment
This commit is contained in:
@@ -50,6 +50,9 @@ func Robots(service func() *classes.Service) http.HandlerFunc {
|
||||
type UpdateRobotsInput struct {
|
||||
Content *string `json:"content,omitempty" jsonschema:"the robots.txt rules; Sitemap lines and the AI bot rules are added when the file is served"`
|
||||
Bots map[string]bool `json:"bots,omitempty" jsonschema:"AI bot switches by user-agent token, for example {\"GPTBot\": false}; a bot switched off is disallowed"`
|
||||
// LlmsEnabled and LlmsIntro are the llms.txt switch and introduction.
|
||||
LlmsEnabled *bool `json:"llms_enabled,omitempty" jsonschema:"serve /llms.txt: the site's indexable pages from the sitemaps, for AI tools (off by default)"`
|
||||
LlmsIntro *string `json:"llms_intro,omitempty" jsonschema:"a short description of the site at the top of llms.txt; empty uses the default meta description"`
|
||||
}
|
||||
|
||||
// TestRobotsInput is the query of GET /_seo/api/v1/robots/test.
|
||||
@@ -66,12 +69,14 @@ func robotsJSON(ctx context.Context, svc *classes.Service, row models.Robots) (m
|
||||
for _, b := range classes.Bots {
|
||||
bots = append(bots, map[string]any{"token": b.Token, "operator": b.Operator, "kind": b.Kind, "allowed": row.Bots.Allowed(b.Token)})
|
||||
}
|
||||
production := classes.ComposeRobots(row.Content, row.Bots, svc.RobotsSitemaps())
|
||||
production := classes.ComposeRobots(row.Content, row.Bots, svc.SitemapLines(ctx))
|
||||
return map[string]any{
|
||||
"content": row.Content,
|
||||
"bots": bots,
|
||||
"production": production,
|
||||
"staging": svc.Staging(),
|
||||
"content": row.Content,
|
||||
"bots": bots,
|
||||
"llms_enabled": row.LlmsEnabled,
|
||||
"llms_intro": row.LlmsIntro,
|
||||
"production": production,
|
||||
"staging": svc.Staging(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -107,7 +112,7 @@ func UpdateRobots(service func() *classes.Service) http.HandlerFunc {
|
||||
var blocked []string
|
||||
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
||||
var err error
|
||||
row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots})
|
||||
row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots, LlmsEnabled: in.LlmsEnabled, LlmsIntro: in.LlmsIntro})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user