feat: site endpoint, llms.txt and the sitemap index in robots.txt

- GET /_seo/api/v1/site (public, noindex): site_url, default_locale,
  locales, unprefixed_locale, links.new_tab, html_sitemap (from the
  sitemap plugin's Listing), routes of every content type, root_files
  (verification files and the IndexNow key file) and llms
- GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by
  default) and in production, the indexable pages from the sitemaps per
  group under the site name and llms_intro (else the default
  description); otherwise 404; seo_update_robots takes both fields
- robots.txt names the sitemap plugin's index unless
  golem15.seo.robots_sitemaps is set; the config default is now a comment
This commit is contained in:
Jakub Zych
2026-11-04 13:32:12 +01:00
parent 49d3879954
commit 742a5b930b
16 changed files with 645 additions and 29 deletions

View File

@@ -12,11 +12,15 @@ import (
// file adds a Disallow group for every switched-off bot and the Sitemap
// lines; outside production it is replaced by the non-production lock.
type Robots struct {
ID uint `gorm:"column:id;primaryKey"`
Content string `gorm:"column:content"`
Bots BotSwitches `gorm:"column:bots"`
CreatedAt *time.Time `gorm:"column:created_at"`
UpdatedAt *time.Time `gorm:"column:updated_at"`
ID uint `gorm:"column:id;primaryKey"`
Content string `gorm:"column:content"`
Bots BotSwitches `gorm:"column:bots"`
// LlmsEnabled serves /llms.txt (D-25, off by default); LlmsIntro is
// its introduction, the default meta description when empty.
LlmsEnabled bool `gorm:"column:llms_enabled"`
LlmsIntro string `gorm:"column:llms_intro"`
CreatedAt *time.Time `gorm:"column:created_at"`
UpdatedAt *time.Time `gorm:"column:updated_at"`
}
func (Robots) TableName() string { return "golem15_seo_robots" }