feat: site endpoint, llms.txt and the sitemap index in robots.txt

- GET /_seo/api/v1/site (public, noindex): site_url, default_locale,
  locales, unprefixed_locale, links.new_tab, html_sitemap (from the
  sitemap plugin's Listing), routes of every content type, root_files
  (verification files and the IndexNow key file) and llms
- GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by
  default) and in production, the indexable pages from the sitemaps per
  group under the site name and llms_intro (else the default
  description); otherwise 404; seo_update_robots takes both fields
- robots.txt names the sitemap plugin's index unless
  golem15.seo.robots_sitemaps is set; the config default is now a comment
This commit is contained in:
Jakub Zych
2026-11-04 13:32:12 +01:00
parent 49d3879954
commit 742a5b930b
16 changed files with 645 additions and 29 deletions

34
controllers/api/llms.go Normal file
View File

@@ -0,0 +1,34 @@
package api
import (
"net/http"
"git.golem15.com/golem15/sm-seo-plugin/classes"
)
// LLMS serves GET /llms.txt at the site root (D-25, UI-SPEC C25): while
// llms.txt is on (the robots.txt screen's AI bots tab) and in production,
// the site's indexable pages from the sitemaps as text/plain; otherwise
// 404.
func LLMS(service func() *classes.Service) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
svc := service()
if svc == nil {
http.Error(w, "SEO is not available", http.StatusServiceUnavailable)
return
}
body, ok, err := svc.LLMSText(r.Context())
if err != nil {
log.Error("golem15.seo: llms.txt", "error", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
_, _ = w.Write([]byte(body))
}
}