Jakub Zych
3dd936202f
- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
56 lines
1.9 KiB
Go
56 lines
1.9 KiB
Go
package classes
|
|
|
|
import "strings"
|
|
|
|
// The kinds of AI bots (D-25): search and answer crawlers, fetchers that
|
|
// load a page when a user asks for it, and crawlers that collect content
|
|
// for training.
|
|
const (
|
|
BotSearch = "search"
|
|
BotUser = "user"
|
|
BotTraining = "training"
|
|
)
|
|
|
|
// Bot is one AI crawler the robots.txt screen can switch off: its
|
|
// user-agent token, its operator and its kind.
|
|
type Bot struct {
|
|
Token string
|
|
Operator string
|
|
Kind string
|
|
}
|
|
|
|
// Bots is the AI bot list in the screen's order (UI-SPEC C10, D-25,
|
|
// D-105): search bots, user-fetch bots, then training bots. Every bot is
|
|
// allowed until an editor switches it off. The list is plugin data and
|
|
// can grow without a layout change (Q-6).
|
|
var Bots = []Bot{
|
|
{Token: "OAI-SearchBot", Operator: "OpenAI", Kind: BotSearch},
|
|
{Token: "Claude-SearchBot", Operator: "Anthropic", Kind: BotSearch},
|
|
{Token: "PerplexityBot", Operator: "Perplexity", Kind: BotSearch},
|
|
{Token: "ChatGPT-User", Operator: "OpenAI", Kind: BotUser},
|
|
{Token: "Claude-User", Operator: "Anthropic", Kind: BotUser},
|
|
{Token: "GPTBot", Operator: "OpenAI", Kind: BotTraining},
|
|
{Token: "ClaudeBot", Operator: "Anthropic", Kind: BotTraining},
|
|
{Token: "Google-Extended", Operator: "Google", Kind: BotTraining},
|
|
{Token: "Applebot-Extended", Operator: "Apple", Kind: BotTraining},
|
|
{Token: "CCBot", Operator: "Common Crawl", Kind: BotTraining},
|
|
{Token: "Meta-ExternalAgent", Operator: "Meta", Kind: BotTraining},
|
|
}
|
|
|
|
// BotField is the robots form's switch field of a bot: bot_ and its token
|
|
// lowercased with - as _ (bot_oai_searchbot).
|
|
func BotField(token string) string {
|
|
return "bot_" + strings.ReplaceAll(strings.ToLower(token), "-", "_")
|
|
}
|
|
|
|
// BotByToken returns the listed bot whose token equals token (letter case
|
|
// ignored).
|
|
func BotByToken(token string) (Bot, bool) {
|
|
for _, b := range Bots {
|
|
if strings.EqualFold(b.Token, token) {
|
|
return b, true
|
|
}
|
|
}
|
|
return Bot{}, false
|
|
}
|