package classes import "strings" // The kinds of AI bots (D-25): search and answer crawlers, fetchers that // load a page when a user asks for it, and crawlers that collect content // for training. const ( BotSearch = "search" BotUser = "user" BotTraining = "training" ) // Bot is one AI crawler the robots.txt screen can switch off: its // user-agent token, its operator and its kind. type Bot struct { Token string Operator string Kind string } // Bots is the AI bot list in the screen's order (UI-SPEC C10, D-25, // D-105): search bots, user-fetch bots, then training bots. Every bot is // allowed until an editor switches it off. The list is plugin data and // can grow without a layout change (Q-6). var Bots = []Bot{ {Token: "OAI-SearchBot", Operator: "OpenAI", Kind: BotSearch}, {Token: "Claude-SearchBot", Operator: "Anthropic", Kind: BotSearch}, {Token: "PerplexityBot", Operator: "Perplexity", Kind: BotSearch}, {Token: "ChatGPT-User", Operator: "OpenAI", Kind: BotUser}, {Token: "Claude-User", Operator: "Anthropic", Kind: BotUser}, {Token: "GPTBot", Operator: "OpenAI", Kind: BotTraining}, {Token: "ClaudeBot", Operator: "Anthropic", Kind: BotTraining}, {Token: "Google-Extended", Operator: "Google", Kind: BotTraining}, {Token: "Applebot-Extended", Operator: "Apple", Kind: BotTraining}, {Token: "CCBot", Operator: "Common Crawl", Kind: BotTraining}, {Token: "Meta-ExternalAgent", Operator: "Meta", Kind: BotTraining}, } // BotField is the robots form's switch field of a bot: bot_ and its token // lowercased with - as _ (bot_oai_searchbot). func BotField(token string) string { return "bot_" + strings.ReplaceAll(strings.ToLower(token), "-", "_") } // BotByToken returns the listed bot whose token equals token (letter case // ignored). func BotByToken(token string) (Bot, bool) { for _, b := range Bots { if strings.EqualFold(b.Token, token) { return b, true } } return Bot{}, false }