- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
34 lines
916 B
Go
34 lines
916 B
Go
package updates
|
|
|
|
import (
|
|
"github.com/go-gormigrate/gormigrate/v2"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// createRobots creates golem15_seo_robots, the robots.txt singleton (D-22,
|
|
// D-25), with its row 1: no rules and every AI bot allowed.
|
|
var createRobots = []*gormigrate.Migration{
|
|
{
|
|
ID: "202610270008_create_golem15_seo_robots",
|
|
Migrate: func(tx *gorm.DB) error {
|
|
if err := tx.Exec(`CREATE TABLE golem15_seo_robots (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
content TEXT NOT NULL DEFAULT '',
|
|
bots JSONB NOT NULL DEFAULT '{}',
|
|
created_at TIMESTAMP(0) NULL,
|
|
updated_at TIMESTAMP(0) NULL
|
|
)`).Error; err != nil {
|
|
return err
|
|
}
|
|
return tx.Exec(`INSERT INTO golem15_seo_robots (id, content, bots, created_at, updated_at) VALUES (1, '', '{}', NOW(), NOW())`).Error
|
|
},
|
|
Rollback: func(tx *gorm.DB) error {
|
|
return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_robots`).Error
|
|
},
|
|
},
|
|
}
|
|
|
|
func init() {
|
|
Register(createRobots...)
|
|
}
|