package classes import ( "errors" "fmt" "regexp" "strconv" "strings" "time" ) // ErrRobotsValue is the refusal of a malformed robots value. var ErrRobotsValue = errors.New("golem15.seo: malformed robots value") // Robots is a parsed robots value (UI-SPEC C5, D-08): comma-separated // directives in the fixed order index|noindex, follow|nofollow, noarchive, // nosnippet, noimageindex, max-snippet:N, max-image-preview:none|standard| // large, max-video-preview:N, unavailable_after:YYYY-MM-DD. An empty string // inherits everything; an unset directive inherits too. type Robots struct { Index string // "index", "noindex" or "" (inherit) Follow string // "follow", "nofollow" or "" (inherit) NoArchive bool NoSnippet bool NoImageIndex bool MaxSnippet *int MaxImagePreview string MaxVideoPreview *int UnavailableAfter string } var robotsDate = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) // ParseRobots reads a robots value; spaces around directives and their // case are ignored. A directive outside the grammar, a repeated directive, // two contradicting directives or a malformed number or date is refused. func ParseRobots(value string) (Robots, error) { var r Robots value = strings.TrimSpace(value) if value == "" { return r, nil } seen := map[string]bool{} for _, part := range strings.Split(value, ",") { d := strings.ToLower(strings.TrimSpace(part)) if d == "" { return Robots{}, fmt.Errorf("%w: empty directive", ErrRobotsValue) } name, arg, hasArg := strings.Cut(d, ":") name, arg = strings.TrimSpace(name), strings.TrimSpace(arg) group := name switch name { case "index", "noindex": group = "index" case "follow", "nofollow": group = "follow" } if seen[group] { return Robots{}, fmt.Errorf("%w: %s given twice", ErrRobotsValue, group) } seen[group] = true switch name { case "index", "noindex": if hasArg { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } r.Index = name case "follow", "nofollow": if hasArg { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } r.Follow = name case "noarchive", "nosnippet", "noimageindex": if hasArg { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } switch name { case "noarchive": r.NoArchive = true case "nosnippet": r.NoSnippet = true default: r.NoImageIndex = true } case "max-snippet", "max-video-preview": n, err := strconv.Atoi(arg) if !hasArg || err != nil || n < -1 { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } if name == "max-snippet" { r.MaxSnippet = &n } else { r.MaxVideoPreview = &n } case "max-image-preview": if arg != "none" && arg != "standard" && arg != "large" { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } r.MaxImagePreview = arg case "unavailable_after": if !robotsDate.MatchString(arg) { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } if _, err := time.Parse("2006-01-02", arg); err != nil { return Robots{}, fmt.Errorf("%w: %s", ErrRobotsValue, d) } r.UnavailableAfter = arg default: return Robots{}, fmt.Errorf("%w: unknown directive %s", ErrRobotsValue, name) } } return r, nil } // directives lists the set directives in the fixed order. func (r Robots) directives() []string { var out []string if r.Index != "" { out = append(out, r.Index) } if r.Follow != "" { out = append(out, r.Follow) } if r.NoArchive { out = append(out, "noarchive") } if r.NoSnippet { out = append(out, "nosnippet") } if r.NoImageIndex { out = append(out, "noimageindex") } if r.MaxSnippet != nil { out = append(out, "max-snippet:"+strconv.Itoa(*r.MaxSnippet)) } if r.MaxImagePreview != "" { out = append(out, "max-image-preview:"+r.MaxImagePreview) } if r.MaxVideoPreview != nil { out = append(out, "max-video-preview:"+strconv.Itoa(*r.MaxVideoPreview)) } if r.UnavailableAfter != "" { out = append(out, "unavailable_after:"+r.UnavailableAfter) } return out } // String serialises the stored form: directives in the fixed order joined // by ",". The empty value inherits everything. func (r Robots) String() string { return strings.Join(r.directives(), ",") } // Header serialises the value for a meta robots tag and the X-Robots-Tag // header: the same directives joined by ", ". func (r Robots) Header() string { return strings.Join(r.directives(), ", ") } // Empty reports whether no directive is set. func (r Robots) Empty() bool { return len(r.directives()) == 0 } // Inherit fills every directive r leaves unset from lower (the next layer // of the precedence chain); a directive r sets wins. func (r Robots) Inherit(lower Robots) Robots { out := r if out.Index == "" { out.Index = lower.Index } if out.Follow == "" { out.Follow = lower.Follow } out.NoArchive = out.NoArchive || lower.NoArchive out.NoSnippet = out.NoSnippet || lower.NoSnippet out.NoImageIndex = out.NoImageIndex || lower.NoImageIndex if out.MaxSnippet == nil { out.MaxSnippet = lower.MaxSnippet } if out.MaxImagePreview == "" { out.MaxImagePreview = lower.MaxImagePreview } if out.MaxVideoPreview == nil { out.MaxVideoPreview = lower.MaxVideoPreview } if out.UnavailableAfter == "" { out.UnavailableAfter = lower.UnavailableAfter } return out } // NormalizeRobots parses and re-serialises a robots value in the fixed // order; a malformed value is an error. func NormalizeRobots(value string) (string, error) { r, err := ParseRobots(value) if err != nil { return "", err } return r.String(), nil }