feat: auto-link phrases in the resolve answer (tracer)

- golem15_seo_autolinks (migration 202610280007) and the settings column
  autolink_max (Links tab, 0 to 20, default 3)
- classes/autolinks.go: checked save and delete with history, ForPage
- ok resolve answers carry autolinks for the page's locale without
  disabled phrases and links to the page itself, and autolink_max

classes/autolinks.go

@@ -0,0 +1,296 @@
+package classes
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"gorm.io/gorm"
+)
+
+// MaxPageAutolinks caps the phrases one resolve answer carries.
+const MaxPageAutolinks = 1000
+
+// AutolinkLink is one phrase of a resolve answer's autolinks (D-82): the
+// layer links the phrase's first occurrence in a text to URL.
+type AutolinkLink struct {
+	Phrase string `json:"phrase"`
+	URL    string `json:"url"`
+}
+
+// AutolinkValues are an auto-link's values for history and the API.
+func AutolinkValues(a models.Autolink) map[string]any {
+	return map[string]any{
+		"phrase": a.Phrase, "url": a.URL, "locale": a.Locale,
+		"enabled": a.Enabled, "origin": a.Origin, "note": a.Note,
+	}
+}
+
+// AutolinkWrite is a create or update of an auto-link: only the fields
+// set change.
+type AutolinkWrite struct {
+	Phrase  *string `json:"phrase,omitempty"`
+	URL     *string `json:"url,omitempty"`
+	Locale  *string `json:"locale,omitempty"`
+	Enabled *bool   `json:"enabled,omitempty"`
+	Note    *string `json:"note,omitempty"`
+}
+
+// Apply copies the fields set into a.
+func (w AutolinkWrite) Apply(a *models.Autolink) {
+	if w.Phrase != nil {
+		a.Phrase = *w.Phrase
+	}
+	if w.URL != nil {
+		a.URL = *w.URL
+	}
+	if w.Locale != nil {
+		a.Locale = *w.Locale
+	}
+	if w.Enabled != nil {
+		a.Enabled = *w.Enabled
+	}
+	if w.Note != nil {
+		a.Note = *w.Note
+	}
+}
+
+// NormalizeAutolink trims the phrase (inner whitespace collapsed to one
+// space), the address, the locale ("all" is every locale) and the note.
+func NormalizeAutolink(a *models.Autolink) {
+	a.Phrase = strings.Join(strings.Fields(a.Phrase), " ")
+	a.URL = strings.TrimSpace(a.URL)
+	a.Locale = strings.TrimSpace(a.Locale)
+	if a.Locale == models.AutolinkLocaleAll {
+		a.Locale = ""
+	}
+	a.Note = strings.TrimSpace(a.Note)
+	if a.Origin == "" {
+		a.Origin = models.AutolinkOriginManual
+	}
+}
+
+// ValidAutolinkURL reports whether u is a path on the site (starting with
+// one /) or an absolute http(s) address, without whitespace or control
+// characters (T-33-50: javascript:, data: and protocol-relative addresses
+// are refused).
+func ValidAutolinkURL(u string) bool {
+	if u == "" || len(u) > models.MaxAutolinkURL || !utf8.ValidString(u) {
+		return false
+	}
+	for _, r := range u {
+		if unicode.IsSpace(r) || unicode.IsControl(r) {
+			return false
+		}
+	}
+	if strings.HasPrefix(u, "/") {
+		return !strings.HasPrefix(u, "//") && !strings.HasPrefix(u, `/\`)
+	}
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return false
+	}
+	scheme := strings.ToLower(parsed.Scheme)
+	return (scheme == "http" || scheme == "https") && parsed.Host != "" && parsed.User == nil
+}
+
+// AutolinkProblems refuses a phrase under 3 characters (or over 200), an
+// address that is not a path or an http(s) address, an unknown locale,
+// and a phrase another row already has in the same locale (a phrase of
+// every locale meets the phrases of each locale), naming that row's
+// address (UI-SPEC C22). a is normalised first.
+func (s *Service) AutolinkProblems(ctx context.Context, db *gorm.DB, a *models.Autolink) (map[string][]models.Problem, error) {
+	NormalizeAutolink(a)
+	out := map[string][]models.Problem{}
+	n := utf8.RuneCountInString(a.Phrase)
+	switch {
+	case n < models.MinAutolinkPhrase:
+		out["phrase"] = append(out["phrase"], models.Problem{Key: "golem15.seo::lang.autolinks.phrase_short"})
+	case n > models.MaxAutolinkPhrase:
+		out["phrase"] = append(out["phrase"], models.Problem{Key: "lagoon::validation.max.string", Params: map[string]string{"attribute": "phrase", "max": "200"}})
+	}
+	if !ValidAutolinkURL(a.URL) {
+		out["url"] = append(out["url"], models.Problem{Key: "golem15.seo::lang.autolinks.url_invalid"})
+	}
+	if a.Locale != "" {
+		known := false
+		for _, code := range s.Locales(ctx, db) {
+			if code == a.Locale {
+				known = true
+				break
+			}
+		}
+		if !known {
+			out["locale"] = append(out["locale"], models.Problem{Key: "golem15.seo::lang.api.value_invalid"})
+		}
+	}
+	if len(out["phrase"]) == 0 {
+		q := db.Model(&models.Autolink{}).Where("lower(phrase) = lower(?)", a.Phrase)
+		if a.Locale != "" {
+			q = q.Where("locale = ? OR locale = ''", a.Locale)
+		}
+		if a.ID != 0 {
+			q = q.Where("id <> ?", a.ID)
+		}
+		var rows []models.Autolink
+		if err := q.Order("id ASC").Limit(1).Find(&rows).Error; err != nil {
+			return nil, err
+		}
+		if len(rows) > 0 {
+			out["phrase"] = append(out["phrase"], models.Problem{Key: "golem15.seo::lang.autolinks.phrase_taken", Params: map[string]string{"url": rows[0].URL}})
+		}
+	}
+	return out, nil
+}
+
+// LoadAutolink returns auto-link id; ErrNoRecord when there is none.
+func LoadAutolink(db *gorm.DB, id uint) (models.Autolink, error) {
+	var rows []models.Autolink
+	if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil {
+		return models.Autolink{}, err
+	}
+	if len(rows) == 0 {
+		return models.Autolink{}, ErrNoRecord
+	}
+	return rows[0], nil
+}
+
+// AutolinkOrigin is the origin of a phrase created through ctx: mcp
+// through MCP, else manual.
+func AutolinkOrigin(ctx context.Context) string {
+	if ViaOf(ctx) == models.ViaMCP {
+		return models.AutolinkOriginMCP
+	}
+	return models.AutolinkOriginManual
+}
+
+// autolinkWhat is an auto-link's history description.
+func (s *Service) autolinkWhat(a models.Autolink) string {
+	return s.Phrase("golem15.seo::lang.history.what_autolink", map[string]string{"phrase": a.Phrase})
+}
+
+// RecordAutolink records an auto-link's change in history inside tx; the
+// write purges the site's cached pages, since the phrase may appear in any
+// text. before is nil for a created phrase.
+func (s *Service) RecordAutolink(ctx context.Context, tx *gorm.DB, before map[string]any, a models.Autolink, deleted bool) error {
+	c := Change{Kind: models.SubjectAutolink, SubjectID: a.IDText(), What: s.autolinkWhat(a), Before: before}
+	if !deleted {
+		c.After = AutolinkValues(a)
+	}
+	return s.Record(ctx, tx, c)
+}
+
+// SaveAutolink checks and writes an auto-link inside tx and records it in
+// history; a refused phrase is a *ProblemsError.
+func (s *Service) SaveAutolink(ctx context.Context, tx *gorm.DB, a *models.Autolink) error {
+	if a == nil {
+		return errors.New("golem15.seo: no auto-link")
+	}
+	problems, err := s.AutolinkProblems(ctx, tx, a)
+	if err != nil {
+		return err
+	}
+	if len(problems) > 0 {
+		return &ProblemsError{Problems: problems}
+	}
+	now := time.Now().UTC()
+	var before map[string]any
+	if a.ID == 0 {
+		a.CreatedAt, a.UpdatedAt = &now, &now
+		if err := tx.Create(a).Error; err != nil {
+			return err
+		}
+	} else {
+		old, err := LoadAutolink(tx, a.ID)
+		if err != nil {
+			return err
+		}
+		before = AutolinkValues(old)
+		a.CreatedAt, a.UpdatedAt = old.CreatedAt, &now
+		if err := tx.Save(a).Error; err != nil {
+			return err
+		}
+	}
+	return s.RecordAutolink(ctx, tx, before, *a, false)
+}
+
+// DeleteAutolinks deletes the auto-links ids inside tx, records each in
+// history and returns the ids deleted (unknown ids are skipped).
+func (s *Service) DeleteAutolinks(ctx context.Context, tx *gorm.DB, ids []uint) ([]uint, error) {
+	if len(ids) == 0 {
+		return []uint{}, nil
+	}
+	var rows []models.Autolink
+	if err := tx.Where("id IN ?", ids).Order("id ASC").Find(&rows).Error; err != nil {
+		return nil, err
+	}
+	out := make([]uint, 0, len(rows))
+	for _, row := range rows {
+		if err := tx.Delete(&models.Autolink{}, row.ID).Error; err != nil {
+			return nil, err
+		}
+		if err := s.RecordAutolink(ctx, tx, AutolinkValues(row), row, true); err != nil {
+			return nil, err
+		}
+		out = append(out, row.ID)
+	}
+	return out, nil
+}
+
+// autolinkPath is the site path an auto-link's address names: the path of
+// a / address, or of an absolute address on the site's host (its www twin
+// included); "" for another site.
+func (s *Service) autolinkPath(target string) string {
+	if strings.HasPrefix(target, "/") {
+		path, _, _ := strings.Cut(target, "#")
+		path, _, _ = strings.Cut(path, "?")
+		return path
+	}
+	u, err := url.Parse(target)
+	if err != nil {
+		return ""
+	}
+	site := strings.TrimPrefix(strings.ToLower(s.host()), "www.")
+	if site == "" || strings.TrimPrefix(strings.ToLower(u.Host), "www.") != site {
+		return ""
+	}
+	if u.Path == "" {
+		return "/"
+	}
+	return u.Path
+}
+
+// ForPage lists the auto-link phrases of a page in locale (D-82): the
+// active phrases of that locale and of every locale, without those that
+// link to the page itself (selfPath, a trailing slash ignored), longest
+// phrase first so a longer phrase wins over a phrase it contains, one row
+// per phrase, at most MaxPageAutolinks.
+func (s *Service) ForPage(ctx context.Context, db *gorm.DB, locale, selfPath string) ([]AutolinkLink, error) {
+	var rows []models.Autolink
+	err := db.WithContext(ctx).Where("enabled AND (locale = '' OR locale = ?)", locale).
+		Order("char_length(phrase) DESC, lower(phrase) ASC, locale DESC, id ASC").
+		Limit(MaxPageAutolinks).Find(&rows).Error
+	if err != nil {
+		return nil, err
+	}
+	self := trimSlash(selfPath)
+	seen := map[string]bool{}
+	out := make([]AutolinkLink, 0, len(rows))
+	for _, row := range rows {
+		key := strings.ToLower(row.Phrase)
+		if seen[key] {
+			continue
+		}
+		if p := s.autolinkPath(row.URL); p != "" && trimSlash(p) == self {
+			continue
+		}
+		seen[key] = true
+		out = append(out, AutolinkLink{Phrase: row.Phrase, URL: row.URL})
+	}
+	return out, nil
+}

classes/history.go

@@ -295,10 +295,11 @@ var (
 	purging map[gorm.ConnPool]bool
 )
 
-// purgeSlug is the purge's slug: "*" for the settings and robots.txt,
-// else the changed object's address.
+// purgeSlug is the purge's slug: "*" for the settings, robots.txt and an
+// auto-link phrase (it may appear in any text), else the changed object's
+// address.
 func purgeSlug(c Change) string {
-	if c.Kind == models.SubjectSettings || c.Kind == models.SubjectRobots {
+	if c.Kind == models.SubjectSettings || c.Kind == models.SubjectRobots || c.Kind == models.SubjectAutolink {
 		return "*"
 	}
 	for _, values := range []map[string]any{c.After, c.Before} {

classes/resolver.go

@@ -67,6 +67,12 @@ type Result struct {
 	// Outbound is how the layer renders links to other sites (P33-16),
 	// present on ok answers.
 	Outbound *Outbound `json:"outbound,omitempty"`
+	// Autolinks are the auto-link phrases of the page's locale without
+	// those linking to the page itself, longest first, and AutolinkMax the
+	// most the layer links in one text (D-82); present on ok answers, no
+	// phrases when AutolinkMax is 0.
+	Autolinks   []AutolinkLink `json:"autolinks,omitempty"`
+	AutolinkMax *int           `json:"autolink_max,omitempty"`
 	// HTMLLang is the page's <html lang> (P33-21); Alternates its hreflang
 	// list, the page itself and x-default included (D-55 to D-57); JSONLD
 	// its one @graph (D-43); Breadcrumbs its trail, the same items as the
@@ -799,8 +805,15 @@ func (r *Resolver) resolveAt(ctx context.Context, db *gorm.DB, settings models.S
 	res.Action, res.Status, res.Locale, res.SEO = ActionOK, 200, locale, seo
 	res.Outbound = OutboundOf(settings)
 	// The sitemap filter (production) decides on robots and canonical
-	// only; it skips the structured data.
+	// only; it skips the auto-links and the structured data.
 	if !r.production {
+		most := min(models.MaxAutolinkMax, max(models.MinAutolinkMax, settings.AutolinkMax))
+		res.AutolinkMax = &most
+		if most > 0 {
+			if res.Autolinks, err = svc.ForPage(ctx, db, locale, path); err != nil {
+				return Result{}, err
+			}
+		}
 		if err := svc.structuredData(ctx, db, settings, &res, structuredIn{
 			key: key, id: res.RecordID, rec: rec, entry: entry, content: content,
 			locale: locale, path: path, url: self, home: home, siteName: svc.SiteName(ctx, settings, locale), seo: seo,

classes/settings_write.go

@@ -48,7 +48,7 @@ var SettingsColumns = []string{
 	"local_latitude", "local_longitude", "local_hours", "page_form",
 	"page_segment", "page_title_suffix", "page_description_suffix",
 	"crawl_schedule", "crawl_max_pages", "crawl_delay", "crawl_keep",
-	"generic_anchors",
+	"generic_anchors", "autolink_max",
 }
 
 // settingsText maps the plain text settings columns to their fields.
@@ -102,6 +102,7 @@ func settingsInts(st *models.Settings) map[string]*int {
 		"crawl_max_pages": &st.CrawlMaxPages,
 		"crawl_delay":     &st.CrawlDelay,
 		"crawl_keep":      &st.CrawlKeep,
+		"autolink_max":    &st.AutolinkMax,
 	}
 }
 
@@ -158,6 +159,7 @@ func SettingsValues(st models.Settings) map[string]any {
 		"crawl_delay":             st.CrawlDelay,
 		"crawl_keep":              st.CrawlKeep,
 		"generic_anchors":         st.GenericAnchors,
+		"autolink_max":            st.AutolinkMax,
 	}
 }