sm-seo-plugin/classes/linkgraph.go

package classes

import (
	"context"
	"encoding/json"
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/cabana"
	"git.golem15.com/golem15/summercms/modules/pact"
	"gorm.io/gorm"
)

// MaxLinkSuggestions caps each suggestion list of the internal links
// widget (UI-SPEC C19).
const MaxLinkSuggestions = 10

// minLinkPhrase is the shortest phrase a link suggestion matches, in
// bytes (the orphan suggestions' limit).
const minLinkPhrase = 3

// LinksActionName is the SEO tab's internal links widget action (E10).
const LinksActionName = "seo_links"

// LinkOut is a page this text could link to: the phrase found in this
// page's crawled text, the target's title and its public address.
type LinkOut struct {
	Phrase string `json:"phrase"`
	Title  string `json:"title"`
	URL    string `json:"url"`
}

// LinkIn is a page that could link here: the phrase of this page found in
// its crawled text, its title and its admin form ("" when no record of
// the admin is behind it).
type LinkIn struct {
	Phrase   string `json:"phrase"`
	Title    string `json:"title"`
	AdminURL string `json:"adminUrl"`
}

// LinkReport is the internal links of a page from the last finished crawl
// (D-82, UI-SPEC C19): the time of that crawl, the pages linking here and
// the pages linked from here (both nil before the first crawl), whether
// nothing links to the page, and up to 10 pages this text could link to
// and 10 that could link here.
type LinkReport struct {
	CrawledAt *time.Time `json:"crawledAt"`
	Inbound   *int       `json:"inbound"`
	Outbound  *int       `json:"outbound"`
	Orphan    bool       `json:"orphan"`
	Out       []LinkOut  `json:"out"`
	In        []LinkIn   `json:"in"`
}

// NormalText is text in lower case with its whitespace collapsed to one
// space: the form the crawler's suggestions compare phrases in.
func NormalText(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) }

// errEnough stops a batched scan once a list is full.
var errEnough = errors.New("golem15.seo: enough suggestions")

// lastFinishedCrawl is the newest finished crawl, nil before the first.
func lastFinishedCrawl(db *gorm.DB) (*models.Crawl, error) {
	var rows []models.Crawl
	if err := db.Where("status = ?", models.CrawlFinished).Order("finished_at DESC NULLS LAST, id DESC").Limit(1).Find(&rows).Error; err != nil {
		return nil, err
	}
	if len(rows) == 0 {
		return nil, nil
	}
	return &rows[0], nil
}

// pageHeading is a crawled page's first H1, else its title.
func pageHeading(p models.CrawlPage) (h1, title string) {
	if list := p.H1.Get(); len(list) > 0 {
		h1 = strings.TrimSpace(list[0])
	}
	title = h1
	if title == "" {
		title = strings.TrimSpace(p.Title)
	}
	return h1, title
}

// entryKeyphrases maps each address with an SEO entry to its focus
// keyphrase.
func entryKeyphrases(db *gorm.DB) (map[string]string, error) {
	var rows []struct {
		URL       string
		Keyphrase string
	}
	if err := db.Raw(`SELECT url, keyphrase FROM golem15_seo_entries WHERE url <> '' AND keyphrase <> ''`).Scan(&rows).Error; err != nil {
		return nil, err
	}
	out := make(map[string]string, len(rows))
	for _, r := range rows {
		out[r.URL] = r.Keyphrase
	}
	return out, nil
}

// phraseList keeps the phrases of at least 3 bytes, once each (by their
// normal form), in order.
func phraseList(phrases ...string) []string {
	var out []string
	seen := map[string]bool{}
	for _, p := range phrases {
		p = strings.Join(strings.Fields(p), " ")
		n := NormalText(p)
		if len(n) < minLinkPhrase || seen[n] {
			continue
		}
		seen[n] = true
		out = append(out, p)
	}
	return out
}

// mentioned returns the first phrase whose normal form normal (a page's
// NormalText) contains, "" when none.
func mentioned(normal string, phrases []string) string {
	for _, p := range phrases {
		if strings.Contains(normal, NormalText(p)) {
			return p
		}
	}
	return ""
}

// adminURLOf is the admin form of the record behind a site path, "" when
// no content type with an admin controller owns it.
func (s *Service) adminURLOf(ctx context.Context, path string) string {
	reg := s.Registry()
	if reg == nil {
		return ""
	}
	p, _, _ := strings.Cut(path, "?")
	key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale())
	if err != nil || match == nil || !match.Exact {
		return ""
	}
	typ, ok := reg.ContentType(key)
	if !ok || typ.Content == nil || strings.TrimSpace(typ.Content.Controller) == "" {
		return ""
	}
	return s.adminFormURL(typ.Content.Controller, match.ID)
}

// LinkSuggestions builds the internal links of the page at path from the
// last finished crawl (D-82, UI-SPEC C19), like the crawl's orphan
// suggestions: the pages whose crawled text contains one of phrases (the
// page's title and focus keyphrase; its crawled H1 is added) and do not
// link here yet, and the pages whose H1 or focus keyphrase this page's
// crawled text contains and that it does not link to yet. Before the
// first crawl the counts are nil and the lists empty.
func (s *Service) LinkSuggestions(ctx context.Context, db *gorm.DB, path string, phrases []string) (LinkReport, error) {
	out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}}
	db = db.WithContext(ctx)
	c, err := lastFinishedCrawl(db)
	if err != nil || c == nil {
		return out, err
	}
	out.CrawledAt = c.FinishedAt
	var links []models.CrawlLink
	if err := db.Select("from_url", "to_url").Where("crawl_id = ? AND internal AND from_url <> to_url AND (from_url = ? OR to_url = ?)", c.ID, path, path).Find(&links).Error; err != nil {
		return out, err
	}
	linkedFrom, linksTo := map[string]bool{}, map[string]bool{}
	for _, l := range links {
		if l.FromURL == path {
			linksTo[l.ToURL] = true
		} else {
			linkedFrom[l.FromURL] = true
		}
	}
	inbound, outbound := len(linkedFrom), len(linksTo)
	out.Inbound, out.Outbound = &inbound, &outbound
	var self []models.CrawlPage
	if err := db.Where("crawl_id = ? AND url = ?", c.ID, path).Limit(1).Find(&self).Error; err != nil {
		return out, err
	}
	keyphrases, err := entryKeyphrases(db)
	if err != nil {
		return out, err
	}
	site := s.SiteURL()
	if len(self) > 0 {
		out.Orphan = inbound == 0 && path != "/"
		h1, _ := pageHeading(self[0])
		phrases = append(phrases, h1)
		text := NormalText(self[0].Text)
		var others []models.CrawlPage
		if err := db.Select("id", "url", "title", "h1").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).Order("url ASC").Find(&others).Error; err != nil {
			return out, err
		}
		for _, o := range others {
			if len(out.Out) >= MaxLinkSuggestions {
				break
			}
			if linksTo[o.URL] {
				continue
			}
			oh1, title := pageHeading(o)
			if p := mentioned(text, phraseList(oh1, keyphrases[o.URL])); p != "" {
				out.Out = append(out.Out, LinkOut{Phrase: p, Title: title, URL: site + o.URL})
			}
		}
	}
	own := phraseList(append(phrases, keyphrases[path])...)
	if len(own) == 0 {
		return out, nil
	}
	var batch []models.CrawlPage
	err = db.Select("id", "url", "title", "h1", "text").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).
		FindInBatches(&batch, 100, func(*gorm.DB, int) error {
			for _, p := range batch {
				if len(out.In) >= MaxLinkSuggestions {
					return errEnough
				}
				if linkedFrom[p.URL] {
					continue
				}
				if phrase := mentioned(NormalText(p.Text), own); phrase != "" {
					_, title := pageHeading(p)
					out.In = append(out.In, LinkIn{Phrase: phrase, Title: title, AdminURL: s.adminURLOf(ctx, p.URL)})
				}
			}
			return nil
		}).Error
	if err != nil && !errors.Is(err, errEnough) {
		return out, err
	}
	return out, nil
}

// sitePath is the path of a path or an absolute URL; ErrBadURL otherwise.
func sitePath(raw string) (string, error) {
	raw = strings.TrimSpace(raw)
	if strings.HasPrefix(raw, "/") && !strings.HasPrefix(raw, "//") {
		path, _, _ := strings.Cut(raw, "#")
		return path, nil
	}
	u, err := url.Parse(raw)
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
		return "", ErrBadURL
	}
	path := u.EscapedPath()
	if path == "" {
		path = "/"
	}
	if u.RawQuery != "" {
		path += "?" + u.RawQuery
	}
	return path, nil
}

// LinksOfURL is the internal links of a page addressed by URL (a path or
// an absolute URL of the site): the record behind it gives its title in
// the matched locale.
func (s *Service) LinksOfURL(ctx context.Context, db *gorm.DB, raw string) (LinkReport, error) {
	path, err := sitePath(raw)
	if err != nil {
		return LinkReport{}, err
	}
	var phrases []string
	if reg := s.Registry(); reg != nil {
		p, _, _ := strings.Cut(path, "?")
		if key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale()); err == nil && match != nil && match.Exact {
			if rec, err := reg.Describe(ctx, key, match.ID, s.UnprefixedLocale()); err == nil && rec != nil {
				for _, l := range rec.Locales {
					if l.Locale == match.Locale {
						phrases = append(phrases, l.Title)
					}
				}
			}
		}
	}
	return s.LinkSuggestions(ctx, db, path, phrases)
}

// LinksOfRecord is the internal links of record id of content type key in
// locale: its address and title in that locale and its focus keyphrase. A
// record without an address in the locale answers the crawl time only.
func (s *Service) LinksOfRecord(ctx context.Context, db *gorm.DB, key, id, locale string) (LinkReport, error) {
	reg := s.Registry()
	if reg == nil {
		return LinkReport{Out: []LinkOut{}, In: []LinkIn{}}, nil
	}
	rec, err := reg.Describe(ctx, key, id, s.UnprefixedLocale())
	if err != nil {
		return LinkReport{}, err
	}
	var loc *pact.ContentLocale
	if rec != nil {
		for i := range rec.Locales {
			if rec.Locales[i].Locale == locale {
				loc = &rec.Locales[i]
			}
		}
	}
	if loc == nil || strings.TrimSpace(loc.URL) == "" {
		out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}}
		c, err := lastFinishedCrawl(db.WithContext(ctx))
		if err == nil && c != nil {
			out.CrawledAt = c.FinishedAt
		}
		return out, err
	}
	phrases := []string{loc.Title}
	if typ, ok := reg.ContentType(key); ok && typ.Content != nil {
		if entry, found, err := LoadEntry(db, typ.Content.Model, id, locale); err == nil && found {
			phrases = append(phrases, entry.Keyphrase)
		}
	}
	return s.LinkSuggestions(ctx, db, loc.URL, phrases)
}

// linksPayload is the seo_links action's payload.
type linksPayload struct {
	Op     string `json:"op"`
	Locale string `json:"locale"`
}

// RecordLinksAction is the SEO tab's internal links widget action (E10,
// UI-SPEC C19) for content type key: {op: links, locale} answers the
// record's LinkReport in that locale (the default when empty).
func RecordLinksAction(svc func() *Service, key, label, permission string) pact.AdminAction {
	return pact.AdminAction{
		Name:        LinksActionName,
		Label:       label,
		Permissions: []string{permission},
		Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
			s := svc()
			if s == nil {
				return pact.AdminActionResult{}, ErrNotBooted
			}
			var p linksPayload
			if len(in.Payload) > 0 {
				if err := json.Unmarshal(in.Payload, &p); err != nil || (p.Op != "" && p.Op != "links") {
					return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.api.value_invalid"}}}
				}
			}
			db, err := s.DB(ctx)
			if err != nil {
				return pact.AdminActionResult{}, err
			}
			if in.RecordID == nil {
				return pact.AdminActionResult{Data: LinkReport{Out: []LinkOut{}, In: []LinkIn{}}}, nil
			}
			locale := p.Locale
			if locale == "" {
				locale = s.DefaultLocale(ctx, db)
			}
			report, err := s.LinksOfRecord(ctx, db, key, strconv.FormatUint(*in.RecordID, 10), locale)
			return pact.AdminActionResult{Data: report}, err
		},
	}
}