e2af5b44c8
Crawl keys, the rendered comparison and the link graph use the decoded path (classes.PathKey), the form the resolver, the descriptors and the SEO entries use; the request URL is percent-encoded again (classes.EscapeKeyPath). A non-ASCII address now matches its CMS record and its stored links. A path that does not decode to text keeps its encoded form.
391 lines
12 KiB
Go
391 lines
12 KiB
Go
package classes
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/cabana"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// MaxLinkSuggestions caps each suggestion list of the internal links
|
|
// widget (UI-SPEC C19).
|
|
const MaxLinkSuggestions = 10
|
|
|
|
// minLinkPhrase is the shortest phrase a link suggestion matches, in
|
|
// bytes (the orphan suggestions' limit).
|
|
const minLinkPhrase = 3
|
|
|
|
// LinksActionName is the SEO tab's internal links widget action (E10).
|
|
const LinksActionName = "seo_links"
|
|
|
|
// LinkOut is a page this text could link to: the phrase found in this
|
|
// page's crawled text, the target's title and its public address.
|
|
type LinkOut struct {
|
|
Phrase string `json:"phrase"`
|
|
Title string `json:"title"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
// LinkIn is a page that could link here: the phrase of this page found in
|
|
// its crawled text, its title and its admin form ("" when no record of
|
|
// the admin is behind it).
|
|
type LinkIn struct {
|
|
Phrase string `json:"phrase"`
|
|
Title string `json:"title"`
|
|
AdminURL string `json:"adminUrl"`
|
|
}
|
|
|
|
// LinkReport is the internal links of a page from the last finished crawl
|
|
// (D-82, UI-SPEC C19): the time of that crawl, the pages linking here and
|
|
// the pages linked from here (both nil before the first crawl), whether
|
|
// nothing links to the page, and up to 10 pages this text could link to
|
|
// and 10 that could link here.
|
|
type LinkReport struct {
|
|
CrawledAt *time.Time `json:"crawledAt"`
|
|
Inbound *int `json:"inbound"`
|
|
Outbound *int `json:"outbound"`
|
|
Orphan bool `json:"orphan"`
|
|
Out []LinkOut `json:"out"`
|
|
In []LinkIn `json:"in"`
|
|
}
|
|
|
|
// NormalText is text in lower case with its whitespace collapsed to one
|
|
// space: the form the crawler's suggestions compare phrases in.
|
|
func NormalText(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) }
|
|
|
|
// errEnough stops a batched scan once a list is full.
|
|
var errEnough = errors.New("golem15.seo: enough suggestions")
|
|
|
|
// lastFinishedCrawl is the newest finished crawl, nil before the first.
|
|
func lastFinishedCrawl(db *gorm.DB) (*models.Crawl, error) {
|
|
var rows []models.Crawl
|
|
if err := db.Where("status = ?", models.CrawlFinished).Order("finished_at DESC NULLS LAST, id DESC").Limit(1).Find(&rows).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
if len(rows) == 0 {
|
|
return nil, nil
|
|
}
|
|
return &rows[0], nil
|
|
}
|
|
|
|
// pageHeading is a crawled page's first H1, else its title.
|
|
func pageHeading(p models.CrawlPage) (h1, title string) {
|
|
if list := p.H1.Get(); len(list) > 0 {
|
|
h1 = strings.TrimSpace(list[0])
|
|
}
|
|
title = h1
|
|
if title == "" {
|
|
title = strings.TrimSpace(p.Title)
|
|
}
|
|
return h1, title
|
|
}
|
|
|
|
// entryKeyphrases maps each address with an SEO entry to its focus
|
|
// keyphrase.
|
|
func entryKeyphrases(db *gorm.DB) (map[string]string, error) {
|
|
var rows []struct {
|
|
URL string
|
|
Keyphrase string
|
|
}
|
|
if err := db.Raw(`SELECT url, keyphrase FROM golem15_seo_entries WHERE url <> '' AND keyphrase <> ''`).Scan(&rows).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
out := make(map[string]string, len(rows))
|
|
for _, r := range rows {
|
|
out[r.URL] = r.Keyphrase
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// phraseList keeps the phrases of at least 3 bytes, once each (by their
|
|
// normal form), in order.
|
|
func phraseList(phrases ...string) []string {
|
|
var out []string
|
|
seen := map[string]bool{}
|
|
for _, p := range phrases {
|
|
p = strings.Join(strings.Fields(p), " ")
|
|
n := NormalText(p)
|
|
if len(n) < minLinkPhrase || seen[n] {
|
|
continue
|
|
}
|
|
seen[n] = true
|
|
out = append(out, p)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// mentioned returns the first phrase whose normal form normal (a page's
|
|
// NormalText) contains, "" when none.
|
|
func mentioned(normal string, phrases []string) string {
|
|
for _, p := range phrases {
|
|
if strings.Contains(normal, NormalText(p)) {
|
|
return p
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// adminURLOf is the admin form of the record behind a site path, "" when
|
|
// no content type with an admin controller owns it.
|
|
func (s *Service) adminURLOf(ctx context.Context, path string) string {
|
|
reg := s.Registry()
|
|
if reg == nil {
|
|
return ""
|
|
}
|
|
p, _, _ := strings.Cut(path, "?")
|
|
key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale())
|
|
if err != nil || match == nil || !match.Exact {
|
|
return ""
|
|
}
|
|
typ, ok := reg.ContentType(key)
|
|
if !ok || typ.Content == nil || strings.TrimSpace(typ.Content.Controller) == "" {
|
|
return ""
|
|
}
|
|
return s.adminFormURL(typ.Content.Controller, match.ID)
|
|
}
|
|
|
|
// LinkSuggestions builds the internal links of the page at path from the
|
|
// last finished crawl (D-82, UI-SPEC C19), like the crawl's orphan
|
|
// suggestions: the pages whose crawled text contains one of phrases (the
|
|
// page's title and focus keyphrase; its crawled H1 is added) and do not
|
|
// link here yet, and the pages whose H1 or focus keyphrase this page's
|
|
// crawled text contains and that it does not link to yet. Before the
|
|
// first crawl the counts are nil and the lists empty.
|
|
func (s *Service) LinkSuggestions(ctx context.Context, db *gorm.DB, path string, phrases []string) (LinkReport, error) {
|
|
out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}}
|
|
db = db.WithContext(ctx)
|
|
c, err := lastFinishedCrawl(db)
|
|
if err != nil || c == nil {
|
|
return out, err
|
|
}
|
|
out.CrawledAt = c.FinishedAt
|
|
var links []models.CrawlLink
|
|
if err := db.Select("from_url", "to_url").Where("crawl_id = ? AND internal AND from_url <> to_url AND (from_url = ? OR to_url = ?)", c.ID, path, path).Find(&links).Error; err != nil {
|
|
return out, err
|
|
}
|
|
linkedFrom, linksTo := map[string]bool{}, map[string]bool{}
|
|
for _, l := range links {
|
|
if l.FromURL == path {
|
|
linksTo[l.ToURL] = true
|
|
} else {
|
|
linkedFrom[l.FromURL] = true
|
|
}
|
|
}
|
|
inbound, outbound := len(linkedFrom), len(linksTo)
|
|
out.Inbound, out.Outbound = &inbound, &outbound
|
|
var self []models.CrawlPage
|
|
if err := db.Where("crawl_id = ? AND url = ?", c.ID, path).Limit(1).Find(&self).Error; err != nil {
|
|
return out, err
|
|
}
|
|
keyphrases, err := entryKeyphrases(db)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
site := s.SiteURL()
|
|
if len(self) > 0 {
|
|
out.Orphan = inbound == 0 && path != "/"
|
|
h1, _ := pageHeading(self[0])
|
|
phrases = append(phrases, h1)
|
|
text := NormalText(self[0].Text)
|
|
var others []models.CrawlPage
|
|
if err := db.Select("id", "url", "title", "h1").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).Order("url ASC").Find(&others).Error; err != nil {
|
|
return out, err
|
|
}
|
|
for _, o := range others {
|
|
if len(out.Out) >= MaxLinkSuggestions {
|
|
break
|
|
}
|
|
if linksTo[o.URL] {
|
|
continue
|
|
}
|
|
oh1, title := pageHeading(o)
|
|
if p := mentioned(text, phraseList(oh1, keyphrases[o.URL])); p != "" {
|
|
out.Out = append(out.Out, LinkOut{Phrase: p, Title: title, URL: site + o.URL})
|
|
}
|
|
}
|
|
}
|
|
own := phraseList(append(phrases, keyphrases[path])...)
|
|
if len(own) == 0 {
|
|
return out, nil
|
|
}
|
|
var batch []models.CrawlPage
|
|
err = db.Select("id", "url", "title", "h1", "text").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).
|
|
FindInBatches(&batch, 100, func(*gorm.DB, int) error {
|
|
for _, p := range batch {
|
|
if len(out.In) >= MaxLinkSuggestions {
|
|
return errEnough
|
|
}
|
|
if linkedFrom[p.URL] {
|
|
continue
|
|
}
|
|
if phrase := mentioned(NormalText(p.Text), own); phrase != "" {
|
|
_, title := pageHeading(p)
|
|
out.In = append(out.In, LinkIn{Phrase: phrase, Title: title, AdminURL: s.adminURLOf(ctx, p.URL)})
|
|
}
|
|
}
|
|
return nil
|
|
}).Error
|
|
if err != nil && !errors.Is(err, errEnough) {
|
|
return out, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// sitePath is the crawl key of a path or an absolute URL (PathKey with
|
|
// the raw query); ErrBadURL otherwise. A path and the absolute URL of the
|
|
// same page give the same answer, written decoded or percent-encoded.
|
|
func sitePath(raw string) (string, error) {
|
|
raw = strings.TrimSpace(raw)
|
|
if strings.HasPrefix(raw, "/") && !strings.HasPrefix(raw, "//") {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return "", ErrBadURL
|
|
}
|
|
return joinQuery(PathKey(u), u.RawQuery), nil
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
|
return "", ErrBadURL
|
|
}
|
|
return joinQuery(PathKey(u), u.RawQuery), nil
|
|
}
|
|
|
|
// PathKey is the path form of the crawl's addresses (P33-61), the form
|
|
// the resolver, the descriptors and the SEO entries use: u's decoded path
|
|
// ("/" when empty). A path whose decoded form is not valid UTF-8 without
|
|
// NUL, or holds "%" or "?" (which would read as an escape or a query),
|
|
// keeps its percent-encoded form.
|
|
func PathKey(u *url.URL) string {
|
|
if u == nil {
|
|
return "/"
|
|
}
|
|
p := u.Path
|
|
if !safeText(p) || strings.ContainsAny(p, "%?") {
|
|
p = u.EscapedPath()
|
|
}
|
|
if p == "" {
|
|
p = "/"
|
|
}
|
|
return p
|
|
}
|
|
|
|
// EscapeKeyPath is the request form of a PathKey path: a decoded path is
|
|
// percent-encoded, an encoded one (it holds "%") is kept.
|
|
func EscapeKeyPath(p string) string {
|
|
if strings.Contains(p, "%") {
|
|
return p
|
|
}
|
|
return (&url.URL{Path: p}).EscapedPath()
|
|
}
|
|
|
|
// LinksOfURL is the internal links of a page addressed by URL (a path or
|
|
// an absolute URL of the site): the record behind it gives its title in
|
|
// the matched locale.
|
|
func (s *Service) LinksOfURL(ctx context.Context, db *gorm.DB, raw string) (LinkReport, error) {
|
|
path, err := sitePath(raw)
|
|
if err != nil {
|
|
return LinkReport{}, err
|
|
}
|
|
var phrases []string
|
|
if reg := s.Registry(); reg != nil {
|
|
p, _, _ := strings.Cut(path, "?")
|
|
if key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale()); err == nil && match != nil && match.Exact {
|
|
if rec, err := reg.Describe(ctx, key, match.ID, s.UnprefixedLocale()); err == nil && rec != nil {
|
|
for _, l := range rec.Locales {
|
|
if l.Locale == match.Locale {
|
|
phrases = append(phrases, l.Title)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return s.LinkSuggestions(ctx, db, path, phrases)
|
|
}
|
|
|
|
// LinksOfRecord is the internal links of record id of content type key in
|
|
// locale: its address and title in that locale and its focus keyphrase. A
|
|
// record without an address in the locale answers the crawl time only.
|
|
func (s *Service) LinksOfRecord(ctx context.Context, db *gorm.DB, key, id, locale string) (LinkReport, error) {
|
|
reg := s.Registry()
|
|
if reg == nil {
|
|
return LinkReport{Out: []LinkOut{}, In: []LinkIn{}}, nil
|
|
}
|
|
rec, err := reg.Describe(ctx, key, id, s.UnprefixedLocale())
|
|
if err != nil {
|
|
return LinkReport{}, err
|
|
}
|
|
var loc *pact.ContentLocale
|
|
if rec != nil {
|
|
for i := range rec.Locales {
|
|
if rec.Locales[i].Locale == locale {
|
|
loc = &rec.Locales[i]
|
|
}
|
|
}
|
|
}
|
|
if loc == nil || strings.TrimSpace(loc.URL) == "" {
|
|
out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}}
|
|
c, err := lastFinishedCrawl(db.WithContext(ctx))
|
|
if err == nil && c != nil {
|
|
out.CrawledAt = c.FinishedAt
|
|
}
|
|
return out, err
|
|
}
|
|
phrases := []string{loc.Title}
|
|
if typ, ok := reg.ContentType(key); ok && typ.Content != nil {
|
|
if entry, found, err := LoadEntry(db, typ.Content.Model, id, locale); err == nil && found {
|
|
phrases = append(phrases, entry.Keyphrase)
|
|
}
|
|
}
|
|
return s.LinkSuggestions(ctx, db, loc.URL, phrases)
|
|
}
|
|
|
|
// linksPayload is the seo_links action's payload.
|
|
type linksPayload struct {
|
|
Op string `json:"op"`
|
|
Locale string `json:"locale"`
|
|
}
|
|
|
|
// RecordLinksAction is the SEO tab's internal links widget action (E10,
|
|
// UI-SPEC C19) for content type key: {op: links, locale} answers the
|
|
// record's LinkReport in that locale (the default when empty).
|
|
func RecordLinksAction(svc func() *Service, key, label, permission string) pact.AdminAction {
|
|
return pact.AdminAction{
|
|
Name: LinksActionName,
|
|
Label: label,
|
|
Permissions: []string{permission},
|
|
Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
|
|
s := svc()
|
|
if s == nil {
|
|
return pact.AdminActionResult{}, ErrNotBooted
|
|
}
|
|
var p linksPayload
|
|
if len(in.Payload) > 0 {
|
|
if err := json.Unmarshal(in.Payload, &p); err != nil || (p.Op != "" && p.Op != "links") {
|
|
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.api.value_invalid"}}}
|
|
}
|
|
}
|
|
db, err := s.DB(ctx)
|
|
if err != nil {
|
|
return pact.AdminActionResult{}, err
|
|
}
|
|
if in.RecordID == nil {
|
|
return pact.AdminActionResult{Data: LinkReport{Out: []LinkOut{}, In: []LinkIn{}}}, nil
|
|
}
|
|
locale := p.Locale
|
|
if locale == "" {
|
|
locale = s.DefaultLocale(ctx, db)
|
|
}
|
|
report, err := s.LinksOfRecord(ctx, db, key, strconv.FormatUint(*in.RecordID, 10), locale)
|
|
return pact.AdminActionResult{Data: report}, err
|
|
},
|
|
}
|
|
}
|