feat: internal links widget with counts and suggestions
- E10 golem15-seo-links in the SEO tab after the graph preview (types with a body): inbound and outbound counts from the last finished crawl, the orphan warning, up to 10 pages to link to (Copy address) and 10 that could link here - classes/linkgraph.go builds the report from the crawl's pages and links; the crawler's orphan suggestions share NormalText - GET /_seo/api/v1/links and seo_get_link_suggestions; widget action seo_links mapped in AdminOperations - element bundle rebuilt
This commit is contained in:
Jakub Zych
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
@@ -142,4 +143,4 @@ func Fingerprint(kind, url, locale, check string) string {
|
||||
}
|
||||
|
||||
// normal is text compared without case and extra whitespace.
|
||||
func normal(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) }
|
||||
func normal(s string) string { return classes.NormalText(s) }
|
||||
|
||||
364
classes/linkgraph.go
Normal file
364
classes/linkgraph.go
Normal file
@@ -0,0 +1,364 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// MaxLinkSuggestions caps each suggestion list of the internal links
|
||||
// widget (UI-SPEC C19).
|
||||
const MaxLinkSuggestions = 10
|
||||
|
||||
// minLinkPhrase is the shortest phrase a link suggestion matches, in
|
||||
// bytes (the orphan suggestions' limit).
|
||||
const minLinkPhrase = 3
|
||||
|
||||
// LinksActionName is the SEO tab's internal links widget action (E10).
|
||||
const LinksActionName = "seo_links"
|
||||
|
||||
// LinkOut is a page this text could link to: the phrase found in this
|
||||
// page's crawled text, the target's title and its public address.
|
||||
type LinkOut struct {
|
||||
Phrase string `json:"phrase"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// LinkIn is a page that could link here: the phrase of this page found in
|
||||
// its crawled text, its title and its admin form ("" when no record of
|
||||
// the admin is behind it).
|
||||
type LinkIn struct {
|
||||
Phrase string `json:"phrase"`
|
||||
Title string `json:"title"`
|
||||
AdminURL string `json:"adminUrl"`
|
||||
}
|
||||
|
||||
// LinkReport is the internal links of a page from the last finished crawl
|
||||
// (D-82, UI-SPEC C19): the time of that crawl, the pages linking here and
|
||||
// the pages linked from here (both nil before the first crawl), whether
|
||||
// nothing links to the page, and up to 10 pages this text could link to
|
||||
// and 10 that could link here.
|
||||
type LinkReport struct {
|
||||
CrawledAt *time.Time `json:"crawledAt"`
|
||||
Inbound *int `json:"inbound"`
|
||||
Outbound *int `json:"outbound"`
|
||||
Orphan bool `json:"orphan"`
|
||||
Out []LinkOut `json:"out"`
|
||||
In []LinkIn `json:"in"`
|
||||
}
|
||||
|
||||
// NormalText is text in lower case with its whitespace collapsed to one
|
||||
// space: the form the crawler's suggestions compare phrases in.
|
||||
func NormalText(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) }
|
||||
|
||||
// errEnough stops a batched scan once a list is full.
|
||||
var errEnough = errors.New("golem15.seo: enough suggestions")
|
||||
|
||||
// lastFinishedCrawl is the newest finished crawl, nil before the first.
|
||||
func lastFinishedCrawl(db *gorm.DB) (*models.Crawl, error) {
|
||||
var rows []models.Crawl
|
||||
if err := db.Where("status = ?", models.CrawlFinished).Order("finished_at DESC NULLS LAST, id DESC").Limit(1).Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &rows[0], nil
|
||||
}
|
||||
|
||||
// pageHeading is a crawled page's first H1, else its title.
|
||||
func pageHeading(p models.CrawlPage) (h1, title string) {
|
||||
if list := p.H1.Get(); len(list) > 0 {
|
||||
h1 = strings.TrimSpace(list[0])
|
||||
}
|
||||
title = h1
|
||||
if title == "" {
|
||||
title = strings.TrimSpace(p.Title)
|
||||
}
|
||||
return h1, title
|
||||
}
|
||||
|
||||
// entryKeyphrases maps each address with an SEO entry to its focus
|
||||
// keyphrase.
|
||||
func entryKeyphrases(db *gorm.DB) (map[string]string, error) {
|
||||
var rows []struct {
|
||||
URL string
|
||||
Keyphrase string
|
||||
}
|
||||
if err := db.Raw(`SELECT url, keyphrase FROM golem15_seo_entries WHERE url <> '' AND keyphrase <> ''`).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]string, len(rows))
|
||||
for _, r := range rows {
|
||||
out[r.URL] = r.Keyphrase
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// phraseList keeps the phrases of at least 3 bytes, once each (by their
|
||||
// normal form), in order.
|
||||
func phraseList(phrases ...string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, p := range phrases {
|
||||
p = strings.Join(strings.Fields(p), " ")
|
||||
n := NormalText(p)
|
||||
if len(n) < minLinkPhrase || seen[n] {
|
||||
continue
|
||||
}
|
||||
seen[n] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mentioned returns the first phrase whose normal form normal (a page's
|
||||
// NormalText) contains, "" when none.
|
||||
func mentioned(normal string, phrases []string) string {
|
||||
for _, p := range phrases {
|
||||
if strings.Contains(normal, NormalText(p)) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// adminURLOf is the admin form of the record behind a site path, "" when
|
||||
// no content type with an admin controller owns it.
|
||||
func (s *Service) adminURLOf(ctx context.Context, path string) string {
|
||||
reg := s.Registry()
|
||||
if reg == nil {
|
||||
return ""
|
||||
}
|
||||
p, _, _ := strings.Cut(path, "?")
|
||||
key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale())
|
||||
if err != nil || match == nil || !match.Exact {
|
||||
return ""
|
||||
}
|
||||
typ, ok := reg.ContentType(key)
|
||||
if !ok || typ.Content == nil || strings.TrimSpace(typ.Content.Controller) == "" {
|
||||
return ""
|
||||
}
|
||||
return s.adminFormURL(typ.Content.Controller, match.ID)
|
||||
}
|
||||
|
||||
// LinkSuggestions builds the internal links of the page at path from the
|
||||
// last finished crawl (D-82, UI-SPEC C19), like the crawl's orphan
|
||||
// suggestions: the pages whose crawled text contains one of phrases (the
|
||||
// page's title and focus keyphrase; its crawled H1 is added) and do not
|
||||
// link here yet, and the pages whose H1 or focus keyphrase this page's
|
||||
// crawled text contains and that it does not link to yet. Before the
|
||||
// first crawl the counts are nil and the lists empty.
|
||||
func (s *Service) LinkSuggestions(ctx context.Context, db *gorm.DB, path string, phrases []string) (LinkReport, error) {
|
||||
out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}}
|
||||
db = db.WithContext(ctx)
|
||||
c, err := lastFinishedCrawl(db)
|
||||
if err != nil || c == nil {
|
||||
return out, err
|
||||
}
|
||||
out.CrawledAt = c.FinishedAt
|
||||
var links []models.CrawlLink
|
||||
if err := db.Select("from_url", "to_url").Where("crawl_id = ? AND internal AND from_url <> to_url AND (from_url = ? OR to_url = ?)", c.ID, path, path).Find(&links).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
linkedFrom, linksTo := map[string]bool{}, map[string]bool{}
|
||||
for _, l := range links {
|
||||
if l.FromURL == path {
|
||||
linksTo[l.ToURL] = true
|
||||
} else {
|
||||
linkedFrom[l.FromURL] = true
|
||||
}
|
||||
}
|
||||
inbound, outbound := len(linkedFrom), len(linksTo)
|
||||
out.Inbound, out.Outbound = &inbound, &outbound
|
||||
var self []models.CrawlPage
|
||||
if err := db.Where("crawl_id = ? AND url = ?", c.ID, path).Limit(1).Find(&self).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
keyphrases, err := entryKeyphrases(db)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
site := s.SiteURL()
|
||||
if len(self) > 0 {
|
||||
out.Orphan = inbound == 0 && path != "/"
|
||||
h1, _ := pageHeading(self[0])
|
||||
phrases = append(phrases, h1)
|
||||
text := NormalText(self[0].Text)
|
||||
var others []models.CrawlPage
|
||||
if err := db.Select("id", "url", "title", "h1").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).Order("url ASC").Find(&others).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, o := range others {
|
||||
if len(out.Out) >= MaxLinkSuggestions {
|
||||
break
|
||||
}
|
||||
if linksTo[o.URL] {
|
||||
continue
|
||||
}
|
||||
oh1, title := pageHeading(o)
|
||||
if p := mentioned(text, phraseList(oh1, keyphrases[o.URL])); p != "" {
|
||||
out.Out = append(out.Out, LinkOut{Phrase: p, Title: title, URL: site + o.URL})
|
||||
}
|
||||
}
|
||||
}
|
||||
own := phraseList(append(phrases, keyphrases[path])...)
|
||||
if len(own) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var batch []models.CrawlPage
|
||||
err = db.Select("id", "url", "title", "h1", "text").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).
|
||||
FindInBatches(&batch, 100, func(*gorm.DB, int) error {
|
||||
for _, p := range batch {
|
||||
if len(out.In) >= MaxLinkSuggestions {
|
||||
return errEnough
|
||||
}
|
||||
if linkedFrom[p.URL] {
|
||||
continue
|
||||
}
|
||||
if phrase := mentioned(NormalText(p.Text), own); phrase != "" {
|
||||
_, title := pageHeading(p)
|
||||
out.In = append(out.In, LinkIn{Phrase: phrase, Title: title, AdminURL: s.adminURLOf(ctx, p.URL)})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}).Error
|
||||
if err != nil && !errors.Is(err, errEnough) {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sitePath is the path of a path or an absolute URL; ErrBadURL otherwise.
|
||||
func sitePath(raw string) (string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if strings.HasPrefix(raw, "/") && !strings.HasPrefix(raw, "//") {
|
||||
path, _, _ := strings.Cut(raw, "#")
|
||||
return path, nil
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return "", ErrBadURL
|
||||
}
|
||||
path := u.EscapedPath()
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
path += "?" + u.RawQuery
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// LinksOfURL is the internal links of a page addressed by URL (a path or
|
||||
// an absolute URL of the site): the record behind it gives its title in
|
||||
// the matched locale.
|
||||
func (s *Service) LinksOfURL(ctx context.Context, db *gorm.DB, raw string) (LinkReport, error) {
|
||||
path, err := sitePath(raw)
|
||||
if err != nil {
|
||||
return LinkReport{}, err
|
||||
}
|
||||
var phrases []string
|
||||
if reg := s.Registry(); reg != nil {
|
||||
p, _, _ := strings.Cut(path, "?")
|
||||
if key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale()); err == nil && match != nil && match.Exact {
|
||||
if rec, err := reg.Describe(ctx, key, match.ID, s.UnprefixedLocale()); err == nil && rec != nil {
|
||||
for _, l := range rec.Locales {
|
||||
if l.Locale == match.Locale {
|
||||
phrases = append(phrases, l.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.LinkSuggestions(ctx, db, path, phrases)
|
||||
}
|
||||
|
||||
// LinksOfRecord is the internal links of record id of content type key in
|
||||
// locale: its address and title in that locale and its focus keyphrase. A
|
||||
// record without an address in the locale answers the crawl time only.
|
||||
func (s *Service) LinksOfRecord(ctx context.Context, db *gorm.DB, key, id, locale string) (LinkReport, error) {
|
||||
reg := s.Registry()
|
||||
if reg == nil {
|
||||
return LinkReport{Out: []LinkOut{}, In: []LinkIn{}}, nil
|
||||
}
|
||||
rec, err := reg.Describe(ctx, key, id, s.UnprefixedLocale())
|
||||
if err != nil {
|
||||
return LinkReport{}, err
|
||||
}
|
||||
var loc *pact.ContentLocale
|
||||
if rec != nil {
|
||||
for i := range rec.Locales {
|
||||
if rec.Locales[i].Locale == locale {
|
||||
loc = &rec.Locales[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if loc == nil || strings.TrimSpace(loc.URL) == "" {
|
||||
out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}}
|
||||
c, err := lastFinishedCrawl(db.WithContext(ctx))
|
||||
if err == nil && c != nil {
|
||||
out.CrawledAt = c.FinishedAt
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
phrases := []string{loc.Title}
|
||||
if typ, ok := reg.ContentType(key); ok && typ.Content != nil {
|
||||
if entry, found, err := LoadEntry(db, typ.Content.Model, id, locale); err == nil && found {
|
||||
phrases = append(phrases, entry.Keyphrase)
|
||||
}
|
||||
}
|
||||
return s.LinkSuggestions(ctx, db, loc.URL, phrases)
|
||||
}
|
||||
|
||||
// linksPayload is the seo_links action's payload.
|
||||
type linksPayload struct {
|
||||
Op string `json:"op"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
// RecordLinksAction is the SEO tab's internal links widget action (E10,
|
||||
// UI-SPEC C19) for content type key: {op: links, locale} answers the
|
||||
// record's LinkReport in that locale (the default when empty).
|
||||
func RecordLinksAction(svc func() *Service, key, label, permission string) pact.AdminAction {
|
||||
return pact.AdminAction{
|
||||
Name: LinksActionName,
|
||||
Label: label,
|
||||
Permissions: []string{permission},
|
||||
Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
|
||||
s := svc()
|
||||
if s == nil {
|
||||
return pact.AdminActionResult{}, ErrNotBooted
|
||||
}
|
||||
var p linksPayload
|
||||
if len(in.Payload) > 0 {
|
||||
if err := json.Unmarshal(in.Payload, &p); err != nil || (p.Op != "" && p.Op != "links") {
|
||||
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.api.value_invalid"}}}
|
||||
}
|
||||
}
|
||||
db, err := s.DB(ctx)
|
||||
if err != nil {
|
||||
return pact.AdminActionResult{}, err
|
||||
}
|
||||
if in.RecordID == nil {
|
||||
return pact.AdminActionResult{Data: LinkReport{Out: []LinkOut{}, In: []LinkIn{}}}, nil
|
||||
}
|
||||
locale := p.Locale
|
||||
if locale == "" {
|
||||
locale = s.DefaultLocale(ctx, db)
|
||||
}
|
||||
report, err := s.LinksOfRecord(ctx, db, key, strconv.FormatUint(*in.RecordID, 10), locale)
|
||||
return pact.AdminActionResult{Data: report}, err
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user