package classes import ( "context" "encoding/json" "errors" "net/url" "strconv" "strings" "time" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" ) // MaxLinkSuggestions caps each suggestion list of the internal links // widget (UI-SPEC C19). const MaxLinkSuggestions = 10 // minLinkPhrase is the shortest phrase a link suggestion matches, in // bytes (the orphan suggestions' limit). const minLinkPhrase = 3 // LinksActionName is the SEO tab's internal links widget action (E10). const LinksActionName = "seo_links" // LinkOut is a page this text could link to: the phrase found in this // page's crawled text, the target's title and its public address. type LinkOut struct { Phrase string `json:"phrase"` Title string `json:"title"` URL string `json:"url"` } // LinkIn is a page that could link here: the phrase of this page found in // its crawled text, its title and its admin form ("" when no record of // the admin is behind it). type LinkIn struct { Phrase string `json:"phrase"` Title string `json:"title"` AdminURL string `json:"adminUrl"` } // LinkReport is the internal links of a page from the last finished crawl // (D-82, UI-SPEC C19): the time of that crawl, the pages linking here and // the pages linked from here (both nil before the first crawl), whether // nothing links to the page, and up to 10 pages this text could link to // and 10 that could link here. type LinkReport struct { CrawledAt *time.Time `json:"crawledAt"` Inbound *int `json:"inbound"` Outbound *int `json:"outbound"` Orphan bool `json:"orphan"` Out []LinkOut `json:"out"` In []LinkIn `json:"in"` } // NormalText is text in lower case with its whitespace collapsed to one // space: the form the crawler's suggestions compare phrases in. func NormalText(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) } // errEnough stops a batched scan once a list is full. var errEnough = errors.New("golem15.seo: enough suggestions") // lastFinishedCrawl is the newest finished crawl, nil before the first. func lastFinishedCrawl(db *gorm.DB) (*models.Crawl, error) { var rows []models.Crawl if err := db.Where("status = ?", models.CrawlFinished).Order("finished_at DESC NULLS LAST, id DESC").Limit(1).Find(&rows).Error; err != nil { return nil, err } if len(rows) == 0 { return nil, nil } return &rows[0], nil } // pageHeading is a crawled page's first H1, else its title. func pageHeading(p models.CrawlPage) (h1, title string) { if list := p.H1.Get(); len(list) > 0 { h1 = strings.TrimSpace(list[0]) } title = h1 if title == "" { title = strings.TrimSpace(p.Title) } return h1, title } // entryKeyphrases maps each address with an SEO entry to its focus // keyphrase. func entryKeyphrases(db *gorm.DB) (map[string]string, error) { var rows []struct { URL string Keyphrase string } if err := db.Raw(`SELECT url, keyphrase FROM golem15_seo_entries WHERE url <> '' AND keyphrase <> ''`).Scan(&rows).Error; err != nil { return nil, err } out := make(map[string]string, len(rows)) for _, r := range rows { out[r.URL] = r.Keyphrase } return out, nil } // phraseList keeps the phrases of at least 3 bytes, once each (by their // normal form), in order. func phraseList(phrases ...string) []string { var out []string seen := map[string]bool{} for _, p := range phrases { p = strings.Join(strings.Fields(p), " ") n := NormalText(p) if len(n) < minLinkPhrase || seen[n] { continue } seen[n] = true out = append(out, p) } return out } // mentioned returns the first phrase whose normal form normal (a page's // NormalText) contains, "" when none. func mentioned(normal string, phrases []string) string { for _, p := range phrases { if strings.Contains(normal, NormalText(p)) { return p } } return "" } // adminURLOf is the admin form of the record behind a site path, "" when // no content type with an admin controller owns it. func (s *Service) adminURLOf(ctx context.Context, path string) string { reg := s.Registry() if reg == nil { return "" } p, _, _ := strings.Cut(path, "?") key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale()) if err != nil || match == nil || !match.Exact { return "" } typ, ok := reg.ContentType(key) if !ok || typ.Content == nil || strings.TrimSpace(typ.Content.Controller) == "" { return "" } return s.adminFormURL(typ.Content.Controller, match.ID) } // LinkSuggestions builds the internal links of the page at path from the // last finished crawl (D-82, UI-SPEC C19), like the crawl's orphan // suggestions: the pages whose crawled text contains one of phrases (the // page's title and focus keyphrase; its crawled H1 is added) and do not // link here yet, and the pages whose H1 or focus keyphrase this page's // crawled text contains and that it does not link to yet. Before the // first crawl the counts are nil and the lists empty. func (s *Service) LinkSuggestions(ctx context.Context, db *gorm.DB, path string, phrases []string) (LinkReport, error) { out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}} db = db.WithContext(ctx) c, err := lastFinishedCrawl(db) if err != nil || c == nil { return out, err } out.CrawledAt = c.FinishedAt var links []models.CrawlLink if err := db.Select("from_url", "to_url").Where("crawl_id = ? AND internal AND from_url <> to_url AND (from_url = ? OR to_url = ?)", c.ID, path, path).Find(&links).Error; err != nil { return out, err } linkedFrom, linksTo := map[string]bool{}, map[string]bool{} for _, l := range links { if l.FromURL == path { linksTo[l.ToURL] = true } else { linkedFrom[l.FromURL] = true } } inbound, outbound := len(linkedFrom), len(linksTo) out.Inbound, out.Outbound = &inbound, &outbound var self []models.CrawlPage if err := db.Where("crawl_id = ? AND url = ?", c.ID, path).Limit(1).Find(&self).Error; err != nil { return out, err } keyphrases, err := entryKeyphrases(db) if err != nil { return out, err } site := s.SiteURL() if len(self) > 0 { out.Orphan = inbound == 0 && path != "/" h1, _ := pageHeading(self[0]) phrases = append(phrases, h1) text := NormalText(self[0].Text) var others []models.CrawlPage if err := db.Select("id", "url", "title", "h1").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path).Order("url ASC").Find(&others).Error; err != nil { return out, err } for _, o := range others { if len(out.Out) >= MaxLinkSuggestions { break } if linksTo[o.URL] { continue } oh1, title := pageHeading(o) if p := mentioned(text, phraseList(oh1, keyphrases[o.URL])); p != "" { out.Out = append(out.Out, LinkOut{Phrase: p, Title: title, URL: site + o.URL}) } } } own := phraseList(append(phrases, keyphrases[path])...) if len(own) == 0 { return out, nil } var batch []models.CrawlPage err = db.Select("id", "url", "title", "h1", "text").Where("crawl_id = ? AND url <> ? AND status = 200", c.ID, path). FindInBatches(&batch, 100, func(*gorm.DB, int) error { for _, p := range batch { if len(out.In) >= MaxLinkSuggestions { return errEnough } if linkedFrom[p.URL] { continue } if phrase := mentioned(NormalText(p.Text), own); phrase != "" { _, title := pageHeading(p) out.In = append(out.In, LinkIn{Phrase: phrase, Title: title, AdminURL: s.adminURLOf(ctx, p.URL)}) } } return nil }).Error if err != nil && !errors.Is(err, errEnough) { return out, err } return out, nil } // sitePath is the crawl key of a path or an absolute URL (PathKey with // the raw query); ErrBadURL otherwise. A path and the absolute URL of the // same page give the same answer, written decoded or percent-encoded. func sitePath(raw string) (string, error) { raw = strings.TrimSpace(raw) if strings.HasPrefix(raw, "/") && !strings.HasPrefix(raw, "//") { u, err := url.Parse(raw) if err != nil { return "", ErrBadURL } return joinQuery(PathKey(u), u.RawQuery), nil } u, err := url.Parse(raw) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return "", ErrBadURL } return joinQuery(PathKey(u), u.RawQuery), nil } // PathKey is the path form of the crawl's addresses (P33-61), the form // the resolver, the descriptors and the SEO entries use: u's decoded path // ("/" when empty). A path whose decoded form is not valid UTF-8 without // NUL, or holds "%" or "?" (which would read as an escape or a query), // keeps its percent-encoded form. func PathKey(u *url.URL) string { if u == nil { return "/" } p := u.Path if !safeText(p) || strings.ContainsAny(p, "%?") { p = u.EscapedPath() } if p == "" { p = "/" } return p } // EscapeKeyPath is the request form of a PathKey path: a decoded path is // percent-encoded, an encoded one (it holds "%") is kept. func EscapeKeyPath(p string) string { if strings.Contains(p, "%") { return p } return (&url.URL{Path: p}).EscapedPath() } // LinksOfURL is the internal links of a page addressed by URL (a path or // an absolute URL of the site): the record behind it gives its title in // the matched locale. func (s *Service) LinksOfURL(ctx context.Context, db *gorm.DB, raw string) (LinkReport, error) { path, err := sitePath(raw) if err != nil { return LinkReport{}, err } var phrases []string if reg := s.Registry(); reg != nil { p, _, _ := strings.Cut(path, "?") if key, match, err := reg.MatchPath(ctx, p, s.UnprefixedLocale()); err == nil && match != nil && match.Exact { if rec, err := reg.Describe(ctx, key, match.ID, s.UnprefixedLocale()); err == nil && rec != nil { for _, l := range rec.Locales { if l.Locale == match.Locale { phrases = append(phrases, l.Title) } } } } } return s.LinkSuggestions(ctx, db, path, phrases) } // LinksOfRecord is the internal links of record id of content type key in // locale: its address and title in that locale and its focus keyphrase. A // record without an address in the locale answers the crawl time only. func (s *Service) LinksOfRecord(ctx context.Context, db *gorm.DB, key, id, locale string) (LinkReport, error) { reg := s.Registry() if reg == nil { return LinkReport{Out: []LinkOut{}, In: []LinkIn{}}, nil } rec, err := reg.Describe(ctx, key, id, s.UnprefixedLocale()) if err != nil { return LinkReport{}, err } var loc *pact.ContentLocale if rec != nil { for i := range rec.Locales { if rec.Locales[i].Locale == locale { loc = &rec.Locales[i] } } } if loc == nil || strings.TrimSpace(loc.URL) == "" { out := LinkReport{Out: []LinkOut{}, In: []LinkIn{}} c, err := lastFinishedCrawl(db.WithContext(ctx)) if err == nil && c != nil { out.CrawledAt = c.FinishedAt } return out, err } phrases := []string{loc.Title} if typ, ok := reg.ContentType(key); ok && typ.Content != nil { if entry, found, err := LoadEntry(db, typ.Content.Model, id, locale); err == nil && found { phrases = append(phrases, entry.Keyphrase) } } return s.LinkSuggestions(ctx, db, loc.URL, phrases) } // linksPayload is the seo_links action's payload. type linksPayload struct { Op string `json:"op"` Locale string `json:"locale"` } // RecordLinksAction is the SEO tab's internal links widget action (E10, // UI-SPEC C19) for content type key: {op: links, locale} answers the // record's LinkReport in that locale (the default when empty). func RecordLinksAction(svc func() *Service, key, label, permission string) pact.AdminAction { return pact.AdminAction{ Name: LinksActionName, Label: label, Permissions: []string{permission}, Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) { s := svc() if s == nil { return pact.AdminActionResult{}, ErrNotBooted } var p linksPayload if len(in.Payload) > 0 { if err := json.Unmarshal(in.Payload, &p); err != nil || (p.Op != "" && p.Op != "links") { return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.api.value_invalid"}}} } } db, err := s.DB(ctx) if err != nil { return pact.AdminActionResult{}, err } if in.RecordID == nil { return pact.AdminActionResult{Data: LinkReport{Out: []LinkOut{}, In: []LinkIn{}}}, nil } locale := p.Locale if locale == "" { locale = s.DefaultLocale(ctx, db) } report, err := s.LinksOfRecord(ctx, db, key, strconv.FormatUint(*in.RecordID, 10), locale) return pact.AdminActionResult{Data: report}, err }, } }