package crawl import ( "strings" "git.golem15.com/golem15/summercms/modules/pact" ) // maxSuggestions caps the pages suggested to link to an orphan. const maxSuggestions = 10 // minPhrase is the shortest phrase a suggestion matches (in bytes). const minPhrase = 3 // checkOrphans reports orphan_page (D-82): an indexable page a sitemap // lists that no crawled page links to (the home page is the crawl's // entry and never an orphan). Its suggestions are the pages whose main // text mentions the orphan's title or its focus keyphrase; the first // suggestion's record names the tool that adds the link, else the fix is // a link in the site's code. func checkOrphans(res *Result, env *Env) []Finding { links := inbound(res) var out []Finding for _, key := range res.Order { p := res.Pages[key] if key == HomePath || !p.InSitemap || !env.indexable(res, p) || len(links[key]) > 0 { continue } rec := env.record(key) var phrases []string add := func(s string) { if s = normal(s); len(s) >= minPhrase { for _, have := range phrases { if have == s { return } } phrases = append(phrases, s) } } if rec != nil { add(rec.Title) } if h1 := p.Facts.H1(); len(h1) > 0 { add(h1[0]) } add(p.Facts.Title) if env.Keyphrases != nil { add(env.Keyphrases[key]) } var suggestions []map[string]any var first *Record for _, other := range res.Order { if len(suggestions) >= maxSuggestions { break } op := res.Pages[other] if other == key || !op.OK() { continue } text := normal(op.Facts.Text) for _, phrase := range phrases { if strings.Contains(text, phrase) { suggestions = append(suggestions, map[string]any{"url": other, "phrase": phrase}) if first == nil { first = env.record(other) } break } } } f := Finding{Kind: KindOrphanPage, URL: key, Locale: locale(rec), Record: first, Details: map[string]any{"count": len(suggestions), "suggestions": suggestions}} if first != nil { f.Content = []string{pact.ContentToolEdit} } out = append(out, f) } return out }