golem15/sm-seo-plugin
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: the crawler and the Chrome guard keep to the site's port

Browse Source 
Fetcher.Allowed and Key, the rendered comparison and the Chrome request
guard accept an address only on the site's effective port (explicit,
else 443 for https and 80 for http), so page code cannot make headless
Chrome reach other services on the site host's address and a link to
another port is not reported as a page of the site.
This commit is contained in:
Jakub Zych
2026-11-04 22:56:02 +01:00
parent cfeb3dff12
commit 3fdb44bce2
9 changed files with 138 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
classes/crawl/compare_test.go
View File

@@ -18,7 +18,15 @@ func linkFacts(words int, h1 string, hrefs ...string) *PageFacts {
return f
}
var siteHosts = []string{"example.test", "www.example.test"}
// siteKey is the crawl key of a link on https://example.test (its www
// twin too, port 443 only).
func siteKey(u *url.URL) string {
f, err := NewFetcher("https://example.test", nil)
if err != nil {
panic(err)
}
return f.Key(u)
}
// TestCompareRendered pins the rendered comparison (P33-55): a same-host
// link (the www twin too), an H1 and growth of at least 50 words and 20 %
@@ -28,7 +36,7 @@ var siteHosts = []string{"example.test", "www.example.test"}
func TestCompareRendered(t *testing.T) {
raw := linkFacts(100, "", "https://example.test/a")
rendered := linkFacts(160, "Grzyby", "https://example.test/a", "https://www.example.test/b?x=1", "https://other.test/c", "https://example.test/b?x=1")
got := compareRendered(raw, rendered, siteHosts)
got := compareRendered(raw, rendered, siteKey)
if len(got) != 1 || got[0].Kind != KindContentJS || got[0].Check != checkRendered {
t.Fatalf("findings = %+v", got)
}
@@ -47,14 +55,14 @@ func TestCompareRendered(t *testing.T) {
"50 words under 20 %": {linkFacts(300, "T"), linkFacts(350, "T")},
"H1 already there": {linkFacts(10, "Raw"), linkFacts(10, "Rendered")},
} {
if got := compareRendered(pair[0], pair[1], siteHosts); got != nil {
if got := compareRendered(pair[0], pair[1], siteKey); got != nil {
t.Errorf("%s: %+v", name, got)
}
}
if got := compareRendered(linkFacts(0, "T"), linkFacts(50, "T"), siteHosts); len(got) != 1 {
if got := compareRendered(linkFacts(0, "T"), linkFacts(50, "T"), siteKey); len(got) != 1 {
t.Fatalf("50 words from an empty page: %+v", got)
}
if compareRendered(nil, rendered, siteHosts) != nil || compareRendered(raw, nil, siteHosts) != nil {
if compareRendered(nil, rendered, siteKey) != nil || compareRendered(raw, nil, siteKey) != nil {
t.Fatal("a missing side compares to nothing")
}
}
@@ -66,7 +74,7 @@ func TestCompareRenderedCap(t *testing.T) {
for _, p := range []string{"", "/1", "/2", "/3", "/4", "/5", "/6"} {
hrefs = append(hrefs, "https://example.test"+p)
}
got := compareRendered(linkFacts(10, "T"), linkFacts(10, "T", hrefs...), siteHosts)
got := compareRendered(linkFacts(10, "T"), linkFacts(10, "T", hrefs...), siteKey)
links := got[0].Details["links"].([]string)
if len(links) != maxJSOnlyURLs || links[0] != "/" {
t.Fatalf("links = %v", links)