golem15/sm-seo-plugin
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: the crawler and the Chrome guard keep to the site's port

Browse Source 
Fetcher.Allowed and Key, the rendered comparison and the Chrome request
guard accept an address only on the site's effective port (explicit,
else 443 for https and 80 for http), so page code cannot make headless
Chrome reach other services on the site host's address and a link to
another port is not reported as a page of the site.
This commit is contained in:
Jakub Zych
2026-11-04 22:56:02 +01:00
parent cfeb3dff12
commit 3fdb44bce2
9 changed files with 138 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
classes/crawl/fetch.go
View File

@@ -65,10 +65,29 @@ type Fetcher struct {
client *fetchguard.Client
site *url.URL
hosts []string
port string
trusted bool
ua string
}
// effectivePort is u's port: the explicit one, else 443 for https and 80
// for http ("" for another scheme without a port).
func effectivePort(u *url.URL) string {
if u == nil {
return ""
}
if p := u.Port(); p != "" {
return p
}
switch strings.ToLower(u.Scheme) {
case "https":
return "443"
case "http":
return "80"
}
return ""
}
// SiteHosts returns the host of site and its www twin.
func SiteHosts(host string) []string {
host = strings.ToLower(strings.TrimSpace(host))
@@ -92,7 +111,7 @@ func NewFetcher(site string, cfg *compass.Config) (*Fetcher, error) {
if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
return nil, ErrNoSiteURL
}
f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)}
f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), port: effectivePort(u), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)}
policy := fetchguard.Policy{Mode: fetchguard.AllowHostsMode, AllowHosts: f.hosts, MaxBytes: MaxBody, Timeout: FetchTimeout}
if f.trusted {
policy.Mode = fetchguard.TrustedMode
@@ -116,13 +135,21 @@ func (f *Fetcher) Site() *url.URL { return f.site }
// UA returns the crawler's User-Agent.
func (f *Fetcher) UA() string { return f.ua }
// Port returns the site's effective port (explicit, else 443 for https
// and 80 for http).
func (f *Fetcher) Port() string { return f.port }
// Allowed reports whether u may be fetched: https (http only in trusted
// mode) on the site host or its www twin, without credentials. In trusted
// mode the host list is checked here, since fetchguard checks none then.
// mode) on the site host or its www twin and on the site's effective port,
// without credentials. In trusted mode the host list is checked here,
// since fetchguard checks none then.
func (f *Fetcher) Allowed(u *url.URL) bool {
if u == nil || u.User != nil {
return false
}
if effectivePort(u) != f.port {
return false
}
switch strings.ToLower(u.Scheme) {
case "https":
case "http":
@@ -142,7 +169,7 @@ func (f *Fetcher) Allowed(u *url.URL) bool {
}
// Key is the address of u on the site: its path with the query, or ""
// when u is not on the site.
// when u is not on the site (another host, scheme or port).
func (f *Fetcher) Key(u *url.URL) string {
if !f.Allowed(u) {
return ""