feat: crawler finds a page with two H1 headings and stores the issue (tracer)
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
This commit is contained in:
43
classes/crawl/render.go
Normal file
43
classes/crawl/render.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package crawl
|
||||
|
||||
import "context"
|
||||
|
||||
// Renderer renders a page of the site in a browser for the raw versus
|
||||
// rendered comparison (D-70, P33-55): Render answers the page's DOM after
|
||||
// JavaScript ran, Close releases the browser. A crawl without a Renderer
|
||||
// records the comparison as skipped; a Render error records it as failed
|
||||
// with the error and never fails the crawl.
|
||||
type Renderer interface {
|
||||
Render(ctx context.Context, pageURL string) (string, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// DefaultRenderMax is the number of pages rendered per crawl when
|
||||
// golem15.seo.crawl_render_max is unset.
|
||||
const DefaultRenderMax = 200
|
||||
|
||||
// The render states of a crawl (models.RenderSkipped and the others).
|
||||
const (
|
||||
renderStatusSkipped = "skipped"
|
||||
renderStatusDone = "done"
|
||||
renderStatusFailed = "failed"
|
||||
)
|
||||
|
||||
type rendererKey struct{}
|
||||
|
||||
// WithRenderer returns a context whose crawls use r instead of the
|
||||
// renderer built from config. It is the test seam of the comparison,
|
||||
// settable only from Go code (like fetchguard.WithTransport): no config
|
||||
// key, setting, request or MCP call reaches it.
|
||||
func WithRenderer(ctx context.Context, r Renderer) context.Context {
|
||||
return context.WithValue(ctx, rendererKey{}, r)
|
||||
}
|
||||
|
||||
// rendererFrom returns the renderer of ctx, nil when none.
|
||||
func rendererFrom(ctx context.Context) Renderer {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
r, _ := ctx.Value(rendererKey{}).(Renderer)
|
||||
return r
|
||||
}
|
||||
Reference in New Issue
Block a user