feat(11.1-02): check module identifiers in docs and READMEs

- go/parser index of every modules/ package and sub-package, with methods,
  fields, interface methods and promoted members
- code spans in docs pages, module READMEs and the root README fail
  Check and docs:build when the named identifier does not exist
- scripts/check-phase11.1.sh with preconditions, deps, docs, forbidden,
  go and a self-test that plants one violation per rule
This commit is contained in:
Jakub Zych
2026-09-30 21:35:56 +02:00
parent 9dcc101776
commit f4605292b5
6 changed files with 757 additions and 0 deletions

View File

@@ -0,0 +1,397 @@
package docsite
import (
"bytes"
"context"
"errors"
"fmt"
"go/ast"
"go/parser"
"go/token"
"io/fs"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
gast "github.com/yuin/goldmark/ast"
)
// identIndex holds the declared names of every Go package under modules/,
// keyed by the package directory's last path element (lagoon, attach,
// centrifugo), so `pkg.Ident` code spans can be checked without a type
// checker.
type identIndex struct {
root string
pkgs map[string]*pkgIdents
// docs caches go doc fallback results by "dir query".
docs map[string]bool
}
// pkgIdents is the declared names of one package.
type pkgIdents struct {
// dir is the repository-relative directory, such as "modules/lagoon".
dir string
// names holds top-level funcs, types, consts and vars.
names map[string]bool
// members maps a type name to its methods, struct fields (embedded
// type names included) and interface methods.
members map[string]map[string]bool
// embeds maps a type name to the types it embeds, for promoted
// members (go doc does not resolve promotion).
embeds map[string][]string
}
func (p *pkgIdents) addMember(typ, name string) {
if p.members[typ] == nil {
p.members[typ] = map[string]bool{}
}
p.members[typ][name] = true
}
// buildIdentIndex parses the non-test Go files of every directory under
// <root>/modules (sub-packages included, testdata, dot and underscore
// directories skipped). Two directories with the same last path element
// are a problem: a span cannot say which one it means.
func buildIdentIndex(root string) (*identIndex, []Problem, error) {
idx := &identIndex{root: root, pkgs: map[string]*pkgIdents{}, docs: map[string]bool{}}
base := filepath.Join(root, "modules")
if _, err := os.Stat(base); errors.Is(err, fs.ErrNotExist) {
return idx, nil, nil
}
var problems []Problem
fset := token.NewFileSet()
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
return nil
}
name := d.Name()
if p != base && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") || name == "testdata") {
return filepath.SkipDir
}
if p == base {
return nil
}
files, err := os.ReadDir(p)
if err != nil {
return err
}
var goFiles []string
for _, f := range files {
n := f.Name()
if !f.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") {
goFiles = append(goFiles, filepath.Join(p, n))
}
}
if len(goFiles) == 0 {
return nil
}
rel, err := filepath.Rel(root, p)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if other, dup := idx.pkgs[name]; dup {
problems = append(problems, Problem{File: rel, Rule: "identifier",
Message: fmt.Sprintf("package name %q is used by %s and %s; spans cannot tell them apart", name, other.dir, rel)})
return nil
}
pkg := &pkgIdents{dir: rel, names: map[string]bool{}, members: map[string]map[string]bool{}, embeds: map[string][]string{}}
for _, file := range goFiles {
f, err := parser.ParseFile(fset, file, nil, parser.SkipObjectResolution)
if err != nil {
return fmt.Errorf("docsite: parse %s: %w", file, err)
}
indexFile(pkg, f)
}
idx.pkgs[name] = pkg
return nil
})
if err != nil {
return nil, nil, fmt.Errorf("docsite: index modules: %w", err)
}
return idx, problems, nil
}
// indexFile records the declarations of one parsed file.
func indexFile(pkg *pkgIdents, f *ast.File) {
for _, decl := range f.Decls {
switch d := decl.(type) {
case *ast.FuncDecl:
if d.Recv != nil && len(d.Recv.List) > 0 {
if typ := receiverType(d.Recv.List[0].Type); typ != "" {
pkg.addMember(typ, d.Name.Name)
}
continue
}
pkg.names[d.Name.Name] = true
case *ast.GenDecl:
for _, spec := range d.Specs {
switch s := spec.(type) {
case *ast.TypeSpec:
pkg.names[s.Name.Name] = true
indexTypeMembers(pkg, s.Name.Name, s.Type)
case *ast.ValueSpec:
for _, n := range s.Names {
pkg.names[n.Name] = true
}
}
}
}
}
}
// indexTypeMembers records struct fields (embedded type names included)
// and interface methods of a type.
func indexTypeMembers(pkg *pkgIdents, typ string, expr ast.Expr) {
var fields *ast.FieldList
switch t := expr.(type) {
case *ast.StructType:
fields = t.Fields
case *ast.InterfaceType:
fields = t.Methods
default:
return
}
if fields == nil {
return
}
for _, field := range fields.List {
if len(field.Names) == 0 {
if n := typeName(field.Type); n != "" {
pkg.addMember(typ, n)
pkg.embeds[typ] = append(pkg.embeds[typ], n)
}
continue
}
for _, n := range field.Names {
pkg.addMember(typ, n.Name)
}
}
}
// receiverType returns the base type name of a method receiver, stripping
// the pointer and any type parameters (Bus, *Bus, Bus[T], *Bus[K, V]).
func receiverType(expr ast.Expr) string {
for {
switch t := expr.(type) {
case *ast.StarExpr:
expr = t.X
case *ast.IndexExpr:
expr = t.X
case *ast.IndexListExpr:
expr = t.X
case *ast.ParenExpr:
expr = t.X
case *ast.Ident:
return t.Name
default:
return ""
}
}
}
// typeName returns the name an embedded field is reached by: the last
// element of the (possibly qualified, pointer or generic) type.
func typeName(expr ast.Expr) string {
for {
switch t := expr.(type) {
case *ast.StarExpr:
expr = t.X
case *ast.IndexExpr:
expr = t.X
case *ast.IndexListExpr:
expr = t.X
case *ast.SelectorExpr:
return t.Sel.Name
case *ast.Ident:
return t.Name
default:
return ""
}
}
}
// identSpan matches the code-span forms that name a module identifier:
// pkg.Ident, pkg.Type.Member, *pkg.Ident, pkg.Ident[...] and pkg.Ident(...).
var identSpan = regexp.MustCompile(`^\*?([a-z][a-z0-9_]*)\.([A-Z][A-Za-z0-9_]*)(?:\.([A-Za-z_][A-Za-z0-9_]*))?(\[[^\]]*\])?(\(.*\))?$`)
// checkSpan returns the problem message for a code span, or "" when the
// span names no module identifier or names one that exists.
func (idx *identIndex) checkSpan(span string) string {
m := identSpan.FindStringSubmatch(strings.TrimSpace(span))
if m == nil {
return ""
}
pkgName, ident, member := m[1], m[2], m[3]
pkg, ok := idx.pkgs[pkgName]
if !ok {
return ""
}
// A lowercase member is a field access or a config key, not an API name.
if member != "" && !isUpper(member) {
member = ""
}
name := pkgName + "." + ident
query := ident
if member != "" {
name += "." + member
query += "." + member
}
if pkg.has(ident, member) || idx.goDoc(pkg.dir, query) {
return ""
}
return fmt.Sprintf("%s does not exist in %s", name, pkg.dir)
}
// has reports whether the package declares ident, and member on it
// directly or promoted through a type embedded in the same package.
func (p *pkgIdents) has(ident, member string) bool {
if !p.names[ident] {
return false
}
if member == "" {
return true
}
seen := map[string]bool{}
queue := []string{ident}
for len(queue) > 0 {
typ := queue[0]
queue = queue[1:]
if seen[typ] {
continue
}
seen[typ] = true
if p.members[typ][member] {
return true
}
queue = append(queue, p.embeds[typ]...)
}
return false
}
func isUpper(s string) bool {
return s != "" && s[0] >= 'A' && s[0] <= 'Z'
}
// goDocIdent limits what reaches the go doc argument list: a Go identifier,
// optionally followed by one ".Member".
var goDocIdent = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$`)
// goDoc is the fallback for an index miss: `go doc ./<dir> <query>` run in
// the repository root, so a declaration the index does not model still
// counts. It runs without a shell, on an argument list whose query matches
// goDocIdent.
func (idx *identIndex) goDoc(dir, query string) bool {
if !goDocIdent.MatchString(query) {
return false
}
key := dir + " " + query
if v, ok := idx.docs[key]; ok {
return v
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "go", "doc", "./"+dir, query)
cmd.Dir = idx.root
cmd.Stdout, cmd.Stderr = nil, nil
ok := cmd.Run() == nil
idx.docs[key] = ok
return ok
}
// codeSpan is one inline code span with its 1-based source line.
type codeSpan struct {
text string
line int
}
// codeSpans lists the inline code spans of a parsed Markdown body (never
// fenced blocks). lineBase is the source line of the body's first line.
func codeSpans(doc gast.Node, body []byte, lineBase int) []codeSpan {
var out []codeSpan
_ = gast.Walk(doc, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
cs, ok := n.(*gast.CodeSpan)
if !entering || !ok {
return gast.WalkContinue, nil
}
var b strings.Builder
start := -1
for c := cs.FirstChild(); c != nil; c = c.NextSibling() {
switch t := c.(type) {
case *gast.Text:
if start < 0 {
start = t.Segment.Start
}
b.Write(t.Segment.Value(body))
case *gast.String:
b.Write(t.Value)
}
}
out = append(out, codeSpan{text: b.String(), line: lineOf(body, start, lineBase)})
return gast.WalkSkipChildren, nil
})
return out
}
// lineOf returns the source line of a byte offset in body.
func lineOf(body []byte, offset, lineBase int) int {
if offset < 0 || offset > len(body) {
return lineBase
}
return lineBase + bytes.Count(body[:offset], []byte("\n"))
}
// checkIdentifiers checks every code span in the docs pages, the ingested
// module READMEs and the root README.md against the module index.
func (s *site) checkIdentifiers(idx *identIndex, docs []parsedDoc) []Problem {
var problems []Problem
for _, d := range docs {
for _, span := range codeSpans(d.doc, d.body, d.line) {
if msg := idx.checkSpan(span.text); msg != "" {
problems = append(problems, Problem{File: d.file, Line: span.line, Rule: "identifier", Message: msg})
}
}
}
return problems
}
// parsedDoc is a Markdown source parsed for the checkers: a page, or the
// root README.md (page == nil).
type parsedDoc struct {
file string
page *Page
body []byte
line int
doc gast.Node
}
// parseDocs parses every page and the root README.md once for the
// checkers, with the renderer's parser and slug IDs but no page context,
// so heading IDs match the rendered pages and link destinations stay as
// written.
func (s *site) parseDocs() ([]parsedDoc, error) {
md := newMarkdown()
var docs []parsedDoc
for _, p := range s.pages {
doc := s.parseRaw(md, p.Body)
docs = append(docs, parsedDoc{file: p.Source, page: p, body: p.Body, line: p.BodyLine, doc: doc})
}
readme := filepath.Join(s.opts.Root, "README.md")
raw, err := os.ReadFile(readme)
switch {
case errors.Is(err, fs.ErrNotExist):
case err != nil:
return nil, fmt.Errorf("docsite: read %s: %w", readme, err)
default:
docs = append(docs, parsedDoc{file: "README.md", body: raw, line: 1, doc: s.parseRaw(md, raw)})
}
slices.SortStableFunc(docs, func(a, b parsedDoc) int { return strings.Compare(a.file, b.file) })
return docs, nil
}

View File

@@ -0,0 +1,134 @@
package docsite
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
// fixtureModule is a small package exercising every declaration kind the
// identifier index records.
const fixtureModule = `package fixture
import "context"
// Bus is a generic-method host with a field and an embedded type.
type Bus struct {
Name string
Base
}
// Base is embedded in Bus.
type Base struct{ ID int }
// Ping is promoted to Bus.
func (Base) Ping() {}
// Handler is an interface.
type Handler interface {
Handle(ctx context.Context) error
}
// Mode is a const.
const Mode = 1
// Default is a var.
var Default = &Bus{}
// New builds a Bus.
func New() *Bus { return &Bus{} }
// Fire is a generic method.
func (b *Bus) Fire[T any](v T) {}
// Close is a value-receiver method.
func (b Bus) Close() error { return nil }
`
func identFixture(t *testing.T, indexBody, readme string) string {
t.Helper()
return writeTree(t, map[string]string{
"docs/site.yaml": fixtureSite,
"docs/index.md": page("Acme docs", "index", 0, indexBody),
"docs/setup/start.md": page("Start", "setup", 10, "Text.\n"),
"modules/fixture/fixture.go": fixtureModule,
"modules/fixture/README.md": "# fixture\n\nFixture does one thing.\n\n" + readme,
"modules/fixture/sub/sub.go": "package sub\n\n// Thing is exported.\ntype Thing struct{}\n",
"modules/fixture/testdata/x.go": "package x\n\n// Hidden is never indexed.\nfunc Hidden() {}\n",
})
}
// writeFile writes one file under root, creating its directory.
func writeFile(t *testing.T, root, name, body string) {
t.Helper()
p := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
func TestIdentifierChecker(t *testing.T) {
passing := strings.Join([]string{
"- `fixture.New()` and `fixture.New`",
"- `*fixture.Bus` and `fixture.Bus.Close`",
"- `fixture.Bus.Fire[string](\"x\")` (generic method)",
"- `fixture.Bus.Name` and `fixture.Bus.Base` (field, embedded)",
"- `fixture.Bus.ID` and `fixture.Bus.Ping()` (promoted through Base)",
"- `fixture.Handler.Handle` (interface method)",
"- `fixture.Mode`, `fixture.Default`, `fixture.Bus.lowercase`",
"- `sub.Thing` (sub-package)",
"- `http.Handler`, `fields.yaml`, `acme.blog`, `summer.yaml`, `fixture.lower`",
"",
"```text",
"fixture.NotChecked() // fenced blocks are skipped",
"```",
"",
}, "\n")
root := identFixture(t, passing, "## Usage\n\nCall `fixture.New()`.\n")
problems, err := Check(Options{Root: root})
if err != nil {
t.Fatal(err)
}
if len(problems) > 0 {
t.Fatalf("passing fixture: %q", problemLines(problems))
}
failing := passing + "Then `fixture.Missing` and `fixture.Bus.Nope`.\n\n`fixture.Hidden` lives in testdata.\n"
root = identFixture(t, failing, "## Usage\n\nCall `fixture.Gone()`.\n")
writeFile(t, root, "README.md", "# Root\n\nSee `x.Y`.\n\n`sub.Nothing`\n")
problems, err = Check(Options{Root: root})
if err != nil {
t.Fatal(err)
}
want := []string{
"README.md:5: identifier: sub.Nothing does not exist in modules/fixture/sub",
"docs/index.md:22: identifier: fixture.Missing does not exist in modules/fixture",
"docs/index.md:22: identifier: fixture.Bus.Nope does not exist in modules/fixture",
"docs/index.md:24: identifier: fixture.Hidden does not exist in modules/fixture",
"modules/fixture/README.md:7: identifier: fixture.Gone does not exist in modules/fixture",
}
if got := problemLines(problems); !slices.Equal(got, want) {
t.Fatalf("problems =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
}
func TestIdentifierIndexDuplicateName(t *testing.T) {
root := writeTree(t, map[string]string{
"modules/alpha/alpha.go": "package alpha\n",
"modules/alpha/util/util.go": "package util\n",
"modules/beta/util/util.go": "package util\n",
})
_, problems, err := buildIdentIndex(root)
if err != nil {
t.Fatal(err)
}
want := `modules/beta/util: identifier: package name "util" is used by modules/alpha/util and modules/beta/util; spans cannot tell them apart`
if got := problemLines(problems); !slices.Equal(got, []string{want}) {
t.Fatalf("problems = %q, want [%q]", got, want)
}
}

View File

@@ -229,6 +229,21 @@ func writeOutputs(out string, files map[string][]byte) error {
return nil
}
// checkContent runs the accuracy checkers over every page and the root
// README.md: module identifiers named in code spans.
func (s *site) checkContent() ([]Problem, error) {
docs, err := s.parseDocs()
if err != nil {
return nil, err
}
idx, problems, err := buildIdentIndex(s.opts.Root)
if err != nil {
return nil, err
}
problems = append(problems, s.checkIdentifiers(idx, docs)...)
return problems, nil
}
func sortProblems(problems []Problem) {
slices.SortStableFunc(problems, func(a, b Problem) int {
return cmp.Or(strings.Compare(a.File, b.File), cmp.Compare(a.Line, b.Line))

View File

@@ -184,6 +184,11 @@ func assemble(opts Options) (*site, []Problem, error) {
return nil, nil, err
}
problems = append(problems, sp...)
cp, err := s.checkContent()
if err != nil {
return nil, nil, err
}
problems = append(problems, cp...)
if len(problems) == 0 {
rp, err := s.render()
if err != nil {

View File

@@ -253,6 +253,13 @@ func (s *site) parsePage(md goldmark.Markdown, p *Page) (ast.Node, *pageContext)
return doc, pctx
}
// parseRaw parses a Markdown body with the renderer's parser and slug IDs
// but no page context, so link destinations stay as written.
func (s *site) parseRaw(md goldmark.Markdown, body []byte) ast.Node {
ctx := parser.NewContext(parser.WithIDs(newSlugIDs()))
return md.Parser().Parse(text.NewReader(body), parser.WithContext(ctx))
}
// renderPage parses and renders a page body to HTML.
func (s *site) renderPage(md goldmark.Markdown, p *Page) (renderedPage, error) {
doc, pctx := s.parsePage(md, p)