feat(11.1-02): check module identifiers in docs and READMEs
- go/parser index of every modules/ package and sub-package, with methods, fields, interface methods and promoted members - code spans in docs pages, module READMEs and the root README fail Check and docs:build when the named identifier does not exist - scripts/check-phase11.1.sh with preconditions, deps, docs, forbidden, go and a self-test that plants one violation per rule
This commit is contained in:
397
internal/docsite/check_identifiers.go
Normal file
397
internal/docsite/check_identifiers.go
Normal file
@@ -0,0 +1,397 @@
|
|||||||
|
package docsite
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"go/ast"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
gast "github.com/yuin/goldmark/ast"
|
||||||
|
)
|
||||||
|
|
||||||
|
// identIndex holds the declared names of every Go package under modules/,
|
||||||
|
// keyed by the package directory's last path element (lagoon, attach,
|
||||||
|
// centrifugo), so `pkg.Ident` code spans can be checked without a type
|
||||||
|
// checker.
|
||||||
|
type identIndex struct {
|
||||||
|
root string
|
||||||
|
pkgs map[string]*pkgIdents
|
||||||
|
// docs caches go doc fallback results by "dir query".
|
||||||
|
docs map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// pkgIdents is the declared names of one package.
|
||||||
|
type pkgIdents struct {
|
||||||
|
// dir is the repository-relative directory, such as "modules/lagoon".
|
||||||
|
dir string
|
||||||
|
// names holds top-level funcs, types, consts and vars.
|
||||||
|
names map[string]bool
|
||||||
|
// members maps a type name to its methods, struct fields (embedded
|
||||||
|
// type names included) and interface methods.
|
||||||
|
members map[string]map[string]bool
|
||||||
|
// embeds maps a type name to the types it embeds, for promoted
|
||||||
|
// members (go doc does not resolve promotion).
|
||||||
|
embeds map[string][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *pkgIdents) addMember(typ, name string) {
|
||||||
|
if p.members[typ] == nil {
|
||||||
|
p.members[typ] = map[string]bool{}
|
||||||
|
}
|
||||||
|
p.members[typ][name] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildIdentIndex parses the non-test Go files of every directory under
|
||||||
|
// <root>/modules (sub-packages included, testdata, dot and underscore
|
||||||
|
// directories skipped). Two directories with the same last path element
|
||||||
|
// are a problem: a span cannot say which one it means.
|
||||||
|
func buildIdentIndex(root string) (*identIndex, []Problem, error) {
|
||||||
|
idx := &identIndex{root: root, pkgs: map[string]*pkgIdents{}, docs: map[string]bool{}}
|
||||||
|
base := filepath.Join(root, "modules")
|
||||||
|
if _, err := os.Stat(base); errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return idx, nil, nil
|
||||||
|
}
|
||||||
|
var problems []Problem
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
name := d.Name()
|
||||||
|
if p != base && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") || name == "testdata") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
if p == base {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
files, err := os.ReadDir(p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var goFiles []string
|
||||||
|
for _, f := range files {
|
||||||
|
n := f.Name()
|
||||||
|
if !f.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") {
|
||||||
|
goFiles = append(goFiles, filepath.Join(p, n))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(goFiles) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(root, p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
if other, dup := idx.pkgs[name]; dup {
|
||||||
|
problems = append(problems, Problem{File: rel, Rule: "identifier",
|
||||||
|
Message: fmt.Sprintf("package name %q is used by %s and %s; spans cannot tell them apart", name, other.dir, rel)})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pkg := &pkgIdents{dir: rel, names: map[string]bool{}, members: map[string]map[string]bool{}, embeds: map[string][]string{}}
|
||||||
|
for _, file := range goFiles {
|
||||||
|
f, err := parser.ParseFile(fset, file, nil, parser.SkipObjectResolution)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("docsite: parse %s: %w", file, err)
|
||||||
|
}
|
||||||
|
indexFile(pkg, f)
|
||||||
|
}
|
||||||
|
idx.pkgs[name] = pkg
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("docsite: index modules: %w", err)
|
||||||
|
}
|
||||||
|
return idx, problems, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// indexFile records the declarations of one parsed file.
|
||||||
|
func indexFile(pkg *pkgIdents, f *ast.File) {
|
||||||
|
for _, decl := range f.Decls {
|
||||||
|
switch d := decl.(type) {
|
||||||
|
case *ast.FuncDecl:
|
||||||
|
if d.Recv != nil && len(d.Recv.List) > 0 {
|
||||||
|
if typ := receiverType(d.Recv.List[0].Type); typ != "" {
|
||||||
|
pkg.addMember(typ, d.Name.Name)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pkg.names[d.Name.Name] = true
|
||||||
|
case *ast.GenDecl:
|
||||||
|
for _, spec := range d.Specs {
|
||||||
|
switch s := spec.(type) {
|
||||||
|
case *ast.TypeSpec:
|
||||||
|
pkg.names[s.Name.Name] = true
|
||||||
|
indexTypeMembers(pkg, s.Name.Name, s.Type)
|
||||||
|
case *ast.ValueSpec:
|
||||||
|
for _, n := range s.Names {
|
||||||
|
pkg.names[n.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// indexTypeMembers records struct fields (embedded type names included)
|
||||||
|
// and interface methods of a type.
|
||||||
|
func indexTypeMembers(pkg *pkgIdents, typ string, expr ast.Expr) {
|
||||||
|
var fields *ast.FieldList
|
||||||
|
switch t := expr.(type) {
|
||||||
|
case *ast.StructType:
|
||||||
|
fields = t.Fields
|
||||||
|
case *ast.InterfaceType:
|
||||||
|
fields = t.Methods
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if fields == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, field := range fields.List {
|
||||||
|
if len(field.Names) == 0 {
|
||||||
|
if n := typeName(field.Type); n != "" {
|
||||||
|
pkg.addMember(typ, n)
|
||||||
|
pkg.embeds[typ] = append(pkg.embeds[typ], n)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, n := range field.Names {
|
||||||
|
pkg.addMember(typ, n.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// receiverType returns the base type name of a method receiver, stripping
|
||||||
|
// the pointer and any type parameters (Bus, *Bus, Bus[T], *Bus[K, V]).
|
||||||
|
func receiverType(expr ast.Expr) string {
|
||||||
|
for {
|
||||||
|
switch t := expr.(type) {
|
||||||
|
case *ast.StarExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.IndexExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.IndexListExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.ParenExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.Ident:
|
||||||
|
return t.Name
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// typeName returns the name an embedded field is reached by: the last
|
||||||
|
// element of the (possibly qualified, pointer or generic) type.
|
||||||
|
func typeName(expr ast.Expr) string {
|
||||||
|
for {
|
||||||
|
switch t := expr.(type) {
|
||||||
|
case *ast.StarExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.IndexExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.IndexListExpr:
|
||||||
|
expr = t.X
|
||||||
|
case *ast.SelectorExpr:
|
||||||
|
return t.Sel.Name
|
||||||
|
case *ast.Ident:
|
||||||
|
return t.Name
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// identSpan matches the code-span forms that name a module identifier:
|
||||||
|
// pkg.Ident, pkg.Type.Member, *pkg.Ident, pkg.Ident[...] and pkg.Ident(...).
|
||||||
|
var identSpan = regexp.MustCompile(`^\*?([a-z][a-z0-9_]*)\.([A-Z][A-Za-z0-9_]*)(?:\.([A-Za-z_][A-Za-z0-9_]*))?(\[[^\]]*\])?(\(.*\))?$`)
|
||||||
|
|
||||||
|
// checkSpan returns the problem message for a code span, or "" when the
|
||||||
|
// span names no module identifier or names one that exists.
|
||||||
|
func (idx *identIndex) checkSpan(span string) string {
|
||||||
|
m := identSpan.FindStringSubmatch(strings.TrimSpace(span))
|
||||||
|
if m == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
pkgName, ident, member := m[1], m[2], m[3]
|
||||||
|
pkg, ok := idx.pkgs[pkgName]
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// A lowercase member is a field access or a config key, not an API name.
|
||||||
|
if member != "" && !isUpper(member) {
|
||||||
|
member = ""
|
||||||
|
}
|
||||||
|
name := pkgName + "." + ident
|
||||||
|
query := ident
|
||||||
|
if member != "" {
|
||||||
|
name += "." + member
|
||||||
|
query += "." + member
|
||||||
|
}
|
||||||
|
if pkg.has(ident, member) || idx.goDoc(pkg.dir, query) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s does not exist in %s", name, pkg.dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// has reports whether the package declares ident, and member on it
|
||||||
|
// directly or promoted through a type embedded in the same package.
|
||||||
|
func (p *pkgIdents) has(ident, member string) bool {
|
||||||
|
if !p.names[ident] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if member == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
queue := []string{ident}
|
||||||
|
for len(queue) > 0 {
|
||||||
|
typ := queue[0]
|
||||||
|
queue = queue[1:]
|
||||||
|
if seen[typ] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[typ] = true
|
||||||
|
if p.members[typ][member] {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
queue = append(queue, p.embeds[typ]...)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isUpper(s string) bool {
|
||||||
|
return s != "" && s[0] >= 'A' && s[0] <= 'Z'
|
||||||
|
}
|
||||||
|
|
||||||
|
// goDocIdent limits what reaches the go doc argument list: a Go identifier,
|
||||||
|
// optionally followed by one ".Member".
|
||||||
|
var goDocIdent = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$`)
|
||||||
|
|
||||||
|
// goDoc is the fallback for an index miss: `go doc ./<dir> <query>` run in
|
||||||
|
// the repository root, so a declaration the index does not model still
|
||||||
|
// counts. It runs without a shell, on an argument list whose query matches
|
||||||
|
// goDocIdent.
|
||||||
|
func (idx *identIndex) goDoc(dir, query string) bool {
|
||||||
|
if !goDocIdent.MatchString(query) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
key := dir + " " + query
|
||||||
|
if v, ok := idx.docs[key]; ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, "go", "doc", "./"+dir, query)
|
||||||
|
cmd.Dir = idx.root
|
||||||
|
cmd.Stdout, cmd.Stderr = nil, nil
|
||||||
|
ok := cmd.Run() == nil
|
||||||
|
idx.docs[key] = ok
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// codeSpan is one inline code span with its 1-based source line.
|
||||||
|
type codeSpan struct {
|
||||||
|
text string
|
||||||
|
line int
|
||||||
|
}
|
||||||
|
|
||||||
|
// codeSpans lists the inline code spans of a parsed Markdown body (never
|
||||||
|
// fenced blocks). lineBase is the source line of the body's first line.
|
||||||
|
func codeSpans(doc gast.Node, body []byte, lineBase int) []codeSpan {
|
||||||
|
var out []codeSpan
|
||||||
|
_ = gast.Walk(doc, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||||
|
cs, ok := n.(*gast.CodeSpan)
|
||||||
|
if !entering || !ok {
|
||||||
|
return gast.WalkContinue, nil
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
start := -1
|
||||||
|
for c := cs.FirstChild(); c != nil; c = c.NextSibling() {
|
||||||
|
switch t := c.(type) {
|
||||||
|
case *gast.Text:
|
||||||
|
if start < 0 {
|
||||||
|
start = t.Segment.Start
|
||||||
|
}
|
||||||
|
b.Write(t.Segment.Value(body))
|
||||||
|
case *gast.String:
|
||||||
|
b.Write(t.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, codeSpan{text: b.String(), line: lineOf(body, start, lineBase)})
|
||||||
|
return gast.WalkSkipChildren, nil
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// lineOf returns the source line of a byte offset in body.
|
||||||
|
func lineOf(body []byte, offset, lineBase int) int {
|
||||||
|
if offset < 0 || offset > len(body) {
|
||||||
|
return lineBase
|
||||||
|
}
|
||||||
|
return lineBase + bytes.Count(body[:offset], []byte("\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkIdentifiers checks every code span in the docs pages, the ingested
|
||||||
|
// module READMEs and the root README.md against the module index.
|
||||||
|
func (s *site) checkIdentifiers(idx *identIndex, docs []parsedDoc) []Problem {
|
||||||
|
var problems []Problem
|
||||||
|
for _, d := range docs {
|
||||||
|
for _, span := range codeSpans(d.doc, d.body, d.line) {
|
||||||
|
if msg := idx.checkSpan(span.text); msg != "" {
|
||||||
|
problems = append(problems, Problem{File: d.file, Line: span.line, Rule: "identifier", Message: msg})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsedDoc is a Markdown source parsed for the checkers: a page, or the
|
||||||
|
// root README.md (page == nil).
|
||||||
|
type parsedDoc struct {
|
||||||
|
file string
|
||||||
|
page *Page
|
||||||
|
body []byte
|
||||||
|
line int
|
||||||
|
doc gast.Node
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseDocs parses every page and the root README.md once for the
|
||||||
|
// checkers, with the renderer's parser and slug IDs but no page context,
|
||||||
|
// so heading IDs match the rendered pages and link destinations stay as
|
||||||
|
// written.
|
||||||
|
func (s *site) parseDocs() ([]parsedDoc, error) {
|
||||||
|
md := newMarkdown()
|
||||||
|
var docs []parsedDoc
|
||||||
|
for _, p := range s.pages {
|
||||||
|
doc := s.parseRaw(md, p.Body)
|
||||||
|
docs = append(docs, parsedDoc{file: p.Source, page: p, body: p.Body, line: p.BodyLine, doc: doc})
|
||||||
|
}
|
||||||
|
readme := filepath.Join(s.opts.Root, "README.md")
|
||||||
|
raw, err := os.ReadFile(readme)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, fs.ErrNotExist):
|
||||||
|
case err != nil:
|
||||||
|
return nil, fmt.Errorf("docsite: read %s: %w", readme, err)
|
||||||
|
default:
|
||||||
|
docs = append(docs, parsedDoc{file: "README.md", body: raw, line: 1, doc: s.parseRaw(md, raw)})
|
||||||
|
}
|
||||||
|
slices.SortStableFunc(docs, func(a, b parsedDoc) int { return strings.Compare(a.file, b.file) })
|
||||||
|
return docs, nil
|
||||||
|
}
|
||||||
134
internal/docsite/checks_test.go
Normal file
134
internal/docsite/checks_test.go
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
package docsite
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fixtureModule is a small package exercising every declaration kind the
|
||||||
|
// identifier index records.
|
||||||
|
const fixtureModule = `package fixture
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// Bus is a generic-method host with a field and an embedded type.
|
||||||
|
type Bus struct {
|
||||||
|
Name string
|
||||||
|
Base
|
||||||
|
}
|
||||||
|
|
||||||
|
// Base is embedded in Bus.
|
||||||
|
type Base struct{ ID int }
|
||||||
|
|
||||||
|
// Ping is promoted to Bus.
|
||||||
|
func (Base) Ping() {}
|
||||||
|
|
||||||
|
// Handler is an interface.
|
||||||
|
type Handler interface {
|
||||||
|
Handle(ctx context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mode is a const.
|
||||||
|
const Mode = 1
|
||||||
|
|
||||||
|
// Default is a var.
|
||||||
|
var Default = &Bus{}
|
||||||
|
|
||||||
|
// New builds a Bus.
|
||||||
|
func New() *Bus { return &Bus{} }
|
||||||
|
|
||||||
|
// Fire is a generic method.
|
||||||
|
func (b *Bus) Fire[T any](v T) {}
|
||||||
|
|
||||||
|
// Close is a value-receiver method.
|
||||||
|
func (b Bus) Close() error { return nil }
|
||||||
|
`
|
||||||
|
|
||||||
|
func identFixture(t *testing.T, indexBody, readme string) string {
|
||||||
|
t.Helper()
|
||||||
|
return writeTree(t, map[string]string{
|
||||||
|
"docs/site.yaml": fixtureSite,
|
||||||
|
"docs/index.md": page("Acme docs", "index", 0, indexBody),
|
||||||
|
"docs/setup/start.md": page("Start", "setup", 10, "Text.\n"),
|
||||||
|
"modules/fixture/fixture.go": fixtureModule,
|
||||||
|
"modules/fixture/README.md": "# fixture\n\nFixture does one thing.\n\n" + readme,
|
||||||
|
"modules/fixture/sub/sub.go": "package sub\n\n// Thing is exported.\ntype Thing struct{}\n",
|
||||||
|
"modules/fixture/testdata/x.go": "package x\n\n// Hidden is never indexed.\nfunc Hidden() {}\n",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeFile writes one file under root, creating its directory.
|
||||||
|
func writeFile(t *testing.T, root, name, body string) {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(root, filepath.FromSlash(name))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentifierChecker(t *testing.T) {
|
||||||
|
passing := strings.Join([]string{
|
||||||
|
"- `fixture.New()` and `fixture.New`",
|
||||||
|
"- `*fixture.Bus` and `fixture.Bus.Close`",
|
||||||
|
"- `fixture.Bus.Fire[string](\"x\")` (generic method)",
|
||||||
|
"- `fixture.Bus.Name` and `fixture.Bus.Base` (field, embedded)",
|
||||||
|
"- `fixture.Bus.ID` and `fixture.Bus.Ping()` (promoted through Base)",
|
||||||
|
"- `fixture.Handler.Handle` (interface method)",
|
||||||
|
"- `fixture.Mode`, `fixture.Default`, `fixture.Bus.lowercase`",
|
||||||
|
"- `sub.Thing` (sub-package)",
|
||||||
|
"- `http.Handler`, `fields.yaml`, `acme.blog`, `summer.yaml`, `fixture.lower`",
|
||||||
|
"",
|
||||||
|
"```text",
|
||||||
|
"fixture.NotChecked() // fenced blocks are skipped",
|
||||||
|
"```",
|
||||||
|
"",
|
||||||
|
}, "\n")
|
||||||
|
root := identFixture(t, passing, "## Usage\n\nCall `fixture.New()`.\n")
|
||||||
|
problems, err := Check(Options{Root: root})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
t.Fatalf("passing fixture: %q", problemLines(problems))
|
||||||
|
}
|
||||||
|
|
||||||
|
failing := passing + "Then `fixture.Missing` and `fixture.Bus.Nope`.\n\n`fixture.Hidden` lives in testdata.\n"
|
||||||
|
root = identFixture(t, failing, "## Usage\n\nCall `fixture.Gone()`.\n")
|
||||||
|
writeFile(t, root, "README.md", "# Root\n\nSee `x.Y`.\n\n`sub.Nothing`\n")
|
||||||
|
problems, err = Check(Options{Root: root})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
"README.md:5: identifier: sub.Nothing does not exist in modules/fixture/sub",
|
||||||
|
"docs/index.md:22: identifier: fixture.Missing does not exist in modules/fixture",
|
||||||
|
"docs/index.md:22: identifier: fixture.Bus.Nope does not exist in modules/fixture",
|
||||||
|
"docs/index.md:24: identifier: fixture.Hidden does not exist in modules/fixture",
|
||||||
|
"modules/fixture/README.md:7: identifier: fixture.Gone does not exist in modules/fixture",
|
||||||
|
}
|
||||||
|
if got := problemLines(problems); !slices.Equal(got, want) {
|
||||||
|
t.Fatalf("problems =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentifierIndexDuplicateName(t *testing.T) {
|
||||||
|
root := writeTree(t, map[string]string{
|
||||||
|
"modules/alpha/alpha.go": "package alpha\n",
|
||||||
|
"modules/alpha/util/util.go": "package util\n",
|
||||||
|
"modules/beta/util/util.go": "package util\n",
|
||||||
|
})
|
||||||
|
_, problems, err := buildIdentIndex(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := `modules/beta/util: identifier: package name "util" is used by modules/alpha/util and modules/beta/util; spans cannot tell them apart`
|
||||||
|
if got := problemLines(problems); !slices.Equal(got, []string{want}) {
|
||||||
|
t.Fatalf("problems = %q, want [%q]", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -229,6 +229,21 @@ func writeOutputs(out string, files map[string][]byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkContent runs the accuracy checkers over every page and the root
|
||||||
|
// README.md: module identifiers named in code spans.
|
||||||
|
func (s *site) checkContent() ([]Problem, error) {
|
||||||
|
docs, err := s.parseDocs()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
idx, problems, err := buildIdentIndex(s.opts.Root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
problems = append(problems, s.checkIdentifiers(idx, docs)...)
|
||||||
|
return problems, nil
|
||||||
|
}
|
||||||
|
|
||||||
func sortProblems(problems []Problem) {
|
func sortProblems(problems []Problem) {
|
||||||
slices.SortStableFunc(problems, func(a, b Problem) int {
|
slices.SortStableFunc(problems, func(a, b Problem) int {
|
||||||
return cmp.Or(strings.Compare(a.File, b.File), cmp.Compare(a.Line, b.Line))
|
return cmp.Or(strings.Compare(a.File, b.File), cmp.Compare(a.Line, b.Line))
|
||||||
|
|||||||
@@ -184,6 +184,11 @@ func assemble(opts Options) (*site, []Problem, error) {
|
|||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
problems = append(problems, sp...)
|
problems = append(problems, sp...)
|
||||||
|
cp, err := s.checkContent()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
problems = append(problems, cp...)
|
||||||
if len(problems) == 0 {
|
if len(problems) == 0 {
|
||||||
rp, err := s.render()
|
rp, err := s.render()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -253,6 +253,13 @@ func (s *site) parsePage(md goldmark.Markdown, p *Page) (ast.Node, *pageContext)
|
|||||||
return doc, pctx
|
return doc, pctx
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseRaw parses a Markdown body with the renderer's parser and slug IDs
|
||||||
|
// but no page context, so link destinations stay as written.
|
||||||
|
func (s *site) parseRaw(md goldmark.Markdown, body []byte) ast.Node {
|
||||||
|
ctx := parser.NewContext(parser.WithIDs(newSlugIDs()))
|
||||||
|
return md.Parser().Parse(text.NewReader(body), parser.WithContext(ctx))
|
||||||
|
}
|
||||||
|
|
||||||
// renderPage parses and renders a page body to HTML.
|
// renderPage parses and renders a page body to HTML.
|
||||||
func (s *site) renderPage(md goldmark.Markdown, p *Page) (renderedPage, error) {
|
func (s *site) renderPage(md goldmark.Markdown, p *Page) (renderedPage, error) {
|
||||||
doc, pctx := s.parsePage(md, p)
|
doc, pctx := s.parsePage(md, p)
|
||||||
|
|||||||
199
scripts/check-phase11.1.sh
Executable file
199
scripts/check-phase11.1.sh
Executable file
@@ -0,0 +1,199 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Phase 11.1 fail-closed gate (documentation for humans and AI agents:
|
||||||
|
# DOCS-01 to DOCS-08).
|
||||||
|
#
|
||||||
|
# The semantic checks (identifiers, links and anchors, src= snippets,
|
||||||
|
# command names, consuming-application names, fence policy) live in Go and
|
||||||
|
# run through `go test` and `summer docs:build --check`. This script only
|
||||||
|
# orchestrates them and adds repository-level assertions: preconditions,
|
||||||
|
# the dependency delta, the built output and the forbidden-name sweep.
|
||||||
|
# --self-test plants one violation per rule in a scratch copy and requires
|
||||||
|
# docs:build --check to refuse it for that rule.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${PHASE11_1_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||||
|
PHASE11_DIR="$ROOT/.planning/phases/11-jobs-realtime-and-search-infrastructure"
|
||||||
|
DEPS_BASE="${PHASE11_1_DEPS_BASE:-9033d81}"
|
||||||
|
# Module paths this phase may add to go.mod (D-15).
|
||||||
|
ALLOWED_NEW_DEPS="github.com/alecthomas/chroma/v2 github.com/dlclark/regexp2/v2"
|
||||||
|
# Phase 11 modules that must be documented before the docs are written.
|
||||||
|
PHASE11_MODULES=(conga lighthouse flare beachcomber)
|
||||||
|
# D-11 consuming-application spellings, byte-level alternation so it works
|
||||||
|
# in any locale (the accented variant matches the Phase 11 hygiene regex).
|
||||||
|
FORBIDDEN_RE='fonoteka|p(l|ł|Ł)(y|ý|Ý)tarium'
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
usage:
|
||||||
|
check-phase11.1.sh --preconditions
|
||||||
|
check-phase11.1.sh --deps
|
||||||
|
check-phase11.1.sh --docs
|
||||||
|
check-phase11.1.sh --forbidden
|
||||||
|
check-phase11.1.sh --self-test
|
||||||
|
check-phase11.1.sh --go
|
||||||
|
check-phase11.1.sh --all
|
||||||
|
EOF
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
refuse() {
|
||||||
|
echo "refuse: $*" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# build_summer compiles the summer tool into $1 once per mode.
|
||||||
|
build_summer() {
|
||||||
|
(cd "$ROOT" && go build -o "$1" ./cmd/summer)
|
||||||
|
}
|
||||||
|
|
||||||
|
run_preconditions() {
|
||||||
|
[ -f "$PHASE11_DIR/11-08-SUMMARY.md" ] || refuse "gap plan 11-08 has no SUMMARY (module READMEs are not stable yet)"
|
||||||
|
local m
|
||||||
|
for m in "${PHASE11_MODULES[@]}"; do
|
||||||
|
[ -f "$ROOT/modules/$m/README.md" ] || refuse "modules/$m has no README.md"
|
||||||
|
grep -qF "| [$m](modules/$m/README.md) |" "$ROOT/README.md" || refuse "root README has no table row for $m"
|
||||||
|
done
|
||||||
|
echo "phase11.1 preconditions passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
# module_paths prints the module paths required by a go.mod file.
|
||||||
|
module_paths() {
|
||||||
|
(cd "$ROOT" && go mod edit -json "$1") | python3 -c '
|
||||||
|
import json, sys
|
||||||
|
for r in json.load(sys.stdin).get("Require") or []:
|
||||||
|
print(r["Path"])
|
||||||
|
' | sort -u
|
||||||
|
}
|
||||||
|
|
||||||
|
run_deps() {
|
||||||
|
local tmp base now removed added dep
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmp"' RETURN
|
||||||
|
git -C "$ROOT" show "$DEPS_BASE:go.mod" >"$tmp/base.mod"
|
||||||
|
cp "$ROOT/go.mod" "$tmp/now.mod"
|
||||||
|
base="$(module_paths "$tmp/base.mod")"
|
||||||
|
now="$(module_paths "$tmp/now.mod")"
|
||||||
|
removed="$(comm -23 <(echo "$base") <(echo "$now"))"
|
||||||
|
[ -z "$removed" ] || refuse "go.mod dropped modules since $DEPS_BASE: $removed"
|
||||||
|
added="$(comm -13 <(echo "$base") <(echo "$now"))"
|
||||||
|
for dep in $added; do
|
||||||
|
case " $ALLOWED_NEW_DEPS " in
|
||||||
|
*" $dep "*) ;;
|
||||||
|
*) refuse "go.mod added $dep, which no phase decision approves" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
echo "phase11.1 deps passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_docs() {
|
||||||
|
local tmp site name
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmp"' RETURN
|
||||||
|
build_summer "$tmp/summer"
|
||||||
|
site="$tmp/site"
|
||||||
|
(cd "$ROOT" && "$tmp/summer" docs:build --root "$ROOT" --out "$site") || refuse "docs:build failed"
|
||||||
|
for name in index.html index.md llms.txt llms-full.txt search-index.json assets/site.css .summer-docs; do
|
||||||
|
[ -f "$site/$name" ] || refuse "docs:build output has no $name"
|
||||||
|
done
|
||||||
|
if find "$site" -name '*.go' | grep -q .; then
|
||||||
|
refuse "docs:build output contains .go files"
|
||||||
|
fi
|
||||||
|
echo "phase11.1 docs passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
# forbidden_hits prints the files under the given paths that name a
|
||||||
|
# consuming application, never the matching lines.
|
||||||
|
forbidden_hits() {
|
||||||
|
grep -rliE "$FORBIDDEN_RE" "$@" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_forbidden() {
|
||||||
|
local tmp hits
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmp"' RETURN
|
||||||
|
build_summer "$tmp/summer"
|
||||||
|
(cd "$ROOT" && "$tmp/summer" docs:build --root "$ROOT" --out "$tmp/site" >/dev/null) || refuse "docs:build failed"
|
||||||
|
hits="$(forbidden_hits "$ROOT/docs" "$tmp/site")"
|
||||||
|
[ -z "$hits" ] || refuse "consuming-application name in: $(echo "$hits" | sed "s|$tmp/||; s|$ROOT/||" | tr '\n' ' ')"
|
||||||
|
echo "phase11.1 forbidden passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_go() {
|
||||||
|
(cd "$ROOT" && go vet ./...)
|
||||||
|
(cd "$ROOT" && go test ./...)
|
||||||
|
echo "phase11.1 go passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
# expect_refusal runs docs:build --check on the scratch root and requires a
|
||||||
|
# non-zero exit whose output names the rule.
|
||||||
|
expect_refusal() {
|
||||||
|
local name="$1" want="$2" summer="$3" root="$4" output
|
||||||
|
if output="$(cd "$root" && GOWORK=off "$summer" docs:build --check --root "$root" 2>&1)"; then
|
||||||
|
echo "refuse: self-test accepted planted $name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! grep -Fq -- "$want" <<<"$output"; then
|
||||||
|
echo "refuse: self-test $name failed for the wrong rule: $output" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# restore copies a pristine file back over its planted scratch copy.
|
||||||
|
restore() {
|
||||||
|
cp "$1/$3" "$2/$3"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_self_test() {
|
||||||
|
bash -n "${BASH_SOURCE[0]}"
|
||||||
|
local tmp scratch pristine summer
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmp"' RETURN
|
||||||
|
scratch="$tmp/root"
|
||||||
|
pristine="$tmp/pristine"
|
||||||
|
summer="$tmp/summer"
|
||||||
|
build_summer "$summer"
|
||||||
|
mkdir -p "$scratch" "$pristine"
|
||||||
|
cp -R "$ROOT/docs" "$ROOT/modules" "$ROOT/go.mod" "$ROOT/go.sum" "$scratch/"
|
||||||
|
cp -R "$ROOT/docs" "$pristine/"
|
||||||
|
|
||||||
|
(cd "$scratch" && GOWORK=off "$summer" docs:build --check --root "$scratch" >/dev/null) ||
|
||||||
|
refuse "self-test baseline: the unplanted scratch copy does not pass docs:build --check"
|
||||||
|
|
||||||
|
printf '\nSee `bonfire.NoSuchThing`.\n' >>"$scratch/docs/index.md"
|
||||||
|
expect_refusal 'unknown identifier' 'identifier: bonfire.NoSuchThing does not exist in modules/bonfire' "$summer" "$scratch"
|
||||||
|
restore "$pristine" "$scratch" docs/index.md
|
||||||
|
|
||||||
|
mkdir "$scratch/modules/zzplant"
|
||||||
|
printf 'package zzplant\n' >"$scratch/modules/zzplant/zzplant.go"
|
||||||
|
expect_refusal 'missing README' 'readme: package has Go files but no README.md' "$summer" "$scratch"
|
||||||
|
rm -rf "$scratch/modules/zzplant"
|
||||||
|
|
||||||
|
sed -i 's/Hello, %s/Hi, %s/' "$scratch/docs/setup/installation.md"
|
||||||
|
expect_refusal 'drifted snippet' 'snippet: body differs' "$summer" "$scratch"
|
||||||
|
restore "$pristine" "$scratch" docs/setup/installation.md
|
||||||
|
|
||||||
|
sed -i '1a colour: red' "$scratch/docs/setup/installation.md"
|
||||||
|
expect_refusal 'frontmatter typo' 'frontmatter: unknown field' "$summer" "$scratch"
|
||||||
|
restore "$pristine" "$scratch" docs/setup/installation.md
|
||||||
|
|
||||||
|
echo "phase11.1 self-test passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
--preconditions) run_preconditions ;;
|
||||||
|
--deps) run_deps ;;
|
||||||
|
--docs) run_docs ;;
|
||||||
|
--forbidden) run_forbidden ;;
|
||||||
|
--self-test) run_self_test ;;
|
||||||
|
--go) run_go ;;
|
||||||
|
--all)
|
||||||
|
run_preconditions
|
||||||
|
run_deps
|
||||||
|
run_self_test
|
||||||
|
run_docs
|
||||||
|
run_forbidden
|
||||||
|
run_go
|
||||||
|
echo "phase11.1 all passed"
|
||||||
|
;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user