feat(11.1-02): check module identifiers in docs and READMEs
- go/parser index of every modules/ package and sub-package, with methods, fields, interface methods and promoted members - code spans in docs pages, module READMEs and the root README fail Check and docs:build when the named identifier does not exist - scripts/check-phase11.1.sh with preconditions, deps, docs, forbidden, go and a self-test that plants one violation per rule
This commit is contained in:
Jakub Zych
397
internal/docsite/check_identifiers.go
Normal file
397
internal/docsite/check_identifiers.go
Normal file
@@ -0,0 +1,397 @@
|
||||
package docsite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gast "github.com/yuin/goldmark/ast"
|
||||
)
|
||||
|
||||
// identIndex holds the declared names of every Go package under modules/,
|
||||
// keyed by the package directory's last path element (lagoon, attach,
|
||||
// centrifugo), so `pkg.Ident` code spans can be checked without a type
|
||||
// checker.
|
||||
type identIndex struct {
|
||||
root string
|
||||
pkgs map[string]*pkgIdents
|
||||
// docs caches go doc fallback results by "dir query".
|
||||
docs map[string]bool
|
||||
}
|
||||
|
||||
// pkgIdents is the declared names of one package.
|
||||
type pkgIdents struct {
|
||||
// dir is the repository-relative directory, such as "modules/lagoon".
|
||||
dir string
|
||||
// names holds top-level funcs, types, consts and vars.
|
||||
names map[string]bool
|
||||
// members maps a type name to its methods, struct fields (embedded
|
||||
// type names included) and interface methods.
|
||||
members map[string]map[string]bool
|
||||
// embeds maps a type name to the types it embeds, for promoted
|
||||
// members (go doc does not resolve promotion).
|
||||
embeds map[string][]string
|
||||
}
|
||||
|
||||
func (p *pkgIdents) addMember(typ, name string) {
|
||||
if p.members[typ] == nil {
|
||||
p.members[typ] = map[string]bool{}
|
||||
}
|
||||
p.members[typ][name] = true
|
||||
}
|
||||
|
||||
// buildIdentIndex parses the non-test Go files of every directory under
|
||||
// <root>/modules (sub-packages included, testdata, dot and underscore
|
||||
// directories skipped). Two directories with the same last path element
|
||||
// are a problem: a span cannot say which one it means.
|
||||
func buildIdentIndex(root string) (*identIndex, []Problem, error) {
|
||||
idx := &identIndex{root: root, pkgs: map[string]*pkgIdents{}, docs: map[string]bool{}}
|
||||
base := filepath.Join(root, "modules")
|
||||
if _, err := os.Stat(base); errors.Is(err, fs.ErrNotExist) {
|
||||
return idx, nil, nil
|
||||
}
|
||||
var problems []Problem
|
||||
fset := token.NewFileSet()
|
||||
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if p != base && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") || name == "testdata") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if p == base {
|
||||
return nil
|
||||
}
|
||||
files, err := os.ReadDir(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var goFiles []string
|
||||
for _, f := range files {
|
||||
n := f.Name()
|
||||
if !f.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") {
|
||||
goFiles = append(goFiles, filepath.Join(p, n))
|
||||
}
|
||||
}
|
||||
if len(goFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if other, dup := idx.pkgs[name]; dup {
|
||||
problems = append(problems, Problem{File: rel, Rule: "identifier",
|
||||
Message: fmt.Sprintf("package name %q is used by %s and %s; spans cannot tell them apart", name, other.dir, rel)})
|
||||
return nil
|
||||
}
|
||||
pkg := &pkgIdents{dir: rel, names: map[string]bool{}, members: map[string]map[string]bool{}, embeds: map[string][]string{}}
|
||||
for _, file := range goFiles {
|
||||
f, err := parser.ParseFile(fset, file, nil, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return fmt.Errorf("docsite: parse %s: %w", file, err)
|
||||
}
|
||||
indexFile(pkg, f)
|
||||
}
|
||||
idx.pkgs[name] = pkg
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("docsite: index modules: %w", err)
|
||||
}
|
||||
return idx, problems, nil
|
||||
}
|
||||
|
||||
// indexFile records the declarations of one parsed file.
|
||||
func indexFile(pkg *pkgIdents, f *ast.File) {
|
||||
for _, decl := range f.Decls {
|
||||
switch d := decl.(type) {
|
||||
case *ast.FuncDecl:
|
||||
if d.Recv != nil && len(d.Recv.List) > 0 {
|
||||
if typ := receiverType(d.Recv.List[0].Type); typ != "" {
|
||||
pkg.addMember(typ, d.Name.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
pkg.names[d.Name.Name] = true
|
||||
case *ast.GenDecl:
|
||||
for _, spec := range d.Specs {
|
||||
switch s := spec.(type) {
|
||||
case *ast.TypeSpec:
|
||||
pkg.names[s.Name.Name] = true
|
||||
indexTypeMembers(pkg, s.Name.Name, s.Type)
|
||||
case *ast.ValueSpec:
|
||||
for _, n := range s.Names {
|
||||
pkg.names[n.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// indexTypeMembers records struct fields (embedded type names included)
|
||||
// and interface methods of a type.
|
||||
func indexTypeMembers(pkg *pkgIdents, typ string, expr ast.Expr) {
|
||||
var fields *ast.FieldList
|
||||
switch t := expr.(type) {
|
||||
case *ast.StructType:
|
||||
fields = t.Fields
|
||||
case *ast.InterfaceType:
|
||||
fields = t.Methods
|
||||
default:
|
||||
return
|
||||
}
|
||||
if fields == nil {
|
||||
return
|
||||
}
|
||||
for _, field := range fields.List {
|
||||
if len(field.Names) == 0 {
|
||||
if n := typeName(field.Type); n != "" {
|
||||
pkg.addMember(typ, n)
|
||||
pkg.embeds[typ] = append(pkg.embeds[typ], n)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, n := range field.Names {
|
||||
pkg.addMember(typ, n.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// receiverType returns the base type name of a method receiver, stripping
|
||||
// the pointer and any type parameters (Bus, *Bus, Bus[T], *Bus[K, V]).
|
||||
func receiverType(expr ast.Expr) string {
|
||||
for {
|
||||
switch t := expr.(type) {
|
||||
case *ast.StarExpr:
|
||||
expr = t.X
|
||||
case *ast.IndexExpr:
|
||||
expr = t.X
|
||||
case *ast.IndexListExpr:
|
||||
expr = t.X
|
||||
case *ast.ParenExpr:
|
||||
expr = t.X
|
||||
case *ast.Ident:
|
||||
return t.Name
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// typeName returns the name an embedded field is reached by: the last
|
||||
// element of the (possibly qualified, pointer or generic) type.
|
||||
func typeName(expr ast.Expr) string {
|
||||
for {
|
||||
switch t := expr.(type) {
|
||||
case *ast.StarExpr:
|
||||
expr = t.X
|
||||
case *ast.IndexExpr:
|
||||
expr = t.X
|
||||
case *ast.IndexListExpr:
|
||||
expr = t.X
|
||||
case *ast.SelectorExpr:
|
||||
return t.Sel.Name
|
||||
case *ast.Ident:
|
||||
return t.Name
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// identSpan matches the code-span forms that name a module identifier:
|
||||
// pkg.Ident, pkg.Type.Member, *pkg.Ident, pkg.Ident[...] and pkg.Ident(...).
|
||||
var identSpan = regexp.MustCompile(`^\*?([a-z][a-z0-9_]*)\.([A-Z][A-Za-z0-9_]*)(?:\.([A-Za-z_][A-Za-z0-9_]*))?(\[[^\]]*\])?(\(.*\))?$`)
|
||||
|
||||
// checkSpan returns the problem message for a code span, or "" when the
|
||||
// span names no module identifier or names one that exists.
|
||||
func (idx *identIndex) checkSpan(span string) string {
|
||||
m := identSpan.FindStringSubmatch(strings.TrimSpace(span))
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
pkgName, ident, member := m[1], m[2], m[3]
|
||||
pkg, ok := idx.pkgs[pkgName]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
// A lowercase member is a field access or a config key, not an API name.
|
||||
if member != "" && !isUpper(member) {
|
||||
member = ""
|
||||
}
|
||||
name := pkgName + "." + ident
|
||||
query := ident
|
||||
if member != "" {
|
||||
name += "." + member
|
||||
query += "." + member
|
||||
}
|
||||
if pkg.has(ident, member) || idx.goDoc(pkg.dir, query) {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s does not exist in %s", name, pkg.dir)
|
||||
}
|
||||
|
||||
// has reports whether the package declares ident, and member on it
|
||||
// directly or promoted through a type embedded in the same package.
|
||||
func (p *pkgIdents) has(ident, member string) bool {
|
||||
if !p.names[ident] {
|
||||
return false
|
||||
}
|
||||
if member == "" {
|
||||
return true
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
queue := []string{ident}
|
||||
for len(queue) > 0 {
|
||||
typ := queue[0]
|
||||
queue = queue[1:]
|
||||
if seen[typ] {
|
||||
continue
|
||||
}
|
||||
seen[typ] = true
|
||||
if p.members[typ][member] {
|
||||
return true
|
||||
}
|
||||
queue = append(queue, p.embeds[typ]...)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isUpper(s string) bool {
|
||||
return s != "" && s[0] >= 'A' && s[0] <= 'Z'
|
||||
}
|
||||
|
||||
// goDocIdent limits what reaches the go doc argument list: a Go identifier,
|
||||
// optionally followed by one ".Member".
|
||||
var goDocIdent = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$`)
|
||||
|
||||
// goDoc is the fallback for an index miss: `go doc ./<dir> <query>` run in
|
||||
// the repository root, so a declaration the index does not model still
|
||||
// counts. It runs without a shell, on an argument list whose query matches
|
||||
// goDocIdent.
|
||||
func (idx *identIndex) goDoc(dir, query string) bool {
|
||||
if !goDocIdent.MatchString(query) {
|
||||
return false
|
||||
}
|
||||
key := dir + " " + query
|
||||
if v, ok := idx.docs[key]; ok {
|
||||
return v
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "go", "doc", "./"+dir, query)
|
||||
cmd.Dir = idx.root
|
||||
cmd.Stdout, cmd.Stderr = nil, nil
|
||||
ok := cmd.Run() == nil
|
||||
idx.docs[key] = ok
|
||||
return ok
|
||||
}
|
||||
|
||||
// codeSpan is one inline code span with its 1-based source line.
|
||||
type codeSpan struct {
|
||||
text string
|
||||
line int
|
||||
}
|
||||
|
||||
// codeSpans lists the inline code spans of a parsed Markdown body (never
|
||||
// fenced blocks). lineBase is the source line of the body's first line.
|
||||
func codeSpans(doc gast.Node, body []byte, lineBase int) []codeSpan {
|
||||
var out []codeSpan
|
||||
_ = gast.Walk(doc, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
|
||||
cs, ok := n.(*gast.CodeSpan)
|
||||
if !entering || !ok {
|
||||
return gast.WalkContinue, nil
|
||||
}
|
||||
var b strings.Builder
|
||||
start := -1
|
||||
for c := cs.FirstChild(); c != nil; c = c.NextSibling() {
|
||||
switch t := c.(type) {
|
||||
case *gast.Text:
|
||||
if start < 0 {
|
||||
start = t.Segment.Start
|
||||
}
|
||||
b.Write(t.Segment.Value(body))
|
||||
case *gast.String:
|
||||
b.Write(t.Value)
|
||||
}
|
||||
}
|
||||
out = append(out, codeSpan{text: b.String(), line: lineOf(body, start, lineBase)})
|
||||
return gast.WalkSkipChildren, nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// lineOf returns the source line of a byte offset in body.
|
||||
func lineOf(body []byte, offset, lineBase int) int {
|
||||
if offset < 0 || offset > len(body) {
|
||||
return lineBase
|
||||
}
|
||||
return lineBase + bytes.Count(body[:offset], []byte("\n"))
|
||||
}
|
||||
|
||||
// checkIdentifiers checks every code span in the docs pages, the ingested
|
||||
// module READMEs and the root README.md against the module index.
|
||||
func (s *site) checkIdentifiers(idx *identIndex, docs []parsedDoc) []Problem {
|
||||
var problems []Problem
|
||||
for _, d := range docs {
|
||||
for _, span := range codeSpans(d.doc, d.body, d.line) {
|
||||
if msg := idx.checkSpan(span.text); msg != "" {
|
||||
problems = append(problems, Problem{File: d.file, Line: span.line, Rule: "identifier", Message: msg})
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// parsedDoc is a Markdown source parsed for the checkers: a page, or the
|
||||
// root README.md (page == nil).
|
||||
type parsedDoc struct {
|
||||
file string
|
||||
page *Page
|
||||
body []byte
|
||||
line int
|
||||
doc gast.Node
|
||||
}
|
||||
|
||||
// parseDocs parses every page and the root README.md once for the
|
||||
// checkers, with the renderer's parser and slug IDs but no page context,
|
||||
// so heading IDs match the rendered pages and link destinations stay as
|
||||
// written.
|
||||
func (s *site) parseDocs() ([]parsedDoc, error) {
|
||||
md := newMarkdown()
|
||||
var docs []parsedDoc
|
||||
for _, p := range s.pages {
|
||||
doc := s.parseRaw(md, p.Body)
|
||||
docs = append(docs, parsedDoc{file: p.Source, page: p, body: p.Body, line: p.BodyLine, doc: doc})
|
||||
}
|
||||
readme := filepath.Join(s.opts.Root, "README.md")
|
||||
raw, err := os.ReadFile(readme)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("docsite: read %s: %w", readme, err)
|
||||
default:
|
||||
docs = append(docs, parsedDoc{file: "README.md", body: raw, line: 1, doc: s.parseRaw(md, raw)})
|
||||
}
|
||||
slices.SortStableFunc(docs, func(a, b parsedDoc) int { return strings.Compare(a.file, b.file) })
|
||||
return docs, nil
|
||||
}
|
||||
134
internal/docsite/checks_test.go
Normal file
134
internal/docsite/checks_test.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package docsite
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fixtureModule is a small package exercising every declaration kind the
|
||||
// identifier index records.
|
||||
const fixtureModule = `package fixture
|
||||
|
||||
import "context"
|
||||
|
||||
// Bus is a generic-method host with a field and an embedded type.
|
||||
type Bus struct {
|
||||
Name string
|
||||
Base
|
||||
}
|
||||
|
||||
// Base is embedded in Bus.
|
||||
type Base struct{ ID int }
|
||||
|
||||
// Ping is promoted to Bus.
|
||||
func (Base) Ping() {}
|
||||
|
||||
// Handler is an interface.
|
||||
type Handler interface {
|
||||
Handle(ctx context.Context) error
|
||||
}
|
||||
|
||||
// Mode is a const.
|
||||
const Mode = 1
|
||||
|
||||
// Default is a var.
|
||||
var Default = &Bus{}
|
||||
|
||||
// New builds a Bus.
|
||||
func New() *Bus { return &Bus{} }
|
||||
|
||||
// Fire is a generic method.
|
||||
func (b *Bus) Fire[T any](v T) {}
|
||||
|
||||
// Close is a value-receiver method.
|
||||
func (b Bus) Close() error { return nil }
|
||||
`
|
||||
|
||||
func identFixture(t *testing.T, indexBody, readme string) string {
|
||||
t.Helper()
|
||||
return writeTree(t, map[string]string{
|
||||
"docs/site.yaml": fixtureSite,
|
||||
"docs/index.md": page("Acme docs", "index", 0, indexBody),
|
||||
"docs/setup/start.md": page("Start", "setup", 10, "Text.\n"),
|
||||
"modules/fixture/fixture.go": fixtureModule,
|
||||
"modules/fixture/README.md": "# fixture\n\nFixture does one thing.\n\n" + readme,
|
||||
"modules/fixture/sub/sub.go": "package sub\n\n// Thing is exported.\ntype Thing struct{}\n",
|
||||
"modules/fixture/testdata/x.go": "package x\n\n// Hidden is never indexed.\nfunc Hidden() {}\n",
|
||||
})
|
||||
}
|
||||
|
||||
// writeFile writes one file under root, creating its directory.
|
||||
func writeFile(t *testing.T, root, name, body string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifierChecker(t *testing.T) {
|
||||
passing := strings.Join([]string{
|
||||
"- `fixture.New()` and `fixture.New`",
|
||||
"- `*fixture.Bus` and `fixture.Bus.Close`",
|
||||
"- `fixture.Bus.Fire[string](\"x\")` (generic method)",
|
||||
"- `fixture.Bus.Name` and `fixture.Bus.Base` (field, embedded)",
|
||||
"- `fixture.Bus.ID` and `fixture.Bus.Ping()` (promoted through Base)",
|
||||
"- `fixture.Handler.Handle` (interface method)",
|
||||
"- `fixture.Mode`, `fixture.Default`, `fixture.Bus.lowercase`",
|
||||
"- `sub.Thing` (sub-package)",
|
||||
"- `http.Handler`, `fields.yaml`, `acme.blog`, `summer.yaml`, `fixture.lower`",
|
||||
"",
|
||||
"```text",
|
||||
"fixture.NotChecked() // fenced blocks are skipped",
|
||||
"```",
|
||||
"",
|
||||
}, "\n")
|
||||
root := identFixture(t, passing, "## Usage\n\nCall `fixture.New()`.\n")
|
||||
problems, err := Check(Options{Root: root})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
t.Fatalf("passing fixture: %q", problemLines(problems))
|
||||
}
|
||||
|
||||
failing := passing + "Then `fixture.Missing` and `fixture.Bus.Nope`.\n\n`fixture.Hidden` lives in testdata.\n"
|
||||
root = identFixture(t, failing, "## Usage\n\nCall `fixture.Gone()`.\n")
|
||||
writeFile(t, root, "README.md", "# Root\n\nSee `x.Y`.\n\n`sub.Nothing`\n")
|
||||
problems, err = Check(Options{Root: root})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"README.md:5: identifier: sub.Nothing does not exist in modules/fixture/sub",
|
||||
"docs/index.md:22: identifier: fixture.Missing does not exist in modules/fixture",
|
||||
"docs/index.md:22: identifier: fixture.Bus.Nope does not exist in modules/fixture",
|
||||
"docs/index.md:24: identifier: fixture.Hidden does not exist in modules/fixture",
|
||||
"modules/fixture/README.md:7: identifier: fixture.Gone does not exist in modules/fixture",
|
||||
}
|
||||
if got := problemLines(problems); !slices.Equal(got, want) {
|
||||
t.Fatalf("problems =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifierIndexDuplicateName(t *testing.T) {
|
||||
root := writeTree(t, map[string]string{
|
||||
"modules/alpha/alpha.go": "package alpha\n",
|
||||
"modules/alpha/util/util.go": "package util\n",
|
||||
"modules/beta/util/util.go": "package util\n",
|
||||
})
|
||||
_, problems, err := buildIdentIndex(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := `modules/beta/util: identifier: package name "util" is used by modules/alpha/util and modules/beta/util; spans cannot tell them apart`
|
||||
if got := problemLines(problems); !slices.Equal(got, []string{want}) {
|
||||
t.Fatalf("problems = %q, want [%q]", got, want)
|
||||
}
|
||||
}
|
||||
@@ -229,6 +229,21 @@ func writeOutputs(out string, files map[string][]byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkContent runs the accuracy checkers over every page and the root
|
||||
// README.md: module identifiers named in code spans.
|
||||
func (s *site) checkContent() ([]Problem, error) {
|
||||
docs, err := s.parseDocs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
idx, problems, err := buildIdentIndex(s.opts.Root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
problems = append(problems, s.checkIdentifiers(idx, docs)...)
|
||||
return problems, nil
|
||||
}
|
||||
|
||||
func sortProblems(problems []Problem) {
|
||||
slices.SortStableFunc(problems, func(a, b Problem) int {
|
||||
return cmp.Or(strings.Compare(a.File, b.File), cmp.Compare(a.Line, b.Line))
|
||||
|
||||
@@ -184,6 +184,11 @@ func assemble(opts Options) (*site, []Problem, error) {
|
||||
return nil, nil, err
|
||||
}
|
||||
problems = append(problems, sp...)
|
||||
cp, err := s.checkContent()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
problems = append(problems, cp...)
|
||||
if len(problems) == 0 {
|
||||
rp, err := s.render()
|
||||
if err != nil {
|
||||
|
||||
@@ -253,6 +253,13 @@ func (s *site) parsePage(md goldmark.Markdown, p *Page) (ast.Node, *pageContext)
|
||||
return doc, pctx
|
||||
}
|
||||
|
||||
// parseRaw parses a Markdown body with the renderer's parser and slug IDs
|
||||
// but no page context, so link destinations stay as written.
|
||||
func (s *site) parseRaw(md goldmark.Markdown, body []byte) ast.Node {
|
||||
ctx := parser.NewContext(parser.WithIDs(newSlugIDs()))
|
||||
return md.Parser().Parse(text.NewReader(body), parser.WithContext(ctx))
|
||||
}
|
||||
|
||||
// renderPage parses and renders a page body to HTML.
|
||||
func (s *site) renderPage(md goldmark.Markdown, p *Page) (renderedPage, error) {
|
||||
doc, pctx := s.parsePage(md, p)
|
||||
|
||||
199
scripts/check-phase11.1.sh
Executable file
199
scripts/check-phase11.1.sh
Executable file
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 11.1 fail-closed gate (documentation for humans and AI agents:
|
||||
# DOCS-01 to DOCS-08).
|
||||
#
|
||||
# The semantic checks (identifiers, links and anchors, src= snippets,
|
||||
# command names, consuming-application names, fence policy) live in Go and
|
||||
# run through `go test` and `summer docs:build --check`. This script only
|
||||
# orchestrates them and adds repository-level assertions: preconditions,
|
||||
# the dependency delta, the built output and the forbidden-name sweep.
|
||||
# --self-test plants one violation per rule in a scratch copy and requires
|
||||
# docs:build --check to refuse it for that rule.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${PHASE11_1_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
PHASE11_DIR="$ROOT/.planning/phases/11-jobs-realtime-and-search-infrastructure"
|
||||
DEPS_BASE="${PHASE11_1_DEPS_BASE:-9033d81}"
|
||||
# Module paths this phase may add to go.mod (D-15).
|
||||
ALLOWED_NEW_DEPS="github.com/alecthomas/chroma/v2 github.com/dlclark/regexp2/v2"
|
||||
# Phase 11 modules that must be documented before the docs are written.
|
||||
PHASE11_MODULES=(conga lighthouse flare beachcomber)
|
||||
# D-11 consuming-application spellings, byte-level alternation so it works
|
||||
# in any locale (the accented variant matches the Phase 11 hygiene regex).
|
||||
FORBIDDEN_RE='fonoteka|p(l|ł|Ł)(y|ý|Ý)tarium'
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase11.1.sh --preconditions
|
||||
check-phase11.1.sh --deps
|
||||
check-phase11.1.sh --docs
|
||||
check-phase11.1.sh --forbidden
|
||||
check-phase11.1.sh --self-test
|
||||
check-phase11.1.sh --go
|
||||
check-phase11.1.sh --all
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
refuse() {
|
||||
echo "refuse: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# build_summer compiles the summer tool into $1 once per mode.
|
||||
build_summer() {
|
||||
(cd "$ROOT" && go build -o "$1" ./cmd/summer)
|
||||
}
|
||||
|
||||
run_preconditions() {
|
||||
[ -f "$PHASE11_DIR/11-08-SUMMARY.md" ] || refuse "gap plan 11-08 has no SUMMARY (module READMEs are not stable yet)"
|
||||
local m
|
||||
for m in "${PHASE11_MODULES[@]}"; do
|
||||
[ -f "$ROOT/modules/$m/README.md" ] || refuse "modules/$m has no README.md"
|
||||
grep -qF "| [$m](modules/$m/README.md) |" "$ROOT/README.md" || refuse "root README has no table row for $m"
|
||||
done
|
||||
echo "phase11.1 preconditions passed"
|
||||
}
|
||||
|
||||
# module_paths prints the module paths required by a go.mod file.
|
||||
module_paths() {
|
||||
(cd "$ROOT" && go mod edit -json "$1") | python3 -c '
|
||||
import json, sys
|
||||
for r in json.load(sys.stdin).get("Require") or []:
|
||||
print(r["Path"])
|
||||
' | sort -u
|
||||
}
|
||||
|
||||
run_deps() {
|
||||
local tmp base now removed added dep
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
git -C "$ROOT" show "$DEPS_BASE:go.mod" >"$tmp/base.mod"
|
||||
cp "$ROOT/go.mod" "$tmp/now.mod"
|
||||
base="$(module_paths "$tmp/base.mod")"
|
||||
now="$(module_paths "$tmp/now.mod")"
|
||||
removed="$(comm -23 <(echo "$base") <(echo "$now"))"
|
||||
[ -z "$removed" ] || refuse "go.mod dropped modules since $DEPS_BASE: $removed"
|
||||
added="$(comm -13 <(echo "$base") <(echo "$now"))"
|
||||
for dep in $added; do
|
||||
case " $ALLOWED_NEW_DEPS " in
|
||||
*" $dep "*) ;;
|
||||
*) refuse "go.mod added $dep, which no phase decision approves" ;;
|
||||
esac
|
||||
done
|
||||
echo "phase11.1 deps passed"
|
||||
}
|
||||
|
||||
run_docs() {
|
||||
local tmp site name
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
build_summer "$tmp/summer"
|
||||
site="$tmp/site"
|
||||
(cd "$ROOT" && "$tmp/summer" docs:build --root "$ROOT" --out "$site") || refuse "docs:build failed"
|
||||
for name in index.html index.md llms.txt llms-full.txt search-index.json assets/site.css .summer-docs; do
|
||||
[ -f "$site/$name" ] || refuse "docs:build output has no $name"
|
||||
done
|
||||
if find "$site" -name '*.go' | grep -q .; then
|
||||
refuse "docs:build output contains .go files"
|
||||
fi
|
||||
echo "phase11.1 docs passed"
|
||||
}
|
||||
|
||||
# forbidden_hits prints the files under the given paths that name a
|
||||
# consuming application, never the matching lines.
|
||||
forbidden_hits() {
|
||||
grep -rliE "$FORBIDDEN_RE" "$@" 2>/dev/null || true
|
||||
}
|
||||
|
||||
run_forbidden() {
|
||||
local tmp hits
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
build_summer "$tmp/summer"
|
||||
(cd "$ROOT" && "$tmp/summer" docs:build --root "$ROOT" --out "$tmp/site" >/dev/null) || refuse "docs:build failed"
|
||||
hits="$(forbidden_hits "$ROOT/docs" "$tmp/site")"
|
||||
[ -z "$hits" ] || refuse "consuming-application name in: $(echo "$hits" | sed "s|$tmp/||; s|$ROOT/||" | tr '\n' ' ')"
|
||||
echo "phase11.1 forbidden passed"
|
||||
}
|
||||
|
||||
run_go() {
|
||||
(cd "$ROOT" && go vet ./...)
|
||||
(cd "$ROOT" && go test ./...)
|
||||
echo "phase11.1 go passed"
|
||||
}
|
||||
|
||||
# expect_refusal runs docs:build --check on the scratch root and requires a
|
||||
# non-zero exit whose output names the rule.
|
||||
expect_refusal() {
|
||||
local name="$1" want="$2" summer="$3" root="$4" output
|
||||
if output="$(cd "$root" && GOWORK=off "$summer" docs:build --check --root "$root" 2>&1)"; then
|
||||
echo "refuse: self-test accepted planted $name" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fq -- "$want" <<<"$output"; then
|
||||
echo "refuse: self-test $name failed for the wrong rule: $output" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# restore copies a pristine file back over its planted scratch copy.
|
||||
restore() {
|
||||
cp "$1/$3" "$2/$3"
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
bash -n "${BASH_SOURCE[0]}"
|
||||
local tmp scratch pristine summer
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
scratch="$tmp/root"
|
||||
pristine="$tmp/pristine"
|
||||
summer="$tmp/summer"
|
||||
build_summer "$summer"
|
||||
mkdir -p "$scratch" "$pristine"
|
||||
cp -R "$ROOT/docs" "$ROOT/modules" "$ROOT/go.mod" "$ROOT/go.sum" "$scratch/"
|
||||
cp -R "$ROOT/docs" "$pristine/"
|
||||
|
||||
(cd "$scratch" && GOWORK=off "$summer" docs:build --check --root "$scratch" >/dev/null) ||
|
||||
refuse "self-test baseline: the unplanted scratch copy does not pass docs:build --check"
|
||||
|
||||
printf '\nSee `bonfire.NoSuchThing`.\n' >>"$scratch/docs/index.md"
|
||||
expect_refusal 'unknown identifier' 'identifier: bonfire.NoSuchThing does not exist in modules/bonfire' "$summer" "$scratch"
|
||||
restore "$pristine" "$scratch" docs/index.md
|
||||
|
||||
mkdir "$scratch/modules/zzplant"
|
||||
printf 'package zzplant\n' >"$scratch/modules/zzplant/zzplant.go"
|
||||
expect_refusal 'missing README' 'readme: package has Go files but no README.md' "$summer" "$scratch"
|
||||
rm -rf "$scratch/modules/zzplant"
|
||||
|
||||
sed -i 's/Hello, %s/Hi, %s/' "$scratch/docs/setup/installation.md"
|
||||
expect_refusal 'drifted snippet' 'snippet: body differs' "$summer" "$scratch"
|
||||
restore "$pristine" "$scratch" docs/setup/installation.md
|
||||
|
||||
sed -i '1a colour: red' "$scratch/docs/setup/installation.md"
|
||||
expect_refusal 'frontmatter typo' 'frontmatter: unknown field' "$summer" "$scratch"
|
||||
restore "$pristine" "$scratch" docs/setup/installation.md
|
||||
|
||||
echo "phase11.1 self-test passed"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--preconditions) run_preconditions ;;
|
||||
--deps) run_deps ;;
|
||||
--docs) run_docs ;;
|
||||
--forbidden) run_forbidden ;;
|
||||
--self-test) run_self_test ;;
|
||||
--go) run_go ;;
|
||||
--all)
|
||||
run_preconditions
|
||||
run_deps
|
||||
run_self_test
|
||||
run_docs
|
||||
run_forbidden
|
||||
run_go
|
||||
echo "phase11.1 all passed"
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user