package docsite import ( "bytes" "context" "errors" "fmt" "go/ast" "go/parser" "go/token" "io/fs" "os" "os/exec" "path/filepath" "regexp" "slices" "strings" "time" gast "github.com/yuin/goldmark/ast" ) // identIndex holds the declared names of every Go package under modules/, // keyed by the package directory's last path element (lagoon, attach, // centrifugo), so `pkg.Ident` code spans can be checked without a type // checker. type identIndex struct { root string pkgs map[string]*pkgIdents // docs caches go doc fallback results by "dir query". docs map[string]bool } // pkgIdents is the declared names of one package. type pkgIdents struct { // dir is the repository-relative directory, such as "modules/lagoon". dir string // names holds top-level funcs, types, consts and vars. names map[string]bool // members maps a type name to its methods, struct fields (embedded // type names included) and interface methods. members map[string]map[string]bool // embeds maps a type name to the types it embeds, for promoted // members (go doc does not resolve promotion). embeds map[string][]string } func (p *pkgIdents) addMember(typ, name string) { if p.members[typ] == nil { p.members[typ] = map[string]bool{} } p.members[typ][name] = true } // buildIdentIndex parses the non-test Go files of every directory under // /modules (sub-packages included, testdata, dot and underscore // directories skipped). Two directories with the same last path element // are a problem: a span cannot say which one it means. func buildIdentIndex(root string) (*identIndex, []Problem, error) { idx := &identIndex{root: root, pkgs: map[string]*pkgIdents{}, docs: map[string]bool{}} base := filepath.Join(root, "modules") if _, err := os.Stat(base); errors.Is(err, fs.ErrNotExist) { return idx, nil, nil } var problems []Problem fset := token.NewFileSet() err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } if !d.IsDir() { return nil } name := d.Name() if p != base && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") || name == "testdata") { return filepath.SkipDir } if p == base { return nil } files, err := os.ReadDir(p) if err != nil { return err } var goFiles []string for _, f := range files { n := f.Name() if !f.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") { goFiles = append(goFiles, filepath.Join(p, n)) } } if len(goFiles) == 0 { return nil } rel, err := filepath.Rel(root, p) if err != nil { return err } rel = filepath.ToSlash(rel) if other, dup := idx.pkgs[name]; dup { problems = append(problems, Problem{File: rel, Rule: "identifier", Message: fmt.Sprintf("package name %q is used by %s and %s; spans cannot tell them apart", name, other.dir, rel)}) return nil } pkg := &pkgIdents{dir: rel, names: map[string]bool{}, members: map[string]map[string]bool{}, embeds: map[string][]string{}} for _, file := range goFiles { f, err := parser.ParseFile(fset, file, nil, parser.SkipObjectResolution) if err != nil { return fmt.Errorf("docsite: parse %s: %w", file, err) } indexFile(pkg, f) } idx.pkgs[name] = pkg return nil }) if err != nil { return nil, nil, fmt.Errorf("docsite: index modules: %w", err) } return idx, problems, nil } // indexFile records the declarations of one parsed file. func indexFile(pkg *pkgIdents, f *ast.File) { for _, decl := range f.Decls { switch d := decl.(type) { case *ast.FuncDecl: if d.Recv != nil && len(d.Recv.List) > 0 { if typ := receiverType(d.Recv.List[0].Type); typ != "" { pkg.addMember(typ, d.Name.Name) } continue } pkg.names[d.Name.Name] = true case *ast.GenDecl: for _, spec := range d.Specs { switch s := spec.(type) { case *ast.TypeSpec: pkg.names[s.Name.Name] = true indexTypeMembers(pkg, s.Name.Name, s.Type) case *ast.ValueSpec: for _, n := range s.Names { pkg.names[n.Name] = true } } } } } } // indexTypeMembers records struct fields (embedded type names included) // and interface methods of a type. func indexTypeMembers(pkg *pkgIdents, typ string, expr ast.Expr) { var fields *ast.FieldList switch t := expr.(type) { case *ast.StructType: fields = t.Fields case *ast.InterfaceType: fields = t.Methods default: return } if fields == nil { return } for _, field := range fields.List { if len(field.Names) == 0 { if n := typeName(field.Type); n != "" { pkg.addMember(typ, n) pkg.embeds[typ] = append(pkg.embeds[typ], n) } continue } for _, n := range field.Names { pkg.addMember(typ, n.Name) } } } // receiverType returns the base type name of a method receiver, stripping // the pointer and any type parameters (Bus, *Bus, Bus[T], *Bus[K, V]). func receiverType(expr ast.Expr) string { for { switch t := expr.(type) { case *ast.StarExpr: expr = t.X case *ast.IndexExpr: expr = t.X case *ast.IndexListExpr: expr = t.X case *ast.ParenExpr: expr = t.X case *ast.Ident: return t.Name default: return "" } } } // typeName returns the name an embedded field is reached by: the last // element of the (possibly qualified, pointer or generic) type. func typeName(expr ast.Expr) string { for { switch t := expr.(type) { case *ast.StarExpr: expr = t.X case *ast.IndexExpr: expr = t.X case *ast.IndexListExpr: expr = t.X case *ast.SelectorExpr: return t.Sel.Name case *ast.Ident: return t.Name default: return "" } } } // identSpan matches the code-span forms that name a module identifier: // pkg.Ident, pkg.Type.Member, *pkg.Ident, pkg.Ident[...] and pkg.Ident(...). var identSpan = regexp.MustCompile(`^\*?([a-z][a-z0-9_]*)\.([A-Z][A-Za-z0-9_]*)(?:\.([A-Za-z_][A-Za-z0-9_]*))?(\[[^\]]*\])?(\(.*\))?$`) // checkSpan returns the problem message for a code span, or "" when the // span names no module identifier or names one that exists. func (idx *identIndex) checkSpan(span string) string { m := identSpan.FindStringSubmatch(strings.TrimSpace(span)) if m == nil { return "" } pkgName, ident, member := m[1], m[2], m[3] pkg, ok := idx.pkgs[pkgName] if !ok { return "" } // A lowercase member is a field access or a config key, not an API name. if member != "" && !isUpper(member) { member = "" } name := pkgName + "." + ident query := ident if member != "" { name += "." + member query += "." + member } if pkg.has(ident, member) || idx.goDoc(pkg.dir, query) { return "" } return fmt.Sprintf("%s does not exist in %s", name, pkg.dir) } // has reports whether the package declares ident, and member on it // directly or promoted through a type embedded in the same package. func (p *pkgIdents) has(ident, member string) bool { if !p.names[ident] { return false } if member == "" { return true } seen := map[string]bool{} queue := []string{ident} for len(queue) > 0 { typ := queue[0] queue = queue[1:] if seen[typ] { continue } seen[typ] = true if p.members[typ][member] { return true } queue = append(queue, p.embeds[typ]...) } return false } func isUpper(s string) bool { return s != "" && s[0] >= 'A' && s[0] <= 'Z' } // goDocIdent limits what reaches the go doc argument list: a Go identifier, // optionally followed by one ".Member". var goDocIdent = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$`) // goDoc is the fallback for an index miss: `go doc ./ ` run in // the repository root, so a declaration the index does not model still // counts. It runs without a shell, on an argument list whose query matches // goDocIdent. func (idx *identIndex) goDoc(dir, query string) bool { if !goDocIdent.MatchString(query) { return false } key := dir + " " + query if v, ok := idx.docs[key]; ok { return v } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "go", "doc", "./"+dir, query) cmd.Dir = idx.root cmd.Stdout, cmd.Stderr = nil, nil ok := cmd.Run() == nil idx.docs[key] = ok return ok } // codeSpan is one inline code span with its 1-based source line. type codeSpan struct { text string line int } // codeSpans lists the inline code spans of a parsed Markdown body (never // fenced blocks). lineBase is the source line of the body's first line. func codeSpans(doc gast.Node, body []byte, lineBase int) []codeSpan { var out []codeSpan _ = gast.Walk(doc, func(n gast.Node, entering bool) (gast.WalkStatus, error) { cs, ok := n.(*gast.CodeSpan) if !entering || !ok { return gast.WalkContinue, nil } var b strings.Builder start := -1 for c := cs.FirstChild(); c != nil; c = c.NextSibling() { switch t := c.(type) { case *gast.Text: if start < 0 { start = t.Segment.Start } b.Write(t.Segment.Value(body)) case *gast.String: b.Write(t.Value) } } out = append(out, codeSpan{text: b.String(), line: lineOf(body, start, lineBase)}) return gast.WalkSkipChildren, nil }) return out } // lineOf returns the source line of a byte offset in body. func lineOf(body []byte, offset, lineBase int) int { if offset < 0 || offset > len(body) { return lineBase } return lineBase + bytes.Count(body[:offset], []byte("\n")) } // checkIdentifiers checks every code span in the docs pages, the ingested // module READMEs and the root README.md against the module index. func (s *site) checkIdentifiers(idx *identIndex, docs []parsedDoc) []Problem { var problems []Problem for _, d := range docs { for _, span := range codeSpans(d.doc, d.body, d.line) { if msg := idx.checkSpan(span.text); msg != "" { problems = append(problems, Problem{File: d.file, Line: span.line, Rule: "identifier", Message: msg}) } } } return problems } // parsedDoc is a Markdown source parsed for the checkers: a page, or the // root README.md (page == nil). type parsedDoc struct { file string page *Page body []byte line int doc gast.Node } // parseDocs parses every page and the root README.md once for the // checkers, with the renderer's parser and slug IDs but no page context, // so heading IDs match the rendered pages and link destinations stay as // written. func (s *site) parseDocs() ([]parsedDoc, error) { md := newMarkdown() var docs []parsedDoc for _, p := range s.pages { doc := s.parseRaw(md, p.Body) docs = append(docs, parsedDoc{file: p.Source, page: p, body: p.Body, line: p.BodyLine, doc: doc}) } readme := filepath.Join(s.opts.Root, "README.md") raw, err := os.ReadFile(readme) switch { case errors.Is(err, fs.ErrNotExist): case err != nil: return nil, fmt.Errorf("docsite: read %s: %w", readme, err) default: docs = append(docs, parsedDoc{file: "README.md", body: raw, line: 1, doc: s.parseRaw(md, raw)}) } slices.SortStableFunc(docs, func(a, b parsedDoc) int { return strings.Compare(a.file, b.file) }) return docs, nil }