feat(11.1-01): publish every module README as an API reference page
- discover modules/<m> with non-test Go files; a missing README is a readme: problem - one GitHub-compatible slug parser.IDs for heading anchors, passed per page - rewrite links to .md pages and module READMEs to site .html and .md URLs - search-index.json gains one entry per H2 with 300-char plain text - add the api section to docs/site.yaml; docsite.Pages exposes reading order - tests: TestSlugIDs, TestReadmeIngestion, TestEveryModuleInSidebar, TestDocsAIOutputsInSync
This commit is contained in:
Jakub Zych
@@ -3,6 +3,7 @@ package docsite
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -139,12 +140,13 @@ func (s Site) hasSection(name string) bool {
|
||||
|
||||
// site is one assembled documentation site.
|
||||
type site struct {
|
||||
opts Options
|
||||
cfg Site
|
||||
cfgRaw []byte
|
||||
base string
|
||||
pages []*Page // reading order
|
||||
outputs map[string][]byte
|
||||
opts Options
|
||||
cfg Site
|
||||
cfgRaw []byte
|
||||
base string
|
||||
pages []*Page // reading order
|
||||
bySource map[string]*Page
|
||||
outputs map[string][]byte
|
||||
}
|
||||
|
||||
// rel returns the display path of an absolute path: repository-relative
|
||||
@@ -156,13 +158,47 @@ func (s *site) rel(abs string) string {
|
||||
return filepath.ToSlash(abs)
|
||||
}
|
||||
|
||||
// Pages loads the docs pages and the module READMEs and returns them in
|
||||
// reading order (the order of the sidebar, the pager and llms-full.txt),
|
||||
// with any load problems. It renders nothing.
|
||||
func Pages(opts Options) ([]Page, []Problem, error) {
|
||||
s, problems, err := load(opts)
|
||||
if err != nil || s == nil {
|
||||
return nil, problems, err
|
||||
}
|
||||
out := make([]Page, len(s.pages))
|
||||
for i, p := range s.pages {
|
||||
out[i] = *p
|
||||
}
|
||||
return out, problems, nil
|
||||
}
|
||||
|
||||
// assemble loads, verifies and renders a site in memory.
|
||||
func assemble(opts Options) (*site, []Problem, error) {
|
||||
s, problems, err := load(opts)
|
||||
if err != nil || s == nil {
|
||||
return nil, problems, err
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
rp, err := s.render()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
problems = append(problems, rp...)
|
||||
}
|
||||
sortProblems(problems)
|
||||
return s, problems, nil
|
||||
}
|
||||
|
||||
// load reads site.yaml, the docs pages and the module READMEs and puts
|
||||
// the pages in reading order. A nil site with problems means site.yaml
|
||||
// itself is invalid.
|
||||
func load(opts Options) (*site, []Problem, error) {
|
||||
opts, err := opts.normalize()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
s := &site{opts: opts, outputs: map[string][]byte{}}
|
||||
s := &site{opts: opts, outputs: map[string][]byte{}, bySource: map[string]*Page{}}
|
||||
cfgPath := filepath.Join(opts.Src, "site.yaml")
|
||||
raw, err := os.ReadFile(cfgPath)
|
||||
if err != nil {
|
||||
@@ -182,19 +218,120 @@ func assemble(opts Options) (*site, []Problem, error) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
s.order(guides)
|
||||
problems = append(problems, s.emptySections()...)
|
||||
if len(problems) == 0 {
|
||||
rp, err := s.render()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
problems = append(problems, rp...)
|
||||
modules, mp, err := s.loadModules()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
problems = append(problems, mp...)
|
||||
if len(modules) > 0 && !cfg.hasSection(apiSection) {
|
||||
problems = append(problems, Problem{File: s.rel(cfgPath), Line: 1, Rule: "section",
|
||||
Message: fmt.Sprintf("%q is not listed (every module README is published there)", apiSection)})
|
||||
}
|
||||
s.order(append(guides, modules...))
|
||||
for _, p := range s.pages {
|
||||
s.bySource[p.Source] = p
|
||||
}
|
||||
problems = append(problems, s.emptySections()...)
|
||||
sortProblems(problems)
|
||||
return s, problems, nil
|
||||
}
|
||||
|
||||
// loadModules turns every modules/<name> directory that holds a non-test Go
|
||||
// file into an API reference page built from its README.md. The module
|
||||
// list is discovered, never hard-coded.
|
||||
func (s *site) loadModules() ([]*Page, []Problem, error) {
|
||||
names, err := moduleNames(s.opts.Root)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var pages []*Page
|
||||
var problems []Problem
|
||||
for i, name := range names {
|
||||
dir := filepath.Join(s.opts.Root, "modules", name)
|
||||
readme := filepath.Join(dir, "README.md")
|
||||
raw, err := os.ReadFile(readme)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
problems = append(problems, Problem{File: s.rel(dir), Rule: "readme", Message: "package has Go files but no README.md"})
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("docsite: read %s: %w", readme, err)
|
||||
}
|
||||
title, desc, body, ok := splitReadme(raw)
|
||||
if !ok {
|
||||
problems = append(problems, Problem{File: s.rel(readme), Line: 1, Rule: "readme", Message: `first line must be the "# <module>" title followed by a summary line`})
|
||||
continue
|
||||
}
|
||||
pages = append(pages, &Page{
|
||||
Source: s.rel(readme),
|
||||
URL: apiSection + "/" + name,
|
||||
Section: apiSection,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Order: i,
|
||||
Module: name,
|
||||
Body: body,
|
||||
BodyLine: 1,
|
||||
abs: readme,
|
||||
})
|
||||
}
|
||||
return pages, problems, nil
|
||||
}
|
||||
|
||||
// moduleNames lists the top-level modules/ directories that hold a non-test
|
||||
// Go file, sorted by name.
|
||||
func moduleNames(root string) ([]string, error) {
|
||||
entries, err := os.ReadDir(filepath.Join(root, "modules"))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("docsite: read modules: %w", err)
|
||||
}
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || strings.HasPrefix(e.Name(), ".") || strings.HasPrefix(e.Name(), "_") {
|
||||
continue
|
||||
}
|
||||
files, err := os.ReadDir(filepath.Join(root, "modules", e.Name()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("docsite: read module %s: %w", e.Name(), err)
|
||||
}
|
||||
if slices.ContainsFunc(files, func(f fs.DirEntry) bool {
|
||||
n := f.Name()
|
||||
return !f.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go")
|
||||
}) {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// splitReadme returns a README's H1 text, its summary line and the body
|
||||
// with the summary line blanked (line numbers are kept so problems still
|
||||
// cite the README's own lines).
|
||||
func splitReadme(raw []byte) (title, desc string, body []byte, ok bool) {
|
||||
lines := strings.Split(string(raw), "\n")
|
||||
if len(lines) == 0 || !strings.HasPrefix(lines[0], "# ") {
|
||||
return "", "", nil, false
|
||||
}
|
||||
title = strings.TrimSpace(strings.TrimPrefix(lines[0], "# "))
|
||||
for i := 1; i < len(lines); i++ {
|
||||
line := strings.TrimSpace(lines[i])
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") || title == "" {
|
||||
return "", "", nil, false
|
||||
}
|
||||
desc = line
|
||||
lines[i] = ""
|
||||
return title, desc, []byte(strings.Join(lines, "\n")), true
|
||||
}
|
||||
return "", "", nil, false
|
||||
}
|
||||
|
||||
// loadGuides walks Src and loads every page with its frontmatter.
|
||||
func (s *site) loadGuides() ([]*Page, []Problem, error) {
|
||||
files, err := walkPages(s.opts.Src)
|
||||
|
||||
Reference in New Issue
Block a user