summercms/internal/docsite/snippet.go

package docsite

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"slices"
	"strings"
	"unicode"
	"unicode/utf8"
)

// Ref is a src= reference in a code fence info string: a
// repository-relative path and an optional fragment (a top-level Go
// identifier or a docs:start/docs:end region name).
type Ref struct {
	Path     string
	Fragment string
}

// String returns "path" or "path#fragment".
func (r Ref) String() string {
	if r.Fragment == "" {
		return r.Path
	}
	return r.Path + "#" + r.Fragment
}

// ParseSrc finds the src= field in a fence info string, such as
// "go src=modules/bonfire/example_test.go#ExampleCall".
func ParseSrc(info string) (Ref, bool) {
	for _, f := range strings.Fields(info) {
		if v, ok := strings.CutPrefix(f, "src="); ok {
			p, frag, _ := strings.Cut(v, "#")
			return Ref{Path: p, Fragment: frag}, true
		}
	}
	return Ref{}, false
}

// errSnippetNotFound marks a missing file, identifier or region.
var errSnippetNotFound = errors.New("not found")

// snippetError is a src= reference that breaks a confinement or run rule.
type snippetError string

func (e snippetError) Error() string { return string(e) }

// Extract returns the text a src= reference names, after checking that
// the path is confined to the repository root and that Go sources are code
// go test compiles and runs:
//
//   - no fragment: the whole file;
//   - #region: the lines between "// docs:start region" and
//     "// docs:end region" ("# docs:..." in YAML), dedented;
//   - #Ident: a top-level Go declaration with its doc comment, verbatim;
//     for an Example function, its body dedented with the // Output: line.
func Extract(root string, ref Ref) (string, error) {
	if err := checkRefPath(ref.Path); err != nil {
		return "", err
	}
	rootAbs, err := filepath.Abs(root)
	if err != nil {
		return "", fmt.Errorf("docsite: %w", err)
	}
	rootReal, err := filepath.EvalSymlinks(rootAbs)
	if err != nil {
		return "", fmt.Errorf("docsite: %w", err)
	}
	real, err := filepath.EvalSymlinks(filepath.Join(rootAbs, filepath.FromSlash(ref.Path)))
	if errors.Is(err, fs.ErrNotExist) {
		return "", errSnippetNotFound
	}
	if err != nil {
		return "", fmt.Errorf("docsite: %w", err)
	}
	if !within(real, rootReal) {
		return "", snippetError("path resolves outside the repository root")
	}
	if rel, err := filepath.Rel(rootReal, real); err == nil {
		if err := checkRefPath(filepath.ToSlash(rel)); err != nil {
			return "", snippetError("path resolves to a dotfile or .env file")
		}
	}
	st, err := os.Stat(real)
	if err != nil {
		return "", fmt.Errorf("docsite: %w", err)
	}
	if st.IsDir() {
		return "", snippetError("path is a directory")
	}
	raw, err := os.ReadFile(real)
	if err != nil {
		return "", fmt.Errorf("docsite: %w", err)
	}

	isGo := strings.HasSuffix(real, ".go")
	isTest := strings.HasSuffix(real, "_test.go")
	if isGo {
		if err := checkRootModule(rootReal, filepath.Dir(real)); err != nil {
			return "", err
		}
		if err := checkBuiltGo(ref.Path, real, isTest); err != nil {
			return "", err
		}
	}
	if ref.Fragment == "" {
		return string(raw), nil
	}

	lines := strings.Split(string(raw), "\n")
	if text, start, end, found, err := extractRegion(lines, ref.Fragment); err != nil {
		return "", err
	} else if found {
		if isTest {
			if err := checkTestRegion(real, start, end); err != nil {
				return "", err
			}
		}
		return text, nil
	}
	if !isGo {
		return "", errSnippetNotFound
	}
	return extractIdent(real, raw, ref.Fragment, isTest)
}

// checkRefPath enforces the form of a src= path: relative, clean, inside
// the root, and never a dotfile or .env file.
func checkRefPath(p string) error {
	if p == "" {
		return snippetError("src= has no path")
	}
	if filepath.IsAbs(p) || strings.HasPrefix(p, "/") || strings.HasPrefix(p, `\`) || filepath.VolumeName(p) != "" ||
		(len(p) > 1 && p[1] == ':') {
		return snippetError("path must be relative to the repository root")
	}
	if strings.Contains(p, `\`) {
		return snippetError("path must use forward slashes")
	}
	for _, seg := range strings.Split(p, "/") {
		if seg == ".." {
			return snippetError("path must not leave the repository root")
		}
		if strings.HasPrefix(seg, ".") || strings.HasSuffix(seg, ".env") {
			return snippetError("path names a dotfile or .env file")
		}
	}
	if path.Clean(p) != p {
		return snippetError("path must be clean (no ./, // or trailing /)")
	}
	return nil
}

// buildPackage loads dir the way go test ./... does: build.Default, so
// the host GOOS, GOARCH, CGO_ENABLED and release tags apply.
func buildPackage(dir string) (*build.Package, error) {
	return build.ImportDir(dir, 0)
}

// checkBuiltGo refuses a .go src= target go test ./... does not compile:
// a testdata or underscore directory, a file left out of the default
// build, or a non-test file in a package with no tests.
func checkBuiltGo(refPath, real string, isTest bool) error {
	for _, seg := range strings.Split(refPath, "/") {
		if seg == "testdata" || strings.HasPrefix(seg, "_") {
			return snippetError("file is in a directory go test ./... skips (testdata or a _-prefixed directory)")
		}
	}
	pkg, err := buildPackage(filepath.Dir(real))
	if err != nil {
		var noGo *build.NoGoError
		if errors.As(err, &noGo) {
			return snippetError("file is excluded from the default build (build constraint or GOOS/GOARCH file-name suffix)")
		}
		return snippetError("cannot load package: " + firstLine(err.Error()))
	}
	base := filepath.Base(real)
	listed := func(groups ...[]string) bool {
		for _, g := range groups {
			if slices.Contains(g, base) {
				return true
			}
		}
		return false
	}
	if isTest {
		if !listed(pkg.TestGoFiles, pkg.XTestGoFiles) {
			return snippetError("file is excluded from the default build (build constraint or GOOS/GOARCH file-name suffix)")
		}
		return nil
	}
	if !listed(pkg.GoFiles, pkg.CgoFiles) {
		return snippetError("file is excluded from the default build (build constraint or GOOS/GOARCH file-name suffix)")
	}
	if len(pkg.TestGoFiles) == 0 && len(pkg.XTestGoFiles) == 0 {
		return snippetError("package has no _test.go files, so go test does not cover it")
	}
	return nil
}

// checkRootModule refuses a Go source that sits in a nested module (a
// directory between it and the root holds a go.mod): root go test does not
// run it.
func checkRootModule(rootReal, dir string) error {
	for d := dir; d != rootReal && within(d, rootReal); d = filepath.Dir(d) {
		if _, err := os.Stat(filepath.Join(d, "go.mod")); err == nil {
			rel, _ := filepath.Rel(rootReal, d)
			return snippetError(fmt.Sprintf("path is inside the nested module %s/go.mod, which root go test does not run", filepath.ToSlash(rel)))
		}
	}
	return nil
}

// extractRegion returns the dedented lines between the docs:start and
// docs:end markers of a region, and the 1-based marker lines.
func extractRegion(lines []string, name string) (text string, start, end int, found bool, err error) {
	isMarker := func(line, kind string) bool {
		t := strings.TrimSpace(line)
		return t == "// docs:"+kind+" "+name || t == "# docs:"+kind+" "+name
	}
	start = -1
	for i, line := range lines {
		if start < 0 && isMarker(line, "start") {
			start = i
			continue
		}
		if start >= 0 && isMarker(line, "end") {
			return strings.Join(dedent(lines[start+1:i]), "\n"), start + 1, i + 1, true, nil
		}
	}
	if start >= 0 {
		return "", 0, 0, false, snippetError(fmt.Sprintf("region %q has no docs:end marker", name))
	}
	return "", 0, 0, false, nil
}

// dedent removes the longest common leading whitespace of the non-blank
// lines and trims leading and trailing blank lines.
func dedent(lines []string) []string {
	for len(lines) > 0 && strings.TrimSpace(lines[0]) == "" {
		lines = lines[1:]
	}
	for len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" {
		lines = lines[:len(lines)-1]
	}
	prefix, set := "", false
	for _, l := range lines {
		if strings.TrimSpace(l) == "" {
			continue
		}
		lead := l[:len(l)-len(strings.TrimLeft(l, " \t"))]
		if !set {
			prefix, set = lead, true
			continue
		}
		for !strings.HasPrefix(lead, prefix) {
			prefix = prefix[:len(prefix)-1]
		}
	}
	out := make([]string, len(lines))
	for i, l := range lines {
		if strings.TrimSpace(l) == "" {
			out[i] = ""
			continue
		}
		out[i] = strings.TrimPrefix(l, prefix)
	}
	return out
}

// extractIdent returns a top-level declaration by name, or an Example
// function's body.
func extractIdent(file string, raw []byte, name string, isTest bool) (string, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, file, raw, parser.ParseComments)
	if err != nil {
		return "", snippetError("cannot parse Go source: " + firstLine(err.Error()))
	}
	offset := func(p token.Pos) int { return fset.Position(p).Offset }
	for _, decl := range f.Decls {
		switch d := decl.(type) {
		case *ast.FuncDecl:
			if funcKey(d) != name {
				continue
			}
			// An Example with output is a root, so it is not also asked
			// to be reachable from some other test. Without output, go
			// test compiles it and never runs it.
			if d.Recv == nil && strings.HasPrefix(name, "Example") && isTest {
				if !exampleHasOutput(f, name) {
					return "", snippetError(name + " has no // Output: comment, so go test compiles it but never runs it")
				}
				body := string(raw[offset(d.Body.Lbrace)+1 : offset(d.Body.Rbrace)])
				return strings.Join(dedent(strings.Split(body, "\n")), "\n"), nil
			}
			if isTest {
				if err := checkTestIdent(file, name, true); err != nil {
					return "", err
				}
			}
			start := d.Pos()
			if d.Doc != nil {
				start = d.Doc.Pos()
			}
			return string(raw[offset(start):offset(d.End())]), nil
		case *ast.GenDecl:
			for _, spec := range d.Specs {
				var names []*ast.Ident
				var specDoc *ast.CommentGroup
				switch sp := spec.(type) {
				case *ast.TypeSpec:
					names, specDoc = []*ast.Ident{sp.Name}, sp.Doc
				case *ast.ValueSpec:
					names, specDoc = sp.Names, sp.Doc
				}
				if !slices.ContainsFunc(names, func(id *ast.Ident) bool { return id.Name == name }) {
					continue
				}
				if isTest {
					if err := checkTestIdent(file, name, false); err != nil {
						return "", err
					}
				}
				start, end := d.Pos(), d.End()
				doc := d.Doc
				if d.Lparen.IsValid() {
					start, end, doc = spec.Pos(), spec.End(), specDoc
				}
				if doc != nil {
					start = doc.Pos()
				}
				return string(raw[offset(start):offset(end)]), nil
			}
		}
	}
	return "", errSnippetNotFound
}

// funcKey is "Name" for a function and "Type.Name" for a method.
func funcKey(d *ast.FuncDecl) string {
	if d.Recv == nil || len(d.Recv.List) == 0 {
		return d.Name.Name
	}
	t := d.Recv.List[0].Type
	for {
		switch x := t.(type) {
		case *ast.StarExpr:
			t = x.X
			continue
		case *ast.IndexExpr:
			t = x.X
			continue
		case *ast.IndexListExpr:
			t = x.X
			continue
		case *ast.ParenExpr:
			t = x.X
			continue
		case *ast.Ident:
			return x.Name + "." + d.Name.Name
		}
		return d.Name.Name
	}
}

func exampleHasOutput(f *ast.File, name string) bool {
	for _, ex := range doc.Examples(f) {
		if "Example"+ex.Name == name {
			return ex.Output != "" || ex.EmptyOutput
		}
	}
	return false
}

// testGraph is the reachability of the functions in one directory's test
// files from its Test and Example functions.
type testGraph struct {
	fset       *token.FileSet
	funcs      map[string][]*ast.FuncDecl // by key, per file order
	reachable  map[string]bool            // func keys
	referenced map[string]bool            // identifier names used by reachable funcs
	files      map[string]*ast.File       // by path
}

// isTestName is the cmd/go rule for a test function: "Test" alone, or
// "Test" followed by a rune that is not a lowercase letter. Testable
// is not a test. Benchmark and Fuzz are not roots.
func isTestName(name string) bool {
	rest, ok := strings.CutPrefix(name, "Test")
	if !ok {
		return false
	}
	if rest == "" {
		return true
	}
	r, _ := utf8.DecodeRuneInString(rest)
	return !unicode.IsLower(r)
}

func loadTestGraph(dir string) (*testGraph, error) {
	pkg, err := buildPackage(dir)
	if err != nil {
		var noGo *build.NoGoError
		if !errors.As(err, &noGo) {
			return nil, snippetError("cannot load package: " + firstLine(err.Error()))
		}
		pkg = &build.Package{}
	}
	g := &testGraph{fset: token.NewFileSet(), funcs: map[string][]*ast.FuncDecl{},
		reachable: map[string]bool{}, referenced: map[string]bool{}, files: map[string]*ast.File{}}
	byName := map[string][]string{} // plain or method name -> keys
	var queue []string
	var files []*ast.File
	for _, name := range append(append([]string{}, pkg.TestGoFiles...), pkg.XTestGoFiles...) {
		p := filepath.Join(dir, name)
		f, err := parser.ParseFile(g.fset, p, nil, parser.ParseComments)
		if err != nil {
			return nil, snippetError("cannot parse " + filepath.Base(p) + ": " + firstLine(err.Error()))
		}
		g.files[p] = f
		files = append(files, f)
		for _, decl := range f.Decls {
			d, ok := decl.(*ast.FuncDecl)
			if !ok {
				continue
			}
			key := funcKey(d)
			g.funcs[key] = append(g.funcs[key], d)
			byName[d.Name.Name] = append(byName[d.Name.Name], key)
			if d.Recv == nil && isTestName(d.Name.Name) && !g.reachable[key] {
				g.reachable[key] = true
				queue = append(queue, key)
			}
		}
	}
	for _, ex := range doc.Examples(files...) {
		if ex.Output == "" && !ex.EmptyOutput {
			continue
		}
		key := "Example" + ex.Name
		if !g.reachable[key] {
			g.reachable[key] = true
			queue = append(queue, key)
		}
	}
	for len(queue) > 0 {
		key := queue[0]
		queue = queue[1:]
		for _, d := range g.funcs[key] {
			if d.Body == nil {
				continue
			}
			ast.Inspect(d.Body, func(n ast.Node) bool {
				id, ok := n.(*ast.Ident)
				if !ok {
					return true
				}
				g.referenced[id.Name] = true
				for _, k := range byName[id.Name] {
					if !g.reachable[k] {
						g.reachable[k] = true
						queue = append(queue, k)
					}
				}
				return true
			})
		}
	}
	return g, nil
}

const notRunMessage = "fragment is not inside a Test or Example function (an Example counts only with an // Output: comment), or a function one of them calls, so go test does not run it"

// checkTestIdent requires a _test.go declaration to be run by go test: a
// function reachable from a Test or Example, or a type, var or const that
// such a function uses.
func checkTestIdent(file, name string, isFunc bool) error {
	g, err := loadTestGraph(filepath.Dir(file))
	if err != nil {
		return err
	}
	if (isFunc && g.reachable[name]) || (!isFunc && g.referenced[name]) {
		return nil
	}
	return snippetError(notRunMessage)
}

// checkTestRegion requires the region's marker lines to sit inside one
// function of the file that a Test or Example function reaches.
func checkTestRegion(file string, start, end int) error {
	g, err := loadTestGraph(filepath.Dir(file))
	if err != nil {
		return err
	}
	f := g.files[file]
	if f == nil {
		return snippetError(notRunMessage)
	}
	for _, decl := range f.Decls {
		d, ok := decl.(*ast.FuncDecl)
		if !ok {
			continue
		}
		from, to := g.fset.Position(d.Pos()).Line, g.fset.Position(d.End()).Line
		if from < start && end < to && g.reachable[funcKey(d)] {
			return nil
		}
	}
	return snippetError(notRunMessage)
}

// drift is a src= fence whose body differs from its source.
type drift struct {
	f    fence
	want string
}

// nestedSrcMessage is the problem for a src= fence that is not a
// top-level block. docs:sync cannot rewrite a line that still carries
// its container marker, so the fence is refused instead of extracted.
const nestedSrcMessage = "src= code block must be a top-level block of the page, not inside a callout, blockquote or list item"

// readmeSrcMessage is the problem for a src= fence in a module README.
// Those fences are rendered as written and never compared, so a src=
// there would claim a check that does not run.
const readmeSrcMessage = "module README code blocks are rendered as written; src= is verified only in docs/ pages"

// checkFences verifies every src= fence in fences. lines are the body
// lines the fences index into. lineBase is the 1-based file line of
// lines[0]; file is the display path for problems.
func checkFences(root, file string, lines []string, lineBase int, fences []mdFence) ([]drift, []Problem, error) {
	var drifts []drift
	var problems []Problem
	for _, f := range fences {
		ref, ok := ParseSrc(f.info)
		if !ok {
			continue
		}
		line := lineBase + f.line
		fail := func(msg string) {
			problems = append(problems, Problem{File: file, Line: line, Rule: "snippet", Message: msg})
		}
		if !f.topLevel {
			fail(ref.String() + ": " + nestedSrcMessage)
			continue
		}
		if f.top.close < 0 {
			fail("code block has no closing fence")
			continue
		}
		want, err := Extract(root, ref)
		var se snippetError
		switch {
		case errors.Is(err, errSnippetNotFound):
			fail(ref.String() + " not found")
			continue
		case errors.As(err, &se):
			fail(ref.String() + ": " + se.Error())
			continue
		case err != nil:
			return nil, nil, err
		}
		if fenceBody(lines, f.top) != strings.TrimRight(want, "\n") {
			fail(fmt.Sprintf("body differs from %s (run: summer docs:sync)", ref))
			drifts = append(drifts, drift{f: f.top, want: want})
		}
	}
	return drifts, problems, nil
}

// fenceBody returns a fence's body with the fence indentation removed and
// trailing newlines trimmed.
func fenceBody(lines []string, f fence) string {
	body := make([]string, 0, f.close-f.open)
	for _, l := range lines[f.open+1 : f.close] {
		body = append(body, unindent(l, f.indent))
	}
	return strings.TrimRight(strings.Join(body, "\n"), "\n")
}

func unindent(l string, n int) string {
	i := 0
	for i < n && i < len(l) && l[i] == ' ' {
		i++
	}
	return l[i:]
}

// checkSnippets verifies the src= fences of every page. A docs/ page is
// compared with its source. A module README is rendered as written, so
// any src= fence there is refused instead of checked.
func (s *site) checkSnippets(docs []parsedDoc) ([]Problem, error) {
	var problems []Problem
	for _, d := range docs {
		if d.page == nil {
			continue
		}
		fences, err := collectFences(d.doc, d.body)
		if err != nil {
			return nil, err
		}
		if d.page.Module != "" {
			for _, f := range fences {
				ref, ok := ParseSrc(f.info)
				if !ok {
					continue
				}
				problems = append(problems, Problem{File: d.file, Line: d.line + f.line, Rule: "snippet",
					Message: ref.String() + ": " + readmeSrcMessage})
			}
			continue
		}
		_, ps, err := checkFences(s.opts.Root, d.file, strings.Split(string(d.body), "\n"), d.line, fences)
		if err != nil {
			return nil, err
		}
		problems = append(problems, ps...)
	}
	return problems, nil
}

// Sync rewrites every drifted src= fence body under Src from its source,
// preserving everything else byte for byte. When a reference is missing or
// breaks a rule, it returns the problems and writes nothing.
func Sync(opts Options) (SyncResult, []Problem, error) {
	opts, err := opts.normalize()
	if err != nil {
		return SyncResult{}, nil, err
	}
	s := &site{opts: opts}
	files, err := walkPages(opts.Src)
	if err != nil {
		return SyncResult{}, nil, err
	}
	type rewrite struct {
		path string
		mode fs.FileMode
		data []byte
	}
	var rewrites []rewrite
	var problems []Problem
	var result SyncResult
	for _, abs := range files {
		raw, err := os.ReadFile(abs)
		if err != nil {
			return SyncResult{}, nil, fmt.Errorf("docs:sync: read %s: %w", abs, err)
		}
		body, bodyLine := raw, 1
		if _, b, bl, ok := splitFrontmatter(raw); ok {
			body, bodyLine = b, bl
		}
		if !bytes.HasSuffix(raw, body) {
			return SyncResult{}, nil, fmt.Errorf("docs:sync: %s: body is not a suffix of the file", abs)
		}
		doc := s.parseRaw(newMarkdown(), body)
		fences, err := collectFences(doc, body)
		if err != nil {
			return SyncResult{}, nil, err
		}
		lines := strings.Split(string(body), "\n")
		drifts, ps, err := checkFences(opts.Root, s.rel(abs), lines, bodyLine, fences)
		if err != nil {
			return SyncResult{}, nil, err
		}
		for _, p := range ps {
			if !strings.HasPrefix(p.Message, "body differs from ") {
				problems = append(problems, p)
			}
		}
		if len(drifts) == 0 {
			continue
		}
		for i := len(drifts) - 1; i >= 0; i-- {
			d := drifts[i]
			pad := strings.Repeat(" ", d.f.indent)
			var rewritten []string
			for _, l := range strings.Split(strings.TrimRight(d.want, "\n"), "\n") {
				if l == "" {
					rewritten = append(rewritten, "")
				} else {
					rewritten = append(rewritten, pad+l)
				}
			}
			lines = slices.Concat(lines[:d.f.open+1], rewritten, lines[d.f.close:])
		}
		st, err := os.Stat(abs)
		if err != nil {
			return SyncResult{}, nil, fmt.Errorf("docs:sync: %w", err)
		}
		rewritten := append(append([]byte{}, raw[:len(raw)-len(body)]...), []byte(strings.Join(lines, "\n"))...)
		rewrites = append(rewrites, rewrite{path: abs, mode: st.Mode().Perm(), data: rewritten})
		result.Snippets += len(drifts)
		result.Files++
	}
	if len(problems) > 0 {
		sortProblems(problems)
		return SyncResult{}, problems, nil
	}
	for _, r := range rewrites {
		if err := os.WriteFile(r.path, r.data, r.mode); err != nil {
			return SyncResult{}, nil, fmt.Errorf("docs:sync: write %s: %w", r.path, err)
		}
	}
	return result, nil, nil
}