summercms/cmd/summer/phase11_1_acceptance_test.go

package main

import (
	"bytes"
	"go/ast"
	"go/doc"
	"go/parser"
	"go/token"
	"io/fs"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"testing"

	"github.com/alecthomas/chroma/v2/lexers"

	"git.golem15.com/golem15/summercms/internal/docsite"
	"git.golem15.com/golem15/summercms/modules/bonfire"
)

// TestPhase11_1Acceptance asserts the Phase 11.1 success criteria on the
// real tree, one subtest per ROADMAP criterion.
func TestPhase11_1Acceptance(t *testing.T) {
	opts := docsite.Options{Root: repoRoot, Commands: docsCommands()}
	pages, loadProblems, err := docsite.Pages(opts)
	if err != nil {
		t.Fatal(err)
	}
	problems, err := docsite.Check(opts)
	if err != nil {
		t.Fatal(err)
	}
	out, _ := buildRealTree(t)

	// SC1: docs/ holds Markdown pages with frontmatter in the
	// Winter-mirroring sections, and every framework module is reachable
	// from the sidebar.
	t.Run("SC1", func(t *testing.T) {
		site, err := docsite.LoadSite(filepath.Join(repoRoot, "docs", "site.yaml"))
		if err != nil {
			t.Fatal(err)
		}
		var names []string
		for _, s := range site.Sections {
			names = append(names, s.Name)
		}
		if !slices.Equal(names, sectionOrder) {
			t.Errorf("site.yaml sections = %v, want %v", names, sectionOrder)
		}
		for _, p := range append(slices.Clone(loadProblems), problems...) {
			switch p.Rule {
			case "frontmatter", "section", "site", "page", "readme":
				t.Errorf("load problem: %s", p)
			}
		}
		guides := 0
		for _, p := range pages {
			if p.Module != "" {
				continue
			}
			guides++
			if p.Title == "" || p.Description == "" || !strings.HasPrefix(p.Source, "docs/") || !strings.HasSuffix(p.Source, ".md") {
				t.Errorf("page %s has incomplete frontmatter: %+v", p.URL, p)
			}
			if p.Section != "index" && (!slices.Contains(sectionOrder, p.Section) || p.Section == "api") {
				t.Errorf("page %s is in section %q", p.URL, p.Section)
			}
		}
		for _, sec := range sectionOrder[:len(sectionOrder)-1] {
			if !slices.ContainsFunc(pages, func(p docsite.Page) bool { return p.Section == sec }) {
				t.Errorf("section %s has no pages", sec)
			}
		}
		if guides < len(requiredPages) {
			t.Errorf("%d guide pages, want at least %d", guides, len(requiredPages))
		}
		sidebar := sidebarOf(t, out, "setup/installation.html")
		for _, m := range frameworkModules(t) {
			if !slices.ContainsFunc(pages, func(p docsite.Page) bool { return p.URL == "api/"+m && p.Module == m }) {
				t.Errorf("module %s has no api/%s page", m, m)
			}
			if !strings.Contains(sidebar, `href="/api/`+m+`.html"`) {
				t.Errorf("the sidebar does not link api/%s", m)
			}
		}
	})

	// SC2: summer builds a self-contained static site with the UI-SPEC
	// theme parts, search and dark mode, with no Node toolchain.
	t.Run("SC2", func(t *testing.T) {
		guide := readFile(t, filepath.Join(out, "setup", "installation.html"))
		for _, marker := range []string{
			`<body class="docs">`,
			`<header class="site-header">`,
			`<nav class="sidebar" aria-label="Documentation">`,
			`<aside class="toc" aria-label="On this page">`,
			`<details class="toc-inline">`,
			`<div class="page-actions">`,
			`<nav class="pager" aria-label="Previous and next page">`,
			`<footer class="site-footer">`,
			`<dialog class="search" id="search">`,
			`<button class="theme-toggle"`,
			`<figure class="code">`,
			`<a class="heading-anchor"`,
			`<svg class="icon icon-`,
			`<link rel="stylesheet" href="/assets/site.css">`,
			`<script src="/assets/theme-init.js"></script>`,
			`<script src="/assets/search.js"`,
		} {
			if !strings.Contains(guide, marker) {
				t.Errorf("setup/installation.html is missing the theme marker %s", marker)
			}
		}
		// Callouts of every type appear on the guide pages.
		guides := guidePagesHTML(t, out, pages)
		for _, kind := range []string{"note", "tip", "warning"} {
			marker := `<aside class="callout callout-` + kind + `" role="note">`
			if !strings.Contains(guides, marker) {
				t.Errorf("no guide page renders %s", marker)
			}
		}
		if nf := readFile(t, filepath.Join(out, "404.html")); !strings.Contains(nf, `<body class="docs docs-404">`) {
			t.Error("404.html has no not-found body marker")
		}
		for _, name := range []string{"site.css", "site.js", "search.js", "theme-init.js"} {
			assertFile(t, filepath.Join(out, "assets", name))
		}
		fonts, err := filepath.Glob(filepath.Join(out, "assets", "fonts", "*.woff2"))
		if err != nil || len(fonts) != 8 {
			t.Errorf("fonts = %v, want 8 woff2 files", fonts)
		}
		for _, name := range []string{"search-index.json", "assets/LICENSE-lucide.txt", "assets/fonts/LICENSE-dm-sans.txt", "assets/fonts/LICENSE-dm-mono.txt"} {
			assertFile(t, filepath.Join(out, filepath.FromSlash(name)))
		}
		if strings.Contains(guide, "//fonts.") || strings.Contains(guide, "cdn.") || strings.Contains(guide, "unpkg") {
			t.Error("the guide page loads an asset from a third-party origin")
		}

		// No Node toolchain: nothing in the generator or the docs commands
		// executes node, npm or npx.
		files, err := filepath.Glob(filepath.Join(repoRoot, "internal", "docsite", "*.go"))
		if err != nil {
			t.Fatal(err)
		}
		files = append(files, "docs.go")
		for _, f := range files {
			if strings.HasSuffix(f, "_test.go") {
				continue
			}
			for _, prog := range execPrograms(t, f) {
				if prog == "" || slices.Contains([]string{"node", "npm", "npx", "pnpm", "yarn"}, filepath.Base(prog)) {
					t.Errorf("%s executes %q (the docs build must not need a Node toolchain)", f, prog)
				}
			}
		}
		if !slices.ContainsFunc(toolCommands(), func(c bonfire.Command) bool { return c.Name == "docs:serve" }) {
			t.Error("docs:serve is not registered")
		}
	})

	// SC3: llms.txt, llms-full.txt and a .md per page stay in sync with
	// the page tree.
	t.Run("SC3", func(t *testing.T) {
		var urls []string
		for _, p := range pages {
			urls = append(urls, p.URL)
		}
		html, md := outputPages(t, out)
		sorted := slices.Sorted(slices.Values(urls))
		if !slices.Equal(html, sorted) || !slices.Equal(md, sorted) {
			t.Errorf("html = %v\nmd = %v\nwant %v", html, md, sorted)
		}
		var linked []string
		for _, line := range strings.Split(readFile(t, filepath.Join(out, "llms.txt")), "\n") {
			if rest, ok := strings.CutPrefix(line, "- ["); ok {
				_, target, _ := strings.Cut(rest, "](/")
				target, _, _ = strings.Cut(target, ".md)")
				linked = append(linked, target)
			}
		}
		if !slices.Equal(linked, urls) {
			t.Errorf("llms.txt links = %v, want %v", linked, urls)
		}
		var sources []string
		for _, line := range strings.Split(readFile(t, filepath.Join(out, "llms-full.txt")), "\n") {
			if rest, ok := strings.CutPrefix(line, "Source: /"); ok {
				sources = append(sources, strings.TrimSuffix(rest, ".html"))
			}
		}
		if !slices.Equal(sources, urls) {
			t.Errorf("llms-full.txt sources = %v, want %v", sources, urls)
		}
		for _, u := range []string{"index", "setup/installation"} {
			md := readFile(t, filepath.Join(out, filepath.FromSlash(u)+".md"))
			if !strings.HasPrefix(md, "# ") || strings.HasPrefix(md, "---") || strings.Contains(md, "src=") {
				t.Errorf("%s.md is not a clean Markdown page", u)
			}
		}
	})

	// SC4: every Go example in a docs/ page is compiled and run by go test,
	// and the checkers report nothing on the real tree.
	t.Run("SC4", func(t *testing.T) {
		for _, p := range problems {
			t.Errorf("docs problem: %s", p)
		}
		refs := docsGoFences(t)
		if len(refs) < 50 {
			t.Fatalf("%d go fences under docs/, expected the full content set", len(refs))
		}
		dirs := map[string]bool{}
		examples := 0
		for _, f := range refs {
			if f.src == "" {
				t.Errorf("%s:%d: go fence without src=", f.file, f.line)
				continue
			}
			path, frag, _ := strings.Cut(f.src, "#")
			if !strings.HasSuffix(path, ".go") {
				t.Errorf("%s:%d: go fence names a non-Go source %s", f.file, f.line, f.src)
				continue
			}
			dirs[filepath.Dir(path)] = true
			if strings.HasPrefix(frag, "Example") {
				examples++
				if !exampleWithOutput(t, filepath.Join(repoRoot, filepath.FromSlash(path)), frag) {
					t.Errorf("%s:%d: %s is not an Example with an // Output: comment", f.file, f.line, f.src)
				}
			}
		}
		if examples < 20 {
			t.Errorf("%d Example fences, expected the full content set", examples)
		}
		// Every referenced directory is a package of the root module, so
		// go test ./... compiles and runs it.
		listed := goList(t, "./...")
		for dir := range dirs {
			if !listed[filepath.ToSlash(dir)] {
				t.Errorf("%s is not a package of the root module", dir)
			}
		}
		// The acceptance scanner is independent of docsite: a src= fence
		// inside a blockquote or list is nested, and a module README has none.
		err := filepath.WalkDir(filepath.Join(repoRoot, "docs"), func(p string, d fs.DirEntry, err error) error {
			if err != nil {
				return err
			}
			if d.IsDir() && d.Name() == "examples" {
				return filepath.SkipDir
			}
			if d.IsDir() || !strings.HasSuffix(p, ".md") {
				return nil
			}
			rel, _ := filepath.Rel(repoRoot, p)
			for _, f := range scanDocFences(filepath.ToSlash(rel), readFile(t, p)) {
				if f.src != "" && f.nested {
					t.Errorf("%s:%d: src= fence is nested (%s)", f.file, f.line, f.src)
				}
			}
			return nil
		})
		if err != nil {
			t.Fatal(err)
		}
		err = filepath.WalkDir(filepath.Join(repoRoot, "modules"), func(p string, d fs.DirEntry, err error) error {
			if err != nil {
				return err
			}
			if d.IsDir() || d.Name() != "README.md" {
				return nil
			}
			rel, _ := filepath.Rel(repoRoot, p)
			for _, f := range scanDocFences(filepath.ToSlash(rel), readFile(t, p)) {
				if f.src != "" {
					t.Errorf("%s:%d: module README has a src= fence", f.file, f.line)
				}
			}
			return nil
		})
		if err != nil {
			t.Fatal(err)
		}
		for _, p := range pages {
			if p.Module != "" {
				continue
			}
			n := 0
			for _, f := range pageFences(t, p.Source) {
				if f.src != "" && !f.nested {
					n++
				}
			}
			html := readFile(t, filepath.Join(out, filepath.FromSlash(p.URL)+".html"))
			if got := strings.Count(html, "<figcaption>"); got != n {
				t.Errorf("%s has %d figcaptions, want %d top-level src= fences", p.URL, got, n)
			}
		}
		api, err := filepath.Glob(filepath.Join(out, "api", "*.html"))
		if err != nil {
			t.Fatal(err)
		}
		for _, p := range api {
			if strings.Contains(readFile(t, p), "<figcaption>") {
				t.Errorf("%s has a figcaption", p)
			}
		}
	})

	// SC5: the concept map and the acme/blog walkthrough exist, and the
	// walkthrough's code is verified under SC4.
	t.Run("SC5", func(t *testing.T) {
		for _, u := range []string{"setup/coming-from-wintercms", "setup/porting-a-plugin"} {
			if !slices.ContainsFunc(pages, func(p docsite.Page) bool { return p.URL == u }) {
				t.Errorf("page %s is missing", u)
			}
		}
		const walkthrough = "docs/setup/porting-a-plugin.md"
		srcs := 0
		for _, f := range pageFences(t, walkthrough) {
			lang, _, _ := strings.Cut(f.info, " ")
			switch {
			case lang == "go" && !strings.HasPrefix(f.src, "docs/examples/blog/"):
				t.Errorf("%s:%d: go fence is not a src= copy of docs/examples/blog (%q)", walkthrough, f.line, f.info)
			case f.src == "" && strings.Contains(f.body, "docs/examples/blog"):
				t.Errorf("%s:%d: fence shows docs/examples/blog code without src=", walkthrough, f.line)
			case strings.HasPrefix(f.src, "docs/examples/blog/"):
				srcs++
			}
		}
		if srcs < 15 {
			t.Errorf("%d src= fences into docs/examples/blog, expected the full walkthrough", srcs)
		}
		concept := readFile(t, filepath.Join(repoRoot, "docs", "setup", "coming-from-wintercms.md"))
		if !strings.Contains(concept, "porting-a-plugin.md") {
			t.Error("the concept map does not link the walkthrough")
		}
		listed := goList(t, "./docs/examples/...")
		for _, pkg := range []string{"docs/examples/blog", "docs/examples/blog/models", "docs/examples/blog/updates", "docs/examples/blog/controllers", "docs/examples/blog/console"} {
			if !listed[pkg] {
				t.Errorf("go list ./docs/examples/... does not include %s", pkg)
			}
		}
		if _, err := os.Stat(filepath.Join(repoRoot, "docs", "examples", "blog", "go.mod")); err == nil {
			t.Error("docs/examples/blog is a nested module; go test ./... would not run it")
		}
	})
}

func readFile(t *testing.T, path string) string {
	t.Helper()
	b, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func assertFile(t *testing.T, path string) {
	t.Helper()
	if st, err := os.Stat(path); err != nil || st.Size() == 0 {
		t.Errorf("missing or empty %s: %v", path, err)
	}
}

// sidebarOf returns the sidebar nav of a built page.
func sidebarOf(t *testing.T, out, name string) string {
	t.Helper()
	html := readFile(t, filepath.Join(out, filepath.FromSlash(name)))
	start := strings.Index(html, `<nav class="sidebar" aria-label="Documentation">`)
	if start < 0 {
		t.Fatalf("%s has no sidebar", name)
	}
	end := strings.Index(html[start:], "</nav>")
	if end < 0 {
		t.Fatalf("%s has an unterminated sidebar", name)
	}
	return html[start : start+end]
}

// guidePagesHTML concatenates the built HTML of every guide page.
func guidePagesHTML(t *testing.T, out string, pages []docsite.Page) string {
	t.Helper()
	var b strings.Builder
	for _, p := range pages {
		if p.Module == "" {
			b.WriteString(readFile(t, filepath.Join(out, filepath.FromSlash(p.URL)+".html")))
		}
	}
	return b.String()
}

// outputPages lists the built .html and .md pages (404.html and assets
// excluded), sorted.
func outputPages(t *testing.T, out string) (html, md []string) {
	t.Helper()
	err := filepath.WalkDir(out, func(p string, d fs.DirEntry, err error) error {
		if err != nil || d.IsDir() {
			return err
		}
		rel, _ := filepath.Rel(out, p)
		rel = filepath.ToSlash(rel)
		if strings.HasPrefix(rel, "assets/") || rel == "404.html" {
			return nil
		}
		switch filepath.Ext(rel) {
		case ".html":
			html = append(html, strings.TrimSuffix(rel, ".html"))
		case ".md":
			md = append(md, strings.TrimSuffix(rel, ".md"))
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	slices.Sort(html)
	slices.Sort(md)
	return html, md
}

// execPrograms returns the program argument of every exec.Command and
// exec.CommandContext call in a Go file: the literal value, or "" when it
// is not a string literal.
func execPrograms(t *testing.T, file string) []string {
	t.Helper()
	f, err := parser.ParseFile(token.NewFileSet(), file, nil, 0)
	if err != nil {
		t.Fatal(err)
	}
	execName := ""
	for _, imp := range f.Imports {
		if p, _ := strconv.Unquote(imp.Path.Value); p == "os/exec" {
			execName = "exec"
			if imp.Name != nil {
				execName = imp.Name.Name
			}
		}
	}
	if execName == "" {
		return nil
	}
	var progs []string
	ast.Inspect(f, func(n ast.Node) bool {
		call, ok := n.(*ast.CallExpr)
		if !ok {
			return true
		}
		sel, ok := call.Fun.(*ast.SelectorExpr)
		if !ok {
			return true
		}
		if x, ok := sel.X.(*ast.Ident); !ok || x.Name != execName {
			return true
		}
		arg := -1
		switch sel.Sel.Name {
		case "Command":
			arg = 0
		case "CommandContext":
			arg = 1
		}
		if arg < 0 || len(call.Args) <= arg {
			return true
		}
		prog := ""
		if lit, ok := call.Args[arg].(*ast.BasicLit); ok && lit.Kind == token.STRING {
			prog, _ = strconv.Unquote(lit.Value)
		}
		progs = append(progs, prog)
		return true
	})
	return progs
}

// docFence is one fenced code block of a docs page, found by an
// independent scanner (not docsite's). nested is true when the opening
// line was indented or a blockquote marker was stripped from it.
type docFence struct {
	file, info, src, body string
	line                  int
	nested                bool
}

// stripMarkers removes a run of leading spaces and blockquote markers
// (each ">" optionally followed by one space). nested is true when the
// line was indented or a marker was removed.
func stripMarkers(line string) (string, bool) {
	nested := false
	for {
		if strings.HasPrefix(line, " ") {
			nested = true
			line = strings.TrimLeft(line, " ")
			continue
		}
		if strings.HasPrefix(line, ">") {
			nested = true
			line = line[1:]
			if strings.HasPrefix(line, " ") {
				line = line[1:]
			}
			continue
		}
		return line, nested
	}
}

// scanDocFences lists the fences in content. Container markers are
// stripped before detection, so a fence inside a blockquote or list is
// visible and marked nested. The closing fence is detected on stripped
// lines.
func scanDocFences(rel, content string) []docFence {
	var fences []docFence
	lines := strings.Split(content, "\n")
	for i := 0; i < len(lines); i++ {
		stripped, nested := stripMarkers(lines[i])
		if !strings.HasPrefix(stripped, "```") && !strings.HasPrefix(stripped, "~~~") {
			continue
		}
		marker := stripped[:3]
		f := docFence{
			file:   rel,
			info:   strings.TrimSpace(strings.TrimLeft(stripped, marker[:1])),
			line:   i + 1,
			nested: nested,
		}
		for _, field := range strings.Fields(f.info) {
			if v, ok := strings.CutPrefix(field, "src="); ok {
				f.src = v
			}
		}
		var body []string
		for i++; i < len(lines); i++ {
			s, _ := stripMarkers(lines[i])
			if strings.HasPrefix(s, marker) {
				break
			}
			body = append(body, s)
		}
		f.body = strings.Join(body, "\n")
		fences = append(fences, f)
	}
	return fences
}

func pageFences(t *testing.T, rel string) []docFence {
	t.Helper()
	return scanDocFences(rel, readFile(t, filepath.Join(repoRoot, filepath.FromSlash(rel))))
}

// goFenceLang reports a Go fence the way the highlighter's chroma
// registry does: names, aliases and file names. It does not call docsite.
func goFenceLang(lang string) bool {
	lexer := lexers.Get(lang)
	return lexer != nil && lexer.Config().Name == "Go"
}

func TestAcceptanceFenceScanner(t *testing.T) {
	body := strings.Join([]string{
		"> [!TIP]",
		"> ```go src=pkg/a.go#A",
		"> BOGUS",
		"> ```",
		"",
		"> ```go",
		"> x := 1",
		"> ```",
		"",
		"```golang",
		"x := 1",
		"```",
		"",
		"```go src=pkg/b.go",
		"package b",
		"```",
		"",
	}, "\n")
	fences := scanDocFences("docs/x.md", body)
	if len(fences) != 4 {
		t.Fatalf("fences = %d, want 4", len(fences))
	}
	if !fences[0].nested || fences[0].src != "pkg/a.go#A" {
		t.Errorf("callout fence = %+v, want nested src= pkg/a.go#A", fences[0])
	}
	if !fences[1].nested || fences[1].src != "" || !strings.HasPrefix(fences[1].info, "go") {
		t.Errorf("blockquote fence = %+v, want nested go", fences[1])
	}
	if fences[2].nested || !strings.HasPrefix(fences[2].info, "golang") {
		t.Errorf("golang fence = %+v, want a top-level golang fence", fences[2])
	}
	if fences[3].nested || fences[3].src != "pkg/b.go" {
		t.Errorf("top-level fence = %+v, want src= pkg/b.go", fences[3])
	}
	if !goFenceLang("go") || !goFenceLang("golang") {
		t.Error("goFenceLang rejected go or golang")
	}
	if goFenceLang("go-html-template") || goFenceLang("text") {
		t.Error("goFenceLang accepted go-html-template or text")
	}
}

// docsGoFences lists every go fence of every page under docs/ (the
// docs/examples tree is code, not pages).
func docsGoFences(t *testing.T) []docFence {
	t.Helper()
	var out []docFence
	err := filepath.WalkDir(filepath.Join(repoRoot, "docs"), func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() && d.Name() == "examples" {
			return filepath.SkipDir
		}
		if d.IsDir() || !strings.HasSuffix(p, ".md") {
			return nil
		}
		rel, _ := filepath.Rel(repoRoot, p)
		for _, f := range pageFences(t, filepath.ToSlash(rel)) {
			lang, _, _ := strings.Cut(f.info, " ")
			if goFenceLang(lang) {
				out = append(out, f)
			}
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	return out
}

// exampleWithOutput reports whether file declares the Example function name
// with an // Output: comment, so go test runs it.
func exampleWithOutput(t *testing.T, file, name string) bool {
	t.Helper()
	f, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ParseComments)
	if err != nil {
		t.Errorf("parse %s: %v", file, err)
		return false
	}
	for _, ex := range doc.Examples(f) {
		if "Example"+ex.Name == name {
			return ex.Output != "" || ex.EmptyOutput
		}
	}
	return false
}

// goList returns the repository-relative directories of the packages a
// go list pattern matches in the root module (workspace off).
func goList(t *testing.T, pattern string) map[string]bool {
	t.Helper()
	cmd := exec.Command("go", "list", "-f", "{{.Dir}}", pattern)
	cmd.Dir = repoRoot
	// Without the workspace, ./... matches the root module's packages only,
	// which is what the root go test ./... runs.
	cmd.Env = append(os.Environ(), "GOWORK=off")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	raw, err := cmd.Output()
	if err != nil {
		t.Fatalf("go list %s: %v\n%s", pattern, err, stderr.String())
	}
	root, err := filepath.Abs(repoRoot)
	if err != nil {
		t.Fatal(err)
	}
	dirs := map[string]bool{}
	for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
		if rel, err := filepath.Rel(root, line); err == nil && !strings.HasPrefix(rel, "..") {
			dirs[filepath.ToSlash(rel)] = true
		}
	}
	return dirs
}