test(11.1-06): acceptance subtests per success criterion and the final docs gate

- TestPhase11_1Acceptance asserts SC1 to SC5 on the real tree: sections
  and module pages, theme markers and no Node exec, AI outputs in sync,
  every go fence a src= copy run by go test, concept map and walkthrough
- check-phase11.1.sh --named runs every named test of the phase by exact
  name (modules' TestDocs* and output Examples derived from source) and
  refuses failures, skips, missing or renamed tests and no tests to run
- --all runs preconditions, deps, self-test, docs, forbidden, claude,
  named and the full go vet and go test
This commit is contained in:
Jakub Zych
2026-10-01 00:10:12 +02:00
parent 28afd4d197
commit c1c9a9f853
2 changed files with 580 additions and 1 deletions

View File

@@ -8,7 +8,10 @@
# orchestrates them and adds repository-level assertions: preconditions,
# the dependency delta, the built output and the forbidden-name sweep.
# --self-test plants one violation per rule in a scratch copy and requires
# docs:build --check to refuse it for that rule.
# docs:build --check to refuse it for that rule, then runs the Go
# planted-violation corpus. --named runs every named test of the phase by
# exact name and requires each to PASS: a failure, a skip, a missing or
# renamed test and "no tests to run" all refuse.
set -euo pipefail
ROOT="${PHASE11_1_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
@@ -31,6 +34,7 @@ usage:
check-phase11.1.sh --forbidden
check-phase11.1.sh --claude
check-phase11.1.sh --self-test
check-phase11.1.sh --named
check-phase11.1.sh --go
check-phase11.1.sh --all
EOF
@@ -302,6 +306,63 @@ run_self_test() {
echo "phase11.1 self-test passed"
}
# docs_example_tests prints the TestDocs* functions and the Example*
# functions with an output comment (the ones go test runs) declared in the
# _test.go files of one package directory.
docs_example_tests() {
python3 - "$ROOT/$1" <<'PY'
import glob, os, re, sys
names = []
for path in sorted(glob.glob(os.path.join(sys.argv[1], "*_test.go"))):
src = open(path, encoding="utf-8").read()
for m in re.finditer(r"^func (TestDocs\w*|Example\w*)\(", src, re.M):
name = m.group(1)
if name.startswith("Example"):
end = re.search(r"^}", src[m.end():], re.M)
body = src[m.end():m.end() + (end.start() if end else len(src))]
if not re.search(r"//\s*(Unordered output|Output):", body):
continue
names.append(name)
print(" ".join(names))
PY
}
# Named tests of the phase, by package. The docs examples in modules are
# derived from the sources (every TestDocs* and every Example with an
# output comment); the generator, CLI and walkthrough tests are listed.
DOCSITE_TESTS=(TestPlantedViolations TestCleanFixture TestBuildOutputGuard TestSlugIDs TestSlugIDsEdgeCases
TestReadmeIngestion TestSnippetForms TestSnippetConfinement TestSyncRewritesDrift TestSnippetGenericsAndGroups
TestSyncPreservesAndReports TestIdentifierChecker TestIdentifierIndexDuplicateName TestIdentifierIndexForms
TestIdentifierGoDocFallback TestLinkChecker TestCommandChecker TestCommandTokenForms TestForbiddenChecker
TestFencePolicy TestFrontmatterEncodingProblems TestParseSite TestLLMSTxtShape TestLLMSFullBlocks
TestMarkdownSiblings TestSearchIndexSchema TestBaseURLPrefixing TestTOCThreshold TestLinkRewriting
TestCalloutRendering TestHighlightGo TestHighlightYAML TestHighlightShell TestHighlightFallback
TestBuildSiteMarkers TestThemeAssetsAndPager TestServeHandler TestServeHandlerBranches TestServeAddrPolicy
TestServeRefusesNonLoopback TestServeRebuildKeepsLastGoodBuild TestServeWatchRebuilds)
SUMMER_TESTS=(TestPhase11_1Acceptance TestDocsTree TestDocsBuildRealTree TestEveryModuleInSidebar
TestDocsAIOutputsInSync TestDocsRequiredPages TestDocsCommandNames TestDocsCommandsMirrorGeneratedMain
TestDocsBuildCheckOutput TestDocsSyncOutput TestDocsServeRefusal TestToolCommandNames)
run_named() {
local dir names
go_json_named ./internal/docsite "${DOCSITE_TESTS[@]}" || refuse "named: internal/docsite"
go_json_named ./cmd/summer "${SUMMER_TESTS[@]}" || refuse "named: cmd/summer"
go_json_named ./docs/examples/blog TestPluginActivates TestRoutesRegistered TestScaffoldLayout \
TestMigrateUpAndRollback TestPostsRouteAgainstDatabase TestPublishCommandAgainstDatabase \
TestPublishCommandOpensDatabase || refuse "named: docs/examples/blog"
go_json_named ./docs/examples/blog/models TestPostTableAndFill || refuse "named: docs/examples/blog/models"
go_json_named ./docs/examples/blog/updates TestMigrationIDs || refuse "named: docs/examples/blog/updates"
go_json_named ./docs/examples/blog/controllers TestPostsControllerDeclaration || refuse "named: docs/examples/blog/controllers"
go_json_named ./docs/examples/blog/console TestPublishCommandShape || refuse "named: docs/examples/blog/console"
while IFS= read -r dir; do
names="$(docs_example_tests "$dir")"
[ -n "$names" ] || refuse "named: $dir has an example_test.go but no Example with output or TestDocs test"
# shellcheck disable=SC2086 # names is a space-separated list of identifiers
go_json_named "./$dir" $names || refuse "named: $dir"
done < <(cd "$ROOT" && find modules -name example_test.go -not -path '*/testdata/*' -printf '%h\n' | sort -u)
echo "phase11.1 named passed"
}
case "${1:-}" in
--preconditions) run_preconditions ;;
--deps) run_deps ;;
@@ -309,6 +370,7 @@ case "${1:-}" in
--forbidden) run_forbidden ;;
--claude) run_claude ;;
--self-test) run_self_test ;;
--named) run_named ;;
--go) run_go ;;
--all)
run_preconditions
@@ -317,6 +379,7 @@ case "${1:-}" in
run_docs
run_forbidden
run_claude
run_named
run_go
echo "phase11.1 all passed"
;;