diff --git a/scripts/apisurface/main.go b/scripts/apisurface/main.go new file mode 100644 index 00000000..949e6f5e --- /dev/null +++ b/scripts/apisurface/main.go @@ -0,0 +1,327 @@ +// Command apisurface prints the exported API of the Go package in one +// directory, one element per line, sorted, each line collapsed to single +// spaces. scripts/check-phase32.sh compares its output for the last release +// tag and for HEAD to prove that a patch release is strictly additive. +// +// go run ./scripts/apisurface +// +// The files are the non-test files go/build's default context selects for +// the directory. Parameter and result names of every func type are cleared +// and every source position is dropped before printing, so renamed +// parameters, realigned fields, reflowed signatures and edited comments +// print identically. +// +// Line kinds: +// +// func F[]() +// method () M() +// type T struct +// field T.Name +// embed T. +// type T interface +// imethod T.M() (unexported methods too) +// iembed T. (embedded interface or type-set term) +// type T +// type T = +// const X = @ +// var X +// +// It exits 2 when the directory has no Go files and 1 on any other error. +package main + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/printer" + "go/token" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: apisurface ") + os.Exit(1) + } + lines, err := surface(os.Args[1]) + if err != nil { + var noGo *build.NoGoError + if errors.As(err, &noGo) { + fmt.Fprintf(os.Stderr, "apisurface: no Go files in %s\n", os.Args[1]) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "apisurface: %v\n", err) + os.Exit(1) + } + for _, line := range lines { + fmt.Println(line) + } +} + +// surface returns the sorted API lines of the package in dir. +func surface(dir string) ([]string, error) { + pkg, err := build.Default.ImportDir(dir, 0) + if err != nil { + return nil, err + } + fset := token.NewFileSet() + var lines []string + for _, name := range pkg.GoFiles { + file, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, parser.SkipObjectResolution) + if err != nil { + return nil, err + } + markAliases(file) + canonicalize(file) + for _, decl := range file.Decls { + lines = append(lines, declLines(decl)...) + } + } + for i, line := range lines { + lines[i] = strings.Join(strings.Fields(line), " ") + } + sort.Strings(lines) + return lines, nil +} + +var posType = reflect.TypeOf(token.NoPos) + +// canonicalize clears the parameter and result names of every func type +// and sets every position to token.NoPos, so the printer lays each node +// out the same way whatever its source formatting was. +func canonicalize(file *ast.File) { + ast.Inspect(file, func(n ast.Node) bool { + if n == nil { + return false + } + if ft, ok := n.(*ast.FuncType); ok { + ft.Params = unnamed(ft.Params) + ft.Results = unnamed(ft.Results) + } + v := reflect.ValueOf(n) + if v.Kind() == reflect.Pointer && !v.IsNil() { + v = v.Elem() + if v.Kind() == reflect.Struct { + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + if f.Type() == posType && f.CanSet() { + f.SetInt(0) + } + } + } + } + return true + }) +} + +// unnamed returns a field list with one unnamed field per name of the +// original (func(a, b int) becomes func(int, int)). +func unnamed(list *ast.FieldList) *ast.FieldList { + if list == nil { + return nil + } + out := &ast.FieldList{} + for _, field := range list.List { + n := len(field.Names) + if n == 0 { + n = 1 + } + for i := 0; i < n; i++ { + out.List = append(out.List, &ast.Field{Type: field.Type}) + } + } + return out +} + +func declLines(decl ast.Decl) []string { + switch d := decl.(type) { + case *ast.FuncDecl: + return funcLines(d) + case *ast.GenDecl: + switch d.Tok { + case token.TYPE: + var out []string + for _, spec := range d.Specs { + out = append(out, typeLines(spec.(*ast.TypeSpec))...) + } + return out + case token.CONST: + return constLines(d) + case token.VAR: + var out []string + for _, spec := range d.Specs { + vs := spec.(*ast.ValueSpec) + for _, name := range vs.Names { + if !name.IsExported() { + continue + } + if vs.Type == nil { + out = append(out, "var "+name.Name) + } else { + out = append(out, "var "+name.Name+" "+node(vs.Type)) + } + } + } + return out + } + } + return nil +} + +func funcLines(d *ast.FuncDecl) []string { + if !d.Name.IsExported() { + return nil + } + sig := strings.TrimPrefix(node(d.Type), "func") + if d.Recv == nil || len(d.Recv.List) == 0 { + return []string{"func " + d.Name.Name + sig} + } + recv := d.Recv.List[0].Type + if !ast.IsExported(baseTypeName(recv)) { + return nil + } + return []string{"method (" + node(recv) + ") " + d.Name.Name + sig} +} + +// baseTypeName returns T for T, *T, T[K] and *T[K, V]. +func baseTypeName(expr ast.Expr) string { + for { + switch e := expr.(type) { + case *ast.StarExpr: + expr = e.X + case *ast.IndexExpr: + expr = e.X + case *ast.IndexListExpr: + expr = e.X + case *ast.ParenExpr: + expr = e.X + case *ast.Ident: + return e.Name + default: + return "" + } + } +} + +func typeLines(ts *ast.TypeSpec) []string { + if !ts.Name.IsExported() { + return nil + } + name := ts.Name.Name + head := "type " + name + typeParams(ts.TypeParams) + if aliases[ts] { + return []string{head + " = " + node(ts.Type)} + } + switch t := ts.Type.(type) { + case *ast.StructType: + out := []string{head + " struct"} + for _, field := range t.Fields.List { + tag := "" + if field.Tag != nil { + tag = " " + field.Tag.Value + } + if len(field.Names) == 0 { + out = append(out, "embed "+name+"."+node(field.Type)+tag) + continue + } + for _, fn := range field.Names { + if fn.IsExported() { + out = append(out, "field "+name+"."+fn.Name+" "+node(field.Type)+tag) + } + } + } + return out + case *ast.InterfaceType: + out := []string{head + " interface"} + for _, field := range t.Methods.List { + if len(field.Names) == 0 { + out = append(out, "iembed "+name+"."+node(field.Type)) + continue + } + sig := strings.TrimPrefix(node(field.Type), "func") + for _, mn := range field.Names { + out = append(out, "imethod "+name+"."+mn.Name+sig) + } + } + return out + } + return []string{head + " " + node(ts.Type)} +} + +// aliases records alias specs: canonicalize drops the Assign position, the +// parser's only alias marker, so markAliases runs before it. +var aliases = map[*ast.TypeSpec]bool{} + +func markAliases(file *ast.File) { + for _, decl := range file.Decls { + if gd, ok := decl.(*ast.GenDecl); ok && gd.Tok == token.TYPE { + for _, spec := range gd.Specs { + if ts := spec.(*ast.TypeSpec); ts.Assign.IsValid() { + aliases[ts] = true + } + } + } + } +} + +func typeParams(list *ast.FieldList) string { + if list == nil || len(list.List) == 0 { + return "" + } + parts := make([]string, 0, len(list.List)) + for _, field := range list.List { + names := make([]string, 0, len(field.Names)) + for _, n := range field.Names { + names = append(names, n.Name) + } + parts = append(parts, strings.Join(names, ", ")+" "+node(field.Type)) + } + return "[" + strings.Join(parts, ", ") + "]" +} + +// constLines prints each exported constant with its type and value; a spec +// without them repeats the previous spec's, as Go does, and @index is the +// spec's position in its group (the value of iota). +func constLines(d *ast.GenDecl) []string { + var out []string + var typ ast.Expr + var values []ast.Expr + for i, spec := range d.Specs { + vs := spec.(*ast.ValueSpec) + if len(vs.Values) > 0 { + typ, values = vs.Type, vs.Values + } + for j, name := range vs.Names { + if !name.IsExported() { + continue + } + line := "const " + name.Name + if typ != nil { + line += " " + node(typ) + } + if j < len(values) { + line += " = " + node(values[j]) + } + out = append(out, line+" @"+strconv.Itoa(i)) + } + } + return out +} + +var printFset = token.NewFileSet() + +func node(n ast.Node) string { + var buf bytes.Buffer + if err := printer.Fprint(&buf, printFset, n); err != nil { + return fmt.Sprintf("<%v>", err) + } + return buf.String() +} diff --git a/scripts/check-phase32.sh b/scripts/check-phase32.sh index 9702cb98..0e8bec08 100755 --- a/scripts/check-phase32.sh +++ b/scripts/check-phase32.sh @@ -35,6 +35,25 @@ # nor a consuming application; the Journal plugin imports # neither the SEO nor the sitemap plugin; the framework go.mod # does not require grobotstxt. +# additive D-106 (the release is a patch tag): the exported API of +# modules/pact, modules/cabana, modules/signpost and every other +# package under modules/ changed since the newest tag, printed +# by scripts/apisurface for a worktree of that tag and for this +# checkout, keeps every line of the tag (no removed +# declaration, field or method, no changed signature, field +# type, struct tag, constant value or underlying type) and adds +# no method or embed to an interface that exists at the tag (a +# new capability goes on a new optional interface). +# compat existing consumers build unchanged against this checkout, +# each through a scratch go.work in a scratch sibling layout: +# sm-sitemap-plugin at HEAD (no change since the phase base; +# build and short tests), sm-journal-plugin at its commit +# before the phase and sm-demo-app at its commit before the +# phase with its pinned submodules (builds, the demo also with +# -tags summercms_mcp, and short tests); every other sibling +# that requires the framework builds at HEAD with its pinned +# submodules ("compat: ok (build)"), or fails at the +# newest tag too ("compat: skipped (fails at too)"). # # A Go stage exits non-zero on a failing command, or on a go test -json run # that fails, skips a test (a subtest included), matches zero tests, prints @@ -43,13 +62,16 @@ # Several stages may be given at once (--go --plugins ...). --all runs # self-test, go, plugins, consumers, admin, docs and marker, then docker # when a Docker daemon answers (NOT RUN docker otherwise, a failure with -# PHASE32_REQUIRE_DOCKER=1). Plan 32-06 adds the additive and compat -# stages. +# PHASE32_REQUIRE_DOCKER=1), then additive and compat. # # Environment overrides (default: the sibling checkouts next to this # repository): PHASE32_ROOT, SEO_PLUGIN, JOURNAL_PLUGIN, DEMO_APP, -# JOURNAL_LAYER, PHASE32_BASE (the commit before the phase, plan 32-01's -# start), PHASE32_ALLOW_UNMOUNTED, PHASE32_REQUIRE_DOCKER. +# SITEMAP_PLUGIN, JOURNAL_LAYER, PHASE32_BASE (the commit before the phase, +# plan 32-01's start), PHASE32_JOURNAL_BASE and PHASE32_DEMO_BASE (the +# Journal and demo commits before the phase, for compat), +# PHASE32_ALLOW_UNMOUNTED, PHASE32_REQUIRE_DOCKER, PHASE32_SKIP_TESTS (a go +# test -skip pattern for the compat stage's short tests, for a test that +# cannot run on this host; each use is printed). set -euo pipefail unset FORCE_COLOR @@ -60,8 +82,13 @@ SEO_PLUGIN="${SEO_PLUGIN:-$ROOT/../sm-seo-plugin}" JOURNAL_PLUGIN="${JOURNAL_PLUGIN:-$ROOT/../sm-journal-plugin}" DEMO_APP="${DEMO_APP:-$ROOT/../sm-demo-app}" JOURNAL_LAYER="${JOURNAL_LAYER:-$ROOT/../vue-journal-layer}" +SITEMAP_PLUGIN="${SITEMAP_PLUGIN:-$ROOT/../sm-sitemap-plugin}" # The framework commit before the phase (plan 32-01's start). PHASE32_BASE="${PHASE32_BASE:-9868838fe3d01e07d578f2b2dd557d4296c48e10}" +# Journal's commit before plan 32-02 and the demo's commit before the phase +# (plan 32-05 did not commit to the demo), for the compat stage. +PHASE32_JOURNAL_BASE="${PHASE32_JOURNAL_BASE:-7fc07d88226cb99091e7fdc1422ee5f4a4509132}" +PHASE32_DEMO_BASE="${PHASE32_DEMO_BASE:-8fcdbdd034ac349431e3775c33cb9ff1826e3e32}" MCP_TAG="summercms_mcp" SEO_ID="golem15.seo" @@ -117,7 +144,7 @@ usage() { cat >&2 <<'EOF' usage: check-phase32.sh --self-test - check-phase32.sh [--go] [--plugins] [--consumers] [--docker] [--admin] [--docs] [--marker] + check-phase32.sh [--go] [--plugins] [--consumers] [--docker] [--admin] [--docs] [--marker] [--additive] [--compat] check-phase32.sh --all EOF exit 2 @@ -545,9 +572,80 @@ run_self_test() { expect_mode docker-required 1 "FAIL docker" --docker-gate "PATH=$tmp/bin:$PATH" PHASE32_REQUIRE_DOCKER=1 expect_mode docker-stage-needs-daemon 1 "needs a reachable Docker daemon" --docker "PATH=$tmp/bin:$PATH" rm -rf "$tmp" + self_test_additive echo "phase32 self-test passed" } +# additive_case TOOL NAME WANT BASE_SRC HEAD_SRC runs a planted package pair +# through scripts/apisurface and additive_compare; WANT is the line the +# comparison must print ("removed or changed", "method added to existing +# interface" or "ok"). +additive_case() { + local tool="$1" name="$2" want="$3" dir out rc=0 ok=0 + dir="$(mktemp -d)" + mkdir -p "$dir/base" "$dir/head" + printf '%s\n' "$4" >"$dir/base/p.go" + printf '%s\n' "$5" >"$dir/head/p.go" + if api_surface "$tool" "$dir/base" "$dir/base.api" && api_surface "$tool" "$dir/head" "$dir/head.api"; then + out="$(additive_compare "selftest/$name" "$dir/base.api" "$dir/head.api" 2>&1)" || rc=$? + case "$want" in + ok) [[ "$rc" -eq 0 ]] && grep -q '^additive: ok selftest/' <<<"$out" && ok=1 ;; + *) [[ "$rc" -ne 0 ]] && grep -q "^additive: $want in selftest/$name: " <<<"$out" && ok=1 ;; + esac + fi + rm -rf -- "${dir:?}" + if [[ "$ok" -ne 1 ]]; then + printf '%s\n' "${out:-}" >&2 + echo "refuse: self-test additive $name (want $want, exit $rc)" + return 1 + fi + echo "PASS self-test additive $name" +} + +# self_test_additive plants the refused and the accepted API changes. +self_test_additive() { + local tmp tool failed=0 + tmp="$(mktemp -d)" + tool="$tmp/apisurface" + (cd "$ROOT" && go build -o "$tool" ./scripts/apisurface) + mkdir -p "$tmp/empty" + local rc=0 + "$tool" "$tmp/empty" >/dev/null 2>&1 || rc=$? + if [[ "$rc" -ne 2 ]]; then + rm -rf -- "${tmp:?}" + echo "refuse: self-test apisurface: exit $rc for a directory without Go files, want 2" >&2 + return 1 + fi + echo "PASS self-test apisurface no-go-files" + local p=$'package p\n\n' + additive_case "$tool" removed-func "removed or changed" "${p}func A() {}"$'\n\n'"func B() {}" "${p}func A() {}" || failed=1 + additive_case "$tool" changed-param "removed or changed" "${p}func F(n int) {}" "${p}func F(n int64) {}" || failed=1 + additive_case "$tool" multiline-param-added "removed or changed" \ + "${p}"$'func F(\n\ta int,\n\tb string,\n) error {\n\treturn nil\n}' \ + "${p}"$'func F(\n\ta int,\n\tb string,\n\tc bool,\n) error {\n\treturn nil\n}' || failed=1 + additive_case "$tool" interface-method-added "method added to existing interface" \ + "${p}"$'// HasThing is an optional capability.\ntype HasThing interface {\n\tThing() string\n}' \ + "${p}"$'// HasThing is an optional capability.\ntype HasThing interface {\n\tThing() string\n\tOther() error\n}' || failed=1 + additive_case "$tool" field-type-changed "removed or changed" "${p}"$'type S struct {\n\tA int\n}' "${p}"$'type S struct {\n\tA int64\n}' || failed=1 + additive_case "$tool" const-value-changed "removed or changed" "${p}const C = 1" "${p}const C = 2" || failed=1 + additive_case "$tool" func-added ok "${p}func A() {}" "${p}func A() {}"$'\n\n'"func B(s string) error { return nil }" || failed=1 + additive_case "$tool" struct-field-added ok \ + "${p}"$'type S struct {\n\tA int\n\tB string `json:"b"`\n}' \ + "${p}"$'type S struct {\n\tA int\n\tB string `json:"b"`\n\tLonger bool\n}' || failed=1 + additive_case "$tool" optional-interface-added ok \ + "${p}"$'type HasThing interface {\n\tThing() string\n}' \ + "${p}"$'type HasThing interface {\n\tThing() string\n}\n\n// HasOther is a new optional capability.\ntype HasOther interface {\n\tOther() error\n\tMore() int\n}' || failed=1 + additive_case "$tool" method-added-to-struct ok \ + "${p}"$'type S struct{}\n\nfunc (S) A() {}' \ + "${p}"$'type S struct{}\n\nfunc (S) A() {}\n\nfunc (s *S) B(n int) error { return nil }' || failed=1 + additive_case "$tool" param-renamed ok "${p}func F(a int, b string) {}" "${p}func F(count int, label string) {}" || failed=1 + additive_case "$tool" comment-edited ok "${p}"$'// F does x.\nfunc F() {}' "${p}"$'// F does y,\n// in two lines.\nfunc F() {}' || failed=1 + rm -rf -- "${tmp:?}" + if [[ "$failed" -ne 0 ]]; then + return 1 + fi +} + run_go() { go_vet "$ROOT" ./... go_short "$ROOT" @@ -711,9 +809,396 @@ run_marker() { echo "phase32 marker passed" } +# --------------------------------------------------------------------------- +# additive and compat (plan 32-06, D-106: the release is a patch tag, so the +# framework change must be strictly additive and existing consumers must +# build unchanged). + +# scratch_init creates the stage's scratch directory and the exit trap that +# removes it and every worktree registered with add_worktree. +SCRATCH="" +CLEANUP_WORKTREES=() +scratch_cleanup() { + local entry + for entry in "${CLEANUP_WORKTREES[@]}"; do + git -c safe.directory='*' -C "${entry%%|*}" worktree remove --force "${entry#*|}" >/dev/null 2>&1 || true + git -c safe.directory='*' -C "${entry%%|*}" worktree prune >/dev/null 2>&1 || true + done + if [[ -n "$SCRATCH" && -d "$SCRATCH" ]]; then + chmod -R u+w -- "${SCRATCH:?}" 2>/dev/null || true + rm -rf -- "${SCRATCH:?}" + fi +} +scratch_init() { + SCRATCH="$(mktemp -d)" + trap scratch_cleanup EXIT +} + +# add_worktree REPO PATH REV adds a detached worktree (git worktree add +# --detach) that the exit trap removes again. +add_worktree() { + git -c safe.directory='*' -C "$1" worktree add --detach -q "$2" "$3" + CLEANUP_WORKTREES+=("$1|$2") +} + +# snapshot GITDIR REV DEST writes the tree of REV into DEST through a +# temporary index: nothing is written to the repository, so read-only +# checkouts (other users' repositories, submodule git dirs) work too. +snapshot() { + local gitdir="$1" rev="$2" dest="$3" index rc=0 + index="${SCRATCH:?}/index.$RANDOM$RANDOM" + mkdir -p "$dest" + GIT_INDEX_FILE="$index" git -c safe.directory='*' --git-dir="$gitdir" read-tree "$rev" >/dev/null 2>&1 && + GIT_INDEX_FILE="$index" git -c safe.directory='*' --git-dir="$gitdir" --work-tree="$dest" checkout-index -a -f >/dev/null 2>&1 || rc=$? + rm -f -- "$index" "$index.lock" + return "$rc" +} + +# git_dir_of DIR prints the absolute git dir of the checkout in DIR. +git_dir_of() { + git -c safe.directory='*' -C "$1" rev-parse --absolute-git-dir 2>/dev/null +} + +# snapshot_submodules REPO REV DEST fills every gitlink of REV under DEST +# with a snapshot of its pinned commit, read from REPO's submodule checkout +# (or REPO/.git/modules/). A gitlink without a local copy of the +# pinned commit is reported and left empty; the build then shows it. +snapshot_submodules() { + local repo="$1" rev="$2" dest="$3" mode type sha path gd + while read -r mode type sha path; do + [[ "$mode" == "160000" ]] || continue + gd="" + if [[ -e "$repo/$path/.git" ]]; then + gd="$(git_dir_of "$repo/$path" || true)" + elif [[ -d "$repo/.git/modules/$path" ]]; then + gd="$repo/.git/modules/$path" + fi + if [[ -z "$gd" ]] || ! snapshot "$gd" "$sha" "$dest/$path"; then + echo "compat: note $(basename "$repo"): submodule $path at ${sha:0:12} is not available locally" >&2 + fi + done < <(git -c safe.directory='*' -C "$repo" ls-tree -r "$rev") +} + +# sibling_farm DIR SKIP creates DIR with a symlink to every sibling of this +# checkout except SKIP (the consumer placed there), so relative references +# of the consumer (go.mod and go.work replaces, tests reading +# ../summercms.go) resolve as they do next to the real checkouts. +sibling_farm() { + local farm="$1" skip="$2" parent entry + parent="$(dirname "$ROOT")" + mkdir -p "$farm" + for entry in "$parent"/*; do + [[ "$(basename "$entry")" == "$skip" ]] && continue + ln -s "$entry" "$farm/$(basename "$entry")" + done +} + +# scratch_gowork APP FRAMEWORK prints a go.work for APP (a checkout placed in +# a sibling farm): APP's own go.work (or "use ." when it has none) with every +# use and replace path made absolute, any replace of the framework dropped, +# a use of this checkout pointed at FRAMEWORK, and the framework replaced by +# FRAMEWORK. +scratch_gowork() { + python3 - "$1" "$2" "$ROOT" <<'PY' +import os, re, sys +app, framework, root = sys.argv[1:4] +module = "git.golem15.com/golem15/summercms" +real_root = os.path.realpath(root) +used = False + +def absolute(path): + global used + full = path if path.startswith("/") else os.path.normpath(os.path.join(app, path)) + if os.path.realpath(full) == real_root: + used = True + return framework + return full + +def replace(spec): + left, right = (part.strip() for part in spec.split("=>", 1)) + if left.split()[0] == module: + return None + target = right.split() + if len(target) == 1 and target[0].startswith((".", "/")): + right = absolute(target[0]) + return f"{left} => {right}" + +src = os.path.join(app, "go.work") +if os.path.exists(src): + text = open(src, encoding="utf-8").read() +else: + gover = "1.27.0" + for line in open(os.path.join(app, "go.mod"), encoding="utf-8"): + m = re.match(r"^go\s+(\S+)", line) + if m: + gover = m.group(1) + break + text = f"go {gover}\n\nuse .\n" +out, block = [], None +for line in text.splitlines(): + code = line.split("//", 1)[0].strip() + if block is not None: + if code == ")": + block = None + out.append(")") + elif code and block == "use": + out.append("\t" + absolute(code)) + elif code: + rewritten = replace(code) + if rewritten is not None: + out.append("\t" + rewritten) + continue + if code in ("use (", "replace ("): + block = code.split()[0] + out.append(code) + continue + m = re.fullmatch(r"use\s+(\S+)", code) + if m: + out.append("use " + absolute(m.group(1))) + continue + m = re.fullmatch(r"replace\s+(.+)", code) + if m: + rewritten = replace(m.group(1)) + if rewritten is not None: + out.append("replace " + rewritten) + continue + out.append(line) +if not used: + out += ["", f"replace {module} => {framework}"] +print("\n".join(out)) +PY +} + +# consumer_run NAME APP FRAMEWORK CMD... writes APP's scratch go.work outside +# APP, checks that the framework resolves to FRAMEWORK and runs CMD in APP +# with that workspace. The output goes to a log whose last lines are printed +# on failure unless COMPAT_QUIET=1. +consumer_run() { + local name="$1" app="$2" framework="$3" work got rc=0 + shift 3 + work="${SCRATCH:?}/work/$name.$(basename "$framework")" + mkdir -p "$work" + if [[ ! -f "$work/go.work" ]]; then + scratch_gowork "$app" "$framework" >"$work/go.work" + fi + got="$( (cd "$app" && GOWORK="$work/go.work" go list -m -f '{{if .Replace}}{{.Replace.Dir}}{{else}}{{.Dir}}{{end}}' git.golem15.com/golem15/summercms 2>/dev/null) || true)" + if [[ -z "$got" || "$(realpath -e -- "$got" 2>/dev/null || true)" != "$(realpath -e -- "$framework")" ]]; then + echo "refuse: compat: $name resolves git.golem15.com/golem15/summercms to ${got:-}, not $framework" >&2 + return 1 + fi + (cd "$app" && GOWORK="$work/go.work" "$@") >"$work/log" 2>&1 || rc=$? + if [[ "$rc" -ne 0 && "${COMPAT_QUIET:-0}" != "1" ]]; then + grep -v '^ok ' "$work/log" | tail -n 40 >&2 || true + echo "refuse: compat: $name: $* (exit $rc)" >&2 + fi + return "$rc" +} + +# last_tag prints the newest release tag of this checkout. +last_tag() { + git -C "$ROOT" tag --sort=-v:refname | head -n 1 +} + +# base_worktree creates the last tag's worktree once (BASE_TREE). +BASE_TREE="" +base_worktree() { + if [[ -z "$BASE_TREE" ]]; then + BASE_TREE="${SCRATCH:?}/base" + add_worktree "$ROOT" "$BASE_TREE" "$(last_tag)" + fi +} + +# additive_compare PKG BASE_API HEAD_API refuses every API line of the base +# missing from HEAD (removed or changed) and every interface method or embed +# HEAD adds to an interface that exists in the base; it prints +# "additive: ok PKG (+N lines)" otherwise. +additive_compare() { + local pkg="$1" base="$2" head="$3" line name refused=0 added + while IFS= read -r line; do + [[ -n "$line" ]] || continue + echo "additive: removed or changed in $pkg: $line" >&2 + refused=1 + done < <(LC_ALL=C comm -23 "$base" "$head") + while IFS= read -r line; do + [[ "$line" =~ ^(imethod|iembed)\ ([A-Za-z0-9_]+)\. ]] || continue + name="${BASH_REMATCH[2]}" + if grep -qE "^type ${name}(\[.*\])? interface\$" "$base"; then + echo "additive: method added to existing interface in $pkg: $line" >&2 + refused=1 + fi + done < <(LC_ALL=C comm -13 "$base" "$head") + if [[ "$refused" -ne 0 ]]; then + return 1 + fi + added="$(LC_ALL=C comm -13 "$base" "$head" | grep -c . || true)" + echo "additive: ok $pkg (+$added lines)" +} + +# api_surface TOOL DIR OUT writes DIR's sorted API to OUT; a directory that +# does not exist or has no Go files has an empty API. +api_surface() { + local tool="$1" dir="$2" out="$3" rc=0 + if [[ ! -d "$dir" ]]; then + : >"$out" + return 0 + fi + "$tool" "$dir" >"$out.raw" 2>"$out.err" || rc=$? + if [[ "$rc" -eq 2 ]]; then + : >"$out.raw" + elif [[ "$rc" -ne 0 ]]; then + cat "$out.err" >&2 + echo "refuse: apisurface $dir (exit $rc)" >&2 + return 1 + fi + LC_ALL=C sort "$out.raw" >"$out" +} + +# additive_packages TAG BASE prints modules/pact, modules/cabana, +# modules/signpost and every other directory under modules/ holding a +# non-test .go file changed since TAG (uncommitted changes included) that +# exists at TAG. +additive_packages() { + local tag="$1" base="$2" dir + { + printf '%s\n' modules/pact modules/cabana modules/signpost + git -C "$ROOT" diff --name-only "$tag" -- modules | + grep -E '\.go$' | grep -vE '_test\.go$' | xargs -r -n1 dirname + } | LC_ALL=C sort -u | while IFS= read -r dir; do + if [[ -d "$base/$dir" ]]; then + echo "$dir" + fi + done +} + +run_additive() { + local tag dir name failed=0 pkgs=() + tag="$(last_tag)" + if [[ -z "$tag" ]]; then + echo "refuse: additive: $ROOT has no release tag" >&2 + return 1 + fi + scratch_init + base_worktree + (cd "$ROOT" && go build -o "$SCRATCH/apisurface" ./scripts/apisurface) + mapfile -t pkgs < <(additive_packages "$tag" "$BASE_TREE") + mkdir -p "$SCRATCH/api" + for dir in "${pkgs[@]}"; do + name="${dir//\//_}" + api_surface "$SCRATCH/apisurface" "$BASE_TREE/$dir" "$SCRATCH/api/$name.base" + api_surface "$SCRATCH/apisurface" "$ROOT/$dir" "$SCRATCH/api/$name.head" + additive_compare "$dir" "$SCRATCH/api/$name.base" "$SCRATCH/api/$name.head" || failed=1 + done + if [[ "$failed" -ne 0 ]]; then + echo "refuse: additive: the framework API changed incompatibly since $tag (D-106: a patch release is strictly additive)" >&2 + return 1 + fi + echo "phase32 additive passed (${#pkgs[@]} packages against $tag)" +} + +# compat_tested NAME APP FRAMEWORK builds APP (also with each build tag in +# COMPAT_TAGS) and runs its short tests. +compat_tested() { + local name="$1" app="$2" framework="$3" tags + consumer_run "$name" "$app" "$framework" go build ./... + for tags in ${COMPAT_TAGS:-}; do + consumer_run "$name" "$app" "$framework" go build -tags "$tags" ./... + done + if [[ -n "${PHASE32_SKIP_TESTS:-}" ]]; then + echo "compat: note $name: tests matching $PHASE32_SKIP_TESTS skipped (PHASE32_SKIP_TESTS)" >&2 + consumer_run "$name" "$app" "$framework" go test -count=1 -short -skip "$PHASE32_SKIP_TESTS" ./... + else + consumer_run "$name" "$app" "$framework" go test -count=1 -short ./... + fi + echo "compat: ok $name" +} + +run_compat() { + local parent farm repo name gd real failed=0 base_date tag skip=() + parent="$(dirname "$ROOT")" + tag="$(last_tag)" + require_dir "$SITEMAP_PLUGIN" "sitemap plugin" + require_dir "$JOURNAL_PLUGIN" "Journal plugin" + require_dir "$DEMO_APP" "demo application" + export GOPRIVATE="${GOPRIVATE:-git.golem15.com/*}" + scratch_init + + # (a) sm-sitemap-plugin, which the phase must not touch, at HEAD. + base_date="$(git -C "$ROOT" log -1 --format=%cI "$PHASE32_BASE")" + if [[ -n "$(git -c safe.directory='*' -C "$SITEMAP_PLUGIN" status --porcelain)" ]] || + [[ -n "$(git -c safe.directory='*' -C "$SITEMAP_PLUGIN" log --oneline --since="$base_date" HEAD)" ]]; then + echo "refuse: compat: $SITEMAP_PLUGIN changed since the phase base ($base_date); it must build unchanged" >&2 + return 1 + fi + name="$(basename "$SITEMAP_PLUGIN")" + farm="$SCRATCH/a" + sibling_farm "$farm" "$name" + snapshot "$(git_dir_of "$SITEMAP_PLUGIN")" HEAD "$farm/$name" + compat_tested "$name" "$farm/$name" "$ROOT" + + # (b) sm-journal-plugin at its commit before the phase. + name="$(basename "$JOURNAL_PLUGIN")" + farm="$SCRATCH/b" + sibling_farm "$farm" "$name" + add_worktree "$JOURNAL_PLUGIN" "$farm/$name" "$PHASE32_JOURNAL_BASE" + compat_tested "$name" "$farm/$name" "$ROOT" + + # (c) sm-demo-app at its commit before the phase, with the submodules + # pinned there. + name="$(basename "$DEMO_APP")" + farm="$SCRATCH/c" + sibling_farm "$farm" "$name" + add_worktree "$DEMO_APP" "$farm/$name" "$PHASE32_DEMO_BASE" + snapshot_submodules "$DEMO_APP" "$PHASE32_DEMO_BASE" "$farm/$name" + COMPAT_TAGS="$MCP_TAG" compat_tested "$name" "$farm/$name" "$ROOT" + + # (d) every other sibling that requires the framework, at HEAD with its + # pinned submodules: it builds, or it fails at the last tag too. + for repo in "$ROOT" "$SITEMAP_PLUGIN" "$JOURNAL_PLUGIN" "$DEMO_APP" "$SEO_PLUGIN"; do + skip+=("$(realpath -m -- "$repo")") + done + for repo in "$parent"/*; do + [[ -f "$repo/go.mod" ]] || continue + grep -qE '^[[:space:]]*(require[[:space:]]+)?git\.golem15\.com/golem15/summercms[[:space:]]+v' "$repo/go.mod" || continue + real="$(realpath -m -- "$repo")" + if printf '%s\n' "${skip[@]}" | grep -qxF -- "$real"; then + continue + fi + name="$(basename "$repo")" + gd="$(git_dir_of "$repo" || true)" + if [[ -z "$gd" ]]; then + echo "compat: skipped $name (not a git checkout)" + continue + fi + farm="$SCRATCH/d/$name" + sibling_farm "$farm" "$name" + if ! snapshot "$gd" HEAD "$farm/$name"; then + echo "refuse: compat: cannot read $name at HEAD" >&2 + failed=1 + continue + fi + snapshot_submodules "$repo" HEAD "$farm/$name" + if COMPAT_QUIET=1 consumer_run "$name" "$farm/$name" "$ROOT" go build ./...; then + echo "compat: ok $name (build)" + continue + fi + base_worktree + if COMPAT_QUIET=1 consumer_run "$name" "$farm/$name" "$BASE_TREE" go build ./...; then + grep -v '^ok ' "$SCRATCH/work/$name.$(basename "$ROOT")/log" | tail -n 40 >&2 || true + echo "refuse: compat: $name builds at $tag but not at HEAD" >&2 + failed=1 + else + echo "compat: skipped $name (fails at $tag too)" + fi + done + if [[ "$failed" -ne 0 ]]; then + return 1 + fi + echo "phase32 compat passed" +} + run_all() { local stage - for stage in self-test go plugins consumers admin docs marker; do + for stage in self-test go plugins consumers admin docs marker additive compat; do if bash "${BASH_SOURCE[0]}" "--$stage"; then echo "PASS $stage" else @@ -737,6 +1222,8 @@ run_stage() { --admin | admin) run_admin ;; --docs | docs) run_docs ;; --marker | marker) run_marker ;; + --additive | additive) run_additive ;; + --compat | compat) run_compat ;; --all | all) run_all ;; *) usage ;; esac