test(surf,conga,lagoon): summaries, reports, stats and checklist flags

- serve summary lines at -v and -q, records and HideOnTTY, DSN labels
- shutdown report: aborted requests fail the stop, a drained stop is nil
- request log: request_id cap, default on, off by config or env, never the
  path for matched, 404 and 405 routes, component http; route:list --json
- WorkerStats through runAttempt: done, failed, in flight, handed back
- queue:clear and lagoon's production question on a pseudo-terminal:
  no cancels with exit 1, --force, -n and other environments skip it
- Docker: PendingMigrations, migrate/rollback/deferred:purge --pretend,
  production runs off a terminal never read stdin, migrate:status --json
- key:generate --json and -q
This commit is contained in:
Jakub Zych
2026-10-21 19:38:18 +02:00
parent fd5286884e
commit 6720caebd3
7 changed files with 1363 additions and 0 deletions

View File

@@ -4,14 +4,19 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"git.golem15.com/golem15/summercms/modules/backpack"
"git.golem15.com/golem15/summercms/modules/compass"
"git.golem15.com/golem15/summercms/modules/diary"
"git.golem15.com/golem15/summercms/modules/pact"
"git.golem15.com/golem15/summercms/modules/party"
)
@@ -413,3 +418,167 @@ func TestAccessLog(t *testing.T) {
t.Fatalf("direct records = %v", recs)
}
}
// sharePlugin serves token-bearing paths: GET /share/{token} and
// POST /invite/{token}.
type sharePlugin struct{}
func (sharePlugin) ID() string { return "acme.share" }
func (sharePlugin) Requires() []string { return nil }
func (sharePlugin) Register(*backpack.App) error { return nil }
func (sharePlugin) Boot(*backpack.App) error { return nil }
func (sharePlugin) Routes(r pact.Router) error {
r.Get("/share/{token}", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) })
r.Post("/invite/{token}", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
return nil
}
// diaryAccessRouter assembles plugin with httpYAML and no published
// logger, so the request log goes to the diary component http, set up on a
// console buffer with environ.
func diaryAccessRouter(t *testing.T, httpYAML string, plugin party.Plugin, environ ...string) (http.Handler, *syncBuffer) {
t.Helper()
app := backpack.New(writeHTTPConfig(t, "body_limits:\n default_bytes: 1024\n upload_bytes: 1024\n"+httpYAML))
h, err := Assemble(app, []party.Plugin{plugin})
if err != nil {
t.Fatal(err)
}
cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Environ: append([]string{"SUMMER_LOG__DEFAULT=console"}, environ...)})
if err != nil {
t.Fatal(err)
}
var buf syncBuffer
t.Cleanup(diary.Reset)
if err := diary.Setup(cfg, diary.Options{Stderr: &buf, Root: t.TempDir(), Environ: []string{}}); err != nil {
t.Fatal(err)
}
return h, &buf
}
// TestAccessLogRequestIDCap: X-Request-Id is trimmed, cut to 128 bytes on
// a rune boundary and stripped of invalid UTF-8.
func TestAccessLogRequestIDCap(t *testing.T) {
h, buf := accessLogRouter(t, true)
cases := map[string]string{
strings.Repeat("a", 300): strings.Repeat("a", errorPageMaxRequestID),
"\t req-7 \t": "req-7",
strings.Repeat("a", 127) + "é" + "tail": strings.Repeat("a", 127),
"bad\xffid": "badid",
}
for header, want := range cases {
buf.Reset()
req := httptest.NewRequest(http.MethodGet, "/blog/posts/1", nil)
req.Header["X-Request-Id"] = []string{header}
h.ServeHTTP(httptest.NewRecorder(), req)
recs := accessRecords(t, buf)
if len(recs) != 1 || recs[0]["request_id"] != want {
t.Errorf("X-Request-Id %q: records %v, want request_id %q", header, recs, want)
}
if id, _ := recs[0]["request_id"].(string); len(id) > errorPageMaxRequestID {
t.Errorf("request_id is %d bytes", len(id))
}
}
buf.Reset()
req := httptest.NewRequest(http.MethodGet, "/blog/posts/1", nil)
req.Header.Set("X-Request-Id", " ")
h.ServeHTTP(httptest.NewRecorder(), req)
if recs := accessRecords(t, buf); len(recs) != 1 || recs[0]["request_id"] != nil {
t.Errorf("a blank X-Request-Id gives %v, want no request_id", recs)
}
}
// TestAccessLogDefaultOn: without http.access_log, and with it true, every
// request is logged once.
func TestAccessLogDefaultOn(t *testing.T) {
for name, yaml := range map[string]string{"unset": "", "true": "access_log: true\n"} {
h, buf := diaryAccessRouter(t, yaml, blogPostsPlugin{})
for range 3 {
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/blog/posts/7", nil))
}
if n := strings.Count(buf.String(), `msg="http request"`); n != 3 {
t.Errorf("%s: %d request records, want 3:\n%s", name, n, buf.String())
}
}
}
// TestAccessLogDisabled: http.access_log false, also as the string an
// environment variable gives, turns the request log off.
func TestAccessLogDisabled(t *testing.T) {
h, buf := diaryAccessRouter(t, "access_log: false\n", blogPostsPlugin{})
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/blog/posts/7", nil))
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/nowhere", nil))
if strings.Contains(buf.String(), "http request") {
t.Errorf("access_log false still logged:\n%s", buf.String())
}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "http.yaml"), []byte("body_limits:\n default_bytes: 1024\n upload_bytes: 1024\n"), 0o644); err != nil {
t.Fatal(err)
}
cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{"SUMMER_ENV=development", "SUMMER_HTTP__ACCESS_LOG=false"}})
if err != nil {
t.Fatal(err)
}
if accessLogEnabled(backpack.New(cfg)) {
t.Error("SUMMER_HTTP__ACCESS_LOG=false does not turn the request log off")
}
if !accessLogEnabled(nil) || !accessLogEnabled(backpack.New(nil)) {
t.Error("the request log is off without a config")
}
}
// TestAccessLogNeverLogsPath: token-bearing paths and queries never reach
// the request log, for a matched route, a 404 (trailing slash) and a 405
// (wrong method), on the default diary console.
func TestAccessLogNeverLogsPath(t *testing.T) {
h, buf := diaryAccessRouter(t, "", sharePlugin{})
cases := []struct {
method, target string
status int
pattern string
}{
{http.MethodGet, "/share/tok-secret-200?sig=query-secret", http.StatusOK, `pattern="GET /share/{token}"`},
{http.MethodGet, "/share/tok-secret-404/?sig=query-secret", http.StatusNotFound, `pattern=""`},
{http.MethodDelete, "/share/tok-secret-405?sig=query-secret", http.StatusMethodNotAllowed, `pattern=""`},
{http.MethodGet, "/invite/tok-secret-invite", http.StatusMethodNotAllowed, `pattern=""`},
{http.MethodPost, "/invite/tok-secret-post", http.StatusNoContent, `pattern="POST /invite/{token}"`},
}
for _, tc := range cases {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(tc.method, tc.target, nil))
if rec.Code != tc.status {
t.Fatalf("%s %s = %d, want %d", tc.method, tc.target, rec.Code, tc.status)
}
}
got := buf.String()
lines := strings.Split(strings.TrimSpace(got), "\n")
if len(lines) != len(cases) {
t.Fatalf("want %d request records, got %d:\n%s", len(cases), len(lines), got)
}
for i, tc := range cases {
if !strings.Contains(lines[i], tc.pattern) || !strings.Contains(lines[i], fmt.Sprintf("status=%d", tc.status)) {
t.Errorf("record %d = %q, want %s status=%d", i, lines[i], tc.pattern, tc.status)
}
}
for _, leak := range []string{"tok-secret", "query-secret", "sig=", "/share/tok", "/invite/tok"} {
if strings.Contains(got, leak) {
t.Errorf("the request log carries %q:\n%s", leak, got)
}
}
}
// TestAccessLogComponentHTTP: without a published logger the request log
// is the diary component http, so log.components.http: warn silences it.
func TestAccessLogComponentHTTP(t *testing.T) {
h, buf := diaryAccessRouter(t, "", blogPostsPlugin{})
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/blog/posts/7", nil))
if !strings.Contains(buf.String(), "component=http") || !strings.Contains(buf.String(), "level=INFO") {
t.Errorf("request record lacks component=http:\n%s", buf.String())
}
quiet, qbuf := diaryAccessRouter(t, "", blogPostsPlugin{}, "SUMMER_LOG__COMPONENTS__HTTP=warn")
quiet.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/blog/posts/7", nil))
if strings.Contains(qbuf.String(), "http request") {
t.Errorf("log.components.http warn still logged:\n%s", qbuf.String())
}
}

View File

@@ -0,0 +1,352 @@
package surf
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.golem15.com/golem15/summercms/modules/backpack"
"git.golem15.com/golem15/summercms/modules/bonfire"
"git.golem15.com/golem15/summercms/modules/compass"
"git.golem15.com/golem15/summercms/modules/diary"
"git.golem15.com/golem15/summercms/modules/lagoon"
"git.golem15.com/golem15/summercms/modules/party"
)
// summaryFixture is buildFacts over summaryApp with two plugins, three
// pending migrations and a worker.
func summaryFixture(t *testing.T) summaryFacts {
t.Helper()
return buildFacts(summaryApp(t), factInputs{
addr: ":8096",
bin: "bin/acme",
plugins: []party.Plugin{summaryPlugin{id: "acme.user"}, summaryPlugin{id: "acme.blog", req: []string{"acme.user"}}},
queues: []string{"default"},
worker: true,
adminPrefix: "/backend",
mounted: true,
admins: adminsPresent,
pending: []lagoon.MigrationRef{
{Plugin: "acme.blog", ID: "0001_posts"},
{Plugin: "acme.blog", ID: "0002_tags"},
{Plugin: "summercms.cabana", ID: "0001"},
},
version: bonfire.Version{App: "v1.4.0", Framework: "v0.4.0"},
})
}
// runSummaryProbe prints the summary through a command run with args and
// returns its output.
func runSummaryProbe(t *testing.T, f summaryFacts, args ...string) string {
t.Helper()
probe := []bonfire.Command{{Name: "probe:summary", Run: func(_ context.Context, _ bonfire.Input, out bonfire.Output) error {
printSummary(out, f)
return nil
}}}
var buf bytes.Buffer
if err := bonfire.Call(context.Background(), probe, "probe:summary", args, &buf); err != nil {
t.Fatal(err)
}
return buf.String()
}
// TestServeSummaryLines: the block at verbosity 0, the per-plugin lines at
// -v, nothing under -q, and never a DSN user or password; the database
// label for URL, keyword, socket, IPv6, empty and unparsable DSNs.
func TestServeSummaryLines(t *testing.T) {
f := summaryFixture(t)
plain := runSummaryProbe(t, f)
for _, want := range []string{
"SummerCMS acme v1.4.0 (framework v0.4.0)\n",
" Environment staging\n",
" URL http://127.0.0.1:8096/\n",
" Admin http://127.0.0.1:8096/backend\n",
" Database db.internal:5433/acme_prod\n",
" Migrations 3 pending — run bin/acme migrate\n",
" Plugins 2 (acme.user, acme.blog)\n",
" Queue default (worker in serve)\n",
" Scheduler in serve\n",
} {
if !strings.Contains(plain, want) {
t.Errorf("verbosity 0 block lacks %q:\n%s", want, plain)
}
}
if strings.Contains(plain, "requires") || strings.Contains(plain, "sign in") {
t.Errorf("verbosity 0 block has per-plugin lines or an admin note with admins present:\n%s", plain)
}
verbose := runSummaryProbe(t, f, "-v")
for _, want := range []string{
" acme.user requires nothing; 0 pending migrations\n",
" acme.blog requires acme.user; 2 pending migrations\n",
} {
if !strings.Contains(verbose, want) {
t.Errorf("-v block lacks %q:\n%s", want, verbose)
}
}
if quiet := runSummaryProbe(t, f, "-q"); quiet != "" {
t.Errorf("-q printed the block:\n%s", quiet)
}
for _, out := range []string{plain, verbose} {
for _, secret := range []string{"summer_user", "pa55-secret", "postgres://"} {
if strings.Contains(out, secret) {
t.Fatalf("summary leaks %q:\n%s", secret, out)
}
}
}
// The worker off, migrations up to date or unknown.
off := buildFacts(summaryApp(t), factInputs{addr: "127.0.0.1:80", bin: "bin/acme", version: bonfire.Version{App: "devel"}})
if off.URL != "http://127.0.0.1/" || !strings.Contains(off.Queue, "queue.work_in_serve is false") || !strings.Contains(off.Scheduler, "run schedule:run") {
t.Errorf("worker off facts = %+v", off)
}
if m, warn := off.migrations(); m != "up to date" || warn {
t.Errorf("no pending migrations = %q, %v", m, warn)
}
unknown := buildFacts(summaryApp(t), factInputs{addr: ":1", pendingErr: errors.New("no history")})
if m, warn := unknown.migrations(); !strings.HasPrefix(m, "unknown") || !warn || unknown.Pending != -1 {
t.Errorf("unreadable migrations = %q, %v, %d", m, warn, unknown.Pending)
}
for dsn, want := range map[string]string{
"postgres://u:secret@db.example:5432/app": "db.example:5432/app",
"host=db port=6543 dbname=acme user=u password=p": "db:6543/acme",
"host=/var/run/postgresql dbname=acme user=u": "socket /var/run/postgresql:5432/acme",
"postgres://u:p@[::1]:5432/acme": "[::1]:5432/acme",
"": "not configured",
"postgres://u:p@[::1:bad": "configured",
} {
if got := lagoon.DatabaseLabel(dsn); got != want {
t.Errorf("DatabaseLabel(%q) = %q, want %q", dsn, got, want)
}
}
}
// TestServeRecords: off a terminal serve logs one "startup summary" record
// with every fact and one record whose message is exactly "listening on
// <addr>"; with HideOnTTY (used only when the block printed) a styled
// console skips both while the daily file keeps them.
func TestServeRecords(t *testing.T) {
f := summaryFixture(t)
var logs bytes.Buffer
logSummary(context.Background(), slog.New(slog.NewJSONHandler(&logs, nil)), f)
var summary map[string]any
var messages []string
for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") {
var rec map[string]any
if err := json.Unmarshal([]byte(line), &rec); err != nil {
t.Fatal(err)
}
messages = append(messages, rec["msg"].(string))
if rec["msg"] == summaryMessage {
summary = rec
}
}
if strings.Join(messages, "|") != "startup summary|listening on :8096" {
t.Fatalf("records = %q, want the summary and the listening line (no admin note with admins present)", messages)
}
keys := []string{"app", "environment", "url", "admin", "database", "migrations_pending", "plugins", "plugin_ids", "queue", "scheduler", "mail", "cache", "realtime", "search"}
for _, k := range keys {
if _, ok := summary[k]; !ok {
t.Errorf("summary record lacks %s: %v", k, summary)
}
}
if summary["app"] != "SummerCMS acme v1.4.0 (framework v0.4.0)" || summary["url"] != "http://127.0.0.1:8096/" {
t.Errorf("summary record = %v", summary)
}
// Admin notes follow, warnings at WARN.
logs.Reset()
noAdmins := f
noAdmins.Admin, noAdmins.AdminNotes = adminStatus(f.URL, "/backend", true, adminsNone)
logSummary(context.Background(), slog.New(slog.NewJSONHandler(&logs, nil)), noAdmins)
if !strings.Contains(logs.String(), `"level":"WARN","msg":"no administrators yet`) {
t.Errorf("admin warning record missing:\n%s", logs.String())
}
// HideOnTTY: hidden on a styled console, kept in the daily file.
root := t.TempDir()
cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Environ: []string{"SUMMER_LOG__CHANNELS__DAILY__LEVEL=info"}})
if err != nil {
t.Fatal(err)
}
var console syncBuffer
t.Cleanup(diary.Reset)
now := func() time.Time { return time.Date(2026, 10, 21, 12, 0, 0, 0, time.Local) }
if err := diary.Setup(cfg, diary.Options{Stderr: &console, Root: root, Color: bonfire.ColorAlways, Environ: []string{}, Now: now}); err != nil {
t.Fatal(err)
}
logSummary(diary.HideOnTTY(context.Background()), serveLog, f)
if strings.Contains(console.String(), summaryMessage) || strings.Contains(console.String(), "listening on") {
t.Errorf("styled console shows the hidden records:\n%s", console.String())
}
logSummary(context.Background(), serveLog, f)
if !strings.Contains(console.String(), "listening on :8096") {
t.Errorf("styled console lacks the records logged without HideOnTTY:\n%s", console.String())
}
if err := diary.Close(); err != nil {
t.Fatal(err)
}
daily, err := os.ReadFile(filepath.Join(root, "storage", "logs", "summer-2026-10-21.log"))
if err != nil {
t.Fatal(err)
}
if n := strings.Count(string(daily), `"msg":"listening on :8096"`); n != 2 {
t.Errorf("daily file has %d listening records, want 2 (hidden and shown):\n%s", n, daily)
}
}
// shutdownServer serves handler through a requestCounter on a loopback
// listener and returns the server, the counter and the base URL.
func shutdownServer(t *testing.T, handler http.Handler) (*http.Server, *requestCounter, chan error, string) {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
counter := newRequestCounter(handler)
srv := &http.Server{Handler: counter}
errCh := make(chan error, 1)
go func() { errCh <- srv.Serve(ln) }()
return srv, counter, errCh, "http://" + ln.Addr().String()
}
// TestShutdownReportAborted: a request still running at the drain deadline
// is counted as aborted, the drain error is in the report and Err returns
// it, which makes serve exit 1.
func TestShutdownReportAborted(t *testing.T) {
started, release := make(chan struct{}), make(chan struct{})
srv, counter, errCh, base := shutdownServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
close(started)
<-release
w.WriteHeader(http.StatusOK)
}))
defer close(release)
go func() {
if resp, err := http.Get(base + "/slow"); err == nil {
_ = resp.Body.Close()
}
}()
<-started
report := shutdownReport{Draining: counter.inFlight.Load()}
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
report.Aborted = counter.inFlight.Load()
_ = srv.Close()
report.Errors = append(report.Errors, fmt.Errorf("surf: drain requests: %w", err))
}
if err := <-errCh; err != http.ErrServerClosed {
t.Fatalf("Serve returned %v", err)
}
report.Served = counter.served.Load()
if report.Draining != 1 || report.Aborted != 1 || report.Drained() != 0 {
t.Fatalf("report = %+v, want 1 draining, 1 aborted, 0 drained", report)
}
if err := report.Err(); err == nil || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Err() = %v, want the drain deadline error", err)
}
var buf bytes.Buffer
bonfire.Details(bonfire.NewOutput(strings.NewReader(""), &buf, &buf), "Shutdown report", report.details())
for _, want := range []string{"0 drained, 1 aborted", "surf: drain requests: context deadline exceeded"} {
if !strings.Contains(buf.String(), want) {
t.Errorf("report lacks %q:\n%s", want, buf.String())
}
}
}
// TestServeGracefulNil: requests that finish within the deadline drain,
// the report counts them as served and drained, and Err is nil (exit 0).
func TestServeGracefulNil(t *testing.T) {
started, release := make(chan struct{}), make(chan struct{})
srv, counter, errCh, base := shutdownServer(t, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/slow" {
close(started)
<-release
}
w.WriteHeader(http.StatusOK)
}))
for range 3 {
resp, err := http.Get(base + "/fast")
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
}
slowDone := make(chan int, 1)
go func() {
resp, err := http.Get(base + "/slow")
if err != nil {
slowDone <- 0
return
}
_ = resp.Body.Close()
slowDone <- resp.StatusCode
}()
<-started
report := shutdownReport{Draining: counter.inFlight.Load()}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
shut := make(chan error, 1)
go func() { shut <- srv.Shutdown(ctx) }()
close(release)
if err := <-shut; err != nil {
t.Fatalf("Shutdown: %v", err)
}
if err := <-errCh; err != http.ErrServerClosed {
t.Fatalf("Serve returned %v", err)
}
if code := <-slowDone; code != http.StatusOK {
t.Fatalf("the in-flight request got %d, want 200", code)
}
report.Served = counter.served.Load()
if report.Draining != 1 || report.Aborted != 0 || report.Drained() != 1 || report.Served != 4 {
t.Fatalf("report = %+v, want 1 draining, 1 drained, 4 served", report)
}
if err := report.Err(); err != nil {
t.Fatalf("Err() = %v, want nil after a graceful stop", err)
}
}
// TestRouteListJSON: route:list --json prints every route as a JSON array
// with method, pattern, plugin, middleware (never null) and raw, under -q
// too; the table and the admin line are left out.
func TestRouteListJSON(t *testing.T) {
app := backpack.New(writeHTTPConfig(t, "body_limits:\n default_bytes: 1024\n upload_bytes: 1024\n"))
cmd := RouteListCommand(app, []party.Plugin{blogPostsPlugin{}})
for _, args := range [][]string{{"--json"}, {"--json", "-q"}} {
var buf bytes.Buffer
if err := bonfire.Call(context.Background(), []bonfire.Command{cmd}, "route:list", args, &buf); err != nil {
t.Fatal(err)
}
var routes []map[string]any
if err := json.Unmarshal(buf.Bytes(), &routes); err != nil {
t.Fatalf("%v: not a JSON array: %v\n%s", args, err, buf.String())
}
if len(routes) != 1 {
t.Fatalf("%v: routes = %v", args, routes)
}
r := routes[0]
if r["method"] != "GET" || r["pattern"] != "/blog/posts/{id}" || r["plugin"] != "acme.blog" || r["raw"] != false {
t.Errorf("%v: route = %v", args, r)
}
if mw, ok := r["middleware"].([]any); !ok || mw == nil {
t.Errorf("%v: middleware is not an array: %v", args, r["middleware"])
}
if strings.Contains(buf.String(), "admin:") {
t.Errorf("%v: JSON output carries the admin line:\n%s", args, buf.String())
}
}
}