package quizzes_test import ( "context" "errors" "math" "strings" "sync" "testing" "time" "git.golem15.com/golem15/sm-quizzes-plugin/classes" "git.golem15.com/golem15/sm-quizzes-plugin/events" "git.golem15.com/golem15/sm-quizzes-plugin/graders" "git.golem15.com/golem15/sm-quizzes-plugin/jobs" "git.golem15.com/golem15/sm-quizzes-plugin/models" "git.golem15.com/golem15/sm-quizzes-plugin/updates" "git.golem15.com/golem15/summercms/modules/conga" "git.golem15.com/golem15/summercms/modules/festival" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" ) // fakeGrader answers with a configurable score or error and counts calls. type fakeGrader struct { mu sync.Mutex points float64 text string err error calls int last graders.Input } func (g *fakeGrader) Code() string { return "fake" } func (g *fakeGrader) Label() string { return "Fake grader" } func (g *fakeGrader) Grade(_ context.Context, in graders.Input) (graders.Result, error) { g.mu.Lock() defer g.mu.Unlock() g.calls++ g.last = in if g.err != nil { return graders.Result{}, g.err } return graders.Result{Points: g.points, Feedback: g.text}, nil } func (g *fakeGrader) set(points float64, text string, err error) { g.mu.Lock() defer g.mu.Unlock() g.points, g.text, g.err = points, text, err } func (g *fakeGrader) callCount() int { g.mu.Lock() defer g.mu.Unlock() return g.calls } // recordingEnqueuer records the grading jobs instead of inserting them. type recordingEnqueuer struct { mu sync.Mutex jobs []jobs.GradeAnswerArgs opts []conga.EnqueueOpts inTx []bool } func (e *recordingEnqueuer) Enqueue(_ context.Context, db *gorm.DB, args pact.JobArgs, o conga.EnqueueOpts) error { e.mu.Lock() defer e.mu.Unlock() a, ok := args.(jobs.GradeAnswerArgs) if !ok { return errors.New("unexpected job args") } _, tx := db.Statement.ConnPool.(gorm.TxCommitter) e.jobs, e.opts, e.inTx = append(e.jobs, a), append(e.opts, o), append(e.inTx, tx) return nil } func (e *recordingEnqueuer) take() []jobs.GradeAnswerArgs { e.mu.Lock() defer e.mu.Unlock() out := e.jobs for i, o := range e.opts { if o.Queue != jobs.Queue || o.MaxAttempts != classes.GradeMaxAttempts || !e.inTx[i] { out = append(out, jobs.GradeAnswerArgs{}) // makes the caller's count check fail } } e.jobs, e.opts, e.inTx = nil, nil, nil return out } // TestOpenQuestionsAgainstDatabase covers open questions and their grading // end to end on a migrated database: the migration, learning answers and // their re-answers, the background grade with clamping, recalculation and // the AnswerGraded event, stale jobs, failures, manual quizzes, test mode // (grading at close), the text limits and the migration's rollback. func TestOpenQuestionsAgainstDatabase(t *testing.T) { if testing.Short() { t.Skip("Postgres test; runs in the verification phase") } _, _, gdb := migrated(t) ctx := t.Context() grader := &fakeGrader{} reg := graders.NewRegistry() must(t, reg.Add("acme.fake", grader)) enq := &recordingEnqueuer{} bus := festival.New() var ( evMu sync.Mutex graded []*events.AnswerGradedEvent seenStatus []string ) bus.Listen[*events.AnswerGradedEvent]("acme.test", func(ctx context.Context, e *events.AnswerGradedEvent) error { var row models.AttemptAnswer status := "missing" if err := gdb.WithContext(ctx).Where("id = ?", e.AnswerID).Take(&row).Error; err == nil && row.GradingStatus != nil { status = *row.GradingStatus } evMu.Lock() defer evMu.Unlock() graded, seenStatus = append(graded, e), append(seenStatus, status) return errors.New("a listener error is logged, never returned") }) takeEvents := func() ([]*events.AnswerGradedEvent, []string) { evMu.Lock() defer evMu.Unlock() out, st := graded, seenStatus graded, seenStatus = nil, nil return out, st } svc := &classes.Service{DB: gdb, Graders: reg, Jobs: enq, Events: bus} seedOpen := func(t *testing.T, slug string, mutate func(*models.Quiz)) (models.Quiz, models.Question) { t.Helper() quiz := models.Quiz{Title: slug, Slug: slug, IsPublished: true, PassingScore: 50, Grader: "fake"} if mutate != nil { mutate(&quiz) } must(t, gdb.Create(&quiz).Error) q := models.Question{QuizID: quiz.ID, Text: "Wyjaśnij odstraszanie.", Type: models.TypeOpen, AnswerText: strPtr("MODEL"), Rubric: strPtr("R-SECRET"), Points: 2, SortOrder: 1} must(t, gdb.Create(&q).Error) return quiz, q } users := 0 startFor := func(t *testing.T, slug string) (uint, uint) { t.Helper() users++ u, _ := newUser(t, gdb, strings.ReplaceAll(t.Name(), "/", "-")+"-"+string(rune('a'+users))+"@example.com") st, err := svc.Start(ctx, u.ID, slug, "") must(t, err) return u.ID, st.Attempt.ID } answer := func(userID, attemptID, questionID uint, text string) (*classes.Feedback, error) { return svc.Answer(ctx, userID, attemptID, classes.AnswerInput{QuestionID: questionID, Text: &text}) } rowOf := func(t *testing.T, attemptID, questionID uint) models.AttemptAnswer { t.Helper() var row models.AttemptAnswer must(t, gdb.Where("attempt_id = ? AND question_id = ?", attemptID, questionID).Take(&row).Error) return row } status := func(r models.AttemptAnswer) string { if r.GradingStatus == nil { return "" } return *r.GradingStatus } fieldKey := func(err error) string { var fe *classes.FieldError if errors.As(err, &fe) { return fe.Field + ":" + fe.Key } return "" } t.Run("migration", func(t *testing.T) { var n int64 must(t, gdb.Raw(`SELECT COUNT(*) FROM pg_constraint WHERE conrelid = 'golem15_quizzes_questions'::regclass AND conname = 'golem15_quizzes_questions_type_check'`).Scan(&n).Error) if n != 1 { t.Errorf("golem15_quizzes_questions_type_check count = %d, want 1", n) } must(t, gdb.Raw(`SELECT COUNT(*) FROM pg_constraint con JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attname = 'type' WHERE con.conrelid = 'golem15_quizzes_questions'::regclass AND con.contype = 'c' AND con.conkey = ARRAY[att.attnum]`).Scan(&n).Error) if n != 1 { t.Errorf("CHECK constraints on questions.type = %d, want exactly the new one", n) } quiz, q := seedOpen(t, "migracja", nil) if q.ID == 0 { t.Fatal("open question not created") } if err := gdb.Exec(`INSERT INTO golem15_quizzes_questions (quiz_id, text, type) VALUES (?, 'Q', 'essay')`, quiz.ID).Error; err == nil { t.Error("a fifth question type was accepted") } if err := gdb.Exec(`INSERT INTO golem15_quizzes_questions (quiz_id, text, type, max_length) VALUES (?, 'Q', 'open', 0)`, quiz.ID).Error; err == nil { t.Error("max_length 0 was accepted") } must(t, gdb.Exec(`INSERT INTO golem15_quizzes_quizzes (title, slug, is_published, passing_score, shuffle_questions, created_at, updated_at) VALUES ('Stary otwarty', 'stary-otwarty', true, 50, false, NOW(), NOW())`).Error) var old models.Quiz must(t, gdb.Where("slug = ?", "stary-otwarty").Take(&old).Error) if old.Grader != graders.Manual { t.Errorf("pre-existing quiz grader = %q, want manual", old.Grader) } zero := models.Quiz{Title: "Zero", Slug: "zero-grader", PassingScore: 50} must(t, gdb.Create(&zero).Error) if zero.Grader != graders.Manual { t.Errorf("zero-value quiz grader = %q, want manual", zero.Grader) } f := seedQuiz(t, gdb, "zwykly-otwarty", nil) uid, aid := startFor(t, "zwykly-otwarty") _, err := svc.Answer(ctx, uid, aid, classes.AnswerInput{QuestionID: f.single.ID, AnswerIDs: []uint{f.singleRight.ID}}) must(t, err) if r := rowOf(t, aid, f.single.ID); r.GradingStatus != nil || r.GradingSeq != 0 || r.TextAnswer != nil || !r.IsCorrect { t.Errorf("single answer row = %+v, want no grading state and the usual score", r) } if jobs := enq.take(); len(jobs) != 0 { t.Errorf("a single answer enqueued %v", jobs) } }) t.Run("learning answer", func(t *testing.T) { _, q := seedOpen(t, "nauka-otwarte", nil) uid, aid := startFor(t, "nauka-otwarte") enq.take() fb, err := answer(uid, aid, q.ID, " moja odpowiedź \x00") must(t, err) if fb.Withheld || fb.Answer == nil || status(*fb.Answer) != models.GradingPending || fb.Answer.GradingSeq != 1 { t.Fatalf("feedback = %+v answer %+v", fb, fb.Answer) } if fb.Answer.TextAnswer == nil || *fb.Answer.TextAnswer != "moja odpowiedź" { t.Errorf("stored text = %v, want trimmed without NUL", fb.Answer.TextAnswer) } row := rowOf(t, aid, q.ID) if got := enq.take(); len(got) != 1 || got[0] != (jobs.GradeAnswerArgs{AnswerID: row.ID, Seq: 1}) { t.Errorf("enqueued %v, want one job for answer %d seq 1 in the answer transaction", got, row.ID) } if _, err := answer(uid, aid, q.ID, "moja odpowiedź"); err != nil { t.Fatal(err) } if got := enq.take(); len(got) != 0 { t.Errorf("unchanged text enqueued %v", got) } if r := rowOf(t, aid, q.ID); r.GradingSeq != 1 || !r.UpdatedAt.Equal(row.UpdatedAt) { t.Errorf("unchanged text rewrote the row: seq %d updated %v -> %v", r.GradingSeq, row.UpdatedAt, r.UpdatedAt) } must(t, gdb.Model(&models.AttemptAnswer{}).Where("id = ?", row.ID).Updates(map[string]any{ "grading_status": models.GradingReviewed, "grader": "fake", "grader_points": 2.0, "points": 2.0, "is_correct": true, "grader_feedback": "stara uwaga", "review_comment": "stary komentarz", "grading_attempts": 1, "graded_at": time.Now(), "reviewed_at": time.Now(), "reviewed_by_user_id": uid, }).Error) fb, err = answer(uid, aid, q.ID, "inna odpowiedź") must(t, err) r := rowOf(t, aid, q.ID) if status(r) != models.GradingPending || r.GradingSeq != 2 || r.GradingAttempts != 0 || r.Points != 0 || r.IsCorrect || r.Grader != nil || r.GraderPoints != nil || r.GraderFeedback != nil || r.GradedAt != nil || r.ReviewComment != nil || r.ReviewedAt != nil || r.ReviewedByUserID != nil || *r.TextAnswer != "inna odpowiedź" { t.Errorf("changed text did not reset the grading state: %+v", r) } if fb.Answer == nil || fb.Answer.GradingSeq != 2 { t.Errorf("feedback answer = %+v, want seq 2", fb.Answer) } if got := enq.take(); len(got) != 1 || got[0].Seq != 2 { t.Errorf("changed text enqueued %v, want one job with seq 2", got) } }) t.Run("grading", func(t *testing.T) { _, q := seedOpen(t, "ocena-otwarte", nil) uid, aid := startFor(t, "ocena-otwarte") _, err := answer(uid, aid, q.ID, "odstraszanie to groźba odwetu") must(t, err) row := rowOf(t, aid, q.ID) enq.take() takeEvents() grader.set(1.26, " Dobrze, ale bez przykładu. ", nil) calls := grader.callCount() must(t, svc.GradeAnswer(ctx, row.ID, 1)) r := rowOf(t, aid, q.ID) if status(r) != models.GradingGraded || r.Points != 1.5 || r.GraderPoints == nil || *r.GraderPoints != 1.5 || r.IsCorrect || r.Grader == nil || *r.Grader != "fake" || r.GraderFeedback == nil || *r.GraderFeedback != "Dobrze, ale bez przykładu." || r.GradingAttempts != 1 || r.GradedAt == nil || r.GradingError != nil { t.Errorf("graded row = %+v", r) } if grader.callCount() != calls+1 || grader.last.Rubric != "R-SECRET" || grader.last.ModelAnswer != "MODEL" || grader.last.Answer != "odstraszanie to groźba odwetu" || grader.last.Points != 2 || grader.last.UserID != uid { t.Errorf("grader input = %+v", grader.last) } evs, st := takeEvents() if len(evs) != 1 || evs[0].AnswerID != row.ID || evs[0].Points != 1.5 || evs[0].MaxPoints != 2 || evs[0].Grader != "fake" || evs[0].UserID != uid || evs[0].AttemptID != aid || evs[0].QuestionID != q.ID { t.Errorf("AnswerGraded events = %+v", evs) } else if st[0] != models.GradingGraded { t.Errorf("the event fired before the commit: the listener saw %q", st[0]) } must(t, svc.GradeAnswer(ctx, row.ID, 1)) if evs, _ := takeEvents(); len(evs) != 0 || grader.callCount() != calls+1 { t.Errorf("a second job for a graded answer ran the grader or fired again: %d events", len(evs)) } // Full points on a completed attempt: the stored score follows. _, q2 := seedOpen(t, "ocena-pelna", nil) uid2, aid2 := startFor(t, "ocena-pelna") _, err = answer(uid2, aid2, q2.ID, "pełna odpowiedź") must(t, err) _, err = svc.Finish(ctx, uid2, aid2) must(t, err) var before models.Attempt must(t, gdb.Where("id = ?", aid2).Take(&before).Error) if before.ScorePoints != 0 || before.MaxPoints != 2 { t.Fatalf("finished attempt = %v / %d, want 0 / 2", before.ScorePoints, before.MaxPoints) } grader.set(7, "", nil) must(t, svc.GradeAnswer(ctx, rowOf(t, aid2, q2.ID).ID, 1)) r2 := rowOf(t, aid2, q2.ID) if r2.Points != 2 || !r2.IsCorrect || r2.GraderFeedback != nil { t.Errorf("full-points row = %+v", r2) } var after models.Attempt must(t, gdb.Where("id = ?", aid2).Take(&after).Error) if after.ScorePoints != 2 || after.ScorePercent == nil || *after.ScorePercent != 100 || after.MaxPoints != 2 { t.Errorf("recalculated attempt = %v / %v / %d, want 2 / 100 / 2", after.ScorePoints, after.ScorePercent, after.MaxPoints) } res, err := svc.Result(ctx, uid2, aid2) must(t, err) if res.OpenAnswers != 1 || res.Unreviewed != 1 { t.Errorf("result open / unreviewed = %d / %d, want 1 / 1", res.OpenAnswers, res.Unreviewed) } hist, _, err := svc.History(ctx, uid2, "", 1, 10) must(t, err) if len(hist) != 1 || hist[0].OpenAnswers != 1 || hist[0].Unreviewed != 1 || hist[0].AnsweredCount != 1 { t.Errorf("history = %+v", hist) } takeEvents() }) t.Run("stale job", func(t *testing.T) { _, q := seedOpen(t, "stary-job", nil) uid, aid := startFor(t, "stary-job") _, err := answer(uid, aid, q.ID, "pierwsza") must(t, err) _, err = answer(uid, aid, q.ID, "druga") must(t, err) row := rowOf(t, aid, q.ID) enq.take() grader.set(2, "", nil) calls := grader.callCount() must(t, svc.GradeAnswer(ctx, row.ID, 1)) r := rowOf(t, aid, q.ID) if status(r) != models.GradingPending || r.GradingSeq != 2 || r.GradingAttempts != 0 || grader.callCount() != calls { t.Errorf("a stale job changed the answer or called the grader: %+v", r) } must(t, svc.GradeAnswer(ctx, row.ID+100000, 1)) }) t.Run("failure", func(t *testing.T) { _, q := seedOpen(t, "porazka", nil) uid, aid := startFor(t, "porazka") _, err := answer(uid, aid, q.ID, "odpowiedź") must(t, err) row := rowOf(t, aid, q.ID) enq.take() takeEvents() grader.set(0, "", errors.New("provider down")) for i := 1; i <= 2; i++ { if err := svc.GradeAnswer(ctx, row.ID, 1); err == nil || !strings.Contains(err.Error(), "provider down") { t.Fatalf("attempt %d: err = %v, want the grader error for a retry", i, err) } if r := rowOf(t, aid, q.ID); status(r) != models.GradingPending || r.GradingAttempts != i { t.Errorf("after attempt %d: %s, attempts %d", i, status(r), r.GradingAttempts) } } if err := svc.GradeAnswer(ctx, row.ID, 1); err != nil { t.Fatalf("last attempt returned %v, want nil", err) } r := rowOf(t, aid, q.ID) if status(r) != models.GradingFailed || r.GradingError == nil || *r.GradingError != "provider down" || r.GradingAttempts != 3 { t.Errorf("failed row = %+v", r) } if evs, _ := takeEvents(); len(evs) != 0 { t.Errorf("a failure fired %d AnswerGraded events", len(evs)) } // A score that is not a number counts as a failure. _, q2 := seedOpen(t, "porazka-nan", nil) uid2, aid2 := startFor(t, "porazka-nan") _, err = answer(uid2, aid2, q2.ID, "odpowiedź") must(t, err) grader.set(math.NaN(), "", nil) if err := svc.GradeAnswer(ctx, rowOf(t, aid2, q2.ID).ID, 1); err == nil || !strings.Contains(err.Error(), "invalid score") { t.Errorf("NaN score: err = %v, want the invalid score error", err) } grader.set(0, "", nil) enq.take() }) t.Run("manual grader", func(t *testing.T) { _, q := seedOpen(t, "recznie", func(qz *models.Quiz) { qz.Grader = "" }) uid, aid := startFor(t, "recznie") _, err := answer(uid, aid, q.ID, "odpowiedź") must(t, err) if got := enq.take(); len(got) != 0 { t.Errorf("a manual quiz enqueued %v", got) } calls := grader.callCount() row := rowOf(t, aid, q.ID) must(t, svc.GradeAnswer(ctx, row.ID, 1)) if r := rowOf(t, aid, q.ID); status(r) != models.GradingPending || r.GradingAttempts != 0 || grader.callCount() != calls { t.Errorf("a manual answer was graded: %+v", r) } _, q2 := seedOpen(t, "nieznany", func(qz *models.Quiz) { qz.Grader = "nobody" }) uid2, aid2 := startFor(t, "nieznany") _, err = answer(uid2, aid2, q2.ID, "odpowiedź") must(t, err) if got := enq.take(); len(got) != 0 { t.Errorf("an unknown grader enqueued %v", got) } }) t.Run("test mode", func(t *testing.T) { quiz, q := seedOpen(t, "test-otwarty", func(qz *models.Quiz) { qz.Mode = models.ModeTest }) q2 := models.Question{QuizID: quiz.ID, Text: "Drugie pytanie", Type: models.TypeOpen, Points: 1, SortOrder: 2} must(t, gdb.Create(&q2).Error) uid, aid := startFor(t, "test-otwarty") enq.take() for _, id := range []uint{q.ID, q2.ID} { fb, err := answer(uid, aid, id, "odpowiedź w teście") must(t, err) if !fb.Withheld { t.Errorf("open test feedback is not withheld") } } _, err := answer(uid, aid, q.ID, "poprawiona odpowiedź") must(t, err) if got := enq.take(); len(got) != 0 { t.Errorf("answers in a running test enqueued %v", got) } res, err := svc.Finish(ctx, uid, aid) must(t, err) got := enq.take() want := map[jobs.GradeAnswerArgs]bool{ {AnswerID: rowOf(t, aid, q.ID).ID, Seq: 2}: true, {AnswerID: rowOf(t, aid, q2.ID).ID, Seq: 1}: true, } if len(got) != 2 || !want[got[0]] || !want[got[1]] || got[0] == got[1] { t.Errorf("Finish enqueued %v, want one job per pending open answer %v", got, want) } if classes.Withheld(&res.Attempt) || res.OpenAnswers != 2 || res.Unreviewed != 2 { t.Errorf("finished result withheld %v open %d unreviewed %d", classes.Withheld(&res.Attempt), res.OpenAnswers, res.Unreviewed) } // An expired test closed lazily by Result enqueues too. limit := 60 _, q3 := seedOpen(t, "test-czasowy", func(qz *models.Quiz) { qz.Mode = models.ModeTest; qz.TimeLimitSeconds = &limit }) uid3, aid3 := startFor(t, "test-czasowy") _, err = answer(uid3, aid3, q3.ID, "zdążyłem") must(t, err) enq.take() late := *svc late.Now = func() time.Time { return time.Now().Add(2 * time.Hour) } _, err = late.Result(ctx, uid3, aid3) must(t, err) if got := enq.take(); len(got) != 1 || got[0].AnswerID != rowOf(t, aid3, q3.ID).ID { t.Errorf("the lazy close enqueued %v, want the one open answer", got) } }) t.Run("text limits", func(t *testing.T) { quiz, _ := seedOpen(t, "limity", nil) five := 5 short := models.Question{QuizID: quiz.ID, Text: "Krótko", Type: models.TypeOpen, Points: 1, MaxLength: &five, SortOrder: 2} must(t, gdb.Create(&short).Error) uid, aid := startFor(t, "limity") if _, err := svc.Answer(ctx, uid, aid, classes.AnswerInput{QuestionID: short.ID}); fieldKey(err) != "text:text_required" { t.Errorf("no text: err = %v", err) } if _, err := answer(uid, aid, short.ID, " "); fieldKey(err) != "text:text_required" { t.Errorf("blank text: err = %v", err) } if _, err := answer(uid, aid, short.ID, "abcdef"); fieldKey(err) != "text:text_too_long" { t.Errorf("6 characters with max 5: err = %v", err) } if _, err := answer(uid, aid, short.ID, "żółtą"); err != nil { t.Errorf("5 characters (8 bytes) with max 5: err = %v", err) } enq.take() }) t.Run("rollback", func(t *testing.T) { m := updates.AddOpenQuestionsAndGrading() err := m.Rollback(gdb) if err == nil || !strings.Contains(err.Error(), "rollback refused") { t.Fatalf("rollback with open questions = %v, want a refusal", err) } must(t, gdb.Exec(`DELETE FROM golem15_quizzes_questions WHERE type = 'open'`).Error) must(t, m.Rollback(gdb)) var quizID uint must(t, gdb.Raw(`SELECT id FROM golem15_quizzes_quizzes ORDER BY id LIMIT 1`).Scan(&quizID).Error) if err := gdb.Exec(`INSERT INTO golem15_quizzes_questions (quiz_id, text, type) VALUES (?, 'Q', 'open')`, quizID).Error; err == nil { t.Error("the rolled-back CHECK still accepts open") } var cols int64 must(t, gdb.Raw(`SELECT COUNT(*) FROM information_schema.columns WHERE (table_name = 'golem15_quizzes_attempt_answers' AND column_name IN ('text_answer', 'grading_status', 'reviewed_by_admin_id')) OR (table_name = 'golem15_quizzes_quizzes' AND column_name = 'grader') OR (table_name = 'golem15_quizzes_questions' AND column_name IN ('rubric', 'max_length'))`).Scan(&cols).Error) if cols != 0 { t.Errorf("%d added columns survived the rollback", cols) } must(t, m.Migrate(gdb)) must(t, gdb.Exec(`INSERT INTO golem15_quizzes_questions (quiz_id, text, type) VALUES (?, 'Q', 'open')`, quizID).Error) }) }