898de758ba
WR-07: when storing the grader's score fails on the last River attempt, record the answer as failed instead of leaving it pending with no job. The terminal failure write runs on a context detached from the job's, so a timed-out or cancelled job still records it.
478 lines
16 KiB
Go
478 lines
16 KiB
Go
package classes
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"git.golem15.com/golem15/sm-quizzes-plugin/events"
|
|
"git.golem15.com/golem15/sm-quizzes-plugin/graders"
|
|
"git.golem15.com/golem15/sm-quizzes-plugin/jobs"
|
|
"git.golem15.com/golem15/sm-quizzes-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/conga"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
)
|
|
|
|
// GradeMaxAttempts is how many times one open answer is handed to its
|
|
// grader before it is marked failed (D-17). The grading job's River attempt
|
|
// limit is the same number.
|
|
const GradeMaxAttempts = 3
|
|
|
|
// GradeTimeout is the deadline of one grading job attempt.
|
|
const GradeTimeout = 3 * time.Minute
|
|
|
|
// Text limits of what a grading writes.
|
|
const (
|
|
maxGraderFeedback = 2000
|
|
maxGradingError = 500
|
|
)
|
|
|
|
// errInvalidScore is a grader result ClampOpenPoints refuses; it counts as
|
|
// a failed attempt.
|
|
var errInvalidScore = errors.New("quizzes: grader returned an invalid score")
|
|
|
|
// Enqueuer inserts a background job, joining db's transaction when there is
|
|
// one. *conga.Manager implements it; tests record the jobs instead.
|
|
type Enqueuer interface {
|
|
Enqueue(ctx context.Context, db *gorm.DB, args pact.JobArgs, o conga.EnqueueOpts) error
|
|
}
|
|
|
|
// GraderFor returns the grader of a quiz: false for a nil registry or quiz,
|
|
// for manual and for a code no activated plugin registered (D-05). An
|
|
// answer without a grader waits for the owner.
|
|
func GraderFor(reg *graders.Registry, quiz *models.Quiz) (graders.Grader, bool) {
|
|
if quiz == nil {
|
|
return nil, false
|
|
}
|
|
return reg.Get(quiz.Grader)
|
|
}
|
|
|
|
// GradingStep is what a grading job does with an answer.
|
|
type GradingStep int
|
|
|
|
// Grading steps.
|
|
const (
|
|
// GradingSkip leaves the answer alone: it is no longer pending or it
|
|
// has a newer text than the job.
|
|
GradingSkip GradingStep = iota
|
|
// GradingRetry returns the grader's error so River retries the job.
|
|
GradingRetry
|
|
// GradingFail marks the answer failed after the last attempt.
|
|
GradingFail
|
|
// GradingStore stores the grader's score.
|
|
GradingStore
|
|
)
|
|
|
|
// NextGrading decides a grading job's step from the answer's status and
|
|
// grading_seq, the job's seq, the attempts used for this text (the current
|
|
// one included) and the grader's error. It is pure.
|
|
func NextGrading(status *string, rowSeq, jobSeq, attempts int, gradeErr error) GradingStep {
|
|
if status == nil || *status != models.GradingPending || rowSeq != jobSeq {
|
|
return GradingSkip
|
|
}
|
|
if gradeErr == nil {
|
|
return GradingStore
|
|
}
|
|
if attempts >= GradeMaxAttempts {
|
|
return GradingFail
|
|
}
|
|
return GradingRetry
|
|
}
|
|
|
|
// enqueueGrading hands a pending open answer to the quiz's grader as a
|
|
// background job inside tx, so the job exists only if the answer commits.
|
|
// Without a job queue or a registered grader (manual, unknown) it does
|
|
// nothing: the answer waits for the owner.
|
|
func (s *Service) enqueueGrading(ctx context.Context, tx *gorm.DB, answerID uint, seq int, attempt *models.Attempt) error {
|
|
if s.Jobs == nil || attempt == nil {
|
|
return nil
|
|
}
|
|
if _, ok := GraderFor(s.Graders, attempt.Quiz); !ok {
|
|
return nil
|
|
}
|
|
return s.Jobs.Enqueue(ctx, tx, jobs.GradeAnswerArgs{AnswerID: answerID, Seq: seq}, conga.EnqueueOpts{
|
|
Queue: jobs.Queue,
|
|
MaxAttempts: GradeMaxAttempts,
|
|
})
|
|
}
|
|
|
|
// enqueuePending enqueues every pending open answer of a test attempt that
|
|
// is being closed, one job per answer (UD-2): answers given during a test
|
|
// are graded only once it ends.
|
|
func (s *Service) enqueuePending(ctx context.Context, tx *gorm.DB, attempt *models.Attempt) error {
|
|
if s.Jobs == nil || !IsTest(attempt.Quiz) {
|
|
return nil
|
|
}
|
|
if _, ok := GraderFor(s.Graders, attempt.Quiz); !ok {
|
|
return nil
|
|
}
|
|
var rows []models.AttemptAnswer
|
|
if err := tx.WithContext(ctx).Where("attempt_id = ? AND grading_status = ?", attempt.ID, models.GradingPending).
|
|
Order("id ASC").Find(&rows).Error; err != nil {
|
|
return err
|
|
}
|
|
for _, r := range rows {
|
|
if err := s.enqueueGrading(ctx, tx, r.ID, r.GradingSeq, attempt); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// lockAttempt locks the attempt row FOR UPDATE, so a grade, a review and
|
|
// Finish of one attempt never interleave. A missing attempt is
|
|
// gorm.ErrRecordNotFound.
|
|
func lockAttempt(ctx context.Context, tx *gorm.DB, attemptID uint) (*models.Attempt, error) {
|
|
var a models.Attempt
|
|
err := tx.WithContext(ctx).Clauses(clause.Locking{Strength: "UPDATE", Table: clause.Table{Name: clause.CurrentTable}}).
|
|
Where("id = ?", attemptID).Take(&a).Error
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &a, nil
|
|
}
|
|
|
|
// recalcAttempt recomputes the score of a completed attempt from its
|
|
// answers after a grade or a review (Pitfall 2), under the attempt row
|
|
// lock. The percentage is taken against the attempt's stored max_points,
|
|
// so a later question edit does not move an old attempt's maximum. An open
|
|
// or missing attempt is left alone: close scores it.
|
|
func (s *Service) recalcAttempt(ctx context.Context, tx *gorm.DB, attemptID uint) error {
|
|
tx = tx.WithContext(ctx)
|
|
a, err := lockAttempt(ctx, tx, attemptID)
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if a.CompletedAt == nil {
|
|
return nil
|
|
}
|
|
var points []float64
|
|
if err := tx.Model(&models.AttemptAnswer{}).Where("attempt_id = ?", attemptID).Pluck("points", &points).Error; err != nil {
|
|
return err
|
|
}
|
|
score, percent := Totals(points, a.MaxPoints)
|
|
return tx.Model(&models.Attempt{}).Where("id = ?", attemptID).Updates(map[string]any{
|
|
"score_points": score,
|
|
"score_percent": percent,
|
|
"updated_at": s.now().UTC(),
|
|
}).Error
|
|
}
|
|
|
|
// fireAfterCommit fires event on s.Events once tx commits (never when it
|
|
// rolls back). A listener error is logged with the event name and never
|
|
// returned: the write it announces stays (D-14).
|
|
func fireAfterCommit[T any](ctx context.Context, s *Service, tx *gorm.DB, name string, event T) {
|
|
if s == nil || s.Events == nil {
|
|
return
|
|
}
|
|
bus := s.Events
|
|
lagoon.AfterCommit(ctx, tx, func(ctx context.Context, _ *gorm.DB) {
|
|
if err := bus.Fire(ctx, event); err != nil {
|
|
pluginLog.Warn("quizzes: event listener failed", "event", name, "error", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
// GradeAnswer is the body of the grading job: it asks the quiz's grader to
|
|
// score one pending open answer (D-06). The grader runs with no transaction
|
|
// open and no row lock held, so a slow grader blocks no student.
|
|
//
|
|
// A missing answer, one that is no longer pending (graded, failed or
|
|
// reviewed by the owner), one whose grading_seq differs from seq (the
|
|
// student wrote a newer text) and one whose question is no longer open are
|
|
// left alone, as is an answer whose quiz has no registered grader. Each
|
|
// run counts an attempt for the current text first. A grader error, or a
|
|
// score ClampOpenPoints refuses, is returned for River to retry while
|
|
// fewer than GradeMaxAttempts attempts were made; the last one marks the
|
|
// answer failed with the error and returns nil (D-17), as does an error
|
|
// storing the score on the last attempt. A score is stored as graded with points clamped to
|
|
// 0..question points in steps of 0.5, the attempt's score is recalculated
|
|
// when it is completed, and AnswerGradedEvent fires after the commit.
|
|
func (s *Service) GradeAnswer(ctx context.Context, answerID uint, seq int) error {
|
|
db := s.db(ctx)
|
|
var row models.AttemptAnswer
|
|
if err := db.Where("id = ?", answerID).Take(&row).Error; err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if NextGrading(row.GradingStatus, row.GradingSeq, seq, 0, nil) == GradingSkip {
|
|
return nil
|
|
}
|
|
var question models.Question
|
|
if err := db.Where("id = ?", row.QuestionID).Take(&question).Error; err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if question.Type != models.TypeOpen {
|
|
// The question stopped being open after the job was queued.
|
|
return nil
|
|
}
|
|
var attempt models.Attempt
|
|
if err := db.Preload("Quiz").Where("id = ?", row.AttemptID).Take(&attempt).Error; err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
grader, ok := GraderFor(s.Graders, attempt.Quiz)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
var attempts int
|
|
if err := db.Raw(`UPDATE golem15_quizzes_attempt_answers SET grading_attempts = grading_attempts + 1
|
|
WHERE id = ? AND grading_status = ? AND grading_seq = ? RETURNING grading_attempts`,
|
|
answerID, models.GradingPending, seq).Scan(&attempts).Error; err != nil {
|
|
return err
|
|
}
|
|
if attempts == 0 {
|
|
return nil
|
|
}
|
|
|
|
res, gradeErr := grader.Grade(ctx, graders.Input{
|
|
QuizID: attempt.QuizID,
|
|
QuestionID: question.ID,
|
|
AttemptID: attempt.ID,
|
|
AnswerID: row.ID,
|
|
UserID: attempt.UserID,
|
|
Question: question.Text,
|
|
ModelAnswer: deref(question.AnswerText),
|
|
Rubric: deref(question.Rubric),
|
|
Answer: deref(row.TextAnswer),
|
|
Points: question.Points,
|
|
})
|
|
var points float64
|
|
if gradeErr == nil {
|
|
p, ok := ClampOpenPoints(res.Points, question.Points)
|
|
if !ok {
|
|
gradeErr = errInvalidScore
|
|
}
|
|
points = p
|
|
}
|
|
pending := models.GradingPending
|
|
switch NextGrading(&pending, seq, seq, attempts, gradeErr) {
|
|
case GradingRetry:
|
|
return gradeErr
|
|
case GradingFail:
|
|
return s.failLast(ctx, row.ID, attempt.ID, seq, grader.Code(), gradeErr)
|
|
case GradingStore:
|
|
err := s.storeGrade(ctx, row.ID, seq, &attempt, question, grader.Code(), points, res.Feedback)
|
|
if err != nil && attempts >= GradeMaxAttempts {
|
|
// River will not run the job again: without this the answer
|
|
// would stay pending with no job (D-17).
|
|
return s.failLast(ctx, row.ID, attempt.ID, seq, grader.Code(), err)
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// failLastTimeout bounds the write that marks an answer failed after its
|
|
// last attempt.
|
|
const failLastTimeout = 30 * time.Second
|
|
|
|
// failLast marks an answer failed after its last grading attempt on a
|
|
// context detached from the job's, so a job that timed out or was
|
|
// cancelled still records the failure. If even that write fails, both
|
|
// errors are returned and the answer stays pending for a regrade.
|
|
func (s *Service) failLast(ctx context.Context, answerID, attemptID uint, seq int, code string, cause error) error {
|
|
fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), failLastTimeout)
|
|
defer cancel()
|
|
if err := s.failGrading(fctx, answerID, attemptID, seq, code, cause); err != nil {
|
|
return errors.Join(cause, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// stillPending re-reads an answer inside the grading write's transaction
|
|
// (the attempt row is locked) and reports whether the job's seq still
|
|
// applies.
|
|
func stillPending(ctx context.Context, tx *gorm.DB, answerID uint, seq int) (bool, error) {
|
|
var cur models.AttemptAnswer
|
|
err := tx.WithContext(ctx).Where("id = ?", answerID).Take(&cur).Error
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return NextGrading(cur.GradingStatus, cur.GradingSeq, seq, 0, nil) != GradingSkip, nil
|
|
}
|
|
|
|
// failGrading marks an answer failed after its last grading attempt.
|
|
func (s *Service) failGrading(ctx context.Context, answerID, attemptID uint, seq int, code string, gradeErr error) error {
|
|
return lagoon.Transaction(ctx, s.DB, func(ctx context.Context, tx *gorm.DB) error {
|
|
tx = tx.WithContext(ctx)
|
|
if _, err := lockAttempt(ctx, tx, attemptID); err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if ok, err := stillPending(ctx, tx, answerID, seq); err != nil || !ok {
|
|
return err
|
|
}
|
|
return tx.Model(&models.AttemptAnswer{}).Where("id = ?", answerID).Updates(map[string]any{
|
|
"grading_status": models.GradingFailed,
|
|
"grader": code,
|
|
"grading_error": cutRunes(gradeErr.Error(), maxGradingError),
|
|
"updated_at": s.now().UTC(),
|
|
}).Error
|
|
})
|
|
}
|
|
|
|
// storeGrade stores a grader's clamped score as graded, recalculates a
|
|
// completed attempt and fires AnswerGradedEvent after the commit. Feedback
|
|
// that repeats the rubric (LeaksRubric) is dropped, for the student and the
|
|
// event alike, and noted for the owner in grading_error.
|
|
func (s *Service) storeGrade(ctx context.Context, answerID uint, seq int, attempt *models.Attempt, question models.Question, code string, points float64, feedback string) error {
|
|
return lagoon.Transaction(ctx, s.DB, func(ctx context.Context, tx *gorm.DB) error {
|
|
tx = tx.WithContext(ctx)
|
|
if _, err := lockAttempt(ctx, tx, attempt.ID); err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if ok, err := stillPending(ctx, tx, answerID, seq); err != nil || !ok {
|
|
return err
|
|
}
|
|
feedback = cutRunes(strings.TrimSpace(feedback), maxGraderFeedback)
|
|
var note any
|
|
if LeaksRubric(feedback, deref(question.Rubric), question.Text, deref(question.AnswerText)) {
|
|
pluginLog.Warn("quizzes: grader feedback withheld, it repeats the rubric", "answer_id", answerID, "grader", code)
|
|
feedback, note = "", feedbackWithheldNote
|
|
}
|
|
var fb any
|
|
if feedback != "" {
|
|
fb = feedback
|
|
}
|
|
now := s.now().UTC()
|
|
if err := tx.Model(&models.AttemptAnswer{}).Where("id = ?", answerID).Updates(map[string]any{
|
|
"grading_status": models.GradingGraded,
|
|
"grader": code,
|
|
"grader_points": points,
|
|
"points": points,
|
|
"is_correct": OpenCorrect(points, question.Points),
|
|
"grader_feedback": fb,
|
|
"grading_error": note,
|
|
"graded_at": now,
|
|
"updated_at": now,
|
|
}).Error; err != nil {
|
|
return err
|
|
}
|
|
if err := s.recalcAttempt(ctx, tx, attempt.ID); err != nil {
|
|
return err
|
|
}
|
|
fireAfterCommit(ctx, s, tx, events.AnswerGradedEventName, &events.AnswerGradedEvent{
|
|
QuizID: attempt.QuizID,
|
|
QuestionID: question.ID,
|
|
AttemptID: attempt.ID,
|
|
AnswerID: answerID,
|
|
UserID: attempt.UserID,
|
|
Grader: code,
|
|
Points: points,
|
|
MaxPoints: question.Points,
|
|
Feedback: feedback,
|
|
})
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// feedbackWithheldNote is the owner's note on a graded answer whose
|
|
// feedback LeaksRubric dropped; students never see grading_error.
|
|
const feedbackWithheldNote = "feedback withheld: it repeated the rubric"
|
|
|
|
// Rubric leak detection: a run of rubricRunWords consecutive words shared
|
|
// by the feedback and the rubric counts as quoting it. A rubric shorter
|
|
// than that counts when the feedback holds it whole, unless it has fewer
|
|
// than minRubricWords words (too short to tell from ordinary wording).
|
|
const (
|
|
rubricRunWords = 6
|
|
minRubricWords = 4
|
|
)
|
|
|
|
// LeaksRubric reports whether a grader's feedback repeats the owner-only
|
|
// rubric (D-01): a run of words of the rubric, compared case-insensitively
|
|
// word by word, that does not also occur in one of the public texts (the
|
|
// question and the model answer, which the student is shown anyway). It is
|
|
// a deterministic guard behind the grader's prompt, so a model talked into
|
|
// listing the rubric never puts it in a student response verbatim.
|
|
func LeaksRubric(feedback, rubric string, public ...string) bool {
|
|
rw := words(rubric)
|
|
if len(rw) < minRubricWords {
|
|
return false
|
|
}
|
|
n := min(rubricRunWords, len(rw))
|
|
fw := words(feedback)
|
|
if len(fw) < n {
|
|
return false
|
|
}
|
|
skip := map[string]bool{}
|
|
for _, p := range public {
|
|
for g := range ngrams(words(p), n) {
|
|
skip[g] = true
|
|
}
|
|
}
|
|
secret := map[string]bool{}
|
|
for g := range ngrams(rw, n) {
|
|
if !skip[g] {
|
|
secret[g] = true
|
|
}
|
|
}
|
|
if len(secret) == 0 {
|
|
return false
|
|
}
|
|
for g := range ngrams(fw, n) {
|
|
if secret[g] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// words splits s into lower-case words of letters and digits.
|
|
func words(s string) []string {
|
|
return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
|
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
|
|
})
|
|
}
|
|
|
|
// ngrams is the set of runs of n consecutive words of w, joined by spaces.
|
|
func ngrams(w []string, n int) map[string]bool {
|
|
out := map[string]bool{}
|
|
for i := 0; i+n <= len(w); i++ {
|
|
out[strings.Join(w[i:i+n], " ")] = true
|
|
}
|
|
return out
|
|
}
|
|
|
|
// cutRunes keeps at most n runes of s.
|
|
func cutRunes(s string, n int) string {
|
|
if utf8.RuneCountInString(s) <= n {
|
|
return s
|
|
}
|
|
r := []rune(s)
|
|
return string(r[:n])
|
|
}
|
|
|
|
func deref(s *string) string {
|
|
if s == nil {
|
|
return ""
|
|
}
|
|
return *s
|
|
}
|