feat: carry open questions and the grader through import and the quiz tools

- import document, manage routes and MCP question tools take rubric and
  max_length; open questions refuse answers, max_length is 1..20000 and
  both apply to open questions only
- quizzes take a grader; the manage routes refuse a code the registry
  does not know, the import accepts any well-formed code
- quiz stats report unreviewed open answers and their average points
- import_quiz warns that replace deletes the grades and reviews
This commit is contained in:
Jakub Zych
2026-11-09 18:36:20 +01:00
parent 2ad4e8b1ec
commit 9688b4d552
10 changed files with 334 additions and 39 deletions

View File

@@ -8,14 +8,16 @@ import (
"slices"
"strings"
"git.golem15.com/golem15/sm-quizzes-plugin/graders"
"git.golem15.com/golem15/sm-quizzes-plugin/models"
"git.golem15.com/golem15/summercms/modules/lagoon"
"gorm.io/gorm"
)
// ImportQuiz is the JSON document quizzes:import reads. Mode, Navigation
// and MaxAttempts keep the quiz's current value (the model default on
// create) when absent.
// ImportQuiz is the JSON document quizzes:import reads. Mode, Navigation,
// MaxAttempts and Grader keep the quiz's current value (the model default
// on create) when absent. Grader is any well-formed grader code: one that
// no activated plugin registers grades as manual (D-05).
type ImportQuiz struct {
Title string `json:"title"`
Slug string `json:"slug"`
@@ -29,10 +31,13 @@ type ImportQuiz struct {
Mode *string `json:"mode"`
Navigation *string `json:"navigation"`
MaxAttempts *int `json:"max_attempts"`
Grader *string `json:"grader"`
Questions []ImportQuestion `json:"questions"`
}
// ImportQuestion is one question of an ImportQuiz.
// ImportQuestion is one question of an ImportQuiz. Rubric (owner-only
// grading notes) and MaxLength (the longest answer in characters) belong
// to open questions; AnswerText is an open question's model answer.
type ImportQuestion struct {
Text string `json:"text"`
Type string `json:"type"`
@@ -40,6 +45,8 @@ type ImportQuestion struct {
AnswerText *string `json:"answer_text"`
Explanation *string `json:"explanation"`
Points *int `json:"points"`
Rubric *string `json:"rubric"`
MaxLength *int `json:"max_length"`
Answers []ImportAnswer `json:"answers"`
}
@@ -93,11 +100,13 @@ func ParseImport(raw []byte) (*ImportQuiz, error) {
if doc.MaxAttempts != nil && *doc.MaxAttempts < 1 {
problems = append(problems, "max_attempts must be at least 1")
}
problems = append(problems, checkGraderFormat(doc.Grader)...)
if len(doc.Questions) == 0 {
problems = append(problems, "questions must not be empty")
}
for i, q := range doc.Questions {
problems = append(problems, CheckQuestion(fmt.Sprintf("questions[%d]", i), q)...)
for i := range doc.Questions {
doc.Questions[i].Rubric = nilIfBlank(deref(doc.Questions[i].Rubric), false)
problems = append(problems, CheckQuestion(fmt.Sprintf("questions[%d]", i), doc.Questions[i])...)
}
if len(problems) > 0 {
return nil, fmt.Errorf("invalid quiz: %s", strings.Join(problems, "; "))
@@ -109,8 +118,11 @@ func ParseImport(raw []byte) (*ImportQuiz, error) {
// at (such as "questions[0]"): the text is required, the type is one of
// models.QuestionTypes, points are not negative, answer texts are
// required, a single question has at least two answers and exactly one
// correct, a multiple question at least two and one or more correct, and a
// self_check question none. The importer and the manage routes share it.
// correct, a multiple question at least two and one or more correct, a
// self_check question none, and an open question none either (answer_text
// is its model answer). rubric and max_length (1..models.MaxMaxLength)
// apply to open questions only. The importer and the manage routes share
// it.
func CheckQuestion(at string, q ImportQuestion) []string {
var problems []string
if strings.TrimSpace(q.Text) == "" {
@@ -144,10 +156,34 @@ func CheckQuestion(at string, q ImportQuestion) []string {
if len(q.Answers) > 0 {
problems = append(problems, at+": a self_check question has no answers; use answer_text")
}
case models.TypeOpen:
if len(q.Answers) > 0 {
problems = append(problems, at+": an open question has no answers; put the model answer in answer_text")
}
if q.MaxLength != nil && (*q.MaxLength < 1 || *q.MaxLength > models.MaxMaxLength) {
problems = append(problems, fmt.Sprintf("%s.max_length must be between 1 and %d", at, models.MaxMaxLength))
}
}
if q.Type != models.TypeOpen {
if q.Rubric != nil && strings.TrimSpace(*q.Rubric) != "" {
problems = append(problems, at+".rubric applies to open questions only")
}
if q.MaxLength != nil {
problems = append(problems, at+".max_length applies to open questions only")
}
}
return problems
}
// checkGraderFormat is the problem of a sent grader code that is not
// well-formed. Whether a plugin registers it is CheckGrader's question.
func checkGraderFormat(code *string) []string {
if code == nil || graders.ValidCode(*code) {
return nil
}
return []string{"grader must be lowercase letters, digits and underscores"}
}
// ImportResult reports what an import wrote.
type ImportResult struct {
Quiz models.Quiz
@@ -198,11 +234,15 @@ func Import(ctx context.Context, db *gorm.DB, doc *ImportQuiz, replace bool) (*I
limit := *doc.MaxAttempts
quiz.MaxAttempts = &limit
}
if doc.Grader != nil {
quiz.Grader = *doc.Grader
}
if err := tx.Save(&quiz).Error; err != nil {
return err
}
if !res.Created {
// The foreign keys cascade to answers and attempt answers.
// The foreign keys cascade to answers and attempt answers,
// with the grades and reviews of open answers.
if err := tx.Where("quiz_id = ?", quiz.ID).Delete(&models.Question{}).Error; err != nil {
return err
}
@@ -215,6 +255,7 @@ func Import(ctx context.Context, db *gorm.DB, doc *ImportQuiz, replace bool) (*I
q := models.Question{
QuizID: quiz.ID, Text: strings.TrimSpace(in.Text), Type: in.Type, Category: trimmed(in.Category),
AnswerText: in.AnswerText, Explanation: in.Explanation, SortOrder: i + 1, Points: points,
Rubric: in.Rubric, MaxLength: in.MaxLength,
}
if err := tx.Create(&q).Error; err != nil {
return err

View File

@@ -88,6 +88,8 @@ func TestParseImportTestMode(t *testing.T) {
// with the manage routes; ParseImport reports exactly these problems.
func TestCheckQuestion(t *testing.T) {
neg := -1
zero, five, tooLong := 0, 500, 20001
rubric := "Mentions sovereignty."
for name, tc := range map[string]struct {
q classes.ImportQuestion
want []string
@@ -100,6 +102,12 @@ func TestCheckQuestion(t *testing.T) {
"self_check answer": {classes.ImportQuestion{Text: "Q", Type: "self_check", Answers: []classes.ImportAnswer{{Text: "a"}}}, []string{"questions[0]: a self_check question has no answers; use answer_text"}},
"blank answer": {classes.ImportQuestion{Text: "Q", Type: "single", Answers: []classes.ImportAnswer{{Text: " ", IsCorrect: true}, {Text: "b"}}}, []string{"questions[0].answers[0].text is required"}},
"valid single": {classes.ImportQuestion{Text: "Q", Type: "single", Answers: []classes.ImportAnswer{{Text: "a", IsCorrect: true}, {Text: "b"}}}, nil},
"valid open": {classes.ImportQuestion{Text: "Q", Type: "open", AnswerText: str("Model"), Rubric: &rubric, MaxLength: &five}, nil},
"open with answer": {classes.ImportQuestion{Text: "Q", Type: "open", Answers: []classes.ImportAnswer{{Text: "a"}}}, []string{"questions[0]: an open question has no answers; put the model answer in answer_text"}},
"open length 0": {classes.ImportQuestion{Text: "Q", Type: "open", MaxLength: &zero}, []string{"questions[0].max_length must be between 1 and 20000"}},
"open length 20001": {classes.ImportQuestion{Text: "Q", Type: "open", MaxLength: &tooLong}, []string{"questions[0].max_length must be between 1 and 20000"}},
"single rubric": {classes.ImportQuestion{Text: "Q", Type: "single", Rubric: &rubric, Answers: []classes.ImportAnswer{{Text: "a", IsCorrect: true}, {Text: "b"}}}, []string{"questions[0].rubric applies to open questions only"}},
"single max_length": {classes.ImportQuestion{Text: "Q", Type: "single", MaxLength: &five, Answers: []classes.ImportAnswer{{Text: "a", IsCorrect: true}, {Text: "b"}}}, []string{"questions[0].max_length applies to open questions only"}},
} {
got := classes.CheckQuestion("questions[0]", tc.q)
if strings.Join(got, "; ") != strings.Join(tc.want, "; ") {
@@ -115,3 +123,32 @@ func TestCheckQuestion(t *testing.T) {
}
}
}
func TestParseImportOpen(t *testing.T) {
for _, grader := range []string{"ai", "manual"} {
raw := `{"title":"O","grader":"` + grader + `","questions":[{"text":"Q","type":"open","answer_text":"M","rubric":"R","max_length":300}]}`
doc, err := classes.ParseImport([]byte(raw))
if err != nil {
t.Fatalf("grader %s: %v", grader, err)
}
q := doc.Questions[0]
if doc.Grader == nil || *doc.Grader != grader || q.Rubric == nil || *q.Rubric != "R" || q.MaxLength == nil || *q.MaxLength != 300 {
t.Errorf("grader %s: doc = %+v, question = %+v", grader, doc, q)
}
}
if _, err := classes.ParseImport([]byte(`{"title":"O","grader":"Bad Grader","questions":[{"text":"Q","type":"open"}]}`)); err == nil ||
!strings.Contains(err.Error(), "grader must be lowercase letters, digits and underscores") {
t.Errorf("bad grader = %v", err)
}
doc, err := classes.ParseImport([]byte(`{"title":"O","questions":[{"text":"Q","type":"open","rubric":" "}]}`))
if err != nil || doc.Questions[0].Rubric != nil {
t.Errorf("blank rubric = %+v %v, want it dropped", doc, err)
}
raw, err := os.ReadFile("../testdata/quiz.json")
if err != nil {
t.Fatal(err)
}
if doc, err := classes.ParseImport(raw); err != nil || doc.Grader != nil {
t.Errorf("sample = %+v %v, want no grader", doc, err)
}
}

View File

@@ -56,6 +56,7 @@ type QuizInput struct {
PassingScore *int `json:"passing_score"`
RoundSize *int `json:"round_size"`
ShuffleQuestions *bool `json:"shuffle_questions"`
Grader *string `json:"grader"`
}
// CheckQuizInput returns the rule problems of in. On create the title is
@@ -92,6 +93,7 @@ func CheckQuizInput(in QuizInput, create bool) []string {
if in.PassingScore != nil && (*in.PassingScore < 0 || *in.PassingScore > 100) {
problems = append(problems, "passing_score must be between 0 and 100")
}
problems = append(problems, checkGraderFormat(in.Grader)...)
least, word := 1, "at least 1"
if !create {
least, word = 0, "0 (clears it) or more"
@@ -107,6 +109,16 @@ func CheckQuizInput(in QuizInput, create bool) []string {
return problems
}
// CheckGrader is the problem of a sent grader code that is not one of
// known, the codes of the registry (manual first), naming the valid codes.
// A nil code is fine.
func CheckGrader(code *string, known []string) []string {
if code == nil || slices.Contains(known, *code) {
return nil
}
return []string{"grader must be one of " + strings.Join(known, ", ")}
}
// positive is v, or nil when v is nil or not positive (0 clears).
func positive(v *int) *int {
if v == nil || *v <= 0 {
@@ -181,6 +193,9 @@ func applyQuizInput(quiz *models.Quiz, in QuizInput) {
if in.ShuffleQuestions != nil {
quiz.ShuffleQuestions = *in.ShuffleQuestions
}
if in.Grader != nil {
quiz.Grader = *in.Grader
}
}
// nilIfBlank is nil for a blank s, else s (trimmed when trim is set).
@@ -414,7 +429,11 @@ type WrongAnswerStat struct {
// QuestionStat is one question's results over every completed attempt's
// answers. CorrectPercent is nil when nobody answered it; MostChosenWrong
// is nil for a self_check question and when no wrong answer was chosen.
// is nil for a self_check or open question and when no wrong answer was
// chosen. Unreviewed counts the open answers the owner has not reviewed
// yet (pending, graded or failed); AveragePoints is the mean score of an
// open question's graded and reviewed answers, nil for other types and
// before any grade.
type QuestionStat struct {
ID uint
Text string
@@ -423,6 +442,8 @@ type QuestionStat struct {
Correct int
CorrectPercent *float64
MostChosenWrong *WrongAnswerStat
Unreviewed int
AveragePoints *float64
}
// Stats is the owner's report on one quiz (D-15).
@@ -431,6 +452,8 @@ type Stats struct {
QuestionCount int
Students []StudentStat
Questions []QuestionStat
// Unreviewed is the sum of the questions' Unreviewed.
Unreviewed int
}
// StudentRow is one row of the per-student stats query.
@@ -447,9 +470,11 @@ type StudentRow struct {
// AnswerRow is one row of the per-question answered/correct query.
type AnswerRow struct {
QuestionID uint
Answered int
Correct int
QuestionID uint
Answered int
Correct int
Unreviewed int
AveragePoints *float64
}
// WrongRow is one wrong answer chosen N times for a question.
@@ -470,7 +495,11 @@ WHERE a.quiz_id = ? AND a.completed_at IS NOT NULL AND a.category IS NULL
GROUP BY a.user_id, u.email, u.name, u.surname
ORDER BY u.surname ASC NULLS LAST, u.name ASC NULLS LAST, u.email ASC, a.user_id ASC`
statsAnswersSQL = `SELECT aa.question_id, COUNT(*) AS answered, COUNT(*) FILTER (WHERE aa.is_correct) AS correct
// average_points counts only scored open answers: a pending or failed
// answer holds 0 points until it is graded.
statsAnswersSQL = `SELECT aa.question_id, COUNT(*) AS answered, COUNT(*) FILTER (WHERE aa.is_correct) AS correct,
COUNT(*) FILTER (WHERE aa.grading_status IN ('pending', 'graded', 'failed')) AS unreviewed,
AVG(aa.points) FILTER (WHERE aa.grading_status IN ('graded', 'reviewed')) AS average_points
FROM golem15_quizzes_attempt_answers aa
JOIN golem15_quizzes_attempts a ON a.id = aa.attempt_id
WHERE a.quiz_id = ? AND a.completed_at IS NOT NULL
@@ -589,9 +618,18 @@ func BuildStats(quiz models.Quiz, questions []models.Question, students []Studen
pct := math.Round(float64(a.Correct)*1000/float64(a.Answered)) / 10
st.CorrectPercent = &pct
}
if q.Type != models.TypeSelfCheck {
switch q.Type {
case models.TypeSelfCheck:
case models.TypeOpen:
st.Unreviewed = a.Unreviewed
if a.AveragePoints != nil {
avg := math.Round(*a.AveragePoints*100) / 100
st.AveragePoints = &avg
}
default:
st.MostChosenWrong = wrongBy[q.ID]
}
out.Unreviewed += st.Unreviewed
out.Questions = append(out.Questions, st)
}
return out
@@ -609,8 +647,9 @@ type ManagedAnswerInput struct {
// QuestionInput is the fields of a question create or partial update; nil
// means not sent. Answers, when sent, is the question's full new answer
// list in order. An empty string clears category, answer_text and
// explanation.
// list in order. An empty string clears category, answer_text,
// explanation and rubric; a max_length of 0 clears it (the default 2000
// applies).
type QuestionInput struct {
Text *string `json:"text"`
Type *string `json:"type"`
@@ -619,6 +658,8 @@ type QuestionInput struct {
Explanation *string `json:"explanation"`
Points *int `json:"points"`
SortOrder *int `json:"sort_order"`
Rubric *string `json:"rubric"`
MaxLength *int `json:"max_length"`
Answers *[]ManagedAnswerInput `json:"answers"`
}
@@ -630,6 +671,7 @@ func MergeQuestion(current models.Question, in QuestionInput) (ImportQuestion, [
q := ImportQuestion{
Text: current.Text, Type: current.Type, Category: current.Category,
AnswerText: current.AnswerText, Explanation: current.Explanation,
Rubric: current.Rubric, MaxLength: current.MaxLength,
}
points := current.Points
if current.ID == 0 {
@@ -653,6 +695,17 @@ func MergeQuestion(current models.Question, in QuestionInput) (ImportQuestion, [
if in.Points != nil {
points = *in.Points
}
if in.Rubric != nil {
q.Rubric = nilIfBlank(*in.Rubric, false)
}
if in.MaxLength != nil {
if *in.MaxLength == 0 {
q.MaxLength = nil
} else {
n := *in.MaxLength
q.MaxLength = &n
}
}
q.Points = &points
var problems []string
if in.Category != nil && utf8.RuneCountInString(strings.TrimSpace(*in.Category)) > 255 {
@@ -747,6 +800,7 @@ func CreateQuestion(ctx context.Context, db *gorm.DB, slug string, in QuestionIn
q = models.Question{
QuizID: quizID, Text: merged.Text, Type: merged.Type, Category: merged.Category,
AnswerText: merged.AnswerText, Explanation: merged.Explanation, SortOrder: order, Points: *merged.Points,
Rubric: merged.Rubric, MaxLength: merged.MaxLength,
}
if err := tx.Omit(clause.Associations).Create(&q).Error; err != nil {
return err
@@ -802,6 +856,7 @@ func UpdateQuestion(ctx context.Context, db *gorm.DB, slug string, id uint, in Q
}
q.Text, q.Type, q.Category = merged.Text, merged.Type, merged.Category
q.AnswerText, q.Explanation, q.Points = merged.AnswerText, merged.Explanation, *merged.Points
q.Rubric, q.MaxLength = merged.Rubric, merged.MaxLength
if in.SortOrder != nil {
q.SortOrder = *in.SortOrder
}

View File

@@ -29,6 +29,8 @@ func TestCheckQuizInput(t *testing.T) {
"long context": {classes.QuizInput{Title: str("A"), Context: str(strings.Repeat("c", 256))}, true, "context must be at most 255"},
"update blank title": {classes.QuizInput{Title: str("")}, false, "title is required"},
"update negative limit": {classes.QuizInput{MaxAttempts: num(-1)}, false, "max_attempts must be 0"},
"bad grader": {classes.QuizInput{Title: str("A"), Grader: str("Bad Grader")}, true, "grader must be lowercase"},
"update empty grader": {classes.QuizInput{Grader: str("")}, false, "grader must be lowercase"},
} {
got := strings.Join(classes.CheckQuizInput(tc.in, tc.create), "; ")
if !strings.Contains(got, tc.want) {
@@ -38,6 +40,9 @@ func TestCheckQuizInput(t *testing.T) {
if got := classes.CheckQuizInput(classes.QuizInput{Title: str("Pojęcia")}, true); len(got) != 0 {
t.Errorf("minimal create: %v", got)
}
if got := classes.CheckQuizInput(classes.QuizInput{Title: str("A"), Grader: str("ai")}, true); len(got) != 0 {
t.Errorf("well-formed grader: %v", got)
}
if got := classes.CheckQuizInput(classes.QuizInput{}, false); len(got) != 0 {
t.Errorf("empty update: %v", got)
}
@@ -107,6 +112,9 @@ func TestBuildStats(t *testing.T) {
if ok := s.Questions[3]; ok.CorrectPercent == nil || *ok.CorrectPercent != 100 || ok.MostChosenWrong != nil {
t.Errorf("all-correct question = %+v", ok)
}
if s.Unreviewed != 0 || s.Questions[0].AveragePoints != nil || s.Questions[0].Unreviewed != 0 {
t.Errorf("closed questions report unreviewed %d, average %v", s.Unreviewed, s.Questions[0].AveragePoints)
}
more := append(wrong, classes.WrongRow{QuestionID: 10, AnswerID: 103, Text: "third", N: 2})
if w := classes.BuildStats(quiz, questions, nil, answers, more).Questions[0].MostChosenWrong; w == nil || w.AnswerID != 103 {
t.Errorf("most_chosen_wrong by count = %+v, want 103", w)
@@ -115,6 +123,52 @@ func TestBuildStats(t *testing.T) {
func uid(n uint) *uint { return &n }
// TestBuildStatsOpen: an open question reports its unreviewed answers and
// the mean score of its scored answers; it has no most chosen wrong answer.
func TestBuildStatsOpen(t *testing.T) {
quiz := models.Quiz{ID: 1, Slug: "q", PassingScore: 60}
questions := []models.Question{
{ID: 20, Text: "essay", Type: models.TypeOpen},
{ID: 21, Text: "single", Type: models.TypeSingle},
}
avg, ignored := 4.0/3.0, 1.5
answers := []classes.AnswerRow{
{QuestionID: 20, Answered: 3, Correct: 1, Unreviewed: 2, AveragePoints: &avg},
{QuestionID: 21, Answered: 2, Correct: 1, Unreviewed: 5, AveragePoints: &ignored},
}
wrong := []classes.WrongRow{{QuestionID: 20, AnswerID: 1, Text: "never", N: 3}}
s := classes.BuildStats(quiz, questions, nil, answers, wrong)
open := s.Questions[0]
if open.Unreviewed != 2 || open.AveragePoints == nil || *open.AveragePoints != 1.33 || open.MostChosenWrong != nil {
t.Errorf("open = %+v (average %v)", open, open.AveragePoints)
}
if single := s.Questions[1]; single.Unreviewed != 0 || single.AveragePoints != nil {
t.Errorf("single = %+v, want no open counts", single)
}
if s.Unreviewed != 2 {
t.Errorf("total unreviewed = %d, want 2", s.Unreviewed)
}
if none := classes.BuildStats(quiz, questions[:1], nil, nil, nil).Questions[0]; none.AveragePoints != nil || none.Unreviewed != 0 {
t.Errorf("unanswered open = %+v", none)
}
}
func TestCheckGrader(t *testing.T) {
known := []string{"manual"}
if got := classes.CheckGrader(str("ai"), known); len(got) != 1 || !strings.Contains(got[0], "grader must be one of manual") {
t.Errorf("unknown grader = %v", got)
}
if got := classes.CheckGrader(str("manual"), known); got != nil {
t.Errorf("manual = %v", got)
}
if got := classes.CheckGrader(nil, known); got != nil {
t.Errorf("nil = %v", got)
}
if got := classes.CheckGrader(str("ai"), []string{"manual", "ai"}); got != nil {
t.Errorf("registered ai = %v", got)
}
}
// TestQuestionInputRules: MergeQuestion applies a partial update and the
// answers list to the current question, and with CheckQuestion refuses
// what the importer would refuse.
@@ -168,6 +222,29 @@ func TestQuestionInputRules(t *testing.T) {
if problems := classes.CheckNewQuestion(classes.QuestionInput{Text: str("Q"), Type: str("self_check")}); len(problems) != 0 {
t.Errorf("minimal self_check create: %v", problems)
}
open := models.Question{ID: 3, Text: "Esej", Type: models.TypeOpen, Points: 2, Rubric: str("Kryteria"), MaxLength: num(500)}
merged, problems = check(open, classes.QuestionInput{Text: str("Esej?")})
if len(problems) != 0 || merged.Rubric == nil || *merged.Rubric != "Kryteria" || merged.MaxLength == nil || *merged.MaxLength != 500 {
t.Errorf("open kept: %+v %v", merged, problems)
}
merged, problems = check(open, classes.QuestionInput{Rubric: str(""), MaxLength: num(0)})
if len(problems) != 0 || merged.Rubric != nil || merged.MaxLength != nil {
t.Errorf("open cleared: %+v %v", merged, problems)
}
if _, problems := check(open, classes.QuestionInput{MaxLength: num(20001)}); !strings.Contains(strings.Join(problems, "; "), "max_length must be between 1 and 20000") {
t.Errorf("open max_length 20001: %v", problems)
}
withAnswers := []classes.ManagedAnswerInput{{Text: "a", IsCorrect: true}}
if problems := classes.CheckNewQuestion(classes.QuestionInput{Text: str("Q"), Type: str("open"), Answers: &withAnswers}); !strings.Contains(strings.Join(problems, "; "), "an open question has no answers") {
t.Errorf("open create with answers: %v", problems)
}
if problems := classes.CheckNewQuestion(classes.QuestionInput{Text: str("Q"), Type: str("single"), Rubric: str("R"), Answers: &toSingle}); !strings.Contains(strings.Join(problems, "; "), "rubric applies to open questions only") {
t.Errorf("single with rubric: %v", problems)
}
if _, problems := check(open, classes.QuestionInput{Type: str(models.TypeSingle), Answers: &toSingle}); !strings.Contains(strings.Join(problems, "; "), "rubric applies to open questions only") {
t.Errorf("open to single keeping the rubric: %v", problems)
}
}
// TestCheckQuestionOrder: an empty list and a repeated id are refused