feat: score single, multiple and self-check answers with configurable partial credit and penalty
This commit is contained in:
Jakub Zych
174
classes/scoring.go
Normal file
174
classes/scoring.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
|
||||
"git.golem15.com/golem15/sm-quizzes-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
)
|
||||
|
||||
// ScoringConfig holds the golem15.quizzes.scoring.* settings.
|
||||
type ScoringConfig struct {
|
||||
// MultiplePartial gives partial credit on a multiple question: (correct
|
||||
// chosen - wrong chosen) / correct answers, floored at 0.
|
||||
MultiplePartial bool
|
||||
// PenaltyWrong is the fraction of a question's points subtracted for a
|
||||
// wrong answer, from 0 (no penalty) to 1.
|
||||
PenaltyWrong float64
|
||||
}
|
||||
|
||||
// ScoringConfigFrom reads golem15.quizzes.scoring.multiple_partial and
|
||||
// golem15.quizzes.scoring.penalty_wrong. A missing config gives the defaults
|
||||
// (no partial credit, no penalty); the penalty is clamped to [0, 1].
|
||||
func ScoringConfigFrom(cfg *compass.Config) ScoringConfig {
|
||||
if cfg == nil {
|
||||
return ScoringConfig{}
|
||||
}
|
||||
out := ScoringConfig{MultiplePartial: cfg.Bool("golem15.quizzes.scoring.multiple_partial")}
|
||||
if v, ok := cfg.Lookup("golem15.quizzes.scoring.penalty_wrong"); ok {
|
||||
out.PenaltyWrong = toFloat(v)
|
||||
}
|
||||
out.PenaltyWrong = math.Min(math.Max(out.PenaltyWrong, 0), 1)
|
||||
return out
|
||||
}
|
||||
|
||||
func toFloat(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case float32:
|
||||
return float64(n)
|
||||
case int:
|
||||
return float64(n)
|
||||
case int64:
|
||||
return float64(n)
|
||||
case uint64:
|
||||
return float64(n)
|
||||
case string:
|
||||
f, err := strconv.ParseFloat(n, 64)
|
||||
if err == nil {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ScoreInput is one answered question, as the scorer sees it.
|
||||
type ScoreInput struct {
|
||||
// Type is models.TypeSingle, models.TypeMultiple or models.TypeSelfCheck.
|
||||
Type string
|
||||
// Points is the question's full value.
|
||||
Points int
|
||||
// CorrectIDs are the IDs of the question's correct answers.
|
||||
CorrectIDs []uint
|
||||
// ChosenIDs are the answer IDs the user chose (single and multiple).
|
||||
ChosenIDs []uint
|
||||
// Known is the user's own mark on a self_check question.
|
||||
Known *bool
|
||||
}
|
||||
|
||||
// ScoreResult is the outcome of one question.
|
||||
type ScoreResult struct {
|
||||
// IsCorrect is true only for a fully correct answer.
|
||||
IsCorrect bool
|
||||
// Points earned; negative when a penalty applies. Totals are floored at
|
||||
// 0 by Totals, never per question.
|
||||
Points float64
|
||||
}
|
||||
|
||||
// Score grades one answer. It is a pure function:
|
||||
//
|
||||
// - single: correct when exactly one answer is chosen and it is a correct
|
||||
// one.
|
||||
// - multiple: correct when the chosen set equals the correct set. With
|
||||
// MultiplePartial a partly right answer earns (correct chosen - wrong
|
||||
// chosen) / correct answers of the points, floored at 0.
|
||||
// - self_check: correct when the user marked it as known.
|
||||
//
|
||||
// An answer that earns nothing and is not empty is wrong: with PenaltyWrong
|
||||
// it loses that fraction of the question's points. An empty answer and a
|
||||
// self_check marked as unknown are never penalised.
|
||||
func Score(in ScoreInput, cfg ScoringConfig) ScoreResult {
|
||||
full := float64(max(in.Points, 0))
|
||||
switch in.Type {
|
||||
case models.TypeSelfCheck:
|
||||
if in.Known != nil && *in.Known {
|
||||
return ScoreResult{IsCorrect: true, Points: full}
|
||||
}
|
||||
return ScoreResult{}
|
||||
case models.TypeSingle, models.TypeMultiple:
|
||||
default:
|
||||
return ScoreResult{}
|
||||
}
|
||||
|
||||
chosen := uniqueIDs(in.ChosenIDs)
|
||||
if len(chosen) == 0 {
|
||||
return ScoreResult{}
|
||||
}
|
||||
correct := make(map[uint]bool, len(in.CorrectIDs))
|
||||
for _, id := range in.CorrectIDs {
|
||||
correct[id] = true
|
||||
}
|
||||
right, wrong := 0, 0
|
||||
for _, id := range chosen {
|
||||
if correct[id] {
|
||||
right++
|
||||
} else {
|
||||
wrong++
|
||||
}
|
||||
}
|
||||
|
||||
var res ScoreResult
|
||||
switch in.Type {
|
||||
case models.TypeSingle:
|
||||
if len(chosen) == 1 && right == 1 {
|
||||
res = ScoreResult{IsCorrect: true, Points: full}
|
||||
}
|
||||
case models.TypeMultiple:
|
||||
switch {
|
||||
case len(correct) > 0 && wrong == 0 && right == len(correct):
|
||||
res = ScoreResult{IsCorrect: true, Points: full}
|
||||
case cfg.MultiplePartial && len(correct) > 0:
|
||||
res.Points = round(full * math.Max(float64(right-wrong), 0) / float64(len(correct)))
|
||||
}
|
||||
}
|
||||
if !res.IsCorrect && res.Points == 0 && cfg.PenaltyWrong > 0 {
|
||||
res.Points = round(-full * cfg.PenaltyWrong)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Totals sums the earned points of an attempt, floored at 0, and the
|
||||
// percentage of maxPoints it is. With no points to earn the percentage is 0.
|
||||
func Totals(points []float64, maxPoints int) (score, percent float64) {
|
||||
for _, p := range points {
|
||||
score += p
|
||||
}
|
||||
score = round(math.Max(score, 0))
|
||||
if maxPoints <= 0 {
|
||||
return score, 0
|
||||
}
|
||||
return score, round(score / float64(maxPoints) * 100)
|
||||
}
|
||||
|
||||
// round keeps two decimals.
|
||||
func round(f float64) float64 {
|
||||
r := math.Round(f*100) / 100
|
||||
if r == 0 {
|
||||
return 0 // no negative zero in JSON
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func uniqueIDs(ids []uint) []uint {
|
||||
seen := make(map[uint]bool, len(ids))
|
||||
out := make([]uint, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if !seen[id] {
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
132
classes/scoring_test.go
Normal file
132
classes/scoring_test.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package classes_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/sm-quizzes-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-quizzes-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
)
|
||||
|
||||
func boolPtr(b bool) *bool { return &b }
|
||||
|
||||
func TestScore(t *testing.T) {
|
||||
none := classes.ScoringConfig{}
|
||||
partial := classes.ScoringConfig{MultiplePartial: true}
|
||||
penalty := classes.ScoringConfig{PenaltyWrong: 0.25}
|
||||
both := classes.ScoringConfig{MultiplePartial: true, PenaltyWrong: 0.5}
|
||||
|
||||
single := func(chosen ...uint) classes.ScoreInput {
|
||||
return classes.ScoreInput{Type: models.TypeSingle, Points: 2, CorrectIDs: []uint{10}, ChosenIDs: chosen}
|
||||
}
|
||||
multi := func(chosen ...uint) classes.ScoreInput {
|
||||
return classes.ScoreInput{Type: models.TypeMultiple, Points: 4, CorrectIDs: []uint{1, 2, 3, 4}, ChosenIDs: chosen}
|
||||
}
|
||||
self := func(known *bool) classes.ScoreInput {
|
||||
return classes.ScoreInput{Type: models.TypeSelfCheck, Points: 3, Known: known}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
in classes.ScoreInput
|
||||
cfg classes.ScoringConfig
|
||||
correct bool
|
||||
points float64
|
||||
}{
|
||||
{"single right", single(10), none, true, 2},
|
||||
{"single wrong", single(11), none, false, 0},
|
||||
{"single two chosen", single(10, 11), none, false, 0},
|
||||
{"single duplicate id counts once", single(10, 10), none, true, 2},
|
||||
{"single empty", single(), none, false, 0},
|
||||
{"single wrong with penalty", single(11), penalty, false, -0.5},
|
||||
{"single empty never penalised", single(), penalty, false, 0},
|
||||
|
||||
{"multiple exact", multi(4, 3, 2, 1), none, true, 4},
|
||||
{"multiple subset without partial", multi(1, 2), none, false, 0},
|
||||
{"multiple subset with partial", multi(1, 2), partial, false, 2},
|
||||
{"multiple partial minus wrong", multi(1, 2, 3, 9), partial, false, 2},
|
||||
{"multiple partial floored at 0", multi(1, 8, 9), partial, false, 0},
|
||||
{"multiple superset with partial", multi(1, 2, 3, 4, 9), partial, false, 3},
|
||||
{"multiple wrong with penalty", multi(9), penalty, false, -1},
|
||||
{"multiple partial earns, no penalty", multi(1), both, false, 1},
|
||||
{"multiple partial nothing, penalty", multi(9), both, false, -2},
|
||||
|
||||
{"self_check known", self(boolPtr(true)), none, true, 3},
|
||||
{"self_check unknown", self(boolPtr(false)), none, false, 0},
|
||||
{"self_check unknown never penalised", self(boolPtr(false)), penalty, false, 0},
|
||||
{"self_check unanswered", self(nil), none, false, 0},
|
||||
|
||||
{"unknown type", classes.ScoreInput{Type: "essay", Points: 5}, none, false, 0},
|
||||
{"negative points treated as 0", classes.ScoreInput{Type: models.TypeSingle, Points: -1, CorrectIDs: []uint{1}, ChosenIDs: []uint{1}}, none, true, 0},
|
||||
{"multiple without correct answers", classes.ScoreInput{Type: models.TypeMultiple, Points: 1, ChosenIDs: []uint{1}}, partial, false, 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := classes.Score(tc.in, tc.cfg)
|
||||
if got.IsCorrect != tc.correct || got.Points != tc.points {
|
||||
t.Errorf("Score = %+v, want correct=%v points=%v", got, tc.correct, tc.points)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTotals(t *testing.T) {
|
||||
cases := []struct {
|
||||
points []float64
|
||||
max int
|
||||
score, pct float64
|
||||
}{
|
||||
{[]float64{1, 1, 0}, 3, 2, 66.67},
|
||||
{[]float64{-1, -0.5}, 2, 0, 0},
|
||||
{[]float64{2, -0.5}, 4, 1.5, 37.5},
|
||||
{nil, 0, 0, 0},
|
||||
{[]float64{1}, 1, 1, 100},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
score, pct := classes.Totals(tc.points, tc.max)
|
||||
if score != tc.score || pct != tc.pct {
|
||||
t.Errorf("Totals(%v, %d) = %v, %v; want %v, %v", tc.points, tc.max, score, pct, tc.score, tc.pct)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoringConfigFrom(t *testing.T) {
|
||||
if got := classes.ScoringConfigFrom(nil); got != (classes.ScoringConfig{}) {
|
||||
t.Errorf("nil config = %+v, want defaults", got)
|
||||
}
|
||||
open := func(yaml string) *compass.Config {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if yaml != "" {
|
||||
if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte(yaml), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cfg, err := compass.Open(compass.Options{Dir: dir, Env: "testing", Environ: []string{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
cfg := open("")
|
||||
if got := classes.ScoringConfigFrom(cfg); got != (classes.ScoringConfig{}) {
|
||||
t.Errorf("empty config = %+v, want defaults", got)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
value any
|
||||
want float64
|
||||
}{{0.25, 0.25}, {1, 1}, {"0.5", 0.5}, {3, 1}, {-2, 0}, {"x", 0}} {
|
||||
if err := cfg.Set("golem15.quizzes.scoring.penalty_wrong", tc.value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cfg.Set("golem15.quizzes.scoring.multiple_partial", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := classes.ScoringConfigFrom(cfg)
|
||||
if !got.MultiplePartial || got.PenaltyWrong != tc.want {
|
||||
t.Errorf("penalty_wrong %v: %+v, want partial and %v", tc.value, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1 +1,10 @@
|
||||
{}
|
||||
# Defaults for golem15.quizzes, merged under the plugin ID: the application
|
||||
# reads these as golem15.quizzes.scoring.* and can override them in its own
|
||||
# config.
|
||||
scoring:
|
||||
# Give partial credit on multiple-answer questions:
|
||||
# (correct chosen - wrong chosen) / correct answers, floored at 0.
|
||||
multiple_partial: false
|
||||
# Fraction of a question's points subtracted for a wrong answer (0 to 1).
|
||||
# An attempt's total never goes below 0.
|
||||
penalty_wrong: 0
|
||||
|
||||
Reference in New Issue
Block a user