package classes import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "slices" "strings" "time" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/jakub/sm-bm-plugin/models" "gorm.io/gorm" "gorm.io/gorm/clause" ) // SeedFile is the JSON document bm:seed reads. Every top-level array is // optional. Courses carry a seed-local key that class sessions, deadlines // and readings use to point at them; meetings are referenced by number. type SeedFile struct { Meetings []SeedMeeting `json:"meetings"` Courses []SeedCourse `json:"courses"` ClassSessions []SeedSession `json:"class_sessions"` Deadlines []SeedDeadline `json:"deadlines"` Readings []SeedReading `json:"readings"` } // SeedMeeting is one meeting; its identity is Number. type SeedMeeting struct { Number int `json:"number"` StartsOn string `json:"starts_on"` EndsOn string `json:"ends_on"` Notes *string `json:"notes"` } // SeedCourse is one course; its identity is Name, Form and Semester. Key is // only a reference inside the file. type SeedCourse struct { Key string `json:"key"` Code *string `json:"code"` Name string `json:"name"` Form string `json:"form"` Lecturer *string `json:"lecturer"` Semester int `json:"semester"` ECTS *float64 `json:"ects"` Assessment *string `json:"assessment"` ElectiveGroup *string `json:"elective_group"` Description *string `json:"description"` SyllabusURL *string `json:"syllabus_url"` SortOrder int `json:"sort_order"` } // SeedSession is one class session; its identity is the course and StartsAt. // Course is a course key, Meeting an optional meeting number. type SeedSession struct { Course string `json:"course"` Meeting *int `json:"meeting"` StartsAt string `json:"starts_at"` EndsAt string `json:"ends_at"` Room *string `json:"room"` Kind *string `json:"kind"` IsOnline bool `json:"is_online"` Notes *string `json:"notes"` } // SeedDeadline is one deadline; its identity is Title and DueAt. Course is // an optional course key. type SeedDeadline struct { Title string `json:"title"` DueAt string `json:"due_at"` Course *string `json:"course"` Kind string `json:"kind"` Description *string `json:"description"` } // SeedReading is one reading; its identity is the course and Title. type SeedReading struct { Course string `json:"course"` Title string `json:"title"` Author *string `json:"author"` Source *string `json:"source"` URL *string `json:"url"` IsRequired bool `json:"is_required"` SortOrder int `json:"sort_order"` } // SeedCounts counts the rows of one kind a seed created, updated, left // unchanged or (with SeedOptions.Prune) deleted. type SeedCounts struct { Created int Updated int Unchanged int Deleted int } // SeedOptions changes how a seed is applied. type SeedOptions struct { // Prune deletes the courses of the file's semesters, and the class // sessions and readings of the file's courses, that the file does not // list. Meetings and deadlines are never pruned. Prune bool // DryRun applies the seed in a transaction that is rolled back, so the // summary shows what would change and nothing is written. DryRun bool } // errDryRun rolls back a dry-run transaction. var errDryRun = errors.New("bm:seed: dry run") func (c *SeedCounts) record(created, changed bool) { switch { case created: c.Created++ case changed: c.Updated++ default: c.Unchanged++ } } // SeedSummary is the outcome of a seed, per kind. type SeedSummary struct { Meetings SeedCounts Courses SeedCounts ClassSessions SeedCounts Deadlines SeedCounts Readings SeedCounts // Removed describes each row Prune deleted, for the command output. Removed []string } // Rows returns the summary as table rows: kind, created, updated, // unchanged, deleted. func (s SeedSummary) Rows() [][]string { row := func(name string, c SeedCounts) []string { return []string{name, fmt.Sprint(c.Created), fmt.Sprint(c.Updated), fmt.Sprint(c.Unchanged), fmt.Sprint(c.Deleted)} } return [][]string{ row("meetings", s.Meetings), row("courses", s.Courses), row("class_sessions", s.ClassSessions), row("deadlines", s.Deadlines), row("readings", s.Readings), } } // ParseSeed decodes a seed document. Unknown fields are an error, so a typo // in a key never drops data silently. func ParseSeed(r io.Reader) (*SeedFile, error) { body, err := io.ReadAll(r) if err != nil { return nil, err } dec := json.NewDecoder(bytes.NewReader(body)) dec.DisallowUnknownFields() var f SeedFile if err := dec.Decode(&f); err != nil { return nil, fmt.Errorf("bm:seed: %w", err) } if dec.More() { return nil, errors.New("bm:seed: trailing data after the JSON document") } return &f, nil } // Validate checks the whole document before anything is written: required // fields, allowed values, dates and times, duplicate identities and that // every course key and meeting number a record points at exists in the // file. Each error names the record. func (f *SeedFile) Validate() error { var errs []error add := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) } meetings := map[int]bool{} for i, m := range f.Meetings { at := fmt.Sprintf("meetings[%d] (number %d)", i, m.Number) if m.Number < 1 { add("%s: number must be a positive integer", at) } if meetings[m.Number] { add("%s: duplicate meeting number", at) } meetings[m.Number] = true start, err1 := models.ParseDate(m.StartsOn) end, err2 := models.ParseDate(m.EndsOn) if err1 != nil { add("%s: starts_on: %v", at, err1) } if err2 != nil { add("%s: ends_on: %v", at, err2) } if err1 == nil && err2 == nil && end.Time(nil).Before(start.Time(nil)) { add("%s: ends_on is before starts_on", at) } } keys := map[string]bool{} identities := map[string]bool{} for i, c := range f.Courses { at := fmt.Sprintf("courses[%d] (key %q)", i, c.Key) if strings.TrimSpace(c.Key) == "" { add("%s: key is required", at) } else if keys[c.Key] { add("%s: duplicate course key", at) } keys[c.Key] = true if strings.TrimSpace(c.Name) == "" { add("%s: name is required", at) } if !slices.Contains(models.CourseForms, c.Form) { add("%s: form %q is not one of %s", at, c.Form, strings.Join(models.CourseForms, ", ")) } if c.Semester < 1 || c.Semester > 4 { add("%s: semester must be between 1 and 4", at) } id := fmt.Sprintf("%s\x00%s\x00%d", c.Name, c.Form, c.Semester) if identities[id] { add("%s: another course has the same name, form and semester", at) } identities[id] = true } sessions := map[string]bool{} for i, s := range f.ClassSessions { at := fmt.Sprintf("class_sessions[%d] (course %q, starts_at %s)", i, s.Course, s.StartsAt) if !keys[s.Course] { add("%s: unknown course key %q", at, s.Course) } if s.Meeting != nil && !meetings[*s.Meeting] { add("%s: unknown meeting number %d", at, *s.Meeting) } start, err1 := models.ParseDateTime(s.StartsAt) end, err2 := models.ParseDateTime(s.EndsAt) if err1 != nil { add("%s: starts_at: %v", at, err1) } if err2 != nil { add("%s: ends_at: %v", at, err2) } if err1 == nil && err2 == nil { if !end.After(start) { add("%s: ends_at is not after starts_at", at) } id := s.Course + "\x00" + start.UTC().String() if sessions[id] { add("%s: duplicate session of the course at this time", at) } sessions[id] = true } } deadlines := map[string]bool{} for i, d := range f.Deadlines { at := fmt.Sprintf("deadlines[%d] (%q)", i, d.Title) if strings.TrimSpace(d.Title) == "" { add("%s: title is required", at) } if d.Course != nil && !keys[*d.Course] { add("%s: unknown course key %q", at, *d.Course) } if !slices.Contains(models.DeadlineKinds, d.Kind) { add("%s: kind %q is not one of %s", at, d.Kind, strings.Join(models.DeadlineKinds, ", ")) } due, err := models.ParseDateTime(d.DueAt) if err != nil { add("%s: due_at: %v", at, err) continue } id := d.Title + "\x00" + due.UTC().String() if deadlines[id] { add("%s: duplicate deadline (same title and due_at)", at) } deadlines[id] = true } readings := map[string]bool{} for i, r := range f.Readings { at := fmt.Sprintf("readings[%d] (course %q, %q)", i, r.Course, r.Title) if !keys[r.Course] { add("%s: unknown course key %q", at, r.Course) } if strings.TrimSpace(r.Title) == "" { add("%s: title is required", at) } id := r.Course + "\x00" + r.Title if readings[id] { add("%s: duplicate reading of the course", at) } readings[id] = true } return errors.Join(errs...) } // Seed validates f and upserts it in one transaction: meetings by number, // courses by name, form and semester, class sessions by course and start // time, deadlines by title and due time, readings by course and title. Rows // absent from the file are left alone. On any error nothing is written. func Seed(ctx context.Context, db *gorm.DB, f *SeedFile) (SeedSummary, error) { return SeedWith(ctx, db, f, SeedOptions{}) } // SeedWith is Seed with options: pruning rows the file no longer lists and // dry runs. func SeedWith(ctx context.Context, db *gorm.DB, f *SeedFile, opts SeedOptions) (SeedSummary, error) { var sum SeedSummary if err := f.Validate(); err != nil { return sum, err } err := lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error { meetingIDs := map[int]uint{} for _, in := range f.Meetings { m := models.Meeting{} created, changed, err := upsert(tx, &m, clause.Eq{Column: "number", Value: in.Number}, func(m *models.Meeting) { m.Number = in.Number m.StartsOn, _ = models.ParseDate(in.StartsOn) m.EndsOn, _ = models.ParseDate(in.EndsOn) m.Notes = nz(in.Notes) }) if err != nil { return fmt.Errorf("meeting %d: %w", in.Number, err) } sum.Meetings.record(created, changed) meetingIDs[in.Number] = m.ID } courseIDs := map[string]uint{} semesters := map[int]bool{} for _, in := range f.Courses { semesters[in.Semester] = true c := models.Course{} where := clause.And( clause.Eq{Column: "name", Value: in.Name}, clause.Eq{Column: "form", Value: in.Form}, clause.Eq{Column: "semester", Value: in.Semester}, ) created, changed, err := upsert(tx, &c, where, func(c *models.Course) { c.Code = nz(in.Code) c.Name = in.Name c.Form = in.Form c.Lecturer = nz(in.Lecturer) c.Semester = in.Semester c.ECTS = in.ECTS c.Assessment = nz(in.Assessment) c.ElectiveGroup = nz(in.ElectiveGroup) c.Description = nz(in.Description) c.SyllabusURL = nz(in.SyllabusURL) c.SortOrder = in.SortOrder }) if err != nil { return fmt.Errorf("course %q: %w", in.Key, err) } sum.Courses.record(created, changed) courseIDs[in.Key] = c.ID } keptSessions := []uint{0} for _, in := range f.ClassSessions { start, _ := models.ParseDateTime(in.StartsAt) s := models.ClassSession{} where := clause.And( clause.Eq{Column: "course_id", Value: courseIDs[in.Course]}, clause.Eq{Column: "starts_at", Value: start.UTC()}, ) created, changed, err := upsert(tx, &s, where, func(s *models.ClassSession) { courseID := courseIDs[in.Course] s.CourseID = &courseID s.MeetingID = nil if in.Meeting != nil { id := meetingIDs[*in.Meeting] s.MeetingID = &id } end, _ := models.ParseDateTime(in.EndsAt) setInstant(&s.StartsAt, start) setInstant(&s.EndsAt, end) s.Room = nz(in.Room) s.Kind = nz(in.Kind) s.IsOnline = in.IsOnline s.Notes = nz(in.Notes) }) if err != nil { return fmt.Errorf("class session %s %s: %w", in.Course, in.StartsAt, err) } sum.ClassSessions.record(created, changed) keptSessions = append(keptSessions, s.ID) } for _, in := range f.Deadlines { due, _ := models.ParseDateTime(in.DueAt) d := models.Deadline{} where := clause.And( clause.Eq{Column: "title", Value: in.Title}, clause.Eq{Column: "due_at", Value: due.UTC()}, ) created, changed, err := upsert(tx, &d, where, func(d *models.Deadline) { d.Title = in.Title setInstant(&d.DueAt, due) d.CourseID = nil if in.Course != nil { id := courseIDs[*in.Course] d.CourseID = &id } d.Kind = in.Kind d.Description = nz(in.Description) }) if err != nil { return fmt.Errorf("deadline %q: %w", in.Title, err) } sum.Deadlines.record(created, changed) } keptReadings := []uint{0} for _, in := range f.Readings { r := models.Reading{} where := clause.And( clause.Eq{Column: "course_id", Value: courseIDs[in.Course]}, clause.Eq{Column: "title", Value: in.Title}, ) created, changed, err := upsert(tx, &r, where, func(r *models.Reading) { courseID := courseIDs[in.Course] r.CourseID = &courseID r.Title = in.Title r.Author = nz(in.Author) r.Source = nz(in.Source) r.URL = nz(in.URL) r.IsRequired = in.IsRequired r.SortOrder = in.SortOrder }) if err != nil { return fmt.Errorf("reading %q of %s: %w", in.Title, in.Course, err) } sum.Readings.record(created, changed) keptReadings = append(keptReadings, r.ID) } if opts.Prune { if err := prune(tx, &sum, semesters, courseIDs, keptSessions, keptReadings); err != nil { return err } } if opts.DryRun { return errDryRun } return nil }) if err != nil && !errors.Is(err, errDryRun) { return SeedSummary{}, err } return sum, nil } // prune deletes what the file no longer lists: courses of its semesters // (their sessions and readings go with them through the foreign keys), then // the sessions and readings of the kept courses that the seed did not touch. func prune(tx *gorm.DB, sum *SeedSummary, semesters map[int]bool, courseIDs map[string]uint, keptSessions, keptReadings []uint) error { kept := []uint{0} for _, id := range courseIDs { kept = append(kept, id) } sems := []int{} for sem := range semesters { sems = append(sems, sem) } var courses []models.Course if len(sems) > 0 { if err := tx.Where("semester IN ? AND id NOT IN ?", sems, kept).Order("id").Find(&courses).Error; err != nil { return err } } for _, c := range courses { var sessions, readings int64 tx.Model(&models.ClassSession{}).Where("course_id = ?", c.ID).Count(&sessions) tx.Model(&models.Reading{}).Where("course_id = ?", c.ID).Count(&readings) sum.Removed = append(sum.Removed, fmt.Sprintf("course %q (%s, semester %d) with %d sessions and %d readings", c.Name, c.Form, c.Semester, sessions, readings)) if err := tx.Delete(&models.Course{}, c.ID).Error; err != nil { return fmt.Errorf("prune course %q: %w", c.Name, err) } sum.Courses.Deleted++ sum.ClassSessions.Deleted += int(sessions) sum.Readings.Deleted += int(readings) } var sessions []models.ClassSession if err := tx.Where("course_id IN ? AND id NOT IN ?", kept, keptSessions).Order("starts_at").Find(&sessions).Error; err != nil { return err } for _, s := range sessions { sum.Removed = append(sum.Removed, fmt.Sprintf("class session of course %d at %s", s.CourseID, models.FormatDateTime(s.StartsAt))) if err := tx.Delete(&models.ClassSession{}, s.ID).Error; err != nil { return fmt.Errorf("prune class session %d: %w", s.ID, err) } sum.ClassSessions.Deleted++ } var readings []models.Reading if err := tx.Where("course_id IN ? AND id NOT IN ?", kept, keptReadings).Order("id").Find(&readings).Error; err != nil { return err } for _, r := range readings { sum.Removed = append(sum.Removed, fmt.Sprintf("reading %q of course %d", r.Title, r.CourseID)) if err := tx.Delete(&models.Reading{}, r.ID).Error; err != nil { return fmt.Errorf("prune reading %d: %w", r.ID, err) } sum.Readings.Deleted++ } return nil } // nz returns nil for a missing or blank optional text, as the models' // BeforeSave hooks store it, so an unchanged row compares equal. func nz(s *string) *string { if s == nil || strings.TrimSpace(*s) == "" { return nil } return s } // upsert loads the row matching where into row, applies set and writes it: // an insert when there was no row, an update when set changed a column, and // nothing otherwise. The comparison is on the row's own fields, before and // after set, as JSON. // setInstant sets *dst to t unless it already holds the same instant, so a // time read back from the database in another zone does not count as a // change in upsert. func setInstant(dst *time.Time, t time.Time) { if !dst.Equal(t) { *dst = t } } func upsert[T any](tx *gorm.DB, row *T, where clause.Expression, set func(*T)) (created, changed bool, err error) { res := tx.Clauses(where).Limit(1).Find(row) if res.Error != nil { return false, false, res.Error } if res.RowsAffected == 0 { set(row) return true, false, tx.Create(row).Error } before, err := json.Marshal(row) if err != nil { return false, false, err } set(row) after, err := json.Marshal(row) if err != nil { return false, false, err } if bytes.Equal(before, after) { return false, false, nil } return false, true, tx.Save(row).Error }