package models import ( "context" "regexp" "strconv" "strings" "gorm.io/gorm" ) // A material is kept in the search index jz_bm_materials (beachcomber). The // index holds every material, drafts and private ones included: visibility // is enforced only in SQL, where the engine's ids are candidates that the // student queries re-gate. This file is hand-written; material.go is // generated. // SearchableAs is the index name without the search.prefix. func (Material) SearchableAs() string { return "jz_bm_materials" } // ShouldBeSearchable is true for every material: the index carries no // visibility, so drafts and private materials are indexed too. func (m *Material) ShouldBeSearchable() bool { return m != nil } // ToSearchableArray builds the document of the material. beachcomber reloads // the row by primary key without preloads, so the course, the meeting number // and the tags are loaded here. func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) { db = db.WithContext(ctx) var course *Course if m.CourseID != nil { var c Course res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c) if res.Error != nil { return nil, res.Error } if res.RowsAffected > 0 { course = &c } } meetingNumber := 0 if m.MeetingID != nil { var mt Meeting res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt) if res.Error != nil { return nil, res.Error } if res.RowsAffected > 0 { meetingNumber = mt.Number } } var tags []Tag err := db.Model(&Tag{}). Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id"). Where("mt.material_id = ?", m.ID). Order("jz_bm_tags.name"). Find(&tags).Error if err != nil { return nil, err } files, err := IndexedFiles(db, []uint{m.ID}, true) if err != nil { return nil, err } texts := make([]string, 0, len(files[m.ID])) for _, f := range files[m.ID] { texts = append(texts, f.Text) } return MaterialDocument(*m, course, meetingNumber, tags, texts), nil } // IndexedFile is one entry of a material document's file_texts: the file, // its name and, when asked for, its extracted text. type IndexedFile struct { FileID uint FileName string Text string } // IndexedFiles returns, per material id, the files whose text the search // document holds, in document order: the current public files of the // material's files field that have a non-empty extracted text, by // sort_order (nulls last) and id. It is the one query that decides what // file_texts holds, so a snippet's array index maps back to a file through // it. A detached file is no longer a current file, so its text drops out // even before the cascade removes it. The text is read only when withText. func IndexedFiles(db *gorm.DB, materialIDs []uint, withText bool) (map[uint][]IndexedFile, error) { out := map[uint][]IndexedFile{} if len(materialIDs) == 0 { return out, nil } ids := make([]string, 0, len(materialIDs)) for _, id := range materialIDs { ids = append(ids, strconv.FormatUint(uint64(id), 10)) } cols := "system_files.id AS file_id, system_files.file_name, system_files.attachment_id" if withText { cols += ", t.text" } var rows []struct { FileID uint FileName string AttachmentID string Text string } err := db.Table("system_files"). Select(cols). Joins("JOIN jz_bm_material_file_texts t ON t.file_id = system_files.id"). Where("system_files.attachment_type = ? AND system_files.field = ? AND system_files.is_public", Material{}.MorphName(), MaterialFilesField). Where("system_files.attachment_id IN ?", ids). Where("t.text <> ''"). Order("system_files.attachment_id, system_files.sort_order NULLS LAST, system_files.id"). Scan(&rows).Error if err != nil { return nil, err } for _, r := range rows { n, err := strconv.ParseUint(r.AttachmentID, 10, 64) if err != nil { continue } id := uint(n) out[id] = append(out[id], IndexedFile{FileID: r.FileID, FileName: r.FileName, Text: r.Text}) } return out, nil } // MaterialDocument is the search document of m with its course, meeting // number (0 when none), tags in order and the texts of its attached files. // Absent numbers are 0, so filters and the sort tiebreak never see a // missing value. func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any { doc := map[string]any{ "id": strconv.FormatUint(uint64(m.ID), 10), "title": m.Title, "type": m.Type, "course_id": int64(0), "meeting_id": int64(0), "published_at": int64(0), } if m.Source != nil && strings.TrimSpace(*m.Source) != "" { doc["source"] = *m.Source } if m.URL != nil && strings.TrimSpace(*m.URL) != "" { doc["url"] = *m.URL } if m.CourseID != nil { doc["course_id"] = int64(*m.CourseID) } if m.MeetingID != nil { doc["meeting_id"] = int64(*m.MeetingID) } if m.PublishedAt != nil { doc["published_at"] = m.PublishedAt.Unix() } if course != nil { label := course.Name if course.Code != nil && strings.TrimSpace(*course.Code) != "" { label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code)) } doc["course"] = label } if meetingNumber > 0 { doc["meeting_number"] = meetingNumber } names := make([]string, 0, len(tags)) slugs := make([]string, 0, len(tags)) for _, t := range tags { names = append(names, t.Name) slugs = append(slugs, t.Slug) } doc["tags"] = names doc["tag_slugs"] = slugs if m.Body != nil { if body := MaterialPlainBody(*m.Body); body != "" { doc["body"] = body } } if len(fileTexts) > 0 { doc["file_texts"] = fileTexts } return doc } // SearchIndexSchema is the Typesense collection schema. It applies only when // the collection is created; bm:materials-reindex --fresh recreates it. // Every field is stored (a search leaves body and file_texts out of the // answer instead), and no field sets a locale: the default one folds // diacritics, so lodz finds Łódź. func (Material) SearchIndexSchema() map[string]any { return map[string]any{ "fields": []map[string]any{ {"name": "id", "type": "string"}, {"name": "title", "type": "string"}, {"name": "source", "type": "string", "optional": true}, {"name": "url", "type": "string", "optional": true}, {"name": "tags", "type": "string[]", "optional": true}, {"name": "tag_slugs", "type": "string[]", "optional": true}, {"name": "course", "type": "string", "optional": true}, {"name": "course_id", "type": "int64"}, {"name": "meeting_id", "type": "int64"}, {"name": "meeting_number", "type": "int32", "optional": true}, {"name": "type", "type": "string"}, {"name": "published_at", "type": "int64"}, {"name": "body", "type": "string", "optional": true}, {"name": "file_texts", "type": "string[]", "optional": true}, }, } } var ( mdImage = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`) mdLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`) mdHeading = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`) mdQuote = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`) mdList = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`) mdEmphasis = regexp.MustCompile(`\*+|~~`) mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`) mdUnderEnd = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`) ) // MaterialPlainBody drops the Markdown syntax of a material body (heading // hashes, blockquote and list markers, emphasis, inline code ticks, link // and image syntax down to their text) and collapses white space, so the // index holds and highlights only words. func MaterialPlainBody(body string) string { s := strings.ReplaceAll(body, "`", "") s = mdImage.ReplaceAllString(s, "$1") s = mdLink.ReplaceAllString(s, "$1") s = mdHeading.ReplaceAllString(s, "") s = mdQuote.ReplaceAllString(s, "") s = mdList.ReplaceAllString(s, "") s = mdEmphasis.ReplaceAllString(s, "") s = mdUnderStart.ReplaceAllString(s, "$1") s = mdUnderEnd.ReplaceAllString(s, "$1") return strings.Join(strings.Fields(s), " ") }