sm-bm-plugin/models/material_search.go

package models

import (
	"context"
	"regexp"
	"strconv"
	"strings"

	"gorm.io/gorm"
)

// A material is kept in the search index jz_bm_materials (beachcomber). The
// index holds every material, drafts and private ones included: visibility
// is enforced only in SQL, where the engine's ids are candidates that the
// student queries re-gate. This file is hand-written; material.go is
// generated.

// SearchableAs is the index name without the search.prefix.
func (Material) SearchableAs() string { return "jz_bm_materials" }

// ShouldBeSearchable is true for every material: the index carries no
// visibility, so drafts and private materials are indexed too.
func (m *Material) ShouldBeSearchable() bool { return m != nil }

// ToSearchableArray builds the document of the material. beachcomber reloads
// the row by primary key without preloads, so the course, the meeting number
// and the tags are loaded here.
func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) {
	db = db.WithContext(ctx)
	var course *Course
	if m.CourseID != nil {
		var c Course
		res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c)
		if res.Error != nil {
			return nil, res.Error
		}
		if res.RowsAffected > 0 {
			course = &c
		}
	}
	meetingNumber := 0
	if m.MeetingID != nil {
		var mt Meeting
		res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt)
		if res.Error != nil {
			return nil, res.Error
		}
		if res.RowsAffected > 0 {
			meetingNumber = mt.Number
		}
	}
	var tags []Tag
	err := db.Model(&Tag{}).
		Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id").
		Where("mt.material_id = ?", m.ID).
		Order("jz_bm_tags.name").
		Find(&tags).Error
	if err != nil {
		return nil, err
	}
	files, err := IndexedFiles(db, []uint{m.ID}, true)
	if err != nil {
		return nil, err
	}
	texts := make([]string, 0, len(files[m.ID]))
	for _, f := range files[m.ID] {
		texts = append(texts, f.Text)
	}
	return MaterialDocument(*m, course, meetingNumber, tags, texts), nil
}

// IndexedFile is one entry of a material document's file_texts: the file,
// its name and, when asked for, its extracted text.
type IndexedFile struct {
	FileID   uint
	FileName string
	Text     string
}

// IndexedFiles returns, per material id, the files whose text the search
// document holds, in document order: the current public files of the
// material's files field that have a non-empty extracted text, by
// sort_order (nulls last) and id. It is the one query that decides what
// file_texts holds, so a snippet's array index maps back to a file through
// it. A detached file is no longer a current file, so its text drops out
// even before the cascade removes it. The text is read only when withText.
func IndexedFiles(db *gorm.DB, materialIDs []uint, withText bool) (map[uint][]IndexedFile, error) {
	out := map[uint][]IndexedFile{}
	if len(materialIDs) == 0 {
		return out, nil
	}
	ids := make([]string, 0, len(materialIDs))
	for _, id := range materialIDs {
		ids = append(ids, strconv.FormatUint(uint64(id), 10))
	}
	cols := "system_files.id AS file_id, system_files.file_name, system_files.attachment_id"
	if withText {
		cols += ", t.text"
	}
	var rows []struct {
		FileID       uint
		FileName     string
		AttachmentID string
		Text         string
	}
	err := db.Table("system_files").
		Select(cols).
		Joins("JOIN jz_bm_material_file_texts t ON t.file_id = system_files.id").
		Where("system_files.attachment_type = ? AND system_files.field = ? AND system_files.is_public", Material{}.MorphName(), MaterialFilesField).
		Where("system_files.attachment_id IN ?", ids).
		Where("t.text <> ''").
		Order("system_files.attachment_id, system_files.sort_order NULLS LAST, system_files.id").
		Scan(&rows).Error
	if err != nil {
		return nil, err
	}
	for _, r := range rows {
		n, err := strconv.ParseUint(r.AttachmentID, 10, 64)
		if err != nil {
			continue
		}
		id := uint(n)
		out[id] = append(out[id], IndexedFile{FileID: r.FileID, FileName: r.FileName, Text: r.Text})
	}
	return out, nil
}

// MaterialDocument is the search document of m with its course, meeting
// number (0 when none), tags in order and the texts of its attached files.
// Absent numbers are 0, so filters and the sort tiebreak never see a
// missing value.
func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any {
	doc := map[string]any{
		"id":           strconv.FormatUint(uint64(m.ID), 10),
		"title":        m.Title,
		"type":         m.Type,
		"course_id":    int64(0),
		"meeting_id":   int64(0),
		"published_at": int64(0),
	}
	if m.Source != nil && strings.TrimSpace(*m.Source) != "" {
		doc["source"] = *m.Source
	}
	if m.URL != nil && strings.TrimSpace(*m.URL) != "" {
		doc["url"] = *m.URL
	}
	if m.CourseID != nil {
		doc["course_id"] = int64(*m.CourseID)
	}
	if m.MeetingID != nil {
		doc["meeting_id"] = int64(*m.MeetingID)
	}
	if m.PublishedAt != nil {
		doc["published_at"] = m.PublishedAt.Unix()
	}
	if course != nil {
		label := course.Name
		if course.Code != nil && strings.TrimSpace(*course.Code) != "" {
			label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code))
		}
		doc["course"] = label
	}
	if meetingNumber > 0 {
		doc["meeting_number"] = meetingNumber
	}
	names := make([]string, 0, len(tags))
	slugs := make([]string, 0, len(tags))
	for _, t := range tags {
		names = append(names, t.Name)
		slugs = append(slugs, t.Slug)
	}
	doc["tags"] = names
	doc["tag_slugs"] = slugs
	if m.Body != nil {
		if body := MaterialPlainBody(*m.Body); body != "" {
			doc["body"] = body
		}
	}
	if len(fileTexts) > 0 {
		doc["file_texts"] = fileTexts
	}
	return doc
}

// SearchIndexSchema is the Typesense collection schema. It applies only when
// the collection is created; bm:materials-reindex --fresh recreates it.
// Every field is stored (a search leaves body and file_texts out of the
// answer instead), and no field sets a locale: the default one folds
// diacritics, so lodz finds Łódź.
func (Material) SearchIndexSchema() map[string]any {
	return map[string]any{
		"fields": []map[string]any{
			{"name": "id", "type": "string"},
			{"name": "title", "type": "string"},
			{"name": "source", "type": "string", "optional": true},
			{"name": "url", "type": "string", "optional": true},
			{"name": "tags", "type": "string[]", "optional": true},
			{"name": "tag_slugs", "type": "string[]", "optional": true},
			{"name": "course", "type": "string", "optional": true},
			{"name": "course_id", "type": "int64"},
			{"name": "meeting_id", "type": "int64"},
			{"name": "meeting_number", "type": "int32", "optional": true},
			{"name": "type", "type": "string"},
			{"name": "published_at", "type": "int64"},
			{"name": "body", "type": "string", "optional": true},
			{"name": "file_texts", "type": "string[]", "optional": true},
		},
	}
}

var (
	mdImage      = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
	mdLink       = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
	mdHeading    = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`)
	mdQuote      = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`)
	mdList       = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`)
	mdEmphasis   = regexp.MustCompile(`\*+|~~`)
	mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`)
	mdUnderEnd   = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`)
)

// MaterialPlainBody drops the Markdown syntax of a material body (heading
// hashes, blockquote and list markers, emphasis, inline code ticks, link
// and image syntax down to their text) and collapses white space, so the
// index holds and highlights only words.
func MaterialPlainBody(body string) string {
	s := strings.ReplaceAll(body, "`", "")
	s = mdImage.ReplaceAllString(s, "$1")
	s = mdLink.ReplaceAllString(s, "$1")
	s = mdHeading.ReplaceAllString(s, "")
	s = mdQuote.ReplaceAllString(s, "")
	s = mdList.ReplaceAllString(s, "")
	s = mdEmphasis.ReplaceAllString(s, "")
	s = mdUnderStart.ReplaceAllString(s, "$1")
	s = mdUnderEnd.ReplaceAllString(s, "$1")
	return strings.Join(strings.Fields(s), " ")
}