sm-bm-plugin/console/materials_reindex.go

package console

import (
	"context"
	"errors"
	"fmt"

	"git.golem15.com/golem15/summercms/modules/beachcomber"
	"git.golem15.com/golem15/summercms/modules/bonfire"
	"git.golem15.com/jakub/sm-bm-plugin/classes"
	"git.golem15.com/jakub/sm-bm-plugin/models"
	"gorm.io/gorm"
)

// MaterialsReindexName is the name of the materials reindex command.
const MaterialsReindexName = "bm:materials-reindex"

// materialsReindexBatch is how many material ids one database read loads.
const materialsReindexBatch = 200

// errSearchNotConfigured is the refusal of a reindex without an engine.
var errSearchNotConfigured = errors.New("jz.bm: the search engine is not configured (set search.driver and search.typesense.api_key)")

// MaterialsReindexDeps are what the reindex command needs. DB returns the
// application database, published on the app (Sync skips silently when no
// database is published), and done, which closes a pool DB opened. Index
// reaches the beachcomber service.
type MaterialsReindexDeps struct {
	DB    func(ctx context.Context) (*gorm.DB, func(), error)
	Index *classes.MaterialIndexer
	// Backfill extracts the text of a material's PDFs that have none yet
	// (classes.EnsureFileTexts) before the material is synced. Nil skips
	// it.
	Backfill func(ctx context.Context, db *gorm.DB, materialID uint) error
}

// MaterialsReindexCommand returns bm:materials-reindex: it ensures the
// materials index (dropping it first with --fresh) and syncs every
// material. It refuses with an error when the engine is not configured,
// unless --if-configured, which the nightly schedule passes, makes that a
// notice.
func MaterialsReindexCommand(deps MaterialsReindexDeps) bonfire.Command {
	return bonfire.Command{
		Name:        MaterialsReindexName,
		Description: "Sync every material into the search index",
		Long: `Upserts the search document of every material, drafts and private ones
included (visibility is enforced in SQL, never by the index), through the same
path as the sync after a save. Before each material is synced, the text of
its attached PDFs that have none yet is extracted with pdftotext, so PDFs
attached before file search existed become searchable; a missing pdftotext
(poppler-utils) is one warning and the run goes on without file text.

--fresh first drops the collection and recreates it from the material schema
(models.Material.SearchIndexSchema). Use it on first deploy and after the
schema changed; search finds nothing while the collection is being refilled.

The command refuses, and sends nothing, when the search engine is not
configured (search.driver null, or no search.typesense.api_key).
--if-configured makes that a notice and a zero exit; the nightly schedule
runs the command with it. It ends with "Reindexed: N synced, M failed" and
exits non-zero when any material failed.`,
		Flags: []bonfire.Flag{
			{Name: "fresh", Description: "Drop and recreate the collection from the schema before syncing", Bare: true},
			{Name: "if-configured", Description: "Exit 0 with a notice when the search engine is not configured", Bare: true},
		},
		Examples: []bonfire.Example{
			{Command: MaterialsReindexName, Description: "Sync every material into the existing collection"},
			{Command: MaterialsReindexName + " --fresh", Description: "Recreate the collection from the schema, then sync every material"},
		},
		Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
			return reindexMaterials(ctx, deps, flagOn(in, "fresh"), flagOn(in, "if-configured"), out)
		},
	}
}

func reindexMaterials(ctx context.Context, deps MaterialsReindexDeps, fresh, ifConfigured bool, out bonfire.Output) error {
	var svc *beachcomber.Service
	if deps.Index != nil && deps.Index.Service != nil {
		svc = deps.Index.Service()
	}
	engine := svc.Engine()
	if engine == nil || !engine.Configured() {
		if ifConfigured {
			out.Info(MaterialsReindexName + ": search is not configured; nothing to do")
			return nil
		}
		return errSearchNotConfigured
	}
	if deps.DB == nil {
		return errors.New("jz.bm: no database")
	}
	gdb, done, err := deps.DB(ctx)
	if err != nil {
		return err
	}
	defer done()

	index := svc.IndexName(&models.Material{})
	if fresh {
		existed, err := beachcomber.DropIndex(ctx, engine, index)
		if err != nil {
			return fmt.Errorf("jz.bm: drop %s: %w", index, err)
		}
		if existed {
			out.Printf("Dropped %s\n", index)
		}
	}
	if err := beachcomber.EnsureIndex(ctx, engine, index, models.Material{}.SearchIndexSchema()); err != nil {
		return fmt.Errorf("jz.bm: create %s: %w", index, err)
	}

	marked := classes.MarkMaterialSearch(ctx)
	run := &reindexRun{
		backfill: deps.Backfill,
		sync: func(id uint) error {
			return svc.Sync(marked, gdb, &models.Material{ID: id})
		},
		out: out,
	}
	var batch []models.Material
	err = gdb.WithContext(ctx).Model(&models.Material{}).
		Select("id").
		Order("id").
		FindInBatches(&batch, materialsReindexBatch, func(_ *gorm.DB, _ int) error {
			for i := range batch {
				run.material(ctx, gdb, batch[i].ID)
			}
			return ctx.Err()
		}).Error
	if err != nil {
		return err
	}
	msg := fmt.Sprintf("Reindexed: %d synced, %d failed", run.synced, run.failed)
	if run.failed > 0 {
		out.Error(msg)
		return fmt.Errorf("jz.bm: %d materials failed to sync", run.failed)
	}
	out.Success(msg)
	return nil
}

// pdfToolMissingWarning is printed once per run when pdftotext is missing.
const pdfToolMissingWarning = "pdftotext not found: PDFs add no search text; install poppler-utils and rerun"

// reindexRun is the state of one reindex walk: the run's counters and
// whether pdftotext turned out to be missing.
type reindexRun struct {
	backfill func(ctx context.Context, db *gorm.DB, materialID uint) error
	sync     func(id uint) error
	out      bonfire.Output
	synced   int
	failed   int
	noTool   bool
}

// material backfills the file texts of material id, then syncs it. A
// missing pdftotext warns once and stops the backfill for the rest of the
// run; another backfill error warns for this material, which is still
// synced.
func (r *reindexRun) material(ctx context.Context, db *gorm.DB, id uint) {
	if r.backfill != nil && !r.noTool {
		err := r.backfill(ctx, db, id)
		switch {
		case errors.Is(err, classes.ErrPDFToolMissing):
			r.noTool = true
			r.out.Warning(pdfToolMissingWarning)
		case err != nil:
			r.out.Warning(fmt.Sprintf("material %d: file text: %v", id, err))
		}
	}
	if err := r.sync(id); err != nil {
		r.failed++
		r.out.Warning(fmt.Sprintf("material %d: %v", id, err))
		return
	}
	r.synced++
}

// flagOn reports whether a bare flag was given.
func flagOn(in bonfire.Input, name string) bool {
	v, _ := in.Flag(name)
	return v == "true"
}