package console import ( "context" "errors" "fmt" "git.golem15.com/golem15/summercms/modules/beachcomber" "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/jakub/sm-bm-plugin/classes" "git.golem15.com/jakub/sm-bm-plugin/models" "gorm.io/gorm" ) // MaterialsReindexName is the name of the materials reindex command. const MaterialsReindexName = "bm:materials-reindex" // materialsReindexBatch is how many material ids one database read loads. const materialsReindexBatch = 200 // errSearchNotConfigured is the refusal of a reindex without an engine. var errSearchNotConfigured = errors.New("jz.bm: the search engine is not configured (set search.driver and search.typesense.api_key)") // MaterialsReindexDeps are what the reindex command needs. DB returns the // application database, published on the app (Sync skips silently when no // database is published), and done, which closes a pool DB opened. Index // reaches the beachcomber service. type MaterialsReindexDeps struct { DB func(ctx context.Context) (*gorm.DB, func(), error) Index *classes.MaterialIndexer // Backfill extracts the text of a material's PDFs that have none yet // (classes.EnsureFileTexts) before the material is synced. Nil skips // it. Backfill func(ctx context.Context, db *gorm.DB, materialID uint) error } // MaterialsReindexCommand returns bm:materials-reindex: it ensures the // materials index (dropping it first with --fresh) and syncs every // material. It refuses with an error when the engine is not configured, // unless --if-configured, which the nightly schedule passes, makes that a // notice. func MaterialsReindexCommand(deps MaterialsReindexDeps) bonfire.Command { return bonfire.Command{ Name: MaterialsReindexName, Description: "Sync every material into the search index", Long: `Upserts the search document of every material, drafts and private ones included (visibility is enforced in SQL, never by the index), through the same path as the sync after a save. Before each material is synced, the text of its attached PDFs that have none yet is extracted with pdftotext, so PDFs attached before file search existed become searchable; a missing pdftotext (poppler-utils) is one warning and the run goes on without file text. --fresh first drops the collection and recreates it from the material schema (models.Material.SearchIndexSchema). Use it on first deploy and after the schema changed; search finds nothing while the collection is being refilled. The command refuses, and sends nothing, when the search engine is not configured (search.driver null, or no search.typesense.api_key). --if-configured makes that a notice and a zero exit; the nightly schedule runs the command with it. It ends with "Reindexed: N synced, M failed" and exits non-zero when any material failed.`, Flags: []bonfire.Flag{ {Name: "fresh", Description: "Drop and recreate the collection from the schema before syncing", Bare: true}, {Name: "if-configured", Description: "Exit 0 with a notice when the search engine is not configured", Bare: true}, }, Examples: []bonfire.Example{ {Command: MaterialsReindexName, Description: "Sync every material into the existing collection"}, {Command: MaterialsReindexName + " --fresh", Description: "Recreate the collection from the schema, then sync every material"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { return reindexMaterials(ctx, deps, flagOn(in, "fresh"), flagOn(in, "if-configured"), out) }, } } func reindexMaterials(ctx context.Context, deps MaterialsReindexDeps, fresh, ifConfigured bool, out bonfire.Output) error { var svc *beachcomber.Service if deps.Index != nil && deps.Index.Service != nil { svc = deps.Index.Service() } engine := svc.Engine() if engine == nil || !engine.Configured() { if ifConfigured { out.Info(MaterialsReindexName + ": search is not configured; nothing to do") return nil } return errSearchNotConfigured } if deps.DB == nil { return errors.New("jz.bm: no database") } gdb, done, err := deps.DB(ctx) if err != nil { return err } defer done() index := svc.IndexName(&models.Material{}) if fresh { existed, err := beachcomber.DropIndex(ctx, engine, index) if err != nil { return fmt.Errorf("jz.bm: drop %s: %w", index, err) } if existed { out.Printf("Dropped %s\n", index) } } if err := beachcomber.EnsureIndex(ctx, engine, index, models.Material{}.SearchIndexSchema()); err != nil { return fmt.Errorf("jz.bm: create %s: %w", index, err) } marked := classes.MarkMaterialSearch(ctx) run := &reindexRun{ backfill: deps.Backfill, sync: func(id uint) error { return svc.Sync(marked, gdb, &models.Material{ID: id}) }, out: out, } var batch []models.Material err = gdb.WithContext(ctx).Model(&models.Material{}). Select("id"). Order("id"). FindInBatches(&batch, materialsReindexBatch, func(_ *gorm.DB, _ int) error { for i := range batch { run.material(ctx, gdb, batch[i].ID) } return ctx.Err() }).Error if err != nil { return err } msg := fmt.Sprintf("Reindexed: %d synced, %d failed", run.synced, run.failed) if run.failed > 0 { out.Error(msg) return fmt.Errorf("jz.bm: %d materials failed to sync", run.failed) } out.Success(msg) return nil } // pdfToolMissingWarning is printed once per run when pdftotext is missing. const pdfToolMissingWarning = "pdftotext not found: PDFs add no search text; install poppler-utils and rerun" // reindexRun is the state of one reindex walk: the run's counters and // whether pdftotext turned out to be missing. type reindexRun struct { backfill func(ctx context.Context, db *gorm.DB, materialID uint) error sync func(id uint) error out bonfire.Output synced int failed int noTool bool } // material backfills the file texts of material id, then syncs it. A // missing pdftotext warns once and stops the backfill for the rest of the // run; another backfill error warns for this material, which is still // synced. func (r *reindexRun) material(ctx context.Context, db *gorm.DB, id uint) { if r.backfill != nil && !r.noTool { err := r.backfill(ctx, db, id) switch { case errors.Is(err, classes.ErrPDFToolMissing): r.noTool = true r.out.Warning(pdfToolMissingWarning) case err != nil: r.out.Warning(fmt.Sprintf("material %d: file text: %v", id, err)) } } if err := r.sync(id); err != nil { r.failed++ r.out.Warning(fmt.Sprintf("material %d: %v", id, err)) return } r.synced++ } // flagOn reports whether a bare flag was given. func flagOn(in bonfire.Input, name string) bool { v, _ := in.Flag(name) return v == "true" }