feat: index the text of attached PDFs

- jz_bm_material_file_texts caches the pdftotext output per file
- the River job jz.bm.extract_material_text extracts and resyncs
- the management multipart attach queues it inside its transaction
- file_texts in the document and a file snippet with its name
This commit is contained in:
Jakub Zych
2026-11-08 20:11:19 +01:00
parent dd13795e6f
commit 412821ec56
21 changed files with 747 additions and 24 deletions

View File

@@ -0,0 +1,34 @@
package updates
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)
// CreateMaterialFileTexts returns the
// 20261108130000_create_jz_bm_material_file_texts gormigrate entry: the text
// pdftotext extracted from each PDF attached to a material, cached per file
// so the search document can carry it without running the tool again. An
// empty text means the file was extracted and held none (a scan or a broken
// PDF), so it is not retried. The foreign key to system_files cascades, so
// detaching a file removes its text; deleting a material removes the texts
// of its files.
func CreateMaterialFileTexts() *gormigrate.Migration {
return &gormigrate.Migration{
ID: "20261108130000_create_jz_bm_material_file_texts",
Migrate: func(tx *gorm.DB) error {
return execAll(tx, `
CREATE TABLE jz_bm_material_file_texts (
file_id INTEGER PRIMARY KEY REFERENCES system_files (id) ON DELETE CASCADE,
material_id BIGINT NOT NULL REFERENCES jz_bm_materials (id) ON DELETE CASCADE,
text TEXT NOT NULL DEFAULT '',
truncated BOOLEAN NOT NULL DEFAULT FALSE,
extracted_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL
);
CREATE INDEX jz_bm_material_file_texts_material_id_index ON jz_bm_material_file_texts (material_id)`)
},
Rollback: func(tx *gorm.DB) error {
return execAll(tx, `DROP TABLE IF EXISTS jz_bm_material_file_texts`)
},
}
}