412821ec56
- jz_bm_material_file_texts caches the pdftotext output per file - the River job jz.bm.extract_material_text extracts and resyncs - the management multipart attach queues it inside its transaction - file_texts in the document and a file snippet with its name
77 lines
2.8 KiB
Go
77 lines
2.8 KiB
Go
package classes
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
|
|
"git.golem15.com/jakub/sm-bm-plugin/models"
|
|
"gocloud.dev/blob"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
)
|
|
|
|
// EnsureFileTexts extracts the text of every current public PDF of material
|
|
// materialID that has no text row yet, and stores one row per file (an
|
|
// empty text for a scan or a broken PDF, so it is not retried). It returns
|
|
// how many rows it added. ErrPDFToolMissing stops the loop and is returned
|
|
// as is, so callers can match it; a blob, extraction or insert error is
|
|
// returned wrapped with the file id. It runs only in the extraction job and
|
|
// bm:materials-reindex, never in a request.
|
|
func EnsureFileTexts(ctx context.Context, db *gorm.DB, bucket *blob.Bucket, pdf PDFText, materialID uint) (int, error) {
|
|
if db == nil {
|
|
return 0, fmt.Errorf("jz.bm: no database for the file texts of material %d", materialID)
|
|
}
|
|
if bucket == nil {
|
|
return 0, fmt.Errorf("jz.bm: no uploads bucket for the file texts of material %d", materialID)
|
|
}
|
|
db = db.WithContext(ctx)
|
|
// The conditions of the API's materialFiles, which classes cannot import.
|
|
var files []attach.File
|
|
err := db.Model(&attach.File{}).
|
|
Select("system_files.*").
|
|
Joins("LEFT JOIN jz_bm_material_file_texts t ON t.file_id = system_files.id").
|
|
Where("system_files.attachment_type = ? AND system_files.attachment_id = ? AND system_files.field = ?",
|
|
models.Material{}.MorphName(), strconv.FormatUint(uint64(materialID), 10), models.MaterialFilesField).
|
|
Where("system_files.is_public").
|
|
Where("(system_files.content_type = 'application/pdf' OR lower(system_files.file_name) LIKE '%.pdf')").
|
|
Where("t.file_id IS NULL").
|
|
Order("system_files.sort_order NULLS LAST, system_files.id").
|
|
Find(&files).Error
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
added := 0
|
|
for _, f := range files {
|
|
text, truncated, err := extractBlob(ctx, bucket, pdf, f)
|
|
if errors.Is(err, ErrPDFToolMissing) {
|
|
return added, err
|
|
}
|
|
if err != nil {
|
|
return added, fmt.Errorf("jz.bm: file %d: %w", f.ID, err)
|
|
}
|
|
row := models.MaterialFileText{FileID: f.ID, MaterialID: materialID, Text: text, Truncated: truncated, ExtractedAt: time.Now().UTC()}
|
|
res := db.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "file_id"}}, DoNothing: true}).Create(&row)
|
|
if res.Error != nil {
|
|
return added, fmt.Errorf("jz.bm: file %d: %w", f.ID, res.Error)
|
|
}
|
|
if res.RowsAffected > 0 {
|
|
added++
|
|
}
|
|
}
|
|
return added, nil
|
|
}
|
|
|
|
// extractBlob runs pdf on the blob of f.
|
|
func extractBlob(ctx context.Context, bucket *blob.Bucket, pdf PDFText, f attach.File) (string, bool, error) {
|
|
rd, err := bucket.NewReader(ctx, attach.BlobKey(f.DiskName), nil)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
defer rd.Close()
|
|
return pdf.Extract(ctx, rd)
|
|
}
|