package classes import ( "context" "errors" "fmt" "strconv" "time" "git.golem15.com/golem15/summercms/modules/lagoon/attach" "git.golem15.com/jakub/sm-bm-plugin/models" "gocloud.dev/blob" "gorm.io/gorm" "gorm.io/gorm/clause" ) // EnsureFileTexts extracts the text of every current public PDF of material // materialID that has no text row yet, and stores one row per file (an // empty text for a scan or a broken PDF, so it is not retried). It returns // how many rows it added. ErrPDFToolMissing stops the loop and is returned // as is, so callers can match it; a blob, extraction or insert error is // returned wrapped with the file id. It runs only in the extraction job and // bm:materials-reindex, never in a request. func EnsureFileTexts(ctx context.Context, db *gorm.DB, bucket *blob.Bucket, pdf PDFText, materialID uint) (int, error) { if db == nil { return 0, fmt.Errorf("jz.bm: no database for the file texts of material %d", materialID) } if bucket == nil { return 0, fmt.Errorf("jz.bm: no uploads bucket for the file texts of material %d", materialID) } db = db.WithContext(ctx) // The conditions of the API's materialFiles, which classes cannot import. var files []attach.File err := db.Model(&attach.File{}). Select("system_files.*"). Joins("LEFT JOIN jz_bm_material_file_texts t ON t.file_id = system_files.id"). Where("system_files.attachment_type = ? AND system_files.attachment_id = ? AND system_files.field = ?", models.Material{}.MorphName(), strconv.FormatUint(uint64(materialID), 10), models.MaterialFilesField). Where("system_files.is_public"). Where("(system_files.content_type = 'application/pdf' OR lower(system_files.file_name) LIKE '%.pdf')"). Where("t.file_id IS NULL"). Order("system_files.sort_order NULLS LAST, system_files.id"). Find(&files).Error if err != nil { return 0, err } added := 0 for _, f := range files { text, truncated, err := extractBlob(ctx, bucket, pdf, f) if errors.Is(err, ErrPDFToolMissing) { return added, err } if err != nil { return added, fmt.Errorf("jz.bm: file %d: %w", f.ID, err) } row := models.MaterialFileText{FileID: f.ID, MaterialID: materialID, Text: text, Truncated: truncated, ExtractedAt: time.Now().UTC()} res := db.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "file_id"}}, DoNothing: true}).Create(&row) if res.Error != nil { return added, fmt.Errorf("jz.bm: file %d: %w", f.ID, res.Error) } if res.RowsAffected > 0 { added++ } } return added, nil } // extractBlob runs pdf on the blob of f. func extractBlob(ctx context.Context, bucket *blob.Bucket, pdf PDFText, f attach.File) (string, bool, error) { rd, err := bucket.NewReader(ctx, attach.BlobKey(f.DiskName), nil) if err != nil { return "", false, err } defer rd.Close() return pdf.Extract(ctx, rd) }