refactor: 重构文档处理流程和任务管理

dao/document_chunk.go

@@ -2,12 +2,17 @@ package dao
 
 import (
 	"context"
+	"fmt"
 	"rag/consts/public"
 	"rag/model/dto"
 	"rag/model/entity"
 
 	"gitea.com/red-future/common/db/gfdb"
+	"gitea.com/red-future/common/full-text-search/meilisearch"
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/text/gstr"
 	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/pgvector/pgvector-go"
 )
 
 var DocumentChunk = new(documentChunkDao)
@@ -55,3 +60,56 @@ func (d *documentChunkDao) List(ctx context.Context, req *dto.ListDocumentChunkR
 	err = r.Structs(&res)
 	return
 }
+
+func (d *documentChunkDao) GetAllByVector(ctx context.Context, datasetId []int64, queryVec pgvector.Vector, topK int) (list gdb.List, err error) {
+	sql := `
+	SELECT id, content, dataset_id, document_id,
+	       vector <-> ? AS distance
+	FROM rag_vector_document_chunk
+	WHERE dataset_id IN (?)       
+	  AND vector IS NOT NULL
+	ORDER BY distance ASC
+	LIMIT ?
+`
+	// 顺序：vector, dataset_id, topK
+	result, err := gfdb.DB(ctx, public.DbNameVector).GetAll(ctx, sql, queryVec, datasetId, topK)
+	if err != nil {
+		return nil, err
+	}
+
+	return result.List(), nil
+}
+
+// SearchByKeywords 通过关键词全文检索文档块
+func (d *documentChunkDao) SearchByKeywords(ctx context.Context, query string, datasetIds []int64, topK int) (list gdb.List, err error) {
+	// 构建 meilisearch 查询参数
+	searchParams := &meilisearch.SearchParams{
+		Query: query,
+		Limit: int64(topK),
+	}
+
+	// 构建 datasetIds 过滤条件
+	if len(datasetIds) > 0 {
+		datasetIdStrs := gconv.Strings(datasetIds)
+		quotedIds := make([]string, len(datasetIdStrs))
+		for i, id := range datasetIdStrs {
+			quotedIds[i] = fmt.Sprintf("%s", id)
+		}
+		searchParams.Filter = fmt.Sprintf("dataset_id IN [%s]", gstr.Implode(", ", quotedIds))
+	}
+
+	// 执行搜索
+	var hits []map[string]interface{}
+	_, err = meilisearch.DB().Search(ctx, searchParams, public.IndexNameDocumentChunk, &hits)
+	if err != nil {
+		return nil, err
+	}
+
+	// 转换查询结果为 gdb.List
+	resultList := make(gdb.List, 0, len(hits))
+	for _, hit := range hits {
+		resultList = append(resultList, hit)
+	}
+
+	return resultList, nil
+}