red-future/rag
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: rag初始版

Browse Source
This commit is contained in:
曲皓东
2026-04-03 09:16:53 +08:00
commit 6f5c80da16
38 changed files with 3840 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,10 @@
# 默认忽略的文件
/shelf/
/workspace.xml
# 已忽略包含查询文件的默认文件夹
/queries/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
# 基于编辑器的 HTTP 客户端请求
/httpRequests/

8
.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/rag.iml" filepath="$PROJECT_DIR$/.idea/rag.iml" />
</modules>
</component>
</project>

9
.idea/rag.iml generated Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

122
config.yml Normal file
View File

@@ -0,0 +1,122 @@
server:
address: :3006
name: rag
workerId: 1
# Database.
database:
default:
- type: "pgsql"
host: "116.204.74.41"
port: "15432"
user: "postgres"
pass: "Bjang09@686^*^"
name: "rag"
role: "master" # (可选)数据库主从角色(master/slave)，默认为master。如果不使用应用主从机制请不配置或留空即可。
debug: false # (可选)开启调试模式
dryRun: false # (可选)ORM空跑(只读不写)
charset: "utf8" # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
timezone: "Asia/Shanghai" # (可选)时区配置，例如:Local
maxIdle: 5 # (可选)连接池最大闲置的连接数(默认10)
maxOpen: 20 # (可选)连接池最大打开的连接数(默认无限制)
maxLifetime: "30s" # (可选)连接对象可重复使用的时间长度(默认30秒)
maxIdleConnTime: "30s" # (可选，v2.10新增)连接池中空闲连接的最大生存时间(默认30秒)。可以通过配置文件或SetConnMaxIdleTime方法设置，避免长时间空闲连接占用资源。
createdAt: "created_at" # (可选)自动创建时间字段名称
updatedAt: "updated_at" # (可选)自动更新时间字段名称
deletedAt: "deleted_at" # (可选)软删除时间字段名称
timeMaintainDisabled: false # (可选)是否完全关闭时间更新特性，为true时CreatedAt/UpdatedAt/DeletedAt都将失效
- type: "pgsql"
host: "116.204.74.41"
port: "15432"
user: "postgres"
pass: "Bjang09@686^*^"
name: "rag"
role: "slave" # (可选)数据库主从角色(master/slave)，默认为master。如果不使用应用主从机制请不配置或留空即可。
debug: false # (可选)开启调试模式
dryRun: false # (可选)ORM空跑(只读不写)
charset: "utf8" # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
timezone: "Asia/Shanghai" # (可选)时区配置，例如:Local
maxIdle: 5 # (可选)连接池最大闲置的连接数(默认10)
maxOpen: 20 # (可选)连接池最大打开的连接数(默认无限制)
maxLifetime: "30s" # (可选)连接对象可重复使用的时间长度(默认30秒)
maxIdleConnTime: "30s" # (可选，v2.10新增)连接池中空闲连接的最大生存时间(默认30秒)。可以通过配置文件或SetConnMaxIdleTime方法设置，避免长时间空闲连接占用资源。
createdAt: "created_at" # (可选)自动创建时间字段名称
updatedAt: "updated_at" # (可选)自动更新时间字段名称
deletedAt: "deleted_at" # (可选)软删除时间字段名称
timeMaintainDisabled: false # (可选)是否完全关闭时间更新特性，为true时CreatedAt/UpdatedAt/DeletedAt都将失效
tenant-1:
- type: "pgsql"
host: "localhost"
port: "5432"
user: "postgres"
pass: "123456"
name: "tenant"
role: "master"
prefix: "rag_" # (可选)表名前缀
debug: true # (可选)开启调试模式
dryRun: false # (可选)ORM空跑(只读不写)
charset: "utf8" # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
timezone: "Asia/Shanghai" # (可选)时区配置，例如:Local
maxIdle: 5 # (可选)连接池最大闲置的连接数(默认10)
maxOpen: 20 # (可选)连接池最大打开的连接数(默认无限制)
maxLifetime: "30s" # (可选)连接对象可重复使用的时间长度(默认30秒)
maxIdleConnTime: "30s" # (可选，v2.10新增)连接池中空闲连接的最大生存时间(默认30秒)。可以通过配置文件或SetConnMaxIdleTime方法设置，避免长时间空闲连接占用资源。
createdAt: "created_at" # (可选)自动创建时间字段名称
updatedAt: "updated_at" # (可选)自动更新时间字段名称
deletedAt: "deleted_at" # (可选)软删除时间字段名称
timeMaintainDisabled: false # (可选)是否完全关闭时间更新特性，为true时CreatedAt/UpdatedAt/DeletedAt都将失效
redis:
default:
address: "localhost:6379"
db: 0
consul:
address: localhost:8500
jaeger:
addr: localhost:4318
# eino框架配置
eino:
# 文件切分配置
splitter:
bufferSize: 1
minChunkSize: 64
percentile: 0.75
# 向量化配置
embedding:
provider: "dashscope"
# apiKey: "d158d896-8c54-40ee-9d61-4c5d37cd545c"
# model: "ep-20260326123502-khmdq"
# apiType: "multi_modal_api"
apiKey: "sk-4a8b82770bf74bc490eb3e4c5a8e2be9"
model: "text-embedding-v3"
# 文件上传服务地址，与oss模块minio中的endpoint一致
filePrefix: "http://116.204.74.41:9000"
gmq:
redis:
primary:
addr: "localhost"
port: "6379"
db: 0
username: ""
password: ""
poolSize: 10
minIdleConn: 5
maxActiveConn: 10
maxRetries: 30
# Meilisearch 全文检索配置
meilisearch:
default:
host: "http://localhost"
port: 7700
apiKey: "admin"
# apiKey: "6b8b6062bcb5e31f150427961d9da1a9e81758aa"
cache:
localTTL: 60
redisTTL: 300

26
consts/document/status.go Normal file
View File

@@ -0,0 +1,26 @@
package document
import "github.com/gogf/gf/v2/util/gconv"
var (
StatusDisable = newStatus(gconv.PtrInt8(0), "disable")
StatusEnable = newStatus(gconv.PtrInt8(1), "enable")
)
type Status *int8
type status struct {
code Status
desc string
}
func (s status) Code() Status {
return s.code
}
func (s status) Desc() string {
return s.desc
}
func newStatus(code Status, desc string) status {
return status{code: code, desc: desc}
}

28
consts/document/vector_status.go Normal file
View File

@@ -0,0 +1,28 @@
package document
import "github.com/gogf/gf/v2/util/gconv"
var (
VectorStatusPending = newVectorStatus(gconv.PtrInt8(1), "pending")
VectorStatusProcessing = newVectorStatus(gconv.PtrInt8(2), "processing")
VectorStatusCompleted = newVectorStatus(gconv.PtrInt8(3), "completed")
VectorStatusFailed = newVectorStatus(gconv.PtrInt8(4), "failed")
)
type VectorStatus *int8
type vectorStatus struct {
code VectorStatus
desc string
}
func (s vectorStatus) Code() VectorStatus {
return s.code
}
func (s vectorStatus) Desc() string {
return s.desc
}
func newVectorStatus(code VectorStatus, desc string) vectorStatus {
return vectorStatus{code: code, desc: desc}
}

20
consts/public/redis_key.go Normal file
View File

@@ -0,0 +1,20 @@
package public
const KnowledgeLockEsKey = "rag:knowledge:lock:knowledgeIdEs-%v"
const KnowledgeLockSqlKey = "rag:knowledge:lock:knowledgeIdSql-%v"
const KnowledgeContentHashEsKey = "rag:knowledge:knowledgeId:contentHashEs-%v"
const KnowledgeContentHashSqlKey = "rag:knowledge:knowledgeId:contentHashSql-%v"
const (
KnowledgeDocumentVectorStatusTopic = "knowledge:document:vector:status:stream"
KnowledgeDocumentVectorStatusConsumer = "knowledge-document-vector-status-consumer"
KnowledgeDocumentVectorStatusBatchSize = 1
KnowledgeDocumentVectorStatusAutoAck = false
)
const (
KnowledgeDocumentChunkTopic = "knowledge:document:chunk:stream" // 请求 Stream 键名（与发消息的key一致）
KnowledgeDocumentChunkConsumer = "knowledge-document-chunk-consumer" // 消费者名称（唯一标识）
KnowledgeDocumentChunkBatchSize = 1 // 批处理大小（每次读取1条）
KnowledgeDocumentChunkAutoAck = false // ACK是否自动确认（true自动确认，false不确认）
)

15
consts/public/table_name.go Normal file
View File

@@ -0,0 +1,15 @@
package public
// sql 数据库表名
const (
TableNameDocument = "document"
TableNameDataset = "dataset"
TableNameKeyword = "keyword"
TableNameDatasetIndex = "dataset_index"
TableNameDocumentChunk = "document_chunk"
)
// es 索引名称
const (
IndexNameDocumentChunk = "document_chunk" // 文档分块索引
)

48
controller/dataset.go Normal file
View File

@@ -0,0 +1,48 @@
package controller
import (
"context"
"rag/model/dto"
"rag/service"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
)
type dataset struct{}
var Dataset = new(dataset)
// Create 创建数据集
func (c *dataset) Create(ctx context.Context, req *dto.CreateDatasetReq) (res *dto.CreateDatasetRes, err error) {
res, err = service.Dataset.Create(ctx, req)
return
}
// Update 更新数据集
func (c *dataset) Update(ctx context.Context, req *dto.UpdateDatasetReq) (res *beans.ResponseEmpty, err error) {
err = service.Dataset.Update(ctx, req)
return
}
// Delete 删除数据集
func (c *dataset) Delete(ctx context.Context, req *dto.DeleteDatasetReq) (res *beans.ResponseEmpty, err error) {
err = service.Dataset.Delete(ctx, req)
return
}
// List 数据集列表
func (c *dataset) List(ctx context.Context, req *dto.ListDatasetReq) (res *dto.ListDatasetRes, err error) {
if !g.IsEmpty(req.Page) {
req.Page = &beans.Page{PageNum: 1, PageSize: 20}
}
res, err = service.Dataset.List(ctx, req)
return
}
// Search 搜索
//func (c *dataset) Search(ctx context.Context, req *dto.SearchReq) (res *dto.SearchRes, err error) {
// res, err = service.Dataset.Search(ctx, req)
// return
//}

5
controller/dataset_index.go Normal file
View File

@@ -0,0 +1,5 @@
package controller
type datasetIndex struct{}
var DatasetIndex = new(datasetIndex)

54
controller/document.go Normal file
View File

@@ -0,0 +1,54 @@
package controller
import (
"context"
"rag/model/dto"
"rag/service"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
)
type document struct{}
var Document = new(document)
// Create 创建文件
func (c *document) Create(ctx context.Context, req *dto.CreateDocumentReq) (res *dto.CreateDocumentRes, err error) {
res, err = service.Document.Create(ctx, req)
return
}
// Update 更新文件
func (c *document) Update(ctx context.Context, req *dto.UpdateDocumentReq) (res *beans.ResponseEmpty, err error) {
err = service.Document.Update(ctx, req)
return
}
// Delete 删除文件
func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (res *beans.ResponseEmpty, err error) {
err = service.Document.Delete(ctx, req)
return
}
// Get 获取文件详情
func (c *document) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.DocumentVO, err error) {
res, err = service.Document.Get(ctx, req)
return
}
// List 文件列表
func (c *document) List(ctx context.Context, req *dto.ListDocumentReq) (res *dto.ListDocumentRes, err error) {
if !g.IsEmpty(req.Page) {
req.Page = &beans.Page{PageNum: 1, PageSize: 20}
}
res, err = service.Document.List(ctx, req)
return
}
// Process 处理文件(向量化)
func (c *document) Process(ctx context.Context, req *dto.ProcessDocumentReq) (res *dto.ProcessDocumentRes, err error) {
res, err = service.Document.Process(ctx, req)
return
}

29
controller/document_chunk.go Normal file
View File

@@ -0,0 +1,29 @@
package controller
import (
"context"
"rag/model/dto"
"rag/service"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
)
type documentChunk struct{}
var DocumentChunk = new(documentChunk)
// Update 更新文件片段
func (c *documentChunk) Update(ctx context.Context, req *dto.UpdateDocumentChunkReq) (res *beans.ResponseEmpty, err error) {
err = service.DocumentChunk.Update(ctx, req)
return
}
// List 文件片段列表
func (c *documentChunk) List(ctx context.Context, req *dto.ListDocumentChunkReq) (res *dto.ListDocumentChunkRes, err error) {
if !g.IsEmpty(req.Page) {
req.Page = &beans.Page{PageNum: 1, PageSize: 20}
}
res, err = service.DocumentChunk.List(ctx, req)
return
}

43
controller/keyword.go Normal file
View File

@@ -0,0 +1,43 @@
package controller
import (
"context"
"rag/model/dto"
"rag/service"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
)
type keyword struct{}
var Keyword = new(keyword)
func (c *keyword) Create(ctx context.Context, req *dto.CreateKeywordReq) (res *dto.CreateKeywordRes, err error) {
res, err = service.Keyword.Create(ctx, req)
return
}
func (c *keyword) Update(ctx context.Context, req *dto.UpdateKeywordReq) (res *beans.ResponseEmpty, err error) {
err = service.Keyword.Update(ctx, req)
return
}
func (c *keyword) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (res *beans.ResponseEmpty, err error) {
err = service.Keyword.Delete(ctx, req)
return
}
func (c *keyword) Get(ctx context.Context, req *dto.GetKeywordReq) (res *dto.KeywordVO, err error) {
res, err = service.Keyword.Get(ctx, req)
return
}
func (c *keyword) List(ctx context.Context, req *dto.ListKeywordReq) (res *dto.ListKeywordRes, err error) {
if !g.IsEmpty(req.Page) {
req.Page = &beans.Page{PageNum: 1, PageSize: 20}
}
res, err = service.Keyword.List(ctx, req)
return
}

88
dao/dataset.go Normal file
View File

@@ -0,0 +1,88 @@
package dao
import (
"context"
"rag/consts/public"
"rag/model/dto"
"rag/model/entity"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
var Dataset = new(datasetDao)
type datasetDao struct{}
// Insert 插入数据集
func (d *datasetDao) Insert(ctx context.Context, req *dto.CreateDatasetReq) (id int64, err error) {
var res *entity.Dataset
if err = gconv.Struct(req, &res); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Data(&res).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// Update 更新数据集
func (d *datasetDao) Update(ctx context.Context, req *dto.UpdateDatasetReq) (rows int64, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).OmitEmpty()
if !g.IsEmpty(req.DocumentCount) {
model.Data(entity.DatasetCol.DocumentCount, &gdb.Counter{
Field: entity.DatasetCol.DocumentCount,
Value: gconv.Float64(req.DocumentCount),
})
}
if !g.IsEmpty(req.DocumentSize) {
model.Data(entity.DatasetCol.DocumentSize, &gdb.Counter{
Field: entity.DatasetCol.DocumentSize,
Value: gconv.Float64(req.DocumentSize),
})
}
r, err := model.Data(&req).Where(entity.DatasetCol.Id, req.Id).Update()
if err != nil {
return
}
return r.RowsAffected()
}
// Delete 删除数据集
func (d *datasetDao) Delete(ctx context.Context, req *dto.DeleteDatasetReq) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Where(entity.DatasetCol.Id, req.Id).Delete()
if err != nil {
return
}
return r.RowsAffected()
}
func (d *datasetDao) GetByID(ctx context.Context, req *dto.GetDatasetReq, fields ...string) (res *entity.Dataset, err error) {
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Where(entity.DatasetCol.Id, req.Id).Fields(fields).One()
if err != nil {
return
}
err = r.Struct(&res)
return
}
// List 获取数据集列表
func (d *datasetDao) List(ctx context.Context, req *dto.ListDatasetReq, fields ...string) (res []*entity.Dataset, total int, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Fields(fields).OmitEmpty()
if !g.IsEmpty(req.Keyword) {
model.WhereLike(entity.DatasetCol.Name, "%"+req.Keyword+"%")
}
model.OrderDesc(entity.DatasetCol.CreatedAt)
if req.Page != nil {
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
}
r, total, err := model.AllAndCount(false)
if err != nil {
return
}
err = r.Structs(&res)
return
}

59
dao/dataset_index.go Normal file
View File

@@ -0,0 +1,59 @@
package dao
import (
"context"
"database/sql"
"fmt"
"rag/consts/public"
"rag/model/entity"
"gitea.com/red-future/common/db/gfdb"
)
var DatasetIndex = new(datasetIndexDao)
type datasetIndexDao struct{}
// Insert 插入数据集索引
func (d *datasetIndexDao) Insert(ctx context.Context, index *entity.DatasetIndex) (id int64, err error) {
_, err = gfdb.DB(ctx).Model(ctx, public.TableNameDatasetIndex).Data(index).Insert()
if err != nil {
return
}
return 0, nil
}
// GetByDatasetId 根据数据集ID获取索引
func (d *datasetIndexDao) GetByDatasetId(ctx context.Context, datasetId int64) (result *entity.DatasetIndex, err error) {
err = gfdb.DB(ctx).Model(ctx, public.TableNameDatasetIndex).Where(entity.DatasetIndexCol.DatasetId, datasetId).Scan(&result)
if err != nil {
if err == sql.ErrNoRows {
return nil, nil
}
return
}
return result, nil
}
// IncVectorCount 增加或减少向量数量
func (d *datasetIndexDao) IncVectorCount(ctx context.Context, id int64, delta int64) (err error) {
_, err = gfdb.DB(ctx).Model(ctx, public.TableNameDatasetIndex).
Where(entity.DatasetIndexCol.Id, id).
Increment(entity.DatasetIndexCol.VectorCount, delta)
return
}
func (d *datasetIndexDao) InsertIndex(ctx context.Context, indexName string) (err error) {
prefix, err := gfdb.GetTablePrefix(ctx)
if err != nil {
return
}
sqlStr := fmt.Sprintf(`
CREATE INDEX IF NOT EXISTS %s
ON %s
USING ivfflat (vector vector_cosine_ops)
WHERE vector IS NOT NULL;
`, indexName, prefix+public.TableNameDocumentChunk)
_, err = gfdb.DB(ctx).Exec(ctx, sqlStr)
return
}

87
dao/document.go Normal file
View File

@@ -0,0 +1,87 @@
package dao
import (
"context"
"rag/consts/public"
"rag/model/dto"
"rag/model/entity"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
var Document = new(documentDao)
type documentDao struct{}
// Insert 插入文件
func (d *documentDao) Insert(ctx context.Context, req *dto.CreateDocumentReq) (id int64, err error) {
var res *entity.Document
if err = gconv.Struct(req, &res); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).Data(&res).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
// Update 更新文件
func (d *documentDao) Update(ctx context.Context, req *dto.UpdateDocumentReq) (rows int64, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).OmitEmpty()
if !g.IsEmpty(req.ChunkCount) {
model.Data(entity.DocumentCol.ChunkCount, &gdb.Counter{
Field: entity.DocumentCol.ChunkCount,
Value: gconv.Float64(req.ChunkCount),
})
}
r, err := model.Data(&req).Where(entity.DocumentCol.Id, req.Id).Update()
if err != nil {
return
}
return r.RowsAffected()
}
// Delete 删除文件
func (d *documentDao) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Delete()
if err != nil {
return
}
return r.RowsAffected()
}
// GetByID 根据ID获取文件
func (d *documentDao) GetByID(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) {
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Fields(fields).One()
if err != nil {
return
}
err = r.Struct(&res)
return
}
// List 获取文件列表
func (d *documentDao) List(ctx context.Context, req *dto.ListDocumentReq, fields ...string) (res []*entity.Document, total int, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).OmitEmpty()
if !g.IsEmpty(req.Keyword) {
model.WhereLike(entity.DocumentCol.Title, "%"+req.Keyword+"%")
}
model.Where(entity.DocumentCol.DatasetId, req.DatasetId)
model.Where(entity.DocumentCol.Status, req.Status)
model.Fields(fields)
model.OrderDesc(entity.DocumentCol.CreatedAt)
if req.Page != nil {
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
}
r, total, err := model.AllAndCount(false)
if err != nil {
return
}
err = r.Structs(&res)
return
}

104
dao/document_chunk.go Normal file
View File

@@ -0,0 +1,104 @@
package dao
import (
"context"
"rag/consts/public"
"rag/model/dto"
"rag/model/entity"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/util/gconv"
)
var DocumentChunk = new(documentChunkDao)
type documentChunkDao struct{}
// BatchInsert 批量插入文件块
func (d *documentChunkDao) BatchInsert(ctx context.Context, req []*dto.VectorDocumentChunkMsg) (rows int64, err error) {
var res []*entity.DocumentChunk
if err = gconv.Structs(req, &res); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocumentChunk).Data(&res).Insert()
if err != nil {
return
}
return r.RowsAffected()
}
// Update 更新文件块
func (d *documentChunkDao) Update(ctx context.Context, req *dto.UpdateDocumentChunkReq) (rows int64, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameDocumentChunk)
r, err := model.Data(&req).Where(entity.DocumentChunkCol.Id, req.Id).Update()
if err != nil {
return
}
return r.RowsAffected()
}
// List 文件块列表
func (d *documentChunkDao) List(ctx context.Context, req *dto.ListDocumentChunkReq, fields ...string) (res []*entity.DocumentChunk, total int, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameDocumentChunk).Fields(fields).OmitEmpty().
Where(entity.DocumentChunkCol.DatasetId, req.DatasetId).
Where(entity.DocumentChunkCol.DocumentId, req.DocumentId).
Where(entity.DocumentChunkCol.Status, req.Status).
Where(entity.DocumentChunkCol.VectorStatus, req.VectorStatus).
OrderDesc(entity.DocumentChunkCol.CreatedAt)
if req.Page != nil {
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
}
r, total, err := model.AllAndCount(false)
if err != nil {
return
}
err = r.Structs(&res)
return
}
//// Insert 插入向量文档
//func (d *vectorDocumentDao) Insert(ctx context.Context, docs []*entity.DocumentChunk) (ids []interface{}, err error) {
// if len(docs) == 0 {
// return
// }
// interfaces := make([]interface{}, len(docs))
// for i := range docs {
// interfaces[i] = docs[i]
// }
// return mongoDB.Insert(ctx, interfaces, CollectionVectorDoc)
//}
//
//// DeleteByIDs 根据ID删除向量文档
//func (d *vectorDocumentDao) DeleteByIDs(ctx context.Context, ids []string) (err error) {
// if len(ids) == 0 {
// return
// }
// objectIDs := make([]bson.ObjectID, len(ids))
// for i, id := range ids {
// objectIDs[i], err = bson.ObjectIDFromHex(id)
// if err != nil {
// return err
// }
// }
// filter := bson.M{"_id": bson.M{"$in": objectIDs}}
// _, err = mongoDB.Delete(ctx, filter, CollectionVectorDoc)
// return
//}
//
//// GetByIndexID 根据索引ID获取向量文档
//func (d *vectorDocumentDao) GetByIndexID(ctx context.Context, indexID string, limit int) (result []*entity.DocumentChunk, err error) {
// filter := bson.M{"indexId": indexID}
// page := &beans.Page{PageNum: 1, PageSize: int64(limit)}
// _, err = mongoDB.Find(ctx, filter, &result, CollectionVectorDoc, page, nil)
// return
//}
//
//// GetByVectorIDs 根据向量ID获取向量文档
//func (d *vectorDocumentDao) GetByVectorIDs(ctx context.Context, vectorIDs []string) (result []*entity.DocumentChunk, err error) {
// if len(vectorIDs) == 0 {
// return
// }
// filter := bson.M{"vectorId": bson.M{"$in": vectorIDs}}
// _, err = mongoDB.Find(ctx, filter, &result, CollectionVectorDoc, &beans.Page{PageSize: -1}, nil)
// return
//}

96
dao/keyword.go Normal file
View File

@@ -0,0 +1,96 @@
package dao
import (
"context"
"rag/consts/public"
"rag/model/dto"
"rag/model/entity"
"gitea.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
var Keyword = new(keywordDao)
type keywordDao struct{}
func (d *keywordDao) Insert(ctx context.Context, req *dto.CreateKeywordReq) (id int64, err error) {
var res *entity.Keyword
if err = gconv.Struct(req, &res); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Data(&res).Insert()
if err != nil {
return
}
return r.LastInsertId()
}
func (d *keywordDao) BatchSaveOrUpdate(ctx context.Context, req []*dto.CreateKeywordReq) (rows int64, err error) {
var res []*entity.Keyword
if err = gconv.Structs(req, &res); err != nil {
return
}
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Data(&res).OnConflict(
entity.KeywordCol.TenantId,
entity.KeywordCol.DatasetId,
entity.KeywordCol.DocumentId,
entity.KeywordCol.Word).Save()
if err != nil {
return
}
return r.RowsAffected()
}
func (d *keywordDao) Update(ctx context.Context, req *dto.UpdateKeywordReq) (rows int64, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword)
r, err := model.Data(&req).Where(entity.KeywordCol.Id, req.Id).Update()
if err != nil {
return
}
return r.RowsAffected()
}
func (d *keywordDao) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Delete()
if err != nil {
return
}
return r.RowsAffected()
}
func (d *keywordDao) Count(ctx context.Context, req *dto.ListKeywordReq) (count int, err error) {
count, err = gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).OmitEmpty().
Where(entity.KeywordCol.DatasetId, req.DatasetId).
Where(entity.KeywordCol.DocumentId, req.DocumentId).
Where(entity.KeywordCol.Word, req.Word).Count()
return
}
func (d *keywordDao) GetByID(ctx context.Context, req *dto.GetKeywordReq, fields ...string) (res *entity.Document, err error) {
r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Fields(fields).One()
if err != nil {
return
}
err = r.Struct(&res)
return
}
func (d *keywordDao) List(ctx context.Context, req *dto.ListKeywordReq, fields ...string) (res []*entity.Keyword, total int, err error) {
model := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Fields(fields).OmitEmpty()
if !g.IsEmpty(req.Keyword) {
model.WhereLike(entity.KeywordCol.Word, "%"+req.Keyword+"%")
}
model.OrderDesc(entity.KeywordCol.Weight)
model.OrderDesc(entity.KeywordCol.CreatedAt)
if req.Page != nil {
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
}
r, total, err := model.AllAndCount(false)
if err != nil {
return
}
err = r.Structs(&res)
return
}

167
go.mod Normal file
View File

@@ -0,0 +1,167 @@
module rag
go 1.26.0
require (
gitea.com/red-future/common v0.0.6
github.com/bjang03/gmq v0.0.0-00010101000000-000000000000
github.com/cloudwego/eino v0.8.6
github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0
github.com/gogf/gf/v2 v2.10.0
github.com/pgvector/pgvector-go v0.3.0
)
replace gitea.com/red-future/common v0.0.6 => ../common
replace github.com/bjang03/gmq => ../gmq
require (
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/PuerkitoBio/goquery v1.8.1 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.15.0 // indirect
github.com/bytedance/sonic/loader v0.5.0 // indirect
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/clbanning/mxj/v2 v2.7.0 // indirect
github.com/clipperhouse/displaywidth v0.11.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20241224063832-9fbcc0e56c28 // indirect
github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/document/parser/xlsx v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1 // indirect
github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419 // indirect
github.com/cloudwego/eino-ext/components/indexer/es8 v0.0.0-20260331071634-4f359694d2d9 // indirect
github.com/cloudwego/eino-ext/components/retriever/es8 v0.0.0-20260331071634-4f359694d2d9 // indirect
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.14 // indirect
github.com/dgraph-io/badger/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dslipak/pdf v0.0.2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/eino-contrib/docx2md v0.0.1 // indirect
github.com/eino-contrib/jsonschema v1.0.3 // indirect
github.com/elastic/elastic-transport-go/v8 v8.10.0 // indirect
github.com/elastic/go-elasticsearch/v8 v8.16.0 // indirect
github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect
github.com/evanphx/json-patch v0.5.2 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/go-ego/gse v1.0.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/goccy/go-json v0.10.6 // indirect
github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.1 // indirect
github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect
github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang/glog v1.2.5 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/goph/emperror v0.17.2 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grokify/html-strip-tags-go v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/hashicorp/consul/api v1.26.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.5.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.18.4 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lib/pq v1.12.1 // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.21 // indirect
github.com/meguminnnnnnnnn/go-openai v0.1.1 // indirect
github.com/meilisearch/meilisearch-go v0.36.1 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/nats-io/nats.go v1.49.0 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/nikolalohinski/gonja v1.5.3 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
github.com/olekukonko/errors v1.2.0 // indirect
github.com/olekukonko/ll v0.1.8 // indirect
github.com/olekukonko/tablewriter v1.1.4 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/redis/go-redis/v9 v9.18.0 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.4 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/tiger1103/gfast-token v1.0.10 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/vcaesar/cedar v0.30.0 // indirect
github.com/volcengine/volc-sdk-golang v1.0.199 // indirect
github.com/volcengine/volcengine-go-sdk v1.0.181 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect
github.com/xuri/excelize/v2 v2.9.0 // indirect
github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
github.com/yargevad/filepathx v1.0.0 // indirect
go.mongodb.org/mongo-driver/v2 v2.4.0 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.42.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.42.0 // indirect
go.opentelemetry.io/otel/sdk v1.42.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/arch v0.15.0 // indirect
golang.org/x/crypto v0.49.0 // indirect
golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect
golang.org/x/net v0.52.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/text v0.35.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

1239
go.sum Normal file
View File

File diff suppressed because it is too large Load Diff

64
main.go Normal file
View File

@@ -0,0 +1,64 @@
package main
import (
"context"
"os"
"os/signal"
"rag/consts/public"
"rag/controller"
"rag/service"
"syscall"
"gitea.com/red-future/common/http"
"gitea.com/red-future/common/jaeger"
gmq "github.com/bjang03/gmq/core/gmq"
"github.com/bjang03/gmq/mq"
"github.com/bjang03/gmq/types"
_ "github.com/gogf/gf/contrib/drivers/pgsql/v2"
"github.com/gogf/gf/v2/frame/g"
)
func main() {
ctx := context.Background()
defer jaeger.ShutDown(ctx)
// 注册路由
http.RouteRegister([]interface{}{
controller.Dataset,
controller.Document,
controller.DocumentChunk,
})
gmq.Init("config.yml")
if err := gmq.GetGmq("primary").GmqSubscribe(ctx, &mq.RedisSubMessage{
SubMessage: types.SubMessage{
Topic: public.KnowledgeDocumentVectorStatusTopic,
ConsumerName: public.KnowledgeDocumentVectorStatusConsumer,
AutoAck: public.KnowledgeDocumentVectorStatusAutoAck,
FetchCount: public.KnowledgeDocumentVectorStatusBatchSize,
HandleFunc: service.Document.DocsVectorStatusMsg,
},
}); err != nil {
return
}
if err := gmq.GetGmq("primary").GmqSubscribe(ctx, &mq.RedisSubMessage{
SubMessage: types.SubMessage{
Topic: public.KnowledgeDocumentChunkTopic,
ConsumerName: public.KnowledgeDocumentChunkConsumer,
AutoAck: public.KnowledgeDocumentChunkAutoAck,
FetchCount: public.KnowledgeDocumentChunkBatchSize,
HandleFunc: service.DocumentChunk.DocsChunkMsg,
},
}); err != nil {
return
}
// 等待退出信号
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
<-quit
g.Log().Info(ctx, "服务正在关闭...")
}

69
model/dto/dataset.go Normal file
View File

@@ -0,0 +1,69 @@
package dto
import (
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
// CreateDatasetReq 创建数据集请求
type CreateDatasetReq struct {
g.Meta `path:"/createDataset" method:"post" tags:"知识库（数据集）管理" summary:"创建知识库（数据集）" dc:"创建知识库（数据集）"`
Name string `json:"name" v:"required#名称不能为空"`
Description string `json:"description"`
}
// CreateDatasetRes 创建数据集响应
type CreateDatasetRes struct {
Id int64 `json:"id,string"`
}
// UpdateDatasetReq 更新数据集请求
type UpdateDatasetReq struct {
g.Meta `path:"/updateDataset" method:"put" tags:"知识库（数据集）管理" summary:"更新知识库（数据集）" dc:"更新知识库（数据集）"`
Id int64 `json:"id" v:"required#ID不能为空"`
Name string `json:"name"`
Description string `json:"description"`
DocumentCount int64 `json:"documentCount"`
DocumentSize int64 `json:"documentSize"`
}
// DeleteDatasetReq 删除数据集请求
type DeleteDatasetReq struct {
g.Meta `path:"/deleteDataset" method:"delete" tags:"知识库（数据集）管理" summary:"删除知识库（数据集）" dc:"删除知识库（数据集）"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// GetDatasetReq 获取数据集请求
type GetDatasetReq struct {
g.Meta `path:"/getDataset" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）详情" dc:"获取知识库（数据集）详情"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// ListDatasetReq 数据集列表请求
type ListDatasetReq struct {
g.Meta `path:"/listDataset" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）列表" dc:"分页查询知识库（数据集）列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
Keyword string `json:"keyword" dc:"关键词搜索"`
}
// ListDatasetRes 数据集列表响应
type ListDatasetRes struct {
List []*DatasetVO `json:"list"`
Total int `json:"total"`
}
type DatasetVO struct {
Id int64 `json:"id,string" dc:"id"`
Name string `json:"name" dc:"数据集名称"`
Description string `json:"description" dc:"数据集描述"`
DocumentCount int64 `json:"documentCount" dc:"文件数量"`
DocumentSize int64 `json:"documentSize" dc:"文件大小(字节)"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}

1
model/dto/dataset_index.go Normal file
View File

@@ -0,0 +1 @@
package dto

108
model/dto/document.go Normal file
View File

@@ -0,0 +1,108 @@
package dto
import (
"rag/consts/document"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
// CreateDocumentReq 创建文件请求
type CreateDocumentReq struct {
g.Meta `path:"/createDocument" method:"post" tags:"文件管理" summary:"创建文件" dc:"创建文件"`
DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
Title string `json:"title" v:"required#标题不能为空"`
Format string `json:"format" v:"required#格式不能为空"`
FileSize int64 `json:"fileSize" v:"required#大小不能为空"`
FilePath string `json:"filePath" v:"required#路径不能为空"`
}
// CreateDocumentRes 创建文件响应
type CreateDocumentRes struct {
Id int64 `json:"id,string"`
}
// UpdateDocumentReq 更新文件请求
type UpdateDocumentReq struct {
g.Meta `path:"/updateDocument" method:"put" tags:"文件管理" summary:"更新文件" dc:"更新文件"`
Id int64 `json:"id" v:"required#ID不能为空"`
Status document.Status `json:"status"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
ChunkCount int64 `json:"chunkCount"`
}
// DeleteDocumentReq 删除文件请求
type DeleteDocumentReq struct {
g.Meta `path:"/deleteDocument" method:"delete" tags:"文件管理" summary:"删除文件" dc:"删除文件"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// GetDocumentReq 获取文件请求
type GetDocumentReq struct {
g.Meta `path:"/getDocument" method:"get" tags:"文件管理" summary:"获取文件详情" dc:"获取文件详情"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// ListDocumentReq 文件列表请求
type ListDocumentReq struct {
g.Meta `path:"/listDocument" method:"get" tags:"文件管理" summary:"获取文件列表" dc:"分页查询文件列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
DatasetId int64 `json:"datasetId"`
Keyword string `json:"keyword" dc:"关键词搜索"`
Status document.Status `json:"status"`
}
// ListDocumentRes 文件列表响应
type ListDocumentRes struct {
List []*DocumentVO `json:"list"`
Total int `json:"total"`
}
type DocumentVO struct {
Id int64 `json:"id,string" dc:"id"`
DatasetId int64 `json:"datasetId,string"`
Title string `json:"title" dc:"文件标题"`
Status document.Status `json:"status" dc:"状态1启用/0停用"`
VectorStatus document.VectorStatus `json:"vectorStatus" dc:"向量化状态 状态: 1 待定, 2 处理, 3 完成, 4 失败"`
ChunkCount int64 `json:"chunkCount" dc:"分块数"`
FileSize int64 `json:"fileSize" dc:"文件大小"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}
// ProcessDocumentReq 处理文件请求(向量化)
type ProcessDocumentReq struct {
g.Meta `path:"/getProcess" method:"get" tags:"文件管理" summary:"文件向量化处理" dc:"文件向量化处理"`
Id int64 `json:"id" v:"required#ID不能为空"`
DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
}
// ProcessDocumentRes 处理文件响应
type ProcessDocumentRes struct {
ChunkCount int64 `json:"chunkCount"`
CostTime int64 `json:"costTime"`
}
type ListDocumentChunkRPC struct {
List []*DocumentChunkRPC `json:"list"`
}
type DocumentChunkRPC struct {
Id int64 `json:"id" dc:"id"`
DatasetId int64 `json:"datasetId" dc:"所属数据集ID"`
ContentHash string `json:"contentHash" dc:"内容hash"`
}
type KnowledgeDocumentMsg struct {
TenantId uint64 `json:"tenantId"`
Creator string `json:"creator"`
Id int64 `json:"id"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
}

64
model/dto/document_chunk.go Normal file
View File

@@ -0,0 +1,64 @@
package dto
import (
"rag/consts/document"
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
"github.com/pgvector/pgvector-go"
)
// UpdateDocumentChunkReq 更新文件块向量请求
type UpdateDocumentChunkReq struct {
g.Meta `path:"/updateDocumentChunk" method:"put" tags:"文件块向量管理" summary:"更新文件块" dc:"更新文件块"`
Id int64 `json:"id" v:"required#ID不能为空"`
Status document.Status `json:"status"`
}
// ListDocumentChunkReq 文件块向量列表请求
type ListDocumentChunkReq struct {
g.Meta `path:"/listDocumentChunk" method:"get" tags:"文件块向量管理" summary:"获取文件块向量列表" dc:"分页查询文件块向量列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
DatasetId int64 `json:"datasetId"`
DocumentId int64 `json:"documentId"`
Status document.Status `json:"status"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
}
// ListDocumentChunkRes 文件块向量列表响应
type ListDocumentChunkRes struct {
List []*DocumentChunkItem `json:"list"`
Total int `json:"total"`
}
type DocumentChunkItem struct {
Id int64 `json:"id,string" dc:"id"`
Status document.Status `json:"status" dc:"状态"`
VectorStatus document.VectorStatus `json:"vectorStatus" dc:"向量状态"`
DatasetId int64 `json:"datasetId,string" dc:"所属数据集ID"`
DocumentId int64 `json:"documentId,string" dc:"所属文档ID"`
Content string `json:"content" dc:"内容"`
ContentHash string `json:"contentHash" dc:"内容hash"`
ChunkIndex int64 `json:"chunkIndex" dc:"块索引"`
Vector []float64 `json:"vector" dc:"向量"`
Metadata map[string]interface{} `json:"metadata" dc:"元信息"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}
type VectorDocumentChunkMsg struct {
TenantId uint64 `json:"tenantId"`
Creator string `json:"creator"`
DatasetId int64 `json:"datasetId"` // 数据集ID
DocumentId int64 `json:"documentId"` // 所属文档ID
Content string `json:"content"` // 原始内容
ContentHash string `json:"contentHash"` // 原始内容hash
ChunkIndex int64 `json:"chunkIndex"` // 第几块
Status document.Status `json:"status"`
VectorStatus document.VectorStatus `json:"vectorStatus"`
Vector pgvector.Vector `json:"vector"`
Metadata map[string]interface{} `json:"metadata"`
}

70
model/dto/keyword.go Normal file
View File

@@ -0,0 +1,70 @@
package dto
import (
"gitea.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
)
// CreateKeywordReq 创建关键词请求
type CreateKeywordReq struct {
g.Meta `path:"/createKeyword" method:"post" tags:"关键词管理" summary:"创建关键词" dc:"创建关键词"`
DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
DocumentId int64 `json:"documentId" v:"required#文档ID不能为空"`
Word string `json:"word" v:"required#名称不能为空"`
Weight int16 `json:"weight" v:"required#权重不能为空"`
}
// CreateKeywordRes 创建关键词响应
type CreateKeywordRes struct {
Id int64 `json:"id,string"`
}
// UpdateKeywordReq 更新关键词请求
type UpdateKeywordReq struct {
g.Meta `path:"/updateKeyword" method:"put" tags:"关键词管理" summary:"更新关键词" dc:"更新关键词"`
Id int64 `json:"id" v:"required#ID不能为空"`
Word string `json:"word"`
Weight int16 `json:"weight"`
}
// DeleteKeywordReq 删除关键词请求
type DeleteKeywordReq struct {
g.Meta `path:"/deleteKeyword" method:"delete" tags:"关键词管理" summary:"删除关键词" dc:"删除关键词"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// GetKeywordReq 获取关键词请求
type GetKeywordReq struct {
g.Meta `path:"/getKeyword" method:"get" tags:"关键词管理" summary:"获取关键词详情" dc:"获取关键词详情"`
Id int64 `json:"id" v:"required#ID不能为空"`
}
// ListKeywordReq 关键词列表请求
type ListKeywordReq struct {
g.Meta `path:"/listKeyword" method:"get" tags:"关键词管理" summary:"获取关键词列表" dc:"分页查询关键词列表，支持多条件筛选"`
Page *beans.Page `json:"page"`
DatasetId int64 `json:"datasetId"`
DocumentId int64 `json:"documentId"`
Word string `json:"word"`
Keyword string `json:"keyword" dc:"关键词搜索"`
}
// ListKeywordRes 关键词列表响应
type ListKeywordRes struct {
List []*KeywordVO `json:"list"`
Total int `json:"total"`
}
type KeywordVO struct {
Id int64 `json:"id,string" dc:"id"`
Word string `json:"word" dc:"关键词名称"`
Weight int16 `json:"weight" dc:"权重"`
CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
}

37
model/entity/dataset.go Normal file
View File

@@ -0,0 +1,37 @@
package entity
import (
"gitea.com/red-future/common/beans"
)
type datasetCol struct {
beans.SQLBaseCol
Name string
Description string
Embedding string
Dimension string
DocumentCount string
DocumentSize string
}
var DatasetCol = datasetCol{
SQLBaseCol: beans.DefSQLBaseCol,
Name: "name",
Description: "description",
Embedding: "embedding",
Dimension: "dimension",
DocumentCount: "document_count",
DocumentSize: "document_size",
}
// Dataset 数据集表
type Dataset struct {
beans.SQLBaseDO `orm:",inline"`
Name string `orm:"name" json:"name" dc:"数据集名称"`
Description string `orm:"description" json:"description" dc:"数据集描述"`
Embedding string `orm:"embedding" json:"embedding" dc:"向量模型"`
Dimension int `orm:"dimension" json:"dimension" dc:"向量维度"`
DocumentCount int64 `orm:"document_count" json:"documentCount" dc:"文档数量"`
DocumentSize int64 `orm:"document_size" json:"documentSize" dc:"文档大小"`
}

46
model/entity/dataset_index.go Normal file
View File

@@ -0,0 +1,46 @@
package entity
import "gitea.com/red-future/common/beans"
type datasetIndexCol struct {
beans.SQLBaseCol
Status string
VectorStatus string
DatasetId string
Name string
Collection string
Dimension string
FieldType string
MetricType string
VectorCount string
Description string
}
var DatasetIndexCol = datasetIndexCol{
SQLBaseCol: beans.DefSQLBaseCol,
Status: "status",
VectorStatus: "vector_status",
DatasetId: "dataset_id",
Name: "name",
Collection: "collection",
Dimension: "dimension",
FieldType: "field_type",
MetricType: "metric_type",
VectorCount: "vector_count",
Description: "description",
}
// DatasetIndex 数据集索引实体
type DatasetIndex struct {
beans.SQLBaseDO `orm:",inline"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
Name string `orm:"name" json:"name" dc:"索引名称"`
Collection string `orm:"collection" json:"collection" dc:"向量集合名称"`
Dimension int `orm:"dimension" json:"dimension" dc:"向量维度"`
FieldType string `orm:"field_type" json:"fieldType" dc:"字段类型: float, binary"`
MetricType string `orm:"metric_type" json:"metricType" dc:"度量类型: L2, IP, COSINE"`
Status *int8 `orm:"status" json:"status" dc:"状态: creating, ready, error"`
VectorCount int64 `orm:"vector_count" json:"vectorCount" dc:"向量数量"`
Description string `orm:"description" json:"description" dc:"描述"`
}

64
model/entity/document.go Normal file
View File

@@ -0,0 +1,64 @@
package entity
import (
"gitea.com/red-future/common/beans"
"rag/consts/document"
)
type documentCol struct {
beans.SQLBaseCol
DatasetId string
Title string
Content string
Format string
Source string
SourceId string
Status string
VectorStatus string
ChunkCount string
FileSize string
FilePath string
Metadata string
}
var DocumentCol = documentCol{
SQLBaseCol: beans.DefSQLBaseCol,
DatasetId: "dataset_id",
Title: "title",
Content: "content",
Format: "format",
Source: "source",
SourceId: "source_id",
Status: "status",
VectorStatus: "vector_status",
ChunkCount: "chunk_count",
FileSize: "file_size",
FilePath: "file_path",
Metadata: "metadata",
}
// Document 文件实体
type Document struct {
beans.SQLBaseDO `orm:",inline"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
Title string `orm:"title" json:"title" dc:"文件标题"`
Content string `orm:"content" json:"content" dc:"文件内容"`
Format string `orm:"format" json:"format" dc:"文件格式"`
Source string `orm:"source" json:"source" dc:"来源"`
SourceId string `orm:"source_id" json:"sourceId" dc:"来源ID"`
Status document.Status `orm:"status" json:"status" dc:"状态"`
VectorStatus document.VectorStatus `orm:"vector_status" json:"vectorStatus" dc:"向量状态"`
ChunkCount int64 `orm:"chunk_count" json:"chunkCount" dc:"切分块数量"`
FileSize int64 `orm:"file_size" json:"fileSize" dc:"文件大小"`
FilePath string `orm:"file_path" json:"filePath" dc:"文件存储路径"`
Metadata *Metadata `orm:"metadata" json:"metadata" dc:"文件元信息"`
}
// Metadata 文件元数据
type Metadata struct {
Author string `orm:"author" json:"author" dc:"作者"`
Tags []string `orm:"tags" json:"tags" dc:"标签"`
Custom map[string]string `orm:"custom" json:"custom" dc:"自定义字段"`
}

49
model/entity/document_chunk.go Normal file
View File

@@ -0,0 +1,49 @@
package entity
import (
"rag/consts/document"
"gitea.com/red-future/common/beans"
"github.com/pgvector/pgvector-go"
)
type documentChunkCol struct {
beans.SQLBaseCol
Status string
VectorStatus string
DatasetId string
DocumentId string
Content string
ContentHash string
ChunkIndex string
Vector string
Metadata string
}
var DocumentChunkCol = documentChunkCol{
SQLBaseCol: beans.DefSQLBaseCol,
Status: "status",
VectorStatus: "vector_status",
DatasetId: "dataset_id",
DocumentId: "document_id",
Content: "content",
ContentHash: "content_hash",
ChunkIndex: "chunk_index",
Vector: "vector",
Metadata: "metadata",
}
// DocumentChunk 文档切分块实体
type DocumentChunk struct {
beans.SQLBaseDO `orm:",inline"`
Status document.Status `orm:"status" json:"status" dc:"状态"`
VectorStatus document.VectorStatus `orm:"vector_status" json:"vectorStatus" dc:"向量状态"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
DocumentId int64 `orm:"document_id" json:"documentId" dc:"文件ID"`
Content string `orm:"content" json:"content" dc:"切分块内容"`
ContentHash string `orm:"content_hash" json:"contentHash" dc:"切分块内容哈希"`
ChunkIndex int64 `orm:"chunk_index" json:"chunkIndex" dc:"切分块索引"`
Vector pgvector.Vector `orm:"vector" json:"vector" dc:"向量"`
Metadata map[string]interface{} `orm:"metadata" json:"metadata" dc:"元信息"`
}

27
model/entity/keyword.go Normal file
View File

@@ -0,0 +1,27 @@
package entity
import "gitea.com/red-future/common/beans"
type keywordCol struct {
beans.SQLBaseCol
DatasetId string
DocumentId string
Word string
Weight string
}
var KeywordCol = keywordCol{
SQLBaseCol: beans.DefSQLBaseCol,
DatasetId: "dataset_id",
DocumentId: "document_id",
Word: "word",
Weight: "weight",
}
type Keyword struct {
beans.SQLBaseDO `orm:",inline"`
DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
DocumentId int64 `orm:"document_id" json:"documentId" dc:"文件ID"`
Word string `orm:"word" json:"word" dc:"关键词"`
Weight int16 `orm:"weight" json:"weight" dc:"权重"`
}

87
service/dataset.go Normal file
View File

@@ -0,0 +1,87 @@
package service
import (
"context"
"rag/dao"
"rag/model/dto"
"github.com/gogf/gf/v2/util/gconv"
)
var Dataset = new(datasetService)
type datasetService struct{}
// Create 创建数据集
func (s *datasetService) Create(ctx context.Context, req *dto.CreateDatasetReq) (res *dto.CreateDatasetRes, err error) {
id, err := dao.Dataset.Insert(ctx, req)
if err != nil {
return
}
return &dto.CreateDatasetRes{Id: id}, nil
}
// Update 更新数据集
func (s *datasetService) Update(ctx context.Context, req *dto.UpdateDatasetReq) (err error) {
_, err = dao.Dataset.Update(ctx, req)
return
}
// Delete 删除数据集
func (s *datasetService) Delete(ctx context.Context, req *dto.DeleteDatasetReq) (err error) {
_, err = dao.Dataset.Delete(ctx, req)
return
}
// List 数据集列表
func (s *datasetService) List(ctx context.Context, req *dto.ListDatasetReq) (res *dto.ListDatasetRes, err error) {
list, total, err := dao.Dataset.List(ctx, req)
if err != nil {
return
}
res = &dto.ListDatasetRes{
Total: total,
}
err = gconv.Struct(list, &res.List)
return
}
//// Search 搜索(示例,实际需要调用向量库)
//func (s *datasetService) Search(ctx context.Context, req *dto.SearchReq) (res *dto.SearchRes, err error) {
// // 1. 获取数据集信息
// kb, err := dao.Dataset.GetByID(ctx, req)
// if err != nil {
// return nil, err
// }
//
// // 2. 获取文件块
// chunks, err := dao.Chunk.FindChunksByKBIDWithLimit(ctx, req.KBID, 0, req.TopK)
// if err != nil {
// return nil, err
// }
//
// // 3. TODO: 使用向量检索(需要集成向量库)
// // 暂时使用简单的关键词匹配
// results := make([]dto.SearchResult, 0)
// for _, chunk := range chunks {
// results = append(results, dto.SearchResult{
// Content: chunk.Content,
// Score: 0.8, // TODO: 计算实际向量相似度
// DocumentID: chunk.DocumentID,
// ChunkIndex: chunk.Index,
// })
// }
//
// g.Log().Infof(ctx, "数据集[%s]搜索完成,查询:%s,结果数:%d", kb.Name, req.Query, len(results))
//
// return &dto.SearchRes{Results: results}, nil
//}
//
//// formatChunks 格式化文件块为上下文
//func (s *datasetService) formatChunks(chunks []*entity.DocumentChunk) string {
// var sb strings.Builder
// for i, chunk := range chunks {
// sb.WriteString(fmt.Sprintf("[%d] %s\n\n", i+1, chunk.Content))
// }
// return sb.String()
//}

5
service/dataset_index.go Normal file
View File

@@ -0,0 +1,5 @@
package service
var DatasetIndex = new(datasetIndexService)
type datasetIndexService struct{}

483
service/document.go Normal file
View File

@@ -0,0 +1,483 @@
package service
import (
"context"
"fmt"
"rag/consts/document"
"rag/consts/public"
"rag/dao"
"rag/model/dto"
"rag/model/entity"
"strings"
"sync"
"time"
"gitea.com/red-future/common/beans"
"gitea.com/red-future/common/db/gfdb"
"gitea.com/red-future/common/full-text-search/meilisearch"
"gitea.com/red-future/common/http"
"gitea.com/red-future/common/rag/eino"
"gitea.com/red-future/common/rag/gse"
"gitea.com/red-future/common/utils"
gmq "github.com/bjang03/gmq/core/gmq"
"github.com/bjang03/gmq/mq"
"github.com/bjang03/gmq/types"
"github.com/cloudwego/eino/schema"
"github.com/gogf/gf/v2/container/gvar"
"github.com/gogf/gf/v2/crypto/gmd5"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/database/gredis"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
var Document = new(documentService)
type documentService struct{}
// Create 创建文件
func (s *documentService) Create(ctx context.Context, req *dto.CreateDocumentReq) (res *dto.CreateDocumentRes, err error) {
err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
var id int64
id, err = dao.Document.Insert(ctx, req)
if err != nil {
return
}
datasetReq := &dto.UpdateDatasetReq{
Id: req.DatasetId,
DocumentCount: 1,
DocumentSize: req.FileSize,
}
_, err = dao.Dataset.Update(ctx, datasetReq)
if err != nil {
return
}
res = &dto.CreateDocumentRes{Id: id}
return
})
return
}
// Update 更新文件
func (s *documentService) Update(ctx context.Context, req *dto.UpdateDocumentReq) (err error) {
_, err = dao.Document.Update(ctx, req)
return
}
// Delete 删除文件
func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (err error) {
docs, err := dao.Document.GetByID(ctx, &dto.GetDocumentReq{Id: req.Id})
if err != nil {
return
}
err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
datasetReq := &dto.UpdateDatasetReq{
Id: docs.DatasetId,
DocumentCount: -1,
DocumentSize: -docs.FileSize,
}
_, err = dao.Dataset.Update(ctx, datasetReq)
if err != nil {
return
}
_, err = dao.Document.Delete(ctx, req)
return
})
return
}
// Get 获取文件详情
func (s *documentService) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.DocumentVO, err error) {
r, err := dao.Document.GetByID(ctx, req)
err = gconv.Struct(r, &res)
return
}
// List 文件列表
func (s *documentService) List(ctx context.Context, req *dto.ListDocumentReq) (res *dto.ListDocumentRes, err error) {
list, total, err := dao.Document.List(ctx, req)
if err != nil {
return nil, err
}
res = &dto.ListDocumentRes{
Total: total,
}
err = gconv.Struct(list, &res.List)
//eino.TestIndexer()
//eino.TestRetriever()
return
}
// Process 处理文件(使用eino框架切分和向量化)
func (s *documentService) Process(ctx context.Context, req *dto.ProcessDocumentReq) (res *dto.ProcessDocumentRes, err error) {
startTime := time.Now()
// 1. 查询文件信息
documentReq := dto.GetDocumentReq{Id: req.Id}
doc, err := dao.Document.GetByID(ctx, &documentReq)
if err != nil {
return nil, err
}
// 2. 使用eino框架进行文件切分（并发执行）
var vectorDocsCount, chunks int64
// 用 gopool 或者简单的错误等待，绝对不用裸 goroutine
var err1, err2, err3 error
var wg sync.WaitGroup
wg.Add(3)
// 任务1
go func() {
defer wg.Done()
vectorDocsCount, chunks, err1 = s.sqlSplitDocument(ctx, doc)
}()
// 任务2
go func() {
defer wg.Done()
err2 = s.esSplitDocument(ctx, doc)
}()
// 任务3
go func() {
defer wg.Done()
err3 = s.extractDocument(ctx, doc)
}()
// 直接等待，不使用通道，避免泄漏
wg.Wait()
updateDocumentReq := new(dto.UpdateDocumentReq)
updateDocumentReq.Id = req.Id
// 统一判断错误
if err1 != nil || err2 != nil || err3 != nil {
// 更新文档状态
updateDocumentReq.VectorStatus = document.VectorStatusFailed.Code()
if _, err = dao.Document.Update(ctx, updateDocumentReq); err != nil {
return nil, err
}
if err1 != nil {
return nil, err1
}
if err2 != nil {
return nil, err2
}
return nil, err3
}
// 4. 更新文件状态为处理中和切分数量
if vectorDocsCount > 0 {
updateDocumentReq.VectorStatus = document.VectorStatusProcessing.Code()
} else {
updateDocumentReq.VectorStatus = document.VectorStatusCompleted.Code()
}
updateDocumentReq.ChunkCount = chunks
if _, err = dao.Document.Update(ctx, updateDocumentReq); err != nil {
return
}
costTime := time.Since(startTime).Milliseconds()
return &dto.ProcessDocumentRes{
ChunkCount: chunks,
CostTime: costTime,
}, nil
}
func (s *documentService) extractDocument(ctx context.Context, doc *entity.Document) (err error) {
// 1. 加载文件
docs, err := s.loadDocument(ctx, doc)
if err != nil {
return
}
var words []gse.Keyword
if len(docs[0].Content) < 500 {
words = gse.GseTool.Extract(docs[0].Content, 4)
} else if len(docs[0].Content) < 2000 {
words = gse.GseTool.Extract(docs[0].Content, 8)
} else if len(docs[0].Content) < 5000 {
words = gse.GseTool.Extract(docs[0].Content, 13)
} else {
var docsSplit []*schema.Document
docsSplit, err = eino.RecursiveSplitDocument(ctx, docs)
if err != nil {
return
}
for _, t := range docsSplit {
words = append(words, gse.GseTool.Extract(t.Content, 6)...)
}
}
var keywordReqs = make([]*dto.CreateKeywordReq, 0)
for _, word := range words {
keywordReqs = append(keywordReqs, &dto.CreateKeywordReq{
DatasetId: doc.DatasetId,
DocumentId: doc.Id,
Word: word.Word,
Weight: gconv.Int16(word.Score),
})
}
if len(keywordReqs) > 0 {
_, err = dao.Keyword.BatchSaveOrUpdate(ctx, keywordReqs)
if err != nil {
return
}
}
return
}
func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Document) (vectorDocsCount, docsSplitCount int64, err error) {
// 1. 加载文件
docs, err := s.loadDocument(ctx, doc)
if err != nil {
return
}
// 2. 语义切分文件
docsSplit, err := eino.SemanticSplitDocument(ctx, docs)
if err != nil {
return
}
docsSplitCount = gconv.Int64(len(docsSplit))
// 2. 获取历史数据
err = s.getHistoryData(ctx, doc, public.KnowledgeLockSqlKey, public.KnowledgeContentHashSqlKey)
if err != nil {
return
}
// 3. 组装向量文档
var vectorDocs = make([]dto.VectorDocumentChunkMsg, 0)
for i, t := range docsSplit {
contentHash := gmd5.MustEncryptString(t.Content)
// 检查是否重复
var success bool
success, err = s.checkRepeat(ctx, public.KnowledgeContentHashSqlKey, contentHash)
if err != nil {
return
}
if !success {
continue
}
vectorDocs = append(vectorDocs, dto.VectorDocumentChunkMsg{
TenantId: doc.TenantId,
Creator: doc.Creator,
DatasetId: doc.DatasetId,
DocumentId: doc.Id,
Content: t.Content,
ContentHash: contentHash,
ChunkIndex: gconv.Int64(i),
})
}
// 4. 发送消息到队列
if len(vectorDocs) > 0 {
err = gmq.GetGmq("primary").GmqPublish(ctx, &mq.RedisPubMessage{
PubMessage: types.PubMessage{
Topic: public.KnowledgeDocumentChunkTopic,
Data: vectorDocs,
},
})
}
vectorDocsCount = gconv.Int64(len(vectorDocs))
return
}
func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
// 1. 加载文件
docs, err := s.loadDocument(ctx, doc)
if err != nil {
return
}
// 2. 递归切分文件
docsSplit, err := eino.RecursiveSplitDocument(ctx, docs)
if err != nil {
return
}
// 2. 获取历史数据
err = s.getHistoryData(ctx, doc, public.KnowledgeLockEsKey, public.KnowledgeContentHashEsKey)
if err != nil {
return
}
// 3. 组装向量文档并同时构建meilisearch文档
var meiliDocs = make([]interface{}, 0)
for i, t := range docsSplit {
contentHash := gmd5.MustEncryptString(t.Content)
// 检查是否重复
var success bool
success, err = s.checkRepeat(ctx, public.KnowledgeContentHashEsKey, contentHash)
if err != nil {
return
}
if !success {
continue
}
// 构建Meilisearch文档
meiliDocs = append(meiliDocs, map[string]interface{}{
"id": contentHash,
"datasetId": doc.DatasetId,
"documentId": doc.Id,
"content": t.Content,
"contentHash": contentHash,
"chunkIndex": i,
})
}
// 4. 写入到meilisearch数据库中
if len(meiliDocs) > 0 {
if _, err = meilisearch.DB().InsertMany(ctx, meiliDocs, public.IndexNameDocumentChunk); err != nil {
g.Log().Errorf(ctx, "写入meilisearch失败: %v", err)
return
}
}
return
}
// loadDocument 加载文件
func (s *documentService) loadDocument(ctx context.Context, doc *entity.Document) (docs []*schema.Document, err error) {
return eino.LoadDocument(ctx, doc.FilePath, doc.Format)
}
// getHistoryData 获取历史数据
func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Document, lockKey, contentKey string) (err error) {
docsLockKey := fmt.Sprintf(lockKey, doc.DatasetId)
success, err := utils.Lock(ctx, docsLockKey, int64(60), func(ctx context.Context) error {
// 1. 扫描 Redis 中所有 前缀为 rag:knowledge:xxx:contentHash 的 key
pattern := fmt.Sprintf(contentKey, "*")
keys, err := g.Redis().Keys(ctx, pattern)
if err != nil {
return err
}
// 2. Redis 有数据：只刷新过期时间，不查库
if len(keys) > 0 {
// 批量刷新过期时间为 60s
for _, key := range keys {
_, err = g.Redis().Expire(ctx, key, 600)
if err != nil {
return err
}
}
return nil
}
// 3. Redis 无数据：根据 contentKey 类型选择查询方式
var dictData = make([]*dto.DocumentChunkRPC, 0)
if public.KnowledgeContentHashSqlKey == contentKey {
// SQL 方式：调用 HTTP 接口查询
dictData, err = s.getHistoryDataFromHttp(ctx, doc)
} else {
// ES 方式：查询 meilisearch
dictData, err = s.getHistoryDataFromMeilisearch(ctx, doc)
}
if err != nil {
return err
}
// 4. 把查询到的数据写入 Redis（600s过期）
for _, item := range dictData {
// 去除可能的 JSON 引号
contentHash := strings.Trim(item.ContentHash, `"`)
key := fmt.Sprintf(contentKey, contentHash)
_, err = g.Redis().Set(ctx, key, true, gredis.SetOption{
TTLOption: gredis.TTLOption{
EX: gconv.PtrInt64(600),
},
NX: true,
})
if err != nil {
return err
}
}
return nil
})
if err != nil && !success {
return
}
return
}
// getHistoryDataFromHttp 通过 HTTP 接口查询历史数据
func (s *documentService) getHistoryDataFromHttp(ctx context.Context, doc *entity.Document) (dictData []*dto.DocumentChunkRPC, err error) {
headers := make(map[string]string)
if r := g.RequestFromCtx(ctx); r != nil {
for k, v := range r.Request.Header {
if len(v) > 0 {
headers[k] = v[0]
}
}
}
// 调用接口获取数据
d := &dto.ListDocumentChunkRPC{}
if err = http.Get(ctx, "rag-vector/document/chunk/listDocumentChunk", headers, &d,
"datasetId", gconv.String(doc.DatasetId),
"status", 1); err != nil {
return
}
dictData = d.List
return
}
// getHistoryDataFromMeilisearch 通过 meilisearch 查询历史数据
func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc *entity.Document) (dictData []*dto.DocumentChunkRPC, err error) {
// 构建 meilisearch 查询参数
searchParams := &meilisearch.SearchParams{
Filter: fmt.Sprintf("datasetId = %d", doc.DatasetId),
Limit: 10000,
}
// 执行搜索
var hits []map[string]interface{}
_, err = meilisearch.DB().Search(ctx, searchParams, public.IndexNameDocumentChunk, &hits)
if err != nil {
return
}
// 转换查询结果
dictData = make([]*dto.DocumentChunkRPC, 0)
for _, hit := range hits {
item := &dto.DocumentChunkRPC{}
if err = gconv.Struct(hit, item); err != nil {
return
}
dictData = append(dictData, item)
}
return
}
// checkRepeat 检查是否重复
func (s *documentService) checkRepeat(ctx context.Context, contentKey, contentHash string) (success bool, err error) {
var val *gvar.Var
if val, err = g.Redis().Set(ctx, fmt.Sprintf(contentKey, contentHash), true, gredis.SetOption{
TTLOption: gredis.TTLOption{
EX: gconv.PtrInt64(600),
},
NX: true,
}); err != nil {
return
}
success = val.Bool()
return
}
func (s *documentService) DocsVectorStatusMsg(ctx context.Context, msg any) (err error) {
var req = new(dto.KnowledgeDocumentMsg)
if err = gconv.Struct(msg, &req); err != nil {
g.Log().Error(ctx, "DocsVectorStatusMsg err:", err)
return
}
ctx = context.WithValue(ctx, "user", &beans.User{
TenantId: req.TenantId,
UserName: req.Creator,
})
_, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{
Id: req.Id,
VectorStatus: req.VectorStatus,
})
return
}

176
service/document_chunk.go Normal file
View File

@@ -0,0 +1,176 @@
package service
import (
"context"
"database/sql"
"errors"
"fmt"
"rag/consts/document"
"rag/consts/public"
"rag/dao"
"rag/model/dto"
"rag/model/entity"
"gitea.com/red-future/common/beans"
"gitea.com/red-future/common/rag/eino"
gmq "github.com/bjang03/gmq/core/gmq"
"github.com/bjang03/gmq/mq"
"github.com/bjang03/gmq/types"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
"github.com/pgvector/pgvector-go"
)
var DocumentChunk = new(documentChunkService)
type documentChunkService struct{}
const (
DatasetIndexStatusReady = "ready"
)
// Update 更新文件块
func (s *documentChunkService) Update(ctx context.Context, req *dto.UpdateDocumentChunkReq) (err error) {
_, err = dao.DocumentChunk.Update(ctx, req)
return
}
// List 获取文件块列表
func (s *documentChunkService) List(ctx context.Context, req *dto.ListDocumentChunkReq) (res *dto.ListDocumentChunkRes, err error) {
list, total, err := dao.DocumentChunk.List(ctx, req)
if err != nil {
return
}
res = &dto.ListDocumentChunkRes{
Total: total,
}
err = gconv.Struct(list, &res.List)
return
}
func (s *documentChunkService) DocsChunkMsg(ctx context.Context, msg any) (err error) {
var req = make([]*dto.VectorDocumentChunkMsg, 0)
msgMap := gconv.Map(msg)
if err = gconv.Structs(msgMap["data"], &req); err != nil {
g.Log().Error(ctx, "DocsChunkMsg err:", err)
return
}
if len(req) == 0 {
g.Log().Error(ctx, "DocsChunkMsg err:", "msg is empty")
return
}
ctx = context.WithValue(ctx, "user", &beans.User{
TenantId: req[0].TenantId,
UserName: req[0].Creator,
})
// 调用eino接口获取向量
var vectorDocsStr = make([]string, 0, len(req))
for _, t := range req {
vectorDocsStr = append(vectorDocsStr, t.Content)
}
embeddings, err := eino.EmbedStrings(ctx, vectorDocsStr)
if err != nil {
g.Log().Error(ctx, "DocsChunkMsg err:", err)
err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusFailed.Code())
return
}
// 获取向量维度
dimension := 0
if len(embeddings) > 0 {
dimension = len(embeddings[0])
}
// 创建或更新DatasetIndex
err = s.createOrUpdateDatasetIndex(ctx, req[0].DatasetId, dimension, int64(len(req)))
if err != nil {
g.Log().Error(ctx, "CreateOrUpdateDatasetIndex err:", err)
err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusFailed.Code())
return
}
// 更新向量文档
for i, embedding := range embeddings {
req[i].Vector = pgvector.NewVector(gconv.Float32s(embedding))
req[i].VectorStatus = document.VectorStatusCompleted.Code()
req[i].Status = document.StatusEnable.Code()
}
_, err = dao.DocumentChunk.BatchInsert(ctx, req)
if err != nil {
g.Log().Error(ctx, "DocsChunkMsg err:", err)
err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusFailed.Code())
return
}
err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusCompleted.Code())
return
}
// createOrUpdateDatasetIndex 创建或更新数据集索引
func (s *documentChunkService) createOrUpdateDatasetIndex(ctx context.Context, datasetId int64, dimension int, vectorCount int64) (err error) {
// 查询数据集是否已有索引
existIndex, err := dao.DatasetIndex.GetByDatasetId(ctx, datasetId)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
// 已有索引 → 只更新数量
if existIndex != nil {
_ = dao.DatasetIndex.IncVectorCount(ctx, existIndex.Id, vectorCount)
return nil
}
// ====================== 创建新索引 ======================
indexName := fmt.Sprintf("idx_dataset_%d_vector", datasetId) // 真实PG索引名
// 1. 插入索引配置
index := &entity.DatasetIndex{
DatasetId: datasetId,
Name: indexName,
Dimension: dimension,
FieldType: "float",
MetricType: "COSINE",
Status: gconv.PtrInt8(1),
VectorCount: vectorCount,
Description: fmt.Sprintf("数据集%d向量索引", datasetId),
}
_, err = dao.DatasetIndex.Insert(ctx, index)
if err != nil {
return err
}
// 2. 真正创建 PGVector 索引（唯一真实索引！）
err = s.createRealPGVectorIndex(ctx, indexName)
return err
}
// createRealPGVectorIndex 真正在PostgreSQL创建向量索引（真实可用）
func (s *documentChunkService) createRealPGVectorIndex(ctx context.Context, indexName string) error {
// 执行真实建索引语句
err := dao.DatasetIndex.InsertIndex(ctx, indexName)
if err != nil {
g.Log().Error(ctx, "创建向量索引失败：", err)
return err
}
g.Log().Info(ctx, "PGVector真实索引创建成功："+indexName)
return nil
}
// publishKnowledgeDocumentMsg 发布消息
func (s *documentChunkService) publishKnowledgeDocumentMsg(ctx context.Context, tenantId uint64, creator string, documentId int64, vectorStatus document.VectorStatus) (err error) {
knowledgeDocumentMsg := dto.KnowledgeDocumentMsg{
TenantId: tenantId,
Creator: creator,
Id: documentId,
VectorStatus: vectorStatus,
}
err = gmq.GetGmq("primary").GmqPublish(ctx, &mq.RedisPubMessage{
PubMessage: types.PubMessage{
Topic: public.KnowledgeDocumentVectorStatusTopic,
Data: knowledgeDocumentMsg,
},
})
return
}

65
service/keyword.go Normal file
View File

@@ -0,0 +1,65 @@
package service
import (
"context"
"rag/dao"
"rag/model/dto"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/util/gconv"
)
var Keyword = new(keywordService)
type keywordService struct{}
func (s *keywordService) Create(ctx context.Context, req *dto.CreateKeywordReq) (res *dto.CreateKeywordRes, err error) {
count, err := dao.Keyword.Count(ctx, &dto.ListKeywordReq{
DatasetId: req.DatasetId,
DocumentId: req.DocumentId,
Word: req.Word,
})
if err != nil {
return
}
if count > 0 {
err = gerror.New("关键词已存在")
return
}
var id int64
id, err = dao.Keyword.Insert(ctx, req)
if err != nil {
return
}
res = &dto.CreateKeywordRes{Id: id}
return
}
func (s *keywordService) Update(ctx context.Context, req *dto.UpdateKeywordReq) (err error) {
_, err = dao.Keyword.Update(ctx, req)
return
}
func (s *keywordService) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (err error) {
_, err = dao.Keyword.Delete(ctx, req)
return
}
func (s *keywordService) Get(ctx context.Context, req *dto.GetKeywordReq) (res *dto.KeywordVO, err error) {
r, err := dao.Keyword.GetByID(ctx, req)
err = gconv.Struct(r, &res)
return
}
func (s *keywordService) List(ctx context.Context, req *dto.ListKeywordReq) (res *dto.ListKeywordRes, err error) {
list, total, err := dao.Keyword.List(ctx, req)
if err != nil {
return nil, err
}
res = &dto.ListKeywordRes{
Total: total,
}
err = gconv.Struct(list, &res.List)
return
}

162
update.sql Normal file
View File

@@ -0,0 +1,162 @@
-----------张斌2025-06-16 15:00:00--------------
--------------------pgsql创建rag_knowledge_dataset表语句---------------------------
-- 数据集表（RAG场景专用）
CREATE TABLE IF NOT EXISTS rag_knowledge_dataset (
-- 基础字段（继承 SQLBaseCol 通用字段，与 SQLBaseDO 对齐）
id BIGINT PRIMARY KEY, -- 主键ID（非自增）
tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8类型
creator VARCHAR(64) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updater VARCHAR(64) NOT NULL,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at timestamp(6),
-- 数据集核心字段（调整为允许为空）
name VARCHAR(128) NOT NULL, -- 数据集名称（核心字段仍非空）
description TEXT DEFAULT '', -- 数据集描述（长文本，适配详细描述场景）
embedding VARCHAR(64), -- 向量模型名称（允许为空）
dimension INT, -- 向量维度（允许为空）
document_count BIGINT, -- 文件数量（int64 映射为 BIGINT，允许为空）
document_size BIGINT -- 文件大小(字节)（int64 映射为 BIGINT，允许为空）
);
-- 索引（针对RAG数据集高频查询优化）
CREATE INDEX idx_dataset_tenant_id ON rag_knowledge_dataset(tenant_id); -- 租户ID索引
CREATE INDEX idx_dataset_name ON rag_knowledge_dataset(name); -- 数据集名称模糊/精准查询
CREATE INDEX idx_dataset_embedding ON rag_knowledge_dataset(embedding); -- 按向量模型筛选（允许空值，索引自动忽略NULL）
CREATE INDEX idx_dataset_deleted_at ON rag_knowledge_dataset(deleted_at); -- 软删字段索引
-- 唯一索引（保证数据集称唯一性，避免重复创建）
CREATE UNIQUE INDEX uk_dataset_name ON rag_knowledge_dataset(name) WHERE deleted_at IS NULL;
-- 表和字段注释
COMMENT ON TABLE rag_knowledge_dataset IS '数据集表（RAG场景专用）';
COMMENT ON COLUMN rag_knowledge_dataset.id IS '主键ID（非自增）';
COMMENT ON COLUMN rag_knowledge_dataset.tenant_id IS '租户ID';
COMMENT ON COLUMN rag_knowledge_dataset.creator IS '创建人';
COMMENT ON COLUMN rag_knowledge_dataset.created_at IS '创建时间';
COMMENT ON COLUMN rag_knowledge_dataset.updater IS '更新人';
COMMENT ON COLUMN rag_knowledge_dataset.updated_at IS '更新时间';
COMMENT ON COLUMN rag_knowledge_dataset.deleted_at IS '删除时间（软删）';
COMMENT ON COLUMN rag_knowledge_dataset.name IS '数据集名称';
COMMENT ON COLUMN rag_knowledge_dataset.description IS '数据集描述';
COMMENT ON COLUMN rag_knowledge_dataset.embedding IS '向量模型名称（如text-embedding-ada-002，允许为空）';
COMMENT ON COLUMN rag_knowledge_dataset.dimension IS '向量维度（对应embedding模型的输出维度，允许为空）';
COMMENT ON COLUMN rag_knowledge_dataset.document_count IS '数据集内文件数量（允许为空）';
COMMENT ON COLUMN rag_knowledge_dataset.document_size IS '数据集内文件总大小（字节，允许为空）';
--------------------pgsql创建rag_knowledge_dataset表语句---------------------------
--------------------pgsql创建rag_knowledge_document表语句---------------------------
-- RAG文件表（存储原始文件及切分相关信息，关联数据集）
CREATE TABLE IF NOT EXISTS rag_knowledge_document (
-- 基础字段（继承 SQLBaseCol 通用字段）
id BIGINT PRIMARY KEY, -- 主键ID（非自增）
tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8类型
creator VARCHAR(64) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updater VARCHAR(64) NOT NULL,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at timestamp(6),
-- 核心关联字段
dataset_id BIGINT NOT NULL, -- 关联数据集ID（新增，非空）
-- 文件核心字段
title VARCHAR(256) NOT NULL, -- 文件标题
content TEXT, -- 文件内容（长文本，允许为空：大文件内容可仅存路径，不存原文）
format VARCHAR(16) DEFAULT '', -- 文件格式: txt, md, pdf, docx, html
source VARCHAR(64) DEFAULT '', -- 来源（如：手动上传/爬虫/API导入）
source_id VARCHAR(64) DEFAULT '', -- 来源ID（如：爬虫任务ID/上传批次ID）
status SMALLINT NOT NULL DEFAULT 1, -- 状态：1启用/0停用
vector_status SMALLINT NOT NULL DEFAULT 1, -- 向量化状态: 1pending, 2processing, 3completed, 4failed,5partCompleted
chunk_count BIGINT, -- 切分后的块数量（int64映射为BIGINT，允许为空）
file_size BIGINT, -- 文件大小(字节)（int64映射为BIGINT，允许为空）
file_path VARCHAR(512) DEFAULT '', -- 文件存储路径（如MinIO路径）
metadata JSONB DEFAULT '{}'::JSONB -- 额外元数据（嵌套Metadata结构体，JSONB存储）
);
-- 单独添加外键约束（避免表定义内写约束导致的语法兼容问题）
-- 注意：执行前确保 rag_knowledge_dataset 表已存在，否则注释此行
ALTER TABLE rag_knowledge_document ADD CONSTRAINT fk_document_dataset_id FOREIGN KEY (dataset_id) REFERENCES rag_knowledge_dataset(id) ON DELETE CASCADE;
-- 索引（针对RAG文件高频查询+数据集关联优化）
CREATE INDEX idx_document_tenant_id ON rag_knowledge_document(tenant_id); -- 租户ID索引
CREATE INDEX idx_document_dataset_id ON rag_knowledge_document(dataset_id); -- 数据集关联查询（核心索引）
CREATE INDEX idx_document_title ON rag_knowledge_document(title); -- 标题模糊查询
CREATE INDEX idx_document_format ON rag_knowledge_document(format); -- 按文件格式筛选
CREATE INDEX idx_document_status ON rag_knowledge_document(status); -- 启用/停用筛选
CREATE INDEX idx_document_vector_status ON rag_knowledge_document(vector_status); -- 向量化状态筛选（核心：监控处理中/失败文件）
CREATE INDEX idx_document_source ON rag_knowledge_document(source, source_id); -- 来源+来源ID组合查询（溯源场景）
CREATE INDEX idx_document_deleted_at ON rag_knowledge_document(deleted_at); -- 软删字段索引
-- 表和字段注释
COMMENT ON TABLE rag_knowledge_document IS 'RAG文件表（存储原始文件及切分、元数据相关信息，关联数据集）';
COMMENT ON COLUMN rag_knowledge_document.id IS '主键ID（非自增）';
COMMENT ON COLUMN rag_knowledge_document.tenant_id IS '租户ID';
COMMENT ON COLUMN rag_knowledge_document.creator IS '创建人';
COMMENT ON COLUMN rag_knowledge_document.created_at IS '创建时间';
COMMENT ON COLUMN rag_knowledge_document.updater IS '更新人';
COMMENT ON COLUMN rag_knowledge_document.updated_at IS '更新时间';
COMMENT ON COLUMN rag_knowledge_document.deleted_at IS '删除时间（软删）';
COMMENT ON COLUMN rag_knowledge_document.dataset_id IS '关联数据集ID';
COMMENT ON COLUMN rag_knowledge_document.title IS '文件标题';
COMMENT ON COLUMN rag_knowledge_document.content IS '文件内容（大文件建议仅存路径，不存储原文）';
COMMENT ON COLUMN rag_knowledge_document.format IS '文件格式：txt/md/pdf/docx/html等';
COMMENT ON COLUMN rag_knowledge_document.source IS '文件来源（手动上传/爬虫/API导入等）';
COMMENT ON COLUMN rag_knowledge_document.source_id IS '来源ID（溯源标识）';
COMMENT ON COLUMN rag_knowledge_document.status IS '文件状态：1启用/0停用';
COMMENT ON COLUMN rag_knowledge_document.vector_status IS '向量化状状态：1pending-待处理/2processing-处理中/3completed-完成/4failed-失败/5partCompleted';
COMMENT ON COLUMN rag_knowledge_document.chunk_count IS '文件切分后的块数量（int64类型，未切分时为空）';
COMMENT ON COLUMN rag_knowledge_document.file_size IS '文件大小（字节，int64类型，允许为空）';
COMMENT ON COLUMN rag_knowledge_document.file_path IS '文件存储路径（如MinIO对象存储路径）';
COMMENT ON COLUMN rag_knowledge_document.metadata IS '文件元数据，结构：{"author":"作者","tags":["标签1","标签2"],"custom":{"key":"值"}}';
--------------------pgsql创建rag_knowledge_document表语句---------------------------
--------------------pgsql创建rag_knowledge_keyword表语句---------------------------
-- 关键词表（文档关键词+权重）
CREATE TABLE IF NOT EXISTS rag_knowledge_keyword (
-- 基础字段（完全对齐项目规范）
id BIGINT PRIMARY KEY, -- 主键ID（非自增）
tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8
creator VARCHAR(64) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updater VARCHAR(64) NOT NULL,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at timestamp(6),
-- 业务字段
dataset_id BIGINT NOT NULL, -- 数据集ID
document_id BIGINT NOT NULL, -- 文件ID
word VARCHAR(255) NOT NULL, -- 关键词
weight SMALLINT NOT NULL DEFAULT 0 -- 权重
);
-- 唯一索引：保证 租户 + 数据集 + 文档 + 关键词 全局唯一
CREATE UNIQUE INDEX uk_rag_knowledge_keyword_tenant_dataset_doc_word
ON rag_knowledge_keyword(tenant_id, dataset_id, document_id, word)
WHERE deleted_at IS NULL;
-- 索引（按业务高频查询）
CREATE INDEX idx_keyword_tenant_id ON rag_knowledge_keyword(tenant_id);
CREATE INDEX idx_keyword_dataset_id ON rag_knowledge_keyword(dataset_id);
CREATE INDEX idx_keyword_document_id ON rag_knowledge_keyword(document_id);
CREATE INDEX idx_keyword_word ON rag_knowledge_keyword(word);
CREATE INDEX idx_keyword_deleted_at ON rag_knowledge_keyword(deleted_at);
-- 表和字段注释
COMMENT ON TABLE rag_knowledge_keyword IS 'RAG关键词表（文档关键词+权重）';
COMMENT ON COLUMN rag_knowledge_keyword.id IS '主键ID（非自增）';
COMMENT ON COLUMN rag_knowledge_keyword.tenant_id IS '租户ID';
COMMENT ON COLUMN rag_knowledge_keyword.creator IS '创建人';
COMMENT ON COLUMN rag_knowledge_keyword.created_at IS '创建时间';
COMMENT ON COLUMN rag_knowledge_keyword.updater IS '更新人';
COMMENT ON COLUMN rag_knowledge_keyword.updated_at IS '更新时间';
COMMENT ON COLUMN rag_knowledge_keyword.deleted_at IS '删除时间（软删）';
COMMENT ON COLUMN rag_knowledge_keyword.dataset_id IS '数据集ID';
COMMENT ON COLUMN rag_knowledge_keyword.document_id IS '文档ID';
COMMENT ON COLUMN rag_knowledge_keyword.word IS '关键词';
COMMENT ON COLUMN rag_knowledge_keyword.weight IS '权重';
--------------------pgsql创建rag_knowledge_keyword表语句---------------------------