diff --git a/common/eino/chat.go b/common/eino/chat.go new file mode 100644 index 0000000..031ab9c --- /dev/null +++ b/common/eino/chat.go @@ -0,0 +1,243 @@ +package eino + +import ( + "context" + "fmt" + "rag/consts/model" + "rag/dao" + "rag/model/dto" + "rag/model/entity" + + "gitea.com/red-future/common/jaeger" + "gitea.com/red-future/common/utils" + "github.com/cloudwego/eino-ext/components/model/ark" + "github.com/cloudwego/eino-ext/components/model/arkbot" + "github.com/cloudwego/eino-ext/components/model/claude" + "github.com/cloudwego/eino-ext/components/model/deepseek" + "github.com/cloudwego/eino-ext/components/model/ollama" + "github.com/cloudwego/eino-ext/components/model/openai" + "github.com/cloudwego/eino-ext/components/model/qianfan" + "github.com/cloudwego/eino-ext/components/model/qwen" + modelChat "github.com/cloudwego/eino/components/model" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" +) + +type ChatModelSet struct { + Ark *ark.ChatModel + ArkBot *arkbot.ChatModel + Claude *claude.ChatModel + DeepSeek *deepseek.ChatModel + Ollama *ollama.ChatModel + OpenAI *openai.ChatModel + Qianfan *qianfan.ChatModel + Qwen *qwen.ChatModel +} + +// 全局租户容器：key=tenantId，value=该租户的对话模型 +var tenantChatModels = make(map[uint64]*ChatModelSet) + +func init() { + ctx := context.Background() + ctx, span := jaeger.NewSpan(ctx, "InitAllChat") + defer span.End() + InitAllChat(ctx) + return +} + +// ===================== 1. 服务启动时：初始化所有租户对话模型 ===================== +func InitAllChat(ctx context.Context) { + list, err := dao.Model.GetNoTenantId(ctx, &dto.GetModelReq{ + ModelType: model.ModelTypeChat.Code(), + }) + if err != nil { + g.Log().Errorf(ctx, "获取所有租户对话模型失败: %v", err) + return + } + + for _, l := range list { + err = InitChat(ctx, l) + if err != nil { + g.Log().Errorf(ctx, "初始化租户[%v]的对话模型失败: %v", l.TenantId, err) + continue + } + } +} + +func InitChat(ctx context.Context, modelDO *entity.Model) (err error) { + set := &ChatModelSet{} + switch *modelDO.ConfigType { + case *model.ModelConfigTypeChatArk.Code(): + var cfg entity.ChatModelConfigArk + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析Ark配置失败: %v", err) + } + set.Ark, err = ark.NewChatModel(ctx, &ark.ChatModelConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + Temperature: gconv.PtrFloat32(0.7), + MaxTokens: gconv.PtrInt(1024), + TopP: gconv.PtrFloat32(1.0), + }) + + case *model.ModelConfigTypeChatArkBot.Code(): + var cfg entity.ChatModelConfigArkBot + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析ArkBot配置失败: %v", err) + } + set.ArkBot, err = arkbot.NewChatModel(ctx, &arkbot.Config{ + APIKey: cfg.APIKey, + Model: cfg.Model, + Temperature: gconv.PtrFloat32(0.7), + MaxTokens: gconv.PtrInt(1024), + TopP: gconv.PtrFloat32(1.0), + }) + + case *model.ModelConfigTypeChatClaude.Code(): + var cfg entity.ChatModelConfigClaude + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析Claude配置失败: %v", err) + } + claudeCfg := claude.Config{ + APIKey: cfg.APIKey, + BaseURL: gconv.PtrString(cfg.BaseURL), + Model: cfg.Model, + Temperature: gconv.PtrFloat32(0.7), + MaxTokens: gconv.Int(1024), + TopP: gconv.PtrFloat32(1.0), + ByBedrock: cfg.ByBedrock, + AccessKey: cfg.AccessKey, + SecretAccessKey: cfg.SecretAccessKey, + Region: cfg.Region, + } + set.Claude, err = claude.NewChatModel(ctx, &claudeCfg) + + case *model.ModelConfigTypeChatDeepSeek.Code(): + var cfg entity.ChatModelConfigDeepSeek + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析DeepSeek配置失败: %v", err) + } + set.DeepSeek, err = deepseek.NewChatModel(ctx, &deepseek.ChatModelConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + BaseURL: cfg.BaseURL, + Temperature: gconv.Float32(0.7), + MaxTokens: gconv.Int(1024), + TopP: gconv.Float32(1.0), + }) + + case *model.ModelConfigTypeChatOllama.Code(): + var cfg entity.ChatModelConfigOllama + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析Ollama配置失败: %v", err) + } + set.Ollama, err = ollama.NewChatModel(ctx, &ollama.ChatModelConfig{ + BaseURL: cfg.BaseURL, + Model: cfg.Model, + }) + + case *model.ModelConfigTypeChatOpenAI.Code(): + var cfg entity.ChatModelConfigOpenAI + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析OpenAI配置失败: %v", err) + } + openAiCfg := openai.ChatModelConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + ByAzure: cfg.ByAzure, + BaseURL: cfg.BaseURL, + APIVersion: cfg.APIVersion, + Temperature: gconv.PtrFloat32(0.7), + MaxCompletionTokens: gconv.PtrInt(1024), + TopP: gconv.PtrFloat32(1.0), + } + set.OpenAI, err = openai.NewChatModel(ctx, &openAiCfg) + + case *model.ModelConfigTypeChatQianfan.Code(): + var cfg entity.ChatModelConfigQianfan + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析千帆配置失败: %v", err) + } + qcfg := qianfan.GetQianfanSingletonConfig() + qcfg.AccessKey = cfg.AccessKey + qcfg.SecretKey = cfg.SecretKey + set.Qianfan, err = qianfan.NewChatModel(ctx, &qianfan.ChatModelConfig{ + Model: cfg.Model, + Temperature: gconv.PtrFloat32(0.7), + MaxCompletionTokens: gconv.PtrInt(1024), + TopP: gconv.PtrFloat32(1.0), + }) + + case *model.ModelConfigTypeChatQwen.Code(): + var cfg entity.ChatModelConfigQwen + if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil { + return fmt.Errorf("解析Qwen配置失败: %v", err) + } + set.Qwen, err = qwen.NewChatModel(ctx, &qwen.ChatModelConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + BaseURL: cfg.BaseURL, + Temperature: gconv.PtrFloat32(0.7), + MaxTokens: gconv.PtrInt(1024), + TopP: gconv.PtrFloat32(1.0), + }) + + default: + return fmt.Errorf("不支持的对话模型类型: %v", *modelDO.ConfigType) + } + + if err != nil { + return fmt.Errorf("初始化对话模型失败: %v", err) + } + + // 无锁存入租户 map + tenantChatModels[modelDO.TenantId] = set + g.Log().Infof(ctx, "租户[%v]对话模型[%v]初始化成功", modelDO.TenantId, *modelDO.ConfigType) + return +} + +func GetTenantChatModel(tenantId uint64) (*ChatModelSet, error) { + set := tenantChatModels[tenantId] + if set == nil { + return nil, fmt.Errorf("租户[%v]对话模型未初始化", tenantId) + } + return set, nil +} + +func GetTenantChatModelByType(ctx context.Context, configType model.ModelConfigType) (modelChat.BaseChatModel, error) { + userInfo, err := utils.GetUserInfo(ctx) + if err != nil { + return nil, err + } + + set, err := GetTenantChatModel(userInfo.TenantId) + if set == nil { + return nil, err + } + + switch *configType { + case *model.ModelConfigTypeChatArk.Code(): + return set.Ark, nil + case *model.ModelConfigTypeChatArkBot.Code(): + return set.ArkBot, nil + case *model.ModelConfigTypeChatClaude.Code(): + return set.Claude, nil + case *model.ModelConfigTypeChatDeepSeek.Code(): + return set.DeepSeek, nil + case *model.ModelConfigTypeChatOllama.Code(): + return set.Ollama, nil + case *model.ModelConfigTypeChatOpenAI.Code(): + return set.OpenAI, nil + case *model.ModelConfigTypeChatQianfan.Code(): + return set.Qianfan, nil + case *model.ModelConfigTypeChatQwen.Code(): + return set.Qwen, nil + default: + return nil, fmt.Errorf("不支持的对话模型类型: %v", configType) + } +} + +func RefreshTenantChatModel(ctx context.Context, modelDO *entity.Model) error { + delete(tenantChatModels, modelDO.TenantId) + return InitChat(ctx, modelDO) +} diff --git a/common/eino/chat_model.go b/common/eino/chat_model.go index 9f36db0..b6aa6ca 100644 --- a/common/eino/chat_model.go +++ b/common/eino/chat_model.go @@ -5,13 +5,10 @@ import ( "errors" "fmt" "io" + "rag/consts/model" - "github.com/cloudwego/eino-ext/components/model/qwen" "github.com/cloudwego/eino/components/prompt" "github.com/cloudwego/eino/schema" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" - "github.com/gogf/gf/v2/util/gconv" ) const ( @@ -19,48 +16,15 @@ const ( ) var ( - globalChatModel *qwen.ChatModel ragPromptTemplate prompt.ChatTemplate // EINO 官方模板 ) func init() { - ctx := context.Background() - // 初始化大模型 - if err := initChatModel(ctx); err != nil { - glog.Errorf(ctx, "初始化大模型失败: %v", err) - } // 初始化 EINO 提示词模板 initRAGPromptTemplate() - return } -// 初始化通义千问 -func initChatModel(ctx context.Context) error { - if globalChatModel != nil { - return nil - } - - apiKey := g.Cfg().MustGet(ctx, "eino.chatmodel.apiKey").String() - model := g.Cfg().MustGet(ctx, "eino.chatmodel.model").String() - - cm, err := qwen.NewChatModel(ctx, &qwen.ChatModelConfig{ - APIKey: apiKey, - Model: model, - BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", - Timeout: 60 * 1e9, - Temperature: gconv.PtrFloat32(0.7), // 客服最佳 - MaxTokens: gconv.PtrInt(1024), // 最长回答 - TopP: gconv.PtrFloat32(1.0), - }) - if err != nil { - return err - } - - globalChatModel = cm - return nil -} - // 初始化 EINO 官方提示词模板（最关键！） func initRAGPromptTemplate() { ragPromptTemplate = prompt.FromMessages( @@ -69,7 +33,7 @@ func initRAGPromptTemplate() { &schema.Message{ Role: schema.System, Content: `你是专业客服，语气友好简洁。 -请严格依据参考知识回答，不知道就说：抱歉，我暂时无法回答这个问题。 +请依据参考知识回答，不知道就说：抱歉，我暂时无法回答这个问题。 参考知识： {knowledge}`, @@ -83,7 +47,7 @@ func initRAGPromptTemplate() { } // NewChatModel 只处理逻辑，不复用创建模型 -func NewChatModel(ctx context.Context, question string, docs []*schema.Document, history []*schema.Message) (replyMsg *schema.Message, err error) { +func NewChatModel(ctx context.Context, question string, docs []*schema.Document, history []*schema.Message, chatModel model.ModelConfigType) (replyMsg *schema.Message, err error) { // 1. 构建参考知识 knowledge := buildKnowledgeAndSources(docs) // 2. 历史精简 @@ -101,7 +65,7 @@ func NewChatModel(ctx context.Context, question string, docs []*schema.Document, msgs = append(msgs[:1], append(history, msgs[1:]...)...) } // 5. 🔥 直接使用全局单例，不重复创建 - replyMsg, err = streamGenerateAnswer(ctx, globalChatModel, msgs) + replyMsg, err = streamGenerateAnswer(ctx, msgs, chatModel) return } @@ -133,9 +97,14 @@ func buildKnowledgeAndSources(docs []*schema.Document) string { } // streamGenerateAnswer 流式生成 -func streamGenerateAnswer(ctx context.Context, chatModel *qwen.ChatModel, msgs []*schema.Message) (reply *schema.Message, err error) { +func streamGenerateAnswer(ctx context.Context, msgs []*schema.Message, chatModel model.ModelConfigType) (reply *schema.Message, err error) { - sr, err := chatModel.Stream(ctx, msgs) + cm, err := GetTenantChatModelByType(ctx, chatModel) + if err != nil { + return nil, err + } + + sr, err := cm.Stream(ctx, msgs) if err != nil { return nil, fmt.Errorf("stream failed: %w", err) } diff --git a/common/eino/consts.go b/common/eino/consts.go deleted file mode 100644 index e766233..0000000 --- a/common/eino/consts.go +++ /dev/null @@ -1,8 +0,0 @@ -package eino - -const ( - providerArk = "ark" - providerOpenai = "openai" - providerQianfan = "qianfan" - providerDashscope = "dashscope" -) diff --git a/common/eino/document_loader.go b/common/eino/document_loader.go index c5b7f65..bd6d143 100644 --- a/common/eino/document_loader.go +++ b/common/eino/document_loader.go @@ -5,6 +5,7 @@ import ( "fmt" "gitea.com/red-future/common/utils" + "github.com/cloudwego/eino-ext/components/document/loader/file" "github.com/cloudwego/eino-ext/components/document/loader/url" "github.com/cloudwego/eino-ext/components/document/parser/docx" "github.com/cloudwego/eino-ext/components/document/parser/pdf" @@ -15,7 +16,7 @@ import ( ) // LoadDocument 业务函数：加载文件 -func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) { +func a(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) { p, err := docsParser(ctx, fileFormat) if err != nil { return @@ -27,12 +28,34 @@ func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*sch if err != nil { return } - docs, err = loader.Load(context.Background(), document.Source{ + docs, err = loader.Load(ctx, document.Source{ URI: fmt.Sprintf("%s%s", imageUrl, filePath), }) return } +func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) { + p, err := docsParser(ctx, fileFormat) + if err != nil { + return + } + // 1. 创建文件加载器 + loader, err := file.NewFileLoader(ctx, &file.FileLoaderConfig{ + UseNameAsID: false, // 使用文件名作为文档ID + Parser: p, + }) + if err != nil { + return + } + + // 2. 加载本地文件 + docs, err = loader.Load(ctx, document.Source{ + URI: "C:\\Users\\AI\\Desktop\\手机发展史.txt", + }) + + return +} + func docsParser(ctx context.Context, fileFormat string) (p parser.Parser, err error) { switch fileFormat { case "docx": diff --git a/common/eino/document_semantic.go b/common/eino/document_semantic.go index 2f39a1a..3041c29 100644 --- a/common/eino/document_semantic.go +++ b/common/eino/document_semantic.go @@ -2,6 +2,7 @@ package eino import ( "context" + "rag/consts/model" "github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive" "github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic" @@ -10,7 +11,7 @@ import ( ) // SemanticSplitDocument 语义分割文档 -func SemanticSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) { +func SemanticSplitDocument(ctx context.Context, docs []*schema.Document, vectorModel model.ModelConfigType) (res []*schema.Document, err error) { // 默认分隔符（支持中英文） separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"} // 读取配置，使用合理的默认值 @@ -18,24 +19,14 @@ func SemanticSplitDocument(ctx context.Context, docs []*schema.Document) (res [] minChunkSize := g.Cfg().MustGet(ctx, "eino.splitter.minChunkSize").Int() percentile := g.Cfg().MustGet(ctx, "eino.splitter.percentile").Float64() batchSize := g.Cfg().MustGet(ctx, "eino.splitter.batchSize").Int() - if batchSize <= 0 { - batchSize = 10 // doubao-embedding-vision 限制每批最多 10 个 - } - // 使用批量包装器 - var batchEmbedder *BatchEmbedder - provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String() - switch provider { - case providerArk: - batchEmbedder = NewBatchEmbedder(EmbedderArk, batchSize) - case providerOpenai: - batchEmbedder = NewBatchEmbedder(EmbedderOpenAI, batchSize) - case providerDashscope: - batchEmbedder = NewBatchEmbedder(EmbedderDashscope, batchSize) + embedder, err := GetTenantEmbedderByType(ctx, vectorModel) + if err != nil { + return nil, err } splitter, err := semantic.NewSplitter(ctx, &semantic.Config{ - Embedding: batchEmbedder, + Embedding: NewBatchEmbedder(embedder, batchSize), BufferSize: bufferSize, MinChunkSize: minChunkSize, Percentile: percentile, diff --git a/common/eino/embedding.go b/common/eino/embedding.go index 7af67e3..f86635f 100644 --- a/common/eino/embedding.go +++ b/common/eino/embedding.go @@ -3,67 +3,211 @@ package eino import ( "context" "fmt" + "rag/consts/model" + "rag/model/entity" + "gitea.com/red-future/common/jaeger" + "gitea.com/red-future/common/utils" "github.com/cloudwego/eino-ext/components/embedding/ark" "github.com/cloudwego/eino-ext/components/embedding/dashscope" + "github.com/cloudwego/eino-ext/components/embedding/ollama" "github.com/cloudwego/eino-ext/components/embedding/openai" + "github.com/cloudwego/eino-ext/components/embedding/qianfan" + "github.com/cloudwego/eino-ext/components/embedding/tencentcloud" + "github.com/cloudwego/eino/components/embedding" "github.com/gogf/gf/v2/frame/g" - "github.com/golang/glog" + "github.com/gogf/gf/v2/util/gconv" ) -// 全局只初始化一次 -var ( - EmbedderArk *ark.Embedder - EmbedderDashscope *dashscope.Embedder - EmbedderOpenAI *openai.Embedder -) +type EmbedderSet struct { + Ark *ark.Embedder + Ollama *ollama.Embedder + OpenAI *openai.Embedder + Qianfan *qianfan.Embedder + TencentCloud *tencentcloud.Embedder + DashScope *dashscope.Embedder +} + +// 全局租户容器：key=tenantId，value=该租户的向量模型 +var tenantEmbedders = make(map[uint64]*EmbedderSet) func init() { ctx := context.Background() - if !g.Cfg().MustGet(ctx, "eino.embedding").IsEmpty() { - var err error - provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String() - switch provider { - case providerArk: - cfg := &ark.EmbeddingConfig{ - APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(), - Model: g.Cfg().MustGet(ctx, "eino.embedding.model").String(), - } - if apiType := g.Cfg().MustGet(ctx, "eino.embedding.apiType").String(); apiType != "" { - apiTypeVal := ark.APIType(apiType) - cfg.APIType = &apiTypeVal - } - EmbedderArk, err = ark.NewEmbedder(ctx, cfg) - case providerOpenai: - chatModelConfig := &openai.EmbeddingConfig{ - APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(), - Model: g.Cfg().MustGet(ctx, "eino.embedding.model").String(), - } - EmbedderOpenAI, err = openai.NewEmbedder(ctx, chatModelConfig) - case providerDashscope: - cfg := &dashscope.EmbeddingConfig{ - APIKey: g.Cfg().MustGet(ctx, "eino.embedding.apiKey").String(), - Model: g.Cfg().MustGet(ctx, "eino.embedding.model").String(), - } - EmbedderDashscope, err = dashscope.NewEmbedder(ctx, cfg) - } - if err != nil { - glog.Fatalf("NewEmbedder of %v error: %v", provider, err) - } - } - + ctx, span := jaeger.NewSpan(ctx, "InitAllVector") + defer span.End() + InitAllVector(ctx) return } -func EmbedStrings(ctx context.Context, texts []string) (embeddings [][]float64, err error) { - provider := g.Cfg().MustGet(ctx, "eino.embedding.provider").String() - switch provider { - case providerArk: - return EmbedderArk.EmbedStrings(ctx, texts) - case providerOpenai: - return EmbedderOpenAI.EmbedStrings(ctx, texts) - case providerDashscope: - return EmbedderDashscope.EmbedStrings(ctx, texts) +// ===================== 1. 服务启动时调用：初始化所有租户 ===================== +func InitAllVector(ctx context.Context) { + //list, err := dao.Model.GetNoTenantId(ctx, &dto.GetModelReq{ + // ModelType: model.ModelTypeVector.Code(), + //}) + //if err != nil { + // g.Log().Errorf(ctx, "获取所有租户ID失败: %v", err) + // return + //} + // + //for _, l := range list { + // err = InitVector(ctx, l) + // if err != nil { + // g.Log().Errorf(ctx, "初始化租户[%v]的向量模型失败: %v", l.TenantId, err) + // continue + // } + //} + modelDO := new(entity.Model) + modelDO.TenantId = 1 + modelDO.ConfigType = model.ModelConfigTypeVectorDashScope.Code() + var cfg entity.VectorModelConfigDashScope + cfg.APIKey = "sk-4a8b82770bf74bc490eb3e4c5a8e2be9" + cfg.Model = "text-embedding-v3" + modelDO.ConfigContent = gconv.Map(&cfg) + err := InitVector(ctx, modelDO) + if err != nil { + g.Log().Errorf(ctx, "初始化向量模型失败: %v", err) + return } - return nil, fmt.Errorf("unsupported provider: %v", provider) +} + +func InitVector(ctx context.Context, modelDO *entity.Model) (err error) { + set := &EmbedderSet{} + switch *modelDO.ConfigType { + case *model.ModelConfigTypeVectorArk.Code(): + // 解析 Ark 向量配置 + var cfg entity.VectorModelConfigArk + err = gconv.Struct(modelDO.ConfigContent, &cfg) + if err != nil { + return fmt.Errorf("解析Ark向量配置失败: %v", err) + } + arkCfg := &ark.EmbeddingConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + } + if !g.IsEmpty(cfg.APIType) { + arkCfg.APIType = new(ark.APIType(cfg.APIType)) + } + set.Ark, err = ark.NewEmbedder(ctx, arkCfg) + + case *model.ModelConfigTypeVectorOllama.Code(): + // 解析 Ollama 向量配置 + var cfg entity.VectorModelConfigOllama + err = gconv.Struct(modelDO.ConfigContent, &cfg) + if err != nil { + return fmt.Errorf("解析Ollama向量配置失败: %v", err) + } + set.Ollama, err = ollama.NewEmbedder(ctx, &ollama.EmbeddingConfig{ + BaseURL: cfg.BaseURL, + Model: cfg.Model, + }) + + case *model.ModelConfigTypeVectorOpenAI.Code(): + // 解析 OpenAI 向量配置 + var cfg entity.VectorModelConfigOpenAI + err = gconv.Struct(modelDO.ConfigContent, &cfg) + if err != nil { + return fmt.Errorf("解析OpenAI向量配置失败: %v", err) + } + openaiCfg := &openai.EmbeddingConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + ByAzure: cfg.ByAzure, + BaseURL: cfg.BaseURL, + APIVersion: cfg.APIVersion, + } + set.OpenAI, err = openai.NewEmbedder(ctx, openaiCfg) + + case *model.ModelConfigTypeVectorQianfan.Code(): + // 解析 千帆 向量配置 + var cfg entity.VectorModelConfigQianfan + err = gconv.Struct(modelDO.ConfigContent, &cfg) + if err != nil { + return fmt.Errorf("解析千帆向量配置失败: %v", err) + } + qcfg := qianfan.GetQianfanSingletonConfig() + qcfg.AccessKey = cfg.AccessKey + qcfg.SecretKey = cfg.SecretKey + set.Qianfan, err = qianfan.NewEmbedder(ctx, &qianfan.EmbeddingConfig{ + Model: cfg.Model, + }) + + case *model.ModelConfigTypeVectorTencentCloud.Code(): + // 解析 腾讯云 向量配置 + var cfg entity.VectorModelConfigTencentCloud + err = gconv.Struct(modelDO.ConfigContent, &cfg) + if err != nil { + return fmt.Errorf("解析腾讯云向量配置失败: %v", err) + } + set.TencentCloud, err = tencentcloud.NewEmbedder(ctx, &tencentcloud.EmbeddingConfig{ + SecretID: cfg.SecretID, + SecretKey: cfg.SecretKey, + Region: cfg.Region, + }) + + case *model.ModelConfigTypeVectorDashScope.Code(): + // 解析 阿里 dashscope 向量配置 + var cfg entity.VectorModelConfigDashScope + err = gconv.Struct(modelDO.ConfigContent, &cfg) + if err != nil { + return fmt.Errorf("解析阿里dashscope向量配置失败: %v", err) + } + set.DashScope, err = dashscope.NewEmbedder(ctx, &dashscope.EmbeddingConfig{ + APIKey: cfg.APIKey, + Model: cfg.Model, + }) + + default: + return fmt.Errorf("不支持的向量模型配置类型: %v", *modelDO.ConfigType) + } + + // 统一错误处理 + if err != nil { + return fmt.Errorf("初始化向量模型失败: %v", err) + } + // 直接存入 map（无锁，重复初始化会直接覆盖） + tenantEmbedders[modelDO.TenantId] = set + g.Log().Infof(ctx, "向量模型[%v]初始化成功", modelDO.ConfigType) + return +} + +func GetTenantEmbedder(tenantId uint64) (*EmbedderSet, error) { + set := tenantEmbedders[tenantId] + if set == nil { + return nil, fmt.Errorf("租户[%v]的向量模型未初始化", tenantId) + } + return set, nil +} + +func GetTenantEmbedderByType(ctx context.Context, configType model.ModelConfigType) (embedding.Embedder, error) { + userInfo, err := utils.GetUserInfo(ctx) + if err != nil { + return nil, err + } + + set, err := GetTenantEmbedder(userInfo.TenantId) + if set == nil { + return nil, err + } + + switch *configType { + case *model.ModelConfigTypeVectorArk.Code(): + return set.Ark, nil + case *model.ModelConfigTypeVectorOllama.Code(): + return set.Ollama, nil + case *model.ModelConfigTypeVectorOpenAI.Code(): + return set.OpenAI, nil + case *model.ModelConfigTypeVectorQianfan.Code(): + return set.Qianfan, nil + case *model.ModelConfigTypeVectorTencentCloud.Code(): + return set.TencentCloud, nil + case *model.ModelConfigTypeVectorDashScope.Code(): + return set.DashScope, nil + default: + return nil, fmt.Errorf("不支持的向量模型配置类型: %v", *configType) + } +} + +func RefreshTenantEmbedder(ctx context.Context, modelDO *entity.Model) error { + delete(tenantEmbedders, modelDO.TenantId) + return InitVector(ctx, modelDO) } diff --git a/common/eino/indexer.go b/common/eino/indexer.go index 76bee6f..4e983b2 100644 --- a/common/eino/indexer.go +++ b/common/eino/indexer.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "rag/consts/model" "rag/dao" "rag/model/dto" "rag/model/entity" @@ -34,7 +35,14 @@ func NewPGVectorIndexer(opts *PGVectorIndexerOptions) *PGVectorIndexer { return &PGVectorIndexer{opts: opts} } -func (i *PGVectorIndexer) Store(ctx context.Context, docs []*schema.Document, opts ...indexer.Option) (rows int64, err error) { +func (i *PGVectorIndexer) Store(ctx context.Context, docs []*schema.Document, configType model.ModelConfigType, opts ...indexer.Option) (rows int64, err error) { + + embedderByType, err := GetTenantEmbedderByType(ctx, configType) + if err != nil { + return + } + indexer.WithEmbedding(embedderByType) + commonOpts := indexer.GetCommonOptions(&indexer.Options{}, opts...) if commonOpts.Embedding == nil { diff --git a/common/eino/rerank.go b/common/eino/rerank.go new file mode 100644 index 0000000..47f2ab1 --- /dev/null +++ b/common/eino/rerank.go @@ -0,0 +1,116 @@ +package eino + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/cloudwego/eino/schema" + "github.com/gogf/gf/v2/frame/g" +) + +// DashScopeReranker 通义百炼 Rerank 精排（Cross-Encoder） +type DashScopeReranker struct { + httpClient *http.Client +} + +func NewDashScopeReranker() *DashScopeReranker { + return &DashScopeReranker{ + httpClient: &http.Client{ + Timeout: 10 * time.Second, + }, + } +} + +// Rerank 对文档进行精排（Cross-Encoder 核心） +func (d *DashScopeReranker) Rerank(ctx context.Context, query string, docs []*schema.Document) ([]*schema.Document, error) { + if len(docs) == 0 { + return docs, nil + } + + // 官方必过 URL + url := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank" + apiKey := g.Cfg().MustGet(ctx, "eino.rerank.apiKey").String() + model := g.Cfg().MustGet(ctx, "eino.rerank.model").String() + + documents := make([]string, len(docs)) + for i, doc := range docs { + documents[i] = doc.Content + } + + reqBody := map[string]any{ + "model": model, + "input": map[string]any{ + "query": query, + "documents": documents, + }, + "parameters": map[string]any{ + "top_n": len(docs), + }, + } + bs, _ := json.Marshal(reqBody) + + req, _ := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(bs)) + req.Header.Set("Authorization", "Bearer "+apiKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := d.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("rerank api error: status=%d, body=%s", resp.StatusCode, string(body)) + } + + // 解析结果 + var result struct { + Output struct { + Results []struct { + Index int `json:"index"` + RelevanceScore float64 `json:"relevance_score"` + } `json:"results"` + } `json:"output"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + // 按分数排序 + type scoredDoc struct { + doc *schema.Document + score float64 + } + scored := make([]scoredDoc, len(docs)) + for i, doc := range docs { + scored[i] = scoredDoc{doc: doc, score: 0} + } + for _, res := range result.Output.Results { + scored[res.Index].score = res.RelevanceScore + } + + // 分数从高到低排序 + for i := 0; i < len(scored); i++ { + for j := i + 1; j < len(scored); j++ { + if scored[j].score > scored[i].score { + scored[i], scored[j] = scored[j], scored[i] + } + } + } + + // 输出最终排好的文档 + ranked := make([]*schema.Document, 0, len(scored)) + for _, s := range scored { + s.doc.MetaData["rerank_score"] = s.score + ranked = append(ranked, s.doc) + } + + return ranked, nil +} diff --git a/common/eino/retriever.go b/common/eino/retriever.go index ac2529e..7be1b73 100644 --- a/common/eino/retriever.go +++ b/common/eino/retriever.go @@ -3,6 +3,8 @@ package eino import ( "context" "errors" + "fmt" + "rag/consts/model" "rag/dao" "sort" "time" @@ -29,21 +31,25 @@ type PGVectorRetriever struct { topK int index string dslInfo map[string]any + reranker *DashScopeReranker // 通义精排 } -func NewPGVectorRetriever(config *PGVectorRetrieverConfig) (*PGVectorRetriever, error) { - if config.Embedder == nil { - return nil, errors.New("embedder is required") - } +func NewPGVectorRetriever(ctx context.Context, config *PGVectorRetrieverConfig, configType model.ModelConfigType) (*PGVectorRetriever, error) { if config.DefaultTopK <= 0 { config.DefaultTopK = 5 } + e, err := GetTenantEmbedderByType(ctx, configType) + if err != nil { + return nil, err + } + return &PGVectorRetriever{ - embedder: config.Embedder, + embedder: e, topK: config.DefaultTopK, index: config.DefaultIndex, dslInfo: config.DSLInfo, + //reranker: NewDashScopeReranker(), // 👈 直接初始化你的精排 }, nil } @@ -138,48 +144,37 @@ func (r *PGVectorRetriever) Retrieve(ctx context.Context, query string, opts ... } // 合并 + 智能去重（保留最优分数） - docs := mergeAndDeduplicate(docsVector, docsFulltext) + mergedDocs := mergeAndDeduplicate(docsVector, docsFulltext) - // 排序：向量优先，同类型按距离升序 - sort.Slice(docs, func(i, j int) bool { - //byI, okI := docs[i].MetaData["retrieve_by"].(string) - //byJ, okJ := docs[j].MetaData["retrieve_by"].(string) - // - //// 有类型标记的优先 - //if okI && !okJ { - // return true - //} - //if !okI && okJ { - // return false - //} - // - //// 向量永远排前面 - //if byI == "vector" && byJ == "fulltext" { - // return true - //} - //if byI == "fulltext" && byJ == "vector" { - // return false - //} - - // 同类型按 distance 升序（越小越相似） - d1 := gconv.Float64(docs[i].MetaData["distance"]) - d2 := gconv.Float64(docs[j].MetaData["distance"]) - return d1 < d2 - }) - - // 在Retrieve方法末尾，增加相关性校验 - validDocs := make([]*schema.Document, 0) - for i, d := range docs { - // 过滤distance过大的垃圾结果（比如distance>0.8的直接丢弃） - if gconv.Float64(docs[i].MetaData["distance"]) < 0.8 { - validDocs = append(validDocs, d) + // ========================= + // 🔥 Cross-Encoder 精排 + // ========================= + var finalDocs []*schema.Document + if r.reranker != nil { + ranked, err := r.reranker.Rerank(ctx, query, mergedDocs) + if err != nil { + return nil, fmt.Errorf("rerank failed: %w", err) } + finalDocs = ranked + } else { + sort.Slice(mergedDocs, func(i, j int) bool { + d1 := gconv.Float64(mergedDocs[i].MetaData["distance"]) + d2 := gconv.Float64(mergedDocs[j].MetaData["distance"]) + return d1 < d2 + }) + finalDocs = mergedDocs } - // 如果没有有效结果，返回空，让LLM回答「暂无相关信息」 - if len(validDocs) == 0 { - callbacks.OnEnd(ctx, &retriever.CallbackOutput{Docs: validDocs}) - return validDocs, nil + // ========================= + // 过滤无效文档 + // ========================= + const maxDistance = 0.8 + validDocs := make([]*schema.Document, 0, len(finalDocs)) + for _, doc := range finalDocs { + dist := gconv.Float64(doc.MetaData["distance"]) + if dist <= maxDistance { + validDocs = append(validDocs, doc) + } } // 最多保留 topK @@ -208,9 +203,15 @@ func (r *PGVectorRetriever) doRetrieveVector(ctx context.Context, query string, if opts.TopK != nil { topK = *opts.TopK } - datasetIds := gconv.Int64s(opts.DSLInfo["dataset_ids"]) + var datasetIds, documentIds []int64 + if g.IsEmpty(opts.DSLInfo["dataset_ids"]) { + datasetIds = gconv.Int64s(opts.DSLInfo["dataset_ids"]) + } + if g.IsEmpty(opts.DSLInfo["document_ids"]) { + documentIds = gconv.Int64s(opts.DSLInfo["document_ids"]) + } - rows, err := dao.DocumentVector.GetAllByVector(ctx, datasetIds, queryVec, topK) + rows, err := dao.DocumentVector.GetAllByVector(ctx, datasetIds, documentIds, queryVec, topK) if err != nil { return nil, err } @@ -236,10 +237,17 @@ func (r *PGVectorRetriever) doRetrieveVector(ctx context.Context, query string, // ========================================== func (r *PGVectorRetriever) doRetrieveMeilisearch(ctx context.Context, query string, opts *retriever.Options) ([]*schema.Document, error) { topK := *opts.TopK - datasetIds := gconv.Int64s(opts.DSLInfo["dataset_ids"]) + + var datasetIds, documentIds []int64 + if g.IsEmpty(opts.DSLInfo["dataset_ids"]) { + datasetIds = gconv.Int64s(opts.DSLInfo["dataset_ids"]) + } + if g.IsEmpty(opts.DSLInfo["document_ids"]) { + documentIds = gconv.Int64s(opts.DSLInfo["document_ids"]) + } // 调用你已有的 Meilisearch DAO - rows, err := dao.DocumentVector.SearchByKeywords(ctx, query, datasetIds, topK) + rows, err := dao.DocumentVector.SearchByKeywords(ctx, query, datasetIds, documentIds, topK) if err != nil { return nil, err } diff --git a/consts/model/config_type.go b/consts/model/config_type.go new file mode 100644 index 0000000..d1f7039 --- /dev/null +++ b/consts/model/config_type.go @@ -0,0 +1,132 @@ +package model + +import ( + "github.com/gogf/gf/v2/util/gconv" +) + +var ( + ModelConfigTypeVectorArk = newModelConfigType(gconv.PtrString("ark"), "字节跳动火山引擎方舟大模型服务") + ModelConfigTypeVectorOllama = newModelConfigType(gconv.PtrString("ollama"), "Ollama 本地大模型运行工具") + ModelConfigTypeVectorOpenAI = newModelConfigType(gconv.PtrString("openAI"), "OpenAI 官方大模型服务") + ModelConfigTypeVectorQianfan = newModelConfigType(gconv.PtrString("qianfan"), "百度文心一言千帆大模型平台") + ModelConfigTypeVectorTencentCloud = newModelConfigType(gconv.PtrString("tencentCloud"), "腾讯云大模型服务") + ModelConfigTypeVectorDashScope = newModelConfigType(gconv.PtrString("dashScope"), "阿里云通义千问 DashScope 平台") + + ModelConfigTypeChatArk = newModelConfigType(gconv.PtrString("ark"), "字节跳动火山引擎方舟大模型服务") + ModelConfigTypeChatArkBot = newModelConfigType(gconv.PtrString("arkBot"), "火山引擎 ARK Bot 智能体服务") + ModelConfigTypeChatClaude = newModelConfigType(gconv.PtrString("claude"), "Anthropic Claude 系列大模型") + ModelConfigTypeChatDeepSeek = newModelConfigType(gconv.PtrString("deepSeek"), "DeepSeek 深度求索大模型") + ModelConfigTypeChatOllama = newModelConfigType(gconv.PtrString("ollama"), "Ollama 本地大模型运行工具") + ModelConfigTypeChatOpenAI = newModelConfigType(gconv.PtrString("openAI"), "OpenAI 官方大模型服务") + ModelConfigTypeChatQianfan = newModelConfigType(gconv.PtrString("qianfan"), "百度文心一言千帆大模型平台") + ModelConfigTypeChatQwen = newModelConfigType(gconv.PtrString("qwen"), "腾讯文心千问大模型平台") +) + +type ModelConfigType *string + +type modelConfigType struct { + code ModelConfigType + desc string +} + +func (s modelConfigType) Code() ModelConfigType { + return s.code +} +func (s modelConfigType) Desc() string { + return s.desc +} + +func newModelConfigType(code ModelConfigType, desc string) modelConfigType { + return modelConfigType{code: code, desc: desc} +} + +func GetVectorDescByCode(code ModelConfigType) string { + switch *code { + case *ModelConfigTypeVectorArk.Code(): + return ModelConfigTypeVectorArk.Desc() + case *ModelConfigTypeVectorOllama.Code(): + return ModelConfigTypeVectorOllama.Desc() + case *ModelConfigTypeVectorOpenAI.Code(): + return ModelConfigTypeVectorOpenAI.Desc() + case *ModelConfigTypeVectorQianfan.Code(): + return ModelConfigTypeVectorQianfan.Desc() + case *ModelConfigTypeVectorTencentCloud.Code(): + return ModelConfigTypeVectorTencentCloud.Desc() + case *ModelConfigTypeVectorDashScope.Code(): + return ModelConfigTypeVectorDashScope.Desc() + } + return "未知类型" +} + +func GetChatDescByCode(code ModelConfigType) string { + switch *code { + case *ModelConfigTypeChatArk.Code(): + return ModelConfigTypeChatArk.Desc() + case *ModelConfigTypeChatArkBot.Code(): + return ModelConfigTypeChatArkBot.Desc() + case *ModelConfigTypeChatClaude.Code(): + return ModelConfigTypeChatClaude.Desc() + case *ModelConfigTypeChatDeepSeek.Code(): + return ModelConfigTypeChatDeepSeek.Desc() + case *ModelConfigTypeChatOllama.Code(): + return ModelConfigTypeChatOllama.Desc() + case *ModelConfigTypeChatOpenAI.Code(): + return ModelConfigTypeChatOpenAI.Desc() + case *ModelConfigTypeChatQianfan.Code(): + return ModelConfigTypeChatQianfan.Desc() + case *ModelConfigTypeChatQwen.Code(): + return ModelConfigTypeChatQwen.Desc() + } + return "未知类型" +} + +type GetModelConfigTypeEnumRes struct { + Options []ConfigTypeKeyValue `json:"options"` +} + +type ConfigTypeKeyValue struct { + Key interface{} `json:"key"` // 对应原有常量值 + Value interface{} `json:"value"` // 对应描述信息 +} + +func GetAllModelConfigTypeEnums(modelType ModelType) *GetModelConfigTypeEnumRes { + // 枚举列表 + var list []modelConfigType + + if *modelType == *ModelTypeVector.Code() { + list = []modelConfigType{ + ModelConfigTypeVectorArk, + ModelConfigTypeVectorOllama, + ModelConfigTypeVectorOpenAI, + ModelConfigTypeVectorQianfan, + ModelConfigTypeVectorTencentCloud, + ModelConfigTypeVectorDashScope, + } + } + + if *modelType == *ModelTypeChat.Code() { + list = []modelConfigType{ + ModelConfigTypeChatArk, + ModelConfigTypeChatArkBot, + ModelConfigTypeChatClaude, + ModelConfigTypeChatDeepSeek, + ModelConfigTypeChatOllama, + ModelConfigTypeChatOpenAI, + ModelConfigTypeChatQianfan, + ModelConfigTypeChatQwen, + } + } + + // 组装返回格式 + options := make([]ConfigTypeKeyValue, 0, len(list)) + for _, item := range list { + options = append(options, ConfigTypeKeyValue{ + Key: *item.Code(), + Value: item.Desc(), + }) + } + + return &GetModelConfigTypeEnumRes{ + Options: options, + } +} diff --git a/consts/model/type.go b/consts/model/type.go new file mode 100644 index 0000000..ef0c17b --- /dev/null +++ b/consts/model/type.go @@ -0,0 +1,66 @@ +package model + +import "github.com/gogf/gf/v2/util/gconv" + +var ( + ModelTypeVector = newModelType(gconv.PtrString("vector"), "向量") + ModelTypeChat = newModelType(gconv.PtrString("chat"), "对话") +) + +type ModelType *string + +type modelType struct { + code ModelType + desc string +} + +func (s modelType) Code() ModelType { + return s.code +} +func (s modelType) Desc() string { + return s.desc +} + +func newModelType(code ModelType, desc string) modelType { + return modelType{code: code, desc: desc} +} + +func GetModelTypeDescByCode(code ModelType) string { + switch *code { + case *ModelTypeVector.Code(): + return ModelTypeVector.Desc() + case *ModelTypeChat.Code(): + return ModelTypeChat.Desc() + } + return "未知类型" +} + +type GetModelTypeEnumRes struct { + Options []TypeKeyValue `json:"options"` +} + +type TypeKeyValue struct { + Key interface{} `json:"key"` // 对应原有常量值 + Value interface{} `json:"value"` // 对应描述信息 +} + +func GetAllModelTypeEnums() *GetModelTypeEnumRes { + // 枚举列表 + list := []modelType{ + //ModelTypeVector, + ModelTypeChat, + } + + // 组装返回格式 + options := make([]TypeKeyValue, 0, len(list)) + for _, item := range list { + options = append(options, TypeKeyValue{ + Key: *item.Code(), + Value: item.Desc(), + }) + } + + return &GetModelTypeEnumRes{ + Options: options, + } +} diff --git a/consts/public/msg_key.go b/consts/public/msg_key.go index 059cafc..e211880 100644 --- a/consts/public/msg_key.go +++ b/consts/public/msg_key.go @@ -2,17 +2,10 @@ package public const GmqMsgPluginsName = "gmq_msg" -const KnowledgeLockEsKey = "rag_binary:knowledge:lock:knowledgeIdEs-%v" -const KnowledgeLockSqlKey = "rag_binary:knowledge:lock:knowledgeIdSql-%v" -const KnowledgeContentHashEsKey = "rag_binary:knowledge:knowledgeId:contentHashEs-%v" -const KnowledgeContentHashSqlKey = "rag_binary:knowledge:knowledgeId:contentHashSql-%v" - -const ( - KnowledgeDocumentVectorStatusTopic = "knowledge:document:vector:status:stream" - KnowledgeDocumentVectorStatusConsumer = "knowledge-document-vector-status-consumer" - KnowledgeDocumentVectorStatusBatchSize = 1 - KnowledgeDocumentVectorStatusAutoAck = false -) +const KnowledgeLockEsKey = "knowledge:lock:knowledgeIdEs-%v" +const KnowledgeLockSqlKey = "knowledge:lock:knowledgeIdSql-%v" +const KnowledgeContentHashEsKey = "knowledge:knowledgeId:contentHashEs-%v" +const KnowledgeContentHashSqlKey = "knowledge:knowledgeId:contentHashSql-%v" const ( KnowledgeDocumentVectorTopic = "knowledge:document:vector:stream" // 请求 Stream 键名（与发消息的key一致） diff --git a/consts/public/table_name.go b/consts/public/table_name.go index 85e2535..ae60f66 100644 --- a/consts/public/table_name.go +++ b/consts/public/table_name.go @@ -12,6 +12,7 @@ const ( TableNameDataset = "dataset" TableNameKeyword = "keyword" TableNameTask = "task" + TableNameModel = "model" TableNameDatasetIndex = "dataset_index" TableNameDocumentVector = "document_vector" ) diff --git a/controller/model.go b/controller/model.go new file mode 100644 index 0000000..ed3ace9 --- /dev/null +++ b/controller/model.go @@ -0,0 +1,52 @@ +package controller + +import ( + "context" + "rag/model/dto" + "rag/service" + + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +type model struct{} + +var Model = new(model) + +func (c *model) GetModelTypeEnums(ctx context.Context, req *dto.GetModelAllEnumsReq) (res *dto.GetModelEnumRes, err error) { + res, err = service.ModelService.GetModelAllEnums(ctx, req) + return +} + +func (c *model) GetModelConfigFormFields(ctx context.Context, req *dto.GetModelConfigFormFieldsReq) (res *dto.GetModelConfigFormFieldsRes, err error) { + res, err = service.ModelService.GetModelConfigFormFields(ctx, req) + return +} + +func (c *model) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) { + res, err = service.ModelService.Create(ctx, req) + return +} + +func (c *model) Update(ctx context.Context, req *dto.UpdateModelReq) (res *beans.ResponseEmpty, err error) { + err = service.ModelService.Update(ctx, req) + return +} + +func (c *model) Delete(ctx context.Context, req *dto.DeleteModelReq) (res *beans.ResponseEmpty, err error) { + err = service.ModelService.Delete(ctx, req) + return +} + +func (c *model) Get(ctx context.Context, req *dto.GetModelReq) (res *dto.ModelVO, err error) { + res, err = service.ModelService.Get(ctx, req) + return +} + +func (c *model) List(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) { + if !g.IsEmpty(req.Page) { + req.Page = &beans.Page{PageNum: 1, PageSize: 20} + } + res, err = service.ModelService.List(ctx, req) + return +} diff --git a/controller/task.go b/controller/task.go new file mode 100644 index 0000000..cf72c17 --- /dev/null +++ b/controller/task.go @@ -0,0 +1,16 @@ +package controller + +import ( + "context" + "rag/model/dto" + "rag/service" +) + +type task struct{} + +var Task = new(task) + +func (c *task) Get(ctx context.Context, req *dto.GetTaskReq) (res *dto.ListTaskRes, err error) { + res, err = service.Task.Get(ctx, req) + return +} diff --git a/dao/document.go b/dao/document.go index 5f943fc..dfcfb28 100644 --- a/dao/document.go +++ b/dao/document.go @@ -48,16 +48,28 @@ func (d *documentDao) Update(ctx context.Context, req *dto.UpdateDocumentReq) (r // Delete 删除文件 func (d *documentDao) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (rows int64, err error) { - r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Delete() + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty(). + Where(entity.DocumentCol.Id, req.Id). + Delete() if err != nil { return } return r.RowsAffected() } -// GetByID 根据ID获取文件 -func (d *documentDao) GetByID(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) { - r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Fields(fields).One() +func (d *documentDao) Count(ctx context.Context, req *dto.ListDocumentReq) (count int, err error) { + count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty(). + Where(entity.DocumentCol.DatasetId, req.DatasetId). + Where(entity.DocumentCol.Title, req.Title).Count() + return +} + +// Get 根据ID获取文件 +func (d *documentDao) Get(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) { + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Fields(fields).OmitEmpty(). + Where(entity.DocumentCol.Id, req.Id). + Where(entity.DocumentCol.Title, req.Title). + Where(entity.DocumentCol.DatasetId, req.DatasetId).One() if err != nil { return } diff --git a/dao/document_vector.go b/dao/document_vector.go index 96ecd5b..0b55556 100644 --- a/dao/document_vector.go +++ b/dao/document_vector.go @@ -43,17 +43,29 @@ func (d *documentVectorDao) Update(ctx context.Context, req *dto.UpdateDocumentV return r.RowsAffected() } +func (d *documentVectorDao) Delete(ctx context.Context, req *dto.DeleteDocumentVectorReq) (rows int64, err error) { + result, err := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).OmitEmpty(). + Where(entity.DocumentVectorCol.Id, req.Id). + Where(entity.DocumentVectorCol.DocumentId, req.DocumentId). + Delete() + if err != nil { + return + } + return result.RowsAffected() +} + // List 文件块列表 func (d *documentVectorDao) List(ctx context.Context, req *dto.ListDocumentVectorReq, fields ...string) (res []*entity.DocumentVector, total int, err error) { model := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).Fields(fields).OmitEmpty(). Where(entity.DocumentVectorCol.DatasetId, req.DatasetId). Where(entity.DocumentVectorCol.DocumentId, req.DocumentId). Where(entity.DocumentVectorCol.Status, req.Status). - Where(entity.DocumentVectorCol.VectorStatus, req.VectorStatus) + Where(entity.DocumentVectorCol.VectorStatus, req.VectorStatus). + WhereIn(entity.DocumentVectorCol.DocumentId, req.DocumentIds) if !g.IsEmpty(req.Keyword) { model.WhereLike(entity.DocumentVectorCol.Content, "%"+req.Keyword+"%") } - model.OrderDesc(entity.DocumentVectorCol.CreatedAt) + model.OrderAsc(entity.DocumentVectorCol.ChunkIndex) if req.Page != nil { model.Page(int(req.Page.PageNum), int(req.Page.PageSize)) } @@ -65,28 +77,32 @@ func (d *documentVectorDao) List(ctx context.Context, req *dto.ListDocumentVecto return } -func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds []int64, vector pgvector.Vector, topK int) (list gdb.List, err error) { - //result, err := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector). - // Fields("id, content, dataset_id, document_id, vector <=> ? AS distance"). - // WhereIn(entity.DocumentVectorCol.DatasetId, datasetIds). - // WhereNotNull(entity.DocumentVectorCol.Vector). - // OrderAsc("distance"). - // Limit(topK). - // All() - //if err != nil { - // return nil, err - //} +func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds, documentIds []int64, vector pgvector.Vector, topK int) (list gdb.List, err error) { + // 动态拼接 WHERE 条件 + var whereCondition string + var queryParams []interface{} + + // 优先使用 documentIds 查询 + if len(documentIds) > 0 { + whereCondition = fmt.Sprintf(" AND %s IN (?) ", entity.DocumentVectorCol.DocumentId) + queryParams = append(queryParams, documentIds) + } + if len(datasetIds) > 0 { + whereCondition = fmt.Sprintf(" AND %s IN (?) ", entity.DocumentVectorCol.DatasetId) + queryParams = append(queryParams, datasetIds) + } + + // 完整 SQL sql := ` - SELECT id, content, dataset_id, document_id, - vector <=> ? AS distance + SELECT id, content, dataset_id, document_id,vector <=> ? AS distance FROM rag_vector_document_vector - WHERE dataset_id IN (?) - AND vector IS NOT NULL - ORDER BY distance ASC - LIMIT ? -` - // 顺序：vector, dataset_id, topK - result, err := gfdb.DB(ctx, public.DbNameVector).GetAll(ctx, sql, vector, datasetIds, topK) + WHERE 1=1 ` + whereCondition + ` AND vector IS NOT NULL ORDER BY distance ASC LIMIT ?` + // 拼接参数：vector + 条件参数 + topK + queryParams = append([]interface{}{vector}, queryParams...) + queryParams = append(queryParams, topK) + + // 执行查询 + result, err := gfdb.DB(ctx, public.DbNameVector).GetAll(ctx, sql, queryParams...) if err != nil { return nil, err } @@ -94,7 +110,7 @@ func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds []int } // SearchByKeywords 通过关键词全文检索文档块 -func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, datasetIds []int64, topK int) (list gdb.List, err error) { +func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, datasetIds, documentIds []int64, topK int) (list gdb.List, err error) { // 构建 meilisearch 查询参数 searchParams := &meilisearch.SearchParams{ Query: query, @@ -102,14 +118,22 @@ func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, ShowRankingScore: true, } - // 构建 datasetIds 过滤条件 if len(datasetIds) > 0 { datasetIdStrs := gconv.Strings(datasetIds) quotedIds := make([]string, len(datasetIdStrs)) for i, id := range datasetIdStrs { quotedIds[i] = fmt.Sprintf("%s", id) } - searchParams.Filter = fmt.Sprintf("dataset_id IN [%s]", gstr.Implode(", ", quotedIds)) + searchParams.Filter = fmt.Sprintf("%s IN [%s]", entity.DocumentVectorCol.DatasetId, gstr.Implode(", ", quotedIds)) + } + + if len(documentIds) > 0 { + documentIdStrs := gconv.Strings(documentIds) + quotedIds := make([]string, len(documentIdStrs)) + for i, id := range documentIdStrs { + quotedIds[i] = fmt.Sprintf("%s", id) + } + searchParams.Filter = fmt.Sprintf("%s IN [%s]", entity.DocumentVectorCol.DocumentId, gstr.Implode(", ", quotedIds)) } // 执行搜索 @@ -124,6 +148,5 @@ func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, for _, hit := range hits { resultList = append(resultList, hit) } - return resultList, nil } diff --git a/dao/keyword.go b/dao/keyword.go index 589799f..445fd2e 100644 --- a/dao/keyword.go +++ b/dao/keyword.go @@ -53,7 +53,10 @@ func (d *keywordDao) Update(ctx context.Context, req *dto.UpdateKeywordReq) (row } func (d *keywordDao) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (rows int64, err error) { - r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Delete() + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).OmitEmpty(). + Where(entity.KeywordCol.Id, req.Id). + Where(entity.KeywordCol.DocumentId, req.DocumentId). + Delete() if err != nil { return } diff --git a/dao/model.go b/dao/model.go new file mode 100644 index 0000000..ccb4eb5 --- /dev/null +++ b/dao/model.go @@ -0,0 +1,88 @@ +package dao + +import ( + "context" + "rag/consts/public" + "rag/model/dto" + "rag/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" +) + +var Model = new(modelDao) + +type modelDao struct{} + +func (d *modelDao) Insert(ctx context.Context, req *dto.CreateModelReq) (id int64, err error) { + var res *entity.Model + if err = gconv.Struct(req, &res); err != nil { + return + } + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Data(&res).Insert() + if err != nil { + return + } + return r.LastInsertId() +} + +func (d *modelDao) Update(ctx context.Context, req *dto.UpdateModelReq) (rows int64, err error) { + model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).OmitEmpty() + r, err := model.Data(&req).Where(entity.ModelCol.Id, req.Id).Update() + if err != nil { + return + } + return r.RowsAffected() +} + +func (d *modelDao) Delete(ctx context.Context, req *dto.DeleteModelReq) (rows int64, err error) { + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Where(entity.ModelCol.Id, req.Id).Delete() + if err != nil { + return + } + return r.RowsAffected() +} + +func (d *modelDao) Count(ctx context.Context, req *dto.GetModelReq) (count int, err error) { + count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).OmitEmpty().Where(entity.ModelCol.ModelType, req.ModelType).Count() + return +} + +func (d *modelDao) Get(ctx context.Context, req *dto.GetModelReq, fields ...string) (res *entity.Model, err error) { + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Fields(fields).OmitEmpty(). + Where(entity.ModelCol.Id, req.Id). + Where(entity.ModelCol.ModelType, req.ModelType).One() + if err != nil { + return + } + err = r.Struct(&res) + return +} + +func (d *modelDao) GetNoTenantId(ctx context.Context, req *dto.GetModelReq, fields ...string) (res []*entity.Model, err error) { + r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).NoTenantId(ctx).Where(entity.ModelCol.ModelType, req.ModelType).Fields(fields).All() + if err != nil { + return + } + err = r.Structs(&res) + return +} + +func (d *modelDao) List(ctx context.Context, req *dto.ListModelReq, fields ...string) (res []*entity.Model, total int, err error) { + model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Fields(fields).OmitEmpty() + if !g.IsEmpty(req.ModelName) { + model.WhereLike(entity.ModelCol.ModelName, "%"+req.ModelName+"%") + } + model.Where(entity.ModelCol.ModelType, req.ModelType) + model.OrderDesc(entity.KeywordCol.CreatedAt) + if req.Page != nil { + model.Page(int(req.Page.PageNum), int(req.Page.PageSize)) + } + r, total, err := model.AllAndCount(false) + if err != nil { + return + } + err = r.Structs(&res) + return +} diff --git a/dao/task.go b/dao/task.go index 023ccb2..a233195 100644 --- a/dao/task.go +++ b/dao/task.go @@ -37,6 +37,15 @@ func (d *taskDao) Update(ctx context.Context, req *dto.UpdateTaskReq) (rows int6 return r.RowsAffected() } +func (d *taskDao) Count(ctx context.Context, req *dto.GetTaskReq) (count int, err error) { + count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty(). + Where(entity.TaskCol.TaskId, req.TaskId). + Where(entity.TaskCol.TaskType, req.TaskType). + Where(entity.TaskCol.Status, req.TaskStatus). + Count() + return +} + func (d *taskDao) Get(ctx context.Context, req *dto.GetTaskReq) (res []*entity.Task, total int, err error) { r, total, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty(). Where(entity.TaskCol.Id, req.Id). diff --git a/go.mod b/go.mod index 028fb43..72673bd 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( gitea.com/red-future/common v0.0.12 github.com/bjang03/gmq v0.0.0-00010101000000-000000000000 github.com/cloudwego/eino v0.8.6 + github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250519091007-282cc7eb18d3 github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419 github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419 github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20260323112355-f061db7e8419 @@ -14,11 +15,20 @@ require ( github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic v0.0.0-20260323112355-f061db7e8419 github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1 github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419 + github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260413110502-8d10f059c9a4 github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419 + github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260413110502-8d10f059c9a4 + github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260413110502-8d10f059c9a4 + github.com/cloudwego/eino-ext/components/model/ark v0.1.65 + github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2 + github.com/cloudwego/eino-ext/components/model/claude v0.1.17 + github.com/cloudwego/eino-ext/components/model/deepseek v0.1.2 + github.com/cloudwego/eino-ext/components/model/ollama v0.1.9 + github.com/cloudwego/eino-ext/components/model/openai v0.1.12 + github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4 github.com/cloudwego/eino-ext/components/model/qwen v0.1.7 github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0 github.com/gogf/gf/v2 v2.10.0 - github.com/golang/glog v1.2.5 github.com/pgvector/pgvector-go v0.3.0 ) @@ -27,13 +37,33 @@ replace gitea.com/red-future/common v0.0.12 => ../common replace github.com/bjang03/gmq => ../gmq require ( + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/anthropics/anthropic-sdk-go v1.26.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14 // indirect + github.com/baidubce/bce-sdk-go v0.9.164 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect @@ -47,7 +77,8 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20241224063832-9fbcc0e56c28 // indirect - github.com/cloudwego/eino-ext/libs/acl/openai v0.1.15 // indirect + github.com/cloudwego/eino-ext/libs/acl/openai v0.1.16 // indirect + github.com/cohesion-org/deepseek-go v1.3.2 // indirect github.com/dgraph-io/badger/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -55,9 +86,11 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/eino-contrib/docx2md v0.0.1 // indirect github.com/eino-contrib/jsonschema v1.0.3 // indirect + github.com/eino-contrib/ollama v0.1.0 // indirect github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect github.com/evanphx/json-patch v0.5.2 // indirect github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-ego/gse v1.0.2 // indirect @@ -72,14 +105,18 @@ require ( github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golang/glog v1.2.5 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect - github.com/google/flatbuffers v1.12.1 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/goph/emperror v0.17.2 // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grokify/html-strip-tags-go v0.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/hashicorp/consul/api v1.26.1 // indirect @@ -90,8 +127,10 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -99,6 +138,7 @@ require ( github.com/lib/pq v1.12.1 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mailru/easyjson v0.9.0 // indirect + github.com/matoous/go-nanoid v1.5.1 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.21 // indirect @@ -108,7 +148,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/nats-io/nats.go v1.49.0 // indirect github.com/nats-io/nkeys v0.4.15 // indirect @@ -118,6 +158,7 @@ require ( github.com/olekukonko/errors v1.2.0 // indirect github.com/olekukonko/ll v0.1.8 // indirect github.com/olekukonko/tablewriter v1.1.4 // indirect + github.com/ollama/ollama v0.9.6 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/r3labs/diff/v2 v2.15.1 // indirect @@ -125,9 +166,22 @@ require ( github.com/redis/go-redis/v9 v9.18.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/spf13/viper v1.18.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093 // indirect + github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/tiger1103/gfast-token v1.0.10 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/vcaesar/cedar v0.30.0 // indirect @@ -140,8 +194,10 @@ require ( github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect github.com/yargevad/filepathx v1.0.0 // indirect go.mongodb.org/mongo-driver/v2 v2.4.0 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect @@ -150,18 +206,23 @@ require ( go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.15.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect golang.org/x/net v0.52.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/api v0.189.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/grpc v1.75.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e6cf737..1608d5e 100644 --- a/go.sum +++ b/go.sum @@ -13,12 +13,18 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -62,6 +68,8 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAfT7CoSYSac11PY= +github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -73,12 +81,44 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= +github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= +github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= +github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14 h1:XNP24illv5CWTLinpdF8Xo73YWQ2ZWbmlNT0BTWFCGg= +github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14/go.mod h1:f/kIWWvAHAcU7bzgkfN30SkpN0I4lLvsJkljVK6v5YY= +github.com/baidubce/bce-sdk-go v0.9.164 h1:7gswLMsdQyarovMKuv3i6wxFQ3BQgvc5CmyGXb/D/xA= +github.com/baidubce/bce-sdk-go v0.9.164/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -134,6 +174,8 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cloudwego/eino v0.8.6 h1:Rc9/ElXNrTrSCv68t/U0yUmNVu5uMmpPyMCb+WyFIQQ= github.com/cloudwego/eino v0.8.6/go.mod h1:+2N4nsMPxA6kGBHpH+75JuTfEcGprAMTdsZESrShKpU= +github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250519091007-282cc7eb18d3 h1:ykb5Nz6WZR6U3CgffUIxdPWi8lLttvhOeA3gYqbXOpY= +github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250519091007-282cc7eb18d3/go.mod h1:I4vbBCIMMKeF436Lc+L3DSPQ3f1nmiHD0JS+LhMYCdQ= github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419 h1:dMr31rw5pjZMKMPEvNvpy+1RI3HnqVUWmk2abNkb+yM= github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419/go.mod h1:/IeSk52Hhym5AUjCs3ESTF5Nb0RLYFWW8llSpWuc/JA= github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419 h1:NkEyzkMvYj1imCNclbL0OeesosinmbU10uW+ZlC0vtA= @@ -152,15 +194,39 @@ github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1 h1:PM/+XAvJtrBqFlB github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1/go.mod h1:6O6x0fHfM3uCLr3lX1DnB/my7fC3WRUA5hpkCkrkZrg= github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419 h1:gGnohcgEaHqp5V826Ay0H3fi4TpK8ReWlUPePAnzvA4= github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419/go.mod h1:ekJmA+GLD9vJyZNeODZDBFMiJ92Suy6nF0OY42X3sao= +github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260413110502-8d10f059c9a4 h1:j17fEjj9dsLFIVYnh7lLyxrGNiuQF2E9iyDAIZ6Ohf8= +github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260413110502-8d10f059c9a4/go.mod h1:mI8QMT4DtgLGUuMTVFDNIgRFmirA//do8UnLmZg0DZ4= github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419 h1:eM29lyMShtFZNoAhE5g96+zHg9PBLckRyd2HtVeeY4E= github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419/go.mod h1:SajSFFRIXJXIbxadAAlSUIS5KTY8R/jzJg9RNSOXCCI= +github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260413110502-8d10f059c9a4 h1:NsZBgdDn2Qmga8B81z+Rbxz0ouy/CDDoMUW5rq8Im/o= +github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260413110502-8d10f059c9a4/go.mod h1:Ga/ANlFO4JGsNPk8vAC+v7hiGYfLmXpYqrihiOpR26I= +github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260413110502-8d10f059c9a4 h1:F1l0lHLLErXnVAKho7or9NTM6RIdT3UWBFQlCEUHmUE= +github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260413110502-8d10f059c9a4/go.mod h1:SM49B6gK7V7nR/7/dXSp0Z2pAy38uu/ZaREbs/9S5Vw= +github.com/cloudwego/eino-ext/components/model/ark v0.1.65 h1:52ukXVU9ntToTa36SwI8be81qskGkpUEZraIFOf0wqk= +github.com/cloudwego/eino-ext/components/model/ark v0.1.65/go.mod h1:aabMR15RTXBSi9Eu13CWavzE+no5BQO4FJUEEdqImbg= +github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2 h1:dpVEadipJKrsQ8JBSpNcosQrhis+lOIivnSmGla9+v4= +github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2/go.mod h1:sEsPSnIiB+zi7MjG++MLOPTJd7cT2gqfcK4pMpAkleI= +github.com/cloudwego/eino-ext/components/model/claude v0.1.17 h1:QcK41lNtNfWBAXnaEGjAUtrbC+t8n1PMH9OSl9qdVu4= +github.com/cloudwego/eino-ext/components/model/claude v0.1.17/go.mod h1:2sGGgwpR60LW+RdG/hcjdGBVEVEJ6EkKxlva8mju9BI= +github.com/cloudwego/eino-ext/components/model/deepseek v0.1.2 h1:PSHIDLUOv3ZCO7G6ZXnuJWb5pvRZV6xnfLLbwbfY704= +github.com/cloudwego/eino-ext/components/model/deepseek v0.1.2/go.mod h1:beCP+L7CsxDz4+DvBjo8iR/v/ZBPpmQfJtrqG280rjw= +github.com/cloudwego/eino-ext/components/model/ollama v0.1.9 h1:+eZbquy5lF3WHvK9+T7UUqI0CTRqDEniP7fzL85lJuk= +github.com/cloudwego/eino-ext/components/model/ollama v0.1.9/go.mod h1:C3rf3yy2nEoXFP/CQJne4gbiu1pREKplHKmFlhuOzPE= +github.com/cloudwego/eino-ext/components/model/openai v0.1.12 h1:vcwNXeT7bpaXMNwUhtcHZwMYY8II2jAihuooyivmEZ0= +github.com/cloudwego/eino-ext/components/model/openai v0.1.12/go.mod h1:ve/+/hLZMvxD5AieQ355xHIFhAZVlsG4rdwTnE16aQU= +github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4 h1:wCl6EDT4eacyBAopQ0N6bQVwRy6wW9HNmIag9zBljkI= +github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4/go.mod h1:ShVCwEhltA7hyc4jYfxMS5TbF6N/RATlNHt/jpGgzWI= github.com/cloudwego/eino-ext/components/model/qwen v0.1.7 h1:8c1LB5lH+dERbf2twp18B1Y822JOQSsS6x7Vnksehk0= github.com/cloudwego/eino-ext/components/model/qwen v0.1.7/go.mod h1:n4iuIUQeL3D8GRsGAhkgceRZpoyPQbqOXFMXM2Q4hNY= -github.com/cloudwego/eino-ext/libs/acl/openai v0.1.15 h1:LbdSG9+qWzzp9RFW6dSFkaUW171JvCoYn/K63zX6dQE= -github.com/cloudwego/eino-ext/libs/acl/openai v0.1.15/go.mod h1:p+l0zBB0GjjX8HTlbTs3g3KfUFwZC11bsCGZOXW/3L0= +github.com/cloudwego/eino-ext/devops v0.1.8 h1:qBg5vjZSDnd9tHzCHG8YsjnGB5vKG2EoZuuQCI8qrGs= +github.com/cloudwego/eino-ext/devops v0.1.8/go.mod h1:8yjvPNTaB5Ve4aJmJ0ysFgB10y3YbIuqMh0/Uwt5Fnw= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.16 h1:q242n5P5Tx3a2QLaBmkfEpfRs/o17Ac6u3EAgItEEOc= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.16/go.mod h1:p+l0zBB0GjjX8HTlbTs3g3KfUFwZC11bsCGZOXW/3L0= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cohesion-org/deepseek-go v1.3.2 h1:WTZ/2346KFYca+n+DL5p+Ar1RQxF2w/wGkU4jDvyXaQ= +github.com/cohesion-org/deepseek-go v1.3.2/go.mod h1:bOVyKj38r90UEYZFrmJOzJKPxuAh8sIzHOCnLOpiXeI= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -177,6 +243,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dslipak/pdf v0.0.2 h1:djAvcM5neg9Ush+zR6QXB+VMJzR6TdnX766HPIg1JmI= github.com/dslipak/pdf v0.0.2/go.mod h1:2L3SnkI9cQwnAS9gfPz2iUoLC0rUZwbucpbKi5R1mUo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -191,6 +259,8 @@ github.com/eino-contrib/docx2md v0.0.1 h1:Clz0sF8jiQRYAIZAUTuTAjh0vF/1KqHQqsMha1 github.com/eino-contrib/docx2md v0.0.1/go.mod h1:b1dupA9cF5yExHjVMCcP6feyE6mwZjsY7Cc9ESO5Y14= github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0= github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/eino-contrib/ollama v0.1.0 h1:z1NaMdKW6X1ftP8g5xGGR5zDRPUtuTKFq35vBQgxsN4= +github.com/eino-contrib/ollama v0.1.0/go.mod h1:mYsQ7b3DeqY8bHPuD3MZJYTqkgyL6LoemxoP/B7ZNhA= github.com/emirpasic/gods/v2 v2.0.0-alpha h1:dwFlh8pBg1VMOXWGipNMRt8v96dKAIvBehtCt6OtunU= github.com/emirpasic/gods/v2 v2.0.0-alpha/go.mod h1:W0y4M2dtBB9U5z3YlghmpuUhiaZT2h6yoeE+C1sCp6A= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -208,6 +278,8 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= @@ -322,8 +394,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= -github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -350,10 +422,14 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= @@ -366,10 +442,12 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4= github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -422,6 +500,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= @@ -460,6 +540,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -504,6 +586,8 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/matoous/go-nanoid v1.5.1 h1:aCjdvTyO9LLnTIi0fgdXhOPPvOHjpXN6Ik9DaNjIct4= +github.com/matoous/go-nanoid v1.5.1/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -553,8 +637,9 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -584,6 +669,8 @@ github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8= github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw= github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= +github.com/ollama/ollama v0.9.6 h1:HZNJmB52pMt6zLkGkkheBuXBXM5478eiSAj7GR75AMc= +github.com/ollama/ollama v0.9.6/go.mod h1:zLwx3iZ3AI4Rc/egsrx3u1w4RU2MHQ/Ylxse48jvyt4= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -657,6 +744,10 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -675,9 +766,17 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -696,10 +795,27 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093 h1:pkz4SrPMy3TLKwqwCH6gIgv6TqEvJKkyiq0sEFl8wb0= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093 h1:YlJETpB0b4KtK3Km8ak+Td1WqY8kQYaF+r3LId/hOc0= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093/go.mod h1:IUc23LMmefQegnwF1l9DlvIbgFY5xD9AzpWVaBiBYBk= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tiger1103/gfast-token v1.0.10 h1:fNiBE/Dq5iTHvTGlCx3DmXa2o4hr0NtumFpffZ39k6s= github.com/tiger1103/gfast-token v1.0.10/go.mod h1:a/21mxmj7zFeNvjhZSC0XpEAFHfb1aT2k6DXnufFU1s= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= @@ -771,10 +887,15 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= @@ -803,6 +924,8 @@ go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= @@ -843,8 +966,8 @@ golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/usc golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/image v0.22.0 h1:UtK5yLUzilVrkjMAZAZ34DXGpASN8i8pj8g+O+yd10g= +golang.org/x/image v0.22.0/go.mod h1:9hPFhljd4zZ1GNSIZJ49sqbp45GKK9t6w+iXvGqZUz4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -924,6 +1047,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1035,6 +1160,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1114,6 +1241,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1204,6 +1333,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/main.go b/main.go index 60efb8e..67fc776 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,9 @@ func main() { controller.Dataset, controller.Document, controller.DocumentVector, + controller.Model, controller.Keyword, + controller.Task, }) if err := utils.InitGseTool(ctx); err != nil { diff --git a/model/dto/document.go b/model/dto/document.go index c597a33..a3b1ad0 100644 --- a/model/dto/document.go +++ b/model/dto/document.go @@ -6,6 +6,7 @@ import ( "gitea.com/red-future/common/beans" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" + "github.com/pgvector/pgvector-go" ) // CreateDocumentReq 创建文件请求 @@ -45,7 +46,9 @@ type DeleteDocumentReq struct { type GetDocumentReq struct { g.Meta `path:"/get" method:"get" tags:"文件管理" summary:"获取文件详情" dc:"获取文件详情"` - Id int64 `json:"id" v:"required#ID不能为空"` + Id int64 `json:"id" v:"required#ID不能为空"` + DatasetId int64 `json:"datasetId"` + Title string `json:"title"` } type GetDocumentRes struct { @@ -60,6 +63,7 @@ type ListDocumentReq struct { Page *beans.Page `json:"page"` DatasetId int64 `json:"datasetId"` Keyword string `json:"keyword" dc:"关键词搜索"` + Title string `json:"title" dc:"文件标题"` Status document.Status `json:"status"` } @@ -92,7 +96,9 @@ type DocumentVectorReq struct { } type DocumentVectorRPC struct { - Id int64 `json:"id" dc:"id"` - DatasetId int64 `json:"datasetId" dc:"所属数据集ID"` - ContentHash string `json:"contentHash" dc:"内容hash"` + Id int64 `json:"id" dc:"id"` + DatasetId int64 `json:"datasetId" dc:"所属数据集ID"` + DocumentId int64 `json:"documentId" dc:"文件ID"` + ContentHash string `json:"contentHash" dc:"内容hash"` + Vector pgvector.Vector `json:"vector" dc:"向量"` } diff --git a/model/dto/document_vector.go b/model/dto/document_vector.go index 31c2acb..2f816cd 100644 --- a/model/dto/document_vector.go +++ b/model/dto/document_vector.go @@ -13,10 +13,11 @@ import ( type RAGQueryReq struct { g.Meta `path:"/ragQuery" method:"post" tags:"RAG查询" summary:"执行RAG查询" dc:"执行RAG查询"` - Content string `json:"content" v:"required#查询内容不能为空" dc:"用户问题"` - DatasetIds []int64 `json:"datasetIds" dc:"数据集ID"` - History []*Message `json:"history" dc:"历史对话"` - TopK int `json:"topK" d:"5" dc:"检索topK，默认5"` + Content string `json:"content" v:"required#查询内容不能为空" dc:"用户问题"` + DatasetIds []int64 `json:"datasetIds" dc:"数据集ID"` + DocumentIds []int64 `json:"documentIds" dc:"文档ID"` + History []*Message `json:"history" dc:"历史对话"` + TopK int `json:"topK" d:"5" dc:"检索topK，默认5"` } type Message struct { @@ -37,6 +38,13 @@ type UpdateDocumentVectorReq struct { Status document.Status `json:"status"` } +type DeleteDocumentVectorReq struct { + g.Meta `path:"/delete" method:"put" tags:"文件块向量管理" summary:"删除文件块" dc:"删除文件块"` + + Id int64 `json:"id"` + DocumentId int64 `json:"documentId"` +} + // ListDocumentVectorReq 文件块向量列表请求 type ListDocumentVectorReq struct { g.Meta `path:"/list" method:"get" tags:"文件块向量管理" summary:"获取文件块向量列表" dc:"分页查询文件块向量列表，支持多条件筛选"` @@ -45,6 +53,8 @@ type ListDocumentVectorReq struct { Keyword string `json:"keyword" dc:"关键词搜索"` DatasetId int64 `json:"datasetId"` DocumentId int64 `json:"documentId"` + DocumentIds []int64 `json:"documentIds"` + ContentHashs []string `json:"contentHash"` Status document.Status `json:"status"` VectorStatus document.VectorStatus `json:"vectorStatus"` } diff --git a/model/dto/keyword.go b/model/dto/keyword.go index cd2ecce..7a0cdfa 100644 --- a/model/dto/keyword.go +++ b/model/dto/keyword.go @@ -37,7 +37,8 @@ type UpdateKeywordReq struct { type DeleteKeywordReq struct { g.Meta `path:"/delete" method:"delete" tags:"关键词管理" summary:"删除关键词" dc:"删除关键词"` - Id int64 `json:"id" v:"required#ID不能为空"` + Id int64 `json:"id"` + DocumentId int64 `json:"documentId"` } // GetKeywordReq 获取关键词请求 diff --git a/model/dto/model.go b/model/dto/model.go new file mode 100644 index 0000000..6f2c63a --- /dev/null +++ b/model/dto/model.go @@ -0,0 +1,114 @@ +package dto + +import ( + "rag/consts/model" + "time" + + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +type GetModelAllEnumsReq struct { + g.Meta `path:"/getAllEnums" method:"get" tags:"模型配置管理" summary:"获取全量模型枚举（类型+配置）"` +} + +type GetModelEnumRes struct { + Options []ModelEnumOption `json:"options"` +} + +// ModelEnumOption 主类型：模型类型（vector/chat） +type ModelEnumOption struct { + Key interface{} `json:"key"` + Value interface{} `json:"value"` + ConfigTypes []ModelKeyValue `json:"configTypes"` // 这里统一！ +} + +// ModelKeyValue 统一的 KV 结构 → 给模型类型 + 配置类型共用 +type ModelKeyValue struct { + Key interface{} `json:"key"` + Value interface{} `json:"value"` +} + +// GetModelConfigFormFieldsReq 获取模型配置表单请求 +type GetModelConfigFormFieldsReq struct { + g.Meta `path:"/getModelFormField" method:"get" tags:"模型配置管理" summary:"获取模型表单" dc:"获取模型表单列表"` + + ModelType model.ModelType `json:"modelType"` // 模型类型 vector/chat + ConfigType model.ModelConfigType `json:"configType"` // 配置类型 ark/ollama/openai... +} + +// GetModelConfigFormFieldsRes 获取模型配置表单响应 +type GetModelConfigFormFieldsRes struct { + ModelType model.ModelType `json:"modelType"` + ConfigType model.ModelConfigType `json:"configType"` + Fields []map[string]interface{} `json:"fields"` +} + +// CreateModelReq 创建模型请求 +type CreateModelReq struct { + g.Meta `path:"/create" method:"post" tags:"模型配置管理" summary:"创建模型配置" dc:"创建模型配置"` + + ModelType model.ModelType `json:"modelType" v:"required#模型类型不能为空"` + ModelName string `json:"modelName" v:"required#模型名称不能为空"` + ModelDesc string `json:"modelDesc"` + ConfigType model.ModelConfigType `json:"configType"` + ConfigContent map[string]interface{} `json:"configContent"` +} + +// CreateModelRes 创建模型响应 +type CreateModelRes struct { + Id int64 `json:"id,string"` +} + +// UpdateModelReq 更新模型请求 +type UpdateModelReq struct { + g.Meta `path:"/update" method:"put" tags:"模型配置管理" summary:"更新模型配置" dc:"更新模型配置"` + + Id int64 `json:"id" v:"required#ID不能为空"` + ModelType model.ModelType `json:"modelType"` + ModelName string `json:"modelName"` + ModelDesc string `json:"modelDesc"` + ConfigType model.ModelConfigType `json:"configType"` + ConfigContent map[string]interface{} `json:"configContent"` +} + +// DeleteModelReq 删除模型请求 +type DeleteModelReq struct { + g.Meta `path:"/delete" method:"delete" tags:"模型配置管理" summary:"删除模型配置" dc:"删除模型配置"` + + Id int64 `json:"id" v:"required#ID不能为空"` +} + +// GetModelReq 获取模型请求 +type GetModelReq struct { + g.Meta `path:"/get" method:"get" tags:"模型配置管理" summary:"获取模型配置详情" dc:"获取模型配置详情"` + + Id int64 `json:"id"` + ModelType model.ModelType `json:"modelType"` +} + +// ListModelReq 获取模型列表请求 +type ListModelReq struct { + g.Meta `path:"/list" method:"get" tags:"模型配置管理" summary:"获取模型配置列表" dc:"分页查询模型配置列表，支持多条件筛选"` + + Page *beans.Page `json:"page"` + ModelType model.ModelType `json:"modelType"` + ModelName string `json:"modelName"` +} + +// ListModelRes 获取模型列表响应 +type ListModelRes struct { + List []*ModelVO `json:"list"` + Total int `json:"total"` +} + +type ModelVO struct { + Id int64 `json:"id,string"` + ModelType model.ModelType `json:"modelType"` + ModelName string `json:"modelName"` + ModelDesc string `json:"modelDesc"` + ConfigType model.ModelConfigType `json:"configType"` + ConfigContent map[string]interface{} `json:"configContent"` + CreateTime time.Time `json:"createTime"` + UpdateTime time.Time `json:"updateTime"` +} diff --git a/model/dto/task.go b/model/dto/task.go index c94a476..8934e54 100644 --- a/model/dto/task.go +++ b/model/dto/task.go @@ -2,6 +2,8 @@ package dto import ( "rag/consts/task" + + "github.com/gogf/gf/v2/frame/g" ) // WriteTaskProgressReq 写入任务进度请求 @@ -35,9 +37,17 @@ type DeleteTaskByTaskIdReq struct { // GetTaskReq 获取任务请求 type GetTaskReq struct { - Id int64 `json:"id" dc:"任务ID"` - TaskId int64 `json:"taskId" dc:"任务ID"` - TaskType task.TaskType `json:"taskType" dc:"任务类型"` + g.Meta `path:"/get" method:"get" tags:"任务管理" summary:"获取任务详情" dc:"获取任务详情"` + + Id int64 `json:"id" dc:"任务ID"` + TaskId int64 `json:"taskId" dc:"任务ID"` + TaskType task.TaskType `json:"taskType" dc:"任务类型"` + TaskStatus task.TaskStatus `json:"taskStatus" dc:"任务状态"` +} + +type ListTaskRes struct { + List []*TaskVO `json:"list"` + Total int `json:"total"` } // TaskVO 任务视图对象 diff --git a/model/entity/model.go b/model/entity/model.go new file mode 100644 index 0000000..27557df --- /dev/null +++ b/model/entity/model.go @@ -0,0 +1,119 @@ +package entity + +import ( + "rag/consts/model" + + "gitea.com/red-future/common/beans" +) + +type modelCol struct { + beans.SQLBaseCol + DatasetId string + ModelType string + ModelName string + ModelDesc string + ConfigType string + ConfigContent string +} + +var ModelCol = modelCol{ + SQLBaseCol: beans.DefSQLBaseCol, + DatasetId: "dataset_id", + ModelType: "model_type", + ModelName: "model_name", + ModelDesc: "model_desc", + ConfigType: "config_type", + ConfigContent: "config_content", +} + +type Model struct { + beans.SQLBaseDO `orm:",inline"` + DatasetId int64 `orm:"dataset_id" json:"datasetId" dc:"数据集ID"` + ModelType model.ModelType `orm:"model_type" json:"modelType" dc:"模型类型"` // 向量/对话 + ModelName string `orm:"model_name" json:"modelName" dc:"模型名称"` + ModelDesc string `orm:"model_desc" json:"modelDesc" dc:"模型描述"` + ConfigType model.ModelConfigType `orm:"config_type" json:"configType" dc:"配置类型"` // ark/ollama等 + ConfigContent map[string]interface{} `orm:"config_content" json:"configContent" dc:"配置详情"` // 存JSON +} + +// -------------------------- 通用配置结构体（抽离重复字段）-------------------------- + +// OllamaConfig 通用配置（向量/对话完全一致） +type OllamaConfig struct { + BaseURL string `json:"base_url"` + Model string `json:"model"` +} + +// OpenAIConfig 通用配置 +type OpenAIConfig struct { + APIKey string `json:"api_key"` + Model string `json:"model"` + ByAzure bool `json:"by_azure"` + BaseURL string `json:"base_url"` + APIVersion string `json:"api_version"` +} + +// QianfanConfig 千帆通用配置 +type QianfanConfig struct { + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + Model string `json:"model"` +} + +// ArkConfig 通用配置 +type ArkConfig struct { + APIKey string `json:"api_key"` + Model string `json:"model"` +} + +// -------------------------- 向量模型配置 -------------------------- + +type VectorModelConfigOllama = OllamaConfig // 直接复用 +type VectorModelConfigOpenAI = OpenAIConfig // 直接复用 +type VectorModelConfigQianfan = QianfanConfig // 直接复用 + +type VectorModelConfigArk struct { + ArkConfig + APIType string `json:"api_type"` +} + +type VectorModelConfigTencentCloud struct { + SecretID string `json:"secret_id"` + SecretKey string `json:"secret_key"` + Region string `json:"region"` +} + +type VectorModelConfigDashScope struct { + APIKey string `json:"api_key"` + Model string `json:"model"` +} + +// -------------------------- 对话模型配置 -------------------------- + +type ChatModelConfigArk = ArkConfig // 直接复用 +type ChatModelConfigArkBot = ArkConfig // 直接复用 +type ChatModelConfigOllama = OllamaConfig // 直接复用 +type ChatModelConfigOpenAI = OpenAIConfig // 直接复用 +type ChatModelConfigQianfan = QianfanConfig // 直接复用 + +type ChatModelConfigClaude struct { + ByBedrock bool `json:"by_bedrock"` + AccessKey string `json:"access_key"` + SecretAccessKey string `json:"secret_access_key"` + Region string `json:"region"` + APIKey string `json:"api_key"` + Model string `json:"model"` + BaseURL string `json:"base_url"` +} + +type ChatModelConfigDeepSeek struct { + APIKey string `json:"api_key"` + Model string `json:"model"` + BaseURL string `json:"base_url"` +} + +type ChatModelConfigQwen struct { + APIKey string `json:"api_key"` + Model string `json:"model"` + BaseURL string `json:"base_url"` +} diff --git a/service/document.go b/service/document.go index 1328eee..50c672e 100644 --- a/service/document.go +++ b/service/document.go @@ -7,6 +7,7 @@ import ( "rag/common/eino" "rag/consts/document" "rag/consts/keyword" + "rag/consts/model" "rag/consts/public" "rag/consts/task" "rag/dao" @@ -22,10 +23,8 @@ import ( "github.com/bjang03/gmq/mq" "github.com/bjang03/gmq/types" "github.com/cloudwego/eino/schema" - "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/crypto/gmd5" "github.com/gogf/gf/v2/database/gdb" - "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/grpool" "github.com/gogf/gf/v2/util/gconv" @@ -37,7 +36,35 @@ type documentService struct{} // Create 创建文件 func (s *documentService) Create(ctx context.Context, req *dto.CreateDocumentReq) (res *dto.CreateDocumentRes, err error) { - err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) { + err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) { + doc, err := dao.Document.Get(ctx, &dto.GetDocumentReq{ + DatasetId: req.DatasetId, + Title: req.Title, + }) + if err != nil { + return + } + if !g.IsEmpty(doc) && doc.Id > 0 { + _, err = dao.Keyword.Delete(ctx, &dto.DeleteKeywordReq{ + DocumentId: doc.Id, + }) + if err != nil { + return err + } + _, err = dao.DocumentVector.Delete(ctx, &dto.DeleteDocumentVectorReq{ + DocumentId: doc.Id, + }) + if err != nil { + return err + } + _, err = dao.Document.Delete(ctx, &dto.DeleteDocumentReq{ + Id: doc.Id, + }) + if err != nil { + return err + } + } + var id int64 id, err = dao.Document.Insert(ctx, req) if err != nil { @@ -74,11 +101,11 @@ func (s *documentService) Update(ctx context.Context, req *dto.UpdateDocumentReq // Delete 删除文件 func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (err error) { - docs, err := dao.Document.GetByID(ctx, &dto.GetDocumentReq{Id: req.Id}) + docs, err := dao.Document.Get(ctx, &dto.GetDocumentReq{Id: req.Id}) if err != nil { return } - err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) { + err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) { datasetReq := &dto.UpdateDatasetReq{ Id: docs.DatasetId, DocumentCount: -1, @@ -92,6 +119,18 @@ func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq return } + if _, err = dao.Keyword.Delete(ctx, &dto.DeleteKeywordReq{ + DocumentId: docs.Id, + }); err != nil { + return err + } + + if _, err = dao.DocumentVector.Delete(ctx, &dto.DeleteDocumentVectorReq{ + DocumentId: docs.Id, + }); err != nil { + return err + } + if _, err = dao.Task.DeleteByTaskId(ctx, &dto.DeleteTaskByTaskIdReq{ TaskId: docs.Id, }); err != nil { @@ -106,7 +145,7 @@ func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq // Get 获取文件详情 func (s *documentService) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.GetDocumentRes, err error) { - r, err := dao.Document.GetByID(ctx, req) + r, err := dao.Document.Get(ctx, req) if err != nil { return } @@ -136,7 +175,7 @@ func (s *documentService) List(ctx context.Context, req *dto.ListDocumentReq) (r func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq) (err error) { // 1. 查询文件信息 documentReq := dto.GetDocumentReq{Id: req.Id} - doc, err := dao.Document.GetByID(ctx, &documentReq) + doc, err := dao.Document.Get(ctx, &documentReq) if err != nil { return err } @@ -172,16 +211,13 @@ func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq if err != nil { return err } - // ====================== - // 核心：grpool + g.Try 最佳实践 - // ====================== // 使用带超时的background context，避免HTTP请求完成后context被取消 taskCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) taskCtx = context.WithValue(taskCtx, "user", user) - // 任务1: SQL 切分文档 + // 任务1: 语义 切分文档 grpool.Add(taskCtx, func(ctx context.Context) { g.TryCatch(ctx, func(ctx context.Context) { - if innerErr := s.sqlSplitDocument(ctx, doc); innerErr != nil { + if innerErr := s.semanticSplitDocument(ctx, doc); innerErr != nil { cancel() } }, func(ctx context.Context, err error) { @@ -189,10 +225,10 @@ func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq }) }) - // 任务2: ES 切分文档 + // 任务2: 递归 切分文档 grpool.Add(taskCtx, func(ctx context.Context) { g.TryCatch(ctx, func(ctx context.Context) { - if innerErr := s.esSplitDocument(ctx, doc); innerErr != nil { + if innerErr := s.recursiveSplitDocument(ctx, doc); innerErr != nil { cancel() } }, func(ctx context.Context, err error) { @@ -327,8 +363,8 @@ func (s *documentService) extractDocument(ctx context.Context, doc *entity.Docum return } -// sqlSplitDocument SQL切分（支持取消） -func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Document) (err error) { +// semanticSplitDocument 语义切分 +func (s *documentService) semanticSplitDocument(ctx context.Context, doc *entity.Document) (err error) { // ========== 取消检查 1：方法入口 ========== if ctx.Err() != nil { err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ @@ -354,7 +390,7 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu } // 2. 语义切分文件 - docsSplit, err := eino.SemanticSplitDocument(ctx, docs) + docsSplit, err := eino.SemanticSplitDocument(ctx, docs, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型 if err != nil { // 写入任务进度失败 任务类型为sql存储 err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ @@ -394,8 +430,8 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu } contentHash := gmd5.MustEncryptString(t.Content) - var success bool - success, err = s.checkRepeat(ctx, public.KnowledgeContentHashSqlKey, contentHash) + var isNew, needCopy bool + isNew, needCopy, err = s.checkRepeatWithDocId(ctx, public.KnowledgeContentHashSqlKey, contentHash, doc.Id) if err != nil { // 写入任务进度失败 任务类型为sql存储 err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ @@ -406,7 +442,7 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu }) return } - if !success { + if !isNew && !needCopy { continue } var metaData = make(map[string]any) @@ -415,7 +451,13 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu metaData[entity.DocumentCol.DatasetId] = doc.DatasetId metaData[entity.DocumentVectorCol.DocumentId] = doc.Id metaData[entity.DocumentVectorCol.ContentHash] = contentHash - metaData[entity.DocumentVectorCol.ChunkIndex] = gconv.Int64(i) + metaData[entity.DocumentVectorCol.ChunkIndex] = gconv.Int64(i + 1) + if isNew { + metaData["isNew"] = true + } + if needCopy { + metaData["isNew"] = false + } t.MetaData = metaData docsChunk = append(docsChunk, t) } @@ -468,8 +510,8 @@ func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Docu return } -// esSplitDocument ES切分（支持取消） -func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Document) (err error) { +// recursiveSplitDocument 递归切分 +func (s *documentService) recursiveSplitDocument(ctx context.Context, doc *entity.Document) (err error) { // ========== 取消检查 1：方法入口 ========== if ctx.Err() != nil { err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ @@ -535,8 +577,8 @@ func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Docum } contentHash := gmd5.MustEncryptString(t.Content) - var success bool - success, err = s.checkRepeat(ctx, public.KnowledgeContentHashEsKey, contentHash) + var isNew, needCopy bool + isNew, needCopy, err = s.checkRepeatWithDocId(ctx, public.KnowledgeContentHashEsKey, contentHash, doc.Id) if err != nil { // 写入任务进度失败 任务类型为es存储 err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ @@ -547,7 +589,7 @@ func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Docum }) return } - if !success { + if !isNew && !needCopy { continue } meiliDocs = append(meiliDocs, map[string]interface{}{ @@ -556,7 +598,7 @@ func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Docum entity.DocumentVectorCol.DocumentId: doc.Id, entity.DocumentVectorCol.Content: t.Content, entity.DocumentVectorCol.ContentHash: contentHash, - entity.DocumentVectorCol.ChunkIndex: i, + entity.DocumentVectorCol.ChunkIndex: i + 1, }) } @@ -632,6 +674,7 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume // 3. Redis 无数据：根据 contentKey 类型选择查询方式 var dictData = make([]*dto.DocumentVectorRPC, 0) + if public.KnowledgeContentHashSqlKey == contentKey { // SQL 方式：调用 HTTP 接口查询 dictData, err = s.getHistoryDataFromHttp(ctx, doc) @@ -643,20 +686,16 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume return err } - // 4. 把查询到的数据写入 Redis（600s过期） for _, item := range dictData { - // 去除可能的 JSON 引号 contentHash := strings.Trim(item.ContentHash, `"`) key := fmt.Sprintf(contentKey, contentHash) - _, err = g.Redis().Set(ctx, key, true, gredis.SetOption{ - TTLOption: gredis.TTLOption{ - EX: gconv.PtrInt64(600), - }, - NX: true, - }) + // SAdd：把文档ID加入集合（自动去重，可存多个） + _, err = g.Redis().SAdd(ctx, key, item.DocumentId) if err != nil { return err } + // 设置过期时间 + _, _ = g.Redis().Expire(ctx, key, 600) } return nil @@ -672,8 +711,10 @@ func (s *documentService) getHistoryDataFromHttp(ctx context.Context, doc *entit // 调用接口获取数据 res, _, err := dao.DocumentVector.List(ctx, &dto.ListDocumentVectorReq{ DatasetId: doc.DatasetId, - Status: gconv.PtrInt8(1), }) + if err != nil { + return + } err = gconv.Struct(res, &dictData) return } @@ -705,17 +746,39 @@ func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc return } -// checkRepeat 检查是否重复 -func (s *documentService) checkRepeat(ctx context.Context, contentKey, contentHash string) (success bool, err error) { - var val *gvar.Var - if val, err = g.Redis().Set(ctx, fmt.Sprintf(contentKey, contentHash), true, gredis.SetOption{ - TTLOption: gredis.TTLOption{ - EX: gconv.PtrInt64(600), - }, - NX: true, - }); err != nil { - return +// checkRepeatWithDocId 正确版：检查当前文档是否已存在该分片 +// 返回：isNew(是否需要生成向量)、isCrossDoc(是否跨文档需拷贝)、err +func (s *documentService) checkRepeatWithDocId(ctx context.Context, contentKey string, contentHash string, currentDocId int64) (isNew bool, needCopy bool, err error) { + key := fmt.Sprintf(contentKey, contentHash) + + // 1. 检查当前文档ID是否在集合中 + exists, err := g.Redis().SIsMember(ctx, key, currentDocId) + if err != nil { + return false, false, err } - success = val.Bool() - return + + // 情况1：当前文档已存在 → 完全跳过，不生成、不拷贝 + if !g.IsEmpty(exists) { + return false, false, nil + } + + // 2. 检查 key 是否存在（是否有任何文档拥有该分片） + keyExists, err := g.Redis().Exists(ctx, key) + if err != nil { + return false, false, err + } + + // 情况2：key 不存在 = 全新数据 → 需要生成向量 + if g.IsEmpty(keyExists) { + // 把当前文档ID加入集合 + _, err = g.Redis().SAdd(ctx, key, currentDocId) + _, _ = g.Redis().Expire(ctx, key, 600) + return true, false, err + } + + // 情况3：key 存在，但当前文档不在集合中 = 跨文档重复 → 不生成，需拷贝 + // 把当前文档ID加入集合（记录归属关系） + _, err = g.Redis().SAdd(ctx, key, currentDocId) + _, _ = g.Redis().Expire(ctx, key, 600) + return false, true, err } diff --git a/service/document_vector.go b/service/document_vector.go index 2a4bb91..ec79a49 100644 --- a/service/document_vector.go +++ b/service/document_vector.go @@ -4,17 +4,18 @@ import ( "context" "fmt" "rag/common/eino" + "rag/consts/model" "rag/consts/task" "rag/dao" "rag/model/dto" "rag/model/entity" "gitea.com/red-future/common/beans" - "github.com/cloudwego/eino/components/indexer" "github.com/cloudwego/eino/components/retriever" "github.com/cloudwego/eino/schema" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/gconv" + "github.com/pgvector/pgvector-go" ) var DocumentVector = new(documentVectorService) @@ -23,23 +24,32 @@ type documentVectorService struct{} // Query 执行RAG查询 func (s *documentVectorService) Query(ctx context.Context, req *dto.RAGQueryReq) (*dto.RAGQueryRes, error) { - if req.TopK <= 0 { - req.TopK = 5 + + modelInfo, err := dao.Model.Get(ctx, &dto.GetModelReq{ + ModelType: model.ModelTypeChat.Code(), + }) + if err != nil { + g.Log().Errorf(ctx, "获取模型失败: %v", err) + return nil, fmt.Errorf("获取模型失败: %w", err) + } + if modelInfo == nil { + g.Log().Errorf(ctx, "模型不存在: %v", model.ModelTypeChat.Code()) + return nil, fmt.Errorf("模型不存在: %w", err) } // 4. 使用向量检索器进行查询 - r, err := eino.NewPGVectorRetriever(&eino.PGVectorRetrieverConfig{ - Embedder: eino.EmbedderDashscope, + r, err := eino.NewPGVectorRetriever(ctx, &eino.PGVectorRetrieverConfig{ DefaultTopK: req.TopK, - }) + }, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型 if err != nil { g.Log().Errorf(ctx, "初始化向量检索器失败: %v", err) return nil, fmt.Errorf("初始化向量检索器失败: %w", err) } // 5. 执行向量检索 - docs, err := r.Retrieve(ctx, req.Content, retriever.WithEmbedding(eino.EmbedderDashscope), retriever.WithDSLInfo(map[string]any{ - "dataset_ids": req.DatasetIds, + docs, err := r.Retrieve(ctx, req.Content, retriever.WithDSLInfo(map[string]any{ + "dataset_ids": req.DatasetIds, + "document_ids": req.DocumentIds, })) if err != nil { g.Log().Errorf(ctx, "向量检索失败: %v", err) @@ -53,7 +63,7 @@ func (s *documentVectorService) Query(ctx context.Context, req *dto.RAGQueryReq) return nil, fmt.Errorf("转换历史消息失败: %w", err) } - replyMsg, err := eino.NewChatModel(ctx, req.Content, docs, messages) + replyMsg, err := eino.NewChatModel(ctx, req.Content, docs, messages, modelInfo.ConfigType) if err != nil { g.Log().Errorf(ctx, "向量检索失败: %v", err) return nil, fmt.Errorf("向量检索失败: %w", err) @@ -98,26 +108,108 @@ func (s *documentVectorService) DocsChunkMsg(ctx context.Context, msg any) (err TenantId: gconv.Uint64(docs[0].MetaData[entity.DocumentVectorCol.TenantId]), UserName: gconv.String(docs[0].MetaData[entity.DocumentVectorCol.Creator]), }) - idx := eino.NewPGVectorIndexer(&eino.PGVectorIndexerOptions{ - BatchSize: 10, - }) + documentId := gconv.Int64(docs[0].MetaData[entity.DocumentVectorCol.DocumentId]) - rows, err := idx.Store(ctx, docs, indexer.WithEmbedding(eino.EmbedderDashscope)) - if err != nil || rows == 0 { - g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err) - // 写入任务进度失败 任务类型为sql存储 - remark := " 向量存储数量: " + gconv.String(rows) - if err != nil { - remark = "向量存储失败: " + err.Error() + + var docsStore = make([]*schema.Document, 0) + var docsInsert = make([]*dto.VectorDocumentVectorMsg, 0) + for _, doc := range docs { + if gconv.Bool(doc.MetaData["isNew"]) { + docsStore = append(docsStore, doc) + } else { + ck := new(dto.VectorDocumentVectorMsg) + err = gconv.Struct(doc.MetaData, ck) + ck.Content = doc.Content + ck.VectorStatus = gconv.PtrInt8(1) + ck.Status = gconv.PtrInt8(1) + docsInsert = append(docsInsert, ck) } - err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ - TaskId: documentId, - TaskType: task.TaskTypeGenerateVector, - Status: task.TaskStatusFailed, - Remark: remark, - }) - return } + + if !g.IsEmpty(docsStore) { + idx := eino.NewPGVectorIndexer(&eino.PGVectorIndexerOptions{ + BatchSize: 10, + }) + var rows int64 + rows, err = idx.Store(ctx, docsStore, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型 + + if err != nil || rows == 0 { + g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err) + // 写入任务进度失败 任务类型为sql存储 + remark := " 向量存储数量: " + gconv.String(rows) + if err != nil { + remark = "向量存储失败: " + err.Error() + } + err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ + TaskId: documentId, + TaskType: task.TaskTypeGenerateVector, + Status: task.TaskStatusFailed, + Remark: remark, + }) + return + } + } + + if !g.IsEmpty(docsInsert) { + // 1. 提取所有 contentHash + contentHashs := make([]string, 0, len(docsInsert)) + for _, d := range docsInsert { + contentHashs = append(contentHashs, d.ContentHash) + } + + // 2. 分页查询已存在的向量（一页1000，避免大查询） + var existVectors []*entity.DocumentVector + for page := 1; ; page++ { + res, total, err := dao.DocumentVector.List(ctx, &dto.ListDocumentVectorReq{ + Page: &beans.Page{PageSize: 1000, PageNum: int64(page)}, + ContentHashs: contentHashs, + }) + if err != nil { + return err + } + if len(res) == 0 { + break + } + existVectors = append(existVectors, res...) + if len(existVectors) >= total { + break + } + } + + // 3. 构建哈希 -> 向量 的映射表（O(1) 查找，性能提升巨大） + vectorMap := make(map[string]pgvector.Vector, len(existVectors)) + for _, v := range existVectors { + vectorMap[v.ContentHash] = v.Vector + } + + // 4. 回填向量 + 过滤掉数据库已存在的数据（避免重复插入） + for _, d := range docsInsert { + // 回填已有向量 + if vec, ok := vectorMap[d.ContentHash]; ok { + d.Vector = vec + } + } + + var rows int64 + rows, err = dao.DocumentVector.BatchInsert(ctx, docsInsert) + + if err != nil || rows == 0 { + g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err) + // 写入任务进度失败 任务类型为sql存储 + remark := " 向量存储数量: " + gconv.String(rows) + if err != nil { + remark = "向量存储失败: " + err.Error() + } + err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ + TaskId: documentId, + TaskType: task.TaskTypeGenerateVector, + Status: task.TaskStatusFailed, + Remark: remark, + }) + return + } + } + // 写入任务进度成功 任务类型为sql存储 err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{ TaskId: documentId, diff --git a/service/model.go b/service/model.go new file mode 100644 index 0000000..1d1239c --- /dev/null +++ b/service/model.go @@ -0,0 +1,299 @@ +package service + +import ( + "context" + "rag/common/eino" + "rag/consts/model" + "rag/consts/task" + "rag/dao" + "rag/model/dto" + "rag/model/entity" + + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" +) + +var ModelService = new(modelService) + +type modelService struct{} + +// GetModelAllEnums 获取模型全量枚举（模型类型 + 配置类型 合并） +func (s *modelService) GetModelAllEnums(ctx context.Context, req *dto.GetModelAllEnumsReq) (res *dto.GetModelEnumRes, err error) { + _, _ = ctx, req + res = new(dto.GetModelEnumRes) + + // 获取所有模型类型 + modelTypeRes := model.GetAllModelTypeEnums() + + var options []dto.ModelEnumOption + for _, mt := range modelTypeRes.Options { + // 构造 modelType + modelTypeStr := gconv.String(mt.Key) + modelType := model.ModelType(gconv.PtrString(modelTypeStr)) + + // 获取对应配置类型 + configRes := model.GetAllModelConfigTypeEnums(modelType) + + // 把 configRes.Options 转成目标类型 + var configList []dto.ModelKeyValue + err = gconv.Structs(configRes.Options, &configList) + if err != nil { + return + } + options = append(options, dto.ModelEnumOption{ + Key: mt.Key, + Value: mt.Value, + ConfigTypes: configList, + }) + } + + res.Options = options + return +} + +func (s *modelService) GetModelConfigFormFields(ctx context.Context, req *dto.GetModelConfigFormFieldsReq) (*dto.GetModelConfigFormFieldsRes, error) { + _ = ctx + + fields := make([]map[string]interface{}, 0) + + // ===================== 固定基础字段（CreateModelReq 前4个）===================== + // 1. 模型类型：固定只读字段 + fields = append(fields, map[string]interface{}{ + "name": "modelType", + "label": "模型类型", + "type": "text", + "disabled": true, + "required": true, + "value": model.GetModelTypeDescByCode(req.ModelType), + }) + + var configTypeValue = "未知类型" + if *req.ModelType == *model.ModelTypeVector.Code() { + configTypeValue = model.GetVectorDescByCode(req.ConfigType) + } else if *req.ModelType == *model.ModelTypeChat.Code() { + configTypeValue = model.GetChatDescByCode(req.ConfigType) + } + + // 2. 配置类型：固定只读字段 + fields = append(fields, map[string]interface{}{ + "name": "configType", + "label": "配置类型", + "type": "text", + "disabled": true, + "required": true, + "value": configTypeValue, + }) + + // 3. 基础信息 + fields = append(fields, []map[string]interface{}{ + { + "name": "modelName", + "label": "模型名称", + "type": "input", + "required": true, + "placeholder": "例如：DeepSeek 对话模型", + }, + { + "name": "modelDesc", + "label": "模型描述", + "type": "textarea", + "required": false, + }, + }...) + + // 4. 通用模型名称字段 + fields = append(fields, map[string]interface{}{ + "name": "model", + "label": "模型类型", + "type": "input", + "required": true, + "placeholder": "例如：deepseek-chat / text-embedding-3-small", + }) + + // ===================== 动态配置内容 ConfigContent ===================== + + // 根据模型类型 + 配置类型生成动态字段 + switch *req.ModelType { + case *model.ModelTypeChat.Code(): + switch *req.ConfigType { + case *model.ModelConfigTypeChatArk.Code(): + fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true}) + + case *model.ModelConfigTypeChatArkBot.Code(): + fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true}) + + case *model.ModelConfigTypeChatClaude.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "by_bedrock", "label": "使用 AWS Bedrock", "type": "switch", "default": true}, + {"name": "access_key", "label": "Access Key", "type": "input"}, + {"name": "secret_access_key", "label": "Secret Access Key", "type": "input"}, + {"name": "region", "label": "Region", "type": "input"}, + {"name": "api_key", "label": "API Key", "type": "input"}, + {"name": "base_url", "label": "Base URL", "type": "input"}, + }...) + + case *model.ModelConfigTypeChatDeepSeek.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "api_key", "label": "API Key", "type": "input", "required": true}, + {"name": "base_url", "label": "Base URL", "type": "input", "default": "https://api.deepseek.com"}, + }...) + + case *model.ModelConfigTypeChatOllama.Code(): + fields = append(fields, map[string]interface{}{"name": "base_url", "label": "Base URL", "type": "input", "required": true, "default": "http://127.0.0.1:11434"}) + + case *model.ModelConfigTypeChatOpenAI.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "api_key", "label": "API Key", "type": "input", "required": true}, + {"name": "by_azure", "label": "使用 Azure", "type": "switch", "default": true}, + {"name": "base_url", "label": "Base URL", "type": "input"}, + {"name": "api_version", "label": "API Version", "type": "input"}, + }...) + + case *model.ModelConfigTypeChatQianfan.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "access_key", "label": "Access Key", "type": "input", "required": true}, + {"name": "secret_key", "label": "Secret Key", "type": "input", "required": true}, + }...) + + case *model.ModelConfigTypeChatQwen.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "api_key", "label": "API Key", "type": "input", "required": true}, + {"name": "base_url", "label": "Base URL", "type": "input"}, + }...) + } + + case *model.ModelTypeVector.Code(): + switch *req.ConfigType { + case *model.ModelConfigTypeVectorArk.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "api_key", "label": "API Key", "type": "input", "required": true}, + {"name": "api_type", "label": "API Type", "type": "input"}, + }...) + + case *model.ModelConfigTypeVectorOllama.Code(): + fields = append(fields, map[string]interface{}{"name": "base_url", "label": "Base URL", "type": "input", "required": true, "default": "http://127.0.0.1:11434"}) + + case *model.ModelConfigTypeVectorOpenAI.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "api_key", "label": "API Key", "type": "input", "required": true}, + {"name": "by_azure", "label": "使用 Azure", "type": "switch", "default": true}, + {"name": "base_url", "label": "Base URL", "type": "input"}, + {"name": "api_version", "label": "API Version", "type": "input"}, + }...) + + case *model.ModelConfigTypeVectorQianfan.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "access_key", "label": "Access Key", "type": "input", "required": true}, + {"name": "secret_key", "label": "Secret Key", "type": "input", "required": true}, + }...) + + case *model.ModelConfigTypeVectorTencentCloud.Code(): + fields = append(fields, []map[string]interface{}{ + {"name": "secret_id", "label": "Secret ID", "type": "input", "required": true}, + {"name": "secret_key", "label": "Secret Key", "type": "input", "required": true}, + {"name": "region", "label": "Region", "type": "input", "required": true, "default": "ap-beijing"}, + }...) + case *model.ModelConfigTypeVectorDashScope.Code(): + fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true}) + } + } + + return &dto.GetModelConfigFormFieldsRes{ + ModelType: req.ModelType, + ConfigType: req.ConfigType, + Fields: fields, + }, nil +} + +func (s *modelService) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) { + count, err := dao.Model.Count(ctx, &dto.GetModelReq{ + ModelType: req.ModelType, + }) + if err != nil { + return + } + if count > 0 { + err = gerror.New("模型配置已存在") + return + } + var id int64 + id, err = dao.Model.Insert(ctx, req) + if err != nil { + return + } + res = &dto.CreateModelRes{Id: id} + err = s.refresh(ctx, id) + return +} + +func (s *modelService) Update(ctx context.Context, req *dto.UpdateModelReq) (err error) { + count, err := dao.Task.Count(ctx, &dto.GetTaskReq{ + TaskStatus: task.TaskStatusRunning, + }) + if err != nil { + return err + } + if !g.IsEmpty(count) { + err = gerror.New("任务正在执行中，模型配置暂时不可修改，请稍后再试") + return + } + var updateCount int64 + updateCount, err = dao.Model.Update(ctx, req) + if err != nil { + return + } + if !g.IsEmpty(updateCount) { + err = s.refresh(ctx, req.Id) + if err != nil { + return err + } + } + return +} + +func (s *modelService) refresh(ctx context.Context, id int64) (err error) { + var modelDO *entity.Model + modelDO, err = dao.Model.Get(ctx, &dto.GetModelReq{ + Id: id, + }) + if err != nil { + return err + } + if *modelDO.ModelType == *model.ModelTypeChat.Code() { + if err = eino.RefreshTenantChatModel(ctx, modelDO); err != nil { + return err + } + } + + if *modelDO.ModelType == *model.ModelTypeVector.Code() { + if err = eino.RefreshTenantEmbedder(ctx, modelDO); err != nil { + return err + } + } + + return +} + +func (s *modelService) Delete(ctx context.Context, req *dto.DeleteModelReq) (err error) { + _, err = dao.Model.Delete(ctx, req) + return +} + +func (s *modelService) Get(ctx context.Context, req *dto.GetModelReq) (res *dto.ModelVO, err error) { + r, err := dao.Model.Get(ctx, req) + err = gconv.Struct(r, &res) + return +} + +func (s *modelService) List(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) { + list, total, err := dao.Model.List(ctx, req) + if err != nil { + return nil, err + } + res = &dto.ListModelRes{ + Total: total, + } + err = gconv.Struct(list, &res.List) + return +} diff --git a/service/task.go b/service/task.go index 39f66d4..a4c74a4 100644 --- a/service/task.go +++ b/service/task.go @@ -2,6 +2,8 @@ package service import ( "context" + "rag/consts/document" + "rag/consts/public" "rag/consts/task" "rag/dao" "rag/model/dto" @@ -37,7 +39,7 @@ func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskP TaskType: req.TaskType, Status: req.Status, }) - completed = IsAllSubTasksCompleted(taskVO) + completed = IsAllSubTasks(taskVO, task.TaskStatusCompleted) } // 1. 查询是否已存在该文档的该类型任务 @@ -49,7 +51,7 @@ func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskP g.Log().Errorf(ctx, "查询任务失败: %v", err) return err } - err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // 2. 如果不存在，则创建新任务 if g.IsEmpty(existTask) { createReq := &dto.CreateTaskReq{ @@ -80,17 +82,36 @@ func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskP Status: task.TaskStatusCompleted, Remark: "文档解析完成", }) + if err != nil { + g.Log().Errorf(ctx, "更新任务失败: %v", err) + return err + } + _, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{ + Id: req.TaskId, + VectorStatus: document.VectorStatusCompleted.Code(), + }) + if err != nil { + return err + } + } else { + if task.TaskStatusFailed == req.Status { + _, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{ + Id: req.TaskId, + VectorStatus: document.VectorStatusFailed.Code(), + }) + if err != nil { + return err + } + } } - return nil }) return } -// IsAllSubTasksCompleted 判断三个子任务是否全部完成 -// 参数：传入当前文档的所有子任务列表 -func IsAllSubTasksCompleted(subTasks []*dto.TaskVO) bool { +// IsAllSubTasks 判断三个子任务 +func IsAllSubTasks(subTasks []*dto.TaskVO, taskStatus task.TaskStatus) bool { // 必须包含 3 种任务类型 hasKeywords := false hasVector := false @@ -98,7 +119,7 @@ func IsAllSubTasksCompleted(subTasks []*dto.TaskVO) bool { for _, t := range subTasks { // 子任务必须是【已完成】状态才计数 - if t.Status == task.TaskStatusCompleted { + if t.Status == taskStatus { switch t.TaskType { case task.TaskTypeExtractKeywords: hasKeywords = true @@ -113,3 +134,15 @@ func IsAllSubTasksCompleted(subTasks []*dto.TaskVO) bool { // 三个任务全部完成 → 返回true return hasKeywords && hasVector && hasFullText } + +func (s *taskService) Get(ctx context.Context, req *dto.GetTaskReq) (res *dto.ListTaskRes, err error) { + list, total, err := dao.Task.Get(ctx, req) + if err != nil { + return + } + res = &dto.ListTaskRes{ + Total: total, + } + err = gconv.Struct(list, &res.List) + return +} diff --git a/update.sql b/update.sql index 1da633f..9a687c3 100644 --- a/update.sql +++ b/update.sql @@ -206,6 +206,49 @@ COMMENT ON COLUMN rag_knowledge_task.remark IS '备注'; --------------------pgsql创建rag_knowledge_task表语句--------------------------- +--------------------pgsql创建rag_knowledge_model表语句--------------------------- +-- 知识库模型配置表 +CREATE TABLE IF NOT EXISTS rag_knowledge_model ( + -- 基础字段（完全对齐项目规范） + id BIGINT PRIMARY KEY, -- 主键ID（非自增） + tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8 + creator VARCHAR(64) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(64) NOT NULL, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted_at timestamp(6), + + -- 业务字段 + dataset_id BIGINT NOT NULL, -- 数据集ID + model_type VARCHAR(32) NOT NULL, -- 模型类型 + model_name VARCHAR(128) NOT NULL, -- 模型名称 + model_desc TEXT DEFAULT '', -- 模型描述 + model_config JSONB DEFAULT '{}'::JSONB -- 模型配置(JSONB) + ); + +-- 索引（高频查询） +CREATE INDEX idx_rkm_tenant_id ON rag_knowledge_model(tenant_id); +CREATE INDEX idx_rkm_dataset_id ON rag_knowledge_model(dataset_id); +CREATE INDEX idx_rkm_model_type ON rag_knowledge_model(model_type); +CREATE INDEX idx_rkm_deleted_at ON rag_knowledge_model(deleted_at); + +-- 表和字段注释 +COMMENT ON TABLE rag_knowledge_model IS '知识库模型配置表'; +COMMENT ON COLUMN rag_knowledge_model.id IS '主键ID（非自增）'; +COMMENT ON COLUMN rag_knowledge_model.tenant_id IS '租户ID'; +COMMENT ON COLUMN rag_knowledge_model.creator IS '创建人'; +COMMENT ON COLUMN rag_knowledge_model.created_at IS '创建时间'; +COMMENT ON COLUMN rag_knowledge_model.updater IS '更新人'; +COMMENT ON COLUMN rag_knowledge_model.updated_at IS '更新时间'; +COMMENT ON COLUMN rag_knowledge_model.deleted_at IS '删除时间（软删）'; +COMMENT ON COLUMN rag_knowledge_model.dataset_id IS '数据集ID'; +COMMENT ON COLUMN rag_knowledge_model.model_type IS '模型类型'; +COMMENT ON COLUMN rag_knowledge_model.model_name IS '模型名称'; +COMMENT ON COLUMN rag_knowledge_model.model_desc IS '模型描述'; +COMMENT ON COLUMN rag_knowledge_model.model_config IS '模型配置(JSONB)'; + +--------------------pgsql创建rag_knowledge_model表语句--------------------------- + --------------------pgsql创建rag_vector_dataset_index表语句--------------------------- -- 向量数据集索引表