feat: HTTP模式支持 - 优化Stream消费逻辑,添加pending消息优先读取

This commit is contained in:
Cold
2025-12-31 16:10:23 +08:00
committed by 张斌
parent 3b8980bb24
commit 4269b4fd79
2 changed files with 49 additions and 9 deletions

View File

@@ -2,6 +2,7 @@ package ragflow
import (
"context"
"runtime/debug"
"strings"
"time"
@@ -81,11 +82,14 @@ func (q *QueueProcessor) Start(ctx context.Context) error {
continue
}
glog.Debugf(ctx, "读取 %d 条消息,开始发送", len(messages))
glog.Infof(ctx, "✅ 从Stream读取 %d 条消息,开始处理", len(messages))
// 2. 使用协程池提交任务复用goroutine避免频繁创建销毁
for _, msg := range messages {
for i, msg := range messages {
m := msg // 捕获循环变量
msgIndex := i + 1
glog.Infof(ctx, "📨 准备处理第 %d/%d 条消息 - ID: %s", msgIndex, len(messages), m.ID)
// 提交到协程池池满时会阻塞等待空闲worker
q.pool.Add(ctx, func(ctx context.Context) {
q.processMessage(ctx, m)
@@ -98,9 +102,29 @@ func (q *QueueProcessor) Start(ctx context.Context) error {
// processMessage 处理单条消息(异步执行)
func (q *QueueProcessor) processMessage(ctx context.Context, message redis.StreamMessage) {
// 捕获panic防止协程崩溃
defer func() {
if r := recover(); r != nil {
glog.Errorf(ctx, "❌ PANIC: 消息处理发生panic - 消息ID: %s, panic内容: %v\n堆栈:\n%s",
message.ID, r, debug.Stack())
}
}()
glog.Infof(ctx, "🔄 开始处理消息 - ID: %s", message.ID)
// 打印实际字段名(调试用)
var fieldNames []string
for key := range message.Values {
fieldNames = append(fieldNames, key)
}
glog.Infof(ctx, "📋 消息字段名列表: %v", fieldNames)
glog.Infof(ctx, "📦 消息完整内容: %+v", message.Values)
// 调用处理函数发送到 RAGFlow
if err := q.handleFunc(ctx, message.Values); err != nil {
glog.Errorf(ctx, "消息处理失败: %v, 消息ID: %s", err, message.ID)
glog.Errorf(ctx, "消息处理失败: %v, 消息ID: %s", err, message.ID)
} else {
glog.Infof(ctx, "✅ 消息处理成功 - ID: %s", message.ID)
}
// 无论成功失败都 ACK避免重复消费