diff --git a/common/util/mapping.go b/common/util/mapping.go index a709b8b..3ea32ed 100644 --- a/common/util/mapping.go +++ b/common/util/mapping.go @@ -20,36 +20,125 @@ import ( tgjson "github.com/tidwall/gjson" ) -// ValidatePromptResult 校验模型返回结果的 JSON 结构完整性 -// 校验逻辑:只校验 requestMapping 中默认值为空的必填字段 -func ValidatePromptResult(raw map[string]any, model *entity.AsynchModel) error { - // 1) 获取校验配置,并取值 - requestMapping := model.RequestMapping - contentStr, ok := raw[model.ResponseBody].(string) - if !ok || contentStr == "" { - return fmt.Errorf("%s 字段为空或不是字符串", model.ResponseBody) +// ParseAndValidate 解析并校验结果 +func ParseAndValidate(raw map[string]any, model *entity.AsynchModel) (map[string]any, error) { + // 1) 解析 content 字符串为 rounds 数组 + contentVal, ok := raw[model.ResponseBody] + if !ok { + return raw, fmt.Errorf("字段 %s 不存在", model.ResponseBody) + } + fmt.Println("model.ResponseBody打印", model.ResponseBody) + contentStr, ok := contentVal.(string) + if !ok || strings.TrimSpace(contentStr) == "" { + return raw, fmt.Errorf("字段 %s 为空或不是字符串", model.ResponseBody) + } + var arr []any + if err := json.Unmarshal([]byte(contentStr), &arr); err != nil { + return raw, fmt.Errorf("JSON解析失败: %w", err) + } + if len(arr) == 0 { + return raw, fmt.Errorf("解析后数组为空") } - // 2) 解析 content 为 JSON 数组 - var rounds []map[string]any - if err := gjson.DecodeTo(contentStr, &rounds); err != nil { - return fmt.Errorf("解析 content JSON 数组失败: %w", err) - } - if len(rounds) == 0 { - return fmt.Errorf("content 数组为空") - } - - // 3) 逐条校验:只检查默认值为空的必填字段是否存在 - for i, round := range rounds { - for path, defaultValue := range requestMapping { - if !g.IsEmpty(defaultValue) { + // 2) 校验必填字段 + if len(model.RequiredFields) > 0 { + for i, r := range arr { + round, ok := r.(map[string]any) + if !ok { continue } - if gjson.New(round).Get(path).IsNil() { - return fmt.Errorf("rounds[%d] 缺少必填字段: %s", i, path) + for _, field := range model.RequiredFields { + if gjson.New(round).Get(field).IsNil() { + return raw, fmt.Errorf("rounds[%d] 缺少必填字段: %s", i, field) + } } } } + + return map[string]any{"total_rounds": len(arr), "rounds": arr}, nil +} + +// ParseStructResult 解析结构结果 +func ParseStructResult(raw map[string]any, responseBody string) map[string]any { + contentVal := raw[responseBody] + // 是字符串,尝试解析 + contentStr := gconv.String(contentVal) + if contentStr == "" || contentStr == "0" { + return map[string]any{ + "total_rounds": 1, + "rounds": []map[string]any{{responseBody: raw}}, + } + } + + // 尝试解析为数组 + var arr []any + if err := json.Unmarshal([]byte(contentStr), &arr); err == nil && len(arr) > 0 { + return map[string]any{ + "total_rounds": 1, + "rounds": []map[string]any{{responseBody: arr}}, + } + } + + // 尝试解析为单个对象 + var parsed any + if err := json.Unmarshal([]byte(contentStr), &parsed); err == nil { + return map[string]any{ + "total_rounds": 1, + "rounds": []map[string]any{{responseBody: parsed}}, + } + } + + // 兜底:原始字符串作为内容 + return map[string]any{ + "total_rounds": 1, + "rounds": []map[string]any{{responseBody: contentStr}}, + } +} + +// ValidatePromptResult 校验模型返回结果的 JSON 结构完整性 +// raw 必须包含 "rounds" 字段,格式为 []map[string]any +func ValidatePromptResult(raw map[string]any, model *entity.AsynchModel) error { + // 1) 获取 rounds + roundsRaw, ok := raw["rounds"] + if !ok { + return fmt.Errorf("缺少 rounds 字段") + } + rounds, ok := roundsRaw.([]any) + if !ok { + return fmt.Errorf("rounds 不是数组") + } + if len(rounds) == 0 { + return fmt.Errorf("rounds 数组为空") + } + + // 2) 没有配置必填字段,跳过 + if len(model.RequiredFields) == 0 { + return nil + } + + // 3) 逐条校验 + for i, r := range rounds { + round, ok := r.(map[string]any) + if !ok { + continue + } + for _, field := range model.RequiredFields { + if gjson.New(round).Get(field).IsNil() { + return fmt.Errorf("rounds[%d] 缺少必填字段: %s", i, field) + } + } + } + + return nil +} + +// validateRequiredFields 校验单个 round 对象的必选字段 +func validateRequiredFields(round map[string]any, requiredFields []string, prefix string) error { + for _, field := range requiredFields { + if gjson.New(round).Get(field).IsNil() { + return fmt.Errorf("%s 缺少必填字段: %s", prefix, field) + } + } return nil } diff --git a/consts/public/public.go b/consts/public/public.go index 6349997..a5c794c 100644 --- a/consts/public/public.go +++ b/consts/public/public.go @@ -1,5 +1,17 @@ package public +const ( + CallModeSync = 0 // 同步调用 + CallModeAsync = 1 // 异步调用 + CallModeStream = 2 // 流式调用 +) + +const ( + BuildTypePrompt = 1 //提示词构建 + BuildTypeNode = 2 //节点构建 + BuildTypeStruct = 3 //结构构建 +) + // ModelType 模型类型常量 const ( ModelTypeInference = 100 // 推理模型 @@ -29,7 +41,6 @@ const ( VideoSubTypeImageToVideo = 602 // 视频模型-图生视频 VideoSubTypeImageTextToVideo = 603 // 视频模型-图文生视频 VideoSubTypeVideoToVideo = 604 // 视频模型-视频生视频 - VideoSubTypeVideoEdit = 605 // 视频模型-视频编辑 ) // ModelTypeName 模型类型名称映射 @@ -61,7 +72,6 @@ var ModelTypeName = map[int]string{ VideoSubTypeImageToVideo: "视频模型-图生视频", VideoSubTypeImageTextToVideo: "视频模型-图文生视频", VideoSubTypeVideoToVideo: "视频模型-视频生视频", - VideoSubTypeVideoEdit: "视频模型-视频编辑", } // 运营商常量 diff --git a/controller/task_controller.go b/controller/task_controller.go index 51bdbe9..9557c76 100644 --- a/controller/task_controller.go +++ b/controller/task_controller.go @@ -43,11 +43,6 @@ func (c *task) ListTask(ctx context.Context, req *dto.ListTaskReq) (res *dto.Lis return taskService.Task.List(ctx, req) } -// RunWork 手动触发一次 worker(由上层定时任务调用) -func (c *task) RunWork(ctx context.Context, req *dto.RunWorkReq) (res *dto.RunWorkRes, err error) { - return taskService.AsyncWorker.RunOnce(ctx, req) -} - // CleanWork 手动触发一次 cleaner(由上层定时任务调用) func (c *task) CleanWork(ctx context.Context, req *dto.CleanWorkReq) (res *dto.CleanWorkRes, err error) { return job.Cleaner.RunOnce(ctx) diff --git a/dao/task_dao_bg.go b/dao/task_dao_bg.go index dfab6d6..4228022 100644 --- a/dao/task_dao_bg.go +++ b/dao/task_dao_bg.go @@ -13,28 +13,53 @@ import ( "github.com/gogf/gf/v2/os/gtime" ) -// ClaimPendingGlobal 后台任务使用:全局抢占 pending 任务(不加 tenant 过滤) -func (d *taskDao) ClaimPendingGlobal(ctx context.Context, batchSize int) (tasks []*entity.AsynchTask, err error) { +// ======================== 查询辅助 ======================== + +// taskColumns 查询用的公共字段 +const taskColumns = `id, tenant_id, creator, model_name, task_id, biz_name, callback_url, model_key, retry_count, input_ref, request_payload, phase, tmp_file` + +// ======================== 事务抢占 ======================== + +// claimTasks 事务内抢占任务并更新 state=1 +func claimTasks(ctx context.Context, where string, args ...any) ([]*entity.AsynchTask, error) { + var tasks []*entity.AsynchTask + err := gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + sql := fmt.Sprintf(`SELECT %s FROM %s WHERE deleted_at IS NULL AND state = 0 %s LIMIT 1 FOR UPDATE SKIP LOCKED`, taskColumns, public.TableNameTask, where) + r, err := tx.GetOne(sql, args...) + if err != nil { + return err + } + if r.IsEmpty() { + return nil + } + var task entity.AsynchTask + if err := r.Struct(&task); err != nil { + return err + } + now := time.Now() + _, err = tx.Exec(fmt.Sprintf(`UPDATE %s SET state=1, started_at=?, updated_at=? WHERE id=?`, public.TableNameTask), now, now, task.Id) + if err != nil { + return err + } + tasks = []*entity.AsynchTask{&task} + return nil + }) + return tasks, err +} + +// ClaimPendingGlobal 批量抢占 pending 任务 +func (d *taskDao) ClaimPendingGlobal(ctx context.Context, batchSize int) ([]*entity.AsynchTask, error) { if batchSize <= 0 { batchSize = 1 } - err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - sql := fmt.Sprintf( - `SELECT id, tenant_id, creator, model_name, task_id, biz_name, callback_url, model_key, retry_count, input_ref, request_payload, phase, tmp_file - FROM %s - WHERE deleted_at IS NULL AND state = 0 - ORDER BY enqueue_at ASC - LIMIT %d - FOR UPDATE SKIP LOCKED`, - public.TableNameTask, - batchSize, - ) + var tasks []*entity.AsynchTask + err := gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + sql := fmt.Sprintf(`SELECT %s FROM %s WHERE deleted_at IS NULL AND state = 0 ORDER BY enqueue_at ASC LIMIT %d FOR UPDATE SKIP LOCKED`, taskColumns, public.TableNameTask, batchSize) r, err := tx.GetAll(sql) if err != nil { return err } if r.IsEmpty() { - tasks = nil return nil } if err := r.Structs(&tasks); err != nil { @@ -42,234 +67,148 @@ func (d *taskDao) ClaimPendingGlobal(ctx context.Context, batchSize int) (tasks } now := time.Now() for _, t := range tasks { - _, err = tx.Exec( - fmt.Sprintf(`UPDATE %s SET state=1, started_at=?, updated_at=? WHERE id=?`, public.TableNameTask), - now, now, t.Id, - ) + _, err = tx.Exec(fmt.Sprintf(`UPDATE %s SET state=1, started_at=?, updated_at=? WHERE id=?`, public.TableNameTask), now, now, t.Id) if err != nil { return err } } return nil }) - return + return tasks, err } -// ClaimPendingByTaskIDGlobal 按 task_id 定向抢占单个 pending 任务(不加 tenant 过滤) -// 用于 createTask 创建成功后立即异步尝试执行当前任务,避免只依赖后续 runWork 扫描队列。 -func (d *taskDao) ClaimPendingByTaskIDGlobal(ctx context.Context, taskID string) (task *entity.AsynchTask, err error) { +// ClaimPendingByTaskIDGlobal 按 task_id 抢占 +func (d *taskDao) ClaimPendingByTaskIDGlobal(ctx context.Context, taskID string) (*entity.AsynchTask, error) { if taskID == "" { return nil, nil } - err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - sql := fmt.Sprintf( - `SELECT id, tenant_id, creator, model_name, task_id, biz_name, callback_url, model_key, retry_count, input_ref, request_payload, phase, tmp_file - FROM %s - WHERE deleted_at IS NULL AND state = 0 AND task_id = ? - LIMIT 1 - FOR UPDATE SKIP LOCKED`, - public.TableNameTask, - ) - r, err := tx.GetOne(sql, taskID) - if err != nil { - return err - } - if r.IsEmpty() { - task = nil - return nil - } - if err := r.Struct(&task); err != nil { - return err - } - now := time.Now() - _, err = tx.Exec( - fmt.Sprintf(`UPDATE %s SET state=1, started_at=?, updated_at=? WHERE id=?`, public.TableNameTask), - now, now, task.Id, - ) - return err - }) - return + tasks, err := claimTasks(ctx, "AND task_id = ?", taskID) + if err != nil || len(tasks) == 0 { + return nil, err + } + return tasks[0], nil +} + +// ======================== 更新辅助 ======================== + +func execSQL(ctx context.Context, sql string, args ...any) error { + _, err := gfdb.DB(ctx).Exec(ctx, sql, args...) + return err +} + +// updateTask 通用更新 +func updateTask(ctx context.Context, id int64, data entity.AsynchTask) error { + _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask).OmitEmpty(). + Where(entity.AsynchTaskCol.Id, id).Data(data).Update() + return err } // UpdateSuccessGlobal 更新任务成功 func (d *taskDao) UpdateSuccessGlobal(ctx context.Context, t *entity.AsynchTask) error { - now := gtime.Now() - _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask).OmitEmpty(). - Where(entity.AsynchTaskCol.Id, t.Id). - Data(entity.AsynchTask{ - State: 2, - OssFile: t.OssFile, - FileType: t.FileType, - TextResult: t.TextResult, - FileSize: t.FileSize, - ErrorMsg: "", - FinishedAt: now, - Phase: 0, - TmpFile: "", - ExpendTokens: t.ExpendTokens, - }). - Update() - return err + return updateTask(ctx, t.Id, entity.AsynchTask{ + State: 2, + OssFile: t.OssFile, + FileType: t.FileType, + TextResult: t.TextResult, + FileSize: t.FileSize, + ErrorMsg: "", + FinishedAt: gtime.Now(), + Phase: 0, + TmpFile: "", + ExpendTokens: t.ExpendTokens, + DurationSeconds: t.DurationSeconds, + }) } // UpdateFailedGlobal 模型调用失败 func (d *taskDao) UpdateFailedGlobal(ctx context.Context, t *entity.AsynchTask) error { - now := gtime.Now() + return updateTask(ctx, t.Id, entity.AsynchTask{ + State: 3, + ErrorMsg: t.ErrorMsg, + FinishedAt: gtime.Now(), + Phase: 0, + TmpFile: "", + TextResult: t.TextResult, + DurationSeconds: t.DurationSeconds, + }) +} + +// UpdateFailedKeepTmpGlobal OSS 上传失败 +func (d *taskDao) UpdateFailedKeepTmpGlobal(ctx context.Context, id int64, errorMsg string) error { + return execSQL(ctx, fmt.Sprintf(`UPDATE %s SET state=3, error_msg=?, finished_at=?, phase=1, updated_at=? WHERE id=?`, public.TableNameTask), errorMsg, gtime.Now(), gtime.Now(), id) +} + +// UpdateTmpAfterModelGlobal 写临时文件 +func (d *taskDao) UpdateTmpAfterModelGlobal(ctx context.Context, id int64, tmpFile string) error { + return execSQL(ctx, fmt.Sprintf(`UPDATE %s SET phase=1, tmp_file=?, updated_at=NOW() WHERE id=?`, public.TableNameTask), tmpFile, id) +} + +// RollbackToPendingGlobal 回滚 +func (d *taskDao) RollbackToPendingGlobal(ctx context.Context, id int64) error { + return execSQL(ctx, fmt.Sprintf(`UPDATE %s SET state=0, enqueue_at=NOW(), updated_at=NOW() WHERE id=? AND state=1`, public.TableNameTask), id) +} + +// IncRetryCountGlobal 重试计数+1 +func (d *taskDao) IncRetryCountGlobal(ctx context.Context, id int64) error { + return execSQL(ctx, fmt.Sprintf(`UPDATE %s SET retry_count=retry_count+1, updated_at=NOW() WHERE id=?`, public.TableNameTask), id) +} + +// RequeueForRetryGlobal 重新入队 +func (d *taskDao) RequeueForRetryGlobal(ctx context.Context, id int64, enqueueAt time.Time) error { + return execSQL(ctx, fmt.Sprintf(`UPDATE %s SET state=0, retry_count=retry_count+1, enqueue_at=?, updated_at=NOW() WHERE id=? AND state=3 AND deleted_at IS NULL`, public.TableNameTask), enqueueAt, id) +} + +// ======================== 列表查询 ======================== + +// ListExpiredDownloadedGlobal +func (d *taskDao) ListExpiredDownloadedGlobal(ctx context.Context, limit int) ([]*entity.AsynchTask, error) { + return queryTasks(ctx, fmt.Sprintf(`SELECT * FROM %s WHERE deleted_at IS NULL AND state=4 AND expire_at IS NOT NULL AND expire_at < ? LIMIT ?`, public.TableNameTask), gtime.Now(), clampLimit(limit, 200)) +} + +// ListFailedRetryableGlobal +func (d *taskDao) ListFailedRetryableGlobal(ctx context.Context, limit int) ([]*entity.AsynchTask, error) { + return queryTasks(ctx, fmt.Sprintf(`SELECT t.*, m.retry_queue_max_seconds FROM %s t JOIN %s m ON t.tenant_id=m.tenant_id AND t.model_name=m.model_name WHERE t.deleted_at IS NULL AND t.state=3 AND t.retry_count < m.retry_times ORDER BY t.updated_at ASC LIMIT ?`, public.TableNameTask, public.TableNameModel), clampLimit(limit, 200)) +} + +// ListFailedExhaustedGlobal +func (d *taskDao) ListFailedExhaustedGlobal(ctx context.Context, limit int) ([]*entity.AsynchTask, error) { + return queryTasks(ctx, fmt.Sprintf(`SELECT t.* FROM %s t JOIN %s m ON t.tenant_id=m.tenant_id AND t.model_name=m.model_name WHERE t.deleted_at IS NULL AND t.state=3 AND t.retry_count >= m.retry_times ORDER BY t.updated_at ASC LIMIT ?`, public.TableNameTask, public.TableNameModel), clampLimit(limit, 200)) +} + +// ListTimeoutTasksGlobal +func (d *taskDao) ListTimeoutTasksGlobal(ctx context.Context, limit int) ([]*entity.AsynchTask, error) { + return queryTasks(ctx, fmt.Sprintf(`SELECT t.* FROM %s t JOIN %s m ON t.tenant_id=m.tenant_id AND t.model_name=m.model_name WHERE t.deleted_at IS NULL AND t.state IN (0,1) AND m.expected_seconds > 0 AND t.created_at < (NOW() - (m.expected_seconds || ' seconds')::interval) LIMIT ?`, public.TableNameTask, public.TableNameModel), clampLimit(limit, 200)) +} + +// HardDeleteByIDGlobal +func (d *taskDao) HardDeleteByIDGlobal(ctx context.Context, id int64) error { + return execSQL(ctx, fmt.Sprintf(`DELETE FROM %s WHERE id=?`, public.TableNameTask), id) +} + +// ======================== 内部辅助 ======================== + +func queryTasks(ctx context.Context, sql string, args ...any) ([]*entity.AsynchTask, error) { + r, err := gfdb.DB(ctx).GetAll(ctx, sql, args...) + if err != nil { + return nil, err + } + var list []*entity.AsynchTask + err = r.Structs(&list) + return list, err +} + +func clampLimit(limit, defaultVal int) int { + if limit <= 0 { + return defaultVal + } + return limit +} + +// UpdateColumns 更新指定字段(结构体版) +func (d *taskDao) UpdateColumns(ctx context.Context, id int64, data entity.AsynchTask) error { _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask).OmitEmpty(). - Where(entity.AsynchTaskCol.Id, t.Id). - Data(entity.AsynchTask{ - State: 3, - ErrorMsg: t.ErrorMsg, - FinishedAt: now, - Phase: 0, - TmpFile: "", - }). + Where(entity.AsynchTaskCol.Id, id). + Data(data). Update() return err } - -// UpdateFailedKeepTmpGlobal OSS 上传失败:保留 phase/tmp_file,下一轮仅重试 OSS 上传 -func (d *taskDao) UpdateFailedKeepTmpGlobal(ctx context.Context, id int64, errorMsg string) error { - now := gtime.Now() - _, err := gfdb.DB(ctx).Exec(ctx, - fmt.Sprintf(`UPDATE %s SET state=3, error_msg=?, finished_at=?, phase=1, updated_at=? WHERE id=?`, public.TableNameTask), - errorMsg, now, now, id, - ) - return err -} - -// UpdateTmpAfterModelGlobal 模型调用成功后,写入临时文件路径并标记 phase=1 -func (d *taskDao) UpdateTmpAfterModelGlobal(ctx context.Context, id int64, tmpFile string) error { - _, err := gfdb.DB(ctx).Exec(ctx, - fmt.Sprintf(`UPDATE %s SET phase=1, tmp_file=?, updated_at=NOW() WHERE id=?`, public.TableNameTask), - tmpFile, id, - ) - return err -} - -func (d *taskDao) RollbackToPendingGlobal(ctx context.Context, id int64) error { - _, err := gfdb.DB(ctx).Exec(ctx, - fmt.Sprintf(`UPDATE %s SET state=0, enqueue_at=NOW(), updated_at=NOW() WHERE id=? AND state=1`, public.TableNameTask), - id, - ) - return err -} - -// ListExpiredDownloadedGlobal 获取已下载(state=4)且过期的任务,用于清理 -func (d *taskDao) ListExpiredDownloadedGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { - if limit <= 0 { - limit = 200 - } - r, err := gfdb.DB(ctx).GetAll(ctx, - fmt.Sprintf(`SELECT * FROM %s WHERE deleted_at IS NULL AND state=4 AND expire_at IS NOT NULL AND expire_at < ? LIMIT ?`, public.TableNameTask), - gtime.Now(), limit, - ) - if err != nil { - return nil, err - } - err = r.Structs(&list) - return -} - -// ListFailedRetryableGlobal 获取失败(state=3)且仍可重试的任务 -// retry_count 不含首次执行;retry_times 表示失败后最多再重试 N 次 -func (d *taskDao) ListFailedRetryableGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { - if limit <= 0 { - limit = 200 - } - r, err := gfdb.DB(ctx).GetAll(ctx, - fmt.Sprintf(` -SELECT t.*, - m.retry_queue_max_seconds AS retry_queue_max_seconds - FROM %s t - JOIN %s m - ON t.tenant_id = m.tenant_id - AND t.model_name = m.model_name - WHERE t.deleted_at IS NULL - AND t.state = 3 - AND t.retry_count < m.retry_times - ORDER BY t.updated_at ASC - LIMIT ?`, public.TableNameTask, public.TableNameModel), - limit, - ) - if err != nil { - return nil, err - } - err = r.Structs(&list) - return -} - -// RequeueForRetryGlobal 将任务重新入队(state=0),并将 retry_count +1 -// enqueueAt 用于控制重试任务在队列中的位置: -// - enqueueAt 越早,越靠前(ClaimPendingGlobal 按 enqueue_at ASC 抢占) -func (d *taskDao) RequeueForRetryGlobal(ctx context.Context, id int64, enqueueAt time.Time) error { - _, err := gfdb.DB(ctx).Exec(ctx, - fmt.Sprintf(`UPDATE %s SET state=0, retry_count=retry_count+1, enqueue_at=?, updated_at=NOW() WHERE id=? AND state=3 AND deleted_at IS NULL`, public.TableNameTask), - enqueueAt, id, - ) - return err -} - -// ListFailedExhaustedGlobal 获取失败(state=3)且超过重试次数的任务,用于硬删除 -func (d *taskDao) ListFailedExhaustedGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { - if limit <= 0 { - limit = 200 - } - r, err := gfdb.DB(ctx).GetAll(ctx, - fmt.Sprintf(` -SELECT t.* - FROM %s t - JOIN %s m - ON t.tenant_id = m.tenant_id - AND t.model_name = m.model_name - WHERE t.deleted_at IS NULL - AND t.state = 3 - AND t.retry_count >= m.retry_times - ORDER BY t.updated_at ASC - LIMIT ?`, public.TableNameTask, public.TableNameModel), - limit, - ) - if err != nil { - return nil, err - } - err = r.Structs(&list) - return -} - -// HardDeleteByIDGlobal 硬删除任务记录 -func (d *taskDao) HardDeleteByIDGlobal(ctx context.Context, id int64) error { - _, err := gfdb.DB(ctx).Exec(ctx, - fmt.Sprintf(`DELETE FROM %s WHERE id=?`, public.TableNameTask), - id, - ) - return err -} - -// ListTimeoutTasksGlobal 根据模型配置 expected_seconds 判定超时任务: -// - state in (0,1) -// - 模型 expected_seconds > 0 -// - now - created_at >= expected_seconds -func (d *taskDao) ListTimeoutTasksGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { - if limit <= 0 { - limit = 200 - } - r, err := gfdb.DB(ctx).GetAll(ctx, - fmt.Sprintf(` -SELECT t.* - FROM %s t - JOIN %s m - ON t.tenant_id = m.tenant_id - AND t.model_name = m.model_name - WHERE t.deleted_at IS NULL - AND t.state IN (0,1) - AND m.expected_seconds > 0 - AND t.created_at < (NOW() - (m.expected_seconds || ' seconds')::interval) - LIMIT ?`, public.TableNameTask, public.TableNameModel), - limit, - ) - if err != nil { - return nil, err - } - err = r.Structs(&list) - return -} diff --git a/go.mod b/go.mod index b7b70f9..efe8e7f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module model-gateway go 1.26.1 require ( - gitea.com/red-future/common v0.0.20 + gitea.com/red-future/common v0.0.21 github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.2 github.com/gogf/gf/contrib/nosql/redis/v2 v2.10.2 github.com/gogf/gf/v2 v2.10.2 @@ -12,22 +12,25 @@ require ( ) require ( - github.com/BurntSushi/toml v1.5.0 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clbanning/mxj/v2 v2.7.0 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dgraph-io/badger/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect - github.com/fatih/color v1.18.0 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-ego/gse v1.0.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -50,39 +53,41 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/lib/pq v1.10.9 // indirect + github.com/lib/pq v1.12.3 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/olekukonko/errors v1.1.0 // indirect - github.com/olekukonko/ll v0.0.9 // indirect - github.com/olekukonko/tablewriter v1.1.0 // indirect + github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect + github.com/olekukonko/errors v1.3.0 // indirect + github.com/olekukonko/ll v0.1.8 // indirect + github.com/olekukonko/tablewriter v1.1.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/r3labs/diff/v2 v2.15.1 // indirect - github.com/redis/go-redis/v9 v9.12.1 // indirect + github.com/redis/go-redis/v9 v9.20.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tiger1103/gfast-token v1.0.10 // indirect github.com/vcaesar/cedar v0.30.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect go.mongodb.org/mongo-driver/v2 v2.4.0 // indirect go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/sdk v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.uber.org/atomic v1.11.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect diff --git a/go.sum b/go.sum index 82cd765..0a1b96e 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= gitea.com/red-future/common v0.0.20 h1:KlKINnJFmOVkDzgkptEAFsdpMUZb0zK9BTdiXRxVfAo= gitea.com/red-future/common v0.0.20/go.mod h1:6/nqIucVzmjOyqDTIq71feYBXXFNBy0rFwzaQ0/Ueoo= +gitea.com/red-future/common v0.0.21 h1:8w30HmCVmFG/hphH3ODJs1KxDEGmRpq+/PXI0pQjJKc= +gitea.com/red-future/common v0.0.21/go.mod h1:6/nqIucVzmjOyqDTIq71feYBXXFNBy0rFwzaQ0/Ueoo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -36,6 +40,10 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -52,6 +60,7 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods/v2 v2.0.0-alpha h1:dwFlh8pBg1VMOXWGipNMRt8v96dKAIvBehtCt6OtunU= github.com/emirpasic/gods/v2 v2.0.0-alpha/go.mod h1:W0y4M2dtBB9U5z3YlghmpuUhiaZT2h6yoeE+C1sCp6A= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -63,8 +72,12 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-ego/gse v1.0.2 h1:+27lYFPhQEhA9igtdOsJPRKYL/k3TwYsxBF5jr6KFv4= github.com/go-ego/gse v1.0.2/go.mod h1:Fy35G+q7VV7Et1zIKO8o/sW1kkugV3znXap/lF/11zc= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -77,6 +90,8 @@ github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.2 h1:u8EpP24GkprogROnJ7htMov9Fc66pTP1eVYrWxiCYOs= github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.2/go.mod h1:GmvM3r8GVByVMi4RD2+MCs5+CfxVXPMeT8mVDkAaAXE= github.com/gogf/gf/contrib/nosql/redis/v2 v2.10.2 h1:iTQegT+lEg/wDKvj2mi3W1wrdrwFarjokf88EXVVgu4= @@ -198,6 +213,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -207,6 +224,8 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -214,8 +233,12 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -232,12 +255,20 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/errors v1.3.0 h1:teJvgLGUEqMzBUms+Dj3/3szNqCG/Jdw9iDbum8fR6U= +github.com/olekukonko/errors v1.3.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.0.9 h1:Y+1YqDfVkqMWuEQMclsF9HUR5+a82+dxJuL1HHSRpxI= github.com/olekukonko/ll v0.0.9/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g= +github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8= +github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw= github.com/olekukonko/tablewriter v1.1.0 h1:N0LHrshF4T39KvI96fn6GT8HEjXRXYNDrDjKFDB7RIY= github.com/olekukonko/tablewriter v1.1.0/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo= +github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= +github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -266,11 +297,14 @@ github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= +github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -292,8 +326,12 @@ github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tiger1103/gfast-token v1.0.10 h1:fNiBE/Dq5iTHvTGlCx3DmXa2o4hr0NtumFpffZ39k6s= github.com/tiger1103/gfast-token v1.0.10/go.mod h1:a/21mxmj7zFeNvjhZSC0XpEAFHfb1aT2k6DXnufFU1s= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -311,22 +349,35 @@ go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -344,6 +395,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -361,6 +413,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -371,6 +425,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -397,6 +452,8 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -404,6 +461,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -415,6 +474,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 75f77ed..cb3edbe 100644 --- a/main.go +++ b/main.go @@ -47,33 +47,6 @@ func main() { } func startAutoRunner(ctx context.Context) { - // worker - if g.Cfg().MustGet(ctx, "asynch.worker.enabled").Bool() { - interval := g.Cfg().MustGet(ctx, "asynch.worker.intervalSeconds").Int() - if interval <= 0 { - interval = 5 - } - batchSize := g.Cfg().MustGet(ctx, "asynch.worker.batchSize").Int() - goroutines := g.Cfg().MustGet(ctx, "asynch.worker.goroutines").Int() - ticker := time.NewTicker(time.Duration(interval) * time.Second) - go func() { - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if _, err := task.AsyncWorker.RunOnce(ctx, &dto.RunWorkReq{ - BatchSize: batchSize, - Goroutines: goroutines, - }); err != nil { - g.Log().Warningf(ctx, "[auto-worker] run once failed: %v", err) - } - } - } - }() - } - // cleaner if g.Cfg().MustGet(ctx, "asynch.cleaner.enabled").Bool() { interval := g.Cfg().MustGet(ctx, "asynch.cleaner.intervalSeconds").Int() diff --git a/model/dto/model_dto.go b/model/dto/model_dto.go index fd972d1..c3774a4 100644 --- a/model/dto/model_dto.go +++ b/model/dto/model_dto.go @@ -18,8 +18,8 @@ type CreateModelReq struct { IsPrivate *int `p:"isPrivate" json:"isPrivate" dc:"是否私有化:0-私有 1-公共"` Enabled *int `p:"enabled" json:"enabled" dc:"是否启用:0-停用 1-启用"` IsChatModel *int `p:"isChatModel" json:"isChatModel" dc:"是否为对话模型:0-否 1-是"` - IsAsync *int `p:"isAsync" json:"isAsync" dc:"是否异步:0-同步 1-异步"` - IsStream *int `p:"isStream" json:"isStream" dc:"是否流式:0-非流式 1-流式"` + CallModel *int `p:"callModel" json:"callModel" dc:"调用模式:0-同步 1-异步 2-流式"` + RequiredFields []string `p:"requiredFields" json:"requiredFields" dc:"必填字段"` IsOwner *int `p:"isOwner" json:"isOwner" dc:"是否为所有者:0-否 1-是"` ApiKey string `p:"apiKey" json:"apiKey" dc:"调用凭证/密钥"` Form []map[string]any `p:"form" json:"form" dc:"动态表单配置"` @@ -56,8 +56,8 @@ type UpdateModelReq struct { IsPrivate *int `p:"isPrivate" json:"isPrivate" dc:"是否私有化:0-私有 1-公共"` Enabled *int `p:"enabled" json:"enabled" dc:"是否启用:0-停用 1-启用"` IsChatModel *int `p:"isChatModel" json:"isChatModel" dc:"是否为对话模型:0-否 1-是"` - IsAsync *int `p:"isAsync" json:"isAsync" dc:"是否异步:0-同步 1-异步"` - IsStream *int `p:"isStream" json:"isStream" dc:"是否流式:0-非流式 1-流式"` + CallModel *int `p:"callModel" json:"callModel" dc:"调用模式:0-同步 1-异步 2-流式"` + RequiredFields []string `p:"requiredFields" json:"requiredFields" dc:"必填字段"` IsOwner *int `p:"isOwner" json:"isOwner" dc:"是否为所有者:0-否 1-是"` ApiKey string `p:"apiKey" json:"apiKey" dc:"调用凭证/密钥"` Form []map[string]any `p:"form" json:"form" dc:"动态表单配置"` diff --git a/model/entity/asynch_model.go b/model/entity/asynch_model.go index 5202e45..6c6ba7c 100644 --- a/model/entity/asynch_model.go +++ b/model/entity/asynch_model.go @@ -14,10 +14,10 @@ type asynchModelCol struct { ResponseMapping string ResponseBody string ResponseTokenField string + RequiredFields string IsPrivate string IsChatModel string - IsAsync string - IsStream string + CallMode string ApiKey string Enabled string MaxConcurrency string @@ -47,10 +47,10 @@ var AsynchModelCol = asynchModelCol{ ResponseMapping: "response_mapping", ResponseBody: "response_body", ResponseTokenField: "response_token_field", + RequiredFields: "required_fields", IsPrivate: "is_private", IsChatModel: "is_chat_model", - IsAsync: "is_async", - IsStream: "is_stream", + CallMode: "call_mode", ApiKey: "api_key", Enabled: "enabled", MaxConcurrency: "max_concurrency", @@ -81,10 +81,10 @@ type AsynchModel struct { ResponseMapping map[string]any `orm:"response_mapping" json:"responseMapping"` ResponseBody string `orm:"response_body" json:"responseBody"` ResponseTokenField string `orm:"response_token_field" json:"responseTokenField"` + RequiredFields []string `orm:"required_fields" json:"requiredFields"` IsPrivate *int `orm:"is_private" json:"isPrivate"` IsChatModel *int `orm:"is_chat_model" json:"isChatModel"` - IsAsync *int `orm:"is_async" json:"isAsync"` - IsStream *int `orm:"is_stream" json:"isStream"` + CallMode *int `orm:"call_mode" json:"callMode"` ApiKey string `orm:"api_key" json:"apiKey"` Enabled *int `orm:"enabled" json:"enabled"` MaxConcurrency int `orm:"max_concurrency" json:"maxConcurrency"` diff --git a/service/task/task_service.go b/service/task/task_service.go index 6aa3366..8aae7af 100644 --- a/service/task/task_service.go +++ b/service/task/task_service.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "model-gateway/common/util" + "model-gateway/consts/public" "model-gateway/service/queue" "time" @@ -61,7 +62,7 @@ func (s *taskService) Create(ctx context.Context, req *dto.CreateTaskReq) (res * } // 3) 插入任务记录 - if model.IsAsync != nil && *model.IsAsync == 1 { + if model.CallMode != nil && *model.CallMode == public.CallModeAsync { // 异步调用:注入回调地址后提交,拿到 task_id 轮询 req.RequestPayload = util.InjectCallbackURL(ctx, req.RequestPayload, model.CallbackUrl) } diff --git a/service/task/worker.go b/service/task/worker.go index 7d49d29..a9c9553 100644 --- a/service/task/worker.go +++ b/service/task/worker.go @@ -4,13 +4,8 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" - "model-gateway/common/util" - "model-gateway/model/dto" - "model-gateway/service/gateway" - "model-gateway/service/queue" "net/http" "os" "strings" @@ -18,12 +13,17 @@ import ( "time" "unicode/utf8" + "model-gateway/common/util" + "model-gateway/consts/public" "model-gateway/dao" + "model-gateway/model/dto" "model-gateway/model/entity" + "model-gateway/service/gateway" + "model-gateway/service/queue" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/grpool" + "github.com/gogf/gf/v2/util/gconv" ) var AsyncWorker = &asyncWorker{} @@ -31,61 +31,21 @@ var AsyncWorker = &asyncWorker{} type asyncWorker struct { } -// RunOnce 由上层定时任务触发:一次性抢占并处理一批任务 -// - batchSize: 本次抢占数量 -// - goroutines: 本次并发数(协程池大小) -func (w *asyncWorker) RunOnce(ctx context.Context, req *dto.RunWorkReq) (res *dto.RunWorkRes, err error) { - if req.BatchSize <= 0 { - req.BatchSize = 10 - } - if req.Goroutines <= 0 { - req.Goroutines = 1 - } - tasks, err := dao.Task.ClaimPendingGlobal(ctx, req.BatchSize) - if err != nil { - return nil, err - } - if len(tasks) == 0 { - return nil, errors.New("no task to run") - } - pool := grpool.New(req.Goroutines) - defer pool.Close() - claimed := len(tasks) - done := make(chan struct{}, claimed) - for _, t := range tasks { - task := t - _ = pool.AddWithRecover(ctx, func(ctx context.Context) { - //w.handleOne(ctx, task, &dto.CreateTaskReq{EpicycleId: 0}) - done <- struct{}{} - }, func(ctx context.Context, e error) { - if e != nil { - task.ErrorMsg = fmt.Sprintf("worker panic: %v", e) - _ = dao.Task.UpdateFailedGlobal(ctx, task) - queue.ReleaseQueueSlot(ctx, task.ModelName, task.TaskID) - } - done <- struct{}{} - }) - } - for i := 0; i < claimed; i++ { - <-done - } - return &dto.RunWorkRes{ - Claimed: claimed, - }, nil -} - // handleOne 执行一次完整的任务 func (w *asyncWorker) handleOne(ctx context.Context, task *entity.AsynchTask, model *entity.AsynchModel, req *dto.CreateTaskReq) { - body := util.GetModelBody(task.RequestPayload) //核心请求参数 - maxRetry := model.RetryTimes //重试次数 + body := util.GetModelBody(task.RequestPayload) // 核心请求参数 + maxRetry := model.RetryTimes // 重试次数 + startTime := time.Now() g.Log().Infof(ctx, "[执行任务][开始] taskId=%s model=%s", task.TaskID, task.ModelName) + // 1) 分布式并发控制 semKey := fmt.Sprintf("asynch:sem:%s", task.ModelName) maxC := queue.GetRuntimeMaxConcurrency(ctx, task.ModelName, model.MaxConcurrency) acquired, err := queue.AcquireSemaphore(ctx, semKey, maxC, 3600) if err != nil { - w.failTask(ctx, task, err.Error()) + task.DurationSeconds = int64(time.Since(startTime).Seconds()) + w.failTask(ctx, task, startTime, err.Error()) return } if !acquired { @@ -95,62 +55,55 @@ func (w *asyncWorker) handleOne(ctx context.Context, task *entity.AsynchTask, mo } defer func() { _ = queue.ReleaseSemaphore(ctx, semKey) }() - // 2) request_payload 校验 - if body == nil { - w.failTask(ctx, task, "请求模型为空") - return - } - - // 3) 调用模型 + // 2) 调用模型 switch { - case model.IsStream != nil && *model.IsStream == 1: // 流式调用 + case model.CallMode != nil && *model.CallMode == public.CallModeStream: rawBytes, err := w.callModelStream(ctx, task, model, body) if err != nil { - w.failTask(ctx, task, err.Error()) + w.failTask(ctx, task, startTime, err.Error()) return } - // 解析流式结果 body, err = util.ParseStreamResponse(rawBytes, model.StreamConfig) if err != nil { - w.failTask(ctx, task, err.Error()) + w.failTask(ctx, task, startTime, err.Error()) return } - case model.IsAsync != nil && *model.IsAsync == 1: // 异步调用:注入回调地址后提交,拿到 task_id 轮询 - // 异步调用:提交任务 + case model.CallMode != nil && *model.CallMode == public.CallModeAsync: body, err = w.callModel(ctx, task, model, body) if err != nil { - w.failTask(ctx, task, err.Error()) + w.failTask(ctx, task, startTime, err.Error()) return } body, err = util.PullTaskResult(ctx, body, model.QueryConfig, model.HeadMsg) if err != nil { - w.failTask(ctx, task, err.Error()) + w.failTask(ctx, task, startTime, err.Error()) return } - default: // 同步调用 + default: body, err = w.callModel(ctx, task, model, body) if err != nil { - w.failTask(ctx, task, err.Error()) + w.failTask(ctx, task, startTime, err.Error()) return } } - // 5) 解析响应映射 - body, err = util.MapResponsePayload(model.ResponseMapping, body) - if err != nil { - w.failTask(ctx, task, err.Error()) - return - } - - // 5) 保存临时文件(通用工具方法) - tmpPath, tmpErr := util.SaveTempFileByType(task.TaskID, body, task.TmpFile) - if tmpErr == nil && tmpPath != "" { + // 3) 保存临时文件 + tmpPath, err := util.SaveTempFileByType(task.TaskID, body, task.TmpFile) + if err == nil && tmpPath != "" { task.TmpFile = tmpPath task.Phase = 1 _ = dao.Task.UpdateTmpAfterModelGlobal(ctx, task.Id, tmpPath) } - // 6) 上传 OSS(可重试) + // 4) 解析校验 + 响应映射(可重试,失败重新调模型) + body, err = w.parseAndRetry(ctx, body, task, model, req, maxRetry, startTime) + if err != nil { + task.TextResult = body + w.failTask(ctx, task, startTime, err.Error()) + return + } + + // 5) 上传 OSS(可重试) var oss *gateway.UploadFileResponse for attempt := 0; attempt <= maxRetry; attempt++ { if attempt > 0 { @@ -164,46 +117,18 @@ func (w *asyncWorker) handleOne(ctx context.Context, task *entity.AsynchTask, mo task.TaskID, attempt, maxRetry, err) if attempt == maxRetry { _ = dao.Task.UpdateFailedKeepTmpGlobal(ctx, task.Id, err.Error()) - w.failTask(ctx, task, fmt.Sprintf("OSS上传重试耗尽: %v", err)) + w.failTask(ctx, task, startTime, fmt.Sprintf("OSS上传重试耗尽: %v", err)) return } } - // 7) 解析校验(可重试,失败重新调模型) - if req.BuildType == 1 { - for attempt := 0; attempt <= maxRetry; attempt++ { - if attempt > 0 { - g.Log().Infof(ctx, "[执行任务][重试] JSON解析 第%d/%d次 taskId=%s", attempt, maxRetry, task.TaskID) - } - // 6.1) 校验数据 - err = util.ValidatePromptResult(body, model) - if err == nil { - break - } - g.Log().Warningf(ctx, "[执行任务][解析失败] taskId=%s attempt=%d/%d err=%v", - task.TaskID, attempt, maxRetry, err) - if attempt == maxRetry { - w.failTask(ctx, task, fmt.Sprintf("JSON解析重试耗尽: %v", err)) - return - } - // 6.2) 重新调模型 - newResult, modelErr := w.callModel(ctx, task, model, body) - if modelErr != nil { - g.Log().Warningf(ctx, "[执行任务][重试] 重新调模型失败 taskId=%s attempt=%d/%d err=%v", - task.TaskID, attempt, maxRetry, modelErr) - continue - } - body = newResult - } - } - - // 8) 成功回调 + // 6) 成功回调 task.State = 2 + task.DurationSeconds = int64(time.Since(startTime).Seconds()) task.OssFile = oss.FileAddressPrefix + oss.FileURL task.FileType = oss.FileFormat task.TextResult = body task.FileSize = int64(oss.FileSize) - task.ExpendTokens = int64(GetExpendTokens(model.ResponseTokenField, body)) if err = dao.Task.UpdateSuccessGlobal(ctx, task); err != nil { g.Log().Errorf(ctx, "[执行任务][失败] 更新数据库失败 taskId=%s err=%v", task.TaskID, err) @@ -216,10 +141,10 @@ func (w *asyncWorker) handleOne(ctx context.Context, task *entity.AsynchTask, mo go gateway.TriggerPromptsCallback(context.WithoutCancel(ctx), task, req.EpicycleId) } - g.Log().Infof(ctx, "[执行任务][成功] taskId=%s fileType=%s textLen=%d callbackUrl=%s", - task.TaskID, oss.FileFormat, len(body), task.CallbackURL) + g.Log().Infof(ctx, "[执行任务][成功] taskId=%s duration=%ds fileType=%s textLen=%d callbackUrl=%s", + task.TaskID, task.DurationSeconds, oss.FileFormat, len(body), task.CallbackURL) - // 9) 删除临时文件 + // 7) 删除临时文件 _ = os.Remove(task.TmpFile) } @@ -263,12 +188,12 @@ var asyncTaskChan = sync.Map{} // taskID → chan asyncResult func (w *asyncWorker) callModelAsync(ctx context.Context, task *entity.AsynchTask, model *entity.AsynchModel, body map[string]any) (map[string]any, error) { // 1. 提交异步任务 - result, err := w.callModel(ctx, task, model, body) + body, err := w.callModel(ctx, task, model, body) if err != nil { return nil, err } // 2. 拿到 task_id - taskID := gjson.New(result).Get(model.ResponseBody).String() + taskID := gjson.New(body).Get(model.ResponseBody).String() // 3. 创建等待通道 ch := make(chan asyncResult, 1) @@ -304,26 +229,31 @@ func NotifyAsyncResult(taskID string, result map[string]any, err error) { } } -// 返回: ossURL(成功时有值), fileType, textResult(失败时是错误信息), retryable(是否可重试) // callModel 调用模型 + 检测文件类型 + 保存临时文件 +// 返回: 解析后的响应体, error func (w *asyncWorker) callModel(ctx context.Context, task *entity.AsynchTask, model *entity.AsynchModel, body map[string]any) (map[string]any, error) { var data []byte - var contentType, ext, textResult string var err error + // 1) 如果已有临时文件且 phase=1,直接读取 if task.Phase == 1 && strings.TrimSpace(task.TmpFile) != "" { data, err = os.ReadFile(task.TmpFile) if err != nil || len(data) == 0 { + g.Log().Warningf(ctx, "[callModel] 读取临时文件失败,重新调用模型 taskId=%s err=%v", task.TaskID, err) data = nil } } + // 2) 没有可用数据,调用模型 if data == nil { _ = dao.Stat.IncRequestCount(ctx, time.Now(), int64(task.TenantId), task.Creator, task.ModelName) data, err = InvokeModel(ctx, model, body, task.ModelKey) if err != nil { return nil, err } + + // 3) 检测文件类型,保存临时文件 + _, ext := util.DetectFileType(data) tmpPath, tmpErr := util.SaveTmpResult(task.TaskID, data, ext) if tmpErr == nil && tmpPath != "" { task.TmpFile = tmpPath @@ -332,17 +262,94 @@ func (w *asyncWorker) callModel(ctx context.Context, task *entity.AsynchTask, mo } } - contentType, ext = util.DetectFileType(data) + // 4) 检测文件类型,提取文本结果 + contentType, _ := util.DetectFileType(data) + var textResult string if utf8.Valid(data) && (strings.HasPrefix(contentType, "text/") || contentType == "application/json") { textResult = string(data) } + + // 5) 非文本内容,返回错误 + if textResult == "" { + return nil, fmt.Errorf("模型返回非文本内容,contentType=%s", contentType) + } + + // 6) 解析并返回 return gjson.New(textResult).Map(), nil } +// parseAndRetry 解析模型返回结果,并重试 +func (w *asyncWorker) parseAndRetry(ctx context.Context, body map[string]any, task *entity.AsynchTask, model *entity.AsynchModel, req *dto.CreateTaskReq, maxRetry int, startTime time.Time) (map[string]any, error) { + for attempt := 0; attempt <= maxRetry; attempt++ { + if attempt > 0 { + g.Log().Infof(ctx, "[执行任务][重试] JSON解析 第%d/%d次 taskId=%s", attempt, maxRetry, task.TaskID) + } + + // 1) 响应映射 + mapped, err := util.MapResponsePayload(model.ResponseMapping, body) + if err != nil { + g.Log().Warningf(ctx, "[执行任务][映射失败] taskId=%s attempt=%d/%d err=%v", task.TaskID, attempt, maxRetry, err) + if attempt == maxRetry { + return nil, fmt.Errorf("响应映射重试耗尽: %w", err) + } + continue + } + + // 2) 先存 token 到数据库,防止后续失败丢失 + if tokens, ok := mapped[model.ResponseTokenField]; ok { + task.ExpendTokens = gconv.Int64(tokens) + _ = dao.Task.UpdateColumns(ctx, task.Id, entity.AsynchTask{ + ExpendTokens: gconv.Int64(body[model.ResponseTokenField]), + }) + } + + // 3) 解析 + 校验 + var parsed map[string]any + switch req.BuildType { + case public.BuildTypePrompt, public.BuildTypeNode: + parsed, err = util.ParseAndValidate(mapped, model) + if err == nil { + return parsed, nil + } + case public.BuildTypeStruct: + parsed = util.ParseStructResult(mapped, model.ResponseBody) + return parsed, nil + default: + return mapped, nil + } + + g.Log().Warningf(ctx, "[执行任务][解析失败] taskId=%s attempt=%d/%d err=%v", task.TaskID, attempt, maxRetry, err) + + if attempt == maxRetry { + return nil, fmt.Errorf("JSON解析重试耗尽: %w", err) + } + + // 4) 重新调模型(直接调,不走缓存) + _ = dao.Task.IncRetryCountGlobal(ctx, task.Id) + reqBody := util.GetModelBody(task.RequestPayload) + rawData, callErr := InvokeModel(ctx, model, reqBody, task.ModelKey) + if callErr != nil { + g.Log().Warningf(ctx, "[执行任务][重调模型失败] taskId=%s attempt=%d/%d err=%v", task.TaskID, attempt, maxRetry, callErr) + continue + } + + // 5) 解析原始响应,覆盖 body 进入下一轮 + var rawResp map[string]any + if err := json.Unmarshal(rawData, &rawResp); err != nil { + g.Log().Warningf(ctx, "[执行任务][Unmarshal失败] taskId=%s err=%v", task.TaskID, err) + continue + } + body = rawResp + } + + return body, nil +} + // InvokeModel 调用模型服务,返回二进制结果 // modelKey 用于覆盖/补充模型配置 head_msg(例如每次请求携带不同的 X-API-Key) func InvokeModel(ctx context.Context, model *entity.AsynchModel, body map[string]any, modelKey string) ([]byte, error) { // 1)请求参数映射:将标准 payload 按模型配置的 requestMapping 转为模型需要的格式 + //—— 请求映射实际处理为提示词构建请求,因为有附加字段及其他字段的拼接。这里不方便做请求映射 //mappedPayload := util.ReverseMap(model.RequestMapping, payload) // 2)构建请求 URL 和超时 @@ -472,9 +479,10 @@ func (w *asyncWorker) uploadOSS(ctx context.Context, t *entity.AsynchTask) (*gat } // failTask 任务失败统一处理:更新数据库 + 释放排队 + 回调 -func (w *asyncWorker) failTask(ctx context.Context, t *entity.AsynchTask, errMsg string) { +func (w *asyncWorker) failTask(ctx context.Context, t *entity.AsynchTask, startTime time.Time, errMsg string) { t.State = 3 t.ErrorMsg = errMsg + t.DurationSeconds = int64(time.Since(startTime).Seconds()) _ = dao.Task.UpdateFailedGlobal(ctx, t) queue.ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) go gateway.TriggerCallback(context.WithoutCancel(ctx), t) @@ -484,12 +492,3 @@ func (w *asyncWorker) failTask(ctx context.Context, t *entity.AsynchTask, errMsg func (w *asyncWorker) rollbackToPending(ctx context.Context, id int64) error { return dao.Task.RollbackToPendingGlobal(ctx, id) } - -// GetExpendTokens 根据映射路径从 result 中提取消耗 token 值 -func GetExpendTokens(responseTokenField string, result map[string]any) int { - val := gjson.New(result).Get(responseTokenField) - if val.IsNil() { - return 0 - } - return val.Int() -}