WangLiZhao
f6c70a451e
- 新增操作日志表（asynch_op_log）及对应DAO，记录任务创建等操作的审计信息 - 新增任务分页查询接口（ListTask）及对应DTO、Service和DAO方法 - 优化模型调用失败重试逻辑：支持配置重试排队策略（插队到队首或队尾） - 新增临时文件存储机制，当模型调用成功但OSS上传失败时，下次仅重试OSS上传 - 模型配置新增retry_queue_max_seconds字段，控制失败重试排队策略 - 更新数据库表结构（asynch_models、asynch_task、新增asynch_op_log）及同步更新SQL - 配置文件调整：超时单位改为秒，更新服务地址和轮询间隔 - 修复模型列表查询支持按名称模糊搜索
105 lines
3.2 KiB
Go
105 lines
3.2 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"time"
|
||
|
||
"model-asynch/dao"
|
||
|
||
"github.com/gogf/gf/v2/frame/g"
|
||
)
|
||
|
||
var Cleaner = &cleaner{}
|
||
|
||
type cleaner struct{}
|
||
|
||
func (c *cleaner) Start(ctx context.Context) {
|
||
if !g.Cfg().MustGet(ctx, "asynch.cleaner.enabled", true).Bool() {
|
||
g.Log().Warningf(ctx, "[cleaner] asynch.cleaner.enabled=false，cleaner 未启动")
|
||
return
|
||
}
|
||
intervalStr := g.Cfg().MustGet(ctx, "asynch.cleaner.interval", "10m").String()
|
||
interval, _ := time.ParseDuration(intervalStr)
|
||
if interval <= 0 {
|
||
interval = 10 * time.Minute
|
||
}
|
||
go func() {
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
g.Log().Infof(ctx, "[cleaner] started, interval=%s", interval)
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
c.runOnce(ctx)
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
|
||
func (c *cleaner) runOnce(ctx context.Context) {
|
||
// 1) 清理已下载(state=4)且过期的任务（硬删除 + OSS）
|
||
expired, err := dao.Task.ListExpiredDownloadedGlobal(ctx, 200)
|
||
if err != nil {
|
||
g.Log().Errorf(ctx, "[cleaner] list expired(downloaded) error: %v", err)
|
||
} else {
|
||
for _, t := range expired {
|
||
deleteTmpResult(t.TmpFile)
|
||
_ = dao.Task.HardDeleteByIDGlobal(ctx, t.Id)
|
||
}
|
||
g.Log().Infof(ctx, "[cleaner] expired(downloaded) cleaned, count=%d", len(expired))
|
||
}
|
||
|
||
// 2) 超时任务标失败
|
||
timeoutStr := g.Cfg().MustGet(ctx, "asynch.worker.taskTimeout", "30m").String()
|
||
timeout, _ := time.ParseDuration(timeoutStr)
|
||
if timeout > 0 {
|
||
list, err := dao.Task.ListTimeoutTasksGlobal(ctx, timeout, 200)
|
||
if err != nil {
|
||
g.Log().Errorf(ctx, "[cleaner] list timeout error: %v", err)
|
||
} else {
|
||
for _, t := range list {
|
||
_ = dao.Task.UpdateFailedGlobal(ctx, t.Id, "任务超时自动失败")
|
||
}
|
||
g.Log().Infof(ctx, "[cleaner] timeout cleaned, count=%d", len(list))
|
||
}
|
||
}
|
||
|
||
// 3) 失败(state=3)的任务按模型配置 retry_times 重新入队（放到队尾）
|
||
retryable, err := dao.Task.ListFailedRetryableGlobal(ctx, 200)
|
||
if err != nil {
|
||
g.Log().Errorf(ctx, "[cleaner] list failed retryable error: %v", err)
|
||
} else {
|
||
for _, t := range retryable {
|
||
// retry_queue_max_seconds 控制失败重试的排队策略：
|
||
// - =0：失败重试插队到队首
|
||
// - >0：当任务从创建到现在的排队时长 >= maxSeconds，则插队到队首；否则仍放到队尾
|
||
now := time.Now()
|
||
enqueueAt := now
|
||
maxSeconds := t.RetryQueueMaxSeconds
|
||
if maxSeconds == 0 {
|
||
enqueueAt = now.Add(-100 * 365 * 24 * time.Hour)
|
||
} else if maxSeconds > 0 && t.CreatedAt != nil {
|
||
if now.Sub(t.CreatedAt.Time) >= time.Duration(maxSeconds)*time.Second {
|
||
enqueueAt = now.Add(-100 * 365 * 24 * time.Hour)
|
||
}
|
||
}
|
||
_ = dao.Task.RequeueForRetryGlobal(ctx, t.Id, enqueueAt)
|
||
}
|
||
g.Log().Infof(ctx, "[cleaner] failed retryable cleaned, count=%d", len(retryable))
|
||
}
|
||
|
||
// 4) 超过重试次数仍失败(state=3)的任务：硬删除
|
||
exhausted, err := dao.Task.ListFailedExhaustedGlobal(ctx, 200)
|
||
if err != nil {
|
||
g.Log().Errorf(ctx, "[cleaner] list failed exhausted error: %v", err)
|
||
} else {
|
||
for _, t := range exhausted {
|
||
deleteTmpResult(t.TmpFile)
|
||
_ = dao.Task.HardDeleteByIDGlobal(ctx, t.Id)
|
||
}
|
||
g.Log().Infof(ctx, "[cleaner] failed exhausted cleaned, count=%d", len(exhausted))
|
||
}
|
||
}
|