diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dab850d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# 最小化Docker镜像 +FROM busybox:uclibc + +WORKDIR /app + +# 复制时区数据 +COPY timezone/localtime /etc/localtime +COPY timezone/timezone /etc/timezone +COPY timezone/Shanghai /usr/share/zoneinfo/Asia/Shanghai + +# 复制预构建的二进制文件和配置文件 +COPY gateway_binary ./main +COPY config.yml ./ + +# 添加执行权限 +RUN chmod +x /app/main + +# 创建日志目录 +RUN mkdir -p /logs /app/resource/log/run /app/resource/log/server + +EXPOSE 8000 + +# 前台运行（确保容器不会立即退出） +CMD ["./main"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..357ddce --- /dev/null +++ b/README.md @@ -0,0 +1,177 @@ +# model-asynch（模型异步中间件） + +一个独立的异步中间件服务：按模型配置路由调用不同模型服务，统一生成 `task_id`，后台异步执行，结果上传 OSS，并提供查询/批量领取/自动重试/自动清理能力，便于业务方“拿走结果并转移”。 + +> 分支约定：`dev` 为开发分支；`main`（或 master）为线上主分支。 + +--- + +## 1. 核心功能 + +### 1.1 模型配置（asynch_models） +- 增删改查模型服务配置（`model_name` 唯一标识） +- 支持配置： + - 请求地址：`base_url + route` + - 请求方式：`http_method`（GET/POST） + - 请求头：`head_msg`（以请求头注入，支持多个 header） + - 超时：`timeout_seconds` + - 并发：`max_concurrency`（按租户+模型的 Redis 分布式信号量限流） + - 重试：`retry_times`（失败后最多再重试 N 次） + - 保留：`auto_clean_seconds`（任务被业务领取到 `state=4` 后的保留秒数，到期清理） + +### 1.2 异步任务（asynch_task） +- 创建任务：生成 `task_id`，入库排队 +- 后台 Worker： + - PostgreSQL `FOR UPDATE SKIP LOCKED` 抢占任务，支持多实例不重复消费 + - 调用模型服务（GET/POST） + - 结果上传 OSS（调用你们的 OSS 文件服务 `oss/file/uploadFile`，透传 `Authorization/X-User-Info`） +- 批量领取结果：批量查询 `task_id` 列表，返回 `task_id/state/oss_file`，并把成功的任务从 `state=2` 更新为 `state=4` +- 自动重试：失败 `state=3` 会由清理器按 `retry_times` 重新入队到队尾 +- 自动清理： + - `state=4` 且 `expire_at` 到期 → 硬删除任务 + - 失败重试耗尽仍失败 → 硬删除任务 + - `state=0/1` 超时 → 标记失败（防止卡死） + +### 1.3 统计（asynch_model_stat） +- 按天统计：`day + tenant_id + creator + model_name -> request_count` +- 统计口径：仅在 Worker 真正调用模型服务时计数（OSS 重试不计数） +- 用途：给其他服务提供全局限流/监控依据 + +--- + +## 2. 使用流程（业务方如何接入） + +### 第一步：创建模型配置 +业务方（或运维）先在中间件里创建/更新模型配置（`model_name` 为唯一键），例如： +- `POST /model/createModel`（或 `/model/updateModel`） + +请求示例（JSON）： +```json +{ + "modelName": "model-service", + "modelsType": "1,2,3", + "baseUrl": "http://127.0.0.1:8000", + "route": "/api/v1/chat", + "httpMethod": "POST", + "headMsg": "API_KEY:model-key,API_STATE:true,API_NUM:123", + "enabled": 1, + "maxConcurrency": 5, + "queueLimit": 20, + "timeoutSeconds": 1800, + "expectedSeconds": 600, + "retryTimes": 3, + "retryQueueMaxSeconds": 600, + "autoCleanSeconds": 3600, + "remark": "Model-Service 模型服务" +} +``` + +参数说明： +- `modelName`：模型名称（唯一标识/路由键） +- `modelsType`：模型类型ID列表（逗号分隔），示例：`1,2,3`（关联 `asynch_models_type.type_id`） +- `form`：动态表单配置（JSON数组），用于前端按模型渲染参数表单（字段示例：field/label/type/required） +- `baseUrl`：模型服务地址（Base URL） +- `route`：模型服务路由（拼接到 baseUrl 后） +- `httpMethod`：请求方式（GET/POST） +- `headMsg`：请求头绑定（支持多个 header，逗号分隔，格式 `Key:Value`；布尔/数字也会以字符串形式注入 header） +- `enabled`：是否启用（0禁用/1启用） +- `maxConcurrency`：单模型最大并发（按租户+模型维度限流） +- `queueLimit`：排队上限（严格控制）。创建任务时通过 Redis Lua 原子闸门校验并占位，保证分布式并发创建不会超限；任务进入成功/失败态后释放占位，失败重试重新入队时会再次占位。 +- `timeoutSeconds`：调用模型服务超时（秒） +- `expectedSeconds`：模型预计执行时间（秒，用于超时判定/排队策略等） +- `retryTimes`：失败后最多再重试 N 次（不含首次） +- `retryQueueMaxSeconds`：失败重试最大排队时间（秒）；0 表示重试插队到队首；>0 表示排队超过该时间后插队，否则仍到队尾 +- `autoCleanSeconds`：任务被领取到 `state=4` 后的保留时间（秒），到期清理 +- `remark`：备注说明 + +### 第二步：创建任务拿到 task_id +业务方发起推理请求时调用： +- `POST /task/createTask`（传 `modelName + requestPayload + bizName + callbackUrl(可选) + modelKey(可选)`） +- 中间件返回 `task_id` +- 业务方将 `task_id` 落到自己的业务表，并把业务状态置为「生成中」 + +> `modelKey` 用于“动态覆盖/补充”模型配置中的 `head_msg`（例如每次请求携带不同的 `X-API-Key:xxx`）。 +> +> `callbackUrl` 用于任务成功后的回调通知：当任务 `state=2` 成功时，中间件会发起一次 GET 请求： +> - 实际回调地址：`callbackUrl/{bizName}` +> - query 参数：`task_id/state/oss_file/file_type` + +### 第三步：同步任务进度（推荐批量） +业务方通过轮询/定时任务同步进度： +- 推荐：`POST /task/getTaskBatch`（批量传 `taskIds`，返回每个任务的 `state + oss_file`） +- 或单条：`GET /task/getTaskResult?taskId=...` + +业务侧拿到 `oss_file` 后自行做资源处理（直接保存或转存），并把业务状态更新为「成功/失败」。 + +> 说明：批量接口对 `state=2(成功)` 的任务会自动标记为 `state=4(已下载)` 并写入 `expire_at`，用于后续清理。 + +### 后台执行（由上层定时任务控制） +本项目不再在服务进程内常驻轮询 worker/cleaner，而是提供两个接口供上层定时任务触发： +- `POST /task/runWork`：执行一次 Worker（抢占并处理一批排队任务） +- `POST /task/cleanWork`：执行一次 Cleaner（清理过期任务、失败重试、超时任务失败等） + +### 动态并发/队列调参（接口请求控制） +为支持根据最近一段时间的耗时与吞吐对 `max_concurrency/queue_limit` 做动态调整，本项目提供接口供上层定时任务触发（建议每小时一次）： +- `POST /model/autoTune` + +请求参数（JSON，可选）： +```json +{ + "windowSeconds": 3600 +} +``` +> `windowSeconds` 不传/<=0 默认 3600（1小时）。 + +动态调参口径（默认近 1 小时窗口，按 `model_name` 维度）： +- 执行耗时：`finished_at - started_at`（取 P90） +- 吞吐：近 1 小时完成数 / 3600 + +调参结果不会覆盖 `asynch_models` 中配置的最大上限（cap），而是写入 Redis 运行时参数（带 TTL，默认 2 小时）： +- `asynch:runtime:max_concurrency:{model_name}` +- `asynch:runtime:queue_limit:{model_name}` + +生效位置： +- CreateTask 入队时，严格 queue_limit 闸门会优先使用运行时 `queue_limit`（若无运行时值则回退 cap）。 +- Worker 获取并发令牌时，优先使用运行时 `max_concurrency`（若无运行时值则回退 cap）。 + +--- + +## 3. 状态机说明（asynch_task.state） + +| state | 含义 | 产生方 | +|---:|---|---| +| 0 | 排队中 | 创建任务/重试入队 | +| 1 | 执行中 | Worker 抢占后 | +| 2 | 成功（已上传 OSS） | Worker | +| 3 | 失败 | Worker / 超时处理 | +| 4 | 已下载（已领取） | 批量领取接口（2→4） | + +字段补充： +- `retry_count`：已重试次数（不含首次） +- `enqueue_at`：入队时间（用于排队顺序，重试会更新为 NOW() 放到队尾） +- `expire_at`：仅对 `state=4` 生效，表示保留到期时间 + +--- + +## 4. 配置说明（config.yml） + +关键配置： +- `database.default`: PostgreSQL 连接 +- `redis.default`: Redis 连接（并发令牌、可扩展用途） + +--- + +## 5. 数据库初始化 + +项目根目录提供 `update.sql`：首次部署执行建表 SQL。 + +--- + +## 6. 开发与发布建议（Git） + +- `dev`：日常开发与联调 +- `main`：线上稳定分支 +- 推荐流程： + 1) 从 `main` 拉出 `dev` + 2) 功能完成后提 MR/PR 合并回 `main` + 3) 打 tag / 发布镜像 diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..5648a2e --- /dev/null +++ b/config.yml @@ -0,0 +1,39 @@ +server: + address: ":8001" + name: "model-asynch" + workerId: 1 # 雪花算法worker ID（用于 common/db/gfdb） + +# PostgreSQL（GoFrame driver pgsql） +database: + default: + - type: "pgsql" + host: "116.204.74.41" + port: "15432" + user: "postgres" + pass: "Bjang09@686^*^" + name: "model-asynch" + prefix: "" # (可选)表名前缀 + role: "master" # (可选)数据库主从角色(master/slave)，默认为master。如果不使用应用主从机制请不配置或留空即可。 + debug: true # (可选)开启调试模式 + dryRun: false # (可选)ORM空跑(只读不写) + charset: "utf8" # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。 + timezone: "Asia/Shanghai" # (可选)时区配置，例如:Local + maxIdle: 5 # (可选)连接池最大闲置的连接数(默认10) + maxOpen: 20 # (可选)连接池最大打开的连接数(默认无限制) + maxLifetime: "30s" # (可选)连接对象可重复使用的时间长度(默认30秒) + maxIdleConnTime: "30s" # (可选，v2.10新增)连接池中空闲连接的最大生存时间(默认30秒)。可以通过配置文件或SetConnMaxIdleTime方法设置，避免长时间空闲连接占用资源。 + createdAt: "created_at" # (可选)自动创建时间字段名称 + updatedAt: "updated_at" # (可选)自动更新时间字段名称 + deletedAt: "deleted_at" # (可选)软删除时间字段名称 + timeMaintainDisabled: false # (可选)是否完全关闭时间更新特性，为true时CreatedAt/UpdatedAt/DeletedAt都将失效 + +redis: + default: + address: 192.168.3.30:6379 + db: 0 + +consul: + address: 192.168.3.30:8500 + +jaeger: + addr: 192.168.3.30:4318 diff --git a/consts/public/table_name.go b/consts/public/table_name.go new file mode 100644 index 0000000..433fcbf --- /dev/null +++ b/consts/public/table_name.go @@ -0,0 +1,9 @@ +package public + +const ( + TableNameModel = "asynch_models" // 异步模型表 + TableNameModelType = "asynch_models_type" // 模型类型表 + TableNameTask = "asynch_task" // 异步任务表 + TableNameOpLog = "asynch_op_log" // 异步操作日志表 + TableNameStat = "asynch_model_stat" // 按天统计表（请求次数） +) diff --git a/controller/base.go b/controller/base.go new file mode 100644 index 0000000..08f113d --- /dev/null +++ b/controller/base.go @@ -0,0 +1,24 @@ +package controller + +import ( + "context" + + "gitea.com/red-future/common/beans" +) + +// ensureUser 用于本地/无网关环境下的兜底用户信息，避免 gfdb Hook 因缺少用户上下文而报错。 +// 生产环境建议由网关透传 X-User-Info 或鉴权中间件注入 ctx.Value("user")。 +func ensureUser(ctx context.Context) context.Context { + if ctx == nil { + ctx = context.Background() + } + if ctx.Value("user") != nil { + return ctx + } + u := &beans.User{ + UserName: "admin", + TenantId: 1, + } + return context.WithValue(ctx, "user", u) +} + diff --git a/controller/model_controller.go b/controller/model_controller.go new file mode 100644 index 0000000..331f7fb --- /dev/null +++ b/controller/model_controller.go @@ -0,0 +1,85 @@ +package controller + +import ( + "context" + + "model-asynch/model/dto" + "model-asynch/service" + + "gitea.com/red-future/common/beans" +) + +type model struct{} + +// Model 模型配置控制器 +var Model = new(model) + +// CreateModel 添加配置 +func (c *model) CreateModel(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) { + ctx = ensureUser(ctx) + return service.Model.Create(ctx, req) +} + +// UpdateModel 更改配置 +func (c *model) UpdateModel(ctx context.Context, req *dto.UpdateModelReq) (res *beans.ResponseEmpty, err error) { + ctx = ensureUser(ctx) + err = service.Model.Update(ctx, req) + return +} + +// DeleteModel 删除配置 +func (c *model) DeleteModel(ctx context.Context, req *dto.DeleteModelReq) (res *beans.ResponseEmpty, err error) { + ctx = ensureUser(ctx) + err = service.Model.Delete(ctx, req.ID) + return +} + +// GetModel 获取配置详情（按 modelName） +func (c *model) GetModel(ctx context.Context, req *dto.GetModelReq) (res *dto.GetModelRes, err error) { + ctx = ensureUser(ctx) + m, err := service.Model.Get(ctx, req.ID) + if err != nil { + return nil, err + } + return &dto.GetModelRes{Model: m}, nil +} + +// ListModel 配置列表 +func (c *model) ListModel(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) { + ctx = ensureUser(ctx) + pageNum, pageSize := 1, 10 //默认分页参数 + if req != nil && req.Page != nil { + if req.Page.PageNum > 0 { + pageNum = int(req.Page.PageNum) + } + if req.Page.PageSize > 0 { + pageSize = int(req.Page.PageSize) + } + } + modelName := "" + if req != nil { + modelName = req.ModelName + } + list, total, err := service.Model.List(ctx, pageNum, pageSize, modelName) + if err != nil { + return nil, err + } + return &dto.ListModelRes{ + List: list, + Total: total, + }, nil +} + +// AutoTune 动态调参（由上层定时任务每小时触发一次） +func (c *model) AutoTune(ctx context.Context, req *dto.AutoTuneReq) (res *dto.AutoTuneRes, err error) { + ctx = ensureUser(ctx) + windowSeconds := 3600 + if req != nil && req.WindowSeconds > 0 { + windowSeconds = req.WindowSeconds + } + list, err := service.AutoTune(ctx, windowSeconds) + if err != nil { + return nil, err + } + return &dto.AutoTuneRes{List: list}, nil +} diff --git a/controller/model_type_controller.go b/controller/model_type_controller.go new file mode 100644 index 0000000..450ef08 --- /dev/null +++ b/controller/model_type_controller.go @@ -0,0 +1,68 @@ +package controller + +import ( + "context" + + "model-asynch/model/dto" + "model-asynch/service" + + "gitea.com/red-future/common/beans" +) + +type modelType struct{} + +// ModelType 模型类型控制器 +var ModelType = new(modelType) + +func (c *modelType) CreateModelType(ctx context.Context, req *dto.CreateModelTypeReq) (res *dto.CreateModelTypeRes, err error) { + ctx = ensureUser(ctx) + return service.ModelType.Create(ctx, req) +} + +func (c *modelType) UpdateModelType(ctx context.Context, req *dto.UpdateModelTypeReq) (res *beans.ResponseEmpty, err error) { + ctx = ensureUser(ctx) + err = service.ModelType.Update(ctx, req) + return +} + +func (c *modelType) DeleteModelType(ctx context.Context, req *dto.DeleteModelTypeReq) (res *beans.ResponseEmpty, err error) { + ctx = ensureUser(ctx) + err = service.ModelType.Delete(ctx, req.ID) + return +} + +func (c *modelType) GetModelType(ctx context.Context, req *dto.GetModelTypeReq) (res *dto.GetModelTypeRes, err error) { + ctx = ensureUser(ctx) + t, err := service.ModelType.Get(ctx, req.ID) + if err != nil { + return nil, err + } + return &dto.GetModelTypeRes{Type: t}, nil +} + +func (c *modelType) ListModelType(ctx context.Context, req *dto.ListModelTypeReq) (res *dto.ListModelTypeRes, err error) { + ctx = ensureUser(ctx) + pageNum, pageSize := 1, 10 + if req != nil && req.Page != nil { + if req.Page.PageNum > 0 { + pageNum = int(req.Page.PageNum) + } + if req.Page.PageSize > 0 { + pageSize = int(req.Page.PageSize) + } + } + typeName := "" + if req != nil { + typeName = req.TypeName + } + list, total, err := service.ModelType.List(ctx, pageNum, pageSize, typeName) + if err != nil { + return nil, err + } + return &dto.ListModelTypeRes{List: list, Total: total}, nil +} + +func (c *modelType) ListModelTypeWithModels(ctx context.Context, req *dto.ListModelTypeWithModelsReq) (res []dto.ModelTypeWithModelsItem, err error) { + ctx = ensureUser(ctx) + return service.ModelType.ListWithModels(ctx, req) +} diff --git a/controller/stat_controller.go b/controller/stat_controller.go new file mode 100644 index 0000000..e0938d9 --- /dev/null +++ b/controller/stat_controller.go @@ -0,0 +1,20 @@ +package controller + +import ( + "context" + + "model-asynch/model/dto" + "model-asynch/service" +) + +type stat struct{} + +// Stat 统计控制器 +var Stat = new(stat) + +// ListModelStat 统计列表 +func (c *stat) ListModelStat(ctx context.Context, req *dto.ListModelStatReq) (res *dto.ListModelStatRes, err error) { + ctx = ensureUser(ctx) + return service.Stat.List(ctx, req) +} + diff --git a/controller/task_controller.go b/controller/task_controller.go new file mode 100644 index 0000000..e5fd8a4 --- /dev/null +++ b/controller/task_controller.go @@ -0,0 +1,63 @@ +package controller + +import ( + "context" + + "model-asynch/model/dto" + "model-asynch/service" +) + +type task struct{} + +// Task 任务控制器 +var Task = new(task) + +// CreateTask 根据 modelName 创建异步任务，返回 taskId +func (c *task) CreateTask(ctx context.Context, req *dto.CreateTaskReq) (res *dto.CreateTaskRes, err error) { + ctx = ensureUser(ctx) + return service.Task.Create(ctx, req) +} + +// GetTaskResult 获取任务结果（只返回 oss 地址 + state） +func (c *task) GetTaskResult(ctx context.Context, req *dto.GetTaskResultReq) (res *dto.GetTaskResultRes, err error) { + ctx = ensureUser(ctx) + return service.Task.GetResult(ctx, req.TaskID) +} + +// GetTaskBatch 批量查询任务（成功任务标记为已下载） +func (c *task) GetTaskBatch(ctx context.Context, req *dto.GetTaskBatchReq) (res *dto.GetTaskBatchRes, err error) { + ctx = ensureUser(ctx) + return service.Task.GetBatch(ctx, req) +} + +// ListTask 任务列表分页查询 +func (c *task) ListTask(ctx context.Context, req *dto.ListTaskReq) (res *dto.ListTaskRes, err error) { + ctx = ensureUser(ctx) + return service.Task.List(ctx, req) +} + +// RunWork 手动触发一次 worker（由上层定时任务调用） +func (c *task) RunWork(ctx context.Context, req *dto.RunWorkReq) (res *dto.RunWorkRes, err error) { + ctx = ensureUser(ctx) + batchSize, goroutines := 10, 1 + if req != nil { + if req.BatchSize > 0 { + batchSize = req.BatchSize + } + if req.Goroutines > 0 { + goroutines = req.Goroutines + } + } + n, err := service.AsyncWorker.RunOnce(ctx, batchSize, goroutines) + if err != nil { + return nil, err + } + return &dto.RunWorkRes{Claimed: n}, nil +} + +// CleanWork 手动触发一次 cleaner（由上层定时任务调用） +func (c *task) CleanWork(ctx context.Context, req *dto.CleanWorkReq) (res *dto.CleanWorkRes, err error) { + ctx = ensureUser(ctx) + service.Cleaner.RunOnce(ctx) + return &dto.CleanWorkRes{Ok: true}, nil +} diff --git a/dao/model_dao.go b/dao/model_dao.go new file mode 100644 index 0000000..c293049 --- /dev/null +++ b/dao/model_dao.go @@ -0,0 +1,104 @@ +package dao + +import ( + "context" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/util/gconv" +) + +var Model = &modelDao{} + +type modelDao struct{} + +func (d *modelDao) Insert(ctx context.Context, m *entity.AsynchModel) (id int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModel).Data(m).Insert() + if err != nil { + return 0, err + } + return r.LastInsertId() +} + +func (d *modelDao) UpdateByID(ctx context.Context, id int64, data map[string]any) (rows int64, err error) { + // 触发 gfdb 的 updateHook 自动填充 updater，需要显式带 updater 字段 + data[entity.AsynchModelCol.Updater] = "" + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModel). + Where(entity.AsynchModelCol.Id, id). + Data(data). + Update() + if err != nil { + return 0, err + } + return r.RowsAffected() +} + +func (d *modelDao) DeleteByID(ctx context.Context, id int64) (rows int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModel). + Where(entity.AsynchModelCol.Id, id). + Delete() + if err != nil { + return 0, err + } + return r.RowsAffected() +} + +func (d *modelDao) GetByModelName(ctx context.Context, modelName string) (m *entity.AsynchModel, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModel). + Where(entity.AsynchModelCol.ModelName, modelName). + One() + if err != nil { + return nil, err + } + if r.IsEmpty() { + return nil, nil + } + err = r.Struct(&m) + return +} + +func (d *modelDao) GetByID(ctx context.Context, id int64) (m *entity.AsynchModel, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModel). + Where(entity.AsynchModelCol.Id, id). + One() + if err != nil { + return nil, err + } + if r.IsEmpty() { + return nil, nil + } + err = r.Struct(&m) + return +} + +func (d *modelDao) List(ctx context.Context, pageNum, pageSize int, modelNameLike string) (list []*entity.AsynchModel, total int64, err error) { + model := gfdb.DB(ctx).Model(ctx, public.TableNameModel).Where("deleted_at IS NULL").OrderDesc(entity.AsynchModelCol.CreatedAt) + if modelNameLike != "" { + model = model.WhereLike(entity.AsynchModelCol.ModelName, "%"+modelNameLike+"%") + } + if pageNum > 0 && pageSize > 0 { + model = model.Page(pageNum, pageSize) + } + r, totalInt, err := model.AllAndCount(false) + if err != nil { + return nil, 0, err + } + total = gconv.Int64(totalInt) + err = r.Structs(&list) + return +} + +// ListAll 用于分组展示：查询全部模型（不按类型过滤，类型拆分在 service 层处理） +func (d *modelDao) ListAll(ctx context.Context) (list []*entity.AsynchModel, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModel). + Where("deleted_at IS NULL"). + OrderDesc(entity.AsynchModelCol.CreatedAt). + All() + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} diff --git a/dao/model_dao_bg.go b/dao/model_dao_bg.go new file mode 100644 index 0000000..34715eb --- /dev/null +++ b/dao/model_dao_bg.go @@ -0,0 +1,32 @@ +package dao + +import ( + "context" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" +) + +// GetByModelNameForTenant 后台任务使用：按 tenant_id + model_name 查询，不依赖 gfdb Hook/Trace/用户上下文 +func (d *modelDao) GetByModelNameForTenant(ctx context.Context, tenantId uint64, modelName string) (m *entity.AsynchModel, err error) { + r, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT * FROM "+public.TableNameModel+" WHERE tenant_id=? AND model_name=? AND deleted_at IS NULL LIMIT 1", + tenantId, modelName, + ) + if err != nil { + return nil, err + } + if r.IsEmpty() { + return nil, nil + } + var list []*entity.AsynchModel + if err := r.Structs(&list); err != nil { + return nil, err + } + if len(list) == 0 { + return nil, nil + } + return list[0], nil +} diff --git a/dao/model_type_dao.go b/dao/model_type_dao.go new file mode 100644 index 0000000..a43c5a0 --- /dev/null +++ b/dao/model_type_dao.go @@ -0,0 +1,74 @@ +package dao + +import ( + "context" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/util/gconv" +) + +type modelTypeDao struct{} + +var ModelType = &modelTypeDao{} + +func (d *modelTypeDao) Insert(ctx context.Context, t *entity.AsynchModelType) (id int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModelType).Data(t).Insert() + if err != nil { + return 0, err + } + return r.LastInsertId() +} + +func (d *modelTypeDao) UpdateByID(ctx context.Context, id int64, data gdb.Map) (rows int64, err error) { + // 触发 gfdb 的 updateHook 自动填充 updater，需要显式带 updater 字段 + data[entity.AsynchModelTypeCol.Updater] = "" + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModelType).Where(entity.AsynchModelTypeCol.Id, id).Data(data).Update() + if err != nil { + return 0, err + } + n, _ := r.RowsAffected() + return n, nil +} + +func (d *modelTypeDao) DeleteByID(ctx context.Context, id int64) (rows int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModelType).Where(entity.AsynchModelTypeCol.Id, id).Delete() + if err != nil { + return 0, err + } + n, _ := r.RowsAffected() + return n, nil +} + +func (d *modelTypeDao) GetByID(ctx context.Context, id int64) (*entity.AsynchModelType, error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameModelType).Where(entity.AsynchModelTypeCol.Id, id).One() + if err != nil { + return nil, err + } + if r.IsEmpty() { + return nil, nil + } + var t *entity.AsynchModelType + _ = r.Struct(&t) + return t, nil +} + +func (d *modelTypeDao) List(ctx context.Context, pageNum, pageSize int, typeNameLike string) (list []*entity.AsynchModelType, total int64, err error) { + m := gfdb.DB(ctx).Model(ctx, public.TableNameModelType).Where("deleted_at IS NULL").OrderAsc(entity.AsynchModelTypeCol.TypeID) + if typeNameLike != "" { + m = m.WhereLike(entity.AsynchModelTypeCol.TypeName, "%"+typeNameLike+"%") + } + if pageNum > 0 && pageSize > 0 { + m = m.Page(pageNum, pageSize) + } + r, totalInt, err := m.AllAndCount(false) + if err != nil { + return nil, 0, err + } + total = gconv.Int64(totalInt) + err = r.Structs(&list) + return +} diff --git a/dao/op_log_dao.go b/dao/op_log_dao.go new file mode 100644 index 0000000..3293cd7 --- /dev/null +++ b/dao/op_log_dao.go @@ -0,0 +1,22 @@ +package dao + +import ( + "context" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" +) + +type opLogDao struct{} + +var OpLog = &opLogDao{} + +func (d *opLogDao) Insert(ctx context.Context, log *entity.AsynchOpLog) (id int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameOpLog).Data(log).Insert() + if err != nil { + return 0, err + } + return r.LastInsertId() +} diff --git a/dao/stat_dao.go b/dao/stat_dao.go new file mode 100644 index 0000000..edce22f --- /dev/null +++ b/dao/stat_dao.go @@ -0,0 +1,61 @@ +package dao + +import ( + "context" + "fmt" + "time" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/os/gtime" +) + +type statDao struct{} + +var Stat = &statDao{} + +// IncRequestCount 原子累加（支持分布式/多协程）：按天+租户+创建人+模型 +1 +func (d *statDao) IncRequestCount(ctx context.Context, day time.Time, tenantId int64, creator, modelName string) error { + sql := fmt.Sprintf(` +INSERT INTO %s(day, tenant_id, creator, model_name, request_count, created_at, updated_at) +VALUES(?, ?, ?, ?, 1, NOW(), NOW()) +ON CONFLICT (day, tenant_id, creator, model_name) +DO UPDATE SET request_count = %s.request_count + 1, updated_at = NOW()`, + public.TableNameStat, public.TableNameStat, + ) + _, err := gfdb.DB(ctx).Exec(ctx, sql, gtime.New(day).Format("Y-m-d"), tenantId, creator, modelName) + return err +} + +func (d *statDao) List(ctx context.Context, pageNum, pageSize int, startDay, endDay string, tenantId *int64, creator, modelName string) (list []*entity.AsynchModelStat, total int64, err error) { + m := gfdb.DB(ctx).Model(ctx, public.TableNameStat).Where("1=1") + if startDay != "" { + m = m.Where("day >= ?", startDay) + } + if endDay != "" { + m = m.Where("day <= ?", endDay) + } + if tenantId != nil { + m = m.Where("tenant_id = ?", *tenantId) + } + if creator != "" { + m = m.WhereLike("creator", "%"+creator+"%") + } + if modelName != "" { + m = m.WhereLike("model_name", "%"+modelName+"%") + } + m = m.OrderDesc("day").OrderDesc("request_count") + if pageNum > 0 && pageSize > 0 { + m = m.Page(pageNum, pageSize) + } + r, totalInt, err := m.AllAndCount(false) + if err != nil { + return nil, 0, err + } + total = int64(totalInt) + err = r.Structs(&list) + return +} + diff --git a/dao/task_dao.go b/dao/task_dao.go new file mode 100644 index 0000000..5f4a7d0 --- /dev/null +++ b/dao/task_dao.go @@ -0,0 +1,250 @@ +package dao + +import ( + "context" + "fmt" + "time" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/util/gconv" +) + +var Task = &taskDao{} + +type taskDao struct{} + +func (d *taskDao) Insert(ctx context.Context, t *entity.AsynchTask) (id int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask).Data(t).Insert() + if err != nil { + return 0, err + } + return r.LastInsertId() +} + +func (d *taskDao) GetByTaskID(ctx context.Context, taskID string) (t *entity.AsynchTask, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.TaskID, taskID). + One() + if err != nil { + return nil, err + } + if r.IsEmpty() { + return nil, nil + } + err = r.Struct(&t) + return +} + +// ListByTaskIDs 批量查询任务（会受 gfdb 的租户 Hook 影响，只返回当前租户数据） +func (d *taskDao) ListByTaskIDs(ctx context.Context, taskIDs []string) (list []*entity.AsynchTask, err error) { + if len(taskIDs) == 0 { + return nil, nil + } + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + WhereIn(entity.AsynchTaskCol.TaskID, taskIDs). + All() + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} + +// MarkDownloadedByID 将成功任务标记为已下载(state=4)，并写入过期时间 +func (d *taskDao) MarkDownloadedByID(ctx context.Context, id int64, expireAt *gtime.Time) error { + data := gdb.Map{ + entity.AsynchTaskCol.State: 4, + entity.AsynchTaskCol.ExpireAt: expireAt, + entity.AsynchTaskCol.Updater: "", + } + _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.Id, id). + Where(entity.AsynchTaskCol.State, 2). + Data(data). + Update() + return err +} + +func (d *taskDao) UpdateRunning(ctx context.Context, id int64) error { + now := gtime.Now() + data := gdb.Map{ + entity.AsynchTaskCol.State: 1, + entity.AsynchTaskCol.StartedAt: now, + entity.AsynchTaskCol.Updater: "", + } + _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.Id, id). + Data(data). + Update() + return err +} + +func (d *taskDao) UpdateSuccess(ctx context.Context, id int64, ossFile, fileType string, fileSize int64, expireAt *gtime.Time) error { + now := gtime.Now() + data := gdb.Map{ + entity.AsynchTaskCol.State: 2, + entity.AsynchTaskCol.OssFile: ossFile, + entity.AsynchTaskCol.FileType: fileType, + entity.AsynchTaskCol.FileSize: fileSize, + entity.AsynchTaskCol.ErrorMsg: "", + entity.AsynchTaskCol.FinishedAt: now, + entity.AsynchTaskCol.ExpireAt: expireAt, + entity.AsynchTaskCol.Updater: "", + } + _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.Id, id). + Data(data). + Update() + return err +} + +func (d *taskDao) UpdateFailed(ctx context.Context, id int64, errorMsg string) error { + now := gtime.Now() + data := gdb.Map{ + entity.AsynchTaskCol.State: 3, + entity.AsynchTaskCol.ErrorMsg: errorMsg, + entity.AsynchTaskCol.FinishedAt: now, + entity.AsynchTaskCol.Updater: "", + } + _, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.Id, id). + Data(data). + Update() + return err +} + +func (d *taskDao) SoftDeleteByTaskID(ctx context.Context, taskID string) (rows int64, err error) { + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.TaskID, taskID). + Delete() + if err != nil { + return 0, err + } + return r.RowsAffected() +} + +// CountActiveByModel 统计某模型排队中/执行中的任务数，用于 queue_limit 限制（近似值） +func (d *taskDao) CountActiveByModel(ctx context.Context, modelName string) (int64, error) { + n, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.ModelName, modelName). + WhereIn(entity.AsynchTaskCol.State, []int{0, 1}). + Count() + return int64(n), err +} + +// List 任务分页查询（受 gfdb 租户 Hook 影响） +func (d *taskDao) List(ctx context.Context, pageNum, pageSize int, modelNameLike, taskIDLike string, state *int) (list []*entity.AsynchTask, total int64, err error) { + m := gfdb.DB(ctx).Model(ctx, public.TableNameTask).Where("deleted_at IS NULL") + if modelNameLike != "" { + m = m.WhereLike(entity.AsynchTaskCol.ModelName, "%"+modelNameLike+"%") + } + if taskIDLike != "" { + m = m.WhereLike(entity.AsynchTaskCol.TaskID, "%"+taskIDLike+"%") + } + if state != nil { + m = m.Where(entity.AsynchTaskCol.State, *state) + } + m = m.OrderDesc(entity.AsynchTaskCol.CreatedAt) + if pageNum > 0 && pageSize > 0 { + m = m.Page(pageNum, pageSize) + } + r, totalInt, err := m.AllAndCount(false) + if err != nil { + return nil, 0, err + } + total = gconv.Int64(totalInt) + err = r.Structs(&list) + return +} + +// ClaimPending 抢占 pending 任务（state=0），并在同一事务中更新为 running（state=1） +// 使用 PostgreSQL: FOR UPDATE SKIP LOCKED 避免多 worker 重复消费 +func (d *taskDao) ClaimPending(ctx context.Context, batchSize int) (tasks []*entity.AsynchTask, err error) { + if batchSize <= 0 { + batchSize = 1 + } + err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + sql := fmt.Sprintf( + `SELECT id, tenant_id, model_name, task_id, input_ref, request_payload + FROM %s + WHERE deleted_at IS NULL AND state = 0 + ORDER BY created_at ASC + LIMIT %d + FOR UPDATE SKIP LOCKED`, + public.TableNameTask, + batchSize, + ) + r, err := tx.GetAll(sql) + if err != nil { + return err + } + if r.IsEmpty() { + tasks = nil + return nil + } + if err := r.Structs(&tasks); err != nil { + return err + } + // 更新为 running + now := time.Now() + for _, t := range tasks { + // tx.Model 不走 gfdb Hook，这里手动更新必要字段 + _, err = tx.Exec( + fmt.Sprintf(`UPDATE %s SET state=1, started_at=?, updated_at=? WHERE id=?`, public.TableNameTask), + now, now, t.Id, + ) + if err != nil { + return err + } + } + return nil + }) + return +} + +// ListExpiredSuccess 获取已成功且过期的任务 +func (d *taskDao) ListExpiredSuccess(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { + if limit <= 0 { + limit = 100 + } + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + Where(entity.AsynchTaskCol.State, 2). + Where(entity.AsynchTaskCol.ExpireAt+" IS NOT NULL"). + Where(entity.AsynchTaskCol.ExpireAt+" < ?", gtime.Now()). + Limit(limit). + All() + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} + +// ListTimeoutTasks 获取超时的排队/执行中任务 +func (d *taskDao) ListTimeoutTasks(ctx context.Context, timeout time.Duration, limit int) (list []*entity.AsynchTask, err error) { + if limit <= 0 { + limit = 100 + } + deadline := gtime.New(time.Now().Add(-timeout)) + r, err := gfdb.DB(ctx).Model(ctx, public.TableNameTask). + WhereIn(entity.AsynchTaskCol.State, []int{0, 1}). + Where(entity.AsynchTaskCol.UpdatedAt+" < ?", deadline). + Limit(limit). + All() + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} + +// DebugPing 用于启动时检测数据库连通性（可选） +func (d *taskDao) DebugPing(ctx context.Context) error { + _, err := gfdb.DB(ctx).GetAll(ctx, "SELECT 1") + return err +} diff --git a/dao/task_dao_bg.go b/dao/task_dao_bg.go new file mode 100644 index 0000000..5c49cba --- /dev/null +++ b/dao/task_dao_bg.go @@ -0,0 +1,248 @@ +package dao + +import ( + "context" + "fmt" + "time" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/os/gtime" +) + +// ClaimPendingGlobal 后台任务使用：全局抢占 pending 任务（不加 tenant 过滤） +func (d *taskDao) ClaimPendingGlobal(ctx context.Context, batchSize int) (tasks []*entity.AsynchTask, err error) { + if batchSize <= 0 { + batchSize = 1 + } + err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + sql := fmt.Sprintf( + `SELECT id, tenant_id, creator, model_name, task_id, model_key, input_ref, request_payload, phase, tmp_file + FROM %s + WHERE deleted_at IS NULL AND state = 0 + ORDER BY enqueue_at ASC + LIMIT %d + FOR UPDATE SKIP LOCKED`, + public.TableNameTask, + batchSize, + ) + r, err := tx.GetAll(sql) + if err != nil { + return err + } + if r.IsEmpty() { + tasks = nil + return nil + } + if err := r.Structs(&tasks); err != nil { + return err + } + now := time.Now() + for _, t := range tasks { + _, err = tx.Exec( + fmt.Sprintf(`UPDATE %s SET state=1, started_at=?, updated_at=? WHERE id=?`, public.TableNameTask), + now, now, t.Id, + ) + if err != nil { + return err + } + } + return nil + }) + return +} + +func (d *taskDao) UpdateSuccessGlobal(ctx context.Context, id int64, ossFile, fileType string, fileSize int64, expireAt *gtime.Time) error { + now := gtime.Now() + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s +SET state=2, + oss_file=?, + file_type=?, + file_size=?, + error_msg='', + finished_at=?, + duration_seconds=EXTRACT(EPOCH FROM (? - created_at))::BIGINT, + expire_at=NULL, + phase=0, + tmp_file='', + updated_at=? +WHERE id=?`, public.TableNameTask), + ossFile, fileType, fileSize, now, now, now, id, + ) + return err +} + +func (d *taskDao) UpdateFailedGlobal(ctx context.Context, id int64, errorMsg string) error { + now := gtime.Now() + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s +SET state=3, + error_msg=?, + finished_at=?, + duration_seconds=EXTRACT(EPOCH FROM (? - created_at))::BIGINT, + phase=0, + tmp_file='', + updated_at=? +WHERE id=?`, public.TableNameTask), + errorMsg, now, now, now, id, + ) + return err +} + +// UpdateFailedKeepTmpGlobal OSS 上传失败：保留 phase/tmp_file，下一轮仅重试 OSS 上传 +func (d *taskDao) UpdateFailedKeepTmpGlobal(ctx context.Context, id int64, errorMsg string) error { + now := gtime.Now() + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s SET state=3, error_msg=?, finished_at=?, phase=1, updated_at=? WHERE id=?`, public.TableNameTask), + errorMsg, now, now, id, + ) + return err +} + +// UpdateTmpAfterModelGlobal 模型调用成功后，写入临时文件路径并标记 phase=1 +func (d *taskDao) UpdateTmpAfterModelGlobal(ctx context.Context, id int64, tmpFile string) error { + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s SET phase=1, tmp_file=?, updated_at=NOW() WHERE id=?`, public.TableNameTask), + tmpFile, id, + ) + return err +} + +func (d *taskDao) SoftDeleteByTaskIDGlobal(ctx context.Context, taskID string) error { + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s SET deleted_at=NOW(), updated_at=NOW() WHERE task_id=? AND deleted_at IS NULL`, public.TableNameTask), + taskID, + ) + return err +} + +func (d *taskDao) RollbackToPendingGlobal(ctx context.Context, id int64) error { + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s SET state=0, enqueue_at=NOW(), updated_at=NOW() WHERE id=? AND state=1`, public.TableNameTask), + id, + ) + return err +} + +// ListExpiredDownloadedGlobal 获取已下载(state=4)且过期的任务，用于清理 +func (d *taskDao) ListExpiredDownloadedGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { + if limit <= 0 { + limit = 200 + } + r, err := gfdb.DB(ctx).GetAll(ctx, + fmt.Sprintf(`SELECT * FROM %s WHERE deleted_at IS NULL AND state=4 AND expire_at IS NOT NULL AND expire_at < ? LIMIT ?`, public.TableNameTask), + gtime.Now(), limit, + ) + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} + +// ListFailedRetryableGlobal 获取失败(state=3)且仍可重试的任务 +// retry_count 不含首次执行；retry_times 表示失败后最多再重试 N 次 +func (d *taskDao) ListFailedRetryableGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { + if limit <= 0 { + limit = 200 + } + r, err := gfdb.DB(ctx).GetAll(ctx, + fmt.Sprintf(` +SELECT t.*, + m.retry_queue_max_seconds AS retry_queue_max_seconds + FROM %s t + JOIN %s m + ON t.tenant_id = m.tenant_id + AND t.model_name = m.model_name + WHERE t.deleted_at IS NULL + AND t.state = 3 + AND t.retry_count < m.retry_times + ORDER BY t.updated_at ASC + LIMIT ?`, public.TableNameTask, public.TableNameModel), + limit, + ) + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} + +// RequeueForRetryGlobal 将任务重新入队（state=0），并将 retry_count +1 +// enqueueAt 用于控制重试任务在队列中的位置： +// - enqueueAt 越早，越靠前（ClaimPendingGlobal 按 enqueue_at ASC 抢占） +func (d *taskDao) RequeueForRetryGlobal(ctx context.Context, id int64, enqueueAt time.Time) error { + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`UPDATE %s SET state=0, retry_count=retry_count+1, enqueue_at=?, updated_at=NOW() WHERE id=? AND state=3 AND deleted_at IS NULL`, public.TableNameTask), + enqueueAt, id, + ) + return err +} + +// ListFailedExhaustedGlobal 获取失败(state=3)且超过重试次数的任务，用于硬删除 +func (d *taskDao) ListFailedExhaustedGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { + if limit <= 0 { + limit = 200 + } + r, err := gfdb.DB(ctx).GetAll(ctx, + fmt.Sprintf(` +SELECT t.* + FROM %s t + JOIN %s m + ON t.tenant_id = m.tenant_id + AND t.model_name = m.model_name + WHERE t.deleted_at IS NULL + AND t.state = 3 + AND t.retry_count >= m.retry_times + ORDER BY t.updated_at ASC + LIMIT ?`, public.TableNameTask, public.TableNameModel), + limit, + ) + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} + +// HardDeleteByIDGlobal 硬删除任务记录 +func (d *taskDao) HardDeleteByIDGlobal(ctx context.Context, id int64) error { + _, err := gfdb.DB(ctx).Exec(ctx, + fmt.Sprintf(`DELETE FROM %s WHERE id=?`, public.TableNameTask), + id, + ) + return err +} + +// ListTimeoutTasksGlobal 根据模型配置 expected_seconds 判定超时任务： +// - state in (0,1) +// - 模型 expected_seconds > 0 +// - now - created_at >= expected_seconds +func (d *taskDao) ListTimeoutTasksGlobal(ctx context.Context, limit int) (list []*entity.AsynchTask, err error) { + if limit <= 0 { + limit = 200 + } + r, err := gfdb.DB(ctx).GetAll(ctx, + fmt.Sprintf(` +SELECT t.* + FROM %s t + JOIN %s m + ON t.tenant_id = m.tenant_id + AND t.model_name = m.model_name + WHERE t.deleted_at IS NULL + AND t.state IN (0,1) + AND m.expected_seconds > 0 + AND t.created_at < (NOW() - (m.expected_seconds || ' seconds')::interval) + LIMIT ?`, public.TableNameTask, public.TableNameModel), + limit, + ) + if err != nil { + return nil, err + } + err = r.Structs(&list) + return +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..aa1ac10 --- /dev/null +++ b/go.mod @@ -0,0 +1,89 @@ +module model-asynch + +go 1.26.0 + +require ( + gitea.com/red-future/common v0.0.12 + github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0 + github.com/gogf/gf/contrib/nosql/redis/v2 v2.10.0 + github.com/gogf/gf/v2 v2.10.0 + github.com/google/uuid v1.6.0 +) + +// replace gitea.com/red-future/common v0.0.12 => ../common + +require ( + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/bwmarrin/snowflake v0.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clbanning/mxj/v2 v2.7.0 // indirect + github.com/dgraph-io/badger/v4 v4.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ego/gse v1.0.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogf/gf v1.16.9 // indirect + github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect + github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golang/glog v1.2.5 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/grokify/html-strip-tags-go v0.1.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/hashicorp/consul/api v1.26.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/olekukonko/errors v1.1.0 // indirect + github.com/olekukonko/ll v0.0.9 // indirect + github.com/olekukonko/tablewriter v1.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/redis/go-redis/v9 v9.12.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/tiger1103/gfast-token v1.0.10 // indirect + github.com/vcaesar/cedar v0.30.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.4.0 // indirect + go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c1b9bcc --- /dev/null +++ b/go.sum @@ -0,0 +1,473 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +gitea.com/red-future/common v0.0.12 h1:whaCAiH33orl0P+oDpxzC4VoNluHKNYKGZ+FcUWw85Q= +gitea.com/red-future/common v0.0.12/go.mod h1:3a7cwZNvgpKw5FzE8x5MZImd7NBePGXRGFSMjt90158= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= +github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= +github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emirpasic/gods/v2 v2.0.0-alpha h1:dwFlh8pBg1VMOXWGipNMRt8v96dKAIvBehtCt6OtunU= +github.com/emirpasic/gods/v2 v2.0.0-alpha/go.mod h1:W0y4M2dtBB9U5z3YlghmpuUhiaZT2h6yoeE+C1sCp6A= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-ego/gse v1.0.2 h1:+27lYFPhQEhA9igtdOsJPRKYL/k3TwYsxBF5jr6KFv4= +github.com/go-ego/gse v1.0.2/go.mod h1:Fy35G+q7VV7Et1zIKO8o/sW1kkugV3znXap/lF/11zc= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogf/gf v1.16.9 h1:Q803UmmRo59+Ws08sMVFOcd8oNpkSWL9vS33hlo/Cyk= +github.com/gogf/gf v1.16.9/go.mod h1:8Q/kw05nlVRp+4vv7XASBsMe9L1tsVKiGoeP2AHnlkk= +github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0 h1:39+jbTenm7KBj4hO2C8ANAxVHpX/7OuRDs1VcGC9ylA= +github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0/go.mod h1:B0s0fVzn0W220E8UTpSGzrrGKsop5KcB90twBeLCiz0= +github.com/gogf/gf/contrib/nosql/redis/v2 v2.10.0 h1:N/F9CuDdUZLoM1nVRqrDE/33pDZuhVxpNY4wYdeIaBs= +github.com/gogf/gf/contrib/nosql/redis/v2 v2.10.0/go.mod h1:x6uoJGfZOtirIRQls8xUlYzC6f7T/eULPUa9er368X0= +github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 h1:eUqwJ/qNH8lJ6yssiqskazgp1ACQuNU6zXlLOZVuXTQ= +github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5/go.mod h1:sjQyMry9+0POYZCA6lHXBxO77WoNKkruJpRB4xKqk5k= +github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 h1:tHUEZYB5GTqEYYVDYnlGobf1xISARKDE4KHVlgjwTec= +github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5/go.mod h1:cfzTn2HS9RDX8f5pUVkbGxUWcSosouqfNQ1G6cY0V88= +github.com/gogf/gf/v2 v2.10.0 h1:rzDROlyqGMe/eM6dCalSR8dZOuMIdLhmxKSH1DGhbFs= +github.com/gogf/gf/v2 v2.10.0/go.mod h1:Svl1N+E8G/QshU2DUbh/3J/AJauqCgUnxHurXWR4Qx0= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= +github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4= +github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/hashicorp/consul/api v1.26.1 h1:5oSXOO5fboPZeW5SN+TdGFP/BILDgBm19OrPZ/pICIM= +github.com/hashicorp/consul/api v1.26.1/go.mod h1:B4sQTeaSO16NtynqrAdwOlahJ7IUDZM9cj2420xYL8A= +github.com/hashicorp/consul/sdk v0.15.0 h1:2qK9nDrr4tiJKRoxPGhm6B7xJjLVIQqkjiab2M4aKjU= +github.com/hashicorp/consul/sdk v0.15.0/go.mod h1:r/OmRRPbHOe0yxNahLw7G9x5WG17E1BIECMtCjcPSNo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= +github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= +github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.0.9 h1:Y+1YqDfVkqMWuEQMclsF9HUR5+a82+dxJuL1HHSRpxI= +github.com/olekukonko/ll v0.0.9/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/olekukonko/tablewriter v1.1.0 h1:N0LHrshF4T39KvI96fn6GT8HEjXRXYNDrDjKFDB7RIY= +github.com/olekukonko/tablewriter v1.1.0/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tiger1103/gfast-token v1.0.10 h1:fNiBE/Dq5iTHvTGlCx3DmXa2o4hr0NtumFpffZ39k6s= +github.com/tiger1103/gfast-token v1.0.10/go.mod h1:a/21mxmj7zFeNvjhZSC0XpEAFHfb1aT2k6DXnufFU1s= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/vcaesar/cedar v0.30.0 h1:9fSDpM7FTjjUdPiBUUa0MWYMRGSEcqgFXvppZcZ4d7Y= +github.com/vcaesar/cedar v0.30.0/go.mod h1:lyuGvALuZZDPNXwpzv/9LyxW+8Y6faN7zauFezNsnik= +github.com/vcaesar/tt v0.20.1 h1:D/jUeeVCNbq3ad8M7hhtB3J9x5RZ6I1n1eZ0BJp7M+4= +github.com/vcaesar/tt v0.20.1/go.mod h1:cH2+AwGAJm19Wa6xvEa+0r+sXDJBT0QgNQey6mwqLeU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver/v2 v2.4.0 h1:Oq6BmUAAFTzMeh6AonuDlgZMuAuEiUxoAD1koK5MuFo= +go.mongodb.org/mongo-driver/v2 v2.4.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..60a7772 --- /dev/null +++ b/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "model-asynch/controller" + + _ "gitea.com/red-future/common/config" + "gitea.com/red-future/common/http" + "gitea.com/red-future/common/jaeger" + _ "gitea.com/red-future/common/swagger" + _ "github.com/gogf/gf/contrib/drivers/pgsql/v2" + _ "github.com/gogf/gf/contrib/nosql/redis/v2" + "github.com/gogf/gf/v2/frame/g" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer jaeger.ShutDown(ctx) + + // 注册路由 + http.RouteRegister([]interface{}{ + controller.Model, + controller.ModelType, + controller.Task, + controller.Stat, + }) + + // 监听退出信号，确保 Ctrl+C 能完整退出（停止 worker/cleaner 并关闭 http server） + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + <-quit + + g.Log().Infof(ctx, "[main] 收到退出信号，开始优雅退出...") + cancel() + // 关闭 http server（RouteRegister 内部是 go Httpserver.Run() 启动的） + _ = http.Httpserver.Shutdown() +} diff --git a/model/dto/model_dto.go b/model/dto/model_dto.go new file mode 100644 index 0000000..ca90f92 --- /dev/null +++ b/model/dto/model_dto.go @@ -0,0 +1,90 @@ +package dto + +import ( + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +// CreateModelReq 添加模型配置 +type CreateModelReq struct { + g.Meta `path:"/createModel" method:"post" tags:"模型管理" summary:"创建模型配置" dc:"添加新的模型配置"` + ModelName string `p:"modelName" json:"modelName" v:"required#modelName不能为空" dc:"模型名称（唯一标识）"` + ModelsType string `p:"modelsType" json:"modelsType" dc:"模型类型ID列表（逗号分隔），示例：1,2,3（关联 asynch_models_type.type_id，可选）"` + BaseURL string `p:"baseUrl" json:"baseUrl" v:"required#baseUrl不能为空" dc:"模型服务基础地址（如 http(s)://host:port）"` + Route string `p:"route" json:"route" dc:"路由/路径（拼接到 BaseURL 之后的可选路径）"` + HttpMethod string `p:"httpMethod" json:"httpMethod" dc:"请求方式：GET/POST（默认POST）"` + HeadMsg string `p:"headMsg" json:"headMsg" dc:"请求头绑定（支持多个，逗号分隔），示例：X-API-Key:xxx,operation:true"` + Form any `p:"form" json:"form" dc:"动态表单配置（JSON），用于前端渲染配置项，示例：[{field,label,type,required},...]"` + Enabled int `p:"enabled" json:"enabled" dc:"是否启用：0-禁用，1-启用"` + MaxConcurrency int `p:"maxConcurrency" json:"maxConcurrency" dc:"最大并发数"` + QueueLimit int `p:"queueLimit" json:"queueLimit" dc:"排队队列上限（超过则拒绝/限流）"` + TimeoutSeconds int `p:"timeoutSeconds" json:"timeoutSeconds" dc:"请求超时时间（秒）"` + ExpectedSeconds int `p:"expectedSeconds" json:"expectedSeconds" dc:"模型预计执行时间（秒，用于超时判定/排队策略等）"` + RetryTimes int `p:"retryTimes" json:"retryTimes" dc:"失败重试次数"` + RetryQueueMaxSeconds int `p:"retryQueueMaxSeconds" json:"retryQueueMaxSeconds" dc:"失败重试最大排队时间（秒）；0表示失败重试插队到队首；>0表示排队超过该时间后插队，否则仍到队尾"` + AutoCleanSeconds int `p:"autoCleanSeconds" json:"autoCleanSeconds" dc:"自动清理间隔（秒）（如清理超时任务/队列）"` + Remark string `p:"remark" json:"remark" dc:"备注说明"` +} + +type CreateModelRes struct { + ID int64 `json:"id,string" dc:"配置ID"` +} + +// UpdateModelReq 更新模型配置 +type UpdateModelReq struct { + g.Meta `path:"/updateModel" method:"put" tags:"模型管理" summary:"更新模型配置" dc:"更新指定ID的模型配置"` + ID int64 `p:"id" json:"id,string" v:"required#id不能为空" dc:"配置ID"` + ModelsType *string `p:"modelsType" json:"modelsType" dc:"模型类型ID列表（逗号分隔）（可选更新）"` + BaseURL string `p:"baseUrl" json:"baseUrl" dc:"模型服务基础地址"` + Route string `p:"route" json:"route" dc:"路由/路径"` + HttpMethod *string `p:"httpMethod" json:"httpMethod" dc:"请求方式：GET/POST（可选更新）"` + HeadMsg *string `p:"headMsg" json:"headMsg" dc:"请求头绑定（可选更新）"` + Form any `p:"form" json:"form" dc:"动态表单配置（JSON）（可选更新）"` + Enabled *int `p:"enabled" json:"enabled" dc:"是否启用：0-禁用，1-启用（可选更新）"` + MaxConcurrency *int `p:"maxConcurrency" json:"maxConcurrency" dc:"最大并发数（可选更新）"` + QueueLimit *int `p:"queueLimit" json:"queueLimit" dc:"排队队列上限（可选更新）"` + TimeoutSeconds *int `p:"timeoutSeconds" json:"timeoutSeconds" dc:"请求超时时间（秒）（可选更新）"` + ExpectedSeconds *int `p:"expectedSeconds" json:"expectedSeconds" dc:"模型预计执行时间（秒）（可选更新）"` + RetryTimes *int `p:"retryTimes" json:"retryTimes" dc:"失败重试次数（可选更新）"` + RetryQueueMaxSeconds *int `p:"retryQueueMaxSeconds" json:"retryQueueMaxSeconds" dc:"失败重试最大排队时间（秒）（可选更新）"` + AutoCleanSeconds *int `p:"autoCleanSeconds" json:"autoCleanSeconds" dc:"自动清理间隔（秒）（可选更新）"` + Remark *string `p:"remark" json:"remark" dc:"备注说明（可选更新）"` +} + +// DeleteModelReq 删除模型配置 +type DeleteModelReq struct { + g.Meta `path:"/deleteModel" method:"delete" tags:"模型管理" summary:"删除模型配置" dc:"删除指定ID的模型配置"` + ID int64 `p:"id" json:"id,string" v:"required#id不能为空" dc:"配置ID"` +} + +// GetModelReq 获取模型配置详情 +type GetModelReq struct { + g.Meta `path:"/getModel" method:"get" tags:"模型管理" summary:"获取模型配置" dc:"根据模型名称获取配置详情"` + ID int64 `p:"id" json:"id,string" v:"required#id不能为空" dc:"配置ID"` +} + +type GetModelRes struct { + Model any `json:"model" dc:"模型配置详情"` +} + +// ListModelReq 配置列表 +type ListModelReq struct { + g.Meta `path:"/listModel" method:"post" tags:"模型管理" summary:"模型配置列表" dc:"分页获取模型配置列表"` + Page *beans.Page `p:"page" json:"page" dc:"分页参数"` + ModelName string `p:"modelName" json:"modelName" dc:"模型名称（模糊查询，可选）"` +} + +type ListModelRes struct { + List any `json:"list" dc:"列表数据"` + Total int64 `json:"total" dc:"总数"` +} + +// AutoTuneReq 动态调参（由上层定时任务每小时触发一次） +type AutoTuneReq struct { + g.Meta `path:"/autoTune" method:"post" tags:"模型管理" summary:"动态调参" dc:"按 model_name 维度统计指定时间窗口内执行耗时(P90)，动态生成运行时 max_concurrency/queue_limit（不超过配置上限），写入 Redis 供 Worker/CreateTask 使用；windowSeconds 不传默认 3600"` + WindowSeconds int `p:"windowSeconds" json:"windowSeconds" dc:"统计窗口秒数；不传/<=0 默认 3600（1小时）"` +} + +type AutoTuneRes struct { + List any `json:"list" dc:"调参结果列表"` +} diff --git a/model/dto/model_type_dto.go b/model/dto/model_type_dto.go new file mode 100644 index 0000000..3b2c606 --- /dev/null +++ b/model/dto/model_type_dto.go @@ -0,0 +1,74 @@ +package dto + +import ( + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +// CreateModelTypeReq 创建模型类型 +type CreateModelTypeReq struct { + g.Meta `path:"/createModelType" method:"post" tags:"模型类型" summary:"创建模型类型" dc:"创建模型类型（图片/音频/视频等）"` + TypeID int `p:"typeId" json:"typeId" v:"required#typeId不能为空" dc:"模型类型ID（业务枚举）"` + TypeName string `p:"type" json:"type" v:"required#type不能为空" dc:"模型类型名称"` + Remark string `p:"remark" json:"remark" dc:"备注"` +} + +type CreateModelTypeRes struct { + ID int64 `json:"id,string" dc:"主键ID"` +} + +// UpdateModelTypeReq 更新模型类型 +type UpdateModelTypeReq struct { + g.Meta `path:"/updateModelType" method:"put" tags:"模型类型" summary:"更新模型类型" dc:"更新模型类型"` + ID int64 `p:"id" json:"id,string" v:"required#id不能为空" dc:"主键ID"` + TypeID *int `p:"typeId" json:"typeId" dc:"模型类型ID（可选更新）"` + TypeName *string `p:"type" json:"type" dc:"模型类型名称（可选更新）"` + Remark *string `p:"remark" json:"remark" dc:"备注（可选更新）"` +} + +// DeleteModelTypeReq 删除模型类型 +type DeleteModelTypeReq struct { + g.Meta `path:"/deleteModelType" method:"delete" tags:"模型类型" summary:"删除模型类型" dc:"删除模型类型"` + ID int64 `p:"id" json:"id,string" v:"required#id不能为空" dc:"主键ID"` +} + +// GetModelTypeReq 获取模型类型 +type GetModelTypeReq struct { + g.Meta `path:"/getModelType" method:"get" tags:"模型类型" summary:"获取模型类型" dc:"获取模型类型详情"` + ID int64 `p:"id" json:"id,string" v:"required#id不能为空" dc:"主键ID"` +} + +type GetModelTypeRes struct { + Type any `json:"type" dc:"模型类型详情"` +} + +// ListModelTypeReq 模型类型列表（分页） +type ListModelTypeReq struct { + g.Meta `path:"/listModelType" method:"post" tags:"模型类型" summary:"模型类型列表" dc:"分页获取模型类型列表"` + Page *beans.Page `p:"page" json:"page" dc:"分页参数（默认10条）"` + TypeName string `p:"type" json:"type" dc:"模型类型名称（模糊查询，可选）"` +} + +type ListModelTypeRes struct { + List any `json:"list" dc:"列表数据"` + Total int64 `json:"total" dc:"总数"` +} + +// ListModelTypeWithModelsReq 按类型分组返回模型列表 +type ListModelTypeWithModelsReq struct { + g.Meta `path:"/listModelTypeWithModels" method:"post" tags:"模型类型" summary:"按类型分组的模型列表" dc:"返回模型类型及其下的模型列表（用于前端分组展示）"` + TypeID int `p:"typeId" json:"typeId" dc:"按类型ID过滤（可选）"` + Type string `p:"type" json:"type" dc:"按类型名称过滤（可选，模糊匹配）"` +} + +type ModelTypeModelItem struct { + ID int64 `json:"id" dc:"模型主键ID"` + Name string `json:"name" dc:"模型名称"` + Form any `json:"form" dc:"动态表单配置（JSON数组），用于前端渲染"` +} + +type ModelTypeWithModelsItem struct { + TypeID int `json:"typeId" dc:"模型类型ID"` + Type string `json:"type" dc:"模型类型名称"` + Items []ModelTypeModelItem `json:"items" dc:"该类型下模型列表"` +} diff --git a/model/dto/stat_dto.go b/model/dto/stat_dto.go new file mode 100644 index 0000000..2341d66 --- /dev/null +++ b/model/dto/stat_dto.go @@ -0,0 +1,23 @@ +package dto + +import ( + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +// ListModelStatReq 统计列表 +type ListModelStatReq struct { + g.Meta `path:"/listModelStat" method:"post" tags:"统计" summary:"模型请求统计列表" dc:"按天统计模型请求次数，支持分页与条件筛选"` + Page *beans.Page `p:"page" json:"page" dc:"分页参数（默认10条）"` + StartDay string `p:"startDay" json:"startDay" dc:"开始日期（YYYY-MM-DD，可选）"` + EndDay string `p:"endDay" json:"endDay" dc:"结束日期（YYYY-MM-DD，可选）"` + TenantID *int64 `p:"tenantId" json:"tenantId" dc:"租户ID（可选）"` + Creator string `p:"creator" json:"creator" dc:"创建人（可选，模糊匹配）"` + ModelName string `p:"modelName" json:"modelName" dc:"模型名称（可选，模糊匹配）"` +} + +type ListModelStatRes struct { + List any `json:"list" dc:"列表数据"` + Total int64 `json:"total" dc:"总数"` +} + diff --git a/model/dto/task_dto.go b/model/dto/task_dto.go new file mode 100644 index 0000000..922073f --- /dev/null +++ b/model/dto/task_dto.go @@ -0,0 +1,82 @@ +package dto + +import ( + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +// CreateTaskReq 创建异步任务 +type CreateTaskReq struct { + g.Meta `path:"/createTask" method:"post" tags:"任务管理" summary:"创建异步任务" dc:"创建异步任务并返回任务ID"` + ModelName string `p:"modelName" json:"modelName" v:"required#modelName不能为空" dc:"模型名称"` + ModelKey string `p:"modelKey" json:"modelKey" dc:"动态请求头（用于覆盖/补充模型配置 head_msg），示例：X-API-Key:xxx"` + BizName string `p:"bizName" json:"bizName" dc:"业务名称（调用方模块/系统，用于统计）"` + CallbackUrl string `p:"callbackUrl" json:"callbackUrl" dc:"回调地址（可选，用于后续业务通知）"` + InputRef string `p:"inputRef" json:"inputRef" dc:"输入引用（如OSS/文件引用等）"` + RequestPayload any `p:"requestPayload" json:"requestPayload" dc:"请求负载（透传给模型服务）"` +} + +type CreateTaskRes struct { + TaskID string `json:"taskId" dc:"任务ID"` +} + +// GetTaskResultReq 获取结果（只返回 oss 地址） +type GetTaskResultReq struct { + g.Meta `path:"/getTaskResult" method:"get" tags:"任务管理" summary:"获取任务结果" dc:"根据任务ID获取结果（只返回OSS地址）"` + TaskID string `p:"taskId" json:"taskId" v:"required#taskId不能为空" dc:"任务ID"` +} + +type GetTaskResultRes struct { + OssFile string `json:"ossFile" dc:"结果文件OSS地址"` + State int `json:"state" dc:"任务状态"` +} + +// GetTaskBatchReq 批量查询任务（并对成功任务标记为已下载） +type GetTaskBatchReq struct { + g.Meta `path:"/getTaskBatch" method:"post" tags:"任务管理" summary:"批量查询任务" dc:"批量查询任务状态与OSS地址；对成功(state=2)的任务自动标记为已下载(state=4)，并写入保留到期时间"` + TaskIDs []string `p:"taskIds" json:"taskIds" v:"required#taskIds不能为空" dc:"任务ID列表"` +} + +type GetTaskBatchItem struct { + TaskID string `json:"taskId" dc:"任务ID"` + State int `json:"state" dc:"任务状态"` + OssFile string `json:"ossFile" dc:"结果文件OSS地址"` +} + +type GetTaskBatchRes struct { + List []GetTaskBatchItem `json:"list" dc:"任务列表"` +} + +// ListTaskReq 任务列表分页查询 +type ListTaskReq struct { + g.Meta `path:"/listTask" method:"post" tags:"任务管理" summary:"任务列表" dc:"分页查询任务列表，支持按状态/模型名称/task_id过滤"` + Page *beans.Page `p:"page" json:"page" dc:"分页参数"` + ModelName string `p:"modelName" json:"modelName" dc:"模型名称（模糊匹配）"` + TaskID string `p:"taskId" json:"taskId" dc:"任务ID（模糊匹配）"` + State *int `p:"state" json:"state" dc:"任务状态（0/1/2/3/4，可选）"` +} + +type ListTaskRes struct { + List any `json:"list" dc:"列表数据"` + Total int64 `json:"total" dc:"总数"` +} + +// RunWorkReq 手动触发 worker 执行一次（由上层定时任务调用） +type RunWorkReq struct { + g.Meta `path:"/runWork" method:"post" tags:"任务管理" summary:"执行一次Worker" dc:"手动触发一次Worker抢占并处理任务（用于由上层定时任务控制）"` + BatchSize int `p:"batchSize" json:"batchSize" dc:"本次抢占任务数量（默认10）"` + Goroutines int `p:"goroutines" json:"goroutines" dc:"本次并发数（默认1）"` +} + +type RunWorkRes struct { + Claimed int `json:"claimed" dc:"本次抢占并处理的任务数"` +} + +// CleanWorkReq 手动触发 cleaner 执行一次（由上层定时任务调用） +type CleanWorkReq struct { + g.Meta `path:"/cleanWork" method:"post" tags:"任务管理" summary:"执行一次Cleaner" dc:"手动触发一次清理/重试（用于由上层定时任务控制）"` +} + +type CleanWorkRes struct { + Ok bool `json:"ok" dc:"是否执行成功"` +} diff --git a/model/entity/asynch_model.go b/model/entity/asynch_model.go new file mode 100644 index 0000000..8ad9c5d --- /dev/null +++ b/model/entity/asynch_model.go @@ -0,0 +1,64 @@ +package entity + +import "gitea.com/red-future/common/beans" + +type asynchModelCol struct { + beans.SQLBaseCol + ModelName string + BaseURL string + Route string + HttpMethod string + HeadMsg string + FormJSON string + ModelsType string + Enabled string + MaxConcurrency string + QueueLimit string + TimeoutSeconds string + ExpectedSeconds string + RetryTimes string + RetryQueueMaxSecs string + AutoCleanSeconds string + Remark string +} + +var AsynchModelCol = asynchModelCol{ + SQLBaseCol: beans.DefSQLBaseCol, + ModelName: "model_name", + BaseURL: "base_url", + Route: "route", + HttpMethod: "http_method", + HeadMsg: "head_msg", + FormJSON: "form_json", + ModelsType: "models_type", + Enabled: "enabled", + MaxConcurrency: "max_concurrency", + QueueLimit: "queue_limit", + TimeoutSeconds: "timeout_seconds", + ExpectedSeconds: "expected_seconds", + RetryTimes: "retry_times", + RetryQueueMaxSecs: "retry_queue_max_seconds", + AutoCleanSeconds: "auto_clean_seconds", + Remark: "remark", +} + +// AsynchModel 异步模型配置 +type AsynchModel struct { + beans.SQLBaseDO `orm:",inline"` + ModelName string `orm:"model_name" json:"modelName"` + BaseURL string `orm:"base_url" json:"baseUrl"` + Route string `orm:"route" json:"route"` + HttpMethod string `orm:"http_method" json:"httpMethod"` + HeadMsg string `orm:"head_msg" json:"headMsg"` + Form any `orm:"form_json" json:"form"` + ModelsType string `orm:"models_type" json:"modelsType"` + Enabled int `orm:"enabled" json:"enabled"` + MaxConcurrency int `orm:"max_concurrency" json:"maxConcurrency"` + QueueLimit int `orm:"queue_limit" json:"queueLimit"` + TimeoutSeconds int `orm:"timeout_seconds" json:"timeoutSeconds"` + ExpectedSeconds int `orm:"expected_seconds" json:"expectedSeconds"` + RetryTimes int `orm:"retry_times" json:"retryTimes"` + RetryQueueMaxSecs int `orm:"retry_queue_max_seconds" json:"retryQueueMaxSeconds"` + AutoCleanSeconds int `orm:"auto_clean_seconds" json:"autoCleanSeconds"` + Remark string `orm:"remark" json:"remark"` +} diff --git a/model/entity/asynch_model_stat.go b/model/entity/asynch_model_stat.go new file mode 100644 index 0000000..7ba8455 --- /dev/null +++ b/model/entity/asynch_model_stat.go @@ -0,0 +1,16 @@ +package entity + +import "github.com/gogf/gf/v2/os/gtime" + +// AsynchModelStat 按天统计：某天/租户/创建人/模型的请求次数 +// 注：这里不走通用 SQLBaseDO，采用联合唯一键（day,tenant_id,creator,model_name）做 UPSERT 原子累加。 +type AsynchModelStat struct { + Day *gtime.Time `orm:"day" json:"day"` // 日期（建议仅使用日期部分） + TenantId int64 `orm:"tenant_id" json:"tenantId,string"` + Creator string `orm:"creator" json:"creator"` + ModelName string `orm:"model_name" json:"modelName"` + RequestCount int64 `orm:"request_count" json:"requestCount"` + CreatedAt *gtime.Time `orm:"created_at" json:"createdAt"` + UpdatedAt *gtime.Time `orm:"updated_at" json:"updatedAt"` +} + diff --git a/model/entity/asynch_model_type.go b/model/entity/asynch_model_type.go new file mode 100644 index 0000000..3a4d47a --- /dev/null +++ b/model/entity/asynch_model_type.go @@ -0,0 +1,26 @@ +package entity + +import "gitea.com/red-future/common/beans" + +type asynchModelTypeCol struct { + beans.SQLBaseCol + TypeID string + TypeName string + Remark string +} + +var AsynchModelTypeCol = asynchModelTypeCol{ + SQLBaseCol: beans.DefSQLBaseCol, + TypeID: "type_id", + TypeName: "type_name", + Remark: "remark", +} + +// AsynchModelType 模型类型（图片/音频/视频等） +type AsynchModelType struct { + beans.SQLBaseDO `orm:",inline"` + TypeID int `orm:"type_id" json:"typeId"` + TypeName string `orm:"type_name" json:"type"` + Remark string `orm:"remark" json:"remark"` +} + diff --git a/model/entity/asynch_op_log.go b/model/entity/asynch_op_log.go new file mode 100644 index 0000000..0b821a4 --- /dev/null +++ b/model/entity/asynch_op_log.go @@ -0,0 +1,57 @@ +package entity + +import ( + "gitea.com/red-future/common/beans" +) + +type asynchOpLogCol struct { + beans.SQLBaseCol + IP string + UserAgent string + APIPath string + HttpMethod string + BizName string + ModelName string + TaskID string + OpType string + Success string + ErrorMsg string + CostMs string + RequestPayload string + ResponsePayload string +} + +var AsynchOpLogCol = asynchOpLogCol{ + SQLBaseCol: beans.DefSQLBaseCol, + IP: "ip", + UserAgent: "user_agent", + APIPath: "api_path", + HttpMethod: "http_method", + BizName: "biz_name", + ModelName: "model_name", + TaskID: "task_id", + OpType: "op_type", + Success: "success", + ErrorMsg: "error_msg", + CostMs: "cost_ms", + RequestPayload: "request_payload", + ResponsePayload: "response_payload", +} + +// AsynchOpLog 操作日志（创建任务等） +type AsynchOpLog struct { + beans.SQLBaseDO `orm:",inline"` + IP string `orm:"ip" json:"ip"` + UserAgent string `orm:"user_agent" json:"userAgent"` + APIPath string `orm:"api_path" json:"apiPath"` + HttpMethod string `orm:"http_method" json:"httpMethod"` + BizName string `orm:"biz_name" json:"bizName"` + ModelName string `orm:"model_name" json:"modelName"` + TaskID string `orm:"task_id" json:"taskId"` + OpType string `orm:"op_type" json:"opType"` + Success int `orm:"success" json:"success"` + ErrorMsg string `orm:"error_msg" json:"errorMsg"` + CostMs int64 `orm:"cost_ms" json:"costMs"` + RequestPayload any `orm:"request_payload" json:"requestPayload"` + ResponsePayload any `orm:"response_payload" json:"responsePayload"` +} diff --git a/model/entity/asynch_task.go b/model/entity/asynch_task.go new file mode 100644 index 0000000..876838d --- /dev/null +++ b/model/entity/asynch_task.go @@ -0,0 +1,81 @@ +package entity + +import ( + "gitea.com/red-future/common/beans" + "github.com/gogf/gf/v2/os/gtime" +) + +type asynchTaskCol struct { + beans.SQLBaseCol + ModelName string + TaskID string + State string + BizName string + CallbackURL string + ModelKey string + OssFile string + FileType string + FileSize string + ErrorMsg string + StartedAt string + FinishedAt string + ExpireAt string + DurationSeconds string + RetryCount string + EnqueueAt string + Phase string + TmpFile string + InputRef string + RequestPayload string +} + +var AsynchTaskCol = asynchTaskCol{ + SQLBaseCol: beans.DefSQLBaseCol, + ModelName: "model_name", + TaskID: "task_id", + State: "state", + BizName: "biz_name", + CallbackURL: "callback_url", + ModelKey: "model_key", + OssFile: "oss_file", + FileType: "file_type", + FileSize: "file_size", + ErrorMsg: "error_msg", + StartedAt: "started_at", + FinishedAt: "finished_at", + ExpireAt: "expire_at", + DurationSeconds: "duration_seconds", + RetryCount: "retry_count", + EnqueueAt: "enqueue_at", + Phase: "phase", + TmpFile: "tmp_file", + InputRef: "input_ref", + RequestPayload: "request_payload", +} + +// AsynchTask 异步任务 +type AsynchTask struct { + beans.SQLBaseDO `orm:",inline"` + ModelName string `orm:"model_name" json:"modelName"` + TaskID string `orm:"task_id" json:"taskId"` + State int `orm:"state" json:"state"` // 0排队中/1执行中/2成功/3失败/4已下载 + BizName string `orm:"biz_name" json:"bizName"` + CallbackURL string `orm:"callback_url" json:"callbackUrl"` + ModelKey string `orm:"model_key" json:"modelKey"` + OssFile string `orm:"oss_file" json:"ossFile"` + FileType string `orm:"file_type" json:"fileType"` + FileSize int64 `orm:"file_size" json:"fileSize"` + ErrorMsg string `orm:"error_msg" json:"errorMsg"` + StartedAt *gtime.Time `orm:"started_at" json:"startedAt"` + FinishedAt *gtime.Time `orm:"finished_at" json:"finishedAt"` + ExpireAt *gtime.Time `orm:"expire_at" json:"expireAt"` // 已下载(state=4)后的过期时间 + DurationSeconds int64 `orm:"duration_seconds" json:"durationSeconds"` + RetryCount int `orm:"retry_count" json:"retryCount"` + EnqueueAt *gtime.Time `orm:"enqueue_at" json:"enqueueAt"` + Phase int `orm:"phase" json:"phase"` // 0模型阶段/1OSS阶段 + TmpFile string `orm:"tmp_file" json:"tmpFile"` // 临时结果文件路径 + // RetryQueueMaxSeconds 为 ListFailedRetryableGlobal 的 join 字段（非任务表字段） + RetryQueueMaxSeconds int `orm:"retry_queue_max_seconds" json:"-"` + InputRef string `orm:"input_ref" json:"inputRef"` + RequestPayload any `orm:"request_payload" json:"requestPayload"` +} diff --git a/service/auto_tune.go b/service/auto_tune.go new file mode 100644 index 0000000..552357c --- /dev/null +++ b/service/auto_tune.go @@ -0,0 +1,194 @@ +package service + +import ( + "context" + "fmt" + "math" + + "model-asynch/consts/public" + "model-asynch/model/entity" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/frame/g" +) + +// AutoTuneResult 单次调参结果（按 model_name） +type AutoTuneResult struct { + ModelName string `json:"modelName"` // 模型名称（asynch_models.model_name） + Samples int `json:"samples"` // 统计样本数（窗口内 state=2/3 且 started_at/finished_at 非空的任务数量） + P90Exec float64 `json:"p90ExecSeconds"` // 执行耗时 P90（秒），口径：finished_at - started_at + + CapMaxConcurrency int `json:"capMaxConcurrency"` // 配置上限：asynch_models.max_concurrency（cap，不会被动态调参覆盖） + OldMaxConcurrency int `json:"oldMaxConcurrency"` // 调参前运行时值（Redis），若无则等于 cap + NewMaxConcurrency int `json:"newMaxConcurrency"` // 本次计算出的运行时值（将写入 Redis），受 ±50% 约束且不超过 cap + + CapQueueLimit int `json:"capQueueLimit"` // 配置上限：asynch_models.queue_limit（cap，不会被动态调参覆盖） + OldQueueLimit int `json:"oldQueueLimit"` // 调参前运行时值（Redis），若无则等于 cap + NewQueueLimit int `json:"newQueueLimit"` // 本次计算出的运行时值（将写入 Redis），受 ±50% 约束且不超过 cap + + ExpectedSeconds int `json:"expectedSeconds"` // 模型预计执行时间（秒）：asynch_models.expected_seconds（用于 queue_limit 计算绑定） +} + +// AutoTune 由上层定时任务通过接口触发： +// - 统计指定时间窗口内该模型任务的执行耗时（finished_at - started_at，取 P90） +// - 基于吞吐与 P90 执行耗时估算 max_concurrency 的运行时值（不超过 cap） +// - queue_limit 与 expected_seconds 绑定（允许排队时间 = expected_seconds * 2），生成运行时值（不超过 cap） +// - 单次调整幅度限制 ±50%，写入 Redis（带 TTL） +func AutoTune(ctx context.Context, windowSeconds int) ([]AutoTuneResult, error) { + if windowSeconds <= 0 { + windowSeconds = 3600 + } + // 1) 读取模型配置（cap），按 model_name 聚合去重（如果表里有多租户重复数据，取较大上限） + var modelRows []*entity.AsynchModel + if err := gfdb.DB(ctx).Model(ctx, public.TableNameModel). + Where("deleted_at IS NULL"). + Where(entity.AsynchModelCol.Enabled, 1). + Scan(&modelRows); err != nil { + return nil, err + } + modelMap := make(map[string]*entity.AsynchModel) + for _, m := range modelRows { + if m == nil || m.ModelName == "" { + continue + } + cur := modelMap[m.ModelName] + if cur == nil { + modelMap[m.ModelName] = m + continue + } + // 取更大的 cap + if m.MaxConcurrency > cur.MaxConcurrency { + cur.MaxConcurrency = m.MaxConcurrency + } + if m.QueueLimit > cur.QueueLimit { + cur.QueueLimit = m.QueueLimit + } + if m.ExpectedSeconds > cur.ExpectedSeconds { + cur.ExpectedSeconds = m.ExpectedSeconds + } + } + if len(modelMap) == 0 { + return []AutoTuneResult{}, nil + } + + // 2) 统计指定窗口：按 model_name 计算 cnt 和 P90 执行耗时 + type statRow struct { + ModelName string + Cnt int + P90Exec float64 + } + var stats []statRow + sql := fmt.Sprintf(` +SELECT model_name, + COUNT(1) AS cnt, + COALESCE(percentile_cont(0.9) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished_at - started_at))), 0) AS p90_exec + FROM %s + WHERE deleted_at IS NULL + AND state IN (2,3) + AND started_at IS NOT NULL + AND finished_at IS NOT NULL + AND finished_at >= (NOW() - (? || ' seconds')::interval) + GROUP BY model_name`, public.TableNameTask) + r, err := gfdb.DB(ctx).GetAll(ctx, sql, windowSeconds) + if err != nil { + return nil, err + } + _ = r.Structs(&stats) + statMap := make(map[string]statRow, len(stats)) + for _, s := range stats { + statMap[s.ModelName] = s + } + + // 3) 调参计算 + const utilization = 0.8 + const maxChangeRatio = 0.5 // ±50% + const queueFactor = 2.0 // 与 expected_seconds 绑定：W_target = expected_seconds * 2 + + out := make([]AutoTuneResult, 0, len(modelMap)) + for modelName, m := range modelMap { + s := statMap[modelName] + capMax := m.MaxConcurrency + capQueue := m.QueueLimit + oldMax := GetRuntimeMaxConcurrency(ctx, modelName, capMax) + oldQueue := GetRuntimeQueueLimit(ctx, modelName, capQueue) + + // 默认：无样本则不调整 + if s.Cnt <= 0 || s.P90Exec <= 0 { + out = append(out, AutoTuneResult{ + ModelName: modelName, + Samples: s.Cnt, + P90Exec: s.P90Exec, + CapMaxConcurrency: capMax, + OldMaxConcurrency: oldMax, + NewMaxConcurrency: oldMax, + CapQueueLimit: capQueue, + OldQueueLimit: oldQueue, + NewQueueLimit: oldQueue, + ExpectedSeconds: m.ExpectedSeconds, + }) + continue + } + + // arrival_rate ≈ 完成数/3600 + arrivalRate := float64(s.Cnt) / 3600.0 + + // desiredMax = ceil(arrivalRate * p90 / utilization) + desiredMax := int(math.Ceil(arrivalRate * s.P90Exec / utilization)) + if desiredMax < 1 { + desiredMax = 1 + } + // 单次变化幅度限制 + minMax := int(math.Floor(float64(oldMax) * (1 - maxChangeRatio))) + maxMax := int(math.Ceil(float64(oldMax) * (1 + maxChangeRatio))) + if minMax < 1 { + minMax = 1 + } + newMax := clampInt(desiredMax, minMax, maxMax) + if capMax > 0 { + newMax = clampInt(newMax, 1, capMax) + } + setRuntimeInt(ctx, runtimeMaxConcurrencyKey(modelName), newMax) + + // queue_limit：W_target = expected_seconds * queueFactor + exp := m.ExpectedSeconds + if exp <= 0 { + exp = 60 + } + wTarget := float64(exp) * queueFactor + desiredQueue := int(math.Ceil(arrivalRate*wTarget)) + newMax + if desiredQueue < newMax { + desiredQueue = newMax + } + + newQueue := oldQueue + if capQueue > 0 { + minQ := int(math.Floor(float64(oldQueue) * (1 - maxChangeRatio))) + maxQ := int(math.Ceil(float64(oldQueue) * (1 + maxChangeRatio))) + if minQ < newMax { + minQ = newMax + } + if maxQ < minQ { + maxQ = minQ + } + newQueue = clampInt(desiredQueue, minQ, maxQ) + newQueue = clampInt(newQueue, newMax, capQueue) + setRuntimeInt(ctx, runtimeQueueLimitKey(modelName), newQueue) + } + + out = append(out, AutoTuneResult{ + ModelName: modelName, + Samples: s.Cnt, + P90Exec: s.P90Exec, + CapMaxConcurrency: capMax, + OldMaxConcurrency: oldMax, + NewMaxConcurrency: newMax, + CapQueueLimit: capQueue, + OldQueueLimit: oldQueue, + NewQueueLimit: newQueue, + ExpectedSeconds: m.ExpectedSeconds, + }) + } + + g.Log().Infof(ctx, "[auto_tune] done models=%d windowSeconds=%d", len(out), windowSeconds) + return out, nil +} diff --git a/service/callback.go b/service/callback.go new file mode 100644 index 0000000..ed9c86a --- /dev/null +++ b/service/callback.go @@ -0,0 +1,88 @@ +package service + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "model-asynch/model/entity" + + "github.com/gogf/gf/v2/frame/g" +) + +// triggerSuccessCallback 任务成功后的回调钩子： +// - 使用 GET 请求 +// - 回调地址为 callbackUrl + "/" + bizName +// - query 参数：task_id/state/oss_file/file_type +// 注意：回调失败不影响任务主流程，只记录日志。 +func triggerSuccessCallback(ctx context.Context, t *entity.AsynchTask) { + if t == nil { + return + } + callbackURL := strings.TrimSpace(t.CallbackURL) + bizName := strings.TrimSpace(t.BizName) + if callbackURL == "" || bizName == "" { + return + } + + u, err := url.Parse(callbackURL) + if err != nil { + g.Log().Warningf(ctx, "[callback] invalid callbackUrl=%s err=%v", callbackURL, err) + return + } + // 必须是可发起 HTTP 请求的绝对地址 + if u.Scheme == "" || u.Host == "" { + g.Log().Warningf(ctx, "[callback] callbackUrl must be absolute http(s) url, got=%s", callbackURL) + return + } + + // path 末尾拼接 bizName + bizSeg := url.PathEscape(bizName) + if strings.HasSuffix(u.Path, "/") || u.Path == "" { + u.Path = strings.TrimRight(u.Path, "/") + "/" + bizSeg + } else { + u.Path = u.Path + "/" + bizSeg + } + + q := u.Query() + q.Set("task_id", t.TaskID) + q.Set("state", fmt.Sprintf("%d", t.State)) + q.Set("oss_file", t.OssFile) + q.Set("file_type", t.FileType) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + g.Log().Warningf(ctx, "[callback] build request failed url=%s err=%v", u.String(), err) + return + } + // 透传必要头部（如 Authorization / X-User-Info） + for k, v := range forwardHeaders(ctx) { + if v != "" { + req.Header.Set(k, v) + } + } + + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Do(req) + if err != nil { + g.Log().Warningf(ctx, "[callback] request failed url=%s err=%v", u.String(), err) + return + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + msg := string(b) + if len(msg) > 2000 { + msg = msg[:2000] + } + g.Log().Warningf(ctx, "[callback] non-2xx url=%s code=%d body=%s", u.String(), resp.StatusCode, msg) + return + } + g.Log().Infof(ctx, "[callback] success url=%s code=%d", u.String(), resp.StatusCode) +} + diff --git a/service/cleaner.go b/service/cleaner.go new file mode 100644 index 0000000..07db871 --- /dev/null +++ b/service/cleaner.go @@ -0,0 +1,92 @@ +package service + +import ( + "context" + "time" + + "model-asynch/dao" + + "github.com/gogf/gf/v2/frame/g" +) + +var Cleaner = &cleaner{} + +type cleaner struct{} + +// RunOnce 由上层定时任务触发：执行一次清理/重试 +func (c *cleaner) RunOnce(ctx context.Context) { + // 1) 清理已下载(state=4)且过期的任务（硬删除 + OSS） + expired, err := dao.Task.ListExpiredDownloadedGlobal(ctx, 200) + if err != nil { + g.Log().Errorf(ctx, "[cleaner] list expired(downloaded) error: %v", err) + } else { + for _, t := range expired { + deleteTmpResult(t.TmpFile) + _ = dao.Task.HardDeleteByIDGlobal(ctx, t.Id) + } + g.Log().Infof(ctx, "[cleaner] expired(downloaded) cleaned, count=%d", len(expired)) + } + + // 2) 超时任务标失败 + list, err := dao.Task.ListTimeoutTasksGlobal(ctx, 200) + if err != nil { + g.Log().Errorf(ctx, "[cleaner] list timeout error: %v", err) + } else { + for _, t := range list { + _ = dao.Task.UpdateFailedGlobal(ctx, t.Id, "任务超时自动失败") + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + } + g.Log().Infof(ctx, "[cleaner] timeout cleaned, count=%d", len(list)) + } + + // 3) 失败(state=3)的任务按模型配置 retry_times 重新入队（放到队尾） + retryable, err := dao.Task.ListFailedRetryableGlobal(ctx, 200) + if err != nil { + g.Log().Errorf(ctx, "[cleaner] list failed retryable error: %v", err) + } else { + for _, t := range retryable { + // 失败任务重新入队（state=3 -> 0）前，先严格占用 queue_limit slot；占用失败则留在失败态，下一轮再尝试 + // 获取模型配置以得到 queue_limit / expected_seconds + m, err := dao.Model.GetByModelNameForTenant(ctx, t.TenantId, t.ModelName) + if err != nil || m == nil { + continue + } + limit := GetRuntimeQueueLimit(ctx, t.ModelName, m.QueueLimit) + if limit > 0 { + ok, _ := AcquireQueueSlot(ctx, t.ModelName, t.TaskID, limit, m.ExpectedSeconds) + if !ok { + continue + } + } + // retry_queue_max_seconds 控制失败重试的排队策略： + // - =0：失败重试插队到队首 + // - >0：当任务从创建到现在的排队时长 >= maxSeconds，则插队到队首；否则仍放到队尾 + now := time.Now() + enqueueAt := now + maxSeconds := t.RetryQueueMaxSeconds + if maxSeconds == 0 { + enqueueAt = now.Add(-100 * 365 * 24 * time.Hour) + } else if maxSeconds > 0 && t.CreatedAt != nil { + if now.Sub(t.CreatedAt.Time) >= time.Duration(maxSeconds)*time.Second { + enqueueAt = now.Add(-100 * 365 * 24 * time.Hour) + } + } + _ = dao.Task.RequeueForRetryGlobal(ctx, t.Id, enqueueAt) + } + g.Log().Infof(ctx, "[cleaner] failed retryable cleaned, count=%d", len(retryable)) + } + + // 4) 超过重试次数仍失败(state=3)的任务：硬删除 + exhausted, err := dao.Task.ListFailedExhaustedGlobal(ctx, 200) + if err != nil { + g.Log().Errorf(ctx, "[cleaner] list failed exhausted error: %v", err) + } else { + for _, t := range exhausted { + deleteTmpResult(t.TmpFile) + // 重试耗尽硬删除：释放闸门占位（兜底，若此前已释放则幂等） + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + _ = dao.Task.HardDeleteByIDGlobal(ctx, t.Id) + } + g.Log().Infof(ctx, "[cleaner] failed exhausted cleaned, count=%d", len(exhausted)) + } +} diff --git a/service/file_detect.go b/service/file_detect.go new file mode 100644 index 0000000..aaf75c7 --- /dev/null +++ b/service/file_detect.go @@ -0,0 +1,35 @@ +package service + +import ( + "net/http" + "strings" +) + +// DetectFileType 根据返回的二进制内容推断 contentType + 扩展名（尽量稳定） +func DetectFileType(data []byte) (contentType string, ext string) { + if len(data) == 0 { + return "application/octet-stream", "" + } + ct := http.DetectContentType(data) + switch ct { + case "audio/mpeg": + return ct, ".mp3" + case "audio/wave", "audio/wav", "audio/x-wav": + return ct, ".wav" + case "video/mp4": + return ct, ".mp4" + case "image/png": + return ct, ".png" + case "image/jpeg": + return ct, ".jpg" + case "application/pdf": + return ct, ".pdf" + default: + // 兜底：尝试从 ct 截取 subtype 作为后缀（例如 application/json） + if parts := strings.Split(ct, "/"); len(parts) == 2 { + return ct, "." + parts[1] + } + return ct, "" + } +} + diff --git a/service/headers.go b/service/headers.go new file mode 100644 index 0000000..b4033ed --- /dev/null +++ b/service/headers.go @@ -0,0 +1,54 @@ +package service + +import ( + "context" + + "gitea.com/red-future/common/utils" + "github.com/gogf/gf/v2/frame/g" +) + +// asyncCtx 固化异步执行所需的 token/user，避免请求结束后丢失（仅在“同请求内起 goroutine”有用）。 +// 本项目当前是“落库 + 后台 worker”模式，因此还会把必要信息持久化到任务表的 request_payload 中。 +func asyncCtx(ctx context.Context) context.Context { + asyncCtx := context.WithoutCancel(ctx) + if r := g.RequestFromCtx(ctx); r != nil { + if token := r.Header.Get("Authorization"); token != "" { + asyncCtx = context.WithValue(asyncCtx, "token", token) + } + if userInfo := r.Header.Get("X-User-Info"); userInfo != "" { + asyncCtx = context.WithValue(asyncCtx, "xUserInfo", userInfo) + } + } + if user, err := utils.GetUserInfo(ctx); err == nil && user != nil { + asyncCtx = context.WithValue(asyncCtx, "user", user) + } + return asyncCtx +} + +// forwardHeaders 透传调用链路中必须的头信息（优先使用 ctx 里固化的 token / xUserInfo）。 +func forwardHeaders(ctx context.Context) map[string]string { + headers := make(map[string]string) + + if token, ok := ctx.Value("token").(string); ok && token != "" { + headers["Authorization"] = token + } + if x, ok := ctx.Value("xUserInfo").(string); ok && x != "" { + headers["X-User-Info"] = x + } + + // 兜底：从请求头拿 + if r := g.RequestFromCtx(ctx); r != nil { + if headers["Authorization"] == "" { + if token := r.Header.Get("Authorization"); token != "" { + headers["Authorization"] = token + } + } + if headers["X-User-Info"] == "" { + if userInfo := r.Header.Get("X-User-Info"); userInfo != "" { + headers["X-User-Info"] = userInfo + } + } + } + return headers +} + diff --git a/service/model_invoker.go b/service/model_invoker.go new file mode 100644 index 0000000..83186ad --- /dev/null +++ b/service/model_invoker.go @@ -0,0 +1,185 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "model-asynch/model/entity" +) + +// parseHeadMsgHeaders 支持多个 header 绑定，逗号分隔： +// 示例： +// - X-API-Key:qwen3-tts-key,operation:true,count:123 +// - X-API-Key:"qwen3-tts-key",operation:"true" +// +// 说明： +// - HTTP Header 最终都是字符串，这里做的是“值的字符串化表达”。 +// - 若 value 用双引号包裹，会去掉外层引号再注入，便于在配置中区分字符串/布尔/数字等表达（以及避免值中包含特殊字符时歧义）。 +func parseHeadMsgHeaders(headMsg string) map[string]string { + headMsg = strings.TrimSpace(headMsg) + if headMsg == "" { + return nil + } + out := map[string]string{} + parts := strings.Split(headMsg, ",") + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + // HeaderName:HeaderValue（推荐） / HeaderName=HeaderValue（兼容） + if strings.Contains(p, ":") { + kv := strings.SplitN(p, ":", 2) + k := strings.TrimSpace(kv[0]) + v := strings.TrimSpace(kv[1]) + v = strings.Trim(v, "\"") + if k != "" && v != "" { + out[k] = v + } + continue + } + if strings.Contains(p, "=") { + kv := strings.SplitN(p, "=", 2) + k := strings.TrimSpace(kv[0]) + v := strings.TrimSpace(kv[1]) + v = strings.Trim(v, "\"") + if k != "" && v != "" { + out[k] = v + } + continue + } + } + if len(out) == 0 { + return nil + } + return out +} + +func payloadToQuery(payload any) (url.Values, error) { + if payload == nil { + return url.Values{}, nil + } + // 统一转成 map[string]any + b, err := json.Marshal(payload) + if err != nil { + return nil, err + } + m := map[string]any{} + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + q := url.Values{} + for k, v := range m { + if v == nil { + continue + } + // 复杂类型直接 json 字符串化 + switch vv := v.(type) { + case string: + q.Set(k, vv) + case float64, bool, int, int64, uint64: + q.Set(k, fmt.Sprintf("%v", vv)) + default: + bs, _ := json.Marshal(v) + q.Set(k, string(bs)) + } + } + return q, nil +} + +// InvokeModel 调用模型服务，返回二进制结果 +// modelKey 用于覆盖/补充模型配置 head_msg（例如每次请求携带不同的 X-API-Key）。 +func InvokeModel(ctx context.Context, m *entity.AsynchModel, payload any, modelKey string) ([]byte, error) { + if m == nil || m.BaseURL == "" { + return nil, fmt.Errorf("模型配置不完整") + } + url := strings.TrimRight(m.BaseURL, "/") + "/" + strings.TrimLeft(m.Route, "/") + if strings.TrimSpace(m.Route) == "" { + url = strings.TrimRight(m.BaseURL, "/") + } + + timeout := time.Duration(m.TimeoutSeconds) * time.Second + if timeout <= 0 { + timeout = 60 * time.Second + } + client := &http.Client{Timeout: timeout} + + method := strings.ToUpper(strings.TrimSpace(m.HttpMethod)) + if method == "" { + method = http.MethodPost + } + + var ( + req *http.Request + err error + ) + switch method { + case http.MethodGet: + q, err := payloadToQuery(payload) + if err != nil { + return nil, err + } + if len(q) > 0 { + if strings.Contains(url, "?") { + url = url + "&" + q.Encode() + } else { + url = url + "?" + q.Encode() + } + } + req, err = http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + default: + bodyBytes, err := json.Marshal(payload) + if err != nil { + return nil, err + } + req, err = http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyBytes)) + } + if err != nil { + return nil, err + } + + // 先注入模型配置 head_msg（静态头部） + for hk, hv := range parseHeadMsgHeaders(m.HeadMsg) { + req.Header.Set(hk, hv) + } + // 透传必要头部（如 Authorization / X-User-Info） + for k, v := range forwardHeaders(ctx) { + if v != "" { + req.Header.Set(k, v) + } + } + // 最后注入动态 modelKey（覆盖/补充静态 head_msg） + for hk, hv := range parseHeadMsgHeaders(modelKey) { + req.Header.Set(hk, hv) + } + if method != http.MethodGet { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // 尽量把错误体带回去，方便排查 + msg := string(b) + if len(msg) > 2000 { + msg = msg[:2000] + } + return nil, fmt.Errorf("模型服务返回非2xx: %d, body=%s", resp.StatusCode, msg) + } + return b, nil +} diff --git a/service/model_service.go b/service/model_service.go new file mode 100644 index 0000000..6f481ac --- /dev/null +++ b/service/model_service.go @@ -0,0 +1,125 @@ +package service + +import ( + "context" + "errors" + + "model-asynch/dao" + "model-asynch/model/dto" + "model-asynch/model/entity" +) + +var Model = &modelService{} + +type modelService struct{} + +func (s *modelService) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) { + m := &entity.AsynchModel{ + ModelName: req.ModelName, + ModelsType: normalizeModelsType(req.ModelsType), + BaseURL: req.BaseURL, + Route: req.Route, + HttpMethod: req.HttpMethod, + HeadMsg: req.HeadMsg, + Form: req.Form, + Enabled: req.Enabled, + MaxConcurrency: req.MaxConcurrency, + QueueLimit: req.QueueLimit, + TimeoutSeconds: req.TimeoutSeconds, + ExpectedSeconds: req.ExpectedSeconds, + RetryTimes: req.RetryTimes, + RetryQueueMaxSecs: req.RetryQueueMaxSeconds, + AutoCleanSeconds: req.AutoCleanSeconds, + Remark: req.Remark, + } + if m.HttpMethod == "" { + m.HttpMethod = "POST" + } + if m.Enabled == 0 { + m.Enabled = 1 + } + if m.MaxConcurrency <= 0 { + m.MaxConcurrency = 10 + } + if m.QueueLimit <= 0 { + m.QueueLimit = 1000 + } + if m.TimeoutSeconds <= 0 { + m.TimeoutSeconds = 60 + } + if m.AutoCleanSeconds <= 0 { + m.AutoCleanSeconds = 86400 + } + id, err := dao.Model.Insert(ctx, m) + if err != nil { + return nil, err + } + return &dto.CreateModelRes{ID: id}, nil +} + +func (s *modelService) Update(ctx context.Context, req *dto.UpdateModelReq) error { + data := map[string]any{} + if req.BaseURL != "" { + data[entity.AsynchModelCol.BaseURL] = req.BaseURL + } + if req.Route != "" { + data[entity.AsynchModelCol.Route] = req.Route + } + if req.HttpMethod != nil && *req.HttpMethod != "" { + data[entity.AsynchModelCol.HttpMethod] = *req.HttpMethod + } + if req.HeadMsg != nil { + data[entity.AsynchModelCol.HeadMsg] = *req.HeadMsg + } + if req.Form != nil { + data[entity.AsynchModelCol.FormJSON] = req.Form + } + if req.ModelsType != nil { + data[entity.AsynchModelCol.ModelsType] = normalizeModelsType(*req.ModelsType) + } + if req.Enabled != nil { + data[entity.AsynchModelCol.Enabled] = *req.Enabled + } + if req.MaxConcurrency != nil { + data[entity.AsynchModelCol.MaxConcurrency] = *req.MaxConcurrency + } + if req.QueueLimit != nil { + data[entity.AsynchModelCol.QueueLimit] = *req.QueueLimit + } + if req.TimeoutSeconds != nil { + data[entity.AsynchModelCol.TimeoutSeconds] = *req.TimeoutSeconds + } + if req.ExpectedSeconds != nil { + data[entity.AsynchModelCol.ExpectedSeconds] = *req.ExpectedSeconds + } + if req.RetryTimes != nil { + data[entity.AsynchModelCol.RetryTimes] = *req.RetryTimes + } + if req.RetryQueueMaxSeconds != nil { + data[entity.AsynchModelCol.RetryQueueMaxSecs] = *req.RetryQueueMaxSeconds + } + if req.AutoCleanSeconds != nil { + data[entity.AsynchModelCol.AutoCleanSeconds] = *req.AutoCleanSeconds + } + if req.Remark != nil { + data[entity.AsynchModelCol.Remark] = *req.Remark + } + if len(data) == 0 { + return errors.New("无可更新字段") + } + _, err := dao.Model.UpdateByID(ctx, req.ID, data) + return err +} + +func (s *modelService) Delete(ctx context.Context, id int64) error { + _, err := dao.Model.DeleteByID(ctx, id) + return err +} + +func (s *modelService) Get(ctx context.Context, id int64) (*entity.AsynchModel, error) { + return dao.Model.GetByID(ctx, id) +} + +func (s *modelService) List(ctx context.Context, pageNum, pageSize int, modelNameLike string) (list []*entity.AsynchModel, total int64, err error) { + return dao.Model.List(ctx, pageNum, pageSize, modelNameLike) +} diff --git a/service/model_type_service.go b/service/model_type_service.go new file mode 100644 index 0000000..d8c405c --- /dev/null +++ b/service/model_type_service.go @@ -0,0 +1,217 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "strings" + + "model-asynch/dao" + "model-asynch/model/dto" + "model-asynch/model/entity" + + "github.com/gogf/gf/v2/container/gvar" +) + +type modelTypeService struct{} + +var ModelType = &modelTypeService{} + +func normalizeFormValue(v any) any { + // 目标：对外永远返回 JSON 数组/对象，而不是字符串。 + if v == nil { + return []any{} + } + switch t := v.(type) { + case string: + s := strings.TrimSpace(t) + if s == "" { + return []any{} + } + return normalizeFormValueFromJSONString(s) + case []byte: + if len(t) == 0 { + return []any{} + } + return normalizeFormValueFromJSONBytes(t) + case *gvar.Var: + // goframe 常见的 DB 返回类型 + if t == nil { + return []any{} + } + b := t.Bytes() + if len(b) > 0 { + return normalizeFormValueFromJSONBytes(b) + } + s := strings.TrimSpace(t.String()) + if s == "" { + return []any{} + } + return normalizeFormValueFromJSONString(s) + default: + // 尝试兼容其他“像 JSON 的值类型”（例如实现了 Bytes/String 的包装类型） + if vb, ok := v.(interface{ Bytes() []byte }); ok { + if b := vb.Bytes(); len(b) > 0 { + return normalizeFormValueFromJSONBytes(b) + } + } + if vs, ok := v.(interface{ String() string }); ok { + if s := strings.TrimSpace(vs.String()); s != "" { + return normalizeFormValueFromJSONString(s) + } + } + // 已经是 []any / map[string]any 等结构 + return v + } +} + +// 兼容“JSONB 里存了 JSON 字符串”的历史数据： +// 例如 form_json = '"[]"' 或 '"[{...}]"'（外层是字符串，内层才是数组/对象） +func normalizeFormValueFromJSONString(s string) any { + var out any + if err := json.Unmarshal([]byte(s), &out); err != nil || out == nil { + return []any{} + } + // 如果解出来还是 string，且看起来是 JSON，再解一层 + if inner, ok := out.(string); ok { + inner = strings.TrimSpace(inner) + if inner == "" { + return []any{} + } + if strings.HasPrefix(inner, "[") || strings.HasPrefix(inner, "{") { + var out2 any + if err := json.Unmarshal([]byte(inner), &out2); err == nil && out2 != nil { + return out2 + } + } + return []any{} + } + return out +} + +func normalizeFormValueFromJSONBytes(b []byte) any { + var out any + if err := json.Unmarshal(b, &out); err != nil || out == nil { + return []any{} + } + // bytes 解出来也可能是 string（同上） + if inner, ok := out.(string); ok { + return normalizeFormValueFromJSONString(inner) + } + return out +} + +func (s *modelTypeService) Create(ctx context.Context, req *dto.CreateModelTypeReq) (res *dto.CreateModelTypeRes, err error) { + t := &entity.AsynchModelType{ + TypeID: req.TypeID, + TypeName: req.TypeName, + Remark: req.Remark, + } + id, err := dao.ModelType.Insert(ctx, t) + if err != nil { + return nil, err + } + return &dto.CreateModelTypeRes{ID: id}, nil +} + +func (s *modelTypeService) Update(ctx context.Context, req *dto.UpdateModelTypeReq) error { + data := map[string]any{} + if req.TypeID != nil { + data[entity.AsynchModelTypeCol.TypeID] = *req.TypeID + } + if req.TypeName != nil { + data[entity.AsynchModelTypeCol.TypeName] = *req.TypeName + } + if req.Remark != nil { + data[entity.AsynchModelTypeCol.Remark] = *req.Remark + } + if len(data) == 0 { + return errors.New("无可更新字段") + } + _, err := dao.ModelType.UpdateByID(ctx, req.ID, data) + return err +} + +func (s *modelTypeService) Delete(ctx context.Context, id int64) error { + _, err := dao.ModelType.DeleteByID(ctx, id) + return err +} + +func (s *modelTypeService) Get(ctx context.Context, id int64) (*entity.AsynchModelType, error) { + return dao.ModelType.GetByID(ctx, id) +} + +func (s *modelTypeService) List(ctx context.Context, pageNum, pageSize int, typeNameLike string) (list []*entity.AsynchModelType, total int64, err error) { + return dao.ModelType.List(ctx, pageNum, pageSize, typeNameLike) +} + +// ListWithModels 按类型分组返回模型（返回数组，便于前端直接渲染） +func (s *modelTypeService) ListWithModels(ctx context.Context, req *dto.ListModelTypeWithModelsReq) (res []dto.ModelTypeWithModelsItem, err error) { + types, _, err := dao.ModelType.List(ctx, 1, 1000, "") + if err != nil { + return nil, err + } + // 过滤类型（按 typeId / typeName 模糊） + filterTypeID := 0 + filterTypeName := "" + if req != nil { + filterTypeID = req.TypeID + filterTypeName = strings.TrimSpace(req.Type) + } + typeIDs := make([]int, 0, len(types)) + typeNameMap := make(map[int]string, len(types)) + for _, t := range types { + if t == nil { + continue + } + if filterTypeID > 0 && t.TypeID != filterTypeID { + continue + } + if filterTypeName != "" && !strings.Contains(t.TypeName, filterTypeName) { + continue + } + typeIDs = append(typeIDs, t.TypeID) + typeNameMap[t.TypeID] = t.TypeName + } + models, err := dao.Model.ListAll(ctx) + if err != nil { + return nil, err + } + itemsMap := map[int][]dto.ModelTypeModelItem{} + for _, m := range models { + if m == nil { + continue + } + form := normalizeFormValue(m.Form) + // 一个模型可能支持多个类型：models_type="1,2,3" + for _, tid := range parseModelsTypeIDs(m.ModelsType) { + // 若请求过滤了类型，则只输出该类型 + if filterTypeID > 0 && tid != filterTypeID { + continue + } + if filterTypeName != "" { + if _, ok := typeNameMap[tid]; !ok { + continue + } + } + itemsMap[tid] = append(itemsMap[tid], dto.ModelTypeModelItem{ + ID: m.Id, + Name: m.ModelName, + Form: form, + }) + } + } + out := make([]dto.ModelTypeWithModelsItem, 0, len(typeIDs)) + for _, tid := range typeIDs { + items := itemsMap[tid] + if items == nil { + items = make([]dto.ModelTypeModelItem, 0) + } + out = append(out, dto.ModelTypeWithModelsItem{ + TypeID: tid, + Type: typeNameMap[tid], + Items: items, + }) + } + return out, nil +} diff --git a/service/model_types_util.go b/service/model_types_util.go new file mode 100644 index 0000000..c280987 --- /dev/null +++ b/service/model_types_util.go @@ -0,0 +1,52 @@ +package service + +import ( + "sort" + "strconv" + "strings" +) + +// normalizeModelsType 将 "1, 2,2,3" 归一化为 "1,2,3" +// - 去空格 +// - 去重 +// - 升序排序 +func normalizeModelsType(v string) string { + ids := parseModelsTypeIDs(v) + if len(ids) == 0 { + return "" + } + parts := make([]string, 0, len(ids)) + for _, id := range ids { + parts = append(parts, strconv.Itoa(id)) + } + return strings.Join(parts, ",") +} + +// parseModelsTypeIDs 解析 models_type 字段（支持 "1,2,3"），返回去重后的 int 列表（升序）。 +func parseModelsTypeIDs(v string) []int { + v = strings.TrimSpace(v) + if v == "" { + return nil + } + raw := strings.Split(v, ",") + seen := map[int]struct{}{} + out := make([]int, 0, len(raw)) + for _, s := range raw { + s = strings.TrimSpace(s) + if s == "" || s == "0" { + continue + } + id, err := strconv.Atoi(s) + if err != nil || id <= 0 { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + sort.Ints(out) + return out +} + diff --git a/service/payload.go b/service/payload.go new file mode 100644 index 0000000..b6873fc --- /dev/null +++ b/service/payload.go @@ -0,0 +1,25 @@ +package service + +import "github.com/gogf/gf/v2/util/gconv" + +// parseStoredPayload 解析入库的 request_payload，拆出模型调用 payload 与透传 headers +// 入库格式：{"payload": , "headers": {"Authorization": "...", "X-User-Info":"..."}} +func parseStoredPayload(v any) (payload any, headers map[string]string) { + if v == nil { + return nil, nil + } + m := gconv.Map(v) + if len(m) == 0 { + return v, nil + } + if h, ok := m["headers"]; ok { + headers = gconv.MapStrStr(h) + } + if p, ok := m["payload"]; ok { + payload = p + } else { + payload = v + } + return +} + diff --git a/service/queue_gate.go b/service/queue_gate.go new file mode 100644 index 0000000..d9998aa --- /dev/null +++ b/service/queue_gate.go @@ -0,0 +1,107 @@ +package service + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" +) + +// ===== 严格 queue_limit：Redis 原子闸门 ===== +// +// 背景：原来的 queue_limit 通过“Count + Insert”做近似控制，分布式并发创建时会短暂超限。 +// 目标：以 Redis Lua 脚本实现原子校验 + 入队占位，做到严格不超限。 +// +// 计数口径与原逻辑保持一致：只统计 state=0/1（排队中/执行中）。 +// - CreateTask 成功入库后占用 1 个 slot +// - 任务成功/失败（state->2/3）释放 slot +// - 失败任务重试（state 3->0）需要再次占用 slot，若占位失败则暂不重试（留在 state=3，下次 cleaner 再尝试） +// +// 说明：为避免极端情况下“占位泄漏”导致永久占满，采用 ZSET + 过期时间的方式自动回收。 +// 只要任务实际生命周期远小于 gateTTLSeconds，就可保持严格。 + +const ( + queueGateKeyPrefix = "asynch:qgate:" // asynch:qgate:{modelName} +) + +// Lua：清理过期 slot，然后按 limit 做原子判定并占位 +var queueGateAcquireLua = ` +local key = KEYS[1] +local now = tonumber(ARGV[1]) +local limit = tonumber(ARGV[2]) +local expireAt = tonumber(ARGV[3]) +local member = ARGV[4] +local keyTTL = tonumber(ARGV[5]) + +-- 先清理过期的占位 +redis.call("ZREMRANGEBYSCORE", key, "-inf", now) + +local current = tonumber(redis.call("ZCARD", key) or "0") +if current >= limit then + return 0 +end +redis.call("ZADD", key, expireAt, member) +redis.call("EXPIRE", key, keyTTL) +return 1 +` + +// Lua：释放 slot（幂等） +var queueGateReleaseLua = ` +local key = KEYS[1] +local member = ARGV[1] +redis.call("ZREM", key, member) +return 1 +` + +func queueGateKey(modelName string) string { + return fmt.Sprintf("%s%s", queueGateKeyPrefix, modelName) +} + +// calcGateTTLSeconds 计算闸门占位的“自动回收 TTL” +// 取 expectedSeconds 的倍数并做上下限，避免任务异常导致永久占位。 +func calcGateTTLSeconds(expectedSeconds int) int { + // 默认至少 1 小时；最多 24 小时 + minTTL := 3600 + maxTTL := 24 * 3600 + if expectedSeconds <= 0 { + return minTTL + } + ttl := int(math.Ceil(float64(expectedSeconds) * 10)) // 预计耗时 * 10 做兜底 + if ttl < minTTL { + ttl = minTTL + } + if ttl > maxTTL { + ttl = maxTTL + } + return ttl +} + +// AcquireQueueSlot 严格入队：原子占位（成功返回 true） +func AcquireQueueSlot(ctx context.Context, modelName, taskId string, limit int, expectedSeconds int) (bool, error) { + if limit <= 0 { + return true, nil + } + key := queueGateKey(modelName) + now := time.Now().Unix() + ttl := calcGateTTLSeconds(expectedSeconds) + expireAt := now + int64(ttl) + // keyTTL 要略大于 member TTL，避免 key 先过期导致计数丢失 + keyTTL := ttl + 60 + r, err := g.Redis().Do(ctx, "EVAL", queueGateAcquireLua, 1, key, now, limit, expireAt, taskId, keyTTL) + if err != nil { + return false, fmt.Errorf("queue gate acquire failed: %w", err) + } + return gconv.Int(r) == 1, nil +} + +// ReleaseQueueSlot 释放占位（幂等） +func ReleaseQueueSlot(ctx context.Context, modelName, taskId string) { + if taskId == "" || modelName == "" { + return + } + key := queueGateKey(modelName) + _, _ = g.Redis().Do(ctx, "EVAL", queueGateReleaseLua, 1, key, taskId) +} diff --git a/service/runtime_tune.go b/service/runtime_tune.go new file mode 100644 index 0000000..276fe8c --- /dev/null +++ b/service/runtime_tune.go @@ -0,0 +1,83 @@ +package service + +import ( + "context" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" +) + +// 运行时调参存储在 Redis，不修改 asynch_models 中的 cap（最大上限）。 +// 上层每小时调用 /model/autoTune 写入运行时值；Worker/CreateTask 读取运行时值生效。 + +const ( + runtimeMaxCKeyPrefix = "asynch:runtime:max_concurrency:" // + model_name + runtimeQueueKeyPrefix = "asynch:runtime:queue_limit:" // + model_name + runtimeTTLSeconds = 2 * 3600 // 2小时，避免一次调参失败导致立即回退 +) + +func runtimeMaxConcurrencyKey(modelName string) string { + return runtimeMaxCKeyPrefix + modelName +} +func runtimeQueueLimitKey(modelName string) string { + return runtimeQueueKeyPrefix + modelName +} + +func getRuntimeInt(ctx context.Context, key string) (int, bool) { + v, err := g.Redis().Do(ctx, "GET", key) + if err != nil || v == nil { + return 0, false + } + iv := gconv.Int(v) + if iv <= 0 { + return 0, false + } + return iv, true +} + +func setRuntimeInt(ctx context.Context, key string, val int) { + if val <= 0 { + return + } + // SETEX key ttl val + _, _ = g.Redis().Do(ctx, "SETEX", key, runtimeTTLSeconds, val) +} + +// GetRuntimeMaxConcurrency 返回运行时并发上限（<= cap）。若不存在运行时值，则返回 cap。 +func GetRuntimeMaxConcurrency(ctx context.Context, modelName string, cap int) int { + if cap <= 0 { + return cap + } + if v, ok := getRuntimeInt(ctx, runtimeMaxConcurrencyKey(modelName)); ok { + if v > cap { + return cap + } + return v + } + return cap +} + +// GetRuntimeQueueLimit 返回运行时队列上限（<= cap）。若不存在运行时值，则返回 cap。 +func GetRuntimeQueueLimit(ctx context.Context, modelName string, cap int) int { + if cap <= 0 { + return cap + } + if v, ok := getRuntimeInt(ctx, runtimeQueueLimitKey(modelName)); ok { + if v > cap { + return cap + } + return v + } + return cap +} + +func clampInt(v, minV, maxV int) int { + if v < minV { + return minV + } + if v > maxV { + return maxV + } + return v +} + diff --git a/service/semaphore.go b/service/semaphore.go new file mode 100644 index 0000000..e97a9d4 --- /dev/null +++ b/service/semaphore.go @@ -0,0 +1,56 @@ +package service + +import ( + "context" + "fmt" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" +) + +var acquireLua = ` +local current = tonumber(redis.call("GET", KEYS[1]) or "0") +local max = tonumber(ARGV[1]) +local ttl = tonumber(ARGV[2]) +if current >= max then + return 0 +end +current = redis.call("INCR", KEYS[1]) +if current == 1 then + redis.call("EXPIRE", KEYS[1], ttl) +end +if current > max then + redis.call("DECR", KEYS[1]) + return 0 +end +return 1 +` + +var releaseLua = ` +local current = tonumber(redis.call("DECR", KEYS[1]) or "0") +if current <= 0 then + redis.call("DEL", KEYS[1]) +end +return 1 +` + +func acquireSemaphore(ctx context.Context, key string, max int, ttlSeconds int64) (bool, error) { + if max <= 0 { + // 不限制 + return true, nil + } + if ttlSeconds <= 0 { + ttlSeconds = 3600 + } + r, err := g.Redis().Do(ctx, "EVAL", acquireLua, 1, key, max, ttlSeconds) + if err != nil { + return false, fmt.Errorf("获取并发令牌失败: %w", err) + } + return gconv.Int(r) == 1, nil +} + +func releaseSemaphore(ctx context.Context, key string) error { + _, err := g.Redis().Do(ctx, "EVAL", releaseLua, 1, key) + return err +} + diff --git a/service/stat_service.go b/service/stat_service.go new file mode 100644 index 0000000..8509e96 --- /dev/null +++ b/service/stat_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "context" + + "model-asynch/dao" + "model-asynch/model/dto" +) + +type statService struct{} + +var Stat = &statService{} + +func (s *statService) List(ctx context.Context, req *dto.ListModelStatReq) (res *dto.ListModelStatRes, err error) { + pageNum, pageSize := 1, 10 + if req != nil && req.Page != nil { + if req.Page.PageNum > 0 { + pageNum = int(req.Page.PageNum) + } + if req.Page.PageSize > 0 { + pageSize = int(req.Page.PageSize) + } + } + startDay, endDay := "", "" + var tenantID *int64 + creator, modelName := "", "" + if req != nil { + startDay = req.StartDay + endDay = req.EndDay + tenantID = req.TenantID + creator = req.Creator + modelName = req.ModelName + } + list, total, err := dao.Stat.List(ctx, pageNum, pageSize, startDay, endDay, tenantID, creator, modelName) + if err != nil { + return nil, err + } + return &dto.ListModelStatRes{List: list, Total: total}, nil +} + diff --git a/service/storage.go b/service/storage.go new file mode 100644 index 0000000..920071f --- /dev/null +++ b/service/storage.go @@ -0,0 +1,18 @@ +package service + +import ( + "context" + "errors" + + "model-asynch/model/entity" +) + +// StorageService 结果存储（OSS/MinIO）抽象 +type StorageService interface { + UploadByTask(ctx context.Context, t *entity.AsynchTask, data []byte, fileExt string, contentType string) (ossURL string, err error) +} + +// Storage 默认存储实现（优先对接你们的 oss 文件服务；必要时也可以切到 MinIO） +var Storage StorageService = &ossStorage{} + +var ErrStorageNotConfigured = errors.New("存储未配置") diff --git a/service/storage_oss.go b/service/storage_oss.go new file mode 100644 index 0000000..60b09f8 --- /dev/null +++ b/service/storage_oss.go @@ -0,0 +1,82 @@ +package service + +import ( + "bytes" + "context" + "fmt" + "mime/multipart" + "time" + + "model-asynch/model/entity" + + commonHttp "gitea.com/red-future/common/http" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + "github.com/gogf/gf/v2/util/guid" +) + +// 对接你们的 oss 文件服务：POST oss/file/uploadFile (multipart/form-data) +type ossStorage struct{} + +type uploadFileResponse struct { + FileURL string `json:"fileURL"` // 文件 URL + FileSize int `json:"fileSize"` // 文件大小（字节） + FileName string `json:"fileName"` // 文件名 + FileFormat string `json:"fileFormat"` // 文件格式 + FileAddressPrefix string `json:"fileAddressPrefix"` // 文件地址前缀 +} + +func (s *ossStorage) UploadByTask(ctx context.Context, _ *entity.AsynchTask, data []byte, fileExt string, _ string) (ossURL string, err error) { + // multipart + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + ext := fileExt + if ext == "" { + ext = ".bin" + } + if ext[0] != '.' { + ext = "." + ext + } + + filename := fmt.Sprintf("asynch_%d_%s%s", time.Now().Unix(), guid.S(), ext) + part, err := writer.CreateFormFile("file", filename) + if err != nil { + return "", err + } + if _, err := part.Write(data); err != nil { + return "", err + } + contentType := writer.FormDataContentType() + if err := writer.Close(); err != nil { + return "", err + } + + headers := forwardHeaders(ctx) + headers["Content-Type"] = contentType + + fullURL := "oss/file/uploadFile" + g.Log().Infof(ctx, "[OSS] upload start url=%s filename=%s size=%d", fullURL, filename, len(data)) + + var resp uploadFileResponse + if err := commonHttp.Post(ctx, fullURL, headers, &resp, body.Bytes()); err != nil { + return "", err + } + fmt.Println("打印结果 resp:", resp) + g.Log().Infof(ctx, "[OSS] upload success url=%s size=%d format=%s", resp.FileURL, resp.FileSize, resp.FileFormat) + return resp.FileURL, nil +} + +// setTaskHeadersToCtx 把任务入库时保存的 header 信息注入 ctx，给 worker 调 OSS 用 +func setTaskHeadersToCtx(ctx context.Context, headers map[string]string) context.Context { + if headers == nil { + return ctx + } + if v := gconv.String(headers["Authorization"]); v != "" { + ctx = context.WithValue(ctx, "token", v) + } + if v := gconv.String(headers["X-User-Info"]); v != "" { + ctx = context.WithValue(ctx, "xUserInfo", v) + } + return ctx +} diff --git a/service/task_service.go b/service/task_service.go new file mode 100644 index 0000000..11b6d5a --- /dev/null +++ b/service/task_service.go @@ -0,0 +1,192 @@ +package service + +import ( + "context" + "errors" + "time" + + "model-asynch/dao" + "model-asynch/model/dto" + "model-asynch/model/entity" + + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gtime" + "github.com/google/uuid" +) + +var Task = &taskService{} + +type taskService struct{} + +func (s *taskService) Create(ctx context.Context, req *dto.CreateTaskReq) (res *dto.CreateTaskRes, err error) { + startAt := time.Now() + // 固化 token/user 等信息 + ctx = asyncCtx(ctx) + + // 1) 检查模型配置 + m, err := dao.Model.GetByModelName(ctx, req.ModelName) + if err != nil { + return nil, err + } + if m == nil || m.Enabled != 1 { + return nil, errors.New("模型不存在或未启用") + } + + taskID := uuid.NewString() + // 2) 排队上限（严格控制：Redis 原子闸门） + limit := GetRuntimeQueueLimit(ctx, req.ModelName, m.QueueLimit) + if limit > 0 { + ok, err := AcquireQueueSlot(ctx, req.ModelName, taskID, limit, m.ExpectedSeconds) + if err != nil { + return nil, err + } + if !ok { + return nil, errors.New("任务排队已满，请稍后再试") + } + } + + // 将调用模型的 payload 与透传头信息一起存入 request_payload，供后台 worker 使用 + storedPayload := map[string]any{ + "payload": req.RequestPayload, + "headers": forwardHeaders(ctx), + } + + t := &entity.AsynchTask{ + ModelName: req.ModelName, + TaskID: taskID, + State: 0, + BizName: req.BizName, + CallbackURL: req.CallbackUrl, + ModelKey: req.ModelKey, + InputRef: req.InputRef, + RequestPayload: storedPayload, + } + _, err = dao.Task.Insert(ctx, t) + if err != nil { + // 入库失败：回滚闸门占位 + ReleaseQueueSlot(ctx, req.ModelName, taskID) + return nil, err + } + + // 3) 写操作日志（尽量不影响主流程，失败忽略） + ip := "" + ua := "" + apiPath := "/task/createTask" + httpMethod := "POST" + if r := g.RequestFromCtx(ctx); r != nil { + ip = r.GetClientIp() + ua = r.UserAgent() + apiPath = r.URL.Path + httpMethod = r.Method + } + _, _ = dao.OpLog.Insert(ctx, &entity.AsynchOpLog{ + IP: ip, + UserAgent: ua, + APIPath: apiPath, + HttpMethod: httpMethod, + BizName: req.BizName, + ModelName: req.ModelName, + TaskID: taskID, + OpType: "createTask", + Success: 1, + ErrorMsg: "", + CostMs: time.Since(startAt).Milliseconds(), + RequestPayload: storedPayload, + ResponsePayload: gdb.Map{ + "taskId": taskID, + }, + }) + return &dto.CreateTaskRes{TaskID: taskID}, nil +} + +func (s *taskService) GetResult(ctx context.Context, taskID string) (res *dto.GetTaskResultRes, err error) { + t, err := dao.Task.GetByTaskID(ctx, taskID) + if err != nil { + return nil, err + } + if t == nil { + return nil, errors.New("任务不存在") + } + return &dto.GetTaskResultRes{ + OssFile: t.OssFile, + State: t.State, + }, nil +} + +// GetBatch 批量查询任务；将成功(state=2)的任务更新为已下载(state=4)，并写入过期时间 +func (s *taskService) GetBatch(ctx context.Context, req *dto.GetTaskBatchReq) (res *dto.GetTaskBatchRes, err error) { + if req == nil || len(req.TaskIDs) == 0 { + return &dto.GetTaskBatchRes{List: []dto.GetTaskBatchItem{}}, nil + } + // 1) 先查当前租户下的任务列表 + list, err := dao.Task.ListByTaskIDs(ctx, req.TaskIDs) + if err != nil { + return nil, err + } + + // 2) 对成功(state=2)的任务：标记为已下载(state=4)并写入 expire_at + now := time.Now() + for _, t := range list { + if t == nil { + continue + } + if t.State != 2 { + continue + } + // 按模型配置决定保留时间 + m, err := dao.Model.GetByModelName(ctx, t.ModelName) + if err != nil { + return nil, err + } + retainSeconds := 86400 + if m != nil && m.AutoCleanSeconds > 0 { + retainSeconds = m.AutoCleanSeconds + } + expireAt := gtime.New(now.Add(time.Duration(retainSeconds) * time.Second)) + _ = dao.Task.MarkDownloadedByID(ctx, t.Id, expireAt) + + // 为了本次返回一致性，内存里也更新 + t.State = 4 + t.ExpireAt = expireAt + } + + // 3) 组装返回 + items := make([]dto.GetTaskBatchItem, 0, len(list)) + for _, t := range list { + if t == nil { + continue + } + items = append(items, dto.GetTaskBatchItem{ + TaskID: t.TaskID, + State: t.State, + OssFile: t.OssFile, + }) + } + return &dto.GetTaskBatchRes{List: items}, nil +} + +func (s *taskService) List(ctx context.Context, req *dto.ListTaskReq) (res *dto.ListTaskRes, err error) { + pageNum, pageSize := 1, 10 + if req != nil && req.Page != nil { + if req.Page.PageNum > 0 { + pageNum = int(req.Page.PageNum) + } + if req.Page.PageSize > 0 { + pageSize = int(req.Page.PageSize) + } + } + modelName := "" + taskID := "" + var state *int + if req != nil { + modelName = req.ModelName + taskID = req.TaskID + state = req.State + } + list, total, err := dao.Task.List(ctx, pageNum, pageSize, modelName, taskID, state) + if err != nil { + return nil, err + } + return &dto.ListTaskRes{List: list, Total: total}, nil +} diff --git a/service/tmp_store.go b/service/tmp_store.go new file mode 100644 index 0000000..9dea56a --- /dev/null +++ b/service/tmp_store.go @@ -0,0 +1,38 @@ +package service + +import ( + "fmt" + "os" + "path/filepath" +) + +// saveTmpResult 将模型输出写入临时文件，用于 OSS 上传失败后的“仅重试 OSS”。 +func saveTmpResult(taskID string, data []byte, ext string) (string, error) { + dir := filepath.Join(os.TempDir(), "model-asynch") + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + if ext == "" { + ext = ".bin" + } + if ext[0] != '.' { + ext = "." + ext + } + path := filepath.Join(dir, fmt.Sprintf("%s%s", taskID, ext)) + if err := os.WriteFile(path, data, 0o644); err != nil { + return "", err + } + return path, nil +} + +func loadTmpResult(path string) ([]byte, error) { + return os.ReadFile(path) +} + +func deleteTmpResult(path string) { + if path == "" { + return + } + _ = os.Remove(path) +} + diff --git a/service/worker.go b/service/worker.go new file mode 100644 index 0000000..243ed61 --- /dev/null +++ b/service/worker.go @@ -0,0 +1,176 @@ +package service + +import ( + "context" + "fmt" + "strings" + "time" + + "model-asynch/dao" + "model-asynch/model/entity" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/grpool" +) + +var AsyncWorker = &asyncWorker{} + +type asyncWorker struct { +} + +// RunOnce 由上层定时任务触发：一次性抢占并处理一批任务 +// - batchSize: 本次抢占数量 +// - goroutines: 本次并发数（协程池大小） +func (w *asyncWorker) RunOnce(ctx context.Context, batchSize, goroutines int) (claimed int, err error) { + if batchSize <= 0 { + batchSize = 10 + } + if goroutines <= 0 { + goroutines = 1 + } + tasks, err := dao.Task.ClaimPendingGlobal(ctx, batchSize) + if err != nil { + return 0, err + } + if len(tasks) == 0 { + return 0, nil + } + pool := grpool.New(goroutines) + defer pool.Close() + + claimed = len(tasks) + done := make(chan struct{}, claimed) + for _, t := range tasks { + task := t + _ = pool.AddWithRecover(ctx, func(ctx context.Context) { + w.handleOne(ctx, task) + done <- struct{}{} + }, func(ctx context.Context, e error) { + if e != nil { + _ = dao.Task.UpdateFailedGlobal(ctx, task.Id, fmt.Sprintf("worker panic: %v", e)) + ReleaseQueueSlot(ctx, task.ModelName, task.TaskID) + } + done <- struct{}{} + }) + } + for i := 0; i < claimed; i++ { + <-done + } + return claimed, nil +} + +func (w *asyncWorker) handleOne(ctx context.Context, t *entity.AsynchTask) { + // 从任务入库的 request_payload 里恢复 payload + headers，给 OSS 上传透传鉴权用 + payload, headers := parseStoredPayload(t.RequestPayload) + if len(headers) > 0 { + ctx = setTaskHeadersToCtx(ctx, headers) + } + + // 1) 拉取模型配置 + m, err := dao.Model.GetByModelNameForTenant(ctx, t.TenantId, t.ModelName) + if err != nil { + _ = dao.Task.UpdateFailedGlobal(ctx, t.Id, err.Error()) + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + return + } + if m == nil || m.Enabled != 1 { + _ = dao.Task.UpdateFailedGlobal(ctx, t.Id, "模型不存在或未启用") + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + return + } + + // 2) 分布式并发限制（按 model_name 全局维度） + semKey := fmt.Sprintf("asynch:sem:%s", t.ModelName) + leaseSeconds := int64(3600) // 兜底1小时 + maxC := GetRuntimeMaxConcurrency(ctx, t.ModelName, m.MaxConcurrency) + acquired, err := acquireSemaphore(ctx, semKey, maxC, leaseSeconds) + if err != nil { + _ = dao.Task.UpdateFailedGlobal(ctx, t.Id, err.Error()) + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + return + } + if !acquired { + // 并发满了：放回排队（重新置回 state=0），下一轮再抢占 + _ = w.rollbackToPending(ctx, t.Id) + return + } + defer func() { + _ = releaseSemaphore(ctx, semKey) + }() + + // 3) 调用模型服务 + if payload == nil { + payload = map[string]any{ + "taskId": t.TaskID, + "inputRef": t.InputRef, + } + } + var ( + data []byte + contentType string + ext string + ) + + // phase=1 表示模型已成功但 OSS 上传失败：优先从临时文件加载，避免重复跑模型 + if t.Phase == 1 && strings.TrimSpace(t.TmpFile) != "" { + data, err = loadTmpResult(t.TmpFile) + if err == nil && len(data) > 0 { + contentType, ext = DetectFileType(data) + } else { + // 临时文件不可用：回退重新调用模型 + data = nil + } + } + if data == nil { + // 统计：仅在真正请求模型时 +1（OSS 重试不计入） + _ = dao.Stat.IncRequestCount(ctx, time.Now(), int64(t.TenantId), t.Creator, t.ModelName) + + data, err = InvokeModel(ctx, m, payload, t.ModelKey) + if err != nil { + _ = dao.Task.UpdateFailedGlobal(ctx, t.Id, err.Error()) + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + return + } + contentType, ext = DetectFileType(data) + // 将模型输出写入临时文件，后续若 OSS 失败可只重试 OSS + tmpPath, err := saveTmpResult(t.TaskID, data, ext) + if err == nil && tmpPath != "" { + t.TmpFile = tmpPath + t.Phase = 1 + _ = dao.Task.UpdateTmpAfterModelGlobal(ctx, t.Id, tmpPath) + } + } + + // 4) 存储 OSS + ossURL, err := Storage.UploadByTask(ctx, t, data, ext, contentType) + if err != nil { + // OSS 阶段失败：保留临时文件，下一轮仅重试 OSS + _ = dao.Task.UpdateFailedKeepTmpGlobal(ctx, t.Id, err.Error()) + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + return + } + + // 5) 更新任务状态成功 + // 注意：expire_at 的计算改为“已下载(state=4)后开始计时”，因此成功(state=2)不写 expire_at。 + fileType := strings.TrimPrefix(ext, ".") + if fileType == "" { + fileType = contentType + } + if err := dao.Task.UpdateSuccessGlobal(ctx, t.Id, ossURL, fileType, int64(len(data)), nil); err != nil { + g.Log().Errorf(ctx, "[worker] update success failed: %v", err) + return + } + // 成功/失败均不再占用 queue_limit（state=0/1 才占用） + ReleaseQueueSlot(ctx, t.ModelName, t.TaskID) + // 6) 成功回调（不影响主流程） + t.State = 2 + t.OssFile = ossURL + t.FileType = fileType + go triggerSuccessCallback(context.WithoutCancel(ctx), t) + // 成功后清理临时文件 + deleteTmpResult(t.TmpFile) +} + +func (w *asyncWorker) rollbackToPending(ctx context.Context, id int64) error { + return dao.Task.RollbackToPendingGlobal(ctx, id) +} diff --git a/timezone/Shanghai b/timezone/Shanghai new file mode 100644 index 0000000..91f6f8b Binary files /dev/null and b/timezone/Shanghai differ diff --git a/timezone/localtime b/timezone/localtime new file mode 100644 index 0000000..91f6f8b Binary files /dev/null and b/timezone/localtime differ diff --git a/timezone/timezone b/timezone/timezone new file mode 100644 index 0000000..421b7a4 --- /dev/null +++ b/timezone/timezone @@ -0,0 +1 @@ +Asia/Shanghai diff --git a/update.sql b/update.sql new file mode 100644 index 0000000..fb1a2e1 --- /dev/null +++ b/update.sql @@ -0,0 +1,294 @@ +-- model-asynch 核心表（pgsql） +-- 1) asynch_models_type：模型类型 +-- 2) asynch_models：模型配置 +-- 3) asynch_task：异步任务 +-- 4) asynch_op_log：操作日志（统计用） +-- 5) asynch_model_stat：按天模型请求统计（限流/监控用） + +-- ========================= +-- 0) asynch_models_type +-- ========================= +CREATE TABLE IF NOT EXISTS asynch_models_type ( + -- 基础字段（与现有表保持一致） + id BIGINT PRIMARY KEY, -- 主键ID（非自增） + tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID + creator VARCHAR(64) NOT NULL, -- 创建人 + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 创建时间 + updater VARCHAR(64) NOT NULL, -- 更新人 + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 更新时间 + deleted_at TIMESTAMP(6), -- 删除时间（软删） + + -- 业务字段 + type_id INT NOT NULL, -- 模型类型ID（业务枚举） + type_name VARCHAR(64) NOT NULL, -- 模型类型名称（图片模型/音频模型/...） + remark TEXT DEFAULT '' -- 备注 +); + +CREATE UNIQUE INDEX IF NOT EXISTS uk_asynch_models_type_tenant_type_id + ON asynch_models_type(tenant_id, type_id); +CREATE INDEX IF NOT EXISTS idx_asynch_models_type_tenant_id ON asynch_models_type(tenant_id); +CREATE INDEX IF NOT EXISTS idx_asynch_models_type_type_name ON asynch_models_type(type_name); +CREATE INDEX IF NOT EXISTS idx_asynch_models_type_deleted_at ON asynch_models_type(deleted_at); + +COMMENT ON TABLE asynch_models_type IS '模型类型表'; +COMMENT ON COLUMN asynch_models_type.id IS '主键ID（非自增）'; +COMMENT ON COLUMN asynch_models_type.tenant_id IS '租户ID'; +COMMENT ON COLUMN asynch_models_type.creator IS '创建人'; +COMMENT ON COLUMN asynch_models_type.created_at IS '创建时间'; +COMMENT ON COLUMN asynch_models_type.updater IS '更新人'; +COMMENT ON COLUMN asynch_models_type.updated_at IS '更新时间'; +COMMENT ON COLUMN asynch_models_type.deleted_at IS '删除时间（软删）'; +COMMENT ON COLUMN asynch_models_type.type_id IS '模型类型ID（业务枚举）'; +COMMENT ON COLUMN asynch_models_type.type_name IS '模型类型名称'; +COMMENT ON COLUMN asynch_models_type.remark IS '备注'; + + +-- ========================= +-- 1) asynch_models +-- ========================= +CREATE TABLE IF NOT EXISTS asynch_models ( + -- 基础字段（与现有表保持一致） + id BIGINT PRIMARY KEY, -- 主键ID（非自增） + tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID + creator VARCHAR(64) NOT NULL, -- 创建人 + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 创建时间 + updater VARCHAR(64) NOT NULL, -- 更新人 + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 更新时间 + deleted_at TIMESTAMP(6), -- 删除时间（软删） + + -- 业务字段 + model_name VARCHAR(128) NOT NULL, -- 模型名称（路由键） + models_type VARCHAR(128) NOT NULL DEFAULT '', -- 模型类型ID列表（逗号分隔），示例：1,2,3（关联 asynch_models_type.type_id） + base_url VARCHAR(256) NOT NULL, -- 模型服务基础地址（如 http://1.2.3.4:8080） + route VARCHAR(256) NOT NULL DEFAULT '',-- 模型服务路由（如 /v1/infer） + http_method VARCHAR(8) NOT NULL DEFAULT 'POST', -- 请求方式：GET/POST + head_msg VARCHAR(1024) DEFAULT '', -- 请求头绑定（支持多个，逗号分隔）：X-API-Key:xxx,operation:true + form_json JSONB NOT NULL DEFAULT '[]'::jsonb, -- 动态表单配置（JSON数组），用于前端渲染 + + enabled SMALLINT NOT NULL DEFAULT 1, -- 是否启用：1启用/0停用 + max_concurrency INT NOT NULL DEFAULT 10, -- 单模型最大并发 + queue_limit INT NOT NULL DEFAULT 1000, -- 排队上限（近似控制） + timeout_seconds INT NOT NULL DEFAULT 60, -- 调用模型服务超时（秒） + expected_seconds INT NOT NULL DEFAULT 0, -- 模型预计执行时间（秒，用于超时判定/排队策略等） + + retry_times SMALLINT NOT NULL DEFAULT 0, -- 失败后最多再重试 N 次（不含首次） + retry_queue_max_seconds INT NOT NULL DEFAULT 0, -- 失败重试最大排队时间（秒）：0=插队到队首；>0=排队超过该时间后插队，否则仍到队尾 + + auto_clean_seconds INT NOT NULL DEFAULT 86400, -- 已下载(state=4)后的保留时间（秒） + remark TEXT DEFAULT '' -- 备注 +); + +CREATE UNIQUE INDEX IF NOT EXISTS uk_asynch_models_tenant_model_name + ON asynch_models(tenant_id, model_name); +CREATE INDEX IF NOT EXISTS idx_asynch_models_tenant_id ON asynch_models(tenant_id); +CREATE INDEX IF NOT EXISTS idx_asynch_models_model_name ON asynch_models(model_name); +CREATE INDEX IF NOT EXISTS idx_asynch_models_models_type ON asynch_models(models_type); +CREATE INDEX IF NOT EXISTS idx_asynch_models_enabled ON asynch_models(enabled); +CREATE INDEX IF NOT EXISTS idx_asynch_models_deleted_at ON asynch_models(deleted_at); + +COMMENT ON TABLE asynch_models IS '异步模型表（模型服务配置）'; +COMMENT ON COLUMN asynch_models.id IS '主键ID（非自增）'; +COMMENT ON COLUMN asynch_models.tenant_id IS '租户ID'; +COMMENT ON COLUMN asynch_models.creator IS '创建人'; +COMMENT ON COLUMN asynch_models.created_at IS '创建时间'; +COMMENT ON COLUMN asynch_models.updater IS '更新人'; +COMMENT ON COLUMN asynch_models.updated_at IS '更新时间'; +COMMENT ON COLUMN asynch_models.deleted_at IS '删除时间（软删）'; +COMMENT ON COLUMN asynch_models.model_name IS '模型名称（路由键）'; +COMMENT ON COLUMN asynch_models.models_type IS '模型类型ID列表（逗号分隔），示例：1,2,3（关联 asynch_models_type.type_id）'; +COMMENT ON COLUMN asynch_models.base_url IS '模型服务基础地址（如 http://1.2.3.4:8080）'; +COMMENT ON COLUMN asynch_models.route IS '模型服务路由（如 /v1/infer）'; +COMMENT ON COLUMN asynch_models.http_method IS '请求方式：GET/POST'; +COMMENT ON COLUMN asynch_models.head_msg IS '请求头绑定（支持多个，逗号分隔）：X-API-Key:xxx,operation:true'; +COMMENT ON COLUMN asynch_models.form_json IS '动态表单配置（JSON数组），用于前端渲染'; +COMMENT ON COLUMN asynch_models.enabled IS '是否启用：1启用/0停用'; +COMMENT ON COLUMN asynch_models.max_concurrency IS '单模型最大并发'; +COMMENT ON COLUMN asynch_models.queue_limit IS '排队上限（近似控制）'; +COMMENT ON COLUMN asynch_models.timeout_seconds IS '调用模型服务超时（秒）'; +COMMENT ON COLUMN asynch_models.expected_seconds IS '模型预计执行时间（秒，用于超时判定/排队策略等）'; +COMMENT ON COLUMN asynch_models.retry_times IS '失败后最多再重试 N 次（不含首次）'; +COMMENT ON COLUMN asynch_models.retry_queue_max_seconds IS '失败重试最大排队时间（秒）：0=插队到队首；>0=排队超过该时间后插队，否则仍到队尾'; +COMMENT ON COLUMN asynch_models.auto_clean_seconds IS '已下载(state=4)后的保留时间（秒），到期清理'; +COMMENT ON COLUMN asynch_models.remark IS '备注'; + + +-- ========================= +-- 2) asynch_task +-- ========================= +CREATE TABLE IF NOT EXISTS asynch_task ( + -- 基础字段（与现有表保持一致） + id BIGINT PRIMARY KEY, -- 主键ID（非自增） + tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID + creator VARCHAR(64) NOT NULL, -- 创建人 + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 创建时间 + updater VARCHAR(64) NOT NULL, -- 更新人 + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 更新时间 + deleted_at TIMESTAMP(6), -- 删除时间（软删） + + -- 任务核心字段 + model_name VARCHAR(128) NOT NULL, -- 模型名称 + task_id VARCHAR(64) NOT NULL, -- 任务ID（对外返回） + biz_name VARCHAR(128) NOT NULL DEFAULT '', -- 业务名称（调用方模块/系统） + callback_url VARCHAR(512) DEFAULT '', -- 回调地址（可选，用于后续业务通知） + model_key VARCHAR(1024) DEFAULT '', -- 动态请求头（用于覆盖/补充模型配置 head_msg），如 X-API-Key:xxx + state SMALLINT NOT NULL DEFAULT 0, -- 0排队中/1执行中/2成功/3失败/4已下载 + + oss_file VARCHAR(512) DEFAULT '', -- 结果文件OSS地址 + file_type VARCHAR(32) DEFAULT '', -- 文件类型（mp3/mp4/png/...） + file_size BIGINT NOT NULL DEFAULT 0, -- 文件大小（字节） + error_msg TEXT DEFAULT '', -- 错误信息 + + started_at TIMESTAMP, -- 开始执行时间 + finished_at TIMESTAMP, -- 执行结束时间 + duration_seconds BIGINT NOT NULL DEFAULT 0, -- 耗时（秒）：从创建到完成（成功/失败）整体耗时 + + expire_at TIMESTAMP, -- state=4 后写入，用于清理 + + -- 重试/排队 + retry_count INT NOT NULL DEFAULT 0, -- 已重试次数（不含首次） + enqueue_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- 入队时间（用于排队顺序） + + -- 任务执行阶段：用于区分“重试模型”与“仅重试 OSS” + phase SMALLINT NOT NULL DEFAULT 0, -- 0模型阶段/1OSS阶段 + tmp_file TEXT DEFAULT '', -- 临时结果文件路径（phase=1 时仅重试 OSS 上传） + + -- 输入信息（可选） + input_ref TEXT DEFAULT '', -- 输入引用（如OSS/业务资源ID等） + request_payload JSONB -- 请求参数（可选） +); + +CREATE UNIQUE INDEX IF NOT EXISTS uk_asynch_task_tenant_task_id + ON asynch_task(tenant_id, task_id); +CREATE INDEX IF NOT EXISTS idx_asynch_task_tenant_id ON asynch_task(tenant_id); +CREATE INDEX IF NOT EXISTS idx_asynch_task_model_name ON asynch_task(model_name); +CREATE INDEX IF NOT EXISTS idx_asynch_task_biz_name ON asynch_task(biz_name); +CREATE INDEX IF NOT EXISTS idx_asynch_task_model_key ON asynch_task(model_key); +CREATE INDEX IF NOT EXISTS idx_asynch_task_state ON asynch_task(state); +CREATE INDEX IF NOT EXISTS idx_asynch_task_enqueue_at ON asynch_task(enqueue_at); +CREATE INDEX IF NOT EXISTS idx_asynch_task_updated_at ON asynch_task(updated_at); +CREATE INDEX IF NOT EXISTS idx_asynch_task_expire_at ON asynch_task(expire_at); +CREATE INDEX IF NOT EXISTS idx_asynch_task_deleted_at ON asynch_task(deleted_at); + +COMMENT ON TABLE asynch_task IS '异步任务表'; +COMMENT ON COLUMN asynch_task.id IS '主键ID（非自增）'; +COMMENT ON COLUMN asynch_task.tenant_id IS '租户ID'; +COMMENT ON COLUMN asynch_task.creator IS '创建人'; +COMMENT ON COLUMN asynch_task.created_at IS '创建时间'; +COMMENT ON COLUMN asynch_task.updater IS '更新人'; +COMMENT ON COLUMN asynch_task.updated_at IS '更新时间'; +COMMENT ON COLUMN asynch_task.deleted_at IS '删除时间（软删）'; +COMMENT ON COLUMN asynch_task.model_name IS '模型名称'; +COMMENT ON COLUMN asynch_task.task_id IS '任务ID（对外返回）'; +COMMENT ON COLUMN asynch_task.biz_name IS '业务名称（调用方模块/系统）'; +COMMENT ON COLUMN asynch_task.callback_url IS '回调地址（可选，用于后续业务通知）'; +COMMENT ON COLUMN asynch_task.model_key IS '动态请求头（用于覆盖/补充模型配置 head_msg），如 X-API-Key:xxx'; +COMMENT ON COLUMN asynch_task.state IS '0排队中/1执行中/2成功/3失败/4已下载'; +COMMENT ON COLUMN asynch_task.oss_file IS '结果文件OSS地址'; +COMMENT ON COLUMN asynch_task.file_type IS '文件类型（mp3/mp4/png/...）'; +COMMENT ON COLUMN asynch_task.file_size IS '文件大小（字节）'; +COMMENT ON COLUMN asynch_task.error_msg IS '错误信息'; +COMMENT ON COLUMN asynch_task.started_at IS '开始执行时间'; +COMMENT ON COLUMN asynch_task.finished_at IS '执行结束时间'; +COMMENT ON COLUMN asynch_task.duration_seconds IS '耗时（秒）：从创建到完成（成功/失败）整体耗时'; +COMMENT ON COLUMN asynch_task.expire_at IS 'state=4 后写入，用于清理'; +COMMENT ON COLUMN asynch_task.retry_count IS '已重试次数（不含首次）'; +COMMENT ON COLUMN asynch_task.enqueue_at IS '入队时间（用于排队顺序）'; +COMMENT ON COLUMN asynch_task.phase IS '执行阶段：0模型阶段/1OSS阶段（模型已成功，等待上传OSS）'; +COMMENT ON COLUMN asynch_task.tmp_file IS '临时结果文件路径（phase=1 时仅重试 OSS 上传）'; +COMMENT ON COLUMN asynch_task.input_ref IS '输入引用（如OSS/业务资源ID等）'; +COMMENT ON COLUMN asynch_task.request_payload IS '请求参数（可选，JSON）'; + + +-- ========================= +-- 3) asynch_op_log +-- ========================= +CREATE TABLE IF NOT EXISTS asynch_op_log ( + -- 基础字段（与现有表保持一致） + id BIGINT PRIMARY KEY, + tenant_id BIGINT NOT NULL DEFAULT 0, + creator VARCHAR(64) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(64) NOT NULL, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP(6), + + -- 基础审计信息 + ip VARCHAR(64) DEFAULT '', + user_agent VARCHAR(256) DEFAULT '', + api_path VARCHAR(256) DEFAULT '', + http_method VARCHAR(16) DEFAULT '', + + -- 业务信息 + biz_name VARCHAR(128) NOT NULL DEFAULT '', -- 调用方业务模块/系统 + model_name VARCHAR(128) NOT NULL DEFAULT '', + task_id VARCHAR(64) NOT NULL DEFAULT '', + + -- 统计字段 + op_type VARCHAR(64) NOT NULL DEFAULT 'createTask', -- 操作类型（默认创建任务） + success SMALLINT NOT NULL DEFAULT 1, -- 1成功/0失败 + error_msg TEXT DEFAULT '', + cost_ms BIGINT NOT NULL DEFAULT 0, -- 耗时（毫秒） + + -- 请求/响应 JSON（用于后期统计分析） + request_payload JSONB, + response_payload JSONB +); + +CREATE INDEX IF NOT EXISTS idx_asynch_op_log_tenant_time ON asynch_op_log(tenant_id, created_at); +CREATE INDEX IF NOT EXISTS idx_asynch_op_log_model_name ON asynch_op_log(model_name); +CREATE INDEX IF NOT EXISTS idx_asynch_op_log_biz_name ON asynch_op_log(biz_name); +CREATE INDEX IF NOT EXISTS idx_asynch_op_log_task_id ON asynch_op_log(task_id); +CREATE INDEX IF NOT EXISTS idx_asynch_op_log_op_type ON asynch_op_log(op_type); +CREATE INDEX IF NOT EXISTS idx_asynch_op_log_deleted_at ON asynch_op_log(deleted_at); + +COMMENT ON TABLE asynch_op_log IS '操作记录日志表（创建任务等，用于统计）'; +COMMENT ON COLUMN asynch_op_log.id IS '主键ID（非自增）'; +COMMENT ON COLUMN asynch_op_log.tenant_id IS '租户ID'; +COMMENT ON COLUMN asynch_op_log.creator IS '创建人'; +COMMENT ON COLUMN asynch_op_log.created_at IS '创建时间'; +COMMENT ON COLUMN asynch_op_log.updater IS '更新人'; +COMMENT ON COLUMN asynch_op_log.updated_at IS '更新时间'; +COMMENT ON COLUMN asynch_op_log.deleted_at IS '删除时间（软删）'; +COMMENT ON COLUMN asynch_op_log.ip IS '客户端IP'; +COMMENT ON COLUMN asynch_op_log.user_agent IS 'User-Agent'; +COMMENT ON COLUMN asynch_op_log.api_path IS '接口路径'; +COMMENT ON COLUMN asynch_op_log.http_method IS 'HTTP方法'; +COMMENT ON COLUMN asynch_op_log.biz_name IS '业务名称（调用方模块/系统）'; +COMMENT ON COLUMN asynch_op_log.model_name IS '模型名称'; +COMMENT ON COLUMN asynch_op_log.task_id IS '任务ID'; +COMMENT ON COLUMN asynch_op_log.op_type IS '操作类型（如 createTask/getTaskResult/getTaskBatch 等）'; +COMMENT ON COLUMN asynch_op_log.success IS '是否成功：1成功/0失败'; +COMMENT ON COLUMN asynch_op_log.error_msg IS '错误信息（失败时）'; +COMMENT ON COLUMN asynch_op_log.cost_ms IS '耗时（毫秒）'; +COMMENT ON COLUMN asynch_op_log.request_payload IS '请求 JSON'; +COMMENT ON COLUMN asynch_op_log.response_payload IS '响应 JSON'; + + +-- ========================= +-- 4) asynch_model_stat +-- ========================= +CREATE TABLE IF NOT EXISTS asynch_model_stat ( + day DATE NOT NULL, -- 天（YYYY-MM-DD） + tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID + creator VARCHAR(64) NOT NULL DEFAULT '', -- 创建人 + model_name VARCHAR(128) NOT NULL DEFAULT '', -- 模型名称 + request_count BIGINT NOT NULL DEFAULT 0, -- 请求次数 + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(day, tenant_id, creator, model_name) +); + +-- 便于时间段/租户/人/模型过滤 +CREATE INDEX IF NOT EXISTS idx_asynch_model_stat_tenant_day ON asynch_model_stat(tenant_id, day); +CREATE INDEX IF NOT EXISTS idx_asynch_model_stat_day ON asynch_model_stat(day); +CREATE INDEX IF NOT EXISTS idx_asynch_model_stat_model_name ON asynch_model_stat(model_name); +CREATE INDEX IF NOT EXISTS idx_asynch_model_stat_creator ON asynch_model_stat(creator); + +COMMENT ON TABLE asynch_model_stat IS '按天模型请求统计（用于限流/监控）'; +COMMENT ON COLUMN asynch_model_stat.day IS '天（YYYY-MM-DD）'; +COMMENT ON COLUMN asynch_model_stat.tenant_id IS '租户ID'; +COMMENT ON COLUMN asynch_model_stat.creator IS '创建人'; +COMMENT ON COLUMN asynch_model_stat.model_name IS '模型名称'; +COMMENT ON COLUMN asynch_model_stat.request_count IS '请求次数'; +COMMENT ON COLUMN asynch_model_stat.created_at IS '创建时间'; +COMMENT ON COLUMN asynch_model_stat.updated_at IS '更新时间';