数据引擎-快手平台数据抽取bug修复

This commit is contained in:
2026-06-16 10:44:10 +08:00
parent e5133eea34
commit b4fc6f54af
22 changed files with 1324 additions and 487 deletions

View File

@@ -7,8 +7,8 @@ import (
dto "dataengine/model/dto/dict"
entity "dataengine/model/entity/dict"
"dataengine/utils"
"time"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
"github.com/olekukonko/errors"
)
@@ -20,8 +20,10 @@ var DatasourcePlatform = new(datasourcePlatformService)
// Create 创建数据源平台
func (s *datasourcePlatformService) Create(ctx context.Context, req *dto.CreateDatasourcePlatformReq) (res *dto.CreateDatasourcePlatformRes, err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台编码是否重复
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode)
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode, tenantId)
if err != nil {
return nil, err
}
@@ -38,7 +40,7 @@ func (s *datasourcePlatformService) Create(ctx context.Context, req *dto.CreateD
currentUser := utils.GetCurrentUser(ctx)
// 插入数据库
id, err := dao.DatasourcePlatform.Insert(ctx, req, currentUser)
id, err := dao.DatasourcePlatform.Insert(ctx, req, currentUser, tenantId)
if err != nil {
return nil, err
}
@@ -51,7 +53,8 @@ func (s *datasourcePlatformService) Create(ctx context.Context, req *dto.CreateD
// List 获取数据源平台列表
func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatasourcePlatformReq) (res *dto.ListDatasourcePlatformRes, err error) {
platformList, total, err := dao.DatasourcePlatform.List(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
platformList, total, err := dao.DatasourcePlatform.List(ctx, req, tenantId)
if err != nil {
return nil, err
}
@@ -60,7 +63,8 @@ func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatas
list := make([]dto.DatasourcePlatformItem, 0, len(platformList))
for _, item := range platformList {
list = append(list, dto.DatasourcePlatformItem{
Id: item.ID,
Id: item.Id,
TenantId: item.TenantId,
PlatformCode: item.PlatformCode,
PlatformName: item.PlatformName,
Description: item.Description,
@@ -75,9 +79,9 @@ func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatas
RequestTimeoutMs: item.RequestTimeoutMs,
MaxRetries: item.MaxRetries,
RetryDelayMs: item.RetryDelayMs,
CreatedBy: item.CreatedBy,
CreatedBy: item.Creator,
CreatedAt: s.safeUnix(item.CreatedAt),
UpdatedBy: item.UpdatedBy,
UpdatedBy: item.Updater,
UpdatedAt: s.safeUnix(item.UpdatedAt),
})
}
@@ -91,7 +95,8 @@ func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatas
// GetOne 获取单个数据源平台
func (s *datasourcePlatformService) GetOne(ctx context.Context, req *dto.GetDatasourcePlatformReq) (res *dto.GetDatasourcePlatformRes, err error) {
platform, err := dao.DatasourcePlatform.GetOne(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
platform, err := dao.DatasourcePlatform.GetOne(ctx, req, tenantId)
if err != nil {
return nil, err
}
@@ -104,10 +109,9 @@ func (s *datasourcePlatformService) GetOne(ctx context.Context, req *dto.GetData
return nil, err
}
// 隐藏敏感信息
platformEntity.Token = ""
platformEntity.ClientSecret = ""
platformEntity.ApiKey = ""
// 注意:编辑时需要保留 Token/ClientSecret/ApiKey否则前端表单回填为空
// 用户保存时会覆盖已有凭据。敏感字段在 List 接口的 DatasourcePlatformItem 中
// 已不返回DTO 不含这些字段),此处保持完整。
return &dto.GetDatasourcePlatformRes{
DatasourcePlatform: platformEntity,
@@ -116,7 +120,8 @@ func (s *datasourcePlatformService) GetOne(ctx context.Context, req *dto.GetData
// GetByPlatformCode 根据平台编码获取数据源平台
func (s *datasourcePlatformService) GetByPlatformCode(ctx context.Context, platformCode string) (res *entity.DatasourcePlatform, err error) {
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode)
tenantId := utils.GetCurrentTenantId(ctx)
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode, tenantId)
if err != nil {
return nil, err
}
@@ -128,8 +133,10 @@ func (s *datasourcePlatformService) GetByPlatformCode(ctx context.Context, platf
// Update 更新数据源平台
func (s *datasourcePlatformService) Update(ctx context.Context, req *dto.UpdateDatasourcePlatformReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台是否存在
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id})
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id}, tenantId)
if err != nil {
return err
}
@@ -139,7 +146,7 @@ func (s *datasourcePlatformService) Update(ctx context.Context, req *dto.UpdateD
// 如果修改了平台编码,检查新编码是否重复
if req.PlatformCode != "" && req.PlatformCode != exist.PlatformCode {
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode, req.Id)
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode, tenantId, req.Id)
if err != nil {
return err
}
@@ -164,14 +171,16 @@ func (s *datasourcePlatformService) Update(ctx context.Context, req *dto.UpdateD
// 从 context 中获取当前用户
currentUser := utils.GetCurrentUser(ctx)
_, err = dao.DatasourcePlatform.Update(ctx, req, currentUser)
_, err = dao.DatasourcePlatform.Update(ctx, req, currentUser, tenantId)
return err
}
// UpdateStatus 更新数据源平台状态
func (s *datasourcePlatformService) UpdateStatus(ctx context.Context, req *dto.UpdateDatasourcePlatformStatusReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台是否存在
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id})
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id}, tenantId)
if err != nil {
return err
}
@@ -185,14 +194,16 @@ func (s *datasourcePlatformService) UpdateStatus(ctx context.Context, req *dto.U
}
currentUser := utils.GetCurrentUser(ctx)
_, err = dao.DatasourcePlatform.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser)
_, err = dao.DatasourcePlatform.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser, tenantId)
return err
}
// Delete 删除数据源平台
func (s *datasourcePlatformService) Delete(ctx context.Context, req *dto.DeleteDatasourcePlatformReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台是否存在
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id})
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id}, tenantId)
if err != nil {
return err
}
@@ -203,13 +214,14 @@ func (s *datasourcePlatformService) Delete(ctx context.Context, req *dto.DeleteD
// TODO: 检查是否存在关联的数据,防止误删
// 例如:检查该平台是否有关联的接口配置等
_, err = dao.DatasourcePlatform.Delete(ctx, req)
_, err = dao.DatasourcePlatform.Delete(ctx, req, tenantId)
return err
}
// GetStatistics 获取平台统计信息
func (s *datasourcePlatformService) GetStatistics(ctx context.Context) (res *dto.GetPlatformStatisticsRes, err error) {
stats, err := dao.DatasourcePlatform.GetPlatformStatistics(ctx)
tenantId := utils.GetCurrentTenantId(ctx)
stats, err := dao.DatasourcePlatform.GetPlatformStatistics(ctx, tenantId)
if err != nil {
return nil, err
}
@@ -228,7 +240,8 @@ func (s *datasourcePlatformService) GetStatistics(ctx context.Context) (res *dto
// ListActivePlatforms 获取所有启用的平台
func (s *datasourcePlatformService) ListActivePlatforms(ctx context.Context) (platforms []entity.DatasourcePlatform, err error) {
return dao.DatasourcePlatform.ListActivePlatforms(ctx)
tenantId := utils.GetCurrentTenantId(ctx)
return dao.DatasourcePlatform.ListActivePlatforms(ctx, tenantId)
}
// validateAuthFields 验证认证类型相关的必填字段
@@ -292,8 +305,8 @@ func (s *datasourcePlatformService) getAuthTypeName(authType string) string {
return authType
}
// safeUnix 安全地从 *time.Time 获取 Unix 时间戳nil 返回 0
func (s *datasourcePlatformService) safeUnix(t *time.Time) int64 {
// safeUnix 安全地从 *gtime.Time 获取 Unix 时间戳nil 返回 0
func (s *datasourcePlatformService) safeUnix(t *gtime.Time) int64 {
if t == nil {
return 0
}
@@ -301,11 +314,13 @@ func (s *datasourcePlatformService) safeUnix(t *time.Time) int64 {
}
// BatchUpdateStatus 批量更新平台状态
func (s *datasourcePlatformService) BatchUpdateStatus(ctx context.Context, ids []int64, status string, updatedBy string) (err error) {
func (s *datasourcePlatformService) BatchUpdateStatus(ctx context.Context, ids []int64, status string) (err error) {
if len(ids) == 0 {
return errors.New("请选择要更新的平台")
}
_, err = dao.DatasourcePlatform.BatchUpdateStatus(ctx, ids, status, updatedBy)
tenantId := utils.GetCurrentTenantId(ctx)
currentUser := utils.GetCurrentUser(ctx)
_, err = dao.DatasourcePlatform.BatchUpdateStatus(ctx, ids, status, currentUser, tenantId)
return err
}

View File

@@ -19,6 +19,8 @@ var ApiInterface = new(apiInterfaceService)
// Create 创建接口
func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInterfaceReq) (res *dto.CreateApiInterfaceRes, err error) {
tenantId := utils.GetCurrentTenantId(ctx)
_, err = DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.PlatformId})
if err != nil {
return nil, errors.New("平台不存在")
@@ -28,7 +30,7 @@ func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInte
interfaces, _, err := dict.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: req.PlatformId,
Code: req.Code,
})
}, tenantId)
if err != nil {
return
}
@@ -40,7 +42,7 @@ func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInte
currentUser := utils.GetCurrentUser(ctx)
// 插入数据库
id, err := dict.ApiInterface.Insert(ctx, req, currentUser)
id, err := dict.ApiInterface.Insert(ctx, req, currentUser, tenantId)
if err != nil {
return
}
@@ -52,7 +54,8 @@ func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInte
// List 获取接口列表
func (s *apiInterfaceService) List(ctx context.Context, req *dto.ListApiInterfaceReq) (res *dto.ListApiInterfaceRes, err error) {
apiList, total, err := dict.ApiInterface.List(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
apiList, total, err := dict.ApiInterface.List(ctx, req, tenantId)
if err != nil {
return
}
@@ -107,7 +110,8 @@ func (s *apiInterfaceService) List(ctx context.Context, req *dto.ListApiInterfac
// GetOne 获取单个接口
func (s *apiInterfaceService) GetOne(ctx context.Context, req *dto.GetApiInterfaceReq) (res *dto.GetApiInterfaceRes, err error) {
apiInterface, err := dict.ApiInterface.GetOne(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
apiInterface, err := dict.ApiInterface.GetOne(ctx, req, tenantId)
if err != nil {
return
}
@@ -128,7 +132,9 @@ func (s *apiInterfaceService) GetOne(ctx context.Context, req *dto.GetApiInterfa
// Update 更新接口
func (s *apiInterfaceService) Update(ctx context.Context, req *dto.UpdateApiInterfaceReq) (err error) {
exist, err := dict.ApiInterface.GetOne(ctx, &dto.GetApiInterfaceReq{Id: req.Id})
tenantId := utils.GetCurrentTenantId(ctx)
exist, err := dict.ApiInterface.GetOne(ctx, &dto.GetApiInterfaceReq{Id: req.Id}, tenantId)
if err != nil || exist == nil {
return errors.New("接口不存在")
}
@@ -148,7 +154,7 @@ func (s *apiInterfaceService) Update(ctx context.Context, req *dto.UpdateApiInte
interfaces, _, err := dict.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: platformId,
Code: req.Code,
})
}, tenantId)
if err != nil {
return err
}
@@ -158,26 +164,29 @@ func (s *apiInterfaceService) Update(ctx context.Context, req *dto.UpdateApiInte
}
currentUser := utils.GetCurrentUser(ctx)
_, err = dict.ApiInterface.Update(ctx, req, currentUser)
_, err = dict.ApiInterface.Update(ctx, req, currentUser, tenantId)
return
}
// UpdateStatus 更新接口状态
func (s *apiInterfaceService) UpdateStatus(ctx context.Context, req *dto.UpdateApiInterfaceStatusReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
currentUser := utils.GetCurrentUser(ctx)
_, err = dict.ApiInterface.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser)
_, err = dict.ApiInterface.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser, tenantId)
return
}
// Delete 删除接口
func (s *apiInterfaceService) Delete(ctx context.Context, req *dto.DeleteApiInterfaceReq) (err error) {
_, err = dict.ApiInterface.Delete(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
_, err = dict.ApiInterface.Delete(ctx, req, tenantId)
return
}
// GetByIds 根据ID列表获取接口
func (s *apiInterfaceService) GetByIds(ctx context.Context, ids []int64) (res []entity.ApiInterface, err error) {
return dict.ApiInterface.GetByIds(ctx, ids)
tenantId := utils.GetCurrentTenantId(ctx)
return dict.ApiInterface.GetByIds(ctx, ids, tenantId)
}
// getStatusName 获取状态名称

View File

@@ -152,7 +152,7 @@ func (s *publicQueryService) GetTableList(ctx context.Context) (*public.TableLis
_ = gfdb.DB(ctx).Model(ctx, "api_datasource_platform").Scan(&platforms)
platformMap := make(map[int64]string)
for _, p := range platforms {
platformMap[p.ID] = p.PlatformName
platformMap[p.Id] = p.PlatformName
}
var list []public.TableInfo

View File

@@ -78,13 +78,7 @@ func (c *ApiClient) Close() {
// Request 通用请求方法(支持 GET/POST支持参数在 query 或 body
func (c *ApiClient) Request(ctx context.Context, method, path string, params map[string]interface{}, paramsInQuery bool) (*ApiResult, error) {
if paramsInQuery {
return c.doRequest(ctx, method, path, params, true)
}
if method == "GET" {
return c.doRequest(ctx, "GET", path, params, true)
}
return c.doRequest(ctx, method, path, params, false)
return c.doRequest(ctx, method, path, params, paramsInQuery)
}
func (c *ApiClient) doRequest(ctx context.Context, method, path string, body interface{}, paramsInQuery bool) (result *ApiResult, err error) {
@@ -124,30 +118,76 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
start := time.Now()
fullURL := c.config.GetApiUrl(path)
// 先注入认证参数
// 先注入认证参数到 URL
fullURL = c.applyAuthURL(fullURL)
// 将 URL 认证参数注入 body 并清除 URL避免重复参数
var reqBody io.Reader
var reqBodyBytes []byte
if body != nil && !paramsInQuery {
b, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("JSON序列化请求体失败: %w", err)
if paramsMap, ok := body.(map[string]interface{}); ok {
// 从 URL 注入认证参数到 body
if parsed, _ := url.Parse(fullURL); parsed != nil {
q := parsed.Query()
for k, vs := range q {
if len(vs) > 0 {
if _, exists := paramsMap[k]; !exists {
paramsMap[k] = vs[0]
}
q.Del(k)
}
}
parsed.RawQuery = q.Encode()
fullURL = parsed.String()
}
// Form body
formStr := c.buildFormBody(paramsMap)
reqBodyBytes = []byte(formStr)
reqBody = strings.NewReader(formStr)
} else {
b, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("JSON序列化请求体失败: %w", err)
}
reqBodyBytes = b
reqBody = bytes.NewBuffer(b)
}
reqBodyBytes = b
reqBody = bytes.NewBuffer(b)
}
// 如果参数在查询字符串中,拼接到 URL
// GET query 模式
if body != nil && paramsInQuery {
if paramsMap, ok := body.(map[string]interface{}); ok {
fullURL = c.buildQueryURL(fullURL, paramsMap)
}
}
// 计算签名并追加(如快手 API 的 MD5 签名)
// 计算固定签名
fullURL = c.applySignature(fullURL, body, paramsInQuery)
logrus.Infof("请求 URL: %s", fullURL)
// 将 sign 注入 body 并从 URL 清除
if !paramsInQuery && reqBodyBytes != nil {
if parsed, _ := url.Parse(fullURL); parsed != nil {
if signVal := parsed.Query().Get("sign"); signVal != "" {
reqBodyBytes = append(reqBodyBytes, []byte("&sign="+signVal)...)
reqBody = bytes.NewReader(reqBodyBytes)
q := parsed.Query()
q.Del("sign")
parsed.RawQuery = q.Encode()
fullURL = parsed.String()
}
}
}
// 打印等效 curl
curlCmd := fmt.Sprintf("curl -X %s '%s'", method, fullURL)
if reqBodyBytes != nil && len(reqBodyBytes) > 0 {
for _, pair := range strings.Split(string(reqBodyBytes), "&") {
if pair != "" {
curlCmd += fmt.Sprintf(" --data-urlencode '%s'", pair)
}
}
}
logrus.Infof("等效curl: %s", curlCmd)
req, err := http.NewRequestWithContext(ctx, method, fullURL, reqBody)
if err != nil {
@@ -157,7 +197,11 @@ func (c *ApiClient) execute(ctx context.Context, method, path string, body inter
c.applyAuthHeader(req, reqBodyBytes)
req.Header.Set("User-Agent", "data-engine/1.0")
if body != nil && !paramsInQuery {
req.Header.Set("Content-Type", "application/json")
if _, ok := body.(map[string]interface{}); ok {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
} else {
req.Header.Set("Content-Type", "application/json")
}
}
resp, err := c.client.Do(req)
@@ -224,6 +268,28 @@ func (c *ApiClient) buildQueryURL(rawURL string, params map[string]interface{})
return parsed.String()
}
// buildFormBody 将 params 编码为 application/x-www-form-urlencoded 字符串
func (c *ApiClient) buildFormBody(params map[string]interface{}) string {
q := make(url.Values)
for k, v := range params {
switch val := v.(type) {
case string:
q.Set(k, val)
case float64:
if val == float64(int64(val)) {
q.Set(k, fmt.Sprintf("%d", int64(val)))
} else {
q.Set(k, fmt.Sprintf("%v", val))
}
case int, int8, int16, int32, int64:
q.Set(k, fmt.Sprintf("%d", val))
default:
q.Set(k, fmt.Sprintf("%v", v))
}
}
return q.Encode()
}
func (c *ApiClient) applyAuthURL(rawURL string) string {
cfg := c.config.AuthConfig
token := c.config.AccessToken
@@ -264,6 +330,10 @@ func (c *ApiClient) applyAuthURL(rawURL string) string {
for k, v := range extraParams {
q.Set(k, v)
}
// 注入 appkey
if appKey, ok := cfg["app_key"].(string); ok && appKey != "" {
q.Set("appkey", appKey)
}
parsed.RawQuery = q.Encode()
return parsed.String()
}
@@ -379,7 +449,8 @@ func generateNonce() string {
return fmt.Sprintf("%012d%04d", nanoPart, r.Int64())
}
// applySignature 计算签名并追加到 URL(支持快手等平台的 MD5 签名)
// applySignature 计算签名并追加到 URL
// 快手签名: 字母序拼接 key=value&...&signSecret=<secret>, 取 MD5
func (c *ApiClient) applySignature(rawURL string, body interface{}, paramsInQuery bool) string {
cfg := c.config.AuthConfig
if cfg == nil {
@@ -390,11 +461,16 @@ func (c *ApiClient) applySignature(rawURL string, body interface{}, paramsInQuer
if signAlgo == "" {
return rawURL
}
appSecret, _ := cfg["app_secret"].(string)
if appSecret == "" && c.config.AppSecret != "" {
appSecret = c.config.AppSecret
// 获取 signSecret签名专用密钥
signSecret, _ := cfg["sign_secret"].(string)
if signSecret == "" {
signSecret, _ = cfg["app_secret"].(string)
}
if appSecret == "" {
if signSecret == "" && c.config.AppSecret != "" {
signSecret = c.config.AppSecret
}
if signSecret == "" {
return rawURL
}
@@ -405,7 +481,16 @@ func (c *ApiClient) applySignature(rawURL string, body interface{}, paramsInQuer
}
q := parsed.Query()
// 收集所有参数并按 key 排序
// POST: 合并 body 参数
if !paramsInQuery {
if bodyMap, ok := body.(map[string]interface{}); ok {
for k, v := range bodyMap {
q.Set(k, fmt.Sprintf("%v", v))
}
}
}
// 收集参数(排除 sign按 key 排序
keys := make([]string, 0, len(q))
for k := range q {
if k == "sign" {
@@ -415,12 +500,20 @@ func (c *ApiClient) applySignature(rawURL string, body interface{}, paramsInQuer
}
sort.Strings(keys)
// 拼接: key1=value1&key2=value2&...
var signStr string
for _, k := range keys {
signStr += k + "=" + q.Get(k) + "&"
for i, k := range keys {
if i > 0 {
signStr += "&"
}
signStr += k + "=" + q.Get(k)
}
signStr += "key=" + appSecret
// 追加 signSecret
signStr += "&signSecret=" + signSecret
logrus.Infof("签名原文: %s", signStr)
// 计算签名
var sign string
switch signAlgo {
case "md5":
@@ -432,6 +525,7 @@ func (c *ApiClient) applySignature(rawURL string, body interface{}, paramsInQuer
default:
return rawURL
}
logrus.Infof("签名值 sign=%s", sign)
q.Set("sign", sign)
parsed.RawQuery = q.Encode()

View File

@@ -132,13 +132,65 @@ func paramsInQuery(iface *entity.ApiInterface) bool {
return false
}
// pageResult 单页处理结果
type pageResult struct {
raw []byte // 原始响应体hasMore 分页需用于判断是否还有下一页)
rows []map[string]interface{}
maxTime int64
nextCursor string
inserted int
err error
}
// processPage 处理单页数据:请求 → 解析 → 注入
func processPage(ctx context.Context, api *ApiClient, iface *entity.ApiInterface,
method string, inQuery bool, page, pageSize int, lastSyncTime int64,
extraParams map[string]interface{}) pageResult {
body := buildReqBody(ctx, iface, page, pageSize, lastSyncTime, extraParams)
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
return pageResult{err: fmt.Errorf("请求失败: %w", err)}
}
rows, _, maxTime, nextCursor, err := parseRespExt(resp.Body, iface.ResponseConfig)
if err != nil {
return pageResult{err: fmt.Errorf("解析失败: %w", err)}
}
injectRowFields(rows, body, iface.RequestConfig)
return pageResult{
raw: resp.Body,
rows: rows,
maxTime: maxTime,
nextCursor: nextCursor,
}
}
// accumPage 将单页结果累加到 SyncResult
func accumPage(result *SyncResult, maxTime *int64, td *TableDefinition,
ctx context.Context, pr pageResult) {
if len(pr.rows) == 0 {
return
}
inserted, _ := savePage(ctx, td, pr.rows)
result.InsertedRows += inserted
result.TotalRows += len(pr.rows)
if pr.maxTime > *maxTime {
*maxTime = pr.maxTime
}
result.TotalPages++
time.Sleep(100 * time.Millisecond)
}
// syncSingleAPI 单接口分页同步
func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig, iface *entity.ApiInterface, td *TableDefinition, isFullSync bool, lastSyncTime int64, start time.Time) (*SyncResult, error) {
pageSize := GetSyncPageSize(ctx)
if ps, ok := iface.RequestConfig["page_size"].(float64); ok {
pageSize = int(ps)
} else if ps, ok := iface.RequestConfig["pageSize"].(float64); ok {
pageSize = int(ps)
if ps, ok := toInt(iface.RequestConfig["page_size"]); ok {
pageSize = ps
} else if ps, ok := toInt(iface.RequestConfig["pageSize"]); ok {
pageSize = ps
}
taskType := "incremental"
@@ -149,159 +201,123 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
inQuery := paramsInQuery(iface)
method := string(iface.Method)
// 游标分页首次请求需要处理初始游标值
firstExtra := map[string]interface{}{}
if isCursorPagination(iface) {
cp := "cursor"
if p, ok := iface.RequestConfig["page_param"].(string); ok && p != "" {
cp = p
}
// 支持 initial_cursor 配置如钉钉HRM首次传 0
if icv, ok := iface.RequestConfig["initial_cursor"]; ok {
firstExtra[cp] = icv
} else {
firstExtra[cp] = ""
}
}
body := buildReqBody(iface, 1, pageSize, lastSyncTime, firstExtra)
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, err.Error())
return nil, fmt.Errorf("获取第一页失败: %w", err)
// 游标参数名
cursorParam := "cursor"
if p, ok := iface.RequestConfig["page_param"].(string); ok && p != "" {
cursorParam = p
}
cursorMode := isCursorPagination(iface)
rows, totalPages, maxTime, nextCursor, err := parseRespExt(resp.Body, iface.ResponseConfig)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("解析第一页响应失败: %v", err))
return nil, err
}
// 检测 range 模式快手时间分片按7天分片循环拉取
timeMode, _ := iface.RequestConfig["time_field_mode"].(string)
isRangeMode := timeMode == "range"
injectRowFields(rows, body, iface.RequestConfig)
result := &SyncResult{TableName: td.TableName, TotalPages: totalPages}
inserted, _ := savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
// 游标分页
if isCursorPagination(iface) {
for nextCursor != "" && nextCursor != "nomore" {
cp := "cursor"
if p, ok := iface.RequestConfig["page_param"].(string); ok && p != "" {
cp = p
}
body := buildReqBody(iface, 1, pageSize, lastSyncTime, map[string]interface{}{
cp: nextCursor,
})
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("游标 %s 请求失败: %v", nextCursor, err)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("游标 %s 请求失败: %v", nextCursor, err))
break
}
rows, _, mt, nc, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf("游标 %s 解析失败: %v", nextCursor, pe)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("游标 %s 解析失败: %v", nextCursor, pe))
break
}
if len(rows) == 0 {
break
}
nextCursor = nc
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ = savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
if mt > maxTime {
maxTime = mt
}
result.TotalPages++
time.Sleep(100 * time.Millisecond)
// 计算时间分片range 模式 + 全量同步)
var timeChunks [][2]int64
if isRangeMode && lastSyncTime <= 0 {
chunkMs := int64(3 * 24 * 3600 * 1000) // 3天
startMs := time.Now().Add(-time.Duration(GetDefaultLookbackDays(ctx)) * 24 * time.Hour).UnixMilli()
if fst, ok := toFloat64(iface.RequestConfig["full_sync_start_time"]); ok && fst > 0 {
startMs = int64(fst)
}
} else if iface.ResponseConfig != nil {
// hasMore 分页(如钉钉 offset/size + hasMore
if hf, _ := iface.ResponseConfig["has_more_field"].(string); hf != "" {
for page := 2; hasMoreCheck(resp.Body, hf); page++ {
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
resp2, e2 := api.Request(ctx, method, iface.Url, body, inQuery)
if e2 != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, e2)
break
}
rows2, _, mt2, _, pe2 := parseRespExt(resp2.Body, iface.ResponseConfig)
if pe2 != nil {
logrus.Errorf("第 %d 页解析失败: %v", page, pe2)
break
}
injectRowFields(rows2, body, iface.RequestConfig)
inserted2, _ := savePage(ctx, td, rows2)
result.InsertedRows += inserted2
result.TotalRows += len(rows2)
if mt2 > maxTime {
maxTime = mt2
}
resp = resp2
time.Sleep(100 * time.Millisecond)
}
} else {
// 普通分页
for page := 2; page <= totalPages; page++ {
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
resp, err = api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, err)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页请求失败: %v", page, err))
continue
}
rows, _, mt, _, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf("第 %d 页解析失败: %v", page, pe)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页解析失败: %v", page, pe))
continue
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ = savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
if mt > maxTime {
maxTime = mt
}
time.Sleep(100 * time.Millisecond)
endMs := time.Now().UnixMilli()
for t := startMs; t < endMs; t += chunkMs {
chunkEnd := t + chunkMs
if chunkEnd > endMs {
chunkEnd = endMs
}
timeChunks = append(timeChunks, [2]int64{t, chunkEnd})
}
logrus.Infof("时间分片模式: %d 个分片(每片7天)", len(timeChunks))
} else {
// 普通分页(无 response_config
for page := 2; page <= totalPages; page++ {
body := buildReqBody(iface, page, pageSize, lastSyncTime, nil)
resp, err = api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, err)
continue
}
rows, _, mt, _, pe := parseRespExt(resp.Body, iface.ResponseConfig)
if pe != nil {
logrus.Errorf("第 %d 页解析失败: %v", page, pe)
continue
}
injectRowFields(rows, body, iface.RequestConfig)
inserted, _ = savePage(ctx, td, rows)
result.InsertedRows += inserted
result.TotalRows += len(rows)
if mt > maxTime {
maxTime = mt
}
time.Sleep(100 * time.Millisecond)
}
timeChunks = [][2]int64{{0, 0}} // 单次,时间由 buildReqBody 决定
}
if maxTime <= 0 {
maxTime = time.Now().Unix()
result := &SyncResult{TableName: td.TableName}
globalMaxTime := int64(0)
for _, chunk := range timeChunks {
if isRangeMode && chunk[0] > 0 {
lastSyncTime = chunk[0]
}
// 构建第一页的额外参数
firstExtra := map[string]interface{}{}
if cursorMode {
if icv, ok := iface.RequestConfig["initial_cursor"]; ok {
firstExtra[cursorParam] = icv
} else {
firstExtra[cursorParam] = ""
}
}
// 处理第一页
pr := processPage(ctx, api, iface, method, inQuery, 1, pageSize, lastSyncTime, firstExtra)
if pr.err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, pr.err.Error())
return nil, fmt.Errorf("获取第一页失败: %w", pr.err)
}
if len(pr.rows) == 0 {
return &SyncResult{TableName: td.TableName, Duration: fmt.Sprintf("%.1fs", time.Since(start).Seconds())}, nil
}
// result initialized above in time chunk loop
maxTime := pr.maxTime
accumPage(result, &maxTime, td, ctx, pr)
// 分页循环:三种模式仅循环控制和参数不同,内核复用 processPage + accumPage
switch {
case cursorMode:
nextCursor := pr.nextCursor
for nextCursor != "" && nextCursor != "nomore" {
pr := processPage(ctx, api, iface, method, inQuery, 1, pageSize, lastSyncTime, map[string]interface{}{
cursorParam: nextCursor,
})
if pr.err != nil {
logrus.Errorf("游标 %s 处理失败: %v", nextCursor, pr.err)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("游标 %s 失败: %v", nextCursor, pr.err))
break
}
if len(pr.rows) == 0 {
break
}
nextCursor = pr.nextCursor
accumPage(result, &maxTime, td, ctx, pr)
}
case iface.ResponseConfig != nil && iface.ResponseConfig["has_more_field"] != nil:
// hasMore 分页(如钉钉 offset/size + hasMore用上一页的 raw body 判断
hf, _ := iface.ResponseConfig["has_more_field"].(string)
lastRaw := pr.raw
for page := 2; hasMoreCheck(lastRaw, hf); page++ {
pr2 := processPage(ctx, api, iface, method, inQuery, page, pageSize, lastSyncTime, nil)
if pr2.err != nil {
logrus.Errorf("第 %d 页处理失败: %v", page, pr2.err)
break
}
accumPage(result, &maxTime, td, ctx, pr2)
lastRaw = pr2.raw
}
default:
// 普通分页:第一页 parseRespExt 返回 totalPagesparseRespExt 第4个返回值被忽略需要从别处获取
totalPages := getTotalPages(pr.raw)
for page := 2; page <= totalPages; page++ {
pr := processPage(ctx, api, iface, method, inQuery, page, pageSize, lastSyncTime, nil)
if pr.err != nil {
logrus.Errorf("第 %d 页请求失败: %v", page, pr.err)
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("第 %d 页失败: %v", page, pr.err))
continue
}
accumPage(result, &maxTime, td, ctx, pr)
}
}
if globalMaxTime <= 0 {
globalMaxTime = time.Now().Unix()
}
}
updateSyncTime(ctx, platform.PlatformCode, iface.Code, maxTime)
updateSyncTime(ctx, platform.PlatformCode, iface.Code, globalMaxTime)
result.Duration = fmt.Sprintf("%.1fs", time.Since(start).Seconds())
logrus.Infof("同步完成 - 表:%s, %d条, 写入%d条, 耗时%s", td.TableName, result.TotalRows, result.InsertedRows, result.Duration)
@@ -350,7 +366,7 @@ func collectPrefetchEntities(rows []map[string]interface{}, prefetch *PrefetchCo
if prefetch.ValueField == "" {
*allEntities = append(*allEntities, item)
} else if v, ok := item[prefetch.ValueField]; ok {
if f, ok := v.(float64); ok {
if f, ok := toFloat64(v); ok {
*allEntities = append(*allEntities, int64(f))
} else {
*allEntities = append(*allEntities, v)
@@ -392,8 +408,8 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
prefetchMethod := strings.ToUpper(prefetch.Method)
prefetchPageSize := 100
if prefetchIface != nil && prefetchIface.RequestConfig != nil {
if ps, ok := prefetchIface.RequestConfig["pageSize"].(float64); ok {
prefetchPageSize = int(ps)
if ps, ok := toInt(prefetchIface.RequestConfig["pageSize"]); ok {
prefetchPageSize = ps
}
}
@@ -409,6 +425,9 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
var prefetchRespCfg map[string]interface{}
if prefetchIface != nil {
prefetchRespCfg = prefetchIface.ResponseConfig
logrus.Debugf("预取接口配置: code=%s, success_field=%v, success_value=%v", prefetchIface.Code, prefetchRespCfg["success_field"], prefetchRespCfg["success_value"])
} else {
logrus.Warnf("未找到预取接口配置: URL=%s将使用默认配置", prefetch.URL)
}
allEntities := make([]interface{}, 0)
@@ -422,8 +441,8 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
if prefetchIface != nil && prefetchRecursiveCfg != nil {
// ----- 递归遍历预取(如钉钉部门树)-----
maxDepth := 20
if md, ok := prefetchIface.RequestConfig["max_recursive_depth"].(float64); ok {
maxDepth = int(md)
if md, ok := toInt(prefetchIface.RequestConfig["max_recursive_depth"]); ok {
maxDepth = md
}
processedKeys := make(map[string]bool)
type rItem struct {
@@ -449,7 +468,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
if item.keyVal != nil {
extra[prefetchRecursiveCfg.TargetParam] = item.keyVal
}
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, 0, extra)
body := buildReqBody(ctx, prefetchReqIface, 1, prefetchPageSize, 0, extra)
r2, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
logrus.Errorf("预取递归 [depth=%d] 请求失败: %v", item.depth, err)
@@ -465,7 +484,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
if prefetch.ValueField == "" {
allEntities = append(allEntities, row)
} else if v, ok := row[prefetch.ValueField]; ok {
if f, ok := v.(float64); ok {
if f, ok := toFloat64(v); ok {
allEntities = append(allEntities, int64(f))
} else {
allEntities = append(allEntities, v)
@@ -481,9 +500,15 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
// ----- 常规分页预取 -----
firstExtra := make(map[string]interface{})
if prefetchIsCursor {
firstExtra[prefetchPageParam] = ""
// 支持 initial_cursor 配置,如果没有则使用空字符串
if icv, ok := prefetchReqIface.RequestConfig["initial_cursor"]; ok {
firstExtra[prefetchPageParam] = icv
} else {
firstExtra[prefetchPageParam] = ""
}
}
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, lastSyncTime, firstExtra)
body := buildReqBody(ctx, prefetchReqIface, 1, prefetchPageSize, lastSyncTime, firstExtra)
logrus.Debugf("预取请求 URL: %s, Method: %s, Body: %+v", prefetch.URL, prefetchMethod, body)
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
recordFailure(ctx, platform.PlatformCode, iface.Code, taskType, fmt.Sprintf("预取第一页请求失败: %v", err))
@@ -499,7 +524,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
if prefetchIsCursor {
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody(prefetchReqIface, 1, prefetchPageSize, lastSyncTime, map[string]interface{}{
body := buildReqBody(ctx, prefetchReqIface, 1, prefetchPageSize, lastSyncTime, map[string]interface{}{
prefetchPageParam: nextCursor,
})
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
@@ -521,7 +546,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
}
} else {
for page := 2; page <= prefetchTotalPages; page++ {
body := buildReqBody(prefetchReqIface, page, prefetchPageSize, lastSyncTime, nil)
body := buildReqBody(ctx, prefetchReqIface, page, prefetchPageSize, lastSyncTime, nil)
resp, err := api.Request(ctx, prefetchMethod, prefetch.URL, body, prefetchInQuery)
if err != nil {
logrus.Errorf("预取第 %d 页请求失败: %v", page, err)
@@ -558,10 +583,10 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
// 并发处理每个实体的数据
result := &SyncResult{TableName: td.TableName}
pageSize := GetSyncPageSize(ctx)
if ps, ok := iface.RequestConfig["page_size"].(float64); ok {
pageSize = int(ps)
} else if ps, ok := iface.RequestConfig["pageSize"].(float64); ok {
pageSize = int(ps)
if ps, ok := toInt(iface.RequestConfig["page_size"]); ok {
pageSize = ps
} else if ps, ok := toInt(iface.RequestConfig["pageSize"]); ok {
pageSize = ps
}
dataMethod := string(iface.Method)
@@ -598,7 +623,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
} else {
firstExtra[cp] = ""
}
body := buildReqBody(iface, 1, pageSize, lastSyncTime, firstExtra)
body := buildReqBody(ctx, iface, 1, pageSize, lastSyncTime, firstExtra)
resp, err := api.Request(ctx, dataMethod, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf(" 实体 %v 首次请求失败: %v", val, err)
@@ -623,7 +648,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
}
nextCursor := nc
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody(iface, 1, pageSize, lastSyncTime, map[string]interface{}{
body := buildReqBody(ctx, iface, 1, pageSize, lastSyncTime, map[string]interface{}{
cp: nextCursor,
prefetch.TargetParam: val,
})
@@ -660,7 +685,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
page := 1
totalPages := 1
for page <= totalPages {
body := buildReqBody(iface, page, pageSize, lastSyncTime, map[string]interface{}{
body := buildReqBody(ctx, iface, page, pageSize, lastSyncTime, map[string]interface{}{
prefetch.TargetParam: val,
})
resp, err := api.Request(ctx, dataMethod, iface.Url, body, inQuery)
@@ -721,8 +746,8 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
// syncRecursive 递归遍历同步(如钉钉部门树:先查根级 → 对每个子部门递归查下级)
func syncRecursive(ctx context.Context, api *ApiClient, platform *PlatformConfig, iface *entity.ApiInterface, td *TableDefinition, recursive *RecursiveConfig, start time.Time) (*SyncResult, error) {
maxDepth := 20
if md, ok := iface.RequestConfig["max_recursive_depth"].(float64); ok {
maxDepth = int(md)
if md, ok := toInt(iface.RequestConfig["max_recursive_depth"]); ok {
maxDepth = md
}
inQuery := paramsInQuery(iface)
@@ -760,7 +785,7 @@ func syncRecursive(ctx context.Context, api *ApiClient, platform *PlatformConfig
extraParams[recursive.TargetParam] = item.keyVal
}
body := buildReqBody(iface, 1, 100, 0, extraParams)
body := buildReqBody(ctx, iface, 1, 100, 0, extraParams)
resp, err := api.Request(ctx, method, iface.Url, body, inQuery)
if err != nil {
logrus.Errorf("递归 [depth=%d] 请求失败: %v", item.depth, err)
@@ -819,6 +844,12 @@ func toFloat64(v interface{}) (float64, bool) {
return float64(val), true
case int64:
return float64(val), true
case json.Number:
f, err := val.Float64()
if err != nil {
return 0, false
}
return f, true
case string:
// 支持字符串类型的成功值(如钉钉智能薪酬返回 code: "200"
if f, err := strconv.ParseFloat(val, 64); err == nil {
@@ -830,6 +861,32 @@ func toFloat64(v interface{}) (float64, bool) {
}
}
// toInt 从 interface{} 安全转换为 int支持 float64/int/int64/json.Number/string 类型
// JSONB 字段通过 pgx 驱动扫描时可能返回 json.Number 而非 float64
func toInt(v interface{}) (int, bool) {
switch val := v.(type) {
case float64:
return int(val), true
case int:
return val, true
case int64:
return int(val), true
case json.Number:
i, err := val.Int64()
if err != nil {
return 0, false
}
return int(i), true
case string:
if i, err := strconv.Atoi(val); err == nil {
return i, true
}
return 0, false
default:
return 0, false
}
}
// buildPrefetchParams 构建预取接口的请求参数
func buildPrefetchParams(iface *entity.ApiInterface) map[string]interface{} {
params := make(map[string]interface{})
@@ -960,8 +1017,39 @@ func extractValues(raw []byte, path, valueField string) ([]interface{}, error) {
return nil, fmt.Errorf("路径 %s 不完整", path)
}
// normalizeJSONNumbers 递归将 json.Number 转换为 Go 原生数值类型
// pgx 驱动扫描 PostgreSQL JSONB 数值字段时可能返回 json.Number底层是 string
// 直接用 json.Marshal 会将其序列化为带引号的字符串(如 "0" 而非 0
// 导致快手等平台的 param JSON 签名校验失败
func normalizeJSONNumbers(v interface{}) interface{} {
switch val := v.(type) {
case json.Number:
if i, err := val.Int64(); err == nil {
return i
}
if f, err := val.Float64(); err == nil {
return f
}
return val.String()
case map[string]interface{}:
result := make(map[string]interface{}, len(val))
for mk, mv := range val {
result[mk] = normalizeJSONNumbers(mv)
}
return result
case []interface{}:
result := make([]interface{}, len(val))
for i, item := range val {
result[i] = normalizeJSONNumbers(item)
}
return result
default:
return v
}
}
// buildReqBody 构建请求参数
func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime int64, extraParams map[string]interface{}) map[string]interface{} {
func buildReqBody(ctx context.Context, iface *entity.ApiInterface, page, pageSize int, lastSyncTime int64, extraParams map[string]interface{}) map[string]interface{} {
body := make(map[string]interface{})
if iface.RequestConfig != nil {
for k, v := range iface.RequestConfig {
@@ -1017,15 +1105,23 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
timeMs := lastSyncTime
if timeMs <= 0 {
// 全量:优先使用配置的 full_sync_start_time否则默认90天前
if fst, ok := iface.RequestConfig["full_sync_start_time"].(float64); ok && fst > 0 {
if fst, ok := toFloat64(iface.RequestConfig["full_sync_start_time"]); ok && fst > 0 {
timeMs = int64(fst)
} else {
timeMs = time.Now().Add(-90 * 24 * time.Hour).UnixMilli()
timeMs = time.Now().Add(-time.Duration(GetDefaultLookbackDays(ctx)) * 24 * time.Hour).UnixMilli()
}
}
body["queryType"] = 2
// 仅在配置未指定 queryType 时设默认值,尊重配置
if _, exists := body["queryType"]; !exists {
body["queryType"] = 2
}
body["beginTime"] = timeMs
body["endTime"] = time.Now().UnixMilli()
endTime := time.Now().UnixMilli()
maxRangeMs := int64(3 * 24 * 3600 * 1000)
if endTime-timeMs > maxRangeMs {
endTime = timeMs + maxRangeMs
}
body["endTime"] = endTime
} else if lastSyncTime > 0 {
// 腾讯 filtering 模式(仅增量时)
timeFilter := map[string]interface{}{
@@ -1038,7 +1134,7 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
} else {
body["filtering"] = []interface{}{timeFilter}
}
} else if fst, ok := iface.RequestConfig["full_sync_start_time"].(float64); ok && fst > 0 {
} else if fst, ok := toFloat64(iface.RequestConfig["full_sync_start_time"]); ok && fst > 0 {
// 全量 filtering 模式:指定了 full_sync_start_time从该时间戳开始拉取
timeFilter := map[string]interface{}{
"field": tf,
@@ -1076,7 +1172,11 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
delete(body, k)
}
}
b, err := json.Marshal(wrapperObj)
// 规范化 json.Number → Go 原生数值类型,避免 json.Marshal 将其序列化为字符串
// pgx 驱动扫描 JSONB 数值时可能返回 json.Number其底层是 string 类型)
normalized := normalizeJSONNumbers(wrapperObj)
b, err := json.Marshal(normalized)
logrus.Infof("body_wrapper param JSON (normalized): %s", string(b))
if err != nil {
logrus.Errorf("JSON序列化 wrapper 失败: %v", err)
} else {
@@ -1127,6 +1227,17 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
actual, _ := toFloat64(v)
if actual != successVal {
msg, _ := respMap[msgField].(string)
// 如果配置的消息字段为空,尝试通用的错误信息字段
if msg == "" {
for _, altField := range []string{"error_msg", "sub_msg", "msg"} {
if altMsg, _ := respMap[altField].(string); altMsg != "" {
msg = altMsg
break
}
}
}
respJSON, _ := json.Marshal(respMap)
logrus.Errorf("API响应校验失败: success_field=%s, expected=%v, actual=%v, message=%s, 完整响应: %s", successField, successVal, actual, msg, string(respJSON))
return nil, 0, 0, "", fmt.Errorf("API错误: %s=%v, %s=%s", successField, v, msgField, msg)
}
}
@@ -1202,7 +1313,7 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
j, _ := json.Marshal(m)
flat["raw_data"] = string(j)
for _, tf := range []string{"last_modified_time", "created_time", "update_time", "createTime", "updateTime", "lastModifiedTime"} {
if t, ok := flat[tf].(float64); ok && int64(t) > maxTime {
if t, ok := toFloat64(flat[tf]); ok && int64(t) > maxTime {
maxTime = int64(t)
}
}
@@ -1217,7 +1328,7 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
if i == len(cp)-1 {
if s, ok := cc[p].(string); ok {
nextCursor = s
} else if f, ok := cc[p].(float64); ok {
} else if f, ok := toFloat64(cc[p]); ok {
// 数字游标(如钉钉 next_cursor=10
nextCursor = fmt.Sprintf("%.0f", f)
}
@@ -1241,8 +1352,8 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
}
}
if pi, ok := dataContainer["page_info"].(map[string]interface{}); ok {
if tp, ok := pi["total_page"].(float64); ok {
totalPages = int(tp)
if tp, ok := toInt(pi["total_page"]); ok {
totalPages = tp
}
}
return rows, totalPages, maxTime, nextCursor, nil

View File

@@ -50,3 +50,12 @@ func GetSyncTimeout(ctx context.Context) int {
}
return t
}
// GetDefaultLookbackDays 获取全量同步默认回溯天数默认90
func GetDefaultLookbackDays(ctx context.Context) int {
d := g.Cfg().MustGet(ctx, "sync.default_lookback_days", 90).Int()
if d < 1 {
return 90
}
return d
}

View File

@@ -7,6 +7,7 @@ import (
dao "dataengine/dao/dict"
dto "dataengine/model/dto/dict"
entity "dataengine/model/entity/dict"
"dataengine/utils"
"github.com/sirupsen/logrus"
)
@@ -32,7 +33,8 @@ type PlatformManager struct{}
// GetPlatform 根据平台编码获取配置
func (m *PlatformManager) GetPlatform(ctx context.Context, platformCode string) (*PlatformConfig, error) {
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode)
tenantId := utils.GetCurrentTenantId(ctx)
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode, tenantId)
if err != nil {
return nil, fmt.Errorf("查询平台配置失败 [%s]: %w", platformCode, err)
}
@@ -81,10 +83,11 @@ func (m *PlatformManager) GetPlatform(ctx context.Context, platformCode string)
// GetInterfaces 获取平台下的活跃接口列表
func (m *PlatformManager) GetInterfaces(ctx context.Context, platformId int64) ([]entity.ApiInterface, error) {
tenantId := utils.GetCurrentTenantId(ctx)
interfaces, _, err := dao.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: platformId,
Status: "active",
})
}, tenantId)
if err != nil {
return nil, err
}
@@ -93,11 +96,12 @@ func (m *PlatformManager) GetInterfaces(ctx context.Context, platformId int64) (
// GetInterfaceByCode 根据编码获取接口定义
func (m *PlatformManager) GetInterfaceByCode(ctx context.Context, platformId int64, code string) (*entity.ApiInterface, error) {
tenantId := utils.GetCurrentTenantId(ctx)
all, _, err := dao.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: platformId,
Code: code,
Status: "active",
})
}, tenantId)
if err != nil {
return nil, err
}
@@ -113,7 +117,7 @@ func (m *PlatformManager) GetPlatformWithInterfaces(ctx context.Context, platfor
if err != nil {
return nil, nil, err
}
interfaces, err := m.GetInterfaces(ctx, cfg.ID)
interfaces, err := m.GetInterfaces(ctx, cfg.Id)
if err != nil {
return nil, nil, err
}

View File

@@ -27,13 +27,16 @@ func StartAutoSync(ctx context.Context) {
func runAutoSync(ctx context.Context) {
logrus.Info("=== 开始自动同步 ===")
// 从配置读取同步租户 ID运维部署时配置
tenantId := g.Cfg().MustGet(ctx, "sync.default_tenant_id", 1).Uint64()
// 注入用户上下文ORM 框架需要用于租户隔离)
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin", TenantId: 1})
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin", TenantId: tenantId})
// 查询所有 ACTIVE 平台
platforms, _, err := dao.DatasourcePlatform.List(ctx, &dto.ListDatasourcePlatformReq{
Status: "ACTIVE",
})
}, tenantId)
if err != nil {
logrus.Errorf("查询平台列表失败: %v", err)
return
@@ -42,9 +45,9 @@ func runAutoSync(ctx context.Context) {
for _, p := range platforms {
// 查询该平台下有 table_definition 的接口
interfaces, _, err := dao.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: p.ID,
PlatformId: p.Id,
Status: "active",
})
}, tenantId)
if err != nil {
logrus.Errorf("查询接口列表失败 [platform=%s]: %v", p.PlatformCode, err)
continue