@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"time"
@@ -13,7 +14,7 @@ import (
taskDto "dataengine/model/dto/copydata"
entity "dataengine/model/entity/dict"
"gitea.com/red-future/common/db/gfdb"
"gitea.redpowerfuture. com/red-future/common/db/gfdb"
"github.com/sirupsen/logrus"
)
@@ -38,8 +39,20 @@ type PrefetchConfig struct {
ValueField string ` json:"value_field" `
}
// RecursiveConfig 递归遍历配置(如钉钉部门树)
type RecursiveConfig struct {
KeyField string ` json:"key_field" `
TargetParam string ` json:"target_param" `
}
// SyncByConfig 执行同步
func SyncByConfig ( ctx context . Context , platformCode , interfaceCode string , isFullSync bool ) ( * SyncResult , error ) {
// 创建超时 context 防止单次同步卡死
timeoutMin := GetSyncTimeout ( ctx )
timeoutCtx , cancel := context . WithTimeout ( ctx , time . Duration ( timeoutMin ) * time . Minute )
defer cancel ( )
ctx = timeoutCtx
// 内存锁:防止同一个接口被并发执行(两个调度周期重叠时跳过)
lockKey := platformCode + "/" + interfaceCode
if _ , loaded := syncRunningMap . LoadOrStore ( lockKey , true ) ; loaded {
@@ -93,11 +106,16 @@ func SyncByConfig(ctx context.Context, platformCode, interfaceCode string, isFul
markSyncRunning ( ctx , platformCode , interfaceCode , lastSyncTime )
api := NewApiClient ( platform )
defer api . Close ( )
prefetch := parsePrefetchConfig ( iface . RequestConfig )
if prefetch != nil {
return syncWithPrefetch ( ctx , api , platform , iface , ifaces , td , prefetch , isFullSync , lastSyncTime , start )
}
recursive := parseRecursiveConfig ( iface . RequestConfig )
if recursive != nil {
return syncRecursive ( ctx , api , platform , iface , td , recursive , start )
}
return syncSingleAPI ( ctx , api , platform , iface , td , isFullSync , lastSyncTime , start )
}
@@ -119,6 +137,8 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
pageSize := GetSyncPageSize ( ctx )
if ps , ok := iface . RequestConfig [ "page_size" ] . ( float64 ) ; ok {
pageSize = int ( ps )
} else if ps , ok := iface . RequestConfig [ "pageSize" ] . ( float64 ) ; ok {
pageSize = int ( ps )
}
taskType := "incremental"
@@ -129,14 +149,19 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
inQuery := paramsInQuery ( iface )
method := string ( iface . Method )
// 游标分页首次请求需要 cursor=""(通过 extraParams 覆盖 buildReqBody 的 page=1 赋值)
// 游标分页首次请求需要处理初始游标值
firstExtra := map [ string ] interface { } { }
if isCursorPagination ( iface ) {
cp := "cursor"
if p , ok := iface . RequestConfig [ "page_param" ] . ( string ) ; ok && p != "" {
cp = p
}
firstExtra [ cp ] = ""
// 支持 initial_cursor 配置( 如钉钉HRM首次传 0)
if icv , ok := iface . RequestConfig [ "initial_cursor" ] ; ok {
firstExtra [ cp ] = icv
} else {
firstExtra [ cp ] = ""
}
}
body := buildReqBody ( iface , 1 , pageSize , lastSyncTime , firstExtra )
resp , err := api . Request ( ctx , method , iface . Url , body , inQuery )
@@ -151,6 +176,8 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
return nil , err
}
injectRowFields ( rows , body , iface . RequestConfig )
result := & SyncResult { TableName : td . TableName , TotalPages : totalPages }
inserted , _ := savePage ( ctx , td , rows )
result . InsertedRows += inserted
@@ -185,6 +212,7 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
}
nextCursor = nc
injectRowFields ( rows , body , iface . RequestConfig )
inserted , _ = savePage ( ctx , td , rows )
result . InsertedRows += inserted
result . TotalRows += len ( rows )
@@ -194,22 +222,72 @@ func syncSingleAPI(ctx context.Context, api *ApiClient, platform *PlatformConfig
result . TotalPages ++
time . Sleep ( 100 * time . Millisecond )
}
} else if iface . ResponseConfig != nil {
// hasMore 分页(如钉钉 offset/size + hasMore)
if hf , _ := iface . ResponseConfig [ "has_more_field" ] . ( string ) ; hf != "" {
for page := 2 ; hasMoreCheck ( resp . Body , hf ) ; page ++ {
body := buildReqBody ( iface , page , pageSize , lastSyncTime , nil )
resp2 , e2 := api . Request ( ctx , method , iface . Url , body , inQuery )
if e2 != nil {
logrus . Errorf ( "第 %d 页请求失败: %v" , page , e2 )
break
}
rows2 , _ , mt2 , _ , pe2 := parseRespExt ( resp2 . Body , iface . ResponseConfig )
if pe2 != nil {
logrus . Errorf ( "第 %d 页解析失败: %v" , page , pe2 )
break
}
injectRowFields ( rows2 , body , iface . RequestConfig )
inserted2 , _ := savePage ( ctx , td , rows2 )
result . InsertedRows += inserted2
result . TotalRows += len ( rows2 )
if mt2 > maxTime {
maxTime = mt2
}
resp = resp2
time . Sleep ( 100 * time . Millisecond )
}
} else {
// 普通分页
for page := 2 ; page <= totalPages ; page ++ {
body := buildReqBody ( iface , page , pageSize , lastSyncTime , nil )
resp , err = api . Request ( ctx , method , iface . Url , body , inQuery )
if err != nil {
logrus . Errorf ( "第 %d 页请求失败: %v" , page , err )
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "第 %d 页请求失败: %v" , page , err ) )
continue
}
rows , _ , mt , _ , pe := parseRespExt ( resp . Body , iface . ResponseConfig )
if pe != nil {
logrus . Errorf ( "第 %d 页解析失败: %v" , page , pe )
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "第 %d 页解析失败: %v" , page , pe ) )
continue
}
injectRowFields ( rows , body , iface . RequestConfig )
inserted , _ = savePage ( ctx , td , rows )
result . InsertedRows += inserted
result . TotalRows += len ( rows )
if mt > maxTime {
maxTime = mt
}
time . Sleep ( 100 * time . Millisecond )
}
}
} else {
// 普通分页
// 普通分页(无 response_config)
for page := 2 ; page <= totalPages ; page ++ {
body := buildReqBody ( iface , page , pageSize , lastSyncTime , nil )
resp , err : = api . Request ( ctx , method , iface . Url , body , inQuery )
resp , err = api . Request ( ctx , method , iface . Url , body , inQuery )
if err != nil {
logrus . Errorf ( "第 %d 页请求失败: %v" , page , err )
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "第 %d 页请求失败: %v" , page , err ) )
continue
}
rows , _ , mt , _ , pe := parseRespExt ( resp . Body , iface . ResponseConfig )
if pe != nil {
logrus . Errorf ( "第 %d 页解析失败: %v" , page , pe )
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "第 %d 页解析失败: %v" , page , pe ) )
continue
}
injectRowFields ( rows , body , iface . RequestConfig )
inserted , _ = savePage ( ctx , td , rows )
result . InsertedRows += inserted
result . TotalRows += len ( rows )
@@ -238,6 +316,33 @@ func isCursorPagination(iface *entity.ApiInterface) bool {
return cp
}
// hasMoreCheck 从响应体中提取 has_more_field 的值
func hasMoreCheck ( raw [ ] byte , hasMorePath string ) bool {
var respMap map [ string ] interface { }
if err := json . Unmarshal ( raw , & respMap ) ; err != nil {
return false
}
parts := strings . Split ( hasMorePath , "." )
cc := respMap
for i , p := range parts {
if i == len ( parts ) - 1 {
if b , ok := cc [ p ] . ( bool ) ; ok {
return b
}
if s , ok := cc [ p ] . ( string ) ; ok {
return s == "true"
}
return false
}
if m , ok := cc [ p ] . ( map [ string ] interface { } ) ; ok {
cc = m
} else {
return false
}
}
return false
}
// collectPrefetchEntities 从 rows 中收集实体和行数据
func collectPrefetchEntities ( rows [ ] map [ string ] interface { } , prefetch * PrefetchConfig , allEntities * [ ] interface { } , allRows * [ ] map [ string ] interface { } ) {
for _ , item := range rows {
@@ -266,6 +371,12 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
// ====== 1. 预取阶段:分页拉取全部实体列表 ======
prefetchIface := findInterfaceByURL ( allIfaces , prefetch . URL )
// 判断预取来源是否有递归配置(如钉钉部门树)
var prefetchRecursiveCfg * RecursiveConfig
if prefetchIface != nil {
prefetchRecursiveCfg = parseRecursiveConfig ( prefetchIface . RequestConfig )
}
// 判断预取来源是否游标分页,以及分页参数名
prefetchIsCursor := false
prefetchPageParam := "page"
@@ -303,69 +414,127 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
allEntities := make ( [ ] interface { } , 0 )
allRows := make ( [ ] map [ string ] interface { } , 0 )
// 第一页(游标分页首次 cursor="")
firstExtra := make ( map [ string ] interface { } )
if prefetchIsCursor {
firstExtra [ prefetchPageParam ] = ""
}
prefetchReqIface := prefetchIface
if prefetchReqIface == nil {
prefetchReqIface = iface
}
body := buildReqBody ( prefetchReqIface , 1 , prefetchPageSize , lastSyncTime , firstExtra )
resp , err := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
if err != nil {
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "预取第一页请求失败: %v" , err ) )
return nil , fmt . Errorf ( "预取第一页失败: %w" , err )
}
rows , prefetchTotalPages , _ , nextCursor , err := parseRespExt ( resp . Body , prefetchRespCfg )
if err != nil {
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "解析预取响应失败: %v" , err ) )
return nil , fmt . Errorf ( "解析预取响应失败: %w" , err )
}
collectPrefetchEntities ( rows , prefetch , & allEntities , & allRows )
if prefetchIface != nil && prefetchRecursiveCfg != nil {
// ----- 递归遍历预取(如钉钉部门树)-----
maxDepth := 20
if md , ok := prefetchIface . RequestConfig [ "max_recursive_depth" ] . ( float64 ) ; ok {
maxDepth = int ( md )
}
processedKeys := make ( map [ string ] bool )
type rItem struct {
depth int
keyVal interface { }
}
queue := [ ] rItem { { depth : 0 , keyVal : nil } }
// 分页循环
if prefetchIsCursor {
// 游标分页
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody ( prefetchReqIface , 1 , prefetchPageSize , lastSyncTime , map [ string ] interface { } {
prefetchPageParam : nextCursor ,
} )
resp , er r := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
for len ( queue ) > 0 {
item := queue [ 0 ]
queue = queue [ 1 : ]
i f item . depth > maxDepth {
continue
}
if item . keyVal != nil {
keySt r := fmt . Sprintf ( "%v" , item . keyVal )
if processedKeys [ keyStr ] {
continue
}
processedKeys [ keyStr ] = true
}
extra := make ( map [ string ] interface { } )
if item . keyVal != nil {
extra [ prefetchRecursiveCfg . TargetParam ] = item . keyVal
}
body := buildReqBody ( prefetchReqIface , 1 , prefetchPageSize , 0 , extra )
r2 , err := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
if err != nil {
logrus . Errorf ( "预取游标 %s 请求失败: %v" , nextCursor , err )
break
logrus . Errorf ( "预取递归 [depth=%d] 请求失败: %v" , item . depth , err )
continue
}
r ows, _ , _ , nc , pe := parseRespExt ( resp . Body , prefetchRespCfg )
itemR ows, _ , _ , _ , pe := parseRespExt ( r2 . Body , prefetchRespCfg )
if pe != nil {
logrus . Errorf ( "预取游标 %s 解析失败: %v" , nextCursor , pe )
break
logrus . Errorf ( "预取递归 [depth=%d] 解析失败: %v" , item . depth , pe )
continue
}
i f len ( rows ) == 0 {
break
for _ , row := range itemRows {
allRows = append ( allRows , row )
if prefetch . ValueField == "" {
allEntities = append ( allEntities , row )
} else if v , ok := row [ prefetch . ValueField ] ; ok {
if f , ok := v . ( float64 ) ; ok {
allEntities = append ( allEntities , int64 ( f ) )
} else {
allEntities = append ( allEntities , v )
}
}
if v , ok := row [ prefetchRecursiveCfg . KeyField ] ; ok {
queue = append ( queue , rItem { depth : item . depth + 1 , keyVal : v } )
}
}
nextCursor = nc
collectPrefetchEntities ( rows , prefetch , & allEntities , & allRows )
time . Sleep ( 100 * time . Millisecond )
}
} else {
// 普通分页
for page := 2 ; page <= prefetchTotalPages ; page ++ {
body := buildReqBody ( prefetchReqIface , page , prefetchPageSize , lastSyncTime , nil )
resp , err := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
if err != nil {
logrus . Errorf ( "预取第 %d 页请求失败: %v" , page , err )
continue
// ----- 常规分页预取 -----
firstExtra := make ( map [ string ] interface { } )
if prefetchIsCursor {
firstExtra [ prefetchPageParam ] = ""
}
body := buildReqBody ( prefetchReqIface , 1 , prefetchPageSize , lastSyncTime , firstExtra )
resp , err := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
if err != nil {
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "预取第一页请求失败: %v" , err ) )
return nil , fmt . Errorf ( "预取第一页失败: %w" , err )
}
rows , prefetchTotalPages , _ , nextCursor , err := parseRespExt ( resp . Body , prefetchRespCfg )
if err != nil {
recordFailure ( ctx , platform . PlatformCode , iface . Code , taskType , fmt . Sprintf ( "解析预取响应失败: %v" , err ) )
return nil , fmt . Errorf ( "解析预取响应失败: %w" , err )
}
collectPrefetchEntities ( rows , prefetch , & allEntities , & allRows )
if prefetchIsCursor {
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody ( prefetchReqIface , 1 , prefetchPageSize , lastSyncTime , map [ string ] interface { } {
prefetchPageParam : nextCursor ,
} )
resp , err := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
if err != nil {
logrus . Errorf ( "预取游标 %s 请求失败: %v" , nextCursor , err )
break
}
rows , _ , _ , nc , pe := parseRespExt ( resp . Body , prefetchRespCfg )
if pe != nil {
logrus . Errorf ( "预取游标 %s 解析失败: %v" , nextCursor , pe )
break
}
if len ( rows ) == 0 {
break
}
nextCursor = nc
collectPrefetchEntities ( rows , prefetch , & allEntities , & allRows )
time . Sleep ( 100 * time . Millisecond )
}
rows , _ , _ , _ , pe := parseRespExt ( resp . Body , prefetchRespCfg )
i f pe != nil {
logrus . Errorf ( "预取第 %d 页解析失败: %v" , page , pe )
continue
} else {
for pag e := 2 ; page <= prefetchTotalPages ; page ++ {
body := buildReqBody ( prefetchReqIface , page , prefetchPageSize , lastSyncTime , nil )
resp , err := api . Request ( ctx , prefetchMethod , prefetch . URL , body , prefetchInQuery )
if err != nil {
logrus . Errorf ( "预取第 %d 页请求失败: %v" , page , err )
continue
}
rows , _ , _ , _ , pe := parseRespExt ( resp . Body , prefetchRespCfg )
if pe != nil {
logrus . Errorf ( "预取第 %d 页解析失败: %v" , page , pe )
continue
}
collectPrefetchEntities ( rows , prefetch , & allEntities , & allRows )
time . Sleep ( 100 * time . Millisecond )
}
collectPrefetchEntities ( rows , prefetch , & allEntities , & allRows )
time . Sleep ( 100 * time . Millisecond )
}
}
@@ -375,7 +544,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
}
logrus . Infof ( "预取到 %d 个实体" , len ( allEntities ) )
// 2. 将预取的数据也存入库(如账户列表存入 tencent_account_relation)
// 将预取的数据也存入库(如账户列表存入 tencent_account_relation)
if prefetchIface != nil && prefetchIface . TableDefinition != nil {
prefetchTd , err := ParseTableDefinition ( prefetchIface . TableDefinition )
if err == nil {
@@ -386,11 +555,13 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
}
}
// 2. 并发处理每个实体的数据
// 并发处理每个实体的数据
result := & SyncResult { TableName : td . TableName }
pageSize := GetSyncPageSize ( ctx )
if ps , ok := iface . RequestConfig [ "page_size" ] . ( float64 ) ; ok {
pageSize = int ( ps )
} else if ps , ok := iface . RequestConfig [ "pageSize" ] . ( float64 ) ; ok {
pageSize = int ( ps )
}
dataMethod := string ( iface . Method )
@@ -411,52 +582,118 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
defer func ( ) { <- sem } ( )
logrus . Infof ( " 处理实体 [%d/%d]: %v" , idx + 1 , len ( allEntities ) , val )
page := 1
totalPages := 1
entityMaxTime := int64 ( 0 )
for page <= totalPages {
body := buildReqBody ( iface , page , pageSize , lastSyncTime , map [ string ] interface { } {
i f isCursorPagination ( iface ) {
// ----- 游标分页(如钉钉 user_list) -----
cp := "cursor"
if p , ok := iface . RequestConfig [ "page_param" ] . ( string ) ; ok && p != "" {
cp = p
}
firstExtra := map [ string ] interface { } {
prefetch . TargetParam : val ,
} )
}
if icv , ok := iface . RequestConfig [ "initial_cursor" ] ; ok {
firstExtra [ cp ] = icv
} else {
firstExtra [ cp ] = ""
}
body := buildReqBody ( iface , 1 , pageSize , lastSyncTime , firstExtra )
resp , err := api . Request ( ctx , dataMethod , iface . Url , body , inQuery )
if err != nil {
logrus . Errorf ( " 实体 %v 第 %d 页 失败: %v" , val , page , err )
page ++
time . Sleep ( 200 * time . Millisecond )
continue
logrus . Errorf ( " 实体 %v 首次请求 失败: %v" , val , err )
return
}
rows , tp , mt , parseErr : = parseResp ( resp . Body , iface . ResponseConfig )
if parseErr != nil {
logrus . Errorf ( " 解析响应失败: %v" , parseErr )
page ++
continue
rows , _ , mt , nc , pe := parseRespExt ( resp . Body , iface . ResponseConfig )
if pe ! = nil {
logrus . Errorf ( " 实体 %v 解析首页失败: %v" , val , pe )
return
}
if page == 1 {
totalPages = tp
}
for i := range rows {
rows [ i ] [ prefetch . TargetParam ] = val
}
injectRowFields ( rows , body , iface . RequestConfig )
inserted , _ := savePage ( ctx , td , rows )
mu . Lock ( )
result . InsertedRows += inserted
result . TotalRows += len ( rows )
mu . Unlock ( )
if mt > entityMaxTime {
entityMaxTime = mt
}
page ++
time . Sleep ( 100 * time . Millisecond )
nextCursor := nc
for nextCursor != "" && nextCursor != "nomore" {
body := buildReqBody ( iface , 1 , pageSize , lastSyncTime , map [ string ] interface { } {
cp : nextCursor ,
prefetch . TargetParam : val ,
} )
resp , err := api . Request ( ctx , dataMethod , iface . Url , body , inQuery )
if err != nil {
logrus . Errorf ( " 实体 %v 游标 %s 失败: %v" , val , nextCursor , err )
break
}
rows , _ , mt , nc , pe := parseRespExt ( resp . Body , iface . ResponseConfig )
if pe != nil {
logrus . Errorf ( " 实体 %v 游标 %s 解析失败: %v" , val , nextCursor , pe )
break
}
if len ( rows ) == 0 {
break
}
nextCursor = nc
for i := range rows {
rows [ i ] [ prefetch . TargetParam ] = val
}
injectRowFields ( rows , body , iface . RequestConfig )
inserted , _ := savePage ( ctx , td , rows )
mu . Lock ( )
result . InsertedRows += inserted
result . TotalRows += len ( rows )
mu . Unlock ( )
if mt > entityMaxTime {
entityMaxTime = mt
}
time . Sleep ( 100 * time . Millisecond )
}
} else {
// ----- 普通分页 -----
page := 1
totalPages := 1
for page <= totalPages {
body := buildReqBody ( iface , page , pageSize , lastSyncTime , map [ string ] interface { } {
prefetch . TargetParam : val ,
} )
resp , err := api . Request ( ctx , dataMethod , iface . Url , body , inQuery )
if err != nil {
logrus . Errorf ( " 实体 %v 第 %d 页失败: %v" , val , page , err )
page ++
time . Sleep ( 200 * time . Millisecond )
continue
}
rows , tp , mt , parseErr := parseResp ( resp . Body , iface . ResponseConfig )
if parseErr != nil {
logrus . Errorf ( " 解析响应失败: %v" , parseErr )
page ++
continue
}
if page == 1 {
totalPages = tp
}
for i := range rows {
rows [ i ] [ prefetch . TargetParam ] = val
}
injectRowFields ( rows , body , iface . RequestConfig )
inserted , _ := savePage ( ctx , td , rows )
mu . Lock ( )
result . InsertedRows += inserted
result . TotalRows += len ( rows )
mu . Unlock ( )
if mt > entityMaxTime {
entityMaxTime = mt
}
page ++
time . Sleep ( 100 * time . Millisecond )
}
}
if entityMaxTime > 0 {
@@ -481,6 +718,90 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon
return result , nil
}
// syncRecursive 递归遍历同步(如钉钉部门树:先查根级 → 对每个子部门递归查下级)
func syncRecursive ( ctx context . Context , api * ApiClient , platform * PlatformConfig , iface * entity . ApiInterface , td * TableDefinition , recursive * RecursiveConfig , start time . Time ) ( * SyncResult , error ) {
maxDepth := 20
if md , ok := iface . RequestConfig [ "max_recursive_depth" ] . ( float64 ) ; ok {
maxDepth = int ( md )
}
inQuery := paramsInQuery ( iface )
method := string ( iface . Method )
allRows := make ( [ ] map [ string ] interface { } , 0 )
processedKeys := make ( map [ string ] bool )
type queueItem struct {
depth int
keyVal interface { } // nil 表示根级
}
queue := [ ] queueItem { { depth : 0 , keyVal : nil } }
for len ( queue ) > 0 {
item := queue [ 0 ]
queue = queue [ 1 : ]
if item . depth > maxDepth {
logrus . Warnf ( "递归已达最大深度 %d, 终止该分支" , maxDepth )
continue
}
// 防重复处理
if item . keyVal != nil {
keyStr := fmt . Sprintf ( "%v" , item . keyVal )
if processedKeys [ keyStr ] {
continue
}
processedKeys [ keyStr ] = true
}
extraParams := make ( map [ string ] interface { } )
if item . keyVal != nil {
extraParams [ recursive . TargetParam ] = item . keyVal
}
body := buildReqBody ( iface , 1 , 100 , 0 , extraParams )
resp , err := api . Request ( ctx , method , iface . Url , body , inQuery )
if err != nil {
logrus . Errorf ( "递归 [depth=%d] 请求失败: %v" , item . depth , err )
recordFailure ( ctx , platform . PlatformCode , iface . Code , "full" , fmt . Sprintf ( "递归深度 %d 请求失败: %v" , item . depth , err ) )
continue
}
rows , _ , _ , _ , err := parseRespExt ( resp . Body , iface . ResponseConfig )
if err != nil {
logrus . Errorf ( "递归 [depth=%d] 解析失败: %v" , item . depth , err )
continue
}
for _ , row := range rows {
allRows = append ( allRows , row )
if v , ok := row [ recursive . KeyField ] ; ok {
queue = append ( queue , queueItem { depth : item . depth + 1 , keyVal : v } )
}
}
time . Sleep ( 100 * time . Millisecond )
}
if len ( allRows ) == 0 {
logrus . Warn ( "递归结果为空,跳过入库" )
return & SyncResult { TableName : td . TableName , Duration : fmt . Sprintf ( "%.1fs" , time . Since ( start ) . Seconds ( ) ) } , nil
}
inserted , _ := savePage ( ctx , td , allRows )
updateSyncTime ( ctx , platform . PlatformCode , iface . Code , time . Now ( ) . Unix ( ) )
result := & SyncResult {
TableName : td . TableName ,
TotalRows : len ( allRows ) ,
InsertedRows : inserted ,
Duration : fmt . Sprintf ( "%.1fs" , time . Since ( start ) . Seconds ( ) ) ,
}
logrus . Infof ( "递归同步完成 - 表:%s, %d条, 写入%d条, 耗时%s" , td . TableName , result . TotalRows , result . InsertedRows , result . Duration )
return result , nil
}
// getTotalPages 从响应中提取总页数
func getTotalPages ( raw [ ] byte ) int {
rows , tp , _ , _ , err := parseRespExt ( raw , nil )
@@ -498,6 +819,12 @@ func toFloat64(v interface{}) (float64, bool) {
return float64 ( val ) , true
case int64 :
return float64 ( val ) , true
case string :
// 支持字符串类型的成功值(如钉钉智能薪酬返回 code: "200")
if f , err := strconv . ParseFloat ( val , 64 ) ; err == nil {
return f , true
}
return 0 , false
default :
return 0 , false
}
@@ -524,7 +851,10 @@ func buildPrefetchParams(iface *entity.ApiInterface) map[string]interface{} {
k == "page_size_param" || k == "time_field" || k == "parameters_location" ||
k == "filtering" || k == "group_by" || k == "date_range" ||
k == "body_wrapper_field" || k == "exclude_from_wrapper" ||
k == "cursor_pagination" || k == "time_field_mode" {
k == "cursor_pagination" || k == "time_field_mode" ||
k == "recursive" || k == "max_recursive_depth" ||
k == "initial_cursor" || k == "pagination_mode" ||
k == "full_sync_start_time" || k == "row_inject" {
continue
}
if k == pageParam || k == psParam {
@@ -567,6 +897,33 @@ func parsePrefetchConfig(requestConfig map[string]interface{}) *PrefetchConfig {
return pc
}
// parseRecursiveConfig 解析递归遍历配置
func parseRecursiveConfig ( requestConfig map [ string ] interface { } ) * RecursiveConfig {
if requestConfig == nil {
return nil
}
raw , ok := requestConfig [ "recursive" ]
if ! ok || raw == nil {
return nil
}
m , ok := raw . ( map [ string ] interface { } )
if ! ok {
return nil
}
rc := & RecursiveConfig { }
if kf , _ := m [ "key_field" ] . ( string ) ; kf != "" {
rc . KeyField = kf
} else {
return nil
}
if tp , _ := m [ "target_param" ] . ( string ) ; tp != "" {
rc . TargetParam = tp
} else {
return nil
}
return rc
}
// extractValues 从 JSON 响应中提取值列表
func extractValues ( raw [ ] byte , path , valueField string ) ( [ ] interface { } , error ) {
var resp map [ string ] interface { }
@@ -612,7 +969,10 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
k == "page_param" || k == "page_size_param" || k == "parameters_location" ||
k == "cursor_pagination" || k == "time_field_mode" ||
k == "body_wrapper_field" || k == "exclude_from_wrapper" ||
k == "top_level_params" {
k == "top_level_params" || k == "recursive" ||
k == "max_recursive_depth" || k == "initial_cursor" ||
k == "pagination_mode" || k == "full_sync_start_time" ||
k == "row_inject" {
continue
}
body [ k ] = v
@@ -628,39 +988,68 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
psParam = p
}
}
body [ pageParam ] = pag e
// 偏移量分页(如钉钉 offset) : offset = (page-1) * pageSiz e
paginationMode := ""
if iface . RequestConfig != nil {
if pm , ok := iface . RequestConfig [ "pagination_mode" ] . ( string ) ; ok {
paginationMode = pm
}
}
if paginationMode == "offset" {
body [ pageParam ] = ( page - 1 ) * pageSize
} else {
body [ pageParam ] = page
}
body [ psParam ] = pageSize
// 时间过滤处理:支持两种模式
// 1. "filtering" 模式(默认):生成 filtering=[{"field":"...","operator":"GREATER_EQUALS","values":["..."]}](腾讯)
// 2. "range" 模式:生成 beginTime/endTime + queryType( 快手)
if tf , ok := iface . RequestConfig [ "time_field" ] . ( string ) ; ok && tf != "" {
timeMode := "filtering"
if tm , ok := iface . RequestConfig [ "time_field_mode" ] . ( string ) ; ok && tm != "" {
timeMode = tm
}
if iface . RequestConfig != nil {
if tf , ok := iface . RequestConfig [ "time_field" ] . ( string ) ; ok && tf != "" {
timeMode := "filtering"
if tm , ok := iface . RequestConfig [ "time_field_mode" ] . ( string ) ; ok && tm != "" {
timeMode = tm
}
if timeMode == "range" {
// 快手模式: beginTime/endTime( 毫秒时间戳)
timeMs := lastSyncTime
if timeMs <= 0 {
// 全量: 默认90天前
timeMs = tim e. Now ( ) . Add ( - 90 * 24 * time . Hour ) . UnixMilli ( )
}
body [ "queryType" ] = 2
body [ "beginT ime" ] = timeMs
body [ "endTime" ] = time . Now ( ) . UnixMilli ( )
} else if lastSyncTime > 0 {
// 腾讯 filtering 模式(仅增量时)
timeFilter : = map [ string ] interface { } {
"field ": tf ,
"operator" : "GREATER_EQUALS" ,
"values" : [ ] interface { } { fmt . Sprintf ( "%d" , lastSyncTime ) } ,
}
if existing , ok := body [ "filtering" ] . ( [ ] interface { } ) ; ok {
body [ "filtering" ] = append ( existing , timeFilter )
} else {
body [ "filtering" ] = [ ] interface { } { timeFilter }
if timeMode == "range" {
// 快手模式: beginTime/endTime( 毫秒时间戳)
timeMs := lastSyncTime
if timeMs <= 0 {
// 全量:优先使用配置的 full_sync_start_time, 否则 默认90天前
if fst , ok : = ifac e. RequestConfig [ "full_sync_start_time" ] . ( float64 ) ; ok && fst > 0 {
timeMs = int64 ( fst )
} else {
timeMs = t ime . Now ( ) . Add ( - 90 * 24 * time . Hour ) . UnixMilli ( )
}
}
body [ "queryType" ] = 2
body [ "beginTime" ] = timeMs
body [ "endTime "] = time . Now ( ) . UnixMilli ( )
} else if lastSyncTime > 0 {
// 腾讯 filtering 模式(仅增量时)
timeFilter := map [ string ] interface { } {
"field" : tf ,
"operator" : "GREATER_EQUALS" ,
"values" : [ ] interface { } { fmt . Sprintf ( "%d" , lastSyncTime ) } ,
}
if existing , ok := body [ "filtering" ] . ( [ ] interface { } ) ; ok {
body [ "filtering" ] = append ( existing , timeFilter )
} else {
body [ "filtering" ] = [ ] interface { } { timeFilter }
}
} else if fst , ok := iface . RequestConfig [ "full_sync_start_time" ] . ( float64 ) ; ok && fst > 0 {
// 全量 filtering 模式:指定了 full_sync_start_time, 从该时间戳开始拉取
timeFilter := map [ string ] interface { } {
"field" : tf ,
"operator" : "GREATER_EQUALS" ,
"values" : [ ] interface { } { fmt . Sprintf ( "%d" , int64 ( fst ) ) } ,
}
if existing , ok := body [ "filtering" ] . ( [ ] interface { } ) ; ok {
body [ "filtering" ] = append ( existing , timeFilter )
} else {
body [ "filtering" ] = [ ] interface { } { timeFilter }
}
}
}
}
@@ -687,8 +1076,12 @@ func buildReqBody(iface *entity.ApiInterface, page, pageSize int, lastSyncTime i
delete ( body , k )
}
}
b , _ := json . Marshal ( wrapperObj )
body [ wf ] = string ( b )
b , err := json . Marshal ( wrapperObj )
if err ! = nil {
logrus . Errorf ( "JSON序列化 wrapper 失败: %v" , err )
} else {
body [ wf ] = string ( b )
}
}
}
@@ -703,6 +1096,7 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
}
successField , successVal := "code" , float64 ( 0 )
msgField , listPath , cursorPath := "message" , "data" , ""
hasMorePath := ""
singleRecord := false
if rc != nil {
if sf , _ := rc [ "success_field" ] . ( string ) ; sf != "" {
@@ -725,6 +1119,9 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
if sr , _ := rc [ "single_record" ] . ( bool ) ; sr {
singleRecord = true
}
if hm , _ := rc [ "has_more_field" ] . ( string ) ; hm != "" {
hasMorePath = hm
}
}
if v , ok := respMap [ successField ] ; ok {
actual , _ := toFloat64 ( v )
@@ -820,6 +1217,23 @@ func parseRespExt(raw []byte, rc map[string]interface{}) ([]map[string]interface
if i == len ( cp ) - 1 {
if s , ok := cc [ p ] . ( string ) ; ok {
nextCursor = s
} else if f , ok := cc [ p ] . ( float64 ) ; ok {
// 数字游标(如钉钉 next_cursor=10)
nextCursor = fmt . Sprintf ( "%.0f" , f )
}
} else if m , ok := cc [ p ] . ( map [ string ] interface { } ) ; ok {
cc = m
}
}
}
// has_more 字段支持: false 时标记游标结束
if hasMorePath != "" {
parts := strings . Split ( hasMorePath , "." )
cc := respMap
for i , p := range parts {
if i == len ( parts ) - 1 {
if b , ok := cc [ p ] . ( bool ) ; ok && ! b {
nextCursor = "nomore"
}
} else if m , ok := cc [ p ] . ( map [ string ] interface { } ) ; ok {
cc = m
@@ -950,3 +1364,32 @@ func findInterfaceByURL(ifaces []entity.ApiInterface, url string) *entity.ApiInt
}
return nil
}
// injectRowFields 将请求参数中 row_inject 指定的字段注入到响应行中
// 用于需要将请求参数(如 statisticsMonth) 持久化到表中, 但响应不含该字段的场景
func injectRowFields ( rows [ ] map [ string ] interface { } , body map [ string ] interface { } , requestConfig map [ string ] interface { } ) {
if requestConfig == nil || body == nil {
return
}
rawInject , ok := requestConfig [ "row_inject" ]
if ! ok {
return
}
injectList , ok := rawInject . ( [ ] interface { } )
if ! ok {
return
}
for _ , item := range injectList {
fieldName , ok := item . ( string )
if ! ok {
continue
}
val , exists := body [ fieldName ]
if ! exists {
continue
}
for i := range rows {
rows [ i ] [ fieldName ] = val
}
}
}