From 4f3ad39eeb3a247aee6adfa6e41d919e94c61dfd Mon Sep 17 00:00:00 2001 From: lmk <1095689763@qq.com> Date: Tue, 7 Apr 2026 09:51:32 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E6=8A=BD?= =?UTF-8?q?=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + scheduler/run_account_report_task.go | 39 ++ .../copydata/api_account_report_service.go | 4 +- sync/base_report_sync.go | 79 ++++ sync/campaign_report_sync.go | 115 +++++ sync/campaign_report_types.go | 235 ++++++++++ sync/data_converter.go | 415 ++++++++++++++++++ sync/http_client.go | 68 +++ sync/mock_generator.go | 272 ++++++++++++ sync/quick_sync.go | 51 +++ sync/sync_service.go | 268 +++++++++++ sync/sync_test.go | 139 ++++++ 12 files changed, 1684 insertions(+), 2 deletions(-) create mode 100644 scheduler/run_account_report_task.go create mode 100644 sync/base_report_sync.go create mode 100644 sync/campaign_report_sync.go create mode 100644 sync/campaign_report_types.go create mode 100644 sync/data_converter.go create mode 100644 sync/http_client.go create mode 100644 sync/mock_generator.go create mode 100644 sync/quick_sync.go create mode 100644 sync/sync_service.go create mode 100644 sync/sync_test.go diff --git a/go.mod b/go.mod index 4a05f03..7b67d66 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.5 github.com/gogf/gf/v2 v2.10.0 github.com/olekukonko/errors v1.1.0 + github.com/sirupsen/logrus v1.9.3 golang.org/x/net v0.47.0 ) diff --git a/scheduler/run_account_report_task.go b/scheduler/run_account_report_task.go new file mode 100644 index 0000000..f264515 --- /dev/null +++ b/scheduler/run_account_report_task.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "time" + + "cid/sync" + + _ "github.com/gogf/gf/contrib/drivers/pgsql/v2" + + "github.com/gogf/gf/v2/os/gctx" + "github.com/sirupsen/logrus" +) + +func main() { + ctx := gctx.New() + syncService := sync.NewSyncService() + + req := &sync.CampaignReportRequest{ + AdvertiserID: 10001, + StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6, + EndTime: time.Now().UnixNano() / 1e6, + SelectColumns: []string{"impression", "click", "cost", "t0GMV"}, + GroupType: 1, + QueryVersion: 1, + } + + logrus.Info("=== 开始执行定时同步任务 ===") + result, err := syncService.SyncCampaignReportWithPagination(ctx, req, true, 3) + if err != nil { + logrus.Errorf("定时同步任务失败：%v", err) + return + } + + fmt.Printf("✓ 定时同步完成：\n") + fmt.Printf(" 汇总数据：成功=%v, ID=%d\n", result.SumSuccess, result.SumID) + fmt.Printf(" 明细数据：总数=%d, 成功=%d, 失败=%d\n", + result.DetailCount, result.DetailSuccessCount, result.DetailFailCount) +} diff --git a/service/copydata/api_account_report_service.go b/service/copydata/api_account_report_service.go index 09ee563..cf76fb1 100644 --- a/service/copydata/api_account_report_service.go +++ b/service/copydata/api_account_report_service.go @@ -68,7 +68,7 @@ func (s *cidAccountReportDetailService) BatchCreate(ctx context.Context, req *dt return } -// Create 创建广告数据报表汇总 +// CreateSum Create 创建广告数据报表汇总 func (s *cidAccountReportDetailService) CreateSum(ctx context.Context, req *dto.CidAccountReportSumItem) (res *dto.CreateCidAccountReportSumRes, err error) { // 验证必要字段 if req.DataType == "" { @@ -90,7 +90,7 @@ func (s *cidAccountReportDetailService) CreateSum(ctx context.Context, req *dto. return } -// BatchCreate 批量创建广告数据报表汇总 +// BatchCreateSum 批量创建广告数据报表汇总 func (s *cidAccountReportDetailService) BatchCreateSum(ctx context.Context, req *dto.BatchCreateCidAccountReportSumReq) (res *dto.BatchCreateCidAccountReportSumRes, err error) { ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"}) // 验证数据 diff --git a/sync/base_report_sync.go b/sync/base_report_sync.go new file mode 100644 index 0000000..5234ddb --- /dev/null +++ b/sync/base_report_sync.go @@ -0,0 +1,79 @@ +package sync + +import ( + "context" +) + +type ReportSyncable interface { + FetchReport(ctx context.Context, params interface{}) (interface{}, error) + ConvertToSum(apiData interface{}, dataType string) interface{} + ConvertToDetails(apiData interface{}, dataType string) []interface{} + SaveSum(ctx context.Context, data interface{}) (int64, error) + SaveDetails(ctx context.Context, data []interface{}) (successCount, failCount int64, err error) +} + +type BaseReportSync struct { + httpClient *HttpClient +} + +func NewBaseReportSync() *BaseReportSync { + return &BaseReportSync{ + httpClient: NewHttpClient("", 0), + } +} + +func (b *BaseReportSync) FetchReport(ctx context.Context, params interface{}) (interface{}, error) { + return nil, nil +} + +func (b *BaseReportSync) ConvertToSum(apiData interface{}, dataType string) interface{} { + return nil +} + +func (b *BaseReportSync) ConvertToDetails(apiData interface{}, dataType string) []interface{} { + return nil +} + +func (b *BaseReportSync) SaveSum(ctx context.Context, data interface{}) (int64, error) { + return 0, nil +} + +func (b *BaseReportSync) SaveDetails(ctx context.Context, data []interface{}) (int64, int64, error) { + return 0, 0, nil +} + +func (b *BaseReportSync) ExecuteSync(ctx context.Context, syncer ReportSyncable, params interface{}, dataType string, useMock bool) (*SyncResult, error) { + result := &SyncResult{} + + apiData, err := syncer.FetchReport(ctx, params) + if err != nil { + result.Error = err + return result, err + } + + sumData := syncer.ConvertToSum(apiData, dataType) + if sumData != nil { + sumID, err := syncer.SaveSum(ctx, sumData) + if err != nil { + result.Error = err + return result, err + } + result.SumSuccess = true + result.SumID = sumID + } + + detailData := syncer.ConvertToDetails(apiData, dataType) + if len(detailData) > 0 { + successCount, failCount, err := syncer.SaveDetails(ctx, detailData) + if err != nil { + result.Error = err + return result, err + } + result.DetailSuccess = true + result.DetailCount = len(detailData) + result.DetailSuccessCount = successCount + result.DetailFailCount = failCount + } + + return result, nil +} diff --git a/sync/campaign_report_sync.go b/sync/campaign_report_sync.go new file mode 100644 index 0000000..69051bb --- /dev/null +++ b/sync/campaign_report_sync.go @@ -0,0 +1,115 @@ +package sync + +import ( + dto "cid/model/dto/copydata" + "cid/service/copydata" + "context" + "encoding/json" + "fmt" + + "github.com/sirupsen/logrus" +) + +type CampaignReportSync struct { + *BaseReportSync + converter *DataConverter + mockGen *MockDataGenerator +} + +func NewCampaignReportSync() *CampaignReportSync { + return &CampaignReportSync{ + BaseReportSync: NewBaseReportSync(), + converter: NewDataConverter(), + mockGen: NewMockDataGenerator(), + } +} + +func (c *CampaignReportSync) FetchReport(ctx context.Context, params interface{}) (interface{}, error) { + req, ok := params.(*CampaignReportRequest) + if !ok { + return nil, fmt.Errorf("参数类型错误，期望 CampaignReportRequest 类型") + } + + useMock := false + + if useMock { + logrus.Info("使用 Mock 数据") + return c.mockGen.GenerateCampaignReportResponse(), nil + } + + respBytes, err := NewHttpClient("https://ad.e.kuaishou.com", 0).Post(ctx, "/rest/openapi/gw/esp/report/campaignReport", req) + if err != nil { + return nil, fmt.Errorf("调用 API 失败：%w", err) + } + + var response CampaignReportResponse + if err := json.Unmarshal(respBytes, &response); err != nil { + return nil, fmt.Errorf("解析响应失败：%w", err) + } + + if response.Code != 0 { + return nil, fmt.Errorf("API 返回错误：code=%d, message=%s", response.Code, response.Message) + } + + return &response, nil +} + +func (c *CampaignReportSync) ConvertToSum(apiData interface{}, dataType string) interface{} { + response, ok := apiData.(*CampaignReportResponse) + if !ok || response.Data == nil || response.Data.Sum == nil { + return nil + } + + return c.converter.ConvertToSumItem(response.Data.Sum, dataType) +} + +func (c *CampaignReportSync) ConvertToDetails(apiData interface{}, dataType string) []interface{} { + response, ok := apiData.(*CampaignReportResponse) + if !ok || response.Data == nil || len(response.Data.Detail) == 0 { + return nil + } + + detailItems := c.converter.ConvertToDetailItems(response.Data.Detail, dataType) + + result := make([]interface{}, len(detailItems)) + for i, item := range detailItems { + result[i] = item + } + return result +} + +func (c *CampaignReportSync) SaveSum(ctx context.Context, data interface{}) (int64, error) { + sumItem, ok := data.(*dto.CidAccountReportSumItem) + if !ok { + return 0, fmt.Errorf("数据类型错误，期望 CidAccountReportSumItem 类型") + } + + res, err := copydata.CidAccountReportDetail.CreateSum(ctx, sumItem) + if err != nil { + return 0, err + } + + return res.Id, nil +} + +func (c *CampaignReportSync) SaveDetails(ctx context.Context, data []interface{}) (int64, int64, error) { + detailItems := make([]*dto.CidAccountReportDetailItem, len(data)) + for i, item := range data { + detailItem, ok := item.(*dto.CidAccountReportDetailItem) + if !ok { + return 0, 0, fmt.Errorf("第 %d 条数据类型错误", i) + } + detailItems[i] = detailItem + } + + req := &dto.BatchCreateCidAccountReportDetailReq{ + Items: detailItems, + } + + res, err := copydata.CidAccountReportDetail.BatchCreate(ctx, req) + if err != nil { + return 0, 0, err + } + + return res.SuccessCount, res.FailCount, nil +} diff --git a/sync/campaign_report_types.go b/sync/campaign_report_types.go new file mode 100644 index 0000000..d8acb2b --- /dev/null +++ b/sync/campaign_report_types.go @@ -0,0 +1,235 @@ +package sync + +type CampaignReportRequest struct { + AdvertiserID int64 `json:"advertiser_id"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + SelectColumns []string `json:"select_columns"` + GroupType int `json:"group_type"` + QueryVersion int `json:"query_version"` + SelectParam *CampaignSelectParam `json:"select_param,omitempty"` + PageInfo *PageInfo `json:"page_info,omitempty"` +} + +type CampaignSelectParam struct { + CampaignIDs []int64 `json:"campaign_ids,omitempty"` + AuthorID int64 `json:"author_id,omitempty"` + AdTypeStr string `json:"ad_type_str,omitempty"` + MarketingObjective int `json:"marketing_objective,omitempty"` + DeliveryScenario int `json:"delivery_scenario,omitempty"` + DeliveryMethod int `json:"delivery_method,omitempty"` + SupportType string `json:"support_type,omitempty"` + OcpcActionType string `json:"ocpc_action_type,omitempty"` + SpeedType string `json:"speed_type,omitempty"` + ItemType string `json:"item_type,omitempty"` + CreativeBuildType string `json:"creative_build_type,omitempty"` + AdScene string `json:"ad_scene,omitempty"` + IncrementExploreType []int `json:"increment_explore_type,omitempty"` +} + +type PageInfo struct { + CurrentPage int `json:"current_page"` + PageSize int `json:"page_size"` + TotalCount int `json:"total_count"` +} + +type CampaignReportResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Data *CampaignReportData `json:"data"` +} + +type CampaignReportData struct { + Sum *CampaignReportSum `json:"sum"` + Detail []*CampaignReportItem `json:"detail"` + TotalCount int `json:"total_count"` +} + +type CampaignReportSum struct { + T0OrderPaymentAmt string `json:"t0_order_payment_amt"` + CreativeMaterialType string `json:"creative_material_type"` + LiveName string `json:"live_name"` + AuthorId string `json:"author_id"` + PicUrl string `json:"pic_url"` + PicName string `json:"pic_name"` + PicId string `json:"pic_id"` + CoverUrl string `json:"cover_url"` + CoverId int64 `json:"cover_id"` + ItemOrderConversionRatio *float64 `json:"item_order_conversion_ratio"` + ItemCardClickRatio *float64 `json:"item_card_click_ratio"` + ItemCardClkCnt *int64 `json:"item_card_clk_cnt"` + LivePlayCntCost *float64 `json:"live_play_cnt_cost"` + AdMerchantFollowCost *float64 `json:"ad_merchant_follow_cost"` + AdMerchantFollow *int64 `json:"ad_merchant_follow"` + NetT0OrderCnt *int64 `json:"net_t0_order_cnt"` + NetT0Roi *float64 `json:"net_t0_roi"` + NetT0Gmv *float64 `json:"net_t0_gmv"` + PhotoName string `json:"photo_name"` + PhotoIdStr string `json:"photo_id_str"` + PhotoId string `json:"photo_id"` + ModPriceSegment string `json:"mod_price_segment"` + AgeSegment string `json:"age_segment"` + Province string `json:"province"` + Gender string `json:"gender"` + AdPhotoPlayedFiveRatio *float64 `json:"ad_photo_played_five_ratio"` + AdPhotoPlayedThreeRatio *float64 `json:"ad_photo_played_three_ratio"` + OrderSubmitRoi *float64 `json:"order_submit_roi"` + OrderSubmitAmt *int64 `json:"order_submit_amt"` + EventOrderSubmitCost *float64 `json:"event_order_submit_cost"` + EventOrderSubmit *int64 `json:"event_order_submit"` + EventOrderPaidRoi *float64 `json:"event_order_paid_roi"` + EventAppInvoked *int64 `json:"event_app_invoked"` + EventAddShoppingCart *int64 `json:"event_add_shopping_cart"` + ConversionNumCost *float64 `json:"conversion_num_cost"` + AdEffectivePlayNum *int64 `json:"ad_effective_play_num"` + AdItemClick *int64 `json:"ad_item_click"` + MerchantProductId string `json:"merchant_product_id"` + CostTotal *float64 `json:"cost_total"` + AdShow *int64 `json:"ad_show"` + AdShow1kCost *float64 `json:"ad_show1k_cost"` + Impression *int64 `json:"impression"` + PhotoClick *int64 `json:"photo_click"` + PhotoClickRatio *float64 `json:"photo_click_ratio"` + Click *int64 `json:"click"` + ActionbarClick *int64 `json:"actionbar_click"` + ActionbarClickCost *float64 `json:"actionbar_click_cost"` + EspClickRatio *float64 `json:"esp_click_ratio"` + ActionRatio *float64 `json:"action_ratio"` + AdItemClickCount *int64 `json:"ad_item_click_count"` + EspLivePlayedSeconds *int64 `json:"esp_live_played_seconds"` + PlayedThreeSeconds *int64 `json:"played_three_seconds"` + Play3sRatio *float64 `json:"play3s_ratio"` + PlayedFiveSeconds *int64 `json:"played_five_seconds"` + Play5sRatio *float64 `json:"play5s_ratio"` + PlayedEnd *int64 `json:"played_end"` + PlayEndRatio *float64 `json:"play_end_ratio"` + Share *int64 `json:"share"` + Comment *int64 `json:"comment"` + Likes *int64 `json:"likes"` + Report *int64 `json:"report"` + Block *int64 `json:"block"` + ItemNegative *int64 `json:"item_negative"` + LiveShare *int64 `json:"live_share"` + LiveComment *int64 `json:"live_comment"` + LiveReward *int64 `json:"live_reward"` + EffectivePlayCount *int64 `json:"effective_play_count"` + EffectivePlayRatio *float64 `json:"effective_play_ratio"` + ConversionNum *int64 `json:"conversion_num"` + ConversionCostEsp *float64 `json:"conversion_cost_esp"` + Roi *float64 `json:"roi"` + Gmv *float64 `json:"gmv"` + T0Gmv *float64 `json:"t0_gmv"` + T1Gmv *float64 `json:"t1_gmv"` + T7Gmv *float64 `json:"t7_gmv"` + T15Gmv *float64 `json:"t15_gmv"` + T30Gmv *float64 `json:"t30_gmv"` + T0Roi *float64 `json:"t0_roi"` + T1Roi *float64 `json:"t1_roi"` + T7Roi *float64 `json:"t7_roi"` + T15Roi *float64 `json:"t15_roi"` + T30Roi *float64 `json:"t30_roi"` + PaiedOrder *int64 `json:"paied_order"` + OrderRatio *float64 `json:"order_ratio"` + T0OrderCnt *int64 `json:"t0_order_cnt"` + T0OrderCntCost *float64 `json:"t0_order_cnt_cost"` + T0OrderCntRatio *float64 `json:"t0_order_cnt_ratio"` + T1OrderCnt *int64 `json:"t1_order_cnt"` + T3OrderCnt *int64 `json:"t3_order_cnt"` + T7OrderCnt *int64 `json:"t7_order_cnt"` + T15OrderCnt *int64 `json:"t15_order_cnt"` + T30OrderCnt *int64 `json:"t30_order_cnt"` + MerchantRecoFans *int64 `json:"merchant_reco_fans"` + T1Retention *float64 `json:"t1_retention"` + T7Retention *float64 `json:"t7_retention"` + T15Retention *float64 `json:"t15_retention"` + T30Retention *float64 `json:"t30_retention"` + T1RetentionRatio *float64 `json:"t1_retention_ratio"` + T7RetentionRatio *float64 `json:"t7_retention_ratio"` + T15RetentionRatio *float64 `json:"t15_retention_ratio"` + T30RetentionRatio *float64 `json:"t30_retention_ratio"` + ReservationSuccess *int64 `json:"reservation_success"` + ReservationCost *float64 `json:"reservation_cost"` + StandardLivePlayedStarted *int64 `json:"standard_live_played_started"` + AdLivePlayCnt *int64 `json:"ad_live_play_cnt"` + AdLivePlayCntCost *float64 `json:"ad_live_play_cnt_cost"` + LiveAudienceCost *float64 `json:"live_audience_cost"` + LiveEventGoodsView *int64 `json:"live_event_goods_view"` + GoodsClickRatio *float64 `json:"goods_click_ratio"` + DirectAttrPlatNewBuyerCnt *int64 `json:"direct_attr_plat_new_buyer_cnt"` + T30AttrPlatTotalBuyerCnt *int64 `json:"t30_attr_plat_total_buyer_cnt"` + DirectAttrSellerNewBuyerCnt *int64 `json:"direct_attr_seller_new_buyer_cnt"` + T30AttrSellerTotalBuyerCnt *int64 `json:"t30_attr_seller_total_buyer_cnt"` + T3Gmv *float64 `json:"t3_gmv"` + T3Roi *float64 `json:"t3_roi"` + T7IndirectOrderAmt *float64 `json:"t7_indirect_order_amt"` + T7IndirectOrderCnt *int64 `json:"t7_indirect_order_cnt"` + FansT0GmvPerFans *float64 `json:"fans_t0_gmv_per_fans"` + FansT3GmvPerFans *float64 `json:"fans_t3_gmv_per_fans"` + FansT7GmvPerFans *float64 `json:"fans_t7_gmv_per_fans"` + FansT15GmvPerFans *float64 `json:"fans_t15_gmv_per_fans"` + FansT30GmvPerFans *float64 `json:"fans_t30_gmv_per_fans"` + RecoFansCost *float64 `json:"reco_fans_cost"` + QcpxWhiteboxDirectOrderPaymentAmt *float64 `json:"qcpx_whitebox_direct_order_payment_amt"` + QcpxWhiteboxDirectOrderCnt *int64 `json:"qcpx_whitebox_direct_order_cnt"` + FansT0Gmv *float64 `json:"fans_t0_gmv"` + FansT1Gmv *float64 `json:"fans_t1_gmv"` + FansT7Gmv *float64 `json:"fans_t7_gmv"` + FansT15Gmv *float64 `json:"fans_t15_gmv"` + FansT30Gmv *float64 `json:"fans_t30_gmv"` + FansT0Roi *float64 `json:"fans_t0_roi"` + FansT1Roi *float64 `json:"fans_t1_roi"` + FansT7Roi *float64 `json:"fans_t7_roi"` + FansT15Roi *float64 `json:"fans_t15_roi"` + FansT30Roi *float64 `json:"fans_t30_roi"` + T0ShopNewBuyerOrderPaymentAmt *float64 `json:"t0_shop_new_buyer_order_payment_amt"` + T1ShopNewBuyerOrderPaymentAmt *float64 `json:"t1_shop_new_buyer_order_payment_amt"` + T3ShopNewBuyerOrderPaymentAmt *float64 `json:"t3_shop_new_buyer_order_payment_amt"` + T7ShopNewBuyerOrderPaymentAmt *float64 `json:"t7_shop_new_buyer_order_payment_amt"` + T15ShopNewBuyerOrderPaymentAmt *float64 `json:"t15_shop_new_buyer_order_payment_amt"` + T30ShopNewBuyerOrderPaymentAmt *float64 `json:"t30_shop_new_buyer_order_payment_amt"` + T0ShopNewBuyerOrderCnt *int64 `json:"t0_shop_new_buyer_order_cnt"` + T1ShopNewBuyerOrderCnt *int64 `json:"t1_shop_new_buyer_order_cnt"` + T3ShopNewBuyerOrderCnt *int64 `json:"t3_shop_new_buyer_order_cnt"` + T7ShopNewBuyerOrderCnt *int64 `json:"t7_shop_new_buyer_order_cnt"` + T15ShopNewBuyerOrderCnt *int64 `json:"t15_shop_new_buyer_order_cnt"` + T30ShopNewBuyerOrderCnt *int64 `json:"t30_shop_new_buyer_order_cnt"` + T1NewBuyerRepurchaseRatio *float64 `json:"t1_new_buyer_repurchase_ratio"` + T3NewBuyerRepurchaseRatio *float64 `json:"t3_new_buyer_repurchase_ratio"` + T7NewBuyerRepurchaseRatio *float64 `json:"t7_new_buyer_repurchase_ratio"` + T15NewBuyerRepurchaseRatio *float64 `json:"t15_new_buyer_repurchase_ratio"` + T30NewBuyerRepurchaseRatio *float64 `json:"t30_new_buyer_repurchase_ratio"` + T0ShopNewBuyerRoi *float64 `json:"t0_shop_new_buyer_roi"` + T1ShopNewBuyerRoi *float64 `json:"t1_shop_new_buyer_roi"` + T3ShopNewBuyerRoi *float64 `json:"t3_shop_new_buyer_roi"` + T7ShopNewBuyerRoi *float64 `json:"t7_shop_new_buyer_roi"` + T15ShopNewBuyerRoi *float64 `json:"t15_shop_new_buyer_roi"` + T30ShopNewBuyerRoi *float64 `json:"t30_shop_new_buyer_roi"` + CreateCardOrderCnt *int64 `json:"create_card_order_cnt"` + ForwardTsCreateCardOrderCnt *int64 `json:"forward_ts_create_card_order_cnt"` + CreateCardOrderCost *float64 `json:"create_card_order_cost"` + ForwardTsCreateCardOrderCost *float64 `json:"forward_ts_create_card_order_cost"` + ActivateCardOrderCnt *int64 `json:"activate_card_order_cnt"` + ForwardTsActivateCardOrderCnt *int64 `json:"forward_ts_activate_card_order_cnt"` + ActivateCardOrderCost *float64 `json:"activate_card_order_cost"` + ForwardTsActivateCardOrderCost *float64 `json:"forward_ts_activate_card_order_cost"` + CreateCardOrderRatio *float64 `json:"create_card_order_ratio"` + ForwardTsCreateCardOrderRatio *float64 `json:"forward_ts_create_card_order_ratio"` + ActivateCardOrderCntRatio *float64 `json:"activate_card_order_cnt_ratio"` + ForwardTsActivateCardOrderRatio *float64 `json:"forward_ts_activate_card_order_ratio"` + LivePlayCnt *int64 `json:"live_play_cnt"` + ItemEntranceClkCnt *int64 `json:"item_entrance_clk_cnt"` + ShowCnt *int64 `json:"show_cnt"` + ReportDateStr string `json:"report_date_str"` + CampaignId *int64 `json:"campaign_id"` + CampaignName string `json:"campaign_name"` + UnitId *int64 `json:"unit_id"` + UnitName string `json:"unit_name"` + CreativeId *int64 `json:"creative_id"` + CreativeName string `json:"creative_name"` + CidActualRoiAfterSubsidy *float64 `json:"cid_actual_roi_after_subsidy"` + CidCouponAmount *int64 `json:"cid_coupon_amount"` + CidCouponCallbackPaidRefundAmount *int64 `json:"cid_coupon_callback_paid_refund_amount"` + CidVoucherCost *float64 `json:"cid_voucher_cost"` +} + +type CampaignReportItem CampaignReportSum diff --git a/sync/data_converter.go b/sync/data_converter.go new file mode 100644 index 0000000..0cfce1a --- /dev/null +++ b/sync/data_converter.go @@ -0,0 +1,415 @@ +package sync + +import ( + "cid/model/dto/copydata" +) + +type DataConverter struct{} + +func NewDataConverter() *DataConverter { + return &DataConverter{} +} + +func (c *DataConverter) ConvertToSumItem(apiData *CampaignReportSum, dataType string) *copydata.CidAccountReportSumItem { + if apiData == nil { + return nil + } + + return ©data.CidAccountReportSumItem{ + DataType: dataType, + T0OrderPaymentAmt: apiData.T0OrderPaymentAmt, + CreativeMaterialType: apiData.CreativeMaterialType, + LiveName: apiData.LiveName, + AuthorId: apiData.AuthorId, + PicUrl: apiData.PicUrl, + PicName: apiData.PicName, + PicId: apiData.PicId, + CoverUrl: apiData.CoverUrl, + CoverId: apiData.CoverId, + ItemOrderConversionRatio: apiData.ItemOrderConversionRatio, + ItemCardClickRatio: apiData.ItemCardClickRatio, + ItemCardClkCnt: apiData.ItemCardClkCnt, + LivePlayCntCost: apiData.LivePlayCntCost, + AdMerchantFollowCost: apiData.AdMerchantFollowCost, + AdMerchantFollow: apiData.AdMerchantFollow, + NetT0OrderCnt: apiData.NetT0OrderCnt, + NetT0Roi: apiData.NetT0Roi, + NetT0Gmv: apiData.NetT0Gmv, + PhotoName: apiData.PhotoName, + PhotoIdStr: apiData.PhotoIdStr, + PhotoId: apiData.PhotoId, + ModPriceSegment: apiData.ModPriceSegment, + AgeSegment: apiData.AgeSegment, + Province: apiData.Province, + Gender: apiData.Gender, + AdPhotoPlayedFiveRatio: apiData.AdPhotoPlayedFiveRatio, + AdPhotoPlayedThreeRatio: apiData.AdPhotoPlayedThreeRatio, + OrderSubmitRoi: apiData.OrderSubmitRoi, + OrderSubmitAmt: apiData.OrderSubmitAmt, + EventOrderSubmitCost: apiData.EventOrderSubmitCost, + EventOrderSubmit: apiData.EventOrderSubmit, + EventOrderPaidRoi: apiData.EventOrderPaidRoi, + EventAppInvoked: apiData.EventAppInvoked, + EventAddShoppingCart: apiData.EventAddShoppingCart, + ConversionNumCost: apiData.ConversionNumCost, + AdEffectivePlayNum: apiData.AdEffectivePlayNum, + AdItemClick: apiData.AdItemClick, + MerchantProductId: apiData.MerchantProductId, + CostTotal: apiData.CostTotal, + AdShow: apiData.AdShow, + AdShow1kCost: apiData.AdShow1kCost, + Impression: apiData.Impression, + PhotoClick: apiData.PhotoClick, + PhotoClickRatio: apiData.PhotoClickRatio, + Click: apiData.Click, + ActionbarClick: apiData.ActionbarClick, + ActionbarClickCost: apiData.ActionbarClickCost, + EspClickRatio: apiData.EspClickRatio, + ActionRatio: apiData.ActionRatio, + AdItemClickCount: apiData.AdItemClickCount, + EspLivePlayedSeconds: apiData.EspLivePlayedSeconds, + PlayedThreeSeconds: apiData.PlayedThreeSeconds, + Play3sRatio: apiData.Play3sRatio, + PlayedFiveSeconds: apiData.PlayedFiveSeconds, + Play5sRatio: apiData.Play5sRatio, + PlayedEnd: apiData.PlayedEnd, + PlayEndRatio: apiData.PlayEndRatio, + Share: apiData.Share, + Comment: apiData.Comment, + Likes: apiData.Likes, + Report: apiData.Report, + Block: apiData.Block, + ItemNegative: apiData.ItemNegative, + LiveShare: apiData.LiveShare, + LiveComment: apiData.LiveComment, + LiveReward: apiData.LiveReward, + EffectivePlayCount: apiData.EffectivePlayCount, + EffectivePlayRatio: apiData.EffectivePlayRatio, + ConversionNum: apiData.ConversionNum, + ConversionCostEsp: apiData.ConversionCostEsp, + Roi: apiData.Roi, + Gmv: apiData.Gmv, + T0Gmv: apiData.T0Gmv, + T1Gmv: apiData.T1Gmv, + T3Gmv: apiData.T3Gmv, + T7Gmv: apiData.T7Gmv, + T15Gmv: apiData.T15Gmv, + T30Gmv: apiData.T30Gmv, + T0Roi: apiData.T0Roi, + T1Roi: apiData.T1Roi, + T3Roi: apiData.T3Roi, + T7Roi: apiData.T7Roi, + T15Roi: apiData.T15Roi, + T30Roi: apiData.T30Roi, + PaiedOrder: apiData.PaiedOrder, + OrderRatio: apiData.OrderRatio, + T0OrderCnt: apiData.T0OrderCnt, + T0OrderCntCost: apiData.T0OrderCntCost, + T0OrderCntRatio: apiData.T0OrderCntRatio, + T1OrderCnt: apiData.T1OrderCnt, + T3OrderCnt: apiData.T3OrderCnt, + T7OrderCnt: apiData.T7OrderCnt, + T15OrderCnt: apiData.T15OrderCnt, + T30OrderCnt: apiData.T30OrderCnt, + MerchantRecoFans: apiData.MerchantRecoFans, + T1Retention: apiData.T1Retention, + T7Retention: apiData.T7Retention, + T15Retention: apiData.T15Retention, + T30Retention: apiData.T30Retention, + T1RetentionRatio: apiData.T1RetentionRatio, + T7RetentionRatio: apiData.T7RetentionRatio, + T15RetentionRatio: apiData.T15RetentionRatio, + T30RetentionRatio: apiData.T30RetentionRatio, + ReservationSuccess: apiData.ReservationSuccess, + ReservationCost: apiData.ReservationCost, + StandardLivePlayedStarted: apiData.StandardLivePlayedStarted, + AdLivePlayCnt: apiData.AdLivePlayCnt, + AdLivePlayCntCost: apiData.AdLivePlayCntCost, + LiveAudienceCost: apiData.LiveAudienceCost, + LiveEventGoodsView: apiData.LiveEventGoodsView, + GoodsClickRatio: apiData.GoodsClickRatio, + DirectAttrPlatNewBuyerCnt: apiData.DirectAttrPlatNewBuyerCnt, + T30AttrPlatTotalBuyerCnt: apiData.T30AttrPlatTotalBuyerCnt, + DirectAttrSellerNewBuyerCnt: apiData.DirectAttrSellerNewBuyerCnt, + T30AttrSellerTotalBuyerCnt: apiData.T30AttrSellerTotalBuyerCnt, + T7IndirectOrderAmt: apiData.T7IndirectOrderAmt, + T7IndirectOrderCnt: apiData.T7IndirectOrderCnt, + FansT0GmvPerFans: apiData.FansT0GmvPerFans, + FansT3GmvPerFans: apiData.FansT3GmvPerFans, + FansT7GmvPerFans: apiData.FansT7GmvPerFans, + FansT15GmvPerFans: apiData.FansT15GmvPerFans, + FansT30GmvPerFans: apiData.FansT30GmvPerFans, + RecoFansCost: apiData.RecoFansCost, + QcpxWhiteboxDirectOrderPaymentAmt: apiData.QcpxWhiteboxDirectOrderPaymentAmt, + QcpxWhiteboxDirectOrderCnt: apiData.QcpxWhiteboxDirectOrderCnt, + FansT0Gmv: apiData.FansT0Gmv, + FansT1Gmv: apiData.FansT1Gmv, + FansT7Gmv: apiData.FansT7Gmv, + FansT15Gmv: apiData.FansT15Gmv, + FansT30Gmv: apiData.FansT30Gmv, + FansT0Roi: apiData.FansT0Roi, + FansT1Roi: apiData.FansT1Roi, + FansT7Roi: apiData.FansT7Roi, + FansT15Roi: apiData.FansT15Roi, + FansT30Roi: apiData.FansT30Roi, + T0ShopNewBuyerOrderPaymentAmt: apiData.T0ShopNewBuyerOrderPaymentAmt, + T1ShopNewBuyerOrderPaymentAmt: apiData.T1ShopNewBuyerOrderPaymentAmt, + T3ShopNewBuyerOrderPaymentAmt: apiData.T3ShopNewBuyerOrderPaymentAmt, + T7ShopNewBuyerOrderPaymentAmt: apiData.T7ShopNewBuyerOrderPaymentAmt, + T15ShopNewBuyerOrderPaymentAmt: apiData.T15ShopNewBuyerOrderPaymentAmt, + T30ShopNewBuyerOrderPaymentAmt: apiData.T30ShopNewBuyerOrderPaymentAmt, + T0ShopNewBuyerOrderCnt: apiData.T0ShopNewBuyerOrderCnt, + T1ShopNewBuyerOrderCnt: apiData.T1ShopNewBuyerOrderCnt, + T3ShopNewBuyerOrderCnt: apiData.T3ShopNewBuyerOrderCnt, + T7ShopNewBuyerOrderCnt: apiData.T7ShopNewBuyerOrderCnt, + T15ShopNewBuyerOrderCnt: apiData.T15ShopNewBuyerOrderCnt, + T30ShopNewBuyerOrderCnt: apiData.T30ShopNewBuyerOrderCnt, + T1NewBuyerRepurchaseRatio: apiData.T1NewBuyerRepurchaseRatio, + T3NewBuyerRepurchaseRatio: apiData.T3NewBuyerRepurchaseRatio, + T7NewBuyerRepurchaseRatio: apiData.T7NewBuyerRepurchaseRatio, + T15NewBuyerRepurchaseRatio: apiData.T15NewBuyerRepurchaseRatio, + T30NewBuyerRepurchaseRatio: apiData.T30NewBuyerRepurchaseRatio, + T0ShopNewBuyerRoi: apiData.T0ShopNewBuyerRoi, + T1ShopNewBuyerRoi: apiData.T1ShopNewBuyerRoi, + T3ShopNewBuyerRoi: apiData.T3ShopNewBuyerRoi, + T7ShopNewBuyerRoi: apiData.T7ShopNewBuyerRoi, + T15ShopNewBuyerRoi: apiData.T15ShopNewBuyerRoi, + T30ShopNewBuyerRoi: apiData.T30ShopNewBuyerRoi, + CreateCardOrderCnt: apiData.CreateCardOrderCnt, + ForwardTsCreateCardOrderCnt: apiData.ForwardTsCreateCardOrderCnt, + CreateCardOrderCost: apiData.CreateCardOrderCost, + ForwardTsCreateCardOrderCost: apiData.ForwardTsCreateCardOrderCost, + ActivateCardOrderCnt: apiData.ActivateCardOrderCnt, + ForwardTsActivateCardOrderCnt: apiData.ForwardTsActivateCardOrderCnt, + ActivateCardOrderCost: apiData.ActivateCardOrderCost, + ForwardTsActivateCardOrderCost: apiData.ForwardTsActivateCardOrderCost, + CreateCardOrderRatio: apiData.CreateCardOrderRatio, + ForwardTsCreateCardOrderRatio: apiData.ForwardTsCreateCardOrderRatio, + ActivateCardOrderCntRatio: apiData.ActivateCardOrderCntRatio, + ForwardTsActivateCardOrderRatio: apiData.ForwardTsActivateCardOrderRatio, + LivePlayCnt: apiData.LivePlayCnt, + ItemEntranceClkCnt: apiData.ItemEntranceClkCnt, + ShowCnt: apiData.ShowCnt, + ReportDateStr: apiData.ReportDateStr, + CampaignId: apiData.CampaignId, + CampaignName: apiData.CampaignName, + UnitId: apiData.UnitId, + UnitName: apiData.UnitName, + CreativeId: apiData.CreativeId, + CreativeName: apiData.CreativeName, + CidActualRoiAfterSubsidy: apiData.CidActualRoiAfterSubsidy, + CidCouponAmount: apiData.CidCouponAmount, + CidCouponCallbackPaidRefundAmount: apiData.CidCouponCallbackPaidRefundAmount, + CidVoucherCost: apiData.CidVoucherCost, + } +} + +func (c *DataConverter) ConvertToDetailItems(apiItems []*CampaignReportItem, dataType string) []*copydata.CidAccountReportDetailItem { + if len(apiItems) == 0 { + return nil + } + + result := make([]*copydata.CidAccountReportDetailItem, 0, len(apiItems)) + for _, item := range apiItems { + detailItem := c.convertItemToDetail(item, dataType) + result = append(result, detailItem) + } + return result +} + +func (c *DataConverter) convertItemToDetail(apiItem *CampaignReportItem, dataType string) *copydata.CidAccountReportDetailItem { + if apiItem == nil { + return nil + } + + item := (*CampaignReportSum)(apiItem) + sumItem := c.ConvertToSumItem(item, dataType) + + return ©data.CidAccountReportDetailItem{ + DataType: sumItem.DataType, + T0OrderPaymentAmt: sumItem.T0OrderPaymentAmt, + CreativeMaterialType: sumItem.CreativeMaterialType, + LiveName: sumItem.LiveName, + AuthorId: sumItem.AuthorId, + PicUrl: sumItem.PicUrl, + PicName: sumItem.PicName, + PicId: sumItem.PicId, + CoverUrl: sumItem.CoverUrl, + CoverId: sumItem.CoverId, + ItemOrderConversionRatio: sumItem.ItemOrderConversionRatio, + ItemCardClickRatio: sumItem.ItemCardClickRatio, + ItemCardClkCnt: sumItem.ItemCardClkCnt, + LivePlayCntCost: sumItem.LivePlayCntCost, + AdMerchantFollowCost: sumItem.AdMerchantFollowCost, + AdMerchantFollow: sumItem.AdMerchantFollow, + NetT0OrderCnt: sumItem.NetT0OrderCnt, + NetT0Roi: sumItem.NetT0Roi, + NetT0Gmv: sumItem.NetT0Gmv, + PhotoName: sumItem.PhotoName, + PhotoIdStr: sumItem.PhotoIdStr, + PhotoId: sumItem.PhotoId, + ModPriceSegment: sumItem.ModPriceSegment, + AgeSegment: sumItem.AgeSegment, + Province: sumItem.Province, + Gender: sumItem.Gender, + AdPhotoPlayedFiveRatio: sumItem.AdPhotoPlayedFiveRatio, + AdPhotoPlayedThreeRatio: sumItem.AdPhotoPlayedThreeRatio, + OrderSubmitRoi: sumItem.OrderSubmitRoi, + OrderSubmitAmt: sumItem.OrderSubmitAmt, + EventOrderSubmitCost: sumItem.EventOrderSubmitCost, + EventOrderSubmit: sumItem.EventOrderSubmit, + EventOrderPaidRoi: sumItem.EventOrderPaidRoi, + EventAppInvoked: sumItem.EventAppInvoked, + EventAddShoppingCart: sumItem.EventAddShoppingCart, + ConversionNumCost: sumItem.ConversionNumCost, + AdEffectivePlayNum: sumItem.AdEffectivePlayNum, + AdItemClick: sumItem.AdItemClick, + MerchantProductId: sumItem.MerchantProductId, + CostTotal: sumItem.CostTotal, + AdShow: sumItem.AdShow, + AdShow1kCost: sumItem.AdShow1kCost, + Impression: sumItem.Impression, + PhotoClick: sumItem.PhotoClick, + PhotoClickRatio: sumItem.PhotoClickRatio, + Click: sumItem.Click, + ActionbarClick: sumItem.ActionbarClick, + ActionbarClickCost: sumItem.ActionbarClickCost, + EspClickRatio: sumItem.EspClickRatio, + ActionRatio: sumItem.ActionRatio, + AdItemClickCount: sumItem.AdItemClickCount, + EspLivePlayedSeconds: sumItem.EspLivePlayedSeconds, + PlayedThreeSeconds: sumItem.PlayedThreeSeconds, + Play3sRatio: sumItem.Play3sRatio, + PlayedFiveSeconds: sumItem.PlayedFiveSeconds, + Play5sRatio: sumItem.Play5sRatio, + PlayedEnd: sumItem.PlayedEnd, + PlayEndRatio: sumItem.PlayEndRatio, + Share: sumItem.Share, + Comment: sumItem.Comment, + Likes: sumItem.Likes, + Report: sumItem.Report, + Block: sumItem.Block, + ItemNegative: sumItem.ItemNegative, + LiveShare: sumItem.LiveShare, + LiveComment: sumItem.LiveComment, + LiveReward: sumItem.LiveReward, + EffectivePlayCount: sumItem.EffectivePlayCount, + EffectivePlayRatio: sumItem.EffectivePlayRatio, + ConversionNum: sumItem.ConversionNum, + ConversionCostEsp: sumItem.ConversionCostEsp, + Roi: sumItem.Roi, + Gmv: sumItem.Gmv, + T0Gmv: sumItem.T0Gmv, + T1Gmv: sumItem.T1Gmv, + T3Gmv: sumItem.T3Gmv, + T7Gmv: sumItem.T7Gmv, + T15Gmv: sumItem.T15Gmv, + T30Gmv: sumItem.T30Gmv, + T0Roi: sumItem.T0Roi, + T1Roi: sumItem.T1Roi, + T3Roi: sumItem.T3Roi, + T7Roi: sumItem.T7Roi, + T15Roi: sumItem.T15Roi, + T30Roi: sumItem.T30Roi, + PaiedOrder: sumItem.PaiedOrder, + OrderRatio: sumItem.OrderRatio, + T0OrderCnt: sumItem.T0OrderCnt, + T0OrderCntCost: sumItem.T0OrderCntCost, + T0OrderCntRatio: sumItem.T0OrderCntRatio, + T1OrderCnt: sumItem.T1OrderCnt, + T3OrderCnt: sumItem.T3OrderCnt, + T7OrderCnt: sumItem.T7OrderCnt, + T15OrderCnt: sumItem.T15OrderCnt, + T30OrderCnt: sumItem.T30OrderCnt, + MerchantRecoFans: sumItem.MerchantRecoFans, + T1Retention: sumItem.T1Retention, + T7Retention: sumItem.T7Retention, + T15Retention: sumItem.T15Retention, + T30Retention: sumItem.T30Retention, + T1RetentionRatio: sumItem.T1RetentionRatio, + T7RetentionRatio: sumItem.T7RetentionRatio, + T15RetentionRatio: sumItem.T15RetentionRatio, + T30RetentionRatio: sumItem.T30RetentionRatio, + ReservationSuccess: sumItem.ReservationSuccess, + ReservationCost: sumItem.ReservationCost, + StandardLivePlayedStarted: sumItem.StandardLivePlayedStarted, + AdLivePlayCnt: sumItem.AdLivePlayCnt, + AdLivePlayCntCost: sumItem.AdLivePlayCntCost, + LiveAudienceCost: sumItem.LiveAudienceCost, + LiveEventGoodsView: sumItem.LiveEventGoodsView, + GoodsClickRatio: sumItem.GoodsClickRatio, + DirectAttrPlatNewBuyerCnt: sumItem.DirectAttrPlatNewBuyerCnt, + T30AttrPlatTotalBuyerCnt: sumItem.T30AttrPlatTotalBuyerCnt, + DirectAttrSellerNewBuyerCnt: sumItem.DirectAttrSellerNewBuyerCnt, + T30AttrSellerTotalBuyerCnt: sumItem.T30AttrSellerTotalBuyerCnt, + T7IndirectOrderAmt: sumItem.T7IndirectOrderAmt, + T7IndirectOrderCnt: sumItem.T7IndirectOrderCnt, + FansT0GmvPerFans: sumItem.FansT0GmvPerFans, + FansT3GmvPerFans: sumItem.FansT3GmvPerFans, + FansT7GmvPerFans: sumItem.FansT7GmvPerFans, + FansT15GmvPerFans: sumItem.FansT15GmvPerFans, + FansT30GmvPerFans: sumItem.FansT30GmvPerFans, + RecoFansCost: sumItem.RecoFansCost, + QcpxWhiteboxDirectOrderPaymentAmt: sumItem.QcpxWhiteboxDirectOrderPaymentAmt, + QcpxWhiteboxDirectOrderCnt: sumItem.QcpxWhiteboxDirectOrderCnt, + FansT0Gmv: sumItem.FansT0Gmv, + FansT1Gmv: sumItem.FansT1Gmv, + FansT7Gmv: sumItem.FansT7Gmv, + FansT15Gmv: sumItem.FansT15Gmv, + FansT30Gmv: sumItem.FansT30Gmv, + FansT0Roi: sumItem.FansT0Roi, + FansT1Roi: sumItem.FansT1Roi, + FansT7Roi: sumItem.FansT7Roi, + FansT15Roi: sumItem.FansT15Roi, + FansT30Roi: sumItem.FansT30Roi, + T0ShopNewBuyerOrderPaymentAmt: sumItem.T0ShopNewBuyerOrderPaymentAmt, + T1ShopNewBuyerOrderPaymentAmt: sumItem.T1ShopNewBuyerOrderPaymentAmt, + T3ShopNewBuyerOrderPaymentAmt: sumItem.T3ShopNewBuyerOrderPaymentAmt, + T7ShopNewBuyerOrderPaymentAmt: sumItem.T7ShopNewBuyerOrderPaymentAmt, + T15ShopNewBuyerOrderPaymentAmt: sumItem.T15ShopNewBuyerOrderPaymentAmt, + T30ShopNewBuyerOrderPaymentAmt: sumItem.T30ShopNewBuyerOrderPaymentAmt, + T0ShopNewBuyerOrderCnt: sumItem.T0ShopNewBuyerOrderCnt, + T1ShopNewBuyerOrderCnt: sumItem.T1ShopNewBuyerOrderCnt, + T3ShopNewBuyerOrderCnt: sumItem.T3ShopNewBuyerOrderCnt, + T7ShopNewBuyerOrderCnt: sumItem.T7ShopNewBuyerOrderCnt, + T15ShopNewBuyerOrderCnt: sumItem.T15ShopNewBuyerOrderCnt, + T30ShopNewBuyerOrderCnt: sumItem.T30ShopNewBuyerOrderCnt, + T1NewBuyerRepurchaseRatio: sumItem.T1NewBuyerRepurchaseRatio, + T3NewBuyerRepurchaseRatio: sumItem.T3NewBuyerRepurchaseRatio, + T7NewBuyerRepurchaseRatio: sumItem.T7NewBuyerRepurchaseRatio, + T15NewBuyerRepurchaseRatio: sumItem.T15NewBuyerRepurchaseRatio, + T30NewBuyerRepurchaseRatio: sumItem.T30NewBuyerRepurchaseRatio, + T0ShopNewBuyerRoi: sumItem.T0ShopNewBuyerRoi, + T1ShopNewBuyerRoi: sumItem.T1ShopNewBuyerRoi, + T3ShopNewBuyerRoi: sumItem.T3ShopNewBuyerRoi, + T7ShopNewBuyerRoi: sumItem.T7ShopNewBuyerRoi, + T15ShopNewBuyerRoi: sumItem.T15ShopNewBuyerRoi, + T30ShopNewBuyerRoi: sumItem.T30ShopNewBuyerRoi, + CreateCardOrderCnt: sumItem.CreateCardOrderCnt, + ForwardTsCreateCardOrderCnt: sumItem.ForwardTsCreateCardOrderCnt, + CreateCardOrderCost: sumItem.CreateCardOrderCost, + ForwardTsCreateCardOrderCost: sumItem.ForwardTsCreateCardOrderCost, + ActivateCardOrderCnt: sumItem.ActivateCardOrderCnt, + ForwardTsActivateCardOrderCnt: sumItem.ForwardTsActivateCardOrderCnt, + ActivateCardOrderCost: sumItem.ActivateCardOrderCost, + ForwardTsActivateCardOrderCost: sumItem.ForwardTsActivateCardOrderCost, + CreateCardOrderRatio: sumItem.CreateCardOrderRatio, + ForwardTsCreateCardOrderRatio: sumItem.ForwardTsCreateCardOrderRatio, + ActivateCardOrderCntRatio: sumItem.ActivateCardOrderCntRatio, + ForwardTsActivateCardOrderRatio: sumItem.ForwardTsActivateCardOrderRatio, + LivePlayCnt: sumItem.LivePlayCnt, + ItemEntranceClkCnt: sumItem.ItemEntranceClkCnt, + ShowCnt: sumItem.ShowCnt, + ReportDateStr: sumItem.ReportDateStr, + CampaignId: sumItem.CampaignId, + CampaignName: sumItem.CampaignName, + UnitId: sumItem.UnitId, + UnitName: sumItem.UnitName, + CreativeId: sumItem.CreativeId, + CreativeName: sumItem.CreativeName, + CidActualRoiAfterSubsidy: sumItem.CidActualRoiAfterSubsidy, + CidCouponAmount: sumItem.CidCouponAmount, + CidCouponCallbackPaidRefundAmount: sumItem.CidCouponCallbackPaidRefundAmount, + CidVoucherCost: sumItem.CidVoucherCost, + } +} diff --git a/sync/http_client.go b/sync/http_client.go new file mode 100644 index 0000000..ffa0626 --- /dev/null +++ b/sync/http_client.go @@ -0,0 +1,68 @@ +package sync + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +type HttpClient struct { + BaseURL string + Timeout time.Duration + HTTPClient *http.Client +} + +func NewHttpClient(baseURL string, timeout time.Duration) *HttpClient { + if timeout == 0 { + timeout = 30 * time.Second + } + return &HttpClient{ + BaseURL: baseURL, + Timeout: timeout, + HTTPClient: &http.Client{ + Timeout: timeout, + }, + } +} + +func (c *HttpClient) Post(ctx context.Context, url string, body interface{}) ([]byte, error) { + jsonData, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("序列化请求失败：%w", err) + } + + fullURL := url + if c.BaseURL != "" && len(url) > 0 && url[0] != '/' { + fullURL = c.BaseURL + url + } else if c.BaseURL != "" { + fullURL = c.BaseURL + url + } + + req, err := http.NewRequestWithContext(ctx, "POST", fullURL, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("创建请求失败：%w", err) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("请求失败：%w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("读取响应失败：%w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP 错误状态码：%d", resp.StatusCode) + } + + return respBody, nil +} diff --git a/sync/mock_generator.go b/sync/mock_generator.go new file mode 100644 index 0000000..8dab89e --- /dev/null +++ b/sync/mock_generator.go @@ -0,0 +1,272 @@ +package sync + +import ( + "math/rand" + "time" +) + +type MockDataGenerator struct { + rand *rand.Rand +} + +func NewMockDataGenerator() *MockDataGenerator { + return &MockDataGenerator{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + } +} + +func (m *MockDataGenerator) GenerateCampaignReportRequest() *CampaignReportRequest { + return &CampaignReportRequest{ + AdvertiserID: 10001, + StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6, + EndTime: time.Now().UnixNano() / 1e6, + SelectColumns: []string{"impression", "click", "cost", "t0GMV"}, + GroupType: 1, + QueryVersion: 1, + SelectParam: &CampaignSelectParam{ + CampaignIDs: []int64{1, 2, 3}, + }, + PageInfo: &PageInfo{ + CurrentPage: 1, + PageSize: 20, + }, + } +} + +func (m *MockDataGenerator) GenerateCampaignReportResponse() *CampaignReportResponse { + sumData := m.generateSumData() + detailData := m.generateDetailData(5) + + return &CampaignReportResponse{ + Code: 0, + Message: "success", + Data: &CampaignReportData{ + Sum: sumData, + Detail: detailData, + TotalCount: len(detailData), + }, + } +} + +func (m *MockDataGenerator) generateSumData() *CampaignReportSum { + cost := m.randomFloat(1000, 10000) + impression := m.randomInt64(10000, 100000) + click := m.randomInt64(100, 1000) + + return &CampaignReportSum{ + T0OrderPaymentAmt: "888.99", + CreativeMaterialType: "视频素材类型", + LiveName: "测试直播间", + AuthorId: "123456", + PicUrl: "http://example.com/pic.jpg", + PicName: "图片名称", + PicId: "pic_123", + CoverUrl: "http://example.com/cover.jpg", + CoverId: 4551122, + ItemOrderConversionRatio: m.randomFloatPtr(0.01, 0.5), + ItemCardClickRatio: m.randomFloatPtr(0.02, 0.3), + ItemCardClkCnt: m.randomIntPtr(10, 100), + LivePlayCntCost: m.randomFloatPtr(0.5, 5.0), + AdMerchantFollowCost: m.randomFloatPtr(1.0, 10.0), + AdMerchantFollow: m.randomIntPtr(50, 500), + NetT0OrderCnt: m.randomIntPtr(10, 100), + NetT0Roi: m.randomFloatPtr(1.5, 5.0), + NetT0Gmv: m.randomFloatPtr(5000, 50000), + PhotoName: "测试视频", + PhotoIdStr: "video_123", + PhotoId: "video_123", + ModPriceSegment: "1000-2000", + AgeSegment: "24-30", + Province: "广东", + Gender: "男", + AdPhotoPlayedFiveRatio: m.randomFloatPtr(0.3, 0.8), + AdPhotoPlayedThreeRatio: m.randomFloatPtr(0.5, 0.9), + OrderSubmitRoi: m.randomFloatPtr(1.0, 3.0), + OrderSubmitAmt: m.randomIntPtr(10, 100), + EventOrderSubmitCost: m.randomFloatPtr(5.0, 20.0), + EventOrderSubmit: m.randomIntPtr(5, 50), + EventOrderPaidRoi: m.randomFloatPtr(0.5, 2.0), + EventAppInvoked: m.randomIntPtr(100, 1000), + EventAddShoppingCart: m.randomIntPtr(50, 500), + ConversionNumCost: m.randomFloatPtr(10.0, 50.0), + AdEffectivePlayNum: m.randomIntPtr(1000, 10000), + AdItemClick: m.randomIntPtr(100, 1000), + MerchantProductId: "product_123", + CostTotal: &cost, + AdShow: m.randomIntPtr(10000, 100000), + AdShow1kCost: m.randomFloatPtr(5.0, 50.0), + Impression: &impression, + PhotoClick: m.randomIntPtr(100, 5000), + PhotoClickRatio: m.randomFloatPtr(0.01, 0.1), + Click: &click, + ActionbarClick: m.randomIntPtr(50, 500), + ActionbarClickCost: m.randomFloatPtr(1.0, 10.0), + EspClickRatio: m.randomFloatPtr(0.01, 0.1), + ActionRatio: m.randomFloatPtr(0.02, 0.2), + AdItemClickCount: m.randomIntPtr(10, 100), + EspLivePlayedSeconds: m.randomIntPtr(30, 300), + PlayedThreeSeconds: m.randomIntPtr(5000, 50000), + Play3sRatio: m.randomFloatPtr(0.3, 0.8), + PlayedFiveSeconds: m.randomIntPtr(3000, 30000), + Play5sRatio: m.randomFloatPtr(0.2, 0.6), + PlayedEnd: m.randomIntPtr(1000, 10000), + PlayEndRatio: m.randomFloatPtr(0.1, 0.4), + Share: m.randomIntPtr(10, 100), + Comment: m.randomIntPtr(20, 200), + Likes: m.randomIntPtr(100, 1000), + Report: m.randomIntPtr(1, 10), + Block: m.randomIntPtr(1, 10), + ItemNegative: m.randomIntPtr(5, 50), + LiveShare: m.randomIntPtr(5, 50), + LiveComment: m.randomIntPtr(10, 100), + LiveReward: m.randomIntPtr(20, 200), + EffectivePlayCount: m.randomIntPtr(1000, 10000), + EffectivePlayRatio: m.randomFloatPtr(0.1, 0.5), + ConversionNum: m.randomIntPtr(5, 50), + ConversionCostEsp: m.randomFloatPtr(10.0, 50.0), + Roi: m.randomFloatPtr(1.0, 3.0), + Gmv: m.randomFloatPtr(1000, 10000), + T0Gmv: m.randomFloatPtr(500, 5000), + T1Gmv: m.randomFloatPtr(800, 8000), + T3Gmv: m.randomFloatPtr(1200, 12000), + T7Gmv: m.randomFloatPtr(2000, 20000), + T15Gmv: m.randomFloatPtr(3000, 30000), + T30Gmv: m.randomFloatPtr(5000, 50000), + T0Roi: m.randomFloatPtr(0.5, 2.0), + T1Roi: m.randomFloatPtr(0.8, 2.5), + T3Roi: m.randomFloatPtr(1.0, 3.0), + T7Roi: m.randomFloatPtr(1.5, 4.0), + T15Roi: m.randomFloatPtr(2.0, 5.0), + T30Roi: m.randomFloatPtr(2.5, 6.0), + PaiedOrder: m.randomIntPtr(5, 50), + OrderRatio: m.randomFloatPtr(0.01, 0.1), + T0OrderCnt: m.randomIntPtr(5, 50), + T0OrderCntCost: m.randomFloatPtr(10.0, 100.0), + T0OrderCntRatio: m.randomFloatPtr(0.5, 0.9), + T1OrderCnt: m.randomIntPtr(10, 100), + T3OrderCnt: m.randomIntPtr(20, 200), + T7OrderCnt: m.randomIntPtr(30, 300), + T15OrderCnt: m.randomIntPtr(40, 400), + T30OrderCnt: m.randomIntPtr(50, 500), + MerchantRecoFans: m.randomIntPtr(100, 1000), + T1Retention: m.randomFloatPtr(0.3, 0.8), + T7Retention: m.randomFloatPtr(0.2, 0.6), + T15Retention: m.randomFloatPtr(0.15, 0.5), + T30Retention: m.randomFloatPtr(0.1, 0.4), + T1RetentionRatio: m.randomFloatPtr(0.3, 0.8), + T7RetentionRatio: m.randomFloatPtr(0.2, 0.6), + T15RetentionRatio: m.randomFloatPtr(0.15, 0.5), + T30RetentionRatio: m.randomFloatPtr(0.1, 0.4), + ReservationSuccess: m.randomIntPtr(10, 100), + ReservationCost: m.randomFloatPtr(5.0, 50.0), + StandardLivePlayedStarted: m.randomIntPtr(100, 1000), + AdLivePlayCnt: m.randomIntPtr(50, 500), + AdLivePlayCntCost: m.randomFloatPtr(1.0, 10.0), + LiveAudienceCost: m.randomFloatPtr(0.5, 5.0), + LiveEventGoodsView: m.randomIntPtr(100, 1000), + GoodsClickRatio: m.randomFloatPtr(0.05, 0.3), + DirectAttrPlatNewBuyerCnt: m.randomIntPtr(10, 100), + T30AttrPlatTotalBuyerCnt: m.randomIntPtr(50, 500), + DirectAttrSellerNewBuyerCnt: m.randomIntPtr(5, 50), + T30AttrSellerTotalBuyerCnt: m.randomIntPtr(20, 200), + T7IndirectOrderAmt: m.randomFloatPtr(500, 5000), + T7IndirectOrderCnt: m.randomIntPtr(5, 50), + FansT0GmvPerFans: m.randomFloatPtr(10.0, 100.0), + FansT3GmvPerFans: m.randomFloatPtr(20.0, 200.0), + FansT7GmvPerFans: m.randomFloatPtr(30.0, 300.0), + FansT15GmvPerFans: m.randomFloatPtr(40.0, 400.0), + FansT30GmvPerFans: m.randomFloatPtr(50.0, 500.0), + RecoFansCost: m.randomFloatPtr(5.0, 50.0), + QcpxWhiteboxDirectOrderPaymentAmt: m.randomFloatPtr(100, 1000), + QcpxWhiteboxDirectOrderCnt: m.randomIntPtr(1, 10), + FansT0Gmv: m.randomFloatPtr(100, 1000), + FansT1Gmv: m.randomFloatPtr(200, 2000), + FansT7Gmv: m.randomFloatPtr(300, 3000), + FansT15Gmv: m.randomFloatPtr(400, 4000), + FansT30Gmv: m.randomFloatPtr(500, 5000), + FansT0Roi: m.randomFloatPtr(0.5, 2.0), + FansT1Roi: m.randomFloatPtr(0.8, 2.5), + FansT7Roi: m.randomFloatPtr(1.0, 3.0), + FansT15Roi: m.randomFloatPtr(1.5, 4.0), + FansT30Roi: m.randomFloatPtr(2.0, 5.0), + T0ShopNewBuyerOrderPaymentAmt: m.randomFloatPtr(100, 1000), + T1ShopNewBuyerOrderPaymentAmt: m.randomFloatPtr(200, 2000), + T3ShopNewBuyerOrderPaymentAmt: m.randomFloatPtr(300, 3000), + T7ShopNewBuyerOrderPaymentAmt: m.randomFloatPtr(400, 4000), + T15ShopNewBuyerOrderPaymentAmt: m.randomFloatPtr(500, 5000), + T30ShopNewBuyerOrderPaymentAmt: m.randomFloatPtr(600, 6000), + T0ShopNewBuyerOrderCnt: m.randomIntPtr(1, 10), + T1ShopNewBuyerOrderCnt: m.randomIntPtr(2, 20), + T3ShopNewBuyerOrderCnt: m.randomIntPtr(3, 30), + T7ShopNewBuyerOrderCnt: m.randomIntPtr(4, 40), + T15ShopNewBuyerOrderCnt: m.randomIntPtr(5, 50), + T30ShopNewBuyerOrderCnt: m.randomIntPtr(6, 60), + T1NewBuyerRepurchaseRatio: m.randomFloatPtr(0.1, 0.5), + T3NewBuyerRepurchaseRatio: m.randomFloatPtr(0.15, 0.55), + T7NewBuyerRepurchaseRatio: m.randomFloatPtr(0.2, 0.6), + T15NewBuyerRepurchaseRatio: m.randomFloatPtr(0.25, 0.65), + T30NewBuyerRepurchaseRatio: m.randomFloatPtr(0.3, 0.7), + T0ShopNewBuyerRoi: m.randomFloatPtr(0.5, 2.0), + T1ShopNewBuyerRoi: m.randomFloatPtr(0.8, 2.5), + T3ShopNewBuyerRoi: m.randomFloatPtr(1.0, 3.0), + T7ShopNewBuyerRoi: m.randomFloatPtr(1.5, 4.0), + T15ShopNewBuyerRoi: m.randomFloatPtr(2.0, 5.0), + T30ShopNewBuyerRoi: m.randomFloatPtr(2.5, 6.0), + CreateCardOrderCnt: m.randomIntPtr(1, 10), + ForwardTsCreateCardOrderCnt: m.randomIntPtr(1, 10), + CreateCardOrderCost: m.randomFloatPtr(10.0, 100.0), + ForwardTsCreateCardOrderCost: m.randomFloatPtr(10.0, 100.0), + ActivateCardOrderCnt: m.randomIntPtr(1, 10), + ForwardTsActivateCardOrderCnt: m.randomIntPtr(1, 10), + ActivateCardOrderCost: m.randomFloatPtr(10.0, 100.0), + ForwardTsActivateCardOrderCost: m.randomFloatPtr(10.0, 100.0), + CreateCardOrderRatio: m.randomFloatPtr(0.01, 0.1), + ForwardTsCreateCardOrderRatio: m.randomFloatPtr(0.01, 0.1), + ActivateCardOrderCntRatio: m.randomFloatPtr(0.01, 0.1), + ForwardTsActivateCardOrderRatio: m.randomFloatPtr(0.01, 0.1), + LivePlayCnt: m.randomIntPtr(100, 1000), + ItemEntranceClkCnt: m.randomIntPtr(50, 500), + ShowCnt: m.randomIntPtr(1000, 10000), + ReportDateStr: time.Now().Format("2006-01-02"), + CampaignId: m.randomIntPtr(1, 100), + CampaignName: "测试计划", + UnitId: m.randomIntPtr(1, 50), + UnitName: "测试单元", + CreativeId: m.randomIntPtr(1, 20), + CreativeName: "测试创意", + CidActualRoiAfterSubsidy: m.randomFloatPtr(1.0, 3.0), + CidCouponAmount: m.randomIntPtr(100, 1000), + CidCouponCallbackPaidRefundAmount: m.randomIntPtr(50, 500), + CidVoucherCost: m.randomFloatPtr(5.0, 50.0), + } +} + +func (m *MockDataGenerator) generateDetailData(count int) []*CampaignReportItem { + items := make([]*CampaignReportItem, count) + for i := 0; i < count; i++ { + items[i] = (*CampaignReportItem)(m.generateSumData()) + } + return items +} + +func (m *MockDataGenerator) randomInt(min, max int) int { + return m.rand.Intn(max-min) + min +} + +func (m *MockDataGenerator) randomInt64(min, max int64) int64 { + return m.rand.Int63n(max-min) + min +} + +func (m *MockDataGenerator) randomFloat(min, max float64) float64 { + return m.rand.Float64()*(max-min) + min +} + +func (m *MockDataGenerator) randomIntPtr(min, max int) *int64 { + v := int64(m.randomInt(min, max)) + return &v +} + +func (m *MockDataGenerator) randomFloatPtr(min, max float64) *float64 { + v := m.randomFloat(min, max) + return &v +} diff --git a/sync/quick_sync.go b/sync/quick_sync.go new file mode 100644 index 0000000..060d143 --- /dev/null +++ b/sync/quick_sync.go @@ -0,0 +1,51 @@ +package sync + +import ( + "context" + "time" + + "github.com/sirupsen/logrus" +) + +func SyncCampaignReportWithMock(ctx context.Context) error { + syncService := NewSyncService() + + req := &CampaignReportRequest{ + AdvertiserID: 10001, + StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6, + EndTime: time.Now().UnixNano() / 1e6, + SelectColumns: []string{"impression", "click", "cost", "t0GMV"}, + GroupType: 1, + QueryVersion: 1, + SelectParam: &CampaignSelectParam{ + CampaignIDs: []int64{1, 2, 3}, + }, + PageInfo: &PageInfo{ + CurrentPage: 1, + PageSize: 20, + }, + } + + result, err := syncService.SyncCampaignReport(ctx, req, true) + if err != nil { + logrus.Errorf("同步失败：%v", err) + return err + } + + logrus.Infof("同步成功 - 汇总 ID: %d, 明细数量：%d", result.SumID, result.DetailCount) + return nil +} + +func SyncCampaignReportWithRealAPI(ctx context.Context, req *CampaignReportRequest) error { + syncService := NewSyncService() + + result, err := syncService.SyncCampaignReport(ctx, req, false) + if err != nil { + logrus.Errorf("同步失败：%v", err) + return err + } + + logrus.Infof("同步成功 - 汇总 ID: %d, 明细数量：%d, 成功：%d, 失败：%d", + result.SumID, result.DetailCount, result.DetailSuccessCount, result.DetailFailCount) + return nil +} diff --git a/sync/sync_service.go b/sync/sync_service.go new file mode 100644 index 0000000..9c6c35f --- /dev/null +++ b/sync/sync_service.go @@ -0,0 +1,268 @@ +package sync + +import ( + dto "cid/model/dto/copydata" + "cid/service/copydata" + "context" + "encoding/json" + "fmt" + "time" + + "gitea.com/red-future/common/beans" + "github.com/sirupsen/logrus" +) + +type SyncService struct { + httpClient *HttpClient + converter *DataConverter + mockGen *MockDataGenerator +} + +func NewSyncService() *SyncService { + return &SyncService{ + httpClient: NewHttpClient("https://ad.e.kuaishou.com", 0), + converter: NewDataConverter(), + mockGen: NewMockDataGenerator(), + } +} + +type SyncResult struct { + SumSuccess bool `json:"sum_success"` + SumID int64 `json:"sum_id"` + DetailSuccess bool `json:"detail_success"` + DetailCount int `json:"detail_count"` + DetailSuccessCount int64 `json:"detail_success_count"` + DetailFailCount int64 `json:"detail_fail_count"` + Error error `json:"error"` +} + +func (s *SyncService) SyncCampaignReport(ctx context.Context, req *CampaignReportRequest, useMock bool) (*SyncResult, error) { + result := &SyncResult{} + + var responseData *CampaignReportResponse + + if useMock { + logrus.Info("使用 Mock 数据同步快手广告计划报表") + responseData = s.mockGen.GenerateCampaignReportResponse() + } else { + logrus.Info("从真实 API 同步快手广告计划报表") + respBytes, err := s.httpClient.Post(ctx, "/rest/openapi/gw/esp/report/campaignReport", req) + if err != nil { + result.Error = fmt.Errorf("调用 API 失败：%w", err) + return result, result.Error + } + + responseData = &CampaignReportResponse{} + if err := json.Unmarshal(respBytes, responseData); err != nil { + result.Error = fmt.Errorf("解析响应失败：%w", err) + return result, result.Error + } + + if responseData.Code != 0 { + result.Error = fmt.Errorf("API 返回错误：code=%d, message=%s", responseData.Code, responseData.Message) + return result, result.Error + } + } + + if responseData.Data.Sum != nil { + sumItem := s.converter.ConvertToSumItem(responseData.Data.Sum, "campaign_report") + ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"}) + + sumResult, saveErr := s.saveSumData(ctx, sumItem) + if saveErr != nil { + logrus.Errorf("保存汇总数据失败：%v", saveErr) + result.Error = fmt.Errorf("保存汇总数据失败：%w", saveErr) + } else { + result.SumSuccess = true + result.SumID = sumResult.Id + logrus.Infof("成功保存汇总数据，ID=%d", sumResult.Id) + } + } + + if len(responseData.Data.Detail) > 0 { + detailItems := s.converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report") + detailResult, saveErr := s.saveDetailData(ctx, detailItems) + if saveErr != nil { + logrus.Errorf("保存明细数据失败：%v", saveErr) + result.Error = fmt.Errorf("保存明细数据失败：%w", saveErr) + } else { + result.DetailSuccess = true + result.DetailCount = len(detailItems) + result.DetailSuccessCount = detailResult.SuccessCount + result.DetailFailCount = detailResult.FailCount + logrus.Infof("成功保存明细数据，成功=%d, 失败=%d", detailResult.SuccessCount, detailResult.FailCount) + } + } + + return result, result.Error +} + +// SyncCampaignReportWithPagination 带分页处理的同步方法（支持全量数据抽取） +func (s *SyncService) SyncCampaignReportWithPagination(ctx context.Context, req *CampaignReportRequest, useMock bool, maxRetries int) (*SyncResult, error) { + aggregatedResult := &SyncResult{ + SumSuccess: false, + SumID: 0, + } + + allDetailItems := make([]*dto.CidAccountReportDetailItem, 0) + totalCount := 0 + currentPage := 1 + pageSize := 100 + + if req.PageInfo == nil { + req.PageInfo = &PageInfo{} + } + + for { + logrus.Infof(">>> 正在同步第 %d 页数据...", currentPage) + + req.PageInfo.CurrentPage = currentPage + req.PageInfo.PageSize = pageSize + + result, err := s.SyncWithRetry(ctx, req, useMock, maxRetries) + if err != nil { + logrus.Errorf("第 %d 页同步失败：%v", currentPage, err) + return aggregatedResult, err + } + + if result.SumSuccess && aggregatedResult.SumID == 0 { + aggregatedResult.SumSuccess = true + aggregatedResult.SumID = result.SumID + logrus.Infof("✓ 汇总数据已保存，ID=%d", result.SumID) + } + + if result.DetailSuccess && result.DetailCount > 0 { + detailItems := s.extractDetailItems(req, useMock) + if len(detailItems) > 0 { + allDetailItems = append(allDetailItems, detailItems...) + totalCount += len(detailItems) + logrus.Infof("✓ 第 %d 页获取到 %d 条明细数据，累计 %d 条", currentPage, len(detailItems), totalCount) + } + } + + currentData := s.fetchCurrentData(req, useMock) + if currentData != nil && currentData.TotalCount > 0 { + totalPages := (currentData.TotalCount + pageSize - 1) / pageSize + logrus.Infof("总记录数：%d, 总页数：%d, 当前页：%d/%d", + currentData.TotalCount, totalPages, currentPage, totalPages) + + if currentPage >= totalPages { + logrus.Infof("✓ 已同步所有页面数据，共 %d 页，%d 条记录", totalPages, currentData.TotalCount) + break + } + } + + if result.DetailCount < pageSize { + logrus.Infof("✓ 当前页数据不足 %d 条，已到达最后一页", pageSize) + break + } + + currentPage++ + time.Sleep(300 * time.Millisecond) + } + + if len(allDetailItems) > 0 { + logrus.Infof("开始批量保存 %d 条明细数据...", len(allDetailItems)) + detailResult, saveErr := s.saveDetailData(ctx, allDetailItems) + if saveErr != nil { + logrus.Errorf("批量保存明细数据失败：%v", saveErr) + aggregatedResult.Error = fmt.Errorf("批量保存明细数据失败：%w", saveErr) + } else { + aggregatedResult.DetailSuccess = true + aggregatedResult.DetailCount = len(allDetailItems) + aggregatedResult.DetailSuccessCount = detailResult.SuccessCount + aggregatedResult.DetailFailCount = detailResult.FailCount + logrus.Infof("✓ 批量保存明细数据完成，成功=%d, 失败=%d", + detailResult.SuccessCount, detailResult.FailCount) + } + } else { + logrus.Info("没有明细数据需要保存") + } + + return aggregatedResult, aggregatedResult.Error +} + +func (s *SyncService) extractDetailItems(req *CampaignReportRequest, useMock bool) []*dto.CidAccountReportDetailItem { + if useMock { + responseData := s.mockGen.GenerateCampaignReportResponse() + if responseData == nil || responseData.Data == nil || len(responseData.Data.Detail) == 0 { + return nil + } + return s.converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report") + } + + respBytes, err := s.httpClient.Post(context.Background(), "/rest/openapi/gw/esp/report/campaignReport", req) + if err != nil { + logrus.Errorf("重新获取数据失败：%v", err) + return nil + } + + responseData := &CampaignReportResponse{} + if err := json.Unmarshal(respBytes, responseData); err != nil { + logrus.Errorf("解析响应失败：%v", err) + return nil + } + + if responseData.Code != 0 || responseData.Data == nil || len(responseData.Data.Detail) == 0 { + return nil + } + + return s.converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report") +} + +func (s *SyncService) fetchCurrentData(req *CampaignReportRequest, useMock bool) *CampaignReportData { + if useMock { + responseData := s.mockGen.GenerateCampaignReportResponse() + if responseData != nil && responseData.Data != nil { + return responseData.Data + } + return nil + } + + respBytes, err := s.httpClient.Post(context.Background(), "/rest/openapi/gw/esp/report/campaignReport", req) + if err != nil { + return nil + } + + responseData := &CampaignReportResponse{} + if err := json.Unmarshal(respBytes, responseData); err != nil { + return nil + } + + if responseData.Code == 0 && responseData.Data != nil { + return responseData.Data + } + + return nil +} + +func (s *SyncService) saveSumData(ctx context.Context, item *dto.CidAccountReportSumItem) (*dto.CreateCidAccountReportSumRes, error) { + return copydata.CidAccountReportDetail.CreateSum(ctx, item) +} + +func (s *SyncService) saveDetailData(ctx context.Context, items []*dto.CidAccountReportDetailItem) (*dto.BatchCreateCidAccountReportDetailRes, error) { + req := &dto.BatchCreateCidAccountReportDetailReq{ + Items: items, + } + return copydata.CidAccountReportDetail.BatchCreate(ctx, req) +} + +func (s *SyncService) SyncWithRetry(ctx context.Context, req *CampaignReportRequest, useMock bool, maxRetries int) (*SyncResult, error) { + var lastResult *SyncResult + var lastErr error + + for attempt := 0; attempt <= maxRetries; attempt++ { + result, err := s.SyncCampaignReport(ctx, req, useMock) + lastResult = result + lastErr = err + + if err == nil { + logrus.Infof("同步成功，尝试次数：%d", attempt+1) + return result, nil + } + + logrus.Warnf("同步失败，第 %d 次重试，错误：%v", attempt+1, err) + } + + return lastResult, lastErr +} diff --git a/sync/sync_test.go b/sync/sync_test.go new file mode 100644 index 0000000..f74005a --- /dev/null +++ b/sync/sync_test.go @@ -0,0 +1,139 @@ +package sync + +import ( + "fmt" + "testing" + "time" + + _ "github.com/gogf/gf/contrib/drivers/pgsql/v2" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gctx" +) + +func init() { + fmt.Println("=== 初始化测试环境 ===") + ctx := gctx.New() + + db := g.DB() + if db != nil { + _, err := db.Query(ctx, "SELECT 1") + if err == nil { + fmt.Println("✓ 数据库连接成功") + } else { + fmt.Printf("⚠️ 数据库连接失败：%v\n", err) + fmt.Println("⚠️ 将跳过数据库相关测试") + } + } else { + fmt.Println("⚠️ 数据库未初始化") + } + fmt.Println("========================") +} + +func TestMockDataGeneration(t *testing.T) { + mockGen := NewMockDataGenerator() + + req := mockGen.GenerateCampaignReportRequest() + if req == nil { + t.Error("请求数据生成失败") + return + } + + fmt.Printf("✓ Mock 请求生成成功：AdvertiserID=%d\n", req.AdvertiserID) +} + +func TestDataConverter(t *testing.T) { + converter := NewDataConverter() + mockGen := NewMockDataGenerator() + + responseData := mockGen.GenerateCampaignReportResponse() + if responseData == nil || responseData.Data.Sum == nil { + t.Fatal("Mock 数据生成失败") + } + + sumItem := converter.ConvertToSumItem(responseData.Data.Sum, "campaign_report") + if sumItem == nil { + t.Fatal("转换为汇总数据失败") + } + + if sumItem.CampaignName == "" { + t.Error("计划名称为空") + } + + fmt.Printf("✓ 汇总数据转换成功：计划=%s\n", sumItem.CampaignName) + + detailItems := converter.ConvertToDetailItems(responseData.Data.Detail, "campaign_report") + if len(detailItems) == 0 { + t.Fatal("转换为明细数据失败") + } + + fmt.Printf("✓ 明细数据转换成功：数量=%d\n", len(detailItems)) +} + +func TestSyncCampaignReportWithDB(t *testing.T) { + ctx := gctx.New() + syncService := NewSyncService() + + req := &CampaignReportRequest{ + AdvertiserID: 10001, + StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6, + EndTime: time.Now().UnixNano() / 1e6, + SelectColumns: []string{"impression", "click", "cost", "t0GMV"}, + GroupType: 1, + QueryVersion: 1, + } + + result, err := syncService.SyncCampaignReport(ctx, req, true) + if err != nil { + t.Logf("同步失败（可能是数据库问题）: %v", err) + return + } + + fmt.Printf("✓ 同步结果：汇总成功=%v, 汇总 ID=%d, 明细数量=%d\n", + result.SumSuccess, result.SumID, result.DetailCount) +} + +// +//// TestScheduledSyncTask 测试定时同步任务（每小时执行一次，全量分页抽取） +//func TestScheduledSyncTask(t *testing.T) { +// ctx := gctx.New() +// syncService := NewSyncService() +// +// req := &CampaignReportRequest{ +// AdvertiserID: 10001, +// StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6, +// EndTime: time.Now().UnixNano() / 1e6, +// SelectColumns: []string{"impression", "click", "cost", "t0GMV"}, +// GroupType: 1, +// QueryVersion: 1, +// } +// +// logrus.Info("=== 开始执行定时同步任务 ===") +// result, err := syncService.SyncCampaignReportWithPagination(ctx, req, true, 3) +// if err != nil { +// t.Logf("定时同步任务失败：%v", err) +// return +// } +// +// fmt.Printf("✓ 定时同步完成：\n") +// fmt.Printf(" 汇总数据：成功=%v, ID=%d\n", result.SumSuccess, result.SumID) +// fmt.Printf(" 明细数据：总数=%d, 成功=%d, 失败=%d\n", +// result.DetailCount, result.DetailSuccessCount, result.DetailFailCount) +//} + +func BenchmarkSyncCampaignReport(b *testing.B) { + ctx := gctx.New() + syncService := NewSyncService() + req := &CampaignReportRequest{ + AdvertiserID: 10001, + StartTime: time.Now().AddDate(0, 0, -30).UnixNano() / 1e6, + EndTime: time.Now().UnixNano() / 1e6, + SelectColumns: []string{"impression", "click", "cost"}, + GroupType: 1, + QueryVersion: 1, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = syncService.SyncCampaignReport(ctx, req, true) + } +}