抽取数据添加补偿机制

2026-04-08 09:03:20 +08:00
parent 14db0dd2fe
commit 5f2c9c3855
19 changed files with 1058 additions and 178 deletions
--- a/dao/copydata/api_account_report_detail_dao.go
+++ b/dao/copydata/api_account_report_detail_dao.go
@@ -9,6 +9,7 @@ import (

 	"gitea.com/red-future/common/db/gfdb"
 	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/sirupsen/logrus"
 )

 var CidAccountReportDetail = new(cidAccountReportDetailDao)
@@ -29,7 +30,7 @@ func (d *cidAccountReportDetailDao) Insert(ctx context.Context, req *dto.CidAcco
 	return r.LastInsertId()
 }

-// BatchInsert 批量插入广告数据报表详情
+// BatchInsert 批量插入广告数据报表详情（使用 OnConflict 实现幂等性）
 func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto.CidAccountReportDetailItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
 	if len(reqs) == 0 {
 		return 0, 0, nil, errors.New("批量插入数据不能为空")
@@ -64,9 +65,19 @@ func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto
 			continue
 		}

-		// 执行批量插入
-		_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).Data(entityList).Insert()
+		// 执行批量插入，使用 OnConflict 实现幂等性
+		_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).
+			Data(entityList).
+			OnConflict(
+				"report_date_str",
+				"page_number",
+				"campaign_id",
+				"creative_id",
+			).
+			Save()
+
 		if err != nil {
+			logrus.Warnf("批量插入失败，尝试逐条插入: %v", err)
 			// 批量插入失败，尝试逐条插入
 			for k := range batch {
 				_, singleErr := d.Insert(ctx, batch[k])
@@ -84,3 +95,66 @@ func (d *cidAccountReportDetailDao) BatchInsert(ctx context.Context, reqs []*dto

 	return successCount, failCount, failedIndexes, nil
 }
+
+// DeleteByDateRange 按日期范围删除数据（用于补偿前去重）
+func (d *cidAccountReportDetailDao) DeleteByDateRange(ctx context.Context, advertiserID int64, startDateStr, endDateStr string) (int64, error) {
+	cols := (&entity.CidAccountReportDetail{}).GetCols()
+
+	result, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).
+		Where(cols.ReportDateStr+" >= ? AND "+cols.ReportDateStr+" <= ?", startDateStr, endDateStr).
+		Delete()
+
+	if err != nil {
+		return 0, err
+	}
+
+	affected, _ := result.RowsAffected()
+	return affected, nil
+}
+
+// BatchInsertInTx 在事务中批量插入
+func (d *cidAccountReportDetailDao) BatchInsertInTx(ctx context.Context, tx interface{}, reqs []*dto.CidAccountReportDetailItem) (successCount int64, failCount int64, err error) {
+	if len(reqs) == 0 {
+		return 0, 0, errors.New("批量插入数据不能为空")
+	}
+
+	batchSize := 100
+	successCount = 0
+	failCount = 0
+
+	for i := 0; i < len(reqs); i += batchSize {
+		end := i + batchSize
+		if end > len(reqs) {
+			end = len(reqs)
+		}
+
+		batch := reqs[i:end]
+		entityList := make([]*entity.CidAccountReportDetail, 0, len(batch))
+
+		for _, req := range batch {
+			var entityData entity.CidAccountReportDetail
+			if err = gconv.Struct(req, &entityData); err != nil {
+				failCount++
+				logrus.Errorf("数据转换失败: %v", err)
+				continue
+			}
+			entityList = append(entityList, &entityData)
+		}
+
+		if len(entityList) == 0 {
+			continue
+		}
+
+		_, txErr := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportDetailTable).Data(entityList).Insert()
+		if txErr != nil {
+			logrus.Errorf("批量插入失败 batch[%d:%d]: %v", i, end, txErr)
+			failCount += int64(len(entityList))
+			err = txErr
+			continue
+		}
+
+		successCount += int64(len(entityList))
+	}
+
+	return successCount, failCount, err
+}
--- a/dao/copydata/api_account_report_sum_dao.go
+++ b/dao/copydata/api_account_report_sum_dao.go
@@ -9,13 +9,14 @@ import (

 	"gitea.com/red-future/common/db/gfdb"
 	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/sirupsen/logrus"
 )

 var CidAccountReportSum = new(CidAccountReportSumDao)

 type CidAccountReportSumDao struct{}

-// Insert 插入广告数据报表详情
+// Insert 插入广告数据报表汇总
 func (d *CidAccountReportSumDao) Insert(ctx context.Context, req *dto.CidAccountReportSumItem) (id int64, err error) {
 	var entityData *entity.CidAccountReportSum
 	if err = gconv.Struct(req, &entityData); err != nil {
@@ -29,7 +30,7 @@ func (d *CidAccountReportSumDao) Insert(ctx context.Context, req *dto.CidAccount
 	return r.LastInsertId()
 }

-// BatchInsert 批量插入广告数据报表详情
+// BatchInsert 批量插入广告数据报表汇总（使用 OnConflict 实现幂等性）
 func (d *CidAccountReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.CidAccountReportSumItem) (successCount int64, failCount int64, failedIndexes []int64, err error) {
 	if len(reqs) == 0 {
 		return 0, 0, nil, errors.New("批量插入数据不能为空")
@@ -64,9 +65,17 @@ func (d *CidAccountReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.Ci
 			continue
 		}

-		// 执行批量插入
-		_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportSumTable).Data(entityList).Insert()
+		// 执行批量插入，使用 OnConflict 实现幂等性
+		_, err = gfdb.DB(ctx).Model(ctx, consts.CidAccountReportSumTable).
+			Data(entityList).
+			OnConflict(
+				"report_date_str",
+				"page_number",
+			).
+			Save()
+
 		if err != nil {
+			logrus.Warnf("批量插入失败，尝试逐条插入: %v", err)
 			// 批量插入失败，尝试逐条插入
 			for k := range batch {
 				_, singleErr := d.Insert(ctx, batch[k])
@@ -84,3 +93,19 @@ func (d *CidAccountReportSumDao) BatchInsert(ctx context.Context, reqs []*dto.Ci

 	return successCount, failCount, failedIndexes, nil
 }
+
+// DeleteByDateRange 按日期范围删除数据（用于补偿前去重）
+func (d *CidAccountReportSumDao) DeleteByDateRange(ctx context.Context, advertiserID int64, startDateStr, endDateStr string) (int64, error) {
+	cols := (&entity.CidAccountReportSum{}).GetCols()
+
+	result, err := gfdb.DB(ctx).Model(ctx, consts.CidAccountReportSumTable).
+		Where(cols.ReportDateStr+" >= ? AND "+cols.ReportDateStr+" <= ?", startDateStr, endDateStr).
+		Delete()
+
+	if err != nil {
+		return 0, err
+	}
+
+	affected, _ := result.RowsAffected()
+	return affected, nil
+}
--- a/dao/copydata/sync_task_log_dao.go
+++ b/dao/copydata/sync_task_log_dao.go
@@ -0,0 +1,126 @@
+package copydata
+
+import (
+	consts "cid/consts/public"
+	dto "cid/model/dto/copydata"
+	entity "cid/model/entity/copydata"
+	"context"
+	"time"
+
+	"gitea.com/red-future/common/db/gfdb"
+	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/util/gconv"
+)
+
+var SyncTaskLog = new(SyncTaskLogDao)
+
+type SyncTaskLogDao struct{}
+
+// Create 创建任务日志
+func (d *SyncTaskLogDao) Create(ctx context.Context, req *dto.CreateSyncTaskLogReq) (int64, error) {
+	var entityData entity.SyncTaskLog
+	if err := gconv.Struct(req, &entityData); err != nil {
+		return 0, err
+	}
+
+	r, err := gfdb.DB(ctx).Model(ctx, consts.SyncTaskLogTable).Data(&entityData).Insert()
+	if err != nil {
+		return 0, err
+	}
+	return r.LastInsertId()
+}
+
+// Update 更新任务日志
+func (d *SyncTaskLogDao) Update(ctx context.Context, req *dto.UpdateSyncTaskLogReq) error {
+	data := make(gdb.Map)
+
+	if req.Status != "" {
+		data["status"] = req.Status
+	}
+	if req.RetryCount != nil {
+		data["retry_count"] = *req.RetryCount
+	}
+	if req.ErrorMessage != "" {
+		data["error_message"] = req.ErrorMessage
+	}
+	if req.ErrorCode != "" {
+		data["error_code"] = req.ErrorCode
+	}
+	if req.ResultSummary != nil {
+		data["result_summary"] = req.ResultSummary
+	}
+	if req.NextRetryTime != nil {
+		data["next_retry_time"] = req.NextRetryTime
+	}
+	if req.CompletedAt != nil {
+		data["completed_at"] = req.CompletedAt
+	}
+	if req.DurationMs != nil {
+		data["duration_ms"] = *req.DurationMs
+	}
+
+	data["updated_at"] = time.Now()
+
+	_, err := gfdb.DB(ctx).Model(ctx, consts.SyncTaskLogTable).
+		Data(data).
+		Where("id", req.ID).
+		Update()
+
+	return err
+}
+
+// QueryFailedTasks 查询需要补偿的失败任务
+func (d *SyncTaskLogDao) QueryFailedTasks(ctx context.Context, req *dto.QueryFailedTasksReq) ([]*dto.SyncTaskLogItem, error) {
+	model := gfdb.DB(ctx).Model(ctx, consts.SyncTaskLogTable).Model
+
+	// 状态过滤
+	if len(req.Status) > 0 {
+		model = model.WhereIn("status", req.Status)
+	}
+
+	// 任务类型过滤
+	if req.TaskType != "" {
+		model = model.Where("task_type", req.TaskType)
+	}
+
+	// 只查询到达重试时间的任务（或从未设置过重试时间）
+	model = model.Where(
+		"(next_retry_time <= ? OR next_retry_time IS NULL)",
+		time.Now(),
+	)
+
+	// 限制数量
+	limit := req.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+	model = model.Limit(limit)
+
+	var results []*entity.SyncTaskLog
+	if err := model.Scan(&results); err != nil {
+		return nil, err
+	}
+
+	items := make([]*dto.SyncTaskLogItem, len(results))
+	for i, r := range results {
+		item := &dto.SyncTaskLogItem{}
+		gconv.Struct(r, item)
+		items[i] = item
+	}
+
+	return items, nil
+}
+
+// GetByTaskID 根据任务ID获取日志
+func (d *SyncTaskLogDao) GetByTaskID(ctx context.Context, taskID, taskType string) (*entity.SyncTaskLog, error) {
+	var result *entity.SyncTaskLog
+	err := gfdb.DB(ctx).Model(ctx, consts.SyncTaskLogTable).
+		Where("task_id", taskID).
+		Where("task_type", taskType).
+		Scan(&result)
+
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}