red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据引擎-快手平台数据抽取bug修复

Browse Source
This commit is contained in:
李明康
2026-06-16 10:44:10 +08:00
parent e5133eea34
commit b4fc6f54af
22 changed files with 1324 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
common/report/api.go
View File

@@ -11,6 +11,7 @@ import (
"dataengine/common/report/extract"
"dataengine/common/report/model"
"gitea.redpowerfuture.com/red-future/common/beans"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
)
@@ -174,13 +175,12 @@ func (s *ReportService) SaveBusiness(ctx context.Context, req *model.SaveBusines
}
biz := &model.BusinessConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
BusinessCode: req.BusinessCode,
BusinessName: req.BusinessName,
Description: req.Description,
Status: req.Status,
Config: req.Config,
Creator: req.Operator,
Updater: req.Operator,
}
if req.Status == "" {
@@ -192,7 +192,7 @@ func (s *ReportService) SaveBusiness(ctx context.Context, req *model.SaveBusines
if req.ID != nil && *req.ID > 0 {
// 更新
biz.ID = *req.ID
biz.Id = *req.ID
if err := s.configLoader.UpdateBusiness(ctx, biz); err != nil {
return nil, err
}
@@ -235,6 +235,7 @@ func (s *ReportService) SaveReport(ctx context.Context, req *model.SaveReportReq
}
rpt := &model.ReportConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
BusinessCode: req.BusinessCode,
ReportCode: req.ReportCode,
ReportName: req.ReportName,
@@ -246,8 +247,6 @@ func (s *ReportService) SaveReport(ctx context.Context, req *model.SaveReportReq
PrimaryKeys: req.PrimaryKeys,
ConflictKeys: req.ConflictKeys,
Config: req.Config,
Creator: req.Operator,
Updater: req.Operator,
}
if req.Status == "" {
@@ -267,7 +266,7 @@ func (s *ReportService) SaveReport(ctx context.Context, req *model.SaveReportReq
}
if req.ID != nil && *req.ID > 0 {
rpt.ID = *req.ID
rpt.Id = *req.ID
if err := s.configLoader.UpdateReport(ctx, rpt); err != nil {
return nil, err
}
@@ -309,6 +308,7 @@ func (s *ReportService) SaveField(ctx context.Context, req *model.SaveFieldReq)
}
field := &model.FieldConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
BusinessCode: req.BusinessCode,
ReportCode: req.ReportCode,
FieldCode: req.FieldCode,
@@ -331,8 +331,6 @@ func (s *ReportService) SaveField(ctx context.Context, req *model.SaveFieldReq)
SortOrder: req.SortOrder,
GroupName: req.GroupName,
Status: req.Status,
Creator: req.Operator,
Updater: req.Operator,
}
if req.Status == "" {
@@ -349,7 +347,7 @@ func (s *ReportService) SaveField(ctx context.Context, req *model.SaveFieldReq)
}
if req.ID != nil && *req.ID > 0 {
field.ID = *req.ID
field.Id = *req.ID
if err := s.configLoader.UpdateField(ctx, field); err != nil {
return nil, err
}
@@ -391,6 +389,7 @@ func (s *ReportService) SaveExtractConfig(ctx context.Context, req *model.SaveEx
}
ec := &model.ExtractConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
BusinessCode: req.BusinessCode,
ReportCode: req.ReportCode,
ExtractCode: req.ExtractCode,
@@ -410,8 +409,6 @@ func (s *ReportService) SaveExtractConfig(ctx context.Context, req *model.SaveEx
TransformRules: req.TransformRules,
BatchSize: req.BatchSize,
Status: req.Status,
Creator: req.Operator,
Updater: req.Operator,
}
if req.Status == "" {
@@ -440,7 +437,7 @@ func (s *ReportService) SaveExtractConfig(ctx context.Context, req *model.SaveEx
}
if req.ID != nil && *req.ID > 0 {
ec.ID = *req.ID
ec.Id = *req.ID
if err := s.configLoader.UpdateExtractConfig(ctx, ec); err != nil {
return nil, err
}

12
common/report/config/loader.go
View File

@@ -323,7 +323,7 @@ func (l *ConfigLoader) UpdateBusiness(ctx context.Context, biz *model.BusinessCo
delete(m, "deleted_at")
_, err := gfdb.DB(ctx).Model(ctx, "report_business_config").
Where("id", biz.ID).
Where("id", biz.Id).
Data(m).
Update()
if err != nil {
@@ -368,8 +368,8 @@ func (l *ConfigLoader) GetBusinessByID(ctx context.Context, id int64) (*model.Bu
if err = r.Struct(&biz); err != nil {
return nil, err
}
g.Log().Infof(ctx, "[GetBusinessByID] id=%d, biz.ID=%d, biz.BusinessCode=%s",
id, biz.ID, biz.BusinessCode)
g.Log().Infof(ctx, "[GetBusinessByID] id=%d, biz.Id=%d, biz.BusinessCode=%s",
id, biz.Id, biz.BusinessCode)
return &biz, nil
}
@@ -406,7 +406,7 @@ func (l *ConfigLoader) UpdateReport(ctx context.Context, rpt *model.ReportConfig
delete(m, "deleted_at")
_, err := gfdb.DB(ctx).Model(ctx, "report_report_config").
Where("id", rpt.ID).
Where("id", rpt.Id).
Data(m).
Update()
if err != nil {
@@ -486,7 +486,7 @@ func (l *ConfigLoader) UpdateField(ctx context.Context, field *model.FieldConfig
delete(m, "deleted_at")
_, err := gfdb.DB(ctx).Model(ctx, "report_field_config").
Where("id", field.ID).
Where("id", field.Id).
Data(m).
Update()
if err != nil {
@@ -566,7 +566,7 @@ func (l *ConfigLoader) UpdateExtractConfig(ctx context.Context, ec *model.Extrac
delete(m, "deleted_at")
_, err := gfdb.DB(ctx).Model(ctx, "report_extract_config").
Where("id", ec.ID).
Where("id", ec.Id).
Data(m).
Update()
if err != nil {

8
common/report/example_usage.go
View File

@@ -317,7 +317,7 @@ func CRUDExample() {
BusinessCode: "DOUYIN", BusinessName: "抖音电商",
Operator: "admin",
})
businessId := result.ID
businessId := result.Id
// 修改（传 ID 即修改）
result, _ = svc.SaveBusiness(ctx, &model.SaveBusinessReq{
@@ -348,7 +348,7 @@ func CRUDExample() {
ConflictKeys: []string{"shop_id", "stat_date"},
Operator: "admin",
})
reportId := reportResult.ID
reportId := reportResult.Id
rpt, _ := svc.GetReport(ctx, reportId)
svc.DeleteReport(ctx, reportId)
@@ -365,7 +365,7 @@ func CRUDExample() {
ValidAggregates: []string{"SUM", "AVG", "MAX", "MIN"},
SortOrder: 10, GroupName: "金额", Operator: "admin",
})
fieldId := fieldResult.ID
fieldId := fieldResult.Id
// 修改字段（传 id = 更新）
svc.SaveField(ctx, &model.SaveFieldReq{
@@ -395,7 +395,7 @@ func CRUDExample() {
},
Operator: "admin",
})
ecId := ecResult.ID
ecId := ecResult.Id
ec, _ := svc.GetExtractConfig(ctx, ecId)
allEc, _ := svc.GetExtractConfigs(ctx, "DOUYIN", "shop_daily_report")

91
common/report/model/model.go
View File

@@ -2,6 +2,8 @@ package model
import (
"time"
"gitea.redpowerfuture.com/red-future/common/beans"
)
// ============================================================
@@ -10,24 +12,17 @@ import (
// BusinessConfig 业务配置
type BusinessConfig struct {
ID int64 `orm:"id" json:"id"`
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
BusinessCode string `orm:"business_code" json:"businessCode"`
BusinessName string `orm:"business_name" json:"businessName"`
Description string `orm:"description" json:"description"`
Status string `orm:"status" json:"status"`
Config map[string]interface{} `orm:"config" json:"config"`
Creator string `orm:"creator" json:"creator"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
Updater string `orm:"updater" json:"updater"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
BusinessName string `orm:"business_name" json:"businessName"`
Description string `orm:"description" json:"description"`
Status string `orm:"status" json:"status"`
Config map[string]interface{} `orm:"config" json:"config"`
}
// ReportConfig 报表配置
type ReportConfig struct {
ID int64 `orm:"id" json:"id"`
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
ReportName string `orm:"report_name" json:"reportName"`
@@ -39,50 +34,38 @@ type ReportConfig struct {
PrimaryKeys []string `orm:"primary_keys" json:"primaryKeys"`
ConflictKeys []string `orm:"conflict_keys" json:"conflictKeys"`
Config map[string]interface{} `orm:"config" json:"config"`
Creator string `orm:"creator" json:"creator"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
Updater string `orm:"updater" json:"updater"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
}
// FieldConfig 字段配置
type FieldConfig struct {
ID int64 `orm:"id" json:"id"`
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
FieldCode string `orm:"field_code" json:"fieldCode"`
FieldName string `orm:"field_name" json:"fieldName"`
FieldType string `orm:"field_type" json:"fieldType"`
DataType string `orm:"data_type" json:"dataType"`
FieldRole string `orm:"field_role" json:"fieldRole"`
IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable"`
IsFilterable bool `orm:"is_filterable" json:"isFilterable"`
IsQueryable bool `orm:"is_queryable" json:"isQueryable"`
IsSortable bool `orm:"is_sortable" json:"isSortable"`
DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate"`
ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates"`
FilterOperators []string `orm:"filter_operators" json:"filterOperators"`
Expression string `orm:"expression" json:"expression"`
ExpressionType string `orm:"expression_type" json:"expressionType"`
FormatPattern string `orm:"format_pattern" json:"formatPattern"`
Unit string `orm:"unit" json:"unit"`
DictCode string `orm:"dict_code" json:"dictCode"`
SortOrder int `orm:"sort_order" json:"sortOrder"`
GroupName string `orm:"group_name" json:"groupName"`
Status string `orm:"status" json:"status"`
Creator string `orm:"creator" json:"creator"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
Updater string `orm:"updater" json:"updater"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
FieldCode string `orm:"field_code" json:"fieldCode"`
FieldName string `orm:"field_name" json:"fieldName"`
FieldType string `orm:"field_type" json:"fieldType"`
DataType string `orm:"data_type" json:"dataType"`
FieldRole string `orm:"field_role" json:"fieldRole"`
IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable"`
IsFilterable bool `orm:"is_filterable" json:"isFilterable"`
IsQueryable bool `orm:"is_queryable" json:"isQueryable"`
IsSortable bool `orm:"is_sortable" json:"isSortable"`
DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate"`
ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates"`
FilterOperators []string `orm:"filter_operators" json:"filterOperators"`
Expression string `orm:"expression" json:"expression"`
ExpressionType string `orm:"expression_type" json:"expressionType"`
FormatPattern string `orm:"format_pattern" json:"formatPattern"`
Unit string `orm:"unit" json:"unit"`
DictCode string `orm:"dict_code" json:"dictCode"`
SortOrder int `orm:"sort_order" json:"sortOrder"`
GroupName string `orm:"group_name" json:"groupName"`
Status string `orm:"status" json:"status"`
}
// ExtractConfig 抽取配置
type ExtractConfig struct {
ID int64 `orm:"id" json:"id"`
TenantId uint64 `orm:"tenant_id" json:"tenant_id"`
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
ExtractCode string `orm:"extract_code" json:"extractCode"`
@@ -102,16 +85,12 @@ type ExtractConfig struct {
TransformRules []TransformRule `orm:"transform_rules" json:"transformRules"`
BatchSize int `orm:"batch_size" json:"batchSize"`
Status string `orm:"status" json:"status"`
Creator string `orm:"creator" json:"creator"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
Updater string `orm:"updater" json:"updater"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
DeletedAt *time.Time `orm:"deleted_at" json:"deletedAt"`
}
// ExtractLog 抽取记录
// ExtractLog 抽取记录（不含 Creator/Updater/DeletedAt，表结构不同）
type ExtractLog struct {
ID int64 `orm:"id" json:"id"`
TenantId uint64 `orm:"tenant_id" json:"tenantId"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
ExtractCode string `orm:"extract_code" json:"extractCode"`