red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据引擎-快手平台数据抽取bug修复

Browse Source
This commit is contained in:
李明康
2026-06-16 10:44:10 +08:00
parent e5133eea34
commit b4fc6f54af
22 changed files with 1324 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
dao/dict/api_datasource_platform_dao.go
View File

@@ -5,11 +5,11 @@ import (
consts "dataengine/consts/public"
dto "dataengine/model/dto/dict"
entity "dataengine/model/entity/dict"
"time"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
)
@@ -18,17 +18,18 @@ var DatasourcePlatform = new(datasourcePlatformDao)
type datasourcePlatformDao struct{}
// Insert 插入数据源平台
func (d *datasourcePlatformDao) Insert(ctx context.Context, req *dto.CreateDatasourcePlatformReq, createdBy string) (ID int64, err error) {
func (d *datasourcePlatformDao) Insert(ctx context.Context, req *dto.CreateDatasourcePlatformReq, createdBy string, tenantId uint64) (ID int64, err error) {
var res *entity.DatasourcePlatform
if err = gconv.Struct(req, &res); err != nil {
return
}
// 设置创建人和时间（服务端生成，不依赖前端入参）
now := time.Now()
res.CreatedBy = createdBy
res.CreatedAt = &now
res.UpdatedBy = createdBy
res.UpdatedAt = &now
// 设置审计字段（服务端生成，不依赖前端入参）
now := gtime.Now()
res.TenantId = tenantId
res.Creator = createdBy
res.CreatedAt = now
res.Updater = createdBy
res.UpdatedAt = now
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).Data(&res).Insert()
if err != nil {
@@ -38,16 +39,17 @@ func (d *datasourcePlatformDao) Insert(ctx context.Context, req *dto.CreateDatas
}
// Update 更新数据源平台
func (d *datasourcePlatformDao) Update(ctx context.Context, req *dto.UpdateDatasourcePlatformReq, updatedBy string) (rows int64, err error) {
func (d *datasourcePlatformDao) Update(ctx context.Context, req *dto.UpdateDatasourcePlatformReq, updatedBy string, tenantId uint64) (rows int64, err error) {
// 设置更新人和更新时间（服务端生成，不依赖前端入参）
data := gconv.Map(req)
data[entity.DatasourcePlatformCols.UpdatedBy] = updatedBy
data[entity.DatasourcePlatformCols.UpdatedAt] = time.Now()
data[entity.DatasourcePlatformCols.Updater] = updatedBy
data[entity.DatasourcePlatformCols.UpdatedAt] = gtime.Now()
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Data(data).
OmitEmpty().
Where(entity.DatasourcePlatformCols.ID, req.Id).
Where(entity.DatasourcePlatformCols.Id, req.Id).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Update()
if err != nil {
return
@@ -56,9 +58,10 @@ func (d *datasourcePlatformDao) Update(ctx context.Context, req *dto.UpdateDatas
}
// Delete 删除数据源平台
func (d *datasourcePlatformDao) Delete(ctx context.Context, req *dto.DeleteDatasourcePlatformReq) (rows int64, err error) {
func (d *datasourcePlatformDao) Delete(ctx context.Context, req *dto.DeleteDatasourcePlatformReq, tenantId uint64) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.ID, req.Id).
Where(entity.DatasourcePlatformCols.Id, req.Id).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Delete()
if err != nil {
return
@@ -67,9 +70,10 @@ func (d *datasourcePlatformDao) Delete(ctx context.Context, req *dto.DeleteDatas
}
// GetOne 获取单个数据源平台
func (d *datasourcePlatformDao) GetOne(ctx context.Context, req *dto.GetDatasourcePlatformReq) (res *entity.DatasourcePlatform, err error) {
func (d *datasourcePlatformDao) GetOne(ctx context.Context, req *dto.GetDatasourcePlatformReq, tenantId uint64) (res *entity.DatasourcePlatform, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.ID, req.Id).
Where(entity.DatasourcePlatformCols.Id, req.Id).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
One()
if err != nil {
return
@@ -79,9 +83,10 @@ func (d *datasourcePlatformDao) GetOne(ctx context.Context, req *dto.GetDatasour
}
// GetByPlatformCode 根据平台编码获取数据源平台
func (d *datasourcePlatformDao) GetByPlatformCode(ctx context.Context, platformCode string) (res *entity.DatasourcePlatform, err error) {
func (d *datasourcePlatformDao) GetByPlatformCode(ctx context.Context, platformCode string, tenantId uint64) (res *entity.DatasourcePlatform, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.PlatformCode, platformCode).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
One()
if err != nil {
return
@@ -91,13 +96,13 @@ func (d *datasourcePlatformDao) GetByPlatformCode(ctx context.Context, platformC
}
// Count 获取数据源平台数量
func (d *datasourcePlatformDao) Count(ctx context.Context, req *dto.ListDatasourcePlatformReq) (count int, err error) {
return d.buildListFilter(ctx, req).Count()
func (d *datasourcePlatformDao) Count(ctx context.Context, req *dto.ListDatasourcePlatformReq, tenantId uint64) (count int, err error) {
return d.buildListFilter(ctx, req, tenantId).Count()
}
// List 获取数据源平台列表
func (d *datasourcePlatformDao) List(ctx context.Context, req *dto.ListDatasourcePlatformReq) (res []entity.DatasourcePlatform, total int, err error) {
model := d.buildListFilter(ctx, req)
func (d *datasourcePlatformDao) List(ctx context.Context, req *dto.ListDatasourcePlatformReq, tenantId uint64) (res []entity.DatasourcePlatform, total int, err error) {
model := d.buildListFilter(ctx, req, tenantId)
model.OrderDesc(entity.DatasourcePlatformCols.CreatedAt)
if req.Page != nil {
model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
@@ -111,9 +116,12 @@ func (d *datasourcePlatformDao) List(ctx context.Context, req *dto.ListDatasourc
}
// buildListFilter 构建列表查询的过滤条件
func (d *datasourcePlatformDao) buildListFilter(ctx context.Context, req *dto.ListDatasourcePlatformReq) *gdb.Model {
func (d *datasourcePlatformDao) buildListFilter(ctx context.Context, req *dto.ListDatasourcePlatformReq, tenantId uint64) *gdb.Model {
model := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).Model
// 租户隔离
model.Where(entity.DatasourcePlatformCols.TenantId, tenantId)
// 关键字搜索（平台名称或编码）
if !g.IsEmpty(req.Keyword) {
model.WhereLike(entity.DatasourcePlatformCols.PlatformName, "%"+req.Keyword+"%")
@@ -140,14 +148,15 @@ func (d *datasourcePlatformDao) buildListFilter(ctx context.Context, req *dto.Li
}
// UpdateStatus 更新数据源平台状态
func (d *datasourcePlatformDao) UpdateStatus(ctx context.Context, ID int64, status string, updatedBy string) (rows int64, err error) {
func (d *datasourcePlatformDao) UpdateStatus(ctx context.Context, ID int64, status string, updatedBy string, tenantId uint64) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Data(map[string]interface{}{
entity.DatasourcePlatformCols.Status: status,
entity.DatasourcePlatformCols.UpdatedBy: updatedBy,
entity.DatasourcePlatformCols.UpdatedAt: time.Now(),
entity.DatasourcePlatformCols.Updater: updatedBy,
entity.DatasourcePlatformCols.UpdatedAt: gtime.Now(),
}).
Where(entity.DatasourcePlatformCols.ID, ID).
Where(entity.DatasourcePlatformCols.Id, ID).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Update()
if err != nil {
return
@@ -156,12 +165,13 @@ func (d *datasourcePlatformDao) UpdateStatus(ctx context.Context, ID int64, stat
}
// ExistsByPlatformCode 检查平台编码是否存在
func (d *datasourcePlatformDao) ExistsByPlatformCode(ctx context.Context, platformCode string, excludeId ...int64) (exists bool, err error) {
func (d *datasourcePlatformDao) ExistsByPlatformCode(ctx context.Context, platformCode string, tenantId uint64, excludeId ...int64) (exists bool, err error) {
model := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.PlatformCode, platformCode)
Where(entity.DatasourcePlatformCols.PlatformCode, platformCode).
Where(entity.DatasourcePlatformCols.TenantId, tenantId)
if len(excludeId) > 0 && excludeId[0] > 0 {
model.WhereNot(entity.DatasourcePlatformCols.ID, excludeId[0])
model.WhereNot(entity.DatasourcePlatformCols.Id, excludeId[0])
}
count, err := model.Count()
@@ -172,9 +182,10 @@ func (d *datasourcePlatformDao) ExistsByPlatformCode(ctx context.Context, platfo
}
// ListActivePlatforms 获取所有启用的平台
func (d *datasourcePlatformDao) ListActivePlatforms(ctx context.Context) (res []entity.DatasourcePlatform, err error) {
func (d *datasourcePlatformDao) ListActivePlatforms(ctx context.Context, tenantId uint64) (res []entity.DatasourcePlatform, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.Status, "ACTIVE").
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
OrderAsc(entity.DatasourcePlatformCols.PlatformName).
All()
if err != nil {
@@ -185,11 +196,13 @@ func (d *datasourcePlatformDao) ListActivePlatforms(ctx context.Context) (res []
}
// GetPlatformStatistics 获取平台统计信息
func (d *datasourcePlatformDao) GetPlatformStatistics(ctx context.Context) (stats map[string]int64, err error) {
func (d *datasourcePlatformDao) GetPlatformStatistics(ctx context.Context, tenantId uint64) (stats map[string]int64, err error) {
stats = make(map[string]int64)
// 总平台数
total, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).Count()
total, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Count()
if err != nil {
return nil, err
}
@@ -198,6 +211,7 @@ func (d *datasourcePlatformDao) GetPlatformStatistics(ctx context.Context) (stat
// 启用平台数
active, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.Status, "ACTIVE").
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Count()
if err != nil {
return nil, err
@@ -212,6 +226,7 @@ func (d *datasourcePlatformDao) GetPlatformStatistics(ctx context.Context) (stat
for _, authType := range authTypes {
count, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Where(entity.DatasourcePlatformCols.AuthType, authType).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Count()
if err != nil {
return nil, err
@@ -223,14 +238,15 @@ func (d *datasourcePlatformDao) GetPlatformStatistics(ctx context.Context) (stat
}
// BatchUpdateStatus 批量更新平台状态
func (d *datasourcePlatformDao) BatchUpdateStatus(ctx context.Context, ids []int64, status string, updatedBy string) (rows int64, err error) {
func (d *datasourcePlatformDao) BatchUpdateStatus(ctx context.Context, ids []int64, status string, updatedBy string, tenantId uint64) (rows int64, err error) {
r, err := gfdb.DB(ctx).Model(ctx, consts.DatasourcePlatformTable).
Data(map[string]interface{}{
entity.DatasourcePlatformCols.Status: status,
entity.DatasourcePlatformCols.UpdatedBy: updatedBy,
entity.DatasourcePlatformCols.UpdatedAt: time.Now(),
entity.DatasourcePlatformCols.Updater: updatedBy,
entity.DatasourcePlatformCols.UpdatedAt: gtime.Now(),
}).
WhereIn(entity.DatasourcePlatformCols.ID, ids).
WhereIn(entity.DatasourcePlatformCols.Id, ids).
Where(entity.DatasourcePlatformCols.TenantId, tenantId).
Update()
if err != nil {
return