red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据引擎-快手平台数据抽取bug修复

Browse Source
This commit is contained in:
李明康
2026-06-16 10:44:10 +08:00
parent e5133eea34
commit b4fc6f54af
22 changed files with 1324 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
service/dict/api_datasource_platform_service.go
View File

@@ -7,8 +7,8 @@ import (
dto "dataengine/model/dto/dict"
entity "dataengine/model/entity/dict"
"dataengine/utils"
"time"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
"github.com/olekukonko/errors"
)
@@ -20,8 +20,10 @@ var DatasourcePlatform = new(datasourcePlatformService)
// Create 创建数据源平台
func (s *datasourcePlatformService) Create(ctx context.Context, req *dto.CreateDatasourcePlatformReq) (res *dto.CreateDatasourcePlatformRes, err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台编码是否重复
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode)
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode, tenantId)
if err != nil {
return nil, err
}
@@ -38,7 +40,7 @@ func (s *datasourcePlatformService) Create(ctx context.Context, req *dto.CreateD
currentUser := utils.GetCurrentUser(ctx)
// 插入数据库
id, err := dao.DatasourcePlatform.Insert(ctx, req, currentUser)
id, err := dao.DatasourcePlatform.Insert(ctx, req, currentUser, tenantId)
if err != nil {
return nil, err
}
@@ -51,7 +53,8 @@ func (s *datasourcePlatformService) Create(ctx context.Context, req *dto.CreateD
// List 获取数据源平台列表
func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatasourcePlatformReq) (res *dto.ListDatasourcePlatformRes, err error) {
platformList, total, err := dao.DatasourcePlatform.List(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
platformList, total, err := dao.DatasourcePlatform.List(ctx, req, tenantId)
if err != nil {
return nil, err
}
@@ -60,7 +63,8 @@ func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatas
list := make([]dto.DatasourcePlatformItem, 0, len(platformList))
for _, item := range platformList {
list = append(list, dto.DatasourcePlatformItem{
Id: item.ID,
Id: item.Id,
TenantId: item.TenantId,
PlatformCode: item.PlatformCode,
PlatformName: item.PlatformName,
Description: item.Description,
@@ -75,9 +79,9 @@ func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatas
RequestTimeoutMs: item.RequestTimeoutMs,
MaxRetries: item.MaxRetries,
RetryDelayMs: item.RetryDelayMs,
CreatedBy: item.CreatedBy,
CreatedBy: item.Creator,
CreatedAt: s.safeUnix(item.CreatedAt),
UpdatedBy: item.UpdatedBy,
UpdatedBy: item.Updater,
UpdatedAt: s.safeUnix(item.UpdatedAt),
})
}
@@ -91,7 +95,8 @@ func (s *datasourcePlatformService) List(ctx context.Context, req *dto.ListDatas
// GetOne 获取单个数据源平台
func (s *datasourcePlatformService) GetOne(ctx context.Context, req *dto.GetDatasourcePlatformReq) (res *dto.GetDatasourcePlatformRes, err error) {
platform, err := dao.DatasourcePlatform.GetOne(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
platform, err := dao.DatasourcePlatform.GetOne(ctx, req, tenantId)
if err != nil {
return nil, err
}
@@ -104,10 +109,9 @@ func (s *datasourcePlatformService) GetOne(ctx context.Context, req *dto.GetData
return nil, err
}
// 隐藏敏感信息
platformEntity.Token = ""
platformEntity.ClientSecret = ""
platformEntity.ApiKey = ""
// 注意：编辑时需要保留 Token/ClientSecret/ApiKey，否则前端表单回填为空，
// 用户保存时会覆盖已有凭据。敏感字段在 List 接口的 DatasourcePlatformItem 中
// 已不返回（DTO 不含这些字段），此处保持完整。
return &dto.GetDatasourcePlatformRes{
DatasourcePlatform: platformEntity,
@@ -116,7 +120,8 @@ func (s *datasourcePlatformService) GetOne(ctx context.Context, req *dto.GetData
// GetByPlatformCode 根据平台编码获取数据源平台
func (s *datasourcePlatformService) GetByPlatformCode(ctx context.Context, platformCode string) (res *entity.DatasourcePlatform, err error) {
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode)
tenantId := utils.GetCurrentTenantId(ctx)
platform, err := dao.DatasourcePlatform.GetByPlatformCode(ctx, platformCode, tenantId)
if err != nil {
return nil, err
}
@@ -128,8 +133,10 @@ func (s *datasourcePlatformService) GetByPlatformCode(ctx context.Context, platf
// Update 更新数据源平台
func (s *datasourcePlatformService) Update(ctx context.Context, req *dto.UpdateDatasourcePlatformReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台是否存在
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id})
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id}, tenantId)
if err != nil {
return err
}
@@ -139,7 +146,7 @@ func (s *datasourcePlatformService) Update(ctx context.Context, req *dto.UpdateD
// 如果修改了平台编码,检查新编码是否重复
if req.PlatformCode != "" && req.PlatformCode != exist.PlatformCode {
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode, req.Id)
exists, err := dao.DatasourcePlatform.ExistsByPlatformCode(ctx, req.PlatformCode, tenantId, req.Id)
if err != nil {
return err
}
@@ -164,14 +171,16 @@ func (s *datasourcePlatformService) Update(ctx context.Context, req *dto.UpdateD
// 从 context 中获取当前用户
currentUser := utils.GetCurrentUser(ctx)
_, err = dao.DatasourcePlatform.Update(ctx, req, currentUser)
_, err = dao.DatasourcePlatform.Update(ctx, req, currentUser, tenantId)
return err
}
// UpdateStatus 更新数据源平台状态
func (s *datasourcePlatformService) UpdateStatus(ctx context.Context, req *dto.UpdateDatasourcePlatformStatusReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台是否存在
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id})
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id}, tenantId)
if err != nil {
return err
}
@@ -185,14 +194,16 @@ func (s *datasourcePlatformService) UpdateStatus(ctx context.Context, req *dto.U
}
currentUser := utils.GetCurrentUser(ctx)
_, err = dao.DatasourcePlatform.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser)
_, err = dao.DatasourcePlatform.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser, tenantId)
return err
}
// Delete 删除数据源平台
func (s *datasourcePlatformService) Delete(ctx context.Context, req *dto.DeleteDatasourcePlatformReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
// 检查平台是否存在
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id})
exist, err := dao.DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.Id}, tenantId)
if err != nil {
return err
}
@@ -203,13 +214,14 @@ func (s *datasourcePlatformService) Delete(ctx context.Context, req *dto.DeleteD
// TODO: 检查是否存在关联的数据，防止误删
// 例如：检查该平台是否有关联的接口配置等
_, err = dao.DatasourcePlatform.Delete(ctx, req)
_, err = dao.DatasourcePlatform.Delete(ctx, req, tenantId)
return err
}
// GetStatistics 获取平台统计信息
func (s *datasourcePlatformService) GetStatistics(ctx context.Context) (res *dto.GetPlatformStatisticsRes, err error) {
stats, err := dao.DatasourcePlatform.GetPlatformStatistics(ctx)
tenantId := utils.GetCurrentTenantId(ctx)
stats, err := dao.DatasourcePlatform.GetPlatformStatistics(ctx, tenantId)
if err != nil {
return nil, err
}
@@ -228,7 +240,8 @@ func (s *datasourcePlatformService) GetStatistics(ctx context.Context) (res *dto
// ListActivePlatforms 获取所有启用的平台
func (s *datasourcePlatformService) ListActivePlatforms(ctx context.Context) (platforms []entity.DatasourcePlatform, err error) {
return dao.DatasourcePlatform.ListActivePlatforms(ctx)
tenantId := utils.GetCurrentTenantId(ctx)
return dao.DatasourcePlatform.ListActivePlatforms(ctx, tenantId)
}
// validateAuthFields 验证认证类型相关的必填字段
@@ -292,8 +305,8 @@ func (s *datasourcePlatformService) getAuthTypeName(authType string) string {
return authType
}
// safeUnix 安全地从 *time.Time 获取 Unix 时间戳，nil 返回 0
func (s *datasourcePlatformService) safeUnix(t *time.Time) int64 {
// safeUnix 安全地从 *gtime.Time 获取 Unix 时间戳，nil 返回 0
func (s *datasourcePlatformService) safeUnix(t *gtime.Time) int64 {
if t == nil {
return 0
}
@@ -301,11 +314,13 @@ func (s *datasourcePlatformService) safeUnix(t *time.Time) int64 {
}
// BatchUpdateStatus 批量更新平台状态
func (s *datasourcePlatformService) BatchUpdateStatus(ctx context.Context, ids []int64, status string, updatedBy string) (err error) {
func (s *datasourcePlatformService) BatchUpdateStatus(ctx context.Context, ids []int64, status string) (err error) {
if len(ids) == 0 {
return errors.New("请选择要更新的平台")
}
_, err = dao.DatasourcePlatform.BatchUpdateStatus(ctx, ids, status, updatedBy)
tenantId := utils.GetCurrentTenantId(ctx)
currentUser := utils.GetCurrentUser(ctx)
_, err = dao.DatasourcePlatform.BatchUpdateStatus(ctx, ids, status, currentUser, tenantId)
return err
}

29
service/dict/api_interface_service.go
View File

@@ -19,6 +19,8 @@ var ApiInterface = new(apiInterfaceService)
// Create 创建接口
func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInterfaceReq) (res *dto.CreateApiInterfaceRes, err error) {
tenantId := utils.GetCurrentTenantId(ctx)
_, err = DatasourcePlatform.GetOne(ctx, &dto.GetDatasourcePlatformReq{Id: req.PlatformId})
if err != nil {
return nil, errors.New("平台不存在")
@@ -28,7 +30,7 @@ func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInte
interfaces, _, err := dict.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: req.PlatformId,
Code: req.Code,
})
}, tenantId)
if err != nil {
return
}
@@ -40,7 +42,7 @@ func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInte
currentUser := utils.GetCurrentUser(ctx)
// 插入数据库
id, err := dict.ApiInterface.Insert(ctx, req, currentUser)
id, err := dict.ApiInterface.Insert(ctx, req, currentUser, tenantId)
if err != nil {
return
}
@@ -52,7 +54,8 @@ func (s *apiInterfaceService) Create(ctx context.Context, req *dto.CreateApiInte
// List 获取接口列表
func (s *apiInterfaceService) List(ctx context.Context, req *dto.ListApiInterfaceReq) (res *dto.ListApiInterfaceRes, err error) {
apiList, total, err := dict.ApiInterface.List(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
apiList, total, err := dict.ApiInterface.List(ctx, req, tenantId)
if err != nil {
return
}
@@ -107,7 +110,8 @@ func (s *apiInterfaceService) List(ctx context.Context, req *dto.ListApiInterfac
// GetOne 获取单个接口
func (s *apiInterfaceService) GetOne(ctx context.Context, req *dto.GetApiInterfaceReq) (res *dto.GetApiInterfaceRes, err error) {
apiInterface, err := dict.ApiInterface.GetOne(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
apiInterface, err := dict.ApiInterface.GetOne(ctx, req, tenantId)
if err != nil {
return
}
@@ -128,7 +132,9 @@ func (s *apiInterfaceService) GetOne(ctx context.Context, req *dto.GetApiInterfa
// Update 更新接口
func (s *apiInterfaceService) Update(ctx context.Context, req *dto.UpdateApiInterfaceReq) (err error) {
exist, err := dict.ApiInterface.GetOne(ctx, &dto.GetApiInterfaceReq{Id: req.Id})
tenantId := utils.GetCurrentTenantId(ctx)
exist, err := dict.ApiInterface.GetOne(ctx, &dto.GetApiInterfaceReq{Id: req.Id}, tenantId)
if err != nil || exist == nil {
return errors.New("接口不存在")
}
@@ -148,7 +154,7 @@ func (s *apiInterfaceService) Update(ctx context.Context, req *dto.UpdateApiInte
interfaces, _, err := dict.ApiInterface.List(ctx, &dto.ListApiInterfaceReq{
PlatformId: platformId,
Code: req.Code,
})
}, tenantId)
if err != nil {
return err
}
@@ -158,26 +164,29 @@ func (s *apiInterfaceService) Update(ctx context.Context, req *dto.UpdateApiInte
}
currentUser := utils.GetCurrentUser(ctx)
_, err = dict.ApiInterface.Update(ctx, req, currentUser)
_, err = dict.ApiInterface.Update(ctx, req, currentUser, tenantId)
return
}
// UpdateStatus 更新接口状态
func (s *apiInterfaceService) UpdateStatus(ctx context.Context, req *dto.UpdateApiInterfaceStatusReq) (err error) {
tenantId := utils.GetCurrentTenantId(ctx)
currentUser := utils.GetCurrentUser(ctx)
_, err = dict.ApiInterface.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser)
_, err = dict.ApiInterface.UpdateStatus(ctx, req.Id, req.Status.String(), currentUser, tenantId)
return
}
// Delete 删除接口
func (s *apiInterfaceService) Delete(ctx context.Context, req *dto.DeleteApiInterfaceReq) (err error) {
_, err = dict.ApiInterface.Delete(ctx, req)
tenantId := utils.GetCurrentTenantId(ctx)
_, err = dict.ApiInterface.Delete(ctx, req, tenantId)
return
}
// GetByIds 根据ID列表获取接口
func (s *apiInterfaceService) GetByIds(ctx context.Context, ids []int64) (res []entity.ApiInterface, err error) {
return dict.ApiInterface.GetByIds(ctx, ids)
tenantId := utils.GetCurrentTenantId(ctx)
return dict.ApiInterface.GetByIds(ctx, ids, tenantId)
}
// getStatusName 获取状态名称