red-future/data-engine
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据引擎-快手平台数据抽取bug修复

Browse Source
This commit is contained in:
李明康
2026-06-16 10:44:10 +08:00
parent e5133eea34
commit b4fc6f54af
22 changed files with 1324 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
model/dto/dict/api_datasource_platform_dto.go
View File

@@ -54,6 +54,7 @@ type ListDatasourcePlatformRes struct {
type DatasourcePlatformItem struct {
Id int64 `json:"id,string"`
TenantId uint64 `json:"tenantId"`
PlatformCode string `json:"platformCode"`
PlatformName string `json:"platformName"`
Description string `json:"description"`

70
model/entity/dict/api_datasource_platform.go
View File

@@ -1,52 +1,36 @@
package dict
import (
"time"
"gitea.redpowerfuture.com/red-future/common/beans"
)
// DatasourcePlatform 数据源平台配置实体
type DatasourcePlatform struct {
// 主键和标识字段
ID int64 `orm:"id" json:"id" description:"主键ID，自增长"`
PlatformCode string `orm:"platform_code" json:"platformCode" description:"平台编码，唯一标识"`
PlatformName string `orm:"platform_name" json:"platformName" description:"平台名称"`
Description string `orm:"description" json:"description" description:"平台描述"`
// 状态和基础配置
Status string `orm:"status" json:"status" description:"状态: ACTIVE启用, INACTIVE停用"`
ApiBaseUrl string `orm:"api_base_url" json:"apiBaseUrl" description:"API基础地址"`
// 认证配置
AuthType string `orm:"auth_type" json:"authType" description:"认证类型: TOKEN/API_KEY/OAUTH2/BASIC"`
Token string `orm:"token" json:"token" description:"认证token/密钥"`
ApiKey string `orm:"api_key" json:"apiKey" description:"API Key"`
ClientId string `orm:"client_id" json:"clientId" description:"OAuth2 Client ID"`
ClientSecret string `orm:"client_secret" json:"clientSecret" description:"OAuth2 Client Secret"`
// 限流配置
RateLimitPerMinute int `orm:"rate_limit_per_minute" json:"rateLimitPerMinute" description:"每分钟请求限制"`
RateLimitPerHour int `orm:"rate_limit_per_hour" json:"rateLimitPerHour" description:"每小时请求限制"`
ConcurrencyLimit int `orm:"concurrency_limit" json:"concurrencyLimit" description:"并发连接限制"`
RequestTimeoutMs int `orm:"request_timeout_ms" json:"requestTimeoutMs" description:"请求超时时间(毫秒)"`
// 重试策略
MaxRetries int `orm:"max_retries" json:"maxRetries" description:"最大重试次数"`
RetryDelayMs int `orm:"retry_delay_ms" json:"retryDelayMs" description:"重试延迟(毫秒)"`
// 自定义认证配置 (JSONB)
AuthConfig map[string]interface{} `orm:"auth_config" json:"authConfig" description:"自定义认证配置，支持各平台特有的认证方式"`
// 元数据（ORM tag 与数据库列名一致：creator/updater）
CreatedBy string `orm:"creator" json:"createdBy" description:"创建人"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt" description:"创建时间"`
UpdatedBy string `orm:"updater" json:"updatedBy" description:"更新人"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt" description:"更新时间"`
Version int `orm:"version" json:"version" description:"版本号(乐观锁)"`
beans.SQLBaseDO `orm:",inherit"`
// 业务字段
PlatformCode string `orm:"platform_code" json:"platformCode" description:"平台编码，唯一标识"`
PlatformName string `orm:"platform_name" json:"platformName" description:"平台名称"`
Description string `orm:"description" json:"description" description:"平台描述"`
Status string `orm:"status" json:"status" description:"状态: ACTIVE启用, INACTIVE停用"`
ApiBaseUrl string `orm:"api_base_url" json:"apiBaseUrl" description:"API基础地址"`
AuthType string `orm:"auth_type" json:"authType" description:"认证类型: TOKEN/API_KEY/OAUTH2/BASIC"`
Token string `orm:"token" json:"token" description:"认证token/密钥"`
ApiKey string `orm:"api_key" json:"apiKey" description:"API Key"`
ClientId string `orm:"client_id" json:"clientId" description:"OAuth2 Client ID"`
ClientSecret string `orm:"client_secret" json:"clientSecret" description:"OAuth2 Client Secret"`
RateLimitPerMinute int `orm:"rate_limit_per_minute" json:"rateLimitPerMinute" description:"每分钟请求限制"`
RateLimitPerHour int `orm:"rate_limit_per_hour" json:"rateLimitPerHour" description:"每小时请求限制"`
ConcurrencyLimit int `orm:"concurrency_limit" json:"concurrencyLimit" description:"并发连接限制"`
RequestTimeoutMs int `orm:"request_timeout_ms" json:"requestTimeoutMs" description:"请求超时时间(毫秒)"`
MaxRetries int `orm:"max_retries" json:"maxRetries" description:"最大重试次数"`
RetryDelayMs int `orm:"retry_delay_ms" json:"retryDelayMs" description:"重试延迟(毫秒)"`
AuthConfig map[string]interface{} `orm:"auth_config" json:"authConfig" description:"自定义认证配置，支持各平台特有的认证方式"`
Version int `orm:"version" json:"version" description:"版本号(乐观锁)"`
}
// DatasourcePlatformCol 数据源平台配置表字段定义
type DatasourcePlatformCol struct {
ID string
beans.SQLBaseCol
PlatformCode string
PlatformName string
Description string
@@ -64,16 +48,12 @@ type DatasourcePlatformCol struct {
MaxRetries string
RetryDelayMs string
AuthConfig string
CreatedBy string // 对应 DB 列 creator
CreatedAt string
UpdatedBy string // 对应 DB 列 updater
UpdatedAt string
Version string
}
// DatasourcePlatformCols 数据源平台配置表字段常量
var DatasourcePlatformCols = DatasourcePlatformCol{
ID: "id",
SQLBaseCol: beans.DefSQLBaseCol,
PlatformCode: "platform_code",
PlatformName: "platform_name",
Description: "description",
@@ -91,9 +71,5 @@ var DatasourcePlatformCols = DatasourcePlatformCol{
MaxRetries: "max_retries",
RetryDelayMs: "retry_delay_ms",
AuthConfig: "auth_config",
CreatedBy: "creator",
CreatedAt: "created_at",
UpdatedBy: "updater",
UpdatedAt: "updated_at",
Version: "version",
}