diff --git a/consts/public/collections.go b/consts/public/collections.go index cf95edf..576bcb8 100644 --- a/consts/public/collections.go +++ b/consts/public/collections.go @@ -22,4 +22,5 @@ const ( TencentAccountRelationTable = "tencent_account_relation" // 腾讯广告账户关系表 TencentAudioTable = "tencent_audio" // 腾讯广告音乐素材表 TencentImageTable = "tencent_image" // 腾讯广告图片素材表 + TencentVideoTable = "tencent_video" // 腾讯广告视频素材表 ) diff --git a/controller/tencent/oauth_controller.go b/controller/tencent/oauth_controller.go index 19fb52b..164963a 100644 --- a/controller/tencent/oauth_controller.go +++ b/controller/tencent/oauth_controller.go @@ -92,3 +92,30 @@ func (c *oauthController) ListImagePage(ctx context.Context, req *dto.ListImageP } return service.ImageService.ListWithPage(ctx, queryReq) } + +// SyncVideo 同步视频素材（遍历所有账户，自动分页） +func (c *oauthController) SyncVideo(ctx context.Context, req *dto.SyncVideoReq) (res *dto.SyncVideoRes, err error) { + ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"}) + return service.VideoService.SyncAll(ctx, req) +} + +// ListVideo 获取所有视频素材（旧接口，保留兼容） +func (c *oauthController) ListVideo(ctx context.Context, req *dto.ListVideoReq) (res []entity.Video, err error) { + ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"}) + return service.VideoService.ListAll(ctx) +} + +// ListVideoPage 分页查询视频素材（支持时间过滤） +func (c *oauthController) ListVideoPage(ctx context.Context, req *dto.ListVideoPageReq) (res *dto.ListVideoRes, err error) { + ctx = context.WithValue(ctx, "user", &beans.User{UserName: "admin"}) + // 转换请求参数为Service层使用的类型 + queryReq := &dto.ListVideoQueryReq{ + Page: req.Page, + PageSize: req.PageSize, + AccountId: req.AccountId, + StartTime: req.StartTime, + EndTime: req.EndTime, + Status: req.Status, + } + return service.VideoService.ListWithPage(ctx, queryReq) +} diff --git a/dao/tencent/video_dao.go b/dao/tencent/video_dao.go new file mode 100644 index 0000000..cdfa78f --- /dev/null +++ b/dao/tencent/video_dao.go @@ -0,0 +1,158 @@ +package tencent + +import ( + "context" + consts "dataengine/consts/public" + entity "dataengine/model/entity/tencent" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/frame/g" + "github.com/sirupsen/logrus" +) + +type videoDao struct{} + +var Video = new(videoDao) + +// BatchUpsert 批量插入或更新（使用 OnConflict 实现 Upsert） +func (d *videoDao) BatchUpsert(ctx context.Context, items []*entity.Video) (successCount int, err error) { + if len(items) == 0 { + return 0, nil + } + + logrus.Infof("开始批量Upsert视频素材: %d 条记录", len(items)) + + // 分批处理，每批100条 + batchSize := 100 + successCount = 0 + + for i := 0; i < len(items); i += batchSize { + end := i + batchSize + if end > len(items) { + end = len(items) + } + + batch := items[i:end] + + logrus.Infof("处理第 %d-%d 条视频素材记录", i+1, end) + + // 执行批量插入，使用 OnConflict 实现 Upsert + result, err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable). + Data(batch). + OnConflict("(video_id, account_id)"). + Save() + + if err != nil { + logrus.Errorf("批量Upsert视频素材失败: %v，尝试逐条处理", err) + // 批量失败，逐条处理 + for _, item := range batch { + if upsertErr := d.upsertSingle(ctx, item); upsertErr != nil { + logrus.Errorf("逐条Upsert视频素材失败: video_id=%s, account_id=%d, err=%v", item.VideoId, item.AccountId, upsertErr) + } else { + successCount++ + } + } + } else { + affected, _ := result.RowsAffected() + successCount += int(affected) + logrus.Infof("批量Upsert视频素材成功: 影响 %d 条记录", affected) + } + } + + logrus.Infof("批量Upsert视频素材完成: 成功 %d 条", successCount) + return successCount, nil +} + +// upsertSingle 单条插入或更新 +func (d *videoDao) upsertSingle(ctx context.Context, item *entity.Video) error { + var existing entity.Video + err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable). + Where(entity.VideoCols.VideoId, item.VideoId). + Where(entity.VideoCols.AccountId, item.AccountId). + WhereNull("deleted_at"). + Scan(&existing) + + if err != nil && existing.Id == 0 { + // 记录不存在，执行插入 + _, err = gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable). + Data(item). + Insert() + return err + } + + // 记录存在，执行更新 + _, err = gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable). + Where("id", existing.Id). + Data(g.Map{ + entity.VideoCols.Width: item.Width, + entity.VideoCols.Height: item.Height, + entity.VideoCols.FileSize: item.FileSize, + entity.VideoCols.Type: item.Type, + entity.VideoCols.Signature: item.Signature, + entity.VideoCols.Description: item.Description, + entity.VideoCols.PreviewUrl: item.PreviewUrl, + entity.VideoCols.Status: item.Status, + entity.VideoCols.LastModifiedTime: item.LastModifiedTime, + }). + Update() + + return err +} + +// ListAll 获取所有视频素材 +func (d *videoDao) ListAll(ctx context.Context) ([]entity.Video, error) { + var list []entity.Video + err := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable). + WhereNull("deleted_at"). + OrderAsc(entity.VideoCols.VideoId). + Scan(&list) + + return list, err +} + +// ListWithPage 分页查询视频素材（支持时间过滤） +func (d *videoDao) ListWithPage(ctx context.Context, page, pageSize int, accountId *int64, startTime, endTime *int64, status string) ([]entity.Video, int, error) { + model := gfdb.DB(ctx).Model(ctx, consts.TencentVideoTable). + WhereNull("deleted_at") + + // 账户ID过滤 + if accountId != nil && *accountId > 0 { + model = model.Where(entity.VideoCols.AccountId, *accountId) + } + + // 状态过滤 + if status != "" { + model = model.Where(entity.VideoCols.Status, status) + } + + // 时间范围过滤（根据 last_modified_time） + if startTime != nil && *startTime > 0 { + model = model.WhereGTE(entity.VideoCols.LastModifiedTime, *startTime) + } + if endTime != nil && *endTime > 0 { + model = model.WhereLTE(entity.VideoCols.LastModifiedTime, *endTime) + } + + // 设置排序（按最后修改时间降序） + model = model.OrderDesc(entity.VideoCols.LastModifiedTime) + + // 获取总数 + total, err := model.Count() + if err != nil { + return nil, 0, err + } + + // 分页查询 + var list []entity.Video + if page > 0 && pageSize > 0 { + err = model.Page(page, pageSize).Scan(&list) + } else { + err = model.Scan(&list) + } + + if err != nil { + return nil, 0, err + } + + return list, total, nil +} diff --git a/model/dto/tencent/video_dto.go b/model/dto/tencent/video_dto.go new file mode 100644 index 0000000..3da1351 --- /dev/null +++ b/model/dto/tencent/video_dto.go @@ -0,0 +1,73 @@ +package tencent + +import "github.com/gogf/gf/v2/frame/g" + +// SyncVideoReq 同步视频素材请求 +type SyncVideoReq struct { + g.Meta `path:"/syncVideo" method:"post" tags:"腾讯广告视频素材" summary:"同步视频素材" dc:"遍历所有账户，自动分页获取视频素材并保存到数据库"` + AccessToken string `json:"access_token" dc:"访问令牌（可选，不传则从配置读取）"` +} + +// SyncVideoRes 同步视频素材响应 +type SyncVideoRes struct { + TotalAccounts int `json:"total_accounts" dc:"处理的账户数"` + TotalVideos int `json:"total_videos" dc:"总视频数"` + SyncedCount int `json:"synced_count" dc:"同步成功数量"` + Message string `json:"message" dc:"消息"` +} + +// ListVideoReq 获取视频素材列表请求（旧接口，无分页） +type ListVideoReq struct { + g.Meta `path:"/listVideo" method:"post" tags:"腾讯广告视频素材" summary:"获取视频素材列表" dc:"从本地数据库查询所有视频素材（无分页）"` +} + +// ListVideoPageReq 分页查询视频素材请求 +type ListVideoPageReq struct { + g.Meta `path:"/listVideoPage" method:"post" tags:"腾讯广告视频素材" summary:"分页查询视频素材" dc:"支持分页、时间过滤、账户过滤等条件查询"` + Page int `json:"page" dc:"页码" d:"1"` + PageSize int `json:"page_size" dc:"每页数量" d:"20"` + AccountId *int64 `json:"account_id,omitempty" dc:"账户ID（可选）"` + StartTime *int64 `json:"start_time,omitempty" dc:"开始时间戳（秒，可选）"` + EndTime *int64 `json:"end_time,omitempty" dc:"结束时间戳（秒，可选）"` + Status string `json:"status,omitempty" dc:"状态筛选（可选）"` +} + +// ListVideoQueryReq 视频素材查询请求（Service层使用） +type ListVideoQueryReq struct { + Page int `json:"page" dc:"页码"` + PageSize int `json:"page_size" dc:"每页数量"` + AccountId *int64 `json:"account_id,omitempty" dc:"账户ID（可选）"` + StartTime *int64 `json:"start_time,omitempty" dc:"开始时间戳（秒，可选）"` + EndTime *int64 `json:"end_time,omitempty" dc:"结束时间戳（秒，可选）"` + Status string `json:"status,omitempty" dc:"状态筛选（可选）"` +} + +// ListVideoRes 获取视频素材列表响应 +type ListVideoRes struct { + List []VideoItem `json:"list" dc:"视频素材列表"` + Total int `json:"total" dc:"总记录数"` + Page int `json:"page" dc:"当前页码"` + PageSize int `json:"page_size" dc:"每页数量"` + TotalPages int `json:"total_pages" dc:"总页数"` +} + +// VideoItem 视频素材项 +type VideoItem struct { + Id int64 `json:"id" dc:"主键ID"` + VideoId string `json:"video_id" dc:"视频ID"` + AccountId int64 `json:"account_id" dc:"账户ID"` + Width int `json:"width" dc:"宽度"` + Height int `json:"height" dc:"高度"` + VideoFrames int `json:"video_frames" dc:"视频帧数"` + VideoFps int `json:"video_fps" dc:"帧率"` + FileSize int64 `json:"file_size" dc:"文件大小"` + Type string `json:"type" dc:"媒体类型"` + Description string `json:"description" dc:"描述"` + PreviewUrl string `json:"preview_url" dc:"预览URL"` + KeyFrameImageUrl string `json:"key_frame_image_url" dc:"关键帧图片URL"` + Status string `json:"status" dc:"状态"` + CreatedTime int64 `json:"created_time" dc:"创建时间戳"` + LastModifiedTime int64 `json:"last_modified_time" dc:"最后修改时间戳"` + CreatedAt string `json:"created_at" dc:"数据库创建时间"` + UpdatedAt string `json:"updated_at" dc:"数据库更新时间"` +} diff --git a/model/entity/tencent/video.go b/model/entity/tencent/video.go new file mode 100644 index 0000000..86de096 --- /dev/null +++ b/model/entity/tencent/video.go @@ -0,0 +1,159 @@ +package tencent + +import ( + "gitea.com/red-future/common/beans" +) + +// Video 腾讯广告视频素材实体 +type Video struct { + beans.SQLBaseDO `orm:",inherit"` + + VideoId string `orm:"video_id" json:"videoId" description:"视频ID"` + AccountId int64 `orm:"account_id" json:"accountId" description:"账户ID"` + Width int `orm:"width" json:"width" description:"宽度"` + Height int `orm:"height" json:"height" description:"高度"` + VideoFrames int `orm:"video_frames" json:"videoFrames" description:"视频帧数"` + VideoFps int `orm:"video_fps" json:"videoFps" description:"帧率"` + VideoCodec string `orm:"video_codec" json:"videoCodec" description:"视频编码"` + VideoBitRate int64 `orm:"video_bit_rate" json:"videoBitRate" description:"视频码率"` + AudioCodec string `orm:"audio_codec" json:"audioCodec" description:"音频编码"` + AudioBitRate int64 `orm:"audio_bit_rate" json:"audioBitRate" description:"音频码率"` + FileSize int64 `orm:"file_size" json:"fileSize" description:"文件大小"` + Type string `orm:"type" json:"type" description:"媒体类型"` + Signature string `orm:"signature" json:"signature" description:"签名"` + SystemStatus string `orm:"system_status" json:"systemStatus" description:"系统状态"` + Description string `orm:"description" json:"description" description:"描述"` + PreviewUrl string `orm:"preview_url" json:"previewUrl" description:"预览URL"` + KeyFrameImageUrl string `orm:"key_frame_image_url" json:"keyFrameImageUrl" description:"关键帧图片URL"` + CreatedTime int64 `orm:"created_time" json:"createdTime" description:"创建时间戳"` + LastModifiedTime int64 `orm:"last_modified_time" json:"lastModifiedTime" description:"最后修改时间戳"` + VideoProfileName string `orm:"video_profile_name" json:"videoProfileName" description:"视频配置名称"` + AudioSampleRate int `orm:"audio_sample_rate" json:"audioSampleRate" description:"音频采样率"` + MaxKeyframeInterval int `orm:"max_keyframe_interval" json:"maxKeyframeInterval" description:"最大关键帧间隔"` + MinKeyframeInterval int `orm:"min_keyframe_interval" json:"minKeyframeInterval" description:"最小关键帧间隔"` + SampleAspectRatio string `orm:"sample_aspect_ratio" json:"sampleAspectRatio" description:"示例宽高比"` + AudioProfileName string `orm:"audio_profile_name" json:"audioProfileName" description:"音频配置名称"` + ScanType string `orm:"scan_type" json:"scanType" description:"扫描类型"` + ImageDurationMillisecond int64 `orm:"image_duration_millisecond" json:"imageDurationMillisecond" description:"图片时长（毫秒）"` + AudioDurationMillisecond int64 `orm:"audio_duration_millisecond" json:"audioDurationMillisecond" description:"音频时长（毫秒）"` + SourceType string `orm:"source_type" json:"sourceType" description:"来源类型"` + ProductCatalogId string `orm:"product_catalog_id" json:"productCatalogId" description:"产品目录ID"` + ProductOuterId string `orm:"product_outer_id" json:"productOuterId" description:"产品外部ID"` + SourceReferenceId string `orm:"source_reference_id" json:"sourceReferenceId" description:"源引用ID"` + OwnerAccountId string `orm:"owner_account_id" json:"ownerAccountId" description:"所有者账户ID"` + Status string `orm:"status" json:"status" description:"状态"` + SourceMaterialId string `orm:"source_material_id" json:"sourceMaterialId" description:"源素材ID"` + NewSourceType string `orm:"new_source_type" json:"newSourceType" description:"新来源类型"` + AigcType int `orm:"aigc_type" json:"aigcType" description:"AIGC类型"` + FirstPublicationStatus string `orm:"first_publication_status" json:"firstPublicationStatus" description:"首次发布状态"` + QualityStatus string `orm:"quality_status" json:"qualityStatus" description:"质量状态"` + CoverId string `orm:"cover_id" json:"coverId" description:"封面ID"` + SimilarityStatus string `orm:"similarity_status" json:"similarityStatus" description:"相似度状态"` + UserAigcStatus string `orm:"user_aigc_status" json:"userAigcStatus" description:"用户AIGC状态"` + SystemAigcStatus string `orm:"system_aigc_status" json:"systemAigcStatus" description:"系统AIGC状态"` + AigcSource string `orm:"aigc_source" json:"aigcSource" description:"AIGC来源"` + AigcFlag string `orm:"aigc_flag" json:"aigcFlag" description:"AIGC标志"` + MuseAigcVersion int `orm:"muse_aigc_version" json:"museAigcVersion" description:"Muse AIGC版本"` +} + +// VideoCol 视频素材表字段定义 +type VideoCol struct { + beans.SQLBaseCol + VideoId string + AccountId string + Width string + Height string + VideoFrames string + VideoFps string + VideoCodec string + VideoBitRate string + AudioCodec string + AudioBitRate string + FileSize string + Type string + Signature string + SystemStatus string + Description string + PreviewUrl string + KeyFrameImageUrl string + CreatedTime string + LastModifiedTime string + VideoProfileName string + AudioSampleRate string + MaxKeyframeInterval string + MinKeyframeInterval string + SampleAspectRatio string + AudioProfileName string + ScanType string + ImageDurationMillisecond string + AudioDurationMillisecond string + SourceType string + ProductCatalogId string + ProductOuterId string + SourceReferenceId string + OwnerAccountId string + Status string + SourceMaterialId string + NewSourceType string + AigcType string + FirstPublicationStatus string + QualityStatus string + CoverId string + SimilarityStatus string + UserAigcStatus string + SystemAigcStatus string + AigcSource string + AigcFlag string + MuseAigcVersion string +} + +// VideoCols 视频素材表字段常量 +var VideoCols = VideoCol{ + SQLBaseCol: beans.DefSQLBaseCol, + VideoId: "video_id", + AccountId: "account_id", + Width: "width", + Height: "height", + VideoFrames: "video_frames", + VideoFps: "video_fps", + VideoCodec: "video_codec", + VideoBitRate: "video_bit_rate", + AudioCodec: "audio_codec", + AudioBitRate: "audio_bit_rate", + FileSize: "file_size", + Type: "type", + Signature: "signature", + SystemStatus: "system_status", + Description: "description", + PreviewUrl: "preview_url", + KeyFrameImageUrl: "key_frame_image_url", + CreatedTime: "created_time", + LastModifiedTime: "last_modified_time", + VideoProfileName: "video_profile_name", + AudioSampleRate: "audio_sample_rate", + MaxKeyframeInterval: "max_keyframe_interval", + MinKeyframeInterval: "min_keyframe_interval", + SampleAspectRatio: "sample_aspect_ratio", + AudioProfileName: "audio_profile_name", + ScanType: "scan_type", + ImageDurationMillisecond: "image_duration_millisecond", + AudioDurationMillisecond: "audio_duration_millisecond", + SourceType: "source_type", + ProductCatalogId: "product_catalog_id", + ProductOuterId: "product_outer_id", + SourceReferenceId: "source_reference_id", + OwnerAccountId: "owner_account_id", + Status: "status", + SourceMaterialId: "source_material_id", + NewSourceType: "new_source_type", + AigcType: "aigc_type", + FirstPublicationStatus: "first_publication_status", + QualityStatus: "quality_status", + CoverId: "cover_id", + SimilarityStatus: "similarity_status", + UserAigcStatus: "user_aigc_status", + SystemAigcStatus: "system_aigc_status", + AigcSource: "aigc_source", + AigcFlag: "aigc_flag", + MuseAigcVersion: "muse_aigc_version", +} diff --git a/service/tencent/video_service.go b/service/tencent/video_service.go new file mode 100644 index 0000000..36b3033 --- /dev/null +++ b/service/tencent/video_service.go @@ -0,0 +1,417 @@ +package tencent + +import ( + "context" + dao "dataengine/dao/tencent" + dto "dataengine/model/dto/tencent" + entity "dataengine/model/entity/tencent" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "time" + + "gitea.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/frame/g" + "github.com/sirupsen/logrus" +) + +type videoService struct{} + +var VideoService = new(videoService) + +// API响应结构 +type videoResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + List []struct { + VideoId int64 `json:"video_id"` + Width int `json:"width"` + Height int `json:"height"` + VideoFrames int `json:"video_frames"` + VideoFps int `json:"video_fps"` + VideoCodec string `json:"video_codec"` + VideoBitRate int64 `json:"video_bit_rate"` + AudioCodec string `json:"audio_codec"` + AudioBitRate int64 `json:"audio_bit_rate"` + FileSize int64 `json:"file_size"` + Type string `json:"type"` + Signature string `json:"signature"` + SystemStatus string `json:"system_status"` + Description string `json:"description"` + PreviewUrl string `json:"preview_url"` + KeyFrameImageUrl string `json:"key_frame_image_url"` + CreatedTime int64 `json:"created_time"` + LastModifiedTime int64 `json:"last_modified_time"` + VideoProfileName string `json:"video_profile_name"` + AudioSampleRate int `json:"audio_sample_rate"` + MaxKeyframeInterval int `json:"max_keyframe_interval"` + MinKeyframeInterval int `json:"min_keyframe_interval"` + SampleAspectRatio string `json:"sample_aspect_ratio"` + AudioProfileName string `json:"audio_profile_name"` + ScanType string `json:"scan_type"` + ImageDurationMs int64 `json:"image_duration_millisecond"` + AudioDurationMs int64 `json:"audio_duration_millisecond"` + SourceType string `json:"source_type"` + ProductCatalogId string `json:"product_catalog_id"` + ProductOuterId string `json:"product_outer_id"` + SourceReferenceId string `json:"source_reference_id"` + OwnerAccountId string `json:"owner_account_id"` + Status string `json:"status"` + SourceMaterialId string `json:"source_material_id"` + NewSourceType string `json:"new_source_type"` + AigcType int `json:"aigc_type"` + FirstPublicationStatus string `json:"first_publication_status"` + QualityStatus string `json:"quality_status"` + CoverId string `json:"cover_id"` + SimilarityStatus string `json:"similarity_status"` + UserAigcStatus string `json:"user_aigc_status"` + SystemAigcStatus string `json:"system_aigc_status"` + AigcSource string `json:"aigc_source"` + AigcFlag string `json:"aigc_flag"` + MuseAigcVersion int `json:"muse_aigc_version"` + } `json:"list"` + PageInfo struct { + Page int `json:"page"` + PageSize int `json:"page_size"` + TotalNumber int `json:"total_number"` + TotalPage int `json:"total_page"` + } `json:"page_info"` + } `json:"data"` + TraceId string `json:"trace_id"` +} + +// SyncAll 同步所有视频素材数据（遍历所有账户，自动分页） +func (s *videoService) SyncAll(ctx context.Context, req *dto.SyncVideoReq) (res *dto.SyncVideoRes, err error) { + // 创建独立的context，避免HTTP请求超时导致context被取消 + // 设置30分钟超时，足够完成所有账户的同步任务 + independentCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + // 保留原context中的user信息，供数据库中间件使用 + if user := ctx.Value("user"); user != nil { + independentCtx = context.WithValue(independentCtx, "user", user) + } + + // 获取access_token + accessToken := req.AccessToken + if accessToken == "" { + accessToken = g.Cfg().MustGet(independentCtx, "tencent.oauth.access_token").String() + } + + if accessToken == "" { + return nil, fmt.Errorf("access_token不能为空") + } + + res = &dto.SyncVideoRes{} + totalSynced := 0 + totalVideos := 0 + + // 获取所有账户列表 + accounts, err := s.getAccountList(independentCtx) + if err != nil { + return nil, fmt.Errorf("获取账户列表失败: %w", err) + } + + res.TotalAccounts = len(accounts) + logrus.Infof("开始同步腾讯广告视频素材 - 账户数: %d", len(accounts)) + + // 遍历每个账户 + for _, account := range accounts { + logrus.Infof("========== 开始处理账户: %d (%s) ==========", account.AccountID, account.CorporationName) + + // 获取该账户的所有视频（分页） + accountVideos, err := s.syncAccountVideos(independentCtx, accessToken, account.AccountID) + if err != nil { + logrus.Errorf("账户 %d 同步失败: %v，继续下一个账户", account.AccountID, err) + continue + } + + totalVideos += accountVideos + totalSynced += accountVideos + + // 避免请求过快，休眠200ms + time.Sleep(200 * time.Millisecond) + } + + res.TotalVideos = totalVideos + res.SyncedCount = totalSynced + res.Message = fmt.Sprintf("同步完成，共处理 %d 个账户，%d 条视频记录", res.TotalAccounts, totalSynced) + + logrus.Infof("同步完成 - 账户数: %d, 总视频数: %d, 成功同步: %d", res.TotalAccounts, totalVideos, totalSynced) + + return res, nil +} + +// getAccountList 获取所有账户列表 +func (s *videoService) getAccountList(ctx context.Context) ([]entity.AccountRelation, error) { + var accounts []entity.AccountRelation + err := gfdb.DB(ctx).Model(ctx, "tencent_account_relation"). + WhereNull("deleted_at"). + Scan(&accounts) + + return accounts, err +} + +// syncAccountVideos 同步单个账户的视频数据 +func (s *videoService) syncAccountVideos(ctx context.Context, accessToken string, accountId int64) (int, error) { + totalSynced := 0 + + // 先获取第一页，得到总页数 + firstPageData, err := s.fetchPage(ctx, accessToken, accountId, 1, 100) + if err != nil { + // 如果是请求失败或API错误，返回友好的提示 + errMsg := err.Error() + if contains(errMsg, "请求失败") || contains(errMsg, "API错误") { + return 0, fmt.Errorf("该账户没有视频或无法访问") + } + return 0, fmt.Errorf("获取第一页数据失败: %w", err) + } + + totalPage := firstPageData.Data.PageInfo.TotalPage + logrus.Infof("账户 %d - 总页数: %d, 总记录数: %d", accountId, totalPage, firstPageData.Data.PageInfo.TotalNumber) + + // 如果没有数据，直接返回 + if totalPage == 0 || firstPageData.Data.PageInfo.TotalNumber == 0 { + logrus.Infof("账户 %d - 没有视频数据", accountId) + return 0, nil + } + + // 处理第一页数据 + synced, err := s.savePageData(ctx, firstPageData, accountId) + if err != nil { + logrus.Errorf("保存第一页数据失败: %v", err) + } + totalSynced += synced + + // 循环获取剩余页 + for page := 2; page <= totalPage; page++ { + logrus.Infof("账户 %d - 正在获取第 %d/%d 页...", accountId, page, totalPage) + + pageData, err := s.fetchPage(ctx, accessToken, accountId, page, 100) + if err != nil { + logrus.Errorf("账户 %d - 获取第 %d 页失败: %v，继续下一页", accountId, page, err) + continue + } + + synced, err := s.savePageData(ctx, pageData, accountId) + if err != nil { + logrus.Errorf("账户 %d - 保存第 %d 页数据失败: %v", accountId, page, err) + continue + } + totalSynced += synced + + // 避免请求过快，休眠100ms + time.Sleep(100 * time.Millisecond) + } + + logrus.Infof("账户 %d - 同步完成，共 %d 条记录", accountId, totalSynced) + return totalSynced, nil +} + +// fetchPage 获取单页数据 +func (s *videoService) fetchPage(ctx context.Context, accessToken string, accountId int64, page, pageSize int) (*videoResponse, error) { + // 构建filtering参数：状态为正常 + filtering := `[{"field":"status","operator":"EQUALS","values":["ADSTATUS_NORMAL"]}]` + + // URL编码filtering参数 + encodedFiltering := url.QueryEscape(filtering) + + // 在发送请求前生成最新的时间戳和nonce，避免时间戳过期 + timestamp := time.Now().Unix() + // 使用时间戳+纳秒后6位+随机数，确保唯一性且不超过32字符 + nanoSuffix := time.Now().UnixNano() % 1000000 // 取纳秒的后6位 + nonce := fmt.Sprintf("%d%06d%d", timestamp, nanoSuffix, rand.Intn(1000)) + + urlStr := fmt.Sprintf("https://api.e.qq.com/v3.0/videos/get?access_token=%s&nonce=%s×tamp=%d&account_id=%d&filtering=%s&page=%d&page_size=%d", + accessToken, nonce, timestamp, accountId, encodedFiltering, page, pageSize) + + logrus.Debugf("请求URL: %s", urlStr) + + httpReq, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, fmt.Errorf("创建请求失败: %w", err) + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("请求失败: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("读取响应失败: %w", err) + } + + logrus.Debugf("API响应: %s", string(body)) + + var result videoResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("解析响应失败: %w", err) + } + + if result.Code != 0 { + return nil, fmt.Errorf("API错误: code=%d, message=%s", result.Code, result.Message) + } + + return &result, nil +} + +// savePageData 保存单页数据到数据库 +func (s *videoService) savePageData(ctx context.Context, data *videoResponse, accountId int64) (int, error) { + if len(data.Data.List) == 0 { + return 0, nil + } + + logrus.Infof("准备保存 %d 条视频素材数据", len(data.Data.List)) + + var items []*entity.Video + for _, item := range data.Data.List { + video := &entity.Video{ + VideoId: fmt.Sprintf("%d", item.VideoId), + AccountId: accountId, + Width: item.Width, + Height: item.Height, + VideoFrames: item.VideoFrames, + VideoFps: item.VideoFps, + VideoCodec: item.VideoCodec, + VideoBitRate: item.VideoBitRate, + AudioCodec: item.AudioCodec, + AudioBitRate: item.AudioBitRate, + FileSize: item.FileSize, + Type: item.Type, + Signature: item.Signature, + SystemStatus: item.SystemStatus, + Description: item.Description, + PreviewUrl: item.PreviewUrl, + KeyFrameImageUrl: item.KeyFrameImageUrl, + CreatedTime: item.CreatedTime, + LastModifiedTime: item.LastModifiedTime, + VideoProfileName: item.VideoProfileName, + AudioSampleRate: item.AudioSampleRate, + MaxKeyframeInterval: item.MaxKeyframeInterval, + MinKeyframeInterval: item.MinKeyframeInterval, + SampleAspectRatio: item.SampleAspectRatio, + AudioProfileName: item.AudioProfileName, + ScanType: item.ScanType, + ImageDurationMillisecond: item.ImageDurationMs, + AudioDurationMillisecond: item.AudioDurationMs, + SourceType: item.SourceType, + ProductCatalogId: item.ProductCatalogId, + ProductOuterId: item.ProductOuterId, + SourceReferenceId: item.SourceReferenceId, + OwnerAccountId: item.OwnerAccountId, + Status: item.Status, + SourceMaterialId: item.SourceMaterialId, + NewSourceType: item.NewSourceType, + AigcType: item.AigcType, + FirstPublicationStatus: item.FirstPublicationStatus, + QualityStatus: item.QualityStatus, + CoverId: item.CoverId, + SimilarityStatus: item.SimilarityStatus, + UserAigcStatus: item.UserAigcStatus, + SystemAigcStatus: item.SystemAigcStatus, + AigcSource: item.AigcSource, + AigcFlag: item.AigcFlag, + MuseAigcVersion: item.MuseAigcVersion, + } + // 设置 TenantID（框架将0视为空值，所以使用1） + video.TenantId = 1 + + items = append(items, video) + } + + logrus.Infof("调用 BatchUpsert...") + successCount, err := dao.Video.BatchUpsert(ctx, items) + logrus.Infof("BatchUpsert 返回: successCount=%d, err=%v", successCount, err) + + return successCount, err +} + +// ListAll 获取所有视频素材 +func (s *videoService) ListAll(ctx context.Context) ([]entity.Video, error) { + return dao.Video.ListAll(ctx) +} + +// ListWithPage 分页查询视频素材（支持时间过滤） +func (s *videoService) ListWithPage(ctx context.Context, req *dto.ListVideoQueryReq) (*dto.ListVideoRes, error) { + // 设置默认值 + page := req.Page + if page <= 0 { + page = 1 + } + pageSize := req.PageSize + if pageSize <= 0 { + pageSize = 20 + } + if pageSize > 100 { + pageSize = 100 // 限制最大每页数量 + } + + // 调用DAO层查询 + list, total, err := dao.Video.ListWithPage(ctx, page, pageSize, req.AccountId, req.StartTime, req.EndTime, req.Status) + if err != nil { + return nil, fmt.Errorf("查询视频素材失败: %w", err) + } + + // 计算总页数 + totalPages := (total + pageSize - 1) / pageSize + if totalPages == 0 && total > 0 { + totalPages = 1 + } + + // 转换为DTO + items := make([]dto.VideoItem, 0, len(list)) + for _, item := range list { + items = append(items, dto.VideoItem{ + Id: item.Id, + VideoId: item.VideoId, + AccountId: item.AccountId, + Width: item.Width, + Height: item.Height, + VideoFrames: item.VideoFrames, + VideoFps: item.VideoFps, + FileSize: item.FileSize, + Type: item.Type, + Description: item.Description, + PreviewUrl: item.PreviewUrl, + KeyFrameImageUrl: item.KeyFrameImageUrl, + Status: item.Status, + CreatedTime: item.CreatedTime, + LastModifiedTime: item.LastModifiedTime, + CreatedAt: item.CreatedAt.Format("2006-01-02 15:04:05"), + UpdatedAt: item.UpdatedAt.Format("2006-01-02 15:04:05"), + }) + } + + res := &dto.ListVideoRes{ + List: items, + Total: total, + Page: page, + PageSize: pageSize, + TotalPages: totalPages, + } + + logrus.Infof("查询视频素材 - 页码: %d, 每页: %d, 总数: %d, 总页数: %d", page, pageSize, total, totalPages) + + return res, nil +} + +// contains 检查字符串是否包含子串 +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || findSubstring(s, substr))) +} + +func findSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/sql/12_tencent_video.sql b/sql/12_tencent_video.sql new file mode 100644 index 0000000..e560039 --- /dev/null +++ b/sql/12_tencent_video.sql @@ -0,0 +1,123 @@ +-- 腾讯广告视频素材表 +CREATE SEQUENCE IF NOT EXISTS tencent_video_id_seq START WITH 1 INCREMENT BY 1; + +CREATE TABLE IF NOT EXISTS tencent_video ( + id BIGINT NOT NULL DEFAULT nextval('tencent_video_id_seq'::regclass), + tenant_id BIGINT NOT NULL DEFAULT 0, + creator VARCHAR(100) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(100) DEFAULT '', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE, + + -- 业务字段 + video_id VARCHAR(100) NOT NULL, + account_id BIGINT NOT NULL, + width INT, + height INT, + video_frames INT, + video_fps INT, + video_codec VARCHAR(50), + video_bit_rate BIGINT, + audio_codec VARCHAR(50), + audio_bit_rate BIGINT, + file_size BIGINT, + type VARCHAR(50), + signature VARCHAR(200), + system_status VARCHAR(50), + description TEXT, + preview_url TEXT, + key_frame_image_url TEXT, + created_time BIGINT, + last_modified_time BIGINT, + video_profile_name VARCHAR(50), + audio_sample_rate INT, + max_keyframe_interval INT, + min_keyframe_interval INT, + sample_aspect_ratio VARCHAR(50), + audio_profile_name VARCHAR(50), + scan_type VARCHAR(50), + image_duration_millisecond BIGINT, + audio_duration_millisecond BIGINT, + source_type VARCHAR(100), + product_catalog_id VARCHAR(200), + product_outer_id VARCHAR(200), + source_reference_id VARCHAR(200), + owner_account_id VARCHAR(100), + status VARCHAR(50), + source_material_id VARCHAR(100), + new_source_type VARCHAR(100), + aigc_type INT, + first_publication_status VARCHAR(100), + quality_status VARCHAR(100), + cover_id VARCHAR(100), + similarity_status VARCHAR(100), + user_aigc_status VARCHAR(100), + system_aigc_status VARCHAR(100), + aigc_source VARCHAR(200), + aigc_flag VARCHAR(50), + muse_aigc_version INT, + + PRIMARY KEY (id) +); + +COMMENT ON TABLE tencent_video IS '腾讯广告视频素材表'; +COMMENT ON COLUMN tencent_video.id IS '主键ID'; +COMMENT ON COLUMN tencent_video.tenant_id IS '租户ID'; +COMMENT ON COLUMN tencent_video.creator IS '创建人'; +COMMENT ON COLUMN tencent_video.created_at IS '创建时间'; +COMMENT ON COLUMN tencent_video.updater IS '更新人'; +COMMENT ON COLUMN tencent_video.updated_at IS '更新时间'; +COMMENT ON COLUMN tencent_video.deleted_at IS '软删除时间'; +COMMENT ON COLUMN tencent_video.video_id IS '视频ID'; +COMMENT ON COLUMN tencent_video.account_id IS '账户ID'; +COMMENT ON COLUMN tencent_video.width IS '宽度'; +COMMENT ON COLUMN tencent_video.height IS '高度'; +COMMENT ON COLUMN tencent_video.video_frames IS '视频帧数'; +COMMENT ON COLUMN tencent_video.video_fps IS '帧率'; +COMMENT ON COLUMN tencent_video.video_codec IS '视频编码'; +COMMENT ON COLUMN tencent_video.video_bit_rate IS '视频码率'; +COMMENT ON COLUMN tencent_video.audio_codec IS '音频编码'; +COMMENT ON COLUMN tencent_video.audio_bit_rate IS '音频码率'; +COMMENT ON COLUMN tencent_video.file_size IS '文件大小'; +COMMENT ON COLUMN tencent_video.type IS '媒体类型'; +COMMENT ON COLUMN tencent_video.signature IS '签名'; +COMMENT ON COLUMN tencent_video.system_status IS '系统状态'; +COMMENT ON COLUMN tencent_video.description IS '描述'; +COMMENT ON COLUMN tencent_video.preview_url IS '预览URL'; +COMMENT ON COLUMN tencent_video.key_frame_image_url IS '关键帧图片URL'; +COMMENT ON COLUMN tencent_video.created_time IS '创建时间戳'; +COMMENT ON COLUMN tencent_video.last_modified_time IS '最后修改时间戳'; +COMMENT ON COLUMN tencent_video.video_profile_name IS '视频配置名称'; +COMMENT ON COLUMN tencent_video.audio_sample_rate IS '音频采样率'; +COMMENT ON COLUMN tencent_video.max_keyframe_interval IS '最大关键帧间隔'; +COMMENT ON COLUMN tencent_video.min_keyframe_interval IS '最小关键帧间隔'; +COMMENT ON COLUMN tencent_video.sample_aspect_ratio IS '示例宽高比'; +COMMENT ON COLUMN tencent_video.audio_profile_name IS '音频配置名称'; +COMMENT ON COLUMN tencent_video.scan_type IS '扫描类型'; +COMMENT ON COLUMN tencent_video.image_duration_millisecond IS '图片时长（毫秒）'; +COMMENT ON COLUMN tencent_video.audio_duration_millisecond IS '音频时长（毫秒）'; +COMMENT ON COLUMN tencent_video.source_type IS '来源类型'; +COMMENT ON COLUMN tencent_video.product_catalog_id IS '产品目录ID'; +COMMENT ON COLUMN tencent_video.product_outer_id IS '产品外部ID'; +COMMENT ON COLUMN tencent_video.source_reference_id IS '源引用ID'; +COMMENT ON COLUMN tencent_video.owner_account_id IS '所有者账户ID'; +COMMENT ON COLUMN tencent_video.status IS '状态'; +COMMENT ON COLUMN tencent_video.source_material_id IS '源素材ID'; +COMMENT ON COLUMN tencent_video.new_source_type IS '新来源类型'; +COMMENT ON COLUMN tencent_video.aigc_type IS 'AIGC类型'; +COMMENT ON COLUMN tencent_video.first_publication_status IS '首次发布状态'; +COMMENT ON COLUMN tencent_video.quality_status IS '质量状态'; +COMMENT ON COLUMN tencent_video.cover_id IS '封面ID'; +COMMENT ON COLUMN tencent_video.similarity_status IS '相似度状态'; +COMMENT ON COLUMN tencent_video.user_aigc_status IS '用户AIGC状态'; +COMMENT ON COLUMN tencent_video.system_aigc_status IS '系统AIGC状态'; +COMMENT ON COLUMN tencent_video.aigc_source IS 'AIGC来源'; +COMMENT ON COLUMN tencent_video.aigc_flag IS 'AIGC标志'; +COMMENT ON COLUMN tencent_video.muse_aigc_version IS 'Muse AIGC版本'; + +-- 唯一索引：根据video_id和account_id判断是否存在 +CREATE UNIQUE INDEX idx_tencent_video_video_account ON tencent_video(tenant_id, video_id, account_id); +CREATE INDEX idx_tencent_video_account_id ON tencent_video(account_id); +CREATE INDEX idx_tencent_video_last_modified ON tencent_video(last_modified_time); +CREATE INDEX idx_tencent_video_status ON tencent_video(status);