etl

package
v0.0.0-...-2e8b363 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 30, 2025 License: MIT Imports: 14 Imported by: 0

Documentation

Overview

internal/etl/etl_goods_aweme_analysis.go

internal/etl/etl_goods_detail.go

Index

Constants

This section is empty.

Variables

View Source 
var ProviderSet = wire.NewSet(
	NewETLUsecase,
	NewVideoRankProcessor,
	NewVideoDetailProcessor,
	NewGoodsAwemeAnalysisETLProcessor,
	NewGoodsDetailProcessor,
)

ProviderSet 保持不变，wire 会自动注入 VideoRepo

Functions

This section is empty.

Types

type AwemeAnalysisItem 

type AwemeAnalysisItem struct {
	BaseBloggerDto struct {
		BloggerId        int64  `json:"BloggerId"`
		BloggerUid       string `json:"BloggerUid"`
		BloggerName      string `json:"BloggerName"`
		BloggerAvatar    string `json:"BloggerAvatar"`
		FansNum          int    `json:"FansNum"`
		Tag              string `json:"Tag"`
		BloggerDetailUrl string `json:"BloggerDetailUrl"`
	} `json:"BaseBloggerDto"`
	BaseAwemeDto struct {
		AwemeId        string `json:"AwemeId"`
		AwemeDesc      string `json:"AwemeDesc"`
		AwemeCoverUrl  string `json:"AwemeCoverUrl"`
		AwemePubTime   string `json:"AwemePubTime"`
		AwemeDetailUrl string `json:"AwemeDetailUrl"`
	} `json:"BaseAwemeDto"`
	AwemeSaleCountStr string `json:"AwemeSaleCountStr"`
	AwemeSaleGmvStr   string `json:"AwemeSaleGmvStr"`
	LikeCountStr      string `json:"LikeCountStr"`
	CommentCountStr   string `json:"CommentCountStr"`
	ShareCountStr     string `json:"ShareCountStr"`
	PlayCountStr      string `json:"PlayCountStr"`
	CollectCountStr   string `json:"CollectCountStr"`
	GPM               string `json:"GPM"`
}

AwemeAnalysisItem 和 AwemeAnalysisSummary 结构体保持不变

type AwemeAnalysisResponse 

type AwemeAnalysisResponse struct {
	FeiguaBaseResponse
	Data struct {
		Items   []AwemeAnalysisItem  `json:"Items"`
		Summary AwemeAnalysisSummary `json:"Summary"`
	} `json:"Data"`
}

AwemeAnalysisResponse 对应 goods_aweme_analysis_headless 的【明文】响应结构

type AwemeAnalysisSummary 

type AwemeAnalysisSummary struct {
	AwemeCount        int64  `json:"AwemeCount"`
	BloggerCount      int64  `json:"BloggerCount"`
	AwemeSaleCountStr string `json:"AwemeSaleCountStr"`
	AwemeSaleGmvStr   string `json:"AwemeSaleGmvStr"`
}

type ChartDataResponse 

type ChartDataResponse struct {
	FeiguaBaseResponse
	Encrypt bool   `json:"Encrypt"`
	Rnd     string `json:"Rnd"`
	Data    string `json:"Data"`
}

ChartDataResponse 对应 goods_chart_data_headless 的【加密】响应结构

type ETLUsecase 

type ETLUsecase struct {
	// contains filtered or unexported fields
}

ETLUsecase orchestrates ETL processors.

func NewETLUsecase 

func NewETLUsecase(
	logger log.Logger,
	sdRepo data.SourceDataRepo,
	vrp *VideoRankProcessor,
	vdp *VideoDetailProcessor,
	gdp *GoodsDetailProcessor,
) *ETLUsecase

NewETLUsecase 是 ETLUsecase 的构造函数，负责依赖注入和初始化 wire 会自动找到并注入 NewVideoRankProcessor 和 NewVideoDetailProcessor 的实例

func (*ETLUsecase) Run 

func (u *ETLUsecase) Run(ctx context.Context, dataType string) error

Run processes all unprocessed source data.

func (*ETLUsecase) RunWithType 

func (u *ETLUsecase) RunWithType(ctx context.Context, dataType string) error

RunWithType 根据指定的数据类型，查找并处理所有未处理的数据

type FeiguaAwemeDto 

type FeiguaAwemeDto struct {
	AwemeId        string `json:"awemeId"`
	AwemeCoverUrl  string `json:"awemeCoverUrl"`
	AwemeDesc      string `json:"awemeDesc"`
	AwemePubTime   string `json:"awemePubTime"`
	AwemeShareUrl  string `json:"awemeShareUrl"`
	DurationStr    string `json:"durationStr"`
	AwemeScoreStr  string `json:"awemeScoreStr"`
	AwemeDetailUrl string `json:"awemeDetailUrl"`
}

type FeiguaBaseResponse 

type FeiguaBaseResponse struct {
	Status bool   `json:"Status"`
	Msg    string `json:"Msg"`
	Code   int    `json:"Code"`
}

FeiguaBaseResponse 通过嵌套(embedding)的方式，为所有飞瓜API响应提供通用字段

type FeiguaBloggerDto 

type FeiguaBloggerDto struct {
	BloggerId        json.Number `json:"bloggerId"`
	BloggerUid       string      `json:"bloggerUid"`
	BloggerName      string      `json:"bloggerName"`
	BloggerAvatar    string      `json:"bloggerAvatar"`
	FansNum          json.Number `json:"fansNum"`
	Tag              string      `json:"tag"`
	BloggerDetailUrl string      `json:"bloggerDetailUrl"`
}

type FeiguaGoodsDto 

type FeiguaGoodsDto struct {
	Gid             string      `json:"gid"`
	Title           string      `json:"title"`
	CoverUrl        string      `json:"coverUrl"`
	PriceRange      string      `json:"priceRange"`
	Price           json.Number `json:"price"`
	CosRatio        string      `json:"cosRatio"`
	CommissionPrice string      `json:"commissionPrice"`
	ShopName        string      `json:"shopName"`
	DouyinBrandName string      `json:"douyinBrandName"`
	CateNames       string      `json:"cateNames"`
	GoodsDetailUrl  string      `json:"goodsDetailUrl"`
}

type FeiguaVideoRankItem 

type FeiguaVideoRankItem struct {
	AwemeDto     FeiguaAwemeDto   `json:"baseAwemeDto"`
	GoodsDto     FeiguaGoodsDto   `json:"baseGoodsDto"`
	BloggerDto   FeiguaBloggerDto `json:"baseBloggerDto"`
	SalesCount   string           `json:"salesCount"`
	TotalSales   string           `json:"totalSales"`
	LikeCountInc string           `json:"likeCountInc"`
	PlayCountInc string           `json:"playCountInc"`
}

type FeiguaVideoRankResponse 

type FeiguaVideoRankResponse struct {
	FeiguaBaseResponse
	Encrypt bool   `json:"Encrypt"`
	Rnd     string `json:"Rnd"`
	Data    string `json:"Data"`
}

FeiguaVideoRankResponse 统一了API响应结构，嵌套通用响应字段 Status, Msg, Code 字段通过 FeiguaBaseResponse 提供 Encrypt, Rnd, Data 为加密数据相关字段

type FeiguaVideoSummaryDTO 

type FeiguaVideoSummaryDTO struct {
	PlayCountStr       string `json:"PlayCountStr"`
	LikeCountStr       string `json:"LikeCountStr"`
	CommentCountStr    string `json:"CommentCountStr"`
	ShareCountStr      string `json:"ShareCountStr"`
	CollectCountStr    string `json:"CollectCountStr"`
	InteractionRateStr string `json:"InteractionRateStr"`
	Score              string `json:"Score"`
	LikeCommentRateStr string `json:"LikeCommentRateStr"`
	SalesGmvStr        string `json:"SalesGmvStr"`
	SalesCountStr      string `json:"SalesCountStr"`
	GoodsCountStr      string `json:"GoodsCountStr"`
	GPM                string `json:"GPM"`
	AwemeType          int32  `json:"AwemeType"`
}

type FeiguaVideoSummaryData 

type FeiguaVideoSummaryData struct {
	FeiguaBaseResponse
	Data FeiguaVideoSummaryDTO `json:"data"`
}

FeiguaVideoSummaryData ...

type FeiguaVideoTrendData 

type FeiguaVideoTrendData struct {
	FeiguaBaseResponse
	Data []*FeiguaVideoTrendItem `json:"data"`
}

FeiguaVideoTrendData ...

type FeiguaVideoTrendDataItem 

type FeiguaVideoTrendDataItem struct {
	RecordTime      string `json:"record_time"`
	PlayCountStr    string `json:"play_count_str"`
	LikeCountStr    string `json:"like_count_str"`
	CommentCountStr string `json:"comment_count_str"`
	ShareCountStr   string `json:"share_count_str"`
	CollectCountStr string `json:"collect_count_str"`
}

FeiguaVideoTrendDataItem ...

type FeiguaVideoTrendItem 

type FeiguaVideoTrendItem struct {
	DateCode           json.Number `json:"DateCode"`
	LikeCount          json.Number `json:"LikeCount"`
	LikeCountStr       string      `json:"LikeCountStr"`
	ShareCount         json.Number `json:"ShareCount"`
	ShareCountStr      string      `json:"ShareCountStr"`
	CommentCount       json.Number `json:"CommentCount"`
	CommentCountStr    string      `json:"CommentCountStr"`
	CollectCount       json.Number `json:"CollectCount"`
	CollectCountStr    string      `json:"CollectCountStr"`
	InteractionRate    json.Number `json:"InteractionRate"`
	InteractionRateStr string      `json:"InteractionRateStr"`
	SalesGmv           json.Number `json:"SalesGmv"`
	SalesGmvStr        string      `json:"SalesGmvStr"`
	SalesCount         json.Number `json:"SalesCount"`
	SalesCountStr      string      `json:"SalesCountStr"`
	Fans               json.Number `json:"Fans"`
	FansStr            string      `json:"FansStr"`
	GPM                json.Number `json:"GPM"`
	GPMStr             string      `json:"GPMStr"`
	IncLikeCount       json.Number `json:"IncLikeCount"`
	IncLikeCountStr    string      `json:"IncLikeCountStr"`
	IncShareCount      json.Number `json:"IncShareCount"`
	IncShareCountStr   string      `json:"IncShareCountStr"`
	IncCommentCount    json.Number `json:"IncCommentCount"`
	IncCommentCountStr string      `json:"IncCommentCountStr"`
	IncCollectCount    json.Number `json:"IncCollectCount"`
	IncCollectCountStr string      `json:"IncCollectCountStr"`
	IncSalesCount      json.Number `json:"IncSalesCount"`
	IncSalesCountStr   string      `json:"IncSalesCountStr"`
	IncSalesGmv        json.Number `json:"IncSalesGmv"`
	IncSalesGmvStr     string      `json:"IncSalesGmvStr"`
	IncFans            json.Number `json:"IncFans"`
	IncFansStr         string      `json:"IncFansStr"`
	ListTimeStr        string      `json:"ListTimeStr"`
	TimeStamp          json.Number `json:"TimeStamp"`
}

FeiguaVideoTrendItem 使用 json.Number 接收所有数值，增加所有字段

type GoodsAwemeAnalysisETLProcessor 

type GoodsAwemeAnalysisETLProcessor struct {
	// contains filtered or unexported fields
}

GoodsAwemeAnalysisETLProcessor 实现了 Processor 接口

func NewGoodsAwemeAnalysisETLProcessor 

func NewGoodsAwemeAnalysisETLProcessor(
	logger log.Logger,
	sdRepo data.SourceDataRepo,
	pRepo data.ProductRepo,
	gaRepo data.GoodsAwemeRepo,
) *GoodsAwemeAnalysisETLProcessor

NewGoodsAwemeAnalysisETLProcessor 构造函数

func (*GoodsAwemeAnalysisETLProcessor) Process 

func (p *GoodsAwemeAnalysisETLProcessor) Process(ctx context.Context, rawData *v1.SourceData) error

Process 是 ETL 的核心处理方法

type GoodsDetailProcessor 

type GoodsDetailProcessor struct {
	// contains filtered or unexported fields
}

GoodsDetailProcessor 统一处理器 (构造函数等保持不变)

func NewGoodsDetailProcessor 

func NewGoodsDetailProcessor(
	logger log.Logger, sdRepo data.SourceDataRepo, pRepo data.ProductRepo,
	gaRepo data.GoodsAwemeRepo, gtRepo data.GoodsTrendRepo, gpRepo data.GoodsPortrayalRepo,
	gbRepo data.GoodsBloggerRepo,

) *GoodsDetailProcessor

func (*GoodsDetailProcessor) Process 

func (p *GoodsDetailProcessor) Process(ctx context.Context, rawData *v1.SourceData) error

Process 路由逻辑保持不变

type ProcessError 

type ProcessError struct {
	Msg      string
	SourceID int64
	Err      error
}

ProcessError 自定义错误类型，方便记录日志

func (*ProcessError) Error 

func (e *ProcessError) Error() string

type Processor 

type Processor interface {
	Process(ctx context.Context, rawData *v1.SourceData) error
}

Processor defines a generic ETL processor.

type VideoDetailProcessor 

type VideoDetailProcessor struct {
	// contains filtered or unexported fields
}

VideoDetailProcessor 负责处理所有视频详情相关的数据，包括 summary 和 trend

func NewVideoDetailProcessor 

func NewVideoDetailProcessor(
	logger log.Logger,
	sourceDataRepo data.SourceDataRepo,
	videoRepo data.VideoRepo,
	bloggerRepo data.BloggerRepo,
	videoTrendRepo data.VideoTrendRepo,
) *VideoDetailProcessor

NewVideoDetailProcessor .

func (*VideoDetailProcessor) Process 

func (p *VideoDetailProcessor) Process(ctx context.Context, rawData *v1.SourceData) error

Process 实现了 Processor 接口，负责DTO到DO的转换和任务分发 Process implements the Processor interface, routing data by type.

type VideoRankProcessor 

type VideoRankProcessor struct {
	// contains filtered or unexported fields
}

VideoRankProcessor implements Processor for video rank data.

func NewVideoRankProcessor 

func NewVideoRankProcessor(vrRepo data.VideoRankRepo, sdRepo data.SourceDataRepo, vRepo data.VideoRepo, pRepo data.ProductRepo, bRepo data.BloggerRepo, logger log.Logger) *VideoRankProcessor

func (*VideoRankProcessor) Process 

func (p *VideoRankProcessor) Process(ctx context.Context, rawData *v1.SourceData) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.