feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投 - 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引 - 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存 - 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
154
internal/task/collection_batch_consumer.go
Normal file
154
internal/task/collection_batch_consumer.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// collectionWriteTimeout 是一批采集记录写库的超时。
|
||||
const collectionWriteTimeout = 30 * time.Second
|
||||
|
||||
// CollectionBatchConsumer 消费采集记录消息(topic card.collection),攒满一批或等满 1 秒写一次库:
|
||||
// 写成功后整批确认,写失败整批延迟重投;重复记录由唯一索引忽略。写库只占一个连接。
|
||||
type CollectionBatchConsumer struct {
|
||||
db *gorm.DB
|
||||
batchSize int
|
||||
logger *zap.Logger
|
||||
items chan *collectionItem
|
||||
}
|
||||
|
||||
type collectionItem struct {
|
||||
msg CollectionMessage
|
||||
done chan error
|
||||
}
|
||||
|
||||
// NewCollectionBatchConsumer 创建采集记录批量消费者。
|
||||
func NewCollectionBatchConsumer(db *gorm.DB, batchSize int, logger *zap.Logger) *CollectionBatchConsumer {
|
||||
if batchSize <= 0 {
|
||||
batchSize = constants.PollingCollectionDefaultBatchSize
|
||||
}
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &CollectionBatchConsumer{db: db, batchSize: batchSize, logger: logger,
|
||||
items: make(chan *collectionItem, constants.PollingCollectionMaxInFlight)}
|
||||
}
|
||||
|
||||
// Start 启动攒批写库循环,ctx 结束时写完已缓冲的一批后退出。
|
||||
func (c *CollectionBatchConsumer) Start(ctx context.Context) {
|
||||
go c.run(ctx)
|
||||
}
|
||||
|
||||
// Handle 处理一条采集记录消息:交给攒批循环并等待本批写库结果。
|
||||
func (c *CollectionBatchConsumer) Handle(ctx context.Context, body []byte, attempts uint16, _ func()) time.Duration {
|
||||
var msg CollectionMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || (msg.Success == nil && msg.Failure == nil) {
|
||||
c.logger.Warn("采集记录消息无效,丢弃", zap.ByteString("body", body))
|
||||
return 0
|
||||
}
|
||||
item := &collectionItem{msg: msg, done: make(chan error, 1)}
|
||||
select {
|
||||
case c.items <- item:
|
||||
case <-ctx.Done():
|
||||
return ingestRetryDelay(attempts)
|
||||
}
|
||||
select {
|
||||
case err := <-item.done:
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
delay := ingestRetryDelay(attempts)
|
||||
if attempts >= constants.PollingIngestErrorLogFromAttempt {
|
||||
c.logger.Error("写采集记录持续失败,继续重投", zap.Uint16("attempts", attempts),
|
||||
zap.String("observation_id", collectionObservationID(msg)), zap.Duration("retry_after", delay), zap.Error(err))
|
||||
}
|
||||
return delay
|
||||
case <-ctx.Done():
|
||||
return ingestRetryDelay(attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CollectionBatchConsumer) run(ctx context.Context) {
|
||||
ticker := time.NewTicker(constants.PollingCollectionFlushInterval)
|
||||
defer ticker.Stop()
|
||||
batch := make([]*collectionItem, 0, c.batchSize)
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
c.flush(batch)
|
||||
batch = make([]*collectionItem, 0, c.batchSize)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
flush()
|
||||
return
|
||||
case item := <-c.items:
|
||||
batch = append(batch, item)
|
||||
if len(batch) >= c.batchSize {
|
||||
flush()
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// flush 一次写入一批成功记录与失败记录,并把结果通知本批每条消息。
|
||||
func (c *CollectionBatchConsumer) flush(batch []*collectionItem) {
|
||||
var successes []*model.CardTrafficCollection
|
||||
var failures []*model.CardTrafficCollectionFailure
|
||||
for _, item := range batch {
|
||||
if item.msg.Success != nil {
|
||||
successes = append(successes, item.msg.Success)
|
||||
}
|
||||
if item.msg.Failure != nil {
|
||||
failures = append(failures, item.msg.Failure)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), collectionWriteTimeout)
|
||||
defer cancel()
|
||||
err := c.write(ctx, successes, failures)
|
||||
if err != nil {
|
||||
c.logger.Warn("批量写采集记录失败,整批重投", zap.Int("success_rows", len(successes)),
|
||||
zap.Int("failure_rows", len(failures)), zap.Error(err))
|
||||
}
|
||||
for _, item := range batch {
|
||||
item.done <- err
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CollectionBatchConsumer) write(ctx context.Context, successes []*model.CardTrafficCollection, failures []*model.CardTrafficCollectionFailure) error {
|
||||
return c.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if len(successes) > 0 {
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(successes, len(successes)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(failures, len(failures)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func collectionObservationID(msg CollectionMessage) string {
|
||||
if msg.Success != nil {
|
||||
return msg.Success.ObservationID
|
||||
}
|
||||
if msg.Failure != nil {
|
||||
return msg.Failure.ObservationID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user