feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s

- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投
- 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引
- 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存
- 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-06 16:33:29 +09:00
parent 7e0e37933e
commit 1214181e78
41 changed files with 2007 additions and 46 deletions

View File

@@ -0,0 +1,154 @@
package task
import (
"context"
"encoding/json"
"time"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// collectionWriteTimeout 是一批采集记录写库的超时。
const collectionWriteTimeout = 30 * time.Second
// CollectionBatchConsumer 消费采集记录消息(topic card.collection),攒满一批或等满 1 秒写一次库:
// 写成功后整批确认,写失败整批延迟重投;重复记录由唯一索引忽略。写库只占一个连接。
type CollectionBatchConsumer struct {
db *gorm.DB
batchSize int
logger *zap.Logger
items chan *collectionItem
}
type collectionItem struct {
msg CollectionMessage
done chan error
}
// NewCollectionBatchConsumer 创建采集记录批量消费者。
func NewCollectionBatchConsumer(db *gorm.DB, batchSize int, logger *zap.Logger) *CollectionBatchConsumer {
if batchSize <= 0 {
batchSize = constants.PollingCollectionDefaultBatchSize
}
if logger == nil {
logger = zap.NewNop()
}
return &CollectionBatchConsumer{db: db, batchSize: batchSize, logger: logger,
items: make(chan *collectionItem, constants.PollingCollectionMaxInFlight)}
}
// Start 启动攒批写库循环,ctx 结束时写完已缓冲的一批后退出。
func (c *CollectionBatchConsumer) Start(ctx context.Context) {
go c.run(ctx)
}
// Handle 处理一条采集记录消息:交给攒批循环并等待本批写库结果。
func (c *CollectionBatchConsumer) Handle(ctx context.Context, body []byte, attempts uint16, _ func()) time.Duration {
var msg CollectionMessage
if err := json.Unmarshal(body, &msg); err != nil || (msg.Success == nil && msg.Failure == nil) {
c.logger.Warn("采集记录消息无效,丢弃", zap.ByteString("body", body))
return 0
}
item := &collectionItem{msg: msg, done: make(chan error, 1)}
select {
case c.items <- item:
case <-ctx.Done():
return ingestRetryDelay(attempts)
}
select {
case err := <-item.done:
if err == nil {
return 0
}
delay := ingestRetryDelay(attempts)
if attempts >= constants.PollingIngestErrorLogFromAttempt {
c.logger.Error("写采集记录持续失败,继续重投", zap.Uint16("attempts", attempts),
zap.String("observation_id", collectionObservationID(msg)), zap.Duration("retry_after", delay), zap.Error(err))
}
return delay
case <-ctx.Done():
return ingestRetryDelay(attempts)
}
}
func (c *CollectionBatchConsumer) run(ctx context.Context) {
ticker := time.NewTicker(constants.PollingCollectionFlushInterval)
defer ticker.Stop()
batch := make([]*collectionItem, 0, c.batchSize)
flush := func() {
if len(batch) == 0 {
return
}
c.flush(batch)
batch = make([]*collectionItem, 0, c.batchSize)
}
for {
select {
case <-ctx.Done():
flush()
return
case item := <-c.items:
batch = append(batch, item)
if len(batch) >= c.batchSize {
flush()
}
case <-ticker.C:
flush()
}
}
}
// flush 一次写入一批成功记录与失败记录,并把结果通知本批每条消息。
func (c *CollectionBatchConsumer) flush(batch []*collectionItem) {
var successes []*model.CardTrafficCollection
var failures []*model.CardTrafficCollectionFailure
for _, item := range batch {
if item.msg.Success != nil {
successes = append(successes, item.msg.Success)
}
if item.msg.Failure != nil {
failures = append(failures, item.msg.Failure)
}
}
ctx, cancel := context.WithTimeout(context.Background(), collectionWriteTimeout)
defer cancel()
err := c.write(ctx, successes, failures)
if err != nil {
c.logger.Warn("批量写采集记录失败,整批重投", zap.Int("success_rows", len(successes)),
zap.Int("failure_rows", len(failures)), zap.Error(err))
}
for _, item := range batch {
item.done <- err
}
}
func (c *CollectionBatchConsumer) write(ctx context.Context, successes []*model.CardTrafficCollection, failures []*model.CardTrafficCollectionFailure) error {
return c.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if len(successes) > 0 {
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(successes, len(successes)).Error; err != nil {
return err
}
}
if len(failures) > 0 {
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(failures, len(failures)).Error; err != nil {
return err
}
}
return nil
})
}
func collectionObservationID(msg CollectionMessage) string {
if msg.Success != nil {
return msg.Success.ObservationID
}
if msg.Failure != nil {
return msg.Failure.ObservationID
}
return ""
}