feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投 - 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引 - 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存 - 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -124,7 +124,7 @@ func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.Card
|
||||
}
|
||||
return set, del
|
||||
}
|
||||
if !card.EnablePolling || (e.base.scopeChecker != nil && !e.base.scopeChecker.ShouldEnqueue(ctx, card)) {
|
||||
if !e.base.scopeAllowsCached(ctx, card) {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
return set, del
|
||||
}
|
||||
@@ -143,7 +143,9 @@ func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.Card
|
||||
if lost.Load() || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow && flowEvaluated {
|
||||
// 异步入库时流量评估发生在入库段,由其写入的评估标记表示套餐检查间隔内已触发过停复机评估。
|
||||
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow &&
|
||||
(flowEvaluated || e.base.flowEvaluatedRecently(ctx, msg.CardID)) {
|
||||
executed[checkType] = stepOK()
|
||||
continue
|
||||
}
|
||||
|
||||
154
internal/task/collection_batch_consumer.go
Normal file
154
internal/task/collection_batch_consumer.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// collectionWriteTimeout 是一批采集记录写库的超时。
|
||||
const collectionWriteTimeout = 30 * time.Second
|
||||
|
||||
// CollectionBatchConsumer 消费采集记录消息(topic card.collection),攒满一批或等满 1 秒写一次库:
|
||||
// 写成功后整批确认,写失败整批延迟重投;重复记录由唯一索引忽略。写库只占一个连接。
|
||||
type CollectionBatchConsumer struct {
|
||||
db *gorm.DB
|
||||
batchSize int
|
||||
logger *zap.Logger
|
||||
items chan *collectionItem
|
||||
}
|
||||
|
||||
type collectionItem struct {
|
||||
msg CollectionMessage
|
||||
done chan error
|
||||
}
|
||||
|
||||
// NewCollectionBatchConsumer 创建采集记录批量消费者。
|
||||
func NewCollectionBatchConsumer(db *gorm.DB, batchSize int, logger *zap.Logger) *CollectionBatchConsumer {
|
||||
if batchSize <= 0 {
|
||||
batchSize = constants.PollingCollectionDefaultBatchSize
|
||||
}
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &CollectionBatchConsumer{db: db, batchSize: batchSize, logger: logger,
|
||||
items: make(chan *collectionItem, constants.PollingCollectionMaxInFlight)}
|
||||
}
|
||||
|
||||
// Start 启动攒批写库循环,ctx 结束时写完已缓冲的一批后退出。
|
||||
func (c *CollectionBatchConsumer) Start(ctx context.Context) {
|
||||
go c.run(ctx)
|
||||
}
|
||||
|
||||
// Handle 处理一条采集记录消息:交给攒批循环并等待本批写库结果。
|
||||
func (c *CollectionBatchConsumer) Handle(ctx context.Context, body []byte, attempts uint16, _ func()) time.Duration {
|
||||
var msg CollectionMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || (msg.Success == nil && msg.Failure == nil) {
|
||||
c.logger.Warn("采集记录消息无效,丢弃", zap.ByteString("body", body))
|
||||
return 0
|
||||
}
|
||||
item := &collectionItem{msg: msg, done: make(chan error, 1)}
|
||||
select {
|
||||
case c.items <- item:
|
||||
case <-ctx.Done():
|
||||
return ingestRetryDelay(attempts)
|
||||
}
|
||||
select {
|
||||
case err := <-item.done:
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
delay := ingestRetryDelay(attempts)
|
||||
if attempts >= constants.PollingIngestErrorLogFromAttempt {
|
||||
c.logger.Error("写采集记录持续失败,继续重投", zap.Uint16("attempts", attempts),
|
||||
zap.String("observation_id", collectionObservationID(msg)), zap.Duration("retry_after", delay), zap.Error(err))
|
||||
}
|
||||
return delay
|
||||
case <-ctx.Done():
|
||||
return ingestRetryDelay(attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CollectionBatchConsumer) run(ctx context.Context) {
|
||||
ticker := time.NewTicker(constants.PollingCollectionFlushInterval)
|
||||
defer ticker.Stop()
|
||||
batch := make([]*collectionItem, 0, c.batchSize)
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
c.flush(batch)
|
||||
batch = make([]*collectionItem, 0, c.batchSize)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
flush()
|
||||
return
|
||||
case item := <-c.items:
|
||||
batch = append(batch, item)
|
||||
if len(batch) >= c.batchSize {
|
||||
flush()
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// flush 一次写入一批成功记录与失败记录,并把结果通知本批每条消息。
|
||||
func (c *CollectionBatchConsumer) flush(batch []*collectionItem) {
|
||||
var successes []*model.CardTrafficCollection
|
||||
var failures []*model.CardTrafficCollectionFailure
|
||||
for _, item := range batch {
|
||||
if item.msg.Success != nil {
|
||||
successes = append(successes, item.msg.Success)
|
||||
}
|
||||
if item.msg.Failure != nil {
|
||||
failures = append(failures, item.msg.Failure)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), collectionWriteTimeout)
|
||||
defer cancel()
|
||||
err := c.write(ctx, successes, failures)
|
||||
if err != nil {
|
||||
c.logger.Warn("批量写采集记录失败,整批重投", zap.Int("success_rows", len(successes)),
|
||||
zap.Int("failure_rows", len(failures)), zap.Error(err))
|
||||
}
|
||||
for _, item := range batch {
|
||||
item.done <- err
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CollectionBatchConsumer) write(ctx context.Context, successes []*model.CardTrafficCollection, failures []*model.CardTrafficCollectionFailure) error {
|
||||
return c.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if len(successes) > 0 {
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(successes, len(successes)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(failures, len(failures)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func collectionObservationID(msg CollectionMessage) string {
|
||||
if msg.Success != nil {
|
||||
return msg.Success.ObservationID
|
||||
}
|
||||
if msg.Failure != nil {
|
||||
return msg.Failure.ObservationID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
177
internal/task/ingest_executor.go
Normal file
177
internal/task/ingest_executor.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
stderrors "errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// IngestExecutor 消费轮询观测入库消息(topic card.ingest):复用卡观测应用服务写库,按采集 ID 去重;
|
||||
// 可重试错误按退避延迟重投且不设次数上限,业务性错误记录后丢弃。
|
||||
type IngestExecutor struct {
|
||||
base *PollingBase
|
||||
observation *cardapp.Service
|
||||
timeout time.Duration
|
||||
// lastLagMS 是最近处理消息的发布到开始处理的等待时长,供积压监控读取。
|
||||
lastLagMS atomic.Int64
|
||||
}
|
||||
|
||||
// NewIngestExecutor 创建入库消费者。
|
||||
func NewIngestExecutor(base *PollingBase, observation *cardapp.Service, timeout time.Duration) *IngestExecutor {
|
||||
if timeout <= 0 {
|
||||
timeout = constants.PollingIngestDefaultTimeout
|
||||
}
|
||||
return &IngestExecutor{base: base, observation: observation, timeout: timeout}
|
||||
}
|
||||
|
||||
// LastLag 返回最近处理的入库消息从发布到开始处理的等待时长。
|
||||
func (e *IngestExecutor) LastLag() time.Duration {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(e.lastLagMS.Load()) * time.Millisecond
|
||||
}
|
||||
|
||||
// Handle 处理一条入库消息,返回 0 表示确认,正数表示按该延迟重投。
|
||||
func (e *IngestExecutor) Handle(ctx context.Context, body []byte, attempts uint16, _ func()) time.Duration {
|
||||
var msg IngestMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || !msg.valid() {
|
||||
e.base.logger.Warn("入库消息无效,丢弃", zap.ByteString("body", body))
|
||||
return 0
|
||||
}
|
||||
if !msg.PublishedAt.IsZero() {
|
||||
e.lastLagMS.Store(time.Since(msg.PublishedAt).Milliseconds())
|
||||
}
|
||||
if e.alreadyDone(ctx, msg.ObservationID) {
|
||||
return 0
|
||||
}
|
||||
taskCtx, cancel := context.WithTimeout(ctx, e.timeout)
|
||||
defer cancel()
|
||||
taskCtx = withPollingWorkerAuditContext(taskCtx, msg.TaskType, pollingPriorityTaskName(msg.TaskType), msg.ObservationID)
|
||||
changed, err := e.apply(taskCtx, msg)
|
||||
if err != nil {
|
||||
return e.onFailure(msg, attempts, err)
|
||||
}
|
||||
e.markDone(msg.ObservationID)
|
||||
if changed && msg.Kind == ingestKindTraffic {
|
||||
e.markFlowEvaluated(msg.CardID)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// apply 按观测类型调用卡观测应用服务,返回流量是否产生增量或跨月。
|
||||
func (e *IngestExecutor) apply(ctx context.Context, msg IngestMessage) (bool, error) {
|
||||
if e.observation == nil {
|
||||
return false, errors.New(errors.CodeInternalError, "卡观测能力未配置")
|
||||
}
|
||||
switch msg.Kind {
|
||||
case ingestKindTraffic:
|
||||
decision, err := e.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||
CardID: msg.CardID, GatewayReadingMB: msg.Traffic.ReadingMB, ReadingResult: msg.Traffic.ReadingResult,
|
||||
ResetDay: msg.Traffic.ResetDay, Raw: msg.Raw, ICCID: msg.ICCID, CarrierID: msg.CarrierID,
|
||||
Failure: &carddomain.CollectionFailure{DurationMS: msg.Traffic.DurationMS},
|
||||
Metadata: msg.metadata(),
|
||||
})
|
||||
return decision.IncrementMB > 0 || decision.CrossMonth, err
|
||||
case ingestKindNetwork:
|
||||
_, err := e.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
||||
CardID: msg.CardID, GatewayStatus: msg.Network.Status, GatewayExtend: msg.Network.Extend,
|
||||
GatewayIMEI: msg.Network.IMEI, Metadata: msg.metadata(),
|
||||
})
|
||||
return false, err
|
||||
case ingestKindRealname:
|
||||
_, err := e.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
||||
CardID: msg.CardID, Verified: msg.Realname.Verified, Metadata: msg.metadata(),
|
||||
})
|
||||
return false, err
|
||||
}
|
||||
return false, errors.New(errors.CodeInvalidParam, "未知的入库观测类型")
|
||||
}
|
||||
|
||||
// onFailure 业务性错误丢弃,其余按退避重投;第 10 次起每次重投记 ERROR。
|
||||
func (e *IngestExecutor) onFailure(msg IngestMessage, attempts uint16, err error) time.Duration {
|
||||
fields := []zap.Field{zap.String("kind", msg.Kind), zap.Uint("card_id", msg.CardID),
|
||||
zap.String("observation_id", msg.ObservationID), zap.Uint16("attempts", attempts), zap.Error(err)}
|
||||
if ingestBusinessError(err) {
|
||||
e.base.logger.Warn("入库遇到业务性错误,丢弃消息", fields...)
|
||||
return 0
|
||||
}
|
||||
delay := ingestRetryDelay(attempts)
|
||||
fields = append(fields, zap.Duration("retry_after", delay))
|
||||
if attempts >= constants.PollingIngestErrorLogFromAttempt {
|
||||
e.base.logger.Error("入库持续失败,继续重投", fields...)
|
||||
} else {
|
||||
e.base.logger.Warn("入库失败,延迟重投", fields...)
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
// ingestBusinessError 判断错误是否为重试也不会成功的业务性错误(卡不存在、观测数据校验失败)。
|
||||
func ingestBusinessError(err error) bool {
|
||||
var appErr *errors.AppError
|
||||
if !stderrors.As(err, &appErr) {
|
||||
return false
|
||||
}
|
||||
switch appErr.Code {
|
||||
case errors.CodeNotFound, errors.CodeInvalidParam, errors.CodeInvalidStatus:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *IngestExecutor) alreadyDone(ctx context.Context, observationID string) bool {
|
||||
if e.base.redis == nil {
|
||||
return false
|
||||
}
|
||||
exists, err := e.base.redis.Exists(ctx, constants.RedisPollingIngestDoneKey(observationID)).Result()
|
||||
return err == nil && exists > 0
|
||||
}
|
||||
|
||||
func (e *IngestExecutor) markDone(observationID string) {
|
||||
if e.base.redis == nil {
|
||||
return
|
||||
}
|
||||
markCtx, cancel := pollingFallbackContext()
|
||||
defer cancel()
|
||||
if err := e.base.redis.Set(markCtx, constants.RedisPollingIngestDoneKey(observationID), "1", constants.PollingIngestDoneTTL).Err(); err != nil {
|
||||
e.base.logger.Warn("写入库已处理标记失败,重投时由业务幂等兜底", zap.String("observation_id", observationID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// markFlowEvaluated 流量入库产生增量或跨月时写评估标记,有效期取该卡套餐检查间隔;套餐步骤据此跳过。
|
||||
func (e *IngestExecutor) markFlowEvaluated(cardID uint) {
|
||||
if e.base.redis == nil {
|
||||
return
|
||||
}
|
||||
markCtx, cancel := pollingFallbackContext()
|
||||
defer cancel()
|
||||
ttl := constants.PollingFlowEvaluatedFallbackTTL
|
||||
if card, err := e.base.getCardWithCache(markCtx, cardID); err == nil && e.base.configMgr != nil {
|
||||
info, ok := e.base.configMgr.MergedTaskIntervals(card)[constants.TaskTypePollingPackage]
|
||||
if !ok || info.Interval <= 0 {
|
||||
return
|
||||
}
|
||||
ttl = time.Duration(info.Interval) * time.Second
|
||||
}
|
||||
if err := e.base.redis.Set(markCtx, constants.RedisPollingFlowEvaluatedKey(cardID), "1", ttl).Err(); err != nil {
|
||||
e.base.logger.Debug("写流量评估标记失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// flowEvaluatedRecently 判断入库段是否在套餐检查间隔内写过流量评估标记。
|
||||
func (b *PollingBase) flowEvaluatedRecently(ctx context.Context, cardID uint) bool {
|
||||
if b == nil || b.redis == nil {
|
||||
return false
|
||||
}
|
||||
exists, err := b.redis.Exists(ctx, constants.RedisPollingFlowEvaluatedKey(cardID)).Result()
|
||||
return err == nil && exists > 0
|
||||
}
|
||||
156
internal/task/ingest_message.go
Normal file
156
internal/task/ingest_message.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// 入库消息的观测类型。
|
||||
const (
|
||||
ingestKindTraffic = "traffic"
|
||||
ingestKindNetwork = "network"
|
||||
ingestKindRealname = "realname"
|
||||
)
|
||||
|
||||
// ingestRawResponseLimit 是入库消息携带原始响应的上限:比采集记录保存上限多 1 字节,使"超过 4KB 截断"判断不变,同时控制消息大小。
|
||||
const ingestRawResponseLimit = 4096 + 1
|
||||
|
||||
// IngestMessage 是一次轮询观测的入库消息(topic card.ingest)。采集时间取网关返回时刻,入库全程按它归属日期与判断新旧。
|
||||
type IngestMessage struct {
|
||||
Kind string `json:"kind"`
|
||||
CardID uint `json:"card_id"`
|
||||
CarrierID uint `json:"carrier_id"`
|
||||
ICCID string `json:"iccid"`
|
||||
ObservationID string `json:"observation_id"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
Source string `json:"source"`
|
||||
Scene string `json:"scene"`
|
||||
CorrelationID string `json:"correlation_id"`
|
||||
// TaskType 是发起观测的轮询任务类型,入库时用于审计上下文与流量评估标记。
|
||||
TaskType string `json:"task_type"`
|
||||
Traffic *IngestTrafficPayload `json:"traffic,omitempty"`
|
||||
Network *IngestNetworkPayload `json:"network,omitempty"`
|
||||
Realname *IngestRealnamePayload `json:"realname,omitempty"`
|
||||
Raw *carddomain.GatewayRaw `json:"raw,omitempty"`
|
||||
PublishedAt time.Time `json:"published_at"`
|
||||
}
|
||||
|
||||
// IngestTrafficPayload 是流量观测内容。
|
||||
type IngestTrafficPayload struct {
|
||||
ReadingMB float64 `json:"reading_mb"`
|
||||
ReadingResult string `json:"reading_result"`
|
||||
ResetDay int `json:"reset_day"`
|
||||
DurationMS int `json:"duration_ms"`
|
||||
}
|
||||
|
||||
// IngestNetworkPayload 是网络状态观测内容。
|
||||
type IngestNetworkPayload struct {
|
||||
Status string `json:"status"`
|
||||
Extend string `json:"extend"`
|
||||
IMEI string `json:"imei"`
|
||||
}
|
||||
|
||||
// IngestRealnamePayload 是实名观测内容。
|
||||
type IngestRealnamePayload struct {
|
||||
Verified bool `json:"verified"`
|
||||
}
|
||||
|
||||
// metadata 还原观测元数据。
|
||||
func (m IngestMessage) metadata() carddomain.ObservationMetadata {
|
||||
return carddomain.ObservationMetadata{
|
||||
ObservationID: m.ObservationID, Source: m.Source, Scene: m.Scene,
|
||||
ObservedAt: m.ObservedAt, CorrelationID: m.CorrelationID,
|
||||
}
|
||||
}
|
||||
|
||||
// valid 校验消息必需字段。
|
||||
func (m IngestMessage) valid() bool {
|
||||
if m.CardID == 0 || m.ObservationID == "" || m.ObservedAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
switch m.Kind {
|
||||
case ingestKindTraffic:
|
||||
return m.Traffic != nil
|
||||
case ingestKindNetwork:
|
||||
return m.Network != nil
|
||||
case ingestKindRealname:
|
||||
return m.Realname != nil
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CollectionMessage 是一条采集记录消息(topic card.collection),成功记录与失败记录二选一。
|
||||
type CollectionMessage struct {
|
||||
Success *model.CardTrafficCollection `json:"success,omitempty"`
|
||||
Failure *model.CardTrafficCollectionFailure `json:"failure,omitempty"`
|
||||
}
|
||||
|
||||
// MessagePublisher 发布 NSQ 消息。
|
||||
type MessagePublisher interface {
|
||||
Publish(topic string, body []byte) error
|
||||
}
|
||||
|
||||
// IngestPublisher 发布入库消息与采集记录消息。
|
||||
type IngestPublisher struct {
|
||||
producer MessagePublisher
|
||||
}
|
||||
|
||||
// NewIngestPublisher 创建入库消息发布器;producer 为 nil 时所有发布都返回错误,调用方降级同步入库。
|
||||
func NewIngestPublisher(producer MessagePublisher) *IngestPublisher {
|
||||
return &IngestPublisher{producer: producer}
|
||||
}
|
||||
|
||||
// PublishIngest 发布一条入库消息。
|
||||
func (p *IngestPublisher) PublishIngest(_ context.Context, msg IngestMessage) error {
|
||||
if p == nil || p.producer == nil {
|
||||
return errors.New(errors.CodeInternalError, "入库消息发布器未配置")
|
||||
}
|
||||
if msg.Raw != nil && len(msg.Raw.Response) > ingestRawResponseLimit {
|
||||
trimmed := *msg.Raw
|
||||
trimmed.Response = trimmed.Response[:ingestRawResponseLimit]
|
||||
msg.Raw = &trimmed
|
||||
}
|
||||
msg.PublishedAt = time.Now()
|
||||
body, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeInternalError, err, "编码入库消息失败")
|
||||
}
|
||||
if err := p.producer.Publish(constants.NSQCardIngestTopic, body); err != nil {
|
||||
return errors.Wrap(errors.CodeInternalError, err, "发布入库消息失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublishCollection 发布一条采集记录消息,实现卡观测服务的 CollectionSink。
|
||||
func (p *IngestPublisher) PublishCollection(_ context.Context, success *model.CardTrafficCollection, failure *model.CardTrafficCollectionFailure) error {
|
||||
if p == nil || p.producer == nil {
|
||||
return errors.New(errors.CodeInternalError, "采集记录消息发布器未配置")
|
||||
}
|
||||
body, err := json.Marshal(CollectionMessage{Success: success, Failure: failure})
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeInternalError, err, "编码采集记录消息失败")
|
||||
}
|
||||
if err := p.producer.Publish(constants.NSQCardCollectionTopic, body); err != nil {
|
||||
return errors.Wrap(errors.CodeInternalError, err, "发布采集记录消息失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ingestRetryDelay 返回第 attempts 次投递失败后的重投延迟;超过序列长度取最后一项,不设次数上限。
|
||||
func ingestRetryDelay(attempts uint16) time.Duration {
|
||||
delays := constants.PollingIngestRetryDelays
|
||||
index := int(attempts) - 1
|
||||
if index < 0 {
|
||||
index = 0
|
||||
}
|
||||
if index >= len(delays) {
|
||||
index = len(delays) - 1
|
||||
}
|
||||
return delays[index]
|
||||
}
|
||||
@@ -73,6 +73,10 @@ type PollingBase struct {
|
||||
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
||||
errorClassifier ErrorClassifier // 上游错误分类(按运营商账户调度的失败退避与告警使用)
|
||||
schedulerMode SchedulerModeReader // 当前调度模式;carrier 时旧 Asynq 轮询任务直接结束
|
||||
ingest *IngestPublisher // 入库消息发布器;为 nil 时轮询步骤同步入库
|
||||
ingestAsync func(ctx context.Context) bool
|
||||
scopeCache scopeCache
|
||||
resetDays resetDayCache
|
||||
}
|
||||
|
||||
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
||||
|
||||
@@ -69,8 +69,8 @@ func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutc
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
card, err := h.base.iotCardStore.GetByID(ctx, cardID)
|
||||
// 卡任务只用到 ICCID、运营商账户与轮询开关;读缓存即可,入库段写库后会失效缓存。
|
||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
@@ -95,6 +95,7 @@ func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutc
|
||||
queryStartedAt := time.Now()
|
||||
result, err := h.gateway.QueryFlow(queryCtx, &gateway.FlowQueryReq{CardNo: card.ICCID})
|
||||
unreadable := unreadableDetail{raw: domainGatewayRaw(rawCapture), failure: collectionFailure(err, time.Since(queryStartedAt))}
|
||||
unreadable.iccid, unreadable.carrierID = card.ICCID, card.CarrierID
|
||||
if gateway.IsRateLimited(err) {
|
||||
h.recordUnreadable(ctx, cardID, attempt.IntegrationID, constants.TrafficCollectionResultRateLimited, unreadable)
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
@@ -120,10 +121,25 @@ func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutc
|
||||
return h.fail(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
resetDay := h.base.carrierResetDay(ctx, card.CarrierID)
|
||||
durationMS := int(time.Since(queryStartedAt).Milliseconds())
|
||||
if h.base.publishIngest(ctx, priority, IngestMessage{
|
||||
Kind: ingestKindTraffic, CardID: cardID, CarrierID: card.CarrierID, ICCID: card.ICCID,
|
||||
ObservationID: attempt.IntegrationID, ObservedAt: now, Source: constants.CardObservationSourcePolling,
|
||||
Scene: constants.CardObservationSceneTrafficPolling, CorrelationID: attempt.IntegrationID,
|
||||
TaskType: constants.TaskTypePollingCarddata, Raw: unreadable.raw,
|
||||
Traffic: &IngestTrafficPayload{ReadingMB: float64(result.Used), ReadingResult: gatewayReadingResult(result),
|
||||
ResetDay: resetDay, DurationMS: durationMS},
|
||||
}) {
|
||||
// 异步入库:网关已成功,入库由入库消费者完成与重试,本步骤按成功结束。
|
||||
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
||||
return stepOK()
|
||||
}
|
||||
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||
CardID: cardID, GatewayReadingMB: float64(result.Used), ReadingResult: gatewayReadingResult(result),
|
||||
ResetDay: h.carrier.GetDataResetDay(ctx, card.CarrierID), Raw: unreadable.raw,
|
||||
Failure: &carddomain.CollectionFailure{DurationMS: int(time.Since(queryStartedAt).Milliseconds())},
|
||||
ResetDay: resetDay, Raw: unreadable.raw, ICCID: card.ICCID, CarrierID: card.CarrierID,
|
||||
Failure: &carddomain.CollectionFailure{DurationMS: durationMS},
|
||||
Metadata: carddomain.ObservationMetadata{
|
||||
ObservationID: attempt.IntegrationID, Source: constants.CardObservationSourcePolling,
|
||||
Scene: constants.CardObservationSceneTrafficPolling, ObservedAt: now, CorrelationID: attempt.IntegrationID,
|
||||
@@ -168,8 +184,10 @@ func (h *PollingCarddataHandler) fail(ctx context.Context, cardID uint, startedA
|
||||
// recordUnreadable 为没拿到有效读数的采集写采集记录(只记采集时刻有套餐的卡),失败只影响留痕。
|
||||
// unreadableDetail 是读数不可用时随采集失败记录保存的原始数据与错误详情。
|
||||
type unreadableDetail struct {
|
||||
raw *carddomain.GatewayRaw
|
||||
failure *carddomain.CollectionFailure
|
||||
raw *carddomain.GatewayRaw
|
||||
failure *carddomain.CollectionFailure
|
||||
iccid string
|
||||
carrierID uint
|
||||
}
|
||||
|
||||
func (h *PollingCarddataHandler) recordUnreadable(ctx context.Context, cardID uint, observationID, result string, detail unreadableDetail) {
|
||||
@@ -178,6 +196,7 @@ func (h *PollingCarddataHandler) recordUnreadable(ctx context.Context, cardID ui
|
||||
}
|
||||
if _, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||
CardID: cardID, ReadingResult: result, ResetDay: 1, Raw: detail.raw, Failure: detail.failure,
|
||||
ICCID: detail.iccid, CarrierID: detail.carrierID,
|
||||
Metadata: carddomain.ObservationMetadata{
|
||||
ObservationID: observationID, Source: constants.CardObservationSourcePolling,
|
||||
Scene: constants.CardObservationSceneTrafficPolling, ObservedAt: time.Now(), CorrelationID: observationID,
|
||||
|
||||
@@ -118,6 +118,16 @@ func (h *PollingCardStatusHandler) Step(ctx context.Context, cardID uint) StepOu
|
||||
return h.fail(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
if h.base.publishIngest(ctx, priority, IngestMessage{
|
||||
Kind: ingestKindNetwork, CardID: cardID, CarrierID: card.CarrierID, ICCID: card.ICCID,
|
||||
ObservationID: attempt.IntegrationID, ObservedAt: now, Source: constants.CardObservationSourcePolling,
|
||||
Scene: constants.CardObservationSceneNetworkPolling, CorrelationID: attempt.IntegrationID,
|
||||
TaskType: constants.TaskTypePollingCardStatus,
|
||||
Network: &IngestNetworkPayload{Status: result.CardStatus, Extend: result.Extend, IMEI: result.IMEI},
|
||||
}) {
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
||||
return asyncNetworkOutcome(card.EnablePolling, result.CardStatus, result.Extend)
|
||||
}
|
||||
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
||||
CardID: cardID, GatewayStatus: result.CardStatus, GatewayExtend: result.Extend, GatewayIMEI: result.IMEI,
|
||||
Metadata: carddomain.ObservationMetadata{
|
||||
@@ -170,3 +180,20 @@ func (h *PollingCardStatusHandler) fail(ctx context.Context, cardID uint, starte
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
||||
return h.base.stepFail(ctx, constants.GatewayOpQueryStatus, message, err)
|
||||
}
|
||||
|
||||
// asyncNetworkOutcome 在异步入库时按网关结果直接得出步骤结果:命中风险停机或已销户时停止轮询(入库段同时关闭卡轮询开关),
|
||||
// 观测到的网络状态按网关状态映射,供延迟确认比较。
|
||||
func asyncNetworkOutcome(enablePolling bool, gatewayStatus, gatewayExtend string) StepOutcome {
|
||||
extend := strings.TrimSpace(gatewayExtend)
|
||||
if enablePolling && carddomain.ShouldStopPollingForRisk(extend) {
|
||||
return StepOutcome{Result: StepStopPolling}
|
||||
}
|
||||
outcome := stepOK()
|
||||
if status, known := carddomain.MapGatewayNetworkStatus(gatewayStatus, extend); known {
|
||||
outcome.Observed = ObservedOffline
|
||||
if status == constants.NetworkStatusOnline {
|
||||
outcome.Observed = ObservedOnline
|
||||
}
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
104
internal/task/polling_ingest.go
Normal file
104
internal/task/polling_ingest.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// SetIngest 注入入库消息发布器与入库模式读取(受控系统配置 polling.ingest.mode,数秒内生效)。
|
||||
func (b *PollingBase) SetIngest(publisher *IngestPublisher, async func(ctx context.Context) bool) {
|
||||
b.ingest = publisher
|
||||
b.ingestAsync = async
|
||||
}
|
||||
|
||||
// publishIngest 在异步入库模式下发布入库消息,返回是否已发布;返回 false 时调用方按原逻辑同步入库。
|
||||
// 本轮同时是优先执行时保持同步入库:加急结果需在本轮写库后才能完成优先项。
|
||||
func (b *PollingBase) publishIngest(ctx context.Context, round *pollingPriorityRound, msg IngestMessage) bool {
|
||||
if round != nil || b.ingest == nil || b.ingestAsync == nil || !b.ingestAsync(ctx) {
|
||||
return false
|
||||
}
|
||||
if err := b.ingest.PublishIngest(ctx, msg); err != nil {
|
||||
b.logger.Warn("发布入库消息失败,本次改为同步入库", zap.String("kind", msg.Kind), zap.Uint("card_id", msg.CardID),
|
||||
zap.String("observation_id", msg.ObservationID), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// scopeCache 是卡轮询范围校验结果的进程内短缓存,只供按运营商账户调度的卡任务使用,减少每个卡任务查绑定与设备。
|
||||
// 设备停用轮询最迟在缓存时长后生效;生命周期事件仍按实时校验移出队列。
|
||||
type scopeCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[uint]scopeEntry
|
||||
}
|
||||
|
||||
type scopeEntry struct {
|
||||
allowed bool
|
||||
loadedAt time.Time
|
||||
}
|
||||
|
||||
// scopeAllowsCached 返回卡是否在轮询范围内(卡自身开关实时判断,设备开关按短缓存)。
|
||||
func (b *PollingBase) scopeAllowsCached(ctx context.Context, card *model.IotCard) bool {
|
||||
if !card.EnablePolling {
|
||||
return false
|
||||
}
|
||||
if b.scopeChecker == nil || card.IsStandalone {
|
||||
return true
|
||||
}
|
||||
b.scopeCache.mu.Lock()
|
||||
entry, ok := b.scopeCache.entries[card.ID]
|
||||
b.scopeCache.mu.Unlock()
|
||||
if ok && time.Since(entry.loadedAt) < constants.PollingScopeCacheTTL {
|
||||
return entry.allowed
|
||||
}
|
||||
allowed := b.scopeChecker.ShouldEnqueue(ctx, card)
|
||||
b.scopeCache.mu.Lock()
|
||||
if b.scopeCache.entries == nil || len(b.scopeCache.entries) > 200000 {
|
||||
b.scopeCache.entries = map[uint]scopeEntry{}
|
||||
}
|
||||
b.scopeCache.entries[card.ID] = scopeEntry{allowed: allowed, loadedAt: time.Now()}
|
||||
b.scopeCache.mu.Unlock()
|
||||
return allowed
|
||||
}
|
||||
|
||||
// resetDayCache 是运营商账户流量重置日的进程内缓存。
|
||||
type resetDayCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[uint]resetDayEntry
|
||||
}
|
||||
|
||||
type resetDayEntry struct {
|
||||
day int
|
||||
loadedAt time.Time
|
||||
}
|
||||
|
||||
// carrierResetDay 返回运营商账户流量重置日(1-28),成功读取的值按 1 分钟缓存;读取失败或未配置时返回 1 且不缓存。
|
||||
func (b *PollingBase) carrierResetDay(ctx context.Context, carrierID uint) int {
|
||||
b.resetDays.mu.Lock()
|
||||
entry, ok := b.resetDays.entries[carrierID]
|
||||
b.resetDays.mu.Unlock()
|
||||
if ok && time.Since(entry.loadedAt) < constants.PollingCarrierResetDayCacheTTL {
|
||||
return entry.day
|
||||
}
|
||||
if b.db == nil {
|
||||
return 1
|
||||
}
|
||||
var days []int
|
||||
if err := b.db.WithContext(ctx).Unscoped().Model(&model.Carrier{}).Where("id = ?", carrierID).Limit(1).
|
||||
Pluck("data_reset_day", &days).Error; err != nil || len(days) == 0 || days[0] == 0 {
|
||||
return 1
|
||||
}
|
||||
b.resetDays.mu.Lock()
|
||||
if b.resetDays.entries == nil {
|
||||
b.resetDays.entries = map[uint]resetDayEntry{}
|
||||
}
|
||||
b.resetDays.entries[carrierID] = resetDayEntry{day: days[0], loadedAt: time.Now()}
|
||||
b.resetDays.mu.Unlock()
|
||||
return days[0]
|
||||
}
|
||||
@@ -53,6 +53,9 @@ func (b *PollingBase) beginPollingPriority(ctx context.Context, cardID uint, tas
|
||||
if b == nil || b.priorityStore == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
if b.priorityIndexAbsent(ctx, cardID, taskType) {
|
||||
return nil, false, nil
|
||||
}
|
||||
item, err := b.priorityStore.FindActive(ctx, cardID, taskType)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
|
||||
73
internal/task/polling_priority_index.go
Normal file
73
internal/task/polling_priority_index.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// priorityIndexAbsent 判断活动优先项索引能否确认该卡该任务类型没有活动优先项。
|
||||
// 索引未建立、读取失败或成员存在时返回 false,调用方照常查库。
|
||||
func (b *PollingBase) priorityIndexAbsent(ctx context.Context, cardID uint, taskType string) bool {
|
||||
if b == nil || b.redis == nil {
|
||||
return false
|
||||
}
|
||||
pipe := b.redis.Pipeline()
|
||||
ready := pipe.Exists(ctx, constants.RedisPollingPriorityReadyKey)
|
||||
member := pipe.SIsMember(ctx, constants.RedisPollingPriorityActiveKey, constants.PollingPriorityIndexMember(cardID, taskType))
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return false
|
||||
}
|
||||
return ready.Val() > 0 && !member.Val()
|
||||
}
|
||||
|
||||
// PriorityIndexRebuilder 定期按库内活动优先项重建索引(并入最近入队的项),使已终态的项退出索引。
|
||||
type PriorityIndexRebuilder struct {
|
||||
store *postgres.PollingPriorityItemStore
|
||||
redis *redis.Client
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewPriorityIndexRebuilder 创建活动优先项索引重建器。
|
||||
func NewPriorityIndexRebuilder(store *postgres.PollingPriorityItemStore, redisClient *redis.Client, logger *zap.Logger) *PriorityIndexRebuilder {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &PriorityIndexRebuilder{store: store, redis: redisClient, logger: logger}
|
||||
}
|
||||
|
||||
// Rebuild 重建一次索引;失败时清除就绪标记,轮询步骤回退为查库。
|
||||
func (r *PriorityIndexRebuilder) Rebuild(ctx context.Context) {
|
||||
if r == nil || r.store == nil || r.redis == nil {
|
||||
return
|
||||
}
|
||||
keys, err := r.store.ListActiveKeys(ctx)
|
||||
if err != nil {
|
||||
r.logger.Warn("重建活动优先项索引失败,轮询步骤回退查库", zap.Error(err))
|
||||
r.redis.Del(ctx, constants.RedisPollingPriorityReadyKey)
|
||||
return
|
||||
}
|
||||
members := make([]any, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
members = append(members, constants.PollingPriorityIndexMember(key.CardID, key.TaskType))
|
||||
}
|
||||
tmp := constants.RedisPollingPriorityRebuildKey
|
||||
pipe := r.redis.TxPipeline()
|
||||
pipe.Del(ctx, tmp)
|
||||
if len(members) > 0 {
|
||||
pipe.SAdd(ctx, tmp, members...)
|
||||
}
|
||||
pipe.SUnionStore(ctx, tmp, tmp, constants.RedisPollingPriorityRecentKey)
|
||||
pipe.Del(ctx, constants.RedisPollingPriorityActiveKey)
|
||||
pipe.SUnionStore(ctx, constants.RedisPollingPriorityActiveKey, tmp)
|
||||
pipe.Del(ctx, tmp)
|
||||
pipe.Set(ctx, constants.RedisPollingPriorityReadyKey, "1", constants.PollingPriorityIndexReadyTTL)
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
r.logger.Warn("写入活动优先项索引失败,轮询步骤回退查库", zap.Error(err))
|
||||
r.redis.Del(ctx, constants.RedisPollingPriorityReadyKey)
|
||||
}
|
||||
}
|
||||
@@ -112,6 +112,15 @@ func (h *PollingRealnameHandler) Step(ctx context.Context, cardID uint) StepOutc
|
||||
return h.fail(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
if h.base.publishIngest(ctx, priority, IngestMessage{
|
||||
Kind: ingestKindRealname, CardID: cardID, CarrierID: card.CarrierID, ICCID: card.ICCID,
|
||||
ObservationID: attempt.IntegrationID, ObservedAt: now, Source: constants.CardObservationSourcePolling,
|
||||
Scene: constants.CardObservationSceneRealnamePolling, CorrelationID: attempt.IntegrationID,
|
||||
TaskType: constants.TaskTypePollingRealname, Realname: &IngestRealnamePayload{Verified: result.RealStatus},
|
||||
}) {
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
||||
return stepOK()
|
||||
}
|
||||
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
||||
CardID: cardID, Verified: result.RealStatus,
|
||||
Metadata: carddomain.ObservationMetadata{
|
||||
|
||||
Reference in New Issue
Block a user