feat: 卡流量读数先过采集队列再入账,扣减按采集时刻确定套餐
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m27s

- 每卡 Redis 采集队列(默认 N=3、跨天清空、回退容差 1 MB),攒满且不下降才写最后一个点
- 测试环境全量切换日快照计量,下线灰度名单、切换水位与卡表基线扣费;全部卡置待建起点
- 读数区分没给值与 0;有套餐的卡读数 0 也写快照与日用量 0 行;重置日比较基准为 0 照常防旧值
- 新增采集记录(按月分区保留 90 天)、人工冲正流水与后台接口、采集告警
- 扣减按采集时刻有效套餐查找,只对生效中套餐做状态迁移,查不到套餐按成功跳过
- 换货新卡(新设备下的卡)打待建起点;采集间隔上限只校验启用中的轮询配置

Change: add-card-traffic-queue-and-observed-deduction

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-29 12:18:47 +09:00
parent c33cc6f6c3
commit 9c3c258f16
55 changed files with 3142 additions and 881 deletions

View File

@@ -74,9 +74,11 @@ type Service struct {
auditWriter StateAuditWriter
// channelThreshold 为可选的通道阈值判定能力;未注入时流量观测不做阈值判定。
channelThreshold ChannelThresholdEvaluator
// trafficCalc 判定卡是否走日快照计量算法;未注入时全部卡走卡表基线算法且不写快照。
trafficCalc *TrafficCalcPolicy
// logger 记录快照写入失败等不影响观测结果的告警;未注入时不输出。
// trafficQueue 是卡流量采集队列;未注入时流量观测报错。
trafficQueue TrafficQueuePort
// trafficQueueSettings 返回采集队列配置;未注入时使用代码默认值。
trafficQueueSettings TrafficQueueSettingsSource
// logger 记录采集记录写入失败等不影响观测结果的告警;未注入时不输出。
logger *zap.Logger
// commandGrace 返回停复机成功后的状态保护窗口;未注入时不保护。
commandGrace func(ctx context.Context) time.Duration
@@ -94,9 +96,10 @@ func (s *Service) currentCommandGrace(ctx context.Context) time.Duration {
return s.commandGrace(ctx)
}
// SetTrafficCalcPolicy 注入卡流量计量切换策略与告警日志。
func (s *Service) SetTrafficCalcPolicy(policy *TrafficCalcPolicy, logger *zap.Logger) {
s.trafficCalc = policy
// SetTrafficQueue 注入卡流量采集队列、队列配置与告警日志。
func (s *Service) SetTrafficQueue(queue TrafficQueuePort, settings TrafficQueueSettingsSource, logger *zap.Logger) {
s.trafficQueue = queue
s.trafficQueueSettings = settings
s.logger = logger
}
@@ -305,11 +308,3 @@ func realnameChangedEventID(cardID uint, observationID string) string {
prefix := "card-realname:"
return outboxid.Stable(prefix, strconv.FormatUint(uint64(cardID), 10)+":"+observationID+":changed")
}
// TrafficSwitched 判定卡是否已切换到日快照计量(激活前流量结算只对已切换的卡执行)。
func (s *Service) TrafficSwitched(cardID uint) bool {
if s == nil {
return false
}
return s.trafficCalc.Switched(cardID)
}

View File

@@ -5,6 +5,7 @@ import (
"strconv"
"time"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -12,6 +13,7 @@ import (
domain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
carrierthresholddomain "github.com/break/junhong_cmp_fiber/internal/domain/carrierthreshold"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/outboxid"
@@ -19,24 +21,56 @@ import (
// TrafficIncrementedEvent 是卡流量正增量的可靠领域事实。
type TrafficIncrementedEvent struct {
EventID string `json:"event_id"`
CardID uint `json:"card_id"`
IncrementMB float64 `json:"increment_mb"`
ObservedAt time.Time `json:"observed_at"`
Source string `json:"source"`
Scene string `json:"scene"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
EventID string `json:"event_id"`
CardID uint `json:"card_id"`
IncrementMB float64 `json:"increment_mb"`
ObservedAt time.Time `json:"observed_at"`
// ObservationID 是采集 ID,扣费审计据此追溯到采集记录。
ObservationID string `json:"observation_id,omitempty"`
Source string `json:"source"`
Scene string `json:"scene"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
}
// ApplyTrafficObservation 串行应用流量读数,并在正增量时同事务写 Outbox。
// TrafficQueuePort 是卡流量采集队列:原子入队并返回入队后的队列与新点是否在队尾。
type TrafficQueuePort interface {
Push(ctx context.Context, cardID uint, point cardflow.QueuePoint, settings TrafficQueueSettings) ([]cardflow.QueuePoint, bool, error)
}
// TrafficQueueSettingsSource 返回当前采集队列配置。
type TrafficQueueSettingsSource interface {
Current(ctx context.Context) TrafficQueueSettings
}
// ApplyTrafficObservation 应用一次流量采集:读数先进采集队列,满足写入条件才写日快照、按月对账、
// 在采集时刻有套餐时同事务写流量事件;有套餐的卡每次采集写一条采集记录。
func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domain.TrafficObservation) (domain.TrafficDecision, error) {
if s == nil || s.db == nil || s.eventWriter == nil {
if s == nil || s.db == nil || s.eventWriter == nil || s.trafficQueue == nil {
return domain.TrafficDecision{}, errors.New(errors.CodeInternalError, "卡流量观测能力未完整配置")
}
var decision domain.TrafficDecision
var auditedCard *model.IotCard
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := domain.ValidateTrafficObservation(observation); err != nil {
return domain.TrafficDecision{}, err
}
record := newCollectionRecord(observation)
decision := domain.TrafficDecision{Result: record.Result}
if record.Result != constants.TrafficCollectionResultOK {
s.writeCollectionRecord(ctx, record, observation.Metadata.ObservedAt)
return decision, nil
}
reading := observation.GatewayReadingMB
decision.ReadingMB = &reading
settings := s.queueSettings()
points, tail, err := s.trafficQueue.Push(ctx, observation.CardID,
cardflow.QueuePoint{ObservedAt: observation.Metadata.ObservedAt, ReadingMB: reading}, settings)
if err != nil {
return decision, err
}
record.setQueue(points)
judge := cardflow.JudgeQueue(points, tail, settings.Size, settings.DropToleranceMB)
record.Decision = judge
var outcome *flowOutcome
err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var card model.IotCard
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
if err == gorm.ErrRecordNotFound {
@@ -44,164 +78,177 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
}
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡流量事实失败")
}
auditedCard = &card
nextDecision, decisionErr := domain.ApplyTraffic(domain.CardTrafficSnapshot{
CardID: card.ID, DataUsageMB: card.DataUsageMB, CurrentMonthUsageMB: card.CurrentMonthUsageMB,
CurrentMonthStartDate: card.CurrentMonthStartDate, LastMonthTotalMB: card.LastMonthTotalMB,
LastGatewayReadingMB: card.LastGatewayReadingMB,
}, observation)
if decisionErr != nil {
return decisionErr
decision.CurrentMonthUsageMB, decision.LastGatewayReadingMB = card.CurrentMonthUsageMB, card.LastGatewayReadingMB
packages, err := postgres.FindEffectivePackagesAt(ctx, tx, card.ID, observation.Metadata.ObservedAt, false)
if err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "查询卡采集时刻有效套餐失败")
}
decision = nextDecision
// 已切换日快照算法的卡:增量、通道阈值与本月用量改由快照决定;读数为 0 或无效时不写快照、不产生增量。
switched := s.trafficCalc.Switched(card.ID)
var flow *switchedFlowOutcome
if switched && cardflow.ValidReading(observation.GatewayReadingMB) {
outcome, flowErr := s.applySwitchedFlow(ctx, tx, &card, observation, decision)
if flowErr != nil {
return flowErr
record.HasPackage = len(packages.Packages) > 0
if judge != cardflow.QueueWrite {
if observation.Metadata.Scene == constants.CardObservationScenePreActivationSettlement {
// 激活前结算没写入:用已有本月最大快照结算,不新写快照。
if err := s.settleExisting(ctx, tx, &card, observation.Metadata.ObservedAt); err != nil {
return err
}
record.Decision = constants.TrafficDecisionSettledExisting
}
flow = outcome
return touchTrafficCheck(tx, card.ID, observation.Metadata.ObservedAt)
}
if switched {
decision.IncrementMB = 0
if flow != nil {
decision.IncrementMB = flow.EmitMB
decision.CurrentMonthUsageMB = flow.MonthMaxMB
}
}
updates := map[string]any{
"last_data_check_at": observation.Metadata.ObservedAt, "last_sync_time": observation.Metadata.ObservedAt,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"current_month_usage_mb": decision.CurrentMonthUsageMB, "data_usage_mb": decision.DataUsageMB,
}
if decision.ReadingAccepted {
updates["last_gateway_reading_mb"] = decision.LastGatewayReadingMB
}
result := tx.Model(&model.IotCard{}).
Where("id = ? AND last_gateway_reading_mb = ?", card.ID, card.LastGatewayReadingMB).
Updates(updates)
if result.Error != nil {
return errors.Wrap(errors.CodeDatabaseError, result.Error, "更新卡流量事实失败")
}
if result.RowsAffected != 1 {
return errors.New(errors.CodeConflict, "卡流量基线已被其他请求更新")
}
if decision.IncrementMB > 0 {
// 未切换卡沿用卡表基线算法的增量;已切换卡的增量已替换为按月对账结果,每次观测最多一条 V1 事件。
eventID := outboxid.Stable("card-traffic:", strconv.FormatUint(uint64(card.ID), 10)+":"+observation.Metadata.ObservationID+":incremented")
if err := s.eventWriter.AppendTraffic(ctx, tx, TrafficIncrementedEvent{
EventID: eventID, CardID: card.ID, IncrementMB: decision.IncrementMB,
ObservedAt: observation.Metadata.ObservedAt, Source: observation.Metadata.Source,
Scene: observation.Metadata.Scene, RequestID: observation.Metadata.RequestID,
CorrelationID: observation.Metadata.CorrelationID,
}); err != nil {
return err
}
}
evaluateThreshold := decision.ReadingAccepted
if switched {
evaluateThreshold = switchedThresholdEvaluable(flow, decision.ReadingAccepted)
}
if observation.Metadata.Scene == constants.CardObservationScenePreActivationSettlement {
// 激活前结算只记已结算用量,不触发停复机评估;下一次正常轮询观测照常判定。
evaluateThreshold = false
}
if evaluateThreshold && s.channelThreshold != nil {
// 未切换卡只使用本次已接受的网关累计读数,异常下降保护命中的观测不参与判定;
// 已切换卡读数有效且未被重置日防旧值处理即判定,不再受卡表基线影响。
// 判定失败必须回滚整个流量事务,与既有流量事实保持同事务语义。
if err := s.channelThreshold.EvaluateInTx(ctx, tx, carrierthresholddomain.Evaluation{
CardID: card.ID, CarrierID: card.CarrierID,
ReadingMB: thresholdReading(switched, flow, observation, decision), ObservedAt: observation.Metadata.ObservedAt,
// 观测来源是判定端获取触发来源的唯一位置:命中留痕据此区分自动判定、人工同步与回调。
Source: observation.Metadata.Source,
}); err != nil {
return err
}
}
stateChanged := decision.IncrementMB != 0 || decision.CrossMonth || decision.LastGatewayReadingMB != card.LastGatewayReadingMB
if actionCode, audited := manualRefreshAuditAction(ctx); observation.Metadata.Source == constants.CardObservationSourceManualSync && stateChanged && audited {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: actionCode,
Summary: "人工刷新 IoT 卡流量",
Card: &card,
BeforeData: map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
"last_gateway_reading_mb": card.LastGatewayReadingMB,
},
AfterData: map[string]any{
"data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB,
},
}); err != nil {
return err
}
} else if workerObservationAudited(ctx) && stateChanged {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
BeforeData: map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
"last_gateway_reading_mb": card.LastGatewayReadingMB,
},
AfterData: map[string]any{
"data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB,
},
}); err != nil {
return err
}
next, err := s.applyWrittenPoint(ctx, tx, &card, observation, packages)
if err != nil {
return err
}
outcome = next
return nil
})
if err != nil {
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实失败", Card: auditedCard,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
s.writeTrafficFailureAudit(ctx, observation, err)
return domain.TrafficDecision{}, err
}
if outcome != nil {
record.applyOutcome(outcome)
decision.ReadingAccepted = outcome.Written
decision.IncrementMB = outcome.EmitMB
decision.CrossMonth = outcome.CrossMonth
decision.CurrentMonthUsageMB = outcome.MonthMaxMB
}
decision.Decision = record.Decision
s.writeCollectionRecord(ctx, record, observation.Metadata.ObservedAt)
if s.cache != nil {
s.cache.Invalidate(ctx, observation.CardID)
}
if auditedCard != nil && !s.trafficCalc.Switched(auditedCard.ID) {
s.writeUnswitchedSnapshot(ctx, auditedCard, observation)
}
return decision, nil
}
// switchedThresholdEvaluable 判定已切换卡本次是否做通道阈值判定:读数有效且本次未被防旧值按 0 处理;
// 第一次切换观测仍按卡表基线算法是否接受读数判定。
func switchedThresholdEvaluable(flow *switchedFlowOutcome, oldAccepted bool) bool {
if flow == nil {
return false
// applyWrittenPoint 在观测事务内处理一个通过队列判断的写入点。
func (s *Service) applyWrittenPoint(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation, packages postgres.EffectivePackages) (*flowOutcome, error) {
hasPackage := len(packages.Packages) > 0
reading := observation.GatewayReadingMB
observedAt := observation.Metadata.ObservedAt
outcome := &flowOutcome{MonthMaxMB: card.CurrentMonthUsageMB, Decision: constants.TrafficDecisionWritten}
if reading == 0 && !hasPackage {
// 没套餐的卡读数为 0 不写快照。
return outcome, touchTrafficCheck(tx, card.ID, observedAt)
}
if flow.FirstSwitch {
return oldAccepted
flow, err := s.writeFlow(ctx, tx, card, reading, observedAt, hasPackage)
if err != nil {
return nil, err
}
return !flow.StaleGuarded
outcome.Written, outcome.MonthMaxMB, outcome.StaleGuarded = true, flow.monthMax, flow.conversion.StaleGuarded
outcome.MonthUsedMB, outcome.DeltaMB = flow.conversion.MonthUsedMB, flow.delta
if card.TrafficBaselinePending {
outcome.Decision = constants.TrafficDecisionBaseline
} else if hasPackage && flow.delta > 0 {
outcome.EmitMB = flow.delta
}
if hasPackage {
if err := ensureDailyUsageRows(ctx, tx, packages.Packages, flow.flowDate); err != nil {
return nil, err
}
}
monthStart := cardflow.MonthStart(flow.flowDate)
outcome.CrossMonth = card.CurrentMonthStartDate == nil || card.CurrentMonthStartDate.Before(monthStart)
updates := map[string]any{
"last_data_check_at": observedAt, "last_sync_time": observedAt,
"current_month_usage_mb": flow.monthMax, "current_month_start_date": monthStart,
"data_usage_mb": card.DataUsageMB + int64(flow.delta), "traffic_baseline_pending": false,
}
if outcome.CrossMonth {
updates["last_month_total_mb"] = card.CurrentMonthUsageMB
}
if err := tx.Model(&model.IotCard{}).Where("id = ?", card.ID).Updates(updates).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "更新卡流量事实失败")
}
if outcome.EmitMB > 0 {
outcome.EventID = outboxid.Stable("card-traffic:", strconv.FormatUint(uint64(card.ID), 10)+":"+observation.Metadata.ObservationID+":incremented")
if err := s.eventWriter.AppendTraffic(ctx, tx, TrafficIncrementedEvent{
EventID: outcome.EventID, CardID: card.ID, IncrementMB: outcome.EmitMB,
ObservedAt: observedAt, ObservationID: observation.Metadata.ObservationID,
Source: observation.Metadata.Source, Scene: observation.Metadata.Scene,
RequestID: observation.Metadata.RequestID, CorrelationID: observation.Metadata.CorrelationID,
}); err != nil {
return nil, err
}
}
// 通道阈值只在写入点判断:读数大于 0 且未被防旧值按 0 处理;激活前结算不触发停复机评估。
if reading > 0 && !outcome.StaleGuarded && s.channelThreshold != nil &&
observation.Metadata.Scene != constants.CardObservationScenePreActivationSettlement {
if err := s.channelThreshold.EvaluateInTx(ctx, tx, carrierthresholddomain.Evaluation{
CardID: card.ID, CarrierID: card.CarrierID, ReadingMB: reading, ObservedAt: observedAt,
// 观测来源是判定端获取触发来源的唯一位置:命中留痕据此区分自动判定、人工同步与回调。
Source: observation.Metadata.Source,
}); err != nil {
return nil, err
}
}
if err := s.writeTrafficStateAudit(ctx, tx, card, observation, outcome); err != nil {
return nil, err
}
return outcome, nil
}
// thresholdReading 返回通道阈值判定使用的网关累计读数:已切换卡(非第一次切换观测)用本次读数。
func thresholdReading(switched bool, flow *switchedFlowOutcome, observation domain.TrafficObservation, decision domain.TrafficDecision) float64 {
if switched && flow != nil && !flow.FirstSwitch {
return observation.GatewayReadingMB
// touchTrafficCheck 只刷新卡的最近流量检查时间(本次没有写入快照)。
func touchTrafficCheck(tx *gorm.DB, cardID uint, observedAt time.Time) error {
if err := tx.Model(&model.IotCard{}).Where("id = ?", cardID).
Updates(map[string]any{"last_data_check_at": observedAt, "last_sync_time": observedAt}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新卡流量检查时间失败")
}
return decision.LastGatewayReadingMB
return nil
}
// writeTrafficStateAudit 在本次写入改变了卡流量事实时写统一审计(人工刷新或 Worker 观测)。
func (s *Service) writeTrafficStateAudit(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation, outcome *flowOutcome) error {
if outcome.DeltaMB <= 0 && !outcome.CrossMonth {
return nil
}
before := map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
}
after := map[string]any{
"data_usage_mb": card.DataUsageMB + int64(outcome.DeltaMB), "current_month_usage_mb": outcome.MonthMaxMB,
"increment_mb": outcome.EmitMB, "delta_mb": outcome.DeltaMB, "decision": outcome.Decision,
"observation_id": observation.Metadata.ObservationID,
}
input := StateAudit{Card: card, BeforeData: before, AfterData: after}
if actionCode, audited := manualRefreshAuditAction(ctx); observation.Metadata.Source == constants.CardObservationSourceManualSync && audited {
input.ActionCode, input.Summary = actionCode, "人工刷新 IoT 卡流量"
} else if workerObservationAudited(ctx) {
input.ActionCode, input.Summary = constants.AuditActionIotCardWorkerTrafficSynced, "Worker 同步 IoT 卡流量事实"
input.IntegrationID = observation.Metadata.ObservationID
} else {
return nil
}
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
return s.auditWriter.WriteCardStateAudit(ctx, tx, input)
}
// writeTrafficFailureAudit 记录 Worker 观测事务失败的审计。
func (s *Service) writeTrafficFailureAudit(ctx context.Context, observation domain.TrafficObservation, err error) {
if !workerObservationAudited(ctx) || s.auditWriter == nil {
return
}
var card model.IotCard
if loadErr := s.db.WithContext(ctx).Where("id = ?", observation.CardID).First(&card).Error; loadErr != nil {
return
}
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实失败", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
func (s *Service) queueSettings() TrafficQueueSettings {
if s.trafficQueueSettings == nil {
return DefaultTrafficQueueSettings()
}
return s.trafficQueueSettings.Current(context.Background())
}
// logTrafficWarn 记录不影响观测结果的告警。
func (s *Service) logTrafficWarn(message string, fields ...zap.Field) {
s.warnLogger().Warn(message, fields...)
}

View File

@@ -1,171 +0,0 @@
package cardobservation
import (
"time"
"context"
"strconv"
"strings"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// 灰度名单中扣减载体的前缀。
const (
trafficCalcCarrierDevice = "device"
trafficCalcCarrierIotCard = "iot_card"
)
// TrafficCalcPolicy 判定卡是否走日快照计量算法,以及未切换卡是否写快照。
//
// 名单在进程启动时展开为内存卡集合,观测事务内只查内存、不查库;API 与全部 Worker 必须使用一致的配置。
// 零值策略等同于 off 模式且不写快照。
type TrafficCalcPolicy struct {
mode string
cards map[uint]struct{}
snapshotEnabled bool
}
// Switched 判定卡是否走日快照算法。
func (p *TrafficCalcPolicy) Switched(cardID uint) bool {
if p == nil {
return false
}
switch p.mode {
case config.TrafficCalcModeFull:
return true
case config.TrafficCalcModeGray:
_, ok := p.cards[cardID]
return ok
default:
return false
}
}
// SnapshotEnabled 判定未切换卡是否写日快照。
func (p *TrafficCalcPolicy) SnapshotEnabled() bool {
return p != nil && p.snapshotEnabled
}
// BuildTrafficCalcPolicy 按配置展开灰度名单:device 展开为当前绑定的全部卡;iot_card 实际使用设备套餐时
// 记录错误并跳过该项(不阻止启动)。展开查询失败时整体退回 off 模式,保证判定出错一律走卡表基线算法。
func BuildTrafficCalcPolicy(ctx context.Context, db *gorm.DB, cfg config.TrafficCalcConfig, logger *zap.Logger) *TrafficCalcPolicy {
if logger == nil {
logger = zap.NewNop()
}
policy := &TrafficCalcPolicy{mode: cfg.Mode, cards: map[uint]struct{}{}, snapshotEnabled: cfg.FlowSnapshotEnabled}
if cfg.Mode != config.TrafficCalcModeGray {
logger.Info("卡流量计量模式", zap.String("mode", cfg.Mode), zap.Bool("flow_snapshot_enabled", cfg.FlowSnapshotEnabled))
return policy
}
for _, item := range strings.Split(cfg.Cards, ",") {
item = strings.TrimSpace(item)
if item == "" {
continue
}
if err := policy.expandItem(ctx, db, item, logger); err != nil {
logger.Error("展开卡流量计量灰度名单失败,本进程全部卡走卡表基线算法", zap.String("item", item), zap.Error(err))
return &TrafficCalcPolicy{mode: config.TrafficCalcModeOff, snapshotEnabled: cfg.FlowSnapshotEnabled}
}
}
logger.Info("卡流量计量灰度名单已展开", zap.String("cards", cfg.Cards), zap.Int("card_count", len(policy.cards)))
return policy
}
// expandItem 展开一项扣减载体;格式非法或单卡实际用设备套餐时只记录错误并跳过,查询失败时返回错误。
func (p *TrafficCalcPolicy) expandItem(ctx context.Context, db *gorm.DB, item string, logger *zap.Logger) error {
kind, rawID, found := strings.Cut(item, ":")
id, parseErr := strconv.ParseUint(strings.TrimSpace(rawID), 10, 64)
if !found || parseErr != nil || id == 0 {
logger.Error("卡流量计量灰度名单项格式非法,已跳过", zap.String("item", item))
return nil
}
switch strings.TrimSpace(kind) {
case trafficCalcCarrierDevice:
var cardIDs []uint
if err := db.WithContext(ctx).Model(&model.DeviceSimBinding{}).
Where("device_id = ? AND bind_status = ?", id, constants.BindStatusBound).
Pluck("iot_card_id", &cardIDs).Error; err != nil {
return err
}
for _, cardID := range cardIDs {
p.cards[cardID] = struct{}{}
}
return nil
case trafficCalcCarrierIotCard:
usesDevice, err := cardUsesDevicePackage(ctx, db, uint(id))
if err != nil {
return err
}
if usesDevice {
logger.Error("灰度名单中的单卡实际从设备套餐扣减,已跳过;请改为按设备整台切换", zap.String("item", item))
return nil
}
p.cards[uint(id)] = struct{}{}
return nil
default:
logger.Error("卡流量计量灰度名单项载体类型非法,已跳过", zap.String("item", item))
return nil
}
}
// cardUsesDevicePackage 与套餐扣减的载体解析同口径:卡自身没有生效中套餐、且当前绑定设备有生效中套餐时,
// 卡的流量扣到设备套餐上。
func cardUsesDevicePackage(ctx context.Context, db *gorm.DB, cardID uint) (bool, error) {
ownActive, err := countActivePackages(ctx, db, "iot_card_id = ?", cardID, time.Time{})
if err != nil || ownActive > 0 {
return false, err
}
deviceID, bound, err := activeBoundDeviceID(ctx, db, cardID)
if err != nil || !bound {
return false, err
}
deviceActive, err := countActivePackages(ctx, db, "device_id = ?", deviceID, time.Time{})
return deviceActive > 0, err
}
// hasActivePackageForCard 判定卡本次增量是否有可扣的生效中套餐:卡自身或当前绑定设备存在生效中套餐。
// observedAt 非零时只认激活时间不晚于观测时间的生效套餐(流量按观测时间归属):
// 观测早于套餐激活的用量属于激活前,只记已结算、不扣新套餐;激活时间为空的历史套餐视为早已生效。
func hasActivePackageForCard(ctx context.Context, db *gorm.DB, cardID uint, observedAt time.Time) (bool, error) {
ownActive, err := countActivePackages(ctx, db, "iot_card_id = ?", cardID, observedAt)
if err != nil || ownActive > 0 {
return ownActive > 0, err
}
deviceID, bound, err := activeBoundDeviceID(ctx, db, cardID)
if err != nil || !bound {
return false, err
}
deviceActive, err := countActivePackages(ctx, db, "device_id = ?", deviceID, observedAt)
return deviceActive > 0, err
}
func countActivePackages(ctx context.Context, db *gorm.DB, carrierCondition string, carrierID uint, observedAt time.Time) (int64, error) {
var count int64
query := db.WithContext(ctx).Model(&model.PackageUsage{}).
Where(carrierCondition, carrierID).
Where("status = ?", constants.PackageUsageStatusActive)
if !observedAt.IsZero() {
query = query.Where("activated_at IS NULL OR activated_at <= ?", observedAt)
}
err := query.Count(&count).Error
return count, err
}
func activeBoundDeviceID(ctx context.Context, db *gorm.DB, cardID uint) (uint, bool, error) {
var deviceIDs []uint
if err := db.WithContext(ctx).Model(&model.DeviceSimBinding{}).
Where("iot_card_id = ? AND bind_status = ?", cardID, constants.BindStatusBound).
Limit(1).Pluck("device_id", &deviceIDs).Error; err != nil {
return 0, false, err
}
if len(deviceIDs) == 0 {
return 0, false, nil
}
return deviceIDs[0], true, nil
}

View File

@@ -0,0 +1,74 @@
package cardobservation
import (
"context"
"encoding/json"
"time"
"go.uber.org/zap"
cardflow "github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
domain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// collectionRecordTimeout 是事务提交后写采集记录的超时;采集记录只用于排查,写失败只告警。
const collectionRecordTimeout = 3 * time.Second
// collectionRecord 是一次采集的留痕内容;HasPackage 在观测事务内判断,读数无效的观测写记录前再查。
type collectionRecord struct {
model.CardTrafficCollection
HasPackage bool
}
func newCollectionRecord(observation domain.TrafficObservation) *collectionRecord {
record := &collectionRecord{CardTrafficCollection: model.CardTrafficCollection{
IotCardID: observation.CardID, ObservedAt: observation.Metadata.ObservedAt,
ObservationID: observation.Metadata.ObservationID, Source: observation.Metadata.Source,
Scene: observation.Metadata.Scene, Result: domain.ResolveReadingResult(observation),
}}
if record.Result == constants.TrafficCollectionResultOK {
reading := observation.GatewayReadingMB
record.VendorUsedMB = &reading
}
return record
}
func (r *collectionRecord) setQueue(points []cardflow.QueuePoint) {
if encoded, err := json.Marshal(points); err == nil {
r.QueuePoints = encoded
}
}
func (r *collectionRecord) applyOutcome(outcome *flowOutcome) {
r.Decision = outcome.Decision
if !outcome.Written {
return
}
monthUsed, delta, stale := outcome.MonthUsedMB, outcome.DeltaMB, outcome.StaleGuarded
r.MonthUsedMB, r.DeltaMB, r.StaleGuarded, r.EventID = &monthUsed, &delta, &stale, outcome.EventID
}
// writeCollectionRecord 在观测事务之外为采集时刻有套餐的卡写采集记录;没套餐的卡不写。
func (s *Service) writeCollectionRecord(ctx context.Context, record *collectionRecord, observedAt time.Time) {
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), collectionRecordTimeout)
defer cancel()
hasPackage := record.HasPackage
if record.Result != constants.TrafficCollectionResultOK {
packages, err := postgres.FindEffectivePackagesAt(writeCtx, s.db, record.IotCardID, observedAt, false)
if err != nil {
s.logTrafficWarn("查询卡采集时刻有效套餐失败,跳过采集记录", zap.Uint("card_id", record.IotCardID), zap.Error(err))
return
}
hasPackage = len(packages.Packages) > 0
}
if !hasPackage {
return
}
if err := postgres.NewCardTrafficCollectionStore(s.db).Create(writeCtx, &record.CardTrafficCollection); err != nil {
s.logTrafficWarn("写卡流量采集记录失败", zap.Uint("card_id", record.IotCardID),
zap.String("observation_id", record.ObservationID), zap.Error(err))
}
}

View File

@@ -8,50 +8,93 @@ import (
"gorm.io/gorm"
cardflow "github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
domain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// unswitchedSnapshotTimeout 是未切换卡在观测事务提交后写快照的超时,超时只告警。
const unswitchedSnapshotTimeout = 3 * time.Second
// switchedFlowOutcome 是已切换卡一次观测的日快照计量结果。
type switchedFlowOutcome struct {
// EmitMB 是本次应写入 V1 流量事件的增量,0 表示不写事件。
// flowOutcome 是一个写入点的日快照计量结果。
type flowOutcome struct {
// Written 为 true 表示本次写入了日快照(没套餐且读数为 0 时为 false)。
Written bool
// Decision 是写入判断:已写入或建起点。
Decision string
// EmitMB 是本次写入流量事件的增量,0 表示不写事件。
EmitMB float64
// MonthMaxMB 是写入本次快照后的本月最大快照,用于覆盖卡表本月用量。
MonthMaxMB float64
// DeltaMB 是本次按月对账计入已结算的增量(含没套餐、建起点时只记已结算的部分)。
DeltaMB float64
// MonthUsedMB 是本次读数换算后的本月累计;MonthMaxMB 是写入后的本月最大快照。
MonthUsedMB float64
MonthMaxMB float64
// StaleGuarded 为 true 表示本次读数在重置日触发防旧值、按 0 计。
StaleGuarded bool
// FirstSwitch 为 true 表示这是卡的第一次切换观测(本次写入了切换水位)。
FirstSwitch bool
// CrossMonth 为 true 表示卡表本月用量跨入新的自然月。
CrossMonth bool
// EventID 是本次写入的流量事件 ID。
EventID string
}
// snapshotResult 是一次快照换算与写入的结果。
type snapshotResult struct {
// flowWrite 是一次快照写入与按月对账的结果。
type flowWrite struct {
flowDate time.Time
conversion cardflow.Conversion
monthMax float64
delta float64
}
// writeSnapshot 按卡所属通道的重置日换算并原子写入当天快照;读数必须已通过 cardflow.ValidReading。
func (s *Service) writeSnapshot(ctx context.Context, db *gorm.DB, card *model.IotCard, readingMB float64, observedAt time.Time) (snapshotResult, error) {
resetDay := s.strictResetDay(ctx, db, card)
// writeFlow 按卡所属通道的重置日换算并原子写入当天快照,再按月对账把增量计入当天已结算。
func (s *Service) writeFlow(ctx context.Context, tx *gorm.DB, card *model.IotCard, readingMB float64, observedAt time.Time, hasPackage bool) (flowWrite, error) {
resetDay := s.strictResetDay(ctx, tx, card)
flowDate := cardflow.ShanghaiDate(observedAt)
days := postgres.NewCardFlowDayStore(db)
days := postgres.NewCardFlowDayStore(tx)
monthCtx, err := days.MonthContext(ctx, card.ID, flowDate, resetDay)
if err != nil {
return snapshotResult{}, errors.Wrap(errors.CodeDatabaseError, err, "查询卡日流量快照失败")
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "查询卡日流量快照失败")
}
conversion := cardflow.Convert(readingMB, flowDate.Day(), resetDay, monthCtx)
if err := days.Upsert(ctx, postgres.CardFlowSnapshotWrite{
CardID: card.ID, FlowDate: flowDate, MonthUsedMB: conversion.MonthUsedMB,
VendorUsedMB: readingMB, StaleGuarded: conversion.StaleGuarded,
}); err != nil {
return snapshotResult{}, errors.Wrap(errors.CodeDatabaseError, err, "写入卡日流量快照失败")
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "写入卡日流量快照失败")
}
return snapshotResult{flowDate: flowDate, conversion: conversion}, nil
monthMax, settled, err := days.MonthTotals(ctx, card.ID, flowDate)
if err != nil {
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "查询卡本月流量结算失败")
}
delta := cardflow.MonthlyDelta(monthMax, settled)
if err := days.AddSettled(ctx, card.ID, flowDate, delta); err != nil {
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "结算卡流量增量失败")
}
return flowWrite{flowDate: flowDate, conversion: conversion, monthMax: monthMax, delta: delta}, nil
}
// settleExisting 用卡上已有的本月最大快照结算(激活前结算本次没写入时):差额全部记为已结算,不新写快照。
func (s *Service) settleExisting(ctx context.Context, tx *gorm.DB, card *model.IotCard, observedAt time.Time) error {
flowDate := cardflow.ShanghaiDate(observedAt)
days := postgres.NewCardFlowDayStore(tx)
monthMax, settled, err := days.MonthTotals(ctx, card.ID, flowDate)
if err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "查询卡本月流量结算失败")
}
if err := days.AddSettledToLatest(ctx, card.ID, flowDate, cardflow.MonthlyDelta(monthMax, settled)); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "激活前结算已有快照失败")
}
return nil
}
// ensureDailyUsageRows 为采集时刻每个有效套餐确保当天一行套餐日用量(没扣也写 0)。
func ensureDailyUsageRows(ctx context.Context, tx *gorm.DB, packages []*model.PackageUsage, flowDate time.Time) error {
ids := make([]uint, 0, len(packages))
cumulative := make(map[uint]int64, len(packages))
for _, pkg := range packages {
ids = append(ids, pkg.ID)
cumulative[pkg.ID] = pkg.DataUsageMB
}
if err := postgres.EnsureDailyUsageRows(ctx, tx, ids, flowDate, cumulative); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "写入套餐当天日用量失败")
}
return nil
}
// strictResetDay 在当前连接内按卡的 carrier_id 读取上游流量重置日,不使用调用方预先计算的值;
@@ -73,78 +116,3 @@ func (s *Service) strictResetDay(ctx context.Context, db *gorm.DB, card *model.I
}
return resetDay
}
// applySwitchedFlow 在观测事务内为已切换卡写快照、按月对账结算增量,并在第一次切换观测时写切换水位。
//
// oldDecision 是卡表基线算法对本次观测的结果:只在第一次切换观测时沿用其增量,此后不再使用。
func (s *Service) applySwitchedFlow(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation, oldDecision domain.TrafficDecision) (*switchedFlowOutcome, error) {
readingMB := observation.GatewayReadingMB
snapshot, err := s.writeSnapshot(ctx, tx, card, readingMB, observation.Metadata.ObservedAt)
if err != nil {
return nil, err
}
days := postgres.NewCardFlowDayStore(tx)
monthMax, settled, err := days.MonthTotals(ctx, card.ID, snapshot.flowDate)
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡本月流量结算失败")
}
hasPackage, pkgErr := hasActivePackageForCard(ctx, tx, card.ID, observation.Metadata.ObservedAt)
if pkgErr != nil {
// 判断出错按有套餐处理,保持不丢量。
s.warnLogger().Warn("判断卡是否有生效中套餐失败,按有套餐处理", zap.Uint("card_id", card.ID), zap.Error(pkgErr))
hasPackage = true
}
outcome := &switchedFlowOutcome{MonthMaxMB: monthMax, StaleGuarded: snapshot.conversion.StaleGuarded}
switches := postgres.NewCardFlowSwitchStore(tx)
watermark, err := switches.Get(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡流量切换水位失败")
}
if watermark == nil {
return outcome, s.firstSwitch(ctx, tx, card, observation, oldDecision, snapshot.flowDate, monthMax, settled, hasPackage, outcome)
}
delta := cardflow.MonthlyDelta(monthMax, settled)
if delta <= 0 {
return outcome, nil
}
if err := days.AddSettled(ctx, card.ID, snapshot.flowDate, delta); err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "结算卡流量增量失败")
}
if hasPackage {
outcome.EmitMB = delta
}
return outcome, nil
}
// firstSwitch 处理第一次切换观测:本次沿用卡表基线算法的增量,此前部分记为已结算,并写入切换水位。
func (s *Service) firstSwitch(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation,
oldDecision domain.TrafficDecision, flowDate time.Time, monthMax, settled float64, hasPackage bool, outcome *switchedFlowOutcome) error {
outcome.FirstSwitch = true
if oldDecision.IncrementMB > 0 && hasPackage {
outcome.EmitMB = oldDecision.IncrementMB
}
if err := postgres.NewCardFlowDayStore(tx).AddSettled(ctx, card.ID, flowDate, cardflow.MonthlyDelta(monthMax, settled)); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "结算卡流量切换前用量失败")
}
if err := postgres.NewCardFlowSwitchStore(tx).Create(ctx, &model.CardFlowSwitch{
IotCardID: card.ID, SwitchedAt: observation.Metadata.ObservedAt, MonthUsedMB: monthMax,
VendorUsedMB: observation.GatewayReadingMB, OldBaselineMB: card.LastGatewayReadingMB,
OldMonthUsageMB: card.CurrentMonthUsageMB, OldReadingAccepted: oldDecision.ReadingAccepted,
OldIncrementMB: oldDecision.IncrementMB,
}); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "写入卡流量切换水位失败")
}
return nil
}
// writeUnswitchedSnapshot 在观测事务提交后为未切换卡写快照;失败只告警,不影响本次观测结果。
func (s *Service) writeUnswitchedSnapshot(ctx context.Context, card *model.IotCard, observation domain.TrafficObservation) {
if card == nil || !s.trafficCalc.SnapshotEnabled() || !cardflow.ValidReading(observation.GatewayReadingMB) {
return
}
writeCtx, cancel := context.WithTimeout(ctx, unswitchedSnapshotTimeout)
defer cancel()
if _, err := s.writeSnapshot(writeCtx, s.db, card, observation.GatewayReadingMB, observation.Metadata.ObservedAt); err != nil {
s.warnLogger().Warn("未切换卡写日流量快照失败,不影响本次观测", zap.Uint("card_id", card.ID), zap.Error(err))
}
}

View File

@@ -0,0 +1,193 @@
package cardobservation
import (
"context"
stderrors "errors"
"math"
"strconv"
"sync"
"time"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// trafficQueueSettingsTTL 是队列配置在进程内的缓存时长;后台修改数秒内生效。
const trafficQueueSettingsTTL = 5 * time.Second
// TrafficQueueSettings 是卡流量采集队列的运行参数。
type TrafficQueueSettings struct {
Size int
ClearOnNewDay bool
MaxPointAgeHours int
DropToleranceMB float64
}
// DefaultTrafficQueueSettings 返回代码默认值。
func DefaultTrafficQueueSettings() TrafficQueueSettings {
return TrafficQueueSettings{
Size: constants.TrafficQueueDefaultSize, ClearOnNewDay: true,
MaxPointAgeHours: constants.TrafficQueueDefaultMaxPointAgeHours, DropToleranceMB: constants.TrafficQueueDefaultDropToleranceMB,
}
}
// TrafficIntervalChecker 校验队列配置变更后启用中轮询配置的流量采集间隔是否仍在上限内。
type TrafficIntervalChecker func(settings TrafficQueueSettings) error
// RegisterTrafficQueueConfigDefinitions 注册采集队列系统配置。checker 非空时,队列长度、跨天清空、最长点龄
// 修改前用它校验启用中的轮询配置(读取其余三项的当前值组合成修改后的配置)。
func RegisterTrafficQueueConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logger, current func() TrafficQueueSettings, checker TrafficIntervalChecker) {
if registry == nil {
return
}
minSize, maxSize := int64(constants.TrafficQueueMinSize), int64(constants.TrafficQueueMaxSize)
minAge, maxAge := int64(0), int64(24*30)
withCheck := func(apply func(*TrafficQueueSettings, string) error) func(string) error {
return func(value string) error {
next := DefaultTrafficQueueSettings()
if current != nil {
next = current()
}
if err := apply(&next, value); err != nil {
return err
}
if checker == nil {
return nil
}
return checker(next)
}
}
definitions := []systemconfig.Definition{
{Key: constants.SystemConfigTrafficQueueSize, Module: constants.SystemConfigModuleTraffic, ValueType: constants.SystemConfigTypeInt,
DefaultValue: strconv.Itoa(constants.TrafficQueueDefaultSize), Min: &minSize, Max: &maxSize,
Description: "卡流量采集队列长度 N:攒满 N 个不下降的读数才写最后一个点",
Validator: withCheck(func(s *TrafficQueueSettings, v string) error {
n, err := strconv.Atoi(v)
s.Size = n
return err
})},
{Key: constants.SystemConfigTrafficQueueClearOnNewDay, Module: constants.SystemConfigModuleTraffic, ValueType: constants.SystemConfigTypeBool,
DefaultValue: "true", Description: "卡流量采集队列跨天清空:新点与队列中的点不在同一上海日期时先清空",
Validator: withCheck(func(s *TrafficQueueSettings, v string) error {
s.ClearOnNewDay = v == "true"
return nil
})},
{Key: constants.SystemConfigTrafficQueueMaxPointAgeHours, Module: constants.SystemConfigModuleTraffic, ValueType: constants.SystemConfigTypeInt,
DefaultValue: strconv.Itoa(constants.TrafficQueueDefaultMaxPointAgeHours), Min: &minAge, Max: &maxAge,
Description: "卡流量采集队列最长点龄(小时):入队时丢弃与新点相差超过该时长的点,0 表示不限",
Validator: withCheck(func(s *TrafficQueueSettings, v string) error {
n, err := strconv.Atoi(v)
s.MaxPointAgeHours = n
return err
})},
{Key: constants.SystemConfigTrafficQueueDropToleranceMB, Module: constants.SystemConfigModuleTraffic, ValueType: constants.SystemConfigTypeString,
DefaultValue: "1", Description: "卡流量采集队列回退容差(MB):相邻两点回退不超过该值算持平,0 表示不允许回退",
Validator: func(v string) error {
_, err := parseDropTolerance(v)
return err
}},
}
for _, definition := range definitions {
if _, exists := registry.Get(definition.Key); exists {
continue
}
if err := registry.Register(definition); err != nil && logger != nil {
logger.Error("注册卡流量采集队列系统配置失败,将使用代码默认值", zap.String("config_key", definition.Key), zap.Error(err))
}
}
}
func parseDropTolerance(value string) (float64, error) {
tolerance, err := strconv.ParseFloat(value, 64)
if err != nil || math.IsNaN(tolerance) || math.IsInf(tolerance, 0) || tolerance < 0 || tolerance > 1024 {
return 0, stderrors.New("回退容差必须是 0~1024 之间的数字(MB)")
}
return tolerance, nil
}
// trafficConfigReader 读取系统配置的最小接缝。
type trafficConfigReader interface {
Get(ctx context.Context, key string) (string, error)
}
// TrafficQueueSettingsProvider 读取采集队列配置,带短缓存;读取失败时使用代码默认值。
type TrafficQueueSettingsProvider struct {
reader trafficConfigReader
logger *zap.Logger
mu sync.Mutex
cached TrafficQueueSettings
expiresAt time.Time
}
// NewTrafficQueueSettingsProvider 创建采集队列配置读取器。
func NewTrafficQueueSettingsProvider(reader trafficConfigReader, logger *zap.Logger) *TrafficQueueSettingsProvider {
if logger == nil {
logger = zap.NewNop()
}
return &TrafficQueueSettingsProvider{reader: reader, logger: logger}
}
// NewDefaultTrafficQueueSettingsProvider 用独立注册表与读取器创建采集队列配置读取器(API 与 Worker 共用同一份数据)。
func NewDefaultTrafficQueueSettingsProvider(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *TrafficQueueSettingsProvider {
registry := systemconfig.NewRegistry()
RegisterTrafficQueueConfigDefinitions(registry, logger, nil, nil)
var cache systemconfig.Cache
if redisClient != nil {
cache = systemconfig.NewRedisCache(redisClient)
}
reader := systemconfig.NewReader(db, registry, cache, systemconfig.NewLogAlertSink(logger))
return NewTrafficQueueSettingsProvider(reader, logger)
}
// Current 返回当前采集队列配置。
func (p *TrafficQueueSettingsProvider) Current(ctx context.Context) TrafficQueueSettings {
if p == nil || p.reader == nil {
return DefaultTrafficQueueSettings()
}
p.mu.Lock()
defer p.mu.Unlock()
now := time.Now()
if now.Before(p.expiresAt) {
return p.cached
}
settings, err := p.load(ctx)
if err != nil {
p.logger.Warn("读取卡流量采集队列配置失败,使用代码默认值", zap.Error(err))
settings = DefaultTrafficQueueSettings()
}
p.cached, p.expiresAt = settings, now.Add(trafficQueueSettingsTTL)
return settings
}
func (p *TrafficQueueSettingsProvider) load(ctx context.Context) (TrafficQueueSettings, error) {
settings := DefaultTrafficQueueSettings()
raw, err := p.reader.Get(ctx, constants.SystemConfigTrafficQueueSize)
if err != nil {
return settings, err
}
if settings.Size, err = strconv.Atoi(raw); err != nil || settings.Size < constants.TrafficQueueMinSize {
return DefaultTrafficQueueSettings(), stderrors.New("采集队列长度配置非法")
}
if raw, err = p.reader.Get(ctx, constants.SystemConfigTrafficQueueClearOnNewDay); err != nil {
return DefaultTrafficQueueSettings(), err
}
settings.ClearOnNewDay = raw == "true"
if raw, err = p.reader.Get(ctx, constants.SystemConfigTrafficQueueMaxPointAgeHours); err != nil {
return DefaultTrafficQueueSettings(), err
}
if settings.MaxPointAgeHours, err = strconv.Atoi(raw); err != nil || settings.MaxPointAgeHours < 0 {
return DefaultTrafficQueueSettings(), stderrors.New("采集队列最长点龄配置非法")
}
if raw, err = p.reader.Get(ctx, constants.SystemConfigTrafficQueueDropToleranceMB); err != nil {
return DefaultTrafficQueueSettings(), err
}
if settings.DropToleranceMB, err = parseDropTolerance(raw); err != nil {
return DefaultTrafficQueueSettings(), err
}
return settings, nil
}