feat: 卡流量读数先过采集队列再入账,扣减按采集时刻确定套餐
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m27s

- 每卡 Redis 采集队列(默认 N=3、跨天清空、回退容差 1 MB),攒满且不下降才写最后一个点
- 测试环境全量切换日快照计量,下线灰度名单、切换水位与卡表基线扣费;全部卡置待建起点
- 读数区分没给值与 0;有套餐的卡读数 0 也写快照与日用量 0 行;重置日比较基准为 0 照常防旧值
- 新增采集记录(按月分区保留 90 天)、人工冲正流水与后台接口、采集告警
- 扣减按采集时刻有效套餐查找,只对生效中套餐做状态迁移,查不到套餐按成功跳过
- 换货新卡(新设备下的卡)打待建起点;采集间隔上限只校验启用中的轮询配置

Change: add-card-traffic-queue-and-observed-deduction

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-29 12:18:47 +09:00
parent c33cc6f6c3
commit 9c3c258f16
55 changed files with 3142 additions and 881 deletions

View File

@@ -5,6 +5,7 @@ import (
"strconv"
"time"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -12,6 +13,7 @@ import (
domain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
carrierthresholddomain "github.com/break/junhong_cmp_fiber/internal/domain/carrierthreshold"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/outboxid"
@@ -19,24 +21,56 @@ import (
// TrafficIncrementedEvent 是卡流量正增量的可靠领域事实。
type TrafficIncrementedEvent struct {
EventID string `json:"event_id"`
CardID uint `json:"card_id"`
IncrementMB float64 `json:"increment_mb"`
ObservedAt time.Time `json:"observed_at"`
Source string `json:"source"`
Scene string `json:"scene"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
EventID string `json:"event_id"`
CardID uint `json:"card_id"`
IncrementMB float64 `json:"increment_mb"`
ObservedAt time.Time `json:"observed_at"`
// ObservationID 是采集 ID,扣费审计据此追溯到采集记录。
ObservationID string `json:"observation_id,omitempty"`
Source string `json:"source"`
Scene string `json:"scene"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
}
// ApplyTrafficObservation 串行应用流量读数,并在正增量时同事务写 Outbox。
// TrafficQueuePort 是卡流量采集队列:原子入队并返回入队后的队列与新点是否在队尾。
type TrafficQueuePort interface {
Push(ctx context.Context, cardID uint, point cardflow.QueuePoint, settings TrafficQueueSettings) ([]cardflow.QueuePoint, bool, error)
}
// TrafficQueueSettingsSource 返回当前采集队列配置。
type TrafficQueueSettingsSource interface {
Current(ctx context.Context) TrafficQueueSettings
}
// ApplyTrafficObservation 应用一次流量采集:读数先进采集队列,满足写入条件才写日快照、按月对账、
// 在采集时刻有套餐时同事务写流量事件;有套餐的卡每次采集写一条采集记录。
func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domain.TrafficObservation) (domain.TrafficDecision, error) {
if s == nil || s.db == nil || s.eventWriter == nil {
if s == nil || s.db == nil || s.eventWriter == nil || s.trafficQueue == nil {
return domain.TrafficDecision{}, errors.New(errors.CodeInternalError, "卡流量观测能力未完整配置")
}
var decision domain.TrafficDecision
var auditedCard *model.IotCard
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := domain.ValidateTrafficObservation(observation); err != nil {
return domain.TrafficDecision{}, err
}
record := newCollectionRecord(observation)
decision := domain.TrafficDecision{Result: record.Result}
if record.Result != constants.TrafficCollectionResultOK {
s.writeCollectionRecord(ctx, record, observation.Metadata.ObservedAt)
return decision, nil
}
reading := observation.GatewayReadingMB
decision.ReadingMB = &reading
settings := s.queueSettings()
points, tail, err := s.trafficQueue.Push(ctx, observation.CardID,
cardflow.QueuePoint{ObservedAt: observation.Metadata.ObservedAt, ReadingMB: reading}, settings)
if err != nil {
return decision, err
}
record.setQueue(points)
judge := cardflow.JudgeQueue(points, tail, settings.Size, settings.DropToleranceMB)
record.Decision = judge
var outcome *flowOutcome
err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var card model.IotCard
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
if err == gorm.ErrRecordNotFound {
@@ -44,164 +78,177 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
}
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡流量事实失败")
}
auditedCard = &card
nextDecision, decisionErr := domain.ApplyTraffic(domain.CardTrafficSnapshot{
CardID: card.ID, DataUsageMB: card.DataUsageMB, CurrentMonthUsageMB: card.CurrentMonthUsageMB,
CurrentMonthStartDate: card.CurrentMonthStartDate, LastMonthTotalMB: card.LastMonthTotalMB,
LastGatewayReadingMB: card.LastGatewayReadingMB,
}, observation)
if decisionErr != nil {
return decisionErr
decision.CurrentMonthUsageMB, decision.LastGatewayReadingMB = card.CurrentMonthUsageMB, card.LastGatewayReadingMB
packages, err := postgres.FindEffectivePackagesAt(ctx, tx, card.ID, observation.Metadata.ObservedAt, false)
if err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "查询卡采集时刻有效套餐失败")
}
decision = nextDecision
// 已切换日快照算法的卡:增量、通道阈值与本月用量改由快照决定;读数为 0 或无效时不写快照、不产生增量。
switched := s.trafficCalc.Switched(card.ID)
var flow *switchedFlowOutcome
if switched && cardflow.ValidReading(observation.GatewayReadingMB) {
outcome, flowErr := s.applySwitchedFlow(ctx, tx, &card, observation, decision)
if flowErr != nil {
return flowErr
record.HasPackage = len(packages.Packages) > 0
if judge != cardflow.QueueWrite {
if observation.Metadata.Scene == constants.CardObservationScenePreActivationSettlement {
// 激活前结算没写入:用已有本月最大快照结算,不新写快照。
if err := s.settleExisting(ctx, tx, &card, observation.Metadata.ObservedAt); err != nil {
return err
}
record.Decision = constants.TrafficDecisionSettledExisting
}
flow = outcome
return touchTrafficCheck(tx, card.ID, observation.Metadata.ObservedAt)
}
if switched {
decision.IncrementMB = 0
if flow != nil {
decision.IncrementMB = flow.EmitMB
decision.CurrentMonthUsageMB = flow.MonthMaxMB
}
}
updates := map[string]any{
"last_data_check_at": observation.Metadata.ObservedAt, "last_sync_time": observation.Metadata.ObservedAt,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"current_month_usage_mb": decision.CurrentMonthUsageMB, "data_usage_mb": decision.DataUsageMB,
}
if decision.ReadingAccepted {
updates["last_gateway_reading_mb"] = decision.LastGatewayReadingMB
}
result := tx.Model(&model.IotCard{}).
Where("id = ? AND last_gateway_reading_mb = ?", card.ID, card.LastGatewayReadingMB).
Updates(updates)
if result.Error != nil {
return errors.Wrap(errors.CodeDatabaseError, result.Error, "更新卡流量事实失败")
}
if result.RowsAffected != 1 {
return errors.New(errors.CodeConflict, "卡流量基线已被其他请求更新")
}
if decision.IncrementMB > 0 {
// 未切换卡沿用卡表基线算法的增量;已切换卡的增量已替换为按月对账结果,每次观测最多一条 V1 事件。
eventID := outboxid.Stable("card-traffic:", strconv.FormatUint(uint64(card.ID), 10)+":"+observation.Metadata.ObservationID+":incremented")
if err := s.eventWriter.AppendTraffic(ctx, tx, TrafficIncrementedEvent{
EventID: eventID, CardID: card.ID, IncrementMB: decision.IncrementMB,
ObservedAt: observation.Metadata.ObservedAt, Source: observation.Metadata.Source,
Scene: observation.Metadata.Scene, RequestID: observation.Metadata.RequestID,
CorrelationID: observation.Metadata.CorrelationID,
}); err != nil {
return err
}
}
evaluateThreshold := decision.ReadingAccepted
if switched {
evaluateThreshold = switchedThresholdEvaluable(flow, decision.ReadingAccepted)
}
if observation.Metadata.Scene == constants.CardObservationScenePreActivationSettlement {
// 激活前结算只记已结算用量,不触发停复机评估;下一次正常轮询观测照常判定。
evaluateThreshold = false
}
if evaluateThreshold && s.channelThreshold != nil {
// 未切换卡只使用本次已接受的网关累计读数,异常下降保护命中的观测不参与判定;
// 已切换卡读数有效且未被重置日防旧值处理即判定,不再受卡表基线影响。
// 判定失败必须回滚整个流量事务,与既有流量事实保持同事务语义。
if err := s.channelThreshold.EvaluateInTx(ctx, tx, carrierthresholddomain.Evaluation{
CardID: card.ID, CarrierID: card.CarrierID,
ReadingMB: thresholdReading(switched, flow, observation, decision), ObservedAt: observation.Metadata.ObservedAt,
// 观测来源是判定端获取触发来源的唯一位置:命中留痕据此区分自动判定、人工同步与回调。
Source: observation.Metadata.Source,
}); err != nil {
return err
}
}
stateChanged := decision.IncrementMB != 0 || decision.CrossMonth || decision.LastGatewayReadingMB != card.LastGatewayReadingMB
if actionCode, audited := manualRefreshAuditAction(ctx); observation.Metadata.Source == constants.CardObservationSourceManualSync && stateChanged && audited {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: actionCode,
Summary: "人工刷新 IoT 卡流量",
Card: &card,
BeforeData: map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
"last_gateway_reading_mb": card.LastGatewayReadingMB,
},
AfterData: map[string]any{
"data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB,
},
}); err != nil {
return err
}
} else if workerObservationAudited(ctx) && stateChanged {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
BeforeData: map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
"last_gateway_reading_mb": card.LastGatewayReadingMB,
},
AfterData: map[string]any{
"data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB,
},
}); err != nil {
return err
}
next, err := s.applyWrittenPoint(ctx, tx, &card, observation, packages)
if err != nil {
return err
}
outcome = next
return nil
})
if err != nil {
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实失败", Card: auditedCard,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
s.writeTrafficFailureAudit(ctx, observation, err)
return domain.TrafficDecision{}, err
}
if outcome != nil {
record.applyOutcome(outcome)
decision.ReadingAccepted = outcome.Written
decision.IncrementMB = outcome.EmitMB
decision.CrossMonth = outcome.CrossMonth
decision.CurrentMonthUsageMB = outcome.MonthMaxMB
}
decision.Decision = record.Decision
s.writeCollectionRecord(ctx, record, observation.Metadata.ObservedAt)
if s.cache != nil {
s.cache.Invalidate(ctx, observation.CardID)
}
if auditedCard != nil && !s.trafficCalc.Switched(auditedCard.ID) {
s.writeUnswitchedSnapshot(ctx, auditedCard, observation)
}
return decision, nil
}
// switchedThresholdEvaluable 判定已切换卡本次是否做通道阈值判定:读数有效且本次未被防旧值按 0 处理;
// 第一次切换观测仍按卡表基线算法是否接受读数判定。
func switchedThresholdEvaluable(flow *switchedFlowOutcome, oldAccepted bool) bool {
if flow == nil {
return false
// applyWrittenPoint 在观测事务内处理一个通过队列判断的写入点。
func (s *Service) applyWrittenPoint(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation, packages postgres.EffectivePackages) (*flowOutcome, error) {
hasPackage := len(packages.Packages) > 0
reading := observation.GatewayReadingMB
observedAt := observation.Metadata.ObservedAt
outcome := &flowOutcome{MonthMaxMB: card.CurrentMonthUsageMB, Decision: constants.TrafficDecisionWritten}
if reading == 0 && !hasPackage {
// 没套餐的卡读数为 0 不写快照。
return outcome, touchTrafficCheck(tx, card.ID, observedAt)
}
if flow.FirstSwitch {
return oldAccepted
flow, err := s.writeFlow(ctx, tx, card, reading, observedAt, hasPackage)
if err != nil {
return nil, err
}
return !flow.StaleGuarded
outcome.Written, outcome.MonthMaxMB, outcome.StaleGuarded = true, flow.monthMax, flow.conversion.StaleGuarded
outcome.MonthUsedMB, outcome.DeltaMB = flow.conversion.MonthUsedMB, flow.delta
if card.TrafficBaselinePending {
outcome.Decision = constants.TrafficDecisionBaseline
} else if hasPackage && flow.delta > 0 {
outcome.EmitMB = flow.delta
}
if hasPackage {
if err := ensureDailyUsageRows(ctx, tx, packages.Packages, flow.flowDate); err != nil {
return nil, err
}
}
monthStart := cardflow.MonthStart(flow.flowDate)
outcome.CrossMonth = card.CurrentMonthStartDate == nil || card.CurrentMonthStartDate.Before(monthStart)
updates := map[string]any{
"last_data_check_at": observedAt, "last_sync_time": observedAt,
"current_month_usage_mb": flow.monthMax, "current_month_start_date": monthStart,
"data_usage_mb": card.DataUsageMB + int64(flow.delta), "traffic_baseline_pending": false,
}
if outcome.CrossMonth {
updates["last_month_total_mb"] = card.CurrentMonthUsageMB
}
if err := tx.Model(&model.IotCard{}).Where("id = ?", card.ID).Updates(updates).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "更新卡流量事实失败")
}
if outcome.EmitMB > 0 {
outcome.EventID = outboxid.Stable("card-traffic:", strconv.FormatUint(uint64(card.ID), 10)+":"+observation.Metadata.ObservationID+":incremented")
if err := s.eventWriter.AppendTraffic(ctx, tx, TrafficIncrementedEvent{
EventID: outcome.EventID, CardID: card.ID, IncrementMB: outcome.EmitMB,
ObservedAt: observedAt, ObservationID: observation.Metadata.ObservationID,
Source: observation.Metadata.Source, Scene: observation.Metadata.Scene,
RequestID: observation.Metadata.RequestID, CorrelationID: observation.Metadata.CorrelationID,
}); err != nil {
return nil, err
}
}
// 通道阈值只在写入点判断:读数大于 0 且未被防旧值按 0 处理;激活前结算不触发停复机评估。
if reading > 0 && !outcome.StaleGuarded && s.channelThreshold != nil &&
observation.Metadata.Scene != constants.CardObservationScenePreActivationSettlement {
if err := s.channelThreshold.EvaluateInTx(ctx, tx, carrierthresholddomain.Evaluation{
CardID: card.ID, CarrierID: card.CarrierID, ReadingMB: reading, ObservedAt: observedAt,
// 观测来源是判定端获取触发来源的唯一位置:命中留痕据此区分自动判定、人工同步与回调。
Source: observation.Metadata.Source,
}); err != nil {
return nil, err
}
}
if err := s.writeTrafficStateAudit(ctx, tx, card, observation, outcome); err != nil {
return nil, err
}
return outcome, nil
}
// thresholdReading 返回通道阈值判定使用的网关累计读数:已切换卡(非第一次切换观测)用本次读数。
func thresholdReading(switched bool, flow *switchedFlowOutcome, observation domain.TrafficObservation, decision domain.TrafficDecision) float64 {
if switched && flow != nil && !flow.FirstSwitch {
return observation.GatewayReadingMB
// touchTrafficCheck 只刷新卡的最近流量检查时间(本次没有写入快照)。
func touchTrafficCheck(tx *gorm.DB, cardID uint, observedAt time.Time) error {
if err := tx.Model(&model.IotCard{}).Where("id = ?", cardID).
Updates(map[string]any{"last_data_check_at": observedAt, "last_sync_time": observedAt}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新卡流量检查时间失败")
}
return decision.LastGatewayReadingMB
return nil
}
// writeTrafficStateAudit 在本次写入改变了卡流量事实时写统一审计(人工刷新或 Worker 观测)。
func (s *Service) writeTrafficStateAudit(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation, outcome *flowOutcome) error {
if outcome.DeltaMB <= 0 && !outcome.CrossMonth {
return nil
}
before := map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
}
after := map[string]any{
"data_usage_mb": card.DataUsageMB + int64(outcome.DeltaMB), "current_month_usage_mb": outcome.MonthMaxMB,
"increment_mb": outcome.EmitMB, "delta_mb": outcome.DeltaMB, "decision": outcome.Decision,
"observation_id": observation.Metadata.ObservationID,
}
input := StateAudit{Card: card, BeforeData: before, AfterData: after}
if actionCode, audited := manualRefreshAuditAction(ctx); observation.Metadata.Source == constants.CardObservationSourceManualSync && audited {
input.ActionCode, input.Summary = actionCode, "人工刷新 IoT 卡流量"
} else if workerObservationAudited(ctx) {
input.ActionCode, input.Summary = constants.AuditActionIotCardWorkerTrafficSynced, "Worker 同步 IoT 卡流量事实"
input.IntegrationID = observation.Metadata.ObservationID
} else {
return nil
}
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
return s.auditWriter.WriteCardStateAudit(ctx, tx, input)
}
// writeTrafficFailureAudit 记录 Worker 观测事务失败的审计。
func (s *Service) writeTrafficFailureAudit(ctx context.Context, observation domain.TrafficObservation, err error) {
if !workerObservationAudited(ctx) || s.auditWriter == nil {
return
}
var card model.IotCard
if loadErr := s.db.WithContext(ctx).Where("id = ?", observation.CardID).First(&card).Error; loadErr != nil {
return
}
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实失败", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
func (s *Service) queueSettings() TrafficQueueSettings {
if s.trafficQueueSettings == nil {
return DefaultTrafficQueueSettings()
}
return s.trafficQueueSettings.Current(context.Background())
}
// logTrafficWarn 记录不影响观测结果的告警。
func (s *Service) logTrafficWarn(message string, fields ...zap.Field) {
s.warnLogger().Warn(message, fields...)
}