feat: 卡流量读数先过采集队列再入账,扣减按采集时刻确定套餐
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m27s

- 每卡 Redis 采集队列(默认 N=3、跨天清空、回退容差 1 MB),攒满且不下降才写最后一个点
- 测试环境全量切换日快照计量,下线灰度名单、切换水位与卡表基线扣费;全部卡置待建起点
- 读数区分没给值与 0;有套餐的卡读数 0 也写快照与日用量 0 行;重置日比较基准为 0 照常防旧值
- 新增采集记录(按月分区保留 90 天)、人工冲正流水与后台接口、采集告警
- 扣减按采集时刻有效套餐查找,只对生效中套餐做状态迁移,查不到套餐按成功跳过
- 换货新卡(新设备下的卡)打待建起点;采集间隔上限只校验启用中的轮询配置

Change: add-card-traffic-queue-and-observed-deduction

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-29 12:18:47 +09:00
parent c33cc6f6c3
commit 9c3c258f16
55 changed files with 3142 additions and 881 deletions

View File

@@ -8,50 +8,93 @@ import (
"gorm.io/gorm"
cardflow "github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
domain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// unswitchedSnapshotTimeout 是未切换卡在观测事务提交后写快照的超时,超时只告警。
const unswitchedSnapshotTimeout = 3 * time.Second
// switchedFlowOutcome 是已切换卡一次观测的日快照计量结果。
type switchedFlowOutcome struct {
// EmitMB 是本次应写入 V1 流量事件的增量,0 表示不写事件。
// flowOutcome 是一个写入点的日快照计量结果。
type flowOutcome struct {
// Written 为 true 表示本次写入了日快照(没套餐且读数为 0 时为 false)。
Written bool
// Decision 是写入判断:已写入或建起点。
Decision string
// EmitMB 是本次写入流量事件的增量,0 表示不写事件。
EmitMB float64
// MonthMaxMB 是写入本次快照后的本月最大快照,用于覆盖卡表本月用量。
MonthMaxMB float64
// DeltaMB 是本次按月对账计入已结算的增量(含没套餐、建起点时只记已结算的部分)。
DeltaMB float64
// MonthUsedMB 是本次读数换算后的本月累计;MonthMaxMB 是写入后的本月最大快照。
MonthUsedMB float64
MonthMaxMB float64
// StaleGuarded 为 true 表示本次读数在重置日触发防旧值、按 0 计。
StaleGuarded bool
// FirstSwitch 为 true 表示这是卡的第一次切换观测(本次写入了切换水位)。
FirstSwitch bool
// CrossMonth 为 true 表示卡表本月用量跨入新的自然月。
CrossMonth bool
// EventID 是本次写入的流量事件 ID。
EventID string
}
// snapshotResult 是一次快照换算与写入的结果。
type snapshotResult struct {
// flowWrite 是一次快照写入与按月对账的结果。
type flowWrite struct {
flowDate time.Time
conversion cardflow.Conversion
monthMax float64
delta float64
}
// writeSnapshot 按卡所属通道的重置日换算并原子写入当天快照;读数必须已通过 cardflow.ValidReading。
func (s *Service) writeSnapshot(ctx context.Context, db *gorm.DB, card *model.IotCard, readingMB float64, observedAt time.Time) (snapshotResult, error) {
resetDay := s.strictResetDay(ctx, db, card)
// writeFlow 按卡所属通道的重置日换算并原子写入当天快照,再按月对账把增量计入当天已结算。
func (s *Service) writeFlow(ctx context.Context, tx *gorm.DB, card *model.IotCard, readingMB float64, observedAt time.Time, hasPackage bool) (flowWrite, error) {
resetDay := s.strictResetDay(ctx, tx, card)
flowDate := cardflow.ShanghaiDate(observedAt)
days := postgres.NewCardFlowDayStore(db)
days := postgres.NewCardFlowDayStore(tx)
monthCtx, err := days.MonthContext(ctx, card.ID, flowDate, resetDay)
if err != nil {
return snapshotResult{}, errors.Wrap(errors.CodeDatabaseError, err, "查询卡日流量快照失败")
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "查询卡日流量快照失败")
}
conversion := cardflow.Convert(readingMB, flowDate.Day(), resetDay, monthCtx)
if err := days.Upsert(ctx, postgres.CardFlowSnapshotWrite{
CardID: card.ID, FlowDate: flowDate, MonthUsedMB: conversion.MonthUsedMB,
VendorUsedMB: readingMB, StaleGuarded: conversion.StaleGuarded,
}); err != nil {
return snapshotResult{}, errors.Wrap(errors.CodeDatabaseError, err, "写入卡日流量快照失败")
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "写入卡日流量快照失败")
}
return snapshotResult{flowDate: flowDate, conversion: conversion}, nil
monthMax, settled, err := days.MonthTotals(ctx, card.ID, flowDate)
if err != nil {
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "查询卡本月流量结算失败")
}
delta := cardflow.MonthlyDelta(monthMax, settled)
if err := days.AddSettled(ctx, card.ID, flowDate, delta); err != nil {
return flowWrite{}, errors.Wrap(errors.CodeDatabaseError, err, "结算卡流量增量失败")
}
return flowWrite{flowDate: flowDate, conversion: conversion, monthMax: monthMax, delta: delta}, nil
}
// settleExisting 用卡上已有的本月最大快照结算(激活前结算本次没写入时):差额全部记为已结算,不新写快照。
func (s *Service) settleExisting(ctx context.Context, tx *gorm.DB, card *model.IotCard, observedAt time.Time) error {
flowDate := cardflow.ShanghaiDate(observedAt)
days := postgres.NewCardFlowDayStore(tx)
monthMax, settled, err := days.MonthTotals(ctx, card.ID, flowDate)
if err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "查询卡本月流量结算失败")
}
if err := days.AddSettledToLatest(ctx, card.ID, flowDate, cardflow.MonthlyDelta(monthMax, settled)); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "激活前结算已有快照失败")
}
return nil
}
// ensureDailyUsageRows 为采集时刻每个有效套餐确保当天一行套餐日用量(没扣也写 0)。
func ensureDailyUsageRows(ctx context.Context, tx *gorm.DB, packages []*model.PackageUsage, flowDate time.Time) error {
ids := make([]uint, 0, len(packages))
cumulative := make(map[uint]int64, len(packages))
for _, pkg := range packages {
ids = append(ids, pkg.ID)
cumulative[pkg.ID] = pkg.DataUsageMB
}
if err := postgres.EnsureDailyUsageRows(ctx, tx, ids, flowDate, cumulative); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "写入套餐当天日用量失败")
}
return nil
}
// strictResetDay 在当前连接内按卡的 carrier_id 读取上游流量重置日,不使用调用方预先计算的值;
@@ -73,78 +116,3 @@ func (s *Service) strictResetDay(ctx context.Context, db *gorm.DB, card *model.I
}
return resetDay
}
// applySwitchedFlow 在观测事务内为已切换卡写快照、按月对账结算增量,并在第一次切换观测时写切换水位。
//
// oldDecision 是卡表基线算法对本次观测的结果:只在第一次切换观测时沿用其增量,此后不再使用。
func (s *Service) applySwitchedFlow(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation, oldDecision domain.TrafficDecision) (*switchedFlowOutcome, error) {
readingMB := observation.GatewayReadingMB
snapshot, err := s.writeSnapshot(ctx, tx, card, readingMB, observation.Metadata.ObservedAt)
if err != nil {
return nil, err
}
days := postgres.NewCardFlowDayStore(tx)
monthMax, settled, err := days.MonthTotals(ctx, card.ID, snapshot.flowDate)
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡本月流量结算失败")
}
hasPackage, pkgErr := hasActivePackageForCard(ctx, tx, card.ID, observation.Metadata.ObservedAt)
if pkgErr != nil {
// 判断出错按有套餐处理,保持不丢量。
s.warnLogger().Warn("判断卡是否有生效中套餐失败,按有套餐处理", zap.Uint("card_id", card.ID), zap.Error(pkgErr))
hasPackage = true
}
outcome := &switchedFlowOutcome{MonthMaxMB: monthMax, StaleGuarded: snapshot.conversion.StaleGuarded}
switches := postgres.NewCardFlowSwitchStore(tx)
watermark, err := switches.Get(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡流量切换水位失败")
}
if watermark == nil {
return outcome, s.firstSwitch(ctx, tx, card, observation, oldDecision, snapshot.flowDate, monthMax, settled, hasPackage, outcome)
}
delta := cardflow.MonthlyDelta(monthMax, settled)
if delta <= 0 {
return outcome, nil
}
if err := days.AddSettled(ctx, card.ID, snapshot.flowDate, delta); err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "结算卡流量增量失败")
}
if hasPackage {
outcome.EmitMB = delta
}
return outcome, nil
}
// firstSwitch 处理第一次切换观测:本次沿用卡表基线算法的增量,此前部分记为已结算,并写入切换水位。
func (s *Service) firstSwitch(ctx context.Context, tx *gorm.DB, card *model.IotCard, observation domain.TrafficObservation,
oldDecision domain.TrafficDecision, flowDate time.Time, monthMax, settled float64, hasPackage bool, outcome *switchedFlowOutcome) error {
outcome.FirstSwitch = true
if oldDecision.IncrementMB > 0 && hasPackage {
outcome.EmitMB = oldDecision.IncrementMB
}
if err := postgres.NewCardFlowDayStore(tx).AddSettled(ctx, card.ID, flowDate, cardflow.MonthlyDelta(monthMax, settled)); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "结算卡流量切换前用量失败")
}
if err := postgres.NewCardFlowSwitchStore(tx).Create(ctx, &model.CardFlowSwitch{
IotCardID: card.ID, SwitchedAt: observation.Metadata.ObservedAt, MonthUsedMB: monthMax,
VendorUsedMB: observation.GatewayReadingMB, OldBaselineMB: card.LastGatewayReadingMB,
OldMonthUsageMB: card.CurrentMonthUsageMB, OldReadingAccepted: oldDecision.ReadingAccepted,
OldIncrementMB: oldDecision.IncrementMB,
}); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "写入卡流量切换水位失败")
}
return nil
}
// writeUnswitchedSnapshot 在观测事务提交后为未切换卡写快照;失败只告警,不影响本次观测结果。
func (s *Service) writeUnswitchedSnapshot(ctx context.Context, card *model.IotCard, observation domain.TrafficObservation) {
if card == nil || !s.trafficCalc.SnapshotEnabled() || !cardflow.ValidReading(observation.GatewayReadingMB) {
return
}
writeCtx, cancel := context.WithTimeout(ctx, unswitchedSnapshotTimeout)
defer cancel()
if _, err := s.writeSnapshot(writeCtx, s.db, card, observation.GatewayReadingMB, observation.Metadata.ObservedAt); err != nil {
s.warnLogger().Warn("未切换卡写日流量快照失败,不影响本次观测", zap.Uint("card_id", card.ID), zap.Error(err))
}
}