feat: 卡流量读数先过采集队列再入账,扣减按采集时刻确定套餐
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m27s

- 每卡 Redis 采集队列(默认 N=3、跨天清空、回退容差 1 MB),攒满且不下降才写最后一个点
- 测试环境全量切换日快照计量,下线灰度名单、切换水位与卡表基线扣费;全部卡置待建起点
- 读数区分没给值与 0;有套餐的卡读数 0 也写快照与日用量 0 行;重置日比较基准为 0 照常防旧值
- 新增采集记录(按月分区保留 90 天)、人工冲正流水与后台接口、采集告警
- 扣减按采集时刻有效套餐查找,只对生效中套餐做状态迁移,查不到套餐按成功跳过
- 换货新卡(新设备下的卡)打待建起点;采集间隔上限只校验启用中的轮询配置

Change: add-card-traffic-queue-and-observed-deduction

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-29 12:18:47 +09:00
parent c33cc6f6c3
commit 9c3c258f16
55 changed files with 3142 additions and 881 deletions

View File

@@ -0,0 +1,120 @@
package cardobservation
import (
"context"
"time"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// 采集告警阈值。
const (
// trafficAlertSlopeRatio 是单卡当天"读数下降"拦截占比告警阈值。
trafficAlertSlopeRatio = 0.5
// trafficAlertSlopeMinRecords 是单卡当天参与拦截占比统计的最少采集记录数。
trafficAlertSlopeMinRecords = 6
// trafficAlertSampleLimit 是每类告警日志里列出的卡数上限。
trafficAlertSampleLimit = 50
)
// TrafficAlertScanner 按天扫描采集记录与日快照,输出卡流量采集告警日志(只看开启轮询的卡)。
type TrafficAlertScanner struct {
db *gorm.DB
logger *zap.Logger
}
// NewTrafficAlertScanner 创建卡流量采集告警扫描器。
func NewTrafficAlertScanner(db *gorm.DB, logger *zap.Logger) *TrafficAlertScanner {
if logger == nil {
logger = zap.NewNop()
}
return &TrafficAlertScanner{db: db, logger: logger}
}
// Scan 扫描 day(上海日期)一整天:有套餐的卡全天没有写入、待建起点仍未清除、单卡读数下降占比过高、
// 乱序数量与重置日防旧值命中数。当天没有写入时不补写快照,只告警。
func (s *TrafficAlertScanner) Scan(ctx context.Context, day time.Time) error {
from := cardflow.ShanghaiDate(day)
to := from.AddDate(0, 0, 1)
db := s.db.WithContext(ctx)
var notWritten []uint
if err := db.Raw(`
SELECT c.iot_card_id FROM tb_card_traffic_collection c
JOIN tb_iot_card card ON card.id = c.iot_card_id AND card.enable_polling = TRUE AND card.deleted_at IS NULL
WHERE c.observed_at >= ? AND c.observed_at < ?
GROUP BY c.iot_card_id
HAVING COUNT(*) FILTER (WHERE c.decision IN (?, ?)) = 0
LIMIT ?`, from, to, constants.TrafficDecisionWritten, constants.TrafficDecisionBaseline, trafficAlertSampleLimit).
Scan(&notWritten).Error; err != nil {
return err
}
if len(notWritten) > 0 {
s.logger.Error("有套餐的卡一整天没有写入流量快照,请核对运营商读数与采集间隔",
zap.String("date", from.Format("2006-01-02")), zap.Uints("card_ids", notWritten),
zap.String("metric", "card_traffic.collection.day_not_written"))
}
var baselinePending []uint
if err := db.Raw(`
SELECT id FROM tb_iot_card
WHERE traffic_baseline_pending = TRUE AND enable_polling = TRUE AND deleted_at IS NULL
AND last_data_check_at IS NOT NULL AND last_data_check_at < ? AND last_data_check_at >= ?
LIMIT ?`, to, from, trafficAlertSampleLimit).Scan(&baselinePending).Error; err != nil {
return err
}
if len(baselinePending) > 0 {
s.logger.Error("开启轮询的卡已被采集一天仍未建起点,请核对采集队列是否一直不满足写入条件",
zap.String("date", from.Format("2006-01-02")), zap.Uints("card_ids", baselinePending),
zap.String("metric", "card_traffic.collection.baseline_pending"))
}
var slopeCards []uint
if err := db.Raw(`
SELECT c.iot_card_id FROM tb_card_traffic_collection c
JOIN tb_iot_card card ON card.id = c.iot_card_id AND card.enable_polling = TRUE AND card.deleted_at IS NULL
WHERE c.observed_at >= ? AND c.observed_at < ? AND c.result = ?
GROUP BY c.iot_card_id
HAVING COUNT(*) >= ? AND COUNT(*) FILTER (WHERE c.decision = ?)::float / COUNT(*) > ?
LIMIT ?`, from, to, constants.TrafficCollectionResultOK, trafficAlertSlopeMinRecords,
constants.TrafficDecisionSlopeNegative, trafficAlertSlopeRatio, trafficAlertSampleLimit).Scan(&slopeCards).Error; err != nil {
return err
}
if len(slopeCards) > 0 {
s.logger.Error("卡当天读数下降拦截占比过高,请核对运营商读数",
zap.String("date", from.Format("2006-01-02")), zap.Uints("card_ids", slopeCards),
zap.Float64("ratio_threshold", trafficAlertSlopeRatio), zap.String("metric", "card_traffic.collection.slope_negative_high"))
}
var totals struct {
Total int64
Slope int64
OutOfOrder int64
}
if err := db.Raw(`
SELECT COUNT(*) AS total,
COUNT(*) FILTER (WHERE decision = ?) AS slope,
COUNT(*) FILTER (WHERE decision = ?) AS out_of_order
FROM tb_card_traffic_collection WHERE observed_at >= ? AND observed_at < ?`,
constants.TrafficDecisionSlopeNegative, constants.TrafficDecisionOutOfOrder, from, to).Scan(&totals).Error; err != nil {
return err
}
var staleHits int64
if err := db.Raw(`SELECT COUNT(*) FROM tb_card_flow_day WHERE flow_date = ?::date AND stale_guarded = TRUE`,
from.Format("2006-01-02")).Scan(&staleHits).Error; err != nil {
return err
}
level := s.logger.Info
if totals.OutOfOrder > 0 {
level = s.logger.Warn
}
level("卡流量采集日统计",
zap.String("date", from.Format("2006-01-02")), zap.Int64("collections", totals.Total),
zap.Int64("slope_negative", totals.Slope), zap.Int64("out_of_order", totals.OutOfOrder),
zap.Int64("stale_guarded_cards", staleHits), zap.String("metric", "card_traffic.collection.daily"))
return nil
}