feat: 分层轮询、重置日前加密与激活前流量结算
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m37s

- 轮询配置新增套餐状态、资产状态、重置日前窗口与窗口内流量间隔(迁移 248),间隔三态:正数按间隔、0 不检查、空继承
- 配置按 priority, id 排序合并;在库与已销售无套餐未匹配时告警不入队
- 新增优先级整体重排、匹配预览、立即重算到期时间(异步只提前)、分层统计与重置日前加密统计接口
- 已切换日快照计量的卡流量按观测时间归属,激活前用量只记已结算
- 订单购买与实名激活补齐资产级激活锁,激活前对已切换卡结算一次流量,失败只告警不阻塞,结算不触发通道阈值判定
This commit is contained in:
邹进
2026-10-24 17:06:38 +09:00
parent 02a842f066
commit 98017ac053
34 changed files with 2232 additions and 182 deletions

View File

@@ -0,0 +1,419 @@
package polling
import (
"context"
"sort"
"strconv"
"strings"
"time"
"github.com/bytedance/sonic"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"gorm.io/gorm"
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/model/dto"
pollingcore "github.com/break/junhong_cmp_fiber/internal/polling"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
)
// tierStatsCacheKey 是分层统计结果的缓存 Key(5 分钟)。
const tierStatsCacheKey = "polling:tier_stats"
// TaskEnqueuer 是提交异步任务的最小端口(由 pkg/queue.Client 实现)。
type TaskEnqueuer interface {
EnqueueTask(ctx context.Context, taskType string, payload interface{}, opts ...asynq.Option) error
}
// SetTierDeps 注入分层轮询所需的配置管理器与异步队列;未注入时预览、重算与统计不可用。
func (s *ConfigService) SetTierDeps(configMgr *pollingcore.PollingConfigManager, queueClient TaskEnqueuer) {
s.configMgr = configMgr
s.queueClient = queueClient
}
// UpdatePriorities 按给定顺序把配置优先级重排为 1、2、3…(事务内完成并逐条写统一审计),仅超级管理员可用。
func (s *ConfigService) UpdatePriorities(ctx context.Context, req *dto.UpdatePollingConfigPrioritiesRequest) error {
if middleware.GetUserTypeFromContext(ctx) != constants.UserTypeSuperAdmin {
return errors.New(errors.CodeForbidden, "仅超级管理员可调整轮询配置优先级")
}
operatorID := middleware.GetUserIDFromContext(ctx)
seen := map[uint]bool{}
for _, id := range req.OrderedIDs {
if id == 0 || seen[id] {
return errors.New(errors.CodeInvalidParam, "轮询配置优先级参数不合法:配置 ID 不能为空或重复")
}
seen[id] = true
}
err := runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
for index, id := range req.OrderedIDs {
var cfg model.PollingConfig
if err := tx.WithContext(ctx).First(&cfg, id).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return errors.New(errors.CodePollingConfigNotFound, "轮询配置不存在:"+strconv.FormatUint(uint64(id), 10))
}
return errors.Wrap(errors.CodeDatabaseError, err, "查询轮询配置失败")
}
priority := index + 1
if cfg.Priority == priority {
continue
}
before := cfg
if err := tx.WithContext(ctx).Model(&model.PollingConfig{}).Where("id = ?", id).
Updates(map[string]any{"priority": priority, "updated_by": operatorID, "updated_at": time.Now()}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "调整轮询配置优先级失败")
}
cfg.Priority = priority
if err := writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingConfigUpdated, Summary: "调整轮询配置优先级",
ResourceType: constants.AuditResourcePollingConfig, ResourceID: cfg.ID,
ResourceKey: pollingManualTriggerKey(cfg.ID), DisplayName: cfg.ConfigName,
OperatorID: operatorID, IdentitySnapshot: pollingConfigIdentity(&cfg),
BeforeData: pollingConfigState(&before), AfterData: pollingConfigState(&cfg),
}); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
s.notifyConfigChanged(ctx, "priorities_updated")
return nil
}
// MatchPreview 返回卡的匹配维度、命中配置与各检查类型最终间隔及来源;可带未保存修改预演,不写任何数据。
func (s *ConfigService) MatchPreview(ctx context.Context, req *dto.PollingConfigMatchPreviewRequest) (*dto.PollingConfigMatchPreviewResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
if s.configMgr == nil {
return nil, errors.New(errors.CodeServiceUnavailable, "轮询配置匹配预览能力未配置")
}
iccid := strings.TrimSpace(req.ICCID)
var card model.IotCard
if err := s.db.WithContext(ctx).Where("iccid = ?", iccid).First(&card).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil, errors.New(errors.CodeNotFound, "IoT 卡不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询 IoT 卡失败")
}
configs, err := s.configStore.ListEnabled(ctx)
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询轮询配置失败")
}
configs, drafts, err := applyPreviewOverrides(configs, req.Overrides)
if err != nil {
return nil, err
}
if tier := s.configMgr.TierResolver(); tier != nil {
tier.Invalidate(card.ID)
}
matched, sources := s.configMgr.PreviewIntervalSources(configs, &card)
facts := s.configMgr.TierFactsOf(&card)
resp := &dto.PollingConfigMatchPreviewResponse{
Card: dto.PollingCardMatchFacts{
CardID: card.ID, ICCID: card.ICCID, CardCondition: pollingcore.CardConditionOf(&card), CardCategory: card.CardCategory,
CarrierID: card.CarrierID, AssetStatus: card.AssetStatus, EnablePolling: card.EnablePolling,
},
Matched: []dto.PollingConfigBrief{}, Intervals: []dto.PollingIntervalPreview{},
}
if facts.Known {
resp.Card.PackageState = facts.PackageState
resp.Card.PackageStateName = constants.PollingPackageStateName(facts.PackageState)
}
for _, cfg := range matched {
resp.Matched = append(resp.Matched, dto.PollingConfigBrief{ID: cfg.ID, ConfigName: cfg.ConfigName, Priority: cfg.Priority, Draft: drafts[cfg]})
}
firstID := uint(0)
if len(matched) > 0 {
firstID = matched[0].ID
}
for _, taskType := range previewTaskTypes {
item := dto.PollingIntervalPreview{TaskType: taskType, TaskTypeName: previewTaskTypeName(taskType), Mode: "unmatched"}
if source, ok := sources[taskType]; ok {
item.ConfigID, item.ConfigName = source.ConfigID, source.Config.ConfigName
item.Inherited = source.ConfigID != firstID
if source.Disabled {
item.Mode = "disabled"
} else {
item.Mode, item.Interval = "interval", source.Interval
}
}
resp.Intervals = append(resp.Intervals, item)
}
return resp, nil
}
var previewTaskTypes = []string{constants.TaskTypePollingCardStatus, constants.TaskTypePollingCarddata, constants.TaskTypePollingRealname, constants.TaskTypePollingPackage, constants.TaskTypePollingProtect}
// applyPreviewOverrides 把未保存修改合并进启用配置快照并重新排序;返回草稿配置标记。
func applyPreviewOverrides(configs []*model.PollingConfig, overrides []dto.PollingConfigOverride) ([]*model.PollingConfig, map[*model.PollingConfig]bool, error) {
drafts := map[*model.PollingConfig]bool{}
if len(overrides) == 0 {
return configs, drafts, nil
}
byID := map[uint]int{}
out := make([]*model.PollingConfig, 0, len(configs)+len(overrides))
for _, cfg := range configs {
byID[cfg.ID] = len(out)
out = append(out, cfg)
}
removed := map[int]bool{}
for i := range overrides {
item := overrides[i]
if item.ID != 0 && item.Disabled {
if idx, ok := byID[item.ID]; ok {
removed[idx] = true
}
continue
}
draft := &model.PollingConfig{
ID: item.ID, ConfigName: item.ConfigName, CardCondition: item.CardCondition, CardCategory: item.CardCategory,
CarrierID: item.CarrierID, Priority: item.Priority, RealnameCheckInterval: item.RealnameCheckInterval,
CarddataCheckInterval: item.CarddataCheckInterval, PackageCheckInterval: item.PackageCheckInterval,
ProtectCheckInterval: item.ProtectCheckInterval, CardStatusCheckInterval: item.CardStatusCheckInterval,
PackageState: item.PackageState, AssetStatus: item.AssetStatus, PreResetWindow: item.PreResetWindow,
PreResetCarddataInterval: item.PreResetCarddataInterval, Status: 1,
}
if draft.ConfigName == "" {
draft.ConfigName = "未保存的配置"
}
if err := validatePollingConfig(draft); err != nil {
return nil, nil, err
}
drafts[draft] = true
if idx, ok := byID[item.ID]; ok && item.ID != 0 {
out[idx] = draft
continue
}
out = append(out, draft)
}
filtered := make([]*model.PollingConfig, 0, len(out))
for idx, cfg := range out {
if !removed[idx] {
filtered = append(filtered, cfg)
}
}
sort.SliceStable(filtered, func(i, j int) bool {
if filtered[i].Priority != filtered[j].Priority {
return filtered[i].Priority < filtered[j].Priority
}
// 草稿(ID 为 0)排在同优先级已有配置之后。
if filtered[i].ID == 0 || filtered[j].ID == 0 {
return filtered[j].ID == 0 && filtered[i].ID != 0
}
return filtered[i].ID < filtered[j].ID
})
return filtered, drafts, nil
}
// CreateRecalculateTask 创建"立即重算到期时间"异步任务(由 Worker 执行),仅超级管理员可用。
func (s *ConfigService) CreateRecalculateTask(ctx context.Context, req *dto.PollingRecalculateDueRequest) (*dto.PollingRecalculateDueResponse, error) {
if middleware.GetUserTypeFromContext(ctx) != constants.UserTypeSuperAdmin {
return nil, errors.New(errors.CodeForbidden, "仅超级管理员可重算轮询到期时间")
}
if s.queueClient == nil || s.redis == nil {
return nil, errors.New(errors.CodeServiceUnavailable, "重算到期时间能力未配置")
}
taskID := uuid.NewString()
carrier := ""
if req.CarrierID != nil {
carrier = strconv.FormatUint(uint64(*req.CarrierID), 10)
}
now := time.Now().Format(time.RFC3339)
key := constants.RedisPollingRecalcProgressKey(taskID)
pipe := s.redis.Pipeline()
pipe.HSet(ctx, key, map[string]any{"status": constants.PollingRecalcStatusPending, "carrier_id": carrier, "created_at": now})
pipe.Expire(ctx, key, constants.PollingRecalcProgressTTLSeconds*time.Second)
if _, err := pipe.Exec(ctx); err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "创建重算任务失败")
}
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypePollingRecalculateDue,
pollingcore.RecalculateDuePayload{TaskID: taskID, CarrierID: req.CarrierID},
asynq.Queue(constants.QueueDataCleanup), asynq.MaxRetry(0), asynq.Timeout(30*time.Minute)); err != nil {
_ = s.redis.HSet(ctx, key, "status", constants.PollingRecalcStatusFailed, "error", "任务入队失败").Err()
return nil, errors.Wrap(errors.CodeServiceUnavailable, err, "重算任务入队失败")
}
return &dto.PollingRecalculateDueResponse{TaskID: taskID, Status: constants.PollingRecalcStatusPending, CarrierID: carrier, CreatedAt: now}, nil
}
// GetRecalculateTask 查询重算到期时间任务进度。
func (s *ConfigService) GetRecalculateTask(ctx context.Context, taskID string) (*dto.PollingRecalculateDueResponse, error) {
if s.redis == nil {
return nil, errors.New(errors.CodeServiceUnavailable, "重算到期时间能力未配置")
}
values, err := s.redis.HGetAll(ctx, constants.RedisPollingRecalcProgressKey(taskID)).Result()
if err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "查询重算任务失败")
}
if len(values) == 0 {
return nil, errors.New(errors.CodeNotFound, "重算任务不存在或已过期")
}
atoi := func(v string) int { n, _ := strconv.Atoi(v); return n }
return &dto.PollingRecalculateDueResponse{
TaskID: taskID, Status: values["status"], CarrierID: values["carrier_id"], Total: atoi(values["total"]),
Processed: atoi(values["processed"]), Advanced: atoi(values["advanced"]), Error: values["error"],
CreatedAt: values["created_at"], StartedAt: values["started_at"], FinishedAt: values["finished_at"],
}, nil
}
// TierStats 统计开启轮询的卡按首个命中配置与套餐状态的分布,以及应告警的未匹配卡数;结果缓存 5 分钟。
func (s *ConfigService) TierStats(ctx context.Context) (*dto.PollingTierStatsResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
if s.configMgr == nil {
return nil, errors.New(errors.CodeServiceUnavailable, "分层统计能力未配置")
}
if s.redis != nil {
if raw, err := s.redis.Get(ctx, tierStatsCacheKey).Bytes(); err == nil {
var cached dto.PollingTierStatsResponse
if sonic.Unmarshal(raw, &cached) == nil {
return &cached, nil
}
}
}
configs, err := s.configStore.ListEnabled(ctx)
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询轮询配置失败")
}
names := map[uint]string{0: "未匹配任何配置"}
for _, cfg := range configs {
names[cfg.ID] = cfg.ConfigName
}
resp := &dto.PollingTierStatsResponse{ByPackageState: map[string]int{}, ByConfig: []dto.PollingTierStatItem{}}
counts := map[uint]int{}
var lastID uint
for {
var cards []*model.IotCard
if err := s.db.WithContext(ctx).Where("id > ? AND enable_polling = true AND deleted_at IS NULL", lastID).
Order("id ASC").Limit(1000).Find(&cards).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询轮询卡失败")
}
if len(cards) == 0 {
break
}
lastID = cards[len(cards)-1].ID
if tier := s.configMgr.TierResolver(); tier != nil {
tier.Prefill(ctx, cards)
}
for _, card := range cards {
resp.TotalCards++
facts := s.configMgr.TierFactsOf(card)
state := "unknown"
if facts.Known {
state = facts.PackageState
}
resp.ByPackageState[state]++
matched, _ := s.configMgr.PreviewIntervalSources(configs, card)
if len(matched) == 0 {
counts[0]++
if card.AssetStatus == constants.AssetStatusInStock {
resp.UnmatchedInStock++
} else if card.AssetStatus == constants.AssetStatusSold && facts.Known && facts.PackageState == constants.PollingPackageStateNone {
resp.UnmatchedSoldNoPackage++
}
continue
}
counts[matched[0].ID]++
}
}
for id, n := range counts {
resp.ByConfig = append(resp.ByConfig, dto.PollingTierStatItem{ConfigID: id, ConfigName: names[id], Cards: n})
}
sort.Slice(resp.ByConfig, func(i, j int) bool { return resp.ByConfig[i].Cards > resp.ByConfig[j].Cards })
resp.ComputedAt = time.Now().Format(time.RFC3339)
if s.redis != nil {
if raw, err := sonic.Marshal(resp); err == nil {
_ = s.redis.Set(ctx, tierStatsCacheKey, raw, 5*time.Minute).Err()
}
}
return resp, nil
}
// previewTaskTypeName 返回检查类型中文名称(保护期检查在优先轮询名称表之外单独处理)。
func previewTaskTypeName(taskType string) string {
if taskType == constants.TaskTypePollingProtect {
return "保护期检查"
}
return constants.PollingPriorityTaskTypeName(taskType)
}
// PreResetStats 按运营商账户统计重置日前加密效果:上一个与即将到来的重置周期内,窗口内查到流量的卡数、
// 有生效套餐但窗口内未查到的卡数,以及最后一次查询距重置时刻的分布。
func (s *ConfigService) PreResetStats(ctx context.Context) (*dto.PollingPreResetStatsResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
if s.configMgr == nil || s.redis == nil {
return nil, errors.New(errors.CodeServiceUnavailable, "重置日前加密统计能力未配置")
}
var rows []struct {
CarrierID uint
CarrierName string
Active int
}
err := s.db.WithContext(ctx).Raw(`
SELECT c.carrier_id, COALESCE(ca.carrier_name, '') AS carrier_name,
COUNT(*) FILTER (WHERE (c.is_standalone AND EXISTS (
SELECT 1 FROM tb_package_usage u WHERE u.iot_card_id = c.id AND u.status = ? AND u.deleted_at IS NULL))
OR (NOT c.is_standalone AND EXISTS (
SELECT 1 FROM tb_device_sim_binding b JOIN tb_package_usage u ON u.device_id = b.device_id AND u.status = ? AND u.deleted_at IS NULL
WHERE b.iot_card_id = c.id AND b.bind_status = 1 AND b.deleted_at IS NULL))) AS active
FROM tb_iot_card c LEFT JOIN tb_carrier ca ON ca.id = c.carrier_id
WHERE c.enable_polling = true AND c.deleted_at IS NULL
GROUP BY c.carrier_id, ca.carrier_name ORDER BY c.carrier_id`,
constants.PackageUsageStatusActive, constants.PackageUsageStatusActive).Scan(&rows).Error
if err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计有生效套餐的卡失败")
}
resp := &dto.PollingPreResetStatsResponse{Carriers: []dto.PollingPreResetCarrierStat{}}
now := time.Now()
location, _ := time.LoadLocation("Asia/Shanghai")
for _, row := range rows {
previous, upcoming := s.configMgr.ResetMoments(ctx, row.CarrierID, now)
item := dto.PollingPreResetCarrierStat{CarrierID: row.CarrierID, CarrierName: row.CarrierName}
item.Previous = s.preResetCycle(ctx, row.CarrierID, previous, row.Active, location)
item.Upcoming = s.preResetCycle(ctx, row.CarrierID, upcoming, row.Active, location)
resp.Carriers = append(resp.Carriers, item)
}
return resp, nil
}
func (s *ConfigService) preResetCycle(ctx context.Context, carrierID uint, reset time.Time, active int, location *time.Location) dto.PollingPreResetCycleStat {
date := reset.Format("20060102")
stat := dto.PollingPreResetCycleStat{ResetAt: reset.In(location).Format("2006-01-02 15:04"), ActivePackageCards: active,
LastOffsetBuckets: map[string]int{"le_5": 0, "le_15": 0, "le_30": 0, "le_60": 0, "gt_60": 0}}
if n, err := s.redis.PFCount(ctx, constants.RedisPollingPreResetCheckedKey(carrierID, date)).Result(); err == nil {
stat.CheckedInWindow = n
}
if missing := int64(active) - stat.CheckedInWindow; missing > 0 {
stat.NotCheckedInWindow = missing
}
if values, err := s.redis.HVals(ctx, constants.RedisPollingPreResetOffsetKey(carrierID, date)).Result(); err == nil {
for _, raw := range values {
minutes, convErr := strconv.Atoi(raw)
if convErr != nil {
continue
}
switch {
case minutes <= 5:
stat.LastOffsetBuckets["le_5"]++
case minutes <= 15:
stat.LastOffsetBuckets["le_15"]++
case minutes <= 30:
stat.LastOffsetBuckets["le_30"]++
case minutes <= 60:
stat.LastOffsetBuckets["le_60"]++
default:
stat.LastOffsetBuckets["gt_60"]++
}
}
}
return stat
}