Files
junhong_cmp_fiber/internal/polling/tier.go
邹进 1214181e78
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投
- 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引
- 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存
- 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 16:33:29 +09:00

176 lines
5.7 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package polling
import (
"context"
"sync"
"time"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// TierFacts 是分层轮询匹配所需、卡表之外的业务事实。
type TierFacts struct {
PackageState string // 套餐状态,见 constants.PollingPackageState*
Known bool // 是否成功解析;解析失败时要求套餐状态的配置不匹配
}
type tierCacheEntry struct {
facts TierFacts
loadedAt time.Time
}
// TierResolver 计算卡的套餐状态(绑定设备的卡按设备、独立卡按卡,与停复机"是否有有效套餐"口径一致),
// 并读取运营商账户的流量重置日。结果在进程内短时缓存,层级最迟在缓存过期后的下一次执行时更新。
type TierResolver struct {
usage *postgres.PackageUsageStore
bindings *postgres.DeviceSimBindingStore
carriers *postgres.CarrierStore
logger *zap.Logger
mu sync.Mutex
cache map[uint]tierCacheEntry
resetDays map[uint]int
resetsAt time.Time
}
// NewTierResolver 创建分层事实解析器。
func NewTierResolver(usage *postgres.PackageUsageStore, bindings *postgres.DeviceSimBindingStore, carriers *postgres.CarrierStore, logger *zap.Logger) *TierResolver {
if logger == nil {
logger = zap.NewNop()
}
return &TierResolver{usage: usage, bindings: bindings, carriers: carriers, logger: logger, cache: map[uint]tierCacheEntry{}, resetDays: map[uint]int{}}
}
// Resolve 返回卡的分层事实;缓存未过期时直接返回。
func (r *TierResolver) Resolve(ctx context.Context, card *model.IotCard) TierFacts {
if r == nil || card == nil {
return TierFacts{}
}
if facts, ok := r.cached(card.ID); ok {
return facts
}
r.Prefill(ctx, []*model.IotCard{card})
facts, _ := r.cached(card.ID)
return facts
}
// Prefill 批量解析一组卡的分层事实并写入缓存,供全量初始化等批处理避免逐卡查库。
func (r *TierResolver) Prefill(ctx context.Context, cards []*model.IotCard) {
if r == nil || r.usage == nil || len(cards) == 0 {
return
}
var standaloneIDs, boundCardIDs []uint
for _, card := range cards {
if card == nil {
continue
}
if card.IsStandalone {
standaloneIDs = append(standaloneIDs, card.ID)
} else {
boundCardIDs = append(boundCardIDs, card.ID)
}
}
results := make(map[uint]TierFacts, len(cards))
if len(standaloneIDs) > 0 {
states, err := r.packageStates(ctx, constants.AssetTypeIotCard, standaloneIDs)
if err != nil {
r.logger.Warn("批量查询卡套餐状态失败,相关卡本次不按套餐状态匹配", zap.Error(err))
}
for _, id := range standaloneIDs {
if err == nil {
results[id] = TierFacts{PackageState: states[id], Known: true}
}
}
}
if len(boundCardIDs) > 0 && r.bindings != nil {
bindCtx, cancel := context.WithTimeout(ctx, constants.PollingTierQueryTimeout)
bindings, err := r.bindings.GetActiveBindingsByCardIDs(bindCtx, boundCardIDs)
cancel()
if err != nil {
r.logger.Warn("批量查询卡绑定设备失败,相关卡本次不按套餐状态匹配", zap.Error(err))
} else {
deviceOf := make(map[uint]uint, len(bindings))
deviceIDs := make([]uint, 0, len(bindings))
for _, binding := range bindings {
deviceOf[binding.IotCardID] = binding.DeviceID
deviceIDs = append(deviceIDs, binding.DeviceID)
}
deviceStates, devErr := r.packageStates(ctx, constants.AssetTypeDevice, deviceIDs)
cardStates, cardErr := r.packageStates(ctx, constants.AssetTypeIotCard, boundCardIDs)
for _, id := range boundCardIDs {
// 有有效设备绑定按设备计算;绑定已失效的卡按卡自身计算(与 resolvePackageCarrier 口径一致)。
if deviceID, bound := deviceOf[id]; bound {
if devErr == nil {
results[id] = TierFacts{PackageState: deviceStates[deviceID], Known: true}
}
} else if cardErr == nil {
results[id] = TierFacts{PackageState: cardStates[id], Known: true}
}
}
}
}
// 只缓存查询成功的卡:查询失败的卡本次按未知处理,下次重新查询,不把"未知"留在缓存里。
now := time.Now()
r.mu.Lock()
for id, facts := range results {
r.cache[id] = tierCacheEntry{facts: facts, loadedAt: now}
}
r.mu.Unlock()
}
// packageStates 以单独的超时查询一批载体的套餐状态,避免多次查询共用一个超时。
func (r *TierResolver) packageStates(ctx context.Context, carrierType string, ids []uint) (map[uint]string, error) {
queryCtx, cancel := context.WithTimeout(ctx, constants.PollingTierQueryTimeout)
defer cancel()
return r.usage.PackageStatesByCarriers(queryCtx, carrierType, ids)
}
// Invalidate 清除某卡的分层事实缓存(套餐变化后可调用,未调用时最迟缓存过期后更新)。
func (r *TierResolver) Invalidate(cardID uint) {
if r == nil {
return
}
r.mu.Lock()
delete(r.cache, cardID)
r.mu.Unlock()
}
func (r *TierResolver) cached(cardID uint) (TierFacts, bool) {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.cache[cardID]
if !ok || time.Since(entry.loadedAt) > constants.PollingTierFactsCacheSeconds*time.Second {
return TierFacts{}, false
}
return entry.facts, true
}
// ResetDay 返回运营商账户的流量重置日(1~28),缓存 10 分钟;读取失败按 1 号。
func (r *TierResolver) ResetDay(ctx context.Context, carrierID uint) int {
if r == nil || r.carriers == nil {
return 1
}
r.mu.Lock()
if time.Since(r.resetsAt) > 10*time.Minute {
r.resetDays = map[uint]int{}
r.resetsAt = time.Now()
}
day, ok := r.resetDays[carrierID]
r.mu.Unlock()
if ok {
return day
}
day = r.carriers.GetDataResetDay(ctx, carrierID)
if day < 1 || day > 28 {
day = 1
}
r.mu.Lock()
r.resetDays[carrierID] = day
r.mu.Unlock()
return day
}