feat: 按运营商账户调度的分发器、卡任务执行器与模式切换
- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
228
cmd/worker/carrier_scheduler.go
Normal file
228
cmd/worker/carrier_scheduler.go
Normal file
@@ -0,0 +1,228 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/infrastructure/nsqbus"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/task"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// carrierInitLockKey 新结构初始化锁:多实例时只由一个实例初始化。
|
||||||
|
const carrierInitLockKey = "polling:carrier:init_lock"
|
||||||
|
|
||||||
|
// carrierSchedulerRuntime 是按运营商账户调度在 Worker 内的运行组件:模式监听、初始化、分发器与 NSQ 消费者。
|
||||||
|
type carrierSchedulerRuntime struct {
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
runtime *workerRuntime
|
||||||
|
initializer *polling.CarrierInitializer
|
||||||
|
dispatcher *carrierqueue.Dispatcher
|
||||||
|
consumers *nsqbus.ConsumerManager
|
||||||
|
producer *nsqbus.Producer
|
||||||
|
alerts *carrierqueue.AlertWriter
|
||||||
|
executor *task.CarrierCardExecutor
|
||||||
|
singleton bool
|
||||||
|
instanceID string
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
lastMode string
|
||||||
|
reinitQueued bool
|
||||||
|
legacyRestarts func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// newCarrierSchedulerParts 在 Worker 运行时初始化阶段创建调度参数、账户队列存储与模式接缝,
|
||||||
|
// 并注入旧调度相关组件(生命周期、轮询基类),使其在启动前即可按模式写入正确结构。
|
||||||
|
func newCarrierSchedulerParts(runtime *workerRuntime, appLogger *zap.Logger) {
|
||||||
|
settings := carrierqueue.NewDefaultSettingsProvider(runtime.db, runtime.redisClient, appLogger)
|
||||||
|
store := carrierqueue.NewStore(runtime.redisClient)
|
||||||
|
scheduling := polling.NewCarrierScheduling(store, settings, runtime.pollingConfigMgr, appLogger)
|
||||||
|
runtime.carrierSettings = settings
|
||||||
|
runtime.carrierStore = store
|
||||||
|
runtime.carrierScheduling = scheduling
|
||||||
|
runtime.lifecycleSvc.SetCarrierScheduling(scheduling)
|
||||||
|
runtime.pollingBase.SetSchedulerModeReader(func(ctx context.Context) string { return settings.Current(ctx).Mode })
|
||||||
|
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Classifier != nil {
|
||||||
|
classifier := runtime.gatewayLimit.Classifier
|
||||||
|
runtime.pollingBase.SetErrorClassifier(func(ctx context.Context, op, message string) string {
|
||||||
|
return classifier.Classify(ctx, op, message).Category
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startCarrierScheduler 启动模式监听;分发器只在 leader/all 运行(多实例安全),消费者在所有处理任务的实例运行。
|
||||||
|
func startCarrierScheduler(ctx context.Context, cfg *config.Config, runtime *workerRuntime, steps queue.PollingStepHandlers,
|
||||||
|
singleton bool, legacyRestart func(), appLogger *zap.Logger) *carrierSchedulerRuntime {
|
||||||
|
carrierCfg := cfg.Polling.Carrier
|
||||||
|
instanceID := cfg.Worker.InstanceName
|
||||||
|
if instanceID == "" {
|
||||||
|
instanceID, _ = os.Hostname()
|
||||||
|
}
|
||||||
|
instanceID += "-" + strconv.Itoa(os.Getpid())
|
||||||
|
alerts := carrierqueue.NewAlertWriter(runtime.db, runtime.redisClient, runtime.carrierSettings, appLogger)
|
||||||
|
alerts.Start(ctx)
|
||||||
|
executor := task.NewCarrierCardExecutor(runtime.pollingBase, runtime.carrierStore, runtime.carrierSettings, alerts,
|
||||||
|
steps.Realname, steps.Carddata, steps.CardStatus, steps.Package, carrierCfg, instanceID)
|
||||||
|
c := &carrierSchedulerRuntime{
|
||||||
|
cfg: carrierCfg, runtime: runtime, singleton: singleton, instanceID: instanceID, logger: appLogger,
|
||||||
|
alerts: alerts, executor: executor, legacyRestarts: legacyRestart,
|
||||||
|
initializer: polling.NewCarrierInitializer(runtime.carrierScheduling, runtime.pollingIotCardStore, appLogger),
|
||||||
|
}
|
||||||
|
if carrierCfg.NSQDAddress != "" {
|
||||||
|
producer, err := nsqbus.NewProducer(carrierCfg.NSQDAddress, appLogger)
|
||||||
|
if err != nil {
|
||||||
|
appLogger.Error("创建 NSQ 生产者失败,按运营商账户调度不可用", zap.Error(err))
|
||||||
|
} else {
|
||||||
|
c.producer = producer
|
||||||
|
}
|
||||||
|
c.consumers = nsqbus.NewConsumerManager(nsqbus.ConsumerConfig{
|
||||||
|
NSQDAddress: carrierCfg.NSQDAddress, NSQLookupdAddresses: nsqbus.SplitAddresses(carrierCfg.NSQLookupdAddresses),
|
||||||
|
MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts,
|
||||||
|
}, constants.NSQChannelWorker, executor.Handle, appLogger)
|
||||||
|
}
|
||||||
|
var budgets carrierqueue.BudgetProvider
|
||||||
|
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Limiter != nil {
|
||||||
|
budgets = runtime.gatewayLimit.Limiter
|
||||||
|
}
|
||||||
|
if c.producer != nil {
|
||||||
|
c.dispatcher = carrierqueue.NewDispatcher(runtime.db, runtime.carrierStore, runtime.carrierSettings, c.producer, budgets,
|
||||||
|
carrierCfg, instanceID, appLogger)
|
||||||
|
if singleton {
|
||||||
|
go c.dispatcher.Run(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go c.watch(ctx)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestReinit 轮询配置变更时请求按新配置重新初始化新结构(仅 carrier 模式生效)。
|
||||||
|
func (c *carrierSchedulerRuntime) RequestReinit() {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.reinitQueued = true
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *carrierSchedulerRuntime) watch(ctx context.Context) {
|
||||||
|
ticker := time.NewTicker(3 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
c.step(ctx)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
c.shutdown()
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *carrierSchedulerRuntime) step(ctx context.Context) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
c.logger.Error("按运营商账户调度模式监听 panic,已恢复", zap.Any("panic", r))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
mode := c.runtime.carrierSettings.Current(ctx).Mode
|
||||||
|
c.mu.Lock()
|
||||||
|
previous := c.lastMode
|
||||||
|
c.lastMode = mode
|
||||||
|
reinit := c.reinitQueued
|
||||||
|
c.reinitQueued = false
|
||||||
|
c.mu.Unlock()
|
||||||
|
if mode == constants.PollingSchedulerModeCarrier {
|
||||||
|
if previous != mode {
|
||||||
|
c.logger.Info("卡轮询调度模式:按运营商账户调度", zap.String("previous", previous))
|
||||||
|
}
|
||||||
|
if c.singleton {
|
||||||
|
c.ensureInitialized(ctx, reinit)
|
||||||
|
}
|
||||||
|
c.syncConsumers(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if previous == constants.PollingSchedulerModeCarrier {
|
||||||
|
c.logger.Info("卡轮询调度模式回到旧分片调度:停止领取与消费,重建旧分片队列")
|
||||||
|
if c.consumers != nil {
|
||||||
|
c.consumers.StopAll(c.grace())
|
||||||
|
c.consumers.Reset()
|
||||||
|
}
|
||||||
|
if c.singleton {
|
||||||
|
c.runtime.redisClient.Del(ctx, constants.RedisCarrierSchedulerInitStateKey)
|
||||||
|
if c.legacyRestarts != nil {
|
||||||
|
c.legacyRestarts()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureInitialized 切到 carrier 后初始化新结构;初始化完成前分发器不领取。多实例时用初始化锁协调。
|
||||||
|
func (c *carrierSchedulerRuntime) ensureInitialized(ctx context.Context, reinit bool) {
|
||||||
|
redisClient := c.runtime.redisClient
|
||||||
|
state, _ := redisClient.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||||
|
if state == carrierqueue.InitStateDone && !reinit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ok, err := redisClient.SetNX(ctx, carrierInitLockKey, c.instanceID, 30*time.Minute).Result()
|
||||||
|
if err != nil || !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer redisClient.Del(context.WithoutCancel(ctx), carrierInitLockKey)
|
||||||
|
if state != carrierqueue.InitStateDone {
|
||||||
|
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateRunning, 0)
|
||||||
|
}
|
||||||
|
started := time.Now()
|
||||||
|
written, initErr := c.initializer.Run(ctx)
|
||||||
|
if initErr != nil {
|
||||||
|
c.logger.Error("按运营商账户调度初始化失败,下一轮重试", zap.Error(initErr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateDone, 0)
|
||||||
|
c.logger.Info("按运营商账户调度初始化完成,开始领取", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)),
|
||||||
|
zap.String("metric", "polling.carrier.initialized"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncConsumers 按当前运营商账户订阅卡任务 topic,max_in_flight 取账户调度参数。
|
||||||
|
func (c *carrierSchedulerRuntime) syncConsumers(ctx context.Context) {
|
||||||
|
if c.consumers == nil || c.dispatcher == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
desired := map[string]int{}
|
||||||
|
for _, carrierID := range c.dispatcher.CarrierIDs(ctx) {
|
||||||
|
desired[constants.NSQCardTopic(carrierID)] = c.dispatcher.ParamsFor(carrierID).CardMaxInFlight
|
||||||
|
}
|
||||||
|
c.consumers.Sync(desired)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *carrierSchedulerRuntime) grace() time.Duration {
|
||||||
|
if c.cfg.ShutdownGrace > 0 {
|
||||||
|
return c.cfg.ShutdownGrace
|
||||||
|
}
|
||||||
|
return 30 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
// shutdown 优雅退出:停止订阅,宽限期内等待执行中的卡任务完成;未完成的由租约到期后重新领取。
|
||||||
|
func (c *carrierSchedulerRuntime) shutdown() {
|
||||||
|
if c.consumers != nil {
|
||||||
|
c.consumers.StopAll(c.grace())
|
||||||
|
}
|
||||||
|
if c.producer != nil {
|
||||||
|
c.producer.Stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitProgress 返回新结构初始化进度。
|
||||||
|
func (c *carrierSchedulerRuntime) InitProgress() polling.InitProgress {
|
||||||
|
return c.initializer.Progress()
|
||||||
|
}
|
||||||
@@ -45,6 +45,7 @@ import (
|
|||||||
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||||
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
||||||
refundSvc "github.com/break/junhong_cmp_fiber/internal/service/refund"
|
refundSvc "github.com/break/junhong_cmp_fiber/internal/service/refund"
|
||||||
@@ -102,6 +103,12 @@ type workerRuntime struct {
|
|||||||
assetAutoRenewalService *assetAutoRenewalApp.Service
|
assetAutoRenewalService *assetAutoRenewalApp.Service
|
||||||
// gatewayLimit 是卡网关运营商账户限流组件(限流、统计、停复机记录)。
|
// gatewayLimit 是卡网关运营商账户限流组件(限流、统计、停复机记录)。
|
||||||
gatewayLimit *gatewaylimit.Runtime
|
gatewayLimit *gatewaylimit.Runtime
|
||||||
|
// 按运营商账户调度:调度参数、账户队列存储与模式接缝(模式为 carrier 时各入口写新结构)。
|
||||||
|
carrierSettings *carrierqueue.SettingsProvider
|
||||||
|
carrierStore *carrierqueue.Store
|
||||||
|
carrierScheduling *polling.CarrierScheduling
|
||||||
|
// carrierReinit 在 carrier 模式下轮询配置变更时请求重新初始化新结构。
|
||||||
|
carrierReinit func()
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -171,6 +178,12 @@ func runWorker(cfg *config.Config) {
|
|||||||
|
|
||||||
taskHandler := createTaskHandler(runtime, appLogger)
|
taskHandler := createTaskHandler(runtime, appLogger)
|
||||||
taskHandler.RegisterHandlers()
|
taskHandler.RegisterHandlers()
|
||||||
|
carrierRuntime := startCarrierScheduler(ctx, cfg, runtime, taskHandler.PollingSteps(), runsSingletonModules(cfg.Worker.Role), func() {
|
||||||
|
if pollingInitializer != nil {
|
||||||
|
pollingInitializer.Restart(ctx)
|
||||||
|
}
|
||||||
|
}, appLogger)
|
||||||
|
runtime.carrierReinit = carrierRuntime.RequestReinit
|
||||||
registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
|
registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
|
||||||
registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
||||||
registerRefundChannelRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
registerRefundChannelRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
||||||
@@ -379,6 +392,7 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L
|
|||||||
// 自动续费的复机执行同样复用既有停复机单一事实源。
|
// 自动续费的复机执行同样复用既有停复机单一事实源。
|
||||||
assetAutoRenewalService: workerResult.Services.AssetAutoRenewal,
|
assetAutoRenewalService: workerResult.Services.AssetAutoRenewal,
|
||||||
}
|
}
|
||||||
|
newCarrierSchedulerParts(runtime, appLogger)
|
||||||
registerNotificationOutboxConsumer(runtime, appLogger)
|
registerNotificationOutboxConsumer(runtime, appLogger)
|
||||||
registerWalletOutboxConsumer(runtime, appLogger)
|
registerWalletOutboxConsumer(runtime, appLogger)
|
||||||
registerCardObservationOutboxConsumer(runtime, appLogger)
|
registerCardObservationOutboxConsumer(runtime, appLogger)
|
||||||
@@ -842,6 +856,13 @@ func startPollingInitializer(ctx context.Context, runtime *workerRuntime, appLog
|
|||||||
pollingInitializer.StartBackground(ctx)
|
pollingInitializer.StartBackground(ctx)
|
||||||
|
|
||||||
runtime.pollingConfigMgr.WatchChanges(ctx, func(hadConfigs, hasConfigs bool) {
|
runtime.pollingConfigMgr.WatchChanges(ctx, func(hadConfigs, hasConfigs bool) {
|
||||||
|
if runtime.carrierScheduling.Active(ctx) {
|
||||||
|
// 按运营商账户调度时按新配置重新初始化新结构,旧分片队列在回退时再重建。
|
||||||
|
if runtime.carrierReinit != nil {
|
||||||
|
runtime.carrierReinit()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
if hasConfigs {
|
if hasConfigs {
|
||||||
appLogger.Info("轮询配置已变更,触发队列重新初始化",
|
appLogger.Info("轮询配置已变更,触发队列重新初始化",
|
||||||
zap.Bool("had_configs", hadConfigs))
|
zap.Bool("had_configs", hadConfigs))
|
||||||
@@ -894,6 +915,7 @@ func startPollingScheduler(
|
|||||||
dataResetHandler,
|
dataResetHandler,
|
||||||
)
|
)
|
||||||
pollingScheduler.SetInitializer(pollingInitializer)
|
pollingScheduler.SetInitializer(pollingInitializer)
|
||||||
|
pollingScheduler.SetCarrierScheduling(runtime.carrierScheduling)
|
||||||
|
|
||||||
if err := pollingScheduler.Start(ctx); err != nil {
|
if err := pollingScheduler.Start(ctx); err != nil {
|
||||||
appLogger.Error("启动轮询调度器失败", zap.Error(err))
|
appLogger.Error("启动轮询调度器失败", zap.Error(err))
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import (
|
|||||||
walletinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wallet"
|
walletinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wallet"
|
||||||
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
assetAutoRenewalQuery "github.com/break/junhong_cmp_fiber/internal/query/assetautorenewal"
|
assetAutoRenewalQuery "github.com/break/junhong_cmp_fiber/internal/query/assetautorenewal"
|
||||||
accountSvc "github.com/break/junhong_cmp_fiber/internal/service/account"
|
accountSvc "github.com/break/junhong_cmp_fiber/internal/service/account"
|
||||||
agentOpenAPISvc "github.com/break/junhong_cmp_fiber/internal/service/agent_open_api"
|
agentOpenAPISvc "github.com/break/junhong_cmp_fiber/internal/service/agent_open_api"
|
||||||
@@ -234,6 +235,10 @@ func initServices(s *stores, deps *Dependencies) *services {
|
|||||||
pollingConfigMgr.Start(context.Background())
|
pollingConfigMgr.Start(context.Background())
|
||||||
pollingQueueMgr := polling.NewPollingQueueManager(deps.Redis, constants.PollingShardCount, deps.Logger)
|
pollingQueueMgr := polling.NewPollingQueueManager(deps.Redis, constants.PollingShardCount, deps.Logger)
|
||||||
pollingLifecycleSvc := polling.NewPollingLifecycleService(pollingQueueMgr, pollingConfigMgr, s.IotCard, s.DeviceSimBinding, s.Device, deps.Logger)
|
pollingLifecycleSvc := polling.NewPollingLifecycleService(pollingQueueMgr, pollingConfigMgr, s.IotCard, s.DeviceSimBinding, s.Device, deps.Logger)
|
||||||
|
// 按运营商账户调度:模式为 carrier 时 API 侧生命周期事件写入账户队列与卡到期信息。
|
||||||
|
carrierSettings := carrierqueue.NewDefaultSettingsProvider(deps.DB, deps.Redis, deps.Logger)
|
||||||
|
carrierStore := carrierqueue.NewStore(deps.Redis)
|
||||||
|
pollingLifecycleSvc.SetCarrierScheduling(polling.NewCarrierScheduling(carrierStore, carrierSettings, pollingConfigMgr, deps.Logger))
|
||||||
iotCard.SetPollingCallback(pollingLifecycleSvc)
|
iotCard.SetPollingCallback(pollingLifecycleSvc)
|
||||||
// 创建支付配置服务(Order 和 Recharge 依赖)
|
// 创建支付配置服务(Order 和 Recharge 依赖)
|
||||||
wechatConfig := wechatConfigSvc.New(s.WechatConfig, s.Order, s.RechargeOrder, s.AgentRecharge, s.Payment, auditWriter, deps.Redis, deps.Logger)
|
wechatConfig := wechatConfigSvc.New(s.WechatConfig, s.Order, s.RechargeOrder, s.AgentRecharge, s.Payment, auditWriter, deps.Redis, deps.Logger)
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ type CallGuard interface {
|
|||||||
type cardCallKey struct{}
|
type cardCallKey struct{}
|
||||||
type carrierKey struct{}
|
type carrierKey struct{}
|
||||||
type exemptKey struct{}
|
type exemptKey struct{}
|
||||||
|
type rateLimitWaitKey struct{}
|
||||||
|
|
||||||
type cardCall struct {
|
type cardCall struct {
|
||||||
operation string
|
operation string
|
||||||
@@ -113,6 +114,21 @@ func WithRateLimitExempt(ctx context.Context, source string) context.Context {
|
|||||||
return context.WithValue(ctx, exemptKey{}, source)
|
return context.WithValue(ctx, exemptKey{}, source)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithRateLimitWait 声明本次调用在拿不到限流名额时等待,总等待不超过 maxWait;超时返回限流错误。
|
||||||
|
// 只用于不占 Asynq 协程的执行方(按运营商账户调度的 NSQ 消费者);未声明时拿不到名额立即返回。
|
||||||
|
func WithRateLimitWait(ctx context.Context, maxWait time.Duration) context.Context {
|
||||||
|
if maxWait <= 0 {
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, rateLimitWaitKey{}, maxWait)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RateLimitWait 返回 ctx 中声明的限流最长等待;未声明返回 0。
|
||||||
|
func RateLimitWait(ctx context.Context) time.Duration {
|
||||||
|
wait, _ := ctx.Value(rateLimitWaitKey{}).(time.Duration)
|
||||||
|
return wait
|
||||||
|
}
|
||||||
|
|
||||||
// RateLimitExemptSource 返回 ctx 中声明的豁免来源,未声明返回空串。
|
// RateLimitExemptSource 返回 ctx 中声明的豁免来源,未声明返回空串。
|
||||||
func RateLimitExemptSource(ctx context.Context) string {
|
func RateLimitExemptSource(ctx context.Context) string {
|
||||||
source, _ := ctx.Value(exemptKey{}).(string)
|
source, _ := ctx.Value(exemptKey{}).(string)
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ const (
|
|||||||
statPrefixDuration = "duration_le:"
|
statPrefixDuration = "duration_le:"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
|
||||||
|
const StatFieldRateLimitWait = "rate_wait"
|
||||||
|
|
||||||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||||||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||||||
|
|
||||||
@@ -60,6 +63,18 @@ func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error
|
|||||||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||||||
}
|
}
|
||||||
if !decision.Allowed {
|
if !decision.Allowed {
|
||||||
|
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
|
||||||
|
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
|
||||||
|
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
|
||||||
|
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
|
||||||
|
if allowed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if waitErr != nil && ctx.Err() == nil {
|
||||||
|
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||||||
return gateway.NewRateLimitedError()
|
return gateway.NewRateLimitedError()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -189,3 +189,50 @@ func windowMember(nowMS int64) string {
|
|||||||
_, _ = rand.Read(buf)
|
_, _ = rand.Read(buf)
|
||||||
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Budget 返回运营商账户各查询类操作所属限流组本秒剩余名额,供分发器按预计调用数领取。
|
||||||
|
// groups 为限流组 → 本秒剩余;opGroup 为操作 → 限流组。只统计模式下返回 limited=false(不按 QPS 限制领取)。
|
||||||
|
// 全局组只在账户组之外再收紧:剩余名额取两者较小值。
|
||||||
|
func (l *Limiter) Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool) {
|
||||||
|
if l == nil || l.redis == nil {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
settings := l.settings.Current(ctx)
|
||||||
|
if !settings.Enforce() {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
groups, opGroup = map[string]int{}, map[string]string{}
|
||||||
|
now := time.Now().UnixMilli()
|
||||||
|
since := now - constants.GatewayRateLimitWindow.Milliseconds()
|
||||||
|
for _, op := range ops {
|
||||||
|
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
|
||||||
|
if !accountOK {
|
||||||
|
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
|
||||||
|
}
|
||||||
|
code := account.GroupCode
|
||||||
|
opGroup[op] = code
|
||||||
|
if _, seen := groups[code]; seen {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
limit := account.MaxQPS
|
||||||
|
if !account.HighPriority {
|
||||||
|
limit -= account.ReservedQPS
|
||||||
|
}
|
||||||
|
used, err := l.redis.ZCount(ctx, constants.RedisGatewayQPSKey(carrierID, code), strconv.FormatInt(since, 10), "+inf").Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
remaining := limit - int(used)
|
||||||
|
if globalOK {
|
||||||
|
globalUsed, gerr := l.redis.ZCount(ctx, constants.RedisGatewayGlobalQPSKey(global.GroupCode), strconv.FormatInt(since, 10), "+inf").Result()
|
||||||
|
if gerr == nil && global.MaxQPS-int(globalUsed) < remaining {
|
||||||
|
remaining = global.MaxQPS - int(globalUsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if remaining < 0 {
|
||||||
|
remaining = 0
|
||||||
|
}
|
||||||
|
groups[code] = remaining
|
||||||
|
}
|
||||||
|
return groups, opGroup, true
|
||||||
|
}
|
||||||
|
|||||||
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
@@ -0,0 +1,215 @@
|
|||||||
|
// Package nsqbus 封装卡轮询使用的 NSQ 生产者与按 topic 动态订阅的消费者管理器。
|
||||||
|
// NSQ 只承担执行分发与消费者并发控制,业务状态与重试全部在 Redis(add-carrier-polling-scheduler)。
|
||||||
|
package nsqbus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nsqio/go-nsq"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Producer 是 nsqd 生产者。
|
||||||
|
type Producer struct {
|
||||||
|
producer *nsq.Producer
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewProducer 连接 nsqd TCP 地址创建生产者。
|
||||||
|
func NewProducer(address string, logger *zap.Logger) (*Producer, error) {
|
||||||
|
cfg := nsq.NewConfig()
|
||||||
|
producer, err := nsq.NewProducer(address, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
producer.SetLogger(nsqLogger{logger: logger}, nsq.LogLevelWarning)
|
||||||
|
return &Producer{producer: producer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publish 同步发布一条消息;topic 不存在时由 nsqd 自动创建。
|
||||||
|
func (p *Producer) Publish(topic string, body []byte) error {
|
||||||
|
return p.producer.Publish(topic, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop 停止生产者。
|
||||||
|
func (p *Producer) Stop() {
|
||||||
|
if p != nil && p.producer != nil {
|
||||||
|
p.producer.Stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler 处理一条消息;返回后由管理器 FIN。消息处理期间可调用 touch 延长 NSQ 消息超时。
|
||||||
|
type Handler func(ctx context.Context, body []byte, touch func())
|
||||||
|
|
||||||
|
// ConsumerConfig 是消费者参数。
|
||||||
|
type ConsumerConfig struct {
|
||||||
|
NSQDAddress string
|
||||||
|
NSQLookupdAddresses []string
|
||||||
|
MsgTimeout time.Duration
|
||||||
|
MaxAttempts uint16
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerManager 按 topic 动态订阅与取消订阅;每个 topic 一个消费者,max_in_flight 控制本进程对该 topic 的并发。
|
||||||
|
type ConsumerManager struct {
|
||||||
|
cfg ConsumerConfig
|
||||||
|
channel string
|
||||||
|
handler Handler
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
consumers map[string]*subscription
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
inflight sync.WaitGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
type subscription struct {
|
||||||
|
consumer *nsq.Consumer
|
||||||
|
maxInFlight int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewConsumerManager 创建消费者管理器。
|
||||||
|
func NewConsumerManager(cfg ConsumerConfig, channel string, handler Handler, logger *zap.Logger) *ConsumerManager {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
return &ConsumerManager{cfg: cfg, channel: channel, handler: handler, logger: logger,
|
||||||
|
consumers: map[string]*subscription{}, ctx: ctx, cancel: cancel}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync 使订阅集合与 desired(topic → max_in_flight)一致:新增订阅、取消多余订阅、调整 max_in_flight。
|
||||||
|
func (m *ConsumerManager) Sync(desired map[string]int) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for topic, sub := range m.consumers {
|
||||||
|
if _, keep := desired[topic]; !keep {
|
||||||
|
sub.consumer.Stop()
|
||||||
|
delete(m.consumers, topic)
|
||||||
|
m.logger.Info("取消订阅卡任务 topic", zap.String("topic", topic))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for topic, maxInFlight := range desired {
|
||||||
|
if maxInFlight <= 0 {
|
||||||
|
maxInFlight = 1
|
||||||
|
}
|
||||||
|
if sub, ok := m.consumers[topic]; ok {
|
||||||
|
if sub.maxInFlight != maxInFlight {
|
||||||
|
sub.consumer.ChangeMaxInFlight(maxInFlight)
|
||||||
|
sub.maxInFlight = maxInFlight
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
consumer, err := m.subscribe(topic, maxInFlight)
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Warn("订阅卡任务 topic 失败,下一轮重试", zap.String("topic", topic), zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.consumers[topic] = &subscription{consumer: consumer, maxInFlight: maxInFlight}
|
||||||
|
m.logger.Info("订阅卡任务 topic", zap.String("topic", topic), zap.Int("max_in_flight", maxInFlight))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *ConsumerManager) subscribe(topic string, maxInFlight int) (*nsq.Consumer, error) {
|
||||||
|
cfg := nsq.NewConfig()
|
||||||
|
cfg.MaxInFlight = maxInFlight
|
||||||
|
if m.cfg.MsgTimeout > 0 {
|
||||||
|
cfg.MsgTimeout = m.cfg.MsgTimeout
|
||||||
|
}
|
||||||
|
if m.cfg.MaxAttempts > 0 {
|
||||||
|
cfg.MaxAttempts = m.cfg.MaxAttempts
|
||||||
|
}
|
||||||
|
consumer, err := nsq.NewConsumer(topic, m.channel, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
consumer.SetLogger(nsqLogger{logger: m.logger}, nsq.LogLevelWarning)
|
||||||
|
handlerCtx := m.ctx // subscribe 在持锁时调用,捕获当前处理上下文
|
||||||
|
consumer.AddConcurrentHandlers(nsq.HandlerFunc(func(message *nsq.Message) error {
|
||||||
|
message.DisableAutoResponse()
|
||||||
|
m.inflight.Add(1)
|
||||||
|
defer m.inflight.Done()
|
||||||
|
defer message.Finish()
|
||||||
|
m.handler(handlerCtx, message.Body, message.Touch)
|
||||||
|
return nil
|
||||||
|
}), maxInFlight)
|
||||||
|
if len(m.cfg.NSQLookupdAddresses) > 0 {
|
||||||
|
err = consumer.ConnectToNSQLookupds(m.cfg.NSQLookupdAddresses)
|
||||||
|
} else {
|
||||||
|
err = consumer.ConnectToNSQD(m.cfg.NSQDAddress)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
consumer.Stop()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return consumer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Topics 返回当前已订阅的 topic。
|
||||||
|
func (m *ConsumerManager) Topics() []string {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
topics := make([]string, 0, len(m.consumers))
|
||||||
|
for topic := range m.consumers {
|
||||||
|
topics = append(topics, topic)
|
||||||
|
}
|
||||||
|
return topics
|
||||||
|
}
|
||||||
|
|
||||||
|
// StopAll 停止全部订阅;在宽限期内等待执行中的消息完成并 FIN,超过宽限期后取消处理上下文。
|
||||||
|
func (m *ConsumerManager) StopAll(grace time.Duration) {
|
||||||
|
m.mu.Lock()
|
||||||
|
for topic, sub := range m.consumers {
|
||||||
|
sub.consumer.Stop()
|
||||||
|
delete(m.consumers, topic)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
m.inflight.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(grace):
|
||||||
|
m.logger.Warn("优雅退出宽限期内仍有卡任务未完成,取消执行(由租约到期后重新领取)")
|
||||||
|
}
|
||||||
|
m.cancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset 在 StopAll 后重新允许订阅(切回 carrier 模式时使用)。
|
||||||
|
func (m *ConsumerManager) Reset() {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.ctx.Err() != nil {
|
||||||
|
m.ctx, m.cancel = context.WithCancel(context.Background())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SplitAddresses 解析逗号分隔的地址列表。
|
||||||
|
func SplitAddresses(raw string) []string {
|
||||||
|
var result []string
|
||||||
|
for _, part := range strings.Split(raw, ",") {
|
||||||
|
if part = strings.TrimSpace(part); part != "" {
|
||||||
|
result = append(result, part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type nsqLogger struct {
|
||||||
|
logger *zap.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output 实现 go-nsq 的日志接口。
|
||||||
|
func (l nsqLogger) Output(_ int, s string) error {
|
||||||
|
if l.logger != nil {
|
||||||
|
l.logger.Warn("NSQ", zap.String("message", s))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
199
internal/polling/carrier_bridge.go
Normal file
199
internal/polling/carrier_bridge.go
Normal file
@@ -0,0 +1,199 @@
|
|||||||
|
package polling
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CarrierScheduling 是按运营商账户调度在轮询包内的接缝:判断当前模式,并按卡计算各检查类型到期时间写入账户队列。
|
||||||
|
// 所有写入轮询结构的入口(初始化、生命周期、优先提示、手动触发)按当前模式写新结构或旧分片结构。
|
||||||
|
type CarrierScheduling struct {
|
||||||
|
store *carrierqueue.Store
|
||||||
|
settings *carrierqueue.SettingsProvider
|
||||||
|
configMgr *PollingConfigManager
|
||||||
|
logger *zap.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierScheduling 创建按运营商账户调度接缝。
|
||||||
|
func NewCarrierScheduling(store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, configMgr *PollingConfigManager, logger *zap.Logger) *CarrierScheduling {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &CarrierScheduling{store: store, settings: settings, configMgr: configMgr, logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active 判断当前是否为按运营商账户调度。
|
||||||
|
func (c *CarrierScheduling) Active(ctx context.Context) bool {
|
||||||
|
return c != nil && c.store != nil && c.settings != nil && c.settings.Current(ctx).CarrierMode()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store 返回账户队列存储。
|
||||||
|
func (c *CarrierScheduling) Store() *carrierqueue.Store {
|
||||||
|
if c == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.store
|
||||||
|
}
|
||||||
|
|
||||||
|
// Settings 返回调度参数提供者。
|
||||||
|
func (c *CarrierScheduling) Settings() *carrierqueue.SettingsProvider {
|
||||||
|
if c == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.settings
|
||||||
|
}
|
||||||
|
|
||||||
|
// DuesForCard 按卡匹配的轮询配置计算各检查类型下次到期时间。
|
||||||
|
// fromLastCheck 为 true 时按各类型上次检查时间计算(初始化);否则按当前时间加抖动(新卡、恢复轮询、状态变化)。
|
||||||
|
// 间隔为 0 或未配置的类型不返回;重置日前加密窗口规则算出的时间更早时以它为准。
|
||||||
|
func (c *CarrierScheduling) DuesForCard(ctx context.Context, card *model.IotCard, fromLastCheck bool) map[string]time.Time {
|
||||||
|
dues := map[string]time.Time{}
|
||||||
|
if c == nil || c.configMgr == nil || card == nil {
|
||||||
|
return dues
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
for taskType, info := range c.configMgr.MergedTaskIntervals(card) {
|
||||||
|
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||||
|
if checkType == "" || info.Interval <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var due time.Time
|
||||||
|
if fromLastCheck {
|
||||||
|
due = calculateNextCheckTime(lastCheckAtByTaskType(card, taskType), info.Interval, now)
|
||||||
|
if windowDue := c.configMgr.NextCheckAt(ctx, card, taskType, info, now); windowDue.Before(due) && windowDue.After(now) {
|
||||||
|
due = windowDue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
due = calcInitialDelay(info.Interval)
|
||||||
|
}
|
||||||
|
dues[checkType] = due
|
||||||
|
}
|
||||||
|
return dues
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertCard 把卡写入所属账户队列;fromLastCheck 含义同 DuesForCard。
|
||||||
|
func (c *CarrierScheduling) UpsertCard(ctx context.Context, card *model.IotCard, fromLastCheck bool) {
|
||||||
|
if c == nil || c.store == nil || card == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := c.store.Upsert(ctx, card.ID, card.CarrierID, c.DuesForCard(ctx, card, fromLastCheck)); err != nil {
|
||||||
|
c.logger.Warn("写入运营商账户轮询队列失败", zap.Uint("card_id", card.ID), zap.Uint("carrier_id", card.CarrierID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveCard 把卡移出所属账户队列;carrierID 为 0 时按到期信息中记录的账户移出。
|
||||||
|
func (c *CarrierScheduling) RemoveCard(ctx context.Context, cardID, carrierID uint) {
|
||||||
|
if c == nil || c.store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if carrierID == 0 {
|
||||||
|
due, err := c.store.ReadDue(ctx, cardID)
|
||||||
|
if err != nil || due.CarrierID == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
carrierID = due.CarrierID
|
||||||
|
}
|
||||||
|
if err := c.store.Remove(ctx, cardID, carrierID); err != nil {
|
||||||
|
c.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestTaskType 把卡的某检查类型(旧任务类型 polling:xxx)设为立即到期放入账户优先队列(优先提示、手动触发)。
|
||||||
|
// 卡不在轮询中时不处理。
|
||||||
|
func (c *CarrierScheduling) RequestTaskType(ctx context.Context, cardID, carrierID uint, taskType string) {
|
||||||
|
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||||
|
if c == nil || c.store == nil || checkType == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := c.store.RequestCheck(ctx, carrierqueue.CheckRequest{CardID: cardID, CarrierID: carrierID, CheckTypes: []string{checkType}}); err != nil {
|
||||||
|
c.logger.Warn("写入运营商账户优先队列失败", zap.Uint("card_id", cardID), zap.String("task_type", taskType), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierInitializer 从数据库按批初始化全部运营商账户队列与卡到期信息;进度与旧初始化器同结构,供 init-progress 展示。
|
||||||
|
type CarrierInitializer struct {
|
||||||
|
scheduling *CarrierScheduling
|
||||||
|
iotCardStore *postgres.IotCardStore
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
progress InitProgress
|
||||||
|
running atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierInitializer 创建新结构初始化器。
|
||||||
|
func NewCarrierInitializer(scheduling *CarrierScheduling, iotCardStore *postgres.IotCardStore, logger *zap.Logger) *CarrierInitializer {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &CarrierInitializer{scheduling: scheduling, iotCardStore: iotCardStore, logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress 返回初始化进度快照。
|
||||||
|
func (i *CarrierInitializer) Progress() InitProgress {
|
||||||
|
i.mu.RLock()
|
||||||
|
defer i.mu.RUnlock()
|
||||||
|
return i.progress
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run 同步执行一次全量初始化:按批读取启用轮询的卡,计算各类型到期并写入账户队列。返回写入的卡数。
|
||||||
|
func (i *CarrierInitializer) Run(ctx context.Context) (int64, error) {
|
||||||
|
if !i.running.CompareAndSwap(false, true) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
defer i.running.Store(false)
|
||||||
|
started := time.Now()
|
||||||
|
total, err := i.iotCardStore.CountForPolling(ctx)
|
||||||
|
if err != nil {
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", err.Error() })
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
i.setProgress(func(p *InitProgress) {
|
||||||
|
*p = InitProgress{TotalCards: total, StartTime: started, Status: "running"}
|
||||||
|
})
|
||||||
|
const batchSize = 2000
|
||||||
|
var lastID uint
|
||||||
|
var written int64
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", err.Error() })
|
||||||
|
return written, err
|
||||||
|
}
|
||||||
|
cards, fetchErr := i.iotCardStore.ListForPollingBatch(ctx, lastID, batchSize)
|
||||||
|
if fetchErr != nil {
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", fetchErr.Error() })
|
||||||
|
return written, fetchErr
|
||||||
|
}
|
||||||
|
if len(cards) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if tier := i.scheduling.configMgr.TierResolver(); tier != nil {
|
||||||
|
tier.Prefill(ctx, cards)
|
||||||
|
}
|
||||||
|
for _, card := range cards {
|
||||||
|
i.scheduling.UpsertCard(ctx, card, true)
|
||||||
|
written++
|
||||||
|
}
|
||||||
|
lastID = cards[len(cards)-1].ID
|
||||||
|
loaded := written
|
||||||
|
i.setProgress(func(p *InitProgress) { p.LoadedCards, p.LastBatchTime = loaded, time.Now() })
|
||||||
|
}
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status = "completed" })
|
||||||
|
i.logger.Info("运营商账户轮询结构初始化完成", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)))
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *CarrierInitializer) setProgress(update func(*InitProgress)) {
|
||||||
|
i.mu.Lock()
|
||||||
|
update(&i.progress)
|
||||||
|
i.mu.Unlock()
|
||||||
|
}
|
||||||
177
internal/polling/carrierqueue/alerts.go
Normal file
177
internal/polling/carrierqueue/alerts.go
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
package carrierqueue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FailureEvent 是一次需要告警的检查失败。
|
||||||
|
type FailureEvent struct {
|
||||||
|
CardID uint
|
||||||
|
ICCID string
|
||||||
|
CarrierID uint
|
||||||
|
CheckType string
|
||||||
|
Category string
|
||||||
|
Message string
|
||||||
|
FailCount int
|
||||||
|
FirstFailed time.Time
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type alertOp struct {
|
||||||
|
failure *FailureEvent
|
||||||
|
recovered bool
|
||||||
|
cardID uint
|
||||||
|
checkType string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AlertWriter 异步写卡轮询失败告警:同一卡同一检查类型只保留一条未处理告警,后续失败更新计数;
|
||||||
|
// 首次成功转为已恢复;同账户窗口内新增告警超阈值时产生账户级告警。写入失败只记日志,不影响轮询。
|
||||||
|
type AlertWriter struct {
|
||||||
|
db *gorm.DB
|
||||||
|
redis *redis.Client
|
||||||
|
settings *SettingsProvider
|
||||||
|
logger *zap.Logger
|
||||||
|
ch chan alertOp
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAlertWriter 创建告警写入器;调用 Start 后开始消费。
|
||||||
|
func NewAlertWriter(db *gorm.DB, redisClient *redis.Client, settings *SettingsProvider, logger *zap.Logger) *AlertWriter {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &AlertWriter{db: db, redis: redisClient, settings: settings, logger: logger, ch: make(chan alertOp, 2048)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start 启动后台写入协程,直到 ctx 结束。
|
||||||
|
func (w *AlertWriter) Start(ctx context.Context) {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case op := <-w.ch:
|
||||||
|
w.apply(op)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReportFailure 提交一条失败告警(达到阈值后才调用)。
|
||||||
|
func (w *AlertWriter) ReportFailure(event FailureEvent) {
|
||||||
|
w.offer(alertOp{failure: &event})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReportRecovered 提交一次恢复:该卡该检查类型的未处理告警转为已恢复。
|
||||||
|
func (w *AlertWriter) ReportRecovered(cardID uint, checkType string) {
|
||||||
|
w.offer(alertOp{recovered: true, cardID: cardID, checkType: checkType})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *AlertWriter) offer(op alertOp) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case w.ch <- op:
|
||||||
|
default:
|
||||||
|
w.logger.Warn("卡轮询失败告警队列已满,丢弃本次写入", zap.String("metric", "polling.failure_alert.dropped"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *AlertWriter) apply(op alertOp) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if op.recovered {
|
||||||
|
err := w.db.WithContext(ctx).Exec(`UPDATE tb_polling_failure_alert SET status = ?, recovered_at = NOW(), updated_at = NOW()
|
||||||
|
WHERE alert_level = ? AND card_id = ? AND check_type = ? AND status = ?`,
|
||||||
|
constants.PollingAlertStatusRecovered, constants.PollingAlertLevelCard, op.cardID, op.checkType, constants.PollingAlertStatusOpen).Error
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("卡轮询告警转为已恢复失败", zap.Uint("card_id", op.cardID), zap.String("check_type", op.checkType), zap.Error(err))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
event := op.failure
|
||||||
|
var inserted bool
|
||||||
|
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||||
|
(alert_level, card_id, iccid, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||||
|
DO UPDATE SET fail_count = EXCLUDED.fail_count, error_category = EXCLUDED.error_category,
|
||||||
|
last_failed_at = EXCLUDED.last_failed_at, last_error = EXCLUDED.last_error, updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0)`,
|
||||||
|
constants.PollingAlertLevelCard, event.CardID, event.ICCID, event.CarrierID, event.CheckType, event.Category, event.FailCount,
|
||||||
|
nullableTime(event.FirstFailed), event.At, truncate(event.Message, 500), constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("写卡轮询失败告警失败", zap.Uint("card_id", event.CardID), zap.String("check_type", event.CheckType), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !inserted {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.logger.Error("卡轮询持续失败告警", zap.Uint("card_id", event.CardID), zap.String("iccid", event.ICCID),
|
||||||
|
zap.Uint("carrier_id", event.CarrierID), zap.String("check_type", event.CheckType), zap.String("category", event.Category),
|
||||||
|
zap.Int("fail_count", event.FailCount), zap.String("metric", "polling.failure_alert.opened"))
|
||||||
|
w.checkAccountAlert(ctx, event.CarrierID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkAccountAlert 同账户窗口内新增卡级告警数超过阈值时,产生(或更新)一条账户级告警。
|
||||||
|
func (w *AlertWriter) checkAccountAlert(ctx context.Context, carrierID uint) {
|
||||||
|
settings := w.settings.Current(ctx)
|
||||||
|
window, limit := settings.AccountAlertWindow, settings.AccountAlertLimit
|
||||||
|
if window <= 0 || limit <= 0 || w.redis == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
key := constants.RedisCarrierAccountAlertKey(carrierID)
|
||||||
|
member := strconv.FormatInt(now.UnixNano(), 36)
|
||||||
|
pipe := w.redis.TxPipeline()
|
||||||
|
pipe.ZAdd(ctx, key, redis.Z{Score: float64(now.UnixMilli()), Member: member})
|
||||||
|
pipe.ZRemRangeByScore(ctx, key, "-inf", strconv.FormatInt(now.Add(-window).UnixMilli(), 10))
|
||||||
|
count := pipe.ZCard(ctx, key)
|
||||||
|
pipe.Expire(ctx, key, window*2)
|
||||||
|
if _, err := pipe.Exec(ctx); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if int(count.Val()) <= limit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var inserted bool
|
||||||
|
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||||
|
(alert_level, card_id, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||||
|
VALUES (?, 0, ?, ?, '', ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||||
|
DO UPDATE SET fail_count = EXCLUDED.fail_count, last_failed_at = EXCLUDED.last_failed_at, updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0)`,
|
||||||
|
constants.PollingAlertLevelCarrier, carrierID, constants.PollingAlertCheckTypeAll, count.Val(), now, now,
|
||||||
|
"窗口内同账户新增失败告警数超过阈值,通常是账户额度、凭证或上游故障,请联系上游处理账户", constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("写账户级轮询告警失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if inserted {
|
||||||
|
w.logger.Error("运营商账户轮询失败告警数超过阈值", zap.Uint("carrier_id", carrierID), zap.Int64("count", count.Val()),
|
||||||
|
zap.Duration("window", window), zap.String("metric", "polling.failure_alert.carrier_opened"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func nullableTime(t time.Time) any {
|
||||||
|
if t.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncate(s string, max int) string {
|
||||||
|
runes := []rune(s)
|
||||||
|
if len(runes) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return string(runes[:max])
|
||||||
|
}
|
||||||
326
internal/polling/carrierqueue/dispatcher.go
Normal file
326
internal/polling/carrierqueue/dispatcher.go
Normal file
@@ -0,0 +1,326 @@
|
|||||||
|
package carrierqueue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
stderrors "errors"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CardMessage 是卡任务 NSQ 消息体。
|
||||||
|
type CardMessage struct {
|
||||||
|
CardID uint `json:"card_id"`
|
||||||
|
CarrierID uint `json:"carrier_id"`
|
||||||
|
Token string `json:"token"`
|
||||||
|
DispatchedAt int64 `json:"dispatched_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publisher 发布 NSQ 消息。
|
||||||
|
type Publisher interface {
|
||||||
|
Publish(topic string, body []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// BudgetProvider 返回账户各查询操作所属限流组本秒剩余名额;limited=false 表示不按 QPS 限制领取。
|
||||||
|
type BudgetProvider interface {
|
||||||
|
Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierParams 是一个运营商账户生效的调度参数(已合并全局默认)。
|
||||||
|
type CarrierParams struct {
|
||||||
|
InflightLimit int
|
||||||
|
CardMaxInFlight int
|
||||||
|
PriorityRatio float64
|
||||||
|
Configured bool // 是否有该账户的配置行
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierEntry 是分发器缓存的账户信息。
|
||||||
|
type carrierEntry struct {
|
||||||
|
id uint
|
||||||
|
realnameControl bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dispatcher 每个分发间隔对每个运营商账户抢分发锁、领取到期卡并发布到账户 topic。多实例安全。
|
||||||
|
type Dispatcher struct {
|
||||||
|
db *gorm.DB
|
||||||
|
store *Store
|
||||||
|
settings *SettingsProvider
|
||||||
|
publisher Publisher
|
||||||
|
budgets BudgetProvider
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
instanceID string
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
carriers []carrierEntry
|
||||||
|
params map[uint]model.CarrierPollingParam
|
||||||
|
loadedAt time.Time
|
||||||
|
tightened map[uint]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDispatcher 创建分发器。
|
||||||
|
func NewDispatcher(db *gorm.DB, store *Store, settings *SettingsProvider, publisher Publisher, budgets BudgetProvider,
|
||||||
|
cfg config.CarrierSchedulerConfig, instanceID string, logger *zap.Logger) *Dispatcher {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &Dispatcher{db: db, store: store, settings: settings, publisher: publisher, budgets: budgets, cfg: cfg,
|
||||||
|
instanceID: instanceID, logger: logger, params: map[uint]model.CarrierPollingParam{}, tightened: map[uint]time.Time{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run 按分发间隔循环,直到 ctx 结束。
|
||||||
|
func (d *Dispatcher) Run(ctx context.Context) {
|
||||||
|
interval := d.cfg.DispatchInterval
|
||||||
|
if interval <= 0 {
|
||||||
|
interval = time.Second
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
d.tick(ctx, interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ready 判断新调度是否可领取:模式为 carrier 且新结构初始化已完成。
|
||||||
|
func (d *Dispatcher) Ready(ctx context.Context) bool {
|
||||||
|
if !d.settings.Current(ctx).CarrierMode() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
state, err := d.store.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||||
|
return err == nil && state == InitStateDone
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
d.logger.Error("分发器 panic,已恢复", zap.Any("panic", r))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if !d.Ready(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.store.redis.HSet(ctx, constants.RedisCarrierSchedulerHeartbeatKey, d.instanceID, time.Now().UnixMilli())
|
||||||
|
d.drainLegacyHints(ctx)
|
||||||
|
d.refreshCarriers(ctx)
|
||||||
|
d.mu.Lock()
|
||||||
|
carriers := append([]carrierEntry(nil), d.carriers...)
|
||||||
|
d.mu.Unlock()
|
||||||
|
lockTTL := interval * 9 / 10
|
||||||
|
if lockTTL <= 0 {
|
||||||
|
lockTTL = 900 * time.Millisecond
|
||||||
|
}
|
||||||
|
for _, carrier := range carriers {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ok, err := d.store.redis.SetNX(ctx, constants.RedisCarrierDispatchLockKey(carrier.id), d.instanceID, lockTTL).Result()
|
||||||
|
if err != nil || !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d.dispatchCarrier(ctx, carrier)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) dispatchCarrier(ctx context.Context, carrier carrierEntry) {
|
||||||
|
params := d.ParamsFor(carrier.id)
|
||||||
|
limit := params.InflightLimit
|
||||||
|
// 执行上限不得大于订阅进程 max_in_flight 之和,否则领出的消息在 NSQ 中排队到租约过期;不满足时收紧本轮上限并上报。
|
||||||
|
processes := d.cfg.SubscriberProcesses
|
||||||
|
if processes <= 0 {
|
||||||
|
processes = 1
|
||||||
|
}
|
||||||
|
if capacity := params.CardMaxInFlight * processes; capacity > 0 && limit > capacity {
|
||||||
|
limit = capacity
|
||||||
|
d.reportTightened(carrier.id, params.InflightLimit, capacity)
|
||||||
|
}
|
||||||
|
input := ClaimInput{
|
||||||
|
CarrierID: carrier.id, Lease: d.lease(), InflightLimit: limit, PriorityRatio: params.PriorityRatio,
|
||||||
|
TokenPrefix: d.instanceID + ":" + strconv.FormatInt(time.Now().UnixNano(), 36),
|
||||||
|
CountRealname: carrier.realnameControl,
|
||||||
|
}
|
||||||
|
if d.budgets != nil {
|
||||||
|
ops := []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname}
|
||||||
|
if groups, opGroup, limited := d.budgets.Budget(ctx, carrier.id, ops); limited {
|
||||||
|
input.GroupBudget = groups
|
||||||
|
input.TypeGroup = map[string]string{
|
||||||
|
constants.CheckTypeCardStatus: opGroup[constants.GatewayOpQueryStatus],
|
||||||
|
constants.CheckTypeCarddata: opGroup[constants.GatewayOpQueryFlow],
|
||||||
|
constants.CheckTypeRealname: opGroup[constants.GatewayOpQueryRealname],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
claimed, expired, err := d.store.Claim(ctx, input)
|
||||||
|
if err != nil {
|
||||||
|
d.logger.Warn("运营商账户领取失败", zap.Uint("carrier_id", carrier.id), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if expired > 0 {
|
||||||
|
minute := strconv.FormatInt(time.Now().Unix()/60, 10)
|
||||||
|
key := constants.RedisCarrierExpiredClaimsKey(carrier.id)
|
||||||
|
d.store.redis.HIncrBy(ctx, key, minute, expired)
|
||||||
|
d.store.redis.Expire(ctx, key, 2*time.Hour)
|
||||||
|
d.logger.Warn("运营商账户处理中标记到期(执行崩溃或超时)", zap.Uint("carrier_id", carrier.id), zap.Int64("count", expired),
|
||||||
|
zap.String("metric", "polling.carrier.claim_expired"))
|
||||||
|
}
|
||||||
|
topic := constants.NSQCardTopic(carrier.id)
|
||||||
|
for _, item := range claimed {
|
||||||
|
body, _ := json.Marshal(CardMessage{CardID: item.CardID, CarrierID: carrier.id, Token: item.Token, DispatchedAt: time.Now().UnixMilli()})
|
||||||
|
if err := d.publisher.Publish(topic, body); err != nil {
|
||||||
|
// 发布失败:用独立短超时上下文按 token 删除处理中标记并恢复 score;恢复也失败则等租约到期后自然重新领取。
|
||||||
|
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||||
|
if releaseErr := d.store.Release(releaseCtx, item.CardID, carrier.id, item.Token); releaseErr != nil {
|
||||||
|
d.logger.Warn("发布失败后释放处理中标记失败", zap.Uint("card_id", item.CardID), zap.Error(releaseErr))
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
d.logger.Warn("发布卡任务失败,已释放", zap.Uint("card_id", item.CardID), zap.String("topic", topic), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) lease() time.Duration {
|
||||||
|
if d.cfg.Lease > 0 {
|
||||||
|
return d.cfg.Lease
|
||||||
|
}
|
||||||
|
return 5 * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) reportTightened(carrierID uint, configured, capacity int) {
|
||||||
|
d.mu.Lock()
|
||||||
|
last := d.tightened[carrierID]
|
||||||
|
if time.Since(last) < 10*time.Minute {
|
||||||
|
d.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.tightened[carrierID] = time.Now()
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.logger.Warn("账户执行上限大于订阅进程 max_in_flight 之和,已按消费能力收紧本轮领取上限",
|
||||||
|
zap.Uint("carrier_id", carrierID), zap.Int("inflight_limit", configured), zap.Int("capacity", capacity),
|
||||||
|
zap.String("metric", "polling.carrier.inflight_tightened"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshCarriers 每 10 秒重新读取运营商账户与调度参数。
|
||||||
|
func (d *Dispatcher) refreshCarriers(ctx context.Context) {
|
||||||
|
d.mu.Lock()
|
||||||
|
fresh := time.Since(d.loadedAt) < 10*time.Second
|
||||||
|
d.mu.Unlock()
|
||||||
|
if fresh {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var carriers []model.Carrier
|
||||||
|
if err := d.db.WithContext(ctx).Select("id", "realname_control_enabled").Order("id").Find(&carriers).Error; err != nil {
|
||||||
|
d.logger.Warn("读取运营商账户失败,沿用上一次列表", zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var rows []model.CarrierPollingParam
|
||||||
|
if err := d.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||||
|
d.logger.Warn("读取运营商账户调度参数失败,沿用上一次参数", zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
entries := make([]carrierEntry, 0, len(carriers))
|
||||||
|
for _, carrier := range carriers {
|
||||||
|
entries = append(entries, carrierEntry{id: carrier.ID, realnameControl: carrier.RealnameControlEnabled == 1})
|
||||||
|
}
|
||||||
|
params := make(map[uint]model.CarrierPollingParam, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
params[row.CarrierID] = row
|
||||||
|
}
|
||||||
|
d.mu.Lock()
|
||||||
|
d.carriers, d.params, d.loadedAt = entries, params, time.Now()
|
||||||
|
d.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierIDs 返回当前缓存的运营商账户 ID(消费者据此订阅)。
|
||||||
|
func (d *Dispatcher) CarrierIDs(ctx context.Context) []uint {
|
||||||
|
d.refreshCarriers(ctx)
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
ids := make([]uint, 0, len(d.carriers))
|
||||||
|
for _, carrier := range d.carriers {
|
||||||
|
ids = append(ids, carrier.id)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParamsFor 返回账户生效调度参数(未配置项用全局默认)。
|
||||||
|
func (d *Dispatcher) ParamsFor(carrierID uint) CarrierParams {
|
||||||
|
d.mu.Lock()
|
||||||
|
row, ok := d.params[carrierID]
|
||||||
|
d.mu.Unlock()
|
||||||
|
return MergeParams(row, ok, d.cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeParams 合并账户配置行与全局默认。
|
||||||
|
func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.CarrierSchedulerConfig) CarrierParams {
|
||||||
|
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured}
|
||||||
|
if params.InflightLimit <= 0 {
|
||||||
|
params.InflightLimit = 20
|
||||||
|
}
|
||||||
|
if params.CardMaxInFlight <= 0 {
|
||||||
|
params.CardMaxInFlight = 20
|
||||||
|
}
|
||||||
|
if !configured {
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
if row.InflightLimit != nil && *row.InflightLimit > 0 {
|
||||||
|
params.InflightLimit = *row.InflightLimit
|
||||||
|
}
|
||||||
|
if row.CardMaxInFlight != nil && *row.CardMaxInFlight > 0 {
|
||||||
|
params.CardMaxInFlight = *row.CardMaxInFlight
|
||||||
|
}
|
||||||
|
if row.PriorityMaxRatio != nil && *row.PriorityMaxRatio > 0 {
|
||||||
|
params.PriorityRatio = *row.PriorityMaxRatio
|
||||||
|
}
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
|
||||||
|
// drainLegacyHints 消费旧优先提示通道与手动触发队列:优先提示放入账户优先队列,手动触发在普通队列中立即到期。
|
||||||
|
func (d *Dispatcher) drainLegacyHints(ctx context.Context) {
|
||||||
|
for _, checkType := range constants.AllCheckTypes() {
|
||||||
|
taskType := constants.TaskTypeFromCheckType(checkType)
|
||||||
|
d.drainList(ctx, constants.RedisPollingPriorityQueueKey(taskType), checkType, false)
|
||||||
|
d.drainList(ctx, constants.RedisPollingManualQueueKey(taskType), checkType, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) drainList(ctx context.Context, key, checkType string, normal bool) {
|
||||||
|
for i := 0; i < 500; i++ {
|
||||||
|
raw, err := d.store.redis.LPop(ctx, key).Result()
|
||||||
|
if err != nil {
|
||||||
|
if !stderrors.Is(err, redis.Nil) {
|
||||||
|
d.logger.Warn("读取旧提示队列失败", zap.String("key", key), zap.Error(err))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cardID, convErr := strconv.ParseUint(raw, 10, 64)
|
||||||
|
if convErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
due, dueErr := d.store.ReadDue(ctx, uint(cardID))
|
||||||
|
if dueErr != nil || due.CarrierID == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := d.store.RequestCheck(ctx, CheckRequest{CardID: uint(cardID), CarrierID: due.CarrierID, CheckTypes: []string{checkType}, Normal: normal}); err != nil {
|
||||||
|
d.logger.Warn("写入账户队列失败", zap.Uint64("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 新结构初始化状态(constants.RedisCarrierSchedulerInitStateKey)。
|
||||||
|
const (
|
||||||
|
InitStateRunning = "running"
|
||||||
|
InitStateDone = "done"
|
||||||
|
)
|
||||||
@@ -182,7 +182,8 @@ return 1
|
|||||||
// 只作用于仍在轮询中的卡(在优先或普通队列中)。冷却期内(最近成功距今未超过冷却)的类型不请求。
|
// 只作用于仍在轮询中的卡(在优先或普通队列中)。冷却期内(最近成功距今未超过冷却)的类型不请求。
|
||||||
//
|
//
|
||||||
// KEYS[1] due;KEYS[2] 优先队列;KEYS[3] 普通队列;KEYS[4] claim。
|
// KEYS[1] due;KEYS[2] 优先队列;KEYS[3] 普通队列;KEYS[4] claim。
|
||||||
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、类型…。
|
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、
|
||||||
|
// 是否放入优先队列(1/0;0 用于手动触发:只把到期设为现在并在普通队列中立即到期)、类型…。
|
||||||
// 返回本次实际请求的类型数;卡不在轮询中返回 -1。
|
// 返回本次实际请求的类型数;卡不在轮询中返回 -1。
|
||||||
const requestCheckLua = `
|
const requestCheckLua = `
|
||||||
local now = tonumber(ARGV[1])
|
local now = tonumber(ARGV[1])
|
||||||
@@ -192,7 +193,8 @@ local inPriority = redis.call('ZSCORE', KEYS[2], cardId)
|
|||||||
local inQueue = redis.call('ZSCORE', KEYS[3], cardId)
|
local inQueue = redis.call('ZSCORE', KEYS[3], cardId)
|
||||||
if not inPriority and not inQueue then return -1 end
|
if not inPriority and not inQueue then return -1 end
|
||||||
local requested = 0
|
local requested = 0
|
||||||
for i = 6, #ARGV do
|
local toPriority = ARGV[6] == '1'
|
||||||
|
for i = 7, #ARGV do
|
||||||
local t = ARGV[i]
|
local t = ARGV[i]
|
||||||
local skip = false
|
local skip = false
|
||||||
if cooldown > 0 then
|
if cooldown > 0 then
|
||||||
@@ -209,6 +211,11 @@ for i = 6, #ARGV do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
if requested == 0 then return 0 end
|
if requested == 0 then return 0 end
|
||||||
|
if not toPriority then
|
||||||
|
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
||||||
|
if inQueue then redis.call('ZADD', KEYS[3], now, cardId) else redis.call('ZADD', KEYS[2], now, cardId) end
|
||||||
|
return requested
|
||||||
|
end
|
||||||
redis.call('HSET', KEYS[1], 'prio', '1')
|
redis.call('HSET', KEYS[1], 'prio', '1')
|
||||||
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
||||||
redis.call('ZREM', KEYS[3], cardId)
|
redis.call('ZREM', KEYS[3], cardId)
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
@@ -376,3 +378,15 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
|||||||
}
|
}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewDefaultSettingsProvider 用独立注册表与读取器创建调度参数提供者(API 与 Worker 共用同一份数据库与缓存数据)。
|
||||||
|
func NewDefaultSettingsProvider(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *SettingsProvider {
|
||||||
|
registry := systemconfig.NewRegistry()
|
||||||
|
RegisterConfigDefinitions(registry, logger)
|
||||||
|
var cache systemconfig.Cache
|
||||||
|
if redisClient != nil {
|
||||||
|
cache = systemconfig.NewRedisCache(redisClient)
|
||||||
|
}
|
||||||
|
reader := systemconfig.NewReader(db, registry, cache, systemconfig.NewLogAlertSink(logger))
|
||||||
|
return NewSettingsProvider(reader, logger)
|
||||||
|
}
|
||||||
|
|||||||
@@ -136,6 +136,8 @@ type CheckRequest struct {
|
|||||||
Cooldown time.Duration
|
Cooldown time.Duration
|
||||||
ConfirmDelays []time.Duration
|
ConfirmDelays []time.Duration
|
||||||
Expected string // 延迟确认的期望结果(card_status 为 online / offline),满足时取消后续确认
|
Expected string // 延迟确认的期望结果(card_status 为 online / offline),满足时取消后续确认
|
||||||
|
// Normal 为 true 时不放入优先队列:只把到期设为现在、在普通队列中立即到期(手动触发走普通通道)。
|
||||||
|
Normal bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// RequestCheck 把卡的指定检查类型设为立即到期放入优先队列;返回实际请求的类型数,卡不在轮询中返回 -1。
|
// RequestCheck 把卡的指定检查类型设为立即到期放入优先队列;返回实际请求的类型数,卡不在轮询中返回 -1。
|
||||||
@@ -147,7 +149,11 @@ func (s *Store) RequestCheck(ctx context.Context, request CheckRequest) (int64,
|
|||||||
for _, delay := range request.ConfirmDelays {
|
for _, delay := range request.ConfirmDelays {
|
||||||
confirm = append(confirm, strconv.FormatInt(int64(delay/time.Second), 10))
|
confirm = append(confirm, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||||
}
|
}
|
||||||
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected}
|
toPriority := "1"
|
||||||
|
if request.Normal {
|
||||||
|
toPriority = "0"
|
||||||
|
}
|
||||||
|
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected, toPriority}
|
||||||
for _, checkType := range request.CheckTypes {
|
for _, checkType := range request.CheckTypes {
|
||||||
args = append(args, checkType)
|
args = append(args, checkType)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,18 @@ type PollingLifecycleService struct {
|
|||||||
deviceBindingStore *postgres.DeviceSimBindingStore
|
deviceBindingStore *postgres.DeviceSimBindingStore
|
||||||
deviceStore *postgres.DeviceStore
|
deviceStore *postgres.DeviceStore
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
|
// carrier 按运营商账户调度接缝;当前模式为 carrier 时生命周期事件写入账户队列,不写旧分片队列。
|
||||||
|
carrier *CarrierScheduling
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||||
|
func (s *PollingLifecycleService) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||||
|
s.carrier = carrier
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierActive 判断当前是否按运营商账户调度。
|
||||||
|
func (s *PollingLifecycleService) carrierActive(ctx context.Context) bool {
|
||||||
|
return s.carrier.Active(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPollingLifecycleService 创建卡生命周期轮询管理服务
|
// NewPollingLifecycleService 创建卡生命周期轮询管理服务
|
||||||
@@ -52,6 +64,10 @@ func (s *PollingLifecycleService) OnCardCreated(ctx context.Context, card *model
|
|||||||
s.logger.Debug("卡禁用轮询,跳过入队", zap.Uint("card_id", card.ID))
|
s.logger.Debug("卡禁用轮询,跳过入队", zap.Uint("card_id", card.ID))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.carrier.UpsertCard(ctx, card, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
s.enqueueCard(ctx, card)
|
s.enqueueCard(ctx, card)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,6 +80,11 @@ func (s *PollingLifecycleService) OnBatchCardsCreated(ctx context.Context, cards
|
|||||||
|
|
||||||
// OnCardStatusChanged 卡状态变化后清理缓存、重新匹配配置并更新队列
|
// OnCardStatusChanged 卡状态变化后清理缓存、重新匹配配置并更新队列
|
||||||
func (s *PollingLifecycleService) OnCardStatusChanged(ctx context.Context, cardID uint) {
|
func (s *PollingLifecycleService) OnCardStatusChanged(ctx context.Context, cardID uint) {
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
s.carrierRefresh(ctx, cardID)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
||||||
s.logger.Warn("卡状态变化:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
s.logger.Warn("卡状态变化:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
}
|
}
|
||||||
@@ -87,6 +108,13 @@ func (s *PollingLifecycleService) OnBatchCardsStatusChanged(ctx context.Context,
|
|||||||
if len(cardIDs) == 0 {
|
if len(cardIDs) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
for _, cardID := range cardIDs {
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
s.carrierRefresh(ctx, cardID)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
// 单次 pipeline:批量从当前分片队列移除并清理缓存
|
// 单次 pipeline:批量从当前分片队列移除并清理缓存
|
||||||
if err := s.queueMgr.RemoveBatchFromCurrentShardQueues(ctx, cardIDs); err != nil {
|
if err := s.queueMgr.RemoveBatchFromCurrentShardQueues(ctx, cardIDs); err != nil {
|
||||||
s.logger.Warn("批量卡状态变化:从队列移除失败", zap.Int("count", len(cardIDs)), zap.Error(err))
|
s.logger.Warn("批量卡状态变化:从队列移除失败", zap.Int("count", len(cardIDs)), zap.Error(err))
|
||||||
@@ -107,6 +135,11 @@ func (s *PollingLifecycleService) OnBatchCardsStatusChanged(ctx context.Context,
|
|||||||
|
|
||||||
// OnCardDeleted 卡删除后移除所有队列并清理缓存
|
// OnCardDeleted 卡删除后移除所有队列并清理缓存
|
||||||
func (s *PollingLifecycleService) OnCardDeleted(ctx context.Context, cardID uint) {
|
func (s *PollingLifecycleService) OnCardDeleted(ctx context.Context, cardID uint) {
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.carrier.RemoveCard(ctx, cardID, 0)
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.queueMgr.OnCardDeleted(ctx, cardID); err != nil {
|
if err := s.queueMgr.OnCardDeleted(ctx, cardID); err != nil {
|
||||||
s.logger.Warn("卡删除:清理队列和缓存失败", zap.Uint("card_id", cardID), zap.Error(err))
|
s.logger.Warn("卡删除:清理队列和缓存失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
}
|
}
|
||||||
@@ -123,11 +156,21 @@ func (s *PollingLifecycleService) OnCardEnabled(ctx context.Context, cardID uint
|
|||||||
if !s.ShouldEnqueue(ctx, card) {
|
if !s.ShouldEnqueue(ctx, card) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
// 恢复轮询时各检查类型到期设为当前时间加抖动。
|
||||||
|
s.carrier.UpsertCard(ctx, card, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
s.enqueueCard(ctx, card)
|
s.enqueueCard(ctx, card)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OnCardDisabled 卡禁用轮询后移除所有队列
|
// OnCardDisabled 卡禁用轮询后移除所有队列
|
||||||
func (s *PollingLifecycleService) OnCardDisabled(ctx context.Context, cardID uint) {
|
func (s *PollingLifecycleService) OnCardDisabled(ctx context.Context, cardID uint) {
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.carrier.RemoveCard(ctx, cardID, 0)
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
||||||
s.logger.Warn("卡禁用:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
s.logger.Warn("卡禁用:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
}
|
}
|
||||||
@@ -183,3 +226,17 @@ func calcInitialDelay(interval int) time.Time {
|
|||||||
}
|
}
|
||||||
return now.Add(time.Duration(rand.Intn(jitterMax)) * time.Second)
|
return now.Add(time.Duration(rand.Intn(jitterMax)) * time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// carrierRefresh 按运营商账户调度时重新匹配配置并写入账户队列:不再轮询的卡移出,其余按当前时间加抖动重排。
|
||||||
|
func (s *PollingLifecycleService) carrierRefresh(ctx context.Context, cardID uint) {
|
||||||
|
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("卡状态变化:加载卡信息失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.ShouldEnqueue(ctx, card) {
|
||||||
|
s.carrier.RemoveCard(ctx, card.ID, card.CarrierID)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.carrier.UpsertCard(ctx, card, false)
|
||||||
|
}
|
||||||
|
|||||||
@@ -30,6 +30,8 @@ type Scheduler struct {
|
|||||||
packageActivationHandler *PackageActivationHandler
|
packageActivationHandler *PackageActivationHandler
|
||||||
dataResetHandler *DataResetHandler
|
dataResetHandler *DataResetHandler
|
||||||
initializer *PollingInitializer // 可选,nil 表示不守卫 Init 完成
|
initializer *PollingInitializer // 可选,nil 表示不守卫 Init 完成
|
||||||
|
// carrier 按运营商账户调度接缝:当前模式为 carrier 时旧分片调度停止出队(套餐激活检查照常执行)。
|
||||||
|
carrier *CarrierScheduling
|
||||||
|
|
||||||
stopChan chan struct{}
|
stopChan chan struct{}
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
@@ -156,6 +158,10 @@ func (s *Scheduler) scheduleLoop(ctx context.Context) {
|
|||||||
// processShardSchedule 处理手动队列和分片定时队列(每 1 秒触发)
|
// processShardSchedule 处理手动队列和分片定时队列(每 1 秒触发)
|
||||||
// 使用 90% 的 tick 间隔作为超时,确保单分片 Redis 挂起时不阻塞下一个 tick
|
// 使用 90% 的 tick 间隔作为超时,确保单分片 Redis 挂起时不阻塞下一个 tick
|
||||||
func (s *Scheduler) processShardSchedule(ctx context.Context) {
|
func (s *Scheduler) processShardSchedule(ctx context.Context) {
|
||||||
|
if s.carrier.Active(ctx) {
|
||||||
|
// 已切到按运营商账户调度:旧分片调度停止出队,优先提示与手动触发由新分发器消费。
|
||||||
|
return
|
||||||
|
}
|
||||||
// 优先提示与手动队列都不受 Init 影响(ConfigManager 已就绪即可)。
|
// 优先提示与手动队列都不受 Init 影响(ConfigManager 已就绪即可)。
|
||||||
// 同一调度周期内先排空优先提示、再排空手动触发队列:优先的强度边界仅为调度入口领先,
|
// 同一调度周期内先排空优先提示、再排空手动触发队列:优先的强度边界仅为调度入口领先,
|
||||||
// 提示排空不参与分片背压判断,因此分片积压不会抑制加急入队。
|
// 提示排空不参与分片背压判断,因此分片积压不会抑制加急入队。
|
||||||
@@ -296,3 +302,8 @@ func (s *Scheduler) enqueueBatch(ctx context.Context, taskType string, cardIDs [
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||||
|
func (s *Scheduler) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||||
|
s.carrier = carrier
|
||||||
|
}
|
||||||
|
|||||||
299
internal/task/carrier_card_executor.go
Normal file
299
internal/task/carrier_card_executor.go
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
package task
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CarrierCardExecutor 按运营商账户调度的卡任务执行器:一张卡一个卡任务,按配置步骤顺序串行执行已到期的检查类型。
|
||||||
|
type CarrierCardExecutor struct {
|
||||||
|
base *PollingBase
|
||||||
|
store *carrierqueue.Store
|
||||||
|
settings *carrierqueue.SettingsProvider
|
||||||
|
alerts *carrierqueue.AlertWriter
|
||||||
|
realname *PollingRealnameHandler
|
||||||
|
carddata *PollingCarddataHandler
|
||||||
|
cardStatus *PollingCardStatusHandler
|
||||||
|
pkg *PollingPackageHandler
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
instanceID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierCardExecutor 创建卡任务执行器。
|
||||||
|
func NewCarrierCardExecutor(base *PollingBase, store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, alerts *carrierqueue.AlertWriter,
|
||||||
|
realname *PollingRealnameHandler, carddata *PollingCarddataHandler, cardStatus *PollingCardStatusHandler, pkg *PollingPackageHandler,
|
||||||
|
cfg config.CarrierSchedulerConfig, instanceID string) *CarrierCardExecutor {
|
||||||
|
return &CarrierCardExecutor{base: base, store: store, settings: settings, alerts: alerts, realname: realname, carddata: carddata,
|
||||||
|
cardStatus: cardStatus, pkg: pkg, cfg: cfg, instanceID: instanceID}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CarrierCardExecutor) renewInterval() time.Duration {
|
||||||
|
if e.cfg.RenewInterval > 0 {
|
||||||
|
return e.cfg.RenewInterval
|
||||||
|
}
|
||||||
|
return 30 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CarrierCardExecutor) lease() time.Duration {
|
||||||
|
if e.cfg.Lease > 0 {
|
||||||
|
return e.cfg.Lease
|
||||||
|
}
|
||||||
|
return 5 * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle 消费一条卡任务消息;返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
|
||||||
|
func (e *CarrierCardExecutor) Handle(ctx context.Context, body []byte, touch func()) {
|
||||||
|
var msg carrierqueue.CardMessage
|
||||||
|
if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
|
||||||
|
e.base.logger.Warn("卡任务消息无效,丢弃", zap.ByteString("body", body))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
execValue := msg.Token + "|" + e.instanceID
|
||||||
|
execTTL := e.renewInterval() * 2
|
||||||
|
got, err := e.store.AcquireExec(ctx, msg.CardID, msg.Token, execValue, execTTL)
|
||||||
|
if err != nil || got != 1 {
|
||||||
|
// 0:处理中标记已不是该 token(过期或重复投递);-1:该 token 已有执行者(NSQ 重投时原消费者仍在执行)。
|
||||||
|
return
|
||||||
|
}
|
||||||
|
settings := e.settings.Current(ctx)
|
||||||
|
taskCtx, cancel := context.WithTimeout(ctx, settings.TaskTimeout())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var lost atomic.Bool
|
||||||
|
renewDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(e.renewInterval())
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-renewDone:
|
||||||
|
return
|
||||||
|
case <-taskCtx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
ok, renewErr := e.store.Renew(context.WithoutCancel(ctx), msg.CardID, msg.CarrierID, msg.Token, execValue, e.lease(), execTTL)
|
||||||
|
if renewErr == nil && !ok {
|
||||||
|
// token 已不一致:立即停止后续步骤,不再发起新的网关调用,只写已产生的业务结果。
|
||||||
|
lost.Store(true)
|
||||||
|
cancel()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if touch != nil {
|
||||||
|
touch()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
set, del := e.execute(taskCtx, msg, settings, &lost)
|
||||||
|
close(renewDone)
|
||||||
|
|
||||||
|
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||||
|
defer finishCancel()
|
||||||
|
finished, finishErr := e.store.Finish(finishCtx, msg.CardID, msg.CarrierID, msg.Token, set, del)
|
||||||
|
if finishErr != nil {
|
||||||
|
e.base.logger.Warn("卡任务收尾失败,等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(finishErr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !finished {
|
||||||
|
e.base.logger.Warn("卡任务收尾时处理中标记已不一致,只写入业务结果", zap.Uint("card_id", msg.CardID),
|
||||||
|
zap.String("metric", "polling.carrier.finish_token_mismatch"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// execute 串行执行到期检查步骤,返回收尾要写入与删除的到期字段。
|
||||||
|
func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.CardMessage, settings carrierqueue.Settings, lost *atomic.Bool) (map[string]string, []string) {
|
||||||
|
set, del := map[string]string{}, []string{}
|
||||||
|
card, err := e.base.getCardWithCache(ctx, msg.CardID)
|
||||||
|
if err != nil {
|
||||||
|
if isNotFound(err) {
|
||||||
|
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||||
|
}
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
if !card.EnablePolling || (e.base.scopeChecker != nil && !e.base.scopeChecker.ShouldEnqueue(ctx, card)) {
|
||||||
|
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
due, err := e.store.ReadDue(ctx, msg.CardID)
|
||||||
|
if err != nil {
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
flowEvaluated := false
|
||||||
|
executed := map[string]StepOutcome{}
|
||||||
|
for _, checkType := range settings.StepOrder {
|
||||||
|
item, ok := due.Types[checkType]
|
||||||
|
if !ok || !item.HasDue || item.Due.After(now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if lost.Load() || ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow && flowEvaluated {
|
||||||
|
executed[checkType] = stepOK()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
outcome := e.runStep(ctx, msg, checkType, settings.StepTimeout(checkType))
|
||||||
|
if checkType == constants.CheckTypeCarddata && outcome.TriggeredEvaluation {
|
||||||
|
flowEvaluated = true
|
||||||
|
}
|
||||||
|
executed[checkType] = outcome
|
||||||
|
if outcome.Result == StepCardGone || outcome.Result == StepStopPolling {
|
||||||
|
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(executed) == 0 {
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
// 收尾时按最新卡数据重新匹配间隔(步骤中卡状态、套餐可能已变化)。
|
||||||
|
e.base.invalidateCardCache(ctx, msg.CardID)
|
||||||
|
if fresh, freshErr := e.base.getCardWithCache(ctx, msg.CardID); freshErr == nil {
|
||||||
|
card = fresh
|
||||||
|
}
|
||||||
|
intervals := e.base.configMgr.MergedTaskIntervals(card)
|
||||||
|
now = time.Now()
|
||||||
|
for checkType, outcome := range executed {
|
||||||
|
e.writeBack(ctx, card, checkType, outcome, due.Types[checkType], intervals, settings, now, set, &del)
|
||||||
|
}
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
|
||||||
|
// runStep 以单步超时、等待模式限流与 panic 恢复执行一个检查步骤。
|
||||||
|
func (e *CarrierCardExecutor) runStep(ctx context.Context, msg carrierqueue.CardMessage, checkType string, timeout time.Duration) (outcome StepOutcome) {
|
||||||
|
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
stepCtx = gateway.WithRateLimitWait(stepCtx, timeout)
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
e.base.logger.Error("卡任务步骤 panic,按失败处理", zap.Uint("card_id", msg.CardID), zap.String("check_type", checkType), zap.Any("panic", r))
|
||||||
|
outcome = StepOutcome{Result: StepFailed, Category: "panic", Message: fmt.Sprint(r)}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
switch checkType {
|
||||||
|
case constants.CheckTypeCardStatus:
|
||||||
|
return e.cardStatus.Step(stepCtx, msg.CardID)
|
||||||
|
case constants.CheckTypeCarddata:
|
||||||
|
return e.carddata.Step(stepCtx, msg.CardID)
|
||||||
|
case constants.CheckTypeRealname:
|
||||||
|
return e.realname.Step(stepCtx, msg.CardID)
|
||||||
|
case constants.CheckTypePackage:
|
||||||
|
stepCtx = withPollingWorkerAuditContext(stepCtx, constants.TaskTypePollingPackage, "套餐状态轮询任务", msg.Token)
|
||||||
|
return e.pkg.Step(stepCtx, msg.CardID)
|
||||||
|
default:
|
||||||
|
return stepSkip()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeBack 按步骤结果计算该检查类型下次到期、失败计数与延迟确认,并触发告警或恢复。
|
||||||
|
func (e *CarrierCardExecutor) writeBack(ctx context.Context, card *model.IotCard, checkType string, outcome StepOutcome, previous carrierqueue.TypeDue,
|
||||||
|
intervals map[string]polling.TaskTypeInterval, settings carrierqueue.Settings, now time.Time, set map[string]string, del *[]string) {
|
||||||
|
info, ok := intervals[constants.TaskTypeFromCheckType(checkType)]
|
||||||
|
if !ok || info.Interval <= 0 {
|
||||||
|
// 该类型已不在配置中(间隔为 0 或未配置):移除到期,不再执行。
|
||||||
|
*del = append(*del, carrierqueue.FieldDue(checkType), carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType),
|
||||||
|
carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
interval := time.Duration(info.Interval) * time.Second
|
||||||
|
normalNext := e.base.configMgr.NextCheckAt(ctx, card, constants.TaskTypeFromCheckType(checkType), info, now)
|
||||||
|
switch outcome.Result {
|
||||||
|
case StepSucceeded:
|
||||||
|
next := normalNext
|
||||||
|
set[carrierqueue.FieldOK(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||||
|
if previous.FailCount > 0 {
|
||||||
|
e.alerts.ReportRecovered(card.ID, checkType)
|
||||||
|
}
|
||||||
|
*del = append(*del, carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType), carrierqueue.FieldError(checkType))
|
||||||
|
// 延迟确认:结果已满足预期时取消,否则按序列取下一次确认 = min(正常间隔, 当前 + 确认延迟)。
|
||||||
|
if len(previous.Confirm) > 0 {
|
||||||
|
if previous.Expected != "" && outcome.Observed == previous.Expected {
|
||||||
|
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||||
|
} else {
|
||||||
|
if confirmAt := now.Add(previous.Confirm[0]); confirmAt.Before(next) {
|
||||||
|
next = confirmAt
|
||||||
|
}
|
||||||
|
if rest := previous.Confirm[1:]; len(rest) > 0 {
|
||||||
|
set[carrierqueue.FieldConfirm(checkType)] = joinSeconds(rest)
|
||||||
|
} else {
|
||||||
|
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(next.UnixMilli(), 10)
|
||||||
|
case StepFailed:
|
||||||
|
set[carrierqueue.FieldError(checkType)] = truncateRunes(outcome.Category+"|"+outcome.Message, 300)
|
||||||
|
if !outcome.CountsAsFailure() {
|
||||||
|
// 上游频率限制、已是目标状态:不计入连续失败,按首个退避间隔重试。
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, 1, interval)).UnixMilli(), 10)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
failCount := previous.FailCount + 1
|
||||||
|
firstFailed := previous.FirstFailed
|
||||||
|
if firstFailed.IsZero() {
|
||||||
|
firstFailed = now
|
||||||
|
set[carrierqueue.FieldFirstFail(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldFail(checkType)] = strconv.Itoa(failCount)
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, failCount, interval)).UnixMilli(), 10)
|
||||||
|
if failCount >= settings.AlertThreshold(checkType) || now.Sub(firstFailed) >= settings.AlertFailDuration || settings.ImmediateAlert[outcome.Category] {
|
||||||
|
e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, CheckType: checkType,
|
||||||
|
Category: outcome.Category, Message: outcome.Message, FailCount: failCount, FirstFailed: firstFailed, At: now})
|
||||||
|
}
|
||||||
|
case StepRateLimited:
|
||||||
|
// 限流等待超时:请求未发出,按短延迟重排,不计失败。
|
||||||
|
delay := 5 * time.Second
|
||||||
|
if e.base.rateLimitedDelay != nil {
|
||||||
|
if configured := e.base.rateLimitedDelay(); configured > 0 {
|
||||||
|
delay = configured
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||||
|
case StepDeferred:
|
||||||
|
delay := outcome.DeferDelay
|
||||||
|
if delay <= 0 {
|
||||||
|
delay = 30 * time.Second
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||||
|
default:
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(normalNext.UnixMilli(), 10)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CarrierCardExecutor) removeCard(ctx context.Context, cardID, carrierID uint) {
|
||||||
|
removeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := e.store.Remove(removeCtx, cardID, carrierID); err != nil {
|
||||||
|
e.base.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinSeconds(delays []time.Duration) string {
|
||||||
|
parts := make([]string, 0, len(delays))
|
||||||
|
for _, delay := range delays {
|
||||||
|
parts = append(parts, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateRunes(s string, max int) string {
|
||||||
|
runes := []rune(s)
|
||||||
|
if len(runes) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return string(runes[:max])
|
||||||
|
}
|
||||||
@@ -71,6 +71,8 @@ type PollingBase struct {
|
|||||||
priorityAudit *audit.Writer
|
priorityAudit *audit.Writer
|
||||||
scopeChecker *polling.PollingLifecycleService
|
scopeChecker *polling.PollingLifecycleService
|
||||||
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
||||||
|
errorClassifier ErrorClassifier // 上游错误分类(按运营商账户调度的失败退避与告警使用)
|
||||||
|
schedulerMode SchedulerModeReader // 当前调度模式;carrier 时旧 Asynq 轮询任务直接结束
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
||||||
|
|||||||
@@ -30,34 +30,42 @@ func NewPollingCarddataHandler(base *PollingBase, gw *gateway.Client, carrier *p
|
|||||||
return &PollingCarddataHandler{base: base, gateway: gw, carrier: carrier, observation: observation, integration: integration}
|
return &PollingCarddataHandler{base: base, gateway: gw, carrier: carrier, observation: observation, integration: integration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理流量检查任务。
|
// Handle 处理流量检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||||
startedAt := time.Now()
|
|
||||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCarddata) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCarddata) {
|
||||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCarddata)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCarddata)
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCarddata, h.Step(ctx, cardID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次流量检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startedAt := time.Now()
|
||||||
lockToken, locked, err := h.base.acquireCardTrafficSyncLock(ctx, cardID)
|
lockToken, locked, err := h.base.acquireCardTrafficSyncLock(ctx, cardID)
|
||||||
if err != nil || !locked {
|
if err != nil || !locked {
|
||||||
h.base.logger.Warn("卡流量同步锁不可用,延迟重入队", zap.Uint("card_id", cardID), zap.Error(err))
|
h.base.logger.Warn("卡流量同步锁不可用,延迟重入队", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
return h.base.queueMgr.Requeue(ctx, cardID, constants.TaskTypePollingCarddata, time.Now().Add(5*time.Second))
|
return stepDefer(5 * time.Second)
|
||||||
}
|
}
|
||||||
defer h.base.releaseCardTrafficSyncLock(ctx, cardID, lockToken)
|
defer h.base.releaseCardTrafficSyncLock(ctx, cardID, lockToken)
|
||||||
|
|
||||||
// 优先轮询互斥接缝:放在流量同步锁获取成功之后,避免认领后拿不到锁导致优先项空占。
|
// 优先轮询互斥接缝:放在流量同步锁获取成功之后,避免认领后拿不到锁导致优先项空占。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCarddata)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCarddata))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCarddata))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
h.base.invalidateCardCache(ctx, cardID)
|
h.base.invalidateCardCache(ctx, cardID)
|
||||||
@@ -65,42 +73,42 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFound(err) {
|
if isNotFound(err) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
if h.gateway == nil {
|
if h.gateway == nil {
|
||||||
priority.FailFinal(ctx, "流量查询能力未配置")
|
priority.FailFinal(ctx, "流量查询能力未配置")
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayTraffic, constants.CardObservationSceneTrafficPolling)
|
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayTraffic, constants.CardObservationSceneTrafficPolling)
|
||||||
result, err := h.gateway.QueryFlow(gateway.WithCarrier(ctx, card.CarrierID), &gateway.FlowQueryReq{CardNo: card.ICCID})
|
result, err := h.gateway.QueryFlow(gateway.WithCarrier(ctx, card.CarrierID), &gateway.FlowQueryReq{CardNo: card.ICCID})
|
||||||
if gateway.IsRateLimited(err) {
|
if gateway.IsRateLimited(err) {
|
||||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCarddata)
|
return StepOutcome{Result: StepRateLimited}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询流量失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询流量失败", priority)
|
||||||
priority.FailRetryable(ctx, "查询流量失败")
|
priority.FailRetryable(ctx, "查询流量失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询流量失败", err)
|
return h.fail(ctx, cardID, startedAt, "查询流量失败", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(result.ICCID) == "" {
|
if strings.TrimSpace(result.ICCID) == "" {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "流量查询响应缺少 ICCID", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "流量查询响应缺少 ICCID", priority)
|
||||||
priority.FailRetryable(ctx, "流量查询响应缺少 ICCID")
|
priority.FailRetryable(ctx, "流量查询响应缺少 ICCID")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
return h.fail(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
||||||
}
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
||||||
if h.observation == nil || h.carrier == nil {
|
if h.observation == nil || h.carrier == nil {
|
||||||
priority.FailFinal(ctx, "卡流量观测能力未配置")
|
priority.FailFinal(ctx, "卡流量观测能力未配置")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
return h.fail(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||||
@@ -113,7 +121,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用流量观测失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用流量观测失败", priority)
|
||||||
priority.FailRetryable(ctx, "应用流量观测失败")
|
priority.FailRetryable(ctx, "应用流量观测失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用流量观测失败", err)
|
return h.fail(ctx, cardID, startedAt, "应用流量观测失败", err)
|
||||||
}
|
}
|
||||||
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
||||||
changed := decision.IncrementMB > 0 || decision.CrossMonth
|
changed := decision.IncrementMB > 0 || decision.CrossMonth
|
||||||
@@ -132,15 +140,15 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
h.base.logger.Info("流量轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
h.base.logger.Info("流量轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||||
priority.Complete(ctx)
|
priority.Complete(ctx)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return StepOutcome{Result: StepSucceeded, TriggeredEvaluation: changed}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *PollingCarddataHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
func (h *PollingCarddataHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fields = append(fields, zap.Error(err))
|
fields = append(fields, zap.Error(err))
|
||||||
}
|
}
|
||||||
h.base.logger.Warn(message, fields...)
|
h.base.logger.Warn(message, fields...)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, false, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, false, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return h.base.stepFail(ctx, constants.GatewayOpQueryFlow, message, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,70 +37,85 @@ func NewPollingCardStatusHandler(base *PollingBase, gw *gateway.Client, observat
|
|||||||
return &PollingCardStatusHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
return &PollingCardStatusHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理卡状态轮询任务。
|
// Handle 处理卡状态轮询任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||||
startedAt := time.Now()
|
|
||||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCardStatus) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCardStatus) {
|
||||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCardStatus)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCardStatus)
|
||||||
|
outcome := h.Step(ctx, cardID)
|
||||||
|
if outcome.Result == StepSucceeded {
|
||||||
|
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
||||||
|
h.base.invalidateCardCache(ctx, cardID)
|
||||||
|
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
||||||
|
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCardStatus, outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次卡状态检查步骤,返回结果分类与观测到的网络状态;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingCardStatusHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startedAt := time.Now()
|
||||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCardStatus)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCardStatus))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCardStatus))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFound(err) {
|
if isNotFound(err) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
if h.gateway == nil {
|
if h.gateway == nil {
|
||||||
priority.FailFinal(ctx, "卡状态查询能力未配置")
|
priority.FailFinal(ctx, "卡状态查询能力未配置")
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayNetwork, constants.CardObservationSceneNetworkPolling)
|
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayNetwork, constants.CardObservationSceneNetworkPolling)
|
||||||
result, err := h.gateway.QueryCardStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
result, err := h.gateway.QueryCardStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||||
if gateway.IsRateLimited(err) {
|
if gateway.IsRateLimited(err) {
|
||||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCardStatus)
|
return StepOutcome{Result: StepRateLimited}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询卡状态失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询卡状态失败", priority)
|
||||||
priority.FailRetryable(ctx, "查询卡状态失败")
|
priority.FailRetryable(ctx, "查询卡状态失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询卡状态失败", err)
|
return h.fail(ctx, cardID, startedAt, "查询卡状态失败", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(result.ICCID) == "" {
|
if strings.TrimSpace(result.ICCID) == "" {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "卡状态查询响应缺少 ICCID", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "卡状态查询响应缺少 ICCID", priority)
|
||||||
priority.FailRetryable(ctx, "卡状态查询响应缺少 ICCID")
|
priority.FailRetryable(ctx, "卡状态查询响应缺少 ICCID")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
return h.fail(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
||||||
}
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
||||||
if h.observation == nil {
|
if h.observation == nil {
|
||||||
priority.FailFinal(ctx, "卡网络观测能力未配置")
|
priority.FailFinal(ctx, "卡网络观测能力未配置")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
return h.fail(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
||||||
@@ -113,7 +128,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用网络状态观测失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用网络状态观测失败", priority)
|
||||||
priority.FailRetryable(ctx, "应用网络状态观测失败")
|
priority.FailRetryable(ctx, "应用网络状态观测失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
return h.fail(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
||||||
}
|
}
|
||||||
changed := decision.StatusChanged || decision.StopReasonChanged || decision.StopPolling
|
changed := decision.StatusChanged || decision.StopReasonChanged || decision.StopPolling
|
||||||
if changed {
|
if changed {
|
||||||
@@ -134,23 +149,24 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
|||||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
||||||
if decision.StopPolling {
|
if decision.StopPolling {
|
||||||
h.base.logger.Info("卡命中风险停机或已销户,已关闭该卡轮询", zap.Uint("card_id", cardID), zap.String("gateway_extend", decision.GatewayExtend))
|
h.base.logger.Info("卡命中风险停机或已销户,已关闭该卡轮询", zap.Uint("card_id", cardID), zap.String("gateway_extend", decision.GatewayExtend))
|
||||||
return nil
|
return StepOutcome{Result: StepStopPolling}
|
||||||
|
}
|
||||||
|
outcome := stepOK()
|
||||||
|
if decision.StatusKnown {
|
||||||
|
outcome.Observed = ObservedOffline
|
||||||
|
if decision.AfterStatus == constants.NetworkStatusOnline {
|
||||||
|
outcome.Observed = ObservedOnline
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
func (h *PollingCardStatusHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||||
h.base.invalidateCardCache(ctx, cardID)
|
|
||||||
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
|
||||||
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
|
||||||
}
|
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *PollingCardStatusHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
|
||||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fields = append(fields, zap.Error(err))
|
fields = append(fields, zap.Error(err))
|
||||||
}
|
}
|
||||||
h.base.logger.Warn(message, fields...)
|
h.base.logger.Warn(message, fields...)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return h.base.stepFail(ctx, constants.GatewayOpQueryStatus, message, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,14 +37,15 @@ func NewPollingPackageHandler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理套餐检查任务
|
// Handle 处理套餐检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error {
|
func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error {
|
||||||
startTime := time.Now()
|
|
||||||
|
|
||||||
cardID, ok := parseTaskPayload(t.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(t.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
||||||
|
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
||||||
@@ -52,42 +53,48 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
|||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingPackage)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingPackage)
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingPackage, h.Step(ctx, cardID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次套餐检查步骤(按规则重新判断停复机),返回结果分类;不负责重入队。
|
||||||
|
// 调用方负责在 ctx 中设置 Worker 审计上下文。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingPackageHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startTime := time.Now()
|
||||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingPackage)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingPackage)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
h.base.logger.Error("查询活动优先轮询项失败", zap.Uint("card_id", cardID), zap.Error(priorityErr))
|
h.base.logger.Error("查询活动优先轮询项失败", zap.Uint("card_id", cardID), zap.Error(priorityErr))
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.stepFail(ctx, "", "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingPackage))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingPackage))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
card, cacheErr := h.base.getCardWithCache(ctx, cardID)
|
card, cacheErr := h.base.getCardWithCache(ctx, cardID)
|
||||||
if cacheErr != nil {
|
if cacheErr != nil {
|
||||||
if isNotFound(cacheErr) {
|
if isNotFound(cacheErr) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
h.base.logger.Error("获取卡信息失败", zap.Uint("card_id", cardID), zap.Error(cacheErr))
|
h.base.logger.Error("获取卡信息失败", zap.Uint("card_id", cardID), zap.Error(cacheErr))
|
||||||
// 本地读卡失败属可恢复失败:与其余三个轮询 handler 统一按尝试上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:与其余三个轮询 handler 统一按尝试上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.stepFail(ctx, "", "获取卡信息失败", cacheErr)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
|
|
||||||
if h.stopResumeSvc == nil {
|
if h.stopResumeSvc == nil {
|
||||||
priority.FailFinal(ctx, "套餐评估能力未配置")
|
priority.FailFinal(ctx, "套餐评估能力未配置")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
|
|
||||||
// 停复机评估就是本任务的上游执行动作:先累加尝试次数再发起。
|
// 停复机评估就是本任务的上游执行动作:先累加尝试次数再发起。
|
||||||
@@ -98,7 +105,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
|||||||
zap.Uint("card_id", cardID), zap.Error(loadErr))
|
zap.Uint("card_id", cardID), zap.Error(loadErr))
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.stepFail(ctx, "", "获取卡信息失败", loadErr)
|
||||||
}
|
}
|
||||||
h.base.logger.Info("套餐检查:执行停复机评估",
|
h.base.logger.Info("套餐检查:执行停复机评估",
|
||||||
zap.Uint("card_id", cardID),
|
zap.Uint("card_id", cardID),
|
||||||
@@ -119,10 +126,11 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
|||||||
zap.Uint("card_id", cardID), zap.Error(evalErr))
|
zap.Uint("card_id", cardID), zap.Error(evalErr))
|
||||||
priority.FailRetryable(ctx, "停复机评估失败")
|
priority.FailRetryable(ctx, "停复机评估失败")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
outcome := h.base.stepFail(ctx, constants.GatewayOpStart, "停复机评估失败", evalErr)
|
||||||
|
return outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
priority.Complete(ctx)
|
priority.Complete(ctx)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepOK()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,81 +28,88 @@ func NewPollingRealnameHandler(base *PollingBase, gw *gateway.Client, observatio
|
|||||||
return &PollingRealnameHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
return &PollingRealnameHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理实名检查任务。
|
// Handle 处理实名检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||||
startedAt := time.Now()
|
|
||||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingRealname) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingRealname) {
|
||||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingRealname)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingRealname)
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingRealname, h.Step(ctx, cardID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次实名检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingRealnameHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startedAt := time.Now()
|
||||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingRealname)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingRealname)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingRealname))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingRealname))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFound(err) {
|
if isNotFound(err) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
if h.gateway == nil {
|
if h.gateway == nil {
|
||||||
priority.FailFinal(ctx, "实名查询能力未配置")
|
priority.FailFinal(ctx, "实名查询能力未配置")
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
// 所属运营商账户未开启实名管控时实名结果不影响停复机,跳过查询并按正常间隔重入队(不再按行业卡跳过)。
|
// 所属运营商账户未开启实名管控时实名结果不影响停复机,跳过查询并按正常间隔重入队(不再按行业卡跳过)。
|
||||||
skippable, scopeErr := h.base.realnameQuerySkippable(ctx, card)
|
skippable, scopeErr := h.base.realnameQuerySkippable(ctx, card)
|
||||||
if scopeErr != nil {
|
if scopeErr != nil {
|
||||||
priority.FailRetryable(ctx, "查询运营商账户实名管控失败")
|
priority.FailRetryable(ctx, "查询运营商账户实名管控失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
return h.fail(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
||||||
}
|
}
|
||||||
if skippable {
|
if skippable {
|
||||||
priority.FailFinal(ctx, "所属运营商账户未开启实名管控,跳过实名查询")
|
priority.FailFinal(ctx, "所属运营商账户未开启实名管控,跳过实名查询")
|
||||||
h.base.logger.Debug("所属运营商账户未开启实名管控,跳过实名查询", zap.Uint("card_id", cardID), zap.Uint("carrier_id", card.CarrierID))
|
h.base.logger.Debug("所属运营商账户未开启实名管控,跳过实名查询", zap.Uint("card_id", cardID), zap.Uint("carrier_id", card.CarrierID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayRealname, constants.CardObservationSceneRealnamePolling)
|
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayRealname, constants.CardObservationSceneRealnamePolling)
|
||||||
result, err := h.gateway.QueryRealnameStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
result, err := h.gateway.QueryRealnameStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||||
if gateway.IsRateLimited(err) {
|
if gateway.IsRateLimited(err) {
|
||||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingRealname)
|
return StepOutcome{Result: StepRateLimited}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询实名状态失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询实名状态失败", priority)
|
||||||
priority.FailRetryable(ctx, "查询实名状态失败")
|
priority.FailRetryable(ctx, "查询实名状态失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询实名状态失败", err)
|
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, "查询实名状态失败", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(result.ICCID) == "" {
|
if strings.TrimSpace(result.ICCID) == "" {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "实名查询响应缺少 ICCID", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "实名查询响应缺少 ICCID", priority)
|
||||||
priority.FailRetryable(ctx, "实名查询响应缺少 ICCID")
|
priority.FailRetryable(ctx, "实名查询响应缺少 ICCID")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
return h.fail(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
||||||
}
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
||||||
if h.observation == nil {
|
if h.observation == nil {
|
||||||
priority.FailFinal(ctx, "卡实名观测能力未配置")
|
priority.FailFinal(ctx, "卡实名观测能力未配置")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
return h.fail(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
||||||
@@ -115,7 +122,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用实名观测失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用实名观测失败", priority)
|
||||||
priority.FailRetryable(ctx, "应用实名观测失败")
|
priority.FailRetryable(ctx, "应用实名观测失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用实名观测失败", err)
|
return h.fail(ctx, cardID, startedAt, "应用实名观测失败", err)
|
||||||
}
|
}
|
||||||
changed := decision.StatusChanged
|
changed := decision.StatusChanged
|
||||||
if changed {
|
if changed {
|
||||||
@@ -133,15 +140,19 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
h.base.logger.Info("实名轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
h.base.logger.Info("实名轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||||
priority.Complete(ctx)
|
priority.Complete(ctx)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepOK()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *PollingRealnameHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
func (h *PollingRealnameHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||||
|
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, message, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *PollingRealnameHandler) failOp(ctx context.Context, cardID uint, startedAt time.Time, op, message string, err error) StepOutcome {
|
||||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fields = append(fields, zap.Error(err))
|
fields = append(fields, zap.Error(err))
|
||||||
}
|
}
|
||||||
h.base.logger.Warn(message, fields...)
|
h.base.logger.Warn(message, fields...)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, false, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingRealname, false, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return h.base.stepFail(ctx, op, message, err)
|
||||||
}
|
}
|
||||||
|
|||||||
123
internal/task/polling_step.go
Normal file
123
internal/task/polling_step.go
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
package task
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StepResult 是一个检查步骤的执行结果分类。
|
||||||
|
type StepResult int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// StepSucceeded 检查成功(含观测入账、评估完成)。
|
||||||
|
StepSucceeded StepResult = iota
|
||||||
|
// StepFailed 检查失败,计入连续失败(是否计入还要看失败类别,见 countsAsFailure)。
|
||||||
|
StepFailed
|
||||||
|
// StepRateLimited 本账户限流拒绝或等待超时,请求未发出:短延迟重排,不计失败。
|
||||||
|
StepRateLimited
|
||||||
|
// StepDeferred 暂不可执行(流量同步锁被占、优先项由其他执行持有):短延迟或按间隔重排,不计失败。
|
||||||
|
StepDeferred
|
||||||
|
// StepSkipped 本轮无需执行(能力未配置、不在轮询范围、所属账户未开启实名管控):按正常间隔重排。
|
||||||
|
StepSkipped
|
||||||
|
// StepCardGone 卡不存在:不再重排。
|
||||||
|
StepCardGone
|
||||||
|
// StepStopPolling 观测后该卡已关闭轮询(风险停机、已销户):不再重排。
|
||||||
|
StepStopPolling
|
||||||
|
)
|
||||||
|
|
||||||
|
// StepOutcome 是一个检查步骤的结果。旧分片调度把它映射为原有的重入队动作;按运营商账户调度据此写回到期与失败计数。
|
||||||
|
type StepOutcome struct {
|
||||||
|
Result StepResult
|
||||||
|
// DeferDelay 为 StepDeferred 时的建议短延迟;0 表示按正常间隔。
|
||||||
|
DeferDelay time.Duration
|
||||||
|
// Category 失败类别:网关错误类别(network / timeout / http / invalid_response),上游业务错误时为错误分类类别。
|
||||||
|
Category string
|
||||||
|
// Message 失败信息(已脱敏截断)。
|
||||||
|
Message string
|
||||||
|
// Observed 卡状态步骤观测到的网络状态(online / offline),用于延迟确认是否已满足预期。
|
||||||
|
Observed string
|
||||||
|
// TriggeredEvaluation 流量步骤本轮观测产生了流量变化(会触发停复机评估),可据此跳过套餐步骤。
|
||||||
|
TriggeredEvaluation bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// 卡状态步骤的观测结果取值。
|
||||||
|
const (
|
||||||
|
ObservedOnline = "online"
|
||||||
|
ObservedOffline = "offline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func stepOK() StepOutcome { return StepOutcome{Result: StepSucceeded} }
|
||||||
|
|
||||||
|
func stepSkip() StepOutcome { return StepOutcome{Result: StepSkipped} }
|
||||||
|
|
||||||
|
func stepDefer(delay time.Duration) StepOutcome {
|
||||||
|
return StepOutcome{Result: StepDeferred, DeferDelay: delay}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepFail 按错误构造失败结果:网关结构化错误取其类别,上游业务错误再按错误分类规则细分(用于判断是否计入连续失败与告警)。
|
||||||
|
func (b *PollingBase) stepFail(ctx context.Context, op, message string, err error) StepOutcome {
|
||||||
|
outcome := StepOutcome{Result: StepFailed, Message: message, Category: "local"}
|
||||||
|
if gwErr, ok := gateway.AsGatewayError(err); ok {
|
||||||
|
outcome.Category = gwErr.Category
|
||||||
|
if gwErr.Category == constants.GatewayErrorUpstream {
|
||||||
|
outcome.Message = message + ":" + gwErr.ProviderMessage
|
||||||
|
if b.errorClassifier != nil {
|
||||||
|
outcome.Category = b.errorClassifier(ctx, op, gwErr.ProviderMessage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountsAsFailure 判断失败结果是否计入连续失败:上游频率限制与已是目标状态不计入。
|
||||||
|
func (o StepOutcome) CountsAsFailure() bool {
|
||||||
|
if o.Result != StepFailed {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch o.Category {
|
||||||
|
case constants.GatewayClassRateLimitedUpstream, constants.GatewayClassAlreadyTarget:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrorClassifier 对上游业务错误信息分类,返回错误分类类别。
|
||||||
|
type ErrorClassifier func(ctx context.Context, op, message string) string
|
||||||
|
|
||||||
|
// SetErrorClassifier 注入上游错误分类器(按运营商账户调度的失败退避与告警使用)。
|
||||||
|
func (b *PollingBase) SetErrorClassifier(classifier ErrorClassifier) {
|
||||||
|
b.errorClassifier = classifier
|
||||||
|
}
|
||||||
|
|
||||||
|
// SchedulerModeReader 返回当前卡轮询调度模式(受控系统配置 polling.scheduler.mode)。
|
||||||
|
type SchedulerModeReader func(ctx context.Context) string
|
||||||
|
|
||||||
|
// SetSchedulerModeReader 注入调度模式读取;未注入时视为旧分片调度。
|
||||||
|
func (b *PollingBase) SetSchedulerModeReader(reader SchedulerModeReader) {
|
||||||
|
b.schedulerMode = reader
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierSchedulerActive 判断当前是否已切到按运营商账户调度:是则旧 Asynq 轮询任务直接结束,不调网关、不重入旧队列。
|
||||||
|
func (b *PollingBase) carrierSchedulerActive(ctx context.Context) bool {
|
||||||
|
return b != nil && b.schedulerMode != nil && b.schedulerMode(ctx) == constants.PollingSchedulerModeCarrier
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyLegacyOutcome 把步骤结果映射为旧分片调度原有的重入队动作。
|
||||||
|
func (b *PollingBase) applyLegacyOutcome(ctx context.Context, cardID uint, taskType string, outcome StepOutcome) error {
|
||||||
|
switch outcome.Result {
|
||||||
|
case StepCardGone, StepStopPolling:
|
||||||
|
return nil
|
||||||
|
case StepRateLimited:
|
||||||
|
return b.requeueRateLimited(cardID, taskType)
|
||||||
|
case StepDeferred:
|
||||||
|
if outcome.DeferDelay > 0 {
|
||||||
|
return b.queueMgr.Requeue(ctx, cardID, taskType, time.Now().Add(outcome.DeferDelay))
|
||||||
|
}
|
||||||
|
return b.requeueCard(ctx, cardID, taskType)
|
||||||
|
default:
|
||||||
|
return b.requeueCard(ctx, cardID, taskType)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,7 @@ type Handler struct {
|
|||||||
workerResult *WorkerBootstrapResult
|
workerResult *WorkerBootstrapResult
|
||||||
asynqClient *asynq.Client
|
asynqClient *asynq.Client
|
||||||
pollingBase *task.PollingBase
|
pollingBase *task.PollingBase
|
||||||
|
pollingSteps PollingStepHandlers
|
||||||
pollingStopResumeSvc iot_card_svc.StopResumeServiceInterface
|
pollingStopResumeSvc iot_card_svc.StopResumeServiceInterface
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -292,6 +293,7 @@ func (h *Handler) registerPollingHandlers() {
|
|||||||
cardStatusHandler := task.NewPollingCardStatusHandler(
|
cardStatusHandler := task.NewPollingCardStatusHandler(
|
||||||
h.pollingBase, h.gatewayClient, h.workerResult.Services.CardObservation, integrationRepository)
|
h.pollingBase, h.gatewayClient, h.workerResult.Services.CardObservation, integrationRepository)
|
||||||
|
|
||||||
|
h.pollingSteps = PollingStepHandlers{Realname: realnameHandler, Carddata: carddataHandler, Package: packageHandler, CardStatus: cardStatusHandler}
|
||||||
h.mux.HandleFunc(constants.TaskTypePollingRealname, realnameHandler.Handle)
|
h.mux.HandleFunc(constants.TaskTypePollingRealname, realnameHandler.Handle)
|
||||||
h.mux.HandleFunc(constants.TaskTypePollingCarddata, carddataHandler.Handle)
|
h.mux.HandleFunc(constants.TaskTypePollingCarddata, carddataHandler.Handle)
|
||||||
h.mux.HandleFunc(constants.TaskTypePollingPackage, packageHandler.Handle)
|
h.mux.HandleFunc(constants.TaskTypePollingPackage, packageHandler.Handle)
|
||||||
@@ -408,3 +410,16 @@ func (h *Handler) registerOperationsReportSnapshotHandler() {
|
|||||||
func (h *Handler) GetMux() *asynq.ServeMux {
|
func (h *Handler) GetMux() *asynq.ServeMux {
|
||||||
return h.mux
|
return h.mux
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PollingStepHandlers 是四个轮询检查步骤处理器,旧分片调度注册为 Asynq handler,按运营商账户调度的卡任务执行器复用其 Step。
|
||||||
|
type PollingStepHandlers struct {
|
||||||
|
Realname *task.PollingRealnameHandler
|
||||||
|
Carddata *task.PollingCarddataHandler
|
||||||
|
Package *task.PollingPackageHandler
|
||||||
|
CardStatus *task.PollingCardStatusHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingSteps 返回轮询检查步骤处理器(RegisterHandlers 之后可用)。
|
||||||
|
func (h *Handler) PollingSteps() PollingStepHandlers {
|
||||||
|
return h.pollingSteps
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user