Files
junhong_cmp_fiber/cmd/worker/carrier_scheduler.go
邹进 2fe7c1f8ac feat: 按运营商账户调度的分发器、卡任务执行器与模式切换
- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
2026-10-25 18:44:23 +09:00

229 lines
8.3 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"os"
"strconv"
"sync"
"time"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/nsqbus"
"github.com/break/junhong_cmp_fiber/internal/polling"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
"github.com/break/junhong_cmp_fiber/internal/task"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/queue"
)
// carrierInitLockKey 新结构初始化锁:多实例时只由一个实例初始化。
const carrierInitLockKey = "polling:carrier:init_lock"
// carrierSchedulerRuntime 是按运营商账户调度在 Worker 内的运行组件:模式监听、初始化、分发器与 NSQ 消费者。
type carrierSchedulerRuntime struct {
cfg config.CarrierSchedulerConfig
runtime *workerRuntime
initializer *polling.CarrierInitializer
dispatcher *carrierqueue.Dispatcher
consumers *nsqbus.ConsumerManager
producer *nsqbus.Producer
alerts *carrierqueue.AlertWriter
executor *task.CarrierCardExecutor
singleton bool
instanceID string
logger *zap.Logger
mu sync.Mutex
lastMode string
reinitQueued bool
legacyRestarts func()
}
// newCarrierSchedulerParts 在 Worker 运行时初始化阶段创建调度参数、账户队列存储与模式接缝,
// 并注入旧调度相关组件(生命周期、轮询基类),使其在启动前即可按模式写入正确结构。
func newCarrierSchedulerParts(runtime *workerRuntime, appLogger *zap.Logger) {
settings := carrierqueue.NewDefaultSettingsProvider(runtime.db, runtime.redisClient, appLogger)
store := carrierqueue.NewStore(runtime.redisClient)
scheduling := polling.NewCarrierScheduling(store, settings, runtime.pollingConfigMgr, appLogger)
runtime.carrierSettings = settings
runtime.carrierStore = store
runtime.carrierScheduling = scheduling
runtime.lifecycleSvc.SetCarrierScheduling(scheduling)
runtime.pollingBase.SetSchedulerModeReader(func(ctx context.Context) string { return settings.Current(ctx).Mode })
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Classifier != nil {
classifier := runtime.gatewayLimit.Classifier
runtime.pollingBase.SetErrorClassifier(func(ctx context.Context, op, message string) string {
return classifier.Classify(ctx, op, message).Category
})
}
}
// startCarrierScheduler 启动模式监听;分发器只在 leader/all 运行(多实例安全),消费者在所有处理任务的实例运行。
func startCarrierScheduler(ctx context.Context, cfg *config.Config, runtime *workerRuntime, steps queue.PollingStepHandlers,
singleton bool, legacyRestart func(), appLogger *zap.Logger) *carrierSchedulerRuntime {
carrierCfg := cfg.Polling.Carrier
instanceID := cfg.Worker.InstanceName
if instanceID == "" {
instanceID, _ = os.Hostname()
}
instanceID += "-" + strconv.Itoa(os.Getpid())
alerts := carrierqueue.NewAlertWriter(runtime.db, runtime.redisClient, runtime.carrierSettings, appLogger)
alerts.Start(ctx)
executor := task.NewCarrierCardExecutor(runtime.pollingBase, runtime.carrierStore, runtime.carrierSettings, alerts,
steps.Realname, steps.Carddata, steps.CardStatus, steps.Package, carrierCfg, instanceID)
c := &carrierSchedulerRuntime{
cfg: carrierCfg, runtime: runtime, singleton: singleton, instanceID: instanceID, logger: appLogger,
alerts: alerts, executor: executor, legacyRestarts: legacyRestart,
initializer: polling.NewCarrierInitializer(runtime.carrierScheduling, runtime.pollingIotCardStore, appLogger),
}
if carrierCfg.NSQDAddress != "" {
producer, err := nsqbus.NewProducer(carrierCfg.NSQDAddress, appLogger)
if err != nil {
appLogger.Error("创建 NSQ 生产者失败,按运营商账户调度不可用", zap.Error(err))
} else {
c.producer = producer
}
c.consumers = nsqbus.NewConsumerManager(nsqbus.ConsumerConfig{
NSQDAddress: carrierCfg.NSQDAddress, NSQLookupdAddresses: nsqbus.SplitAddresses(carrierCfg.NSQLookupdAddresses),
MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts,
}, constants.NSQChannelWorker, executor.Handle, appLogger)
}
var budgets carrierqueue.BudgetProvider
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Limiter != nil {
budgets = runtime.gatewayLimit.Limiter
}
if c.producer != nil {
c.dispatcher = carrierqueue.NewDispatcher(runtime.db, runtime.carrierStore, runtime.carrierSettings, c.producer, budgets,
carrierCfg, instanceID, appLogger)
if singleton {
go c.dispatcher.Run(ctx)
}
}
go c.watch(ctx)
return c
}
// RequestReinit 轮询配置变更时请求按新配置重新初始化新结构(仅 carrier 模式生效)。
func (c *carrierSchedulerRuntime) RequestReinit() {
if c == nil {
return
}
c.mu.Lock()
c.reinitQueued = true
c.mu.Unlock()
}
func (c *carrierSchedulerRuntime) watch(ctx context.Context) {
ticker := time.NewTicker(3 * time.Second)
defer ticker.Stop()
for {
c.step(ctx)
select {
case <-ctx.Done():
c.shutdown()
return
case <-ticker.C:
}
}
}
func (c *carrierSchedulerRuntime) step(ctx context.Context) {
defer func() {
if r := recover(); r != nil {
c.logger.Error("按运营商账户调度模式监听 panic,已恢复", zap.Any("panic", r))
}
}()
mode := c.runtime.carrierSettings.Current(ctx).Mode
c.mu.Lock()
previous := c.lastMode
c.lastMode = mode
reinit := c.reinitQueued
c.reinitQueued = false
c.mu.Unlock()
if mode == constants.PollingSchedulerModeCarrier {
if previous != mode {
c.logger.Info("卡轮询调度模式:按运营商账户调度", zap.String("previous", previous))
}
if c.singleton {
c.ensureInitialized(ctx, reinit)
}
c.syncConsumers(ctx)
return
}
if previous == constants.PollingSchedulerModeCarrier {
c.logger.Info("卡轮询调度模式回到旧分片调度:停止领取与消费,重建旧分片队列")
if c.consumers != nil {
c.consumers.StopAll(c.grace())
c.consumers.Reset()
}
if c.singleton {
c.runtime.redisClient.Del(ctx, constants.RedisCarrierSchedulerInitStateKey)
if c.legacyRestarts != nil {
c.legacyRestarts()
}
}
}
}
// ensureInitialized 切到 carrier 后初始化新结构;初始化完成前分发器不领取。多实例时用初始化锁协调。
func (c *carrierSchedulerRuntime) ensureInitialized(ctx context.Context, reinit bool) {
redisClient := c.runtime.redisClient
state, _ := redisClient.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
if state == carrierqueue.InitStateDone && !reinit {
return
}
ok, err := redisClient.SetNX(ctx, carrierInitLockKey, c.instanceID, 30*time.Minute).Result()
if err != nil || !ok {
return
}
defer redisClient.Del(context.WithoutCancel(ctx), carrierInitLockKey)
if state != carrierqueue.InitStateDone {
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateRunning, 0)
}
started := time.Now()
written, initErr := c.initializer.Run(ctx)
if initErr != nil {
c.logger.Error("按运营商账户调度初始化失败,下一轮重试", zap.Error(initErr))
return
}
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateDone, 0)
c.logger.Info("按运营商账户调度初始化完成,开始领取", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)),
zap.String("metric", "polling.carrier.initialized"))
}
// syncConsumers 按当前运营商账户订阅卡任务 topic,max_in_flight 取账户调度参数。
func (c *carrierSchedulerRuntime) syncConsumers(ctx context.Context) {
if c.consumers == nil || c.dispatcher == nil {
return
}
desired := map[string]int{}
for _, carrierID := range c.dispatcher.CarrierIDs(ctx) {
desired[constants.NSQCardTopic(carrierID)] = c.dispatcher.ParamsFor(carrierID).CardMaxInFlight
}
c.consumers.Sync(desired)
}
func (c *carrierSchedulerRuntime) grace() time.Duration {
if c.cfg.ShutdownGrace > 0 {
return c.cfg.ShutdownGrace
}
return 30 * time.Second
}
// shutdown 优雅退出:停止订阅,宽限期内等待执行中的卡任务完成;未完成的由租约到期后重新领取。
func (c *carrierSchedulerRuntime) shutdown() {
if c.consumers != nil {
c.consumers.StopAll(c.grace())
}
if c.producer != nil {
c.producer.Stop()
}
}
// InitProgress 返回新结构初始化进度。
func (c *carrierSchedulerRuntime) InitProgress() polling.InitProgress {
return c.initializer.Progress()
}