feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled

自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭):
- 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机
- 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制
- 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认
- 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后
- 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估
- 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令
- 迁移 000255 插入开关与告警、退避参数

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-26 00:36:26 +09:00
parent 1369fed642
commit 934a3a27a1
31 changed files with 2275 additions and 117 deletions

View File

@@ -12,10 +12,13 @@ import (
"go.uber.org/zap"
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/gatewaylimit"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/nsqbus"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/polling"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
"github.com/break/junhong_cmp_fiber/internal/task"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
@@ -35,9 +38,13 @@ type carrierSchedulerRuntime struct {
producer *nsqbus.Producer
alerts *carrierqueue.AlertWriter
executor *task.CarrierCardExecutor
singleton bool
instanceID string
logger *zap.Logger
// 停复机指令与设备判断的专用消费者:只看停复机出口开关,不受 scheduler_mode 限制。
srConsumers *nsqbus.ConsumerManager
devConsumers *nsqbus.ConsumerManager
srActive bool
singleton bool
instanceID string
logger *zap.Logger
mu sync.Mutex
lastMode string
@@ -56,6 +63,9 @@ func newCarrierSchedulerParts(runtime *workerRuntime, appLogger *zap.Logger) {
runtime.carrierScheduling = scheduling
runtime.lifecycleSvc.SetCarrierScheduling(scheduling)
runtime.pollingBase.SetSchedulerModeReader(func(ctx context.Context) string { return settings.Current(ctx).Mode })
if stopResumeSvc, ok := runtime.workerResult.Services.StopResumeService.(*iot_card_svc.StopResumeService); ok {
stopResumeSvc.SetCommandQueue(carrierqueue.NewCommandQueue(store, settings))
}
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Classifier != nil {
classifier := runtime.gatewayLimit.Classifier
runtime.pollingBase.SetErrorClassifier(func(ctx context.Context, op, message string) string {
@@ -94,6 +104,7 @@ func startCarrierScheduler(ctx context.Context, cfg *config.Config, runtime *wor
MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts,
}, constants.NSQChannelWorker, executor.Handle, appLogger)
}
c.setupStopResume(carrierCfg, alerts)
var budgets carrierqueue.BudgetProvider
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Limiter != nil {
budgets = runtime.gatewayLimit.Limiter
@@ -139,6 +150,7 @@ func (c *carrierSchedulerRuntime) step(ctx context.Context) {
c.logger.Error("按运营商账户调度模式监听 panic,已恢复", zap.Any("panic", r))
}
}()
c.syncStopResume(ctx)
mode := c.runtime.carrierSettings.Current(ctx).Mode
c.mu.Lock()
previous := c.lastMode
@@ -216,11 +228,74 @@ func (c *carrierSchedulerRuntime) grace() time.Duration {
return 30 * time.Second
}
// setupStopResume 创建停复机指令执行器、设备判断执行器与两个专用消费者。
func (c *carrierSchedulerRuntime) setupStopResume(carrierCfg config.CarrierSchedulerConfig, alerts *carrierqueue.AlertWriter) {
service, ok := c.runtime.workerResult.Services.StopResumeService.(*iot_card_svc.StopResumeService)
if !ok || carrierCfg.NSQDAddress == "" {
return
}
var classifier *gatewaylimit.Classifier
if c.runtime.gatewayLimit != nil {
classifier = c.runtime.gatewayLimit.Classifier
}
params := func(carrierID uint) carrierqueue.CarrierParams {
if c.dispatcher != nil {
return c.dispatcher.ParamsFor(carrierID)
}
return carrierqueue.MergeParams(model.CarrierPollingParam{}, false, carrierCfg)
}
srExecutor := task.NewStopResumeExecutor(c.runtime.carrierStore, c.runtime.carrierSettings, params, alerts, service, classifier,
c.instanceID, c.logger)
devExecutor := task.NewDeviceEvalExecutor(c.runtime.carrierStore, service, c.instanceID, c.logger)
consumerCfg := nsqbus.ConsumerConfig{
NSQDAddress: carrierCfg.NSQDAddress, NSQLookupdAddresses: nsqbus.SplitAddresses(carrierCfg.NSQLookupdAddresses),
MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts,
}
c.srConsumers = nsqbus.NewConsumerManager(consumerCfg, constants.NSQChannelWorker, srExecutor.Handle, c.logger)
c.devConsumers = nsqbus.NewConsumerManager(consumerCfg, constants.NSQChannelWorker, devExecutor.Handle, c.logger)
}
// syncStopResume 停复机出口开关开启时订阅各账户停复机 topic 与设备判断 topic;关闭时停止订阅(队列中的指令保留)。
func (c *carrierSchedulerRuntime) syncStopResume(ctx context.Context) {
if c.srConsumers == nil || c.dispatcher == nil {
return
}
enabled := c.runtime.carrierSettings.Current(ctx).StopResumeQueueEnabled
if !enabled {
if c.srActive {
c.logger.Info("停复机出口开关已关闭:停止停复机指令消费,未执行的指令保留")
c.srConsumers.StopAll(c.grace())
c.srConsumers.Reset()
c.devConsumers.StopAll(c.grace())
c.devConsumers.Reset()
c.srActive = false
}
return
}
if !c.srActive {
c.logger.Info("停复机出口开关已开启:自动停复机经停复机指令队列执行")
c.srActive = true
}
desired := map[string]int{}
for _, carrierID := range c.dispatcher.CarrierIDs(ctx) {
desired[constants.NSQStopResumeTopic(carrierID)] = c.dispatcher.ParamsFor(carrierID).SRMaxInFlight
}
c.srConsumers.Sync(desired)
c.devConsumers.Sync(map[string]int{constants.NSQDeviceEvalTopic: deviceEvalMaxInFlight})
}
// deviceEvalMaxInFlight 是每进程设备判断消费者并发。
const deviceEvalMaxInFlight = 10
// shutdown 优雅退出:停止订阅,宽限期内等待执行中的卡任务完成;未完成的由租约到期后重新领取。
func (c *carrierSchedulerRuntime) shutdown() {
if c.consumers != nil {
c.consumers.StopAll(c.grace())
}
if c.srConsumers != nil {
c.srConsumers.StopAll(c.grace())
c.devConsumers.StopAll(c.grace())
}
if c.producer != nil {
c.producer.Stop()
}