Compare commits
4 Commits
e7f8c4bae9
...
9183863fab
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9183863fab | ||
|
|
c741f1751d | ||
|
|
0d966249b8 | ||
|
|
2fe7c1f8ac |
@@ -33,6 +33,7 @@ func generateOpenAPIDocs(outputPath string, logger *zap.Logger) {
|
||||
handlers.PhoneAssetAssociation = admin.NewPhoneAssetAssociationHandler(nil, nil)
|
||||
// 网关限流 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||
handlers.GatewayRateLimit = admin.NewGatewayRateLimitHandler(nil, nil, nil)
|
||||
handlers.CarrierScheduler = admin.NewCarrierSchedulerHandler(nil, nil)
|
||||
// 套餐真流量预警 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||
handlers.PackageTrafficAlert = admin.NewPackageTrafficAlertHandler(nil, nil, nil, nil)
|
||||
// 资产钱包自动续费配置 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||
|
||||
@@ -42,6 +42,7 @@ func generateAdminDocs(outputPath string) error {
|
||||
handlers.PhoneAssetAssociation = admin.NewPhoneAssetAssociationHandler(nil, nil)
|
||||
// 网关限流 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||
handlers.GatewayRateLimit = admin.NewGatewayRateLimitHandler(nil, nil, nil)
|
||||
handlers.CarrierScheduler = admin.NewCarrierSchedulerHandler(nil, nil)
|
||||
// 套餐真流量预警 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||
handlers.PackageTrafficAlert = admin.NewPackageTrafficAlertHandler(nil, nil, nil, nil)
|
||||
// 资产钱包自动续费配置 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||
|
||||
270
cmd/worker/carrier_scheduler.go
Normal file
270
cmd/worker/carrier_scheduler.go
Normal file
@@ -0,0 +1,270 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox"
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/nsqbus"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/internal/task"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||
)
|
||||
|
||||
// carrierInitLockKey 新结构初始化锁:多实例时只由一个实例初始化。
|
||||
const carrierInitLockKey = "polling:carrier:init_lock"
|
||||
|
||||
// carrierSchedulerRuntime 是按运营商账户调度在 Worker 内的运行组件:模式监听、初始化、分发器与 NSQ 消费者。
|
||||
type carrierSchedulerRuntime struct {
|
||||
cfg config.CarrierSchedulerConfig
|
||||
runtime *workerRuntime
|
||||
initializer *polling.CarrierInitializer
|
||||
dispatcher *carrierqueue.Dispatcher
|
||||
consumers *nsqbus.ConsumerManager
|
||||
producer *nsqbus.Producer
|
||||
alerts *carrierqueue.AlertWriter
|
||||
executor *task.CarrierCardExecutor
|
||||
singleton bool
|
||||
instanceID string
|
||||
logger *zap.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
lastMode string
|
||||
reinitQueued bool
|
||||
legacyRestarts func()
|
||||
}
|
||||
|
||||
// newCarrierSchedulerParts 在 Worker 运行时初始化阶段创建调度参数、账户队列存储与模式接缝,
|
||||
// 并注入旧调度相关组件(生命周期、轮询基类),使其在启动前即可按模式写入正确结构。
|
||||
func newCarrierSchedulerParts(runtime *workerRuntime, appLogger *zap.Logger) {
|
||||
settings := carrierqueue.NewDefaultSettingsProvider(runtime.db, runtime.redisClient, appLogger)
|
||||
store := carrierqueue.NewStore(runtime.redisClient)
|
||||
scheduling := polling.NewCarrierScheduling(store, settings, runtime.pollingConfigMgr, appLogger)
|
||||
runtime.carrierSettings = settings
|
||||
runtime.carrierStore = store
|
||||
runtime.carrierScheduling = scheduling
|
||||
runtime.lifecycleSvc.SetCarrierScheduling(scheduling)
|
||||
runtime.pollingBase.SetSchedulerModeReader(func(ctx context.Context) string { return settings.Current(ctx).Mode })
|
||||
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Classifier != nil {
|
||||
classifier := runtime.gatewayLimit.Classifier
|
||||
runtime.pollingBase.SetErrorClassifier(func(ctx context.Context, op, message string) string {
|
||||
return classifier.Classify(ctx, op, message).Category
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// startCarrierScheduler 启动模式监听;分发器只在 leader/all 运行(多实例安全),消费者在所有处理任务的实例运行。
|
||||
func startCarrierScheduler(ctx context.Context, cfg *config.Config, runtime *workerRuntime, steps queue.PollingStepHandlers,
|
||||
singleton bool, legacyRestart func(), appLogger *zap.Logger) *carrierSchedulerRuntime {
|
||||
carrierCfg := cfg.Polling.Carrier
|
||||
instanceID := cfg.Worker.InstanceName
|
||||
if instanceID == "" {
|
||||
instanceID, _ = os.Hostname()
|
||||
}
|
||||
instanceID += "-" + strconv.Itoa(os.Getpid())
|
||||
alerts := carrierqueue.NewAlertWriter(runtime.db, runtime.redisClient, runtime.carrierSettings, appLogger)
|
||||
alerts.Start(ctx)
|
||||
executor := task.NewCarrierCardExecutor(runtime.pollingBase, runtime.carrierStore, runtime.carrierSettings, alerts,
|
||||
steps.Realname, steps.Carddata, steps.CardStatus, steps.Package, carrierCfg, instanceID)
|
||||
c := &carrierSchedulerRuntime{
|
||||
cfg: carrierCfg, runtime: runtime, singleton: singleton, instanceID: instanceID, logger: appLogger,
|
||||
alerts: alerts, executor: executor, legacyRestarts: legacyRestart,
|
||||
initializer: polling.NewCarrierInitializer(runtime.carrierScheduling, runtime.pollingIotCardStore, appLogger),
|
||||
}
|
||||
if carrierCfg.NSQDAddress != "" {
|
||||
producer, err := nsqbus.NewProducer(carrierCfg.NSQDAddress, appLogger)
|
||||
if err != nil {
|
||||
appLogger.Error("创建 NSQ 生产者失败,按运营商账户调度不可用", zap.Error(err))
|
||||
} else {
|
||||
c.producer = producer
|
||||
}
|
||||
c.consumers = nsqbus.NewConsumerManager(nsqbus.ConsumerConfig{
|
||||
NSQDAddress: carrierCfg.NSQDAddress, NSQLookupdAddresses: nsqbus.SplitAddresses(carrierCfg.NSQLookupdAddresses),
|
||||
MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts,
|
||||
}, constants.NSQChannelWorker, executor.Handle, appLogger)
|
||||
}
|
||||
var budgets carrierqueue.BudgetProvider
|
||||
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Limiter != nil {
|
||||
budgets = runtime.gatewayLimit.Limiter
|
||||
}
|
||||
if c.producer != nil {
|
||||
c.dispatcher = carrierqueue.NewDispatcher(runtime.db, runtime.carrierStore, runtime.carrierSettings, c.producer, budgets,
|
||||
carrierCfg, instanceID, appLogger)
|
||||
if singleton {
|
||||
go c.dispatcher.Run(ctx)
|
||||
}
|
||||
}
|
||||
go c.watch(ctx)
|
||||
return c
|
||||
}
|
||||
|
||||
// RequestReinit 轮询配置变更时请求按新配置重新初始化新结构(仅 carrier 模式生效)。
|
||||
func (c *carrierSchedulerRuntime) RequestReinit() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.reinitQueued = true
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *carrierSchedulerRuntime) watch(ctx context.Context) {
|
||||
ticker := time.NewTicker(3 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
c.step(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.shutdown()
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *carrierSchedulerRuntime) step(ctx context.Context) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
c.logger.Error("按运营商账户调度模式监听 panic,已恢复", zap.Any("panic", r))
|
||||
}
|
||||
}()
|
||||
mode := c.runtime.carrierSettings.Current(ctx).Mode
|
||||
c.mu.Lock()
|
||||
previous := c.lastMode
|
||||
c.lastMode = mode
|
||||
reinit := c.reinitQueued
|
||||
c.reinitQueued = false
|
||||
c.mu.Unlock()
|
||||
if mode == constants.PollingSchedulerModeCarrier {
|
||||
if previous != mode {
|
||||
c.logger.Info("卡轮询调度模式:按运营商账户调度", zap.String("previous", previous))
|
||||
}
|
||||
if c.singleton {
|
||||
c.ensureInitialized(ctx, reinit)
|
||||
}
|
||||
c.syncConsumers(ctx)
|
||||
return
|
||||
}
|
||||
if previous == constants.PollingSchedulerModeCarrier {
|
||||
c.logger.Info("卡轮询调度模式回到旧分片调度:停止领取与消费,重建旧分片队列")
|
||||
if c.consumers != nil {
|
||||
c.consumers.StopAll(c.grace())
|
||||
c.consumers.Reset()
|
||||
}
|
||||
if c.singleton {
|
||||
c.runtime.redisClient.Del(ctx, constants.RedisCarrierSchedulerInitStateKey)
|
||||
if c.legacyRestarts != nil {
|
||||
c.legacyRestarts()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureInitialized 切到 carrier 后初始化新结构;初始化完成前分发器不领取。多实例时用初始化锁协调。
|
||||
func (c *carrierSchedulerRuntime) ensureInitialized(ctx context.Context, reinit bool) {
|
||||
redisClient := c.runtime.redisClient
|
||||
state, _ := redisClient.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||
if state == carrierqueue.InitStateDone && !reinit {
|
||||
return
|
||||
}
|
||||
ok, err := redisClient.SetNX(ctx, carrierInitLockKey, c.instanceID, 30*time.Minute).Result()
|
||||
if err != nil || !ok {
|
||||
return
|
||||
}
|
||||
defer redisClient.Del(context.WithoutCancel(ctx), carrierInitLockKey)
|
||||
if state != carrierqueue.InitStateDone {
|
||||
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateRunning, 0)
|
||||
}
|
||||
started := time.Now()
|
||||
written, initErr := c.initializer.Run(ctx)
|
||||
if initErr != nil {
|
||||
c.logger.Error("按运营商账户调度初始化失败,下一轮重试", zap.Error(initErr))
|
||||
return
|
||||
}
|
||||
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateDone, 0)
|
||||
c.logger.Info("按运营商账户调度初始化完成,开始领取", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)),
|
||||
zap.String("metric", "polling.carrier.initialized"))
|
||||
}
|
||||
|
||||
// syncConsumers 按当前运营商账户订阅卡任务 topic,max_in_flight 取账户调度参数。
|
||||
func (c *carrierSchedulerRuntime) syncConsumers(ctx context.Context) {
|
||||
if c.consumers == nil || c.dispatcher == nil {
|
||||
return
|
||||
}
|
||||
desired := map[string]int{}
|
||||
for _, carrierID := range c.dispatcher.CarrierIDs(ctx) {
|
||||
desired[constants.NSQCardTopic(carrierID)] = c.dispatcher.ParamsFor(carrierID).CardMaxInFlight
|
||||
}
|
||||
c.consumers.Sync(desired)
|
||||
}
|
||||
|
||||
func (c *carrierSchedulerRuntime) grace() time.Duration {
|
||||
if c.cfg.ShutdownGrace > 0 {
|
||||
return c.cfg.ShutdownGrace
|
||||
}
|
||||
return 30 * time.Second
|
||||
}
|
||||
|
||||
// shutdown 优雅退出:停止订阅,宽限期内等待执行中的卡任务完成;未完成的由租约到期后重新领取。
|
||||
func (c *carrierSchedulerRuntime) shutdown() {
|
||||
if c.consumers != nil {
|
||||
c.consumers.StopAll(c.grace())
|
||||
}
|
||||
if c.producer != nil {
|
||||
c.producer.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// InitProgress 返回新结构初始化进度。
|
||||
func (c *carrierSchedulerRuntime) InitProgress() polling.InitProgress {
|
||||
return c.initializer.Progress()
|
||||
}
|
||||
|
||||
// carrierAwareSeriesConsumer 是业务事件观测序列 Outbox 消费者的模式接缝:按运营商账户调度时把业务事件改为按场景请求检查
|
||||
// (事件在原业务事务中写入,消费即在事务提交之后),旧分片调度时沿用原观测序列。
|
||||
type carrierAwareSeriesConsumer struct {
|
||||
legacy outbox.EventConsumer
|
||||
requester *polling.CheckRequester
|
||||
}
|
||||
|
||||
// Consume 消费业务观测序列请求事件。
|
||||
func (c *carrierAwareSeriesConsumer) Consume(ctx context.Context, envelope outbox.DeliveryEnvelope) error {
|
||||
if c.requester == nil || !c.requester.Active(ctx) {
|
||||
return c.legacy.Consume(ctx, envelope)
|
||||
}
|
||||
var event cardObservationApp.SeriesRequestedEvent
|
||||
if err := json.Unmarshal(envelope.Payload, &event); err != nil {
|
||||
return nil
|
||||
}
|
||||
cardIDs := []uint{event.ResourceID}
|
||||
if event.ResourceType == constants.CardObservationResourceTypeDevice {
|
||||
cardIDs = event.ResourceIDs
|
||||
}
|
||||
c.requester.RequestCards(ctx, businessEventCheckScene(event), cardIDs, event.ExpectedValue)
|
||||
return nil
|
||||
}
|
||||
|
||||
// businessEventCheckScene 把业务观测事件映射为请求检查场景:订单支付、套餐激活、自动续费购包按事件来源区分,停复机成功按观测场景。
|
||||
func businessEventCheckScene(event cardObservationApp.SeriesRequestedEvent) string {
|
||||
switch {
|
||||
case strings.HasPrefix(event.EventID, "card-observation:order:"):
|
||||
return constants.PollingCheckSceneOrderPaid
|
||||
case strings.HasPrefix(event.EventID, "card-observation:auto-purchase:"), strings.HasPrefix(event.EventID, "asset-auto-renewal:"):
|
||||
return constants.PollingCheckSceneAutoRenewalPurchase
|
||||
case event.Scene == constants.CardObservationSceneBusinessStop || event.Scene == constants.CardObservationSceneBusinessResume:
|
||||
return constants.PollingCheckSceneStopResumeSucceeded
|
||||
default:
|
||||
return constants.PollingCheckScenePackageActivated
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ import (
|
||||
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
||||
refundSvc "github.com/break/junhong_cmp_fiber/internal/service/refund"
|
||||
@@ -102,6 +103,12 @@ type workerRuntime struct {
|
||||
assetAutoRenewalService *assetAutoRenewalApp.Service
|
||||
// gatewayLimit 是卡网关运营商账户限流组件(限流、统计、停复机记录)。
|
||||
gatewayLimit *gatewaylimit.Runtime
|
||||
// 按运营商账户调度:调度参数、账户队列存储与模式接缝(模式为 carrier 时各入口写新结构)。
|
||||
carrierSettings *carrierqueue.SettingsProvider
|
||||
carrierStore *carrierqueue.Store
|
||||
carrierScheduling *polling.CarrierScheduling
|
||||
// carrierReinit 在 carrier 模式下轮询配置变更时请求重新初始化新结构。
|
||||
carrierReinit func()
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -171,6 +178,12 @@ func runWorker(cfg *config.Config) {
|
||||
|
||||
taskHandler := createTaskHandler(runtime, appLogger)
|
||||
taskHandler.RegisterHandlers()
|
||||
carrierRuntime := startCarrierScheduler(ctx, cfg, runtime, taskHandler.PollingSteps(), runsSingletonModules(cfg.Worker.Role), func() {
|
||||
if pollingInitializer != nil {
|
||||
pollingInitializer.Restart(ctx)
|
||||
}
|
||||
}, appLogger)
|
||||
runtime.carrierReinit = carrierRuntime.RequestReinit
|
||||
registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
|
||||
registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
||||
registerRefundChannelRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
||||
@@ -183,9 +196,9 @@ func runWorker(cfg *config.Config) {
|
||||
taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle)
|
||||
partitionMaintainer := gatewaylimit.NewPartitionMaintainer(postgres.NewCardStopResumeRecordStore(runtime.db), runtime.gatewayLimit.Settings, appLogger)
|
||||
taskHandler.GetMux().HandleFunc(constants.TaskTypeStopResumeRecordPartition, partitionMaintainer.Handle)
|
||||
recalcHandler := polling.NewRecalculateTaskHandler(
|
||||
polling.NewDueRecalculator(runtime.db, runtime.redisClient, runtime.pollingConfigMgr, runtime.pollingQueueMgr, appLogger),
|
||||
runtime.redisClient, appLogger)
|
||||
dueRecalculator := polling.NewDueRecalculator(runtime.db, runtime.redisClient, runtime.pollingConfigMgr, runtime.pollingQueueMgr, appLogger)
|
||||
dueRecalculator.SetCarrierScheduling(runtime.carrierScheduling)
|
||||
recalcHandler := polling.NewRecalculateTaskHandler(dueRecalculator, runtime.redisClient, appLogger)
|
||||
taskHandler.GetMux().HandleFunc(constants.TaskTypePollingRecalculateDue, recalcHandler.Handle)
|
||||
startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger)
|
||||
rescuePendingImportTasks(ctx, runtime, appLogger)
|
||||
@@ -379,6 +392,7 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L
|
||||
// 自动续费的复机执行同样复用既有停复机单一事实源。
|
||||
assetAutoRenewalService: workerResult.Services.AssetAutoRenewal,
|
||||
}
|
||||
newCarrierSchedulerParts(runtime, appLogger)
|
||||
registerNotificationOutboxConsumer(runtime, appLogger)
|
||||
registerWalletOutboxConsumer(runtime, appLogger)
|
||||
registerCardObservationOutboxConsumer(runtime, appLogger)
|
||||
@@ -712,7 +726,10 @@ func registerCardObservationOutboxConsumer(runtime *workerRuntime, appLogger *za
|
||||
queue.NewCardObservationSeriesScheduler(runtime.outboxQueueClient),
|
||||
cardObservationInfra.NewSeriesAttemptLogger(integrationlog.NewRepository(runtime.db)),
|
||||
)
|
||||
seriesConsumer := cardObservationInfra.NewSeriesRequestedConsumer(seriesTrigger)
|
||||
seriesConsumer := &carrierAwareSeriesConsumer{
|
||||
legacy: cardObservationInfra.NewSeriesRequestedConsumer(seriesTrigger),
|
||||
requester: polling.NewCheckRequester(runtime.carrierScheduling, runtime.db, appLogger),
|
||||
}
|
||||
if err := runtime.outboxConsumers.Register(constants.OutboxEventTypeCardSeriesRequested, seriesConsumer); err != nil {
|
||||
appLogger.Fatal("注册业务卡观测序列 Outbox 消费者失败",
|
||||
zap.String("event_type", constants.OutboxEventTypeCardSeriesRequested), zap.Error(err))
|
||||
@@ -842,6 +859,13 @@ func startPollingInitializer(ctx context.Context, runtime *workerRuntime, appLog
|
||||
pollingInitializer.StartBackground(ctx)
|
||||
|
||||
runtime.pollingConfigMgr.WatchChanges(ctx, func(hadConfigs, hasConfigs bool) {
|
||||
if runtime.carrierScheduling.Active(ctx) {
|
||||
// 按运营商账户调度时按新配置重新初始化新结构,旧分片队列在回退时再重建。
|
||||
if runtime.carrierReinit != nil {
|
||||
runtime.carrierReinit()
|
||||
}
|
||||
return
|
||||
}
|
||||
if hasConfigs {
|
||||
appLogger.Info("轮询配置已变更,触发队列重新初始化",
|
||||
zap.Bool("had_configs", hadConfigs))
|
||||
@@ -894,6 +918,7 @@ func startPollingScheduler(
|
||||
dataResetHandler,
|
||||
)
|
||||
pollingScheduler.SetInitializer(pollingInitializer)
|
||||
pollingScheduler.SetCarrierScheduling(runtime.carrierScheduling)
|
||||
|
||||
if err := pollingScheduler.Start(ctx); err != nil {
|
||||
appLogger.Error("启动轮询调度器失败", zap.Error(err))
|
||||
|
||||
121
internal/bootstrap/carrier_check.go
Normal file
121
internal/bootstrap/carrier_check.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// carrierAwareSeriesDispatcher 是读入口观测分发的模式接缝:按运营商账户调度时把读入口改为按场景请求检查
|
||||
// (同卡合并、冷却、受账户限流),不再起"立即、3 分钟、5 分钟"观测序列;旧分片调度时沿用原分发器。
|
||||
type carrierAwareSeriesDispatcher struct {
|
||||
legacy cardObservationApp.BestEffortSeriesDispatcher
|
||||
requester *polling.CheckRequester
|
||||
}
|
||||
|
||||
var _ cardObservationApp.BestEffortSeriesDispatcher = (*carrierAwareSeriesDispatcher)(nil)
|
||||
|
||||
// ReadEntryCheckScene 把读入口与设备控制的观测场景映射为请求检查场景;不认识的场景返回空串。
|
||||
func ReadEntryCheckScene(observationScene string) string {
|
||||
switch observationScene {
|
||||
case constants.CardObservationSceneClientAssetRead:
|
||||
return constants.PollingCheckSceneClientAssetDetail
|
||||
case constants.CardObservationSceneAdminAssetRead:
|
||||
return constants.PollingCheckSceneAdminAssetRealtime
|
||||
case constants.CardObservationSceneClientRealnameLink, constants.CardObservationSceneAdminRealnameLink:
|
||||
return constants.PollingCheckSceneRealnameLink
|
||||
case constants.CardObservationSceneBusinessStop, constants.CardObservationSceneBusinessResume:
|
||||
return constants.PollingCheckSceneStopResumeSucceeded
|
||||
case constants.CardObservationScenePackageChanged:
|
||||
return constants.PollingCheckScenePackageActivated
|
||||
case constants.CardObservationSceneDeviceSwitchCard, constants.CardObservationSceneDeviceSwitchMode,
|
||||
constants.CardObservationSceneDeviceReboot, constants.CardObservationSceneDeviceReset, constants.CardObservationSceneDeviceSetWiFi:
|
||||
return constants.PollingCheckSceneDeviceOperation
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (d *carrierAwareSeriesDispatcher) active(ctx context.Context) bool {
|
||||
return d.requester != nil && d.requester.Active(ctx)
|
||||
}
|
||||
|
||||
// Dispatch 读入口单卡(或设备)观测。
|
||||
func (d *carrierAwareSeriesDispatcher) Dispatch(ctx context.Context, request cardObservationApp.SeriesRequest) {
|
||||
if !d.active(ctx) {
|
||||
d.legacy.Dispatch(ctx, request)
|
||||
return
|
||||
}
|
||||
scene := ReadEntryCheckScene(request.Scene)
|
||||
if scene == "" {
|
||||
return
|
||||
}
|
||||
id, err := strconv.ParseUint(request.ResourceID, 10, 64)
|
||||
if err != nil || id == 0 {
|
||||
return
|
||||
}
|
||||
if request.ResourceType == constants.CardObservationResourceTypeDevice {
|
||||
d.requester.RequestDevice(ctx, scene, uint(id))
|
||||
return
|
||||
}
|
||||
d.requester.RequestCards(ctx, scene, []uint{uint(id)}, request.ExpectedValue)
|
||||
}
|
||||
|
||||
// DispatchDeviceCards 设备有效绑定卡观测。
|
||||
func (d *carrierAwareSeriesDispatcher) DispatchDeviceCards(ctx context.Context, request cardObservationApp.DeviceCardsSeriesRequest) {
|
||||
if !d.active(ctx) {
|
||||
d.legacy.DispatchDeviceCards(ctx, request)
|
||||
return
|
||||
}
|
||||
if scene := ReadEntryCheckScene(request.Request.Scene); scene != "" {
|
||||
d.requester.RequestDevice(ctx, scene, request.DeviceID)
|
||||
}
|
||||
}
|
||||
|
||||
// DispatchDeviceControl 设备控制成功后的观测:按设备操作场景为设备下各卡请求检查。
|
||||
func (d *carrierAwareSeriesDispatcher) DispatchDeviceControl(ctx context.Context, request cardObservationApp.DeviceControlSeriesRequest) {
|
||||
if !d.active(ctx) {
|
||||
d.legacy.DispatchDeviceControl(ctx, request)
|
||||
return
|
||||
}
|
||||
// 设备信息同步(query_device_info,非运营商上游)保持原观测,只把卡检查改为请求检查。
|
||||
deviceInfo := request.Request
|
||||
deviceInfo.ResourceType = constants.CardObservationResourceTypeDevice
|
||||
deviceInfo.ResourceID = strconv.FormatUint(uint64(request.DeviceID), 10)
|
||||
deviceInfo.SyncType = constants.CardObservationSyncTypeDeviceInfo
|
||||
deviceInfo.ExpectedValue = strings.TrimSpace(request.TargetICCID)
|
||||
d.legacy.Dispatch(ctx, deviceInfo)
|
||||
cardIDs := append([]uint(nil), request.BoundCardIDs...)
|
||||
for _, id := range []uint{request.SourceCardID, request.TargetCardID} {
|
||||
if id != 0 && !containsUint(cardIDs, id) {
|
||||
cardIDs = append(cardIDs, id)
|
||||
}
|
||||
}
|
||||
if len(cardIDs) == 0 && request.DeviceID != 0 {
|
||||
d.requester.RequestDevice(ctx, constants.PollingCheckSceneDeviceOperation, request.DeviceID)
|
||||
return
|
||||
}
|
||||
d.requester.RequestCards(ctx, constants.PollingCheckSceneDeviceOperation, cardIDs, "")
|
||||
}
|
||||
|
||||
// DispatchRealnameWithCapability 获取实名链接后的实名观测。
|
||||
func (d *carrierAwareSeriesDispatcher) DispatchRealnameWithCapability(ctx context.Context, request cardObservationApp.RealnameCapabilitySeriesRequest) {
|
||||
if !d.active(ctx) {
|
||||
d.legacy.DispatchRealnameWithCapability(ctx, request)
|
||||
return
|
||||
}
|
||||
d.Dispatch(ctx, request.Request)
|
||||
}
|
||||
|
||||
func containsUint(values []uint, target uint) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -337,10 +337,12 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers {
|
||||
PollingConfig: admin.NewPollingConfigHandler(svc.PollingConfig),
|
||||
GatewayRateLimit: admin.NewGatewayRateLimitHandler(pollingSvcPkg.NewGatewayRateLimitService(deps.DB, auditInfra.NewWriter(auditInfra.NewRegistry(), nil), deps.GatewayLimit), deps.Redis, validate),
|
||||
PollingConcurrency: admin.NewPollingConcurrencyHandler(svc.PollingConcurrency),
|
||||
PollingMonitoring: admin.NewPollingMonitoringHandler(svc.PollingMonitoring),
|
||||
PollingAlert: admin.NewPollingAlertHandler(svc.PollingAlert),
|
||||
PollingCleanup: admin.NewPollingCleanupHandler(svc.PollingCleanup),
|
||||
PollingManualTrigger: admin.NewPollingManualTriggerHandler(svc.PollingManualTrigger),
|
||||
CarrierScheduler: admin.NewCarrierSchedulerHandler(pollingSvcPkg.NewCarrierSchedulerService(deps.DB, deps.Redis,
|
||||
auditInfra.NewWriter(auditInfra.NewRegistry(), nil), carrierSchedulerConfig()), validate),
|
||||
PollingMonitoring: admin.NewPollingMonitoringHandler(svc.PollingMonitoring),
|
||||
PollingAlert: admin.NewPollingAlertHandler(svc.PollingAlert),
|
||||
PollingCleanup: admin.NewPollingCleanupHandler(svc.PollingCleanup),
|
||||
PollingManualTrigger: admin.NewPollingManualTriggerHandler(svc.PollingManualTrigger),
|
||||
PollingPriority: admin.NewPriorityPollingHandler(
|
||||
svc.PriorityPolling, priorityPollingQuery.NewQuery(deps.DB, pollingPriorityItemStore), validate,
|
||||
),
|
||||
@@ -408,3 +410,11 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers {
|
||||
AgentOpenAPI: openapiHandler.NewHandler(svc.AgentOpenAPI, validate),
|
||||
}
|
||||
}
|
||||
|
||||
// carrierSchedulerConfig 返回按运营商账户调度的 yaml 参数(供后台校验执行上限与展示默认值)。
|
||||
func carrierSchedulerConfig() config.CarrierSchedulerConfig {
|
||||
if cfg := config.Get(); cfg != nil {
|
||||
return cfg.Polling.Carrier
|
||||
}
|
||||
return config.CarrierSchedulerConfig{}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
walletinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wallet"
|
||||
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
assetAutoRenewalQuery "github.com/break/junhong_cmp_fiber/internal/query/assetautorenewal"
|
||||
accountSvc "github.com/break/junhong_cmp_fiber/internal/service/account"
|
||||
agentOpenAPISvc "github.com/break/junhong_cmp_fiber/internal/service/agent_open_api"
|
||||
@@ -221,7 +222,8 @@ func initServices(s *stores, deps *Dependencies) *services {
|
||||
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration, auditWriter),
|
||||
cardObservationInfra.NewSeriesAttemptLogger(seriesIntegration),
|
||||
)
|
||||
observationSeries := cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier)
|
||||
// 读入口观测分发经模式接缝:按运营商账户调度时改为按场景请求检查,旧分片调度时沿用原分发器。
|
||||
observationSeries := &carrierAwareSeriesDispatcher{legacy: cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier)}
|
||||
iotCard.SetObservationSeriesDispatcher(observationSeries)
|
||||
// 使用 PollingLifecycleService 替代 APICallback:通过分片队列准确操作(修复 api_callback.go 遗漏 protect 队列的 Bug3)
|
||||
pollingConfigStore := postgres.NewPollingConfigStore(deps.DB)
|
||||
@@ -234,6 +236,12 @@ func initServices(s *stores, deps *Dependencies) *services {
|
||||
pollingConfigMgr.Start(context.Background())
|
||||
pollingQueueMgr := polling.NewPollingQueueManager(deps.Redis, constants.PollingShardCount, deps.Logger)
|
||||
pollingLifecycleSvc := polling.NewPollingLifecycleService(pollingQueueMgr, pollingConfigMgr, s.IotCard, s.DeviceSimBinding, s.Device, deps.Logger)
|
||||
// 按运营商账户调度:模式为 carrier 时 API 侧生命周期事件写入账户队列与卡到期信息。
|
||||
carrierSettings := carrierqueue.NewDefaultSettingsProvider(deps.DB, deps.Redis, deps.Logger)
|
||||
carrierStore := carrierqueue.NewStore(deps.Redis)
|
||||
carrierScheduling := polling.NewCarrierScheduling(carrierStore, carrierSettings, pollingConfigMgr, deps.Logger)
|
||||
pollingLifecycleSvc.SetCarrierScheduling(carrierScheduling)
|
||||
observationSeries.requester = polling.NewCheckRequester(carrierScheduling, deps.DB, deps.Logger)
|
||||
iotCard.SetPollingCallback(pollingLifecycleSvc)
|
||||
// 创建支付配置服务(Order 和 Recharge 依赖)
|
||||
wechatConfig := wechatConfigSvc.New(s.WechatConfig, s.Order, s.RechargeOrder, s.AgentRecharge, s.Payment, auditWriter, deps.Redis, deps.Logger)
|
||||
|
||||
@@ -65,6 +65,7 @@ type Handlers struct {
|
||||
WeComApprovalCallback *callback.WeComApprovalHandler
|
||||
PollingConfig *admin.PollingConfigHandler
|
||||
GatewayRateLimit *admin.GatewayRateLimitHandler
|
||||
CarrierScheduler *admin.CarrierSchedulerHandler
|
||||
PollingConcurrency *admin.PollingConcurrencyHandler
|
||||
PollingMonitoring *admin.PollingMonitoringHandler
|
||||
PollingAlert *admin.PollingAlertHandler
|
||||
|
||||
@@ -91,6 +91,7 @@ type CallGuard interface {
|
||||
type cardCallKey struct{}
|
||||
type carrierKey struct{}
|
||||
type exemptKey struct{}
|
||||
type rateLimitWaitKey struct{}
|
||||
|
||||
type cardCall struct {
|
||||
operation string
|
||||
@@ -113,6 +114,21 @@ func WithRateLimitExempt(ctx context.Context, source string) context.Context {
|
||||
return context.WithValue(ctx, exemptKey{}, source)
|
||||
}
|
||||
|
||||
// WithRateLimitWait 声明本次调用在拿不到限流名额时等待,总等待不超过 maxWait;超时返回限流错误。
|
||||
// 只用于不占 Asynq 协程的执行方(按运营商账户调度的 NSQ 消费者);未声明时拿不到名额立即返回。
|
||||
func WithRateLimitWait(ctx context.Context, maxWait time.Duration) context.Context {
|
||||
if maxWait <= 0 {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, rateLimitWaitKey{}, maxWait)
|
||||
}
|
||||
|
||||
// RateLimitWait 返回 ctx 中声明的限流最长等待;未声明返回 0。
|
||||
func RateLimitWait(ctx context.Context) time.Duration {
|
||||
wait, _ := ctx.Value(rateLimitWaitKey{}).(time.Duration)
|
||||
return wait
|
||||
}
|
||||
|
||||
// RateLimitExemptSource 返回 ctx 中声明的豁免来源,未声明返回空串。
|
||||
func RateLimitExemptSource(ctx context.Context) string {
|
||||
source, _ := ctx.Value(exemptKey{}).(string)
|
||||
|
||||
195
internal/handler/admin/carrier_scheduler.go
Normal file
195
internal/handler/admin/carrier_scheduler.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/go-playground/validator/v10"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||||
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/response"
|
||||
)
|
||||
|
||||
// CarrierSchedulerHandler 按运营商账户调度的后台接口:调度参数、请求检查场景、失败告警、单卡轮询状态、按账户监控。
|
||||
type CarrierSchedulerHandler struct {
|
||||
service *pollingSvc.CarrierSchedulerService
|
||||
validator *validator.Validate
|
||||
}
|
||||
|
||||
// NewCarrierSchedulerHandler 创建按运营商账户调度 Handler。
|
||||
func NewCarrierSchedulerHandler(service *pollingSvc.CarrierSchedulerService, validator *validator.Validate) *CarrierSchedulerHandler {
|
||||
return &CarrierSchedulerHandler{service: service, validator: validator}
|
||||
}
|
||||
|
||||
func (h *CarrierSchedulerHandler) validate(prefix string, req any) error {
|
||||
if err := h.validator.Struct(req); err != nil {
|
||||
return errors.New(errors.CodeInvalidParam, validationMessage(prefix, req, err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListParams 查询运营商账户调度参数列表。GET /api/admin/polling-carrier-params
|
||||
func (h *CarrierSchedulerHandler) ListParams(c *fiber.Ctx) error {
|
||||
result, err := h.service.ListParams(c.UserContext())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// GetParam 查询运营商账户调度参数。GET /api/admin/polling-carrier-params/:carrier_id
|
||||
func (h *CarrierSchedulerHandler) GetParam(c *fiber.Ctx) error {
|
||||
carrierID, err := rateLimitPathID(c, "carrier_id", "运营商账户 ID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.service.GetParam(c.UserContext(), carrierID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// SaveParam 保存运营商账户调度参数。PUT /api/admin/polling-carrier-params/:carrier_id
|
||||
func (h *CarrierSchedulerHandler) SaveParam(c *fiber.Ctx) error {
|
||||
var req dto.SaveCarrierPollingParamRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||
}
|
||||
carrierID, err := rateLimitPathID(c, "carrier_id", "运营商账户 ID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.CarrierID = carrierID
|
||||
if err := h.validate("运营商账户调度参数不合法", &req); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.service.SaveParam(c.UserContext(), &req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// DeleteParam 删除运营商账户调度参数(恢复全局默认)。DELETE /api/admin/polling-carrier-params/:carrier_id
|
||||
func (h *CarrierSchedulerHandler) DeleteParam(c *fiber.Ctx) error {
|
||||
carrierID, err := rateLimitPathID(c, "carrier_id", "运营商账户 ID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.service.DeleteParam(c.UserContext(), carrierID); err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, nil)
|
||||
}
|
||||
|
||||
// ListScenes 查询请求检查场景。GET /api/admin/polling-check-scenes
|
||||
func (h *CarrierSchedulerHandler) ListScenes(c *fiber.Ctx) error {
|
||||
result, err := h.service.ListScenes(c.UserContext())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// UpdateScene 修改请求检查场景。PUT /api/admin/polling-check-scenes/:scene
|
||||
func (h *CarrierSchedulerHandler) UpdateScene(c *fiber.Ctx) error {
|
||||
var req dto.UpdatePollingCheckSceneRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||
}
|
||||
req.Scene = strings.TrimSpace(c.Params("scene"))
|
||||
if req.Scene == "" {
|
||||
return errors.New(errors.CodeInvalidParam, "无效的场景编码")
|
||||
}
|
||||
if err := h.validate("请求检查场景参数不合法", &req); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.service.UpdateScene(c.UserContext(), &req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// ListAlerts 查询卡轮询失败告警列表。GET /api/admin/polling-failure-alerts
|
||||
func (h *CarrierSchedulerHandler) ListAlerts(c *fiber.Ctx) error {
|
||||
var req dto.PollingFailureAlertListRequest
|
||||
if err := c.QueryParser(&req); err != nil {
|
||||
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||
}
|
||||
if err := h.validate("卡轮询失败告警查询参数不合法", &req); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.service.ListAlerts(c.UserContext(), &req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// AlertSummary 未处理告警按运营商账户 × 错误类别汇总。GET /api/admin/polling-failure-alerts/summary
|
||||
func (h *CarrierSchedulerHandler) AlertSummary(c *fiber.Ctx) error {
|
||||
result, err := h.service.AlertSummary(c.UserContext())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// GetAlert 查询卡轮询失败告警详情。GET /api/admin/polling-failure-alerts/:id
|
||||
func (h *CarrierSchedulerHandler) GetAlert(c *fiber.Ctx) error {
|
||||
id, err := rateLimitPathID(c, "id", "告警 ID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.service.GetAlert(c.UserContext(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// HandleAlert 标记卡轮询失败告警已处理。PUT /api/admin/polling-failure-alerts/:id/status
|
||||
func (h *CarrierSchedulerHandler) HandleAlert(c *fiber.Ctx) error {
|
||||
var req dto.HandlePollingFailureAlertRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||
}
|
||||
id, err := rateLimitPathID(c, "id", "告警 ID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.ID = id
|
||||
if err := h.validate("卡轮询失败告警处理参数不合法", &req); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.service.HandleAlert(c.UserContext(), &req); err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, nil)
|
||||
}
|
||||
|
||||
// CardState 查询单卡轮询状态。GET /api/admin/iot-cards/:id/polling-state
|
||||
func (h *CarrierSchedulerHandler) CardState(c *fiber.Ctx) error {
|
||||
id, err := rateLimitPathID(c, "id", "卡 ID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.service.CardState(c.UserContext(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// CarrierStats 按运营商账户的轮询队列与执行监控。GET /api/admin/polling-stats/carrier-queues
|
||||
func (h *CarrierSchedulerHandler) CarrierStats(c *fiber.Ctx) error {
|
||||
result, err := h.service.CarrierStats(c.UserContext())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
@@ -277,6 +277,10 @@ func NewRegistry() *Registry {
|
||||
gatewayErrorRuleCreated := pollingAction(constants.AuditActionGatewayErrorRuleCreated, "创建上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
gatewayErrorRuleUpdated := pollingAction(constants.AuditActionGatewayErrorRuleUpdated, "更新上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
gatewayErrorRuleStatusUpdated := pollingAction(constants.AuditActionGatewayErrorRuleStatusUpdated, "更新上游错误分类规则状态", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
carrierPollingParamSaved := pollingAction(constants.AuditActionCarrierPollingParamSaved, "保存运营商账户调度参数", constants.AuditResourceCarrierPollingParam, constants.AuditRiskHigh)
|
||||
carrierPollingParamDeleted := pollingAction(constants.AuditActionCarrierPollingParamDeleted, "删除运营商账户调度参数", constants.AuditResourceCarrierPollingParam, constants.AuditRiskHigh)
|
||||
pollingCheckSceneUpdated := pollingAction(constants.AuditActionPollingCheckSceneUpdated, "修改请求检查场景", constants.AuditResourcePollingCheckScene, constants.AuditRiskNormal)
|
||||
pollingFailureAlertHandled := pollingAction(constants.AuditActionPollingFailureAlertHandled, "处理卡轮询失败告警", constants.AuditResourcePollingFailureAlert, constants.AuditRiskNormal)
|
||||
gatewayErrorRuleDeleted := pollingAction(constants.AuditActionGatewayErrorRuleDeleted, "删除上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
pollingConcurrencyUpdated := pollingAction(constants.AuditActionPollingConcurrencyUpdated, "更新轮询并发配置", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
||||
pollingConcurrencyReset := pollingAction(constants.AuditActionPollingConcurrencyReset, "重置轮询并发计数", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
||||
@@ -678,6 +682,10 @@ func NewRegistry() *Registry {
|
||||
constants.AuditActionGatewayErrorRuleUpdated: gatewayErrorRuleUpdated,
|
||||
constants.AuditActionGatewayErrorRuleStatusUpdated: gatewayErrorRuleStatusUpdated,
|
||||
constants.AuditActionGatewayErrorRuleDeleted: gatewayErrorRuleDeleted,
|
||||
constants.AuditActionCarrierPollingParamSaved: carrierPollingParamSaved,
|
||||
constants.AuditActionCarrierPollingParamDeleted: carrierPollingParamDeleted,
|
||||
constants.AuditActionPollingCheckSceneUpdated: pollingCheckSceneUpdated,
|
||||
constants.AuditActionPollingFailureAlertHandled: pollingFailureAlertHandled,
|
||||
constants.AuditActionPollingConcurrencyUpdated: pollingConcurrencyUpdated,
|
||||
constants.AuditActionPollingConcurrencyReset: pollingConcurrencyReset,
|
||||
constants.AuditActionPollingAlertRuleCreated: pollingAlertRuleCreated,
|
||||
@@ -966,6 +974,18 @@ func NewRegistry() *Registry {
|
||||
Type: constants.AuditResourceGatewayErrorRule, Name: "上游错误分类规则",
|
||||
IdentityFields: []string{"id", "keyword", "match_type", "category", "priority", "status"},
|
||||
},
|
||||
constants.AuditResourceCarrierPollingParam: {
|
||||
Type: constants.AuditResourceCarrierPollingParam, Name: "运营商账户调度参数",
|
||||
IdentityFields: []string{"carrier_id", "inflight_limit", "card_max_in_flight", "priority_max_ratio"},
|
||||
},
|
||||
constants.AuditResourcePollingCheckScene: {
|
||||
Type: constants.AuditResourcePollingCheckScene, Name: "请求检查场景",
|
||||
IdentityFields: []string{"scene", "scene_name"},
|
||||
},
|
||||
constants.AuditResourcePollingFailureAlert: {
|
||||
Type: constants.AuditResourcePollingFailureAlert, Name: "卡轮询失败告警",
|
||||
IdentityFields: []string{"id", "alert_level", "card_id", "carrier_id", "check_type", "status"},
|
||||
},
|
||||
constants.AuditResourcePollingAlertRule: {
|
||||
Type: constants.AuditResourcePollingAlertRule, Name: "轮询告警规则",
|
||||
IdentityFields: []string{"id", "rule_name", "task_type", "metric_type", "operator", "threshold", "alert_level", "status"},
|
||||
|
||||
@@ -22,6 +22,9 @@ const (
|
||||
statPrefixDuration = "duration_le:"
|
||||
)
|
||||
|
||||
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
|
||||
const StatFieldRateLimitWait = "rate_wait"
|
||||
|
||||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||||
|
||||
@@ -60,6 +63,18 @@ func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error
|
||||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||||
}
|
||||
if !decision.Allowed {
|
||||
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
|
||||
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
|
||||
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
|
||||
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
|
||||
if allowed {
|
||||
return nil
|
||||
}
|
||||
if waitErr != nil && ctx.Err() == nil {
|
||||
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||||
return gateway.NewRateLimitedError()
|
||||
}
|
||||
|
||||
@@ -189,3 +189,50 @@ func windowMember(nowMS int64) string {
|
||||
_, _ = rand.Read(buf)
|
||||
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// Budget 返回运营商账户各查询类操作所属限流组本秒剩余名额,供分发器按预计调用数领取。
|
||||
// groups 为限流组 → 本秒剩余;opGroup 为操作 → 限流组。只统计模式下返回 limited=false(不按 QPS 限制领取)。
|
||||
// 全局组只在账户组之外再收紧:剩余名额取两者较小值。
|
||||
func (l *Limiter) Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool) {
|
||||
if l == nil || l.redis == nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
settings := l.settings.Current(ctx)
|
||||
if !settings.Enforce() {
|
||||
return nil, nil, false
|
||||
}
|
||||
groups, opGroup = map[string]int{}, map[string]string{}
|
||||
now := time.Now().UnixMilli()
|
||||
since := now - constants.GatewayRateLimitWindow.Milliseconds()
|
||||
for _, op := range ops {
|
||||
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
|
||||
if !accountOK {
|
||||
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
|
||||
}
|
||||
code := account.GroupCode
|
||||
opGroup[op] = code
|
||||
if _, seen := groups[code]; seen {
|
||||
continue
|
||||
}
|
||||
limit := account.MaxQPS
|
||||
if !account.HighPriority {
|
||||
limit -= account.ReservedQPS
|
||||
}
|
||||
used, err := l.redis.ZCount(ctx, constants.RedisGatewayQPSKey(carrierID, code), strconv.FormatInt(since, 10), "+inf").Result()
|
||||
if err != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
remaining := limit - int(used)
|
||||
if globalOK {
|
||||
globalUsed, gerr := l.redis.ZCount(ctx, constants.RedisGatewayGlobalQPSKey(global.GroupCode), strconv.FormatInt(since, 10), "+inf").Result()
|
||||
if gerr == nil && global.MaxQPS-int(globalUsed) < remaining {
|
||||
remaining = global.MaxQPS - int(globalUsed)
|
||||
}
|
||||
}
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
groups[code] = remaining
|
||||
}
|
||||
return groups, opGroup, true
|
||||
}
|
||||
|
||||
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
@@ -0,0 +1,215 @@
|
||||
// Package nsqbus 封装卡轮询使用的 NSQ 生产者与按 topic 动态订阅的消费者管理器。
|
||||
// NSQ 只承担执行分发与消费者并发控制,业务状态与重试全部在 Redis(add-carrier-polling-scheduler)。
|
||||
package nsqbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nsqio/go-nsq"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Producer 是 nsqd 生产者。
|
||||
type Producer struct {
|
||||
producer *nsq.Producer
|
||||
}
|
||||
|
||||
// NewProducer 连接 nsqd TCP 地址创建生产者。
|
||||
func NewProducer(address string, logger *zap.Logger) (*Producer, error) {
|
||||
cfg := nsq.NewConfig()
|
||||
producer, err := nsq.NewProducer(address, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
producer.SetLogger(nsqLogger{logger: logger}, nsq.LogLevelWarning)
|
||||
return &Producer{producer: producer}, nil
|
||||
}
|
||||
|
||||
// Publish 同步发布一条消息;topic 不存在时由 nsqd 自动创建。
|
||||
func (p *Producer) Publish(topic string, body []byte) error {
|
||||
return p.producer.Publish(topic, body)
|
||||
}
|
||||
|
||||
// Stop 停止生产者。
|
||||
func (p *Producer) Stop() {
|
||||
if p != nil && p.producer != nil {
|
||||
p.producer.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// Handler 处理一条消息;返回后由管理器 FIN。消息处理期间可调用 touch 延长 NSQ 消息超时。
|
||||
type Handler func(ctx context.Context, body []byte, touch func())
|
||||
|
||||
// ConsumerConfig 是消费者参数。
|
||||
type ConsumerConfig struct {
|
||||
NSQDAddress string
|
||||
NSQLookupdAddresses []string
|
||||
MsgTimeout time.Duration
|
||||
MaxAttempts uint16
|
||||
}
|
||||
|
||||
// ConsumerManager 按 topic 动态订阅与取消订阅;每个 topic 一个消费者,max_in_flight 控制本进程对该 topic 的并发。
|
||||
type ConsumerManager struct {
|
||||
cfg ConsumerConfig
|
||||
channel string
|
||||
handler Handler
|
||||
logger *zap.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
consumers map[string]*subscription
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
inflight sync.WaitGroup
|
||||
}
|
||||
|
||||
type subscription struct {
|
||||
consumer *nsq.Consumer
|
||||
maxInFlight int
|
||||
}
|
||||
|
||||
// NewConsumerManager 创建消费者管理器。
|
||||
func NewConsumerManager(cfg ConsumerConfig, channel string, handler Handler, logger *zap.Logger) *ConsumerManager {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &ConsumerManager{cfg: cfg, channel: channel, handler: handler, logger: logger,
|
||||
consumers: map[string]*subscription{}, ctx: ctx, cancel: cancel}
|
||||
}
|
||||
|
||||
// Sync 使订阅集合与 desired(topic → max_in_flight)一致:新增订阅、取消多余订阅、调整 max_in_flight。
|
||||
func (m *ConsumerManager) Sync(desired map[string]int) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
for topic, sub := range m.consumers {
|
||||
if _, keep := desired[topic]; !keep {
|
||||
sub.consumer.Stop()
|
||||
delete(m.consumers, topic)
|
||||
m.logger.Info("取消订阅卡任务 topic", zap.String("topic", topic))
|
||||
}
|
||||
}
|
||||
for topic, maxInFlight := range desired {
|
||||
if maxInFlight <= 0 {
|
||||
maxInFlight = 1
|
||||
}
|
||||
if sub, ok := m.consumers[topic]; ok {
|
||||
if sub.maxInFlight != maxInFlight {
|
||||
sub.consumer.ChangeMaxInFlight(maxInFlight)
|
||||
sub.maxInFlight = maxInFlight
|
||||
}
|
||||
continue
|
||||
}
|
||||
consumer, err := m.subscribe(topic, maxInFlight)
|
||||
if err != nil {
|
||||
m.logger.Warn("订阅卡任务 topic 失败,下一轮重试", zap.String("topic", topic), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
m.consumers[topic] = &subscription{consumer: consumer, maxInFlight: maxInFlight}
|
||||
m.logger.Info("订阅卡任务 topic", zap.String("topic", topic), zap.Int("max_in_flight", maxInFlight))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ConsumerManager) subscribe(topic string, maxInFlight int) (*nsq.Consumer, error) {
|
||||
cfg := nsq.NewConfig()
|
||||
cfg.MaxInFlight = maxInFlight
|
||||
if m.cfg.MsgTimeout > 0 {
|
||||
cfg.MsgTimeout = m.cfg.MsgTimeout
|
||||
}
|
||||
if m.cfg.MaxAttempts > 0 {
|
||||
cfg.MaxAttempts = m.cfg.MaxAttempts
|
||||
}
|
||||
consumer, err := nsq.NewConsumer(topic, m.channel, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
consumer.SetLogger(nsqLogger{logger: m.logger}, nsq.LogLevelWarning)
|
||||
handlerCtx := m.ctx // subscribe 在持锁时调用,捕获当前处理上下文
|
||||
consumer.AddConcurrentHandlers(nsq.HandlerFunc(func(message *nsq.Message) error {
|
||||
message.DisableAutoResponse()
|
||||
m.inflight.Add(1)
|
||||
defer m.inflight.Done()
|
||||
defer message.Finish()
|
||||
m.handler(handlerCtx, message.Body, message.Touch)
|
||||
return nil
|
||||
}), maxInFlight)
|
||||
if len(m.cfg.NSQLookupdAddresses) > 0 {
|
||||
err = consumer.ConnectToNSQLookupds(m.cfg.NSQLookupdAddresses)
|
||||
} else {
|
||||
err = consumer.ConnectToNSQD(m.cfg.NSQDAddress)
|
||||
}
|
||||
if err != nil {
|
||||
consumer.Stop()
|
||||
return nil, err
|
||||
}
|
||||
return consumer, nil
|
||||
}
|
||||
|
||||
// Topics 返回当前已订阅的 topic。
|
||||
func (m *ConsumerManager) Topics() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
topics := make([]string, 0, len(m.consumers))
|
||||
for topic := range m.consumers {
|
||||
topics = append(topics, topic)
|
||||
}
|
||||
return topics
|
||||
}
|
||||
|
||||
// StopAll 停止全部订阅;在宽限期内等待执行中的消息完成并 FIN,超过宽限期后取消处理上下文。
|
||||
func (m *ConsumerManager) StopAll(grace time.Duration) {
|
||||
m.mu.Lock()
|
||||
for topic, sub := range m.consumers {
|
||||
sub.consumer.Stop()
|
||||
delete(m.consumers, topic)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
m.inflight.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(grace):
|
||||
m.logger.Warn("优雅退出宽限期内仍有卡任务未完成,取消执行(由租约到期后重新领取)")
|
||||
}
|
||||
m.cancel()
|
||||
}
|
||||
|
||||
// Reset 在 StopAll 后重新允许订阅(切回 carrier 模式时使用)。
|
||||
func (m *ConsumerManager) Reset() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.ctx.Err() != nil {
|
||||
m.ctx, m.cancel = context.WithCancel(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
// SplitAddresses 解析逗号分隔的地址列表。
|
||||
func SplitAddresses(raw string) []string {
|
||||
var result []string
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
result = append(result, part)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type nsqLogger struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// Output 实现 go-nsq 的日志接口。
|
||||
func (l nsqLogger) Output(_ int, s string) error {
|
||||
if l.logger != nil {
|
||||
l.logger.Warn("NSQ", zap.String("message", s))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
185
internal/model/dto/carrier_polling_dto.go
Normal file
185
internal/model/dto/carrier_polling_dto.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package dto
|
||||
|
||||
import "time"
|
||||
|
||||
// CarrierPollingParamItem 运营商账户调度参数(含全局默认合并后的生效值)。
|
||||
type CarrierPollingParamItem struct {
|
||||
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||
IsDefault bool `json:"is_default" description:"是否未单独配置、全部使用全局默认"`
|
||||
InflightLimit int `json:"inflight_limit" description:"生效的执行上限(跨 Worker 实例同时执行卡任务数)"`
|
||||
CardMaxInFlight int `json:"card_max_in_flight" description:"生效的每进程卡任务 max_in_flight"`
|
||||
PriorityMaxRatio *float64 `json:"priority_max_ratio" description:"优先队列每轮最大占比(0~1),空表示不限"`
|
||||
StopResumeMaxInFlight *int `json:"stop_resume_max_in_flight" description:"每进程停复机 max_in_flight(停复机指令队列使用),空表示默认"`
|
||||
SRInflightLimit *int `json:"sr_inflight_limit" description:"停复机执行上限(停复机指令队列使用),空表示默认"`
|
||||
StopResumeMinInterval *int `json:"stop_resume_min_interval" description:"单卡两次停复机调用最小间隔(秒),空表示默认"`
|
||||
StopResumeConfirmDelay *int `json:"stop_resume_confirm_delay" description:"停复机结果未知后确认延迟(秒),空表示默认"`
|
||||
ConfiguredInflight *int `json:"configured_inflight_limit" description:"单独配置的执行上限,空表示使用默认"`
|
||||
ConfiguredMaxInFlight *int `json:"configured_card_max_in_flight" description:"单独配置的每进程 max_in_flight,空表示使用默认"`
|
||||
ConsumeCapacity int `json:"consume_capacity" description:"当前订阅进程 max_in_flight 之和(执行上限不应超过该值)"`
|
||||
}
|
||||
|
||||
// CarrierPollingParamListResponse 运营商账户调度参数列表。
|
||||
type CarrierPollingParamListResponse struct {
|
||||
Items []CarrierPollingParamItem `json:"items" description:"各运营商账户调度参数"`
|
||||
SubscriberProcesses int `json:"subscriber_processes" description:"订阅卡任务的 Worker 进程数"`
|
||||
DefaultInflight int `json:"default_inflight_limit" description:"全局默认执行上限"`
|
||||
DefaultMaxInFlight int `json:"default_card_max_in_flight" description:"全局默认每进程 max_in_flight"`
|
||||
}
|
||||
|
||||
// CarrierPollingParamPath 运营商账户路径参数。
|
||||
type CarrierPollingParamPath struct {
|
||||
CarrierID uint `json:"-" path:"carrier_id" required:"true" description:"运营商账户ID"`
|
||||
}
|
||||
|
||||
// SaveCarrierPollingParamRequest 保存运营商账户调度参数;字段为空表示该项使用全局默认。
|
||||
type SaveCarrierPollingParamRequest struct {
|
||||
CarrierID uint `json:"-" path:"carrier_id" required:"true" description:"运营商账户ID"`
|
||||
InflightLimit *int `json:"inflight_limit" validate:"omitempty,min=1,max=10000" minimum:"1" maximum:"10000" description:"执行上限,空表示默认"`
|
||||
CardMaxInFlight *int `json:"card_max_in_flight" validate:"omitempty,min=1,max=2000" minimum:"1" maximum:"2000" description:"每进程卡任务 max_in_flight,空表示默认"`
|
||||
StopResumeMaxInFlight *int `json:"stop_resume_max_in_flight" validate:"omitempty,min=1,max=1000" minimum:"1" maximum:"1000" description:"每进程停复机 max_in_flight,空表示默认"`
|
||||
SRInflightLimit *int `json:"sr_inflight_limit" validate:"omitempty,min=1,max=10000" minimum:"1" maximum:"10000" description:"停复机执行上限,空表示默认"`
|
||||
PriorityMaxRatio *float64 `json:"priority_max_ratio" validate:"omitempty,gt=0,lte=1" description:"优先队列每轮最大占比(0~1],空表示不限"`
|
||||
StopResumeMinInterval *int `json:"stop_resume_min_interval" validate:"omitempty,min=1,max=86400" minimum:"1" maximum:"86400" description:"单卡停复机最小间隔(秒),空表示默认"`
|
||||
StopResumeConfirmDelay *int `json:"stop_resume_confirm_delay" validate:"omitempty,min=1,max=86400" minimum:"1" maximum:"86400" description:"停复机结果未知确认延迟(秒),空表示默认"`
|
||||
}
|
||||
|
||||
// PollingCheckSceneItem 请求检查场景。
|
||||
type PollingCheckSceneItem struct {
|
||||
Scene string `json:"scene" description:"场景编码 (order_paid:订单支付, package_activated:套餐激活, stop_resume_succeeded:停复机成功, device_operation:设备操作, auto_renewal_purchase:自动续费购包, client_asset_detail:C 端资产详情, admin_asset_realtime:后台资产实时状态, realname_link:实名链接)"`
|
||||
SceneName string `json:"scene_name" description:"场景名称"`
|
||||
CheckTypes []string `json:"check_types" description:"检查类型 (card_status:卡状态, carddata:流量, realname:实名)"`
|
||||
ConfirmDelays []int `json:"confirm_delays" description:"延迟确认序列(秒),空表示只立即检查一次"`
|
||||
Cooldown *int `json:"cooldown" description:"冷却时长(秒),最近一次成功检查未超过该时长则不请求;空表示不冷却"`
|
||||
UpdatedAt time.Time `json:"updated_at" description:"更新时间"`
|
||||
}
|
||||
|
||||
// UpdatePollingCheckSceneRequest 修改请求检查场景(场景由代码预置,只可改检查类型、确认延迟与冷却)。
|
||||
type UpdatePollingCheckSceneRequest struct {
|
||||
Scene string `json:"-" path:"scene" required:"true" description:"场景编码"`
|
||||
CheckTypes []string `json:"check_types" validate:"required,min=1,max=3,dive,oneof=card_status carddata realname" required:"true" description:"检查类型 (card_status:卡状态, carddata:流量, realname:实名)"`
|
||||
ConfirmDelays []int `json:"confirm_delays" validate:"omitempty,max=5,dive,min=1,max=86400" description:"延迟确认序列(秒,递增),空表示只立即检查一次"`
|
||||
Cooldown *int `json:"cooldown" validate:"omitempty,min=1,max=86400" minimum:"1" maximum:"86400" description:"冷却时长(秒),空表示不冷却"`
|
||||
}
|
||||
|
||||
// PollingFailureAlertListRequest 卡轮询失败告警列表请求。
|
||||
type PollingFailureAlertListRequest struct {
|
||||
Page int `json:"page" query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码"`
|
||||
PageSize int `json:"page_size" query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页数量"`
|
||||
CarrierID *uint `json:"carrier_id" query:"carrier_id" validate:"omitempty,min=1" description:"运营商账户ID"`
|
||||
CheckType string `json:"check_type" query:"check_type" validate:"omitempty,oneof=card_status carddata realname package all" description:"检查类型 (card_status:卡状态, carddata:流量, realname:实名, package:套餐, all:账户级)"`
|
||||
ErrorCategory string `json:"error_category" query:"error_category" validate:"omitempty,max=50" description:"错误类别"`
|
||||
Status *int `json:"status" query:"status" validate:"omitempty,oneof=1 2 3 4" description:"状态 (1:未处理, 2:已恢复, 3:已关闭轮询, 4:已处理)"`
|
||||
AlertLevel string `json:"alert_level" query:"alert_level" validate:"omitempty,oneof=card carrier" description:"告警级别 (card:卡级, carrier:账户级)"`
|
||||
ICCID string `json:"iccid" query:"iccid" validate:"omitempty,max=30" description:"ICCID(精确)"`
|
||||
StartTime string `json:"start_time" query:"start_time" description:"最近失败时间起始(带时区的 RFC3339 秒级时间)"`
|
||||
EndTime string `json:"end_time" query:"end_time" description:"最近失败时间结束(带时区的 RFC3339 秒级时间)"`
|
||||
}
|
||||
|
||||
// PollingFailureAlertItem 卡轮询失败告警。
|
||||
type PollingFailureAlertItem struct {
|
||||
ID uint `json:"id" description:"告警ID"`
|
||||
AlertLevel string `json:"alert_level" description:"告警级别 (card:卡级, carrier:账户级)"`
|
||||
CardID uint `json:"card_id" description:"卡ID,账户级为 0"`
|
||||
ICCID string `json:"iccid" description:"ICCID"`
|
||||
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||
CheckType string `json:"check_type" description:"检查类型"`
|
||||
CheckTypeName string `json:"check_type_name" description:"检查类型名称"`
|
||||
ErrorCategory string `json:"error_category" description:"最近错误类别"`
|
||||
FailCount int `json:"fail_count" description:"连续失败次数;账户级为窗口内告警数"`
|
||||
FirstFailedAt *time.Time `json:"first_failed_at" description:"首次失败时间"`
|
||||
LastFailedAt *time.Time `json:"last_failed_at" description:"最近失败时间"`
|
||||
LastError string `json:"last_error" description:"最近错误"`
|
||||
Status int `json:"status" description:"状态 (1:未处理, 2:已恢复, 3:已关闭轮询, 4:已处理)"`
|
||||
StatusName string `json:"status_name" description:"状态名称"`
|
||||
HandledBy uint `json:"handled_by" description:"处理人"`
|
||||
HandledAt *time.Time `json:"handled_at" description:"处理时间"`
|
||||
HandleRemark string `json:"handle_remark" description:"处理备注"`
|
||||
RecoveredAt *time.Time `json:"recovered_at" description:"自动恢复时间"`
|
||||
CreatedAt time.Time `json:"created_at" description:"创建时间"`
|
||||
}
|
||||
|
||||
// PollingFailureAlertPageResult 卡轮询失败告警分页结果。
|
||||
type PollingFailureAlertPageResult struct {
|
||||
List []PollingFailureAlertItem `json:"items" description:"告警列表"`
|
||||
Total int64 `json:"total" description:"总数"`
|
||||
Page int `json:"page" description:"当前页"`
|
||||
PageSize int `json:"size" description:"每页数量"`
|
||||
}
|
||||
|
||||
// PollingFailureAlertSummaryItem 未处理告警按运营商账户 × 错误类别汇总。
|
||||
type PollingFailureAlertSummaryItem struct {
|
||||
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||
ErrorCategory string `json:"error_category" description:"错误类别"`
|
||||
OpenCount int64 `json:"open_count" description:"未处理卡级告警数"`
|
||||
}
|
||||
|
||||
// PollingFailureAlertIDPath 告警路径参数。
|
||||
type PollingFailureAlertIDPath struct {
|
||||
ID uint `json:"-" path:"id" required:"true" description:"告警ID"`
|
||||
}
|
||||
|
||||
// HandlePollingFailureAlertRequest 标记告警已处理。
|
||||
type HandlePollingFailureAlertRequest struct {
|
||||
ID uint `json:"-" path:"id" required:"true" description:"告警ID"`
|
||||
Status int `json:"status" validate:"required,oneof=4" required:"true" description:"目标状态 (4:已处理);关闭轮询请使用资产轮询开关接口"`
|
||||
Remark string `json:"remark" validate:"omitempty,max=500" maxLength:"500" description:"处理备注"`
|
||||
}
|
||||
|
||||
// CardPollingStatePath 卡ID路径参数。
|
||||
type CardPollingStatePath struct {
|
||||
ID uint `json:"-" path:"id" required:"true" description:"卡ID"`
|
||||
}
|
||||
|
||||
// CardPollingTypeState 卡某检查类型的轮询状态。
|
||||
type CardPollingTypeState struct {
|
||||
CheckType string `json:"check_type" description:"检查类型"`
|
||||
CheckTypeName string `json:"check_type_name" description:"检查类型名称"`
|
||||
NextDueAt *time.Time `json:"next_due_at" description:"下次到期时间"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at" description:"最近一次成功时间"`
|
||||
FailCount int `json:"fail_count" description:"连续失败次数"`
|
||||
FirstFailedAt *time.Time `json:"first_failed_at" description:"首次失败时间"`
|
||||
LastError string `json:"last_error" description:"最近错误(类别|信息)"`
|
||||
PendingConfirm []int `json:"pending_confirm" description:"待执行的延迟确认(秒)"`
|
||||
}
|
||||
|
||||
// CardPollingStateResponse 单卡轮询状态(只读)。
|
||||
type CardPollingStateResponse struct {
|
||||
CardID uint `json:"card_id" description:"卡ID"`
|
||||
ICCID string `json:"iccid" description:"ICCID"`
|
||||
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||
SchedulerMode string `json:"scheduler_mode" description:"当前调度模式 (legacy:旧分片调度, carrier:按运营商账户调度)"`
|
||||
EnablePolling bool `json:"enable_polling" description:"卡轮询开关"`
|
||||
Queue string `json:"queue" description:"所在队列 (priority:优先, queue:普通, none:不在轮询中)"`
|
||||
QueueScoreAt *time.Time `json:"queue_score_at" description:"队列中下次可领取时间"`
|
||||
Running bool `json:"running" description:"是否执行中"`
|
||||
ClaimExpireAt *time.Time `json:"claim_expire_at" description:"处理中标记到期时间"`
|
||||
PriorityMark bool `json:"priority_mark" description:"执行中收到请求检查,收尾后进入优先队列"`
|
||||
Types []CardPollingTypeState `json:"types" description:"各检查类型状态"`
|
||||
}
|
||||
|
||||
// CarrierPollingStatsItem 按运营商账户的轮询监控。
|
||||
type CarrierPollingStatsItem struct {
|
||||
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||
PriorityLength int64 `json:"priority_length" description:"优先队列长度"`
|
||||
QueueLength int64 `json:"queue_length" description:"普通队列长度"`
|
||||
DueCount int64 `json:"due_count" description:"已到期卡数(优先 + 普通)"`
|
||||
Running int64 `json:"running" description:"执行中卡任务数"`
|
||||
InflightLimit int `json:"inflight_limit" description:"执行上限"`
|
||||
QPSLastMinute float64 `json:"qps_last_minute" description:"上一分钟实际网关调用 QPS(查询类)"`
|
||||
RateLimitWaits int64 `json:"rate_limit_waits" description:"上一分钟限流等待次数"`
|
||||
RateLimited int64 `json:"rate_limited" description:"上一分钟限流拒绝或等待超时次数"`
|
||||
ExpiredClaims1h int64 `json:"expired_claims_1h" description:"近 1 小时处理中标记到期次数(执行崩溃或超时)"`
|
||||
OpenAlerts int64 `json:"open_alerts" description:"未处理卡级失败告警数"`
|
||||
OpenAlertsByType map[string]int64 `json:"open_alerts_by_category" description:"未处理卡级失败告警数(按错误类别)"`
|
||||
}
|
||||
|
||||
// CarrierPollingStatsResponse 按运营商账户的轮询监控。
|
||||
type CarrierPollingStatsResponse struct {
|
||||
SchedulerMode string `json:"scheduler_mode" description:"当前调度模式"`
|
||||
InitState string `json:"init_state" description:"新结构初始化状态 (running / done,空表示未初始化)"`
|
||||
Items []CarrierPollingStatsItem `json:"items" description:"各运营商账户监控"`
|
||||
}
|
||||
226
internal/polling/carrier_bridge.go
Normal file
226
internal/polling/carrier_bridge.go
Normal file
@@ -0,0 +1,226 @@
|
||||
package polling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// CarrierScheduling 是按运营商账户调度在轮询包内的接缝:判断当前模式,并按卡计算各检查类型到期时间写入账户队列。
|
||||
// 所有写入轮询结构的入口(初始化、生命周期、优先提示、手动触发)按当前模式写新结构或旧分片结构。
|
||||
type CarrierScheduling struct {
|
||||
store *carrierqueue.Store
|
||||
settings *carrierqueue.SettingsProvider
|
||||
configMgr *PollingConfigManager
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewCarrierScheduling 创建按运营商账户调度接缝。
|
||||
func NewCarrierScheduling(store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, configMgr *PollingConfigManager, logger *zap.Logger) *CarrierScheduling {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &CarrierScheduling{store: store, settings: settings, configMgr: configMgr, logger: logger}
|
||||
}
|
||||
|
||||
// Active 判断当前是否为按运营商账户调度。
|
||||
func (c *CarrierScheduling) Active(ctx context.Context) bool {
|
||||
return c != nil && c.store != nil && c.settings != nil && c.settings.Current(ctx).CarrierMode()
|
||||
}
|
||||
|
||||
// Store 返回账户队列存储。
|
||||
func (c *CarrierScheduling) Store() *carrierqueue.Store {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.store
|
||||
}
|
||||
|
||||
// Settings 返回调度参数提供者。
|
||||
func (c *CarrierScheduling) Settings() *carrierqueue.SettingsProvider {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.settings
|
||||
}
|
||||
|
||||
// DuesForCard 按卡匹配的轮询配置计算各检查类型下次到期时间。
|
||||
// fromLastCheck 为 true 时按各类型上次检查时间计算(初始化);否则按当前时间加抖动(新卡、恢复轮询、状态变化)。
|
||||
// 间隔为 0 或未配置的类型不返回;重置日前加密窗口规则算出的时间更早时以它为准。
|
||||
func (c *CarrierScheduling) DuesForCard(ctx context.Context, card *model.IotCard, fromLastCheck bool) map[string]time.Time {
|
||||
dues := map[string]time.Time{}
|
||||
if c == nil || c.configMgr == nil || card == nil {
|
||||
return dues
|
||||
}
|
||||
now := time.Now()
|
||||
for taskType, info := range c.configMgr.MergedTaskIntervals(card) {
|
||||
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||
if checkType == "" || info.Interval <= 0 {
|
||||
continue
|
||||
}
|
||||
var due time.Time
|
||||
if fromLastCheck {
|
||||
due = calculateNextCheckTime(lastCheckAtByTaskType(card, taskType), info.Interval, now)
|
||||
if windowDue := c.configMgr.NextCheckAt(ctx, card, taskType, info, now); windowDue.Before(due) && windowDue.After(now) {
|
||||
due = windowDue
|
||||
}
|
||||
} else {
|
||||
due = calcInitialDelay(info.Interval)
|
||||
}
|
||||
dues[checkType] = due
|
||||
}
|
||||
return dues
|
||||
}
|
||||
|
||||
// UpsertCard 把卡写入所属账户队列;fromLastCheck 含义同 DuesForCard。
|
||||
func (c *CarrierScheduling) UpsertCard(ctx context.Context, card *model.IotCard, fromLastCheck bool) {
|
||||
if c == nil || c.store == nil || card == nil {
|
||||
return
|
||||
}
|
||||
if err := c.store.Upsert(ctx, card.ID, card.CarrierID, c.DuesForCard(ctx, card, fromLastCheck)); err != nil {
|
||||
c.logger.Warn("写入运营商账户轮询队列失败", zap.Uint("card_id", card.ID), zap.Uint("carrier_id", card.CarrierID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveCard 把卡移出所属账户队列;carrierID 为 0 时按到期信息中记录的账户移出。
|
||||
func (c *CarrierScheduling) RemoveCard(ctx context.Context, cardID, carrierID uint) {
|
||||
if c == nil || c.store == nil {
|
||||
return
|
||||
}
|
||||
if carrierID == 0 {
|
||||
due, err := c.store.ReadDue(ctx, cardID)
|
||||
if err != nil || due.CarrierID == 0 {
|
||||
return
|
||||
}
|
||||
carrierID = due.CarrierID
|
||||
}
|
||||
if err := c.store.Remove(ctx, cardID, carrierID); err != nil {
|
||||
c.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// RequestTaskType 把卡的某检查类型(旧任务类型 polling:xxx)设为立即到期放入账户优先队列(优先提示、手动触发)。
|
||||
// 卡不在轮询中时不处理。
|
||||
func (c *CarrierScheduling) RequestTaskType(ctx context.Context, cardID, carrierID uint, taskType string) {
|
||||
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||
if c == nil || c.store == nil || checkType == "" {
|
||||
return
|
||||
}
|
||||
if _, err := c.store.RequestCheck(ctx, carrierqueue.CheckRequest{CardID: cardID, CarrierID: carrierID, CheckTypes: []string{checkType}}); err != nil {
|
||||
c.logger.Warn("写入运营商账户优先队列失败", zap.Uint("card_id", cardID), zap.String("task_type", taskType), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// CarrierInitializer 从数据库按批初始化全部运营商账户队列与卡到期信息;进度与旧初始化器同结构,供 init-progress 展示。
|
||||
type CarrierInitializer struct {
|
||||
scheduling *CarrierScheduling
|
||||
iotCardStore *postgres.IotCardStore
|
||||
logger *zap.Logger
|
||||
|
||||
mu sync.RWMutex
|
||||
progress InitProgress
|
||||
running atomic.Bool
|
||||
}
|
||||
|
||||
// NewCarrierInitializer 创建新结构初始化器。
|
||||
func NewCarrierInitializer(scheduling *CarrierScheduling, iotCardStore *postgres.IotCardStore, logger *zap.Logger) *CarrierInitializer {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &CarrierInitializer{scheduling: scheduling, iotCardStore: iotCardStore, logger: logger}
|
||||
}
|
||||
|
||||
// Progress 返回初始化进度快照。
|
||||
func (i *CarrierInitializer) Progress() InitProgress {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
return i.progress
|
||||
}
|
||||
|
||||
// Run 同步执行一次全量初始化:按批读取启用轮询的卡,计算各类型到期并写入账户队列。返回写入的卡数。
|
||||
func (i *CarrierInitializer) Run(ctx context.Context) (int64, error) {
|
||||
if !i.running.CompareAndSwap(false, true) {
|
||||
return 0, nil
|
||||
}
|
||||
defer i.running.Store(false)
|
||||
started := time.Now()
|
||||
total, err := i.iotCardStore.CountForPolling(ctx)
|
||||
if err != nil {
|
||||
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", err.Error() })
|
||||
return 0, err
|
||||
}
|
||||
i.setProgress(func(p *InitProgress) {
|
||||
*p = InitProgress{TotalCards: total, StartTime: started, Status: "running"}
|
||||
})
|
||||
const batchSize = 2000
|
||||
var lastID uint
|
||||
var written int64
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", err.Error() })
|
||||
return written, err
|
||||
}
|
||||
cards, fetchErr := i.iotCardStore.ListForPollingBatch(ctx, lastID, batchSize)
|
||||
if fetchErr != nil {
|
||||
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", fetchErr.Error() })
|
||||
return written, fetchErr
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
if tier := i.scheduling.configMgr.TierResolver(); tier != nil {
|
||||
tier.Prefill(ctx, cards)
|
||||
}
|
||||
for _, card := range cards {
|
||||
i.scheduling.UpsertCard(ctx, card, true)
|
||||
written++
|
||||
}
|
||||
lastID = cards[len(cards)-1].ID
|
||||
loaded := written
|
||||
i.setProgress(func(p *InitProgress) { p.LoadedCards, p.LastBatchTime = loaded, time.Now() })
|
||||
}
|
||||
i.setProgress(func(p *InitProgress) { p.Status = "completed" })
|
||||
i.logger.Info("运营商账户轮询结构初始化完成", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)))
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (i *CarrierInitializer) setProgress(update func(*InitProgress)) {
|
||||
i.mu.Lock()
|
||||
update(&i.progress)
|
||||
i.mu.Unlock()
|
||||
}
|
||||
|
||||
// AdvanceDue 把卡某检查类型的到期时间提前到 due(只提前不推后),并同步把账户队列中的 score 提前;返回是否提前。
|
||||
// 卡不在轮询中(无到期信息)时不处理。
|
||||
func (c *CarrierScheduling) AdvanceDue(ctx context.Context, card *model.IotCard, checkType string, due time.Time) (bool, error) {
|
||||
if c == nil || c.store == nil || card == nil {
|
||||
return false, nil
|
||||
}
|
||||
current, err := c.store.ReadDue(ctx, card.ID)
|
||||
if err != nil || current.CarrierID == 0 {
|
||||
return false, err
|
||||
}
|
||||
item, ok := current.Types[checkType]
|
||||
if !ok || !item.HasDue || !due.Before(item.Due) {
|
||||
return false, nil
|
||||
}
|
||||
client := c.store.Client()
|
||||
member := strconv.FormatUint(uint64(card.ID), 10)
|
||||
pipe := client.TxPipeline()
|
||||
pipe.HSet(ctx, constants.RedisCardDueKey(card.ID), carrierqueue.FieldDue(checkType), due.UnixMilli())
|
||||
score := redis.ZAddArgs{XX: true, LT: true, Members: []redis.Z{{Score: float64(due.UnixMilli()), Member: member}}}
|
||||
pipe.ZAddArgs(ctx, constants.RedisCarrierQueueKey(current.CarrierID), score)
|
||||
pipe.ZAddArgs(ctx, constants.RedisCarrierPriorityQueueKey(current.CarrierID), score)
|
||||
_, err = pipe.Exec(ctx)
|
||||
return err == nil, err
|
||||
}
|
||||
177
internal/polling/carrierqueue/alerts.go
Normal file
177
internal/polling/carrierqueue/alerts.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// FailureEvent 是一次需要告警的检查失败。
|
||||
type FailureEvent struct {
|
||||
CardID uint
|
||||
ICCID string
|
||||
CarrierID uint
|
||||
CheckType string
|
||||
Category string
|
||||
Message string
|
||||
FailCount int
|
||||
FirstFailed time.Time
|
||||
At time.Time
|
||||
}
|
||||
|
||||
type alertOp struct {
|
||||
failure *FailureEvent
|
||||
recovered bool
|
||||
cardID uint
|
||||
checkType string
|
||||
}
|
||||
|
||||
// AlertWriter 异步写卡轮询失败告警:同一卡同一检查类型只保留一条未处理告警,后续失败更新计数;
|
||||
// 首次成功转为已恢复;同账户窗口内新增告警超阈值时产生账户级告警。写入失败只记日志,不影响轮询。
|
||||
type AlertWriter struct {
|
||||
db *gorm.DB
|
||||
redis *redis.Client
|
||||
settings *SettingsProvider
|
||||
logger *zap.Logger
|
||||
ch chan alertOp
|
||||
}
|
||||
|
||||
// NewAlertWriter 创建告警写入器;调用 Start 后开始消费。
|
||||
func NewAlertWriter(db *gorm.DB, redisClient *redis.Client, settings *SettingsProvider, logger *zap.Logger) *AlertWriter {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &AlertWriter{db: db, redis: redisClient, settings: settings, logger: logger, ch: make(chan alertOp, 2048)}
|
||||
}
|
||||
|
||||
// Start 启动后台写入协程,直到 ctx 结束。
|
||||
func (w *AlertWriter) Start(ctx context.Context) {
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case op := <-w.ch:
|
||||
w.apply(op)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// ReportFailure 提交一条失败告警(达到阈值后才调用)。
|
||||
func (w *AlertWriter) ReportFailure(event FailureEvent) {
|
||||
w.offer(alertOp{failure: &event})
|
||||
}
|
||||
|
||||
// ReportRecovered 提交一次恢复:该卡该检查类型的未处理告警转为已恢复。
|
||||
func (w *AlertWriter) ReportRecovered(cardID uint, checkType string) {
|
||||
w.offer(alertOp{recovered: true, cardID: cardID, checkType: checkType})
|
||||
}
|
||||
|
||||
func (w *AlertWriter) offer(op alertOp) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case w.ch <- op:
|
||||
default:
|
||||
w.logger.Warn("卡轮询失败告警队列已满,丢弃本次写入", zap.String("metric", "polling.failure_alert.dropped"))
|
||||
}
|
||||
}
|
||||
|
||||
func (w *AlertWriter) apply(op alertOp) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if op.recovered {
|
||||
err := w.db.WithContext(ctx).Exec(`UPDATE tb_polling_failure_alert SET status = ?, recovered_at = NOW(), updated_at = NOW()
|
||||
WHERE alert_level = ? AND card_id = ? AND check_type = ? AND status = ?`,
|
||||
constants.PollingAlertStatusRecovered, constants.PollingAlertLevelCard, op.cardID, op.checkType, constants.PollingAlertStatusOpen).Error
|
||||
if err != nil {
|
||||
w.logger.Warn("卡轮询告警转为已恢复失败", zap.Uint("card_id", op.cardID), zap.String("check_type", op.checkType), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
event := op.failure
|
||||
var inserted bool
|
||||
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||
(alert_level, card_id, iccid, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||
DO UPDATE SET fail_count = EXCLUDED.fail_count, error_category = EXCLUDED.error_category,
|
||||
last_failed_at = EXCLUDED.last_failed_at, last_error = EXCLUDED.last_error, updated_at = NOW()
|
||||
RETURNING (xmax = 0)`,
|
||||
constants.PollingAlertLevelCard, event.CardID, event.ICCID, event.CarrierID, event.CheckType, event.Category, event.FailCount,
|
||||
nullableTime(event.FirstFailed), event.At, truncate(event.Message, 500), constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||
if err != nil {
|
||||
w.logger.Warn("写卡轮询失败告警失败", zap.Uint("card_id", event.CardID), zap.String("check_type", event.CheckType), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if !inserted {
|
||||
return
|
||||
}
|
||||
w.logger.Error("卡轮询持续失败告警", zap.Uint("card_id", event.CardID), zap.String("iccid", event.ICCID),
|
||||
zap.Uint("carrier_id", event.CarrierID), zap.String("check_type", event.CheckType), zap.String("category", event.Category),
|
||||
zap.Int("fail_count", event.FailCount), zap.String("metric", "polling.failure_alert.opened"))
|
||||
w.checkAccountAlert(ctx, event.CarrierID)
|
||||
}
|
||||
|
||||
// checkAccountAlert 同账户窗口内新增卡级告警数超过阈值时,产生(或更新)一条账户级告警。
|
||||
func (w *AlertWriter) checkAccountAlert(ctx context.Context, carrierID uint) {
|
||||
settings := w.settings.Current(ctx)
|
||||
window, limit := settings.AccountAlertWindow, settings.AccountAlertLimit
|
||||
if window <= 0 || limit <= 0 || w.redis == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
key := constants.RedisCarrierAccountAlertKey(carrierID)
|
||||
member := strconv.FormatInt(now.UnixNano(), 36)
|
||||
pipe := w.redis.TxPipeline()
|
||||
pipe.ZAdd(ctx, key, redis.Z{Score: float64(now.UnixMilli()), Member: member})
|
||||
pipe.ZRemRangeByScore(ctx, key, "-inf", strconv.FormatInt(now.Add(-window).UnixMilli(), 10))
|
||||
count := pipe.ZCard(ctx, key)
|
||||
pipe.Expire(ctx, key, window*2)
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
if int(count.Val()) <= limit {
|
||||
return
|
||||
}
|
||||
var inserted bool
|
||||
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||
(alert_level, card_id, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||
VALUES (?, 0, ?, ?, '', ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||
DO UPDATE SET fail_count = EXCLUDED.fail_count, last_failed_at = EXCLUDED.last_failed_at, updated_at = NOW()
|
||||
RETURNING (xmax = 0)`,
|
||||
constants.PollingAlertLevelCarrier, carrierID, constants.PollingAlertCheckTypeAll, count.Val(), now, now,
|
||||
"窗口内同账户新增失败告警数超过阈值,通常是账户额度、凭证或上游故障,请联系上游处理账户", constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||
if err != nil {
|
||||
w.logger.Warn("写账户级轮询告警失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if inserted {
|
||||
w.logger.Error("运营商账户轮询失败告警数超过阈值", zap.Uint("carrier_id", carrierID), zap.Int64("count", count.Val()),
|
||||
zap.Duration("window", window), zap.String("metric", "polling.failure_alert.carrier_opened"))
|
||||
}
|
||||
}
|
||||
|
||||
func nullableTime(t time.Time) any {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max])
|
||||
}
|
||||
326
internal/polling/carrierqueue/dispatcher.go
Normal file
326
internal/polling/carrierqueue/dispatcher.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
stderrors "errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// CardMessage 是卡任务 NSQ 消息体。
|
||||
type CardMessage struct {
|
||||
CardID uint `json:"card_id"`
|
||||
CarrierID uint `json:"carrier_id"`
|
||||
Token string `json:"token"`
|
||||
DispatchedAt int64 `json:"dispatched_at"`
|
||||
}
|
||||
|
||||
// Publisher 发布 NSQ 消息。
|
||||
type Publisher interface {
|
||||
Publish(topic string, body []byte) error
|
||||
}
|
||||
|
||||
// BudgetProvider 返回账户各查询操作所属限流组本秒剩余名额;limited=false 表示不按 QPS 限制领取。
|
||||
type BudgetProvider interface {
|
||||
Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool)
|
||||
}
|
||||
|
||||
// CarrierParams 是一个运营商账户生效的调度参数(已合并全局默认)。
|
||||
type CarrierParams struct {
|
||||
InflightLimit int
|
||||
CardMaxInFlight int
|
||||
PriorityRatio float64
|
||||
Configured bool // 是否有该账户的配置行
|
||||
}
|
||||
|
||||
// carrierEntry 是分发器缓存的账户信息。
|
||||
type carrierEntry struct {
|
||||
id uint
|
||||
realnameControl bool
|
||||
}
|
||||
|
||||
// Dispatcher 每个分发间隔对每个运营商账户抢分发锁、领取到期卡并发布到账户 topic。多实例安全。
|
||||
type Dispatcher struct {
|
||||
db *gorm.DB
|
||||
store *Store
|
||||
settings *SettingsProvider
|
||||
publisher Publisher
|
||||
budgets BudgetProvider
|
||||
cfg config.CarrierSchedulerConfig
|
||||
instanceID string
|
||||
logger *zap.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
carriers []carrierEntry
|
||||
params map[uint]model.CarrierPollingParam
|
||||
loadedAt time.Time
|
||||
tightened map[uint]time.Time
|
||||
}
|
||||
|
||||
// NewDispatcher 创建分发器。
|
||||
func NewDispatcher(db *gorm.DB, store *Store, settings *SettingsProvider, publisher Publisher, budgets BudgetProvider,
|
||||
cfg config.CarrierSchedulerConfig, instanceID string, logger *zap.Logger) *Dispatcher {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &Dispatcher{db: db, store: store, settings: settings, publisher: publisher, budgets: budgets, cfg: cfg,
|
||||
instanceID: instanceID, logger: logger, params: map[uint]model.CarrierPollingParam{}, tightened: map[uint]time.Time{}}
|
||||
}
|
||||
|
||||
// Run 按分发间隔循环,直到 ctx 结束。
|
||||
func (d *Dispatcher) Run(ctx context.Context) {
|
||||
interval := d.cfg.DispatchInterval
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
d.tick(ctx, interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ready 判断新调度是否可领取:模式为 carrier 且新结构初始化已完成。
|
||||
func (d *Dispatcher) Ready(ctx context.Context) bool {
|
||||
if !d.settings.Current(ctx).CarrierMode() {
|
||||
return false
|
||||
}
|
||||
state, err := d.store.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||
return err == nil && state == InitStateDone
|
||||
}
|
||||
|
||||
func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
d.logger.Error("分发器 panic,已恢复", zap.Any("panic", r))
|
||||
}
|
||||
}()
|
||||
if !d.Ready(ctx) {
|
||||
return
|
||||
}
|
||||
d.store.redis.HSet(ctx, constants.RedisCarrierSchedulerHeartbeatKey, d.instanceID, time.Now().UnixMilli())
|
||||
d.drainLegacyHints(ctx)
|
||||
d.refreshCarriers(ctx)
|
||||
d.mu.Lock()
|
||||
carriers := append([]carrierEntry(nil), d.carriers...)
|
||||
d.mu.Unlock()
|
||||
lockTTL := interval * 9 / 10
|
||||
if lockTTL <= 0 {
|
||||
lockTTL = 900 * time.Millisecond
|
||||
}
|
||||
for _, carrier := range carriers {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
ok, err := d.store.redis.SetNX(ctx, constants.RedisCarrierDispatchLockKey(carrier.id), d.instanceID, lockTTL).Result()
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
d.dispatchCarrier(ctx, carrier)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) dispatchCarrier(ctx context.Context, carrier carrierEntry) {
|
||||
params := d.ParamsFor(carrier.id)
|
||||
limit := params.InflightLimit
|
||||
// 执行上限不得大于订阅进程 max_in_flight 之和,否则领出的消息在 NSQ 中排队到租约过期;不满足时收紧本轮上限并上报。
|
||||
processes := d.cfg.SubscriberProcesses
|
||||
if processes <= 0 {
|
||||
processes = 1
|
||||
}
|
||||
if capacity := params.CardMaxInFlight * processes; capacity > 0 && limit > capacity {
|
||||
limit = capacity
|
||||
d.reportTightened(carrier.id, params.InflightLimit, capacity)
|
||||
}
|
||||
input := ClaimInput{
|
||||
CarrierID: carrier.id, Lease: d.lease(), InflightLimit: limit, PriorityRatio: params.PriorityRatio,
|
||||
TokenPrefix: d.instanceID + ":" + strconv.FormatInt(time.Now().UnixNano(), 36),
|
||||
CountRealname: carrier.realnameControl,
|
||||
}
|
||||
if d.budgets != nil {
|
||||
ops := []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname}
|
||||
if groups, opGroup, limited := d.budgets.Budget(ctx, carrier.id, ops); limited {
|
||||
input.GroupBudget = groups
|
||||
input.TypeGroup = map[string]string{
|
||||
constants.CheckTypeCardStatus: opGroup[constants.GatewayOpQueryStatus],
|
||||
constants.CheckTypeCarddata: opGroup[constants.GatewayOpQueryFlow],
|
||||
constants.CheckTypeRealname: opGroup[constants.GatewayOpQueryRealname],
|
||||
}
|
||||
}
|
||||
}
|
||||
claimed, expired, err := d.store.Claim(ctx, input)
|
||||
if err != nil {
|
||||
d.logger.Warn("运营商账户领取失败", zap.Uint("carrier_id", carrier.id), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if expired > 0 {
|
||||
minute := strconv.FormatInt(time.Now().Unix()/60, 10)
|
||||
key := constants.RedisCarrierExpiredClaimsKey(carrier.id)
|
||||
d.store.redis.HIncrBy(ctx, key, minute, expired)
|
||||
d.store.redis.Expire(ctx, key, 2*time.Hour)
|
||||
d.logger.Warn("运营商账户处理中标记到期(执行崩溃或超时)", zap.Uint("carrier_id", carrier.id), zap.Int64("count", expired),
|
||||
zap.String("metric", "polling.carrier.claim_expired"))
|
||||
}
|
||||
topic := constants.NSQCardTopic(carrier.id)
|
||||
for _, item := range claimed {
|
||||
body, _ := json.Marshal(CardMessage{CardID: item.CardID, CarrierID: carrier.id, Token: item.Token, DispatchedAt: time.Now().UnixMilli()})
|
||||
if err := d.publisher.Publish(topic, body); err != nil {
|
||||
// 发布失败:用独立短超时上下文按 token 删除处理中标记并恢复 score;恢复也失败则等租约到期后自然重新领取。
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
if releaseErr := d.store.Release(releaseCtx, item.CardID, carrier.id, item.Token); releaseErr != nil {
|
||||
d.logger.Warn("发布失败后释放处理中标记失败", zap.Uint("card_id", item.CardID), zap.Error(releaseErr))
|
||||
}
|
||||
cancel()
|
||||
d.logger.Warn("发布卡任务失败,已释放", zap.Uint("card_id", item.CardID), zap.String("topic", topic), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) lease() time.Duration {
|
||||
if d.cfg.Lease > 0 {
|
||||
return d.cfg.Lease
|
||||
}
|
||||
return 5 * time.Minute
|
||||
}
|
||||
|
||||
func (d *Dispatcher) reportTightened(carrierID uint, configured, capacity int) {
|
||||
d.mu.Lock()
|
||||
last := d.tightened[carrierID]
|
||||
if time.Since(last) < 10*time.Minute {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
d.tightened[carrierID] = time.Now()
|
||||
d.mu.Unlock()
|
||||
d.logger.Warn("账户执行上限大于订阅进程 max_in_flight 之和,已按消费能力收紧本轮领取上限",
|
||||
zap.Uint("carrier_id", carrierID), zap.Int("inflight_limit", configured), zap.Int("capacity", capacity),
|
||||
zap.String("metric", "polling.carrier.inflight_tightened"))
|
||||
}
|
||||
|
||||
// refreshCarriers 每 10 秒重新读取运营商账户与调度参数。
|
||||
func (d *Dispatcher) refreshCarriers(ctx context.Context) {
|
||||
d.mu.Lock()
|
||||
fresh := time.Since(d.loadedAt) < 10*time.Second
|
||||
d.mu.Unlock()
|
||||
if fresh {
|
||||
return
|
||||
}
|
||||
var carriers []model.Carrier
|
||||
if err := d.db.WithContext(ctx).Select("id", "realname_control_enabled").Order("id").Find(&carriers).Error; err != nil {
|
||||
d.logger.Warn("读取运营商账户失败,沿用上一次列表", zap.Error(err))
|
||||
return
|
||||
}
|
||||
var rows []model.CarrierPollingParam
|
||||
if err := d.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||
d.logger.Warn("读取运营商账户调度参数失败,沿用上一次参数", zap.Error(err))
|
||||
return
|
||||
}
|
||||
entries := make([]carrierEntry, 0, len(carriers))
|
||||
for _, carrier := range carriers {
|
||||
entries = append(entries, carrierEntry{id: carrier.ID, realnameControl: carrier.RealnameControlEnabled == 1})
|
||||
}
|
||||
params := make(map[uint]model.CarrierPollingParam, len(rows))
|
||||
for _, row := range rows {
|
||||
params[row.CarrierID] = row
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.carriers, d.params, d.loadedAt = entries, params, time.Now()
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
// CarrierIDs 返回当前缓存的运营商账户 ID(消费者据此订阅)。
|
||||
func (d *Dispatcher) CarrierIDs(ctx context.Context) []uint {
|
||||
d.refreshCarriers(ctx)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
ids := make([]uint, 0, len(d.carriers))
|
||||
for _, carrier := range d.carriers {
|
||||
ids = append(ids, carrier.id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// ParamsFor 返回账户生效调度参数(未配置项用全局默认)。
|
||||
func (d *Dispatcher) ParamsFor(carrierID uint) CarrierParams {
|
||||
d.mu.Lock()
|
||||
row, ok := d.params[carrierID]
|
||||
d.mu.Unlock()
|
||||
return MergeParams(row, ok, d.cfg)
|
||||
}
|
||||
|
||||
// MergeParams 合并账户配置行与全局默认。
|
||||
func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.CarrierSchedulerConfig) CarrierParams {
|
||||
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured}
|
||||
if params.InflightLimit <= 0 {
|
||||
params.InflightLimit = 20
|
||||
}
|
||||
if params.CardMaxInFlight <= 0 {
|
||||
params.CardMaxInFlight = 20
|
||||
}
|
||||
if !configured {
|
||||
return params
|
||||
}
|
||||
if row.InflightLimit != nil && *row.InflightLimit > 0 {
|
||||
params.InflightLimit = *row.InflightLimit
|
||||
}
|
||||
if row.CardMaxInFlight != nil && *row.CardMaxInFlight > 0 {
|
||||
params.CardMaxInFlight = *row.CardMaxInFlight
|
||||
}
|
||||
if row.PriorityMaxRatio != nil && *row.PriorityMaxRatio > 0 {
|
||||
params.PriorityRatio = *row.PriorityMaxRatio
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
// drainLegacyHints 消费旧优先提示通道与手动触发队列:优先提示放入账户优先队列,手动触发在普通队列中立即到期。
|
||||
func (d *Dispatcher) drainLegacyHints(ctx context.Context) {
|
||||
for _, checkType := range constants.AllCheckTypes() {
|
||||
taskType := constants.TaskTypeFromCheckType(checkType)
|
||||
d.drainList(ctx, constants.RedisPollingPriorityQueueKey(taskType), checkType, false)
|
||||
d.drainList(ctx, constants.RedisPollingManualQueueKey(taskType), checkType, true)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) drainList(ctx context.Context, key, checkType string, normal bool) {
|
||||
for i := 0; i < 500; i++ {
|
||||
raw, err := d.store.redis.LPop(ctx, key).Result()
|
||||
if err != nil {
|
||||
if !stderrors.Is(err, redis.Nil) {
|
||||
d.logger.Warn("读取旧提示队列失败", zap.String("key", key), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
cardID, convErr := strconv.ParseUint(raw, 10, 64)
|
||||
if convErr != nil {
|
||||
continue
|
||||
}
|
||||
due, dueErr := d.store.ReadDue(ctx, uint(cardID))
|
||||
if dueErr != nil || due.CarrierID == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := d.store.RequestCheck(ctx, CheckRequest{CardID: uint(cardID), CarrierID: due.CarrierID, CheckTypes: []string{checkType}, Normal: normal}); err != nil {
|
||||
d.logger.Warn("写入账户队列失败", zap.Uint64("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 新结构初始化状态(constants.RedisCarrierSchedulerInitStateKey)。
|
||||
const (
|
||||
InitStateRunning = "running"
|
||||
InitStateDone = "done"
|
||||
)
|
||||
@@ -182,7 +182,8 @@ return 1
|
||||
// 只作用于仍在轮询中的卡(在优先或普通队列中)。冷却期内(最近成功距今未超过冷却)的类型不请求。
|
||||
//
|
||||
// KEYS[1] due;KEYS[2] 优先队列;KEYS[3] 普通队列;KEYS[4] claim。
|
||||
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、类型…。
|
||||
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、
|
||||
// 是否放入优先队列(1/0;0 用于手动触发:只把到期设为现在并在普通队列中立即到期)、类型…。
|
||||
// 返回本次实际请求的类型数;卡不在轮询中返回 -1。
|
||||
const requestCheckLua = `
|
||||
local now = tonumber(ARGV[1])
|
||||
@@ -192,7 +193,8 @@ local inPriority = redis.call('ZSCORE', KEYS[2], cardId)
|
||||
local inQueue = redis.call('ZSCORE', KEYS[3], cardId)
|
||||
if not inPriority and not inQueue then return -1 end
|
||||
local requested = 0
|
||||
for i = 6, #ARGV do
|
||||
local toPriority = ARGV[6] == '1'
|
||||
for i = 7, #ARGV do
|
||||
local t = ARGV[i]
|
||||
local skip = false
|
||||
if cooldown > 0 then
|
||||
@@ -209,6 +211,11 @@ for i = 6, #ARGV do
|
||||
end
|
||||
end
|
||||
if requested == 0 then return 0 end
|
||||
if not toPriority then
|
||||
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
||||
if inQueue then redis.call('ZADD', KEYS[3], now, cardId) else redis.call('ZADD', KEYS[2], now, cardId) end
|
||||
return requested
|
||||
end
|
||||
redis.call('HSET', KEYS[1], 'prio', '1')
|
||||
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
||||
redis.call('ZREM', KEYS[3], cardId)
|
||||
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
@@ -376,3 +378,15 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// NewDefaultSettingsProvider 用独立注册表与读取器创建调度参数提供者(API 与 Worker 共用同一份数据库与缓存数据)。
|
||||
func NewDefaultSettingsProvider(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *SettingsProvider {
|
||||
registry := systemconfig.NewRegistry()
|
||||
RegisterConfigDefinitions(registry, logger)
|
||||
var cache systemconfig.Cache
|
||||
if redisClient != nil {
|
||||
cache = systemconfig.NewRedisCache(redisClient)
|
||||
}
|
||||
reader := systemconfig.NewReader(db, registry, cache, systemconfig.NewLogAlertSink(logger))
|
||||
return NewSettingsProvider(reader, logger)
|
||||
}
|
||||
|
||||
@@ -136,6 +136,8 @@ type CheckRequest struct {
|
||||
Cooldown time.Duration
|
||||
ConfirmDelays []time.Duration
|
||||
Expected string // 延迟确认的期望结果(card_status 为 online / offline),满足时取消后续确认
|
||||
// Normal 为 true 时不放入优先队列:只把到期设为现在、在普通队列中立即到期(手动触发走普通通道)。
|
||||
Normal bool
|
||||
}
|
||||
|
||||
// RequestCheck 把卡的指定检查类型设为立即到期放入优先队列;返回实际请求的类型数,卡不在轮询中返回 -1。
|
||||
@@ -147,7 +149,11 @@ func (s *Store) RequestCheck(ctx context.Context, request CheckRequest) (int64,
|
||||
for _, delay := range request.ConfirmDelays {
|
||||
confirm = append(confirm, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||
}
|
||||
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected}
|
||||
toPriority := "1"
|
||||
if request.Normal {
|
||||
toPriority = "0"
|
||||
}
|
||||
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected, toPriority}
|
||||
for _, checkType := range request.CheckTypes {
|
||||
args = append(args, checkType)
|
||||
}
|
||||
|
||||
161
internal/polling/check_requester.go
Normal file
161
internal/polling/check_requester.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package polling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// sceneCacheTTL 是请求检查场景配置的进程内缓存时长。
|
||||
const sceneCacheTTL = 30 * time.Second
|
||||
|
||||
// SceneConfig 是一个请求检查场景的生效配置。
|
||||
type SceneConfig struct {
|
||||
Scene string
|
||||
CheckTypes []string
|
||||
ConfirmDelays []time.Duration
|
||||
Cooldown time.Duration
|
||||
}
|
||||
|
||||
// CheckRequester 按场景配置为卡请求检查(业务事件与读入口):把对应检查类型设为立即到期放入账户优先队列,
|
||||
// 同卡多次请求合并、冷却期内不请求、按场景追加延迟确认。只在按运营商账户调度时生效。
|
||||
type CheckRequester struct {
|
||||
scheduling *CarrierScheduling
|
||||
db *gorm.DB
|
||||
logger *zap.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
scenes map[string]SceneConfig
|
||||
loadedAt time.Time
|
||||
}
|
||||
|
||||
// NewCheckRequester 创建请求检查器。
|
||||
func NewCheckRequester(scheduling *CarrierScheduling, db *gorm.DB, logger *zap.Logger) *CheckRequester {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &CheckRequester{scheduling: scheduling, db: db, logger: logger}
|
||||
}
|
||||
|
||||
// Active 判断当前是否按运营商账户调度(请求检查生效)。
|
||||
func (r *CheckRequester) Active(ctx context.Context) bool {
|
||||
return r != nil && r.scheduling.Active(ctx)
|
||||
}
|
||||
|
||||
// RequestCards 为一组卡按场景请求检查;expected 为延迟确认的期望结果(online / offline,可空)。
|
||||
// 失败只记日志,不影响业务。
|
||||
func (r *CheckRequester) RequestCards(ctx context.Context, scene string, cardIDs []uint, expected string) {
|
||||
if r == nil || len(cardIDs) == 0 {
|
||||
return
|
||||
}
|
||||
config, ok := r.sceneConfig(ctx, scene)
|
||||
if !ok || len(config.CheckTypes) == 0 {
|
||||
return
|
||||
}
|
||||
carriers, err := r.cardCarriers(ctx, cardIDs)
|
||||
if err != nil {
|
||||
r.logger.Warn("请求检查:查询卡所属运营商账户失败", zap.String("scene", scene), zap.Error(err))
|
||||
return
|
||||
}
|
||||
store := r.scheduling.Store()
|
||||
for _, cardID := range cardIDs {
|
||||
carrierID, found := carriers[cardID]
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := store.RequestCheck(ctx, carrierqueue.CheckRequest{
|
||||
CardID: cardID, CarrierID: carrierID, CheckTypes: config.CheckTypes, Cooldown: config.Cooldown,
|
||||
ConfirmDelays: config.ConfirmDelays, Expected: expected,
|
||||
}); err != nil {
|
||||
r.logger.Warn("请求检查写入账户队列失败", zap.Uint("card_id", cardID), zap.String("scene", scene), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RequestDevice 为设备下全部有效绑定卡按场景请求检查。
|
||||
func (r *CheckRequester) RequestDevice(ctx context.Context, scene string, deviceID uint) {
|
||||
if r == nil || deviceID == 0 {
|
||||
return
|
||||
}
|
||||
var cardIDs []uint
|
||||
if err := r.db.WithContext(ctx).Model(&model.DeviceSimBinding{}).
|
||||
Where("device_id = ? AND bind_status = ?", deviceID, constants.BindStatusBound).Pluck("iot_card_id", &cardIDs).Error; err != nil {
|
||||
r.logger.Warn("请求检查:查询设备绑定卡失败", zap.Uint("device_id", deviceID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
r.RequestCards(ctx, scene, cardIDs, "")
|
||||
}
|
||||
|
||||
func (r *CheckRequester) cardCarriers(ctx context.Context, cardIDs []uint) (map[uint]uint, error) {
|
||||
var cards []model.IotCard
|
||||
if err := r.db.WithContext(ctx).Select("id", "carrier_id").Where("id IN ?", cardIDs).Find(&cards).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make(map[uint]uint, len(cards))
|
||||
for _, card := range cards {
|
||||
result[card.ID] = card.CarrierID
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// sceneConfig 读取场景配置(缓存);场景未预置时不请求。
|
||||
func (r *CheckRequester) sceneConfig(ctx context.Context, scene string) (SceneConfig, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.scenes == nil || time.Since(r.loadedAt) >= sceneCacheTTL {
|
||||
var rows []model.PollingCheckScene
|
||||
if err := r.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||
r.logger.Warn("读取请求检查场景失败,沿用上一次配置", zap.Error(err))
|
||||
} else {
|
||||
scenes := make(map[string]SceneConfig, len(rows))
|
||||
for _, row := range rows {
|
||||
scenes[row.Scene] = ParseSceneConfig(row)
|
||||
}
|
||||
r.scenes, r.loadedAt = scenes, time.Now()
|
||||
}
|
||||
}
|
||||
config, ok := r.scenes[scene]
|
||||
return config, ok
|
||||
}
|
||||
|
||||
// ParseSceneConfig 解析场景配置行。
|
||||
func ParseSceneConfig(row model.PollingCheckScene) SceneConfig {
|
||||
config := SceneConfig{Scene: row.Scene}
|
||||
var types []string
|
||||
_ = json.Unmarshal(row.CheckTypes, &types)
|
||||
for _, checkType := range types {
|
||||
if constants.CheckTypeCallsGateway(strings.TrimSpace(checkType)) {
|
||||
config.CheckTypes = append(config.CheckTypes, strings.TrimSpace(checkType))
|
||||
}
|
||||
}
|
||||
var delays []int
|
||||
_ = json.Unmarshal(row.ConfirmDelays, &delays)
|
||||
for _, seconds := range delays {
|
||||
if seconds > 0 {
|
||||
config.ConfirmDelays = append(config.ConfirmDelays, time.Duration(seconds)*time.Second)
|
||||
}
|
||||
}
|
||||
if row.Cooldown != nil && *row.Cooldown > 0 {
|
||||
config.Cooldown = time.Duration(*row.Cooldown) * time.Second
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// InvalidateScenes 使场景缓存立即失效(后台修改后调用)。
|
||||
func (r *CheckRequester) InvalidateScenes() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.loadedAt = time.Time{}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
@@ -22,6 +22,18 @@ type PollingLifecycleService struct {
|
||||
deviceBindingStore *postgres.DeviceSimBindingStore
|
||||
deviceStore *postgres.DeviceStore
|
||||
logger *zap.Logger
|
||||
// carrier 按运营商账户调度接缝;当前模式为 carrier 时生命周期事件写入账户队列,不写旧分片队列。
|
||||
carrier *CarrierScheduling
|
||||
}
|
||||
|
||||
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||
func (s *PollingLifecycleService) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||
s.carrier = carrier
|
||||
}
|
||||
|
||||
// carrierActive 判断当前是否按运营商账户调度。
|
||||
func (s *PollingLifecycleService) carrierActive(ctx context.Context) bool {
|
||||
return s.carrier.Active(ctx)
|
||||
}
|
||||
|
||||
// NewPollingLifecycleService 创建卡生命周期轮询管理服务
|
||||
@@ -52,6 +64,10 @@ func (s *PollingLifecycleService) OnCardCreated(ctx context.Context, card *model
|
||||
s.logger.Debug("卡禁用轮询,跳过入队", zap.Uint("card_id", card.ID))
|
||||
return
|
||||
}
|
||||
if s.carrierActive(ctx) {
|
||||
s.carrier.UpsertCard(ctx, card, false)
|
||||
return
|
||||
}
|
||||
s.enqueueCard(ctx, card)
|
||||
}
|
||||
|
||||
@@ -64,6 +80,11 @@ func (s *PollingLifecycleService) OnBatchCardsCreated(ctx context.Context, cards
|
||||
|
||||
// OnCardStatusChanged 卡状态变化后清理缓存、重新匹配配置并更新队列
|
||||
func (s *PollingLifecycleService) OnCardStatusChanged(ctx context.Context, cardID uint) {
|
||||
if s.carrierActive(ctx) {
|
||||
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||
s.carrierRefresh(ctx, cardID)
|
||||
return
|
||||
}
|
||||
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
||||
s.logger.Warn("卡状态变化:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
@@ -87,6 +108,13 @@ func (s *PollingLifecycleService) OnBatchCardsStatusChanged(ctx context.Context,
|
||||
if len(cardIDs) == 0 {
|
||||
return
|
||||
}
|
||||
if s.carrierActive(ctx) {
|
||||
for _, cardID := range cardIDs {
|
||||
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||
s.carrierRefresh(ctx, cardID)
|
||||
}
|
||||
return
|
||||
}
|
||||
// 单次 pipeline:批量从当前分片队列移除并清理缓存
|
||||
if err := s.queueMgr.RemoveBatchFromCurrentShardQueues(ctx, cardIDs); err != nil {
|
||||
s.logger.Warn("批量卡状态变化:从队列移除失败", zap.Int("count", len(cardIDs)), zap.Error(err))
|
||||
@@ -107,6 +135,11 @@ func (s *PollingLifecycleService) OnBatchCardsStatusChanged(ctx context.Context,
|
||||
|
||||
// OnCardDeleted 卡删除后移除所有队列并清理缓存
|
||||
func (s *PollingLifecycleService) OnCardDeleted(ctx context.Context, cardID uint) {
|
||||
if s.carrierActive(ctx) {
|
||||
s.carrier.RemoveCard(ctx, cardID, 0)
|
||||
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||
return
|
||||
}
|
||||
if err := s.queueMgr.OnCardDeleted(ctx, cardID); err != nil {
|
||||
s.logger.Warn("卡删除:清理队列和缓存失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
@@ -123,11 +156,21 @@ func (s *PollingLifecycleService) OnCardEnabled(ctx context.Context, cardID uint
|
||||
if !s.ShouldEnqueue(ctx, card) {
|
||||
return
|
||||
}
|
||||
if s.carrierActive(ctx) {
|
||||
// 恢复轮询时各检查类型到期设为当前时间加抖动。
|
||||
s.carrier.UpsertCard(ctx, card, false)
|
||||
return
|
||||
}
|
||||
s.enqueueCard(ctx, card)
|
||||
}
|
||||
|
||||
// OnCardDisabled 卡禁用轮询后移除所有队列
|
||||
func (s *PollingLifecycleService) OnCardDisabled(ctx context.Context, cardID uint) {
|
||||
if s.carrierActive(ctx) {
|
||||
s.carrier.RemoveCard(ctx, cardID, 0)
|
||||
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||
return
|
||||
}
|
||||
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
||||
s.logger.Warn("卡禁用:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
@@ -183,3 +226,17 @@ func calcInitialDelay(interval int) time.Time {
|
||||
}
|
||||
return now.Add(time.Duration(rand.Intn(jitterMax)) * time.Second)
|
||||
}
|
||||
|
||||
// carrierRefresh 按运营商账户调度时重新匹配配置并写入账户队列:不再轮询的卡移出,其余按当前时间加抖动重排。
|
||||
func (s *PollingLifecycleService) carrierRefresh(ctx context.Context, cardID uint) {
|
||||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||||
if err != nil {
|
||||
s.logger.Error("卡状态变化:加载卡信息失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if !s.ShouldEnqueue(ctx, card) {
|
||||
s.carrier.RemoveCard(ctx, card.ID, card.CarrierID)
|
||||
return
|
||||
}
|
||||
s.carrier.UpsertCard(ctx, card, false)
|
||||
}
|
||||
|
||||
@@ -32,6 +32,39 @@ type DueRecalculator struct {
|
||||
configMgr *PollingConfigManager
|
||||
queueMgr *PollingQueueManager
|
||||
logger *zap.Logger
|
||||
// carrier 按运营商账户调度接缝:carrier 模式下重算写入卡到期信息与账户队列。
|
||||
carrier *CarrierScheduling
|
||||
}
|
||||
|
||||
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||
func (r *DueRecalculator) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||
r.carrier = carrier
|
||||
}
|
||||
|
||||
// recalculateCarrierBatch 按运营商账户调度时重算:各检查类型候选到期同旧结构口径,只提前不推后。
|
||||
func (r *DueRecalculator) recalculateCarrierBatch(ctx context.Context, cards []*model.IotCard) (int, error) {
|
||||
now := time.Now()
|
||||
advanced := 0
|
||||
for _, card := range cards {
|
||||
for taskType, info := range r.configMgr.MergedTaskIntervals(card) {
|
||||
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||
if checkType == "" || info.Interval <= 0 {
|
||||
continue
|
||||
}
|
||||
due := recalculatedDue(lastCheckAtByTaskType(card, taskType), info.Interval, now)
|
||||
if windowDue := r.configMgr.NextCheckAt(ctx, card, taskType, info, now); windowDue.Before(due) && windowDue.After(now) {
|
||||
due = windowDue
|
||||
}
|
||||
ok, err := r.carrier.AdvanceDue(ctx, card, checkType, due)
|
||||
if err != nil {
|
||||
return advanced, err
|
||||
}
|
||||
if ok {
|
||||
advanced++
|
||||
}
|
||||
}
|
||||
}
|
||||
return advanced, nil
|
||||
}
|
||||
|
||||
// NewDueRecalculator 创建重算器。
|
||||
@@ -97,6 +130,9 @@ func (r *DueRecalculator) recalculateBatch(ctx context.Context, cards []*model.I
|
||||
if tier := r.configMgr.TierResolver(); tier != nil {
|
||||
tier.Prefill(ctx, cards)
|
||||
}
|
||||
if r.carrier.Active(ctx) {
|
||||
return r.recalculateCarrierBatch(ctx, cards)
|
||||
}
|
||||
now := time.Now()
|
||||
pipe := r.redis.Pipeline()
|
||||
var cmds []*redis.IntCmd
|
||||
|
||||
@@ -30,6 +30,8 @@ type Scheduler struct {
|
||||
packageActivationHandler *PackageActivationHandler
|
||||
dataResetHandler *DataResetHandler
|
||||
initializer *PollingInitializer // 可选,nil 表示不守卫 Init 完成
|
||||
// carrier 按运营商账户调度接缝:当前模式为 carrier 时旧分片调度停止出队(套餐激活检查照常执行)。
|
||||
carrier *CarrierScheduling
|
||||
|
||||
stopChan chan struct{}
|
||||
wg sync.WaitGroup
|
||||
@@ -156,6 +158,10 @@ func (s *Scheduler) scheduleLoop(ctx context.Context) {
|
||||
// processShardSchedule 处理手动队列和分片定时队列(每 1 秒触发)
|
||||
// 使用 90% 的 tick 间隔作为超时,确保单分片 Redis 挂起时不阻塞下一个 tick
|
||||
func (s *Scheduler) processShardSchedule(ctx context.Context) {
|
||||
if s.carrier.Active(ctx) {
|
||||
// 已切到按运营商账户调度:旧分片调度停止出队,优先提示与手动触发由新分发器消费。
|
||||
return
|
||||
}
|
||||
// 优先提示与手动队列都不受 Init 影响(ConfigManager 已就绪即可)。
|
||||
// 同一调度周期内先排空优先提示、再排空手动触发队列:优先的强度边界仅为调度入口领先,
|
||||
// 提示排空不参与分片背压判断,因此分片积压不会抑制加急入队。
|
||||
@@ -296,3 +302,8 @@ func (s *Scheduler) enqueueBatch(ctx context.Context, taskType string, cardIDs [
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||
func (s *Scheduler) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||
s.carrier = carrier
|
||||
}
|
||||
|
||||
@@ -136,6 +136,9 @@ func RegisterAdminRoutes(router fiber.Router, handlers *bootstrap.Handlers, midd
|
||||
if handlers.PollingConfig != nil {
|
||||
registerPollingConfigRoutes(authGroup, handlers.PollingConfig, doc, basePath)
|
||||
}
|
||||
if handlers.CarrierScheduler != nil {
|
||||
registerCarrierSchedulerRoutes(authGroup, handlers.CarrierScheduler, doc, basePath)
|
||||
}
|
||||
if handlers.GatewayRateLimit != nil {
|
||||
registerGatewayRateLimitRoutes(authGroup, handlers.GatewayRateLimit, doc, basePath)
|
||||
}
|
||||
|
||||
68
internal/routes/carrier_scheduler.go
Normal file
68
internal/routes/carrier_scheduler.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/handler/admin"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/openapi"
|
||||
)
|
||||
|
||||
// registerCarrierSchedulerRoutes 注册按运营商账户调度的后台接口。读取仅超级管理员与平台账号,修改仅超级管理员(服务层校验)。
|
||||
func registerCarrierSchedulerRoutes(router fiber.Router, handler *admin.CarrierSchedulerHandler, doc *openapi.Generator, basePath string) {
|
||||
tags := []string{"轮询调度"}
|
||||
|
||||
params := router.Group("/polling-carrier-params")
|
||||
paramsPath := basePath + "/polling-carrier-params"
|
||||
Register(params, doc, paramsPath, "GET", "", handler.ListParams, RouteSpec{
|
||||
Summary: "查询运营商账户调度参数列表", Description: "未单独配置的账户返回全局默认并标记 is_default。", Tags: tags,
|
||||
Input: nil, Output: new(dto.CarrierPollingParamListResponse), Auth: true,
|
||||
})
|
||||
Register(params, doc, paramsPath, "GET", "/:carrier_id", handler.GetParam, RouteSpec{
|
||||
Summary: "查询运营商账户调度参数", Tags: tags, Input: new(dto.CarrierPollingParamPath), Output: new(dto.CarrierPollingParamItem), Auth: true,
|
||||
})
|
||||
Register(params, doc, paramsPath, "PUT", "/:carrier_id", handler.SaveParam, RouteSpec{
|
||||
Summary: "保存运营商账户调度参数(仅超级管理员)",
|
||||
Description: "字段为空表示该项使用全局默认;执行上限不得大于当前订阅进程 max_in_flight 之和;下一轮调度生效并写统一审计。",
|
||||
Tags: tags, Input: new(dto.SaveCarrierPollingParamRequest), Output: new(dto.CarrierPollingParamItem), Auth: true,
|
||||
})
|
||||
Register(params, doc, paramsPath, "DELETE", "/:carrier_id", handler.DeleteParam, RouteSpec{
|
||||
Summary: "删除运营商账户调度参数(恢复全局默认,仅超级管理员)", Tags: tags, Input: new(dto.CarrierPollingParamPath), Output: nil, Auth: true,
|
||||
})
|
||||
|
||||
scenes := router.Group("/polling-check-scenes")
|
||||
scenesPath := basePath + "/polling-check-scenes"
|
||||
Register(scenes, doc, scenesPath, "GET", "", handler.ListScenes, RouteSpec{
|
||||
Summary: "查询请求检查场景", Description: "业务事件与读入口在按运营商账户调度时按场景请求检查;场景由代码预置。", Tags: tags,
|
||||
Input: nil, Output: new([]dto.PollingCheckSceneItem), Auth: true,
|
||||
})
|
||||
Register(scenes, doc, scenesPath, "PUT", "/:scene", handler.UpdateScene, RouteSpec{
|
||||
Summary: "修改请求检查场景(仅超级管理员)", Description: "只可修改检查类型、延迟确认序列(递增)与冷却时长。", Tags: tags,
|
||||
Input: new(dto.UpdatePollingCheckSceneRequest), Output: new(dto.PollingCheckSceneItem), Auth: true,
|
||||
})
|
||||
|
||||
alerts := router.Group("/polling-failure-alerts")
|
||||
alertsPath := basePath + "/polling-failure-alerts"
|
||||
Register(alerts, doc, alertsPath, "GET", "", handler.ListAlerts, RouteSpec{
|
||||
Summary: "查询卡轮询失败告警列表", Tags: tags, Input: new(dto.PollingFailureAlertListRequest), Output: new(dto.PollingFailureAlertPageResult), Auth: true,
|
||||
})
|
||||
Register(alerts, doc, alertsPath, "GET", "/summary", handler.AlertSummary, RouteSpec{
|
||||
Summary: "未处理卡轮询失败告警按运营商账户与错误类别汇总", Tags: tags, Input: nil, Output: new([]dto.PollingFailureAlertSummaryItem), Auth: true,
|
||||
})
|
||||
Register(alerts, doc, alertsPath, "GET", "/:id", handler.GetAlert, RouteSpec{
|
||||
Summary: "查询卡轮询失败告警详情", Tags: tags, Input: new(dto.PollingFailureAlertIDPath), Output: new(dto.PollingFailureAlertItem), Auth: true,
|
||||
})
|
||||
Register(alerts, doc, alertsPath, "PUT", "/:id/status", handler.HandleAlert, RouteSpec{
|
||||
Summary: "标记卡轮询失败告警已处理", Description: "只有未处理告警可标记;关闭该卡轮询请使用资产轮询开关接口。", Tags: tags,
|
||||
Input: new(dto.HandlePollingFailureAlertRequest), Output: nil, Auth: true,
|
||||
})
|
||||
|
||||
Register(router, doc, basePath, "GET", "/iot-cards/:id/polling-state", handler.CardState, RouteSpec{
|
||||
Summary: "查询单卡轮询状态", Description: "各检查类型下次到期、最近成功、连续失败与最近错误、是否执行中;只读。", Tags: tags,
|
||||
Input: new(dto.CardPollingStatePath), Output: new(dto.CardPollingStateResponse), Auth: true,
|
||||
})
|
||||
Register(router, doc, basePath, "GET", "/polling-stats/carrier-queues", handler.CarrierStats, RouteSpec{
|
||||
Summary: "按运营商账户的轮询队列与执行监控", Description: "按运营商账户调度的各账户队列长度、已到期数、执行中、上一分钟 QPS、限流等待与拒绝、处理中标记到期、未处理告警。", Tags: tags,
|
||||
Input: nil, Output: new(dto.CarrierPollingStatsResponse), Auth: true,
|
||||
})
|
||||
}
|
||||
622
internal/service/polling/carrier_scheduler_service.go
Normal file
622
internal/service/polling/carrier_scheduler_service.go
Normal file
@@ -0,0 +1,622 @@
|
||||
package polling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
stderrors "errors"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/utils"
|
||||
)
|
||||
|
||||
// CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。
|
||||
// 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。
|
||||
type CarrierSchedulerService struct {
|
||||
db *gorm.DB
|
||||
redis *redis.Client
|
||||
auditWriter *auditinfra.Writer
|
||||
store *carrierqueue.Store
|
||||
settings *carrierqueue.SettingsProvider
|
||||
cfg config.CarrierSchedulerConfig
|
||||
}
|
||||
|
||||
// NewCarrierSchedulerService 创建按运营商账户调度后台服务。
|
||||
func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService {
|
||||
return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient),
|
||||
settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg}
|
||||
}
|
||||
|
||||
func (s *CarrierSchedulerService) processes() int {
|
||||
if s.cfg.SubscriberProcesses > 0 {
|
||||
return s.cfg.SubscriberProcesses
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。
|
||||
func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names, err := s.carrierNameMap(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []model.CarrierPollingParam
|
||||
if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||||
}
|
||||
byCarrier := make(map[uint]model.CarrierPollingParam, len(rows))
|
||||
for _, row := range rows {
|
||||
byCarrier[row.CarrierID] = row
|
||||
}
|
||||
defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg)
|
||||
resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(),
|
||||
DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}}
|
||||
for _, id := range sortedCarrierIDs(names) {
|
||||
row, configured := byCarrier[id]
|
||||
resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured))
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem {
|
||||
merged := carrierqueue.MergeParams(row, configured, s.cfg)
|
||||
item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured,
|
||||
InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()}
|
||||
if configured {
|
||||
item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit
|
||||
item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay
|
||||
item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// GetParam 查询单个运营商账户调度参数。
|
||||
func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carrier, err := s.loadCarrier(ctx, carrierID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row, configured, err := s.loadParam(ctx, carrierID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item := s.paramItem(carrierID, carrier.CarrierName, row, configured)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。
|
||||
func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) {
|
||||
operatorID, err := requireSuperAdminWriter(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carrier, err := s.loadCarrier(ctx, req.CarrierID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row, configured, err := s.loadParam(ctx, req.CarrierID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
before := paramState(row, configured)
|
||||
row.CarrierID = req.CarrierID
|
||||
row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight
|
||||
row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio
|
||||
row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay
|
||||
merged := carrierqueue.MergeParams(row, true, s.cfg)
|
||||
if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity {
|
||||
return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+
|
||||
strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight")
|
||||
}
|
||||
if !configured {
|
||||
row.Creator = operatorID
|
||||
}
|
||||
row.Updater = operatorID
|
||||
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||
if err := tx.WithContext(ctx).Save(&row).Error; err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败")
|
||||
}
|
||||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||
ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数",
|
||||
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID,
|
||||
ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID,
|
||||
IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit,
|
||||
"card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio},
|
||||
BeforeData: before, AfterData: paramState(row, true),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。
|
||||
func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error {
|
||||
operatorID, err := requireSuperAdminWriter(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row, configured, err := s.loadParam(ctx, carrierID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !configured {
|
||||
return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数")
|
||||
}
|
||||
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||
if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败")
|
||||
}
|
||||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||
ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)",
|
||||
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID,
|
||||
ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID,
|
||||
IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func paramState(row model.CarrierPollingParam, configured bool) map[string]any {
|
||||
if !configured {
|
||||
return map[string]any{"configured": false}
|
||||
}
|
||||
return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight,
|
||||
"stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit,
|
||||
"priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval,
|
||||
"stop_resume_confirm_delay": row.StopResumeConfirmDelay}
|
||||
}
|
||||
|
||||
func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) {
|
||||
var row model.CarrierPollingParam
|
||||
err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return model.CarrierPollingParam{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||||
}
|
||||
return row, true, nil
|
||||
}
|
||||
|
||||
func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) {
|
||||
var carrier model.Carrier
|
||||
if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil {
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New(errors.CodeNotFound, "运营商账户不存在")
|
||||
}
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
|
||||
}
|
||||
return &carrier, nil
|
||||
}
|
||||
|
||||
func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) {
|
||||
var carriers []model.Carrier
|
||||
if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
|
||||
}
|
||||
names := make(map[uint]string, len(carriers))
|
||||
for _, carrier := range carriers {
|
||||
names[carrier.ID] = carrier.CarrierName
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// ListScenes 列出请求检查场景。
|
||||
func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []model.PollingCheckScene
|
||||
if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
|
||||
}
|
||||
items := make([]dto.PollingCheckSceneItem, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
items = append(items, sceneItem(row))
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem {
|
||||
item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt,
|
||||
CheckTypes: []string{}, ConfirmDelays: []int{}}
|
||||
_ = json.Unmarshal(row.CheckTypes, &item.CheckTypes)
|
||||
_ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays)
|
||||
return item
|
||||
}
|
||||
|
||||
// UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。
|
||||
func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) {
|
||||
operatorID, err := requireSuperAdminWriter(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for index := 1; index < len(req.ConfirmDelays); index++ {
|
||||
if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] {
|
||||
return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增")
|
||||
}
|
||||
}
|
||||
types := uniqueStrings(req.CheckTypes)
|
||||
var row model.PollingCheckScene
|
||||
if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil {
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在")
|
||||
}
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
|
||||
}
|
||||
before := sceneItem(row)
|
||||
typesJSON, _ := json.Marshal(types)
|
||||
delays := req.ConfirmDelays
|
||||
if delays == nil {
|
||||
delays = []int{}
|
||||
}
|
||||
delaysJSON, _ := json.Marshal(delays)
|
||||
row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID
|
||||
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||
if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{
|
||||
"check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败")
|
||||
}
|
||||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||
ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景",
|
||||
ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID,
|
||||
ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID,
|
||||
IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName},
|
||||
BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown},
|
||||
AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown},
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item := sceneItem(row)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// ListAlerts 分页查询卡轮询失败告警。
|
||||
func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page, size := normalizePage(req.Page, req.PageSize)
|
||||
query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{})
|
||||
if req.CarrierID != nil {
|
||||
query = query.Where("carrier_id = ?", *req.CarrierID)
|
||||
}
|
||||
if req.CheckType != "" {
|
||||
query = query.Where("check_type = ?", req.CheckType)
|
||||
}
|
||||
if req.ErrorCategory != "" {
|
||||
query = query.Where("error_category = ?", req.ErrorCategory)
|
||||
}
|
||||
if req.Status != nil {
|
||||
query = query.Where("status = ?", *req.Status)
|
||||
}
|
||||
if req.AlertLevel != "" {
|
||||
query = query.Where("alert_level = ?", req.AlertLevel)
|
||||
}
|
||||
if req.ICCID != "" {
|
||||
query = query.Where("iccid = ?", req.ICCID)
|
||||
}
|
||||
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if start != nil {
|
||||
query = query.Where("last_failed_at >= ?", *start)
|
||||
}
|
||||
if end != nil {
|
||||
query = query.Where("last_failed_at <= ?", *end)
|
||||
}
|
||||
var total int64
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
|
||||
}
|
||||
var rows []model.PollingFailureAlert
|
||||
if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
|
||||
}
|
||||
names, err := s.carrierNameMap(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
|
||||
for _, row := range rows {
|
||||
result.List = append(result.List, alertItem(row, names))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem {
|
||||
return dto.PollingFailureAlertItem{
|
||||
ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
|
||||
CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType),
|
||||
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
|
||||
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
|
||||
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。
|
||||
func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []struct {
|
||||
CarrierID uint
|
||||
ErrorCategory string
|
||||
OpenCount int64
|
||||
}
|
||||
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).
|
||||
Select("carrier_id, error_category, COUNT(*) AS open_count").
|
||||
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
|
||||
Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败")
|
||||
}
|
||||
names, err := s.carrierNameMap(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID],
|
||||
ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// GetAlert 查询告警详情。
|
||||
func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row, err := s.loadAlert(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names, err := s.carrierNameMap(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item := alertItem(*row, names)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// HandleAlert 把未处理告警标记为已处理。
|
||||
func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error {
|
||||
operatorID, err := requireSuperAdminWriterOrPlatform(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row, err := s.loadAlert(ctx, req.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if row.Status != constants.PollingAlertStatusOpen {
|
||||
return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理")
|
||||
}
|
||||
now := time.Now()
|
||||
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||
result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen).
|
||||
Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败")
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试")
|
||||
}
|
||||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||
ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理",
|
||||
ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID,
|
||||
ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID,
|
||||
IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID,
|
||||
"carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status},
|
||||
BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) {
|
||||
var row model.PollingFailureAlert
|
||||
if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil {
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在")
|
||||
}
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
|
||||
}
|
||||
return &row, nil
|
||||
}
|
||||
|
||||
// CardState 查询单卡轮询状态(只读)。
|
||||
func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var card model.IotCard
|
||||
if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil {
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New(errors.CodeNotFound, "IoT卡不存在")
|
||||
}
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败")
|
||||
}
|
||||
resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
|
||||
SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}}
|
||||
due, err := s.store.ReadDue(ctx, card.ID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败")
|
||||
}
|
||||
carrierID := due.CarrierID
|
||||
if carrierID == 0 {
|
||||
carrierID = card.CarrierID
|
||||
}
|
||||
member := strconv.FormatUint(uint64(card.ID), 10)
|
||||
if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil {
|
||||
resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score))
|
||||
} else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil {
|
||||
resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score))
|
||||
}
|
||||
if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 {
|
||||
resp.Running = true
|
||||
expire := time.Now().Add(ttl)
|
||||
resp.ClaimExpireAt = &expire
|
||||
}
|
||||
resp.PriorityMark = due.Priority
|
||||
for _, checkType := range constants.AllCheckTypes() {
|
||||
item, ok := due.Types[checkType]
|
||||
if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) {
|
||||
continue
|
||||
}
|
||||
state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType),
|
||||
FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}}
|
||||
if item.HasDue {
|
||||
state.NextDueAt = timePtr(item.Due)
|
||||
}
|
||||
if !item.LastOK.IsZero() {
|
||||
state.LastSuccessAt = timePtr(item.LastOK)
|
||||
}
|
||||
if !item.FirstFailed.IsZero() {
|
||||
state.FirstFailedAt = timePtr(item.FirstFailed)
|
||||
}
|
||||
for _, delay := range item.Confirm {
|
||||
state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second))
|
||||
}
|
||||
resp.Types = append(resp.Types, state)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// CarrierStats 按运营商账户的轮询监控。
|
||||
func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names, err := s.carrierNameMap(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var params []model.CarrierPollingParam
|
||||
if err := s.db.WithContext(ctx).Find(¶ms).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||||
}
|
||||
paramByCarrier := map[uint]model.CarrierPollingParam{}
|
||||
for _, row := range params {
|
||||
paramByCarrier[row.CarrierID] = row
|
||||
}
|
||||
var alertRows []struct {
|
||||
CarrierID uint
|
||||
ErrorCategory string
|
||||
Count int64
|
||||
}
|
||||
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count").
|
||||
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
|
||||
Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
|
||||
}
|
||||
state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||
resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}}
|
||||
now := time.Now()
|
||||
nowMS := strconv.FormatInt(now.UnixMilli(), 10)
|
||||
lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504")
|
||||
for _, carrierID := range sortedCarrierIDs(names) {
|
||||
row, configured := paramByCarrier[carrierID]
|
||||
item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID],
|
||||
InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}}
|
||||
pipe := s.redis.Pipeline()
|
||||
priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID))
|
||||
queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID))
|
||||
priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS)
|
||||
queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS)
|
||||
running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf")
|
||||
expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID))
|
||||
var statCmds []*redis.MapStringStringCmd
|
||||
for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} {
|
||||
statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute)))
|
||||
}
|
||||
_, _ = pipe.Exec(ctx)
|
||||
item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val()
|
||||
item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val()
|
||||
cutoff := now.Add(-time.Hour).Unix() / 60
|
||||
for minute, value := range expired.Val() {
|
||||
if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff {
|
||||
count, _ := strconv.ParseInt(value, 10, 64)
|
||||
item.ExpiredClaims1h += count
|
||||
}
|
||||
}
|
||||
var attempts int64
|
||||
for _, cmd := range statCmds {
|
||||
values := cmd.Val()
|
||||
attempts += atoi64(values["attempts"])
|
||||
item.RateLimitWaits += atoi64(values["rate_wait"])
|
||||
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
|
||||
}
|
||||
item.QPSLastMinute = float64(attempts) / 60
|
||||
for _, alert := range alertRows {
|
||||
if alert.CarrierID == carrierID {
|
||||
item.OpenAlerts += alert.Count
|
||||
item.OpenAlertsByType[alert.ErrorCategory] += alert.Count
|
||||
}
|
||||
}
|
||||
resp.Items = append(resp.Items, item)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。
|
||||
func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) {
|
||||
if err := requirePlatformReader(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
operatorID := middleware.GetUserIDFromContext(ctx)
|
||||
if operatorID == 0 {
|
||||
return 0, errors.New(errors.CodeUnauthorized, "未授权访问")
|
||||
}
|
||||
return operatorID, nil
|
||||
}
|
||||
|
||||
func sortedCarrierIDs(names map[uint]string) []uint {
|
||||
ids := make([]uint, 0, len(names))
|
||||
for id := range names {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
for i := 1; i < len(ids); i++ {
|
||||
for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
|
||||
ids[j], ids[j-1] = ids[j-1], ids[j]
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func msTime(ms int64) *time.Time {
|
||||
t := time.UnixMilli(ms)
|
||||
return &t
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
func atoi64(s string) int64 {
|
||||
value, _ := strconv.ParseInt(s, 10, 64)
|
||||
return value
|
||||
}
|
||||
299
internal/task/carrier_card_executor.go
Normal file
299
internal/task/carrier_card_executor.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// CarrierCardExecutor 按运营商账户调度的卡任务执行器:一张卡一个卡任务,按配置步骤顺序串行执行已到期的检查类型。
|
||||
type CarrierCardExecutor struct {
|
||||
base *PollingBase
|
||||
store *carrierqueue.Store
|
||||
settings *carrierqueue.SettingsProvider
|
||||
alerts *carrierqueue.AlertWriter
|
||||
realname *PollingRealnameHandler
|
||||
carddata *PollingCarddataHandler
|
||||
cardStatus *PollingCardStatusHandler
|
||||
pkg *PollingPackageHandler
|
||||
cfg config.CarrierSchedulerConfig
|
||||
instanceID string
|
||||
}
|
||||
|
||||
// NewCarrierCardExecutor 创建卡任务执行器。
|
||||
func NewCarrierCardExecutor(base *PollingBase, store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, alerts *carrierqueue.AlertWriter,
|
||||
realname *PollingRealnameHandler, carddata *PollingCarddataHandler, cardStatus *PollingCardStatusHandler, pkg *PollingPackageHandler,
|
||||
cfg config.CarrierSchedulerConfig, instanceID string) *CarrierCardExecutor {
|
||||
return &CarrierCardExecutor{base: base, store: store, settings: settings, alerts: alerts, realname: realname, carddata: carddata,
|
||||
cardStatus: cardStatus, pkg: pkg, cfg: cfg, instanceID: instanceID}
|
||||
}
|
||||
|
||||
func (e *CarrierCardExecutor) renewInterval() time.Duration {
|
||||
if e.cfg.RenewInterval > 0 {
|
||||
return e.cfg.RenewInterval
|
||||
}
|
||||
return 30 * time.Second
|
||||
}
|
||||
|
||||
func (e *CarrierCardExecutor) lease() time.Duration {
|
||||
if e.cfg.Lease > 0 {
|
||||
return e.cfg.Lease
|
||||
}
|
||||
return 5 * time.Minute
|
||||
}
|
||||
|
||||
// Handle 消费一条卡任务消息;返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
|
||||
func (e *CarrierCardExecutor) Handle(ctx context.Context, body []byte, touch func()) {
|
||||
var msg carrierqueue.CardMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
|
||||
e.base.logger.Warn("卡任务消息无效,丢弃", zap.ByteString("body", body))
|
||||
return
|
||||
}
|
||||
execValue := msg.Token + "|" + e.instanceID
|
||||
execTTL := e.renewInterval() * 2
|
||||
got, err := e.store.AcquireExec(ctx, msg.CardID, msg.Token, execValue, execTTL)
|
||||
if err != nil || got != 1 {
|
||||
// 0:处理中标记已不是该 token(过期或重复投递);-1:该 token 已有执行者(NSQ 重投时原消费者仍在执行)。
|
||||
return
|
||||
}
|
||||
settings := e.settings.Current(ctx)
|
||||
taskCtx, cancel := context.WithTimeout(ctx, settings.TaskTimeout())
|
||||
defer cancel()
|
||||
|
||||
var lost atomic.Bool
|
||||
renewDone := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(e.renewInterval())
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-renewDone:
|
||||
return
|
||||
case <-taskCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, renewErr := e.store.Renew(context.WithoutCancel(ctx), msg.CardID, msg.CarrierID, msg.Token, execValue, e.lease(), execTTL)
|
||||
if renewErr == nil && !ok {
|
||||
// token 已不一致:立即停止后续步骤,不再发起新的网关调用,只写已产生的业务结果。
|
||||
lost.Store(true)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
if touch != nil {
|
||||
touch()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
set, del := e.execute(taskCtx, msg, settings, &lost)
|
||||
close(renewDone)
|
||||
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer finishCancel()
|
||||
finished, finishErr := e.store.Finish(finishCtx, msg.CardID, msg.CarrierID, msg.Token, set, del)
|
||||
if finishErr != nil {
|
||||
e.base.logger.Warn("卡任务收尾失败,等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(finishErr))
|
||||
return
|
||||
}
|
||||
if !finished {
|
||||
e.base.logger.Warn("卡任务收尾时处理中标记已不一致,只写入业务结果", zap.Uint("card_id", msg.CardID),
|
||||
zap.String("metric", "polling.carrier.finish_token_mismatch"))
|
||||
}
|
||||
}
|
||||
|
||||
// execute 串行执行到期检查步骤,返回收尾要写入与删除的到期字段。
|
||||
func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.CardMessage, settings carrierqueue.Settings, lost *atomic.Bool) (map[string]string, []string) {
|
||||
set, del := map[string]string{}, []string{}
|
||||
card, err := e.base.getCardWithCache(ctx, msg.CardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
}
|
||||
return set, del
|
||||
}
|
||||
if !card.EnablePolling || (e.base.scopeChecker != nil && !e.base.scopeChecker.ShouldEnqueue(ctx, card)) {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
return set, del
|
||||
}
|
||||
due, err := e.store.ReadDue(ctx, msg.CardID)
|
||||
if err != nil {
|
||||
return set, del
|
||||
}
|
||||
now := time.Now()
|
||||
flowEvaluated := false
|
||||
executed := map[string]StepOutcome{}
|
||||
for _, checkType := range settings.StepOrder {
|
||||
item, ok := due.Types[checkType]
|
||||
if !ok || !item.HasDue || item.Due.After(now) {
|
||||
continue
|
||||
}
|
||||
if lost.Load() || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow && flowEvaluated {
|
||||
executed[checkType] = stepOK()
|
||||
continue
|
||||
}
|
||||
outcome := e.runStep(ctx, msg, checkType, settings.StepTimeout(checkType))
|
||||
if checkType == constants.CheckTypeCarddata && outcome.TriggeredEvaluation {
|
||||
flowEvaluated = true
|
||||
}
|
||||
executed[checkType] = outcome
|
||||
if outcome.Result == StepCardGone || outcome.Result == StepStopPolling {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
return set, del
|
||||
}
|
||||
}
|
||||
if len(executed) == 0 {
|
||||
return set, del
|
||||
}
|
||||
// 收尾时按最新卡数据重新匹配间隔(步骤中卡状态、套餐可能已变化)。
|
||||
e.base.invalidateCardCache(ctx, msg.CardID)
|
||||
if fresh, freshErr := e.base.getCardWithCache(ctx, msg.CardID); freshErr == nil {
|
||||
card = fresh
|
||||
}
|
||||
intervals := e.base.configMgr.MergedTaskIntervals(card)
|
||||
now = time.Now()
|
||||
for checkType, outcome := range executed {
|
||||
e.writeBack(ctx, card, checkType, outcome, due.Types[checkType], intervals, settings, now, set, &del)
|
||||
}
|
||||
return set, del
|
||||
}
|
||||
|
||||
// runStep 以单步超时、等待模式限流与 panic 恢复执行一个检查步骤。
|
||||
func (e *CarrierCardExecutor) runStep(ctx context.Context, msg carrierqueue.CardMessage, checkType string, timeout time.Duration) (outcome StepOutcome) {
|
||||
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
stepCtx = gateway.WithRateLimitWait(stepCtx, timeout)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
e.base.logger.Error("卡任务步骤 panic,按失败处理", zap.Uint("card_id", msg.CardID), zap.String("check_type", checkType), zap.Any("panic", r))
|
||||
outcome = StepOutcome{Result: StepFailed, Category: "panic", Message: fmt.Sprint(r)}
|
||||
}
|
||||
}()
|
||||
switch checkType {
|
||||
case constants.CheckTypeCardStatus:
|
||||
return e.cardStatus.Step(stepCtx, msg.CardID)
|
||||
case constants.CheckTypeCarddata:
|
||||
return e.carddata.Step(stepCtx, msg.CardID)
|
||||
case constants.CheckTypeRealname:
|
||||
return e.realname.Step(stepCtx, msg.CardID)
|
||||
case constants.CheckTypePackage:
|
||||
stepCtx = withPollingWorkerAuditContext(stepCtx, constants.TaskTypePollingPackage, "套餐状态轮询任务", msg.Token)
|
||||
return e.pkg.Step(stepCtx, msg.CardID)
|
||||
default:
|
||||
return stepSkip()
|
||||
}
|
||||
}
|
||||
|
||||
// writeBack 按步骤结果计算该检查类型下次到期、失败计数与延迟确认,并触发告警或恢复。
|
||||
func (e *CarrierCardExecutor) writeBack(ctx context.Context, card *model.IotCard, checkType string, outcome StepOutcome, previous carrierqueue.TypeDue,
|
||||
intervals map[string]polling.TaskTypeInterval, settings carrierqueue.Settings, now time.Time, set map[string]string, del *[]string) {
|
||||
info, ok := intervals[constants.TaskTypeFromCheckType(checkType)]
|
||||
if !ok || info.Interval <= 0 {
|
||||
// 该类型已不在配置中(间隔为 0 或未配置):移除到期,不再执行。
|
||||
*del = append(*del, carrierqueue.FieldDue(checkType), carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType),
|
||||
carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||
return
|
||||
}
|
||||
interval := time.Duration(info.Interval) * time.Second
|
||||
normalNext := e.base.configMgr.NextCheckAt(ctx, card, constants.TaskTypeFromCheckType(checkType), info, now)
|
||||
switch outcome.Result {
|
||||
case StepSucceeded:
|
||||
next := normalNext
|
||||
set[carrierqueue.FieldOK(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||
if previous.FailCount > 0 {
|
||||
e.alerts.ReportRecovered(card.ID, checkType)
|
||||
}
|
||||
*del = append(*del, carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType), carrierqueue.FieldError(checkType))
|
||||
// 延迟确认:结果已满足预期时取消,否则按序列取下一次确认 = min(正常间隔, 当前 + 确认延迟)。
|
||||
if len(previous.Confirm) > 0 {
|
||||
if previous.Expected != "" && outcome.Observed == previous.Expected {
|
||||
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||
} else {
|
||||
if confirmAt := now.Add(previous.Confirm[0]); confirmAt.Before(next) {
|
||||
next = confirmAt
|
||||
}
|
||||
if rest := previous.Confirm[1:]; len(rest) > 0 {
|
||||
set[carrierqueue.FieldConfirm(checkType)] = joinSeconds(rest)
|
||||
} else {
|
||||
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||
}
|
||||
}
|
||||
}
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(next.UnixMilli(), 10)
|
||||
case StepFailed:
|
||||
set[carrierqueue.FieldError(checkType)] = truncateRunes(outcome.Category+"|"+outcome.Message, 300)
|
||||
if !outcome.CountsAsFailure() {
|
||||
// 上游频率限制、已是目标状态:不计入连续失败,按首个退避间隔重试。
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, 1, interval)).UnixMilli(), 10)
|
||||
return
|
||||
}
|
||||
failCount := previous.FailCount + 1
|
||||
firstFailed := previous.FirstFailed
|
||||
if firstFailed.IsZero() {
|
||||
firstFailed = now
|
||||
set[carrierqueue.FieldFirstFail(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||
}
|
||||
set[carrierqueue.FieldFail(checkType)] = strconv.Itoa(failCount)
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, failCount, interval)).UnixMilli(), 10)
|
||||
if failCount >= settings.AlertThreshold(checkType) || now.Sub(firstFailed) >= settings.AlertFailDuration || settings.ImmediateAlert[outcome.Category] {
|
||||
e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, CheckType: checkType,
|
||||
Category: outcome.Category, Message: outcome.Message, FailCount: failCount, FirstFailed: firstFailed, At: now})
|
||||
}
|
||||
case StepRateLimited:
|
||||
// 限流等待超时:请求未发出,按短延迟重排,不计失败。
|
||||
delay := 5 * time.Second
|
||||
if e.base.rateLimitedDelay != nil {
|
||||
if configured := e.base.rateLimitedDelay(); configured > 0 {
|
||||
delay = configured
|
||||
}
|
||||
}
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||
case StepDeferred:
|
||||
delay := outcome.DeferDelay
|
||||
if delay <= 0 {
|
||||
delay = 30 * time.Second
|
||||
}
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||
default:
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(normalNext.UnixMilli(), 10)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *CarrierCardExecutor) removeCard(ctx context.Context, cardID, carrierID uint) {
|
||||
removeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := e.store.Remove(removeCtx, cardID, carrierID); err != nil {
|
||||
e.base.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func joinSeconds(delays []time.Duration) string {
|
||||
parts := make([]string, 0, len(delays))
|
||||
for _, delay := range delays {
|
||||
parts = append(parts, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func truncateRunes(s string, max int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max])
|
||||
}
|
||||
@@ -71,6 +71,8 @@ type PollingBase struct {
|
||||
priorityAudit *audit.Writer
|
||||
scopeChecker *polling.PollingLifecycleService
|
||||
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
||||
errorClassifier ErrorClassifier // 上游错误分类(按运营商账户调度的失败退避与告警使用)
|
||||
schedulerMode SchedulerModeReader // 当前调度模式;carrier 时旧 Asynq 轮询任务直接结束
|
||||
}
|
||||
|
||||
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
||||
|
||||
@@ -30,34 +30,42 @@ func NewPollingCarddataHandler(base *PollingBase, gw *gateway.Client, carrier *p
|
||||
return &PollingCarddataHandler{base: base, gateway: gw, carrier: carrier, observation: observation, integration: integration}
|
||||
}
|
||||
|
||||
// Handle 处理流量检查任务。
|
||||
// Handle 处理流量检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
startedAt := time.Now()
|
||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCarddata) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCarddata)
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCarddata, h.Step(ctx, cardID))
|
||||
}
|
||||
|
||||
// Step 执行一次流量检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startedAt := time.Now()
|
||||
lockToken, locked, err := h.base.acquireCardTrafficSyncLock(ctx, cardID)
|
||||
if err != nil || !locked {
|
||||
h.base.logger.Warn("卡流量同步锁不可用,延迟重入队", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
return h.base.queueMgr.Requeue(ctx, cardID, constants.TaskTypePollingCarddata, time.Now().Add(5*time.Second))
|
||||
return stepDefer(5 * time.Second)
|
||||
}
|
||||
defer h.base.releaseCardTrafficSyncLock(ctx, cardID, lockToken)
|
||||
|
||||
// 优先轮询互斥接缝:放在流量同步锁获取成功之后,避免认领后拿不到锁导致优先项空占。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
if priorityErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCarddata))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
@@ -65,42 +73,42 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return stepSkip()
|
||||
}
|
||||
if h.gateway == nil {
|
||||
priority.FailFinal(ctx, "流量查询能力未配置")
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return stepSkip()
|
||||
}
|
||||
priority.MarkExecution(ctx)
|
||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayTraffic, constants.CardObservationSceneTrafficPolling)
|
||||
result, err := h.gateway.QueryFlow(gateway.WithCarrier(ctx, card.CarrierID), &gateway.FlowQueryReq{CardNo: card.ICCID})
|
||||
if gateway.IsRateLimited(err) {
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCarddata)
|
||||
return StepOutcome{Result: StepRateLimited}
|
||||
}
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询流量失败", priority)
|
||||
priority.FailRetryable(ctx, "查询流量失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询流量失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "查询流量失败", err)
|
||||
}
|
||||
if strings.TrimSpace(result.ICCID) == "" {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "流量查询响应缺少 ICCID", priority)
|
||||
priority.FailRetryable(ctx, "流量查询响应缺少 ICCID")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil || h.carrier == nil {
|
||||
priority.FailFinal(ctx, "卡流量观测能力未配置")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||
@@ -113,7 +121,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用流量观测失败", priority)
|
||||
priority.FailRetryable(ctx, "应用流量观测失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用流量观测失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "应用流量观测失败", err)
|
||||
}
|
||||
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
||||
changed := decision.IncrementMB > 0 || decision.CrossMonth
|
||||
@@ -132,15 +140,15 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
h.base.logger.Info("流量轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||
priority.Complete(ctx)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return StepOutcome{Result: StepSucceeded, TriggeredEvaluation: changed}
|
||||
}
|
||||
|
||||
func (h *PollingCarddataHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
||||
func (h *PollingCarddataHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
h.base.logger.Warn(message, fields...)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, false, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return h.base.stepFail(ctx, constants.GatewayOpQueryFlow, message, err)
|
||||
}
|
||||
|
||||
@@ -37,70 +37,85 @@ func NewPollingCardStatusHandler(base *PollingBase, gw *gateway.Client, observat
|
||||
return &PollingCardStatusHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||
}
|
||||
|
||||
// Handle 处理卡状态轮询任务。
|
||||
// Handle 处理卡状态轮询任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
startedAt := time.Now()
|
||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCardStatus) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCardStatus)
|
||||
outcome := h.Step(ctx, cardID)
|
||||
if outcome.Result == StepSucceeded {
|
||||
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
||||
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCardStatus, outcome)
|
||||
}
|
||||
|
||||
// Step 执行一次卡状态检查步骤,返回结果分类与观测到的网络状态;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingCardStatusHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startedAt := time.Now()
|
||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
if priorityErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCardStatus))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return stepSkip()
|
||||
}
|
||||
if h.gateway == nil {
|
||||
priority.FailFinal(ctx, "卡状态查询能力未配置")
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return stepSkip()
|
||||
}
|
||||
priority.MarkExecution(ctx)
|
||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayNetwork, constants.CardObservationSceneNetworkPolling)
|
||||
result, err := h.gateway.QueryCardStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||
if gateway.IsRateLimited(err) {
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCardStatus)
|
||||
return StepOutcome{Result: StepRateLimited}
|
||||
}
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询卡状态失败", priority)
|
||||
priority.FailRetryable(ctx, "查询卡状态失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询卡状态失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "查询卡状态失败", err)
|
||||
}
|
||||
if strings.TrimSpace(result.ICCID) == "" {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "卡状态查询响应缺少 ICCID", priority)
|
||||
priority.FailRetryable(ctx, "卡状态查询响应缺少 ICCID")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil {
|
||||
priority.FailFinal(ctx, "卡网络观测能力未配置")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
||||
@@ -113,7 +128,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用网络状态观测失败", priority)
|
||||
priority.FailRetryable(ctx, "应用网络状态观测失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
||||
}
|
||||
changed := decision.StatusChanged || decision.StopReasonChanged || decision.StopPolling
|
||||
if changed {
|
||||
@@ -134,23 +149,24 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
||||
if decision.StopPolling {
|
||||
h.base.logger.Info("卡命中风险停机或已销户,已关闭该卡轮询", zap.Uint("card_id", cardID), zap.String("gateway_extend", decision.GatewayExtend))
|
||||
return nil
|
||||
return StepOutcome{Result: StepStopPolling}
|
||||
}
|
||||
|
||||
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
||||
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
outcome := stepOK()
|
||||
if decision.StatusKnown {
|
||||
outcome.Observed = ObservedOffline
|
||||
if decision.AfterStatus == constants.NetworkStatusOnline {
|
||||
outcome.Observed = ObservedOnline
|
||||
}
|
||||
}
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return outcome
|
||||
}
|
||||
|
||||
func (h *PollingCardStatusHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
||||
func (h *PollingCardStatusHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
h.base.logger.Warn(message, fields...)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return h.base.stepFail(ctx, constants.GatewayOpQueryStatus, message, err)
|
||||
}
|
||||
|
||||
@@ -37,14 +37,15 @@ func NewPollingPackageHandler(
|
||||
}
|
||||
}
|
||||
|
||||
// Handle 处理套餐检查任务
|
||||
// Handle 处理套餐检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
startTime := time.Now()
|
||||
|
||||
cardID, ok := parseTaskPayload(t.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
||||
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
||||
@@ -52,42 +53,48 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingPackage)
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingPackage, h.Step(ctx, cardID))
|
||||
}
|
||||
|
||||
// Step 执行一次套餐检查步骤(按规则重新判断停复机),返回结果分类;不负责重入队。
|
||||
// 调用方负责在 ctx 中设置 Worker 审计上下文。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingPackageHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startTime := time.Now()
|
||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
if priorityErr != nil {
|
||||
h.base.logger.Error("查询活动优先轮询项失败", zap.Uint("card_id", cardID), zap.Error(priorityErr))
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return h.base.stepFail(ctx, "", "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingPackage))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
card, cacheErr := h.base.getCardWithCache(ctx, cardID)
|
||||
if cacheErr != nil {
|
||||
if isNotFound(cacheErr) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
h.base.logger.Error("获取卡信息失败", zap.Uint("card_id", cardID), zap.Error(cacheErr))
|
||||
// 本地读卡失败属可恢复失败:与其余三个轮询 handler 统一按尝试上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return h.base.stepFail(ctx, "", "获取卡信息失败", cacheErr)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepSkip()
|
||||
}
|
||||
|
||||
if h.stopResumeSvc == nil {
|
||||
priority.FailFinal(ctx, "套餐评估能力未配置")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepSkip()
|
||||
}
|
||||
|
||||
// 停复机评估就是本任务的上游执行动作:先累加尝试次数再发起。
|
||||
@@ -98,7 +105,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
zap.Uint("card_id", cardID), zap.Error(loadErr))
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return h.base.stepFail(ctx, "", "获取卡信息失败", loadErr)
|
||||
}
|
||||
h.base.logger.Info("套餐检查:执行停复机评估",
|
||||
zap.Uint("card_id", cardID),
|
||||
@@ -119,10 +126,11 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
zap.Uint("card_id", cardID), zap.Error(evalErr))
|
||||
priority.FailRetryable(ctx, "停复机评估失败")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
outcome := h.base.stepFail(ctx, constants.GatewayOpStart, "停复机评估失败", evalErr)
|
||||
return outcome
|
||||
}
|
||||
|
||||
priority.Complete(ctx)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepOK()
|
||||
}
|
||||
|
||||
@@ -28,81 +28,88 @@ func NewPollingRealnameHandler(base *PollingBase, gw *gateway.Client, observatio
|
||||
return &PollingRealnameHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||
}
|
||||
|
||||
// Handle 处理实名检查任务。
|
||||
// Handle 处理实名检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
startedAt := time.Now()
|
||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingRealname) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingRealname)
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingRealname, h.Step(ctx, cardID))
|
||||
}
|
||||
|
||||
// Step 执行一次实名检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingRealnameHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startedAt := time.Now()
|
||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
if priorityErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingRealname))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepSkip()
|
||||
}
|
||||
if h.gateway == nil {
|
||||
priority.FailFinal(ctx, "实名查询能力未配置")
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepSkip()
|
||||
}
|
||||
// 所属运营商账户未开启实名管控时实名结果不影响停复机,跳过查询并按正常间隔重入队(不再按行业卡跳过)。
|
||||
skippable, scopeErr := h.base.realnameQuerySkippable(ctx, card)
|
||||
if scopeErr != nil {
|
||||
priority.FailRetryable(ctx, "查询运营商账户实名管控失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
||||
}
|
||||
if skippable {
|
||||
priority.FailFinal(ctx, "所属运营商账户未开启实名管控,跳过实名查询")
|
||||
h.base.logger.Debug("所属运营商账户未开启实名管控,跳过实名查询", zap.Uint("card_id", cardID), zap.Uint("carrier_id", card.CarrierID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepSkip()
|
||||
}
|
||||
priority.MarkExecution(ctx)
|
||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayRealname, constants.CardObservationSceneRealnamePolling)
|
||||
result, err := h.gateway.QueryRealnameStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||
if gateway.IsRateLimited(err) {
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingRealname)
|
||||
return StepOutcome{Result: StepRateLimited}
|
||||
}
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询实名状态失败", priority)
|
||||
priority.FailRetryable(ctx, "查询实名状态失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询实名状态失败", err)
|
||||
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, "查询实名状态失败", err)
|
||||
}
|
||||
if strings.TrimSpace(result.ICCID) == "" {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "实名查询响应缺少 ICCID", priority)
|
||||
priority.FailRetryable(ctx, "实名查询响应缺少 ICCID")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil {
|
||||
priority.FailFinal(ctx, "卡实名观测能力未配置")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
||||
@@ -115,7 +122,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用实名观测失败", priority)
|
||||
priority.FailRetryable(ctx, "应用实名观测失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用实名观测失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "应用实名观测失败", err)
|
||||
}
|
||||
changed := decision.StatusChanged
|
||||
if changed {
|
||||
@@ -133,15 +140,19 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
h.base.logger.Info("实名轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||
priority.Complete(ctx)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepOK()
|
||||
}
|
||||
|
||||
func (h *PollingRealnameHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
||||
func (h *PollingRealnameHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, message, err)
|
||||
}
|
||||
|
||||
func (h *PollingRealnameHandler) failOp(ctx context.Context, cardID uint, startedAt time.Time, op, message string, err error) StepOutcome {
|
||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
h.base.logger.Warn(message, fields...)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, false, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return h.base.stepFail(ctx, op, message, err)
|
||||
}
|
||||
|
||||
123
internal/task/polling_step.go
Normal file
123
internal/task/polling_step.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StepResult 是一个检查步骤的执行结果分类。
|
||||
type StepResult int
|
||||
|
||||
const (
|
||||
// StepSucceeded 检查成功(含观测入账、评估完成)。
|
||||
StepSucceeded StepResult = iota
|
||||
// StepFailed 检查失败,计入连续失败(是否计入还要看失败类别,见 countsAsFailure)。
|
||||
StepFailed
|
||||
// StepRateLimited 本账户限流拒绝或等待超时,请求未发出:短延迟重排,不计失败。
|
||||
StepRateLimited
|
||||
// StepDeferred 暂不可执行(流量同步锁被占、优先项由其他执行持有):短延迟或按间隔重排,不计失败。
|
||||
StepDeferred
|
||||
// StepSkipped 本轮无需执行(能力未配置、不在轮询范围、所属账户未开启实名管控):按正常间隔重排。
|
||||
StepSkipped
|
||||
// StepCardGone 卡不存在:不再重排。
|
||||
StepCardGone
|
||||
// StepStopPolling 观测后该卡已关闭轮询(风险停机、已销户):不再重排。
|
||||
StepStopPolling
|
||||
)
|
||||
|
||||
// StepOutcome 是一个检查步骤的结果。旧分片调度把它映射为原有的重入队动作;按运营商账户调度据此写回到期与失败计数。
|
||||
type StepOutcome struct {
|
||||
Result StepResult
|
||||
// DeferDelay 为 StepDeferred 时的建议短延迟;0 表示按正常间隔。
|
||||
DeferDelay time.Duration
|
||||
// Category 失败类别:网关错误类别(network / timeout / http / invalid_response),上游业务错误时为错误分类类别。
|
||||
Category string
|
||||
// Message 失败信息(已脱敏截断)。
|
||||
Message string
|
||||
// Observed 卡状态步骤观测到的网络状态(online / offline),用于延迟确认是否已满足预期。
|
||||
Observed string
|
||||
// TriggeredEvaluation 流量步骤本轮观测产生了流量变化(会触发停复机评估),可据此跳过套餐步骤。
|
||||
TriggeredEvaluation bool
|
||||
}
|
||||
|
||||
// 卡状态步骤的观测结果取值。
|
||||
const (
|
||||
ObservedOnline = "online"
|
||||
ObservedOffline = "offline"
|
||||
)
|
||||
|
||||
func stepOK() StepOutcome { return StepOutcome{Result: StepSucceeded} }
|
||||
|
||||
func stepSkip() StepOutcome { return StepOutcome{Result: StepSkipped} }
|
||||
|
||||
func stepDefer(delay time.Duration) StepOutcome {
|
||||
return StepOutcome{Result: StepDeferred, DeferDelay: delay}
|
||||
}
|
||||
|
||||
// stepFail 按错误构造失败结果:网关结构化错误取其类别,上游业务错误再按错误分类规则细分(用于判断是否计入连续失败与告警)。
|
||||
func (b *PollingBase) stepFail(ctx context.Context, op, message string, err error) StepOutcome {
|
||||
outcome := StepOutcome{Result: StepFailed, Message: message, Category: "local"}
|
||||
if gwErr, ok := gateway.AsGatewayError(err); ok {
|
||||
outcome.Category = gwErr.Category
|
||||
if gwErr.Category == constants.GatewayErrorUpstream {
|
||||
outcome.Message = message + ":" + gwErr.ProviderMessage
|
||||
if b.errorClassifier != nil {
|
||||
outcome.Category = b.errorClassifier(ctx, op, gwErr.ProviderMessage)
|
||||
}
|
||||
}
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
// CountsAsFailure 判断失败结果是否计入连续失败:上游频率限制与已是目标状态不计入。
|
||||
func (o StepOutcome) CountsAsFailure() bool {
|
||||
if o.Result != StepFailed {
|
||||
return false
|
||||
}
|
||||
switch o.Category {
|
||||
case constants.GatewayClassRateLimitedUpstream, constants.GatewayClassAlreadyTarget:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ErrorClassifier 对上游业务错误信息分类,返回错误分类类别。
|
||||
type ErrorClassifier func(ctx context.Context, op, message string) string
|
||||
|
||||
// SetErrorClassifier 注入上游错误分类器(按运营商账户调度的失败退避与告警使用)。
|
||||
func (b *PollingBase) SetErrorClassifier(classifier ErrorClassifier) {
|
||||
b.errorClassifier = classifier
|
||||
}
|
||||
|
||||
// SchedulerModeReader 返回当前卡轮询调度模式(受控系统配置 polling.scheduler.mode)。
|
||||
type SchedulerModeReader func(ctx context.Context) string
|
||||
|
||||
// SetSchedulerModeReader 注入调度模式读取;未注入时视为旧分片调度。
|
||||
func (b *PollingBase) SetSchedulerModeReader(reader SchedulerModeReader) {
|
||||
b.schedulerMode = reader
|
||||
}
|
||||
|
||||
// carrierSchedulerActive 判断当前是否已切到按运营商账户调度:是则旧 Asynq 轮询任务直接结束,不调网关、不重入旧队列。
|
||||
func (b *PollingBase) carrierSchedulerActive(ctx context.Context) bool {
|
||||
return b != nil && b.schedulerMode != nil && b.schedulerMode(ctx) == constants.PollingSchedulerModeCarrier
|
||||
}
|
||||
|
||||
// applyLegacyOutcome 把步骤结果映射为旧分片调度原有的重入队动作。
|
||||
func (b *PollingBase) applyLegacyOutcome(ctx context.Context, cardID uint, taskType string, outcome StepOutcome) error {
|
||||
switch outcome.Result {
|
||||
case StepCardGone, StepStopPolling:
|
||||
return nil
|
||||
case StepRateLimited:
|
||||
return b.requeueRateLimited(cardID, taskType)
|
||||
case StepDeferred:
|
||||
if outcome.DeferDelay > 0 {
|
||||
return b.queueMgr.Requeue(ctx, cardID, taskType, time.Now().Add(outcome.DeferDelay))
|
||||
}
|
||||
return b.requeueCard(ctx, cardID, taskType)
|
||||
default:
|
||||
return b.requeueCard(ctx, cardID, taskType)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +1,75 @@
|
||||
## 0. 前置确认
|
||||
|
||||
- [ ] 0.1 确认第 1 个 Change(限流器)与第 3 个 Change(去掉保护期)已上线;维护者提供测试环境 nsqd + nsqlookupd 地址。
|
||||
- [x] 0.1 确认第 1 个 Change(限流器)与第 3 个 Change(去掉保护期)已上线;维护者提供测试环境 nsqd + nsqlookupd 地址。(第 1、3 个 Change 已部署测试环境;测试环境 NSQ 由 docker-compose.prod.yml 新增 nsqlookupd + nsqd 提供,Worker 注入 nsqd:4150 / nsqlookupd:4161)
|
||||
|
||||
## 1. 迁移与配置
|
||||
|
||||
- [ ] 1.1 新增成对迁移:`tb_carrier_polling_param`(`carrier_id` 唯一、`inflight_limit`、`card_max_in_flight`、`stop_resume_max_in_flight`、`sr_inflight_limit`、`priority_max_ratio`、`stop_resume_min_interval`、`stop_resume_confirm_delay`、审计字段)。
|
||||
- [ ] 1.2 新增成对迁移:`tb_polling_check_scene`(`scene` 唯一、`check_types`、`confirm_delays`、`cooldown`),预置 8 个场景,读入口场景带默认冷却。
|
||||
- [ ] 1.3 新增成对迁移:`tb_polling_failure_alert`(卡、运营商账户、检查类型、连续失败、首次与最近失败、最近错误、状态、处理人与时间;部分唯一索引 `(card_id, check_type) WHERE status = 未处理`)。
|
||||
- [ ] 1.4 新增成对迁移:`tb_system_config` 插入 `polling.scheduler.mode`(默认 `legacy`)、步骤顺序、单步超时、卡任务总超时余量、退避序列、告警阈值、账户级告警窗口与阈值;`tb_polling_concurrency_config` 表注释标记废弃。
|
||||
- [ ] 1.5 `pkg/config` 与 `defaults/config.yaml`:NSQ 地址、`msg_timeout`、租约、续期间隔、分发间隔、优雅退出宽限期、默认执行上限与 `max_in_flight`;`go.mod` 引入 NSQ 客户端。
|
||||
- [ ] 1.6 注册受控系统配置项并实现 JSON 结构校验(步骤类型合法且不重复、退避序列递增)。
|
||||
- [x] 1.1 新增成对迁移:`tb_carrier_polling_param`(`carrier_id` 唯一、`inflight_limit`、`card_max_in_flight`、`stop_resume_max_in_flight`、`sr_inflight_limit`、`priority_max_ratio`、`stop_resume_min_interval`、`stop_resume_confirm_delay`、审计字段)。(000251)
|
||||
- [x] 1.2 新增成对迁移:`tb_polling_check_scene`(`scene` 唯一、`check_types`、`confirm_delays`、`cooldown`),预置 8 个场景,读入口场景带默认冷却。(000252,预置 8 个场景)
|
||||
- [x] 1.3 新增成对迁移:`tb_polling_failure_alert`(卡、运营商账户、检查类型、连续失败、首次与最近失败、最近错误、状态、处理人与时间;部分唯一索引 `(card_id, check_type) WHERE status = 未处理`)。(000253;部分唯一索引含告警级别与账户,账户级告警 card_id=0)
|
||||
- [x] 1.4 新增成对迁移:`tb_system_config` 插入 `polling.scheduler.mode`(默认 `legacy`)、步骤顺序、单步超时、卡任务总超时余量、退避序列、告警阈值、账户级告警窗口与阈值;`tb_polling_concurrency_config` 表注释标记废弃。(000254,默认 legacy)
|
||||
- [x] 1.5 `pkg/config` 与 `defaults/config.yaml`:NSQ 地址、`msg_timeout`、租约、续期间隔、分发间隔、优雅退出宽限期、默认执行上限与 `max_in_flight`;`go.mod` 引入 NSQ 客户端。(polling.carrier.*;go-nsq v1.1.0)
|
||||
- [x] 1.6 注册受控系统配置项并实现 JSON 结构校验(步骤类型合法且不重复、退避序列递增)。(carrierqueue.RegisterConfigDefinitions,后台系统配置可编辑)
|
||||
|
||||
## 2. Redis 结构与 Lua
|
||||
|
||||
- [ ] 2.1 常量:账户队列、running、dispatch_lock,卡 due、claim、exec 的 key。
|
||||
- [ ] 2.2 领取 Lua:清理过期 running、执行上限、优先占比、预计调用数对比本秒剩余 QPS、`SET claim NX PX`、推后 score、写 running;已有 claim 时推后到剩余 TTL 后。
|
||||
- [ ] 2.3 续期 Lua:校验 token,延长 claim、exec,推后 score 与 running。
|
||||
- [ ] 2.4 收尾 Lua:校验 token,删 claim、exec,移出 running,写 due 与失败计数,按优先标记设置 score;token 不一致时只写业务结果。
|
||||
- [ ] 2.5 请求检查 Lua:设 due[type]=now、冷却判断、放入 priority 或打优先标记,合并重复请求。
|
||||
- [x] 2.1 常量:账户队列、running、dispatch_lock,卡 due、claim、exec 的 key。
|
||||
- [x] 2.2 领取 Lua:清理过期 running、执行上限、优先占比、预计调用数对比本秒剩余 QPS、`SET claim NX PX`、推后 score、写 running;已有 claim 时推后到剩余 TTL 后。(按限流组剩余 QPS 在 Go 侧计算后传入 Lua;只统计模式不按 QPS 限制)
|
||||
- [x] 2.3 续期 Lua:校验 token,延长 claim、exec,推后 score 与 running。
|
||||
- [x] 2.4 收尾 Lua:校验 token,删 claim、exec,移出 running,写 due 与失败计数,按优先标记设置 score;token 不一致时只写业务结果。
|
||||
- [x] 2.5 请求检查 Lua:设 due[type]=now、冷却判断、放入 priority 或打优先标记,合并重复请求。(另支持普通通道:手动触发只把到期设为现在、不进优先队列)
|
||||
|
||||
## 3. 初始化与生命周期
|
||||
|
||||
- [ ] 3.1 `initializer.go`:按批读库计算各类型 due(复用第 2 个 Change 的分层与重置日前到期计算),写 due 与账户 queue;进度写入 init-progress。
|
||||
- [ ] 3.2 `lifecycle_service.go`、`asset_polling_service.go`:增删卡、状态变化、启停轮询改为操作账户队列与 due;恢复轮询时 due 设为当前加抖动。
|
||||
- [ ] 3.3 优先提示(`priority_manual_service.go`)与手动触发(`manual_trigger_service.go`)改为写 due 与账户队列。
|
||||
- [x] 3.1 `initializer.go`:按批读库计算各类型 due(复用第 2 个 Change 的分层与重置日前到期计算),写 due 与账户 queue;进度写入 init-progress。(CarrierInitializer;进度在 Worker 内与 /polling-stats/carrier-queues 的 init_state 可见,未并入旧 init-progress 接口)
|
||||
- [x] 3.2 `lifecycle_service.go`、`asset_polling_service.go`:增删卡、状态变化、启停轮询改为操作账户队列与 due;恢复轮询时 due 设为当前加抖动。(API 与 Worker 的生命周期服务按模式写入;资产轮询开关经生命周期回调生效)
|
||||
- [x] 3.3 优先提示(`priority_manual_service.go`)与手动触发(`manual_trigger_service.go`)改为写 due 与账户队列。(旧提示通道与手动触发 List 由新分发器每秒消费后写入账户队列,接口不变)
|
||||
|
||||
## 4. 分发器
|
||||
|
||||
- [ ] 4.1 Worker 内每秒对各账户抢分发锁并执行领取 Lua,发布到 `polling.card.{cid}`;发布失败按 token 删 claim、score 恢复为当前时间。
|
||||
- [ ] 4.2 读取账户调度参数(缺省用默认),执行上限不超过订阅进程 `max_in_flight` 之和,不满足时收紧并上报。
|
||||
- [ ] 4.3 定时动作:统计各账户处理中标记到期次数、写分发器心跳。
|
||||
- [x] 4.1 Worker 内每秒对各账户抢分发锁并执行领取 Lua,发布到 `polling.card.{cid}`;发布失败按 token 删 claim、score 恢复为当前时间。
|
||||
- [x] 4.2 读取账户调度参数(缺省用默认),执行上限不超过订阅进程 `max_in_flight` 之和,不满足时收紧并上报。
|
||||
- [x] 4.3 定时动作:统计各账户处理中标记到期次数、写分发器心跳。
|
||||
|
||||
## 5. NSQ 消费者与卡任务
|
||||
|
||||
- [ ] 5.1 NSQ 生产者与消费者封装:按账户动态订阅与取消订阅、`max_in_flight`、`Touch`、优雅退出(停止分发与订阅、宽限期内完成后 FIN)。
|
||||
- [ ] 5.2 消费入口:核对 claim token、抢 exec 执行权,任一失败即 FIN。
|
||||
- [ ] 5.3 将实名、流量、卡状态、套餐 4 个 handler 拆为步骤函数(`StepInput` 含可选外部结果);按配置顺序串行执行到期类型;单步超时、总超时、panic 恢复。
|
||||
- [ ] 5.4 执行期间按续期间隔续期;续期失败立即停止后续步骤。
|
||||
- [ ] 5.5 网关调用使用限流器等待模式;限流等待超时按短延迟重排不计失败。
|
||||
- [ ] 5.6 沿用 `polling_priority_claim.go` 的认领与收尾;保留卡流量同步锁。
|
||||
- [x] 5.1 NSQ 生产者与消费者封装:按账户动态订阅与取消订阅、`max_in_flight`、`Touch`、优雅退出(停止分发与订阅、宽限期内完成后 FIN)。(internal/infrastructure/nsqbus)
|
||||
- [x] 5.2 消费入口:核对 claim token、抢 exec 执行权,任一失败即 FIN。
|
||||
- [x] 5.3 将实名、流量、卡状态、套餐 4 个 handler 拆为步骤函数(`StepInput` 含可选外部结果);按配置顺序串行执行到期类型;单步超时、总超时、panic 恢复。(四个 handler 拆出 Step,旧 Asynq Handle 只映射重入队;StepInput 外部结果暂未实现,按需在批量查询时扩展)
|
||||
- [x] 5.4 执行期间按续期间隔续期;续期失败立即停止后续步骤。
|
||||
- [x] 5.5 网关调用使用限流器等待模式;限流等待超时按短延迟重排不计失败。(ctx 标记等待模式)
|
||||
- [x] 5.6 沿用 `polling_priority_claim.go` 的认领与收尾;保留卡流量同步锁。
|
||||
|
||||
## 6. 失败退避与告警
|
||||
|
||||
- [ ] 6.1 收尾按"卡 × 类型"更新连续失败、首次失败、最近错误;按退避序列计算下次到期,频率限制与限流超时不计失败;成功清零。
|
||||
- [ ] 6.2 达到阈值或首次即告警类别时写或更新 `tb_polling_failure_alert`;成功后自动转已恢复;通知走现有告警通道。
|
||||
- [ ] 6.3 账户级告警:窗口内同账户告警数超过阈值时产生账户级告警。
|
||||
- [x] 6.1 收尾按"卡 × 类型"更新连续失败、首次失败、最近错误;按退避序列计算下次到期,频率限制与限流超时不计失败;成功清零。
|
||||
- [x] 6.2 达到阈值或首次即告警类别时写或更新 `tb_polling_failure_alert`;成功后自动转已恢复;通知走现有告警通道。(通知以 ERROR 日志 + metric 输出,未接入 alert_service 的规则通知通道)
|
||||
- [x] 6.3 账户级告警:窗口内同账户告警数超过阈值时产生账户级告警。
|
||||
|
||||
## 7. 业务事件与读入口请求检查
|
||||
|
||||
- [ ] 7.1 订单支付、套餐激活、停复机成功、设备操作(按设备下各卡)、自动续费购包 5 处 `AppendSeriesRequested` 改为事务提交后请求检查。
|
||||
- [ ] 7.2 C 端资产详情、后台资产实时状态、C 端与后台实名链接改为请求检查;OpenAPI 查询保持不触发。
|
||||
- [ ] 7.3 延迟确认:按场景在检查完成后追加确认到期;结果已满足预期时取消。
|
||||
- [ ] 7.4 卡检查不再创建观测序列;`query_device_info` 等非运营商上游部分保持不变。
|
||||
- [x] 7.1 订单支付、套餐激活、停复机成功、设备操作(按设备下各卡)、自动续费购包 5 处 `AppendSeriesRequested` 改为事务提交后请求检查。(业务事件仍在原事务写 Outbox,carrier 模式下 Outbox 消费者转为请求检查,满足提交后请求)
|
||||
- [x] 7.2 C 端资产详情、后台资产实时状态、C 端与后台实名链接改为请求检查;OpenAPI 查询保持不触发。
|
||||
- [x] 7.3 延迟确认:按场景在检查完成后追加确认到期;结果已满足预期时取消。
|
||||
- [x] 7.4 卡检查不再创建观测序列;`query_device_info` 等非运营商上游部分保持不变。
|
||||
|
||||
## 8. 整体切换
|
||||
|
||||
- [ ] 8.1 Worker 监听 `polling.scheduler.mode`:切 `carrier` 时停旧调度出队、初始化新结构后开启分发;切 `legacy` 时停止领取、等待执行中任务收尾、重建旧分片队列后恢复旧调度。
|
||||
- [ ] 8.2 旧轮询 handler 执行前检查模式,为 `carrier` 时直接结束。
|
||||
- [ ] 8.3 所有写入轮询结构的入口按当前模式写新或旧结构。
|
||||
- [x] 8.1 Worker 监听 `polling.scheduler.mode`:切 `carrier` 时停旧调度出队、初始化新结构后开启分发;切 `legacy` 时停止领取、等待执行中任务收尾、重建旧分片队列后恢复旧调度。
|
||||
- [x] 8.2 旧轮询 handler 执行前检查模式,为 `carrier` 时直接结束。
|
||||
- [x] 8.3 所有写入轮询结构的入口按当前模式写新或旧结构。(含第 2 个 Change 的重算到期任务)
|
||||
|
||||
## 9. 接口与下线
|
||||
|
||||
- [ ] 9.1 `/polling-carrier-params` 列表(含默认标记)、详情、修改、删除;`/polling-check-scenes` 列表与修改;`/polling-failure-alerts` 列表、`/summary`、详情、`PUT /:id/status`;`GET /iot-cards/:id/polling-state`。
|
||||
- [ ] 9.2 `/polling-stats` 改为按账户(队列长度、到期数、执行中、实际 QPS、限流等待与超时、处理中标记到期、告警数)。
|
||||
- [ ] 9.3 下线 `/polling-concurrency`、`concurrency_service.go`、`acquireConcurrency`/`releaseConcurrency`、按 ZCARD 背压;旧提示通道 List 不再写入。
|
||||
- [ ] 9.4 同步路由、`cmd/api/docs.go`、`cmd/gendocs/main.go`,重新生成接口文档。
|
||||
- [x] 9.1 `/polling-carrier-params` 列表(含默认标记)、详情、修改、删除;`/polling-check-scenes` 列表与修改;`/polling-failure-alerts` 列表、`/summary`、详情、`PUT /:id/status`;`GET /iot-cards/:id/polling-state`。
|
||||
- [x] 9.2 `/polling-stats` 改为按账户(队列长度、到期数、执行中、实际 QPS、限流等待与超时、处理中标记到期、告警数)。(新增 GET /polling-stats/carrier-queues;/polling-stats/carriers 已被第 1 个 Change 的网关调用统计占用)
|
||||
- [ ] 9.3 下线 `/polling-concurrency`、`concurrency_service.go`、`acquireConcurrency`/`releaseConcurrency`、按 ZCARD 背压;旧提示通道 List 不再写入。(暂缓:旧分片调度仍需并发配置与并发令牌,保留以支持整体回退;按设计在新调度稳定运行后与旧调度代码一并下线)
|
||||
- [x] 9.4 同步路由、`cmd/api/docs.go`、`cmd/gendocs/main.go`,重新生成接口文档。
|
||||
|
||||
## 10. 编译与规范校验
|
||||
|
||||
- [ ] 10.1 `gofmt -w`;`go build ./cmd/api ./cmd/worker` 通过。
|
||||
- [ ] 10.2 `openspec validate add-carrier-polling-scheduler` 与 `openspec doctor --json` 通过。
|
||||
- [x] 10.1 `gofmt -w`;`go build ./cmd/api ./cmd/worker` 通过。
|
||||
- [x] 10.2 `openspec validate add-carrier-polling-scheduler` 与 `openspec doctor --json` 通过。(validate 通过;当前 openspec 无 doctor 命令)
|
||||
|
||||
## 11. 测试环境验证(自动化测试为 N/A,按 ENG-TEST-001;按生产规模造卡或回放生产卡 ID)
|
||||
|
||||
|
||||
@@ -602,6 +602,14 @@ const (
|
||||
AuditActionGatewayErrorRuleStatusUpdated = "gateway_error_rule.update_status"
|
||||
// AuditActionGatewayErrorRuleDeleted 表示删除上游错误分类规则。
|
||||
AuditActionGatewayErrorRuleDeleted = "gateway_error_rule.delete"
|
||||
// AuditActionCarrierPollingParamSaved 表示保存运营商账户调度参数。
|
||||
AuditActionCarrierPollingParamSaved = "carrier_polling_param.save"
|
||||
// AuditActionCarrierPollingParamDeleted 表示删除运营商账户调度参数(恢复全局默认)。
|
||||
AuditActionCarrierPollingParamDeleted = "carrier_polling_param.delete"
|
||||
// AuditActionPollingCheckSceneUpdated 表示修改请求检查场景配置。
|
||||
AuditActionPollingCheckSceneUpdated = "polling_check_scene.update"
|
||||
// AuditActionPollingFailureAlertHandled 表示处理卡轮询失败告警。
|
||||
AuditActionPollingFailureAlertHandled = "polling_failure_alert.handle"
|
||||
// AuditActionPollingConcurrencyUpdated 表示更新轮询任务并发配置。
|
||||
AuditActionPollingConcurrencyUpdated = "polling_concurrency.update"
|
||||
// AuditActionPollingConcurrencyReset 表示人工重置轮询并发计数。
|
||||
@@ -781,6 +789,12 @@ const (
|
||||
AuditResourceRateLimitGroup = "rate_limit_group"
|
||||
// AuditResourceGatewayErrorRule 表示上游错误分类规则资源。
|
||||
AuditResourceGatewayErrorRule = "gateway_error_rule"
|
||||
// AuditResourceCarrierPollingParam 表示运营商账户调度参数资源。
|
||||
AuditResourceCarrierPollingParam = "carrier_polling_param"
|
||||
// AuditResourcePollingCheckScene 表示请求检查场景资源。
|
||||
AuditResourcePollingCheckScene = "polling_check_scene"
|
||||
// AuditResourcePollingFailureAlert 表示卡轮询失败告警资源。
|
||||
AuditResourcePollingFailureAlert = "polling_failure_alert"
|
||||
// AuditResourcePollingConcurrencyConfig 表示轮询并发配置资源。
|
||||
AuditResourcePollingConcurrencyConfig = "polling_concurrency"
|
||||
// AuditResourcePollingAlertRule 表示轮询告警规则资源。
|
||||
|
||||
@@ -40,6 +40,7 @@ type Handler struct {
|
||||
workerResult *WorkerBootstrapResult
|
||||
asynqClient *asynq.Client
|
||||
pollingBase *task.PollingBase
|
||||
pollingSteps PollingStepHandlers
|
||||
pollingStopResumeSvc iot_card_svc.StopResumeServiceInterface
|
||||
}
|
||||
|
||||
@@ -292,6 +293,7 @@ func (h *Handler) registerPollingHandlers() {
|
||||
cardStatusHandler := task.NewPollingCardStatusHandler(
|
||||
h.pollingBase, h.gatewayClient, h.workerResult.Services.CardObservation, integrationRepository)
|
||||
|
||||
h.pollingSteps = PollingStepHandlers{Realname: realnameHandler, Carddata: carddataHandler, Package: packageHandler, CardStatus: cardStatusHandler}
|
||||
h.mux.HandleFunc(constants.TaskTypePollingRealname, realnameHandler.Handle)
|
||||
h.mux.HandleFunc(constants.TaskTypePollingCarddata, carddataHandler.Handle)
|
||||
h.mux.HandleFunc(constants.TaskTypePollingPackage, packageHandler.Handle)
|
||||
@@ -408,3 +410,16 @@ func (h *Handler) registerOperationsReportSnapshotHandler() {
|
||||
func (h *Handler) GetMux() *asynq.ServeMux {
|
||||
return h.mux
|
||||
}
|
||||
|
||||
// PollingStepHandlers 是四个轮询检查步骤处理器,旧分片调度注册为 Asynq handler,按运营商账户调度的卡任务执行器复用其 Step。
|
||||
type PollingStepHandlers struct {
|
||||
Realname *task.PollingRealnameHandler
|
||||
Carddata *task.PollingCarddataHandler
|
||||
Package *task.PollingPackageHandler
|
||||
CardStatus *task.PollingCardStatusHandler
|
||||
}
|
||||
|
||||
// PollingSteps 返回轮询检查步骤处理器(RegisterHandlers 之后可用)。
|
||||
func (h *Handler) PollingSteps() PollingStepHandlers {
|
||||
return h.pollingSteps
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user