Compare commits
4 Commits
e7f8c4bae9
...
9183863fab
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9183863fab | ||
|
|
c741f1751d | ||
|
|
0d966249b8 | ||
|
|
2fe7c1f8ac |
@@ -33,6 +33,7 @@ func generateOpenAPIDocs(outputPath string, logger *zap.Logger) {
|
|||||||
handlers.PhoneAssetAssociation = admin.NewPhoneAssetAssociationHandler(nil, nil)
|
handlers.PhoneAssetAssociation = admin.NewPhoneAssetAssociationHandler(nil, nil)
|
||||||
// 网关限流 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
// 网关限流 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||||
handlers.GatewayRateLimit = admin.NewGatewayRateLimitHandler(nil, nil, nil)
|
handlers.GatewayRateLimit = admin.NewGatewayRateLimitHandler(nil, nil, nil)
|
||||||
|
handlers.CarrierScheduler = admin.NewCarrierSchedulerHandler(nil, nil)
|
||||||
// 套餐真流量预警 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
// 套餐真流量预警 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||||
handlers.PackageTrafficAlert = admin.NewPackageTrafficAlertHandler(nil, nil, nil, nil)
|
handlers.PackageTrafficAlert = admin.NewPackageTrafficAlertHandler(nil, nil, nil, nil)
|
||||||
// 资产钱包自动续费配置 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
// 资产钱包自动续费配置 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ func generateAdminDocs(outputPath string) error {
|
|||||||
handlers.PhoneAssetAssociation = admin.NewPhoneAssetAssociationHandler(nil, nil)
|
handlers.PhoneAssetAssociation = admin.NewPhoneAssetAssociationHandler(nil, nil)
|
||||||
// 网关限流 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
// 网关限流 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||||
handlers.GatewayRateLimit = admin.NewGatewayRateLimitHandler(nil, nil, nil)
|
handlers.GatewayRateLimit = admin.NewGatewayRateLimitHandler(nil, nil, nil)
|
||||||
|
handlers.CarrierScheduler = admin.NewCarrierSchedulerHandler(nil, nil)
|
||||||
// 套餐真流量预警 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
// 套餐真流量预警 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||||
handlers.PackageTrafficAlert = admin.NewPackageTrafficAlertHandler(nil, nil, nil, nil)
|
handlers.PackageTrafficAlert = admin.NewPackageTrafficAlertHandler(nil, nil, nil, nil)
|
||||||
// 资产钱包自动续费配置 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
// 资产钱包自动续费配置 Handler 必须同时进入文档工厂,避免新增管理接口遗漏文档注册。
|
||||||
|
|||||||
270
cmd/worker/carrier_scheduler.go
Normal file
270
cmd/worker/carrier_scheduler.go
Normal file
@@ -0,0 +1,270 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/infrastructure/nsqbus"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/task"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// carrierInitLockKey 新结构初始化锁:多实例时只由一个实例初始化。
|
||||||
|
const carrierInitLockKey = "polling:carrier:init_lock"
|
||||||
|
|
||||||
|
// carrierSchedulerRuntime 是按运营商账户调度在 Worker 内的运行组件:模式监听、初始化、分发器与 NSQ 消费者。
|
||||||
|
type carrierSchedulerRuntime struct {
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
runtime *workerRuntime
|
||||||
|
initializer *polling.CarrierInitializer
|
||||||
|
dispatcher *carrierqueue.Dispatcher
|
||||||
|
consumers *nsqbus.ConsumerManager
|
||||||
|
producer *nsqbus.Producer
|
||||||
|
alerts *carrierqueue.AlertWriter
|
||||||
|
executor *task.CarrierCardExecutor
|
||||||
|
singleton bool
|
||||||
|
instanceID string
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
lastMode string
|
||||||
|
reinitQueued bool
|
||||||
|
legacyRestarts func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// newCarrierSchedulerParts 在 Worker 运行时初始化阶段创建调度参数、账户队列存储与模式接缝,
|
||||||
|
// 并注入旧调度相关组件(生命周期、轮询基类),使其在启动前即可按模式写入正确结构。
|
||||||
|
func newCarrierSchedulerParts(runtime *workerRuntime, appLogger *zap.Logger) {
|
||||||
|
settings := carrierqueue.NewDefaultSettingsProvider(runtime.db, runtime.redisClient, appLogger)
|
||||||
|
store := carrierqueue.NewStore(runtime.redisClient)
|
||||||
|
scheduling := polling.NewCarrierScheduling(store, settings, runtime.pollingConfigMgr, appLogger)
|
||||||
|
runtime.carrierSettings = settings
|
||||||
|
runtime.carrierStore = store
|
||||||
|
runtime.carrierScheduling = scheduling
|
||||||
|
runtime.lifecycleSvc.SetCarrierScheduling(scheduling)
|
||||||
|
runtime.pollingBase.SetSchedulerModeReader(func(ctx context.Context) string { return settings.Current(ctx).Mode })
|
||||||
|
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Classifier != nil {
|
||||||
|
classifier := runtime.gatewayLimit.Classifier
|
||||||
|
runtime.pollingBase.SetErrorClassifier(func(ctx context.Context, op, message string) string {
|
||||||
|
return classifier.Classify(ctx, op, message).Category
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startCarrierScheduler 启动模式监听;分发器只在 leader/all 运行(多实例安全),消费者在所有处理任务的实例运行。
|
||||||
|
func startCarrierScheduler(ctx context.Context, cfg *config.Config, runtime *workerRuntime, steps queue.PollingStepHandlers,
|
||||||
|
singleton bool, legacyRestart func(), appLogger *zap.Logger) *carrierSchedulerRuntime {
|
||||||
|
carrierCfg := cfg.Polling.Carrier
|
||||||
|
instanceID := cfg.Worker.InstanceName
|
||||||
|
if instanceID == "" {
|
||||||
|
instanceID, _ = os.Hostname()
|
||||||
|
}
|
||||||
|
instanceID += "-" + strconv.Itoa(os.Getpid())
|
||||||
|
alerts := carrierqueue.NewAlertWriter(runtime.db, runtime.redisClient, runtime.carrierSettings, appLogger)
|
||||||
|
alerts.Start(ctx)
|
||||||
|
executor := task.NewCarrierCardExecutor(runtime.pollingBase, runtime.carrierStore, runtime.carrierSettings, alerts,
|
||||||
|
steps.Realname, steps.Carddata, steps.CardStatus, steps.Package, carrierCfg, instanceID)
|
||||||
|
c := &carrierSchedulerRuntime{
|
||||||
|
cfg: carrierCfg, runtime: runtime, singleton: singleton, instanceID: instanceID, logger: appLogger,
|
||||||
|
alerts: alerts, executor: executor, legacyRestarts: legacyRestart,
|
||||||
|
initializer: polling.NewCarrierInitializer(runtime.carrierScheduling, runtime.pollingIotCardStore, appLogger),
|
||||||
|
}
|
||||||
|
if carrierCfg.NSQDAddress != "" {
|
||||||
|
producer, err := nsqbus.NewProducer(carrierCfg.NSQDAddress, appLogger)
|
||||||
|
if err != nil {
|
||||||
|
appLogger.Error("创建 NSQ 生产者失败,按运营商账户调度不可用", zap.Error(err))
|
||||||
|
} else {
|
||||||
|
c.producer = producer
|
||||||
|
}
|
||||||
|
c.consumers = nsqbus.NewConsumerManager(nsqbus.ConsumerConfig{
|
||||||
|
NSQDAddress: carrierCfg.NSQDAddress, NSQLookupdAddresses: nsqbus.SplitAddresses(carrierCfg.NSQLookupdAddresses),
|
||||||
|
MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts,
|
||||||
|
}, constants.NSQChannelWorker, executor.Handle, appLogger)
|
||||||
|
}
|
||||||
|
var budgets carrierqueue.BudgetProvider
|
||||||
|
if runtime.gatewayLimit != nil && runtime.gatewayLimit.Limiter != nil {
|
||||||
|
budgets = runtime.gatewayLimit.Limiter
|
||||||
|
}
|
||||||
|
if c.producer != nil {
|
||||||
|
c.dispatcher = carrierqueue.NewDispatcher(runtime.db, runtime.carrierStore, runtime.carrierSettings, c.producer, budgets,
|
||||||
|
carrierCfg, instanceID, appLogger)
|
||||||
|
if singleton {
|
||||||
|
go c.dispatcher.Run(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
go c.watch(ctx)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestReinit 轮询配置变更时请求按新配置重新初始化新结构(仅 carrier 模式生效)。
|
||||||
|
func (c *carrierSchedulerRuntime) RequestReinit() {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.reinitQueued = true
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *carrierSchedulerRuntime) watch(ctx context.Context) {
|
||||||
|
ticker := time.NewTicker(3 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
c.step(ctx)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
c.shutdown()
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *carrierSchedulerRuntime) step(ctx context.Context) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
c.logger.Error("按运营商账户调度模式监听 panic,已恢复", zap.Any("panic", r))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
mode := c.runtime.carrierSettings.Current(ctx).Mode
|
||||||
|
c.mu.Lock()
|
||||||
|
previous := c.lastMode
|
||||||
|
c.lastMode = mode
|
||||||
|
reinit := c.reinitQueued
|
||||||
|
c.reinitQueued = false
|
||||||
|
c.mu.Unlock()
|
||||||
|
if mode == constants.PollingSchedulerModeCarrier {
|
||||||
|
if previous != mode {
|
||||||
|
c.logger.Info("卡轮询调度模式:按运营商账户调度", zap.String("previous", previous))
|
||||||
|
}
|
||||||
|
if c.singleton {
|
||||||
|
c.ensureInitialized(ctx, reinit)
|
||||||
|
}
|
||||||
|
c.syncConsumers(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if previous == constants.PollingSchedulerModeCarrier {
|
||||||
|
c.logger.Info("卡轮询调度模式回到旧分片调度:停止领取与消费,重建旧分片队列")
|
||||||
|
if c.consumers != nil {
|
||||||
|
c.consumers.StopAll(c.grace())
|
||||||
|
c.consumers.Reset()
|
||||||
|
}
|
||||||
|
if c.singleton {
|
||||||
|
c.runtime.redisClient.Del(ctx, constants.RedisCarrierSchedulerInitStateKey)
|
||||||
|
if c.legacyRestarts != nil {
|
||||||
|
c.legacyRestarts()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureInitialized 切到 carrier 后初始化新结构;初始化完成前分发器不领取。多实例时用初始化锁协调。
|
||||||
|
func (c *carrierSchedulerRuntime) ensureInitialized(ctx context.Context, reinit bool) {
|
||||||
|
redisClient := c.runtime.redisClient
|
||||||
|
state, _ := redisClient.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||||
|
if state == carrierqueue.InitStateDone && !reinit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ok, err := redisClient.SetNX(ctx, carrierInitLockKey, c.instanceID, 30*time.Minute).Result()
|
||||||
|
if err != nil || !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer redisClient.Del(context.WithoutCancel(ctx), carrierInitLockKey)
|
||||||
|
if state != carrierqueue.InitStateDone {
|
||||||
|
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateRunning, 0)
|
||||||
|
}
|
||||||
|
started := time.Now()
|
||||||
|
written, initErr := c.initializer.Run(ctx)
|
||||||
|
if initErr != nil {
|
||||||
|
c.logger.Error("按运营商账户调度初始化失败,下一轮重试", zap.Error(initErr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
redisClient.Set(ctx, constants.RedisCarrierSchedulerInitStateKey, carrierqueue.InitStateDone, 0)
|
||||||
|
c.logger.Info("按运营商账户调度初始化完成,开始领取", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)),
|
||||||
|
zap.String("metric", "polling.carrier.initialized"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncConsumers 按当前运营商账户订阅卡任务 topic,max_in_flight 取账户调度参数。
|
||||||
|
func (c *carrierSchedulerRuntime) syncConsumers(ctx context.Context) {
|
||||||
|
if c.consumers == nil || c.dispatcher == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
desired := map[string]int{}
|
||||||
|
for _, carrierID := range c.dispatcher.CarrierIDs(ctx) {
|
||||||
|
desired[constants.NSQCardTopic(carrierID)] = c.dispatcher.ParamsFor(carrierID).CardMaxInFlight
|
||||||
|
}
|
||||||
|
c.consumers.Sync(desired)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *carrierSchedulerRuntime) grace() time.Duration {
|
||||||
|
if c.cfg.ShutdownGrace > 0 {
|
||||||
|
return c.cfg.ShutdownGrace
|
||||||
|
}
|
||||||
|
return 30 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
// shutdown 优雅退出:停止订阅,宽限期内等待执行中的卡任务完成;未完成的由租约到期后重新领取。
|
||||||
|
func (c *carrierSchedulerRuntime) shutdown() {
|
||||||
|
if c.consumers != nil {
|
||||||
|
c.consumers.StopAll(c.grace())
|
||||||
|
}
|
||||||
|
if c.producer != nil {
|
||||||
|
c.producer.Stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitProgress 返回新结构初始化进度。
|
||||||
|
func (c *carrierSchedulerRuntime) InitProgress() polling.InitProgress {
|
||||||
|
return c.initializer.Progress()
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierAwareSeriesConsumer 是业务事件观测序列 Outbox 消费者的模式接缝:按运营商账户调度时把业务事件改为按场景请求检查
|
||||||
|
// (事件在原业务事务中写入,消费即在事务提交之后),旧分片调度时沿用原观测序列。
|
||||||
|
type carrierAwareSeriesConsumer struct {
|
||||||
|
legacy outbox.EventConsumer
|
||||||
|
requester *polling.CheckRequester
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consume 消费业务观测序列请求事件。
|
||||||
|
func (c *carrierAwareSeriesConsumer) Consume(ctx context.Context, envelope outbox.DeliveryEnvelope) error {
|
||||||
|
if c.requester == nil || !c.requester.Active(ctx) {
|
||||||
|
return c.legacy.Consume(ctx, envelope)
|
||||||
|
}
|
||||||
|
var event cardObservationApp.SeriesRequestedEvent
|
||||||
|
if err := json.Unmarshal(envelope.Payload, &event); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cardIDs := []uint{event.ResourceID}
|
||||||
|
if event.ResourceType == constants.CardObservationResourceTypeDevice {
|
||||||
|
cardIDs = event.ResourceIDs
|
||||||
|
}
|
||||||
|
c.requester.RequestCards(ctx, businessEventCheckScene(event), cardIDs, event.ExpectedValue)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// businessEventCheckScene 把业务观测事件映射为请求检查场景:订单支付、套餐激活、自动续费购包按事件来源区分,停复机成功按观测场景。
|
||||||
|
func businessEventCheckScene(event cardObservationApp.SeriesRequestedEvent) string {
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(event.EventID, "card-observation:order:"):
|
||||||
|
return constants.PollingCheckSceneOrderPaid
|
||||||
|
case strings.HasPrefix(event.EventID, "card-observation:auto-purchase:"), strings.HasPrefix(event.EventID, "asset-auto-renewal:"):
|
||||||
|
return constants.PollingCheckSceneAutoRenewalPurchase
|
||||||
|
case event.Scene == constants.CardObservationSceneBusinessStop || event.Scene == constants.CardObservationSceneBusinessResume:
|
||||||
|
return constants.PollingCheckSceneStopResumeSucceeded
|
||||||
|
default:
|
||||||
|
return constants.PollingCheckScenePackageActivated
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -45,6 +45,7 @@ import (
|
|||||||
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||||
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
||||||
refundSvc "github.com/break/junhong_cmp_fiber/internal/service/refund"
|
refundSvc "github.com/break/junhong_cmp_fiber/internal/service/refund"
|
||||||
@@ -102,6 +103,12 @@ type workerRuntime struct {
|
|||||||
assetAutoRenewalService *assetAutoRenewalApp.Service
|
assetAutoRenewalService *assetAutoRenewalApp.Service
|
||||||
// gatewayLimit 是卡网关运营商账户限流组件(限流、统计、停复机记录)。
|
// gatewayLimit 是卡网关运营商账户限流组件(限流、统计、停复机记录)。
|
||||||
gatewayLimit *gatewaylimit.Runtime
|
gatewayLimit *gatewaylimit.Runtime
|
||||||
|
// 按运营商账户调度:调度参数、账户队列存储与模式接缝(模式为 carrier 时各入口写新结构)。
|
||||||
|
carrierSettings *carrierqueue.SettingsProvider
|
||||||
|
carrierStore *carrierqueue.Store
|
||||||
|
carrierScheduling *polling.CarrierScheduling
|
||||||
|
// carrierReinit 在 carrier 模式下轮询配置变更时请求重新初始化新结构。
|
||||||
|
carrierReinit func()
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -171,6 +178,12 @@ func runWorker(cfg *config.Config) {
|
|||||||
|
|
||||||
taskHandler := createTaskHandler(runtime, appLogger)
|
taskHandler := createTaskHandler(runtime, appLogger)
|
||||||
taskHandler.RegisterHandlers()
|
taskHandler.RegisterHandlers()
|
||||||
|
carrierRuntime := startCarrierScheduler(ctx, cfg, runtime, taskHandler.PollingSteps(), runsSingletonModules(cfg.Worker.Role), func() {
|
||||||
|
if pollingInitializer != nil {
|
||||||
|
pollingInitializer.Restart(ctx)
|
||||||
|
}
|
||||||
|
}, appLogger)
|
||||||
|
runtime.carrierReinit = carrierRuntime.RequestReinit
|
||||||
registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
|
registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
|
||||||
registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
||||||
registerRefundChannelRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
registerRefundChannelRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
|
||||||
@@ -183,9 +196,9 @@ func runWorker(cfg *config.Config) {
|
|||||||
taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle)
|
taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle)
|
||||||
partitionMaintainer := gatewaylimit.NewPartitionMaintainer(postgres.NewCardStopResumeRecordStore(runtime.db), runtime.gatewayLimit.Settings, appLogger)
|
partitionMaintainer := gatewaylimit.NewPartitionMaintainer(postgres.NewCardStopResumeRecordStore(runtime.db), runtime.gatewayLimit.Settings, appLogger)
|
||||||
taskHandler.GetMux().HandleFunc(constants.TaskTypeStopResumeRecordPartition, partitionMaintainer.Handle)
|
taskHandler.GetMux().HandleFunc(constants.TaskTypeStopResumeRecordPartition, partitionMaintainer.Handle)
|
||||||
recalcHandler := polling.NewRecalculateTaskHandler(
|
dueRecalculator := polling.NewDueRecalculator(runtime.db, runtime.redisClient, runtime.pollingConfigMgr, runtime.pollingQueueMgr, appLogger)
|
||||||
polling.NewDueRecalculator(runtime.db, runtime.redisClient, runtime.pollingConfigMgr, runtime.pollingQueueMgr, appLogger),
|
dueRecalculator.SetCarrierScheduling(runtime.carrierScheduling)
|
||||||
runtime.redisClient, appLogger)
|
recalcHandler := polling.NewRecalculateTaskHandler(dueRecalculator, runtime.redisClient, appLogger)
|
||||||
taskHandler.GetMux().HandleFunc(constants.TaskTypePollingRecalculateDue, recalcHandler.Handle)
|
taskHandler.GetMux().HandleFunc(constants.TaskTypePollingRecalculateDue, recalcHandler.Handle)
|
||||||
startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger)
|
startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger)
|
||||||
rescuePendingImportTasks(ctx, runtime, appLogger)
|
rescuePendingImportTasks(ctx, runtime, appLogger)
|
||||||
@@ -379,6 +392,7 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L
|
|||||||
// 自动续费的复机执行同样复用既有停复机单一事实源。
|
// 自动续费的复机执行同样复用既有停复机单一事实源。
|
||||||
assetAutoRenewalService: workerResult.Services.AssetAutoRenewal,
|
assetAutoRenewalService: workerResult.Services.AssetAutoRenewal,
|
||||||
}
|
}
|
||||||
|
newCarrierSchedulerParts(runtime, appLogger)
|
||||||
registerNotificationOutboxConsumer(runtime, appLogger)
|
registerNotificationOutboxConsumer(runtime, appLogger)
|
||||||
registerWalletOutboxConsumer(runtime, appLogger)
|
registerWalletOutboxConsumer(runtime, appLogger)
|
||||||
registerCardObservationOutboxConsumer(runtime, appLogger)
|
registerCardObservationOutboxConsumer(runtime, appLogger)
|
||||||
@@ -712,7 +726,10 @@ func registerCardObservationOutboxConsumer(runtime *workerRuntime, appLogger *za
|
|||||||
queue.NewCardObservationSeriesScheduler(runtime.outboxQueueClient),
|
queue.NewCardObservationSeriesScheduler(runtime.outboxQueueClient),
|
||||||
cardObservationInfra.NewSeriesAttemptLogger(integrationlog.NewRepository(runtime.db)),
|
cardObservationInfra.NewSeriesAttemptLogger(integrationlog.NewRepository(runtime.db)),
|
||||||
)
|
)
|
||||||
seriesConsumer := cardObservationInfra.NewSeriesRequestedConsumer(seriesTrigger)
|
seriesConsumer := &carrierAwareSeriesConsumer{
|
||||||
|
legacy: cardObservationInfra.NewSeriesRequestedConsumer(seriesTrigger),
|
||||||
|
requester: polling.NewCheckRequester(runtime.carrierScheduling, runtime.db, appLogger),
|
||||||
|
}
|
||||||
if err := runtime.outboxConsumers.Register(constants.OutboxEventTypeCardSeriesRequested, seriesConsumer); err != nil {
|
if err := runtime.outboxConsumers.Register(constants.OutboxEventTypeCardSeriesRequested, seriesConsumer); err != nil {
|
||||||
appLogger.Fatal("注册业务卡观测序列 Outbox 消费者失败",
|
appLogger.Fatal("注册业务卡观测序列 Outbox 消费者失败",
|
||||||
zap.String("event_type", constants.OutboxEventTypeCardSeriesRequested), zap.Error(err))
|
zap.String("event_type", constants.OutboxEventTypeCardSeriesRequested), zap.Error(err))
|
||||||
@@ -842,6 +859,13 @@ func startPollingInitializer(ctx context.Context, runtime *workerRuntime, appLog
|
|||||||
pollingInitializer.StartBackground(ctx)
|
pollingInitializer.StartBackground(ctx)
|
||||||
|
|
||||||
runtime.pollingConfigMgr.WatchChanges(ctx, func(hadConfigs, hasConfigs bool) {
|
runtime.pollingConfigMgr.WatchChanges(ctx, func(hadConfigs, hasConfigs bool) {
|
||||||
|
if runtime.carrierScheduling.Active(ctx) {
|
||||||
|
// 按运营商账户调度时按新配置重新初始化新结构,旧分片队列在回退时再重建。
|
||||||
|
if runtime.carrierReinit != nil {
|
||||||
|
runtime.carrierReinit()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
if hasConfigs {
|
if hasConfigs {
|
||||||
appLogger.Info("轮询配置已变更,触发队列重新初始化",
|
appLogger.Info("轮询配置已变更,触发队列重新初始化",
|
||||||
zap.Bool("had_configs", hadConfigs))
|
zap.Bool("had_configs", hadConfigs))
|
||||||
@@ -894,6 +918,7 @@ func startPollingScheduler(
|
|||||||
dataResetHandler,
|
dataResetHandler,
|
||||||
)
|
)
|
||||||
pollingScheduler.SetInitializer(pollingInitializer)
|
pollingScheduler.SetInitializer(pollingInitializer)
|
||||||
|
pollingScheduler.SetCarrierScheduling(runtime.carrierScheduling)
|
||||||
|
|
||||||
if err := pollingScheduler.Start(ctx); err != nil {
|
if err := pollingScheduler.Start(ctx); err != nil {
|
||||||
appLogger.Error("启动轮询调度器失败", zap.Error(err))
|
appLogger.Error("启动轮询调度器失败", zap.Error(err))
|
||||||
|
|||||||
121
internal/bootstrap/carrier_check.go
Normal file
121
internal/bootstrap/carrier_check.go
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// carrierAwareSeriesDispatcher 是读入口观测分发的模式接缝:按运营商账户调度时把读入口改为按场景请求检查
|
||||||
|
// (同卡合并、冷却、受账户限流),不再起"立即、3 分钟、5 分钟"观测序列;旧分片调度时沿用原分发器。
|
||||||
|
type carrierAwareSeriesDispatcher struct {
|
||||||
|
legacy cardObservationApp.BestEffortSeriesDispatcher
|
||||||
|
requester *polling.CheckRequester
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ cardObservationApp.BestEffortSeriesDispatcher = (*carrierAwareSeriesDispatcher)(nil)
|
||||||
|
|
||||||
|
// ReadEntryCheckScene 把读入口与设备控制的观测场景映射为请求检查场景;不认识的场景返回空串。
|
||||||
|
func ReadEntryCheckScene(observationScene string) string {
|
||||||
|
switch observationScene {
|
||||||
|
case constants.CardObservationSceneClientAssetRead:
|
||||||
|
return constants.PollingCheckSceneClientAssetDetail
|
||||||
|
case constants.CardObservationSceneAdminAssetRead:
|
||||||
|
return constants.PollingCheckSceneAdminAssetRealtime
|
||||||
|
case constants.CardObservationSceneClientRealnameLink, constants.CardObservationSceneAdminRealnameLink:
|
||||||
|
return constants.PollingCheckSceneRealnameLink
|
||||||
|
case constants.CardObservationSceneBusinessStop, constants.CardObservationSceneBusinessResume:
|
||||||
|
return constants.PollingCheckSceneStopResumeSucceeded
|
||||||
|
case constants.CardObservationScenePackageChanged:
|
||||||
|
return constants.PollingCheckScenePackageActivated
|
||||||
|
case constants.CardObservationSceneDeviceSwitchCard, constants.CardObservationSceneDeviceSwitchMode,
|
||||||
|
constants.CardObservationSceneDeviceReboot, constants.CardObservationSceneDeviceReset, constants.CardObservationSceneDeviceSetWiFi:
|
||||||
|
return constants.PollingCheckSceneDeviceOperation
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *carrierAwareSeriesDispatcher) active(ctx context.Context) bool {
|
||||||
|
return d.requester != nil && d.requester.Active(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dispatch 读入口单卡(或设备)观测。
|
||||||
|
func (d *carrierAwareSeriesDispatcher) Dispatch(ctx context.Context, request cardObservationApp.SeriesRequest) {
|
||||||
|
if !d.active(ctx) {
|
||||||
|
d.legacy.Dispatch(ctx, request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
scene := ReadEntryCheckScene(request.Scene)
|
||||||
|
if scene == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
id, err := strconv.ParseUint(request.ResourceID, 10, 64)
|
||||||
|
if err != nil || id == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if request.ResourceType == constants.CardObservationResourceTypeDevice {
|
||||||
|
d.requester.RequestDevice(ctx, scene, uint(id))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.requester.RequestCards(ctx, scene, []uint{uint(id)}, request.ExpectedValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DispatchDeviceCards 设备有效绑定卡观测。
|
||||||
|
func (d *carrierAwareSeriesDispatcher) DispatchDeviceCards(ctx context.Context, request cardObservationApp.DeviceCardsSeriesRequest) {
|
||||||
|
if !d.active(ctx) {
|
||||||
|
d.legacy.DispatchDeviceCards(ctx, request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if scene := ReadEntryCheckScene(request.Request.Scene); scene != "" {
|
||||||
|
d.requester.RequestDevice(ctx, scene, request.DeviceID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DispatchDeviceControl 设备控制成功后的观测:按设备操作场景为设备下各卡请求检查。
|
||||||
|
func (d *carrierAwareSeriesDispatcher) DispatchDeviceControl(ctx context.Context, request cardObservationApp.DeviceControlSeriesRequest) {
|
||||||
|
if !d.active(ctx) {
|
||||||
|
d.legacy.DispatchDeviceControl(ctx, request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// 设备信息同步(query_device_info,非运营商上游)保持原观测,只把卡检查改为请求检查。
|
||||||
|
deviceInfo := request.Request
|
||||||
|
deviceInfo.ResourceType = constants.CardObservationResourceTypeDevice
|
||||||
|
deviceInfo.ResourceID = strconv.FormatUint(uint64(request.DeviceID), 10)
|
||||||
|
deviceInfo.SyncType = constants.CardObservationSyncTypeDeviceInfo
|
||||||
|
deviceInfo.ExpectedValue = strings.TrimSpace(request.TargetICCID)
|
||||||
|
d.legacy.Dispatch(ctx, deviceInfo)
|
||||||
|
cardIDs := append([]uint(nil), request.BoundCardIDs...)
|
||||||
|
for _, id := range []uint{request.SourceCardID, request.TargetCardID} {
|
||||||
|
if id != 0 && !containsUint(cardIDs, id) {
|
||||||
|
cardIDs = append(cardIDs, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(cardIDs) == 0 && request.DeviceID != 0 {
|
||||||
|
d.requester.RequestDevice(ctx, constants.PollingCheckSceneDeviceOperation, request.DeviceID)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.requester.RequestCards(ctx, constants.PollingCheckSceneDeviceOperation, cardIDs, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// DispatchRealnameWithCapability 获取实名链接后的实名观测。
|
||||||
|
func (d *carrierAwareSeriesDispatcher) DispatchRealnameWithCapability(ctx context.Context, request cardObservationApp.RealnameCapabilitySeriesRequest) {
|
||||||
|
if !d.active(ctx) {
|
||||||
|
d.legacy.DispatchRealnameWithCapability(ctx, request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.Dispatch(ctx, request.Request)
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsUint(values []uint, target uint) bool {
|
||||||
|
for _, value := range values {
|
||||||
|
if value == target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -337,6 +337,8 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers {
|
|||||||
PollingConfig: admin.NewPollingConfigHandler(svc.PollingConfig),
|
PollingConfig: admin.NewPollingConfigHandler(svc.PollingConfig),
|
||||||
GatewayRateLimit: admin.NewGatewayRateLimitHandler(pollingSvcPkg.NewGatewayRateLimitService(deps.DB, auditInfra.NewWriter(auditInfra.NewRegistry(), nil), deps.GatewayLimit), deps.Redis, validate),
|
GatewayRateLimit: admin.NewGatewayRateLimitHandler(pollingSvcPkg.NewGatewayRateLimitService(deps.DB, auditInfra.NewWriter(auditInfra.NewRegistry(), nil), deps.GatewayLimit), deps.Redis, validate),
|
||||||
PollingConcurrency: admin.NewPollingConcurrencyHandler(svc.PollingConcurrency),
|
PollingConcurrency: admin.NewPollingConcurrencyHandler(svc.PollingConcurrency),
|
||||||
|
CarrierScheduler: admin.NewCarrierSchedulerHandler(pollingSvcPkg.NewCarrierSchedulerService(deps.DB, deps.Redis,
|
||||||
|
auditInfra.NewWriter(auditInfra.NewRegistry(), nil), carrierSchedulerConfig()), validate),
|
||||||
PollingMonitoring: admin.NewPollingMonitoringHandler(svc.PollingMonitoring),
|
PollingMonitoring: admin.NewPollingMonitoringHandler(svc.PollingMonitoring),
|
||||||
PollingAlert: admin.NewPollingAlertHandler(svc.PollingAlert),
|
PollingAlert: admin.NewPollingAlertHandler(svc.PollingAlert),
|
||||||
PollingCleanup: admin.NewPollingCleanupHandler(svc.PollingCleanup),
|
PollingCleanup: admin.NewPollingCleanupHandler(svc.PollingCleanup),
|
||||||
@@ -408,3 +410,11 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers {
|
|||||||
AgentOpenAPI: openapiHandler.NewHandler(svc.AgentOpenAPI, validate),
|
AgentOpenAPI: openapiHandler.NewHandler(svc.AgentOpenAPI, validate),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// carrierSchedulerConfig 返回按运营商账户调度的 yaml 参数(供后台校验执行上限与展示默认值)。
|
||||||
|
func carrierSchedulerConfig() config.CarrierSchedulerConfig {
|
||||||
|
if cfg := config.Get(); cfg != nil {
|
||||||
|
return cfg.Polling.Carrier
|
||||||
|
}
|
||||||
|
return config.CarrierSchedulerConfig{}
|
||||||
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import (
|
|||||||
walletinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wallet"
|
walletinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wallet"
|
||||||
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
assetAutoRenewalQuery "github.com/break/junhong_cmp_fiber/internal/query/assetautorenewal"
|
assetAutoRenewalQuery "github.com/break/junhong_cmp_fiber/internal/query/assetautorenewal"
|
||||||
accountSvc "github.com/break/junhong_cmp_fiber/internal/service/account"
|
accountSvc "github.com/break/junhong_cmp_fiber/internal/service/account"
|
||||||
agentOpenAPISvc "github.com/break/junhong_cmp_fiber/internal/service/agent_open_api"
|
agentOpenAPISvc "github.com/break/junhong_cmp_fiber/internal/service/agent_open_api"
|
||||||
@@ -221,7 +222,8 @@ func initServices(s *stores, deps *Dependencies) *services {
|
|||||||
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration, auditWriter),
|
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration, auditWriter),
|
||||||
cardObservationInfra.NewSeriesAttemptLogger(seriesIntegration),
|
cardObservationInfra.NewSeriesAttemptLogger(seriesIntegration),
|
||||||
)
|
)
|
||||||
observationSeries := cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier)
|
// 读入口观测分发经模式接缝:按运营商账户调度时改为按场景请求检查,旧分片调度时沿用原分发器。
|
||||||
|
observationSeries := &carrierAwareSeriesDispatcher{legacy: cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier)}
|
||||||
iotCard.SetObservationSeriesDispatcher(observationSeries)
|
iotCard.SetObservationSeriesDispatcher(observationSeries)
|
||||||
// 使用 PollingLifecycleService 替代 APICallback:通过分片队列准确操作(修复 api_callback.go 遗漏 protect 队列的 Bug3)
|
// 使用 PollingLifecycleService 替代 APICallback:通过分片队列准确操作(修复 api_callback.go 遗漏 protect 队列的 Bug3)
|
||||||
pollingConfigStore := postgres.NewPollingConfigStore(deps.DB)
|
pollingConfigStore := postgres.NewPollingConfigStore(deps.DB)
|
||||||
@@ -234,6 +236,12 @@ func initServices(s *stores, deps *Dependencies) *services {
|
|||||||
pollingConfigMgr.Start(context.Background())
|
pollingConfigMgr.Start(context.Background())
|
||||||
pollingQueueMgr := polling.NewPollingQueueManager(deps.Redis, constants.PollingShardCount, deps.Logger)
|
pollingQueueMgr := polling.NewPollingQueueManager(deps.Redis, constants.PollingShardCount, deps.Logger)
|
||||||
pollingLifecycleSvc := polling.NewPollingLifecycleService(pollingQueueMgr, pollingConfigMgr, s.IotCard, s.DeviceSimBinding, s.Device, deps.Logger)
|
pollingLifecycleSvc := polling.NewPollingLifecycleService(pollingQueueMgr, pollingConfigMgr, s.IotCard, s.DeviceSimBinding, s.Device, deps.Logger)
|
||||||
|
// 按运营商账户调度:模式为 carrier 时 API 侧生命周期事件写入账户队列与卡到期信息。
|
||||||
|
carrierSettings := carrierqueue.NewDefaultSettingsProvider(deps.DB, deps.Redis, deps.Logger)
|
||||||
|
carrierStore := carrierqueue.NewStore(deps.Redis)
|
||||||
|
carrierScheduling := polling.NewCarrierScheduling(carrierStore, carrierSettings, pollingConfigMgr, deps.Logger)
|
||||||
|
pollingLifecycleSvc.SetCarrierScheduling(carrierScheduling)
|
||||||
|
observationSeries.requester = polling.NewCheckRequester(carrierScheduling, deps.DB, deps.Logger)
|
||||||
iotCard.SetPollingCallback(pollingLifecycleSvc)
|
iotCard.SetPollingCallback(pollingLifecycleSvc)
|
||||||
// 创建支付配置服务(Order 和 Recharge 依赖)
|
// 创建支付配置服务(Order 和 Recharge 依赖)
|
||||||
wechatConfig := wechatConfigSvc.New(s.WechatConfig, s.Order, s.RechargeOrder, s.AgentRecharge, s.Payment, auditWriter, deps.Redis, deps.Logger)
|
wechatConfig := wechatConfigSvc.New(s.WechatConfig, s.Order, s.RechargeOrder, s.AgentRecharge, s.Payment, auditWriter, deps.Redis, deps.Logger)
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ type Handlers struct {
|
|||||||
WeComApprovalCallback *callback.WeComApprovalHandler
|
WeComApprovalCallback *callback.WeComApprovalHandler
|
||||||
PollingConfig *admin.PollingConfigHandler
|
PollingConfig *admin.PollingConfigHandler
|
||||||
GatewayRateLimit *admin.GatewayRateLimitHandler
|
GatewayRateLimit *admin.GatewayRateLimitHandler
|
||||||
|
CarrierScheduler *admin.CarrierSchedulerHandler
|
||||||
PollingConcurrency *admin.PollingConcurrencyHandler
|
PollingConcurrency *admin.PollingConcurrencyHandler
|
||||||
PollingMonitoring *admin.PollingMonitoringHandler
|
PollingMonitoring *admin.PollingMonitoringHandler
|
||||||
PollingAlert *admin.PollingAlertHandler
|
PollingAlert *admin.PollingAlertHandler
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ type CallGuard interface {
|
|||||||
type cardCallKey struct{}
|
type cardCallKey struct{}
|
||||||
type carrierKey struct{}
|
type carrierKey struct{}
|
||||||
type exemptKey struct{}
|
type exemptKey struct{}
|
||||||
|
type rateLimitWaitKey struct{}
|
||||||
|
|
||||||
type cardCall struct {
|
type cardCall struct {
|
||||||
operation string
|
operation string
|
||||||
@@ -113,6 +114,21 @@ func WithRateLimitExempt(ctx context.Context, source string) context.Context {
|
|||||||
return context.WithValue(ctx, exemptKey{}, source)
|
return context.WithValue(ctx, exemptKey{}, source)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithRateLimitWait 声明本次调用在拿不到限流名额时等待,总等待不超过 maxWait;超时返回限流错误。
|
||||||
|
// 只用于不占 Asynq 协程的执行方(按运营商账户调度的 NSQ 消费者);未声明时拿不到名额立即返回。
|
||||||
|
func WithRateLimitWait(ctx context.Context, maxWait time.Duration) context.Context {
|
||||||
|
if maxWait <= 0 {
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, rateLimitWaitKey{}, maxWait)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RateLimitWait 返回 ctx 中声明的限流最长等待;未声明返回 0。
|
||||||
|
func RateLimitWait(ctx context.Context) time.Duration {
|
||||||
|
wait, _ := ctx.Value(rateLimitWaitKey{}).(time.Duration)
|
||||||
|
return wait
|
||||||
|
}
|
||||||
|
|
||||||
// RateLimitExemptSource 返回 ctx 中声明的豁免来源,未声明返回空串。
|
// RateLimitExemptSource 返回 ctx 中声明的豁免来源,未声明返回空串。
|
||||||
func RateLimitExemptSource(ctx context.Context) string {
|
func RateLimitExemptSource(ctx context.Context) string {
|
||||||
source, _ := ctx.Value(exemptKey{}).(string)
|
source, _ := ctx.Value(exemptKey{}).(string)
|
||||||
|
|||||||
195
internal/handler/admin/carrier_scheduler.go
Normal file
195
internal/handler/admin/carrier_scheduler.go
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
package admin
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/go-playground/validator/v10"
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||||||
|
pollingSvc "github.com/break/junhong_cmp_fiber/internal/service/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/response"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CarrierSchedulerHandler 按运营商账户调度的后台接口:调度参数、请求检查场景、失败告警、单卡轮询状态、按账户监控。
|
||||||
|
type CarrierSchedulerHandler struct {
|
||||||
|
service *pollingSvc.CarrierSchedulerService
|
||||||
|
validator *validator.Validate
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierSchedulerHandler 创建按运营商账户调度 Handler。
|
||||||
|
func NewCarrierSchedulerHandler(service *pollingSvc.CarrierSchedulerService, validator *validator.Validate) *CarrierSchedulerHandler {
|
||||||
|
return &CarrierSchedulerHandler{service: service, validator: validator}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *CarrierSchedulerHandler) validate(prefix string, req any) error {
|
||||||
|
if err := h.validator.Struct(req); err != nil {
|
||||||
|
return errors.New(errors.CodeInvalidParam, validationMessage(prefix, req, err))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListParams 查询运营商账户调度参数列表。GET /api/admin/polling-carrier-params
|
||||||
|
func (h *CarrierSchedulerHandler) ListParams(c *fiber.Ctx) error {
|
||||||
|
result, err := h.service.ListParams(c.UserContext())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetParam 查询运营商账户调度参数。GET /api/admin/polling-carrier-params/:carrier_id
|
||||||
|
func (h *CarrierSchedulerHandler) GetParam(c *fiber.Ctx) error {
|
||||||
|
carrierID, err := rateLimitPathID(c, "carrier_id", "运营商账户 ID")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := h.service.GetParam(c.UserContext(), carrierID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveParam 保存运营商账户调度参数。PUT /api/admin/polling-carrier-params/:carrier_id
|
||||||
|
func (h *CarrierSchedulerHandler) SaveParam(c *fiber.Ctx) error {
|
||||||
|
var req dto.SaveCarrierPollingParamRequest
|
||||||
|
if err := c.BodyParser(&req); err != nil {
|
||||||
|
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||||
|
}
|
||||||
|
carrierID, err := rateLimitPathID(c, "carrier_id", "运营商账户 ID")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.CarrierID = carrierID
|
||||||
|
if err := h.validate("运营商账户调度参数不合法", &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := h.service.SaveParam(c.UserContext(), &req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteParam 删除运营商账户调度参数(恢复全局默认)。DELETE /api/admin/polling-carrier-params/:carrier_id
|
||||||
|
func (h *CarrierSchedulerHandler) DeleteParam(c *fiber.Ctx) error {
|
||||||
|
carrierID, err := rateLimitPathID(c, "carrier_id", "运营商账户 ID")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := h.service.DeleteParam(c.UserContext(), carrierID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListScenes 查询请求检查场景。GET /api/admin/polling-check-scenes
|
||||||
|
func (h *CarrierSchedulerHandler) ListScenes(c *fiber.Ctx) error {
|
||||||
|
result, err := h.service.ListScenes(c.UserContext())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateScene 修改请求检查场景。PUT /api/admin/polling-check-scenes/:scene
|
||||||
|
func (h *CarrierSchedulerHandler) UpdateScene(c *fiber.Ctx) error {
|
||||||
|
var req dto.UpdatePollingCheckSceneRequest
|
||||||
|
if err := c.BodyParser(&req); err != nil {
|
||||||
|
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||||
|
}
|
||||||
|
req.Scene = strings.TrimSpace(c.Params("scene"))
|
||||||
|
if req.Scene == "" {
|
||||||
|
return errors.New(errors.CodeInvalidParam, "无效的场景编码")
|
||||||
|
}
|
||||||
|
if err := h.validate("请求检查场景参数不合法", &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := h.service.UpdateScene(c.UserContext(), &req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAlerts 查询卡轮询失败告警列表。GET /api/admin/polling-failure-alerts
|
||||||
|
func (h *CarrierSchedulerHandler) ListAlerts(c *fiber.Ctx) error {
|
||||||
|
var req dto.PollingFailureAlertListRequest
|
||||||
|
if err := c.QueryParser(&req); err != nil {
|
||||||
|
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||||
|
}
|
||||||
|
if err := h.validate("卡轮询失败告警查询参数不合法", &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := h.service.ListAlerts(c.UserContext(), &req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AlertSummary 未处理告警按运营商账户 × 错误类别汇总。GET /api/admin/polling-failure-alerts/summary
|
||||||
|
func (h *CarrierSchedulerHandler) AlertSummary(c *fiber.Ctx) error {
|
||||||
|
result, err := h.service.AlertSummary(c.UserContext())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAlert 查询卡轮询失败告警详情。GET /api/admin/polling-failure-alerts/:id
|
||||||
|
func (h *CarrierSchedulerHandler) GetAlert(c *fiber.Ctx) error {
|
||||||
|
id, err := rateLimitPathID(c, "id", "告警 ID")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := h.service.GetAlert(c.UserContext(), id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleAlert 标记卡轮询失败告警已处理。PUT /api/admin/polling-failure-alerts/:id/status
|
||||||
|
func (h *CarrierSchedulerHandler) HandleAlert(c *fiber.Ctx) error {
|
||||||
|
var req dto.HandlePollingFailureAlertRequest
|
||||||
|
if err := c.BodyParser(&req); err != nil {
|
||||||
|
return errors.New(errors.CodeInvalidParam, "请求参数解析失败")
|
||||||
|
}
|
||||||
|
id, err := rateLimitPathID(c, "id", "告警 ID")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.ID = id
|
||||||
|
if err := h.validate("卡轮询失败告警处理参数不合法", &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := h.service.HandleAlert(c.UserContext(), &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CardState 查询单卡轮询状态。GET /api/admin/iot-cards/:id/polling-state
|
||||||
|
func (h *CarrierSchedulerHandler) CardState(c *fiber.Ctx) error {
|
||||||
|
id, err := rateLimitPathID(c, "id", "卡 ID")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := h.service.CardState(c.UserContext(), id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierStats 按运营商账户的轮询队列与执行监控。GET /api/admin/polling-stats/carrier-queues
|
||||||
|
func (h *CarrierSchedulerHandler) CarrierStats(c *fiber.Ctx) error {
|
||||||
|
result, err := h.service.CarrierStats(c.UserContext())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return response.Success(c, result)
|
||||||
|
}
|
||||||
@@ -277,6 +277,10 @@ func NewRegistry() *Registry {
|
|||||||
gatewayErrorRuleCreated := pollingAction(constants.AuditActionGatewayErrorRuleCreated, "创建上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
gatewayErrorRuleCreated := pollingAction(constants.AuditActionGatewayErrorRuleCreated, "创建上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||||
gatewayErrorRuleUpdated := pollingAction(constants.AuditActionGatewayErrorRuleUpdated, "更新上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
gatewayErrorRuleUpdated := pollingAction(constants.AuditActionGatewayErrorRuleUpdated, "更新上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||||
gatewayErrorRuleStatusUpdated := pollingAction(constants.AuditActionGatewayErrorRuleStatusUpdated, "更新上游错误分类规则状态", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
gatewayErrorRuleStatusUpdated := pollingAction(constants.AuditActionGatewayErrorRuleStatusUpdated, "更新上游错误分类规则状态", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||||
|
carrierPollingParamSaved := pollingAction(constants.AuditActionCarrierPollingParamSaved, "保存运营商账户调度参数", constants.AuditResourceCarrierPollingParam, constants.AuditRiskHigh)
|
||||||
|
carrierPollingParamDeleted := pollingAction(constants.AuditActionCarrierPollingParamDeleted, "删除运营商账户调度参数", constants.AuditResourceCarrierPollingParam, constants.AuditRiskHigh)
|
||||||
|
pollingCheckSceneUpdated := pollingAction(constants.AuditActionPollingCheckSceneUpdated, "修改请求检查场景", constants.AuditResourcePollingCheckScene, constants.AuditRiskNormal)
|
||||||
|
pollingFailureAlertHandled := pollingAction(constants.AuditActionPollingFailureAlertHandled, "处理卡轮询失败告警", constants.AuditResourcePollingFailureAlert, constants.AuditRiskNormal)
|
||||||
gatewayErrorRuleDeleted := pollingAction(constants.AuditActionGatewayErrorRuleDeleted, "删除上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
gatewayErrorRuleDeleted := pollingAction(constants.AuditActionGatewayErrorRuleDeleted, "删除上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||||
pollingConcurrencyUpdated := pollingAction(constants.AuditActionPollingConcurrencyUpdated, "更新轮询并发配置", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
pollingConcurrencyUpdated := pollingAction(constants.AuditActionPollingConcurrencyUpdated, "更新轮询并发配置", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
||||||
pollingConcurrencyReset := pollingAction(constants.AuditActionPollingConcurrencyReset, "重置轮询并发计数", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
pollingConcurrencyReset := pollingAction(constants.AuditActionPollingConcurrencyReset, "重置轮询并发计数", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
||||||
@@ -678,6 +682,10 @@ func NewRegistry() *Registry {
|
|||||||
constants.AuditActionGatewayErrorRuleUpdated: gatewayErrorRuleUpdated,
|
constants.AuditActionGatewayErrorRuleUpdated: gatewayErrorRuleUpdated,
|
||||||
constants.AuditActionGatewayErrorRuleStatusUpdated: gatewayErrorRuleStatusUpdated,
|
constants.AuditActionGatewayErrorRuleStatusUpdated: gatewayErrorRuleStatusUpdated,
|
||||||
constants.AuditActionGatewayErrorRuleDeleted: gatewayErrorRuleDeleted,
|
constants.AuditActionGatewayErrorRuleDeleted: gatewayErrorRuleDeleted,
|
||||||
|
constants.AuditActionCarrierPollingParamSaved: carrierPollingParamSaved,
|
||||||
|
constants.AuditActionCarrierPollingParamDeleted: carrierPollingParamDeleted,
|
||||||
|
constants.AuditActionPollingCheckSceneUpdated: pollingCheckSceneUpdated,
|
||||||
|
constants.AuditActionPollingFailureAlertHandled: pollingFailureAlertHandled,
|
||||||
constants.AuditActionPollingConcurrencyUpdated: pollingConcurrencyUpdated,
|
constants.AuditActionPollingConcurrencyUpdated: pollingConcurrencyUpdated,
|
||||||
constants.AuditActionPollingConcurrencyReset: pollingConcurrencyReset,
|
constants.AuditActionPollingConcurrencyReset: pollingConcurrencyReset,
|
||||||
constants.AuditActionPollingAlertRuleCreated: pollingAlertRuleCreated,
|
constants.AuditActionPollingAlertRuleCreated: pollingAlertRuleCreated,
|
||||||
@@ -966,6 +974,18 @@ func NewRegistry() *Registry {
|
|||||||
Type: constants.AuditResourceGatewayErrorRule, Name: "上游错误分类规则",
|
Type: constants.AuditResourceGatewayErrorRule, Name: "上游错误分类规则",
|
||||||
IdentityFields: []string{"id", "keyword", "match_type", "category", "priority", "status"},
|
IdentityFields: []string{"id", "keyword", "match_type", "category", "priority", "status"},
|
||||||
},
|
},
|
||||||
|
constants.AuditResourceCarrierPollingParam: {
|
||||||
|
Type: constants.AuditResourceCarrierPollingParam, Name: "运营商账户调度参数",
|
||||||
|
IdentityFields: []string{"carrier_id", "inflight_limit", "card_max_in_flight", "priority_max_ratio"},
|
||||||
|
},
|
||||||
|
constants.AuditResourcePollingCheckScene: {
|
||||||
|
Type: constants.AuditResourcePollingCheckScene, Name: "请求检查场景",
|
||||||
|
IdentityFields: []string{"scene", "scene_name"},
|
||||||
|
},
|
||||||
|
constants.AuditResourcePollingFailureAlert: {
|
||||||
|
Type: constants.AuditResourcePollingFailureAlert, Name: "卡轮询失败告警",
|
||||||
|
IdentityFields: []string{"id", "alert_level", "card_id", "carrier_id", "check_type", "status"},
|
||||||
|
},
|
||||||
constants.AuditResourcePollingAlertRule: {
|
constants.AuditResourcePollingAlertRule: {
|
||||||
Type: constants.AuditResourcePollingAlertRule, Name: "轮询告警规则",
|
Type: constants.AuditResourcePollingAlertRule, Name: "轮询告警规则",
|
||||||
IdentityFields: []string{"id", "rule_name", "task_type", "metric_type", "operator", "threshold", "alert_level", "status"},
|
IdentityFields: []string{"id", "rule_name", "task_type", "metric_type", "operator", "threshold", "alert_level", "status"},
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ const (
|
|||||||
statPrefixDuration = "duration_le:"
|
statPrefixDuration = "duration_le:"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
|
||||||
|
const StatFieldRateLimitWait = "rate_wait"
|
||||||
|
|
||||||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||||||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||||||
|
|
||||||
@@ -60,6 +63,18 @@ func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error
|
|||||||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||||||
}
|
}
|
||||||
if !decision.Allowed {
|
if !decision.Allowed {
|
||||||
|
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
|
||||||
|
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
|
||||||
|
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
|
||||||
|
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
|
||||||
|
if allowed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if waitErr != nil && ctx.Err() == nil {
|
||||||
|
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||||||
return gateway.NewRateLimitedError()
|
return gateway.NewRateLimitedError()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -189,3 +189,50 @@ func windowMember(nowMS int64) string {
|
|||||||
_, _ = rand.Read(buf)
|
_, _ = rand.Read(buf)
|
||||||
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Budget 返回运营商账户各查询类操作所属限流组本秒剩余名额,供分发器按预计调用数领取。
|
||||||
|
// groups 为限流组 → 本秒剩余;opGroup 为操作 → 限流组。只统计模式下返回 limited=false(不按 QPS 限制领取)。
|
||||||
|
// 全局组只在账户组之外再收紧:剩余名额取两者较小值。
|
||||||
|
func (l *Limiter) Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool) {
|
||||||
|
if l == nil || l.redis == nil {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
settings := l.settings.Current(ctx)
|
||||||
|
if !settings.Enforce() {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
groups, opGroup = map[string]int{}, map[string]string{}
|
||||||
|
now := time.Now().UnixMilli()
|
||||||
|
since := now - constants.GatewayRateLimitWindow.Milliseconds()
|
||||||
|
for _, op := range ops {
|
||||||
|
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
|
||||||
|
if !accountOK {
|
||||||
|
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
|
||||||
|
}
|
||||||
|
code := account.GroupCode
|
||||||
|
opGroup[op] = code
|
||||||
|
if _, seen := groups[code]; seen {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
limit := account.MaxQPS
|
||||||
|
if !account.HighPriority {
|
||||||
|
limit -= account.ReservedQPS
|
||||||
|
}
|
||||||
|
used, err := l.redis.ZCount(ctx, constants.RedisGatewayQPSKey(carrierID, code), strconv.FormatInt(since, 10), "+inf").Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
remaining := limit - int(used)
|
||||||
|
if globalOK {
|
||||||
|
globalUsed, gerr := l.redis.ZCount(ctx, constants.RedisGatewayGlobalQPSKey(global.GroupCode), strconv.FormatInt(since, 10), "+inf").Result()
|
||||||
|
if gerr == nil && global.MaxQPS-int(globalUsed) < remaining {
|
||||||
|
remaining = global.MaxQPS - int(globalUsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if remaining < 0 {
|
||||||
|
remaining = 0
|
||||||
|
}
|
||||||
|
groups[code] = remaining
|
||||||
|
}
|
||||||
|
return groups, opGroup, true
|
||||||
|
}
|
||||||
|
|||||||
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
@@ -0,0 +1,215 @@
|
|||||||
|
// Package nsqbus 封装卡轮询使用的 NSQ 生产者与按 topic 动态订阅的消费者管理器。
|
||||||
|
// NSQ 只承担执行分发与消费者并发控制,业务状态与重试全部在 Redis(add-carrier-polling-scheduler)。
|
||||||
|
package nsqbus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nsqio/go-nsq"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Producer 是 nsqd 生产者。
|
||||||
|
type Producer struct {
|
||||||
|
producer *nsq.Producer
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewProducer 连接 nsqd TCP 地址创建生产者。
|
||||||
|
func NewProducer(address string, logger *zap.Logger) (*Producer, error) {
|
||||||
|
cfg := nsq.NewConfig()
|
||||||
|
producer, err := nsq.NewProducer(address, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
producer.SetLogger(nsqLogger{logger: logger}, nsq.LogLevelWarning)
|
||||||
|
return &Producer{producer: producer}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publish 同步发布一条消息;topic 不存在时由 nsqd 自动创建。
|
||||||
|
func (p *Producer) Publish(topic string, body []byte) error {
|
||||||
|
return p.producer.Publish(topic, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop 停止生产者。
|
||||||
|
func (p *Producer) Stop() {
|
||||||
|
if p != nil && p.producer != nil {
|
||||||
|
p.producer.Stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler 处理一条消息;返回后由管理器 FIN。消息处理期间可调用 touch 延长 NSQ 消息超时。
|
||||||
|
type Handler func(ctx context.Context, body []byte, touch func())
|
||||||
|
|
||||||
|
// ConsumerConfig 是消费者参数。
|
||||||
|
type ConsumerConfig struct {
|
||||||
|
NSQDAddress string
|
||||||
|
NSQLookupdAddresses []string
|
||||||
|
MsgTimeout time.Duration
|
||||||
|
MaxAttempts uint16
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerManager 按 topic 动态订阅与取消订阅;每个 topic 一个消费者,max_in_flight 控制本进程对该 topic 的并发。
|
||||||
|
type ConsumerManager struct {
|
||||||
|
cfg ConsumerConfig
|
||||||
|
channel string
|
||||||
|
handler Handler
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
consumers map[string]*subscription
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
inflight sync.WaitGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
type subscription struct {
|
||||||
|
consumer *nsq.Consumer
|
||||||
|
maxInFlight int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewConsumerManager 创建消费者管理器。
|
||||||
|
func NewConsumerManager(cfg ConsumerConfig, channel string, handler Handler, logger *zap.Logger) *ConsumerManager {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
return &ConsumerManager{cfg: cfg, channel: channel, handler: handler, logger: logger,
|
||||||
|
consumers: map[string]*subscription{}, ctx: ctx, cancel: cancel}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync 使订阅集合与 desired(topic → max_in_flight)一致:新增订阅、取消多余订阅、调整 max_in_flight。
|
||||||
|
func (m *ConsumerManager) Sync(desired map[string]int) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for topic, sub := range m.consumers {
|
||||||
|
if _, keep := desired[topic]; !keep {
|
||||||
|
sub.consumer.Stop()
|
||||||
|
delete(m.consumers, topic)
|
||||||
|
m.logger.Info("取消订阅卡任务 topic", zap.String("topic", topic))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for topic, maxInFlight := range desired {
|
||||||
|
if maxInFlight <= 0 {
|
||||||
|
maxInFlight = 1
|
||||||
|
}
|
||||||
|
if sub, ok := m.consumers[topic]; ok {
|
||||||
|
if sub.maxInFlight != maxInFlight {
|
||||||
|
sub.consumer.ChangeMaxInFlight(maxInFlight)
|
||||||
|
sub.maxInFlight = maxInFlight
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
consumer, err := m.subscribe(topic, maxInFlight)
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Warn("订阅卡任务 topic 失败,下一轮重试", zap.String("topic", topic), zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.consumers[topic] = &subscription{consumer: consumer, maxInFlight: maxInFlight}
|
||||||
|
m.logger.Info("订阅卡任务 topic", zap.String("topic", topic), zap.Int("max_in_flight", maxInFlight))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *ConsumerManager) subscribe(topic string, maxInFlight int) (*nsq.Consumer, error) {
|
||||||
|
cfg := nsq.NewConfig()
|
||||||
|
cfg.MaxInFlight = maxInFlight
|
||||||
|
if m.cfg.MsgTimeout > 0 {
|
||||||
|
cfg.MsgTimeout = m.cfg.MsgTimeout
|
||||||
|
}
|
||||||
|
if m.cfg.MaxAttempts > 0 {
|
||||||
|
cfg.MaxAttempts = m.cfg.MaxAttempts
|
||||||
|
}
|
||||||
|
consumer, err := nsq.NewConsumer(topic, m.channel, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
consumer.SetLogger(nsqLogger{logger: m.logger}, nsq.LogLevelWarning)
|
||||||
|
handlerCtx := m.ctx // subscribe 在持锁时调用,捕获当前处理上下文
|
||||||
|
consumer.AddConcurrentHandlers(nsq.HandlerFunc(func(message *nsq.Message) error {
|
||||||
|
message.DisableAutoResponse()
|
||||||
|
m.inflight.Add(1)
|
||||||
|
defer m.inflight.Done()
|
||||||
|
defer message.Finish()
|
||||||
|
m.handler(handlerCtx, message.Body, message.Touch)
|
||||||
|
return nil
|
||||||
|
}), maxInFlight)
|
||||||
|
if len(m.cfg.NSQLookupdAddresses) > 0 {
|
||||||
|
err = consumer.ConnectToNSQLookupds(m.cfg.NSQLookupdAddresses)
|
||||||
|
} else {
|
||||||
|
err = consumer.ConnectToNSQD(m.cfg.NSQDAddress)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
consumer.Stop()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return consumer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Topics 返回当前已订阅的 topic。
|
||||||
|
func (m *ConsumerManager) Topics() []string {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
topics := make([]string, 0, len(m.consumers))
|
||||||
|
for topic := range m.consumers {
|
||||||
|
topics = append(topics, topic)
|
||||||
|
}
|
||||||
|
return topics
|
||||||
|
}
|
||||||
|
|
||||||
|
// StopAll 停止全部订阅;在宽限期内等待执行中的消息完成并 FIN,超过宽限期后取消处理上下文。
|
||||||
|
func (m *ConsumerManager) StopAll(grace time.Duration) {
|
||||||
|
m.mu.Lock()
|
||||||
|
for topic, sub := range m.consumers {
|
||||||
|
sub.consumer.Stop()
|
||||||
|
delete(m.consumers, topic)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
m.inflight.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(grace):
|
||||||
|
m.logger.Warn("优雅退出宽限期内仍有卡任务未完成,取消执行(由租约到期后重新领取)")
|
||||||
|
}
|
||||||
|
m.cancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset 在 StopAll 后重新允许订阅(切回 carrier 模式时使用)。
|
||||||
|
func (m *ConsumerManager) Reset() {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.ctx.Err() != nil {
|
||||||
|
m.ctx, m.cancel = context.WithCancel(context.Background())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SplitAddresses 解析逗号分隔的地址列表。
|
||||||
|
func SplitAddresses(raw string) []string {
|
||||||
|
var result []string
|
||||||
|
for _, part := range strings.Split(raw, ",") {
|
||||||
|
if part = strings.TrimSpace(part); part != "" {
|
||||||
|
result = append(result, part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type nsqLogger struct {
|
||||||
|
logger *zap.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output 实现 go-nsq 的日志接口。
|
||||||
|
func (l nsqLogger) Output(_ int, s string) error {
|
||||||
|
if l.logger != nil {
|
||||||
|
l.logger.Warn("NSQ", zap.String("message", s))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
185
internal/model/dto/carrier_polling_dto.go
Normal file
185
internal/model/dto/carrier_polling_dto.go
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
package dto
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// CarrierPollingParamItem 运营商账户调度参数(含全局默认合并后的生效值)。
|
||||||
|
type CarrierPollingParamItem struct {
|
||||||
|
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||||
|
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||||
|
IsDefault bool `json:"is_default" description:"是否未单独配置、全部使用全局默认"`
|
||||||
|
InflightLimit int `json:"inflight_limit" description:"生效的执行上限(跨 Worker 实例同时执行卡任务数)"`
|
||||||
|
CardMaxInFlight int `json:"card_max_in_flight" description:"生效的每进程卡任务 max_in_flight"`
|
||||||
|
PriorityMaxRatio *float64 `json:"priority_max_ratio" description:"优先队列每轮最大占比(0~1),空表示不限"`
|
||||||
|
StopResumeMaxInFlight *int `json:"stop_resume_max_in_flight" description:"每进程停复机 max_in_flight(停复机指令队列使用),空表示默认"`
|
||||||
|
SRInflightLimit *int `json:"sr_inflight_limit" description:"停复机执行上限(停复机指令队列使用),空表示默认"`
|
||||||
|
StopResumeMinInterval *int `json:"stop_resume_min_interval" description:"单卡两次停复机调用最小间隔(秒),空表示默认"`
|
||||||
|
StopResumeConfirmDelay *int `json:"stop_resume_confirm_delay" description:"停复机结果未知后确认延迟(秒),空表示默认"`
|
||||||
|
ConfiguredInflight *int `json:"configured_inflight_limit" description:"单独配置的执行上限,空表示使用默认"`
|
||||||
|
ConfiguredMaxInFlight *int `json:"configured_card_max_in_flight" description:"单独配置的每进程 max_in_flight,空表示使用默认"`
|
||||||
|
ConsumeCapacity int `json:"consume_capacity" description:"当前订阅进程 max_in_flight 之和(执行上限不应超过该值)"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierPollingParamListResponse 运营商账户调度参数列表。
|
||||||
|
type CarrierPollingParamListResponse struct {
|
||||||
|
Items []CarrierPollingParamItem `json:"items" description:"各运营商账户调度参数"`
|
||||||
|
SubscriberProcesses int `json:"subscriber_processes" description:"订阅卡任务的 Worker 进程数"`
|
||||||
|
DefaultInflight int `json:"default_inflight_limit" description:"全局默认执行上限"`
|
||||||
|
DefaultMaxInFlight int `json:"default_card_max_in_flight" description:"全局默认每进程 max_in_flight"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierPollingParamPath 运营商账户路径参数。
|
||||||
|
type CarrierPollingParamPath struct {
|
||||||
|
CarrierID uint `json:"-" path:"carrier_id" required:"true" description:"运营商账户ID"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveCarrierPollingParamRequest 保存运营商账户调度参数;字段为空表示该项使用全局默认。
|
||||||
|
type SaveCarrierPollingParamRequest struct {
|
||||||
|
CarrierID uint `json:"-" path:"carrier_id" required:"true" description:"运营商账户ID"`
|
||||||
|
InflightLimit *int `json:"inflight_limit" validate:"omitempty,min=1,max=10000" minimum:"1" maximum:"10000" description:"执行上限,空表示默认"`
|
||||||
|
CardMaxInFlight *int `json:"card_max_in_flight" validate:"omitempty,min=1,max=2000" minimum:"1" maximum:"2000" description:"每进程卡任务 max_in_flight,空表示默认"`
|
||||||
|
StopResumeMaxInFlight *int `json:"stop_resume_max_in_flight" validate:"omitempty,min=1,max=1000" minimum:"1" maximum:"1000" description:"每进程停复机 max_in_flight,空表示默认"`
|
||||||
|
SRInflightLimit *int `json:"sr_inflight_limit" validate:"omitempty,min=1,max=10000" minimum:"1" maximum:"10000" description:"停复机执行上限,空表示默认"`
|
||||||
|
PriorityMaxRatio *float64 `json:"priority_max_ratio" validate:"omitempty,gt=0,lte=1" description:"优先队列每轮最大占比(0~1],空表示不限"`
|
||||||
|
StopResumeMinInterval *int `json:"stop_resume_min_interval" validate:"omitempty,min=1,max=86400" minimum:"1" maximum:"86400" description:"单卡停复机最小间隔(秒),空表示默认"`
|
||||||
|
StopResumeConfirmDelay *int `json:"stop_resume_confirm_delay" validate:"omitempty,min=1,max=86400" minimum:"1" maximum:"86400" description:"停复机结果未知确认延迟(秒),空表示默认"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingCheckSceneItem 请求检查场景。
|
||||||
|
type PollingCheckSceneItem struct {
|
||||||
|
Scene string `json:"scene" description:"场景编码 (order_paid:订单支付, package_activated:套餐激活, stop_resume_succeeded:停复机成功, device_operation:设备操作, auto_renewal_purchase:自动续费购包, client_asset_detail:C 端资产详情, admin_asset_realtime:后台资产实时状态, realname_link:实名链接)"`
|
||||||
|
SceneName string `json:"scene_name" description:"场景名称"`
|
||||||
|
CheckTypes []string `json:"check_types" description:"检查类型 (card_status:卡状态, carddata:流量, realname:实名)"`
|
||||||
|
ConfirmDelays []int `json:"confirm_delays" description:"延迟确认序列(秒),空表示只立即检查一次"`
|
||||||
|
Cooldown *int `json:"cooldown" description:"冷却时长(秒),最近一次成功检查未超过该时长则不请求;空表示不冷却"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at" description:"更新时间"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePollingCheckSceneRequest 修改请求检查场景(场景由代码预置,只可改检查类型、确认延迟与冷却)。
|
||||||
|
type UpdatePollingCheckSceneRequest struct {
|
||||||
|
Scene string `json:"-" path:"scene" required:"true" description:"场景编码"`
|
||||||
|
CheckTypes []string `json:"check_types" validate:"required,min=1,max=3,dive,oneof=card_status carddata realname" required:"true" description:"检查类型 (card_status:卡状态, carddata:流量, realname:实名)"`
|
||||||
|
ConfirmDelays []int `json:"confirm_delays" validate:"omitempty,max=5,dive,min=1,max=86400" description:"延迟确认序列(秒,递增),空表示只立即检查一次"`
|
||||||
|
Cooldown *int `json:"cooldown" validate:"omitempty,min=1,max=86400" minimum:"1" maximum:"86400" description:"冷却时长(秒),空表示不冷却"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingFailureAlertListRequest 卡轮询失败告警列表请求。
|
||||||
|
type PollingFailureAlertListRequest struct {
|
||||||
|
Page int `json:"page" query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码"`
|
||||||
|
PageSize int `json:"page_size" query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页数量"`
|
||||||
|
CarrierID *uint `json:"carrier_id" query:"carrier_id" validate:"omitempty,min=1" description:"运营商账户ID"`
|
||||||
|
CheckType string `json:"check_type" query:"check_type" validate:"omitempty,oneof=card_status carddata realname package all" description:"检查类型 (card_status:卡状态, carddata:流量, realname:实名, package:套餐, all:账户级)"`
|
||||||
|
ErrorCategory string `json:"error_category" query:"error_category" validate:"omitempty,max=50" description:"错误类别"`
|
||||||
|
Status *int `json:"status" query:"status" validate:"omitempty,oneof=1 2 3 4" description:"状态 (1:未处理, 2:已恢复, 3:已关闭轮询, 4:已处理)"`
|
||||||
|
AlertLevel string `json:"alert_level" query:"alert_level" validate:"omitempty,oneof=card carrier" description:"告警级别 (card:卡级, carrier:账户级)"`
|
||||||
|
ICCID string `json:"iccid" query:"iccid" validate:"omitempty,max=30" description:"ICCID(精确)"`
|
||||||
|
StartTime string `json:"start_time" query:"start_time" description:"最近失败时间起始(带时区的 RFC3339 秒级时间)"`
|
||||||
|
EndTime string `json:"end_time" query:"end_time" description:"最近失败时间结束(带时区的 RFC3339 秒级时间)"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingFailureAlertItem 卡轮询失败告警。
|
||||||
|
type PollingFailureAlertItem struct {
|
||||||
|
ID uint `json:"id" description:"告警ID"`
|
||||||
|
AlertLevel string `json:"alert_level" description:"告警级别 (card:卡级, carrier:账户级)"`
|
||||||
|
CardID uint `json:"card_id" description:"卡ID,账户级为 0"`
|
||||||
|
ICCID string `json:"iccid" description:"ICCID"`
|
||||||
|
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||||
|
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||||
|
CheckType string `json:"check_type" description:"检查类型"`
|
||||||
|
CheckTypeName string `json:"check_type_name" description:"检查类型名称"`
|
||||||
|
ErrorCategory string `json:"error_category" description:"最近错误类别"`
|
||||||
|
FailCount int `json:"fail_count" description:"连续失败次数;账户级为窗口内告警数"`
|
||||||
|
FirstFailedAt *time.Time `json:"first_failed_at" description:"首次失败时间"`
|
||||||
|
LastFailedAt *time.Time `json:"last_failed_at" description:"最近失败时间"`
|
||||||
|
LastError string `json:"last_error" description:"最近错误"`
|
||||||
|
Status int `json:"status" description:"状态 (1:未处理, 2:已恢复, 3:已关闭轮询, 4:已处理)"`
|
||||||
|
StatusName string `json:"status_name" description:"状态名称"`
|
||||||
|
HandledBy uint `json:"handled_by" description:"处理人"`
|
||||||
|
HandledAt *time.Time `json:"handled_at" description:"处理时间"`
|
||||||
|
HandleRemark string `json:"handle_remark" description:"处理备注"`
|
||||||
|
RecoveredAt *time.Time `json:"recovered_at" description:"自动恢复时间"`
|
||||||
|
CreatedAt time.Time `json:"created_at" description:"创建时间"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingFailureAlertPageResult 卡轮询失败告警分页结果。
|
||||||
|
type PollingFailureAlertPageResult struct {
|
||||||
|
List []PollingFailureAlertItem `json:"items" description:"告警列表"`
|
||||||
|
Total int64 `json:"total" description:"总数"`
|
||||||
|
Page int `json:"page" description:"当前页"`
|
||||||
|
PageSize int `json:"size" description:"每页数量"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingFailureAlertSummaryItem 未处理告警按运营商账户 × 错误类别汇总。
|
||||||
|
type PollingFailureAlertSummaryItem struct {
|
||||||
|
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||||
|
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||||
|
ErrorCategory string `json:"error_category" description:"错误类别"`
|
||||||
|
OpenCount int64 `json:"open_count" description:"未处理卡级告警数"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingFailureAlertIDPath 告警路径参数。
|
||||||
|
type PollingFailureAlertIDPath struct {
|
||||||
|
ID uint `json:"-" path:"id" required:"true" description:"告警ID"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandlePollingFailureAlertRequest 标记告警已处理。
|
||||||
|
type HandlePollingFailureAlertRequest struct {
|
||||||
|
ID uint `json:"-" path:"id" required:"true" description:"告警ID"`
|
||||||
|
Status int `json:"status" validate:"required,oneof=4" required:"true" description:"目标状态 (4:已处理);关闭轮询请使用资产轮询开关接口"`
|
||||||
|
Remark string `json:"remark" validate:"omitempty,max=500" maxLength:"500" description:"处理备注"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CardPollingStatePath 卡ID路径参数。
|
||||||
|
type CardPollingStatePath struct {
|
||||||
|
ID uint `json:"-" path:"id" required:"true" description:"卡ID"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CardPollingTypeState 卡某检查类型的轮询状态。
|
||||||
|
type CardPollingTypeState struct {
|
||||||
|
CheckType string `json:"check_type" description:"检查类型"`
|
||||||
|
CheckTypeName string `json:"check_type_name" description:"检查类型名称"`
|
||||||
|
NextDueAt *time.Time `json:"next_due_at" description:"下次到期时间"`
|
||||||
|
LastSuccessAt *time.Time `json:"last_success_at" description:"最近一次成功时间"`
|
||||||
|
FailCount int `json:"fail_count" description:"连续失败次数"`
|
||||||
|
FirstFailedAt *time.Time `json:"first_failed_at" description:"首次失败时间"`
|
||||||
|
LastError string `json:"last_error" description:"最近错误(类别|信息)"`
|
||||||
|
PendingConfirm []int `json:"pending_confirm" description:"待执行的延迟确认(秒)"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CardPollingStateResponse 单卡轮询状态(只读)。
|
||||||
|
type CardPollingStateResponse struct {
|
||||||
|
CardID uint `json:"card_id" description:"卡ID"`
|
||||||
|
ICCID string `json:"iccid" description:"ICCID"`
|
||||||
|
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||||
|
SchedulerMode string `json:"scheduler_mode" description:"当前调度模式 (legacy:旧分片调度, carrier:按运营商账户调度)"`
|
||||||
|
EnablePolling bool `json:"enable_polling" description:"卡轮询开关"`
|
||||||
|
Queue string `json:"queue" description:"所在队列 (priority:优先, queue:普通, none:不在轮询中)"`
|
||||||
|
QueueScoreAt *time.Time `json:"queue_score_at" description:"队列中下次可领取时间"`
|
||||||
|
Running bool `json:"running" description:"是否执行中"`
|
||||||
|
ClaimExpireAt *time.Time `json:"claim_expire_at" description:"处理中标记到期时间"`
|
||||||
|
PriorityMark bool `json:"priority_mark" description:"执行中收到请求检查,收尾后进入优先队列"`
|
||||||
|
Types []CardPollingTypeState `json:"types" description:"各检查类型状态"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierPollingStatsItem 按运营商账户的轮询监控。
|
||||||
|
type CarrierPollingStatsItem struct {
|
||||||
|
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
|
||||||
|
CarrierName string `json:"carrier_name" description:"运营商账户名称"`
|
||||||
|
PriorityLength int64 `json:"priority_length" description:"优先队列长度"`
|
||||||
|
QueueLength int64 `json:"queue_length" description:"普通队列长度"`
|
||||||
|
DueCount int64 `json:"due_count" description:"已到期卡数(优先 + 普通)"`
|
||||||
|
Running int64 `json:"running" description:"执行中卡任务数"`
|
||||||
|
InflightLimit int `json:"inflight_limit" description:"执行上限"`
|
||||||
|
QPSLastMinute float64 `json:"qps_last_minute" description:"上一分钟实际网关调用 QPS(查询类)"`
|
||||||
|
RateLimitWaits int64 `json:"rate_limit_waits" description:"上一分钟限流等待次数"`
|
||||||
|
RateLimited int64 `json:"rate_limited" description:"上一分钟限流拒绝或等待超时次数"`
|
||||||
|
ExpiredClaims1h int64 `json:"expired_claims_1h" description:"近 1 小时处理中标记到期次数(执行崩溃或超时)"`
|
||||||
|
OpenAlerts int64 `json:"open_alerts" description:"未处理卡级失败告警数"`
|
||||||
|
OpenAlertsByType map[string]int64 `json:"open_alerts_by_category" description:"未处理卡级失败告警数(按错误类别)"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierPollingStatsResponse 按运营商账户的轮询监控。
|
||||||
|
type CarrierPollingStatsResponse struct {
|
||||||
|
SchedulerMode string `json:"scheduler_mode" description:"当前调度模式"`
|
||||||
|
InitState string `json:"init_state" description:"新结构初始化状态 (running / done,空表示未初始化)"`
|
||||||
|
Items []CarrierPollingStatsItem `json:"items" description:"各运营商账户监控"`
|
||||||
|
}
|
||||||
226
internal/polling/carrier_bridge.go
Normal file
226
internal/polling/carrier_bridge.go
Normal file
@@ -0,0 +1,226 @@
|
|||||||
|
package polling
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CarrierScheduling 是按运营商账户调度在轮询包内的接缝:判断当前模式,并按卡计算各检查类型到期时间写入账户队列。
|
||||||
|
// 所有写入轮询结构的入口(初始化、生命周期、优先提示、手动触发)按当前模式写新结构或旧分片结构。
|
||||||
|
type CarrierScheduling struct {
|
||||||
|
store *carrierqueue.Store
|
||||||
|
settings *carrierqueue.SettingsProvider
|
||||||
|
configMgr *PollingConfigManager
|
||||||
|
logger *zap.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierScheduling 创建按运营商账户调度接缝。
|
||||||
|
func NewCarrierScheduling(store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, configMgr *PollingConfigManager, logger *zap.Logger) *CarrierScheduling {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &CarrierScheduling{store: store, settings: settings, configMgr: configMgr, logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active 判断当前是否为按运营商账户调度。
|
||||||
|
func (c *CarrierScheduling) Active(ctx context.Context) bool {
|
||||||
|
return c != nil && c.store != nil && c.settings != nil && c.settings.Current(ctx).CarrierMode()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store 返回账户队列存储。
|
||||||
|
func (c *CarrierScheduling) Store() *carrierqueue.Store {
|
||||||
|
if c == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.store
|
||||||
|
}
|
||||||
|
|
||||||
|
// Settings 返回调度参数提供者。
|
||||||
|
func (c *CarrierScheduling) Settings() *carrierqueue.SettingsProvider {
|
||||||
|
if c == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.settings
|
||||||
|
}
|
||||||
|
|
||||||
|
// DuesForCard 按卡匹配的轮询配置计算各检查类型下次到期时间。
|
||||||
|
// fromLastCheck 为 true 时按各类型上次检查时间计算(初始化);否则按当前时间加抖动(新卡、恢复轮询、状态变化)。
|
||||||
|
// 间隔为 0 或未配置的类型不返回;重置日前加密窗口规则算出的时间更早时以它为准。
|
||||||
|
func (c *CarrierScheduling) DuesForCard(ctx context.Context, card *model.IotCard, fromLastCheck bool) map[string]time.Time {
|
||||||
|
dues := map[string]time.Time{}
|
||||||
|
if c == nil || c.configMgr == nil || card == nil {
|
||||||
|
return dues
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
for taskType, info := range c.configMgr.MergedTaskIntervals(card) {
|
||||||
|
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||||
|
if checkType == "" || info.Interval <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var due time.Time
|
||||||
|
if fromLastCheck {
|
||||||
|
due = calculateNextCheckTime(lastCheckAtByTaskType(card, taskType), info.Interval, now)
|
||||||
|
if windowDue := c.configMgr.NextCheckAt(ctx, card, taskType, info, now); windowDue.Before(due) && windowDue.After(now) {
|
||||||
|
due = windowDue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
due = calcInitialDelay(info.Interval)
|
||||||
|
}
|
||||||
|
dues[checkType] = due
|
||||||
|
}
|
||||||
|
return dues
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertCard 把卡写入所属账户队列;fromLastCheck 含义同 DuesForCard。
|
||||||
|
func (c *CarrierScheduling) UpsertCard(ctx context.Context, card *model.IotCard, fromLastCheck bool) {
|
||||||
|
if c == nil || c.store == nil || card == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := c.store.Upsert(ctx, card.ID, card.CarrierID, c.DuesForCard(ctx, card, fromLastCheck)); err != nil {
|
||||||
|
c.logger.Warn("写入运营商账户轮询队列失败", zap.Uint("card_id", card.ID), zap.Uint("carrier_id", card.CarrierID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveCard 把卡移出所属账户队列;carrierID 为 0 时按到期信息中记录的账户移出。
|
||||||
|
func (c *CarrierScheduling) RemoveCard(ctx context.Context, cardID, carrierID uint) {
|
||||||
|
if c == nil || c.store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if carrierID == 0 {
|
||||||
|
due, err := c.store.ReadDue(ctx, cardID)
|
||||||
|
if err != nil || due.CarrierID == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
carrierID = due.CarrierID
|
||||||
|
}
|
||||||
|
if err := c.store.Remove(ctx, cardID, carrierID); err != nil {
|
||||||
|
c.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestTaskType 把卡的某检查类型(旧任务类型 polling:xxx)设为立即到期放入账户优先队列(优先提示、手动触发)。
|
||||||
|
// 卡不在轮询中时不处理。
|
||||||
|
func (c *CarrierScheduling) RequestTaskType(ctx context.Context, cardID, carrierID uint, taskType string) {
|
||||||
|
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||||
|
if c == nil || c.store == nil || checkType == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := c.store.RequestCheck(ctx, carrierqueue.CheckRequest{CardID: cardID, CarrierID: carrierID, CheckTypes: []string{checkType}}); err != nil {
|
||||||
|
c.logger.Warn("写入运营商账户优先队列失败", zap.Uint("card_id", cardID), zap.String("task_type", taskType), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierInitializer 从数据库按批初始化全部运营商账户队列与卡到期信息;进度与旧初始化器同结构,供 init-progress 展示。
|
||||||
|
type CarrierInitializer struct {
|
||||||
|
scheduling *CarrierScheduling
|
||||||
|
iotCardStore *postgres.IotCardStore
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
progress InitProgress
|
||||||
|
running atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierInitializer 创建新结构初始化器。
|
||||||
|
func NewCarrierInitializer(scheduling *CarrierScheduling, iotCardStore *postgres.IotCardStore, logger *zap.Logger) *CarrierInitializer {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &CarrierInitializer{scheduling: scheduling, iotCardStore: iotCardStore, logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress 返回初始化进度快照。
|
||||||
|
func (i *CarrierInitializer) Progress() InitProgress {
|
||||||
|
i.mu.RLock()
|
||||||
|
defer i.mu.RUnlock()
|
||||||
|
return i.progress
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run 同步执行一次全量初始化:按批读取启用轮询的卡,计算各类型到期并写入账户队列。返回写入的卡数。
|
||||||
|
func (i *CarrierInitializer) Run(ctx context.Context) (int64, error) {
|
||||||
|
if !i.running.CompareAndSwap(false, true) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
defer i.running.Store(false)
|
||||||
|
started := time.Now()
|
||||||
|
total, err := i.iotCardStore.CountForPolling(ctx)
|
||||||
|
if err != nil {
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", err.Error() })
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
i.setProgress(func(p *InitProgress) {
|
||||||
|
*p = InitProgress{TotalCards: total, StartTime: started, Status: "running"}
|
||||||
|
})
|
||||||
|
const batchSize = 2000
|
||||||
|
var lastID uint
|
||||||
|
var written int64
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", err.Error() })
|
||||||
|
return written, err
|
||||||
|
}
|
||||||
|
cards, fetchErr := i.iotCardStore.ListForPollingBatch(ctx, lastID, batchSize)
|
||||||
|
if fetchErr != nil {
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status, p.ErrorMessage = "failed", fetchErr.Error() })
|
||||||
|
return written, fetchErr
|
||||||
|
}
|
||||||
|
if len(cards) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if tier := i.scheduling.configMgr.TierResolver(); tier != nil {
|
||||||
|
tier.Prefill(ctx, cards)
|
||||||
|
}
|
||||||
|
for _, card := range cards {
|
||||||
|
i.scheduling.UpsertCard(ctx, card, true)
|
||||||
|
written++
|
||||||
|
}
|
||||||
|
lastID = cards[len(cards)-1].ID
|
||||||
|
loaded := written
|
||||||
|
i.setProgress(func(p *InitProgress) { p.LoadedCards, p.LastBatchTime = loaded, time.Now() })
|
||||||
|
}
|
||||||
|
i.setProgress(func(p *InitProgress) { p.Status = "completed" })
|
||||||
|
i.logger.Info("运营商账户轮询结构初始化完成", zap.Int64("cards", written), zap.Duration("duration", time.Since(started)))
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *CarrierInitializer) setProgress(update func(*InitProgress)) {
|
||||||
|
i.mu.Lock()
|
||||||
|
update(&i.progress)
|
||||||
|
i.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdvanceDue 把卡某检查类型的到期时间提前到 due(只提前不推后),并同步把账户队列中的 score 提前;返回是否提前。
|
||||||
|
// 卡不在轮询中(无到期信息)时不处理。
|
||||||
|
func (c *CarrierScheduling) AdvanceDue(ctx context.Context, card *model.IotCard, checkType string, due time.Time) (bool, error) {
|
||||||
|
if c == nil || c.store == nil || card == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
current, err := c.store.ReadDue(ctx, card.ID)
|
||||||
|
if err != nil || current.CarrierID == 0 {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
item, ok := current.Types[checkType]
|
||||||
|
if !ok || !item.HasDue || !due.Before(item.Due) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
client := c.store.Client()
|
||||||
|
member := strconv.FormatUint(uint64(card.ID), 10)
|
||||||
|
pipe := client.TxPipeline()
|
||||||
|
pipe.HSet(ctx, constants.RedisCardDueKey(card.ID), carrierqueue.FieldDue(checkType), due.UnixMilli())
|
||||||
|
score := redis.ZAddArgs{XX: true, LT: true, Members: []redis.Z{{Score: float64(due.UnixMilli()), Member: member}}}
|
||||||
|
pipe.ZAddArgs(ctx, constants.RedisCarrierQueueKey(current.CarrierID), score)
|
||||||
|
pipe.ZAddArgs(ctx, constants.RedisCarrierPriorityQueueKey(current.CarrierID), score)
|
||||||
|
_, err = pipe.Exec(ctx)
|
||||||
|
return err == nil, err
|
||||||
|
}
|
||||||
177
internal/polling/carrierqueue/alerts.go
Normal file
177
internal/polling/carrierqueue/alerts.go
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
package carrierqueue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FailureEvent 是一次需要告警的检查失败。
|
||||||
|
type FailureEvent struct {
|
||||||
|
CardID uint
|
||||||
|
ICCID string
|
||||||
|
CarrierID uint
|
||||||
|
CheckType string
|
||||||
|
Category string
|
||||||
|
Message string
|
||||||
|
FailCount int
|
||||||
|
FirstFailed time.Time
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type alertOp struct {
|
||||||
|
failure *FailureEvent
|
||||||
|
recovered bool
|
||||||
|
cardID uint
|
||||||
|
checkType string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AlertWriter 异步写卡轮询失败告警:同一卡同一检查类型只保留一条未处理告警,后续失败更新计数;
|
||||||
|
// 首次成功转为已恢复;同账户窗口内新增告警超阈值时产生账户级告警。写入失败只记日志,不影响轮询。
|
||||||
|
type AlertWriter struct {
|
||||||
|
db *gorm.DB
|
||||||
|
redis *redis.Client
|
||||||
|
settings *SettingsProvider
|
||||||
|
logger *zap.Logger
|
||||||
|
ch chan alertOp
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAlertWriter 创建告警写入器;调用 Start 后开始消费。
|
||||||
|
func NewAlertWriter(db *gorm.DB, redisClient *redis.Client, settings *SettingsProvider, logger *zap.Logger) *AlertWriter {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &AlertWriter{db: db, redis: redisClient, settings: settings, logger: logger, ch: make(chan alertOp, 2048)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start 启动后台写入协程,直到 ctx 结束。
|
||||||
|
func (w *AlertWriter) Start(ctx context.Context) {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case op := <-w.ch:
|
||||||
|
w.apply(op)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReportFailure 提交一条失败告警(达到阈值后才调用)。
|
||||||
|
func (w *AlertWriter) ReportFailure(event FailureEvent) {
|
||||||
|
w.offer(alertOp{failure: &event})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReportRecovered 提交一次恢复:该卡该检查类型的未处理告警转为已恢复。
|
||||||
|
func (w *AlertWriter) ReportRecovered(cardID uint, checkType string) {
|
||||||
|
w.offer(alertOp{recovered: true, cardID: cardID, checkType: checkType})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *AlertWriter) offer(op alertOp) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case w.ch <- op:
|
||||||
|
default:
|
||||||
|
w.logger.Warn("卡轮询失败告警队列已满,丢弃本次写入", zap.String("metric", "polling.failure_alert.dropped"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *AlertWriter) apply(op alertOp) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if op.recovered {
|
||||||
|
err := w.db.WithContext(ctx).Exec(`UPDATE tb_polling_failure_alert SET status = ?, recovered_at = NOW(), updated_at = NOW()
|
||||||
|
WHERE alert_level = ? AND card_id = ? AND check_type = ? AND status = ?`,
|
||||||
|
constants.PollingAlertStatusRecovered, constants.PollingAlertLevelCard, op.cardID, op.checkType, constants.PollingAlertStatusOpen).Error
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("卡轮询告警转为已恢复失败", zap.Uint("card_id", op.cardID), zap.String("check_type", op.checkType), zap.Error(err))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
event := op.failure
|
||||||
|
var inserted bool
|
||||||
|
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||||
|
(alert_level, card_id, iccid, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||||
|
DO UPDATE SET fail_count = EXCLUDED.fail_count, error_category = EXCLUDED.error_category,
|
||||||
|
last_failed_at = EXCLUDED.last_failed_at, last_error = EXCLUDED.last_error, updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0)`,
|
||||||
|
constants.PollingAlertLevelCard, event.CardID, event.ICCID, event.CarrierID, event.CheckType, event.Category, event.FailCount,
|
||||||
|
nullableTime(event.FirstFailed), event.At, truncate(event.Message, 500), constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("写卡轮询失败告警失败", zap.Uint("card_id", event.CardID), zap.String("check_type", event.CheckType), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !inserted {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.logger.Error("卡轮询持续失败告警", zap.Uint("card_id", event.CardID), zap.String("iccid", event.ICCID),
|
||||||
|
zap.Uint("carrier_id", event.CarrierID), zap.String("check_type", event.CheckType), zap.String("category", event.Category),
|
||||||
|
zap.Int("fail_count", event.FailCount), zap.String("metric", "polling.failure_alert.opened"))
|
||||||
|
w.checkAccountAlert(ctx, event.CarrierID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkAccountAlert 同账户窗口内新增卡级告警数超过阈值时,产生(或更新)一条账户级告警。
|
||||||
|
func (w *AlertWriter) checkAccountAlert(ctx context.Context, carrierID uint) {
|
||||||
|
settings := w.settings.Current(ctx)
|
||||||
|
window, limit := settings.AccountAlertWindow, settings.AccountAlertLimit
|
||||||
|
if window <= 0 || limit <= 0 || w.redis == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
key := constants.RedisCarrierAccountAlertKey(carrierID)
|
||||||
|
member := strconv.FormatInt(now.UnixNano(), 36)
|
||||||
|
pipe := w.redis.TxPipeline()
|
||||||
|
pipe.ZAdd(ctx, key, redis.Z{Score: float64(now.UnixMilli()), Member: member})
|
||||||
|
pipe.ZRemRangeByScore(ctx, key, "-inf", strconv.FormatInt(now.Add(-window).UnixMilli(), 10))
|
||||||
|
count := pipe.ZCard(ctx, key)
|
||||||
|
pipe.Expire(ctx, key, window*2)
|
||||||
|
if _, err := pipe.Exec(ctx); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if int(count.Val()) <= limit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var inserted bool
|
||||||
|
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||||
|
(alert_level, card_id, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||||
|
VALUES (?, 0, ?, ?, '', ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||||
|
DO UPDATE SET fail_count = EXCLUDED.fail_count, last_failed_at = EXCLUDED.last_failed_at, updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0)`,
|
||||||
|
constants.PollingAlertLevelCarrier, carrierID, constants.PollingAlertCheckTypeAll, count.Val(), now, now,
|
||||||
|
"窗口内同账户新增失败告警数超过阈值,通常是账户额度、凭证或上游故障,请联系上游处理账户", constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("写账户级轮询告警失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if inserted {
|
||||||
|
w.logger.Error("运营商账户轮询失败告警数超过阈值", zap.Uint("carrier_id", carrierID), zap.Int64("count", count.Val()),
|
||||||
|
zap.Duration("window", window), zap.String("metric", "polling.failure_alert.carrier_opened"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func nullableTime(t time.Time) any {
|
||||||
|
if t.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncate(s string, max int) string {
|
||||||
|
runes := []rune(s)
|
||||||
|
if len(runes) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return string(runes[:max])
|
||||||
|
}
|
||||||
326
internal/polling/carrierqueue/dispatcher.go
Normal file
326
internal/polling/carrierqueue/dispatcher.go
Normal file
@@ -0,0 +1,326 @@
|
|||||||
|
package carrierqueue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
stderrors "errors"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CardMessage 是卡任务 NSQ 消息体。
|
||||||
|
type CardMessage struct {
|
||||||
|
CardID uint `json:"card_id"`
|
||||||
|
CarrierID uint `json:"carrier_id"`
|
||||||
|
Token string `json:"token"`
|
||||||
|
DispatchedAt int64 `json:"dispatched_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publisher 发布 NSQ 消息。
|
||||||
|
type Publisher interface {
|
||||||
|
Publish(topic string, body []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// BudgetProvider 返回账户各查询操作所属限流组本秒剩余名额;limited=false 表示不按 QPS 限制领取。
|
||||||
|
type BudgetProvider interface {
|
||||||
|
Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierParams 是一个运营商账户生效的调度参数(已合并全局默认)。
|
||||||
|
type CarrierParams struct {
|
||||||
|
InflightLimit int
|
||||||
|
CardMaxInFlight int
|
||||||
|
PriorityRatio float64
|
||||||
|
Configured bool // 是否有该账户的配置行
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierEntry 是分发器缓存的账户信息。
|
||||||
|
type carrierEntry struct {
|
||||||
|
id uint
|
||||||
|
realnameControl bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dispatcher 每个分发间隔对每个运营商账户抢分发锁、领取到期卡并发布到账户 topic。多实例安全。
|
||||||
|
type Dispatcher struct {
|
||||||
|
db *gorm.DB
|
||||||
|
store *Store
|
||||||
|
settings *SettingsProvider
|
||||||
|
publisher Publisher
|
||||||
|
budgets BudgetProvider
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
instanceID string
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
carriers []carrierEntry
|
||||||
|
params map[uint]model.CarrierPollingParam
|
||||||
|
loadedAt time.Time
|
||||||
|
tightened map[uint]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDispatcher 创建分发器。
|
||||||
|
func NewDispatcher(db *gorm.DB, store *Store, settings *SettingsProvider, publisher Publisher, budgets BudgetProvider,
|
||||||
|
cfg config.CarrierSchedulerConfig, instanceID string, logger *zap.Logger) *Dispatcher {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &Dispatcher{db: db, store: store, settings: settings, publisher: publisher, budgets: budgets, cfg: cfg,
|
||||||
|
instanceID: instanceID, logger: logger, params: map[uint]model.CarrierPollingParam{}, tightened: map[uint]time.Time{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run 按分发间隔循环,直到 ctx 结束。
|
||||||
|
func (d *Dispatcher) Run(ctx context.Context) {
|
||||||
|
interval := d.cfg.DispatchInterval
|
||||||
|
if interval <= 0 {
|
||||||
|
interval = time.Second
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
d.tick(ctx, interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ready 判断新调度是否可领取:模式为 carrier 且新结构初始化已完成。
|
||||||
|
func (d *Dispatcher) Ready(ctx context.Context) bool {
|
||||||
|
if !d.settings.Current(ctx).CarrierMode() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
state, err := d.store.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||||
|
return err == nil && state == InitStateDone
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
d.logger.Error("分发器 panic,已恢复", zap.Any("panic", r))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if !d.Ready(ctx) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.store.redis.HSet(ctx, constants.RedisCarrierSchedulerHeartbeatKey, d.instanceID, time.Now().UnixMilli())
|
||||||
|
d.drainLegacyHints(ctx)
|
||||||
|
d.refreshCarriers(ctx)
|
||||||
|
d.mu.Lock()
|
||||||
|
carriers := append([]carrierEntry(nil), d.carriers...)
|
||||||
|
d.mu.Unlock()
|
||||||
|
lockTTL := interval * 9 / 10
|
||||||
|
if lockTTL <= 0 {
|
||||||
|
lockTTL = 900 * time.Millisecond
|
||||||
|
}
|
||||||
|
for _, carrier := range carriers {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ok, err := d.store.redis.SetNX(ctx, constants.RedisCarrierDispatchLockKey(carrier.id), d.instanceID, lockTTL).Result()
|
||||||
|
if err != nil || !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d.dispatchCarrier(ctx, carrier)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) dispatchCarrier(ctx context.Context, carrier carrierEntry) {
|
||||||
|
params := d.ParamsFor(carrier.id)
|
||||||
|
limit := params.InflightLimit
|
||||||
|
// 执行上限不得大于订阅进程 max_in_flight 之和,否则领出的消息在 NSQ 中排队到租约过期;不满足时收紧本轮上限并上报。
|
||||||
|
processes := d.cfg.SubscriberProcesses
|
||||||
|
if processes <= 0 {
|
||||||
|
processes = 1
|
||||||
|
}
|
||||||
|
if capacity := params.CardMaxInFlight * processes; capacity > 0 && limit > capacity {
|
||||||
|
limit = capacity
|
||||||
|
d.reportTightened(carrier.id, params.InflightLimit, capacity)
|
||||||
|
}
|
||||||
|
input := ClaimInput{
|
||||||
|
CarrierID: carrier.id, Lease: d.lease(), InflightLimit: limit, PriorityRatio: params.PriorityRatio,
|
||||||
|
TokenPrefix: d.instanceID + ":" + strconv.FormatInt(time.Now().UnixNano(), 36),
|
||||||
|
CountRealname: carrier.realnameControl,
|
||||||
|
}
|
||||||
|
if d.budgets != nil {
|
||||||
|
ops := []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname}
|
||||||
|
if groups, opGroup, limited := d.budgets.Budget(ctx, carrier.id, ops); limited {
|
||||||
|
input.GroupBudget = groups
|
||||||
|
input.TypeGroup = map[string]string{
|
||||||
|
constants.CheckTypeCardStatus: opGroup[constants.GatewayOpQueryStatus],
|
||||||
|
constants.CheckTypeCarddata: opGroup[constants.GatewayOpQueryFlow],
|
||||||
|
constants.CheckTypeRealname: opGroup[constants.GatewayOpQueryRealname],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
claimed, expired, err := d.store.Claim(ctx, input)
|
||||||
|
if err != nil {
|
||||||
|
d.logger.Warn("运营商账户领取失败", zap.Uint("carrier_id", carrier.id), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if expired > 0 {
|
||||||
|
minute := strconv.FormatInt(time.Now().Unix()/60, 10)
|
||||||
|
key := constants.RedisCarrierExpiredClaimsKey(carrier.id)
|
||||||
|
d.store.redis.HIncrBy(ctx, key, minute, expired)
|
||||||
|
d.store.redis.Expire(ctx, key, 2*time.Hour)
|
||||||
|
d.logger.Warn("运营商账户处理中标记到期(执行崩溃或超时)", zap.Uint("carrier_id", carrier.id), zap.Int64("count", expired),
|
||||||
|
zap.String("metric", "polling.carrier.claim_expired"))
|
||||||
|
}
|
||||||
|
topic := constants.NSQCardTopic(carrier.id)
|
||||||
|
for _, item := range claimed {
|
||||||
|
body, _ := json.Marshal(CardMessage{CardID: item.CardID, CarrierID: carrier.id, Token: item.Token, DispatchedAt: time.Now().UnixMilli()})
|
||||||
|
if err := d.publisher.Publish(topic, body); err != nil {
|
||||||
|
// 发布失败:用独立短超时上下文按 token 删除处理中标记并恢复 score;恢复也失败则等租约到期后自然重新领取。
|
||||||
|
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||||
|
if releaseErr := d.store.Release(releaseCtx, item.CardID, carrier.id, item.Token); releaseErr != nil {
|
||||||
|
d.logger.Warn("发布失败后释放处理中标记失败", zap.Uint("card_id", item.CardID), zap.Error(releaseErr))
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
d.logger.Warn("发布卡任务失败,已释放", zap.Uint("card_id", item.CardID), zap.String("topic", topic), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) lease() time.Duration {
|
||||||
|
if d.cfg.Lease > 0 {
|
||||||
|
return d.cfg.Lease
|
||||||
|
}
|
||||||
|
return 5 * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) reportTightened(carrierID uint, configured, capacity int) {
|
||||||
|
d.mu.Lock()
|
||||||
|
last := d.tightened[carrierID]
|
||||||
|
if time.Since(last) < 10*time.Minute {
|
||||||
|
d.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d.tightened[carrierID] = time.Now()
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.logger.Warn("账户执行上限大于订阅进程 max_in_flight 之和,已按消费能力收紧本轮领取上限",
|
||||||
|
zap.Uint("carrier_id", carrierID), zap.Int("inflight_limit", configured), zap.Int("capacity", capacity),
|
||||||
|
zap.String("metric", "polling.carrier.inflight_tightened"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshCarriers 每 10 秒重新读取运营商账户与调度参数。
|
||||||
|
func (d *Dispatcher) refreshCarriers(ctx context.Context) {
|
||||||
|
d.mu.Lock()
|
||||||
|
fresh := time.Since(d.loadedAt) < 10*time.Second
|
||||||
|
d.mu.Unlock()
|
||||||
|
if fresh {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var carriers []model.Carrier
|
||||||
|
if err := d.db.WithContext(ctx).Select("id", "realname_control_enabled").Order("id").Find(&carriers).Error; err != nil {
|
||||||
|
d.logger.Warn("读取运营商账户失败,沿用上一次列表", zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var rows []model.CarrierPollingParam
|
||||||
|
if err := d.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||||
|
d.logger.Warn("读取运营商账户调度参数失败,沿用上一次参数", zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
entries := make([]carrierEntry, 0, len(carriers))
|
||||||
|
for _, carrier := range carriers {
|
||||||
|
entries = append(entries, carrierEntry{id: carrier.ID, realnameControl: carrier.RealnameControlEnabled == 1})
|
||||||
|
}
|
||||||
|
params := make(map[uint]model.CarrierPollingParam, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
params[row.CarrierID] = row
|
||||||
|
}
|
||||||
|
d.mu.Lock()
|
||||||
|
d.carriers, d.params, d.loadedAt = entries, params, time.Now()
|
||||||
|
d.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierIDs 返回当前缓存的运营商账户 ID(消费者据此订阅)。
|
||||||
|
func (d *Dispatcher) CarrierIDs(ctx context.Context) []uint {
|
||||||
|
d.refreshCarriers(ctx)
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
ids := make([]uint, 0, len(d.carriers))
|
||||||
|
for _, carrier := range d.carriers {
|
||||||
|
ids = append(ids, carrier.id)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParamsFor 返回账户生效调度参数(未配置项用全局默认)。
|
||||||
|
func (d *Dispatcher) ParamsFor(carrierID uint) CarrierParams {
|
||||||
|
d.mu.Lock()
|
||||||
|
row, ok := d.params[carrierID]
|
||||||
|
d.mu.Unlock()
|
||||||
|
return MergeParams(row, ok, d.cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeParams 合并账户配置行与全局默认。
|
||||||
|
func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.CarrierSchedulerConfig) CarrierParams {
|
||||||
|
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured}
|
||||||
|
if params.InflightLimit <= 0 {
|
||||||
|
params.InflightLimit = 20
|
||||||
|
}
|
||||||
|
if params.CardMaxInFlight <= 0 {
|
||||||
|
params.CardMaxInFlight = 20
|
||||||
|
}
|
||||||
|
if !configured {
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
if row.InflightLimit != nil && *row.InflightLimit > 0 {
|
||||||
|
params.InflightLimit = *row.InflightLimit
|
||||||
|
}
|
||||||
|
if row.CardMaxInFlight != nil && *row.CardMaxInFlight > 0 {
|
||||||
|
params.CardMaxInFlight = *row.CardMaxInFlight
|
||||||
|
}
|
||||||
|
if row.PriorityMaxRatio != nil && *row.PriorityMaxRatio > 0 {
|
||||||
|
params.PriorityRatio = *row.PriorityMaxRatio
|
||||||
|
}
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
|
||||||
|
// drainLegacyHints 消费旧优先提示通道与手动触发队列:优先提示放入账户优先队列,手动触发在普通队列中立即到期。
|
||||||
|
func (d *Dispatcher) drainLegacyHints(ctx context.Context) {
|
||||||
|
for _, checkType := range constants.AllCheckTypes() {
|
||||||
|
taskType := constants.TaskTypeFromCheckType(checkType)
|
||||||
|
d.drainList(ctx, constants.RedisPollingPriorityQueueKey(taskType), checkType, false)
|
||||||
|
d.drainList(ctx, constants.RedisPollingManualQueueKey(taskType), checkType, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Dispatcher) drainList(ctx context.Context, key, checkType string, normal bool) {
|
||||||
|
for i := 0; i < 500; i++ {
|
||||||
|
raw, err := d.store.redis.LPop(ctx, key).Result()
|
||||||
|
if err != nil {
|
||||||
|
if !stderrors.Is(err, redis.Nil) {
|
||||||
|
d.logger.Warn("读取旧提示队列失败", zap.String("key", key), zap.Error(err))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cardID, convErr := strconv.ParseUint(raw, 10, 64)
|
||||||
|
if convErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
due, dueErr := d.store.ReadDue(ctx, uint(cardID))
|
||||||
|
if dueErr != nil || due.CarrierID == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := d.store.RequestCheck(ctx, CheckRequest{CardID: uint(cardID), CarrierID: due.CarrierID, CheckTypes: []string{checkType}, Normal: normal}); err != nil {
|
||||||
|
d.logger.Warn("写入账户队列失败", zap.Uint64("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 新结构初始化状态(constants.RedisCarrierSchedulerInitStateKey)。
|
||||||
|
const (
|
||||||
|
InitStateRunning = "running"
|
||||||
|
InitStateDone = "done"
|
||||||
|
)
|
||||||
@@ -182,7 +182,8 @@ return 1
|
|||||||
// 只作用于仍在轮询中的卡(在优先或普通队列中)。冷却期内(最近成功距今未超过冷却)的类型不请求。
|
// 只作用于仍在轮询中的卡(在优先或普通队列中)。冷却期内(最近成功距今未超过冷却)的类型不请求。
|
||||||
//
|
//
|
||||||
// KEYS[1] due;KEYS[2] 优先队列;KEYS[3] 普通队列;KEYS[4] claim。
|
// KEYS[1] due;KEYS[2] 优先队列;KEYS[3] 普通队列;KEYS[4] claim。
|
||||||
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、类型…。
|
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、
|
||||||
|
// 是否放入优先队列(1/0;0 用于手动触发:只把到期设为现在并在普通队列中立即到期)、类型…。
|
||||||
// 返回本次实际请求的类型数;卡不在轮询中返回 -1。
|
// 返回本次实际请求的类型数;卡不在轮询中返回 -1。
|
||||||
const requestCheckLua = `
|
const requestCheckLua = `
|
||||||
local now = tonumber(ARGV[1])
|
local now = tonumber(ARGV[1])
|
||||||
@@ -192,7 +193,8 @@ local inPriority = redis.call('ZSCORE', KEYS[2], cardId)
|
|||||||
local inQueue = redis.call('ZSCORE', KEYS[3], cardId)
|
local inQueue = redis.call('ZSCORE', KEYS[3], cardId)
|
||||||
if not inPriority and not inQueue then return -1 end
|
if not inPriority and not inQueue then return -1 end
|
||||||
local requested = 0
|
local requested = 0
|
||||||
for i = 6, #ARGV do
|
local toPriority = ARGV[6] == '1'
|
||||||
|
for i = 7, #ARGV do
|
||||||
local t = ARGV[i]
|
local t = ARGV[i]
|
||||||
local skip = false
|
local skip = false
|
||||||
if cooldown > 0 then
|
if cooldown > 0 then
|
||||||
@@ -209,6 +211,11 @@ for i = 6, #ARGV do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
if requested == 0 then return 0 end
|
if requested == 0 then return 0 end
|
||||||
|
if not toPriority then
|
||||||
|
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
||||||
|
if inQueue then redis.call('ZADD', KEYS[3], now, cardId) else redis.call('ZADD', KEYS[2], now, cardId) end
|
||||||
|
return requested
|
||||||
|
end
|
||||||
redis.call('HSET', KEYS[1], 'prio', '1')
|
redis.call('HSET', KEYS[1], 'prio', '1')
|
||||||
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
|
||||||
redis.call('ZREM', KEYS[3], cardId)
|
redis.call('ZREM', KEYS[3], cardId)
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
@@ -376,3 +378,15 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
|||||||
}
|
}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewDefaultSettingsProvider 用独立注册表与读取器创建调度参数提供者(API 与 Worker 共用同一份数据库与缓存数据)。
|
||||||
|
func NewDefaultSettingsProvider(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *SettingsProvider {
|
||||||
|
registry := systemconfig.NewRegistry()
|
||||||
|
RegisterConfigDefinitions(registry, logger)
|
||||||
|
var cache systemconfig.Cache
|
||||||
|
if redisClient != nil {
|
||||||
|
cache = systemconfig.NewRedisCache(redisClient)
|
||||||
|
}
|
||||||
|
reader := systemconfig.NewReader(db, registry, cache, systemconfig.NewLogAlertSink(logger))
|
||||||
|
return NewSettingsProvider(reader, logger)
|
||||||
|
}
|
||||||
|
|||||||
@@ -136,6 +136,8 @@ type CheckRequest struct {
|
|||||||
Cooldown time.Duration
|
Cooldown time.Duration
|
||||||
ConfirmDelays []time.Duration
|
ConfirmDelays []time.Duration
|
||||||
Expected string // 延迟确认的期望结果(card_status 为 online / offline),满足时取消后续确认
|
Expected string // 延迟确认的期望结果(card_status 为 online / offline),满足时取消后续确认
|
||||||
|
// Normal 为 true 时不放入优先队列:只把到期设为现在、在普通队列中立即到期(手动触发走普通通道)。
|
||||||
|
Normal bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// RequestCheck 把卡的指定检查类型设为立即到期放入优先队列;返回实际请求的类型数,卡不在轮询中返回 -1。
|
// RequestCheck 把卡的指定检查类型设为立即到期放入优先队列;返回实际请求的类型数,卡不在轮询中返回 -1。
|
||||||
@@ -147,7 +149,11 @@ func (s *Store) RequestCheck(ctx context.Context, request CheckRequest) (int64,
|
|||||||
for _, delay := range request.ConfirmDelays {
|
for _, delay := range request.ConfirmDelays {
|
||||||
confirm = append(confirm, strconv.FormatInt(int64(delay/time.Second), 10))
|
confirm = append(confirm, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||||
}
|
}
|
||||||
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected}
|
toPriority := "1"
|
||||||
|
if request.Normal {
|
||||||
|
toPriority = "0"
|
||||||
|
}
|
||||||
|
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected, toPriority}
|
||||||
for _, checkType := range request.CheckTypes {
|
for _, checkType := range request.CheckTypes {
|
||||||
args = append(args, checkType)
|
args = append(args, checkType)
|
||||||
}
|
}
|
||||||
|
|||||||
161
internal/polling/check_requester.go
Normal file
161
internal/polling/check_requester.go
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
package polling
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sceneCacheTTL 是请求检查场景配置的进程内缓存时长。
|
||||||
|
const sceneCacheTTL = 30 * time.Second
|
||||||
|
|
||||||
|
// SceneConfig 是一个请求检查场景的生效配置。
|
||||||
|
type SceneConfig struct {
|
||||||
|
Scene string
|
||||||
|
CheckTypes []string
|
||||||
|
ConfirmDelays []time.Duration
|
||||||
|
Cooldown time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckRequester 按场景配置为卡请求检查(业务事件与读入口):把对应检查类型设为立即到期放入账户优先队列,
|
||||||
|
// 同卡多次请求合并、冷却期内不请求、按场景追加延迟确认。只在按运营商账户调度时生效。
|
||||||
|
type CheckRequester struct {
|
||||||
|
scheduling *CarrierScheduling
|
||||||
|
db *gorm.DB
|
||||||
|
logger *zap.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
scenes map[string]SceneConfig
|
||||||
|
loadedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCheckRequester 创建请求检查器。
|
||||||
|
func NewCheckRequester(scheduling *CarrierScheduling, db *gorm.DB, logger *zap.Logger) *CheckRequester {
|
||||||
|
if logger == nil {
|
||||||
|
logger = zap.NewNop()
|
||||||
|
}
|
||||||
|
return &CheckRequester{scheduling: scheduling, db: db, logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active 判断当前是否按运营商账户调度(请求检查生效)。
|
||||||
|
func (r *CheckRequester) Active(ctx context.Context) bool {
|
||||||
|
return r != nil && r.scheduling.Active(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestCards 为一组卡按场景请求检查;expected 为延迟确认的期望结果(online / offline,可空)。
|
||||||
|
// 失败只记日志,不影响业务。
|
||||||
|
func (r *CheckRequester) RequestCards(ctx context.Context, scene string, cardIDs []uint, expected string) {
|
||||||
|
if r == nil || len(cardIDs) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
config, ok := r.sceneConfig(ctx, scene)
|
||||||
|
if !ok || len(config.CheckTypes) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
carriers, err := r.cardCarriers(ctx, cardIDs)
|
||||||
|
if err != nil {
|
||||||
|
r.logger.Warn("请求检查:查询卡所属运营商账户失败", zap.String("scene", scene), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
store := r.scheduling.Store()
|
||||||
|
for _, cardID := range cardIDs {
|
||||||
|
carrierID, found := carriers[cardID]
|
||||||
|
if !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := store.RequestCheck(ctx, carrierqueue.CheckRequest{
|
||||||
|
CardID: cardID, CarrierID: carrierID, CheckTypes: config.CheckTypes, Cooldown: config.Cooldown,
|
||||||
|
ConfirmDelays: config.ConfirmDelays, Expected: expected,
|
||||||
|
}); err != nil {
|
||||||
|
r.logger.Warn("请求检查写入账户队列失败", zap.Uint("card_id", cardID), zap.String("scene", scene), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestDevice 为设备下全部有效绑定卡按场景请求检查。
|
||||||
|
func (r *CheckRequester) RequestDevice(ctx context.Context, scene string, deviceID uint) {
|
||||||
|
if r == nil || deviceID == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var cardIDs []uint
|
||||||
|
if err := r.db.WithContext(ctx).Model(&model.DeviceSimBinding{}).
|
||||||
|
Where("device_id = ? AND bind_status = ?", deviceID, constants.BindStatusBound).Pluck("iot_card_id", &cardIDs).Error; err != nil {
|
||||||
|
r.logger.Warn("请求检查:查询设备绑定卡失败", zap.Uint("device_id", deviceID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.RequestCards(ctx, scene, cardIDs, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *CheckRequester) cardCarriers(ctx context.Context, cardIDs []uint) (map[uint]uint, error) {
|
||||||
|
var cards []model.IotCard
|
||||||
|
if err := r.db.WithContext(ctx).Select("id", "carrier_id").Where("id IN ?", cardIDs).Find(&cards).Error; err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result := make(map[uint]uint, len(cards))
|
||||||
|
for _, card := range cards {
|
||||||
|
result[card.ID] = card.CarrierID
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sceneConfig 读取场景配置(缓存);场景未预置时不请求。
|
||||||
|
func (r *CheckRequester) sceneConfig(ctx context.Context, scene string) (SceneConfig, bool) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
if r.scenes == nil || time.Since(r.loadedAt) >= sceneCacheTTL {
|
||||||
|
var rows []model.PollingCheckScene
|
||||||
|
if err := r.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||||
|
r.logger.Warn("读取请求检查场景失败,沿用上一次配置", zap.Error(err))
|
||||||
|
} else {
|
||||||
|
scenes := make(map[string]SceneConfig, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
scenes[row.Scene] = ParseSceneConfig(row)
|
||||||
|
}
|
||||||
|
r.scenes, r.loadedAt = scenes, time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
config, ok := r.scenes[scene]
|
||||||
|
return config, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseSceneConfig 解析场景配置行。
|
||||||
|
func ParseSceneConfig(row model.PollingCheckScene) SceneConfig {
|
||||||
|
config := SceneConfig{Scene: row.Scene}
|
||||||
|
var types []string
|
||||||
|
_ = json.Unmarshal(row.CheckTypes, &types)
|
||||||
|
for _, checkType := range types {
|
||||||
|
if constants.CheckTypeCallsGateway(strings.TrimSpace(checkType)) {
|
||||||
|
config.CheckTypes = append(config.CheckTypes, strings.TrimSpace(checkType))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var delays []int
|
||||||
|
_ = json.Unmarshal(row.ConfirmDelays, &delays)
|
||||||
|
for _, seconds := range delays {
|
||||||
|
if seconds > 0 {
|
||||||
|
config.ConfirmDelays = append(config.ConfirmDelays, time.Duration(seconds)*time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if row.Cooldown != nil && *row.Cooldown > 0 {
|
||||||
|
config.Cooldown = time.Duration(*row.Cooldown) * time.Second
|
||||||
|
}
|
||||||
|
return config
|
||||||
|
}
|
||||||
|
|
||||||
|
// InvalidateScenes 使场景缓存立即失效(后台修改后调用)。
|
||||||
|
func (r *CheckRequester) InvalidateScenes() {
|
||||||
|
if r == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
r.loadedAt = time.Time{}
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
@@ -22,6 +22,18 @@ type PollingLifecycleService struct {
|
|||||||
deviceBindingStore *postgres.DeviceSimBindingStore
|
deviceBindingStore *postgres.DeviceSimBindingStore
|
||||||
deviceStore *postgres.DeviceStore
|
deviceStore *postgres.DeviceStore
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
|
// carrier 按运营商账户调度接缝;当前模式为 carrier 时生命周期事件写入账户队列,不写旧分片队列。
|
||||||
|
carrier *CarrierScheduling
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||||
|
func (s *PollingLifecycleService) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||||
|
s.carrier = carrier
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierActive 判断当前是否按运营商账户调度。
|
||||||
|
func (s *PollingLifecycleService) carrierActive(ctx context.Context) bool {
|
||||||
|
return s.carrier.Active(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPollingLifecycleService 创建卡生命周期轮询管理服务
|
// NewPollingLifecycleService 创建卡生命周期轮询管理服务
|
||||||
@@ -52,6 +64,10 @@ func (s *PollingLifecycleService) OnCardCreated(ctx context.Context, card *model
|
|||||||
s.logger.Debug("卡禁用轮询,跳过入队", zap.Uint("card_id", card.ID))
|
s.logger.Debug("卡禁用轮询,跳过入队", zap.Uint("card_id", card.ID))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.carrier.UpsertCard(ctx, card, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
s.enqueueCard(ctx, card)
|
s.enqueueCard(ctx, card)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,6 +80,11 @@ func (s *PollingLifecycleService) OnBatchCardsCreated(ctx context.Context, cards
|
|||||||
|
|
||||||
// OnCardStatusChanged 卡状态变化后清理缓存、重新匹配配置并更新队列
|
// OnCardStatusChanged 卡状态变化后清理缓存、重新匹配配置并更新队列
|
||||||
func (s *PollingLifecycleService) OnCardStatusChanged(ctx context.Context, cardID uint) {
|
func (s *PollingLifecycleService) OnCardStatusChanged(ctx context.Context, cardID uint) {
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
s.carrierRefresh(ctx, cardID)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
||||||
s.logger.Warn("卡状态变化:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
s.logger.Warn("卡状态变化:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
}
|
}
|
||||||
@@ -87,6 +108,13 @@ func (s *PollingLifecycleService) OnBatchCardsStatusChanged(ctx context.Context,
|
|||||||
if len(cardIDs) == 0 {
|
if len(cardIDs) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
for _, cardID := range cardIDs {
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
s.carrierRefresh(ctx, cardID)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
// 单次 pipeline:批量从当前分片队列移除并清理缓存
|
// 单次 pipeline:批量从当前分片队列移除并清理缓存
|
||||||
if err := s.queueMgr.RemoveBatchFromCurrentShardQueues(ctx, cardIDs); err != nil {
|
if err := s.queueMgr.RemoveBatchFromCurrentShardQueues(ctx, cardIDs); err != nil {
|
||||||
s.logger.Warn("批量卡状态变化:从队列移除失败", zap.Int("count", len(cardIDs)), zap.Error(err))
|
s.logger.Warn("批量卡状态变化:从队列移除失败", zap.Int("count", len(cardIDs)), zap.Error(err))
|
||||||
@@ -107,6 +135,11 @@ func (s *PollingLifecycleService) OnBatchCardsStatusChanged(ctx context.Context,
|
|||||||
|
|
||||||
// OnCardDeleted 卡删除后移除所有队列并清理缓存
|
// OnCardDeleted 卡删除后移除所有队列并清理缓存
|
||||||
func (s *PollingLifecycleService) OnCardDeleted(ctx context.Context, cardID uint) {
|
func (s *PollingLifecycleService) OnCardDeleted(ctx context.Context, cardID uint) {
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.carrier.RemoveCard(ctx, cardID, 0)
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.queueMgr.OnCardDeleted(ctx, cardID); err != nil {
|
if err := s.queueMgr.OnCardDeleted(ctx, cardID); err != nil {
|
||||||
s.logger.Warn("卡删除:清理队列和缓存失败", zap.Uint("card_id", cardID), zap.Error(err))
|
s.logger.Warn("卡删除:清理队列和缓存失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
}
|
}
|
||||||
@@ -123,11 +156,21 @@ func (s *PollingLifecycleService) OnCardEnabled(ctx context.Context, cardID uint
|
|||||||
if !s.ShouldEnqueue(ctx, card) {
|
if !s.ShouldEnqueue(ctx, card) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
// 恢复轮询时各检查类型到期设为当前时间加抖动。
|
||||||
|
s.carrier.UpsertCard(ctx, card, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
s.enqueueCard(ctx, card)
|
s.enqueueCard(ctx, card)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OnCardDisabled 卡禁用轮询后移除所有队列
|
// OnCardDisabled 卡禁用轮询后移除所有队列
|
||||||
func (s *PollingLifecycleService) OnCardDisabled(ctx context.Context, cardID uint) {
|
func (s *PollingLifecycleService) OnCardDisabled(ctx context.Context, cardID uint) {
|
||||||
|
if s.carrierActive(ctx) {
|
||||||
|
s.carrier.RemoveCard(ctx, cardID, 0)
|
||||||
|
s.queueMgr.InvalidateCardCache(ctx, cardID)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
if err := s.queueMgr.RemoveFromAllQueues(ctx, cardID); err != nil {
|
||||||
s.logger.Warn("卡禁用:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
s.logger.Warn("卡禁用:从队列移除失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
}
|
}
|
||||||
@@ -183,3 +226,17 @@ func calcInitialDelay(interval int) time.Time {
|
|||||||
}
|
}
|
||||||
return now.Add(time.Duration(rand.Intn(jitterMax)) * time.Second)
|
return now.Add(time.Duration(rand.Intn(jitterMax)) * time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// carrierRefresh 按运营商账户调度时重新匹配配置并写入账户队列:不再轮询的卡移出,其余按当前时间加抖动重排。
|
||||||
|
func (s *PollingLifecycleService) carrierRefresh(ctx context.Context, cardID uint) {
|
||||||
|
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("卡状态变化:加载卡信息失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.ShouldEnqueue(ctx, card) {
|
||||||
|
s.carrier.RemoveCard(ctx, card.ID, card.CarrierID)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.carrier.UpsertCard(ctx, card, false)
|
||||||
|
}
|
||||||
|
|||||||
@@ -32,6 +32,39 @@ type DueRecalculator struct {
|
|||||||
configMgr *PollingConfigManager
|
configMgr *PollingConfigManager
|
||||||
queueMgr *PollingQueueManager
|
queueMgr *PollingQueueManager
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
|
// carrier 按运营商账户调度接缝:carrier 模式下重算写入卡到期信息与账户队列。
|
||||||
|
carrier *CarrierScheduling
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||||
|
func (r *DueRecalculator) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||||
|
r.carrier = carrier
|
||||||
|
}
|
||||||
|
|
||||||
|
// recalculateCarrierBatch 按运营商账户调度时重算:各检查类型候选到期同旧结构口径,只提前不推后。
|
||||||
|
func (r *DueRecalculator) recalculateCarrierBatch(ctx context.Context, cards []*model.IotCard) (int, error) {
|
||||||
|
now := time.Now()
|
||||||
|
advanced := 0
|
||||||
|
for _, card := range cards {
|
||||||
|
for taskType, info := range r.configMgr.MergedTaskIntervals(card) {
|
||||||
|
checkType := constants.CheckTypeFromTaskType(taskType)
|
||||||
|
if checkType == "" || info.Interval <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
due := recalculatedDue(lastCheckAtByTaskType(card, taskType), info.Interval, now)
|
||||||
|
if windowDue := r.configMgr.NextCheckAt(ctx, card, taskType, info, now); windowDue.Before(due) && windowDue.After(now) {
|
||||||
|
due = windowDue
|
||||||
|
}
|
||||||
|
ok, err := r.carrier.AdvanceDue(ctx, card, checkType, due)
|
||||||
|
if err != nil {
|
||||||
|
return advanced, err
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
advanced++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return advanced, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDueRecalculator 创建重算器。
|
// NewDueRecalculator 创建重算器。
|
||||||
@@ -97,6 +130,9 @@ func (r *DueRecalculator) recalculateBatch(ctx context.Context, cards []*model.I
|
|||||||
if tier := r.configMgr.TierResolver(); tier != nil {
|
if tier := r.configMgr.TierResolver(); tier != nil {
|
||||||
tier.Prefill(ctx, cards)
|
tier.Prefill(ctx, cards)
|
||||||
}
|
}
|
||||||
|
if r.carrier.Active(ctx) {
|
||||||
|
return r.recalculateCarrierBatch(ctx, cards)
|
||||||
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
pipe := r.redis.Pipeline()
|
pipe := r.redis.Pipeline()
|
||||||
var cmds []*redis.IntCmd
|
var cmds []*redis.IntCmd
|
||||||
|
|||||||
@@ -30,6 +30,8 @@ type Scheduler struct {
|
|||||||
packageActivationHandler *PackageActivationHandler
|
packageActivationHandler *PackageActivationHandler
|
||||||
dataResetHandler *DataResetHandler
|
dataResetHandler *DataResetHandler
|
||||||
initializer *PollingInitializer // 可选,nil 表示不守卫 Init 完成
|
initializer *PollingInitializer // 可选,nil 表示不守卫 Init 完成
|
||||||
|
// carrier 按运营商账户调度接缝:当前模式为 carrier 时旧分片调度停止出队(套餐激活检查照常执行)。
|
||||||
|
carrier *CarrierScheduling
|
||||||
|
|
||||||
stopChan chan struct{}
|
stopChan chan struct{}
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
@@ -156,6 +158,10 @@ func (s *Scheduler) scheduleLoop(ctx context.Context) {
|
|||||||
// processShardSchedule 处理手动队列和分片定时队列(每 1 秒触发)
|
// processShardSchedule 处理手动队列和分片定时队列(每 1 秒触发)
|
||||||
// 使用 90% 的 tick 间隔作为超时,确保单分片 Redis 挂起时不阻塞下一个 tick
|
// 使用 90% 的 tick 间隔作为超时,确保单分片 Redis 挂起时不阻塞下一个 tick
|
||||||
func (s *Scheduler) processShardSchedule(ctx context.Context) {
|
func (s *Scheduler) processShardSchedule(ctx context.Context) {
|
||||||
|
if s.carrier.Active(ctx) {
|
||||||
|
// 已切到按运营商账户调度:旧分片调度停止出队,优先提示与手动触发由新分发器消费。
|
||||||
|
return
|
||||||
|
}
|
||||||
// 优先提示与手动队列都不受 Init 影响(ConfigManager 已就绪即可)。
|
// 优先提示与手动队列都不受 Init 影响(ConfigManager 已就绪即可)。
|
||||||
// 同一调度周期内先排空优先提示、再排空手动触发队列:优先的强度边界仅为调度入口领先,
|
// 同一调度周期内先排空优先提示、再排空手动触发队列:优先的强度边界仅为调度入口领先,
|
||||||
// 提示排空不参与分片背压判断,因此分片积压不会抑制加急入队。
|
// 提示排空不参与分片背压判断,因此分片积压不会抑制加急入队。
|
||||||
@@ -296,3 +302,8 @@ func (s *Scheduler) enqueueBatch(ctx context.Context, taskType string, cardIDs [
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetCarrierScheduling 注入按运营商账户调度接缝。
|
||||||
|
func (s *Scheduler) SetCarrierScheduling(carrier *CarrierScheduling) {
|
||||||
|
s.carrier = carrier
|
||||||
|
}
|
||||||
|
|||||||
@@ -136,6 +136,9 @@ func RegisterAdminRoutes(router fiber.Router, handlers *bootstrap.Handlers, midd
|
|||||||
if handlers.PollingConfig != nil {
|
if handlers.PollingConfig != nil {
|
||||||
registerPollingConfigRoutes(authGroup, handlers.PollingConfig, doc, basePath)
|
registerPollingConfigRoutes(authGroup, handlers.PollingConfig, doc, basePath)
|
||||||
}
|
}
|
||||||
|
if handlers.CarrierScheduler != nil {
|
||||||
|
registerCarrierSchedulerRoutes(authGroup, handlers.CarrierScheduler, doc, basePath)
|
||||||
|
}
|
||||||
if handlers.GatewayRateLimit != nil {
|
if handlers.GatewayRateLimit != nil {
|
||||||
registerGatewayRateLimitRoutes(authGroup, handlers.GatewayRateLimit, doc, basePath)
|
registerGatewayRateLimitRoutes(authGroup, handlers.GatewayRateLimit, doc, basePath)
|
||||||
}
|
}
|
||||||
|
|||||||
68
internal/routes/carrier_scheduler.go
Normal file
68
internal/routes/carrier_scheduler.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package routes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/handler/admin"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/openapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
// registerCarrierSchedulerRoutes 注册按运营商账户调度的后台接口。读取仅超级管理员与平台账号,修改仅超级管理员(服务层校验)。
|
||||||
|
func registerCarrierSchedulerRoutes(router fiber.Router, handler *admin.CarrierSchedulerHandler, doc *openapi.Generator, basePath string) {
|
||||||
|
tags := []string{"轮询调度"}
|
||||||
|
|
||||||
|
params := router.Group("/polling-carrier-params")
|
||||||
|
paramsPath := basePath + "/polling-carrier-params"
|
||||||
|
Register(params, doc, paramsPath, "GET", "", handler.ListParams, RouteSpec{
|
||||||
|
Summary: "查询运营商账户调度参数列表", Description: "未单独配置的账户返回全局默认并标记 is_default。", Tags: tags,
|
||||||
|
Input: nil, Output: new(dto.CarrierPollingParamListResponse), Auth: true,
|
||||||
|
})
|
||||||
|
Register(params, doc, paramsPath, "GET", "/:carrier_id", handler.GetParam, RouteSpec{
|
||||||
|
Summary: "查询运营商账户调度参数", Tags: tags, Input: new(dto.CarrierPollingParamPath), Output: new(dto.CarrierPollingParamItem), Auth: true,
|
||||||
|
})
|
||||||
|
Register(params, doc, paramsPath, "PUT", "/:carrier_id", handler.SaveParam, RouteSpec{
|
||||||
|
Summary: "保存运营商账户调度参数(仅超级管理员)",
|
||||||
|
Description: "字段为空表示该项使用全局默认;执行上限不得大于当前订阅进程 max_in_flight 之和;下一轮调度生效并写统一审计。",
|
||||||
|
Tags: tags, Input: new(dto.SaveCarrierPollingParamRequest), Output: new(dto.CarrierPollingParamItem), Auth: true,
|
||||||
|
})
|
||||||
|
Register(params, doc, paramsPath, "DELETE", "/:carrier_id", handler.DeleteParam, RouteSpec{
|
||||||
|
Summary: "删除运营商账户调度参数(恢复全局默认,仅超级管理员)", Tags: tags, Input: new(dto.CarrierPollingParamPath), Output: nil, Auth: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
scenes := router.Group("/polling-check-scenes")
|
||||||
|
scenesPath := basePath + "/polling-check-scenes"
|
||||||
|
Register(scenes, doc, scenesPath, "GET", "", handler.ListScenes, RouteSpec{
|
||||||
|
Summary: "查询请求检查场景", Description: "业务事件与读入口在按运营商账户调度时按场景请求检查;场景由代码预置。", Tags: tags,
|
||||||
|
Input: nil, Output: new([]dto.PollingCheckSceneItem), Auth: true,
|
||||||
|
})
|
||||||
|
Register(scenes, doc, scenesPath, "PUT", "/:scene", handler.UpdateScene, RouteSpec{
|
||||||
|
Summary: "修改请求检查场景(仅超级管理员)", Description: "只可修改检查类型、延迟确认序列(递增)与冷却时长。", Tags: tags,
|
||||||
|
Input: new(dto.UpdatePollingCheckSceneRequest), Output: new(dto.PollingCheckSceneItem), Auth: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
alerts := router.Group("/polling-failure-alerts")
|
||||||
|
alertsPath := basePath + "/polling-failure-alerts"
|
||||||
|
Register(alerts, doc, alertsPath, "GET", "", handler.ListAlerts, RouteSpec{
|
||||||
|
Summary: "查询卡轮询失败告警列表", Tags: tags, Input: new(dto.PollingFailureAlertListRequest), Output: new(dto.PollingFailureAlertPageResult), Auth: true,
|
||||||
|
})
|
||||||
|
Register(alerts, doc, alertsPath, "GET", "/summary", handler.AlertSummary, RouteSpec{
|
||||||
|
Summary: "未处理卡轮询失败告警按运营商账户与错误类别汇总", Tags: tags, Input: nil, Output: new([]dto.PollingFailureAlertSummaryItem), Auth: true,
|
||||||
|
})
|
||||||
|
Register(alerts, doc, alertsPath, "GET", "/:id", handler.GetAlert, RouteSpec{
|
||||||
|
Summary: "查询卡轮询失败告警详情", Tags: tags, Input: new(dto.PollingFailureAlertIDPath), Output: new(dto.PollingFailureAlertItem), Auth: true,
|
||||||
|
})
|
||||||
|
Register(alerts, doc, alertsPath, "PUT", "/:id/status", handler.HandleAlert, RouteSpec{
|
||||||
|
Summary: "标记卡轮询失败告警已处理", Description: "只有未处理告警可标记;关闭该卡轮询请使用资产轮询开关接口。", Tags: tags,
|
||||||
|
Input: new(dto.HandlePollingFailureAlertRequest), Output: nil, Auth: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
Register(router, doc, basePath, "GET", "/iot-cards/:id/polling-state", handler.CardState, RouteSpec{
|
||||||
|
Summary: "查询单卡轮询状态", Description: "各检查类型下次到期、最近成功、连续失败与最近错误、是否执行中;只读。", Tags: tags,
|
||||||
|
Input: new(dto.CardPollingStatePath), Output: new(dto.CardPollingStateResponse), Auth: true,
|
||||||
|
})
|
||||||
|
Register(router, doc, basePath, "GET", "/polling-stats/carrier-queues", handler.CarrierStats, RouteSpec{
|
||||||
|
Summary: "按运营商账户的轮询队列与执行监控", Description: "按运营商账户调度的各账户队列长度、已到期数、执行中、上一分钟 QPS、限流等待与拒绝、处理中标记到期、未处理告警。", Tags: tags,
|
||||||
|
Input: nil, Output: new(dto.CarrierPollingStatsResponse), Auth: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
622
internal/service/polling/carrier_scheduler_service.go
Normal file
622
internal/service/polling/carrier_scheduler_service.go
Normal file
@@ -0,0 +1,622 @@
|
|||||||
|
package polling
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
stderrors "errors"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"gorm.io/datatypes"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
|
||||||
|
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。
|
||||||
|
// 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。
|
||||||
|
type CarrierSchedulerService struct {
|
||||||
|
db *gorm.DB
|
||||||
|
redis *redis.Client
|
||||||
|
auditWriter *auditinfra.Writer
|
||||||
|
store *carrierqueue.Store
|
||||||
|
settings *carrierqueue.SettingsProvider
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierSchedulerService 创建按运营商账户调度后台服务。
|
||||||
|
func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService {
|
||||||
|
return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient),
|
||||||
|
settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CarrierSchedulerService) processes() int {
|
||||||
|
if s.cfg.SubscriberProcesses > 0 {
|
||||||
|
return s.cfg.SubscriberProcesses
|
||||||
|
}
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。
|
||||||
|
func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names, err := s.carrierNameMap(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var rows []model.CarrierPollingParam
|
||||||
|
if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||||||
|
}
|
||||||
|
byCarrier := make(map[uint]model.CarrierPollingParam, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
byCarrier[row.CarrierID] = row
|
||||||
|
}
|
||||||
|
defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg)
|
||||||
|
resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(),
|
||||||
|
DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}}
|
||||||
|
for _, id := range sortedCarrierIDs(names) {
|
||||||
|
row, configured := byCarrier[id]
|
||||||
|
resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured))
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem {
|
||||||
|
merged := carrierqueue.MergeParams(row, configured, s.cfg)
|
||||||
|
item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured,
|
||||||
|
InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()}
|
||||||
|
if configured {
|
||||||
|
item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit
|
||||||
|
item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay
|
||||||
|
item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight
|
||||||
|
}
|
||||||
|
return item
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetParam 查询单个运营商账户调度参数。
|
||||||
|
func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
carrier, err := s.loadCarrier(ctx, carrierID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
row, configured, err := s.loadParam(ctx, carrierID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
item := s.paramItem(carrierID, carrier.CarrierName, row, configured)
|
||||||
|
return &item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。
|
||||||
|
func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) {
|
||||||
|
operatorID, err := requireSuperAdminWriter(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
carrier, err := s.loadCarrier(ctx, req.CarrierID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
row, configured, err := s.loadParam(ctx, req.CarrierID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
before := paramState(row, configured)
|
||||||
|
row.CarrierID = req.CarrierID
|
||||||
|
row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight
|
||||||
|
row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio
|
||||||
|
row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay
|
||||||
|
merged := carrierqueue.MergeParams(row, true, s.cfg)
|
||||||
|
if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity {
|
||||||
|
return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+
|
||||||
|
strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight")
|
||||||
|
}
|
||||||
|
if !configured {
|
||||||
|
row.Creator = operatorID
|
||||||
|
}
|
||||||
|
row.Updater = operatorID
|
||||||
|
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||||
|
if err := tx.WithContext(ctx).Save(&row).Error; err != nil {
|
||||||
|
return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败")
|
||||||
|
}
|
||||||
|
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||||
|
ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数",
|
||||||
|
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID,
|
||||||
|
ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID,
|
||||||
|
IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit,
|
||||||
|
"card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio},
|
||||||
|
BeforeData: before, AfterData: paramState(row, true),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true)
|
||||||
|
return &item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。
|
||||||
|
func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error {
|
||||||
|
operatorID, err := requireSuperAdminWriter(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
row, configured, err := s.loadParam(ctx, carrierID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !configured {
|
||||||
|
return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数")
|
||||||
|
}
|
||||||
|
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||||
|
if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil {
|
||||||
|
return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败")
|
||||||
|
}
|
||||||
|
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||||
|
ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)",
|
||||||
|
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID,
|
||||||
|
ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID,
|
||||||
|
IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func paramState(row model.CarrierPollingParam, configured bool) map[string]any {
|
||||||
|
if !configured {
|
||||||
|
return map[string]any{"configured": false}
|
||||||
|
}
|
||||||
|
return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight,
|
||||||
|
"stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit,
|
||||||
|
"priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval,
|
||||||
|
"stop_resume_confirm_delay": row.StopResumeConfirmDelay}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) {
|
||||||
|
var row model.CarrierPollingParam
|
||||||
|
err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error
|
||||||
|
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return model.CarrierPollingParam{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||||||
|
}
|
||||||
|
return row, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) {
|
||||||
|
var carrier model.Carrier
|
||||||
|
if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil {
|
||||||
|
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return nil, errors.New(errors.CodeNotFound, "运营商账户不存在")
|
||||||
|
}
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
|
||||||
|
}
|
||||||
|
return &carrier, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) {
|
||||||
|
var carriers []model.Carrier
|
||||||
|
if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
|
||||||
|
}
|
||||||
|
names := make(map[uint]string, len(carriers))
|
||||||
|
for _, carrier := range carriers {
|
||||||
|
names[carrier.ID] = carrier.CarrierName
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListScenes 列出请求检查场景。
|
||||||
|
func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var rows []model.PollingCheckScene
|
||||||
|
if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
|
||||||
|
}
|
||||||
|
items := make([]dto.PollingCheckSceneItem, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
items = append(items, sceneItem(row))
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem {
|
||||||
|
item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt,
|
||||||
|
CheckTypes: []string{}, ConfirmDelays: []int{}}
|
||||||
|
_ = json.Unmarshal(row.CheckTypes, &item.CheckTypes)
|
||||||
|
_ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays)
|
||||||
|
return item
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。
|
||||||
|
func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) {
|
||||||
|
operatorID, err := requireSuperAdminWriter(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for index := 1; index < len(req.ConfirmDelays); index++ {
|
||||||
|
if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] {
|
||||||
|
return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
types := uniqueStrings(req.CheckTypes)
|
||||||
|
var row model.PollingCheckScene
|
||||||
|
if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil {
|
||||||
|
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在")
|
||||||
|
}
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
|
||||||
|
}
|
||||||
|
before := sceneItem(row)
|
||||||
|
typesJSON, _ := json.Marshal(types)
|
||||||
|
delays := req.ConfirmDelays
|
||||||
|
if delays == nil {
|
||||||
|
delays = []int{}
|
||||||
|
}
|
||||||
|
delaysJSON, _ := json.Marshal(delays)
|
||||||
|
row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID
|
||||||
|
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||||
|
if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{
|
||||||
|
"check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(),
|
||||||
|
}).Error; err != nil {
|
||||||
|
return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败")
|
||||||
|
}
|
||||||
|
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||||
|
ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景",
|
||||||
|
ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID,
|
||||||
|
ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID,
|
||||||
|
IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName},
|
||||||
|
BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown},
|
||||||
|
AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
item := sceneItem(row)
|
||||||
|
return &item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAlerts 分页查询卡轮询失败告警。
|
||||||
|
func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
page, size := normalizePage(req.Page, req.PageSize)
|
||||||
|
query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{})
|
||||||
|
if req.CarrierID != nil {
|
||||||
|
query = query.Where("carrier_id = ?", *req.CarrierID)
|
||||||
|
}
|
||||||
|
if req.CheckType != "" {
|
||||||
|
query = query.Where("check_type = ?", req.CheckType)
|
||||||
|
}
|
||||||
|
if req.ErrorCategory != "" {
|
||||||
|
query = query.Where("error_category = ?", req.ErrorCategory)
|
||||||
|
}
|
||||||
|
if req.Status != nil {
|
||||||
|
query = query.Where("status = ?", *req.Status)
|
||||||
|
}
|
||||||
|
if req.AlertLevel != "" {
|
||||||
|
query = query.Where("alert_level = ?", req.AlertLevel)
|
||||||
|
}
|
||||||
|
if req.ICCID != "" {
|
||||||
|
query = query.Where("iccid = ?", req.ICCID)
|
||||||
|
}
|
||||||
|
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if start != nil {
|
||||||
|
query = query.Where("last_failed_at >= ?", *start)
|
||||||
|
}
|
||||||
|
if end != nil {
|
||||||
|
query = query.Where("last_failed_at <= ?", *end)
|
||||||
|
}
|
||||||
|
var total int64
|
||||||
|
if err := query.Count(&total).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
|
||||||
|
}
|
||||||
|
var rows []model.PollingFailureAlert
|
||||||
|
if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
|
||||||
|
}
|
||||||
|
names, err := s.carrierNameMap(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
|
||||||
|
for _, row := range rows {
|
||||||
|
result.List = append(result.List, alertItem(row, names))
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem {
|
||||||
|
return dto.PollingFailureAlertItem{
|
||||||
|
ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
|
||||||
|
CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType),
|
||||||
|
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
|
||||||
|
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
|
||||||
|
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。
|
||||||
|
func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var rows []struct {
|
||||||
|
CarrierID uint
|
||||||
|
ErrorCategory string
|
||||||
|
OpenCount int64
|
||||||
|
}
|
||||||
|
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).
|
||||||
|
Select("carrier_id, error_category, COUNT(*) AS open_count").
|
||||||
|
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
|
||||||
|
Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败")
|
||||||
|
}
|
||||||
|
names, err := s.carrierNameMap(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID],
|
||||||
|
ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount})
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAlert 查询告警详情。
|
||||||
|
func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
row, err := s.loadAlert(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names, err := s.carrierNameMap(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
item := alertItem(*row, names)
|
||||||
|
return &item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleAlert 把未处理告警标记为已处理。
|
||||||
|
func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error {
|
||||||
|
operatorID, err := requireSuperAdminWriterOrPlatform(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
row, err := s.loadAlert(ctx, req.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if row.Status != constants.PollingAlertStatusOpen {
|
||||||
|
return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理")
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||||||
|
result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen).
|
||||||
|
Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now})
|
||||||
|
if result.Error != nil {
|
||||||
|
return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败")
|
||||||
|
}
|
||||||
|
if result.RowsAffected == 0 {
|
||||||
|
return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试")
|
||||||
|
}
|
||||||
|
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||||||
|
ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理",
|
||||||
|
ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID,
|
||||||
|
ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID,
|
||||||
|
IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID,
|
||||||
|
"carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status},
|
||||||
|
BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) {
|
||||||
|
var row model.PollingFailureAlert
|
||||||
|
if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil {
|
||||||
|
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在")
|
||||||
|
}
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
|
||||||
|
}
|
||||||
|
return &row, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CardState 查询单卡轮询状态(只读)。
|
||||||
|
func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var card model.IotCard
|
||||||
|
if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil {
|
||||||
|
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return nil, errors.New(errors.CodeNotFound, "IoT卡不存在")
|
||||||
|
}
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败")
|
||||||
|
}
|
||||||
|
resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
|
||||||
|
SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}}
|
||||||
|
due, err := s.store.ReadDue(ctx, card.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败")
|
||||||
|
}
|
||||||
|
carrierID := due.CarrierID
|
||||||
|
if carrierID == 0 {
|
||||||
|
carrierID = card.CarrierID
|
||||||
|
}
|
||||||
|
member := strconv.FormatUint(uint64(card.ID), 10)
|
||||||
|
if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil {
|
||||||
|
resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score))
|
||||||
|
} else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil {
|
||||||
|
resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score))
|
||||||
|
}
|
||||||
|
if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 {
|
||||||
|
resp.Running = true
|
||||||
|
expire := time.Now().Add(ttl)
|
||||||
|
resp.ClaimExpireAt = &expire
|
||||||
|
}
|
||||||
|
resp.PriorityMark = due.Priority
|
||||||
|
for _, checkType := range constants.AllCheckTypes() {
|
||||||
|
item, ok := due.Types[checkType]
|
||||||
|
if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType),
|
||||||
|
FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}}
|
||||||
|
if item.HasDue {
|
||||||
|
state.NextDueAt = timePtr(item.Due)
|
||||||
|
}
|
||||||
|
if !item.LastOK.IsZero() {
|
||||||
|
state.LastSuccessAt = timePtr(item.LastOK)
|
||||||
|
}
|
||||||
|
if !item.FirstFailed.IsZero() {
|
||||||
|
state.FirstFailedAt = timePtr(item.FirstFailed)
|
||||||
|
}
|
||||||
|
for _, delay := range item.Confirm {
|
||||||
|
state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second))
|
||||||
|
}
|
||||||
|
resp.Types = append(resp.Types, state)
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarrierStats 按运营商账户的轮询监控。
|
||||||
|
func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names, err := s.carrierNameMap(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var params []model.CarrierPollingParam
|
||||||
|
if err := s.db.WithContext(ctx).Find(¶ms).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||||||
|
}
|
||||||
|
paramByCarrier := map[uint]model.CarrierPollingParam{}
|
||||||
|
for _, row := range params {
|
||||||
|
paramByCarrier[row.CarrierID] = row
|
||||||
|
}
|
||||||
|
var alertRows []struct {
|
||||||
|
CarrierID uint
|
||||||
|
ErrorCategory string
|
||||||
|
Count int64
|
||||||
|
}
|
||||||
|
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count").
|
||||||
|
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
|
||||||
|
Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil {
|
||||||
|
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
|
||||||
|
}
|
||||||
|
state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||||||
|
resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}}
|
||||||
|
now := time.Now()
|
||||||
|
nowMS := strconv.FormatInt(now.UnixMilli(), 10)
|
||||||
|
lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504")
|
||||||
|
for _, carrierID := range sortedCarrierIDs(names) {
|
||||||
|
row, configured := paramByCarrier[carrierID]
|
||||||
|
item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID],
|
||||||
|
InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}}
|
||||||
|
pipe := s.redis.Pipeline()
|
||||||
|
priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID))
|
||||||
|
queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID))
|
||||||
|
priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS)
|
||||||
|
queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS)
|
||||||
|
running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf")
|
||||||
|
expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID))
|
||||||
|
var statCmds []*redis.MapStringStringCmd
|
||||||
|
for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} {
|
||||||
|
statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute)))
|
||||||
|
}
|
||||||
|
_, _ = pipe.Exec(ctx)
|
||||||
|
item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val()
|
||||||
|
item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val()
|
||||||
|
cutoff := now.Add(-time.Hour).Unix() / 60
|
||||||
|
for minute, value := range expired.Val() {
|
||||||
|
if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff {
|
||||||
|
count, _ := strconv.ParseInt(value, 10, 64)
|
||||||
|
item.ExpiredClaims1h += count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var attempts int64
|
||||||
|
for _, cmd := range statCmds {
|
||||||
|
values := cmd.Val()
|
||||||
|
attempts += atoi64(values["attempts"])
|
||||||
|
item.RateLimitWaits += atoi64(values["rate_wait"])
|
||||||
|
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
|
||||||
|
}
|
||||||
|
item.QPSLastMinute = float64(attempts) / 60
|
||||||
|
for _, alert := range alertRows {
|
||||||
|
if alert.CarrierID == carrierID {
|
||||||
|
item.OpenAlerts += alert.Count
|
||||||
|
item.OpenAlertsByType[alert.ErrorCategory] += alert.Count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resp.Items = append(resp.Items, item)
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。
|
||||||
|
func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) {
|
||||||
|
if err := requirePlatformReader(ctx); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
operatorID := middleware.GetUserIDFromContext(ctx)
|
||||||
|
if operatorID == 0 {
|
||||||
|
return 0, errors.New(errors.CodeUnauthorized, "未授权访问")
|
||||||
|
}
|
||||||
|
return operatorID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedCarrierIDs(names map[uint]string) []uint {
|
||||||
|
ids := make([]uint, 0, len(names))
|
||||||
|
for id := range names {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
for i := 1; i < len(ids); i++ {
|
||||||
|
for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
|
||||||
|
ids[j], ids[j-1] = ids[j-1], ids[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func msTime(ms int64) *time.Time {
|
||||||
|
t := time.UnixMilli(ms)
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
|
||||||
|
func timePtr(t time.Time) *time.Time {
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
|
||||||
|
func atoi64(s string) int64 {
|
||||||
|
value, _ := strconv.ParseInt(s, 10, 64)
|
||||||
|
return value
|
||||||
|
}
|
||||||
299
internal/task/carrier_card_executor.go
Normal file
299
internal/task/carrier_card_executor.go
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
package task
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CarrierCardExecutor 按运营商账户调度的卡任务执行器:一张卡一个卡任务,按配置步骤顺序串行执行已到期的检查类型。
|
||||||
|
type CarrierCardExecutor struct {
|
||||||
|
base *PollingBase
|
||||||
|
store *carrierqueue.Store
|
||||||
|
settings *carrierqueue.SettingsProvider
|
||||||
|
alerts *carrierqueue.AlertWriter
|
||||||
|
realname *PollingRealnameHandler
|
||||||
|
carddata *PollingCarddataHandler
|
||||||
|
cardStatus *PollingCardStatusHandler
|
||||||
|
pkg *PollingPackageHandler
|
||||||
|
cfg config.CarrierSchedulerConfig
|
||||||
|
instanceID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCarrierCardExecutor 创建卡任务执行器。
|
||||||
|
func NewCarrierCardExecutor(base *PollingBase, store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, alerts *carrierqueue.AlertWriter,
|
||||||
|
realname *PollingRealnameHandler, carddata *PollingCarddataHandler, cardStatus *PollingCardStatusHandler, pkg *PollingPackageHandler,
|
||||||
|
cfg config.CarrierSchedulerConfig, instanceID string) *CarrierCardExecutor {
|
||||||
|
return &CarrierCardExecutor{base: base, store: store, settings: settings, alerts: alerts, realname: realname, carddata: carddata,
|
||||||
|
cardStatus: cardStatus, pkg: pkg, cfg: cfg, instanceID: instanceID}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CarrierCardExecutor) renewInterval() time.Duration {
|
||||||
|
if e.cfg.RenewInterval > 0 {
|
||||||
|
return e.cfg.RenewInterval
|
||||||
|
}
|
||||||
|
return 30 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CarrierCardExecutor) lease() time.Duration {
|
||||||
|
if e.cfg.Lease > 0 {
|
||||||
|
return e.cfg.Lease
|
||||||
|
}
|
||||||
|
return 5 * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle 消费一条卡任务消息;返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
|
||||||
|
func (e *CarrierCardExecutor) Handle(ctx context.Context, body []byte, touch func()) {
|
||||||
|
var msg carrierqueue.CardMessage
|
||||||
|
if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
|
||||||
|
e.base.logger.Warn("卡任务消息无效,丢弃", zap.ByteString("body", body))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
execValue := msg.Token + "|" + e.instanceID
|
||||||
|
execTTL := e.renewInterval() * 2
|
||||||
|
got, err := e.store.AcquireExec(ctx, msg.CardID, msg.Token, execValue, execTTL)
|
||||||
|
if err != nil || got != 1 {
|
||||||
|
// 0:处理中标记已不是该 token(过期或重复投递);-1:该 token 已有执行者(NSQ 重投时原消费者仍在执行)。
|
||||||
|
return
|
||||||
|
}
|
||||||
|
settings := e.settings.Current(ctx)
|
||||||
|
taskCtx, cancel := context.WithTimeout(ctx, settings.TaskTimeout())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var lost atomic.Bool
|
||||||
|
renewDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(e.renewInterval())
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-renewDone:
|
||||||
|
return
|
||||||
|
case <-taskCtx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
ok, renewErr := e.store.Renew(context.WithoutCancel(ctx), msg.CardID, msg.CarrierID, msg.Token, execValue, e.lease(), execTTL)
|
||||||
|
if renewErr == nil && !ok {
|
||||||
|
// token 已不一致:立即停止后续步骤,不再发起新的网关调用,只写已产生的业务结果。
|
||||||
|
lost.Store(true)
|
||||||
|
cancel()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if touch != nil {
|
||||||
|
touch()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
set, del := e.execute(taskCtx, msg, settings, &lost)
|
||||||
|
close(renewDone)
|
||||||
|
|
||||||
|
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||||
|
defer finishCancel()
|
||||||
|
finished, finishErr := e.store.Finish(finishCtx, msg.CardID, msg.CarrierID, msg.Token, set, del)
|
||||||
|
if finishErr != nil {
|
||||||
|
e.base.logger.Warn("卡任务收尾失败,等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(finishErr))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !finished {
|
||||||
|
e.base.logger.Warn("卡任务收尾时处理中标记已不一致,只写入业务结果", zap.Uint("card_id", msg.CardID),
|
||||||
|
zap.String("metric", "polling.carrier.finish_token_mismatch"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// execute 串行执行到期检查步骤,返回收尾要写入与删除的到期字段。
|
||||||
|
func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.CardMessage, settings carrierqueue.Settings, lost *atomic.Bool) (map[string]string, []string) {
|
||||||
|
set, del := map[string]string{}, []string{}
|
||||||
|
card, err := e.base.getCardWithCache(ctx, msg.CardID)
|
||||||
|
if err != nil {
|
||||||
|
if isNotFound(err) {
|
||||||
|
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||||
|
}
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
if !card.EnablePolling || (e.base.scopeChecker != nil && !e.base.scopeChecker.ShouldEnqueue(ctx, card)) {
|
||||||
|
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
due, err := e.store.ReadDue(ctx, msg.CardID)
|
||||||
|
if err != nil {
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
flowEvaluated := false
|
||||||
|
executed := map[string]StepOutcome{}
|
||||||
|
for _, checkType := range settings.StepOrder {
|
||||||
|
item, ok := due.Types[checkType]
|
||||||
|
if !ok || !item.HasDue || item.Due.After(now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if lost.Load() || ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow && flowEvaluated {
|
||||||
|
executed[checkType] = stepOK()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
outcome := e.runStep(ctx, msg, checkType, settings.StepTimeout(checkType))
|
||||||
|
if checkType == constants.CheckTypeCarddata && outcome.TriggeredEvaluation {
|
||||||
|
flowEvaluated = true
|
||||||
|
}
|
||||||
|
executed[checkType] = outcome
|
||||||
|
if outcome.Result == StepCardGone || outcome.Result == StepStopPolling {
|
||||||
|
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(executed) == 0 {
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
// 收尾时按最新卡数据重新匹配间隔(步骤中卡状态、套餐可能已变化)。
|
||||||
|
e.base.invalidateCardCache(ctx, msg.CardID)
|
||||||
|
if fresh, freshErr := e.base.getCardWithCache(ctx, msg.CardID); freshErr == nil {
|
||||||
|
card = fresh
|
||||||
|
}
|
||||||
|
intervals := e.base.configMgr.MergedTaskIntervals(card)
|
||||||
|
now = time.Now()
|
||||||
|
for checkType, outcome := range executed {
|
||||||
|
e.writeBack(ctx, card, checkType, outcome, due.Types[checkType], intervals, settings, now, set, &del)
|
||||||
|
}
|
||||||
|
return set, del
|
||||||
|
}
|
||||||
|
|
||||||
|
// runStep 以单步超时、等待模式限流与 panic 恢复执行一个检查步骤。
|
||||||
|
func (e *CarrierCardExecutor) runStep(ctx context.Context, msg carrierqueue.CardMessage, checkType string, timeout time.Duration) (outcome StepOutcome) {
|
||||||
|
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
stepCtx = gateway.WithRateLimitWait(stepCtx, timeout)
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
e.base.logger.Error("卡任务步骤 panic,按失败处理", zap.Uint("card_id", msg.CardID), zap.String("check_type", checkType), zap.Any("panic", r))
|
||||||
|
outcome = StepOutcome{Result: StepFailed, Category: "panic", Message: fmt.Sprint(r)}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
switch checkType {
|
||||||
|
case constants.CheckTypeCardStatus:
|
||||||
|
return e.cardStatus.Step(stepCtx, msg.CardID)
|
||||||
|
case constants.CheckTypeCarddata:
|
||||||
|
return e.carddata.Step(stepCtx, msg.CardID)
|
||||||
|
case constants.CheckTypeRealname:
|
||||||
|
return e.realname.Step(stepCtx, msg.CardID)
|
||||||
|
case constants.CheckTypePackage:
|
||||||
|
stepCtx = withPollingWorkerAuditContext(stepCtx, constants.TaskTypePollingPackage, "套餐状态轮询任务", msg.Token)
|
||||||
|
return e.pkg.Step(stepCtx, msg.CardID)
|
||||||
|
default:
|
||||||
|
return stepSkip()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeBack 按步骤结果计算该检查类型下次到期、失败计数与延迟确认,并触发告警或恢复。
|
||||||
|
func (e *CarrierCardExecutor) writeBack(ctx context.Context, card *model.IotCard, checkType string, outcome StepOutcome, previous carrierqueue.TypeDue,
|
||||||
|
intervals map[string]polling.TaskTypeInterval, settings carrierqueue.Settings, now time.Time, set map[string]string, del *[]string) {
|
||||||
|
info, ok := intervals[constants.TaskTypeFromCheckType(checkType)]
|
||||||
|
if !ok || info.Interval <= 0 {
|
||||||
|
// 该类型已不在配置中(间隔为 0 或未配置):移除到期,不再执行。
|
||||||
|
*del = append(*del, carrierqueue.FieldDue(checkType), carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType),
|
||||||
|
carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
interval := time.Duration(info.Interval) * time.Second
|
||||||
|
normalNext := e.base.configMgr.NextCheckAt(ctx, card, constants.TaskTypeFromCheckType(checkType), info, now)
|
||||||
|
switch outcome.Result {
|
||||||
|
case StepSucceeded:
|
||||||
|
next := normalNext
|
||||||
|
set[carrierqueue.FieldOK(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||||
|
if previous.FailCount > 0 {
|
||||||
|
e.alerts.ReportRecovered(card.ID, checkType)
|
||||||
|
}
|
||||||
|
*del = append(*del, carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType), carrierqueue.FieldError(checkType))
|
||||||
|
// 延迟确认:结果已满足预期时取消,否则按序列取下一次确认 = min(正常间隔, 当前 + 确认延迟)。
|
||||||
|
if len(previous.Confirm) > 0 {
|
||||||
|
if previous.Expected != "" && outcome.Observed == previous.Expected {
|
||||||
|
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||||
|
} else {
|
||||||
|
if confirmAt := now.Add(previous.Confirm[0]); confirmAt.Before(next) {
|
||||||
|
next = confirmAt
|
||||||
|
}
|
||||||
|
if rest := previous.Confirm[1:]; len(rest) > 0 {
|
||||||
|
set[carrierqueue.FieldConfirm(checkType)] = joinSeconds(rest)
|
||||||
|
} else {
|
||||||
|
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(next.UnixMilli(), 10)
|
||||||
|
case StepFailed:
|
||||||
|
set[carrierqueue.FieldError(checkType)] = truncateRunes(outcome.Category+"|"+outcome.Message, 300)
|
||||||
|
if !outcome.CountsAsFailure() {
|
||||||
|
// 上游频率限制、已是目标状态:不计入连续失败,按首个退避间隔重试。
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, 1, interval)).UnixMilli(), 10)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
failCount := previous.FailCount + 1
|
||||||
|
firstFailed := previous.FirstFailed
|
||||||
|
if firstFailed.IsZero() {
|
||||||
|
firstFailed = now
|
||||||
|
set[carrierqueue.FieldFirstFail(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldFail(checkType)] = strconv.Itoa(failCount)
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, failCount, interval)).UnixMilli(), 10)
|
||||||
|
if failCount >= settings.AlertThreshold(checkType) || now.Sub(firstFailed) >= settings.AlertFailDuration || settings.ImmediateAlert[outcome.Category] {
|
||||||
|
e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, CheckType: checkType,
|
||||||
|
Category: outcome.Category, Message: outcome.Message, FailCount: failCount, FirstFailed: firstFailed, At: now})
|
||||||
|
}
|
||||||
|
case StepRateLimited:
|
||||||
|
// 限流等待超时:请求未发出,按短延迟重排,不计失败。
|
||||||
|
delay := 5 * time.Second
|
||||||
|
if e.base.rateLimitedDelay != nil {
|
||||||
|
if configured := e.base.rateLimitedDelay(); configured > 0 {
|
||||||
|
delay = configured
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||||
|
case StepDeferred:
|
||||||
|
delay := outcome.DeferDelay
|
||||||
|
if delay <= 0 {
|
||||||
|
delay = 30 * time.Second
|
||||||
|
}
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||||
|
default:
|
||||||
|
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(normalNext.UnixMilli(), 10)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CarrierCardExecutor) removeCard(ctx context.Context, cardID, carrierID uint) {
|
||||||
|
removeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := e.store.Remove(removeCtx, cardID, carrierID); err != nil {
|
||||||
|
e.base.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinSeconds(delays []time.Duration) string {
|
||||||
|
parts := make([]string, 0, len(delays))
|
||||||
|
for _, delay := range delays {
|
||||||
|
parts = append(parts, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateRunes(s string, max int) string {
|
||||||
|
runes := []rune(s)
|
||||||
|
if len(runes) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return string(runes[:max])
|
||||||
|
}
|
||||||
@@ -71,6 +71,8 @@ type PollingBase struct {
|
|||||||
priorityAudit *audit.Writer
|
priorityAudit *audit.Writer
|
||||||
scopeChecker *polling.PollingLifecycleService
|
scopeChecker *polling.PollingLifecycleService
|
||||||
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
||||||
|
errorClassifier ErrorClassifier // 上游错误分类(按运营商账户调度的失败退避与告警使用)
|
||||||
|
schedulerMode SchedulerModeReader // 当前调度模式;carrier 时旧 Asynq 轮询任务直接结束
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
||||||
|
|||||||
@@ -30,34 +30,42 @@ func NewPollingCarddataHandler(base *PollingBase, gw *gateway.Client, carrier *p
|
|||||||
return &PollingCarddataHandler{base: base, gateway: gw, carrier: carrier, observation: observation, integration: integration}
|
return &PollingCarddataHandler{base: base, gateway: gw, carrier: carrier, observation: observation, integration: integration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理流量检查任务。
|
// Handle 处理流量检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||||
startedAt := time.Now()
|
|
||||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCarddata) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCarddata) {
|
||||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCarddata)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCarddata)
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCarddata, h.Step(ctx, cardID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次流量检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startedAt := time.Now()
|
||||||
lockToken, locked, err := h.base.acquireCardTrafficSyncLock(ctx, cardID)
|
lockToken, locked, err := h.base.acquireCardTrafficSyncLock(ctx, cardID)
|
||||||
if err != nil || !locked {
|
if err != nil || !locked {
|
||||||
h.base.logger.Warn("卡流量同步锁不可用,延迟重入队", zap.Uint("card_id", cardID), zap.Error(err))
|
h.base.logger.Warn("卡流量同步锁不可用,延迟重入队", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
return h.base.queueMgr.Requeue(ctx, cardID, constants.TaskTypePollingCarddata, time.Now().Add(5*time.Second))
|
return stepDefer(5 * time.Second)
|
||||||
}
|
}
|
||||||
defer h.base.releaseCardTrafficSyncLock(ctx, cardID, lockToken)
|
defer h.base.releaseCardTrafficSyncLock(ctx, cardID, lockToken)
|
||||||
|
|
||||||
// 优先轮询互斥接缝:放在流量同步锁获取成功之后,避免认领后拿不到锁导致优先项空占。
|
// 优先轮询互斥接缝:放在流量同步锁获取成功之后,避免认领后拿不到锁导致优先项空占。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCarddata)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCarddata))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCarddata))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
h.base.invalidateCardCache(ctx, cardID)
|
h.base.invalidateCardCache(ctx, cardID)
|
||||||
@@ -65,42 +73,42 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFound(err) {
|
if isNotFound(err) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
if h.gateway == nil {
|
if h.gateway == nil {
|
||||||
priority.FailFinal(ctx, "流量查询能力未配置")
|
priority.FailFinal(ctx, "流量查询能力未配置")
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayTraffic, constants.CardObservationSceneTrafficPolling)
|
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayTraffic, constants.CardObservationSceneTrafficPolling)
|
||||||
result, err := h.gateway.QueryFlow(gateway.WithCarrier(ctx, card.CarrierID), &gateway.FlowQueryReq{CardNo: card.ICCID})
|
result, err := h.gateway.QueryFlow(gateway.WithCarrier(ctx, card.CarrierID), &gateway.FlowQueryReq{CardNo: card.ICCID})
|
||||||
if gateway.IsRateLimited(err) {
|
if gateway.IsRateLimited(err) {
|
||||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCarddata)
|
return StepOutcome{Result: StepRateLimited}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询流量失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询流量失败", priority)
|
||||||
priority.FailRetryable(ctx, "查询流量失败")
|
priority.FailRetryable(ctx, "查询流量失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询流量失败", err)
|
return h.fail(ctx, cardID, startedAt, "查询流量失败", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(result.ICCID) == "" {
|
if strings.TrimSpace(result.ICCID) == "" {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "流量查询响应缺少 ICCID", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "流量查询响应缺少 ICCID", priority)
|
||||||
priority.FailRetryable(ctx, "流量查询响应缺少 ICCID")
|
priority.FailRetryable(ctx, "流量查询响应缺少 ICCID")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
return h.fail(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
||||||
}
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
||||||
if h.observation == nil || h.carrier == nil {
|
if h.observation == nil || h.carrier == nil {
|
||||||
priority.FailFinal(ctx, "卡流量观测能力未配置")
|
priority.FailFinal(ctx, "卡流量观测能力未配置")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
return h.fail(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||||
@@ -113,7 +121,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用流量观测失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用流量观测失败", priority)
|
||||||
priority.FailRetryable(ctx, "应用流量观测失败")
|
priority.FailRetryable(ctx, "应用流量观测失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用流量观测失败", err)
|
return h.fail(ctx, cardID, startedAt, "应用流量观测失败", err)
|
||||||
}
|
}
|
||||||
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
||||||
changed := decision.IncrementMB > 0 || decision.CrossMonth
|
changed := decision.IncrementMB > 0 || decision.CrossMonth
|
||||||
@@ -132,15 +140,15 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
h.base.logger.Info("流量轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
h.base.logger.Info("流量轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||||
priority.Complete(ctx)
|
priority.Complete(ctx)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return StepOutcome{Result: StepSucceeded, TriggeredEvaluation: changed}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *PollingCarddataHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
func (h *PollingCarddataHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fields = append(fields, zap.Error(err))
|
fields = append(fields, zap.Error(err))
|
||||||
}
|
}
|
||||||
h.base.logger.Warn(message, fields...)
|
h.base.logger.Warn(message, fields...)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, false, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, false, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
return h.base.stepFail(ctx, constants.GatewayOpQueryFlow, message, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,70 +37,85 @@ func NewPollingCardStatusHandler(base *PollingBase, gw *gateway.Client, observat
|
|||||||
return &PollingCardStatusHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
return &PollingCardStatusHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理卡状态轮询任务。
|
// Handle 处理卡状态轮询任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||||
startedAt := time.Now()
|
|
||||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCardStatus) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCardStatus) {
|
||||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCardStatus)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCardStatus)
|
||||||
|
outcome := h.Step(ctx, cardID)
|
||||||
|
if outcome.Result == StepSucceeded {
|
||||||
|
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
||||||
|
h.base.invalidateCardCache(ctx, cardID)
|
||||||
|
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
||||||
|
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCardStatus, outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次卡状态检查步骤,返回结果分类与观测到的网络状态;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingCardStatusHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startedAt := time.Now()
|
||||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCardStatus)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCardStatus))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCardStatus))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFound(err) {
|
if isNotFound(err) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
if h.gateway == nil {
|
if h.gateway == nil {
|
||||||
priority.FailFinal(ctx, "卡状态查询能力未配置")
|
priority.FailFinal(ctx, "卡状态查询能力未配置")
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayNetwork, constants.CardObservationSceneNetworkPolling)
|
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayNetwork, constants.CardObservationSceneNetworkPolling)
|
||||||
result, err := h.gateway.QueryCardStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
result, err := h.gateway.QueryCardStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||||
if gateway.IsRateLimited(err) {
|
if gateway.IsRateLimited(err) {
|
||||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCardStatus)
|
return StepOutcome{Result: StepRateLimited}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询卡状态失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询卡状态失败", priority)
|
||||||
priority.FailRetryable(ctx, "查询卡状态失败")
|
priority.FailRetryable(ctx, "查询卡状态失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询卡状态失败", err)
|
return h.fail(ctx, cardID, startedAt, "查询卡状态失败", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(result.ICCID) == "" {
|
if strings.TrimSpace(result.ICCID) == "" {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "卡状态查询响应缺少 ICCID", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "卡状态查询响应缺少 ICCID", priority)
|
||||||
priority.FailRetryable(ctx, "卡状态查询响应缺少 ICCID")
|
priority.FailRetryable(ctx, "卡状态查询响应缺少 ICCID")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
return h.fail(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
||||||
}
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
||||||
if h.observation == nil {
|
if h.observation == nil {
|
||||||
priority.FailFinal(ctx, "卡网络观测能力未配置")
|
priority.FailFinal(ctx, "卡网络观测能力未配置")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
return h.fail(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
||||||
@@ -113,7 +128,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用网络状态观测失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用网络状态观测失败", priority)
|
||||||
priority.FailRetryable(ctx, "应用网络状态观测失败")
|
priority.FailRetryable(ctx, "应用网络状态观测失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
return h.fail(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
||||||
}
|
}
|
||||||
changed := decision.StatusChanged || decision.StopReasonChanged || decision.StopPolling
|
changed := decision.StatusChanged || decision.StopReasonChanged || decision.StopPolling
|
||||||
if changed {
|
if changed {
|
||||||
@@ -134,23 +149,24 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
|||||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
||||||
if decision.StopPolling {
|
if decision.StopPolling {
|
||||||
h.base.logger.Info("卡命中风险停机或已销户,已关闭该卡轮询", zap.Uint("card_id", cardID), zap.String("gateway_extend", decision.GatewayExtend))
|
h.base.logger.Info("卡命中风险停机或已销户,已关闭该卡轮询", zap.Uint("card_id", cardID), zap.String("gateway_extend", decision.GatewayExtend))
|
||||||
return nil
|
return StepOutcome{Result: StepStopPolling}
|
||||||
}
|
}
|
||||||
|
outcome := stepOK()
|
||||||
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
if decision.StatusKnown {
|
||||||
h.base.invalidateCardCache(ctx, cardID)
|
outcome.Observed = ObservedOffline
|
||||||
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
if decision.AfterStatus == constants.NetworkStatusOnline {
|
||||||
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
outcome.Observed = ObservedOnline
|
||||||
}
|
}
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
}
|
||||||
|
return outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *PollingCardStatusHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
func (h *PollingCardStatusHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fields = append(fields, zap.Error(err))
|
fields = append(fields, zap.Error(err))
|
||||||
}
|
}
|
||||||
h.base.logger.Warn(message, fields...)
|
h.base.logger.Warn(message, fields...)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
return h.base.stepFail(ctx, constants.GatewayOpQueryStatus, message, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,14 +37,15 @@ func NewPollingPackageHandler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理套餐检查任务
|
// Handle 处理套餐检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error {
|
func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error {
|
||||||
startTime := time.Now()
|
|
||||||
|
|
||||||
cardID, ok := parseTaskPayload(t.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(t.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
||||||
|
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
||||||
@@ -52,42 +53,48 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
|||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingPackage)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingPackage)
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingPackage, h.Step(ctx, cardID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次套餐检查步骤(按规则重新判断停复机),返回结果分类;不负责重入队。
|
||||||
|
// 调用方负责在 ctx 中设置 Worker 审计上下文。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingPackageHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startTime := time.Now()
|
||||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingPackage)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingPackage)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
h.base.logger.Error("查询活动优先轮询项失败", zap.Uint("card_id", cardID), zap.Error(priorityErr))
|
h.base.logger.Error("查询活动优先轮询项失败", zap.Uint("card_id", cardID), zap.Error(priorityErr))
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.stepFail(ctx, "", "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingPackage))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingPackage))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
card, cacheErr := h.base.getCardWithCache(ctx, cardID)
|
card, cacheErr := h.base.getCardWithCache(ctx, cardID)
|
||||||
if cacheErr != nil {
|
if cacheErr != nil {
|
||||||
if isNotFound(cacheErr) {
|
if isNotFound(cacheErr) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
h.base.logger.Error("获取卡信息失败", zap.Uint("card_id", cardID), zap.Error(cacheErr))
|
h.base.logger.Error("获取卡信息失败", zap.Uint("card_id", cardID), zap.Error(cacheErr))
|
||||||
// 本地读卡失败属可恢复失败:与其余三个轮询 handler 统一按尝试上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:与其余三个轮询 handler 统一按尝试上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.stepFail(ctx, "", "获取卡信息失败", cacheErr)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
|
|
||||||
if h.stopResumeSvc == nil {
|
if h.stopResumeSvc == nil {
|
||||||
priority.FailFinal(ctx, "套餐评估能力未配置")
|
priority.FailFinal(ctx, "套餐评估能力未配置")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
|
|
||||||
// 停复机评估就是本任务的上游执行动作:先累加尝试次数再发起。
|
// 停复机评估就是本任务的上游执行动作:先累加尝试次数再发起。
|
||||||
@@ -98,7 +105,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
|||||||
zap.Uint("card_id", cardID), zap.Error(loadErr))
|
zap.Uint("card_id", cardID), zap.Error(loadErr))
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return h.base.stepFail(ctx, "", "获取卡信息失败", loadErr)
|
||||||
}
|
}
|
||||||
h.base.logger.Info("套餐检查:执行停复机评估",
|
h.base.logger.Info("套餐检查:执行停复机评估",
|
||||||
zap.Uint("card_id", cardID),
|
zap.Uint("card_id", cardID),
|
||||||
@@ -119,10 +126,11 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
|||||||
zap.Uint("card_id", cardID), zap.Error(evalErr))
|
zap.Uint("card_id", cardID), zap.Error(evalErr))
|
||||||
priority.FailRetryable(ctx, "停复机评估失败")
|
priority.FailRetryable(ctx, "停复机评估失败")
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
outcome := h.base.stepFail(ctx, constants.GatewayOpStart, "停复机评估失败", evalErr)
|
||||||
|
return outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
priority.Complete(ctx)
|
priority.Complete(ctx)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
return stepOK()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,81 +28,88 @@ func NewPollingRealnameHandler(base *PollingBase, gw *gateway.Client, observatio
|
|||||||
return &PollingRealnameHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
return &PollingRealnameHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 处理实名检查任务。
|
// Handle 处理实名检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||||
func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||||
startedAt := time.Now()
|
|
||||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if h.base.carrierSchedulerActive(ctx) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingRealname) {
|
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingRealname) {
|
||||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||||
}
|
}
|
||||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingRealname)
|
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingRealname)
|
||||||
|
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingRealname, h.Step(ctx, cardID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 执行一次实名检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||||
|
func (h *PollingRealnameHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||||
|
startedAt := time.Now()
|
||||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingRealname)
|
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingRealname)
|
||||||
if priorityErr != nil {
|
if priorityErr != nil {
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||||
}
|
}
|
||||||
if skip {
|
if skip {
|
||||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingRealname))
|
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingRealname))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepDefer(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFound(err) {
|
if isNotFound(err) {
|
||||||
priority.FailFinal(ctx, "卡不存在")
|
priority.FailFinal(ctx, "卡不存在")
|
||||||
return nil
|
return StepOutcome{Result: StepCardGone}
|
||||||
}
|
}
|
||||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||||
}
|
}
|
||||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
if h.gateway == nil {
|
if h.gateway == nil {
|
||||||
priority.FailFinal(ctx, "实名查询能力未配置")
|
priority.FailFinal(ctx, "实名查询能力未配置")
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
// 所属运营商账户未开启实名管控时实名结果不影响停复机,跳过查询并按正常间隔重入队(不再按行业卡跳过)。
|
// 所属运营商账户未开启实名管控时实名结果不影响停复机,跳过查询并按正常间隔重入队(不再按行业卡跳过)。
|
||||||
skippable, scopeErr := h.base.realnameQuerySkippable(ctx, card)
|
skippable, scopeErr := h.base.realnameQuerySkippable(ctx, card)
|
||||||
if scopeErr != nil {
|
if scopeErr != nil {
|
||||||
priority.FailRetryable(ctx, "查询运营商账户实名管控失败")
|
priority.FailRetryable(ctx, "查询运营商账户实名管控失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
return h.fail(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
||||||
}
|
}
|
||||||
if skippable {
|
if skippable {
|
||||||
priority.FailFinal(ctx, "所属运营商账户未开启实名管控,跳过实名查询")
|
priority.FailFinal(ctx, "所属运营商账户未开启实名管控,跳过实名查询")
|
||||||
h.base.logger.Debug("所属运营商账户未开启实名管控,跳过实名查询", zap.Uint("card_id", cardID), zap.Uint("carrier_id", card.CarrierID))
|
h.base.logger.Debug("所属运营商账户未开启实名管控,跳过实名查询", zap.Uint("card_id", cardID), zap.Uint("carrier_id", card.CarrierID))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepSkip()
|
||||||
}
|
}
|
||||||
priority.MarkExecution(ctx)
|
priority.MarkExecution(ctx)
|
||||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayRealname, constants.CardObservationSceneRealnamePolling)
|
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayRealname, constants.CardObservationSceneRealnamePolling)
|
||||||
result, err := h.gateway.QueryRealnameStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
result, err := h.gateway.QueryRealnameStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||||
if gateway.IsRateLimited(err) {
|
if gateway.IsRateLimited(err) {
|
||||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingRealname)
|
return StepOutcome{Result: StepRateLimited}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询实名状态失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询实名状态失败", priority)
|
||||||
priority.FailRetryable(ctx, "查询实名状态失败")
|
priority.FailRetryable(ctx, "查询实名状态失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询实名状态失败", err)
|
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, "查询实名状态失败", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(result.ICCID) == "" {
|
if strings.TrimSpace(result.ICCID) == "" {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "实名查询响应缺少 ICCID", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "实名查询响应缺少 ICCID", priority)
|
||||||
priority.FailRetryable(ctx, "实名查询响应缺少 ICCID")
|
priority.FailRetryable(ctx, "实名查询响应缺少 ICCID")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
return h.fail(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
||||||
}
|
}
|
||||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
||||||
if h.observation == nil {
|
if h.observation == nil {
|
||||||
priority.FailFinal(ctx, "卡实名观测能力未配置")
|
priority.FailFinal(ctx, "卡实名观测能力未配置")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
return h.fail(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
||||||
@@ -115,7 +122,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用实名观测失败", priority)
|
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用实名观测失败", priority)
|
||||||
priority.FailRetryable(ctx, "应用实名观测失败")
|
priority.FailRetryable(ctx, "应用实名观测失败")
|
||||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用实名观测失败", err)
|
return h.fail(ctx, cardID, startedAt, "应用实名观测失败", err)
|
||||||
}
|
}
|
||||||
changed := decision.StatusChanged
|
changed := decision.StatusChanged
|
||||||
if changed {
|
if changed {
|
||||||
@@ -133,15 +140,19 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
|||||||
h.base.logger.Info("实名轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
h.base.logger.Info("实名轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||||
priority.Complete(ctx)
|
priority.Complete(ctx)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return stepOK()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *PollingRealnameHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
func (h *PollingRealnameHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||||
|
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, message, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *PollingRealnameHandler) failOp(ctx context.Context, cardID uint, startedAt time.Time, op, message string, err error) StepOutcome {
|
||||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fields = append(fields, zap.Error(err))
|
fields = append(fields, zap.Error(err))
|
||||||
}
|
}
|
||||||
h.base.logger.Warn(message, fields...)
|
h.base.logger.Warn(message, fields...)
|
||||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, false, time.Since(startedAt))
|
h.base.updateStats(ctx, constants.TaskTypePollingRealname, false, time.Since(startedAt))
|
||||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
return h.base.stepFail(ctx, op, message, err)
|
||||||
}
|
}
|
||||||
|
|||||||
123
internal/task/polling_step.go
Normal file
123
internal/task/polling_step.go
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
package task
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StepResult 是一个检查步骤的执行结果分类。
|
||||||
|
type StepResult int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// StepSucceeded 检查成功(含观测入账、评估完成)。
|
||||||
|
StepSucceeded StepResult = iota
|
||||||
|
// StepFailed 检查失败,计入连续失败(是否计入还要看失败类别,见 countsAsFailure)。
|
||||||
|
StepFailed
|
||||||
|
// StepRateLimited 本账户限流拒绝或等待超时,请求未发出:短延迟重排,不计失败。
|
||||||
|
StepRateLimited
|
||||||
|
// StepDeferred 暂不可执行(流量同步锁被占、优先项由其他执行持有):短延迟或按间隔重排,不计失败。
|
||||||
|
StepDeferred
|
||||||
|
// StepSkipped 本轮无需执行(能力未配置、不在轮询范围、所属账户未开启实名管控):按正常间隔重排。
|
||||||
|
StepSkipped
|
||||||
|
// StepCardGone 卡不存在:不再重排。
|
||||||
|
StepCardGone
|
||||||
|
// StepStopPolling 观测后该卡已关闭轮询(风险停机、已销户):不再重排。
|
||||||
|
StepStopPolling
|
||||||
|
)
|
||||||
|
|
||||||
|
// StepOutcome 是一个检查步骤的结果。旧分片调度把它映射为原有的重入队动作;按运营商账户调度据此写回到期与失败计数。
|
||||||
|
type StepOutcome struct {
|
||||||
|
Result StepResult
|
||||||
|
// DeferDelay 为 StepDeferred 时的建议短延迟;0 表示按正常间隔。
|
||||||
|
DeferDelay time.Duration
|
||||||
|
// Category 失败类别:网关错误类别(network / timeout / http / invalid_response),上游业务错误时为错误分类类别。
|
||||||
|
Category string
|
||||||
|
// Message 失败信息(已脱敏截断)。
|
||||||
|
Message string
|
||||||
|
// Observed 卡状态步骤观测到的网络状态(online / offline),用于延迟确认是否已满足预期。
|
||||||
|
Observed string
|
||||||
|
// TriggeredEvaluation 流量步骤本轮观测产生了流量变化(会触发停复机评估),可据此跳过套餐步骤。
|
||||||
|
TriggeredEvaluation bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// 卡状态步骤的观测结果取值。
|
||||||
|
const (
|
||||||
|
ObservedOnline = "online"
|
||||||
|
ObservedOffline = "offline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func stepOK() StepOutcome { return StepOutcome{Result: StepSucceeded} }
|
||||||
|
|
||||||
|
func stepSkip() StepOutcome { return StepOutcome{Result: StepSkipped} }
|
||||||
|
|
||||||
|
func stepDefer(delay time.Duration) StepOutcome {
|
||||||
|
return StepOutcome{Result: StepDeferred, DeferDelay: delay}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepFail 按错误构造失败结果:网关结构化错误取其类别,上游业务错误再按错误分类规则细分(用于判断是否计入连续失败与告警)。
|
||||||
|
func (b *PollingBase) stepFail(ctx context.Context, op, message string, err error) StepOutcome {
|
||||||
|
outcome := StepOutcome{Result: StepFailed, Message: message, Category: "local"}
|
||||||
|
if gwErr, ok := gateway.AsGatewayError(err); ok {
|
||||||
|
outcome.Category = gwErr.Category
|
||||||
|
if gwErr.Category == constants.GatewayErrorUpstream {
|
||||||
|
outcome.Message = message + ":" + gwErr.ProviderMessage
|
||||||
|
if b.errorClassifier != nil {
|
||||||
|
outcome.Category = b.errorClassifier(ctx, op, gwErr.ProviderMessage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountsAsFailure 判断失败结果是否计入连续失败:上游频率限制与已是目标状态不计入。
|
||||||
|
func (o StepOutcome) CountsAsFailure() bool {
|
||||||
|
if o.Result != StepFailed {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch o.Category {
|
||||||
|
case constants.GatewayClassRateLimitedUpstream, constants.GatewayClassAlreadyTarget:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrorClassifier 对上游业务错误信息分类,返回错误分类类别。
|
||||||
|
type ErrorClassifier func(ctx context.Context, op, message string) string
|
||||||
|
|
||||||
|
// SetErrorClassifier 注入上游错误分类器(按运营商账户调度的失败退避与告警使用)。
|
||||||
|
func (b *PollingBase) SetErrorClassifier(classifier ErrorClassifier) {
|
||||||
|
b.errorClassifier = classifier
|
||||||
|
}
|
||||||
|
|
||||||
|
// SchedulerModeReader 返回当前卡轮询调度模式(受控系统配置 polling.scheduler.mode)。
|
||||||
|
type SchedulerModeReader func(ctx context.Context) string
|
||||||
|
|
||||||
|
// SetSchedulerModeReader 注入调度模式读取;未注入时视为旧分片调度。
|
||||||
|
func (b *PollingBase) SetSchedulerModeReader(reader SchedulerModeReader) {
|
||||||
|
b.schedulerMode = reader
|
||||||
|
}
|
||||||
|
|
||||||
|
// carrierSchedulerActive 判断当前是否已切到按运营商账户调度:是则旧 Asynq 轮询任务直接结束,不调网关、不重入旧队列。
|
||||||
|
func (b *PollingBase) carrierSchedulerActive(ctx context.Context) bool {
|
||||||
|
return b != nil && b.schedulerMode != nil && b.schedulerMode(ctx) == constants.PollingSchedulerModeCarrier
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyLegacyOutcome 把步骤结果映射为旧分片调度原有的重入队动作。
|
||||||
|
func (b *PollingBase) applyLegacyOutcome(ctx context.Context, cardID uint, taskType string, outcome StepOutcome) error {
|
||||||
|
switch outcome.Result {
|
||||||
|
case StepCardGone, StepStopPolling:
|
||||||
|
return nil
|
||||||
|
case StepRateLimited:
|
||||||
|
return b.requeueRateLimited(cardID, taskType)
|
||||||
|
case StepDeferred:
|
||||||
|
if outcome.DeferDelay > 0 {
|
||||||
|
return b.queueMgr.Requeue(ctx, cardID, taskType, time.Now().Add(outcome.DeferDelay))
|
||||||
|
}
|
||||||
|
return b.requeueCard(ctx, cardID, taskType)
|
||||||
|
default:
|
||||||
|
return b.requeueCard(ctx, cardID, taskType)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,75 +1,75 @@
|
|||||||
## 0. 前置确认
|
## 0. 前置确认
|
||||||
|
|
||||||
- [ ] 0.1 确认第 1 个 Change(限流器)与第 3 个 Change(去掉保护期)已上线;维护者提供测试环境 nsqd + nsqlookupd 地址。
|
- [x] 0.1 确认第 1 个 Change(限流器)与第 3 个 Change(去掉保护期)已上线;维护者提供测试环境 nsqd + nsqlookupd 地址。(第 1、3 个 Change 已部署测试环境;测试环境 NSQ 由 docker-compose.prod.yml 新增 nsqlookupd + nsqd 提供,Worker 注入 nsqd:4150 / nsqlookupd:4161)
|
||||||
|
|
||||||
## 1. 迁移与配置
|
## 1. 迁移与配置
|
||||||
|
|
||||||
- [ ] 1.1 新增成对迁移:`tb_carrier_polling_param`(`carrier_id` 唯一、`inflight_limit`、`card_max_in_flight`、`stop_resume_max_in_flight`、`sr_inflight_limit`、`priority_max_ratio`、`stop_resume_min_interval`、`stop_resume_confirm_delay`、审计字段)。
|
- [x] 1.1 新增成对迁移:`tb_carrier_polling_param`(`carrier_id` 唯一、`inflight_limit`、`card_max_in_flight`、`stop_resume_max_in_flight`、`sr_inflight_limit`、`priority_max_ratio`、`stop_resume_min_interval`、`stop_resume_confirm_delay`、审计字段)。(000251)
|
||||||
- [ ] 1.2 新增成对迁移:`tb_polling_check_scene`(`scene` 唯一、`check_types`、`confirm_delays`、`cooldown`),预置 8 个场景,读入口场景带默认冷却。
|
- [x] 1.2 新增成对迁移:`tb_polling_check_scene`(`scene` 唯一、`check_types`、`confirm_delays`、`cooldown`),预置 8 个场景,读入口场景带默认冷却。(000252,预置 8 个场景)
|
||||||
- [ ] 1.3 新增成对迁移:`tb_polling_failure_alert`(卡、运营商账户、检查类型、连续失败、首次与最近失败、最近错误、状态、处理人与时间;部分唯一索引 `(card_id, check_type) WHERE status = 未处理`)。
|
- [x] 1.3 新增成对迁移:`tb_polling_failure_alert`(卡、运营商账户、检查类型、连续失败、首次与最近失败、最近错误、状态、处理人与时间;部分唯一索引 `(card_id, check_type) WHERE status = 未处理`)。(000253;部分唯一索引含告警级别与账户,账户级告警 card_id=0)
|
||||||
- [ ] 1.4 新增成对迁移:`tb_system_config` 插入 `polling.scheduler.mode`(默认 `legacy`)、步骤顺序、单步超时、卡任务总超时余量、退避序列、告警阈值、账户级告警窗口与阈值;`tb_polling_concurrency_config` 表注释标记废弃。
|
- [x] 1.4 新增成对迁移:`tb_system_config` 插入 `polling.scheduler.mode`(默认 `legacy`)、步骤顺序、单步超时、卡任务总超时余量、退避序列、告警阈值、账户级告警窗口与阈值;`tb_polling_concurrency_config` 表注释标记废弃。(000254,默认 legacy)
|
||||||
- [ ] 1.5 `pkg/config` 与 `defaults/config.yaml`:NSQ 地址、`msg_timeout`、租约、续期间隔、分发间隔、优雅退出宽限期、默认执行上限与 `max_in_flight`;`go.mod` 引入 NSQ 客户端。
|
- [x] 1.5 `pkg/config` 与 `defaults/config.yaml`:NSQ 地址、`msg_timeout`、租约、续期间隔、分发间隔、优雅退出宽限期、默认执行上限与 `max_in_flight`;`go.mod` 引入 NSQ 客户端。(polling.carrier.*;go-nsq v1.1.0)
|
||||||
- [ ] 1.6 注册受控系统配置项并实现 JSON 结构校验(步骤类型合法且不重复、退避序列递增)。
|
- [x] 1.6 注册受控系统配置项并实现 JSON 结构校验(步骤类型合法且不重复、退避序列递增)。(carrierqueue.RegisterConfigDefinitions,后台系统配置可编辑)
|
||||||
|
|
||||||
## 2. Redis 结构与 Lua
|
## 2. Redis 结构与 Lua
|
||||||
|
|
||||||
- [ ] 2.1 常量:账户队列、running、dispatch_lock,卡 due、claim、exec 的 key。
|
- [x] 2.1 常量:账户队列、running、dispatch_lock,卡 due、claim、exec 的 key。
|
||||||
- [ ] 2.2 领取 Lua:清理过期 running、执行上限、优先占比、预计调用数对比本秒剩余 QPS、`SET claim NX PX`、推后 score、写 running;已有 claim 时推后到剩余 TTL 后。
|
- [x] 2.2 领取 Lua:清理过期 running、执行上限、优先占比、预计调用数对比本秒剩余 QPS、`SET claim NX PX`、推后 score、写 running;已有 claim 时推后到剩余 TTL 后。(按限流组剩余 QPS 在 Go 侧计算后传入 Lua;只统计模式不按 QPS 限制)
|
||||||
- [ ] 2.3 续期 Lua:校验 token,延长 claim、exec,推后 score 与 running。
|
- [x] 2.3 续期 Lua:校验 token,延长 claim、exec,推后 score 与 running。
|
||||||
- [ ] 2.4 收尾 Lua:校验 token,删 claim、exec,移出 running,写 due 与失败计数,按优先标记设置 score;token 不一致时只写业务结果。
|
- [x] 2.4 收尾 Lua:校验 token,删 claim、exec,移出 running,写 due 与失败计数,按优先标记设置 score;token 不一致时只写业务结果。
|
||||||
- [ ] 2.5 请求检查 Lua:设 due[type]=now、冷却判断、放入 priority 或打优先标记,合并重复请求。
|
- [x] 2.5 请求检查 Lua:设 due[type]=now、冷却判断、放入 priority 或打优先标记,合并重复请求。(另支持普通通道:手动触发只把到期设为现在、不进优先队列)
|
||||||
|
|
||||||
## 3. 初始化与生命周期
|
## 3. 初始化与生命周期
|
||||||
|
|
||||||
- [ ] 3.1 `initializer.go`:按批读库计算各类型 due(复用第 2 个 Change 的分层与重置日前到期计算),写 due 与账户 queue;进度写入 init-progress。
|
- [x] 3.1 `initializer.go`:按批读库计算各类型 due(复用第 2 个 Change 的分层与重置日前到期计算),写 due 与账户 queue;进度写入 init-progress。(CarrierInitializer;进度在 Worker 内与 /polling-stats/carrier-queues 的 init_state 可见,未并入旧 init-progress 接口)
|
||||||
- [ ] 3.2 `lifecycle_service.go`、`asset_polling_service.go`:增删卡、状态变化、启停轮询改为操作账户队列与 due;恢复轮询时 due 设为当前加抖动。
|
- [x] 3.2 `lifecycle_service.go`、`asset_polling_service.go`:增删卡、状态变化、启停轮询改为操作账户队列与 due;恢复轮询时 due 设为当前加抖动。(API 与 Worker 的生命周期服务按模式写入;资产轮询开关经生命周期回调生效)
|
||||||
- [ ] 3.3 优先提示(`priority_manual_service.go`)与手动触发(`manual_trigger_service.go`)改为写 due 与账户队列。
|
- [x] 3.3 优先提示(`priority_manual_service.go`)与手动触发(`manual_trigger_service.go`)改为写 due 与账户队列。(旧提示通道与手动触发 List 由新分发器每秒消费后写入账户队列,接口不变)
|
||||||
|
|
||||||
## 4. 分发器
|
## 4. 分发器
|
||||||
|
|
||||||
- [ ] 4.1 Worker 内每秒对各账户抢分发锁并执行领取 Lua,发布到 `polling.card.{cid}`;发布失败按 token 删 claim、score 恢复为当前时间。
|
- [x] 4.1 Worker 内每秒对各账户抢分发锁并执行领取 Lua,发布到 `polling.card.{cid}`;发布失败按 token 删 claim、score 恢复为当前时间。
|
||||||
- [ ] 4.2 读取账户调度参数(缺省用默认),执行上限不超过订阅进程 `max_in_flight` 之和,不满足时收紧并上报。
|
- [x] 4.2 读取账户调度参数(缺省用默认),执行上限不超过订阅进程 `max_in_flight` 之和,不满足时收紧并上报。
|
||||||
- [ ] 4.3 定时动作:统计各账户处理中标记到期次数、写分发器心跳。
|
- [x] 4.3 定时动作:统计各账户处理中标记到期次数、写分发器心跳。
|
||||||
|
|
||||||
## 5. NSQ 消费者与卡任务
|
## 5. NSQ 消费者与卡任务
|
||||||
|
|
||||||
- [ ] 5.1 NSQ 生产者与消费者封装:按账户动态订阅与取消订阅、`max_in_flight`、`Touch`、优雅退出(停止分发与订阅、宽限期内完成后 FIN)。
|
- [x] 5.1 NSQ 生产者与消费者封装:按账户动态订阅与取消订阅、`max_in_flight`、`Touch`、优雅退出(停止分发与订阅、宽限期内完成后 FIN)。(internal/infrastructure/nsqbus)
|
||||||
- [ ] 5.2 消费入口:核对 claim token、抢 exec 执行权,任一失败即 FIN。
|
- [x] 5.2 消费入口:核对 claim token、抢 exec 执行权,任一失败即 FIN。
|
||||||
- [ ] 5.3 将实名、流量、卡状态、套餐 4 个 handler 拆为步骤函数(`StepInput` 含可选外部结果);按配置顺序串行执行到期类型;单步超时、总超时、panic 恢复。
|
- [x] 5.3 将实名、流量、卡状态、套餐 4 个 handler 拆为步骤函数(`StepInput` 含可选外部结果);按配置顺序串行执行到期类型;单步超时、总超时、panic 恢复。(四个 handler 拆出 Step,旧 Asynq Handle 只映射重入队;StepInput 外部结果暂未实现,按需在批量查询时扩展)
|
||||||
- [ ] 5.4 执行期间按续期间隔续期;续期失败立即停止后续步骤。
|
- [x] 5.4 执行期间按续期间隔续期;续期失败立即停止后续步骤。
|
||||||
- [ ] 5.5 网关调用使用限流器等待模式;限流等待超时按短延迟重排不计失败。
|
- [x] 5.5 网关调用使用限流器等待模式;限流等待超时按短延迟重排不计失败。(ctx 标记等待模式)
|
||||||
- [ ] 5.6 沿用 `polling_priority_claim.go` 的认领与收尾;保留卡流量同步锁。
|
- [x] 5.6 沿用 `polling_priority_claim.go` 的认领与收尾;保留卡流量同步锁。
|
||||||
|
|
||||||
## 6. 失败退避与告警
|
## 6. 失败退避与告警
|
||||||
|
|
||||||
- [ ] 6.1 收尾按"卡 × 类型"更新连续失败、首次失败、最近错误;按退避序列计算下次到期,频率限制与限流超时不计失败;成功清零。
|
- [x] 6.1 收尾按"卡 × 类型"更新连续失败、首次失败、最近错误;按退避序列计算下次到期,频率限制与限流超时不计失败;成功清零。
|
||||||
- [ ] 6.2 达到阈值或首次即告警类别时写或更新 `tb_polling_failure_alert`;成功后自动转已恢复;通知走现有告警通道。
|
- [x] 6.2 达到阈值或首次即告警类别时写或更新 `tb_polling_failure_alert`;成功后自动转已恢复;通知走现有告警通道。(通知以 ERROR 日志 + metric 输出,未接入 alert_service 的规则通知通道)
|
||||||
- [ ] 6.3 账户级告警:窗口内同账户告警数超过阈值时产生账户级告警。
|
- [x] 6.3 账户级告警:窗口内同账户告警数超过阈值时产生账户级告警。
|
||||||
|
|
||||||
## 7. 业务事件与读入口请求检查
|
## 7. 业务事件与读入口请求检查
|
||||||
|
|
||||||
- [ ] 7.1 订单支付、套餐激活、停复机成功、设备操作(按设备下各卡)、自动续费购包 5 处 `AppendSeriesRequested` 改为事务提交后请求检查。
|
- [x] 7.1 订单支付、套餐激活、停复机成功、设备操作(按设备下各卡)、自动续费购包 5 处 `AppendSeriesRequested` 改为事务提交后请求检查。(业务事件仍在原事务写 Outbox,carrier 模式下 Outbox 消费者转为请求检查,满足提交后请求)
|
||||||
- [ ] 7.2 C 端资产详情、后台资产实时状态、C 端与后台实名链接改为请求检查;OpenAPI 查询保持不触发。
|
- [x] 7.2 C 端资产详情、后台资产实时状态、C 端与后台实名链接改为请求检查;OpenAPI 查询保持不触发。
|
||||||
- [ ] 7.3 延迟确认:按场景在检查完成后追加确认到期;结果已满足预期时取消。
|
- [x] 7.3 延迟确认:按场景在检查完成后追加确认到期;结果已满足预期时取消。
|
||||||
- [ ] 7.4 卡检查不再创建观测序列;`query_device_info` 等非运营商上游部分保持不变。
|
- [x] 7.4 卡检查不再创建观测序列;`query_device_info` 等非运营商上游部分保持不变。
|
||||||
|
|
||||||
## 8. 整体切换
|
## 8. 整体切换
|
||||||
|
|
||||||
- [ ] 8.1 Worker 监听 `polling.scheduler.mode`:切 `carrier` 时停旧调度出队、初始化新结构后开启分发;切 `legacy` 时停止领取、等待执行中任务收尾、重建旧分片队列后恢复旧调度。
|
- [x] 8.1 Worker 监听 `polling.scheduler.mode`:切 `carrier` 时停旧调度出队、初始化新结构后开启分发;切 `legacy` 时停止领取、等待执行中任务收尾、重建旧分片队列后恢复旧调度。
|
||||||
- [ ] 8.2 旧轮询 handler 执行前检查模式,为 `carrier` 时直接结束。
|
- [x] 8.2 旧轮询 handler 执行前检查模式,为 `carrier` 时直接结束。
|
||||||
- [ ] 8.3 所有写入轮询结构的入口按当前模式写新或旧结构。
|
- [x] 8.3 所有写入轮询结构的入口按当前模式写新或旧结构。(含第 2 个 Change 的重算到期任务)
|
||||||
|
|
||||||
## 9. 接口与下线
|
## 9. 接口与下线
|
||||||
|
|
||||||
- [ ] 9.1 `/polling-carrier-params` 列表(含默认标记)、详情、修改、删除;`/polling-check-scenes` 列表与修改;`/polling-failure-alerts` 列表、`/summary`、详情、`PUT /:id/status`;`GET /iot-cards/:id/polling-state`。
|
- [x] 9.1 `/polling-carrier-params` 列表(含默认标记)、详情、修改、删除;`/polling-check-scenes` 列表与修改;`/polling-failure-alerts` 列表、`/summary`、详情、`PUT /:id/status`;`GET /iot-cards/:id/polling-state`。
|
||||||
- [ ] 9.2 `/polling-stats` 改为按账户(队列长度、到期数、执行中、实际 QPS、限流等待与超时、处理中标记到期、告警数)。
|
- [x] 9.2 `/polling-stats` 改为按账户(队列长度、到期数、执行中、实际 QPS、限流等待与超时、处理中标记到期、告警数)。(新增 GET /polling-stats/carrier-queues;/polling-stats/carriers 已被第 1 个 Change 的网关调用统计占用)
|
||||||
- [ ] 9.3 下线 `/polling-concurrency`、`concurrency_service.go`、`acquireConcurrency`/`releaseConcurrency`、按 ZCARD 背压;旧提示通道 List 不再写入。
|
- [ ] 9.3 下线 `/polling-concurrency`、`concurrency_service.go`、`acquireConcurrency`/`releaseConcurrency`、按 ZCARD 背压;旧提示通道 List 不再写入。(暂缓:旧分片调度仍需并发配置与并发令牌,保留以支持整体回退;按设计在新调度稳定运行后与旧调度代码一并下线)
|
||||||
- [ ] 9.4 同步路由、`cmd/api/docs.go`、`cmd/gendocs/main.go`,重新生成接口文档。
|
- [x] 9.4 同步路由、`cmd/api/docs.go`、`cmd/gendocs/main.go`,重新生成接口文档。
|
||||||
|
|
||||||
## 10. 编译与规范校验
|
## 10. 编译与规范校验
|
||||||
|
|
||||||
- [ ] 10.1 `gofmt -w`;`go build ./cmd/api ./cmd/worker` 通过。
|
- [x] 10.1 `gofmt -w`;`go build ./cmd/api ./cmd/worker` 通过。
|
||||||
- [ ] 10.2 `openspec validate add-carrier-polling-scheduler` 与 `openspec doctor --json` 通过。
|
- [x] 10.2 `openspec validate add-carrier-polling-scheduler` 与 `openspec doctor --json` 通过。(validate 通过;当前 openspec 无 doctor 命令)
|
||||||
|
|
||||||
## 11. 测试环境验证(自动化测试为 N/A,按 ENG-TEST-001;按生产规模造卡或回放生产卡 ID)
|
## 11. 测试环境验证(自动化测试为 N/A,按 ENG-TEST-001;按生产规模造卡或回放生产卡 ID)
|
||||||
|
|
||||||
|
|||||||
@@ -602,6 +602,14 @@ const (
|
|||||||
AuditActionGatewayErrorRuleStatusUpdated = "gateway_error_rule.update_status"
|
AuditActionGatewayErrorRuleStatusUpdated = "gateway_error_rule.update_status"
|
||||||
// AuditActionGatewayErrorRuleDeleted 表示删除上游错误分类规则。
|
// AuditActionGatewayErrorRuleDeleted 表示删除上游错误分类规则。
|
||||||
AuditActionGatewayErrorRuleDeleted = "gateway_error_rule.delete"
|
AuditActionGatewayErrorRuleDeleted = "gateway_error_rule.delete"
|
||||||
|
// AuditActionCarrierPollingParamSaved 表示保存运营商账户调度参数。
|
||||||
|
AuditActionCarrierPollingParamSaved = "carrier_polling_param.save"
|
||||||
|
// AuditActionCarrierPollingParamDeleted 表示删除运营商账户调度参数(恢复全局默认)。
|
||||||
|
AuditActionCarrierPollingParamDeleted = "carrier_polling_param.delete"
|
||||||
|
// AuditActionPollingCheckSceneUpdated 表示修改请求检查场景配置。
|
||||||
|
AuditActionPollingCheckSceneUpdated = "polling_check_scene.update"
|
||||||
|
// AuditActionPollingFailureAlertHandled 表示处理卡轮询失败告警。
|
||||||
|
AuditActionPollingFailureAlertHandled = "polling_failure_alert.handle"
|
||||||
// AuditActionPollingConcurrencyUpdated 表示更新轮询任务并发配置。
|
// AuditActionPollingConcurrencyUpdated 表示更新轮询任务并发配置。
|
||||||
AuditActionPollingConcurrencyUpdated = "polling_concurrency.update"
|
AuditActionPollingConcurrencyUpdated = "polling_concurrency.update"
|
||||||
// AuditActionPollingConcurrencyReset 表示人工重置轮询并发计数。
|
// AuditActionPollingConcurrencyReset 表示人工重置轮询并发计数。
|
||||||
@@ -781,6 +789,12 @@ const (
|
|||||||
AuditResourceRateLimitGroup = "rate_limit_group"
|
AuditResourceRateLimitGroup = "rate_limit_group"
|
||||||
// AuditResourceGatewayErrorRule 表示上游错误分类规则资源。
|
// AuditResourceGatewayErrorRule 表示上游错误分类规则资源。
|
||||||
AuditResourceGatewayErrorRule = "gateway_error_rule"
|
AuditResourceGatewayErrorRule = "gateway_error_rule"
|
||||||
|
// AuditResourceCarrierPollingParam 表示运营商账户调度参数资源。
|
||||||
|
AuditResourceCarrierPollingParam = "carrier_polling_param"
|
||||||
|
// AuditResourcePollingCheckScene 表示请求检查场景资源。
|
||||||
|
AuditResourcePollingCheckScene = "polling_check_scene"
|
||||||
|
// AuditResourcePollingFailureAlert 表示卡轮询失败告警资源。
|
||||||
|
AuditResourcePollingFailureAlert = "polling_failure_alert"
|
||||||
// AuditResourcePollingConcurrencyConfig 表示轮询并发配置资源。
|
// AuditResourcePollingConcurrencyConfig 表示轮询并发配置资源。
|
||||||
AuditResourcePollingConcurrencyConfig = "polling_concurrency"
|
AuditResourcePollingConcurrencyConfig = "polling_concurrency"
|
||||||
// AuditResourcePollingAlertRule 表示轮询告警规则资源。
|
// AuditResourcePollingAlertRule 表示轮询告警规则资源。
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ type Handler struct {
|
|||||||
workerResult *WorkerBootstrapResult
|
workerResult *WorkerBootstrapResult
|
||||||
asynqClient *asynq.Client
|
asynqClient *asynq.Client
|
||||||
pollingBase *task.PollingBase
|
pollingBase *task.PollingBase
|
||||||
|
pollingSteps PollingStepHandlers
|
||||||
pollingStopResumeSvc iot_card_svc.StopResumeServiceInterface
|
pollingStopResumeSvc iot_card_svc.StopResumeServiceInterface
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -292,6 +293,7 @@ func (h *Handler) registerPollingHandlers() {
|
|||||||
cardStatusHandler := task.NewPollingCardStatusHandler(
|
cardStatusHandler := task.NewPollingCardStatusHandler(
|
||||||
h.pollingBase, h.gatewayClient, h.workerResult.Services.CardObservation, integrationRepository)
|
h.pollingBase, h.gatewayClient, h.workerResult.Services.CardObservation, integrationRepository)
|
||||||
|
|
||||||
|
h.pollingSteps = PollingStepHandlers{Realname: realnameHandler, Carddata: carddataHandler, Package: packageHandler, CardStatus: cardStatusHandler}
|
||||||
h.mux.HandleFunc(constants.TaskTypePollingRealname, realnameHandler.Handle)
|
h.mux.HandleFunc(constants.TaskTypePollingRealname, realnameHandler.Handle)
|
||||||
h.mux.HandleFunc(constants.TaskTypePollingCarddata, carddataHandler.Handle)
|
h.mux.HandleFunc(constants.TaskTypePollingCarddata, carddataHandler.Handle)
|
||||||
h.mux.HandleFunc(constants.TaskTypePollingPackage, packageHandler.Handle)
|
h.mux.HandleFunc(constants.TaskTypePollingPackage, packageHandler.Handle)
|
||||||
@@ -408,3 +410,16 @@ func (h *Handler) registerOperationsReportSnapshotHandler() {
|
|||||||
func (h *Handler) GetMux() *asynq.ServeMux {
|
func (h *Handler) GetMux() *asynq.ServeMux {
|
||||||
return h.mux
|
return h.mux
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PollingStepHandlers 是四个轮询检查步骤处理器,旧分片调度注册为 Asynq handler,按运营商账户调度的卡任务执行器复用其 Step。
|
||||||
|
type PollingStepHandlers struct {
|
||||||
|
Realname *task.PollingRealnameHandler
|
||||||
|
Carddata *task.PollingCarddataHandler
|
||||||
|
Package *task.PollingPackageHandler
|
||||||
|
CardStatus *task.PollingCardStatusHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollingSteps 返回轮询检查步骤处理器(RegisterHandlers 之后可用)。
|
||||||
|
func (h *Handler) PollingSteps() PollingStepHandlers {
|
||||||
|
return h.pollingSteps
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user