diff --git a/cmd/worker/carrier_scheduler.go b/cmd/worker/carrier_scheduler.go index 075db499..6ec462fa 100644 --- a/cmd/worker/carrier_scheduler.go +++ b/cmd/worker/carrier_scheduler.go @@ -12,10 +12,13 @@ import ( "go.uber.org/zap" cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation" + "github.com/break/junhong_cmp_fiber/internal/infrastructure/gatewaylimit" "github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox" "github.com/break/junhong_cmp_fiber/internal/infrastructure/nsqbus" + "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/internal/polling" "github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue" + iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card" "github.com/break/junhong_cmp_fiber/internal/task" "github.com/break/junhong_cmp_fiber/pkg/config" "github.com/break/junhong_cmp_fiber/pkg/constants" @@ -35,9 +38,13 @@ type carrierSchedulerRuntime struct { producer *nsqbus.Producer alerts *carrierqueue.AlertWriter executor *task.CarrierCardExecutor - singleton bool - instanceID string - logger *zap.Logger + // 停复机指令与设备判断的专用消费者：只看停复机出口开关，不受 scheduler_mode 限制。 + srConsumers *nsqbus.ConsumerManager + devConsumers *nsqbus.ConsumerManager + srActive bool + singleton bool + instanceID string + logger *zap.Logger mu sync.Mutex lastMode string @@ -56,6 +63,9 @@ func newCarrierSchedulerParts(runtime *workerRuntime, appLogger *zap.Logger) { runtime.carrierScheduling = scheduling runtime.lifecycleSvc.SetCarrierScheduling(scheduling) runtime.pollingBase.SetSchedulerModeReader(func(ctx context.Context) string { return settings.Current(ctx).Mode }) + if stopResumeSvc, ok := runtime.workerResult.Services.StopResumeService.(*iot_card_svc.StopResumeService); ok { + stopResumeSvc.SetCommandQueue(carrierqueue.NewCommandQueue(store, settings)) + } if runtime.gatewayLimit != nil && runtime.gatewayLimit.Classifier != nil { classifier := runtime.gatewayLimit.Classifier runtime.pollingBase.SetErrorClassifier(func(ctx context.Context, op, message string) string { @@ -94,6 +104,7 @@ func startCarrierScheduler(ctx context.Context, cfg *config.Config, runtime *wor MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts, }, constants.NSQChannelWorker, executor.Handle, appLogger) } + c.setupStopResume(carrierCfg, alerts) var budgets carrierqueue.BudgetProvider if runtime.gatewayLimit != nil && runtime.gatewayLimit.Limiter != nil { budgets = runtime.gatewayLimit.Limiter @@ -139,6 +150,7 @@ func (c *carrierSchedulerRuntime) step(ctx context.Context) { c.logger.Error("按运营商账户调度模式监听 panic，已恢复", zap.Any("panic", r)) } }() + c.syncStopResume(ctx) mode := c.runtime.carrierSettings.Current(ctx).Mode c.mu.Lock() previous := c.lastMode @@ -216,11 +228,74 @@ func (c *carrierSchedulerRuntime) grace() time.Duration { return 30 * time.Second } +// setupStopResume 创建停复机指令执行器、设备判断执行器与两个专用消费者。 +func (c *carrierSchedulerRuntime) setupStopResume(carrierCfg config.CarrierSchedulerConfig, alerts *carrierqueue.AlertWriter) { + service, ok := c.runtime.workerResult.Services.StopResumeService.(*iot_card_svc.StopResumeService) + if !ok || carrierCfg.NSQDAddress == "" { + return + } + var classifier *gatewaylimit.Classifier + if c.runtime.gatewayLimit != nil { + classifier = c.runtime.gatewayLimit.Classifier + } + params := func(carrierID uint) carrierqueue.CarrierParams { + if c.dispatcher != nil { + return c.dispatcher.ParamsFor(carrierID) + } + return carrierqueue.MergeParams(model.CarrierPollingParam{}, false, carrierCfg) + } + srExecutor := task.NewStopResumeExecutor(c.runtime.carrierStore, c.runtime.carrierSettings, params, alerts, service, classifier, + c.instanceID, c.logger) + devExecutor := task.NewDeviceEvalExecutor(c.runtime.carrierStore, service, c.instanceID, c.logger) + consumerCfg := nsqbus.ConsumerConfig{ + NSQDAddress: carrierCfg.NSQDAddress, NSQLookupdAddresses: nsqbus.SplitAddresses(carrierCfg.NSQLookupdAddresses), + MsgTimeout: carrierCfg.MsgTimeout, MaxAttempts: carrierCfg.MaxAttempts, + } + c.srConsumers = nsqbus.NewConsumerManager(consumerCfg, constants.NSQChannelWorker, srExecutor.Handle, c.logger) + c.devConsumers = nsqbus.NewConsumerManager(consumerCfg, constants.NSQChannelWorker, devExecutor.Handle, c.logger) +} + +// syncStopResume 停复机出口开关开启时订阅各账户停复机 topic 与设备判断 topic；关闭时停止订阅（队列中的指令保留）。 +func (c *carrierSchedulerRuntime) syncStopResume(ctx context.Context) { + if c.srConsumers == nil || c.dispatcher == nil { + return + } + enabled := c.runtime.carrierSettings.Current(ctx).StopResumeQueueEnabled + if !enabled { + if c.srActive { + c.logger.Info("停复机出口开关已关闭：停止停复机指令消费，未执行的指令保留") + c.srConsumers.StopAll(c.grace()) + c.srConsumers.Reset() + c.devConsumers.StopAll(c.grace()) + c.devConsumers.Reset() + c.srActive = false + } + return + } + if !c.srActive { + c.logger.Info("停复机出口开关已开启：自动停复机经停复机指令队列执行") + c.srActive = true + } + desired := map[string]int{} + for _, carrierID := range c.dispatcher.CarrierIDs(ctx) { + desired[constants.NSQStopResumeTopic(carrierID)] = c.dispatcher.ParamsFor(carrierID).SRMaxInFlight + } + c.srConsumers.Sync(desired) + c.devConsumers.Sync(map[string]int{constants.NSQDeviceEvalTopic: deviceEvalMaxInFlight}) +} + +// deviceEvalMaxInFlight 是每进程设备判断消费者并发。 +const deviceEvalMaxInFlight = 10 + // shutdown 优雅退出：停止订阅，宽限期内等待执行中的卡任务完成；未完成的由租约到期后重新领取。 func (c *carrierSchedulerRuntime) shutdown() { if c.consumers != nil { c.consumers.StopAll(c.grace()) } + if c.srConsumers != nil { + c.srConsumers.StopAll(c.grace()) + c.devConsumers.StopAll(c.grace()) + } if c.producer != nil { c.producer.Stop() } diff --git a/internal/bootstrap/services.go b/internal/bootstrap/services.go index 355f6156..95821afa 100644 --- a/internal/bootstrap/services.go +++ b/internal/bootstrap/services.go @@ -283,6 +283,8 @@ func initServices(s *stores, deps *Dependencies) *services { } // 持通道阈值锁的卡在周期内拒绝一切复机（自动、手动、机卡分离）。 stopResumeService.SetChannelThresholdLockGuard(carrierThresholdService) + // 停复机指令队列：开关开启时自动入口只请求评估，由 Worker 指令执行器调用网关。 + stopResumeService.SetCommandQueue(carrierqueue.NewCommandQueue(carrierStore, carrierSettings)) iotCard.SetRealnameActivator(packageActivation) iotCard.SetStopResumeService(stopResumeService) iotCard.SetDeviceSimBindingStore(s.DeviceSimBinding) @@ -543,7 +545,7 @@ func initServices(s *stores, deps *Dependencies) *services { Device: device, DeviceImport: deviceImportSvc.New(deps.DB, s.DeviceImportTask, deps.QueueClient, auditWriter), AssetAllocationRecord: assetAllocationRecordSvc.New(deps.DB, s.AssetAllocationRecord, s.Shop, s.Account), - Carrier: carrierSvc.New(s.Carrier, auditWriter), + Carrier: newCarrierService(s.Carrier, auditWriter, stopResumeService, deps.Logger), PackageSeries: packageSeriesService, Package: packageService, PackageDailyRecord: packageSvc.NewDailyRecordService(deps.DB, deps.Redis, s.PackageUsageDailyRecord, deps.Logger), @@ -597,3 +599,12 @@ func initServices(s *stores, deps *Dependencies) *services { CardObservationSeries: cardObservationSeries, } } + +// newCarrierService 创建运营商配置服务并注入实名管控修改后的批量停复机评估。 +func newCarrierService(store *postgres.CarrierStore, auditWriter *auditInfra.Writer, stopResume *iotCardSvc.StopResumeService, logger *zap.Logger) *carrierSvc.Service { + service := carrierSvc.New(store, auditWriter) + if stopResume != nil { + service.SetRealnameControlEvaluator(stopResume, logger) + } + return service +} diff --git a/internal/domain/stopresume/scene.go b/internal/domain/stopresume/scene.go new file mode 100644 index 00000000..11ee4523 --- /dev/null +++ b/internal/domain/stopresume/scene.go @@ -0,0 +1,41 @@ +// Package stopresume 提供自动停复机请求评估场景在 ctx 中的传递，供各自动入口与停复机服务共用（无业务依赖）。 +package stopresume + +import ( + "context" + + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +type sceneKey struct{} + +// WithScene 声明本次自动停复机评估的触发场景（constants.StopResumeScene*）；未声明时按套餐步骤（周期评估）处理。 +func WithScene(ctx context.Context, scene string) context.Context { + if scene == "" { + return ctx + } + return context.WithValue(ctx, sceneKey{}, scene) +} + +// Scene 返回 ctx 中声明的场景；未声明返回套餐步骤场景。 +func Scene(ctx context.Context) string { + if scene, ok := ctx.Value(sceneKey{}).(string); ok && scene != "" { + return scene + } + return constants.StopResumeScenePolling +} + +// QueueAware 由停复机服务实现：停复机出口开关开启时请求评估只写共享缓存。 +type QueueAware interface { + CommandQueueEnabled(ctx context.Context) bool +} + +// RunAfterCommit 在业务事务提交后执行自动停复机请求：开关开启时同步执行（只写共享缓存，不起 goroutine）； +// 关闭时沿用旧路径在 goroutine 中当场执行（含网关调用与重试，不阻塞业务流程）。 +func RunAfterCommit(ctx context.Context, callback any, fn func()) { + if aware, ok := callback.(QueueAware); ok && aware.CommandQueueEnabled(ctx) { + fn() + return + } + go fn() +} diff --git a/internal/infrastructure/cardobservation/event.go b/internal/infrastructure/cardobservation/event.go index c4b3ed32..db0aa9c1 100644 --- a/internal/infrastructure/cardobservation/event.go +++ b/internal/infrastructure/cardobservation/event.go @@ -3,6 +3,7 @@ package cardobservation import ( "context" + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" "strconv" "time" @@ -168,7 +169,7 @@ func (c *NetworkChangedConsumer) Consume(ctx context.Context, envelope outbox.De } return errors.Wrap(errors.CodeDatabaseError, err, "查询网络变化后的卡事实失败") } - return c.evaluator.EvaluateAndAct(ctx, &card) + return c.evaluator.EvaluateAndAct(stopresumedomain.WithScene(ctx, constants.StopResumeSceneNetworkObserved), &card) } // NewRealnameChangedConsumer 创建卡实名变化事件消费者。 @@ -217,7 +218,7 @@ func (c *RealnameChangedConsumer) Consume(ctx context.Context, envelope outbox.D } return errors.Wrap(errors.CodeDatabaseError, err, "查询实名变化后的卡事实失败") } - return c.evaluator.EvaluateAndAct(ctx, &card) + return c.evaluator.EvaluateAndAct(stopresumedomain.WithScene(ctx, constants.StopResumeSceneRealnameObserved), &card) } // TrafficIncrementedConsumer 消费卡流量正增量并推进幂等副作用进度。 @@ -286,7 +287,7 @@ func (c *TrafficIncrementedConsumer) Consume(ctx context.Context, envelope outbo if err := c.db.WithContext(ctx).Where("id = ?", event.CardID).First(&card).Error; err != nil { return errors.Wrap(errors.CodeDatabaseError, err, "查询流量变化后的卡事实失败") } - if err := c.evaluator.EvaluateAndAct(ctx, &card); err != nil { + if err := c.evaluator.EvaluateAndAct(stopresumedomain.WithScene(ctx, constants.StopResumeSceneFlowObserved), &card); err != nil { return err } } diff --git a/internal/model/dto/carrier_polling_dto.go b/internal/model/dto/carrier_polling_dto.go index da3a9873..e240d2d7 100644 --- a/internal/model/dto/carrier_polling_dto.go +++ b/internal/model/dto/carrier_polling_dto.go @@ -145,6 +145,21 @@ type CardPollingTypeState struct { PendingConfirm []int `json:"pending_confirm" description:"待执行的延迟确认（秒）"` } +// CardStopResumeCommandState 卡当前停复机指令（停复机指令队列）。 +type CardStopResumeCommandState struct { + Action string `json:"action" description:"动作 (stop:停机, start:复机)"` + Reason string `json:"reason" description:"停机原因"` + Scenes []string `json:"scenes" description:"触发场景"` + Source string `json:"source" description:"请求来源"` + Version int64 `json:"version" description:"指令版本"` + FailCount int `json:"fail_count" description:"连续失败次数"` + LastError string `json:"last_error" description:"最近错误（类别|信息）"` + Pending bool `json:"pending" description:"结果未知，待确认"` + Unknown int `json:"unknown" description:"连续结果未知次数"` + WrittenAt *time.Time `json:"written_at" description:"最近写入时间"` + NextAt *time.Time `json:"next_at" description:"下次可执行时间（空表示不在队列中）"` +} + // CardPollingStateResponse 单卡轮询状态（只读）。 type CardPollingStateResponse struct { CardID uint `json:"card_id" description:"卡ID"` @@ -158,6 +173,10 @@ type CardPollingStateResponse struct { ClaimExpireAt *time.Time `json:"claim_expire_at" description:"处理中标记到期时间"` PriorityMark bool `json:"priority_mark" description:"执行中收到请求检查，收尾后进入优先队列"` Types []CardPollingTypeState `json:"types" description:"各检查类型状态"` + + StopResumeQueueEnabled bool `json:"stop_resume_queue_enabled" description:"停复机出口开关是否开启"` + StopResumeCommand *CardStopResumeCommandState `json:"stop_resume_command" description:"当前停复机指令（无指令为空）"` + LastStopResumeCallAt *time.Time `json:"last_stop_resume_call_at" description:"最近一次停复机网关调用时间（单卡最小重试间隔依据）"` } // CarrierPollingStatsItem 按运营商账户的轮询监控。 @@ -175,6 +194,15 @@ type CarrierPollingStatsItem struct { ExpiredClaims1h int64 `json:"expired_claims_1h" description:"近 1 小时处理中标记到期次数（执行崩溃或超时）"` OpenAlerts int64 `json:"open_alerts" description:"未处理卡级失败告警数"` OpenAlertsByType map[string]int64 `json:"open_alerts_by_category" description:"未处理卡级失败告警数（按错误类别）"` + + StopResumeQueueLength int64 `json:"stop_resume_queue_length" description:"停复机指令队列长度"` + StopResumeDue int64 `json:"stop_resume_due" description:"已到期停复机指令数"` + StopResumeRunning int64 `json:"stop_resume_running" description:"执行中停复机指令数"` + StopResumePending int64 `json:"stop_resume_pending" description:"结果未知待确认的停复机指令数（按队列前 500 条统计）"` + StopResumeSuccess1h int64 `json:"stop_resume_success_1h" description:"近 1 小时停复机成功数"` + StopResumeFailed1h int64 `json:"stop_resume_failed_1h" description:"近 1 小时停复机失败数（计入连续失败或终态）"` + StopResumeRetry1h int64 `json:"stop_resume_retry_1h" description:"近 1 小时停复机推后重试数（限流、频率限制、最小间隔）"` + StopResumeUnknown1h int64 `json:"stop_resume_unknown_1h" description:"近 1 小时停复机结果未知数"` } // CarrierPollingStatsResponse 按运营商账户的轮询监控。 diff --git a/internal/polling/carrierqueue/dispatcher.go b/internal/polling/carrierqueue/dispatcher.go index 02695bd8..5459de24 100644 --- a/internal/polling/carrierqueue/dispatcher.go +++ b/internal/polling/carrierqueue/dispatcher.go @@ -41,6 +41,12 @@ type CarrierParams struct { CardMaxInFlight int PriorityRatio float64 Configured bool // 是否有该账户的配置行 + + // 停复机指令（add-stop-resume-command-queue）。 + SRInflightLimit int // 账户停复机执行上限 + SRMaxInFlight int // 每进程停复机消费者 max_in_flight + SRMinInterval time.Duration // 单卡最小重试间隔（停机与复机合计） + SRConfirmDelay time.Duration // 结果未知后的确认延迟 } // carrierEntry 是分发器缓存的账户信息。 @@ -110,11 +116,16 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) { d.logger.Error("分发器 panic，已恢复", zap.Any("panic", r)) } }() - if !d.Ready(ctx) { + // 停复机指令与设备判断的分发只看停复机出口开关，不受 scheduler_mode 限制；卡任务只在新调度就绪时领取。 + stopResumeOn := d.settings.Current(ctx).StopResumeQueueEnabled + cardsReady := d.Ready(ctx) + if !stopResumeOn && !cardsReady { return } d.store.redis.HSet(ctx, constants.RedisCarrierSchedulerHeartbeatKey, d.instanceID, time.Now().UnixMilli()) - d.drainLegacyHints(ctx) + if cardsReady { + d.drainLegacyHints(ctx) + } d.refreshCarriers(ctx) d.mu.Lock() carriers := append([]carrierEntry(nil), d.carriers...) @@ -123,6 +134,9 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) { if lockTTL <= 0 { lockTTL = 900 * time.Millisecond } + if stopResumeOn { + d.dispatchDeviceEvals(ctx, lockTTL) + } for _, carrier := range carriers { if ctx.Err() != nil { return @@ -131,10 +145,21 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) { if err != nil || !ok { continue } - d.dispatchCarrier(ctx, carrier) + // 停复机优先：先领取停复机指令占用本秒名额，再领取卡任务。 + if stopResumeOn { + d.dispatchStopResume(ctx, carrier.id) + } + if cardsReady { + d.dispatchCarrier(ctx, carrier) + } } } +// StopResumeQueueEnabled 返回停复机出口开关当前值。 +func (d *Dispatcher) StopResumeQueueEnabled(ctx context.Context) bool { + return d.settings.Current(ctx).StopResumeQueueEnabled +} + func (d *Dispatcher) dispatchCarrier(ctx context.Context, carrier carrierEntry) { params := d.ParamsFor(carrier.id) limit := params.InflightLimit @@ -265,7 +290,10 @@ func (d *Dispatcher) ParamsFor(carrierID uint) CarrierParams { // MergeParams 合并账户配置行与全局默认。 func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.CarrierSchedulerConfig) CarrierParams { - params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured} + params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured, + SRInflightLimit: constants.DefaultStopResumeInflightLimit, SRMaxInFlight: constants.DefaultStopResumeMaxInFlight, + SRMinInterval: constants.DefaultStopResumeMinIntervalSeconds * time.Second, + SRConfirmDelay: constants.DefaultStopResumeConfirmDelaySeconds * time.Second} if params.InflightLimit <= 0 { params.InflightLimit = 20 } @@ -284,6 +312,18 @@ func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.Carr if row.PriorityMaxRatio != nil && *row.PriorityMaxRatio > 0 { params.PriorityRatio = *row.PriorityMaxRatio } + if row.SRInflightLimit != nil && *row.SRInflightLimit > 0 { + params.SRInflightLimit = *row.SRInflightLimit + } + if row.StopResumeMaxInFlight != nil && *row.StopResumeMaxInFlight > 0 { + params.SRMaxInFlight = *row.StopResumeMaxInFlight + } + if row.StopResumeMinInterval != nil && *row.StopResumeMinInterval >= 0 { + params.SRMinInterval = time.Duration(*row.StopResumeMinInterval) * time.Second + } + if row.StopResumeConfirmDelay != nil && *row.StopResumeConfirmDelay > 0 { + params.SRConfirmDelay = time.Duration(*row.StopResumeConfirmDelay) * time.Second + } return params } diff --git a/internal/polling/carrierqueue/settings.go b/internal/polling/carrierqueue/settings.go index d113307a..1b44f524 100644 --- a/internal/polling/carrierqueue/settings.go +++ b/internal/polling/carrierqueue/settings.go @@ -51,6 +51,14 @@ func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logg DefaultValue: "600", Description: "账户级告警统计窗口（秒）", Min: &minOne, Max: &maxSeconds}, {Key: constants.SystemConfigPollingAccountAlertThreshold, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, DefaultValue: "50", Description: "账户级告警阈值：窗口内同账户新增失败告警数", Min: &minOne, Max: &maxCount}, + {Key: constants.SystemConfigStopResumeQueueEnabled, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeBool, + DefaultValue: "false", Description: "自动停复机经停复机指令队列执行（全局开关，关闭即各入口当场执行）"}, + {Key: constants.SystemConfigStopResumeUnknownAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, + DefaultValue: "3", Description: "停复机连续结果未知告警阈值", Min: &minOne, Max: &maxCount}, + {Key: constants.SystemConfigStopResumeBackoff, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON, + DefaultValue: `[600,900,1800,3600]`, Description: "停复机失败退避序列（秒，递增）", Validator: validateSequence}, + {Key: constants.SystemConfigStopResumeAlertFailCount, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, + DefaultValue: "5", Description: "停复机连续失败告警阈值", Min: &minOne, Max: &maxCount}, } for _, definition := range definitions { if _, exists := registry.Get(definition.Key); exists { @@ -107,6 +115,23 @@ func validateStepTimeout(raw string) error { return nil } +// validateSequence 校验非空递增的正整数秒序列（停复机退避）。 +func validateSequence(raw string) error { + var values []int + if err := json.Unmarshal([]byte(raw), &values); err != nil || len(values) == 0 { + return stderrors.New("退避序列必须是非空的秒数组") + } + for index, value := range values { + if value <= 0 { + return stderrors.New("退避间隔必须为正数") + } + if index > 0 && value <= values[index-1] { + return stderrors.New("退避序列必须递增") + } + } + return nil +} + func validateBackoff(raw string) error { var backoff map[string][]int if err := json.Unmarshal([]byte(raw), &backoff); err != nil { @@ -171,6 +196,28 @@ type Settings struct { ImmediateAlert map[string]bool AccountAlertWindow time.Duration AccountAlertLimit int + + // 停复机指令队列（add-stop-resume-command-queue）。 + StopResumeQueueEnabled bool + SRUnknownAlert int + SRBackoff []time.Duration + SRAlertFailCount int +} + +// SRBackoffFor 返回停复机第 n 次连续失败（n 从 1 起）的退避间隔；超过序列长度取最后一项。 +func (s Settings) SRBackoffFor(failCount int) time.Duration { + sequence := s.SRBackoff + if len(sequence) == 0 { + sequence = []time.Duration{10 * time.Minute} + } + index := failCount - 1 + if index < 0 { + index = 0 + } + if index >= len(sequence) { + index = len(sequence) - 1 + } + return sequence[index] } // CarrierMode 判断当前是否为按运营商账户调度。 @@ -243,6 +290,9 @@ func DefaultSettings() Settings { AlertFailCount: map[string]int{"default": 5}, AlertFailDuration: 6 * time.Hour, ImmediateAlert: map[string]bool{constants.GatewayClassAccountConfig: true}, AccountAlertWindow: 10 * time.Minute, AccountAlertLimit: 50, + SRUnknownAlert: 3, + SRBackoff: []time.Duration{10 * time.Minute, 15 * time.Minute, 30 * time.Minute, time.Hour}, + SRAlertFailCount: 5, } } @@ -364,6 +414,8 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) { {constants.SystemConfigPollingAlertFailDuration, func(v int) { s.AlertFailDuration = time.Duration(v) * time.Second }}, {constants.SystemConfigPollingAccountAlertWindow, func(v int) { s.AccountAlertWindow = time.Duration(v) * time.Second }}, {constants.SystemConfigPollingAccountAlertThreshold, func(v int) { s.AccountAlertLimit = v }}, + {constants.SystemConfigStopResumeUnknownAlert, func(v int) { s.SRUnknownAlert = v }}, + {constants.SystemConfigStopResumeAlertFailCount, func(v int) { s.SRAlertFailCount = v }}, } for _, item := range ints { raw, err := get(item.key) @@ -376,6 +428,21 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) { } item.set(value) } + if raw, err = get(constants.SystemConfigStopResumeQueueEnabled); err != nil { + return s, err + } + s.StopResumeQueueEnabled = raw == "true" + if raw, err = get(constants.SystemConfigStopResumeBackoff); err != nil { + return s, err + } + var srBackoff []int + if err := json.Unmarshal([]byte(raw), &srBackoff); err != nil || validateSequence(raw) != nil { + return s, stderrors.New("停复机退避序列配置非法") + } + s.SRBackoff = nil + for _, value := range srBackoff { + s.SRBackoff = append(s.SRBackoff, time.Duration(value)*time.Second) + } return s, nil } diff --git a/internal/polling/carrierqueue/sr_dispatch.go b/internal/polling/carrierqueue/sr_dispatch.go new file mode 100644 index 00000000..b39fefa7 --- /dev/null +++ b/internal/polling/carrierqueue/sr_dispatch.go @@ -0,0 +1,120 @@ +package carrierqueue + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// StopResumeMessage 是停复机指令 NSQ 消息体。 +type StopResumeMessage struct { + CardID uint `json:"card_id"` + CarrierID uint `json:"carrier_id"` + Token string `json:"token"` + Version int64 `json:"version"` + Action string `json:"action"` + Pending bool `json:"pending"` + DispatchedAt int64 `json:"dispatched_at"` +} + +// DeviceEvalMessage 是设备判断 NSQ 消息体。 +type DeviceEvalMessage struct { + DeviceID uint `json:"device_id"` + Token string `json:"token"` + Version string `json:"version"` + Scenes []string `json:"scenes"` + DispatchedAt int64 `json:"dispatched_at"` +} + +// deviceEvalLimit 是全局同时执行的设备判断上限：判断只读数据库与写指令，毫秒级。 +const deviceEvalLimit = 50 + +// stopResumeLease 是停复机指令与设备判断的租约：单次执行只调一次网关，远小于该值。 +func (d *Dispatcher) stopResumeLease() time.Duration { + return 2 * time.Minute +} + +// dispatchStopResume 在卡任务之前领取账户停复机指令：领取数 = min(停复机所在限流组本秒剩余, 执行上限 − 执行中)。 +func (d *Dispatcher) dispatchStopResume(ctx context.Context, carrierID uint) { + params := d.ParamsFor(carrierID) + capacity := params.SRInflightLimit + processes := d.cfg.SubscriberProcesses + if processes <= 0 { + processes = 1 + } + if consume := params.SRMaxInFlight * processes; consume > 0 && capacity > consume { + capacity = consume + } + if d.budgets != nil { + ops := []string{constants.GatewayOpStop, constants.GatewayOpStart} + if groups, _, limited := d.budgets.Budget(ctx, carrierID, ops); limited { + for _, remaining := range groups { + if remaining < capacity { + capacity = remaining + } + } + } + } + if capacity <= 0 { + return + } + claimed, expired, err := d.store.ClaimStopResume(ctx, SRClaimInput{ + CarrierID: carrierID, Lease: d.stopResumeLease(), Capacity: capacity, MinInterval: params.SRMinInterval, + TokenPrefix: d.instanceID + ":sr:" + strconv.FormatInt(time.Now().UnixNano(), 36), + }) + if err != nil { + d.logger.Warn("运营商账户停复机指令领取失败", zap.Uint("carrier_id", carrierID), zap.Error(err)) + return + } + if expired > 0 { + d.logger.Warn("停复机指令处理中标记到期（执行崩溃或超时）", zap.Uint("carrier_id", carrierID), zap.Int64("count", expired), + zap.String("metric", "stop_resume.claim_expired")) + } + topic := constants.NSQStopResumeTopic(carrierID) + for _, item := range claimed { + body, _ := json.Marshal(StopResumeMessage{CardID: item.CardID, CarrierID: carrierID, Token: item.Token, Version: item.Version, + Action: item.Action, Pending: item.Pending, DispatchedAt: time.Now().UnixMilli()}) + if err := d.publisher.Publish(topic, body); err != nil { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + if releaseErr := d.store.ReleaseStopResume(releaseCtx, item.CardID, carrierID, item.Token); releaseErr != nil { + d.logger.Warn("发布失败后释放停复机指令失败", zap.Uint("card_id", item.CardID), zap.Error(releaseErr)) + } + cancel() + d.logger.Warn("发布停复机指令失败，已释放", zap.Uint("card_id", item.CardID), zap.String("topic", topic), zap.Error(err)) + } + } +} + +// dispatchDeviceEvals 领取到期设备判断并发布；全局分发锁保证每轮只有一个实例领取。 +func (d *Dispatcher) dispatchDeviceEvals(ctx context.Context, lockTTL time.Duration) { + ok, err := d.store.redis.SetNX(ctx, constants.RedisDeviceEvalDispatchLockKey, d.instanceID, lockTTL).Result() + if err != nil || !ok { + return + } + claimed, expired, err := d.store.ClaimDeviceEvals(ctx, d.stopResumeLease(), deviceEvalLimit, + d.instanceID+":dev:"+strconv.FormatInt(time.Now().UnixNano(), 36)) + if err != nil { + d.logger.Warn("设备判断领取失败", zap.Error(err)) + return + } + if expired > 0 { + d.logger.Warn("设备判断处理中标记到期（执行崩溃或超时）", zap.Int64("count", expired), zap.String("metric", "stop_resume.device_eval_expired")) + } + for _, item := range claimed { + body, _ := json.Marshal(DeviceEvalMessage{DeviceID: item.DeviceID, Token: item.Token, Version: item.Version, Scenes: item.Scenes, + DispatchedAt: time.Now().UnixMilli()}) + if err := d.publisher.Publish(constants.NSQDeviceEvalTopic, body); err != nil { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + if releaseErr := d.store.ReleaseDeviceEval(releaseCtx, item.DeviceID, item.Token); releaseErr != nil { + d.logger.Warn("发布失败后释放设备判断失败", zap.Uint("device_id", item.DeviceID), zap.Error(releaseErr)) + } + cancel() + d.logger.Warn("发布设备判断失败，已释放", zap.Uint("device_id", item.DeviceID), zap.Error(err)) + } + } +} diff --git a/internal/polling/carrierqueue/sr_queue.go b/internal/polling/carrierqueue/sr_queue.go new file mode 100644 index 00000000..eee98b24 --- /dev/null +++ b/internal/polling/carrierqueue/sr_queue.go @@ -0,0 +1,29 @@ +package carrierqueue + +import "context" + +// CommandQueue 是停复机指令队列写入端：开关读取受控系统配置，写入走共享缓存 Lua。 +type CommandQueue struct { + store *Store + settings *SettingsProvider +} + +// NewCommandQueue 创建停复机指令队列写入端。 +func NewCommandQueue(store *Store, settings *SettingsProvider) *CommandQueue { + return &CommandQueue{store: store, settings: settings} +} + +// Enabled 返回停复机出口开关 polling.stop_resume.queue_enabled。 +func (q *CommandQueue) Enabled(ctx context.Context) bool { + return q != nil && q.store != nil && q.settings.Current(ctx).StopResumeQueueEnabled +} + +// WriteCommand 写入或覆盖卡的停复机指令。 +func (q *CommandQueue) WriteCommand(ctx context.Context, cmd StopResumeCommand) (int64, error) { + return q.store.WriteCommand(ctx, cmd) +} + +// RequestDeviceEval 请求一次设备维度判断。 +func (q *CommandQueue) RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error { + return q.store.RequestDeviceEval(ctx, deviceID, scenes) +} diff --git a/internal/polling/carrierqueue/sr_scripts.go b/internal/polling/carrierqueue/sr_scripts.go new file mode 100644 index 00000000..c1935721 --- /dev/null +++ b/internal/polling/carrierqueue/sr_scripts.go @@ -0,0 +1,225 @@ +package carrierqueue + +import "github.com/redis/go-redis/v9" + +// 停复机指令 Hash 字段（见 constants.RedisCardStopResumeCmdKey）。 +const ( + srFieldAction = "action" // stop / start + srFieldReason = "reason" // 停机原因（复机为空） + srFieldScenes = "scenes" // 触发场景（逗号分隔） + srFieldSource = "source" // 请求来源 + srFieldCarrier = "carrier" // 所属运营商账户 + srFieldVersion = "ver" // 指令版本，每次写入 HINCRBY + srFieldFail = "fail" // 连续失败次数 + srFieldFirst = "ffail" // 首次失败（毫秒） + srFieldError = "err" // 最近错误（类别|信息） + srFieldPending = "pending" // 1 表示结果未知、待确认 + srFieldUnknown = "unknown" // 连续结果未知次数 + srFieldAt = "at" // 最近写入时间（毫秒） +) + +// srWriteLua 写入或覆盖停复机指令：方向相反时清零失败与待确认，递增版本，放入账户停复机队列。 +// 指令执行中（存在处理中标记）时只更新指令内容与版本，由收尾按版本保留；待确认中且方向不变时保持确认时间。 +// +// KEYS[1] 指令 Hash；KEYS[2] 账户停复机队列；KEYS[3] 处理中标记。 +// ARGV：action、reason、scenes、source、carrier、now_ms、card_id。返回新版本号。 +const srWriteLua = ` +local prev = redis.call('HGET', KEYS[1], 'action') +if prev and prev ~= ARGV[1] then + redis.call('HDEL', KEYS[1], 'fail', 'ffail', 'err', 'pending', 'unknown') +end +redis.call('HSET', KEYS[1], 'action', ARGV[1], 'reason', ARGV[2], 'scenes', ARGV[3], 'source', ARGV[4], 'carrier', ARGV[5], 'at', ARGV[6]) +local ver = redis.call('HINCRBY', KEYS[1], 'ver', 1) +if redis.call('EXISTS', KEYS[3]) == 1 then return ver end +if redis.call('HGET', KEYS[1], 'pending') == '1' then + redis.call('ZADD', KEYS[2], 'NX', tonumber(ARGV[6]), ARGV[7]) +else + redis.call('ZADD', KEYS[2], 'LT', tonumber(ARGV[6]), ARGV[7]) +end +return ver +` + +// srClaimLua 领取账户到期停复机指令：先清理过期执行中，按容量领取；未到单卡最小重试间隔的推后到可调用时间。 +// 待确认的指令只查状态，不受最小间隔约束。指令已不存在的成员直接移出队列。 +// +// KEYS[1] 账户停复机队列；KEYS[2] 账户停复机执行中计数。 +// ARGV：now_ms、lease_ms、容量（执行上限与剩余 QPS 的较小值）、token 前缀、单卡最小间隔 ms。 +// 返回 {过期数, card_id, token, ver, action, pending, ...}。 +const srClaimLua = ` +local now = tonumber(ARGV[1]) +local lease = tonumber(ARGV[2]) +local limit = tonumber(ARGV[3]) +local prefix = ARGV[4] +local minInterval = tonumber(ARGV[5]) +local expired = redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now) +local result = {expired} +local capacity = limit - redis.call('ZCARD', KEYS[2]) +if capacity <= 0 then return result end +local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', now, 'LIMIT', 0, capacity * 2 + 10) +local taken = 0 +for _, id in ipairs(ids) do + if taken >= capacity then break end + local cmdKey = 'polling:card:' .. id .. ':stop_resume_cmd' + local fields = redis.call('HMGET', cmdKey, 'action', 'ver', 'pending') + if not fields[1] then + redis.call('ZREM', KEYS[1], id) + else + local pending = fields[3] == '1' + local blockedUntil = 0 + if not pending then + local last = tonumber(redis.call('GET', 'polling:card:' .. id .. ':sr_last_call') or '0') + if last > 0 and last + minInterval > now then blockedUntil = last + minInterval end + end + if blockedUntil > 0 then + redis.call('ZADD', KEYS[1], 'XX', blockedUntil, id) + else + local claimKey = 'polling:card:' .. id .. ':sr_claim' + local token = prefix .. ':' .. id + if redis.call('SET', claimKey, token, 'NX', 'PX', lease) then + redis.call('ZADD', KEYS[1], 'XX', now + lease, id) + redis.call('ZADD', KEYS[2], now + lease, token) + taken = taken + 1 + table.insert(result, id) + table.insert(result, token) + table.insert(result, fields[2] or '0') + table.insert(result, fields[1]) + table.insert(result, pending and '1' or '0') + else + local ttl = redis.call('PTTL', claimKey) + if ttl < 0 then ttl = lease end + redis.call('ZADD', KEYS[1], 'XX', now + ttl, id) + end + end + end +end +return result +` + +// srFinishLua 停复机指令收尾：token 不一致返回 0 不做任何修改；版本已变大（执行期间写入了新指令）时保留新指令、立即可领取（返回 2）； +// 否则 done=1 删除指令并移出队列，done=0 写入字段并按 next_ms 重新排队（返回 1）。 +// +// KEYS[1] 指令 Hash；KEYS[2] 处理中标记；KEYS[3] 执行权；KEYS[4] 执行中计数；KEYS[5] 账户停复机队列。 +// ARGV：token、card_id、领取时版本、done(1/0)、next_ms、now_ms、set 字段对数 n、字段1、值1、…、删除字段…。 +const srFinishLua = ` +if redis.call('GET', KEYS[2]) ~= ARGV[1] then return 0 end +redis.call('DEL', KEYS[2]) +redis.call('DEL', KEYS[3]) +redis.call('ZREM', KEYS[4], ARGV[1]) +local cardId = ARGV[2] +if (redis.call('HGET', KEYS[1], 'ver') or '0') ~= ARGV[3] then + if redis.call('EXISTS', KEYS[1]) == 1 then + redis.call('ZADD', KEYS[5], tonumber(ARGV[6]), cardId) + else + redis.call('ZREM', KEYS[5], cardId) + end + return 2 +end +if ARGV[4] == '1' then + redis.call('DEL', KEYS[1]) + redis.call('ZREM', KEYS[5], cardId) + return 1 +end +local n = tonumber(ARGV[7]) +local idx = 8 +for i = 1, n do + redis.call('HSET', KEYS[1], ARGV[idx], ARGV[idx + 1]) + idx = idx + 2 +end +while idx <= #ARGV do + redis.call('HDEL', KEYS[1], ARGV[idx]) + idx = idx + 1 +end +redis.call('ZADD', KEYS[5], tonumber(ARGV[5]), cardId) +return 1 +` + +// srReleaseLua 发布失败：按 token 删除处理中标记、移出执行中计数，score 恢复为当前时间。 +// KEYS[1] 处理中标记；KEYS[2] 执行中计数；KEYS[3] 队列。ARGV：token、member、now_ms。 +const srReleaseLua = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end +redis.call('DEL', KEYS[1]) +redis.call('ZREM', KEYS[2], ARGV[1]) +redis.call('ZADD', KEYS[3], 'XX', tonumber(ARGV[3]), ARGV[2]) +return 1 +` + +// deviceEvalRequestLua 请求设备判断：递增版本、合并场景、设备进入判断队列（同一设备一个成员，ZADD LT 不推后已有更早时间）。 +// KEYS[1] 版本；KEYS[2] 场景集合；KEYS[3] 设备判断队列；KEYS[4] 处理中标记。ARGV：device_id、now_ms、场景…。 +const deviceEvalRequestLua = ` +redis.call('INCR', KEYS[1]) +for i = 3, #ARGV do redis.call('SADD', KEYS[2], ARGV[i]) end +if redis.call('EXISTS', KEYS[4]) == 1 then return 1 end +redis.call('ZADD', KEYS[3], 'LT', tonumber(ARGV[2]), ARGV[1]) +return 1 +` + +// deviceEvalClaimLua 领取到期设备判断：写处理中标记并记下版本与场景快照。 +// KEYS[1] 设备判断队列；KEYS[2] 执行中计数。ARGV：now_ms、lease_ms、上限、token 前缀。 +// 返回 {过期数, device_id, token, ver, scenes, ...}。 +const deviceEvalClaimLua = ` +local now = tonumber(ARGV[1]) +local lease = tonumber(ARGV[2]) +local limit = tonumber(ARGV[3]) +local expired = redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now) +local result = {expired} +local capacity = limit - redis.call('ZCARD', KEYS[2]) +if capacity <= 0 then return result end +local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', now, 'LIMIT', 0, capacity * 2 + 10) +local taken = 0 +for _, id in ipairs(ids) do + if taken >= capacity then break end + local claimKey = 'polling:device:' .. id .. ':eval_claim' + local token = ARGV[4] .. ':' .. id + if redis.call('SET', claimKey, token, 'NX', 'PX', lease) then + redis.call('ZADD', KEYS[1], 'XX', now + lease, id) + redis.call('ZADD', KEYS[2], now + lease, token) + taken = taken + 1 + local ver = redis.call('GET', 'polling:device:' .. id .. ':eval_ver') or '0' + local scenes = redis.call('SMEMBERS', 'polling:device:' .. id .. ':eval_scenes') + table.insert(result, id) + table.insert(result, token) + table.insert(result, ver) + table.insert(result, table.concat(scenes, ',')) + else + local ttl = redis.call('PTTL', claimKey) + if ttl < 0 then ttl = lease end + redis.call('ZADD', KEYS[1], 'XX', now + ttl, id) + end +end +return result +` + +// deviceEvalFinishLua 设备判断收尾：只移除本次已处理的场景；版本变大时立即再判断（返回 2）； +// defer_ms > 0 表示本次未判断（旧路径持设备锁），推后再判断，场景不移除。 +// KEYS[1] 处理中标记；KEYS[2] 执行权；KEYS[3] 执行中计数；KEYS[4] 版本；KEYS[5] 场景集合；KEYS[6] 队列。 +// ARGV：token、device_id、领取时版本、now_ms、defer_ms、已处理场景…。 +const deviceEvalFinishLua = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end +redis.call('DEL', KEYS[1]) +redis.call('DEL', KEYS[2]) +redis.call('ZREM', KEYS[3], ARGV[1]) +local now = tonumber(ARGV[4]) +local deferMs = tonumber(ARGV[5]) +if deferMs > 0 then + redis.call('ZADD', KEYS[6], now + deferMs, ARGV[2]) + return 3 +end +for i = 6, #ARGV do redis.call('SREM', KEYS[5], ARGV[i]) end +if (redis.call('GET', KEYS[4]) or '0') ~= ARGV[3] then + redis.call('ZADD', KEYS[6], now, ARGV[2]) + return 2 +end +redis.call('ZREM', KEYS[6], ARGV[2]) +redis.call('DEL', KEYS[5]) +return 1 +` + +var ( + srWriteScript = redis.NewScript(srWriteLua) + srClaimScript = redis.NewScript(srClaimLua) + srFinishScript = redis.NewScript(srFinishLua) + srReleaseScript = redis.NewScript(srReleaseLua) + deviceRequestScript = redis.NewScript(deviceEvalRequestLua) + deviceClaimScript = redis.NewScript(deviceEvalClaimLua) + deviceFinishScript = redis.NewScript(deviceEvalFinishLua) +) diff --git a/internal/polling/carrierqueue/sr_store.go b/internal/polling/carrierqueue/sr_store.go new file mode 100644 index 00000000..703deb9b --- /dev/null +++ b/internal/polling/carrierqueue/sr_store.go @@ -0,0 +1,348 @@ +package carrierqueue + +import ( + "context" + stderrors "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// StopResumeCommand 是写入停复机队列的一条指令。 +type StopResumeCommand struct { + CardID uint + CarrierID uint + Action string // constants.StopResumeActionStop / StopResumeActionStart + Reason string // 停机原因 + Scenes []string // 触发场景 + Source string // 请求来源（日志与排查） +} + +// CommandState 是卡当前停复机指令的解析结果。 +type CommandState struct { + Exists bool + Action string + Reason string + Scenes []string + Source string + CarrierID uint + Version int64 + FailCount int + FirstFailed time.Time + LastError string + Pending bool + Unknown int + WrittenAt time.Time + NextAt time.Time // 队列中的可执行时间（零值表示不在队列中） + LastCallAt time.Time +} + +// WriteCommand 写入或覆盖卡的停复机指令，返回新版本号。 +func (s *Store) WriteCommand(ctx context.Context, cmd StopResumeCommand) (int64, error) { + if cmd.CardID == 0 || cmd.CarrierID == 0 { + return 0, stderrors.New("停复机指令缺少卡或运营商账户") + } + return srWriteScript.Run(ctx, s.redis, + []string{constants.RedisCardStopResumeCmdKey(cmd.CardID), constants.RedisCarrierStopResumeQueueKey(cmd.CarrierID), + constants.RedisCardStopResumeClaimKey(cmd.CardID)}, + cmd.Action, cmd.Reason, constants.JoinStopResumeScenes(cmd.Scenes), cmd.Source, cmd.CarrierID, time.Now().UnixMilli(), cmd.CardID, + ).Int64() +} + +// SRClaimInput 是一次账户停复机领取的参数。 +type SRClaimInput struct { + CarrierID uint + Lease time.Duration + Capacity int // 执行上限与剩余 QPS 的较小值 + TokenPrefix string + MinInterval time.Duration +} + +// SRClaimed 是一条被领取的停复机指令。 +type SRClaimed struct { + CardID uint + Token string + Version int64 + Action string + Pending bool +} + +// ClaimStopResume 领取账户到期停复机指令，返回领取结果与本轮清理的过期执行数。 +func (s *Store) ClaimStopResume(ctx context.Context, input SRClaimInput) ([]SRClaimed, int64, error) { + raw, err := srClaimScript.Run(ctx, s.redis, + []string{constants.RedisCarrierStopResumeQueueKey(input.CarrierID), constants.RedisCarrierStopResumeRunningKey(input.CarrierID)}, + time.Now().UnixMilli(), input.Lease.Milliseconds(), input.Capacity, input.TokenPrefix, input.MinInterval.Milliseconds(), + ).Slice() + if err != nil || len(raw) == 0 { + return nil, 0, err + } + expired, _ := raw[0].(int64) + claimed := make([]SRClaimed, 0, (len(raw)-1)/5) + for index := 1; index+4 < len(raw); index += 5 { + cardID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64) + version, _ := strconv.ParseInt(fmt.Sprint(raw[index+2]), 10, 64) + claimed = append(claimed, SRClaimed{CardID: uint(cardID), Token: fmt.Sprint(raw[index+1]), Version: version, + Action: fmt.Sprint(raw[index+3]), Pending: fmt.Sprint(raw[index+4]) == "1"}) + } + return claimed, expired, nil +} + +// AcquireStopResumeExec 抢停复机指令执行权：1 抢到，0 处理中标记已不是该 token，-1 已有执行者。 +func (s *Store) AcquireStopResumeExec(ctx context.Context, cardID uint, token, execValue string, ttl time.Duration) (int64, error) { + return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisCardStopResumeClaimKey(cardID), constants.RedisCardStopResumeExecKey(cardID)}, + token, execValue, ttl.Milliseconds()).Int64() +} + +// SRFinish 是一次停复机指令收尾。 +type SRFinish struct { + CardID uint + CarrierID uint + Token string + Version int64 + Done bool // true 结束并删除指令 + NextAt time.Time + Set map[string]string + Del []string +} + +// FinishStopResume 停复机指令收尾：返回 0 token 不一致、1 正常、2 版本已变大保留新指令。 +func (s *Store) FinishStopResume(ctx context.Context, finish SRFinish) (int64, error) { + done := "0" + if finish.Done { + done = "1" + } + now := time.Now() + next := finish.NextAt + if next.IsZero() || next.Before(now) { + next = now + } + args := []any{finish.Token, finish.CardID, finish.Version, done, next.UnixMilli(), now.UnixMilli(), len(finish.Set)} + for field, value := range finish.Set { + args = append(args, field, value) + } + for _, field := range finish.Del { + args = append(args, field) + } + return srFinishScript.Run(ctx, s.redis, + []string{constants.RedisCardStopResumeCmdKey(finish.CardID), constants.RedisCardStopResumeClaimKey(finish.CardID), + constants.RedisCardStopResumeExecKey(finish.CardID), constants.RedisCarrierStopResumeRunningKey(finish.CarrierID), + constants.RedisCarrierStopResumeQueueKey(finish.CarrierID)}, + args...).Int64() +} + +// ReleaseStopResume 发布失败时释放停复机指令处理中标记并恢复为立即可领取。 +func (s *Store) ReleaseStopResume(ctx context.Context, cardID, carrierID uint, token string) error { + return srReleaseScript.Run(ctx, s.redis, + []string{constants.RedisCardStopResumeClaimKey(cardID), constants.RedisCarrierStopResumeRunningKey(carrierID), + constants.RedisCarrierStopResumeQueueKey(carrierID)}, + token, cardID, time.Now().UnixMilli()).Err() +} + +// MarkStopResumeCall 记录本卡一次停复机网关调用（停机与复机合计），供单卡最小重试间隔判断。 +func (s *Store) MarkStopResumeCall(ctx context.Context, cardID uint, at time.Time, keep time.Duration) error { + if keep < time.Hour { + keep = time.Hour + } + return s.redis.Set(ctx, constants.RedisCardStopResumeLastCallKey(cardID), at.UnixMilli(), keep).Err() +} + +// LastStopResumeCall 返回本卡最近一次停复机网关调用时间；没有记录返回零值。 +func (s *Store) LastStopResumeCall(ctx context.Context, cardID uint) (time.Time, error) { + ms, err := s.redis.Get(ctx, constants.RedisCardStopResumeLastCallKey(cardID)).Int64() + if stderrors.Is(err, redis.Nil) { + return time.Time{}, nil + } + if err != nil { + return time.Time{}, err + } + return time.UnixMilli(ms), nil +} + +// ReadCommand 读取卡当前停复机指令与队列位置。 +func (s *Store) ReadCommand(ctx context.Context, cardID uint) (CommandState, error) { + raw, err := s.redis.HGetAll(ctx, constants.RedisCardStopResumeCmdKey(cardID)).Result() + if err != nil { + return CommandState{}, err + } + state := ParseCommand(raw) + if last, lastErr := s.LastStopResumeCall(ctx, cardID); lastErr == nil { + state.LastCallAt = last + } + if state.Exists && state.CarrierID > 0 { + member := strconv.FormatUint(uint64(cardID), 10) + if score, scoreErr := s.redis.ZScore(ctx, constants.RedisCarrierStopResumeQueueKey(state.CarrierID), member).Result(); scoreErr == nil { + state.NextAt = time.UnixMilli(int64(score)) + } + } + return state, nil +} + +// ParseCommand 解析停复机指令 Hash。 +func ParseCommand(raw map[string]string) CommandState { + state := CommandState{Exists: raw[srFieldAction] != ""} + state.Action, state.Reason, state.Source, state.LastError = raw[srFieldAction], raw[srFieldReason], raw[srFieldSource], raw[srFieldError] + state.Scenes = constants.SplitStopResumeScenes(raw[srFieldScenes]) + if value, err := strconv.ParseUint(raw[srFieldCarrier], 10, 64); err == nil { + state.CarrierID = uint(value) + } + state.Version, _ = strconv.ParseInt(raw[srFieldVersion], 10, 64) + state.FailCount, _ = strconv.Atoi(raw[srFieldFail]) + state.Unknown, _ = strconv.Atoi(raw[srFieldUnknown]) + state.Pending = raw[srFieldPending] == "1" + if ms, err := strconv.ParseInt(raw[srFieldFirst], 10, 64); err == nil && ms > 0 { + state.FirstFailed = time.UnixMilli(ms) + } + if ms, err := strconv.ParseInt(raw[srFieldAt], 10, 64); err == nil && ms > 0 { + state.WrittenAt = time.UnixMilli(ms) + } + return state +} + +// 停复机指令字段名导出给执行器组装收尾写入。 +const ( + SRFieldFail = srFieldFail + SRFieldFirst = srFieldFirst + SRFieldError = srFieldError + SRFieldPending = srFieldPending + SRFieldUnknown = srFieldUnknown +) + +// RequestDeviceEval 请求一次设备维度判断：递增版本、合并场景、设备进入判断队列。 +func (s *Store) RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error { + if deviceID == 0 { + return stderrors.New("设备判断缺少设备") + } + args := []any{deviceID, time.Now().UnixMilli()} + for _, scene := range constants.SplitStopResumeScenes(constants.JoinStopResumeScenes(scenes)) { + args = append(args, scene) + } + return deviceRequestScript.Run(ctx, s.redis, + []string{constants.RedisDeviceEvalVersionKey(deviceID), constants.RedisDeviceEvalScenesKey(deviceID), constants.RedisDeviceEvalQueueKey, + constants.RedisDeviceEvalClaimKey(deviceID)}, + args...).Err() +} + +// DeviceEvalClaimed 是一次被领取的设备判断。 +type DeviceEvalClaimed struct { + DeviceID uint + Token string + Version string + Scenes []string +} + +// ClaimDeviceEvals 领取到期设备判断。 +func (s *Store) ClaimDeviceEvals(ctx context.Context, lease time.Duration, limit int, tokenPrefix string) ([]DeviceEvalClaimed, int64, error) { + raw, err := deviceClaimScript.Run(ctx, s.redis, []string{constants.RedisDeviceEvalQueueKey, constants.RedisDeviceEvalRunningKey}, + time.Now().UnixMilli(), lease.Milliseconds(), limit, tokenPrefix).Slice() + if err != nil || len(raw) == 0 { + return nil, 0, err + } + expired, _ := raw[0].(int64) + claimed := make([]DeviceEvalClaimed, 0, (len(raw)-1)/4) + for index := 1; index+3 < len(raw); index += 4 { + deviceID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64) + claimed = append(claimed, DeviceEvalClaimed{DeviceID: uint(deviceID), Token: fmt.Sprint(raw[index+1]), + Version: fmt.Sprint(raw[index+2]), Scenes: constants.SplitStopResumeScenes(fmt.Sprint(raw[index+3]))}) + } + return claimed, expired, nil +} + +// AcquireDeviceEvalLock 抢设备判断锁（执行权）：1 抢到，0 处理中标记已不是该 token，-1 已有判断在执行。 +func (s *Store) AcquireDeviceEvalLock(ctx context.Context, deviceID uint, token, value string, ttl time.Duration) (int64, error) { + return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisDeviceEvalClaimKey(deviceID), constants.RedisDeviceEvalLockKey(deviceID)}, + token, value, ttl.Milliseconds()).Int64() +} + +// FinishDeviceEval 设备判断收尾：返回 0 token 不一致、1 完成、2 版本变大已重新排队、3 已推后。 +func (s *Store) FinishDeviceEval(ctx context.Context, claimed DeviceEvalClaimed, deferFor time.Duration) (int64, error) { + args := []any{claimed.Token, claimed.DeviceID, claimed.Version, time.Now().UnixMilli(), deferFor.Milliseconds()} + for _, scene := range claimed.Scenes { + args = append(args, scene) + } + return deviceFinishScript.Run(ctx, s.redis, + []string{constants.RedisDeviceEvalClaimKey(claimed.DeviceID), constants.RedisDeviceEvalLockKey(claimed.DeviceID), + constants.RedisDeviceEvalRunningKey, constants.RedisDeviceEvalVersionKey(claimed.DeviceID), + constants.RedisDeviceEvalScenesKey(claimed.DeviceID), constants.RedisDeviceEvalQueueKey}, + args...).Int64() +} + +// ReleaseDeviceEval 发布失败时释放设备判断处理中标记。 +func (s *Store) ReleaseDeviceEval(ctx context.Context, deviceID uint, token string) error { + return srReleaseScript.Run(ctx, s.redis, + []string{constants.RedisDeviceEvalClaimKey(deviceID), constants.RedisDeviceEvalRunningKey, constants.RedisDeviceEvalQueueKey}, + token, deviceID, time.Now().UnixMilli()).Err() +} + +// StopResumeQueueStats 是账户停复机队列的即时统计。 +type StopResumeQueueStats struct { + Queued int64 + Due int64 + Running int64 + Pending int64 +} + +// StopResumeStats 读取账户停复机队列长度、到期数与执行中数；待确认数按队列成员抽样统计（最多 500 个）。 +func (s *Store) StopResumeStats(ctx context.Context, carrierID uint) (StopResumeQueueStats, error) { + now := strconv.FormatInt(time.Now().UnixMilli(), 10) + queueKey := constants.RedisCarrierStopResumeQueueKey(carrierID) + pipe := s.redis.Pipeline() + queued := pipe.ZCard(ctx, queueKey) + due := pipe.ZCount(ctx, queueKey, "-inf", now) + running := pipe.ZCount(ctx, constants.RedisCarrierStopResumeRunningKey(carrierID), now, "+inf") + members := pipe.ZRange(ctx, queueKey, 0, 499) + if _, err := pipe.Exec(ctx); err != nil && !stderrors.Is(err, redis.Nil) { + return StopResumeQueueStats{}, err + } + stats := StopResumeQueueStats{Queued: queued.Val(), Due: due.Val(), Running: running.Val()} + if ids := members.Val(); len(ids) > 0 { + check := s.redis.Pipeline() + results := make([]*redis.StringCmd, 0, len(ids)) + for _, id := range ids { + cardID, convErr := strconv.ParseUint(strings.TrimSpace(id), 10, 64) + if convErr != nil { + continue + } + results = append(results, check.HGet(ctx, constants.RedisCardStopResumeCmdKey(uint(cardID)), srFieldPending)) + } + _, _ = check.Exec(ctx) + for _, result := range results { + if result.Val() == "1" { + stats.Pending++ + } + } + } + return stats, nil +} + +// IncrStopResumeStat 累加账户停复机执行计数（10 分钟分桶，保留 2 小时）。 +func (s *Store) IncrStopResumeStat(ctx context.Context, carrierID uint, field string) { + key := constants.RedisCarrierStopResumeStatKey(carrierID, time.Now().Unix()/constants.StopResumeStatBucketSeconds) + pipe := s.redis.Pipeline() + pipe.HIncrBy(ctx, key, field, 1) + pipe.Expire(ctx, key, 2*time.Hour) + _, _ = pipe.Exec(ctx) +} + +// StopResumeStatLastHour 汇总账户近 1 小时（最近 6 个分桶）的停复机执行计数。 +func (s *Store) StopResumeStatLastHour(ctx context.Context, carrierID uint) map[string]int64 { + current := time.Now().Unix() / constants.StopResumeStatBucketSeconds + pipe := s.redis.Pipeline() + cmds := make([]*redis.MapStringStringCmd, 0, 6) + for bucket := current - 5; bucket <= current; bucket++ { + cmds = append(cmds, pipe.HGetAll(ctx, constants.RedisCarrierStopResumeStatKey(carrierID, bucket))) + } + _, _ = pipe.Exec(ctx) + totals := map[string]int64{} + for _, cmd := range cmds { + for field, value := range cmd.Val() { + count, _ := strconv.ParseInt(value, 10, 64) + totals[field] += count + } + } + return totals +} diff --git a/internal/polling/package_activation_handler.go b/internal/polling/package_activation_handler.go index b7fdfed3..f6afe879 100644 --- a/internal/polling/package_activation_handler.go +++ b/internal/polling/package_activation_handler.go @@ -2,6 +2,7 @@ package polling import ( "context" + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" "time" "github.com/bytedance/sonic" @@ -435,16 +436,17 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c if h.stopResumeCallback == nil { return } - detachedCtx := context.WithoutCancel(ctx) + // 套餐到期即停：以"套餐到期"场景请求评估，暂停轮询的卡同样停机（只触发一次，不做补偿扫描）。 + detachedCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeScenePackageExpired) if carrierType == "iot_card" { - go func() { + stopresumedomain.RunAfterCommit(detachedCtx, h.stopResumeCallback, func() { if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, carrierID); err != nil { h.logger.Error("套餐过期后停机失败", zap.Uint("card_id", carrierID), zap.Error(err)) } - }() + }) return } @@ -457,14 +459,14 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c return } for _, b := range bindings { - cardID := b.IotCardID - go func(cID uint) { + cID := b.IotCardID + stopresumedomain.RunAfterCommit(detachedCtx, h.stopResumeCallback, func() { if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, cID); err != nil { h.logger.Error("套餐过期后停机失败", zap.Uint("card_id", cID), zap.Error(err)) } - }(cardID) + }) } } } @@ -482,6 +484,7 @@ func (h *PackageActivationHandler) reconcileCarrierAfterActivation(ctx context.C if carrierID == 0 { return errors.New(errors.CodeInvalidParam, "套餐使用记录缺少有效载体") } + ctx = stopresumedomain.WithScene(ctx, constants.StopResumeSceneActivation) if carrierType == constants.AssetTypeIotCard { if err := h.stopResumeCallback.CheckAndStopCard(ctx, carrierID); err != nil { diff --git a/internal/service/carrier/service.go b/internal/service/carrier/service.go index 5383b9e3..fcc4a32b 100644 --- a/internal/service/carrier/service.go +++ b/internal/service/carrier/service.go @@ -3,6 +3,7 @@ package carrier import ( "context" "fmt" + "go.uber.org/zap" "strconv" "time" @@ -26,6 +27,22 @@ type Service struct { audit systemconfigapp.AuditWriter // thresholdHits 是通道阈值命中记录只读查询；与运营商配置共用同一数据库连接。 thresholdHits *carrierthresholdquery.Query + // realnameEvaluator 实名管控修改后对受影响的卡批量请求停复机评估；未注入时不触发。 + realnameEvaluator RealnameControlEvaluator + logger *zap.Logger +} + +// RealnameControlEvaluator 是实名管控修改后批量请求停复机评估的能力（停复机出口开关关闭时不执行）。 +type RealnameControlEvaluator interface { + RequestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) (int, error) +} + +// SetRealnameControlEvaluator 注入实名管控修改后的批量评估能力。 +func (s *Service) SetRealnameControlEvaluator(evaluator RealnameControlEvaluator, logger *zap.Logger) { + if logger == nil { + logger = zap.NewNop() + } + s.realnameEvaluator, s.logger = evaluator, logger } func New(carrierStore *postgres.CarrierStore, audit systemconfigapp.AuditWriter) *Service { @@ -210,10 +227,24 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateCarrierReq s.recordFailure(ctx, constants.AuditOperationCarrierUpdate, "更新运营商配置失败", carrier) return nil, errors.Wrap(errors.CodeInternalError, err, "更新运营商失败") } + if before.RealnameControlEnabled != carrier.RealnameControlEnabled && s.realnameEvaluator != nil { + s.requestRealnameControlEvaluation(ctx, carrier.ID, carrier.RealnameControlEnabled == constants.StatusEnabled) + } return s.toResponse(ctx, carrier), nil } +// requestRealnameControlEvaluation 在事务提交后异步批量请求评估（只读库与写共享缓存，不阻塞保存接口）。 +func (s *Service) requestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) { + evalCtx := context.WithoutCancel(ctx) + go func() { + defer func() { _ = recover() }() + if _, err := s.realnameEvaluator.RequestRealnameControlEvaluation(evalCtx, carrierID, enabled); err != nil { + s.logger.Error("实名管控修改后批量请求停复机评估失败", zap.Uint("carrier_id", carrierID), zap.Error(err)) + } + }() +} + func (s *Service) Delete(ctx context.Context, id uint) error { carrier, err := s.carrierStore.GetByID(ctx, id) if err != nil { diff --git a/internal/service/iot_card/service.go b/internal/service/iot_card/service.go index 9e876561..cbabcd18 100644 --- a/internal/service/iot_card/service.go +++ b/internal/service/iot_card/service.go @@ -2006,7 +2006,7 @@ func (s *Service) handleManualRealnameStatusChanged(ctx context.Context, oldStat } if s.stopResumeService != nil { - if err := s.stopResumeService.EvaluateAndAct(ctx, card); err != nil { + if err := s.stopResumeService.EvaluateAndAct(WithStopResumeScene(ctx, constants.StopResumeSceneRealnameObserved), card); err != nil { s.logger.Warn("手动实名后触发停复机评估失败", zap.Uint("card_id", card.ID), zap.Error(err)) diff --git a/internal/service/iot_card/stop_resume_command.go b/internal/service/iot_card/stop_resume_command.go new file mode 100644 index 00000000..9617cd3d --- /dev/null +++ b/internal/service/iot_card/stop_resume_command.go @@ -0,0 +1,190 @@ +package iot_card + +import ( + "context" + stderrors "errors" + "time" + + "github.com/google/uuid" + "go.uber.org/zap" + "gorm.io/gorm" + + carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation" + "github.com/break/junhong_cmp_fiber/internal/gateway" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// 停复机指令执行器使用的服务能力（add-stop-resume-command-queue）。执行器只调用一次网关，不 sleep 重试。 + +// CommandDecision 是执行前按指令场景重新判断的结果。 +type CommandDecision struct { + Card *model.IotCard + Action string // 空表示条件不再满足或已是目标状态，结束指令 + Reason string + Gone bool // 卡已不存在 +} + +// DecideCommand 执行前按指令记录的场景重新判断该卡（只看本卡，不扩散到设备）。 +// 复机持通道阈值锁时视为不再满足。 +func (s *StopResumeService) DecideCommand(ctx context.Context, cardID uint, scenes []string) (CommandDecision, error) { + card, err := s.iotCardStore.GetByID(ctx, cardID) + if err != nil { + if stderrors.Is(err, gorm.ErrRecordNotFound) { + return CommandDecision{Gone: true}, nil + } + return CommandDecision{}, errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败") + } + decision, err := s.decideCard(ctx, card, scenes) + if err != nil { + return CommandDecision{Card: card}, err + } + if decision.Action == constants.StopResumeActionStart { + blocked, blockErr := s.channelThresholdBlocked(ctx, card) + if blockErr != nil { + return CommandDecision{Card: card}, blockErr + } + if blocked { + return CommandDecision{Card: card}, nil + } + } + return CommandDecision{Card: card, Action: decision.Action, Reason: decision.Reason}, nil +} + +// CommandCallResult 是一次停复机网关调用的结果。 +type CommandCallResult struct { + // Unknown 为 true 表示结果未知：超时、连接失败，或上游已成功但本地回写失败。 + Unknown bool +} + +// ExecuteCommand 对卡调用一次停机或复机（限流等待模式由调用方 ctx 声明），成功后按现有回写更新卡状态、 +// 停机原因与网络状态序列，并写停复机记录。返回的 error 为网关错误或回写错误，由执行器按错误分类处理。 +func (s *StopResumeService) ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (CommandCallResult, error) { + if card == nil { + return CommandCallResult{}, errors.New(errors.CodeInvalidParam) + } + if action == constants.StopResumeActionStop { + outcome, err := s.stopCardAttempts(ctx, card, reason, 1) + return CommandCallResult{Unknown: err != nil && outcome.Result == constants.AuditResultUnknown && !gateway.IsRateLimited(err)}, err + } + actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络" + attempt, integrationID, err := s.resumeCardAttempts(ctx, card, actionCode, summary, 1) + if err != nil { + return CommandCallResult{Unknown: isUnknownGatewayError(err)}, err + } + if err := s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary); err != nil { + return CommandCallResult{Unknown: true}, err + } + return CommandCallResult{}, nil +} + +// isUnknownGatewayError 判断网关错误是否为结果未知（超时、连接失败）：请求可能已到达上游。 +func isUnknownGatewayError(err error) bool { + gwErr, ok := gateway.AsGatewayError(err) + if !ok { + return false + } + return gwErr.Category == constants.GatewayErrorTimeout || gwErr.Category == constants.GatewayErrorNetwork +} + +// QueryCommandCardStatus 结果未知后的确认查询：只查一次卡状态（受限流、不受单卡最小重试间隔约束）。 +// 查询失败时 known=false 并返回错误，调用方保持待确认、不发起停复机。 +func (s *StopResumeService) QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error) { + if s.gatewayClient == nil { + return 0, false, errors.New(errors.CodeInternalError, "Gateway 未配置，无法查询卡状态") + } + queryCtx := gateway.WithCarrier(ctx, card.CarrierID) + response, err := s.gatewayClient.QueryCardStatus(queryCtx, &gateway.CardStatusReq{CardNo: card.ICCID}) + if err != nil { + return 0, false, err + } + status, known := carddomain.MapGatewayNetworkStatus(response.CardStatus, response.Extend) + return status, known, nil +} + +// ConfirmCommandState 按已确认的上游结果（确认查询已是目标状态、上游返回已是目标状态、停机遇上游终态）回写卡状态。 +func (s *StopResumeService) ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error { + if card == nil { + return errors.New(errors.CodeInvalidParam) + } + actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络（确认）" + scene, expected := constants.CardObservationSceneBusinessResume, "online" + fields := map[string]any{"network_status": constants.NetworkStatusOnline, "resumed_at": time.Now(), "stop_reason": ""} + if action == constants.StopResumeActionStop { + actionCode, summary = stopAuditAction(ctx, reason) + summary += "（确认）" + scene, expected = constants.CardObservationSceneBusinessStop, "offline" + fields = map[string]any{"network_status": constants.NetworkStatusOffline, "stopped_at": time.Now(), "stop_reason": reason} + } + if err := s.updateCardAndAppendNetworkSeries(ctx, card, fields, scene, expected, uuid.NewString(), actionCode, summary, ""); err != nil { + return err + } + s.reschedulePolling(ctx, card.ID) + return nil +} + +// EvaluateDeviceCommands 设备维度判断：读取设备下各卡最新数据，按统一判断函数（各卡按所属运营商账户判断实名， +// 暂停轮询按场景集合判断）为需要动作的卡生成指令，进入各卡所属账户的停复机队列。只读数据库与写共享缓存，不调网关。 +func (s *StopResumeService) EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error { + if s.commandQueue == nil { + return errors.New(errors.CodeInternalError, "停复机指令队列未配置") + } + cards, err := s.deviceCards(ctx, deviceID) + if err != nil { + return err + } + var failed []error + for _, card := range cards { + decision, decideErr := s.decideCard(ctx, card, scenes) + if decideErr != nil { + failed = append(failed, decideErr) + continue + } + if decision.Action == "" { + s.markNotRealnameIfBlocked(ctx, card, scenes) + continue + } + decision.DeviceID = 0 + if writeErr := s.writeCardCommand(ctx, card, decision, scenes, "device_eval"); writeErr != nil { + failed = append(failed, writeErr) + } + } + if len(failed) > 0 { + return errors.Wrap(errors.CodeInternalError, stderrors.Join(failed...), "设备判断存在未写入指令的卡") + } + return nil +} + +// markNotRealnameIfBlocked 设备下停机卡套餐满足但实名不满足时，把停机原因更新为未实名（与旧设备复机路径一致）。 +func (s *StopResumeService) markNotRealnameIfBlocked(ctx context.Context, card *model.IotCard, scenes []string) { + if card.NetworkStatus != constants.NetworkStatusOffline || card.StopReason == constants.StopReasonNotRealname { + return + } + if suspended, err := s.autoControlSuspended(ctx, card); err != nil || suspended { + return + } + hasPackage, err := s.hasValidPackage(ctx, card) + if err != nil || !hasPackage { + return + } + if exhausted, err := s.isTrafficExhausted(ctx, card); err != nil || exhausted { + return + } + realnameOK, err := s.isRealnameOK(ctx, card) + if err != nil || realnameOK { + return + } + if err := s.updateCardStopReasonWithAudit(ctx, card, constants.StopReasonNotRealname); err != nil { + s.logger.Warn("更新未实名卡停机原因失败", zap.Uint("card_id", card.ID), zap.Strings("scenes", scenes), zap.Error(err)) + } +} + +// LegacyDeviceOpLocked 判断旧路径是否正在对该设备执行停复机（持设备锁）；设备判断须推后，不并行执行。 +func (s *StopResumeService) LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool { + if s.redis == nil { + return false + } + exists, err := s.redis.Exists(ctx, constants.RedisPollingDeviceOpLockKey(deviceID)).Result() + return err == nil && exists > 0 +} diff --git a/internal/service/iot_card/stop_resume_decide.go b/internal/service/iot_card/stop_resume_decide.go new file mode 100644 index 00000000..56571184 --- /dev/null +++ b/internal/service/iot_card/stop_resume_decide.go @@ -0,0 +1,92 @@ +package iot_card + +import ( + "context" + + "go.uber.org/zap" + + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// WithStopResumeScene 声明本次自动停复机评估的触发场景；未声明时按套餐步骤（周期评估）处理。 +func WithStopResumeScene(ctx context.Context, scene string) context.Context { + return stopresumedomain.WithScene(ctx, scene) +} + +// stopResumeScenes 返回 ctx 中声明的场景集合（单个场景）。 +func stopResumeScenes(ctx context.Context) []string { + return []string{stopresumedomain.Scene(ctx)} +} + +// stopResumeDecision 是统一判断函数的结果。 +type stopResumeDecision struct { + Action string // constants.StopResumeActionStop / StopResumeActionStart；空表示不动作 + Reason string // 停机原因 + Reasons []string // 全部命中的停机原因（按优先级） + DeviceID uint // 设备维度动作的设备；0 表示单卡 +} + +// decideCard 统一判断函数：暂停轮询（按场景） → 停机条件 → 复机条件。 +// 暂停轮询的卡只在套餐失效类场景（套餐到期、退款）下放行无套餐停机，其余自动停复机一律不做。 +// 设备维度原因（无套餐、流量耗尽）与绑定设备的复机返回 DeviceID，由调用方按设备处理。通道阈值锁由复机执行方判断。 +func (s *StopResumeService) decideCard(ctx context.Context, card *model.IotCard, scenes []string) (stopResumeDecision, error) { + suspended, err := s.autoControlSuspended(ctx, card) + if err != nil { + return stopResumeDecision{}, err + } + allowSuspendedStop := constants.StopResumeSceneAllowsSuspendedStop(scenes) + if suspended && !allowSuspendedStop { + return stopResumeDecision{}, nil + } + switch card.NetworkStatus { + case constants.NetworkStatusOnline: + reasons, err := s.checkStopReasons(ctx, card) + if err != nil || len(reasons) == 0 { + return stopResumeDecision{}, err + } + primary := reasons[0] + if suspended { + // 暂停轮询：只有套餐失效（无可用主套餐）才停机。 + if !containsStopReason(reasons, constants.StopReasonNoPackage) { + return stopResumeDecision{}, nil + } + primary = constants.StopReasonNoPackage + } + decision := stopResumeDecision{Action: constants.StopResumeActionStop, Reason: primary, Reasons: reasons} + if !card.IsStandalone && isDeviceScopeReason(primary) { + decision.DeviceID = s.boundDeviceID(ctx, card) + } + return decision, nil + case constants.NetworkStatusOffline: + if suspended { + return stopResumeDecision{}, nil + } + // 不看停机原因，是否复机只由本系统规则决定。 + resume, err := s.shouldResume(ctx, card) + if err != nil || !resume { + return stopResumeDecision{}, err + } + decision := stopResumeDecision{Action: constants.StopResumeActionStart} + if !card.IsStandalone { + decision.DeviceID = s.boundDeviceID(ctx, card) + } + return decision, nil + default: + return stopResumeDecision{}, nil + } +} + +// boundDeviceID 返回卡的有效绑定设备；查询失败时降级为单卡处理并记日志。 +func (s *StopResumeService) boundDeviceID(ctx context.Context, card *model.IotCard) uint { + deviceID, bound, err := s.getCardDeviceID(ctx, card) + if err != nil { + s.logger.Error("查询设备绑定关系失败，降级为单卡处理", zap.Uint("card_id", card.ID), zap.Error(err)) + return 0 + } + if !bound { + return 0 + } + return deviceID +} diff --git a/internal/service/iot_card/stop_resume_realname_control.go b/internal/service/iot_card/stop_resume_realname_control.go new file mode 100644 index 00000000..c84b90d4 --- /dev/null +++ b/internal/service/iot_card/stop_resume_realname_control.go @@ -0,0 +1,70 @@ +package iot_card + +import ( + "context" + + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// realnameControlBatchSize 是实名管控修改后分批读取受影响卡的批大小。 +const realnameControlBatchSize = 500 + +// RequestRealnameControlEvaluation 运营商账户实名管控修改后对受影响的卡批量请求评估（只在停复机出口开关开启时执行）： +// 0 → 1 时为该账户在线且未实名的卡，1 → 0 时为停机且未实名的卡；绑定设备的卡按设备请求设备判断。 +// 停复机按账户 QPS 由指令队列平滑执行。返回请求评估的卡与设备数。 +func (s *StopResumeService) RequestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) (int, error) { + if !s.commandQueueEnabled(ctx) || s.db == nil { + return 0, nil + } + ctx = WithStopResumeScene(ctx, constants.StopResumeSceneRealnameControl) + scenes := []string{constants.StopResumeSceneRealnameControl} + networkStatus := constants.NetworkStatusOffline + if enabled { + networkStatus = constants.NetworkStatusOnline + } + requested := 0 + devices := map[uint]bool{} + lastID := uint(0) + for { + var cards []*model.IotCard + if err := s.db.WithContext(ctx).Where("carrier_id = ? AND network_status = ? AND real_name_status <> ? AND id > ?", + carrierID, networkStatus, constants.RealNameStatusVerified, lastID). + Order("id").Limit(realnameControlBatchSize).Find(&cards).Error; err != nil { + return requested, errors.Wrap(errors.CodeDatabaseError, err, "查询实名管控受影响的卡失败") + } + if len(cards) == 0 { + break + } + for _, card := range cards { + lastID = card.ID + if deviceID := s.boundDeviceID(ctx, card); deviceID > 0 { + if !devices[deviceID] { + devices[deviceID] = true + if err := s.requestDeviceEval(ctx, deviceID, scenes, "realname_control"); err == nil { + requested++ + } + } + continue + } + decision, err := s.decideCard(ctx, card, scenes) + if err != nil { + s.logger.Warn("实名管控修改后判断卡失败", zap.Uint("card_id", card.ID), zap.Error(err)) + continue + } + if decision.Action == "" { + continue + } + decision.DeviceID = 0 + if err := s.writeCardCommand(ctx, card, decision, scenes, "realname_control"); err == nil { + requested++ + } + } + } + s.logger.Info("运营商账户实名管控修改后已批量请求评估", zap.Uint("carrier_id", carrierID), zap.Bool("enabled", enabled), + zap.Int("requested", requested), zap.String("metric", "stop_resume.realname_control_requested")) + return requested, nil +} diff --git a/internal/service/iot_card/stop_resume_request.go b/internal/service/iot_card/stop_resume_request.go new file mode 100644 index 00000000..1b32da4e --- /dev/null +++ b/internal/service/iot_card/stop_resume_request.go @@ -0,0 +1,144 @@ +package iot_card + +import ( + "context" + + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// StopResumeCommandQueue 是停复机指令队列的写入接缝（共享缓存）。 +type StopResumeCommandQueue interface { + // Enabled 返回停复机出口开关 polling.stop_resume.queue_enabled。 + Enabled(ctx context.Context) bool + WriteCommand(ctx context.Context, cmd carrierqueue.StopResumeCommand) (int64, error) + RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error +} + +// SetCommandQueue 注入停复机指令队列；未注入时始终当场执行。 +func (s *StopResumeService) SetCommandQueue(queue StopResumeCommandQueue) { + s.commandQueue = queue +} + +// CommandQueueEnabled 返回停复机出口开关是否开启（供调用方决定是否还需要异步执行）。 +func (s *StopResumeService) CommandQueueEnabled(ctx context.Context) bool { + return s.commandQueueEnabled(ctx) +} + +func (s *StopResumeService) commandQueueEnabled(ctx context.Context) bool { + return s != nil && s.commandQueue != nil && s.commandQueue.Enabled(ctx) +} + +// RequestAssetEvaluation 请求评估门面：开关开启时单卡生成指令、设备请求设备判断；关闭时走旧路径当场执行。 +// assetType 为 constants.AssetTypeIotCard / AssetTypeDevice。 +func (s *StopResumeService) RequestAssetEvaluation(ctx context.Context, assetType string, assetID uint, scene, source string) error { + ctx = WithStopResumeScene(ctx, scene) + if !s.commandQueueEnabled(ctx) { + return s.legacyAssetEvaluation(ctx, assetType, assetID) + } + if assetType == constants.AssetTypeDevice { + return s.requestDeviceEval(ctx, assetID, []string{scene}, source) + } + card, err := s.iotCardStore.GetByID(ctx, assetID) + if err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败") + } + scenes := []string{scene} + decision, err := s.decideCard(ctx, card, scenes) + if err != nil { + return err + } + return s.submitDecisionWithSource(ctx, card, decision, scenes, source) +} + +// legacyAssetEvaluation 开关关闭时的当场执行：单卡按统一判断执行，设备按设备复机（套餐激活、重置、订单）或逐卡评估（其他场景）。 +func (s *StopResumeService) legacyAssetEvaluation(ctx context.Context, assetType string, assetID uint) error { + if assetType == constants.AssetTypeDevice { + cards, err := s.deviceCards(ctx, assetID) + if err != nil { + return err + } + for _, card := range cards { + if err := s.EvaluateAndAct(ctx, card); err != nil { + return err + } + } + return nil + } + card, err := s.iotCardStore.GetByID(ctx, assetID) + if err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败") + } + return s.EvaluateAndAct(ctx, card) +} + +// submitDecision 把统一判断结果写入指令队列（来源取场景）。 +func (s *StopResumeService) submitDecision(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string) error { + return s.submitDecisionWithSource(ctx, card, decision, scenes, "") +} + +func (s *StopResumeService) submitDecisionWithSource(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string, source string) error { + if decision.Action == "" { + return nil + } + if decision.DeviceID > 0 { + return s.requestDeviceEval(ctx, decision.DeviceID, scenes, source) + } + return s.writeCardCommand(ctx, card, decision, scenes, source) +} + +// writeCardCommand 生成单卡停复机指令；复机前检查通道阈值锁，持锁时记录拒绝事实不生成指令。 +func (s *StopResumeService) writeCardCommand(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string, source string) error { + if decision.Action == constants.StopResumeActionStart { + blocked, err := s.channelThresholdBlocked(ctx, card) + if err != nil { + return err + } + if blocked { + s.recordChannelThresholdDenial(ctx, card, constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络") + return nil + } + } + if source == "" { + source = constants.JoinStopResumeScenes(scenes) + } + version, err := s.commandQueue.WriteCommand(ctx, carrierqueue.StopResumeCommand{ + CardID: card.ID, CarrierID: card.CarrierID, Action: decision.Action, Reason: decision.Reason, Scenes: scenes, Source: source, + }) + if err != nil { + // 写入失败只记日志：由该卡下一次套餐步骤兜底。 + s.logger.Error("写入停复机指令失败", zap.Uint("card_id", card.ID), zap.String("action", decision.Action), + zap.Strings("scenes", scenes), zap.String("metric", "stop_resume.request_failed"), zap.Error(err)) + return errors.Wrap(errors.CodeRedisError, err, "写入停复机指令失败") + } + s.logger.Info("已生成停复机指令", zap.Uint("card_id", card.ID), zap.Uint("carrier_id", card.CarrierID), + zap.String("action", decision.Action), zap.String("reason", decision.Reason), zap.Strings("scenes", scenes), zap.Int64("version", version)) + return nil +} + +// requestDeviceEval 请求一次设备维度判断。 +func (s *StopResumeService) requestDeviceEval(ctx context.Context, deviceID uint, scenes []string, source string) error { + if err := s.commandQueue.RequestDeviceEval(ctx, deviceID, scenes); err != nil { + s.logger.Error("请求设备判断失败", zap.Uint("device_id", deviceID), zap.Strings("scenes", scenes), zap.String("source", source), + zap.String("metric", "stop_resume.request_failed"), zap.Error(err)) + return errors.Wrap(errors.CodeRedisError, err, "请求设备判断失败") + } + return nil +} + +// deviceCards 返回设备下有效绑定的全部卡（在线与停机）。 +func (s *StopResumeService) deviceCards(ctx context.Context, deviceID uint) ([]*model.IotCard, error) { + online, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOnline) + if err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询设备在线卡失败") + } + offline, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOffline) + if err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询设备停机卡失败") + } + return append(online, offline...), nil +} diff --git a/internal/service/iot_card/stop_resume_service.go b/internal/service/iot_card/stop_resume_service.go index 6eebbd23..a3bca40e 100644 --- a/internal/service/iot_card/stop_resume_service.go +++ b/internal/service/iot_card/stop_resume_service.go @@ -60,6 +60,8 @@ type StopResumeService struct { stopResumeRecorder *gatewaylimit.StopResumeRecorder // terminalPolling 停复机命中"自动关闭轮询"终态规则时关闭该卡轮询；未注入时按普通失败处理。 terminalPolling *gatewaylimit.TerminalPollingDisabler + // commandQueue 停复机指令队列；开关开启时自动入口只请求评估。 + commandQueue StopResumeCommandQueue } // SetStopResumeRecorder 注入停复机记录写入器。 @@ -128,69 +130,36 @@ func (s *StopResumeService) SetPollingCallback(callback PollingCallback) { s.pollingCallback = callback } -// EvaluateAndAct 停复机统一入口 -// 根据卡的当前网络状态，自动判断并执行停机或复机操作 -// 设备/单卡维度通过 card.IsStandalone 推导（false 则为设备维度，停/复机覆盖设备下所有卡） +// EvaluateAndAct 停复机统一入口：按统一判断函数决定停机或复机。 +// 停复机出口开关开启时只请求评估（生成指令或请求设备判断），不当场调用网关；关闭时当场执行。 +// 设备/单卡维度通过 card.IsStandalone 推导（false 则为设备维度，停/复机覆盖设备下所有卡）。触发场景见 WithStopResumeScene。 func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error { - // 总开关开启时，卡或所在设备暂停轮询的不自动停复机。 - suspended, err := s.autoControlSuspended(ctx, card) + scenes := stopResumeScenes(ctx) + decision, err := s.decideCard(ctx, card, scenes) if err != nil { return err } - if suspended { - return nil + // 无有效套餐且存在待生效套餐使用记录：仅由普通套餐轮询来源追加优先轮询请求。 + if containsStopReason(decision.Reasons, constants.StopReasonNoPackage) { + if err := s.appendNoValidPackagePriority(ctx, card); err != nil { + return err + } } - switch card.NetworkStatus { - case constants.NetworkStatusOnline: - // 卡在线，检查是否需要停机 - reasons, err := s.checkStopReasons(ctx, card) - if err != nil { - return err + if s.commandQueueEnabled(ctx) { + return s.submitDecision(ctx, card, decision, scenes) + } + switch decision.Action { + case constants.StopResumeActionStop: + if decision.DeviceID > 0 { + return s.stopDeviceCards(ctx, decision.DeviceID, decision.Reason) } - if len(reasons) == 0 { - return nil - } - // 无有效套餐且存在待生效套餐使用记录：仅由普通套餐轮询来源追加优先轮询请求。 - if containsStopReason(reasons, constants.StopReasonNoPackage) { - if err := s.appendNoValidPackagePriority(ctx, card); err != nil { - return err - } - } - // 取最高优先级原因（第一个） - primaryReason := reasons[0] - if !card.IsStandalone && isDeviceScopeReason(primaryReason) { - deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card) - if lookupErr != nil { - s.logger.Error("查询设备绑定关系失败，降级为单卡停机", - zap.Uint("card_id", card.ID), zap.Error(lookupErr)) - // 降级：查不到绑定时按单卡处理 - } else if bound { - return s.stopDeviceCards(ctx, deviceID, primaryReason) - } - } - _, stopErr := s.stopCardWithRetry(ctx, card, primaryReason) + _, stopErr := s.stopCardWithRetry(ctx, card, decision.Reason) return stopErr - - case constants.NetworkStatusOffline: - // 卡停机，检查是否可以复机：不看停机原因，是否复机只由本系统规则决定 - shouldResume, err := s.shouldResume(ctx, card) - if err != nil { - return err - } - if !shouldResume { - return nil - } - if !card.IsStandalone { - deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card) - if lookupErr != nil { - s.logger.Error("查询设备绑定关系失败，降级为单卡复机", - zap.Uint("card_id", card.ID), zap.Error(lookupErr)) - } else if bound { - return s.resumeDeviceCards(ctx, deviceID) - } + case constants.StopResumeActionStart: + if decision.DeviceID > 0 { + return s.resumeDeviceCards(ctx, decision.DeviceID) } return s.resumeSingleCard(ctx, card.ID) - default: return nil } @@ -414,11 +383,11 @@ func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint, var cardErrors []error for _, card := range cards { - // 总开关开启时，暂停轮询的卡不随设备自动停机。 + // 暂停轮询的卡不随设备自动停机；套餐失效类场景（套餐到期、退款）照常停机。 if suspended, suspendErr := s.autoControlSuspended(ctx, card); suspendErr != nil { cardErrors = append(cardErrors, suspendErr) continue - } else if suspended { + } else if suspended && !(stopReason == constants.StopReasonNoPackage && constants.StopResumeSceneAllowsSuspendedStop(stopResumeScenes(ctx))) { continue } if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil { @@ -503,6 +472,9 @@ func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) e // ResumeCardIfStopped 套餐激活或流量重置后自动复机入口 // 支持 iot_card 和 device 两种载体类型 func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error { + if s.commandQueueEnabled(ctx) { + return s.RequestAssetEvaluation(ctx, carrierType, carrierID, stopResumeScenes(ctx)[0], "resume_if_stopped") + } switch carrierType { case "iot_card": return s.resumeSingleCard(ctx, carrierID) @@ -582,7 +554,12 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e zap.Error(err)) return err } + return s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary) +} +// completeAutoResume 自动复机网关成功后回写在线状态、观测序列与统一审计，并重排轮询。 +// 回写失败时返回错误，调用方按结果未知处理。 +func (s *StopResumeService) completeAutoResume(ctx context.Context, card *model.IotCard, attempt *gatewayAttempt, integrationID, actionCode, summary string) error { now := time.Now() if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{ "network_status": constants.NetworkStatusOnline, @@ -593,7 +570,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr)) } s.logger.Error("复机 Gateway 成功但 DB 更新失败", - zap.Uint("card_id", cardID), + zap.Uint("card_id", card.ID), zap.String("iccid", card.ICCID), zap.Error(err)) s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown, @@ -608,7 +585,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e } s.logger.Info("卡已自动复机", - zap.Uint("card_id", cardID), + zap.Uint("card_id", card.ID), zap.String("iccid", card.ICCID)) return nil @@ -619,6 +596,11 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e // 返回的 outcome 始终携带可回填可靠任务的结果分类（成功/失败/未知）与最后一次尝试的 // Integration Log 标识；error 与既有语义一致，表示本次停机未成功。 func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) { + return s.stopCardAttempts(ctx, card, stopReason, s.maxRetries) +} + +// stopCardAttempts 调用运营商停机接口最多 attempts 次（停复机指令执行器只调用一次），成功后回写停机状态。 +func (s *StopResumeService) stopCardAttempts(ctx context.Context, card *model.IotCard, stopReason string, attempts int) (carrierthresholddomain.CommandOutcome, error) { actionCode, summary := stopAuditAction(ctx, stopReason) if s.gatewayClient == nil { failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用") @@ -641,7 +623,7 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID) var lastErr error lastIntegrationID := "" - for i := 0; i < s.maxRetries; i++ { + for i := 0; i < attempts; i++ { if i > 0 { s.logger.Debug("重试调用停机接口", zap.Int("attempt", i+1), @@ -732,6 +714,11 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I // resumeCardWithRetry 调用运营商复机接口（带重试机制）。 // 第二个返回值是最后一次尝试的 Integration Log 标识（无论成功或失败），供调用方回填可靠任务状态。 func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) { + return s.resumeCardAttempts(ctx, card, actionCode, summary, s.maxRetries) +} + +// resumeCardAttempts 调用运营商复机接口最多 attempts 次，不回写卡状态。 +func (s *StopResumeService) resumeCardAttempts(ctx context.Context, card *model.IotCard, actionCode, summary string, attempts int) (*gatewayAttempt, string, error) { if s.gatewayClient == nil { failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用") s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr) @@ -751,7 +738,7 @@ func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID) var lastErr error lastIntegrationID := "" - for i := 0; i < s.maxRetries; i++ { + for i := 0; i < attempts; i++ { if i > 0 { s.logger.Debug("重试调用复机接口", zap.Int("attempt", i+1), diff --git a/internal/service/order/service.go b/internal/service/order/service.go index c721c283..5cd2ec4b 100644 --- a/internal/service/order/service.go +++ b/internal/service/order/service.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" "sort" "strconv" "strings" @@ -2202,8 +2203,9 @@ func (s *Service) tryResumeAfterPayment(ctx context.Context, order *model.Order) return } - resumeCtx := context.WithoutCancel(ctx) - go func(ct string, cid uint) { + resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneOrder) + ct, cid := resumeCarrierType, resumeCarrierID + stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() { if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, ct, cid); err != nil { s.logger.Error("支付后自动复机失败", zap.String("carrier_type", ct), @@ -2211,7 +2213,7 @@ func (s *Service) tryResumeAfterPayment(ctx context.Context, order *model.Order) zap.Uint("order_id", order.ID), zap.Error(err)) } - }(resumeCarrierType, resumeCarrierID) + }) } func (s *Service) activatePackage(ctx context.Context, tx *gorm.DB, order *model.Order) error { diff --git a/internal/service/package/activation_service.go b/internal/service/package/activation_service.go index 01836b12..fc438926 100644 --- a/internal/service/package/activation_service.go +++ b/internal/service/package/activation_service.go @@ -2,6 +2,7 @@ package packagepkg import ( "context" + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" "strconv" "time" @@ -778,15 +779,15 @@ func (s *ActivationService) resumeAfterActivation(ctx context.Context, carrierTy if s.resumeCallback == nil { return } - resumeCtx := context.WithoutCancel(ctx) - go func() { + resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneActivation) + stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() { if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, carrierType, carrierID); err != nil { s.logger.Error(errorMessage, zap.String("carrier_type", carrierType), zap.Uint("carrier_id", carrierID), zap.Error(err)) } - }() + }) } func (s *ActivationService) appendActivationObservation(ctx context.Context, tx *gorm.DB, usage *model.PackageUsage, carrierType string, carrierID uint, occurredAt time.Time) error { diff --git a/internal/service/package/reset_service.go b/internal/service/package/reset_service.go index ee87a8d1..d7b9a207 100644 --- a/internal/service/package/reset_service.go +++ b/internal/service/package/reset_service.go @@ -2,6 +2,7 @@ package packagepkg import ( "context" + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" "time" "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit" @@ -381,7 +382,7 @@ func (s *ResetService) triggerResumeForPackages(ctx context.Context, packages [] return } - resumeCtx := context.WithoutCancel(ctx) + resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneReset) for _, pkg := range packages { var carrierType string var carrierID uint @@ -396,13 +397,14 @@ func (s *ResetService) triggerResumeForPackages(ctx context.Context, packages [] continue } - go func(ct string, cid uint) { + ct, cid := carrierType, carrierID + stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() { if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, ct, cid); err != nil { s.logger.Warn("流量重置后自动复机失败", zap.String("carrier_type", ct), zap.Uint("carrier_id", cid), zap.Error(err)) } - }(carrierType, carrierID) + }) } } diff --git a/internal/service/package/usage_service.go b/internal/service/package/usage_service.go index 76e3371e..800d28dd 100644 --- a/internal/service/package/usage_service.go +++ b/internal/service/package/usage_service.go @@ -2,6 +2,7 @@ package packagepkg import ( "context" + stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume" "math" "strconv" "time" @@ -449,15 +450,15 @@ func (s *UsageService) triggerSuspensionAfterCommit(ctx context.Context, carrier return } - stopCtx := context.WithoutCancel(ctx) + stopCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneUsage) if carrierType == constants.AssetTypeIotCard { - go func() { + stopresumedomain.RunAfterCommit(stopCtx, s.stopResumeCallback, func() { if err := s.stopResumeCallback.CheckAndStopCard(stopCtx, carrierID); err != nil { s.logger.Error("调用停机服务失败", zap.Uint("card_id", carrierID), zap.Error(err)) } - }() + }) return } @@ -479,14 +480,14 @@ func (s *UsageService) triggerSuspensionAfterCommit(ctx context.Context, carrier } for _, b := range bindings { - cardID := b.IotCardID - go func(cID uint) { + cID := b.IotCardID + stopresumedomain.RunAfterCommit(stopCtx, s.stopResumeCallback, func() { if err := s.stopResumeCallback.CheckAndStopCard(stopCtx, cID); err != nil { s.logger.Error("调用停机服务失败", zap.Uint("card_id", cID), zap.Error(err)) } - }(cardID) + }) } } diff --git a/internal/service/polling/carrier_scheduler_service.go b/internal/service/polling/carrier_scheduler_service.go index 53c5aaf2..58b1c0c9 100644 --- a/internal/service/polling/carrier_scheduler_service.go +++ b/internal/service/polling/carrier_scheduler_service.go @@ -503,6 +503,28 @@ func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (* } resp.Types = append(resp.Types, state) } + resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled + command, err := s.store.ReadCommand(ctx, card.ID) + if err != nil { + return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败") + } + if !command.LastCallAt.IsZero() { + resp.LastStopResumeCallAt = timePtr(command.LastCallAt) + } + if command.Exists { + item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source, + Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown} + if item.Scenes == nil { + item.Scenes = []string{} + } + if !command.WrittenAt.IsZero() { + item.WrittenAt = timePtr(command.WrittenAt) + } + if !command.NextAt.IsZero() { + item.NextAt = timePtr(command.NextAt) + } + resp.StopResumeCommand = item + } return resp, nil } @@ -571,6 +593,13 @@ func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.Carrie item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited]) } item.QPSLastMinute = float64(attempts) / 60 + if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil { + item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due + item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending + } + srCounts := s.store.StopResumeStatLastHour(ctx, carrierID) + item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed] + item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown] for _, alert := range alertRows { if alert.CarrierID == carrierID { item.OpenAlerts += alert.Count diff --git a/internal/service/refund/service.go b/internal/service/refund/service.go index 2c93caf6..34ab53e9 100644 --- a/internal/service/refund/service.go +++ b/internal/service/refund/service.go @@ -1243,6 +1243,20 @@ func (s *Service) handleRefundAssetProcessing(ctx context.Context, refundID uint func (s *Service) stopAsset(ctx context.Context, assetType string, assetID uint) bool { logger := s.logger + // 停复机出口开关开启：只请求评估（"退款"场景，暂停轮询的资产同样停机），写入成功即视为资产后处理完成； + // 停机失败由指令队列退避与告警处理，不再触发 Outbox 重投与补偿扫描。 + if s.stopResumeService != nil && s.stopResumeService.CommandQueueEnabled(ctx) { + evalType := constants.AssetTypeIotCard + if assetType == constants.AssetTypeDevice { + evalType = constants.AssetTypeDevice + } + if err := s.stopResumeService.RequestAssetEvaluation(ctx, evalType, assetID, constants.StopResumeSceneRefund, "refund"); err != nil { + logger.Error("退款资产处理：请求停机评估失败", zap.String("asset_type", assetType), zap.Uint("asset_id", assetID), zap.Error(err)) + return false + } + return true + } + switch assetType { case "iot_card": // 单卡停机：需要先查卡获取 iccid diff --git a/internal/task/carrier_device_eval_executor.go b/internal/task/carrier_device_eval_executor.go new file mode 100644 index 00000000..0157b056 --- /dev/null +++ b/internal/task/carrier_device_eval_executor.go @@ -0,0 +1,70 @@ +package task + +import ( + "context" + "encoding/json" + "time" + + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue" +) + +// 设备判断的推后时长：旧路径持设备锁时、判断失败时。 +const ( + deviceEvalLegacyLockDelay = 5 * time.Second + deviceEvalFailureDelay = 30 * time.Second + deviceEvalTimeout = 30 * time.Second +) + +// DeviceEvalExecutor 设备维度判断执行器：抢设备判断锁，读取最新数据为设备下各卡生成指令；锁内不调网关。 +type DeviceEvalExecutor struct { + store *carrierqueue.Store + service StopResumeCommandService + instanceID string + logger *zap.Logger +} + +// NewDeviceEvalExecutor 创建设备判断执行器。 +func NewDeviceEvalExecutor(store *carrierqueue.Store, service StopResumeCommandService, instanceID string, logger *zap.Logger) *DeviceEvalExecutor { + if logger == nil { + logger = zap.NewNop() + } + return &DeviceEvalExecutor{store: store, service: service, instanceID: instanceID, logger: logger} +} + +// Handle 消费一条设备判断消息。收尾只移除本次已处理的场景；版本变大时用最新数据再判断一次。 +func (e *DeviceEvalExecutor) Handle(ctx context.Context, body []byte, _ func()) { + var msg carrierqueue.DeviceEvalMessage + if err := json.Unmarshal(body, &msg); err != nil || msg.DeviceID == 0 || msg.Token == "" { + e.logger.Warn("设备判断消息无效，丢弃", zap.ByteString("body", body)) + return + } + got, err := e.store.AcquireDeviceEvalLock(ctx, msg.DeviceID, msg.Token, msg.Token+"|"+e.instanceID, 2*deviceEvalTimeout) + if err != nil || got != 1 { + return + } + claimed := carrierqueue.DeviceEvalClaimed{DeviceID: msg.DeviceID, Token: msg.Token, Version: msg.Version, Scenes: msg.Scenes} + deferFor := time.Duration(0) + if e.service.LegacyDeviceOpLocked(ctx, msg.DeviceID) { + // 旧路径正在对该设备执行停复机：推后，不并行判断。 + deferFor = deviceEvalLegacyLockDelay + } else { + evalCtx, cancel := context.WithTimeout(ctx, deviceEvalTimeout) + if evalErr := e.service.EvaluateDeviceCommands(evalCtx, msg.DeviceID, msg.Scenes); evalErr != nil { + e.logger.Warn("设备判断失败，稍后再判断", zap.Uint("device_id", msg.DeviceID), zap.Strings("scenes", msg.Scenes), zap.Error(evalErr)) + deferFor = deviceEvalFailureDelay + } + cancel() + } + finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer finishCancel() + result, err := e.store.FinishDeviceEval(finishCtx, claimed, deferFor) + if err != nil { + e.logger.Warn("设备判断收尾失败，等待租约到期后重新领取", zap.Uint("device_id", msg.DeviceID), zap.Error(err)) + return + } + if result == 2 { + e.logger.Info("设备判断期间有新请求，按最新数据再判断", zap.Uint("device_id", msg.DeviceID)) + } +} diff --git a/internal/task/carrier_stop_resume_executor.go b/internal/task/carrier_stop_resume_executor.go new file mode 100644 index 00000000..971e38e5 --- /dev/null +++ b/internal/task/carrier_stop_resume_executor.go @@ -0,0 +1,343 @@ +package task + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/gateway" + "github.com/break/junhong_cmp_fiber/internal/infrastructure/gatewaylimit" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue" + iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// StopResumeCommandService 是停复机指令执行器依赖的停复机服务能力。 +type StopResumeCommandService interface { + DecideCommand(ctx context.Context, cardID uint, scenes []string) (iot_card_svc.CommandDecision, error) + ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (iot_card_svc.CommandCallResult, error) + QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error) + ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error + EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error + LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool +} + +// stopResumeTaskTimeout 是一条停复机指令的总执行时长上限（一次判断 + 一次网关调用）。 +const stopResumeTaskTimeout = 60 * time.Second + +// stopResumeRateLimitedDelay 本账户限流拒绝或等待超时（请求未发出）后的重排延迟，不计失败。 +const stopResumeRateLimitedDelay = 5 * time.Second + +// StopResumeExecutor 停复机指令执行器：执行前按场景重新判断，只调用一次网关，按错误分类退避与告警，结果未知在队列内确认。 +type StopResumeExecutor struct { + store *carrierqueue.Store + settings *carrierqueue.SettingsProvider + params func(carrierID uint) carrierqueue.CarrierParams + alerts *carrierqueue.AlertWriter + service StopResumeCommandService + classifier *gatewaylimit.Classifier + instanceID string + logger *zap.Logger +} + +// NewStopResumeExecutor 创建停复机指令执行器。 +func NewStopResumeExecutor(store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, params func(uint) carrierqueue.CarrierParams, + alerts *carrierqueue.AlertWriter, service StopResumeCommandService, classifier *gatewaylimit.Classifier, instanceID string, logger *zap.Logger) *StopResumeExecutor { + if logger == nil { + logger = zap.NewNop() + } + return &StopResumeExecutor{store: store, settings: settings, params: params, alerts: alerts, service: service, + classifier: classifier, instanceID: instanceID, logger: logger} +} + +// srOutcome 是一次执行的收尾决定。 +type srOutcome struct { + stat string // 计数字段覆盖；statSkip 表示不计数 + done bool + nextAt time.Time + set map[string]string + del []string +} + +// statSkip 表示本次执行不计入监控计数（指令不再需要、卡已不存在）。 +const statSkip = "-" + +func srDone() srOutcome { return srOutcome{done: true} } + +func srEnded() srOutcome { return srOutcome{done: true, stat: statSkip} } + +func srRetry(at time.Time) srOutcome { + return srOutcome{nextAt: at, set: map[string]string{}} +} + +// Handle 消费一条停复机指令消息；返回后由消费者 FIN。重试与退避全部由 Redis score 表达。 +func (e *StopResumeExecutor) Handle(ctx context.Context, body []byte, _ func()) { + var msg carrierqueue.StopResumeMessage + if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" { + e.logger.Warn("停复机指令消息无效，丢弃", zap.ByteString("body", body)) + return + } + execValue := msg.Token + "|" + e.instanceID + got, err := e.store.AcquireStopResumeExec(ctx, msg.CardID, msg.Token, execValue, 2*stopResumeTaskTimeout) + if err != nil || got != 1 { + return + } + taskCtx, cancel := context.WithTimeout(ctx, stopResumeTaskTimeout) + outcome := e.execute(taskCtx, msg) + cancel() + + finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer finishCancel() + e.countOutcome(finishCtx, msg.CarrierID, outcome) + result, err := e.store.FinishStopResume(finishCtx, carrierqueue.SRFinish{ + CardID: msg.CardID, CarrierID: msg.CarrierID, Token: msg.Token, Version: msg.Version, + Done: outcome.done, NextAt: outcome.nextAt, Set: outcome.set, Del: outcome.del, + }) + switch { + case err != nil: + e.logger.Warn("停复机指令收尾失败，等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(err)) + case result == 0: + e.logger.Warn("停复机指令收尾时处理中标记已不一致", zap.Uint("card_id", msg.CardID), zap.String("metric", "stop_resume.finish_token_mismatch")) + case result == 2: + e.logger.Info("停复机指令执行期间已被新指令覆盖，保留新指令", zap.Uint("card_id", msg.CardID), zap.Int64("claimed_version", msg.Version)) + } +} + +func (e *StopResumeExecutor) execute(ctx context.Context, msg carrierqueue.StopResumeMessage) (outcome srOutcome) { + defer func() { + if r := recover(); r != nil { + e.logger.Error("停复机指令执行 panic，稍后重试", zap.Uint("card_id", msg.CardID), zap.Any("panic", r)) + outcome = srRetry(time.Now().Add(time.Minute)) + } + }() + cmd, err := e.store.ReadCommand(ctx, msg.CardID) + if err != nil { + return srRetry(time.Now().Add(time.Minute)) + } + if !cmd.Exists { + return srEnded() + } + decision, err := e.service.DecideCommand(ctx, msg.CardID, cmd.Scenes) + if err != nil { + e.logger.Warn("停复机指令执行前判断失败，稍后重试", zap.Uint("card_id", msg.CardID), zap.Error(err)) + return srRetry(time.Now().Add(time.Minute)) + } + if decision.Gone { + return srEnded() + } + if decision.Action != cmd.Action { + // 条件不再满足或已是目标状态：结束指令，不调用上游。 + e.logger.Info("停复机指令执行前重新判断不再需要，结束指令", zap.Uint("card_id", msg.CardID), + zap.String("action", cmd.Action), zap.String("decided", decision.Action), zap.Strings("scenes", cmd.Scenes)) + e.recovered(cmd, decision.Card) + return srEnded() + } + card := decision.Card + reason := decision.Reason + params := e.params(card.CarrierID) + settings := e.settings.Current(ctx) + gatewayCtx := gateway.WithRateLimitWait(ctx, 20*time.Second) + + if !cmd.Pending { + return e.call(gatewayCtx, card, cmd, reason, params, settings) + } + if confirmed, next := e.confirm(gatewayCtx, card, cmd, reason, params, settings); confirmed != nil { + return *confirmed + } else if next != nil { + return *next + } + // 确认不是目标状态且已满足单卡最小间隔：清除待确认后重新调用。 + result := e.call(gatewayCtx, card, cmd, reason, params, settings) + if !result.done && result.set[carrierqueue.SRFieldPending] != "1" { + result.del = append(result.del, carrierqueue.SRFieldPending) + } + return result +} + +// confirm 处理待确认指令：查一次卡状态。已是目标状态返回结束；查询失败保持待确认按退避重查； +// 不是目标状态时清除待确认，未到单卡最小间隔则推后，否则返回 (nil, nil) 由调用方重新发起停复机。 +func (e *StopResumeExecutor) confirm(ctx context.Context, card *model.IotCard, cmd carrierqueue.CommandState, reason string, + params carrierqueue.CarrierParams, settings carrierqueue.Settings) (*srOutcome, *srOutcome) { + status, known, err := e.service.QueryCommandCardStatus(ctx, card) + if gateway.IsRateLimited(err) { + retry := srRetry(time.Now().Add(stopResumeRateLimitedDelay)) + return nil, &retry + } + if err != nil || !known { + retry := srRetry(time.Now().Add(settings.SRBackoffFor(cmd.Unknown))) + if err != nil { + retry.set[carrierqueue.SRFieldError] = truncateRunes(constants.StopResumeFailureUnknown+"|确认查询失败: "+err.Error(), 300) + } + return nil, &retry + } + if status == targetNetworkStatus(cmd.Action) { + if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil { + e.logger.Warn("确认已是目标状态但回写失败，稍后再确认", zap.Uint("card_id", card.ID), zap.Error(confirmErr)) + retry := srRetry(time.Now().Add(time.Minute)) + return nil, &retry + } + e.logger.Info("停复机结果未知，确认查询已是目标状态，按成功结束", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action)) + e.recovered(cmd, card) + done := srDone() + return &done, nil + } + last, _ := e.store.LastStopResumeCall(ctx, card.ID) + if allowed := last.Add(params.SRMinInterval); !last.IsZero() && allowed.After(time.Now()) { + retry := srRetry(allowed) + retry.del = []string{carrierqueue.SRFieldPending} + return nil, &retry + } + return nil, nil +} + +// call 调用一次停机或复机并按结果分类处理。 +func (e *StopResumeExecutor) call(ctx context.Context, card *model.IotCard, cmd carrierqueue.CommandState, reason string, + params carrierqueue.CarrierParams, settings carrierqueue.Settings) srOutcome { + previousCall, _ := e.store.LastStopResumeCall(ctx, card.ID) + now := time.Now() + if err := e.store.MarkStopResumeCall(ctx, card.ID, now, 2*params.SRMinInterval); err != nil { + return srRetry(now.Add(time.Minute)) + } + result, err := e.service.ExecuteCommand(ctx, card, cmd.Action, reason) + if err == nil { + e.logger.Info("停复机指令执行成功", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action), zap.String("reason", reason), + zap.Strings("scenes", cmd.Scenes), zap.String("metric", "stop_resume.success")) + e.recovered(cmd, card) + return srDone() + } + if gateway.IsRateLimited(err) { + // 本账户限流：请求未发出，恢复最近调用时间，短延迟重排，不计失败。 + e.restoreLastCall(ctx, card.ID, previousCall, params) + return srRetry(time.Now().Add(stopResumeRateLimitedDelay)) + } + classification := e.classify(ctx, cmd.Action, err) + if result.Unknown || classification.MaybeAccepted { + return e.unknown(card, cmd, err, params, settings) + } + message := truncateRunes(classification.Category+"|"+err.Error(), 300) + switch classification.Category { + case constants.GatewayClassAlreadyTarget: + if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil { + return e.unknown(card, cmd, confirmErr, params, settings) + } + e.recovered(cmd, card) + return srDone() + case constants.GatewayClassTerminal: + // 终态不可操作：不重试、结束指令、写告警；停机方向上游已不可用时按目标状态视为已停。 + if cmd.Action == constants.StopResumeActionStop { + if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil { + e.logger.Warn("停机遇上游终态，回写已停失败", zap.Uint("card_id", card.ID), zap.Error(confirmErr)) + } + } + e.reportFailure(card, cmd, classification.Category, err.Error(), cmd.FailCount+1, now) + return srOutcome{done: true, stat: constants.StopResumeStatFailed} + case constants.GatewayClassRateLimitedUpstream: + retry := srRetry(now.Add(maxDuration(params.SRMinInterval, settings.SRBackoffFor(1)))) + retry.set[carrierqueue.SRFieldError] = message + return retry + } + failCount := cmd.FailCount + 1 + retry := srRetry(now.Add(settings.SRBackoffFor(failCount))) + retry.set[carrierqueue.SRFieldFail] = strconv.Itoa(failCount) + retry.set[carrierqueue.SRFieldError] = message + firstFailed := cmd.FirstFailed + if firstFailed.IsZero() { + firstFailed = now + retry.set[carrierqueue.SRFieldFirst] = strconv.FormatInt(now.UnixMilli(), 10) + } + if failCount >= settings.SRAlertFailCount || settings.ImmediateAlert[classification.Category] { + e.reportFailure(card, cmd, classification.Category, err.Error(), failCount, firstFailed) + } + e.logger.Warn("停复机指令执行失败，按退避重试", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action), + zap.String("category", classification.Category), zap.Int("fail_count", failCount), zap.Time("next_at", retry.nextAt), + zap.String("metric", "stop_resume.failed"), zap.Error(err)) + return retry +} + +// unknown 结果未知：置待确认，确认延迟后先判断再查状态；连续未知达阈值告警，指令不丢弃。 +func (e *StopResumeExecutor) unknown(card *model.IotCard, cmd carrierqueue.CommandState, err error, params carrierqueue.CarrierParams, + settings carrierqueue.Settings) srOutcome { + unknown := cmd.Unknown + 1 + retry := srRetry(time.Now().Add(params.SRConfirmDelay)) + retry.set[carrierqueue.SRFieldPending] = "1" + retry.set[carrierqueue.SRFieldUnknown] = strconv.Itoa(unknown) + retry.set[carrierqueue.SRFieldError] = truncateRunes(constants.StopResumeFailureUnknown+"|"+err.Error(), 300) + if unknown >= settings.SRUnknownAlert { + e.reportFailure(card, cmd, constants.StopResumeFailureUnknown, err.Error(), unknown, cmd.WrittenAt) + } + e.logger.Warn("停复机结果未知，确认延迟后查状态", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action), + zap.Int("unknown", unknown), zap.String("metric", "stop_resume.unknown"), zap.Error(err)) + return retry +} + +func (e *StopResumeExecutor) classify(ctx context.Context, action string, err error) gatewaylimit.Classification { + gwErr, ok := gateway.AsGatewayError(err) + if !ok || gwErr.Category != constants.GatewayErrorUpstream { + return gatewaylimit.Classification{Category: constants.GatewayClassTransient} + } + op := constants.GatewayOpStop + if action == constants.StopResumeActionStart { + op = constants.GatewayOpStart + } + return e.classifier.Classify(ctx, op, gwErr.ProviderMessage) +} + +func (e *StopResumeExecutor) restoreLastCall(ctx context.Context, cardID uint, previous time.Time, params carrierqueue.CarrierParams) { + if previous.IsZero() { + e.store.Client().Del(ctx, constants.RedisCardStopResumeLastCallKey(cardID)) + return + } + _ = e.store.MarkStopResumeCall(ctx, cardID, previous, 2*params.SRMinInterval) +} + +func (e *StopResumeExecutor) reportFailure(card *model.IotCard, cmd carrierqueue.CommandState, category, message string, failCount int, firstFailed time.Time) { + if firstFailed.IsZero() { + firstFailed = time.Now() + } + e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, + CheckType: constants.CheckTypeStopResume, Category: category, Message: cmd.Action + ": " + message, + FailCount: failCount, FirstFailed: firstFailed, At: time.Now()}) +} + +func (e *StopResumeExecutor) recovered(cmd carrierqueue.CommandState, card *model.IotCard) { + if card != nil && (cmd.FailCount > 0 || cmd.Unknown > 0) { + e.alerts.ReportRecovered(card.ID, constants.CheckTypeStopResume) + } +} + +func targetNetworkStatus(action string) int { + if action == constants.StopResumeActionStop { + return constants.NetworkStatusOffline + } + return constants.NetworkStatusOnline +} + +func maxDuration(a, b time.Duration) time.Duration { + if a > b { + return a + } + return b +} + +// countOutcome 累加执行计数：结束为成功，置待确认为结果未知，带失败次数为失败，其余推后为重试。 +func (e *StopResumeExecutor) countOutcome(ctx context.Context, carrierID uint, outcome srOutcome) { + if outcome.stat == statSkip { + return + } + field := constants.StopResumeStatRetry + switch { + case outcome.stat != "": + field = outcome.stat + case outcome.done: + field = constants.StopResumeStatSuccess + case outcome.set[carrierqueue.SRFieldPending] == "1": + field = constants.StopResumeStatUnknown + case outcome.set[carrierqueue.SRFieldFail] != "": + field = constants.StopResumeStatFailed + } + e.store.IncrStopResumeStat(ctx, carrierID, field) +} diff --git a/migrations/000255_add_stop_resume_queue_system_config.down.sql b/migrations/000255_add_stop_resume_queue_system_config.down.sql new file mode 100644 index 00000000..f8030332 --- /dev/null +++ b/migrations/000255_add_stop_resume_queue_system_config.down.sql @@ -0,0 +1,4 @@ +-- 回滚停复机指令队列的受控系统配置。 +DELETE FROM tb_system_config WHERE config_key IN ( + 'polling.stop_resume.queue_enabled', 'polling.stop_resume.unknown_alert_threshold', + 'polling.stop_resume.backoff_seconds', 'polling.stop_resume.alert_fail_count'); diff --git a/migrations/000255_add_stop_resume_queue_system_config.up.sql b/migrations/000255_add_stop_resume_queue_system_config.up.sql new file mode 100644 index 00000000..1e7e3915 --- /dev/null +++ b/migrations/000255_add_stop_resume_queue_system_config.up.sql @@ -0,0 +1,16 @@ +-- 停复机指令队列的受控系统配置（add-stop-resume-command-queue），module = polling。 +-- 配置的权威定义（类型、取值范围、默认值、JSON 结构校验）在代码注册表中；本迁移只落库初始值。 +-- 已存在同名 Key 时不覆盖（ON CONFLICT DO NOTHING）。 +-- +-- 1. polling.stop_resume.queue_enabled：自动停复机经停复机指令队列执行的全局开关，默认关闭；关闭即各入口当场执行（回退）。 +-- 2. polling.stop_resume.unknown_alert_threshold：停复机连续结果未知达到该次数时写失败告警，指令不丢弃。 +-- 3. polling.stop_resume.backoff_seconds：停复机失败退避序列（秒，递增），超过序列长度按最后一项重试。 +-- 4. polling.stop_resume.alert_fail_count：停复机连续失败告警阈值。 +-- 账户级停复机参数（执行上限、max_in_flight、单卡最小重试间隔、确认延迟）已在 tb_carrier_polling_param 建列（000251）。 + +INSERT INTO tb_system_config (config_key, config_value, value_type, module, description, is_readonly, is_sensitive, creator, updater, created_at, updated_at) VALUES + ('polling.stop_resume.queue_enabled', 'false', 'bool', 'polling', '自动停复机经停复机指令队列执行（全局开关，关闭即各入口当场执行）', FALSE, FALSE, 0, 0, NOW(), NOW()), + ('polling.stop_resume.unknown_alert_threshold', '3', 'int', 'polling', '停复机连续结果未知告警阈值', FALSE, FALSE, 0, 0, NOW(), NOW()), + ('polling.stop_resume.backoff_seconds', '[600,900,1800,3600]', 'json', 'polling', '停复机失败退避序列（秒，递增）', FALSE, FALSE, 0, 0, NOW(), NOW()), + ('polling.stop_resume.alert_fail_count', '5', 'int', 'polling', '停复机连续失败告警阈值', FALSE, FALSE, 0, 0, NOW(), NOW()) +ON CONFLICT (config_key) DO NOTHING; diff --git a/openspec/changes/add-stop-resume-command-queue/tasks.md b/openspec/changes/add-stop-resume-command-queue/tasks.md index a7271521..d2cac0db 100644 --- a/openspec/changes/add-stop-resume-command-queue/tasks.md +++ b/openspec/changes/add-stop-resume-command-queue/tasks.md @@ -1,50 +1,50 @@ ## 0. 前置确认 -- [ ] 0.1 确认第 1、4、5 个 Change 已上线。 +- [x] 0.1 确认第 1、4、5 个 Change 已上线。（第 1、4、5 个 Change 已部署测试环境；生产按第五部分第 3 节依赖顺序上线） ## 1. 迁移与配置 -- [ ] 1.1 新增成对迁移：`tb_system_config` 插入 `polling.stop_resume.queue_enabled`（默认 false）、结果未知告警阈值、停复机退避序列与告警阈值；注册受控系统配置项。 +- [x] 1.1 新增成对迁移：`tb_system_config` 插入 `polling.stop_resume.queue_enabled`（默认 false）、结果未知告警阈值、停复机退避序列与告警阈值；注册受控系统配置项。（000255；carrierqueue.RegisterConfigDefinitions 注册 queue_enabled、unknown_alert_threshold、backoff_seconds、alert_fail_count，退避序列校验非空递增） ## 2. 统一判断与请求评估 -- [ ] 2.1 抽出统一判断函数（暂停轮询按场景 → 停机条件 → 复机条件 → 通道阈值锁），`EvaluateAndAct`、`resumeSingleCard`、`resumeDeviceCards`、`ResumeCardIfStopped` 改为调用它。 -- [ ] 2.2 新增"请求评估（卡或设备、场景、来源）"门面：开关开启时单卡生成指令、设备请求设备判断；关闭时走旧路径当场执行。 -- [ ] 2.3 指令写入 Lua：覆盖并 `HINCRBY ver`、方向相反清零失败计数、放入卡所属账户 `stop_resume`。 +- [x] 2.1 抽出统一判断函数（暂停轮询按场景 → 停机条件 → 复机条件 → 通道阈值锁），`EvaluateAndAct`、`resumeSingleCard`、`resumeDeviceCards`、`ResumeCardIfStopped` 改为调用它。（decideCard：EvaluateAndAct 新旧两条路径共用；暂停轮询的卡只在套餐到期、退款场景放行无套餐停机；通道阈值锁在复机写指令与执行前判断） +- [x] 2.2 新增"请求评估（卡或设备、场景、来源）"门面：开关开启时单卡生成指令、设备请求设备判断；关闭时走旧路径当场执行。（RequestAssetEvaluation 门面；EvaluateAndAct、ResumeCardIfStopped、CheckAndStopCard 签名不变，开关开启时改为写指令或请求设备判断，场景经 ctx 传入） +- [x] 2.3 指令写入 Lua：覆盖并 `HINCRBY ver`、方向相反清零失败计数、放入卡所属账户 `stop_resume`。（srWriteLua：HINCRBY ver、方向相反清零 fail/pending/unknown、ZADD LT；执行中只更新内容与版本） ## 3. 指令领取与执行 -- [ ] 3.1 分发器在卡任务之前领取 `stop_resume`：清理过期 `sr_running`，领取数 = min(剩余 QPS, `sr_inflight_limit` − 执行中)，校验 `sr_last_call` 推后，写 `sr_claim` 并记版本；开关开启后对全部账户领取。 -- [ ] 3.2 NSQ `polling.stop_resume.{cid}` 专用消费者：核对 token、抢 `sr_exec`、续期。 -- [ ] 3.3 执行器：按场景重新判断 → 写 `sr_last_call` → 调用一次网关（限流等待模式）→ 成功沿用现有回写与停复机记录；收尾按版本移除或保留。 -- [ ] 3.4 结果未知：置待确认，确认延迟后先判断再查状态，按规则结束、重发或推后；查询失败不重发；连续未知达阈值告警。 -- [ ] 3.5 失败分类处理：频率限制、终态（含自动关闭轮询）、已是目标状态、账户配置错误、临时故障与业务错误；停复机方向写入 `tb_polling_failure_alert`；指令不丢弃。 +- [x] 3.1 分发器在卡任务之前领取 `stop_resume`：清理过期 `sr_running`，领取数 = min(剩余 QPS, `sr_inflight_limit` − 执行中)，校验 `sr_last_call` 推后，写 `sr_claim` 并记版本；开关开启后对全部账户领取。（dispatchStopResume 在 dispatchCarrier 前执行，容量 = min(停机/复机所在限流组剩余, sr_inflight_limit, 消费能力)；领取 Lua 校验 sr_last_call 推后；只看开关，不受 scheduler_mode 限制） +- [x] 3.2 NSQ `polling.stop_resume.{cid}` 专用消费者：核对 token、抢 `sr_exec`、续期。（polling.stop_resume.{cid} 专用 ConsumerManager，核对 token 抢 sr_exec；单次执行 ≤60 秒、租约 2 分钟，不续期） +- [x] 3.3 执行器：按场景重新判断 → 写 `sr_last_call` → 调用一次网关（限流等待模式）→ 成功沿用现有回写与停复机记录；收尾按版本移除或保留。（StopResumeExecutor：DecideCommand → MarkStopResumeCall → ExecuteCommand 单次调用（限流等待模式）→ 现有回写与停复机记录；srFinishLua 按版本移除或保留） +- [x] 3.4 结果未知：置待确认，确认延迟后先判断再查状态，按规则结束、重发或推后；查询失败不重发；连续未知达阈值告警。（超时、连接失败、MaybeAccepted、成功后回写失败 → pending；确认延迟后先判断再查状态，查询失败不重发；unknown 达阈值告警） +- [x] 3.5 失败分类处理：频率限制、终态（含自动关闭轮询）、已是目标状态、账户配置错误、临时故障与业务错误；停复机方向写入 `tb_polling_failure_alert`；指令不丢弃。（频率限制不计失败按 max(最小间隔, 首个退避) 推后；终态结束并告警，停机方向回写已停，自动关闭轮询沿用 TerminalPollingDisabler；已是目标状态按成功回写；account_config 首次告警；其余计失败按停复机退避序列；check_type=stop_resume 写入 tb_polling_failure_alert） ## 4. 设备维度判断 -- [ ] 4.1 请求设备判断 Lua：`INCR eval_ver`、`SADD eval_scenes`、`ZADD LT device_eval:queue`。 -- [ ] 4.2 分发与消费：抢 `device_eval:dispatch_lock` 领取，NSQ `polling.device_eval`；消费者抢 `eval_lock`（检查旧路径设备锁），读最新数据判断，为各卡写指令（记录场景）。 -- [ ] 4.3 收尾：`SREM` 已处理场景，比较版本，变大则 score 设为当前时间再判断。 -- [ ] 4.4 轮询路径不再使用 `RedisPollingDeviceOpLockKey`；人工设备停复机保持不变。 +- [x] 4.1 请求设备判断 Lua：`INCR eval_ver`、`SADD eval_scenes`、`ZADD LT device_eval:queue`。（deviceEvalRequestLua：INCR eval_ver、SADD eval_scenes、ZADD LT） +- [x] 4.2 分发与消费：抢 `device_eval:dispatch_lock` 领取，NSQ `polling.device_eval`；消费者抢 `eval_lock`（检查旧路径设备锁），读最新数据判断，为各卡写指令（记录场景）。（device_eval 全局分发锁领取，polling.device_eval 消费者抢 eval_lock；旧路径设备锁存在时推后 5 秒；EvaluateDeviceCommands 逐卡统一判断并写指令） +- [x] 4.3 收尾：`SREM` 已处理场景，比较版本，变大则 score 设为当前时间再判断。（deviceEvalFinishLua：SREM 已处理场景，版本变大立即再判断） +- [x] 4.4 轮询路径不再使用 `RedisPollingDeviceOpLockKey`；人工设备停复机保持不变。（开关开启时轮询路径经设备判断，不再抢 RedisPollingDeviceOpLockKey；人工设备停复机不变） ## 5. 自动入口改造 -- [ ] 5.1 套餐步骤与 Outbox 三个卡观测消费者改为请求评估，不再同步调网关、不 sleep 重试。 -- [ ] 5.2 `resumeAfterActivation`（4 处）、`reset_service.go:400`、`order/service.go:2196` 去掉 goroutine，事务提交后请求评估。 -- [ ] 5.3 `refund/service.go` `stopAsset` 改为请求评估，写入成功即标记 `asset_reset`；`RecoverRefundPostProcessing` 只补偿"请求未写入"。 -- [ ] 5.4 `package_activation_handler.go` 到期失效处：无其他有效主套餐时以"套餐到期"场景请求评估；接续激活路径不变。 -- [ ] 5.5 `carrier/service.go` 实名管控修改后按 0 → 1、1 → 0 筛选受影响卡批量请求评估（设备卡按设备）。 +- [x] 5.1 套餐步骤与 Outbox 三个卡观测消费者改为请求评估，不再同步调网关、不 sleep 重试。（套餐步骤经 EvaluateAndAct；Outbox 网络、实名、流量三个消费者标场景 network_observed / realname_observed / flow_observed） +- [x] 5.2 `resumeAfterActivation`（4 处）、`reset_service.go:400`、`order/service.go:2196` 去掉 goroutine，事务提交后请求评估。（stopresume.RunAfterCommit：开关开启时事务提交后同步请求评估、不起 goroutine；关闭时沿用 goroutine 当场执行；流量扣减后停机同样处理） +- [x] 5.3 `refund/service.go` `stopAsset` 改为请求评估，写入成功即标记 `asset_reset`；`RecoverRefundPostProcessing` 只补偿"请求未写入"。（开关开启时 stopAsset 以 refund 场景请求评估，写入成功即标记 asset_reset；关闭时沿用 ManualStopCard；补偿扫描只会补"请求未写入"的退款） +- [x] 5.4 `package_activation_handler.go` 到期失效处：无其他有效主套餐时以"套餐到期"场景请求评估；接续激活路径不变。（triggerStopAfterExpiry 以 package_expired 场景请求评估，暂停轮询的卡同样停机；接续激活路径不变） +- [x] 5.5 `carrier/service.go` 实名管控修改后按 0 → 1、1 → 0 筛选受影响卡批量请求评估（设备卡按设备）。（RequestRealnameControlEvaluation：0→1 在线未实名、1→0 停机未实名，设备卡按设备，分批 500；仅开关开启时执行，保存接口提交后异步触发） ## 6. 监控与接口 -- [ ] 6.1 `/polling-stats` 按账户增加停复机队列长度、执行中、成功、失败、重试、结果未知数。 -- [ ] 6.2 `/iot-cards/:id/polling-state` 返回当前指令（动作、原因、场景、版本、失败次数、待确认）。 -- [ ] 6.3 重新生成接口文档。 +- [x] 6.1 `/polling-stats` 按账户增加停复机队列长度、执行中、成功、失败、重试、结果未知数。（/polling-stats/carrier-queues 增加停复机队列长度、到期、执行中、待确认，近 1 小时成功、失败、重试、结果未知） +- [x] 6.2 `/iot-cards/:id/polling-state` 返回当前指令（动作、原因、场景、版本、失败次数、待确认）。（返回 stop_resume_queue_enabled、stop_resume_command、last_stop_resume_call_at） +- [x] 6.3 重新生成接口文档。 ## 7. 编译与规范校验 -- [ ] 7.1 `gofmt -w`；`go build ./cmd/api ./cmd/worker` 通过。 -- [ ] 7.2 `openspec validate add-stop-resume-command-queue` 与 `openspec doctor --json` 通过。 +- [x] 7.1 `gofmt -w`；`go build ./cmd/api ./cmd/worker` 通过。 +- [x] 7.2 `openspec validate add-stop-resume-command-queue` 与 `openspec doctor --json` 通过。（validate 通过；当前 openspec 无 doctor 命令） ## 8. 测试环境验证（自动化测试为 N/A，按 ENG-TEST-001） diff --git a/pkg/constants/stop_resume_queue.go b/pkg/constants/stop_resume_queue.go new file mode 100644 index 00000000..67917c78 --- /dev/null +++ b/pkg/constants/stop_resume_queue.go @@ -0,0 +1,174 @@ +package constants + +import ( + "fmt" + "strings" +) + +// 停复机指令队列（add-stop-resume-command-queue）。 + +// 停复机请求评估场景：随请求评估与指令保存，执行前重新判断沿用同一场景。 +const ( + StopResumeScenePolling = "polling" // 套餐步骤（周期评估） + StopResumeSceneFlowObserved = "flow_observed" // 流量增加观测 + StopResumeSceneNetworkObserved = "network_observed" // 网络状态变化观测 + StopResumeSceneRealnameObserved = "realname_observed" // 实名变化观测（含人工实名） + StopResumeSceneActivation = "activation" // 套餐激活 + StopResumeSceneReset = "reset" // 流量重置 + StopResumeSceneOrder = "order" // 订单支付后套餐生效 + StopResumeSceneRefund = "refund" // 退款后资产无可用主套餐 + StopResumeScenePackageExpired = "package_expired" // 套餐到期且无其他有效主套餐 + StopResumeSceneRealnameControl = "realname_control" // 运营商账户实名管控修改 + StopResumeSceneUsage = "usage" // 流量扣减后 +) + +// StopResumeSceneAllowsSuspendedStop 判断场景集合中是否含套餐失效类场景：暂停轮询的卡在这些场景下照常停机。 +func StopResumeSceneAllowsSuspendedStop(scenes []string) bool { + for _, scene := range scenes { + if scene == StopResumeScenePackageExpired || scene == StopResumeSceneRefund { + return true + } + } + return false +} + +// JoinStopResumeScenes 把场景集合编码为逗号分隔串（去重、去空）。 +func JoinStopResumeScenes(scenes []string) string { + seen := map[string]bool{} + parts := make([]string, 0, len(scenes)) + for _, scene := range scenes { + scene = strings.TrimSpace(scene) + if scene == "" || seen[scene] { + continue + } + seen[scene] = true + parts = append(parts, scene) + } + return strings.Join(parts, ",") +} + +// SplitStopResumeScenes 解析逗号分隔的场景串。 +func SplitStopResumeScenes(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + parts := strings.Split(raw, ",") + scenes := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + scenes = append(scenes, part) + } + } + return scenes +} + +// CheckTypeStopResume 是失败告警表中停复机方向的检查类型。 +const CheckTypeStopResume = "stop_resume" + +// 停复机指令失败类别（不来自上游错误分类的部分）。 +const ( + StopResumeFailureUnknown = "unknown" // 结果未知（超时、连接失败、可能已受理） +) + +// 停复机指令默认参数（账户未配置时使用）。 +const ( + DefaultStopResumeMinIntervalSeconds = 600 // 单卡最小重试间隔：上游卡状态变更每十分钟只允许调用一次 + DefaultStopResumeConfirmDelaySeconds = 60 // 结果未知后的确认延迟 + DefaultStopResumeInflightLimit = 10 // 账户停复机执行上限 + DefaultStopResumeMaxInFlight = 10 // 每进程停复机消费者 max_in_flight +) + +// 停复机指令队列受控系统配置键（module = polling）。 +const ( + SystemConfigStopResumeQueueEnabled = "polling.stop_resume.queue_enabled" + SystemConfigStopResumeUnknownAlert = "polling.stop_resume.unknown_alert_threshold" + SystemConfigStopResumeBackoff = "polling.stop_resume.backoff_seconds" + SystemConfigStopResumeAlertFailCount = "polling.stop_resume.alert_fail_count" +) + +// ======================================== +// 停复机指令队列 Redis Key +// ======================================== + +// RedisCarrierStopResumeQueueKey 账户停复机指令队列（ZSET，member=card_id，score=可执行时间毫秒）。 +func RedisCarrierStopResumeQueueKey(carrierID uint) string { + return fmt.Sprintf("polling:carrier:%d:stop_resume", carrierID) +} + +// RedisCarrierStopResumeRunningKey 账户停复机执行中计数（ZSET，member=token，score=租约到期毫秒）。 +func RedisCarrierStopResumeRunningKey(carrierID uint) string { + return fmt.Sprintf("polling:carrier:%d:sr_running", carrierID) +} + +// RedisCardStopResumeCmdKey 卡停复机指令（Hash：action、reason、scenes、source、carrier、ver、fail、ffail、err、pending、unknown、at）。 +func RedisCardStopResumeCmdKey(cardID uint) string { + return fmt.Sprintf("polling:card:%d:stop_resume_cmd", cardID) +} + +// RedisCardStopResumeClaimKey 停复机指令处理中标记，值=派发 token，TTL=租约。 +func RedisCardStopResumeClaimKey(cardID uint) string { + return fmt.Sprintf("polling:card:%d:sr_claim", cardID) +} + +// RedisCardStopResumeExecKey 停复机指令执行权，值=token + 实例。 +func RedisCardStopResumeExecKey(cardID uint) string { + return fmt.Sprintf("polling:card:%d:sr_exec", cardID) +} + +// RedisCardStopResumeLastCallKey 本卡最近一次停复机网关调用时间（毫秒，停机与复机合计）。 +func RedisCardStopResumeLastCallKey(cardID uint) string { + return fmt.Sprintf("polling:card:%d:sr_last_call", cardID) +} + +// RedisDeviceEvalQueueKey 设备判断队列（ZSET，member=device_id，score=可执行时间毫秒）。 +const RedisDeviceEvalQueueKey = "polling:device_eval:queue" + +// RedisDeviceEvalRunningKey 设备判断执行中计数（ZSET，member=token）。 +const RedisDeviceEvalRunningKey = "polling:device_eval:running" + +// RedisDeviceEvalDispatchLockKey 设备判断分发锁。 +const RedisDeviceEvalDispatchLockKey = "polling:device_eval:dispatch_lock" + +// RedisDeviceEvalVersionKey 设备判断版本号（每次请求 INCR）。 +func RedisDeviceEvalVersionKey(deviceID uint) string { + return fmt.Sprintf("polling:device:%d:eval_ver", deviceID) +} + +// RedisDeviceEvalScenesKey 设备待判断场景集合（Set）。 +func RedisDeviceEvalScenesKey(deviceID uint) string { + return fmt.Sprintf("polling:device:%d:eval_scenes", deviceID) +} + +// RedisDeviceEvalClaimKey 设备判断处理中标记，值=token。 +func RedisDeviceEvalClaimKey(deviceID uint) string { + return fmt.Sprintf("polling:device:%d:eval_claim", deviceID) +} + +// RedisDeviceEvalLockKey 设备判断锁（执行权），同一设备同一时刻只有一个判断在执行。 +func RedisDeviceEvalLockKey(deviceID uint) string { + return fmt.Sprintf("polling:device:%d:eval_lock", deviceID) +} + +// NSQStopResumeTopic 返回账户停复机指令 topic。 +func NSQStopResumeTopic(carrierID uint) string { + return fmt.Sprintf("polling.stop_resume.%d", carrierID) +} + +// NSQDeviceEvalTopic 设备判断 topic。 +const NSQDeviceEvalTopic = "polling.device_eval" + +// 停复机指令执行计数字段（按 10 分钟分桶，供监控汇总近 1 小时）。 +const ( + StopResumeStatSuccess = "success" + StopResumeStatFailed = "failed" + StopResumeStatRetry = "retry" + StopResumeStatUnknown = "unknown" +) + +// StopResumeStatBucketSeconds 是停复机执行计数的分桶时长。 +const StopResumeStatBucketSeconds = 600 + +// RedisCarrierStopResumeStatKey 账户停复机执行计数（Hash，field=计数字段），bucket = unix 秒 / 分桶时长。 +func RedisCarrierStopResumeStatKey(carrierID uint, bucket int64) string { + return fmt.Sprintf("polling:carrier:%d:sr_stats:%d", carrierID, bucket) +}