feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Has been cancelled
Some checks failed
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Has been cancelled
自动停复机统一经停复机指令队列执行（全局开关 polling.stop_resume.queue_enabled，默认关闭）： - 统一判断函数 decideCard，新旧路径共用；暂停轮询的卡在套餐到期、退款场景照常停机 - 同卡一条指令、版本号覆盖；领取校验单卡最小重试间隔；先于卡任务领取，不受 scheduler_mode 限制 - 执行器只调用一次网关，按错误分类退避与告警，结果未知在队列内确认 - 设备维度判断合并请求与场景，版本变化再判断，旧路径持设备锁时推后 - 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估 - 监控增加停复机队列与近 1 小时执行计数，单卡轮询状态返回当前指令 - 迁移 000255 插入开关与告警、退避参数 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
@@ -283,6 +283,8 @@ func initServices(s *stores, deps *Dependencies) *services {
|
||||
}
|
||||
// 持通道阈值锁的卡在周期内拒绝一切复机（自动、手动、机卡分离）。
|
||||
stopResumeService.SetChannelThresholdLockGuard(carrierThresholdService)
|
||||
// 停复机指令队列：开关开启时自动入口只请求评估，由 Worker 指令执行器调用网关。
|
||||
stopResumeService.SetCommandQueue(carrierqueue.NewCommandQueue(carrierStore, carrierSettings))
|
||||
iotCard.SetRealnameActivator(packageActivation)
|
||||
iotCard.SetStopResumeService(stopResumeService)
|
||||
iotCard.SetDeviceSimBindingStore(s.DeviceSimBinding)
|
||||
@@ -543,7 +545,7 @@ func initServices(s *stores, deps *Dependencies) *services {
|
||||
Device: device,
|
||||
DeviceImport: deviceImportSvc.New(deps.DB, s.DeviceImportTask, deps.QueueClient, auditWriter),
|
||||
AssetAllocationRecord: assetAllocationRecordSvc.New(deps.DB, s.AssetAllocationRecord, s.Shop, s.Account),
|
||||
Carrier: carrierSvc.New(s.Carrier, auditWriter),
|
||||
Carrier: newCarrierService(s.Carrier, auditWriter, stopResumeService, deps.Logger),
|
||||
PackageSeries: packageSeriesService,
|
||||
Package: packageService,
|
||||
PackageDailyRecord: packageSvc.NewDailyRecordService(deps.DB, deps.Redis, s.PackageUsageDailyRecord, deps.Logger),
|
||||
@@ -597,3 +599,12 @@ func initServices(s *stores, deps *Dependencies) *services {
|
||||
CardObservationSeries: cardObservationSeries,
|
||||
}
|
||||
}
|
||||
|
||||
// newCarrierService 创建运营商配置服务并注入实名管控修改后的批量停复机评估。
|
||||
func newCarrierService(store *postgres.CarrierStore, auditWriter *auditInfra.Writer, stopResume *iotCardSvc.StopResumeService, logger *zap.Logger) *carrierSvc.Service {
|
||||
service := carrierSvc.New(store, auditWriter)
|
||||
if stopResume != nil {
|
||||
service.SetRealnameControlEvaluator(stopResume, logger)
|
||||
}
|
||||
return service
|
||||
}
|
||||
|
||||
41
internal/domain/stopresume/scene.go
Normal file
41
internal/domain/stopresume/scene.go
Normal file
@@ -0,0 +1,41 @@
|
||||
// Package stopresume 提供自动停复机请求评估场景在 ctx 中的传递，供各自动入口与停复机服务共用（无业务依赖）。
|
||||
package stopresume
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
type sceneKey struct{}
|
||||
|
||||
// WithScene 声明本次自动停复机评估的触发场景（constants.StopResumeScene*）；未声明时按套餐步骤（周期评估）处理。
|
||||
func WithScene(ctx context.Context, scene string) context.Context {
|
||||
if scene == "" {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, sceneKey{}, scene)
|
||||
}
|
||||
|
||||
// Scene 返回 ctx 中声明的场景；未声明返回套餐步骤场景。
|
||||
func Scene(ctx context.Context) string {
|
||||
if scene, ok := ctx.Value(sceneKey{}).(string); ok && scene != "" {
|
||||
return scene
|
||||
}
|
||||
return constants.StopResumeScenePolling
|
||||
}
|
||||
|
||||
// QueueAware 由停复机服务实现：停复机出口开关开启时请求评估只写共享缓存。
|
||||
type QueueAware interface {
|
||||
CommandQueueEnabled(ctx context.Context) bool
|
||||
}
|
||||
|
||||
// RunAfterCommit 在业务事务提交后执行自动停复机请求：开关开启时同步执行（只写共享缓存，不起 goroutine）；
|
||||
// 关闭时沿用旧路径在 goroutine 中当场执行（含网关调用与重试，不阻塞业务流程）。
|
||||
func RunAfterCommit(ctx context.Context, callback any, fn func()) {
|
||||
if aware, ok := callback.(QueueAware); ok && aware.CommandQueueEnabled(ctx) {
|
||||
fn()
|
||||
return
|
||||
}
|
||||
go fn()
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package cardobservation
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -168,7 +169,7 @@ func (c *NetworkChangedConsumer) Consume(ctx context.Context, envelope outbox.De
|
||||
}
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "查询网络变化后的卡事实失败")
|
||||
}
|
||||
return c.evaluator.EvaluateAndAct(ctx, &card)
|
||||
return c.evaluator.EvaluateAndAct(stopresumedomain.WithScene(ctx, constants.StopResumeSceneNetworkObserved), &card)
|
||||
}
|
||||
|
||||
// NewRealnameChangedConsumer 创建卡实名变化事件消费者。
|
||||
@@ -217,7 +218,7 @@ func (c *RealnameChangedConsumer) Consume(ctx context.Context, envelope outbox.D
|
||||
}
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "查询实名变化后的卡事实失败")
|
||||
}
|
||||
return c.evaluator.EvaluateAndAct(ctx, &card)
|
||||
return c.evaluator.EvaluateAndAct(stopresumedomain.WithScene(ctx, constants.StopResumeSceneRealnameObserved), &card)
|
||||
}
|
||||
|
||||
// TrafficIncrementedConsumer 消费卡流量正增量并推进幂等副作用进度。
|
||||
@@ -286,7 +287,7 @@ func (c *TrafficIncrementedConsumer) Consume(ctx context.Context, envelope outbo
|
||||
if err := c.db.WithContext(ctx).Where("id = ?", event.CardID).First(&card).Error; err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "查询流量变化后的卡事实失败")
|
||||
}
|
||||
if err := c.evaluator.EvaluateAndAct(ctx, &card); err != nil {
|
||||
if err := c.evaluator.EvaluateAndAct(stopresumedomain.WithScene(ctx, constants.StopResumeSceneFlowObserved), &card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,6 +145,21 @@ type CardPollingTypeState struct {
|
||||
PendingConfirm []int `json:"pending_confirm" description:"待执行的延迟确认（秒）"`
|
||||
}
|
||||
|
||||
// CardStopResumeCommandState 卡当前停复机指令（停复机指令队列）。
|
||||
type CardStopResumeCommandState struct {
|
||||
Action string `json:"action" description:"动作 (stop:停机, start:复机)"`
|
||||
Reason string `json:"reason" description:"停机原因"`
|
||||
Scenes []string `json:"scenes" description:"触发场景"`
|
||||
Source string `json:"source" description:"请求来源"`
|
||||
Version int64 `json:"version" description:"指令版本"`
|
||||
FailCount int `json:"fail_count" description:"连续失败次数"`
|
||||
LastError string `json:"last_error" description:"最近错误（类别|信息）"`
|
||||
Pending bool `json:"pending" description:"结果未知，待确认"`
|
||||
Unknown int `json:"unknown" description:"连续结果未知次数"`
|
||||
WrittenAt *time.Time `json:"written_at" description:"最近写入时间"`
|
||||
NextAt *time.Time `json:"next_at" description:"下次可执行时间（空表示不在队列中）"`
|
||||
}
|
||||
|
||||
// CardPollingStateResponse 单卡轮询状态（只读）。
|
||||
type CardPollingStateResponse struct {
|
||||
CardID uint `json:"card_id" description:"卡ID"`
|
||||
@@ -158,6 +173,10 @@ type CardPollingStateResponse struct {
|
||||
ClaimExpireAt *time.Time `json:"claim_expire_at" description:"处理中标记到期时间"`
|
||||
PriorityMark bool `json:"priority_mark" description:"执行中收到请求检查，收尾后进入优先队列"`
|
||||
Types []CardPollingTypeState `json:"types" description:"各检查类型状态"`
|
||||
|
||||
StopResumeQueueEnabled bool `json:"stop_resume_queue_enabled" description:"停复机出口开关是否开启"`
|
||||
StopResumeCommand *CardStopResumeCommandState `json:"stop_resume_command" description:"当前停复机指令（无指令为空）"`
|
||||
LastStopResumeCallAt *time.Time `json:"last_stop_resume_call_at" description:"最近一次停复机网关调用时间（单卡最小重试间隔依据）"`
|
||||
}
|
||||
|
||||
// CarrierPollingStatsItem 按运营商账户的轮询监控。
|
||||
@@ -175,6 +194,15 @@ type CarrierPollingStatsItem struct {
|
||||
ExpiredClaims1h int64 `json:"expired_claims_1h" description:"近 1 小时处理中标记到期次数（执行崩溃或超时）"`
|
||||
OpenAlerts int64 `json:"open_alerts" description:"未处理卡级失败告警数"`
|
||||
OpenAlertsByType map[string]int64 `json:"open_alerts_by_category" description:"未处理卡级失败告警数（按错误类别）"`
|
||||
|
||||
StopResumeQueueLength int64 `json:"stop_resume_queue_length" description:"停复机指令队列长度"`
|
||||
StopResumeDue int64 `json:"stop_resume_due" description:"已到期停复机指令数"`
|
||||
StopResumeRunning int64 `json:"stop_resume_running" description:"执行中停复机指令数"`
|
||||
StopResumePending int64 `json:"stop_resume_pending" description:"结果未知待确认的停复机指令数（按队列前 500 条统计）"`
|
||||
StopResumeSuccess1h int64 `json:"stop_resume_success_1h" description:"近 1 小时停复机成功数"`
|
||||
StopResumeFailed1h int64 `json:"stop_resume_failed_1h" description:"近 1 小时停复机失败数（计入连续失败或终态）"`
|
||||
StopResumeRetry1h int64 `json:"stop_resume_retry_1h" description:"近 1 小时停复机推后重试数（限流、频率限制、最小间隔）"`
|
||||
StopResumeUnknown1h int64 `json:"stop_resume_unknown_1h" description:"近 1 小时停复机结果未知数"`
|
||||
}
|
||||
|
||||
// CarrierPollingStatsResponse 按运营商账户的轮询监控。
|
||||
|
||||
@@ -41,6 +41,12 @@ type CarrierParams struct {
|
||||
CardMaxInFlight int
|
||||
PriorityRatio float64
|
||||
Configured bool // 是否有该账户的配置行
|
||||
|
||||
// 停复机指令（add-stop-resume-command-queue）。
|
||||
SRInflightLimit int // 账户停复机执行上限
|
||||
SRMaxInFlight int // 每进程停复机消费者 max_in_flight
|
||||
SRMinInterval time.Duration // 单卡最小重试间隔（停机与复机合计）
|
||||
SRConfirmDelay time.Duration // 结果未知后的确认延迟
|
||||
}
|
||||
|
||||
// carrierEntry 是分发器缓存的账户信息。
|
||||
@@ -110,11 +116,16 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
d.logger.Error("分发器 panic，已恢复", zap.Any("panic", r))
|
||||
}
|
||||
}()
|
||||
if !d.Ready(ctx) {
|
||||
// 停复机指令与设备判断的分发只看停复机出口开关，不受 scheduler_mode 限制；卡任务只在新调度就绪时领取。
|
||||
stopResumeOn := d.settings.Current(ctx).StopResumeQueueEnabled
|
||||
cardsReady := d.Ready(ctx)
|
||||
if !stopResumeOn && !cardsReady {
|
||||
return
|
||||
}
|
||||
d.store.redis.HSet(ctx, constants.RedisCarrierSchedulerHeartbeatKey, d.instanceID, time.Now().UnixMilli())
|
||||
d.drainLegacyHints(ctx)
|
||||
if cardsReady {
|
||||
d.drainLegacyHints(ctx)
|
||||
}
|
||||
d.refreshCarriers(ctx)
|
||||
d.mu.Lock()
|
||||
carriers := append([]carrierEntry(nil), d.carriers...)
|
||||
@@ -123,6 +134,9 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
if lockTTL <= 0 {
|
||||
lockTTL = 900 * time.Millisecond
|
||||
}
|
||||
if stopResumeOn {
|
||||
d.dispatchDeviceEvals(ctx, lockTTL)
|
||||
}
|
||||
for _, carrier := range carriers {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
@@ -131,10 +145,21 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
d.dispatchCarrier(ctx, carrier)
|
||||
// 停复机优先：先领取停复机指令占用本秒名额，再领取卡任务。
|
||||
if stopResumeOn {
|
||||
d.dispatchStopResume(ctx, carrier.id)
|
||||
}
|
||||
if cardsReady {
|
||||
d.dispatchCarrier(ctx, carrier)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StopResumeQueueEnabled 返回停复机出口开关当前值。
|
||||
func (d *Dispatcher) StopResumeQueueEnabled(ctx context.Context) bool {
|
||||
return d.settings.Current(ctx).StopResumeQueueEnabled
|
||||
}
|
||||
|
||||
func (d *Dispatcher) dispatchCarrier(ctx context.Context, carrier carrierEntry) {
|
||||
params := d.ParamsFor(carrier.id)
|
||||
limit := params.InflightLimit
|
||||
@@ -265,7 +290,10 @@ func (d *Dispatcher) ParamsFor(carrierID uint) CarrierParams {
|
||||
|
||||
// MergeParams 合并账户配置行与全局默认。
|
||||
func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.CarrierSchedulerConfig) CarrierParams {
|
||||
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured}
|
||||
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured,
|
||||
SRInflightLimit: constants.DefaultStopResumeInflightLimit, SRMaxInFlight: constants.DefaultStopResumeMaxInFlight,
|
||||
SRMinInterval: constants.DefaultStopResumeMinIntervalSeconds * time.Second,
|
||||
SRConfirmDelay: constants.DefaultStopResumeConfirmDelaySeconds * time.Second}
|
||||
if params.InflightLimit <= 0 {
|
||||
params.InflightLimit = 20
|
||||
}
|
||||
@@ -284,6 +312,18 @@ func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.Carr
|
||||
if row.PriorityMaxRatio != nil && *row.PriorityMaxRatio > 0 {
|
||||
params.PriorityRatio = *row.PriorityMaxRatio
|
||||
}
|
||||
if row.SRInflightLimit != nil && *row.SRInflightLimit > 0 {
|
||||
params.SRInflightLimit = *row.SRInflightLimit
|
||||
}
|
||||
if row.StopResumeMaxInFlight != nil && *row.StopResumeMaxInFlight > 0 {
|
||||
params.SRMaxInFlight = *row.StopResumeMaxInFlight
|
||||
}
|
||||
if row.StopResumeMinInterval != nil && *row.StopResumeMinInterval >= 0 {
|
||||
params.SRMinInterval = time.Duration(*row.StopResumeMinInterval) * time.Second
|
||||
}
|
||||
if row.StopResumeConfirmDelay != nil && *row.StopResumeConfirmDelay > 0 {
|
||||
params.SRConfirmDelay = time.Duration(*row.StopResumeConfirmDelay) * time.Second
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,14 @@ func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logg
|
||||
DefaultValue: "600", Description: "账户级告警统计窗口（秒）", Min: &minOne, Max: &maxSeconds},
|
||||
{Key: constants.SystemConfigPollingAccountAlertThreshold, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "50", Description: "账户级告警阈值：窗口内同账户新增失败告警数", Min: &minOne, Max: &maxCount},
|
||||
{Key: constants.SystemConfigStopResumeQueueEnabled, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeBool,
|
||||
DefaultValue: "false", Description: "自动停复机经停复机指令队列执行（全局开关，关闭即各入口当场执行）"},
|
||||
{Key: constants.SystemConfigStopResumeUnknownAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "3", Description: "停复机连续结果未知告警阈值", Min: &minOne, Max: &maxCount},
|
||||
{Key: constants.SystemConfigStopResumeBackoff, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
|
||||
DefaultValue: `[600,900,1800,3600]`, Description: "停复机失败退避序列（秒，递增）", Validator: validateSequence},
|
||||
{Key: constants.SystemConfigStopResumeAlertFailCount, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "5", Description: "停复机连续失败告警阈值", Min: &minOne, Max: &maxCount},
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if _, exists := registry.Get(definition.Key); exists {
|
||||
@@ -107,6 +115,23 @@ func validateStepTimeout(raw string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateSequence 校验非空递增的正整数秒序列（停复机退避）。
|
||||
func validateSequence(raw string) error {
|
||||
var values []int
|
||||
if err := json.Unmarshal([]byte(raw), &values); err != nil || len(values) == 0 {
|
||||
return stderrors.New("退避序列必须是非空的秒数组")
|
||||
}
|
||||
for index, value := range values {
|
||||
if value <= 0 {
|
||||
return stderrors.New("退避间隔必须为正数")
|
||||
}
|
||||
if index > 0 && value <= values[index-1] {
|
||||
return stderrors.New("退避序列必须递增")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBackoff(raw string) error {
|
||||
var backoff map[string][]int
|
||||
if err := json.Unmarshal([]byte(raw), &backoff); err != nil {
|
||||
@@ -171,6 +196,28 @@ type Settings struct {
|
||||
ImmediateAlert map[string]bool
|
||||
AccountAlertWindow time.Duration
|
||||
AccountAlertLimit int
|
||||
|
||||
// 停复机指令队列（add-stop-resume-command-queue）。
|
||||
StopResumeQueueEnabled bool
|
||||
SRUnknownAlert int
|
||||
SRBackoff []time.Duration
|
||||
SRAlertFailCount int
|
||||
}
|
||||
|
||||
// SRBackoffFor 返回停复机第 n 次连续失败（n 从 1 起）的退避间隔；超过序列长度取最后一项。
|
||||
func (s Settings) SRBackoffFor(failCount int) time.Duration {
|
||||
sequence := s.SRBackoff
|
||||
if len(sequence) == 0 {
|
||||
sequence = []time.Duration{10 * time.Minute}
|
||||
}
|
||||
index := failCount - 1
|
||||
if index < 0 {
|
||||
index = 0
|
||||
}
|
||||
if index >= len(sequence) {
|
||||
index = len(sequence) - 1
|
||||
}
|
||||
return sequence[index]
|
||||
}
|
||||
|
||||
// CarrierMode 判断当前是否为按运营商账户调度。
|
||||
@@ -243,6 +290,9 @@ func DefaultSettings() Settings {
|
||||
AlertFailCount: map[string]int{"default": 5}, AlertFailDuration: 6 * time.Hour,
|
||||
ImmediateAlert: map[string]bool{constants.GatewayClassAccountConfig: true},
|
||||
AccountAlertWindow: 10 * time.Minute, AccountAlertLimit: 50,
|
||||
SRUnknownAlert: 3,
|
||||
SRBackoff: []time.Duration{10 * time.Minute, 15 * time.Minute, 30 * time.Minute, time.Hour},
|
||||
SRAlertFailCount: 5,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +414,8 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
{constants.SystemConfigPollingAlertFailDuration, func(v int) { s.AlertFailDuration = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingAccountAlertWindow, func(v int) { s.AccountAlertWindow = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingAccountAlertThreshold, func(v int) { s.AccountAlertLimit = v }},
|
||||
{constants.SystemConfigStopResumeUnknownAlert, func(v int) { s.SRUnknownAlert = v }},
|
||||
{constants.SystemConfigStopResumeAlertFailCount, func(v int) { s.SRAlertFailCount = v }},
|
||||
}
|
||||
for _, item := range ints {
|
||||
raw, err := get(item.key)
|
||||
@@ -376,6 +428,21 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
}
|
||||
item.set(value)
|
||||
}
|
||||
if raw, err = get(constants.SystemConfigStopResumeQueueEnabled); err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.StopResumeQueueEnabled = raw == "true"
|
||||
if raw, err = get(constants.SystemConfigStopResumeBackoff); err != nil {
|
||||
return s, err
|
||||
}
|
||||
var srBackoff []int
|
||||
if err := json.Unmarshal([]byte(raw), &srBackoff); err != nil || validateSequence(raw) != nil {
|
||||
return s, stderrors.New("停复机退避序列配置非法")
|
||||
}
|
||||
s.SRBackoff = nil
|
||||
for _, value := range srBackoff {
|
||||
s.SRBackoff = append(s.SRBackoff, time.Duration(value)*time.Second)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
||||
120
internal/polling/carrierqueue/sr_dispatch.go
Normal file
120
internal/polling/carrierqueue/sr_dispatch.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StopResumeMessage 是停复机指令 NSQ 消息体。
|
||||
type StopResumeMessage struct {
|
||||
CardID uint `json:"card_id"`
|
||||
CarrierID uint `json:"carrier_id"`
|
||||
Token string `json:"token"`
|
||||
Version int64 `json:"version"`
|
||||
Action string `json:"action"`
|
||||
Pending bool `json:"pending"`
|
||||
DispatchedAt int64 `json:"dispatched_at"`
|
||||
}
|
||||
|
||||
// DeviceEvalMessage 是设备判断 NSQ 消息体。
|
||||
type DeviceEvalMessage struct {
|
||||
DeviceID uint `json:"device_id"`
|
||||
Token string `json:"token"`
|
||||
Version string `json:"version"`
|
||||
Scenes []string `json:"scenes"`
|
||||
DispatchedAt int64 `json:"dispatched_at"`
|
||||
}
|
||||
|
||||
// deviceEvalLimit 是全局同时执行的设备判断上限：判断只读数据库与写指令，毫秒级。
|
||||
const deviceEvalLimit = 50
|
||||
|
||||
// stopResumeLease 是停复机指令与设备判断的租约：单次执行只调一次网关，远小于该值。
|
||||
func (d *Dispatcher) stopResumeLease() time.Duration {
|
||||
return 2 * time.Minute
|
||||
}
|
||||
|
||||
// dispatchStopResume 在卡任务之前领取账户停复机指令：领取数 = min(停复机所在限流组本秒剩余, 执行上限 − 执行中)。
|
||||
func (d *Dispatcher) dispatchStopResume(ctx context.Context, carrierID uint) {
|
||||
params := d.ParamsFor(carrierID)
|
||||
capacity := params.SRInflightLimit
|
||||
processes := d.cfg.SubscriberProcesses
|
||||
if processes <= 0 {
|
||||
processes = 1
|
||||
}
|
||||
if consume := params.SRMaxInFlight * processes; consume > 0 && capacity > consume {
|
||||
capacity = consume
|
||||
}
|
||||
if d.budgets != nil {
|
||||
ops := []string{constants.GatewayOpStop, constants.GatewayOpStart}
|
||||
if groups, _, limited := d.budgets.Budget(ctx, carrierID, ops); limited {
|
||||
for _, remaining := range groups {
|
||||
if remaining < capacity {
|
||||
capacity = remaining
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if capacity <= 0 {
|
||||
return
|
||||
}
|
||||
claimed, expired, err := d.store.ClaimStopResume(ctx, SRClaimInput{
|
||||
CarrierID: carrierID, Lease: d.stopResumeLease(), Capacity: capacity, MinInterval: params.SRMinInterval,
|
||||
TokenPrefix: d.instanceID + ":sr:" + strconv.FormatInt(time.Now().UnixNano(), 36),
|
||||
})
|
||||
if err != nil {
|
||||
d.logger.Warn("运营商账户停复机指令领取失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if expired > 0 {
|
||||
d.logger.Warn("停复机指令处理中标记到期（执行崩溃或超时）", zap.Uint("carrier_id", carrierID), zap.Int64("count", expired),
|
||||
zap.String("metric", "stop_resume.claim_expired"))
|
||||
}
|
||||
topic := constants.NSQStopResumeTopic(carrierID)
|
||||
for _, item := range claimed {
|
||||
body, _ := json.Marshal(StopResumeMessage{CardID: item.CardID, CarrierID: carrierID, Token: item.Token, Version: item.Version,
|
||||
Action: item.Action, Pending: item.Pending, DispatchedAt: time.Now().UnixMilli()})
|
||||
if err := d.publisher.Publish(topic, body); err != nil {
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
if releaseErr := d.store.ReleaseStopResume(releaseCtx, item.CardID, carrierID, item.Token); releaseErr != nil {
|
||||
d.logger.Warn("发布失败后释放停复机指令失败", zap.Uint("card_id", item.CardID), zap.Error(releaseErr))
|
||||
}
|
||||
cancel()
|
||||
d.logger.Warn("发布停复机指令失败，已释放", zap.Uint("card_id", item.CardID), zap.String("topic", topic), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchDeviceEvals 领取到期设备判断并发布；全局分发锁保证每轮只有一个实例领取。
|
||||
func (d *Dispatcher) dispatchDeviceEvals(ctx context.Context, lockTTL time.Duration) {
|
||||
ok, err := d.store.redis.SetNX(ctx, constants.RedisDeviceEvalDispatchLockKey, d.instanceID, lockTTL).Result()
|
||||
if err != nil || !ok {
|
||||
return
|
||||
}
|
||||
claimed, expired, err := d.store.ClaimDeviceEvals(ctx, d.stopResumeLease(), deviceEvalLimit,
|
||||
d.instanceID+":dev:"+strconv.FormatInt(time.Now().UnixNano(), 36))
|
||||
if err != nil {
|
||||
d.logger.Warn("设备判断领取失败", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if expired > 0 {
|
||||
d.logger.Warn("设备判断处理中标记到期（执行崩溃或超时）", zap.Int64("count", expired), zap.String("metric", "stop_resume.device_eval_expired"))
|
||||
}
|
||||
for _, item := range claimed {
|
||||
body, _ := json.Marshal(DeviceEvalMessage{DeviceID: item.DeviceID, Token: item.Token, Version: item.Version, Scenes: item.Scenes,
|
||||
DispatchedAt: time.Now().UnixMilli()})
|
||||
if err := d.publisher.Publish(constants.NSQDeviceEvalTopic, body); err != nil {
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
if releaseErr := d.store.ReleaseDeviceEval(releaseCtx, item.DeviceID, item.Token); releaseErr != nil {
|
||||
d.logger.Warn("发布失败后释放设备判断失败", zap.Uint("device_id", item.DeviceID), zap.Error(releaseErr))
|
||||
}
|
||||
cancel()
|
||||
d.logger.Warn("发布设备判断失败，已释放", zap.Uint("device_id", item.DeviceID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
29
internal/polling/carrierqueue/sr_queue.go
Normal file
29
internal/polling/carrierqueue/sr_queue.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package carrierqueue
|
||||
|
||||
import "context"
|
||||
|
||||
// CommandQueue 是停复机指令队列写入端：开关读取受控系统配置，写入走共享缓存 Lua。
|
||||
type CommandQueue struct {
|
||||
store *Store
|
||||
settings *SettingsProvider
|
||||
}
|
||||
|
||||
// NewCommandQueue 创建停复机指令队列写入端。
|
||||
func NewCommandQueue(store *Store, settings *SettingsProvider) *CommandQueue {
|
||||
return &CommandQueue{store: store, settings: settings}
|
||||
}
|
||||
|
||||
// Enabled 返回停复机出口开关 polling.stop_resume.queue_enabled。
|
||||
func (q *CommandQueue) Enabled(ctx context.Context) bool {
|
||||
return q != nil && q.store != nil && q.settings.Current(ctx).StopResumeQueueEnabled
|
||||
}
|
||||
|
||||
// WriteCommand 写入或覆盖卡的停复机指令。
|
||||
func (q *CommandQueue) WriteCommand(ctx context.Context, cmd StopResumeCommand) (int64, error) {
|
||||
return q.store.WriteCommand(ctx, cmd)
|
||||
}
|
||||
|
||||
// RequestDeviceEval 请求一次设备维度判断。
|
||||
func (q *CommandQueue) RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error {
|
||||
return q.store.RequestDeviceEval(ctx, deviceID, scenes)
|
||||
}
|
||||
225
internal/polling/carrierqueue/sr_scripts.go
Normal file
225
internal/polling/carrierqueue/sr_scripts.go
Normal file
@@ -0,0 +1,225 @@
|
||||
package carrierqueue
|
||||
|
||||
import "github.com/redis/go-redis/v9"
|
||||
|
||||
// 停复机指令 Hash 字段（见 constants.RedisCardStopResumeCmdKey）。
|
||||
const (
|
||||
srFieldAction = "action" // stop / start
|
||||
srFieldReason = "reason" // 停机原因（复机为空）
|
||||
srFieldScenes = "scenes" // 触发场景（逗号分隔）
|
||||
srFieldSource = "source" // 请求来源
|
||||
srFieldCarrier = "carrier" // 所属运营商账户
|
||||
srFieldVersion = "ver" // 指令版本，每次写入 HINCRBY
|
||||
srFieldFail = "fail" // 连续失败次数
|
||||
srFieldFirst = "ffail" // 首次失败（毫秒）
|
||||
srFieldError = "err" // 最近错误（类别|信息）
|
||||
srFieldPending = "pending" // 1 表示结果未知、待确认
|
||||
srFieldUnknown = "unknown" // 连续结果未知次数
|
||||
srFieldAt = "at" // 最近写入时间（毫秒）
|
||||
)
|
||||
|
||||
// srWriteLua 写入或覆盖停复机指令：方向相反时清零失败与待确认，递增版本，放入账户停复机队列。
|
||||
// 指令执行中（存在处理中标记）时只更新指令内容与版本，由收尾按版本保留；待确认中且方向不变时保持确认时间。
|
||||
//
|
||||
// KEYS[1] 指令 Hash；KEYS[2] 账户停复机队列；KEYS[3] 处理中标记。
|
||||
// ARGV：action、reason、scenes、source、carrier、now_ms、card_id。返回新版本号。
|
||||
const srWriteLua = `
|
||||
local prev = redis.call('HGET', KEYS[1], 'action')
|
||||
if prev and prev ~= ARGV[1] then
|
||||
redis.call('HDEL', KEYS[1], 'fail', 'ffail', 'err', 'pending', 'unknown')
|
||||
end
|
||||
redis.call('HSET', KEYS[1], 'action', ARGV[1], 'reason', ARGV[2], 'scenes', ARGV[3], 'source', ARGV[4], 'carrier', ARGV[5], 'at', ARGV[6])
|
||||
local ver = redis.call('HINCRBY', KEYS[1], 'ver', 1)
|
||||
if redis.call('EXISTS', KEYS[3]) == 1 then return ver end
|
||||
if redis.call('HGET', KEYS[1], 'pending') == '1' then
|
||||
redis.call('ZADD', KEYS[2], 'NX', tonumber(ARGV[6]), ARGV[7])
|
||||
else
|
||||
redis.call('ZADD', KEYS[2], 'LT', tonumber(ARGV[6]), ARGV[7])
|
||||
end
|
||||
return ver
|
||||
`
|
||||
|
||||
// srClaimLua 领取账户到期停复机指令：先清理过期执行中，按容量领取；未到单卡最小重试间隔的推后到可调用时间。
|
||||
// 待确认的指令只查状态，不受最小间隔约束。指令已不存在的成员直接移出队列。
|
||||
//
|
||||
// KEYS[1] 账户停复机队列；KEYS[2] 账户停复机执行中计数。
|
||||
// ARGV：now_ms、lease_ms、容量（执行上限与剩余 QPS 的较小值）、token 前缀、单卡最小间隔 ms。
|
||||
// 返回 {过期数, card_id, token, ver, action, pending, ...}。
|
||||
const srClaimLua = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local lease = tonumber(ARGV[2])
|
||||
local limit = tonumber(ARGV[3])
|
||||
local prefix = ARGV[4]
|
||||
local minInterval = tonumber(ARGV[5])
|
||||
local expired = redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now)
|
||||
local result = {expired}
|
||||
local capacity = limit - redis.call('ZCARD', KEYS[2])
|
||||
if capacity <= 0 then return result end
|
||||
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', now, 'LIMIT', 0, capacity * 2 + 10)
|
||||
local taken = 0
|
||||
for _, id in ipairs(ids) do
|
||||
if taken >= capacity then break end
|
||||
local cmdKey = 'polling:card:' .. id .. ':stop_resume_cmd'
|
||||
local fields = redis.call('HMGET', cmdKey, 'action', 'ver', 'pending')
|
||||
if not fields[1] then
|
||||
redis.call('ZREM', KEYS[1], id)
|
||||
else
|
||||
local pending = fields[3] == '1'
|
||||
local blockedUntil = 0
|
||||
if not pending then
|
||||
local last = tonumber(redis.call('GET', 'polling:card:' .. id .. ':sr_last_call') or '0')
|
||||
if last > 0 and last + minInterval > now then blockedUntil = last + minInterval end
|
||||
end
|
||||
if blockedUntil > 0 then
|
||||
redis.call('ZADD', KEYS[1], 'XX', blockedUntil, id)
|
||||
else
|
||||
local claimKey = 'polling:card:' .. id .. ':sr_claim'
|
||||
local token = prefix .. ':' .. id
|
||||
if redis.call('SET', claimKey, token, 'NX', 'PX', lease) then
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + lease, id)
|
||||
redis.call('ZADD', KEYS[2], now + lease, token)
|
||||
taken = taken + 1
|
||||
table.insert(result, id)
|
||||
table.insert(result, token)
|
||||
table.insert(result, fields[2] or '0')
|
||||
table.insert(result, fields[1])
|
||||
table.insert(result, pending and '1' or '0')
|
||||
else
|
||||
local ttl = redis.call('PTTL', claimKey)
|
||||
if ttl < 0 then ttl = lease end
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + ttl, id)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return result
|
||||
`
|
||||
|
||||
// srFinishLua 停复机指令收尾：token 不一致返回 0 不做任何修改；版本已变大（执行期间写入了新指令）时保留新指令、立即可领取（返回 2）；
|
||||
// 否则 done=1 删除指令并移出队列，done=0 写入字段并按 next_ms 重新排队（返回 1）。
|
||||
//
|
||||
// KEYS[1] 指令 Hash；KEYS[2] 处理中标记；KEYS[3] 执行权；KEYS[4] 执行中计数；KEYS[5] 账户停复机队列。
|
||||
// ARGV：token、card_id、领取时版本、done(1/0)、next_ms、now_ms、set 字段对数 n、字段1、值1、…、删除字段…。
|
||||
const srFinishLua = `
|
||||
if redis.call('GET', KEYS[2]) ~= ARGV[1] then return 0 end
|
||||
redis.call('DEL', KEYS[2])
|
||||
redis.call('DEL', KEYS[3])
|
||||
redis.call('ZREM', KEYS[4], ARGV[1])
|
||||
local cardId = ARGV[2]
|
||||
if (redis.call('HGET', KEYS[1], 'ver') or '0') ~= ARGV[3] then
|
||||
if redis.call('EXISTS', KEYS[1]) == 1 then
|
||||
redis.call('ZADD', KEYS[5], tonumber(ARGV[6]), cardId)
|
||||
else
|
||||
redis.call('ZREM', KEYS[5], cardId)
|
||||
end
|
||||
return 2
|
||||
end
|
||||
if ARGV[4] == '1' then
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('ZREM', KEYS[5], cardId)
|
||||
return 1
|
||||
end
|
||||
local n = tonumber(ARGV[7])
|
||||
local idx = 8
|
||||
for i = 1, n do
|
||||
redis.call('HSET', KEYS[1], ARGV[idx], ARGV[idx + 1])
|
||||
idx = idx + 2
|
||||
end
|
||||
while idx <= #ARGV do
|
||||
redis.call('HDEL', KEYS[1], ARGV[idx])
|
||||
idx = idx + 1
|
||||
end
|
||||
redis.call('ZADD', KEYS[5], tonumber(ARGV[5]), cardId)
|
||||
return 1
|
||||
`
|
||||
|
||||
// srReleaseLua 发布失败：按 token 删除处理中标记、移出执行中计数，score 恢复为当前时间。
|
||||
// KEYS[1] 处理中标记；KEYS[2] 执行中计数；KEYS[3] 队列。ARGV：token、member、now_ms。
|
||||
const srReleaseLua = `
|
||||
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('ZREM', KEYS[2], ARGV[1])
|
||||
redis.call('ZADD', KEYS[3], 'XX', tonumber(ARGV[3]), ARGV[2])
|
||||
return 1
|
||||
`
|
||||
|
||||
// deviceEvalRequestLua 请求设备判断：递增版本、合并场景、设备进入判断队列（同一设备一个成员，ZADD LT 不推后已有更早时间）。
|
||||
// KEYS[1] 版本；KEYS[2] 场景集合；KEYS[3] 设备判断队列；KEYS[4] 处理中标记。ARGV：device_id、now_ms、场景…。
|
||||
const deviceEvalRequestLua = `
|
||||
redis.call('INCR', KEYS[1])
|
||||
for i = 3, #ARGV do redis.call('SADD', KEYS[2], ARGV[i]) end
|
||||
if redis.call('EXISTS', KEYS[4]) == 1 then return 1 end
|
||||
redis.call('ZADD', KEYS[3], 'LT', tonumber(ARGV[2]), ARGV[1])
|
||||
return 1
|
||||
`
|
||||
|
||||
// deviceEvalClaimLua 领取到期设备判断：写处理中标记并记下版本与场景快照。
|
||||
// KEYS[1] 设备判断队列；KEYS[2] 执行中计数。ARGV：now_ms、lease_ms、上限、token 前缀。
|
||||
// 返回 {过期数, device_id, token, ver, scenes, ...}。
|
||||
const deviceEvalClaimLua = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local lease = tonumber(ARGV[2])
|
||||
local limit = tonumber(ARGV[3])
|
||||
local expired = redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now)
|
||||
local result = {expired}
|
||||
local capacity = limit - redis.call('ZCARD', KEYS[2])
|
||||
if capacity <= 0 then return result end
|
||||
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', now, 'LIMIT', 0, capacity * 2 + 10)
|
||||
local taken = 0
|
||||
for _, id in ipairs(ids) do
|
||||
if taken >= capacity then break end
|
||||
local claimKey = 'polling:device:' .. id .. ':eval_claim'
|
||||
local token = ARGV[4] .. ':' .. id
|
||||
if redis.call('SET', claimKey, token, 'NX', 'PX', lease) then
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + lease, id)
|
||||
redis.call('ZADD', KEYS[2], now + lease, token)
|
||||
taken = taken + 1
|
||||
local ver = redis.call('GET', 'polling:device:' .. id .. ':eval_ver') or '0'
|
||||
local scenes = redis.call('SMEMBERS', 'polling:device:' .. id .. ':eval_scenes')
|
||||
table.insert(result, id)
|
||||
table.insert(result, token)
|
||||
table.insert(result, ver)
|
||||
table.insert(result, table.concat(scenes, ','))
|
||||
else
|
||||
local ttl = redis.call('PTTL', claimKey)
|
||||
if ttl < 0 then ttl = lease end
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + ttl, id)
|
||||
end
|
||||
end
|
||||
return result
|
||||
`
|
||||
|
||||
// deviceEvalFinishLua 设备判断收尾：只移除本次已处理的场景；版本变大时立即再判断（返回 2）；
|
||||
// defer_ms > 0 表示本次未判断（旧路径持设备锁），推后再判断，场景不移除。
|
||||
// KEYS[1] 处理中标记；KEYS[2] 执行权；KEYS[3] 执行中计数；KEYS[4] 版本；KEYS[5] 场景集合；KEYS[6] 队列。
|
||||
// ARGV：token、device_id、领取时版本、now_ms、defer_ms、已处理场景…。
|
||||
const deviceEvalFinishLua = `
|
||||
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('DEL', KEYS[2])
|
||||
redis.call('ZREM', KEYS[3], ARGV[1])
|
||||
local now = tonumber(ARGV[4])
|
||||
local deferMs = tonumber(ARGV[5])
|
||||
if deferMs > 0 then
|
||||
redis.call('ZADD', KEYS[6], now + deferMs, ARGV[2])
|
||||
return 3
|
||||
end
|
||||
for i = 6, #ARGV do redis.call('SREM', KEYS[5], ARGV[i]) end
|
||||
if (redis.call('GET', KEYS[4]) or '0') ~= ARGV[3] then
|
||||
redis.call('ZADD', KEYS[6], now, ARGV[2])
|
||||
return 2
|
||||
end
|
||||
redis.call('ZREM', KEYS[6], ARGV[2])
|
||||
redis.call('DEL', KEYS[5])
|
||||
return 1
|
||||
`
|
||||
|
||||
var (
|
||||
srWriteScript = redis.NewScript(srWriteLua)
|
||||
srClaimScript = redis.NewScript(srClaimLua)
|
||||
srFinishScript = redis.NewScript(srFinishLua)
|
||||
srReleaseScript = redis.NewScript(srReleaseLua)
|
||||
deviceRequestScript = redis.NewScript(deviceEvalRequestLua)
|
||||
deviceClaimScript = redis.NewScript(deviceEvalClaimLua)
|
||||
deviceFinishScript = redis.NewScript(deviceEvalFinishLua)
|
||||
)
|
||||
348
internal/polling/carrierqueue/sr_store.go
Normal file
348
internal/polling/carrierqueue/sr_store.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StopResumeCommand 是写入停复机队列的一条指令。
|
||||
type StopResumeCommand struct {
|
||||
CardID uint
|
||||
CarrierID uint
|
||||
Action string // constants.StopResumeActionStop / StopResumeActionStart
|
||||
Reason string // 停机原因
|
||||
Scenes []string // 触发场景
|
||||
Source string // 请求来源（日志与排查）
|
||||
}
|
||||
|
||||
// CommandState 是卡当前停复机指令的解析结果。
|
||||
type CommandState struct {
|
||||
Exists bool
|
||||
Action string
|
||||
Reason string
|
||||
Scenes []string
|
||||
Source string
|
||||
CarrierID uint
|
||||
Version int64
|
||||
FailCount int
|
||||
FirstFailed time.Time
|
||||
LastError string
|
||||
Pending bool
|
||||
Unknown int
|
||||
WrittenAt time.Time
|
||||
NextAt time.Time // 队列中的可执行时间（零值表示不在队列中）
|
||||
LastCallAt time.Time
|
||||
}
|
||||
|
||||
// WriteCommand 写入或覆盖卡的停复机指令，返回新版本号。
|
||||
func (s *Store) WriteCommand(ctx context.Context, cmd StopResumeCommand) (int64, error) {
|
||||
if cmd.CardID == 0 || cmd.CarrierID == 0 {
|
||||
return 0, stderrors.New("停复机指令缺少卡或运营商账户")
|
||||
}
|
||||
return srWriteScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCardStopResumeCmdKey(cmd.CardID), constants.RedisCarrierStopResumeQueueKey(cmd.CarrierID),
|
||||
constants.RedisCardStopResumeClaimKey(cmd.CardID)},
|
||||
cmd.Action, cmd.Reason, constants.JoinStopResumeScenes(cmd.Scenes), cmd.Source, cmd.CarrierID, time.Now().UnixMilli(), cmd.CardID,
|
||||
).Int64()
|
||||
}
|
||||
|
||||
// SRClaimInput 是一次账户停复机领取的参数。
|
||||
type SRClaimInput struct {
|
||||
CarrierID uint
|
||||
Lease time.Duration
|
||||
Capacity int // 执行上限与剩余 QPS 的较小值
|
||||
TokenPrefix string
|
||||
MinInterval time.Duration
|
||||
}
|
||||
|
||||
// SRClaimed 是一条被领取的停复机指令。
|
||||
type SRClaimed struct {
|
||||
CardID uint
|
||||
Token string
|
||||
Version int64
|
||||
Action string
|
||||
Pending bool
|
||||
}
|
||||
|
||||
// ClaimStopResume 领取账户到期停复机指令，返回领取结果与本轮清理的过期执行数。
|
||||
func (s *Store) ClaimStopResume(ctx context.Context, input SRClaimInput) ([]SRClaimed, int64, error) {
|
||||
raw, err := srClaimScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCarrierStopResumeQueueKey(input.CarrierID), constants.RedisCarrierStopResumeRunningKey(input.CarrierID)},
|
||||
time.Now().UnixMilli(), input.Lease.Milliseconds(), input.Capacity, input.TokenPrefix, input.MinInterval.Milliseconds(),
|
||||
).Slice()
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil, 0, err
|
||||
}
|
||||
expired, _ := raw[0].(int64)
|
||||
claimed := make([]SRClaimed, 0, (len(raw)-1)/5)
|
||||
for index := 1; index+4 < len(raw); index += 5 {
|
||||
cardID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64)
|
||||
version, _ := strconv.ParseInt(fmt.Sprint(raw[index+2]), 10, 64)
|
||||
claimed = append(claimed, SRClaimed{CardID: uint(cardID), Token: fmt.Sprint(raw[index+1]), Version: version,
|
||||
Action: fmt.Sprint(raw[index+3]), Pending: fmt.Sprint(raw[index+4]) == "1"})
|
||||
}
|
||||
return claimed, expired, nil
|
||||
}
|
||||
|
||||
// AcquireStopResumeExec 抢停复机指令执行权：1 抢到，0 处理中标记已不是该 token，-1 已有执行者。
|
||||
func (s *Store) AcquireStopResumeExec(ctx context.Context, cardID uint, token, execValue string, ttl time.Duration) (int64, error) {
|
||||
return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisCardStopResumeClaimKey(cardID), constants.RedisCardStopResumeExecKey(cardID)},
|
||||
token, execValue, ttl.Milliseconds()).Int64()
|
||||
}
|
||||
|
||||
// SRFinish 是一次停复机指令收尾。
|
||||
type SRFinish struct {
|
||||
CardID uint
|
||||
CarrierID uint
|
||||
Token string
|
||||
Version int64
|
||||
Done bool // true 结束并删除指令
|
||||
NextAt time.Time
|
||||
Set map[string]string
|
||||
Del []string
|
||||
}
|
||||
|
||||
// FinishStopResume 停复机指令收尾：返回 0 token 不一致、1 正常、2 版本已变大保留新指令。
|
||||
func (s *Store) FinishStopResume(ctx context.Context, finish SRFinish) (int64, error) {
|
||||
done := "0"
|
||||
if finish.Done {
|
||||
done = "1"
|
||||
}
|
||||
now := time.Now()
|
||||
next := finish.NextAt
|
||||
if next.IsZero() || next.Before(now) {
|
||||
next = now
|
||||
}
|
||||
args := []any{finish.Token, finish.CardID, finish.Version, done, next.UnixMilli(), now.UnixMilli(), len(finish.Set)}
|
||||
for field, value := range finish.Set {
|
||||
args = append(args, field, value)
|
||||
}
|
||||
for _, field := range finish.Del {
|
||||
args = append(args, field)
|
||||
}
|
||||
return srFinishScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCardStopResumeCmdKey(finish.CardID), constants.RedisCardStopResumeClaimKey(finish.CardID),
|
||||
constants.RedisCardStopResumeExecKey(finish.CardID), constants.RedisCarrierStopResumeRunningKey(finish.CarrierID),
|
||||
constants.RedisCarrierStopResumeQueueKey(finish.CarrierID)},
|
||||
args...).Int64()
|
||||
}
|
||||
|
||||
// ReleaseStopResume 发布失败时释放停复机指令处理中标记并恢复为立即可领取。
|
||||
func (s *Store) ReleaseStopResume(ctx context.Context, cardID, carrierID uint, token string) error {
|
||||
return srReleaseScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCardStopResumeClaimKey(cardID), constants.RedisCarrierStopResumeRunningKey(carrierID),
|
||||
constants.RedisCarrierStopResumeQueueKey(carrierID)},
|
||||
token, cardID, time.Now().UnixMilli()).Err()
|
||||
}
|
||||
|
||||
// MarkStopResumeCall 记录本卡一次停复机网关调用（停机与复机合计），供单卡最小重试间隔判断。
|
||||
func (s *Store) MarkStopResumeCall(ctx context.Context, cardID uint, at time.Time, keep time.Duration) error {
|
||||
if keep < time.Hour {
|
||||
keep = time.Hour
|
||||
}
|
||||
return s.redis.Set(ctx, constants.RedisCardStopResumeLastCallKey(cardID), at.UnixMilli(), keep).Err()
|
||||
}
|
||||
|
||||
// LastStopResumeCall 返回本卡最近一次停复机网关调用时间；没有记录返回零值。
|
||||
func (s *Store) LastStopResumeCall(ctx context.Context, cardID uint) (time.Time, error) {
|
||||
ms, err := s.redis.Get(ctx, constants.RedisCardStopResumeLastCallKey(cardID)).Int64()
|
||||
if stderrors.Is(err, redis.Nil) {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.UnixMilli(ms), nil
|
||||
}
|
||||
|
||||
// ReadCommand 读取卡当前停复机指令与队列位置。
|
||||
func (s *Store) ReadCommand(ctx context.Context, cardID uint) (CommandState, error) {
|
||||
raw, err := s.redis.HGetAll(ctx, constants.RedisCardStopResumeCmdKey(cardID)).Result()
|
||||
if err != nil {
|
||||
return CommandState{}, err
|
||||
}
|
||||
state := ParseCommand(raw)
|
||||
if last, lastErr := s.LastStopResumeCall(ctx, cardID); lastErr == nil {
|
||||
state.LastCallAt = last
|
||||
}
|
||||
if state.Exists && state.CarrierID > 0 {
|
||||
member := strconv.FormatUint(uint64(cardID), 10)
|
||||
if score, scoreErr := s.redis.ZScore(ctx, constants.RedisCarrierStopResumeQueueKey(state.CarrierID), member).Result(); scoreErr == nil {
|
||||
state.NextAt = time.UnixMilli(int64(score))
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// ParseCommand 解析停复机指令 Hash。
|
||||
func ParseCommand(raw map[string]string) CommandState {
|
||||
state := CommandState{Exists: raw[srFieldAction] != ""}
|
||||
state.Action, state.Reason, state.Source, state.LastError = raw[srFieldAction], raw[srFieldReason], raw[srFieldSource], raw[srFieldError]
|
||||
state.Scenes = constants.SplitStopResumeScenes(raw[srFieldScenes])
|
||||
if value, err := strconv.ParseUint(raw[srFieldCarrier], 10, 64); err == nil {
|
||||
state.CarrierID = uint(value)
|
||||
}
|
||||
state.Version, _ = strconv.ParseInt(raw[srFieldVersion], 10, 64)
|
||||
state.FailCount, _ = strconv.Atoi(raw[srFieldFail])
|
||||
state.Unknown, _ = strconv.Atoi(raw[srFieldUnknown])
|
||||
state.Pending = raw[srFieldPending] == "1"
|
||||
if ms, err := strconv.ParseInt(raw[srFieldFirst], 10, 64); err == nil && ms > 0 {
|
||||
state.FirstFailed = time.UnixMilli(ms)
|
||||
}
|
||||
if ms, err := strconv.ParseInt(raw[srFieldAt], 10, 64); err == nil && ms > 0 {
|
||||
state.WrittenAt = time.UnixMilli(ms)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// 停复机指令字段名导出给执行器组装收尾写入。
|
||||
const (
|
||||
SRFieldFail = srFieldFail
|
||||
SRFieldFirst = srFieldFirst
|
||||
SRFieldError = srFieldError
|
||||
SRFieldPending = srFieldPending
|
||||
SRFieldUnknown = srFieldUnknown
|
||||
)
|
||||
|
||||
// RequestDeviceEval 请求一次设备维度判断：递增版本、合并场景、设备进入判断队列。
|
||||
func (s *Store) RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error {
|
||||
if deviceID == 0 {
|
||||
return stderrors.New("设备判断缺少设备")
|
||||
}
|
||||
args := []any{deviceID, time.Now().UnixMilli()}
|
||||
for _, scene := range constants.SplitStopResumeScenes(constants.JoinStopResumeScenes(scenes)) {
|
||||
args = append(args, scene)
|
||||
}
|
||||
return deviceRequestScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisDeviceEvalVersionKey(deviceID), constants.RedisDeviceEvalScenesKey(deviceID), constants.RedisDeviceEvalQueueKey,
|
||||
constants.RedisDeviceEvalClaimKey(deviceID)},
|
||||
args...).Err()
|
||||
}
|
||||
|
||||
// DeviceEvalClaimed 是一次被领取的设备判断。
|
||||
type DeviceEvalClaimed struct {
|
||||
DeviceID uint
|
||||
Token string
|
||||
Version string
|
||||
Scenes []string
|
||||
}
|
||||
|
||||
// ClaimDeviceEvals 领取到期设备判断。
|
||||
func (s *Store) ClaimDeviceEvals(ctx context.Context, lease time.Duration, limit int, tokenPrefix string) ([]DeviceEvalClaimed, int64, error) {
|
||||
raw, err := deviceClaimScript.Run(ctx, s.redis, []string{constants.RedisDeviceEvalQueueKey, constants.RedisDeviceEvalRunningKey},
|
||||
time.Now().UnixMilli(), lease.Milliseconds(), limit, tokenPrefix).Slice()
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil, 0, err
|
||||
}
|
||||
expired, _ := raw[0].(int64)
|
||||
claimed := make([]DeviceEvalClaimed, 0, (len(raw)-1)/4)
|
||||
for index := 1; index+3 < len(raw); index += 4 {
|
||||
deviceID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64)
|
||||
claimed = append(claimed, DeviceEvalClaimed{DeviceID: uint(deviceID), Token: fmt.Sprint(raw[index+1]),
|
||||
Version: fmt.Sprint(raw[index+2]), Scenes: constants.SplitStopResumeScenes(fmt.Sprint(raw[index+3]))})
|
||||
}
|
||||
return claimed, expired, nil
|
||||
}
|
||||
|
||||
// AcquireDeviceEvalLock 抢设备判断锁（执行权）：1 抢到，0 处理中标记已不是该 token，-1 已有判断在执行。
|
||||
func (s *Store) AcquireDeviceEvalLock(ctx context.Context, deviceID uint, token, value string, ttl time.Duration) (int64, error) {
|
||||
return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisDeviceEvalClaimKey(deviceID), constants.RedisDeviceEvalLockKey(deviceID)},
|
||||
token, value, ttl.Milliseconds()).Int64()
|
||||
}
|
||||
|
||||
// FinishDeviceEval 设备判断收尾：返回 0 token 不一致、1 完成、2 版本变大已重新排队、3 已推后。
|
||||
func (s *Store) FinishDeviceEval(ctx context.Context, claimed DeviceEvalClaimed, deferFor time.Duration) (int64, error) {
|
||||
args := []any{claimed.Token, claimed.DeviceID, claimed.Version, time.Now().UnixMilli(), deferFor.Milliseconds()}
|
||||
for _, scene := range claimed.Scenes {
|
||||
args = append(args, scene)
|
||||
}
|
||||
return deviceFinishScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisDeviceEvalClaimKey(claimed.DeviceID), constants.RedisDeviceEvalLockKey(claimed.DeviceID),
|
||||
constants.RedisDeviceEvalRunningKey, constants.RedisDeviceEvalVersionKey(claimed.DeviceID),
|
||||
constants.RedisDeviceEvalScenesKey(claimed.DeviceID), constants.RedisDeviceEvalQueueKey},
|
||||
args...).Int64()
|
||||
}
|
||||
|
||||
// ReleaseDeviceEval 发布失败时释放设备判断处理中标记。
|
||||
func (s *Store) ReleaseDeviceEval(ctx context.Context, deviceID uint, token string) error {
|
||||
return srReleaseScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisDeviceEvalClaimKey(deviceID), constants.RedisDeviceEvalRunningKey, constants.RedisDeviceEvalQueueKey},
|
||||
token, deviceID, time.Now().UnixMilli()).Err()
|
||||
}
|
||||
|
||||
// StopResumeQueueStats 是账户停复机队列的即时统计。
|
||||
type StopResumeQueueStats struct {
|
||||
Queued int64
|
||||
Due int64
|
||||
Running int64
|
||||
Pending int64
|
||||
}
|
||||
|
||||
// StopResumeStats 读取账户停复机队列长度、到期数与执行中数；待确认数按队列成员抽样统计（最多 500 个）。
|
||||
func (s *Store) StopResumeStats(ctx context.Context, carrierID uint) (StopResumeQueueStats, error) {
|
||||
now := strconv.FormatInt(time.Now().UnixMilli(), 10)
|
||||
queueKey := constants.RedisCarrierStopResumeQueueKey(carrierID)
|
||||
pipe := s.redis.Pipeline()
|
||||
queued := pipe.ZCard(ctx, queueKey)
|
||||
due := pipe.ZCount(ctx, queueKey, "-inf", now)
|
||||
running := pipe.ZCount(ctx, constants.RedisCarrierStopResumeRunningKey(carrierID), now, "+inf")
|
||||
members := pipe.ZRange(ctx, queueKey, 0, 499)
|
||||
if _, err := pipe.Exec(ctx); err != nil && !stderrors.Is(err, redis.Nil) {
|
||||
return StopResumeQueueStats{}, err
|
||||
}
|
||||
stats := StopResumeQueueStats{Queued: queued.Val(), Due: due.Val(), Running: running.Val()}
|
||||
if ids := members.Val(); len(ids) > 0 {
|
||||
check := s.redis.Pipeline()
|
||||
results := make([]*redis.StringCmd, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
cardID, convErr := strconv.ParseUint(strings.TrimSpace(id), 10, 64)
|
||||
if convErr != nil {
|
||||
continue
|
||||
}
|
||||
results = append(results, check.HGet(ctx, constants.RedisCardStopResumeCmdKey(uint(cardID)), srFieldPending))
|
||||
}
|
||||
_, _ = check.Exec(ctx)
|
||||
for _, result := range results {
|
||||
if result.Val() == "1" {
|
||||
stats.Pending++
|
||||
}
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// IncrStopResumeStat 累加账户停复机执行计数（10 分钟分桶，保留 2 小时）。
|
||||
func (s *Store) IncrStopResumeStat(ctx context.Context, carrierID uint, field string) {
|
||||
key := constants.RedisCarrierStopResumeStatKey(carrierID, time.Now().Unix()/constants.StopResumeStatBucketSeconds)
|
||||
pipe := s.redis.Pipeline()
|
||||
pipe.HIncrBy(ctx, key, field, 1)
|
||||
pipe.Expire(ctx, key, 2*time.Hour)
|
||||
_, _ = pipe.Exec(ctx)
|
||||
}
|
||||
|
||||
// StopResumeStatLastHour 汇总账户近 1 小时（最近 6 个分桶）的停复机执行计数。
|
||||
func (s *Store) StopResumeStatLastHour(ctx context.Context, carrierID uint) map[string]int64 {
|
||||
current := time.Now().Unix() / constants.StopResumeStatBucketSeconds
|
||||
pipe := s.redis.Pipeline()
|
||||
cmds := make([]*redis.MapStringStringCmd, 0, 6)
|
||||
for bucket := current - 5; bucket <= current; bucket++ {
|
||||
cmds = append(cmds, pipe.HGetAll(ctx, constants.RedisCarrierStopResumeStatKey(carrierID, bucket)))
|
||||
}
|
||||
_, _ = pipe.Exec(ctx)
|
||||
totals := map[string]int64{}
|
||||
for _, cmd := range cmds {
|
||||
for field, value := range cmd.Val() {
|
||||
count, _ := strconv.ParseInt(value, 10, 64)
|
||||
totals[field] += count
|
||||
}
|
||||
}
|
||||
return totals
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package polling
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
@@ -435,16 +436,17 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
if h.stopResumeCallback == nil {
|
||||
return
|
||||
}
|
||||
detachedCtx := context.WithoutCancel(ctx)
|
||||
// 套餐到期即停：以"套餐到期"场景请求评估，暂停轮询的卡同样停机（只触发一次，不做补偿扫描）。
|
||||
detachedCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeScenePackageExpired)
|
||||
|
||||
if carrierType == "iot_card" {
|
||||
go func() {
|
||||
stopresumedomain.RunAfterCommit(detachedCtx, h.stopResumeCallback, func() {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, carrierID); err != nil {
|
||||
h.logger.Error("套餐过期后停机失败",
|
||||
zap.Uint("card_id", carrierID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}()
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -457,14 +459,14 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
return
|
||||
}
|
||||
for _, b := range bindings {
|
||||
cardID := b.IotCardID
|
||||
go func(cID uint) {
|
||||
cID := b.IotCardID
|
||||
stopresumedomain.RunAfterCommit(detachedCtx, h.stopResumeCallback, func() {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, cID); err != nil {
|
||||
h.logger.Error("套餐过期后停机失败",
|
||||
zap.Uint("card_id", cID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(cardID)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -482,6 +484,7 @@ func (h *PackageActivationHandler) reconcileCarrierAfterActivation(ctx context.C
|
||||
if carrierID == 0 {
|
||||
return errors.New(errors.CodeInvalidParam, "套餐使用记录缺少有效载体")
|
||||
}
|
||||
ctx = stopresumedomain.WithScene(ctx, constants.StopResumeSceneActivation)
|
||||
|
||||
if carrierType == constants.AssetTypeIotCard {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(ctx, carrierID); err != nil {
|
||||
|
||||
@@ -3,6 +3,7 @@ package carrier
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -26,6 +27,22 @@ type Service struct {
|
||||
audit systemconfigapp.AuditWriter
|
||||
// thresholdHits 是通道阈值命中记录只读查询；与运营商配置共用同一数据库连接。
|
||||
thresholdHits *carrierthresholdquery.Query
|
||||
// realnameEvaluator 实名管控修改后对受影响的卡批量请求停复机评估；未注入时不触发。
|
||||
realnameEvaluator RealnameControlEvaluator
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// RealnameControlEvaluator 是实名管控修改后批量请求停复机评估的能力（停复机出口开关关闭时不执行）。
|
||||
type RealnameControlEvaluator interface {
|
||||
RequestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) (int, error)
|
||||
}
|
||||
|
||||
// SetRealnameControlEvaluator 注入实名管控修改后的批量评估能力。
|
||||
func (s *Service) SetRealnameControlEvaluator(evaluator RealnameControlEvaluator, logger *zap.Logger) {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
s.realnameEvaluator, s.logger = evaluator, logger
|
||||
}
|
||||
|
||||
func New(carrierStore *postgres.CarrierStore, audit systemconfigapp.AuditWriter) *Service {
|
||||
@@ -210,10 +227,24 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateCarrierReq
|
||||
s.recordFailure(ctx, constants.AuditOperationCarrierUpdate, "更新运营商配置失败", carrier)
|
||||
return nil, errors.Wrap(errors.CodeInternalError, err, "更新运营商失败")
|
||||
}
|
||||
if before.RealnameControlEnabled != carrier.RealnameControlEnabled && s.realnameEvaluator != nil {
|
||||
s.requestRealnameControlEvaluation(ctx, carrier.ID, carrier.RealnameControlEnabled == constants.StatusEnabled)
|
||||
}
|
||||
|
||||
return s.toResponse(ctx, carrier), nil
|
||||
}
|
||||
|
||||
// requestRealnameControlEvaluation 在事务提交后异步批量请求评估（只读库与写共享缓存，不阻塞保存接口）。
|
||||
func (s *Service) requestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) {
|
||||
evalCtx := context.WithoutCancel(ctx)
|
||||
go func() {
|
||||
defer func() { _ = recover() }()
|
||||
if _, err := s.realnameEvaluator.RequestRealnameControlEvaluation(evalCtx, carrierID, enabled); err != nil {
|
||||
s.logger.Error("实名管控修改后批量请求停复机评估失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Service) Delete(ctx context.Context, id uint) error {
|
||||
carrier, err := s.carrierStore.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
|
||||
@@ -2006,7 +2006,7 @@ func (s *Service) handleManualRealnameStatusChanged(ctx context.Context, oldStat
|
||||
}
|
||||
|
||||
if s.stopResumeService != nil {
|
||||
if err := s.stopResumeService.EvaluateAndAct(ctx, card); err != nil {
|
||||
if err := s.stopResumeService.EvaluateAndAct(WithStopResumeScene(ctx, constants.StopResumeSceneRealnameObserved), card); err != nil {
|
||||
s.logger.Warn("手动实名后触发停复机评估失败",
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.Error(err))
|
||||
|
||||
190
internal/service/iot_card/stop_resume_command.go
Normal file
190
internal/service/iot_card/stop_resume_command.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// 停复机指令执行器使用的服务能力（add-stop-resume-command-queue）。执行器只调用一次网关，不 sleep 重试。
|
||||
|
||||
// CommandDecision 是执行前按指令场景重新判断的结果。
|
||||
type CommandDecision struct {
|
||||
Card *model.IotCard
|
||||
Action string // 空表示条件不再满足或已是目标状态，结束指令
|
||||
Reason string
|
||||
Gone bool // 卡已不存在
|
||||
}
|
||||
|
||||
// DecideCommand 执行前按指令记录的场景重新判断该卡（只看本卡，不扩散到设备）。
|
||||
// 复机持通道阈值锁时视为不再满足。
|
||||
func (s *StopResumeService) DecideCommand(ctx context.Context, cardID uint, scenes []string) (CommandDecision, error) {
|
||||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||||
if err != nil {
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return CommandDecision{Gone: true}, nil
|
||||
}
|
||||
return CommandDecision{}, errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
|
||||
}
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return CommandDecision{Card: card}, err
|
||||
}
|
||||
if decision.Action == constants.StopResumeActionStart {
|
||||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||||
if blockErr != nil {
|
||||
return CommandDecision{Card: card}, blockErr
|
||||
}
|
||||
if blocked {
|
||||
return CommandDecision{Card: card}, nil
|
||||
}
|
||||
}
|
||||
return CommandDecision{Card: card, Action: decision.Action, Reason: decision.Reason}, nil
|
||||
}
|
||||
|
||||
// CommandCallResult 是一次停复机网关调用的结果。
|
||||
type CommandCallResult struct {
|
||||
// Unknown 为 true 表示结果未知：超时、连接失败，或上游已成功但本地回写失败。
|
||||
Unknown bool
|
||||
}
|
||||
|
||||
// ExecuteCommand 对卡调用一次停机或复机（限流等待模式由调用方 ctx 声明），成功后按现有回写更新卡状态、
|
||||
// 停机原因与网络状态序列，并写停复机记录。返回的 error 为网关错误或回写错误，由执行器按错误分类处理。
|
||||
func (s *StopResumeService) ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (CommandCallResult, error) {
|
||||
if card == nil {
|
||||
return CommandCallResult{}, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
if action == constants.StopResumeActionStop {
|
||||
outcome, err := s.stopCardAttempts(ctx, card, reason, 1)
|
||||
return CommandCallResult{Unknown: err != nil && outcome.Result == constants.AuditResultUnknown && !gateway.IsRateLimited(err)}, err
|
||||
}
|
||||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
|
||||
attempt, integrationID, err := s.resumeCardAttempts(ctx, card, actionCode, summary, 1)
|
||||
if err != nil {
|
||||
return CommandCallResult{Unknown: isUnknownGatewayError(err)}, err
|
||||
}
|
||||
if err := s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary); err != nil {
|
||||
return CommandCallResult{Unknown: true}, err
|
||||
}
|
||||
return CommandCallResult{}, nil
|
||||
}
|
||||
|
||||
// isUnknownGatewayError 判断网关错误是否为结果未知（超时、连接失败）：请求可能已到达上游。
|
||||
func isUnknownGatewayError(err error) bool {
|
||||
gwErr, ok := gateway.AsGatewayError(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return gwErr.Category == constants.GatewayErrorTimeout || gwErr.Category == constants.GatewayErrorNetwork
|
||||
}
|
||||
|
||||
// QueryCommandCardStatus 结果未知后的确认查询：只查一次卡状态（受限流、不受单卡最小重试间隔约束）。
|
||||
// 查询失败时 known=false 并返回错误，调用方保持待确认、不发起停复机。
|
||||
func (s *StopResumeService) QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error) {
|
||||
if s.gatewayClient == nil {
|
||||
return 0, false, errors.New(errors.CodeInternalError, "Gateway 未配置，无法查询卡状态")
|
||||
}
|
||||
queryCtx := gateway.WithCarrier(ctx, card.CarrierID)
|
||||
response, err := s.gatewayClient.QueryCardStatus(queryCtx, &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
status, known := carddomain.MapGatewayNetworkStatus(response.CardStatus, response.Extend)
|
||||
return status, known, nil
|
||||
}
|
||||
|
||||
// ConfirmCommandState 按已确认的上游结果（确认查询已是目标状态、上游返回已是目标状态、停机遇上游终态）回写卡状态。
|
||||
func (s *StopResumeService) ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error {
|
||||
if card == nil {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络（确认）"
|
||||
scene, expected := constants.CardObservationSceneBusinessResume, "online"
|
||||
fields := map[string]any{"network_status": constants.NetworkStatusOnline, "resumed_at": time.Now(), "stop_reason": ""}
|
||||
if action == constants.StopResumeActionStop {
|
||||
actionCode, summary = stopAuditAction(ctx, reason)
|
||||
summary += "（确认）"
|
||||
scene, expected = constants.CardObservationSceneBusinessStop, "offline"
|
||||
fields = map[string]any{"network_status": constants.NetworkStatusOffline, "stopped_at": time.Now(), "stop_reason": reason}
|
||||
}
|
||||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, fields, scene, expected, uuid.NewString(), actionCode, summary, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
s.reschedulePolling(ctx, card.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EvaluateDeviceCommands 设备维度判断：读取设备下各卡最新数据，按统一判断函数（各卡按所属运营商账户判断实名，
|
||||
// 暂停轮询按场景集合判断）为需要动作的卡生成指令，进入各卡所属账户的停复机队列。只读数据库与写共享缓存，不调网关。
|
||||
func (s *StopResumeService) EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error {
|
||||
if s.commandQueue == nil {
|
||||
return errors.New(errors.CodeInternalError, "停复机指令队列未配置")
|
||||
}
|
||||
cards, err := s.deviceCards(ctx, deviceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var failed []error
|
||||
for _, card := range cards {
|
||||
decision, decideErr := s.decideCard(ctx, card, scenes)
|
||||
if decideErr != nil {
|
||||
failed = append(failed, decideErr)
|
||||
continue
|
||||
}
|
||||
if decision.Action == "" {
|
||||
s.markNotRealnameIfBlocked(ctx, card, scenes)
|
||||
continue
|
||||
}
|
||||
decision.DeviceID = 0
|
||||
if writeErr := s.writeCardCommand(ctx, card, decision, scenes, "device_eval"); writeErr != nil {
|
||||
failed = append(failed, writeErr)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return errors.Wrap(errors.CodeInternalError, stderrors.Join(failed...), "设备判断存在未写入指令的卡")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// markNotRealnameIfBlocked 设备下停机卡套餐满足但实名不满足时，把停机原因更新为未实名（与旧设备复机路径一致）。
|
||||
func (s *StopResumeService) markNotRealnameIfBlocked(ctx context.Context, card *model.IotCard, scenes []string) {
|
||||
if card.NetworkStatus != constants.NetworkStatusOffline || card.StopReason == constants.StopReasonNotRealname {
|
||||
return
|
||||
}
|
||||
if suspended, err := s.autoControlSuspended(ctx, card); err != nil || suspended {
|
||||
return
|
||||
}
|
||||
hasPackage, err := s.hasValidPackage(ctx, card)
|
||||
if err != nil || !hasPackage {
|
||||
return
|
||||
}
|
||||
if exhausted, err := s.isTrafficExhausted(ctx, card); err != nil || exhausted {
|
||||
return
|
||||
}
|
||||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||||
if err != nil || realnameOK {
|
||||
return
|
||||
}
|
||||
if err := s.updateCardStopReasonWithAudit(ctx, card, constants.StopReasonNotRealname); err != nil {
|
||||
s.logger.Warn("更新未实名卡停机原因失败", zap.Uint("card_id", card.ID), zap.Strings("scenes", scenes), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// LegacyDeviceOpLocked 判断旧路径是否正在对该设备执行停复机（持设备锁）；设备判断须推后，不并行执行。
|
||||
func (s *StopResumeService) LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool {
|
||||
if s.redis == nil {
|
||||
return false
|
||||
}
|
||||
exists, err := s.redis.Exists(ctx, constants.RedisPollingDeviceOpLockKey(deviceID)).Result()
|
||||
return err == nil && exists > 0
|
||||
}
|
||||
92
internal/service/iot_card/stop_resume_decide.go
Normal file
92
internal/service/iot_card/stop_resume_decide.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// WithStopResumeScene 声明本次自动停复机评估的触发场景；未声明时按套餐步骤（周期评估）处理。
|
||||
func WithStopResumeScene(ctx context.Context, scene string) context.Context {
|
||||
return stopresumedomain.WithScene(ctx, scene)
|
||||
}
|
||||
|
||||
// stopResumeScenes 返回 ctx 中声明的场景集合（单个场景）。
|
||||
func stopResumeScenes(ctx context.Context) []string {
|
||||
return []string{stopresumedomain.Scene(ctx)}
|
||||
}
|
||||
|
||||
// stopResumeDecision 是统一判断函数的结果。
|
||||
type stopResumeDecision struct {
|
||||
Action string // constants.StopResumeActionStop / StopResumeActionStart；空表示不动作
|
||||
Reason string // 停机原因
|
||||
Reasons []string // 全部命中的停机原因（按优先级）
|
||||
DeviceID uint // 设备维度动作的设备；0 表示单卡
|
||||
}
|
||||
|
||||
// decideCard 统一判断函数：暂停轮询（按场景） → 停机条件 → 复机条件。
|
||||
// 暂停轮询的卡只在套餐失效类场景（套餐到期、退款）下放行无套餐停机，其余自动停复机一律不做。
|
||||
// 设备维度原因（无套餐、流量耗尽）与绑定设备的复机返回 DeviceID，由调用方按设备处理。通道阈值锁由复机执行方判断。
|
||||
func (s *StopResumeService) decideCard(ctx context.Context, card *model.IotCard, scenes []string) (stopResumeDecision, error) {
|
||||
suspended, err := s.autoControlSuspended(ctx, card)
|
||||
if err != nil {
|
||||
return stopResumeDecision{}, err
|
||||
}
|
||||
allowSuspendedStop := constants.StopResumeSceneAllowsSuspendedStop(scenes)
|
||||
if suspended && !allowSuspendedStop {
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
switch card.NetworkStatus {
|
||||
case constants.NetworkStatusOnline:
|
||||
reasons, err := s.checkStopReasons(ctx, card)
|
||||
if err != nil || len(reasons) == 0 {
|
||||
return stopResumeDecision{}, err
|
||||
}
|
||||
primary := reasons[0]
|
||||
if suspended {
|
||||
// 暂停轮询：只有套餐失效（无可用主套餐）才停机。
|
||||
if !containsStopReason(reasons, constants.StopReasonNoPackage) {
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
primary = constants.StopReasonNoPackage
|
||||
}
|
||||
decision := stopResumeDecision{Action: constants.StopResumeActionStop, Reason: primary, Reasons: reasons}
|
||||
if !card.IsStandalone && isDeviceScopeReason(primary) {
|
||||
decision.DeviceID = s.boundDeviceID(ctx, card)
|
||||
}
|
||||
return decision, nil
|
||||
case constants.NetworkStatusOffline:
|
||||
if suspended {
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
// 不看停机原因，是否复机只由本系统规则决定。
|
||||
resume, err := s.shouldResume(ctx, card)
|
||||
if err != nil || !resume {
|
||||
return stopResumeDecision{}, err
|
||||
}
|
||||
decision := stopResumeDecision{Action: constants.StopResumeActionStart}
|
||||
if !card.IsStandalone {
|
||||
decision.DeviceID = s.boundDeviceID(ctx, card)
|
||||
}
|
||||
return decision, nil
|
||||
default:
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// boundDeviceID 返回卡的有效绑定设备；查询失败时降级为单卡处理并记日志。
|
||||
func (s *StopResumeService) boundDeviceID(ctx context.Context, card *model.IotCard) uint {
|
||||
deviceID, bound, err := s.getCardDeviceID(ctx, card)
|
||||
if err != nil {
|
||||
s.logger.Error("查询设备绑定关系失败，降级为单卡处理", zap.Uint("card_id", card.ID), zap.Error(err))
|
||||
return 0
|
||||
}
|
||||
if !bound {
|
||||
return 0
|
||||
}
|
||||
return deviceID
|
||||
}
|
||||
70
internal/service/iot_card/stop_resume_realname_control.go
Normal file
70
internal/service/iot_card/stop_resume_realname_control.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// realnameControlBatchSize 是实名管控修改后分批读取受影响卡的批大小。
|
||||
const realnameControlBatchSize = 500
|
||||
|
||||
// RequestRealnameControlEvaluation 运营商账户实名管控修改后对受影响的卡批量请求评估（只在停复机出口开关开启时执行）：
|
||||
// 0 → 1 时为该账户在线且未实名的卡，1 → 0 时为停机且未实名的卡；绑定设备的卡按设备请求设备判断。
|
||||
// 停复机按账户 QPS 由指令队列平滑执行。返回请求评估的卡与设备数。
|
||||
func (s *StopResumeService) RequestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) (int, error) {
|
||||
if !s.commandQueueEnabled(ctx) || s.db == nil {
|
||||
return 0, nil
|
||||
}
|
||||
ctx = WithStopResumeScene(ctx, constants.StopResumeSceneRealnameControl)
|
||||
scenes := []string{constants.StopResumeSceneRealnameControl}
|
||||
networkStatus := constants.NetworkStatusOffline
|
||||
if enabled {
|
||||
networkStatus = constants.NetworkStatusOnline
|
||||
}
|
||||
requested := 0
|
||||
devices := map[uint]bool{}
|
||||
lastID := uint(0)
|
||||
for {
|
||||
var cards []*model.IotCard
|
||||
if err := s.db.WithContext(ctx).Where("carrier_id = ? AND network_status = ? AND real_name_status <> ? AND id > ?",
|
||||
carrierID, networkStatus, constants.RealNameStatusVerified, lastID).
|
||||
Order("id").Limit(realnameControlBatchSize).Find(&cards).Error; err != nil {
|
||||
return requested, errors.Wrap(errors.CodeDatabaseError, err, "查询实名管控受影响的卡失败")
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
for _, card := range cards {
|
||||
lastID = card.ID
|
||||
if deviceID := s.boundDeviceID(ctx, card); deviceID > 0 {
|
||||
if !devices[deviceID] {
|
||||
devices[deviceID] = true
|
||||
if err := s.requestDeviceEval(ctx, deviceID, scenes, "realname_control"); err == nil {
|
||||
requested++
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
s.logger.Warn("实名管控修改后判断卡失败", zap.Uint("card_id", card.ID), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if decision.Action == "" {
|
||||
continue
|
||||
}
|
||||
decision.DeviceID = 0
|
||||
if err := s.writeCardCommand(ctx, card, decision, scenes, "realname_control"); err == nil {
|
||||
requested++
|
||||
}
|
||||
}
|
||||
}
|
||||
s.logger.Info("运营商账户实名管控修改后已批量请求评估", zap.Uint("carrier_id", carrierID), zap.Bool("enabled", enabled),
|
||||
zap.Int("requested", requested), zap.String("metric", "stop_resume.realname_control_requested"))
|
||||
return requested, nil
|
||||
}
|
||||
144
internal/service/iot_card/stop_resume_request.go
Normal file
144
internal/service/iot_card/stop_resume_request.go
Normal file
@@ -0,0 +1,144 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// StopResumeCommandQueue 是停复机指令队列的写入接缝（共享缓存）。
|
||||
type StopResumeCommandQueue interface {
|
||||
// Enabled 返回停复机出口开关 polling.stop_resume.queue_enabled。
|
||||
Enabled(ctx context.Context) bool
|
||||
WriteCommand(ctx context.Context, cmd carrierqueue.StopResumeCommand) (int64, error)
|
||||
RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error
|
||||
}
|
||||
|
||||
// SetCommandQueue 注入停复机指令队列；未注入时始终当场执行。
|
||||
func (s *StopResumeService) SetCommandQueue(queue StopResumeCommandQueue) {
|
||||
s.commandQueue = queue
|
||||
}
|
||||
|
||||
// CommandQueueEnabled 返回停复机出口开关是否开启（供调用方决定是否还需要异步执行）。
|
||||
func (s *StopResumeService) CommandQueueEnabled(ctx context.Context) bool {
|
||||
return s.commandQueueEnabled(ctx)
|
||||
}
|
||||
|
||||
func (s *StopResumeService) commandQueueEnabled(ctx context.Context) bool {
|
||||
return s != nil && s.commandQueue != nil && s.commandQueue.Enabled(ctx)
|
||||
}
|
||||
|
||||
// RequestAssetEvaluation 请求评估门面：开关开启时单卡生成指令、设备请求设备判断；关闭时走旧路径当场执行。
|
||||
// assetType 为 constants.AssetTypeIotCard / AssetTypeDevice。
|
||||
func (s *StopResumeService) RequestAssetEvaluation(ctx context.Context, assetType string, assetID uint, scene, source string) error {
|
||||
ctx = WithStopResumeScene(ctx, scene)
|
||||
if !s.commandQueueEnabled(ctx) {
|
||||
return s.legacyAssetEvaluation(ctx, assetType, assetID)
|
||||
}
|
||||
if assetType == constants.AssetTypeDevice {
|
||||
return s.requestDeviceEval(ctx, assetID, []string{scene}, source)
|
||||
}
|
||||
card, err := s.iotCardStore.GetByID(ctx, assetID)
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
|
||||
}
|
||||
scenes := []string{scene}
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.submitDecisionWithSource(ctx, card, decision, scenes, source)
|
||||
}
|
||||
|
||||
// legacyAssetEvaluation 开关关闭时的当场执行：单卡按统一判断执行，设备按设备复机（套餐激活、重置、订单）或逐卡评估（其他场景）。
|
||||
func (s *StopResumeService) legacyAssetEvaluation(ctx context.Context, assetType string, assetID uint) error {
|
||||
if assetType == constants.AssetTypeDevice {
|
||||
cards, err := s.deviceCards(ctx, assetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, card := range cards {
|
||||
if err := s.EvaluateAndAct(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
card, err := s.iotCardStore.GetByID(ctx, assetID)
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
|
||||
}
|
||||
return s.EvaluateAndAct(ctx, card)
|
||||
}
|
||||
|
||||
// submitDecision 把统一判断结果写入指令队列（来源取场景）。
|
||||
func (s *StopResumeService) submitDecision(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string) error {
|
||||
return s.submitDecisionWithSource(ctx, card, decision, scenes, "")
|
||||
}
|
||||
|
||||
func (s *StopResumeService) submitDecisionWithSource(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string, source string) error {
|
||||
if decision.Action == "" {
|
||||
return nil
|
||||
}
|
||||
if decision.DeviceID > 0 {
|
||||
return s.requestDeviceEval(ctx, decision.DeviceID, scenes, source)
|
||||
}
|
||||
return s.writeCardCommand(ctx, card, decision, scenes, source)
|
||||
}
|
||||
|
||||
// writeCardCommand 生成单卡停复机指令；复机前检查通道阈值锁，持锁时记录拒绝事实不生成指令。
|
||||
func (s *StopResumeService) writeCardCommand(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string, source string) error {
|
||||
if decision.Action == constants.StopResumeActionStart {
|
||||
blocked, err := s.channelThresholdBlocked(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if blocked {
|
||||
s.recordChannelThresholdDenial(ctx, card, constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if source == "" {
|
||||
source = constants.JoinStopResumeScenes(scenes)
|
||||
}
|
||||
version, err := s.commandQueue.WriteCommand(ctx, carrierqueue.StopResumeCommand{
|
||||
CardID: card.ID, CarrierID: card.CarrierID, Action: decision.Action, Reason: decision.Reason, Scenes: scenes, Source: source,
|
||||
})
|
||||
if err != nil {
|
||||
// 写入失败只记日志：由该卡下一次套餐步骤兜底。
|
||||
s.logger.Error("写入停复机指令失败", zap.Uint("card_id", card.ID), zap.String("action", decision.Action),
|
||||
zap.Strings("scenes", scenes), zap.String("metric", "stop_resume.request_failed"), zap.Error(err))
|
||||
return errors.Wrap(errors.CodeRedisError, err, "写入停复机指令失败")
|
||||
}
|
||||
s.logger.Info("已生成停复机指令", zap.Uint("card_id", card.ID), zap.Uint("carrier_id", card.CarrierID),
|
||||
zap.String("action", decision.Action), zap.String("reason", decision.Reason), zap.Strings("scenes", scenes), zap.Int64("version", version))
|
||||
return nil
|
||||
}
|
||||
|
||||
// requestDeviceEval 请求一次设备维度判断。
|
||||
func (s *StopResumeService) requestDeviceEval(ctx context.Context, deviceID uint, scenes []string, source string) error {
|
||||
if err := s.commandQueue.RequestDeviceEval(ctx, deviceID, scenes); err != nil {
|
||||
s.logger.Error("请求设备判断失败", zap.Uint("device_id", deviceID), zap.Strings("scenes", scenes), zap.String("source", source),
|
||||
zap.String("metric", "stop_resume.request_failed"), zap.Error(err))
|
||||
return errors.Wrap(errors.CodeRedisError, err, "请求设备判断失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deviceCards 返回设备下有效绑定的全部卡（在线与停机）。
|
||||
func (s *StopResumeService) deviceCards(ctx context.Context, deviceID uint) ([]*model.IotCard, error) {
|
||||
online, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOnline)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询设备在线卡失败")
|
||||
}
|
||||
offline, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOffline)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询设备停机卡失败")
|
||||
}
|
||||
return append(online, offline...), nil
|
||||
}
|
||||
@@ -60,6 +60,8 @@ type StopResumeService struct {
|
||||
stopResumeRecorder *gatewaylimit.StopResumeRecorder
|
||||
// terminalPolling 停复机命中"自动关闭轮询"终态规则时关闭该卡轮询；未注入时按普通失败处理。
|
||||
terminalPolling *gatewaylimit.TerminalPollingDisabler
|
||||
// commandQueue 停复机指令队列；开关开启时自动入口只请求评估。
|
||||
commandQueue StopResumeCommandQueue
|
||||
}
|
||||
|
||||
// SetStopResumeRecorder 注入停复机记录写入器。
|
||||
@@ -128,69 +130,36 @@ func (s *StopResumeService) SetPollingCallback(callback PollingCallback) {
|
||||
s.pollingCallback = callback
|
||||
}
|
||||
|
||||
// EvaluateAndAct 停复机统一入口
|
||||
// 根据卡的当前网络状态，自动判断并执行停机或复机操作
|
||||
// 设备/单卡维度通过 card.IsStandalone 推导（false 则为设备维度，停/复机覆盖设备下所有卡）
|
||||
// EvaluateAndAct 停复机统一入口：按统一判断函数决定停机或复机。
|
||||
// 停复机出口开关开启时只请求评估（生成指令或请求设备判断），不当场调用网关；关闭时当场执行。
|
||||
// 设备/单卡维度通过 card.IsStandalone 推导（false 则为设备维度，停/复机覆盖设备下所有卡）。触发场景见 WithStopResumeScene。
|
||||
func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error {
|
||||
// 总开关开启时，卡或所在设备暂停轮询的不自动停复机。
|
||||
suspended, err := s.autoControlSuspended(ctx, card)
|
||||
scenes := stopResumeScenes(ctx)
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if suspended {
|
||||
return nil
|
||||
// 无有效套餐且存在待生效套餐使用记录：仅由普通套餐轮询来源追加优先轮询请求。
|
||||
if containsStopReason(decision.Reasons, constants.StopReasonNoPackage) {
|
||||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch card.NetworkStatus {
|
||||
case constants.NetworkStatusOnline:
|
||||
// 卡在线，检查是否需要停机
|
||||
reasons, err := s.checkStopReasons(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
if s.commandQueueEnabled(ctx) {
|
||||
return s.submitDecision(ctx, card, decision, scenes)
|
||||
}
|
||||
switch decision.Action {
|
||||
case constants.StopResumeActionStop:
|
||||
if decision.DeviceID > 0 {
|
||||
return s.stopDeviceCards(ctx, decision.DeviceID, decision.Reason)
|
||||
}
|
||||
if len(reasons) == 0 {
|
||||
return nil
|
||||
}
|
||||
// 无有效套餐且存在待生效套餐使用记录：仅由普通套餐轮询来源追加优先轮询请求。
|
||||
if containsStopReason(reasons, constants.StopReasonNoPackage) {
|
||||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// 取最高优先级原因（第一个）
|
||||
primaryReason := reasons[0]
|
||||
if !card.IsStandalone && isDeviceScopeReason(primaryReason) {
|
||||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||||
if lookupErr != nil {
|
||||
s.logger.Error("查询设备绑定关系失败，降级为单卡停机",
|
||||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||||
// 降级：查不到绑定时按单卡处理
|
||||
} else if bound {
|
||||
return s.stopDeviceCards(ctx, deviceID, primaryReason)
|
||||
}
|
||||
}
|
||||
_, stopErr := s.stopCardWithRetry(ctx, card, primaryReason)
|
||||
_, stopErr := s.stopCardWithRetry(ctx, card, decision.Reason)
|
||||
return stopErr
|
||||
|
||||
case constants.NetworkStatusOffline:
|
||||
// 卡停机，检查是否可以复机：不看停机原因，是否复机只由本系统规则决定
|
||||
shouldResume, err := s.shouldResume(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !shouldResume {
|
||||
return nil
|
||||
}
|
||||
if !card.IsStandalone {
|
||||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||||
if lookupErr != nil {
|
||||
s.logger.Error("查询设备绑定关系失败，降级为单卡复机",
|
||||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||||
} else if bound {
|
||||
return s.resumeDeviceCards(ctx, deviceID)
|
||||
}
|
||||
case constants.StopResumeActionStart:
|
||||
if decision.DeviceID > 0 {
|
||||
return s.resumeDeviceCards(ctx, decision.DeviceID)
|
||||
}
|
||||
return s.resumeSingleCard(ctx, card.ID)
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
@@ -414,11 +383,11 @@ func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint,
|
||||
|
||||
var cardErrors []error
|
||||
for _, card := range cards {
|
||||
// 总开关开启时，暂停轮询的卡不随设备自动停机。
|
||||
// 暂停轮询的卡不随设备自动停机；套餐失效类场景（套餐到期、退款）照常停机。
|
||||
if suspended, suspendErr := s.autoControlSuspended(ctx, card); suspendErr != nil {
|
||||
cardErrors = append(cardErrors, suspendErr)
|
||||
continue
|
||||
} else if suspended {
|
||||
} else if suspended && !(stopReason == constants.StopReasonNoPackage && constants.StopResumeSceneAllowsSuspendedStop(stopResumeScenes(ctx))) {
|
||||
continue
|
||||
}
|
||||
if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
|
||||
@@ -503,6 +472,9 @@ func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) e
|
||||
// ResumeCardIfStopped 套餐激活或流量重置后自动复机入口
|
||||
// 支持 iot_card 和 device 两种载体类型
|
||||
func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error {
|
||||
if s.commandQueueEnabled(ctx) {
|
||||
return s.RequestAssetEvaluation(ctx, carrierType, carrierID, stopResumeScenes(ctx)[0], "resume_if_stopped")
|
||||
}
|
||||
switch carrierType {
|
||||
case "iot_card":
|
||||
return s.resumeSingleCard(ctx, carrierID)
|
||||
@@ -582,7 +554,12 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
zap.Error(err))
|
||||
return err
|
||||
}
|
||||
return s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary)
|
||||
}
|
||||
|
||||
// completeAutoResume 自动复机网关成功后回写在线状态、观测序列与统一审计，并重排轮询。
|
||||
// 回写失败时返回错误，调用方按结果未知处理。
|
||||
func (s *StopResumeService) completeAutoResume(ctx context.Context, card *model.IotCard, attempt *gatewayAttempt, integrationID, actionCode, summary string) error {
|
||||
now := time.Now()
|
||||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||||
"network_status": constants.NetworkStatusOnline,
|
||||
@@ -593,7 +570,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||||
}
|
||||
s.logger.Error("复机 Gateway 成功但 DB 更新失败",
|
||||
zap.Uint("card_id", cardID),
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.String("iccid", card.ICCID),
|
||||
zap.Error(err))
|
||||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||||
@@ -608,7 +585,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
}
|
||||
|
||||
s.logger.Info("卡已自动复机",
|
||||
zap.Uint("card_id", cardID),
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.String("iccid", card.ICCID))
|
||||
|
||||
return nil
|
||||
@@ -619,6 +596,11 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
// 返回的 outcome 始终携带可回填可靠任务的结果分类（成功/失败/未知）与最后一次尝试的
|
||||
// Integration Log 标识；error 与既有语义一致，表示本次停机未成功。
|
||||
func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) {
|
||||
return s.stopCardAttempts(ctx, card, stopReason, s.maxRetries)
|
||||
}
|
||||
|
||||
// stopCardAttempts 调用运营商停机接口最多 attempts 次（停复机指令执行器只调用一次），成功后回写停机状态。
|
||||
func (s *StopResumeService) stopCardAttempts(ctx context.Context, card *model.IotCard, stopReason string, attempts int) (carrierthresholddomain.CommandOutcome, error) {
|
||||
actionCode, summary := stopAuditAction(ctx, stopReason)
|
||||
if s.gatewayClient == nil {
|
||||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用")
|
||||
@@ -641,7 +623,7 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
|
||||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||||
var lastErr error
|
||||
lastIntegrationID := ""
|
||||
for i := 0; i < s.maxRetries; i++ {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
s.logger.Debug("重试调用停机接口",
|
||||
zap.Int("attempt", i+1),
|
||||
@@ -732,6 +714,11 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
|
||||
// resumeCardWithRetry 调用运营商复机接口（带重试机制）。
|
||||
// 第二个返回值是最后一次尝试的 Integration Log 标识（无论成功或失败），供调用方回填可靠任务状态。
|
||||
func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) {
|
||||
return s.resumeCardAttempts(ctx, card, actionCode, summary, s.maxRetries)
|
||||
}
|
||||
|
||||
// resumeCardAttempts 调用运营商复机接口最多 attempts 次，不回写卡状态。
|
||||
func (s *StopResumeService) resumeCardAttempts(ctx context.Context, card *model.IotCard, actionCode, summary string, attempts int) (*gatewayAttempt, string, error) {
|
||||
if s.gatewayClient == nil {
|
||||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用")
|
||||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||||
@@ -751,7 +738,7 @@ func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model
|
||||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||||
var lastErr error
|
||||
lastIntegrationID := ""
|
||||
for i := 0; i < s.maxRetries; i++ {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
s.logger.Debug("重试调用复机接口",
|
||||
zap.Int("attempt", i+1),
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -2202,8 +2203,9 @@ func (s *Service) tryResumeAfterPayment(ctx context.Context, order *model.Order)
|
||||
return
|
||||
}
|
||||
|
||||
resumeCtx := context.WithoutCancel(ctx)
|
||||
go func(ct string, cid uint) {
|
||||
resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneOrder)
|
||||
ct, cid := resumeCarrierType, resumeCarrierID
|
||||
stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() {
|
||||
if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, ct, cid); err != nil {
|
||||
s.logger.Error("支付后自动复机失败",
|
||||
zap.String("carrier_type", ct),
|
||||
@@ -2211,7 +2213,7 @@ func (s *Service) tryResumeAfterPayment(ctx context.Context, order *model.Order)
|
||||
zap.Uint("order_id", order.ID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(resumeCarrierType, resumeCarrierID)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) activatePackage(ctx context.Context, tx *gorm.DB, order *model.Order) error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package packagepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -778,15 +779,15 @@ func (s *ActivationService) resumeAfterActivation(ctx context.Context, carrierTy
|
||||
if s.resumeCallback == nil {
|
||||
return
|
||||
}
|
||||
resumeCtx := context.WithoutCancel(ctx)
|
||||
go func() {
|
||||
resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneActivation)
|
||||
stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() {
|
||||
if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, carrierType, carrierID); err != nil {
|
||||
s.logger.Error(errorMessage,
|
||||
zap.String("carrier_type", carrierType),
|
||||
zap.Uint("carrier_id", carrierID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (s *ActivationService) appendActivationObservation(ctx context.Context, tx *gorm.DB, usage *model.PackageUsage, carrierType string, carrierID uint, occurredAt time.Time) error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package packagepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||||
@@ -381,7 +382,7 @@ func (s *ResetService) triggerResumeForPackages(ctx context.Context, packages []
|
||||
return
|
||||
}
|
||||
|
||||
resumeCtx := context.WithoutCancel(ctx)
|
||||
resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneReset)
|
||||
for _, pkg := range packages {
|
||||
var carrierType string
|
||||
var carrierID uint
|
||||
@@ -396,13 +397,14 @@ func (s *ResetService) triggerResumeForPackages(ctx context.Context, packages []
|
||||
continue
|
||||
}
|
||||
|
||||
go func(ct string, cid uint) {
|
||||
ct, cid := carrierType, carrierID
|
||||
stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() {
|
||||
if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, ct, cid); err != nil {
|
||||
s.logger.Warn("流量重置后自动复机失败",
|
||||
zap.String("carrier_type", ct),
|
||||
zap.Uint("carrier_id", cid),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(carrierType, carrierID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package packagepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -449,15 +450,15 @@ func (s *UsageService) triggerSuspensionAfterCommit(ctx context.Context, carrier
|
||||
return
|
||||
}
|
||||
|
||||
stopCtx := context.WithoutCancel(ctx)
|
||||
stopCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneUsage)
|
||||
if carrierType == constants.AssetTypeIotCard {
|
||||
go func() {
|
||||
stopresumedomain.RunAfterCommit(stopCtx, s.stopResumeCallback, func() {
|
||||
if err := s.stopResumeCallback.CheckAndStopCard(stopCtx, carrierID); err != nil {
|
||||
s.logger.Error("调用停机服务失败",
|
||||
zap.Uint("card_id", carrierID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}()
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -479,14 +480,14 @@ func (s *UsageService) triggerSuspensionAfterCommit(ctx context.Context, carrier
|
||||
}
|
||||
|
||||
for _, b := range bindings {
|
||||
cardID := b.IotCardID
|
||||
go func(cID uint) {
|
||||
cID := b.IotCardID
|
||||
stopresumedomain.RunAfterCommit(stopCtx, s.stopResumeCallback, func() {
|
||||
if err := s.stopResumeCallback.CheckAndStopCard(stopCtx, cID); err != nil {
|
||||
s.logger.Error("调用停机服务失败",
|
||||
zap.Uint("card_id", cID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(cardID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -503,6 +503,28 @@ func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*
|
||||
}
|
||||
resp.Types = append(resp.Types, state)
|
||||
}
|
||||
resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled
|
||||
command, err := s.store.ReadCommand(ctx, card.ID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败")
|
||||
}
|
||||
if !command.LastCallAt.IsZero() {
|
||||
resp.LastStopResumeCallAt = timePtr(command.LastCallAt)
|
||||
}
|
||||
if command.Exists {
|
||||
item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source,
|
||||
Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown}
|
||||
if item.Scenes == nil {
|
||||
item.Scenes = []string{}
|
||||
}
|
||||
if !command.WrittenAt.IsZero() {
|
||||
item.WrittenAt = timePtr(command.WrittenAt)
|
||||
}
|
||||
if !command.NextAt.IsZero() {
|
||||
item.NextAt = timePtr(command.NextAt)
|
||||
}
|
||||
resp.StopResumeCommand = item
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
@@ -571,6 +593,13 @@ func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.Carrie
|
||||
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
|
||||
}
|
||||
item.QPSLastMinute = float64(attempts) / 60
|
||||
if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil {
|
||||
item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due
|
||||
item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending
|
||||
}
|
||||
srCounts := s.store.StopResumeStatLastHour(ctx, carrierID)
|
||||
item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed]
|
||||
item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown]
|
||||
for _, alert := range alertRows {
|
||||
if alert.CarrierID == carrierID {
|
||||
item.OpenAlerts += alert.Count
|
||||
|
||||
@@ -1243,6 +1243,20 @@ func (s *Service) handleRefundAssetProcessing(ctx context.Context, refundID uint
|
||||
func (s *Service) stopAsset(ctx context.Context, assetType string, assetID uint) bool {
|
||||
logger := s.logger
|
||||
|
||||
// 停复机出口开关开启：只请求评估（"退款"场景，暂停轮询的资产同样停机），写入成功即视为资产后处理完成；
|
||||
// 停机失败由指令队列退避与告警处理，不再触发 Outbox 重投与补偿扫描。
|
||||
if s.stopResumeService != nil && s.stopResumeService.CommandQueueEnabled(ctx) {
|
||||
evalType := constants.AssetTypeIotCard
|
||||
if assetType == constants.AssetTypeDevice {
|
||||
evalType = constants.AssetTypeDevice
|
||||
}
|
||||
if err := s.stopResumeService.RequestAssetEvaluation(ctx, evalType, assetID, constants.StopResumeSceneRefund, "refund"); err != nil {
|
||||
logger.Error("退款资产处理：请求停机评估失败", zap.String("asset_type", assetType), zap.Uint("asset_id", assetID), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
switch assetType {
|
||||
case "iot_card":
|
||||
// 单卡停机：需要先查卡获取 iccid
|
||||
|
||||
70
internal/task/carrier_device_eval_executor.go
Normal file
70
internal/task/carrier_device_eval_executor.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
)
|
||||
|
||||
// 设备判断的推后时长：旧路径持设备锁时、判断失败时。
|
||||
const (
|
||||
deviceEvalLegacyLockDelay = 5 * time.Second
|
||||
deviceEvalFailureDelay = 30 * time.Second
|
||||
deviceEvalTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// DeviceEvalExecutor 设备维度判断执行器：抢设备判断锁，读取最新数据为设备下各卡生成指令；锁内不调网关。
|
||||
type DeviceEvalExecutor struct {
|
||||
store *carrierqueue.Store
|
||||
service StopResumeCommandService
|
||||
instanceID string
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewDeviceEvalExecutor 创建设备判断执行器。
|
||||
func NewDeviceEvalExecutor(store *carrierqueue.Store, service StopResumeCommandService, instanceID string, logger *zap.Logger) *DeviceEvalExecutor {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &DeviceEvalExecutor{store: store, service: service, instanceID: instanceID, logger: logger}
|
||||
}
|
||||
|
||||
// Handle 消费一条设备判断消息。收尾只移除本次已处理的场景；版本变大时用最新数据再判断一次。
|
||||
func (e *DeviceEvalExecutor) Handle(ctx context.Context, body []byte, _ func()) {
|
||||
var msg carrierqueue.DeviceEvalMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || msg.DeviceID == 0 || msg.Token == "" {
|
||||
e.logger.Warn("设备判断消息无效，丢弃", zap.ByteString("body", body))
|
||||
return
|
||||
}
|
||||
got, err := e.store.AcquireDeviceEvalLock(ctx, msg.DeviceID, msg.Token, msg.Token+"|"+e.instanceID, 2*deviceEvalTimeout)
|
||||
if err != nil || got != 1 {
|
||||
return
|
||||
}
|
||||
claimed := carrierqueue.DeviceEvalClaimed{DeviceID: msg.DeviceID, Token: msg.Token, Version: msg.Version, Scenes: msg.Scenes}
|
||||
deferFor := time.Duration(0)
|
||||
if e.service.LegacyDeviceOpLocked(ctx, msg.DeviceID) {
|
||||
// 旧路径正在对该设备执行停复机：推后，不并行判断。
|
||||
deferFor = deviceEvalLegacyLockDelay
|
||||
} else {
|
||||
evalCtx, cancel := context.WithTimeout(ctx, deviceEvalTimeout)
|
||||
if evalErr := e.service.EvaluateDeviceCommands(evalCtx, msg.DeviceID, msg.Scenes); evalErr != nil {
|
||||
e.logger.Warn("设备判断失败，稍后再判断", zap.Uint("device_id", msg.DeviceID), zap.Strings("scenes", msg.Scenes), zap.Error(evalErr))
|
||||
deferFor = deviceEvalFailureDelay
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer finishCancel()
|
||||
result, err := e.store.FinishDeviceEval(finishCtx, claimed, deferFor)
|
||||
if err != nil {
|
||||
e.logger.Warn("设备判断收尾失败，等待租约到期后重新领取", zap.Uint("device_id", msg.DeviceID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if result == 2 {
|
||||
e.logger.Info("设备判断期间有新请求，按最新数据再判断", zap.Uint("device_id", msg.DeviceID))
|
||||
}
|
||||
}
|
||||
343
internal/task/carrier_stop_resume_executor.go
Normal file
343
internal/task/carrier_stop_resume_executor.go
Normal file
@@ -0,0 +1,343 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/gatewaylimit"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StopResumeCommandService 是停复机指令执行器依赖的停复机服务能力。
|
||||
type StopResumeCommandService interface {
|
||||
DecideCommand(ctx context.Context, cardID uint, scenes []string) (iot_card_svc.CommandDecision, error)
|
||||
ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (iot_card_svc.CommandCallResult, error)
|
||||
QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error)
|
||||
ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error
|
||||
EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error
|
||||
LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool
|
||||
}
|
||||
|
||||
// stopResumeTaskTimeout 是一条停复机指令的总执行时长上限（一次判断 + 一次网关调用）。
|
||||
const stopResumeTaskTimeout = 60 * time.Second
|
||||
|
||||
// stopResumeRateLimitedDelay 本账户限流拒绝或等待超时（请求未发出）后的重排延迟，不计失败。
|
||||
const stopResumeRateLimitedDelay = 5 * time.Second
|
||||
|
||||
// StopResumeExecutor 停复机指令执行器：执行前按场景重新判断，只调用一次网关，按错误分类退避与告警，结果未知在队列内确认。
|
||||
type StopResumeExecutor struct {
|
||||
store *carrierqueue.Store
|
||||
settings *carrierqueue.SettingsProvider
|
||||
params func(carrierID uint) carrierqueue.CarrierParams
|
||||
alerts *carrierqueue.AlertWriter
|
||||
service StopResumeCommandService
|
||||
classifier *gatewaylimit.Classifier
|
||||
instanceID string
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewStopResumeExecutor 创建停复机指令执行器。
|
||||
func NewStopResumeExecutor(store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, params func(uint) carrierqueue.CarrierParams,
|
||||
alerts *carrierqueue.AlertWriter, service StopResumeCommandService, classifier *gatewaylimit.Classifier, instanceID string, logger *zap.Logger) *StopResumeExecutor {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &StopResumeExecutor{store: store, settings: settings, params: params, alerts: alerts, service: service,
|
||||
classifier: classifier, instanceID: instanceID, logger: logger}
|
||||
}
|
||||
|
||||
// srOutcome 是一次执行的收尾决定。
|
||||
type srOutcome struct {
|
||||
stat string // 计数字段覆盖；statSkip 表示不计数
|
||||
done bool
|
||||
nextAt time.Time
|
||||
set map[string]string
|
||||
del []string
|
||||
}
|
||||
|
||||
// statSkip 表示本次执行不计入监控计数（指令不再需要、卡已不存在）。
|
||||
const statSkip = "-"
|
||||
|
||||
func srDone() srOutcome { return srOutcome{done: true} }
|
||||
|
||||
func srEnded() srOutcome { return srOutcome{done: true, stat: statSkip} }
|
||||
|
||||
func srRetry(at time.Time) srOutcome {
|
||||
return srOutcome{nextAt: at, set: map[string]string{}}
|
||||
}
|
||||
|
||||
// Handle 消费一条停复机指令消息；返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
|
||||
func (e *StopResumeExecutor) Handle(ctx context.Context, body []byte, _ func()) {
|
||||
var msg carrierqueue.StopResumeMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
|
||||
e.logger.Warn("停复机指令消息无效，丢弃", zap.ByteString("body", body))
|
||||
return
|
||||
}
|
||||
execValue := msg.Token + "|" + e.instanceID
|
||||
got, err := e.store.AcquireStopResumeExec(ctx, msg.CardID, msg.Token, execValue, 2*stopResumeTaskTimeout)
|
||||
if err != nil || got != 1 {
|
||||
return
|
||||
}
|
||||
taskCtx, cancel := context.WithTimeout(ctx, stopResumeTaskTimeout)
|
||||
outcome := e.execute(taskCtx, msg)
|
||||
cancel()
|
||||
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer finishCancel()
|
||||
e.countOutcome(finishCtx, msg.CarrierID, outcome)
|
||||
result, err := e.store.FinishStopResume(finishCtx, carrierqueue.SRFinish{
|
||||
CardID: msg.CardID, CarrierID: msg.CarrierID, Token: msg.Token, Version: msg.Version,
|
||||
Done: outcome.done, NextAt: outcome.nextAt, Set: outcome.set, Del: outcome.del,
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
e.logger.Warn("停复机指令收尾失败，等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(err))
|
||||
case result == 0:
|
||||
e.logger.Warn("停复机指令收尾时处理中标记已不一致", zap.Uint("card_id", msg.CardID), zap.String("metric", "stop_resume.finish_token_mismatch"))
|
||||
case result == 2:
|
||||
e.logger.Info("停复机指令执行期间已被新指令覆盖，保留新指令", zap.Uint("card_id", msg.CardID), zap.Int64("claimed_version", msg.Version))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *StopResumeExecutor) execute(ctx context.Context, msg carrierqueue.StopResumeMessage) (outcome srOutcome) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
e.logger.Error("停复机指令执行 panic，稍后重试", zap.Uint("card_id", msg.CardID), zap.Any("panic", r))
|
||||
outcome = srRetry(time.Now().Add(time.Minute))
|
||||
}
|
||||
}()
|
||||
cmd, err := e.store.ReadCommand(ctx, msg.CardID)
|
||||
if err != nil {
|
||||
return srRetry(time.Now().Add(time.Minute))
|
||||
}
|
||||
if !cmd.Exists {
|
||||
return srEnded()
|
||||
}
|
||||
decision, err := e.service.DecideCommand(ctx, msg.CardID, cmd.Scenes)
|
||||
if err != nil {
|
||||
e.logger.Warn("停复机指令执行前判断失败，稍后重试", zap.Uint("card_id", msg.CardID), zap.Error(err))
|
||||
return srRetry(time.Now().Add(time.Minute))
|
||||
}
|
||||
if decision.Gone {
|
||||
return srEnded()
|
||||
}
|
||||
if decision.Action != cmd.Action {
|
||||
// 条件不再满足或已是目标状态：结束指令，不调用上游。
|
||||
e.logger.Info("停复机指令执行前重新判断不再需要，结束指令", zap.Uint("card_id", msg.CardID),
|
||||
zap.String("action", cmd.Action), zap.String("decided", decision.Action), zap.Strings("scenes", cmd.Scenes))
|
||||
e.recovered(cmd, decision.Card)
|
||||
return srEnded()
|
||||
}
|
||||
card := decision.Card
|
||||
reason := decision.Reason
|
||||
params := e.params(card.CarrierID)
|
||||
settings := e.settings.Current(ctx)
|
||||
gatewayCtx := gateway.WithRateLimitWait(ctx, 20*time.Second)
|
||||
|
||||
if !cmd.Pending {
|
||||
return e.call(gatewayCtx, card, cmd, reason, params, settings)
|
||||
}
|
||||
if confirmed, next := e.confirm(gatewayCtx, card, cmd, reason, params, settings); confirmed != nil {
|
||||
return *confirmed
|
||||
} else if next != nil {
|
||||
return *next
|
||||
}
|
||||
// 确认不是目标状态且已满足单卡最小间隔：清除待确认后重新调用。
|
||||
result := e.call(gatewayCtx, card, cmd, reason, params, settings)
|
||||
if !result.done && result.set[carrierqueue.SRFieldPending] != "1" {
|
||||
result.del = append(result.del, carrierqueue.SRFieldPending)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// confirm 处理待确认指令：查一次卡状态。已是目标状态返回结束；查询失败保持待确认按退避重查；
|
||||
// 不是目标状态时清除待确认，未到单卡最小间隔则推后，否则返回 (nil, nil) 由调用方重新发起停复机。
|
||||
func (e *StopResumeExecutor) confirm(ctx context.Context, card *model.IotCard, cmd carrierqueue.CommandState, reason string,
|
||||
params carrierqueue.CarrierParams, settings carrierqueue.Settings) (*srOutcome, *srOutcome) {
|
||||
status, known, err := e.service.QueryCommandCardStatus(ctx, card)
|
||||
if gateway.IsRateLimited(err) {
|
||||
retry := srRetry(time.Now().Add(stopResumeRateLimitedDelay))
|
||||
return nil, &retry
|
||||
}
|
||||
if err != nil || !known {
|
||||
retry := srRetry(time.Now().Add(settings.SRBackoffFor(cmd.Unknown)))
|
||||
if err != nil {
|
||||
retry.set[carrierqueue.SRFieldError] = truncateRunes(constants.StopResumeFailureUnknown+"|确认查询失败: "+err.Error(), 300)
|
||||
}
|
||||
return nil, &retry
|
||||
}
|
||||
if status == targetNetworkStatus(cmd.Action) {
|
||||
if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil {
|
||||
e.logger.Warn("确认已是目标状态但回写失败，稍后再确认", zap.Uint("card_id", card.ID), zap.Error(confirmErr))
|
||||
retry := srRetry(time.Now().Add(time.Minute))
|
||||
return nil, &retry
|
||||
}
|
||||
e.logger.Info("停复机结果未知，确认查询已是目标状态，按成功结束", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action))
|
||||
e.recovered(cmd, card)
|
||||
done := srDone()
|
||||
return &done, nil
|
||||
}
|
||||
last, _ := e.store.LastStopResumeCall(ctx, card.ID)
|
||||
if allowed := last.Add(params.SRMinInterval); !last.IsZero() && allowed.After(time.Now()) {
|
||||
retry := srRetry(allowed)
|
||||
retry.del = []string{carrierqueue.SRFieldPending}
|
||||
return nil, &retry
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// call 调用一次停机或复机并按结果分类处理。
|
||||
func (e *StopResumeExecutor) call(ctx context.Context, card *model.IotCard, cmd carrierqueue.CommandState, reason string,
|
||||
params carrierqueue.CarrierParams, settings carrierqueue.Settings) srOutcome {
|
||||
previousCall, _ := e.store.LastStopResumeCall(ctx, card.ID)
|
||||
now := time.Now()
|
||||
if err := e.store.MarkStopResumeCall(ctx, card.ID, now, 2*params.SRMinInterval); err != nil {
|
||||
return srRetry(now.Add(time.Minute))
|
||||
}
|
||||
result, err := e.service.ExecuteCommand(ctx, card, cmd.Action, reason)
|
||||
if err == nil {
|
||||
e.logger.Info("停复机指令执行成功", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action), zap.String("reason", reason),
|
||||
zap.Strings("scenes", cmd.Scenes), zap.String("metric", "stop_resume.success"))
|
||||
e.recovered(cmd, card)
|
||||
return srDone()
|
||||
}
|
||||
if gateway.IsRateLimited(err) {
|
||||
// 本账户限流：请求未发出，恢复最近调用时间，短延迟重排，不计失败。
|
||||
e.restoreLastCall(ctx, card.ID, previousCall, params)
|
||||
return srRetry(time.Now().Add(stopResumeRateLimitedDelay))
|
||||
}
|
||||
classification := e.classify(ctx, cmd.Action, err)
|
||||
if result.Unknown || classification.MaybeAccepted {
|
||||
return e.unknown(card, cmd, err, params, settings)
|
||||
}
|
||||
message := truncateRunes(classification.Category+"|"+err.Error(), 300)
|
||||
switch classification.Category {
|
||||
case constants.GatewayClassAlreadyTarget:
|
||||
if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil {
|
||||
return e.unknown(card, cmd, confirmErr, params, settings)
|
||||
}
|
||||
e.recovered(cmd, card)
|
||||
return srDone()
|
||||
case constants.GatewayClassTerminal:
|
||||
// 终态不可操作：不重试、结束指令、写告警；停机方向上游已不可用时按目标状态视为已停。
|
||||
if cmd.Action == constants.StopResumeActionStop {
|
||||
if confirmErr := e.service.ConfirmCommandState(ctx, card, cmd.Action, reason); confirmErr != nil {
|
||||
e.logger.Warn("停机遇上游终态，回写已停失败", zap.Uint("card_id", card.ID), zap.Error(confirmErr))
|
||||
}
|
||||
}
|
||||
e.reportFailure(card, cmd, classification.Category, err.Error(), cmd.FailCount+1, now)
|
||||
return srOutcome{done: true, stat: constants.StopResumeStatFailed}
|
||||
case constants.GatewayClassRateLimitedUpstream:
|
||||
retry := srRetry(now.Add(maxDuration(params.SRMinInterval, settings.SRBackoffFor(1))))
|
||||
retry.set[carrierqueue.SRFieldError] = message
|
||||
return retry
|
||||
}
|
||||
failCount := cmd.FailCount + 1
|
||||
retry := srRetry(now.Add(settings.SRBackoffFor(failCount)))
|
||||
retry.set[carrierqueue.SRFieldFail] = strconv.Itoa(failCount)
|
||||
retry.set[carrierqueue.SRFieldError] = message
|
||||
firstFailed := cmd.FirstFailed
|
||||
if firstFailed.IsZero() {
|
||||
firstFailed = now
|
||||
retry.set[carrierqueue.SRFieldFirst] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||
}
|
||||
if failCount >= settings.SRAlertFailCount || settings.ImmediateAlert[classification.Category] {
|
||||
e.reportFailure(card, cmd, classification.Category, err.Error(), failCount, firstFailed)
|
||||
}
|
||||
e.logger.Warn("停复机指令执行失败，按退避重试", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action),
|
||||
zap.String("category", classification.Category), zap.Int("fail_count", failCount), zap.Time("next_at", retry.nextAt),
|
||||
zap.String("metric", "stop_resume.failed"), zap.Error(err))
|
||||
return retry
|
||||
}
|
||||
|
||||
// unknown 结果未知：置待确认，确认延迟后先判断再查状态；连续未知达阈值告警，指令不丢弃。
|
||||
func (e *StopResumeExecutor) unknown(card *model.IotCard, cmd carrierqueue.CommandState, err error, params carrierqueue.CarrierParams,
|
||||
settings carrierqueue.Settings) srOutcome {
|
||||
unknown := cmd.Unknown + 1
|
||||
retry := srRetry(time.Now().Add(params.SRConfirmDelay))
|
||||
retry.set[carrierqueue.SRFieldPending] = "1"
|
||||
retry.set[carrierqueue.SRFieldUnknown] = strconv.Itoa(unknown)
|
||||
retry.set[carrierqueue.SRFieldError] = truncateRunes(constants.StopResumeFailureUnknown+"|"+err.Error(), 300)
|
||||
if unknown >= settings.SRUnknownAlert {
|
||||
e.reportFailure(card, cmd, constants.StopResumeFailureUnknown, err.Error(), unknown, cmd.WrittenAt)
|
||||
}
|
||||
e.logger.Warn("停复机结果未知，确认延迟后查状态", zap.Uint("card_id", card.ID), zap.String("action", cmd.Action),
|
||||
zap.Int("unknown", unknown), zap.String("metric", "stop_resume.unknown"), zap.Error(err))
|
||||
return retry
|
||||
}
|
||||
|
||||
func (e *StopResumeExecutor) classify(ctx context.Context, action string, err error) gatewaylimit.Classification {
|
||||
gwErr, ok := gateway.AsGatewayError(err)
|
||||
if !ok || gwErr.Category != constants.GatewayErrorUpstream {
|
||||
return gatewaylimit.Classification{Category: constants.GatewayClassTransient}
|
||||
}
|
||||
op := constants.GatewayOpStop
|
||||
if action == constants.StopResumeActionStart {
|
||||
op = constants.GatewayOpStart
|
||||
}
|
||||
return e.classifier.Classify(ctx, op, gwErr.ProviderMessage)
|
||||
}
|
||||
|
||||
func (e *StopResumeExecutor) restoreLastCall(ctx context.Context, cardID uint, previous time.Time, params carrierqueue.CarrierParams) {
|
||||
if previous.IsZero() {
|
||||
e.store.Client().Del(ctx, constants.RedisCardStopResumeLastCallKey(cardID))
|
||||
return
|
||||
}
|
||||
_ = e.store.MarkStopResumeCall(ctx, cardID, previous, 2*params.SRMinInterval)
|
||||
}
|
||||
|
||||
func (e *StopResumeExecutor) reportFailure(card *model.IotCard, cmd carrierqueue.CommandState, category, message string, failCount int, firstFailed time.Time) {
|
||||
if firstFailed.IsZero() {
|
||||
firstFailed = time.Now()
|
||||
}
|
||||
e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
|
||||
CheckType: constants.CheckTypeStopResume, Category: category, Message: cmd.Action + ": " + message,
|
||||
FailCount: failCount, FirstFailed: firstFailed, At: time.Now()})
|
||||
}
|
||||
|
||||
func (e *StopResumeExecutor) recovered(cmd carrierqueue.CommandState, card *model.IotCard) {
|
||||
if card != nil && (cmd.FailCount > 0 || cmd.Unknown > 0) {
|
||||
e.alerts.ReportRecovered(card.ID, constants.CheckTypeStopResume)
|
||||
}
|
||||
}
|
||||
|
||||
func targetNetworkStatus(action string) int {
|
||||
if action == constants.StopResumeActionStop {
|
||||
return constants.NetworkStatusOffline
|
||||
}
|
||||
return constants.NetworkStatusOnline
|
||||
}
|
||||
|
||||
func maxDuration(a, b time.Duration) time.Duration {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// countOutcome 累加执行计数：结束为成功，置待确认为结果未知，带失败次数为失败，其余推后为重试。
|
||||
func (e *StopResumeExecutor) countOutcome(ctx context.Context, carrierID uint, outcome srOutcome) {
|
||||
if outcome.stat == statSkip {
|
||||
return
|
||||
}
|
||||
field := constants.StopResumeStatRetry
|
||||
switch {
|
||||
case outcome.stat != "":
|
||||
field = outcome.stat
|
||||
case outcome.done:
|
||||
field = constants.StopResumeStatSuccess
|
||||
case outcome.set[carrierqueue.SRFieldPending] == "1":
|
||||
field = constants.StopResumeStatUnknown
|
||||
case outcome.set[carrierqueue.SRFieldFail] != "":
|
||||
field = constants.StopResumeStatFailed
|
||||
}
|
||||
e.store.IncrStopResumeStat(ctx, carrierID, field)
|
||||
}
|
||||
Reference in New Issue
Block a user