feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled

自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭):
- 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机
- 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制
- 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认
- 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后
- 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估
- 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令
- 迁移 000255 插入开关与告警、退避参数

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-26 00:36:26 +09:00
parent 1369fed642
commit 934a3a27a1
31 changed files with 2275 additions and 117 deletions

View File

@@ -51,6 +51,14 @@ func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logg
DefaultValue: "600", Description: "账户级告警统计窗口(秒)", Min: &minOne, Max: &maxSeconds},
{Key: constants.SystemConfigPollingAccountAlertThreshold, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "50", Description: "账户级告警阈值:窗口内同账户新增失败告警数", Min: &minOne, Max: &maxCount},
{Key: constants.SystemConfigStopResumeQueueEnabled, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeBool,
DefaultValue: "false", Description: "自动停复机经停复机指令队列执行(全局开关,关闭即各入口当场执行)"},
{Key: constants.SystemConfigStopResumeUnknownAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "3", Description: "停复机连续结果未知告警阈值", Min: &minOne, Max: &maxCount},
{Key: constants.SystemConfigStopResumeBackoff, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
DefaultValue: `[600,900,1800,3600]`, Description: "停复机失败退避序列(秒,递增)", Validator: validateSequence},
{Key: constants.SystemConfigStopResumeAlertFailCount, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "5", Description: "停复机连续失败告警阈值", Min: &minOne, Max: &maxCount},
}
for _, definition := range definitions {
if _, exists := registry.Get(definition.Key); exists {
@@ -107,6 +115,23 @@ func validateStepTimeout(raw string) error {
return nil
}
// validateSequence 校验非空递增的正整数秒序列(停复机退避)。
func validateSequence(raw string) error {
var values []int
if err := json.Unmarshal([]byte(raw), &values); err != nil || len(values) == 0 {
return stderrors.New("退避序列必须是非空的秒数组")
}
for index, value := range values {
if value <= 0 {
return stderrors.New("退避间隔必须为正数")
}
if index > 0 && value <= values[index-1] {
return stderrors.New("退避序列必须递增")
}
}
return nil
}
func validateBackoff(raw string) error {
var backoff map[string][]int
if err := json.Unmarshal([]byte(raw), &backoff); err != nil {
@@ -171,6 +196,28 @@ type Settings struct {
ImmediateAlert map[string]bool
AccountAlertWindow time.Duration
AccountAlertLimit int
// 停复机指令队列(add-stop-resume-command-queue)。
StopResumeQueueEnabled bool
SRUnknownAlert int
SRBackoff []time.Duration
SRAlertFailCount int
}
// SRBackoffFor 返回停复机第 n 次连续失败(n 从 1 起)的退避间隔;超过序列长度取最后一项。
func (s Settings) SRBackoffFor(failCount int) time.Duration {
sequence := s.SRBackoff
if len(sequence) == 0 {
sequence = []time.Duration{10 * time.Minute}
}
index := failCount - 1
if index < 0 {
index = 0
}
if index >= len(sequence) {
index = len(sequence) - 1
}
return sequence[index]
}
// CarrierMode 判断当前是否为按运营商账户调度。
@@ -243,6 +290,9 @@ func DefaultSettings() Settings {
AlertFailCount: map[string]int{"default": 5}, AlertFailDuration: 6 * time.Hour,
ImmediateAlert: map[string]bool{constants.GatewayClassAccountConfig: true},
AccountAlertWindow: 10 * time.Minute, AccountAlertLimit: 50,
SRUnknownAlert: 3,
SRBackoff: []time.Duration{10 * time.Minute, 15 * time.Minute, 30 * time.Minute, time.Hour},
SRAlertFailCount: 5,
}
}
@@ -364,6 +414,8 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
{constants.SystemConfigPollingAlertFailDuration, func(v int) { s.AlertFailDuration = time.Duration(v) * time.Second }},
{constants.SystemConfigPollingAccountAlertWindow, func(v int) { s.AccountAlertWindow = time.Duration(v) * time.Second }},
{constants.SystemConfigPollingAccountAlertThreshold, func(v int) { s.AccountAlertLimit = v }},
{constants.SystemConfigStopResumeUnknownAlert, func(v int) { s.SRUnknownAlert = v }},
{constants.SystemConfigStopResumeAlertFailCount, func(v int) { s.SRAlertFailCount = v }},
}
for _, item := range ints {
raw, err := get(item.key)
@@ -376,6 +428,21 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
}
item.set(value)
}
if raw, err = get(constants.SystemConfigStopResumeQueueEnabled); err != nil {
return s, err
}
s.StopResumeQueueEnabled = raw == "true"
if raw, err = get(constants.SystemConfigStopResumeBackoff); err != nil {
return s, err
}
var srBackoff []int
if err := json.Unmarshal([]byte(raw), &srBackoff); err != nil || validateSequence(raw) != nil {
return s, stderrors.New("停复机退避序列配置非法")
}
s.SRBackoff = nil
for _, value := range srBackoff {
s.SRBackoff = append(s.SRBackoff, time.Duration(value)*time.Second)
}
return s, nil
}