feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭): - 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机 - 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制 - 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认 - 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后 - 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估 - 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令 - 迁移 000255 插入开关与告警、退避参数 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -51,6 +51,14 @@ func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logg
|
||||
DefaultValue: "600", Description: "账户级告警统计窗口(秒)", Min: &minOne, Max: &maxSeconds},
|
||||
{Key: constants.SystemConfigPollingAccountAlertThreshold, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "50", Description: "账户级告警阈值:窗口内同账户新增失败告警数", Min: &minOne, Max: &maxCount},
|
||||
{Key: constants.SystemConfigStopResumeQueueEnabled, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeBool,
|
||||
DefaultValue: "false", Description: "自动停复机经停复机指令队列执行(全局开关,关闭即各入口当场执行)"},
|
||||
{Key: constants.SystemConfigStopResumeUnknownAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "3", Description: "停复机连续结果未知告警阈值", Min: &minOne, Max: &maxCount},
|
||||
{Key: constants.SystemConfigStopResumeBackoff, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
|
||||
DefaultValue: `[600,900,1800,3600]`, Description: "停复机失败退避序列(秒,递增)", Validator: validateSequence},
|
||||
{Key: constants.SystemConfigStopResumeAlertFailCount, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "5", Description: "停复机连续失败告警阈值", Min: &minOne, Max: &maxCount},
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if _, exists := registry.Get(definition.Key); exists {
|
||||
@@ -107,6 +115,23 @@ func validateStepTimeout(raw string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateSequence 校验非空递增的正整数秒序列(停复机退避)。
|
||||
func validateSequence(raw string) error {
|
||||
var values []int
|
||||
if err := json.Unmarshal([]byte(raw), &values); err != nil || len(values) == 0 {
|
||||
return stderrors.New("退避序列必须是非空的秒数组")
|
||||
}
|
||||
for index, value := range values {
|
||||
if value <= 0 {
|
||||
return stderrors.New("退避间隔必须为正数")
|
||||
}
|
||||
if index > 0 && value <= values[index-1] {
|
||||
return stderrors.New("退避序列必须递增")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBackoff(raw string) error {
|
||||
var backoff map[string][]int
|
||||
if err := json.Unmarshal([]byte(raw), &backoff); err != nil {
|
||||
@@ -171,6 +196,28 @@ type Settings struct {
|
||||
ImmediateAlert map[string]bool
|
||||
AccountAlertWindow time.Duration
|
||||
AccountAlertLimit int
|
||||
|
||||
// 停复机指令队列(add-stop-resume-command-queue)。
|
||||
StopResumeQueueEnabled bool
|
||||
SRUnknownAlert int
|
||||
SRBackoff []time.Duration
|
||||
SRAlertFailCount int
|
||||
}
|
||||
|
||||
// SRBackoffFor 返回停复机第 n 次连续失败(n 从 1 起)的退避间隔;超过序列长度取最后一项。
|
||||
func (s Settings) SRBackoffFor(failCount int) time.Duration {
|
||||
sequence := s.SRBackoff
|
||||
if len(sequence) == 0 {
|
||||
sequence = []time.Duration{10 * time.Minute}
|
||||
}
|
||||
index := failCount - 1
|
||||
if index < 0 {
|
||||
index = 0
|
||||
}
|
||||
if index >= len(sequence) {
|
||||
index = len(sequence) - 1
|
||||
}
|
||||
return sequence[index]
|
||||
}
|
||||
|
||||
// CarrierMode 判断当前是否为按运营商账户调度。
|
||||
@@ -243,6 +290,9 @@ func DefaultSettings() Settings {
|
||||
AlertFailCount: map[string]int{"default": 5}, AlertFailDuration: 6 * time.Hour,
|
||||
ImmediateAlert: map[string]bool{constants.GatewayClassAccountConfig: true},
|
||||
AccountAlertWindow: 10 * time.Minute, AccountAlertLimit: 50,
|
||||
SRUnknownAlert: 3,
|
||||
SRBackoff: []time.Duration{10 * time.Minute, 15 * time.Minute, 30 * time.Minute, time.Hour},
|
||||
SRAlertFailCount: 5,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +414,8 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
{constants.SystemConfigPollingAlertFailDuration, func(v int) { s.AlertFailDuration = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingAccountAlertWindow, func(v int) { s.AccountAlertWindow = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingAccountAlertThreshold, func(v int) { s.AccountAlertLimit = v }},
|
||||
{constants.SystemConfigStopResumeUnknownAlert, func(v int) { s.SRUnknownAlert = v }},
|
||||
{constants.SystemConfigStopResumeAlertFailCount, func(v int) { s.SRAlertFailCount = v }},
|
||||
}
|
||||
for _, item := range ints {
|
||||
raw, err := get(item.key)
|
||||
@@ -376,6 +428,21 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
}
|
||||
item.set(value)
|
||||
}
|
||||
if raw, err = get(constants.SystemConfigStopResumeQueueEnabled); err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.StopResumeQueueEnabled = raw == "true"
|
||||
if raw, err = get(constants.SystemConfigStopResumeBackoff); err != nil {
|
||||
return s, err
|
||||
}
|
||||
var srBackoff []int
|
||||
if err := json.Unmarshal([]byte(raw), &srBackoff); err != nil || validateSequence(raw) != nil {
|
||||
return s, stderrors.New("停复机退避序列配置非法")
|
||||
}
|
||||
s.SRBackoff = nil
|
||||
for _, value := range srBackoff {
|
||||
s.SRBackoff = append(s.SRBackoff, time.Duration(value)*time.Second)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user