feat: 按运营商账户调度的迁移、配置与账户队列 Lua

- 迁移 000251~000254:运营商账户调度参数、请求检查场景(预置 8 个)、卡轮询失败告警表,以及调度模式、步骤、退避、告警等受控系统配置(默认 legacy,行为不变);并发配置表注释标记废弃
- 受控系统配置注册与 JSON 结构校验:步骤只能是 4 种检查类型且不重复、退避序列递增
- yaml 增加 NSQ 地址、msg_timeout、租约、续期间隔、分发间隔、默认执行上限与 max_in_flight;引入 go-nsq
- 测试环境 docker-compose 增加 nsqlookupd 与 nsqd,Worker 注入 NSQ 地址
- 账户队列 Redis 结构与 Lua:领取(执行上限、优先占比、按限流组剩余 QPS 估算)、抢执行权、续期、收尾、请求检查(合并、冷却、执行中打优先标记)、移出、发布失败释放
This commit is contained in:
邹进
2026-10-25 18:28:42 +09:00
parent f49baff7c2
commit dae44bed30
20 changed files with 1454 additions and 0 deletions

View File

@@ -29,6 +29,7 @@ import (
systemConfigInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
wecomInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/wecom"
pollingPkg "github.com/break/junhong_cmp_fiber/internal/polling"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
agentRechargeQuery "github.com/break/junhong_cmp_fiber/internal/query/agentrecharge"
assetQuery "github.com/break/junhong_cmp_fiber/internal/query/asset"
auditQuery "github.com/break/junhong_cmp_fiber/internal/query/audit"
@@ -123,6 +124,7 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers {
registerCarrierCallbackConfigDefinitions(systemConfigRegistry, deps.Logger)
registerPaymentMethodConfigDefinitions(systemConfigRegistry, deps.Logger)
gatewaylimit.RegisterConfigDefinitions(systemConfigRegistry, deps.Logger)
carrierqueue.RegisterConfigDefinitions(systemConfigRegistry, deps.Logger)
systemConfigAlerts := systemConfigInfra.NewLogAlertSink(deps.Logger)
var systemConfigCache systemConfigInfra.Cache
if deps.Redis != nil {

View File

@@ -0,0 +1,76 @@
package model
import (
"time"
"gorm.io/datatypes"
"gorm.io/gorm"
)
// CarrierPollingParam 运营商账户调度参数;未配置的账户没有记录,使用全局默认。
type CarrierPollingParam struct {
ID uint `gorm:"column:id;primaryKey" json:"id"`
CarrierID uint `gorm:"column:carrier_id;not null" json:"carrier_id"`
InflightLimit *int `gorm:"column:inflight_limit" json:"inflight_limit"`
CardMaxInFlight *int `gorm:"column:card_max_in_flight" json:"card_max_in_flight"`
StopResumeMaxInFlight *int `gorm:"column:stop_resume_max_in_flight" json:"stop_resume_max_in_flight"`
SRInflightLimit *int `gorm:"column:sr_inflight_limit" json:"sr_inflight_limit"`
PriorityMaxRatio *float64 `gorm:"column:priority_max_ratio" json:"priority_max_ratio"`
StopResumeMinInterval *int `gorm:"column:stop_resume_min_interval" json:"stop_resume_min_interval"`
StopResumeConfirmDelay *int `gorm:"column:stop_resume_confirm_delay" json:"stop_resume_confirm_delay"`
Creator uint `gorm:"column:creator;not null;default:0" json:"creator"`
Updater uint `gorm:"column:updater;not null;default:0" json:"updater"`
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index" json:"-"`
}
// TableName 返回运营商账户调度参数表名。
func (CarrierPollingParam) TableName() string {
return "tb_carrier_polling_param"
}
// PollingCheckScene 业务事件与读入口请求检查场景;场景由代码预置,后台只可修改检查类型、确认延迟与冷却。
type PollingCheckScene struct {
ID uint `gorm:"column:id;primaryKey" json:"id"`
Scene string `gorm:"column:scene;type:varchar(50);not null" json:"scene"`
SceneName string `gorm:"column:scene_name;type:varchar(100);not null" json:"scene_name"`
CheckTypes datatypes.JSON `gorm:"column:check_types;type:jsonb;not null" json:"check_types"`
ConfirmDelays datatypes.JSON `gorm:"column:confirm_delays;type:jsonb;not null" json:"confirm_delays"`
Cooldown *int `gorm:"column:cooldown" json:"cooldown"`
Updater uint `gorm:"column:updater;not null;default:0" json:"updater"`
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
}
// TableName 返回请求检查场景表名。
func (PollingCheckScene) TableName() string {
return "tb_polling_check_scene"
}
// PollingFailureAlert 卡轮询失败告警;同一卡同一检查类型只保留一条未处理告警。
type PollingFailureAlert struct {
ID uint `gorm:"column:id;primaryKey" json:"id"`
AlertLevel string `gorm:"column:alert_level;type:varchar(20);not null;default:card" json:"alert_level"`
CardID uint `gorm:"column:card_id;not null;default:0" json:"card_id"`
ICCID string `gorm:"column:iccid;type:varchar(30);not null;default:''" json:"iccid"`
CarrierID uint `gorm:"column:carrier_id;not null;default:0" json:"carrier_id"`
CheckType string `gorm:"column:check_type;type:varchar(30);not null" json:"check_type"`
ErrorCategory string `gorm:"column:error_category;type:varchar(50);not null;default:''" json:"error_category"`
FailCount int `gorm:"column:fail_count;not null;default:0" json:"fail_count"`
FirstFailedAt *time.Time `gorm:"column:first_failed_at" json:"first_failed_at"`
LastFailedAt *time.Time `gorm:"column:last_failed_at" json:"last_failed_at"`
LastError string `gorm:"column:last_error;type:varchar(500);not null;default:''" json:"last_error"`
Status int `gorm:"column:status;type:smallint;not null;default:1" json:"status"`
HandledBy uint `gorm:"column:handled_by;not null;default:0" json:"handled_by"`
HandledAt *time.Time `gorm:"column:handled_at" json:"handled_at"`
HandleRemark string `gorm:"column:handle_remark;type:varchar(500);not null;default:''" json:"handle_remark"`
RecoveredAt *time.Time `gorm:"column:recovered_at" json:"recovered_at"`
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
}
// TableName 返回卡轮询失败告警表名。
func (PollingFailureAlert) TableName() string {
return "tb_polling_failure_alert"
}

View File

@@ -0,0 +1,247 @@
package carrierqueue
import "github.com/redis/go-redis/v9"
// 卡到期 Hash 字段前缀(见 constants.RedisCardDueKey)。
const (
fieldDue = "due:" // 各检查类型下次到期(毫秒)
fieldOK = "ok:" // 最近一次成功(毫秒),用于冷却判断
fieldFail = "fail:" // 连续失败次数
fieldFirstFail = "ffail:" // 首次失败(毫秒)
fieldError = "err:" // 最近错误(类别|信息)
fieldConfirm = "cfm:" // 待执行的延迟确认序列(秒,逗号分隔)
fieldExpected = "exp:" // 延迟确认的期望结果(如 offline / online),满足时取消后续确认
fieldPriority = "prio" // 优先标记:1 表示应在优先队列(执行中收到请求检查时由收尾移入优先队列)
fieldCarrier = "carrier"
)
// claimLua 账户领取(分发器抢到账户分发锁后执行)。
//
// KEYS[1] 优先队列;KEYS[2] 普通队列;KEYS[3] 执行中计数。
// ARGV:now_ms、lease_ms、执行上限、优先占比(0 表示不限)、token 前缀、
// 限流组本秒剩余("g=n;g=n",空串表示不按 QPS 限制)、检查类型所属限流组("type=g;...")、是否计入实名(1/0)。
// 返回 {本轮清理的过期执行数, card_id, token, 是否来自优先(1/0), ...}。
const claimLua = `
local now = tonumber(ARGV[1])
local lease = tonumber(ARGV[2])
local limit = tonumber(ARGV[3])
local ratio = tonumber(ARGV[4])
local prefix = ARGV[5]
local unlimited = ARGV[6] == ''
local countRealname = ARGV[8] == '1'
local expired = redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now)
local result = {expired}
local capacity = limit - redis.call('ZCARD', KEYS[3])
if capacity <= 0 then
return result
end
local budget = {}
for g, n in string.gmatch(ARGV[6], '([^=;]+)=(%-?%d+)') do budget[g] = tonumber(n) end
local typeGroup = {}
for t, g in string.gmatch(ARGV[7], '([^=;]+)=([^;]+)') do typeGroup[t] = g end
local gatewayTypes = {'card_status', 'carddata', 'realname'}
local used = {}
local taken = 0
local stop = false
local function tryClaim(cardId, fromPriority, qkey)
local dueKey = 'polling:card:' .. cardId .. ':due'
local need = {}
if not unlimited then
for _, t in ipairs(gatewayTypes) do
if t ~= 'realname' or countRealname then
local d = tonumber(redis.call('HGET', dueKey, 'due:' .. t) or '-1')
if d >= 0 and d <= now then
local g = typeGroup[t]
if g then need[g] = (need[g] or 0) + 1 end
end
end
end
for g, n in pairs(need) do
if budget[g] ~= nil and (used[g] or 0) + n > budget[g] then
stop = true
return false
end
end
end
local claimKey = 'polling:card:' .. cardId .. ':claim'
local token = prefix .. ':' .. cardId
local ok = redis.call('SET', claimKey, token, 'NX', 'PX', lease)
if not ok then
local ttl = redis.call('PTTL', claimKey)
if ttl < 0 then ttl = lease end
redis.call('ZADD', qkey, 'XX', now + ttl, cardId)
return false
end
for g, n in pairs(need) do used[g] = (used[g] or 0) + n end
redis.call('ZADD', qkey, 'XX', now + lease, cardId)
redis.call('ZADD', KEYS[3], now + lease, token)
if fromPriority then
redis.call('HDEL', dueKey, 'prio')
end
taken = taken + 1
table.insert(result, cardId)
table.insert(result, token)
table.insert(result, fromPriority and '1' or '0')
return true
end
local function drain(qkey, fromPriority, quota)
if quota <= 0 or stop then return end
local ids = redis.call('ZRANGEBYSCORE', qkey, '-inf', now, 'LIMIT', 0, quota * 2 + 10)
for _, id in ipairs(ids) do
if stop or quota <= 0 or taken >= capacity then return end
if tryClaim(id, fromPriority, qkey) then quota = quota - 1 end
end
end
local priorityQuota = capacity
if ratio > 0 then
priorityQuota = math.floor(capacity * ratio)
if priorityQuota < 1 then priorityQuota = 1 end
end
drain(KEYS[1], true, priorityQuota)
drain(KEYS[2], false, capacity - taken)
drain(KEYS[1], true, capacity - taken)
return result
`
// renewLua 续期:token 与执行权一致时延长租约与执行权,并推后 score 与执行中计数到期时间。
//
// KEYS[1] claim;KEYS[2] exec;KEYS[3] 执行中计数;KEYS[4] 优先队列;KEYS[5] 普通队列。
// ARGV:token、执行权值、lease_ms、exec_ttl_ms、now_ms、card_id。返回 1 成功,0 token 或执行权已不一致。
const renewLua = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
if redis.call('GET', KEYS[2]) ~= ARGV[2] then return 0 end
local lease = tonumber(ARGV[3])
local now = tonumber(ARGV[5])
redis.call('PEXPIRE', KEYS[1], lease)
redis.call('PEXPIRE', KEYS[2], tonumber(ARGV[4]))
redis.call('ZADD', KEYS[3], 'XX', now + lease, ARGV[1])
redis.call('ZADD', KEYS[4], 'XX', now + lease, ARGV[6])
redis.call('ZADD', KEYS[5], 'XX', now + lease, ARGV[6])
return 1
`
// acquireExecLua 抢执行权:claim 等于 token 时 SET exec NX PX。返回 1 抢到,0 claim 不一致,-1 已有执行者。
//
// KEYS[1] claim;KEYS[2] exec。ARGV:token、执行权值、exec_ttl_ms。
const acquireExecLua = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
if redis.call('SET', KEYS[2], ARGV[2], 'NX', 'PX', tonumber(ARGV[3])) then return 1 end
return -1
`
// finishLua 收尾:写回业务结果;token 一致时删除处理中标记与执行权、移出执行中计数,
// 按优先标记把卡放入优先或普通队列,score = 各检查类型最早到期。卡已不在任一队列(已移出轮询)时不重新加入。
//
// KEYS[1] due;KEYS[2] claim;KEYS[3] exec;KEYS[4] 执行中计数;KEYS[5] 优先队列;KEYS[6] 普通队列。
// ARGV:token、card_id、now_ms、set 字段对数 n、字段1、值1、…、删除字段…。
// 返回 1 正常收尾,0 token 不一致(只写了业务结果)。
const finishLua = `
local n = tonumber(ARGV[4])
local idx = 5
for i = 1, n do
redis.call('HSET', KEYS[1], ARGV[idx], ARGV[idx + 1])
idx = idx + 2
end
while idx <= #ARGV do
redis.call('HDEL', KEYS[1], ARGV[idx])
idx = idx + 1
end
if redis.call('GET', KEYS[2]) ~= ARGV[1] then return 0 end
redis.call('DEL', KEYS[2])
redis.call('DEL', KEYS[3])
redis.call('ZREM', KEYS[4], ARGV[1])
local cardId = ARGV[2]
local inPriority = redis.call('ZSCORE', KEYS[5], cardId)
local inQueue = redis.call('ZSCORE', KEYS[6], cardId)
if not inPriority and not inQueue then return 1 end
local fields = redis.call('HGETALL', KEYS[1])
local nextDue = nil
for i = 1, #fields, 2 do
if string.sub(fields[i], 1, 4) == 'due:' then
local v = tonumber(fields[i + 1])
if v and (nextDue == nil or v < nextDue) then nextDue = v end
end
end
if nextDue == nil then nextDue = tonumber(ARGV[3]) + 3600000 end
if redis.call('HGET', KEYS[1], 'prio') == '1' then
redis.call('ZREM', KEYS[6], cardId)
redis.call('ZADD', KEYS[5], nextDue, cardId)
else
redis.call('ZREM', KEYS[5], cardId)
redis.call('ZADD', KEYS[6], nextDue, cardId)
end
return 1
`
// requestCheckLua 请求检查:把指定检查类型设为立即到期并放入优先队列;卡执行中时只打优先标记,收尾后立即再执行。
// 只作用于仍在轮询中的卡(在优先或普通队列中)。冷却期内(最近成功距今未超过冷却)的类型不请求。
//
// KEYS[1] due;KEYS[2] 优先队列;KEYS[3] 普通队列;KEYS[4] claim。
// ARGV:now_ms、card_id、cooldown_ms(0 不冷却)、确认延迟(秒,逗号分隔,可空)、期望结果(可空)、类型…。
// 返回本次实际请求的类型数;卡不在轮询中返回 -1。
const requestCheckLua = `
local now = tonumber(ARGV[1])
local cardId = ARGV[2]
local cooldown = tonumber(ARGV[3])
local inPriority = redis.call('ZSCORE', KEYS[2], cardId)
local inQueue = redis.call('ZSCORE', KEYS[3], cardId)
if not inPriority and not inQueue then return -1 end
local requested = 0
for i = 6, #ARGV do
local t = ARGV[i]
local skip = false
if cooldown > 0 then
local ok = tonumber(redis.call('HGET', KEYS[1], 'ok:' .. t) or '0')
if ok > 0 and now - ok < cooldown then skip = true end
end
if not skip then
redis.call('HSET', KEYS[1], 'due:' .. t, now)
if ARGV[4] ~= '' then
redis.call('HSET', KEYS[1], 'cfm:' .. t, ARGV[4])
if ARGV[5] ~= '' then redis.call('HSET', KEYS[1], 'exp:' .. t, ARGV[5]) else redis.call('HDEL', KEYS[1], 'exp:' .. t) end
end
requested = requested + 1
end
end
if requested == 0 then return 0 end
redis.call('HSET', KEYS[1], 'prio', '1')
if redis.call('EXISTS', KEYS[4]) == 1 then return requested end
redis.call('ZREM', KEYS[3], cardId)
redis.call('ZADD', KEYS[2], now, cardId)
return requested
`
// removeLua 把卡移出所属账户队列(关闭轮询、删除卡等);执行中的卡收尾时因不在队列中而不会被重新加入。
// KEYS[1] 优先队列;KEYS[2] 普通队列;KEYS[3] due。ARGV:card_id。
const removeLua = `
redis.call('ZREM', KEYS[1], ARGV[1])
redis.call('ZREM', KEYS[2], ARGV[1])
redis.call('DEL', KEYS[3])
return 1
`
// releaseLua 发布失败时按 token 删除处理中标记、移出执行中计数,并把 score 恢复为当前时间。
// KEYS[1] claim;KEYS[2] 执行中计数;KEYS[3] 优先队列;KEYS[4] 普通队列。ARGV:token、card_id、now_ms。
const releaseLua = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
redis.call('DEL', KEYS[1])
redis.call('ZREM', KEYS[2], ARGV[1])
redis.call('ZADD', KEYS[3], 'XX', tonumber(ARGV[3]), ARGV[2])
redis.call('ZADD', KEYS[4], 'XX', tonumber(ARGV[3]), ARGV[2])
return 1
`
var (
claimScript = redis.NewScript(claimLua)
renewScript = redis.NewScript(renewLua)
acquireExecScript = redis.NewScript(acquireExecLua)
finishScript = redis.NewScript(finishLua)
requestCheckScript = redis.NewScript(requestCheckLua)
removeScript = redis.NewScript(removeLua)
releaseScript = redis.NewScript(releaseLua)
)

View File

@@ -0,0 +1,378 @@
// Package carrierqueue 实现按运营商账户调度的卡轮询结构:账户延时队列、卡到期信息、领取 / 续期 / 收尾 / 请求检查 Lua、
// 分发器、初始化与受控系统配置。Redis 是唯一权威状态,NSQ 只做传输(add-carrier-polling-scheduler)。
package carrierqueue
import (
"context"
"encoding/json"
stderrors "errors"
"strconv"
"sync"
"time"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// settingsTTL 是调度参数在进程内的缓存时长;后台修改数秒内生效。
const settingsTTL = 5 * time.Second
// RegisterConfigDefinitions 注册按运营商账户调度的受控系统配置定义(含 JSON 结构校验)。
func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logger) {
if registry == nil {
return
}
minOne, maxSeconds, maxCount := int64(1), int64(86400*7), int64(100000)
definitions := []systemconfig.Definition{
{Key: constants.SystemConfigPollingSchedulerMode, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeString,
DefaultValue: constants.PollingSchedulerModeLegacy, Description: "卡轮询调度模式:legacy 旧分片调度,carrier 按运营商账户调度(整体切换)",
EnumValues: []string{constants.PollingSchedulerModeLegacy, constants.PollingSchedulerModeCarrier}},
{Key: constants.SystemConfigPollingStepOrder, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
DefaultValue: `["card_status","carddata","realname","package"]`, Description: "卡任务步骤顺序", Validator: validateStepOrder},
{Key: constants.SystemConfigPollingStepTimeout, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
DefaultValue: `{"card_status":15,"carddata":15,"realname":15,"package":20}`, Description: "各检查类型单步超时(秒)", Validator: validateStepTimeout},
{Key: constants.SystemConfigPollingTaskTimeoutMargin, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "10", Description: "卡任务总超时余量(秒)", Min: &minOne, Max: &maxSeconds},
{Key: constants.SystemConfigPollingSkipPackageAfterFlow, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeBool,
DefaultValue: "true", Description: "同轮流量步骤已触发停复机评估时跳过套餐步骤"},
{Key: constants.SystemConfigPollingFailureBackoff, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
DefaultValue: `{"default":[60,300,900,1800,3600]}`, Description: "失败退避序列(秒,递增),可按检查类型覆盖 default", Validator: validateBackoff},
{Key: constants.SystemConfigPollingAlertFailCount, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
DefaultValue: `{"default":5}`, Description: "连续失败告警阈值,可按检查类型覆盖 default", Validator: validateAlertCount},
{Key: constants.SystemConfigPollingAlertFailDuration, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "21600", Description: "持续失败时长告警阈值(秒)", Min: &minOne, Max: &maxSeconds},
{Key: constants.SystemConfigPollingImmediateAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
DefaultValue: `["account_config"]`, Description: "首次出现即告警的错误类别", Validator: validateCategories},
{Key: constants.SystemConfigPollingAccountAlertWindow, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "600", Description: "账户级告警统计窗口(秒)", Min: &minOne, Max: &maxSeconds},
{Key: constants.SystemConfigPollingAccountAlertThreshold, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
DefaultValue: "50", Description: "账户级告警阈值:窗口内同账户新增失败告警数", Min: &minOne, Max: &maxCount},
}
for _, definition := range definitions {
if _, exists := registry.Get(definition.Key); exists {
continue
}
if err := registry.Register(definition); err != nil && logger != nil {
logger.Error("注册按运营商账户调度系统配置失败,将使用代码默认值", zap.String("config_key", definition.Key), zap.Error(err))
}
}
}
func validCheckType(value string) bool {
switch value {
case constants.CheckTypeCardStatus, constants.CheckTypeCarddata, constants.CheckTypeRealname, constants.CheckTypePackage:
return true
}
return false
}
func validateStepOrder(raw string) error {
var steps []string
if err := json.Unmarshal([]byte(raw), &steps); err != nil {
return stderrors.New("步骤顺序必须是字符串数组")
}
if len(steps) == 0 {
return stderrors.New("步骤顺序不能为空")
}
seen := map[string]bool{}
for _, step := range steps {
if !validCheckType(step) {
return stderrors.New("步骤只能是 card_status、carddata、realname、package")
}
if seen[step] {
return stderrors.New("步骤不能重复")
}
seen[step] = true
}
return nil
}
func validateStepTimeout(raw string) error {
var timeouts map[string]int
if err := json.Unmarshal([]byte(raw), &timeouts); err != nil {
return stderrors.New("单步超时必须是 {检查类型: 秒} 对象")
}
for key, value := range timeouts {
if !validCheckType(key) {
return stderrors.New("单步超时的检查类型非法")
}
if value <= 0 || value > 600 {
return stderrors.New("单步超时必须在 1~600 秒之间")
}
}
return nil
}
func validateBackoff(raw string) error {
var backoff map[string][]int
if err := json.Unmarshal([]byte(raw), &backoff); err != nil {
return stderrors.New("退避序列必须是 {default 或检查类型: [秒...]} 对象")
}
if len(backoff["default"]) == 0 {
return stderrors.New("退避序列必须包含非空的 default")
}
for key, values := range backoff {
if key != "default" && !validCheckType(key) {
return stderrors.New("退避序列的检查类型非法")
}
for index, value := range values {
if value <= 0 {
return stderrors.New("退避间隔必须为正数")
}
if index > 0 && value <= values[index-1] {
return stderrors.New("退避序列必须递增")
}
}
}
return nil
}
func validateAlertCount(raw string) error {
var counts map[string]int
if err := json.Unmarshal([]byte(raw), &counts); err != nil {
return stderrors.New("告警阈值必须是 {default 或检查类型: 次数} 对象")
}
if counts["default"] <= 0 {
return stderrors.New("告警阈值必须包含正数 default")
}
for key, value := range counts {
if key != "default" && !validCheckType(key) {
return stderrors.New("告警阈值的检查类型非法")
}
if value <= 0 {
return stderrors.New("告警阈值必须为正数")
}
}
return nil
}
func validateCategories(raw string) error {
var categories []string
if err := json.Unmarshal([]byte(raw), &categories); err != nil {
return stderrors.New("错误类别必须是字符串数组")
}
return nil
}
// Settings 是按运营商账户调度的运行参数快照。
type Settings struct {
Mode string
StepOrder []string
StepTimeouts map[string]time.Duration
TaskTimeoutMargin time.Duration
SkipPackageAfterFlow bool
Backoff map[string][]time.Duration
AlertFailCount map[string]int
AlertFailDuration time.Duration
ImmediateAlert map[string]bool
AccountAlertWindow time.Duration
AccountAlertLimit int
}
// CarrierMode 判断当前是否为按运营商账户调度。
func (s Settings) CarrierMode() bool {
return s.Mode == constants.PollingSchedulerModeCarrier
}
// StepTimeout 返回检查类型单步超时;未配置时 15 秒。
func (s Settings) StepTimeout(checkType string) time.Duration {
if value, ok := s.StepTimeouts[checkType]; ok && value > 0 {
return value
}
return 15 * time.Second
}
// TaskTimeout 返回卡任务总超时 = 各步骤超时之和 + 余量。
func (s Settings) TaskTimeout() time.Duration {
total := s.TaskTimeoutMargin
for _, step := range s.StepOrder {
total += s.StepTimeout(step)
}
return total
}
// BackoffFor 返回检查类型第 n 次连续失败(n 从 1 起)的退避间隔,不超过 maxInterval(> 0 时)。
func (s Settings) BackoffFor(checkType string, failCount int, maxInterval time.Duration) time.Duration {
sequence := s.Backoff[checkType]
if len(sequence) == 0 {
sequence = s.Backoff["default"]
}
if len(sequence) == 0 {
sequence = []time.Duration{time.Minute}
}
index := failCount - 1
if index < 0 {
index = 0
}
if index >= len(sequence) {
index = len(sequence) - 1
}
delay := sequence[index]
if maxInterval > 0 && delay > maxInterval {
delay = maxInterval
}
return delay
}
// AlertThreshold 返回检查类型连续失败告警阈值。
func (s Settings) AlertThreshold(checkType string) int {
if value, ok := s.AlertFailCount[checkType]; ok && value > 0 {
return value
}
if value := s.AlertFailCount["default"]; value > 0 {
return value
}
return 5
}
// DefaultSettings 是配置读取失败时的安全默认值(旧分片调度)。
func DefaultSettings() Settings {
return Settings{
Mode: constants.PollingSchedulerModeLegacy,
StepOrder: constants.AllCheckTypes(),
StepTimeouts: map[string]time.Duration{
constants.CheckTypeCardStatus: 15 * time.Second, constants.CheckTypeCarddata: 15 * time.Second,
constants.CheckTypeRealname: 15 * time.Second, constants.CheckTypePackage: 20 * time.Second,
},
TaskTimeoutMargin: 10 * time.Second, SkipPackageAfterFlow: true,
Backoff: map[string][]time.Duration{"default": {time.Minute, 5 * time.Minute, 15 * time.Minute, 30 * time.Minute, time.Hour}},
AlertFailCount: map[string]int{"default": 5}, AlertFailDuration: 6 * time.Hour,
ImmediateAlert: map[string]bool{constants.GatewayClassAccountConfig: true},
AccountAlertWindow: 10 * time.Minute, AccountAlertLimit: 50,
}
}
// ConfigReader 读取已注册的受控系统配置。
type ConfigReader interface {
Get(ctx context.Context, key string) (string, error)
}
// SettingsProvider 在进程内缓存调度参数,超过 TTL 重新读取。
type SettingsProvider struct {
reader ConfigReader
logger *zap.Logger
mu sync.Mutex
cached Settings
loaded time.Time
}
// NewSettingsProvider 创建调度参数提供者;reader 为 nil 时始终返回默认值。
func NewSettingsProvider(reader ConfigReader, logger *zap.Logger) *SettingsProvider {
if logger == nil {
logger = zap.NewNop()
}
return &SettingsProvider{reader: reader, logger: logger, cached: DefaultSettings()}
}
// Current 返回当前调度参数;读取失败时保留上一次成功值。
func (p *SettingsProvider) Current(ctx context.Context) Settings {
if p == nil {
return DefaultSettings()
}
p.mu.Lock()
defer p.mu.Unlock()
if p.reader == nil || time.Since(p.loaded) < settingsTTL {
return p.cached
}
next, err := p.load(ctx)
p.loaded = time.Now()
if err != nil {
p.logger.Warn("读取按运营商账户调度配置失败,沿用上一次配置", zap.Error(err))
return p.cached
}
p.cached = next
return next
}
// Invalidate 使缓存立即失效(后台保存后调用)。
func (p *SettingsProvider) Invalidate() {
if p == nil {
return
}
p.mu.Lock()
p.loaded = time.Time{}
p.mu.Unlock()
}
func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
s := DefaultSettings()
get := func(key string) (string, error) { return p.reader.Get(ctx, key) }
raw, err := get(constants.SystemConfigPollingSchedulerMode)
if err != nil {
return s, err
}
s.Mode = raw
if raw, err = get(constants.SystemConfigPollingStepOrder); err != nil {
return s, err
}
if err := json.Unmarshal([]byte(raw), &s.StepOrder); err != nil || validateStepOrder(raw) != nil {
return s, stderrors.New("步骤顺序配置非法")
}
if raw, err = get(constants.SystemConfigPollingStepTimeout); err != nil {
return s, err
}
var timeouts map[string]int
if err := json.Unmarshal([]byte(raw), &timeouts); err != nil {
return s, stderrors.New("单步超时配置非法")
}
for key, value := range timeouts {
s.StepTimeouts[key] = time.Duration(value) * time.Second
}
if raw, err = get(constants.SystemConfigPollingFailureBackoff); err != nil {
return s, err
}
var backoff map[string][]int
if err := json.Unmarshal([]byte(raw), &backoff); err != nil {
return s, stderrors.New("退避序列配置非法")
}
s.Backoff = map[string][]time.Duration{}
for key, values := range backoff {
for _, value := range values {
s.Backoff[key] = append(s.Backoff[key], time.Duration(value)*time.Second)
}
}
if raw, err = get(constants.SystemConfigPollingAlertFailCount); err != nil {
return s, err
}
if err := json.Unmarshal([]byte(raw), &s.AlertFailCount); err != nil {
return s, stderrors.New("告警阈值配置非法")
}
if raw, err = get(constants.SystemConfigPollingImmediateAlert); err != nil {
return s, err
}
var categories []string
if err := json.Unmarshal([]byte(raw), &categories); err != nil {
return s, stderrors.New("首次告警类别配置非法")
}
s.ImmediateAlert = map[string]bool{}
for _, category := range categories {
s.ImmediateAlert[category] = true
}
if raw, err = get(constants.SystemConfigPollingSkipPackageAfterFlow); err != nil {
return s, err
}
s.SkipPackageAfterFlow = raw == "true"
ints := []struct {
key string
set func(int)
}{
{constants.SystemConfigPollingTaskTimeoutMargin, func(v int) { s.TaskTimeoutMargin = time.Duration(v) * time.Second }},
{constants.SystemConfigPollingAlertFailDuration, func(v int) { s.AlertFailDuration = time.Duration(v) * time.Second }},
{constants.SystemConfigPollingAccountAlertWindow, func(v int) { s.AccountAlertWindow = time.Duration(v) * time.Second }},
{constants.SystemConfigPollingAccountAlertThreshold, func(v int) { s.AccountAlertLimit = v }},
}
for _, item := range ints {
raw, err := get(item.key)
if err != nil {
return s, err
}
value, convErr := strconv.Atoi(raw)
if convErr != nil {
return s, stderrors.New("调度整数配置非法")
}
item.set(value)
}
return s, nil
}

View File

@@ -0,0 +1,278 @@
package carrierqueue
import (
"context"
stderrors "errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// Store 封装按运营商账户调度的 Redis 结构与 Lua 原子操作。
type Store struct {
redis *redis.Client
}
// NewStore 创建账户队列存储。
func NewStore(redisClient *redis.Client) *Store {
return &Store{redis: redisClient}
}
// Client 返回底层 Redis 客户端。
func (s *Store) Client() *redis.Client {
return s.redis
}
// ClaimInput 是一次账户领取的参数。
type ClaimInput struct {
CarrierID uint
Lease time.Duration
InflightLimit int
PriorityRatio float64 // 0 表示不限
TokenPrefix string // 本轮派发 token 前缀(实例 + 轮次),token = 前缀:card_id
GroupBudget map[string]int // 限流组本秒剩余;nil 表示不按 QPS 限制
TypeGroup map[string]string // 检查类型 → 限流组
CountRealname bool // 预计调用数是否计入实名(账户未开实名管控时不计)
}
// Claimed 是一张被领取的卡。
type Claimed struct {
CardID uint
Token string
FromPriority bool
}
// Claim 执行一次账户领取,返回领取到的卡与本轮清理的过期执行数(处理中崩溃或超时次数)。
func (s *Store) Claim(ctx context.Context, input ClaimInput) ([]Claimed, int64, error) {
budget := ""
if input.GroupBudget != nil {
parts := make([]string, 0, len(input.GroupBudget))
for group, remaining := range input.GroupBudget {
parts = append(parts, group+"="+strconv.Itoa(remaining))
}
budget = strings.Join(parts, ";")
if budget == "" {
budget = "none=0"
}
}
typeParts := make([]string, 0, len(input.TypeGroup))
for checkType, group := range input.TypeGroup {
typeParts = append(typeParts, checkType+"="+group)
}
countRealname := "0"
if input.CountRealname {
countRealname = "1"
}
raw, err := claimScript.Run(ctx, s.redis,
[]string{constants.RedisCarrierPriorityQueueKey(input.CarrierID), constants.RedisCarrierQueueKey(input.CarrierID), constants.RedisCarrierRunningKey(input.CarrierID)},
time.Now().UnixMilli(), input.Lease.Milliseconds(), input.InflightLimit, strconv.FormatFloat(input.PriorityRatio, 'f', 3, 64),
input.TokenPrefix, budget, strings.Join(typeParts, ";"), countRealname,
).Slice()
if err != nil {
return nil, 0, err
}
if len(raw) == 0 {
return nil, 0, nil
}
expired, _ := raw[0].(int64)
claimed := make([]Claimed, 0, (len(raw)-1)/3)
for index := 1; index+2 < len(raw); index += 3 {
cardID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64)
claimed = append(claimed, Claimed{CardID: uint(cardID), Token: fmt.Sprint(raw[index+1]), FromPriority: fmt.Sprint(raw[index+2]) == "1"})
}
return claimed, expired, nil
}
// AcquireExec 抢执行权:返回 1 抢到,0 处理中标记已不是该 token(过期或重复投递),-1 该 token 已有执行者。
func (s *Store) AcquireExec(ctx context.Context, cardID uint, token, execValue string, ttl time.Duration) (int64, error) {
return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisCardClaimKey(cardID), constants.RedisCardExecKey(cardID)},
token, execValue, ttl.Milliseconds()).Int64()
}
// Renew 续期租约与执行权;返回 false 表示 token 或执行权已不一致,调用方必须停止后续步骤。
func (s *Store) Renew(ctx context.Context, cardID, carrierID uint, token, execValue string, lease, execTTL time.Duration) (bool, error) {
result, err := renewScript.Run(ctx, s.redis,
[]string{constants.RedisCardClaimKey(cardID), constants.RedisCardExecKey(cardID), constants.RedisCarrierRunningKey(carrierID),
constants.RedisCarrierPriorityQueueKey(carrierID), constants.RedisCarrierQueueKey(carrierID)},
token, execValue, lease.Milliseconds(), execTTL.Milliseconds(), time.Now().UnixMilli(), cardID).Int64()
return result == 1, err
}
// Finish 收尾:写回业务结果(set 字段与删除字段),token 一致时释放处理中标记并按最早到期设置 score。
// 返回 false 表示 token 已不一致,只写入了业务结果。
func (s *Store) Finish(ctx context.Context, cardID, carrierID uint, token string, set map[string]string, del []string) (bool, error) {
args := []any{token, cardID, time.Now().UnixMilli(), len(set)}
for field, value := range set {
args = append(args, field, value)
}
for _, field := range del {
args = append(args, field)
}
result, err := finishScript.Run(ctx, s.redis,
[]string{constants.RedisCardDueKey(cardID), constants.RedisCardClaimKey(cardID), constants.RedisCardExecKey(cardID),
constants.RedisCarrierRunningKey(carrierID), constants.RedisCarrierPriorityQueueKey(carrierID), constants.RedisCarrierQueueKey(carrierID)},
args...).Int64()
return result == 1, err
}
// Release 发布失败时按 token 删除处理中标记并把 score 恢复为当前时间。
func (s *Store) Release(ctx context.Context, cardID, carrierID uint, token string) error {
return releaseScript.Run(ctx, s.redis,
[]string{constants.RedisCardClaimKey(cardID), constants.RedisCarrierRunningKey(carrierID),
constants.RedisCarrierPriorityQueueKey(carrierID), constants.RedisCarrierQueueKey(carrierID)},
token, cardID, time.Now().UnixMilli()).Err()
}
// CheckRequest 是一次请求检查。
type CheckRequest struct {
CardID uint
CarrierID uint
CheckTypes []string
Cooldown time.Duration
ConfirmDelays []time.Duration
Expected string // 延迟确认的期望结果(card_status 为 online / offline),满足时取消后续确认
}
// RequestCheck 把卡的指定检查类型设为立即到期放入优先队列;返回实际请求的类型数,卡不在轮询中返回 -1。
func (s *Store) RequestCheck(ctx context.Context, request CheckRequest) (int64, error) {
if len(request.CheckTypes) == 0 {
return 0, nil
}
confirm := make([]string, 0, len(request.ConfirmDelays))
for _, delay := range request.ConfirmDelays {
confirm = append(confirm, strconv.FormatInt(int64(delay/time.Second), 10))
}
args := []any{time.Now().UnixMilli(), request.CardID, request.Cooldown.Milliseconds(), strings.Join(confirm, ","), request.Expected}
for _, checkType := range request.CheckTypes {
args = append(args, checkType)
}
return requestCheckScript.Run(ctx, s.redis,
[]string{constants.RedisCardDueKey(request.CardID), constants.RedisCarrierPriorityQueueKey(request.CarrierID),
constants.RedisCarrierQueueKey(request.CarrierID), constants.RedisCardClaimKey(request.CardID)},
args...).Int64()
}
// Remove 把卡移出所属账户队列并删除到期信息(关闭轮询、删除卡)。
func (s *Store) Remove(ctx context.Context, cardID, carrierID uint) error {
return removeScript.Run(ctx, s.redis,
[]string{constants.RedisCarrierPriorityQueueKey(carrierID), constants.RedisCarrierQueueKey(carrierID), constants.RedisCardDueKey(cardID)},
cardID).Err()
}
// Upsert 写入卡各检查类型到期时间并按最早到期放入账户队列(已在优先队列中的保留在优先队列)。
// 卡所属账户变化时先移出旧账户队列。dues 为空表示该卡没有任何需要执行的检查类型,移出队列。
func (s *Store) Upsert(ctx context.Context, cardID, carrierID uint, dues map[string]time.Time) error {
dueKey := constants.RedisCardDueKey(cardID)
previous, err := s.redis.HGet(ctx, dueKey, fieldCarrier).Uint64()
if err != nil && !stderrors.Is(err, redis.Nil) {
return err
}
pipe := s.redis.TxPipeline()
if previous != 0 && uint(previous) != carrierID {
pipe.ZRem(ctx, constants.RedisCarrierPriorityQueueKey(uint(previous)), cardID)
pipe.ZRem(ctx, constants.RedisCarrierQueueKey(uint(previous)), cardID)
}
if len(dues) == 0 {
pipe.ZRem(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), cardID)
pipe.ZRem(ctx, constants.RedisCarrierQueueKey(carrierID), cardID)
pipe.Del(ctx, dueKey)
_, err := pipe.Exec(ctx)
return err
}
values := map[string]any{fieldCarrier: carrierID}
var earliest time.Time
for _, checkType := range constants.AllCheckTypes() {
due, ok := dues[checkType]
if !ok {
pipe.HDel(ctx, dueKey, fieldDue+checkType)
continue
}
values[fieldDue+checkType] = due.UnixMilli()
if earliest.IsZero() || due.Before(earliest) {
earliest = due
}
}
pipe.HSet(ctx, dueKey, values)
_, err = pipe.Exec(ctx)
if err != nil {
return err
}
priorityKey := constants.RedisCarrierPriorityQueueKey(carrierID)
if inPriority, _ := s.redis.ZScore(ctx, priorityKey, strconv.FormatUint(uint64(cardID), 10)).Result(); inPriority > 0 {
return s.redis.ZAdd(ctx, priorityKey, redis.Z{Score: float64(earliest.UnixMilli()), Member: cardID}).Err()
}
return s.redis.ZAdd(ctx, constants.RedisCarrierQueueKey(carrierID), redis.Z{Score: float64(earliest.UnixMilli()), Member: cardID}).Err()
}
// CardDue 是卡到期信息的解析结果。
type CardDue struct {
CarrierID uint
Priority bool
Types map[string]TypeDue
}
// TypeDue 是卡某检查类型的到期与失败信息。
type TypeDue struct {
Due time.Time
HasDue bool
LastOK time.Time
FailCount int
FirstFailed time.Time
LastError string
Confirm []time.Duration
Expected string
}
// ReadDue 读取并解析卡到期信息;Hash 不存在时返回空结构。
func (s *Store) ReadDue(ctx context.Context, cardID uint) (CardDue, error) {
raw, err := s.redis.HGetAll(ctx, constants.RedisCardDueKey(cardID)).Result()
if err != nil {
return CardDue{}, err
}
return ParseDue(raw), nil
}
// ParseDue 解析卡到期 Hash。
func ParseDue(raw map[string]string) CardDue {
result := CardDue{Types: map[string]TypeDue{}}
if value, err := strconv.ParseUint(raw[fieldCarrier], 10, 64); err == nil {
result.CarrierID = uint(value)
}
result.Priority = raw[fieldPriority] == "1"
for _, checkType := range constants.AllCheckTypes() {
item := TypeDue{}
if ms, err := strconv.ParseInt(raw[fieldDue+checkType], 10, 64); err == nil {
item.Due, item.HasDue = time.UnixMilli(ms), true
}
if ms, err := strconv.ParseInt(raw[fieldOK+checkType], 10, 64); err == nil && ms > 0 {
item.LastOK = time.UnixMilli(ms)
}
item.FailCount, _ = strconv.Atoi(raw[fieldFail+checkType])
if ms, err := strconv.ParseInt(raw[fieldFirstFail+checkType], 10, 64); err == nil && ms > 0 {
item.FirstFailed = time.UnixMilli(ms)
}
item.LastError = raw[fieldError+checkType]
for _, part := range strings.Split(raw[fieldConfirm+checkType], ",") {
if seconds, err := strconv.Atoi(strings.TrimSpace(part)); err == nil && seconds > 0 {
item.Confirm = append(item.Confirm, time.Duration(seconds)*time.Second)
}
}
item.Expected = raw[fieldExpected+checkType]
result.Types[checkType] = item
}
return result
}
// Field 名称导出给执行器组装收尾写入。
func FieldDue(checkType string) string { return fieldDue + checkType }
func FieldOK(checkType string) string { return fieldOK + checkType }
func FieldFail(checkType string) string { return fieldFail + checkType }
func FieldFirstFail(checkType string) string { return fieldFirstFail + checkType }
func FieldError(checkType string) string { return fieldError + checkType }
func FieldConfirm(checkType string) string { return fieldConfirm + checkType }
func FieldExpected(checkType string) string { return fieldExpected + checkType }