feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭): - 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机 - 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制 - 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认 - 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后 - 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估 - 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令 - 迁移 000255 插入开关与告警、退避参数 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -41,6 +41,12 @@ type CarrierParams struct {
|
||||
CardMaxInFlight int
|
||||
PriorityRatio float64
|
||||
Configured bool // 是否有该账户的配置行
|
||||
|
||||
// 停复机指令(add-stop-resume-command-queue)。
|
||||
SRInflightLimit int // 账户停复机执行上限
|
||||
SRMaxInFlight int // 每进程停复机消费者 max_in_flight
|
||||
SRMinInterval time.Duration // 单卡最小重试间隔(停机与复机合计)
|
||||
SRConfirmDelay time.Duration // 结果未知后的确认延迟
|
||||
}
|
||||
|
||||
// carrierEntry 是分发器缓存的账户信息。
|
||||
@@ -110,11 +116,16 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
d.logger.Error("分发器 panic,已恢复", zap.Any("panic", r))
|
||||
}
|
||||
}()
|
||||
if !d.Ready(ctx) {
|
||||
// 停复机指令与设备判断的分发只看停复机出口开关,不受 scheduler_mode 限制;卡任务只在新调度就绪时领取。
|
||||
stopResumeOn := d.settings.Current(ctx).StopResumeQueueEnabled
|
||||
cardsReady := d.Ready(ctx)
|
||||
if !stopResumeOn && !cardsReady {
|
||||
return
|
||||
}
|
||||
d.store.redis.HSet(ctx, constants.RedisCarrierSchedulerHeartbeatKey, d.instanceID, time.Now().UnixMilli())
|
||||
d.drainLegacyHints(ctx)
|
||||
if cardsReady {
|
||||
d.drainLegacyHints(ctx)
|
||||
}
|
||||
d.refreshCarriers(ctx)
|
||||
d.mu.Lock()
|
||||
carriers := append([]carrierEntry(nil), d.carriers...)
|
||||
@@ -123,6 +134,9 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
if lockTTL <= 0 {
|
||||
lockTTL = 900 * time.Millisecond
|
||||
}
|
||||
if stopResumeOn {
|
||||
d.dispatchDeviceEvals(ctx, lockTTL)
|
||||
}
|
||||
for _, carrier := range carriers {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
@@ -131,10 +145,21 @@ func (d *Dispatcher) tick(ctx context.Context, interval time.Duration) {
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
d.dispatchCarrier(ctx, carrier)
|
||||
// 停复机优先:先领取停复机指令占用本秒名额,再领取卡任务。
|
||||
if stopResumeOn {
|
||||
d.dispatchStopResume(ctx, carrier.id)
|
||||
}
|
||||
if cardsReady {
|
||||
d.dispatchCarrier(ctx, carrier)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StopResumeQueueEnabled 返回停复机出口开关当前值。
|
||||
func (d *Dispatcher) StopResumeQueueEnabled(ctx context.Context) bool {
|
||||
return d.settings.Current(ctx).StopResumeQueueEnabled
|
||||
}
|
||||
|
||||
func (d *Dispatcher) dispatchCarrier(ctx context.Context, carrier carrierEntry) {
|
||||
params := d.ParamsFor(carrier.id)
|
||||
limit := params.InflightLimit
|
||||
@@ -265,7 +290,10 @@ func (d *Dispatcher) ParamsFor(carrierID uint) CarrierParams {
|
||||
|
||||
// MergeParams 合并账户配置行与全局默认。
|
||||
func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.CarrierSchedulerConfig) CarrierParams {
|
||||
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured}
|
||||
params := CarrierParams{InflightLimit: cfg.DefaultInflightLimit, CardMaxInFlight: cfg.DefaultCardMaxInFlight, Configured: configured,
|
||||
SRInflightLimit: constants.DefaultStopResumeInflightLimit, SRMaxInFlight: constants.DefaultStopResumeMaxInFlight,
|
||||
SRMinInterval: constants.DefaultStopResumeMinIntervalSeconds * time.Second,
|
||||
SRConfirmDelay: constants.DefaultStopResumeConfirmDelaySeconds * time.Second}
|
||||
if params.InflightLimit <= 0 {
|
||||
params.InflightLimit = 20
|
||||
}
|
||||
@@ -284,6 +312,18 @@ func MergeParams(row model.CarrierPollingParam, configured bool, cfg config.Carr
|
||||
if row.PriorityMaxRatio != nil && *row.PriorityMaxRatio > 0 {
|
||||
params.PriorityRatio = *row.PriorityMaxRatio
|
||||
}
|
||||
if row.SRInflightLimit != nil && *row.SRInflightLimit > 0 {
|
||||
params.SRInflightLimit = *row.SRInflightLimit
|
||||
}
|
||||
if row.StopResumeMaxInFlight != nil && *row.StopResumeMaxInFlight > 0 {
|
||||
params.SRMaxInFlight = *row.StopResumeMaxInFlight
|
||||
}
|
||||
if row.StopResumeMinInterval != nil && *row.StopResumeMinInterval >= 0 {
|
||||
params.SRMinInterval = time.Duration(*row.StopResumeMinInterval) * time.Second
|
||||
}
|
||||
if row.StopResumeConfirmDelay != nil && *row.StopResumeConfirmDelay > 0 {
|
||||
params.SRConfirmDelay = time.Duration(*row.StopResumeConfirmDelay) * time.Second
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,14 @@ func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logg
|
||||
DefaultValue: "600", Description: "账户级告警统计窗口(秒)", Min: &minOne, Max: &maxSeconds},
|
||||
{Key: constants.SystemConfigPollingAccountAlertThreshold, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "50", Description: "账户级告警阈值:窗口内同账户新增失败告警数", Min: &minOne, Max: &maxCount},
|
||||
{Key: constants.SystemConfigStopResumeQueueEnabled, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeBool,
|
||||
DefaultValue: "false", Description: "自动停复机经停复机指令队列执行(全局开关,关闭即各入口当场执行)"},
|
||||
{Key: constants.SystemConfigStopResumeUnknownAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "3", Description: "停复机连续结果未知告警阈值", Min: &minOne, Max: &maxCount},
|
||||
{Key: constants.SystemConfigStopResumeBackoff, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeJSON,
|
||||
DefaultValue: `[600,900,1800,3600]`, Description: "停复机失败退避序列(秒,递增)", Validator: validateSequence},
|
||||
{Key: constants.SystemConfigStopResumeAlertFailCount, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt,
|
||||
DefaultValue: "5", Description: "停复机连续失败告警阈值", Min: &minOne, Max: &maxCount},
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if _, exists := registry.Get(definition.Key); exists {
|
||||
@@ -107,6 +115,23 @@ func validateStepTimeout(raw string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateSequence 校验非空递增的正整数秒序列(停复机退避)。
|
||||
func validateSequence(raw string) error {
|
||||
var values []int
|
||||
if err := json.Unmarshal([]byte(raw), &values); err != nil || len(values) == 0 {
|
||||
return stderrors.New("退避序列必须是非空的秒数组")
|
||||
}
|
||||
for index, value := range values {
|
||||
if value <= 0 {
|
||||
return stderrors.New("退避间隔必须为正数")
|
||||
}
|
||||
if index > 0 && value <= values[index-1] {
|
||||
return stderrors.New("退避序列必须递增")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBackoff(raw string) error {
|
||||
var backoff map[string][]int
|
||||
if err := json.Unmarshal([]byte(raw), &backoff); err != nil {
|
||||
@@ -171,6 +196,28 @@ type Settings struct {
|
||||
ImmediateAlert map[string]bool
|
||||
AccountAlertWindow time.Duration
|
||||
AccountAlertLimit int
|
||||
|
||||
// 停复机指令队列(add-stop-resume-command-queue)。
|
||||
StopResumeQueueEnabled bool
|
||||
SRUnknownAlert int
|
||||
SRBackoff []time.Duration
|
||||
SRAlertFailCount int
|
||||
}
|
||||
|
||||
// SRBackoffFor 返回停复机第 n 次连续失败(n 从 1 起)的退避间隔;超过序列长度取最后一项。
|
||||
func (s Settings) SRBackoffFor(failCount int) time.Duration {
|
||||
sequence := s.SRBackoff
|
||||
if len(sequence) == 0 {
|
||||
sequence = []time.Duration{10 * time.Minute}
|
||||
}
|
||||
index := failCount - 1
|
||||
if index < 0 {
|
||||
index = 0
|
||||
}
|
||||
if index >= len(sequence) {
|
||||
index = len(sequence) - 1
|
||||
}
|
||||
return sequence[index]
|
||||
}
|
||||
|
||||
// CarrierMode 判断当前是否为按运营商账户调度。
|
||||
@@ -243,6 +290,9 @@ func DefaultSettings() Settings {
|
||||
AlertFailCount: map[string]int{"default": 5}, AlertFailDuration: 6 * time.Hour,
|
||||
ImmediateAlert: map[string]bool{constants.GatewayClassAccountConfig: true},
|
||||
AccountAlertWindow: 10 * time.Minute, AccountAlertLimit: 50,
|
||||
SRUnknownAlert: 3,
|
||||
SRBackoff: []time.Duration{10 * time.Minute, 15 * time.Minute, 30 * time.Minute, time.Hour},
|
||||
SRAlertFailCount: 5,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +414,8 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
{constants.SystemConfigPollingAlertFailDuration, func(v int) { s.AlertFailDuration = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingAccountAlertWindow, func(v int) { s.AccountAlertWindow = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingAccountAlertThreshold, func(v int) { s.AccountAlertLimit = v }},
|
||||
{constants.SystemConfigStopResumeUnknownAlert, func(v int) { s.SRUnknownAlert = v }},
|
||||
{constants.SystemConfigStopResumeAlertFailCount, func(v int) { s.SRAlertFailCount = v }},
|
||||
}
|
||||
for _, item := range ints {
|
||||
raw, err := get(item.key)
|
||||
@@ -376,6 +428,21 @@ func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
}
|
||||
item.set(value)
|
||||
}
|
||||
if raw, err = get(constants.SystemConfigStopResumeQueueEnabled); err != nil {
|
||||
return s, err
|
||||
}
|
||||
s.StopResumeQueueEnabled = raw == "true"
|
||||
if raw, err = get(constants.SystemConfigStopResumeBackoff); err != nil {
|
||||
return s, err
|
||||
}
|
||||
var srBackoff []int
|
||||
if err := json.Unmarshal([]byte(raw), &srBackoff); err != nil || validateSequence(raw) != nil {
|
||||
return s, stderrors.New("停复机退避序列配置非法")
|
||||
}
|
||||
s.SRBackoff = nil
|
||||
for _, value := range srBackoff {
|
||||
s.SRBackoff = append(s.SRBackoff, time.Duration(value)*time.Second)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
||||
120
internal/polling/carrierqueue/sr_dispatch.go
Normal file
120
internal/polling/carrierqueue/sr_dispatch.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StopResumeMessage 是停复机指令 NSQ 消息体。
|
||||
type StopResumeMessage struct {
|
||||
CardID uint `json:"card_id"`
|
||||
CarrierID uint `json:"carrier_id"`
|
||||
Token string `json:"token"`
|
||||
Version int64 `json:"version"`
|
||||
Action string `json:"action"`
|
||||
Pending bool `json:"pending"`
|
||||
DispatchedAt int64 `json:"dispatched_at"`
|
||||
}
|
||||
|
||||
// DeviceEvalMessage 是设备判断 NSQ 消息体。
|
||||
type DeviceEvalMessage struct {
|
||||
DeviceID uint `json:"device_id"`
|
||||
Token string `json:"token"`
|
||||
Version string `json:"version"`
|
||||
Scenes []string `json:"scenes"`
|
||||
DispatchedAt int64 `json:"dispatched_at"`
|
||||
}
|
||||
|
||||
// deviceEvalLimit 是全局同时执行的设备判断上限:判断只读数据库与写指令,毫秒级。
|
||||
const deviceEvalLimit = 50
|
||||
|
||||
// stopResumeLease 是停复机指令与设备判断的租约:单次执行只调一次网关,远小于该值。
|
||||
func (d *Dispatcher) stopResumeLease() time.Duration {
|
||||
return 2 * time.Minute
|
||||
}
|
||||
|
||||
// dispatchStopResume 在卡任务之前领取账户停复机指令:领取数 = min(停复机所在限流组本秒剩余, 执行上限 − 执行中)。
|
||||
func (d *Dispatcher) dispatchStopResume(ctx context.Context, carrierID uint) {
|
||||
params := d.ParamsFor(carrierID)
|
||||
capacity := params.SRInflightLimit
|
||||
processes := d.cfg.SubscriberProcesses
|
||||
if processes <= 0 {
|
||||
processes = 1
|
||||
}
|
||||
if consume := params.SRMaxInFlight * processes; consume > 0 && capacity > consume {
|
||||
capacity = consume
|
||||
}
|
||||
if d.budgets != nil {
|
||||
ops := []string{constants.GatewayOpStop, constants.GatewayOpStart}
|
||||
if groups, _, limited := d.budgets.Budget(ctx, carrierID, ops); limited {
|
||||
for _, remaining := range groups {
|
||||
if remaining < capacity {
|
||||
capacity = remaining
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if capacity <= 0 {
|
||||
return
|
||||
}
|
||||
claimed, expired, err := d.store.ClaimStopResume(ctx, SRClaimInput{
|
||||
CarrierID: carrierID, Lease: d.stopResumeLease(), Capacity: capacity, MinInterval: params.SRMinInterval,
|
||||
TokenPrefix: d.instanceID + ":sr:" + strconv.FormatInt(time.Now().UnixNano(), 36),
|
||||
})
|
||||
if err != nil {
|
||||
d.logger.Warn("运营商账户停复机指令领取失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if expired > 0 {
|
||||
d.logger.Warn("停复机指令处理中标记到期(执行崩溃或超时)", zap.Uint("carrier_id", carrierID), zap.Int64("count", expired),
|
||||
zap.String("metric", "stop_resume.claim_expired"))
|
||||
}
|
||||
topic := constants.NSQStopResumeTopic(carrierID)
|
||||
for _, item := range claimed {
|
||||
body, _ := json.Marshal(StopResumeMessage{CardID: item.CardID, CarrierID: carrierID, Token: item.Token, Version: item.Version,
|
||||
Action: item.Action, Pending: item.Pending, DispatchedAt: time.Now().UnixMilli()})
|
||||
if err := d.publisher.Publish(topic, body); err != nil {
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
if releaseErr := d.store.ReleaseStopResume(releaseCtx, item.CardID, carrierID, item.Token); releaseErr != nil {
|
||||
d.logger.Warn("发布失败后释放停复机指令失败", zap.Uint("card_id", item.CardID), zap.Error(releaseErr))
|
||||
}
|
||||
cancel()
|
||||
d.logger.Warn("发布停复机指令失败,已释放", zap.Uint("card_id", item.CardID), zap.String("topic", topic), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchDeviceEvals 领取到期设备判断并发布;全局分发锁保证每轮只有一个实例领取。
|
||||
func (d *Dispatcher) dispatchDeviceEvals(ctx context.Context, lockTTL time.Duration) {
|
||||
ok, err := d.store.redis.SetNX(ctx, constants.RedisDeviceEvalDispatchLockKey, d.instanceID, lockTTL).Result()
|
||||
if err != nil || !ok {
|
||||
return
|
||||
}
|
||||
claimed, expired, err := d.store.ClaimDeviceEvals(ctx, d.stopResumeLease(), deviceEvalLimit,
|
||||
d.instanceID+":dev:"+strconv.FormatInt(time.Now().UnixNano(), 36))
|
||||
if err != nil {
|
||||
d.logger.Warn("设备判断领取失败", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if expired > 0 {
|
||||
d.logger.Warn("设备判断处理中标记到期(执行崩溃或超时)", zap.Int64("count", expired), zap.String("metric", "stop_resume.device_eval_expired"))
|
||||
}
|
||||
for _, item := range claimed {
|
||||
body, _ := json.Marshal(DeviceEvalMessage{DeviceID: item.DeviceID, Token: item.Token, Version: item.Version, Scenes: item.Scenes,
|
||||
DispatchedAt: time.Now().UnixMilli()})
|
||||
if err := d.publisher.Publish(constants.NSQDeviceEvalTopic, body); err != nil {
|
||||
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
if releaseErr := d.store.ReleaseDeviceEval(releaseCtx, item.DeviceID, item.Token); releaseErr != nil {
|
||||
d.logger.Warn("发布失败后释放设备判断失败", zap.Uint("device_id", item.DeviceID), zap.Error(releaseErr))
|
||||
}
|
||||
cancel()
|
||||
d.logger.Warn("发布设备判断失败,已释放", zap.Uint("device_id", item.DeviceID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
29
internal/polling/carrierqueue/sr_queue.go
Normal file
29
internal/polling/carrierqueue/sr_queue.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package carrierqueue
|
||||
|
||||
import "context"
|
||||
|
||||
// CommandQueue 是停复机指令队列写入端:开关读取受控系统配置,写入走共享缓存 Lua。
|
||||
type CommandQueue struct {
|
||||
store *Store
|
||||
settings *SettingsProvider
|
||||
}
|
||||
|
||||
// NewCommandQueue 创建停复机指令队列写入端。
|
||||
func NewCommandQueue(store *Store, settings *SettingsProvider) *CommandQueue {
|
||||
return &CommandQueue{store: store, settings: settings}
|
||||
}
|
||||
|
||||
// Enabled 返回停复机出口开关 polling.stop_resume.queue_enabled。
|
||||
func (q *CommandQueue) Enabled(ctx context.Context) bool {
|
||||
return q != nil && q.store != nil && q.settings.Current(ctx).StopResumeQueueEnabled
|
||||
}
|
||||
|
||||
// WriteCommand 写入或覆盖卡的停复机指令。
|
||||
func (q *CommandQueue) WriteCommand(ctx context.Context, cmd StopResumeCommand) (int64, error) {
|
||||
return q.store.WriteCommand(ctx, cmd)
|
||||
}
|
||||
|
||||
// RequestDeviceEval 请求一次设备维度判断。
|
||||
func (q *CommandQueue) RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error {
|
||||
return q.store.RequestDeviceEval(ctx, deviceID, scenes)
|
||||
}
|
||||
225
internal/polling/carrierqueue/sr_scripts.go
Normal file
225
internal/polling/carrierqueue/sr_scripts.go
Normal file
@@ -0,0 +1,225 @@
|
||||
package carrierqueue
|
||||
|
||||
import "github.com/redis/go-redis/v9"
|
||||
|
||||
// 停复机指令 Hash 字段(见 constants.RedisCardStopResumeCmdKey)。
|
||||
const (
|
||||
srFieldAction = "action" // stop / start
|
||||
srFieldReason = "reason" // 停机原因(复机为空)
|
||||
srFieldScenes = "scenes" // 触发场景(逗号分隔)
|
||||
srFieldSource = "source" // 请求来源
|
||||
srFieldCarrier = "carrier" // 所属运营商账户
|
||||
srFieldVersion = "ver" // 指令版本,每次写入 HINCRBY
|
||||
srFieldFail = "fail" // 连续失败次数
|
||||
srFieldFirst = "ffail" // 首次失败(毫秒)
|
||||
srFieldError = "err" // 最近错误(类别|信息)
|
||||
srFieldPending = "pending" // 1 表示结果未知、待确认
|
||||
srFieldUnknown = "unknown" // 连续结果未知次数
|
||||
srFieldAt = "at" // 最近写入时间(毫秒)
|
||||
)
|
||||
|
||||
// srWriteLua 写入或覆盖停复机指令:方向相反时清零失败与待确认,递增版本,放入账户停复机队列。
|
||||
// 指令执行中(存在处理中标记)时只更新指令内容与版本,由收尾按版本保留;待确认中且方向不变时保持确认时间。
|
||||
//
|
||||
// KEYS[1] 指令 Hash;KEYS[2] 账户停复机队列;KEYS[3] 处理中标记。
|
||||
// ARGV:action、reason、scenes、source、carrier、now_ms、card_id。返回新版本号。
|
||||
const srWriteLua = `
|
||||
local prev = redis.call('HGET', KEYS[1], 'action')
|
||||
if prev and prev ~= ARGV[1] then
|
||||
redis.call('HDEL', KEYS[1], 'fail', 'ffail', 'err', 'pending', 'unknown')
|
||||
end
|
||||
redis.call('HSET', KEYS[1], 'action', ARGV[1], 'reason', ARGV[2], 'scenes', ARGV[3], 'source', ARGV[4], 'carrier', ARGV[5], 'at', ARGV[6])
|
||||
local ver = redis.call('HINCRBY', KEYS[1], 'ver', 1)
|
||||
if redis.call('EXISTS', KEYS[3]) == 1 then return ver end
|
||||
if redis.call('HGET', KEYS[1], 'pending') == '1' then
|
||||
redis.call('ZADD', KEYS[2], 'NX', tonumber(ARGV[6]), ARGV[7])
|
||||
else
|
||||
redis.call('ZADD', KEYS[2], 'LT', tonumber(ARGV[6]), ARGV[7])
|
||||
end
|
||||
return ver
|
||||
`
|
||||
|
||||
// srClaimLua 领取账户到期停复机指令:先清理过期执行中,按容量领取;未到单卡最小重试间隔的推后到可调用时间。
|
||||
// 待确认的指令只查状态,不受最小间隔约束。指令已不存在的成员直接移出队列。
|
||||
//
|
||||
// KEYS[1] 账户停复机队列;KEYS[2] 账户停复机执行中计数。
|
||||
// ARGV:now_ms、lease_ms、容量(执行上限与剩余 QPS 的较小值)、token 前缀、单卡最小间隔 ms。
|
||||
// 返回 {过期数, card_id, token, ver, action, pending, ...}。
|
||||
const srClaimLua = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local lease = tonumber(ARGV[2])
|
||||
local limit = tonumber(ARGV[3])
|
||||
local prefix = ARGV[4]
|
||||
local minInterval = tonumber(ARGV[5])
|
||||
local expired = redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now)
|
||||
local result = {expired}
|
||||
local capacity = limit - redis.call('ZCARD', KEYS[2])
|
||||
if capacity <= 0 then return result end
|
||||
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', now, 'LIMIT', 0, capacity * 2 + 10)
|
||||
local taken = 0
|
||||
for _, id in ipairs(ids) do
|
||||
if taken >= capacity then break end
|
||||
local cmdKey = 'polling:card:' .. id .. ':stop_resume_cmd'
|
||||
local fields = redis.call('HMGET', cmdKey, 'action', 'ver', 'pending')
|
||||
if not fields[1] then
|
||||
redis.call('ZREM', KEYS[1], id)
|
||||
else
|
||||
local pending = fields[3] == '1'
|
||||
local blockedUntil = 0
|
||||
if not pending then
|
||||
local last = tonumber(redis.call('GET', 'polling:card:' .. id .. ':sr_last_call') or '0')
|
||||
if last > 0 and last + minInterval > now then blockedUntil = last + minInterval end
|
||||
end
|
||||
if blockedUntil > 0 then
|
||||
redis.call('ZADD', KEYS[1], 'XX', blockedUntil, id)
|
||||
else
|
||||
local claimKey = 'polling:card:' .. id .. ':sr_claim'
|
||||
local token = prefix .. ':' .. id
|
||||
if redis.call('SET', claimKey, token, 'NX', 'PX', lease) then
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + lease, id)
|
||||
redis.call('ZADD', KEYS[2], now + lease, token)
|
||||
taken = taken + 1
|
||||
table.insert(result, id)
|
||||
table.insert(result, token)
|
||||
table.insert(result, fields[2] or '0')
|
||||
table.insert(result, fields[1])
|
||||
table.insert(result, pending and '1' or '0')
|
||||
else
|
||||
local ttl = redis.call('PTTL', claimKey)
|
||||
if ttl < 0 then ttl = lease end
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + ttl, id)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return result
|
||||
`
|
||||
|
||||
// srFinishLua 停复机指令收尾:token 不一致返回 0 不做任何修改;版本已变大(执行期间写入了新指令)时保留新指令、立即可领取(返回 2);
|
||||
// 否则 done=1 删除指令并移出队列,done=0 写入字段并按 next_ms 重新排队(返回 1)。
|
||||
//
|
||||
// KEYS[1] 指令 Hash;KEYS[2] 处理中标记;KEYS[3] 执行权;KEYS[4] 执行中计数;KEYS[5] 账户停复机队列。
|
||||
// ARGV:token、card_id、领取时版本、done(1/0)、next_ms、now_ms、set 字段对数 n、字段1、值1、…、删除字段…。
|
||||
const srFinishLua = `
|
||||
if redis.call('GET', KEYS[2]) ~= ARGV[1] then return 0 end
|
||||
redis.call('DEL', KEYS[2])
|
||||
redis.call('DEL', KEYS[3])
|
||||
redis.call('ZREM', KEYS[4], ARGV[1])
|
||||
local cardId = ARGV[2]
|
||||
if (redis.call('HGET', KEYS[1], 'ver') or '0') ~= ARGV[3] then
|
||||
if redis.call('EXISTS', KEYS[1]) == 1 then
|
||||
redis.call('ZADD', KEYS[5], tonumber(ARGV[6]), cardId)
|
||||
else
|
||||
redis.call('ZREM', KEYS[5], cardId)
|
||||
end
|
||||
return 2
|
||||
end
|
||||
if ARGV[4] == '1' then
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('ZREM', KEYS[5], cardId)
|
||||
return 1
|
||||
end
|
||||
local n = tonumber(ARGV[7])
|
||||
local idx = 8
|
||||
for i = 1, n do
|
||||
redis.call('HSET', KEYS[1], ARGV[idx], ARGV[idx + 1])
|
||||
idx = idx + 2
|
||||
end
|
||||
while idx <= #ARGV do
|
||||
redis.call('HDEL', KEYS[1], ARGV[idx])
|
||||
idx = idx + 1
|
||||
end
|
||||
redis.call('ZADD', KEYS[5], tonumber(ARGV[5]), cardId)
|
||||
return 1
|
||||
`
|
||||
|
||||
// srReleaseLua 发布失败:按 token 删除处理中标记、移出执行中计数,score 恢复为当前时间。
|
||||
// KEYS[1] 处理中标记;KEYS[2] 执行中计数;KEYS[3] 队列。ARGV:token、member、now_ms。
|
||||
const srReleaseLua = `
|
||||
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('ZREM', KEYS[2], ARGV[1])
|
||||
redis.call('ZADD', KEYS[3], 'XX', tonumber(ARGV[3]), ARGV[2])
|
||||
return 1
|
||||
`
|
||||
|
||||
// deviceEvalRequestLua 请求设备判断:递增版本、合并场景、设备进入判断队列(同一设备一个成员,ZADD LT 不推后已有更早时间)。
|
||||
// KEYS[1] 版本;KEYS[2] 场景集合;KEYS[3] 设备判断队列;KEYS[4] 处理中标记。ARGV:device_id、now_ms、场景…。
|
||||
const deviceEvalRequestLua = `
|
||||
redis.call('INCR', KEYS[1])
|
||||
for i = 3, #ARGV do redis.call('SADD', KEYS[2], ARGV[i]) end
|
||||
if redis.call('EXISTS', KEYS[4]) == 1 then return 1 end
|
||||
redis.call('ZADD', KEYS[3], 'LT', tonumber(ARGV[2]), ARGV[1])
|
||||
return 1
|
||||
`
|
||||
|
||||
// deviceEvalClaimLua 领取到期设备判断:写处理中标记并记下版本与场景快照。
|
||||
// KEYS[1] 设备判断队列;KEYS[2] 执行中计数。ARGV:now_ms、lease_ms、上限、token 前缀。
|
||||
// 返回 {过期数, device_id, token, ver, scenes, ...}。
|
||||
const deviceEvalClaimLua = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local lease = tonumber(ARGV[2])
|
||||
local limit = tonumber(ARGV[3])
|
||||
local expired = redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now)
|
||||
local result = {expired}
|
||||
local capacity = limit - redis.call('ZCARD', KEYS[2])
|
||||
if capacity <= 0 then return result end
|
||||
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', now, 'LIMIT', 0, capacity * 2 + 10)
|
||||
local taken = 0
|
||||
for _, id in ipairs(ids) do
|
||||
if taken >= capacity then break end
|
||||
local claimKey = 'polling:device:' .. id .. ':eval_claim'
|
||||
local token = ARGV[4] .. ':' .. id
|
||||
if redis.call('SET', claimKey, token, 'NX', 'PX', lease) then
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + lease, id)
|
||||
redis.call('ZADD', KEYS[2], now + lease, token)
|
||||
taken = taken + 1
|
||||
local ver = redis.call('GET', 'polling:device:' .. id .. ':eval_ver') or '0'
|
||||
local scenes = redis.call('SMEMBERS', 'polling:device:' .. id .. ':eval_scenes')
|
||||
table.insert(result, id)
|
||||
table.insert(result, token)
|
||||
table.insert(result, ver)
|
||||
table.insert(result, table.concat(scenes, ','))
|
||||
else
|
||||
local ttl = redis.call('PTTL', claimKey)
|
||||
if ttl < 0 then ttl = lease end
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + ttl, id)
|
||||
end
|
||||
end
|
||||
return result
|
||||
`
|
||||
|
||||
// deviceEvalFinishLua 设备判断收尾:只移除本次已处理的场景;版本变大时立即再判断(返回 2);
|
||||
// defer_ms > 0 表示本次未判断(旧路径持设备锁),推后再判断,场景不移除。
|
||||
// KEYS[1] 处理中标记;KEYS[2] 执行权;KEYS[3] 执行中计数;KEYS[4] 版本;KEYS[5] 场景集合;KEYS[6] 队列。
|
||||
// ARGV:token、device_id、领取时版本、now_ms、defer_ms、已处理场景…。
|
||||
const deviceEvalFinishLua = `
|
||||
if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('DEL', KEYS[2])
|
||||
redis.call('ZREM', KEYS[3], ARGV[1])
|
||||
local now = tonumber(ARGV[4])
|
||||
local deferMs = tonumber(ARGV[5])
|
||||
if deferMs > 0 then
|
||||
redis.call('ZADD', KEYS[6], now + deferMs, ARGV[2])
|
||||
return 3
|
||||
end
|
||||
for i = 6, #ARGV do redis.call('SREM', KEYS[5], ARGV[i]) end
|
||||
if (redis.call('GET', KEYS[4]) or '0') ~= ARGV[3] then
|
||||
redis.call('ZADD', KEYS[6], now, ARGV[2])
|
||||
return 2
|
||||
end
|
||||
redis.call('ZREM', KEYS[6], ARGV[2])
|
||||
redis.call('DEL', KEYS[5])
|
||||
return 1
|
||||
`
|
||||
|
||||
var (
|
||||
srWriteScript = redis.NewScript(srWriteLua)
|
||||
srClaimScript = redis.NewScript(srClaimLua)
|
||||
srFinishScript = redis.NewScript(srFinishLua)
|
||||
srReleaseScript = redis.NewScript(srReleaseLua)
|
||||
deviceRequestScript = redis.NewScript(deviceEvalRequestLua)
|
||||
deviceClaimScript = redis.NewScript(deviceEvalClaimLua)
|
||||
deviceFinishScript = redis.NewScript(deviceEvalFinishLua)
|
||||
)
|
||||
348
internal/polling/carrierqueue/sr_store.go
Normal file
348
internal/polling/carrierqueue/sr_store.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StopResumeCommand 是写入停复机队列的一条指令。
|
||||
type StopResumeCommand struct {
|
||||
CardID uint
|
||||
CarrierID uint
|
||||
Action string // constants.StopResumeActionStop / StopResumeActionStart
|
||||
Reason string // 停机原因
|
||||
Scenes []string // 触发场景
|
||||
Source string // 请求来源(日志与排查)
|
||||
}
|
||||
|
||||
// CommandState 是卡当前停复机指令的解析结果。
|
||||
type CommandState struct {
|
||||
Exists bool
|
||||
Action string
|
||||
Reason string
|
||||
Scenes []string
|
||||
Source string
|
||||
CarrierID uint
|
||||
Version int64
|
||||
FailCount int
|
||||
FirstFailed time.Time
|
||||
LastError string
|
||||
Pending bool
|
||||
Unknown int
|
||||
WrittenAt time.Time
|
||||
NextAt time.Time // 队列中的可执行时间(零值表示不在队列中)
|
||||
LastCallAt time.Time
|
||||
}
|
||||
|
||||
// WriteCommand 写入或覆盖卡的停复机指令,返回新版本号。
|
||||
func (s *Store) WriteCommand(ctx context.Context, cmd StopResumeCommand) (int64, error) {
|
||||
if cmd.CardID == 0 || cmd.CarrierID == 0 {
|
||||
return 0, stderrors.New("停复机指令缺少卡或运营商账户")
|
||||
}
|
||||
return srWriteScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCardStopResumeCmdKey(cmd.CardID), constants.RedisCarrierStopResumeQueueKey(cmd.CarrierID),
|
||||
constants.RedisCardStopResumeClaimKey(cmd.CardID)},
|
||||
cmd.Action, cmd.Reason, constants.JoinStopResumeScenes(cmd.Scenes), cmd.Source, cmd.CarrierID, time.Now().UnixMilli(), cmd.CardID,
|
||||
).Int64()
|
||||
}
|
||||
|
||||
// SRClaimInput 是一次账户停复机领取的参数。
|
||||
type SRClaimInput struct {
|
||||
CarrierID uint
|
||||
Lease time.Duration
|
||||
Capacity int // 执行上限与剩余 QPS 的较小值
|
||||
TokenPrefix string
|
||||
MinInterval time.Duration
|
||||
}
|
||||
|
||||
// SRClaimed 是一条被领取的停复机指令。
|
||||
type SRClaimed struct {
|
||||
CardID uint
|
||||
Token string
|
||||
Version int64
|
||||
Action string
|
||||
Pending bool
|
||||
}
|
||||
|
||||
// ClaimStopResume 领取账户到期停复机指令,返回领取结果与本轮清理的过期执行数。
|
||||
func (s *Store) ClaimStopResume(ctx context.Context, input SRClaimInput) ([]SRClaimed, int64, error) {
|
||||
raw, err := srClaimScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCarrierStopResumeQueueKey(input.CarrierID), constants.RedisCarrierStopResumeRunningKey(input.CarrierID)},
|
||||
time.Now().UnixMilli(), input.Lease.Milliseconds(), input.Capacity, input.TokenPrefix, input.MinInterval.Milliseconds(),
|
||||
).Slice()
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil, 0, err
|
||||
}
|
||||
expired, _ := raw[0].(int64)
|
||||
claimed := make([]SRClaimed, 0, (len(raw)-1)/5)
|
||||
for index := 1; index+4 < len(raw); index += 5 {
|
||||
cardID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64)
|
||||
version, _ := strconv.ParseInt(fmt.Sprint(raw[index+2]), 10, 64)
|
||||
claimed = append(claimed, SRClaimed{CardID: uint(cardID), Token: fmt.Sprint(raw[index+1]), Version: version,
|
||||
Action: fmt.Sprint(raw[index+3]), Pending: fmt.Sprint(raw[index+4]) == "1"})
|
||||
}
|
||||
return claimed, expired, nil
|
||||
}
|
||||
|
||||
// AcquireStopResumeExec 抢停复机指令执行权:1 抢到,0 处理中标记已不是该 token,-1 已有执行者。
|
||||
func (s *Store) AcquireStopResumeExec(ctx context.Context, cardID uint, token, execValue string, ttl time.Duration) (int64, error) {
|
||||
return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisCardStopResumeClaimKey(cardID), constants.RedisCardStopResumeExecKey(cardID)},
|
||||
token, execValue, ttl.Milliseconds()).Int64()
|
||||
}
|
||||
|
||||
// SRFinish 是一次停复机指令收尾。
|
||||
type SRFinish struct {
|
||||
CardID uint
|
||||
CarrierID uint
|
||||
Token string
|
||||
Version int64
|
||||
Done bool // true 结束并删除指令
|
||||
NextAt time.Time
|
||||
Set map[string]string
|
||||
Del []string
|
||||
}
|
||||
|
||||
// FinishStopResume 停复机指令收尾:返回 0 token 不一致、1 正常、2 版本已变大保留新指令。
|
||||
func (s *Store) FinishStopResume(ctx context.Context, finish SRFinish) (int64, error) {
|
||||
done := "0"
|
||||
if finish.Done {
|
||||
done = "1"
|
||||
}
|
||||
now := time.Now()
|
||||
next := finish.NextAt
|
||||
if next.IsZero() || next.Before(now) {
|
||||
next = now
|
||||
}
|
||||
args := []any{finish.Token, finish.CardID, finish.Version, done, next.UnixMilli(), now.UnixMilli(), len(finish.Set)}
|
||||
for field, value := range finish.Set {
|
||||
args = append(args, field, value)
|
||||
}
|
||||
for _, field := range finish.Del {
|
||||
args = append(args, field)
|
||||
}
|
||||
return srFinishScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCardStopResumeCmdKey(finish.CardID), constants.RedisCardStopResumeClaimKey(finish.CardID),
|
||||
constants.RedisCardStopResumeExecKey(finish.CardID), constants.RedisCarrierStopResumeRunningKey(finish.CarrierID),
|
||||
constants.RedisCarrierStopResumeQueueKey(finish.CarrierID)},
|
||||
args...).Int64()
|
||||
}
|
||||
|
||||
// ReleaseStopResume 发布失败时释放停复机指令处理中标记并恢复为立即可领取。
|
||||
func (s *Store) ReleaseStopResume(ctx context.Context, cardID, carrierID uint, token string) error {
|
||||
return srReleaseScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisCardStopResumeClaimKey(cardID), constants.RedisCarrierStopResumeRunningKey(carrierID),
|
||||
constants.RedisCarrierStopResumeQueueKey(carrierID)},
|
||||
token, cardID, time.Now().UnixMilli()).Err()
|
||||
}
|
||||
|
||||
// MarkStopResumeCall 记录本卡一次停复机网关调用(停机与复机合计),供单卡最小重试间隔判断。
|
||||
func (s *Store) MarkStopResumeCall(ctx context.Context, cardID uint, at time.Time, keep time.Duration) error {
|
||||
if keep < time.Hour {
|
||||
keep = time.Hour
|
||||
}
|
||||
return s.redis.Set(ctx, constants.RedisCardStopResumeLastCallKey(cardID), at.UnixMilli(), keep).Err()
|
||||
}
|
||||
|
||||
// LastStopResumeCall 返回本卡最近一次停复机网关调用时间;没有记录返回零值。
|
||||
func (s *Store) LastStopResumeCall(ctx context.Context, cardID uint) (time.Time, error) {
|
||||
ms, err := s.redis.Get(ctx, constants.RedisCardStopResumeLastCallKey(cardID)).Int64()
|
||||
if stderrors.Is(err, redis.Nil) {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.UnixMilli(ms), nil
|
||||
}
|
||||
|
||||
// ReadCommand 读取卡当前停复机指令与队列位置。
|
||||
func (s *Store) ReadCommand(ctx context.Context, cardID uint) (CommandState, error) {
|
||||
raw, err := s.redis.HGetAll(ctx, constants.RedisCardStopResumeCmdKey(cardID)).Result()
|
||||
if err != nil {
|
||||
return CommandState{}, err
|
||||
}
|
||||
state := ParseCommand(raw)
|
||||
if last, lastErr := s.LastStopResumeCall(ctx, cardID); lastErr == nil {
|
||||
state.LastCallAt = last
|
||||
}
|
||||
if state.Exists && state.CarrierID > 0 {
|
||||
member := strconv.FormatUint(uint64(cardID), 10)
|
||||
if score, scoreErr := s.redis.ZScore(ctx, constants.RedisCarrierStopResumeQueueKey(state.CarrierID), member).Result(); scoreErr == nil {
|
||||
state.NextAt = time.UnixMilli(int64(score))
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// ParseCommand 解析停复机指令 Hash。
|
||||
func ParseCommand(raw map[string]string) CommandState {
|
||||
state := CommandState{Exists: raw[srFieldAction] != ""}
|
||||
state.Action, state.Reason, state.Source, state.LastError = raw[srFieldAction], raw[srFieldReason], raw[srFieldSource], raw[srFieldError]
|
||||
state.Scenes = constants.SplitStopResumeScenes(raw[srFieldScenes])
|
||||
if value, err := strconv.ParseUint(raw[srFieldCarrier], 10, 64); err == nil {
|
||||
state.CarrierID = uint(value)
|
||||
}
|
||||
state.Version, _ = strconv.ParseInt(raw[srFieldVersion], 10, 64)
|
||||
state.FailCount, _ = strconv.Atoi(raw[srFieldFail])
|
||||
state.Unknown, _ = strconv.Atoi(raw[srFieldUnknown])
|
||||
state.Pending = raw[srFieldPending] == "1"
|
||||
if ms, err := strconv.ParseInt(raw[srFieldFirst], 10, 64); err == nil && ms > 0 {
|
||||
state.FirstFailed = time.UnixMilli(ms)
|
||||
}
|
||||
if ms, err := strconv.ParseInt(raw[srFieldAt], 10, 64); err == nil && ms > 0 {
|
||||
state.WrittenAt = time.UnixMilli(ms)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// 停复机指令字段名导出给执行器组装收尾写入。
|
||||
const (
|
||||
SRFieldFail = srFieldFail
|
||||
SRFieldFirst = srFieldFirst
|
||||
SRFieldError = srFieldError
|
||||
SRFieldPending = srFieldPending
|
||||
SRFieldUnknown = srFieldUnknown
|
||||
)
|
||||
|
||||
// RequestDeviceEval 请求一次设备维度判断:递增版本、合并场景、设备进入判断队列。
|
||||
func (s *Store) RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error {
|
||||
if deviceID == 0 {
|
||||
return stderrors.New("设备判断缺少设备")
|
||||
}
|
||||
args := []any{deviceID, time.Now().UnixMilli()}
|
||||
for _, scene := range constants.SplitStopResumeScenes(constants.JoinStopResumeScenes(scenes)) {
|
||||
args = append(args, scene)
|
||||
}
|
||||
return deviceRequestScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisDeviceEvalVersionKey(deviceID), constants.RedisDeviceEvalScenesKey(deviceID), constants.RedisDeviceEvalQueueKey,
|
||||
constants.RedisDeviceEvalClaimKey(deviceID)},
|
||||
args...).Err()
|
||||
}
|
||||
|
||||
// DeviceEvalClaimed 是一次被领取的设备判断。
|
||||
type DeviceEvalClaimed struct {
|
||||
DeviceID uint
|
||||
Token string
|
||||
Version string
|
||||
Scenes []string
|
||||
}
|
||||
|
||||
// ClaimDeviceEvals 领取到期设备判断。
|
||||
func (s *Store) ClaimDeviceEvals(ctx context.Context, lease time.Duration, limit int, tokenPrefix string) ([]DeviceEvalClaimed, int64, error) {
|
||||
raw, err := deviceClaimScript.Run(ctx, s.redis, []string{constants.RedisDeviceEvalQueueKey, constants.RedisDeviceEvalRunningKey},
|
||||
time.Now().UnixMilli(), lease.Milliseconds(), limit, tokenPrefix).Slice()
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil, 0, err
|
||||
}
|
||||
expired, _ := raw[0].(int64)
|
||||
claimed := make([]DeviceEvalClaimed, 0, (len(raw)-1)/4)
|
||||
for index := 1; index+3 < len(raw); index += 4 {
|
||||
deviceID, _ := strconv.ParseUint(fmt.Sprint(raw[index]), 10, 64)
|
||||
claimed = append(claimed, DeviceEvalClaimed{DeviceID: uint(deviceID), Token: fmt.Sprint(raw[index+1]),
|
||||
Version: fmt.Sprint(raw[index+2]), Scenes: constants.SplitStopResumeScenes(fmt.Sprint(raw[index+3]))})
|
||||
}
|
||||
return claimed, expired, nil
|
||||
}
|
||||
|
||||
// AcquireDeviceEvalLock 抢设备判断锁(执行权):1 抢到,0 处理中标记已不是该 token,-1 已有判断在执行。
|
||||
func (s *Store) AcquireDeviceEvalLock(ctx context.Context, deviceID uint, token, value string, ttl time.Duration) (int64, error) {
|
||||
return acquireExecScript.Run(ctx, s.redis, []string{constants.RedisDeviceEvalClaimKey(deviceID), constants.RedisDeviceEvalLockKey(deviceID)},
|
||||
token, value, ttl.Milliseconds()).Int64()
|
||||
}
|
||||
|
||||
// FinishDeviceEval 设备判断收尾:返回 0 token 不一致、1 完成、2 版本变大已重新排队、3 已推后。
|
||||
func (s *Store) FinishDeviceEval(ctx context.Context, claimed DeviceEvalClaimed, deferFor time.Duration) (int64, error) {
|
||||
args := []any{claimed.Token, claimed.DeviceID, claimed.Version, time.Now().UnixMilli(), deferFor.Milliseconds()}
|
||||
for _, scene := range claimed.Scenes {
|
||||
args = append(args, scene)
|
||||
}
|
||||
return deviceFinishScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisDeviceEvalClaimKey(claimed.DeviceID), constants.RedisDeviceEvalLockKey(claimed.DeviceID),
|
||||
constants.RedisDeviceEvalRunningKey, constants.RedisDeviceEvalVersionKey(claimed.DeviceID),
|
||||
constants.RedisDeviceEvalScenesKey(claimed.DeviceID), constants.RedisDeviceEvalQueueKey},
|
||||
args...).Int64()
|
||||
}
|
||||
|
||||
// ReleaseDeviceEval 发布失败时释放设备判断处理中标记。
|
||||
func (s *Store) ReleaseDeviceEval(ctx context.Context, deviceID uint, token string) error {
|
||||
return srReleaseScript.Run(ctx, s.redis,
|
||||
[]string{constants.RedisDeviceEvalClaimKey(deviceID), constants.RedisDeviceEvalRunningKey, constants.RedisDeviceEvalQueueKey},
|
||||
token, deviceID, time.Now().UnixMilli()).Err()
|
||||
}
|
||||
|
||||
// StopResumeQueueStats 是账户停复机队列的即时统计。
|
||||
type StopResumeQueueStats struct {
|
||||
Queued int64
|
||||
Due int64
|
||||
Running int64
|
||||
Pending int64
|
||||
}
|
||||
|
||||
// StopResumeStats 读取账户停复机队列长度、到期数与执行中数;待确认数按队列成员抽样统计(最多 500 个)。
|
||||
func (s *Store) StopResumeStats(ctx context.Context, carrierID uint) (StopResumeQueueStats, error) {
|
||||
now := strconv.FormatInt(time.Now().UnixMilli(), 10)
|
||||
queueKey := constants.RedisCarrierStopResumeQueueKey(carrierID)
|
||||
pipe := s.redis.Pipeline()
|
||||
queued := pipe.ZCard(ctx, queueKey)
|
||||
due := pipe.ZCount(ctx, queueKey, "-inf", now)
|
||||
running := pipe.ZCount(ctx, constants.RedisCarrierStopResumeRunningKey(carrierID), now, "+inf")
|
||||
members := pipe.ZRange(ctx, queueKey, 0, 499)
|
||||
if _, err := pipe.Exec(ctx); err != nil && !stderrors.Is(err, redis.Nil) {
|
||||
return StopResumeQueueStats{}, err
|
||||
}
|
||||
stats := StopResumeQueueStats{Queued: queued.Val(), Due: due.Val(), Running: running.Val()}
|
||||
if ids := members.Val(); len(ids) > 0 {
|
||||
check := s.redis.Pipeline()
|
||||
results := make([]*redis.StringCmd, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
cardID, convErr := strconv.ParseUint(strings.TrimSpace(id), 10, 64)
|
||||
if convErr != nil {
|
||||
continue
|
||||
}
|
||||
results = append(results, check.HGet(ctx, constants.RedisCardStopResumeCmdKey(uint(cardID)), srFieldPending))
|
||||
}
|
||||
_, _ = check.Exec(ctx)
|
||||
for _, result := range results {
|
||||
if result.Val() == "1" {
|
||||
stats.Pending++
|
||||
}
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// IncrStopResumeStat 累加账户停复机执行计数(10 分钟分桶,保留 2 小时)。
|
||||
func (s *Store) IncrStopResumeStat(ctx context.Context, carrierID uint, field string) {
|
||||
key := constants.RedisCarrierStopResumeStatKey(carrierID, time.Now().Unix()/constants.StopResumeStatBucketSeconds)
|
||||
pipe := s.redis.Pipeline()
|
||||
pipe.HIncrBy(ctx, key, field, 1)
|
||||
pipe.Expire(ctx, key, 2*time.Hour)
|
||||
_, _ = pipe.Exec(ctx)
|
||||
}
|
||||
|
||||
// StopResumeStatLastHour 汇总账户近 1 小时(最近 6 个分桶)的停复机执行计数。
|
||||
func (s *Store) StopResumeStatLastHour(ctx context.Context, carrierID uint) map[string]int64 {
|
||||
current := time.Now().Unix() / constants.StopResumeStatBucketSeconds
|
||||
pipe := s.redis.Pipeline()
|
||||
cmds := make([]*redis.MapStringStringCmd, 0, 6)
|
||||
for bucket := current - 5; bucket <= current; bucket++ {
|
||||
cmds = append(cmds, pipe.HGetAll(ctx, constants.RedisCarrierStopResumeStatKey(carrierID, bucket)))
|
||||
}
|
||||
_, _ = pipe.Exec(ctx)
|
||||
totals := map[string]int64{}
|
||||
for _, cmd := range cmds {
|
||||
for field, value := range cmd.Val() {
|
||||
count, _ := strconv.ParseInt(value, 10, 64)
|
||||
totals[field] += count
|
||||
}
|
||||
}
|
||||
return totals
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package polling
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
@@ -435,16 +436,17 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
if h.stopResumeCallback == nil {
|
||||
return
|
||||
}
|
||||
detachedCtx := context.WithoutCancel(ctx)
|
||||
// 套餐到期即停:以"套餐到期"场景请求评估,暂停轮询的卡同样停机(只触发一次,不做补偿扫描)。
|
||||
detachedCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeScenePackageExpired)
|
||||
|
||||
if carrierType == "iot_card" {
|
||||
go func() {
|
||||
stopresumedomain.RunAfterCommit(detachedCtx, h.stopResumeCallback, func() {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, carrierID); err != nil {
|
||||
h.logger.Error("套餐过期后停机失败",
|
||||
zap.Uint("card_id", carrierID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}()
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -457,14 +459,14 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
return
|
||||
}
|
||||
for _, b := range bindings {
|
||||
cardID := b.IotCardID
|
||||
go func(cID uint) {
|
||||
cID := b.IotCardID
|
||||
stopresumedomain.RunAfterCommit(detachedCtx, h.stopResumeCallback, func() {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, cID); err != nil {
|
||||
h.logger.Error("套餐过期后停机失败",
|
||||
zap.Uint("card_id", cID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(cardID)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -482,6 +484,7 @@ func (h *PackageActivationHandler) reconcileCarrierAfterActivation(ctx context.C
|
||||
if carrierID == 0 {
|
||||
return errors.New(errors.CodeInvalidParam, "套餐使用记录缺少有效载体")
|
||||
}
|
||||
ctx = stopresumedomain.WithScene(ctx, constants.StopResumeSceneActivation)
|
||||
|
||||
if carrierType == constants.AssetTypeIotCard {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(ctx, carrierID); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user