feat: 按运营商账户调度的分发器、卡任务执行器与模式切换

- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
邹进
2026-10-25 18:44:23 +09:00
parent e7f8c4bae9
commit 2fe7c1f8ac
23 changed files with 1899 additions and 72 deletions

View File

@@ -22,6 +22,9 @@ const (
statPrefixDuration = "duration_le:"
)
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
const StatFieldRateLimitWait = "rate_wait"
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
@@ -60,6 +63,18 @@ func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
}
if !decision.Allowed {
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
if allowed {
return nil
}
if waitErr != nil && ctx.Err() == nil {
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
return nil
}
}
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
return gateway.NewRateLimitedError()
}

View File

@@ -189,3 +189,50 @@ func windowMember(nowMS int64) string {
_, _ = rand.Read(buf)
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
}
// Budget 返回运营商账户各查询类操作所属限流组本秒剩余名额,供分发器按预计调用数领取。
// groups 为限流组 → 本秒剩余;opGroup 为操作 → 限流组。只统计模式下返回 limited=false(不按 QPS 限制领取)。
// 全局组只在账户组之外再收紧:剩余名额取两者较小值。
func (l *Limiter) Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool) {
if l == nil || l.redis == nil {
return nil, nil, false
}
settings := l.settings.Current(ctx)
if !settings.Enforce() {
return nil, nil, false
}
groups, opGroup = map[string]int{}, map[string]string{}
now := time.Now().UnixMilli()
since := now - constants.GatewayRateLimitWindow.Milliseconds()
for _, op := range ops {
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
if !accountOK {
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
}
code := account.GroupCode
opGroup[op] = code
if _, seen := groups[code]; seen {
continue
}
limit := account.MaxQPS
if !account.HighPriority {
limit -= account.ReservedQPS
}
used, err := l.redis.ZCount(ctx, constants.RedisGatewayQPSKey(carrierID, code), strconv.FormatInt(since, 10), "+inf").Result()
if err != nil {
return nil, nil, false
}
remaining := limit - int(used)
if globalOK {
globalUsed, gerr := l.redis.ZCount(ctx, constants.RedisGatewayGlobalQPSKey(global.GroupCode), strconv.FormatInt(since, 10), "+inf").Result()
if gerr == nil && global.MaxQPS-int(globalUsed) < remaining {
remaining = global.MaxQPS - int(globalUsed)
}
}
if remaining < 0 {
remaining = 0
}
groups[code] = remaining
}
return groups, opGroup, true
}