feat: 按运营商账户调度的分发器、卡任务执行器与模式切换
- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
@@ -22,6 +22,9 @@ const (
|
||||
statPrefixDuration = "duration_le:"
|
||||
)
|
||||
|
||||
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
|
||||
const StatFieldRateLimitWait = "rate_wait"
|
||||
|
||||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||||
|
||||
@@ -60,6 +63,18 @@ func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error
|
||||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||||
}
|
||||
if !decision.Allowed {
|
||||
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
|
||||
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
|
||||
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
|
||||
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
|
||||
if allowed {
|
||||
return nil
|
||||
}
|
||||
if waitErr != nil && ctx.Err() == nil {
|
||||
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||||
return gateway.NewRateLimitedError()
|
||||
}
|
||||
|
||||
@@ -189,3 +189,50 @@ func windowMember(nowMS int64) string {
|
||||
_, _ = rand.Read(buf)
|
||||
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// Budget 返回运营商账户各查询类操作所属限流组本秒剩余名额,供分发器按预计调用数领取。
|
||||
// groups 为限流组 → 本秒剩余;opGroup 为操作 → 限流组。只统计模式下返回 limited=false(不按 QPS 限制领取)。
|
||||
// 全局组只在账户组之外再收紧:剩余名额取两者较小值。
|
||||
func (l *Limiter) Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool) {
|
||||
if l == nil || l.redis == nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
settings := l.settings.Current(ctx)
|
||||
if !settings.Enforce() {
|
||||
return nil, nil, false
|
||||
}
|
||||
groups, opGroup = map[string]int{}, map[string]string{}
|
||||
now := time.Now().UnixMilli()
|
||||
since := now - constants.GatewayRateLimitWindow.Milliseconds()
|
||||
for _, op := range ops {
|
||||
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
|
||||
if !accountOK {
|
||||
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
|
||||
}
|
||||
code := account.GroupCode
|
||||
opGroup[op] = code
|
||||
if _, seen := groups[code]; seen {
|
||||
continue
|
||||
}
|
||||
limit := account.MaxQPS
|
||||
if !account.HighPriority {
|
||||
limit -= account.ReservedQPS
|
||||
}
|
||||
used, err := l.redis.ZCount(ctx, constants.RedisGatewayQPSKey(carrierID, code), strconv.FormatInt(since, 10), "+inf").Result()
|
||||
if err != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
remaining := limit - int(used)
|
||||
if globalOK {
|
||||
globalUsed, gerr := l.redis.ZCount(ctx, constants.RedisGatewayGlobalQPSKey(global.GroupCode), strconv.FormatInt(since, 10), "+inf").Result()
|
||||
if gerr == nil && global.MaxQPS-int(globalUsed) < remaining {
|
||||
remaining = global.MaxQPS - int(globalUsed)
|
||||
}
|
||||
}
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
groups[code] = remaining
|
||||
}
|
||||
return groups, opGroup, true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user