- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
185 lines
6.5 KiB
Go
185 lines
6.5 KiB
Go
package gatewaylimit
|
||
|
||
import (
|
||
"context"
|
||
"strconv"
|
||
"time"
|
||
|
||
"github.com/redis/go-redis/v9"
|
||
"go.uber.org/zap"
|
||
|
||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||
)
|
||
|
||
// 分钟计数 Hash 的字段名。
|
||
const (
|
||
statFieldWouldLimit = "would_limit"
|
||
statFieldDurationSum = "duration_ms_sum"
|
||
statFieldAttempts = "attempts"
|
||
statPrefixExempt = "exempt:"
|
||
statPrefixUpstream = "upstream:"
|
||
statPrefixDuration = "duration_le:"
|
||
)
|
||
|
||
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
|
||
const StatFieldRateLimitWait = "rate_wait"
|
||
|
||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||
|
||
// Guard 实现 gateway.CallGuard:每次卡网关 HTTP 尝试前按运营商账户限流,结束后写分钟计数与卡最近错误。
|
||
type Guard struct {
|
||
limiter *Limiter
|
||
resolver *CarrierResolver
|
||
classifier *Classifier
|
||
redis *redis.Client
|
||
logger *zap.Logger
|
||
}
|
||
|
||
// NewGuard 创建卡网关调用守卫。
|
||
func NewGuard(limiter *Limiter, resolver *CarrierResolver, classifier *Classifier, redisClient *redis.Client, logger *zap.Logger) *Guard {
|
||
if logger == nil {
|
||
logger = zap.NewNop()
|
||
}
|
||
return &Guard{limiter: limiter, resolver: resolver, classifier: classifier, redis: redisClient, logger: logger}
|
||
}
|
||
|
||
var _ gateway.CallGuard = (*Guard)(nil)
|
||
|
||
// BeforeAttempt 取限流名额;豁免调用直接放行。拦截模式下拿不到名额返回限流错误,本次方法调用随即结束。
|
||
func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error {
|
||
if g == nil || g.limiter == nil || call.ExemptSource != "" {
|
||
return nil
|
||
}
|
||
carrierID := g.carrierOf(ctx, call)
|
||
decision, err := g.limiter.Check(ctx, carrierID, call.Operation)
|
||
if err != nil {
|
||
// 限流器故障时失败开放,避免 Redis 抖动阻断全部网关调用。
|
||
g.logger.Warn("网关限流判断失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(err))
|
||
return nil
|
||
}
|
||
if decision.Exceed {
|
||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||
}
|
||
if !decision.Allowed {
|
||
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
|
||
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
|
||
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
|
||
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
|
||
if allowed {
|
||
return nil
|
||
}
|
||
if waitErr != nil && ctx.Err() == nil {
|
||
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
|
||
return nil
|
||
}
|
||
}
|
||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||
return gateway.NewRateLimitedError()
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// AfterAttempt 写分钟计数(按结果类别与耗时分桶);查询失败时写卡该操作最近一次错误。
|
||
func (g *Guard) AfterAttempt(ctx context.Context, call gateway.CallInfo, callErr error, duration time.Duration) {
|
||
if g == nil || g.redis == nil {
|
||
return
|
||
}
|
||
ctx = context.WithoutCancel(ctx)
|
||
carrierID := g.carrierOf(ctx, call)
|
||
field := constants.GatewayResultSuccess
|
||
var classification Classification
|
||
gwErr, isGatewayErr := gateway.AsGatewayError(callErr)
|
||
if callErr != nil {
|
||
field = constants.GatewayErrorNetwork
|
||
if isGatewayErr {
|
||
field = gwErr.Category
|
||
if gwErr.Category == constants.GatewayErrorUpstream {
|
||
classification = g.classifier.Classify(ctx, call.Operation, gwErr.ProviderMessage)
|
||
field = statPrefixUpstream + classification.Category
|
||
}
|
||
}
|
||
}
|
||
key := constants.RedisGatewayCallStatKey(carrierID, call.Operation, statMinute(time.Now()))
|
||
durationMS := duration.Milliseconds()
|
||
pipe := g.redis.Pipeline()
|
||
pipe.HIncrBy(ctx, key, field, 1)
|
||
pipe.HIncrBy(ctx, key, statFieldAttempts, 1)
|
||
pipe.HIncrBy(ctx, key, statFieldDurationSum, durationMS)
|
||
pipe.HIncrBy(ctx, key, statPrefixDuration+durationBucket(durationMS), 1)
|
||
if call.ExemptSource != "" {
|
||
pipe.HIncrBy(ctx, key, statPrefixExempt+call.ExemptSource, 1)
|
||
}
|
||
pipe.Expire(ctx, key, constants.GatewayCallStatTTL)
|
||
if callErr != nil && isQueryOperation(call.Operation) && call.CardNo != "" {
|
||
errKey := constants.RedisGatewayCardLastErrorKey(call.CardNo, call.Operation)
|
||
values := map[string]any{"category": field, "at": time.Now().Format(time.RFC3339)}
|
||
if isGatewayErr {
|
||
values["provider_code"] = strconv.Itoa(gwErr.ProviderCode)
|
||
values["provider_message"] = gwErr.ProviderMessage
|
||
}
|
||
pipe.HSet(ctx, errKey, values)
|
||
pipe.Expire(ctx, errKey, constants.GatewayCardLastErrorTTL)
|
||
}
|
||
if _, err := pipe.Exec(ctx); err != nil {
|
||
g.logger.Debug("写网关调用统计失败", zap.Error(err))
|
||
}
|
||
}
|
||
|
||
// Classify 暴露分类器供停复机记录等调用方对上游错误分类。
|
||
func (g *Guard) Classify(ctx context.Context, op, message string) Classification {
|
||
if g == nil {
|
||
return Classification{Category: constants.GatewayClassBusiness}
|
||
}
|
||
return g.classifier.Classify(ctx, op, message)
|
||
}
|
||
|
||
func (g *Guard) carrierOf(ctx context.Context, call gateway.CallInfo) uint {
|
||
if call.CarrierID != 0 || g.resolver == nil {
|
||
return call.CarrierID
|
||
}
|
||
carrierID, err := g.resolver.Resolve(ctx, call.CardNo)
|
||
if err != nil {
|
||
g.logger.Debug("解析卡所属运营商账户失败,按未知账户计量", zap.String("iccid", call.CardNo), zap.Error(err))
|
||
}
|
||
return carrierID
|
||
}
|
||
|
||
func (g *Guard) incr(ctx context.Context, carrierID uint, op, field string) {
|
||
if g.redis == nil {
|
||
return
|
||
}
|
||
key := constants.RedisGatewayCallStatKey(carrierID, op, statMinute(time.Now()))
|
||
pipe := g.redis.Pipeline()
|
||
pipe.HIncrBy(ctx, key, field, 1)
|
||
pipe.Expire(ctx, key, constants.GatewayCallStatTTL)
|
||
_, _ = pipe.Exec(context.WithoutCancel(ctx))
|
||
}
|
||
|
||
func isQueryOperation(op string) bool {
|
||
return op == constants.GatewayOpQueryFlow || op == constants.GatewayOpQueryStatus || op == constants.GatewayOpQueryRealname
|
||
}
|
||
|
||
// statMinute 返回上海时区的分钟标识(yyyyMMddHHmm)。
|
||
func statMinute(t time.Time) string {
|
||
return t.In(shanghai()).Format("200601021504")
|
||
}
|
||
|
||
func durationBucket(ms int64) string {
|
||
for _, upper := range durationBuckets {
|
||
if ms <= upper {
|
||
return strconv.FormatInt(upper, 10)
|
||
}
|
||
}
|
||
return "inf"
|
||
}
|
||
|
||
func shanghai() *time.Location {
|
||
location, err := time.LoadLocation("Asia/Shanghai")
|
||
if err != nil {
|
||
return time.FixedZone("CST", 8*3600)
|
||
}
|
||
return location
|
||
}
|