feat: 按运营商账户调度的分发器、卡任务执行器与模式切换
- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
@@ -22,6 +22,9 @@ const (
|
||||
statPrefixDuration = "duration_le:"
|
||||
)
|
||||
|
||||
// StatFieldRateLimitWait 是等待模式下进入限流等待的次数字段(等待超时另计入 rate_limited)。
|
||||
const StatFieldRateLimitWait = "rate_wait"
|
||||
|
||||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||||
|
||||
@@ -60,6 +63,18 @@ func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error
|
||||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||||
}
|
||||
if !decision.Allowed {
|
||||
// 等待模式(NSQ 消费者):等名额直到超时;超时按限流错误返回,调用方按短延迟重排、不计失败。
|
||||
if maxWait := gateway.RateLimitWait(ctx); maxWait > 0 {
|
||||
g.incr(ctx, carrierID, call.Operation, StatFieldRateLimitWait)
|
||||
allowed, waitErr := g.limiter.Wait(ctx, carrierID, call.Operation, maxWait)
|
||||
if allowed {
|
||||
return nil
|
||||
}
|
||||
if waitErr != nil && ctx.Err() == nil {
|
||||
g.logger.Warn("网关限流等待失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(waitErr))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||||
return gateway.NewRateLimitedError()
|
||||
}
|
||||
|
||||
@@ -189,3 +189,50 @@ func windowMember(nowMS int64) string {
|
||||
_, _ = rand.Read(buf)
|
||||
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// Budget 返回运营商账户各查询类操作所属限流组本秒剩余名额,供分发器按预计调用数领取。
|
||||
// groups 为限流组 → 本秒剩余;opGroup 为操作 → 限流组。只统计模式下返回 limited=false(不按 QPS 限制领取)。
|
||||
// 全局组只在账户组之外再收紧:剩余名额取两者较小值。
|
||||
func (l *Limiter) Budget(ctx context.Context, carrierID uint, ops []string) (groups map[string]int, opGroup map[string]string, limited bool) {
|
||||
if l == nil || l.redis == nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
settings := l.settings.Current(ctx)
|
||||
if !settings.Enforce() {
|
||||
return nil, nil, false
|
||||
}
|
||||
groups, opGroup = map[string]int{}, map[string]string{}
|
||||
now := time.Now().UnixMilli()
|
||||
since := now - constants.GatewayRateLimitWindow.Milliseconds()
|
||||
for _, op := range ops {
|
||||
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
|
||||
if !accountOK {
|
||||
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
|
||||
}
|
||||
code := account.GroupCode
|
||||
opGroup[op] = code
|
||||
if _, seen := groups[code]; seen {
|
||||
continue
|
||||
}
|
||||
limit := account.MaxQPS
|
||||
if !account.HighPriority {
|
||||
limit -= account.ReservedQPS
|
||||
}
|
||||
used, err := l.redis.ZCount(ctx, constants.RedisGatewayQPSKey(carrierID, code), strconv.FormatInt(since, 10), "+inf").Result()
|
||||
if err != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
remaining := limit - int(used)
|
||||
if globalOK {
|
||||
globalUsed, gerr := l.redis.ZCount(ctx, constants.RedisGatewayGlobalQPSKey(global.GroupCode), strconv.FormatInt(since, 10), "+inf").Result()
|
||||
if gerr == nil && global.MaxQPS-int(globalUsed) < remaining {
|
||||
remaining = global.MaxQPS - int(globalUsed)
|
||||
}
|
||||
}
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
groups[code] = remaining
|
||||
}
|
||||
return groups, opGroup, true
|
||||
}
|
||||
|
||||
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
215
internal/infrastructure/nsqbus/nsqbus.go
Normal file
@@ -0,0 +1,215 @@
|
||||
// Package nsqbus 封装卡轮询使用的 NSQ 生产者与按 topic 动态订阅的消费者管理器。
|
||||
// NSQ 只承担执行分发与消费者并发控制,业务状态与重试全部在 Redis(add-carrier-polling-scheduler)。
|
||||
package nsqbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nsqio/go-nsq"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Producer 是 nsqd 生产者。
|
||||
type Producer struct {
|
||||
producer *nsq.Producer
|
||||
}
|
||||
|
||||
// NewProducer 连接 nsqd TCP 地址创建生产者。
|
||||
func NewProducer(address string, logger *zap.Logger) (*Producer, error) {
|
||||
cfg := nsq.NewConfig()
|
||||
producer, err := nsq.NewProducer(address, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
producer.SetLogger(nsqLogger{logger: logger}, nsq.LogLevelWarning)
|
||||
return &Producer{producer: producer}, nil
|
||||
}
|
||||
|
||||
// Publish 同步发布一条消息;topic 不存在时由 nsqd 自动创建。
|
||||
func (p *Producer) Publish(topic string, body []byte) error {
|
||||
return p.producer.Publish(topic, body)
|
||||
}
|
||||
|
||||
// Stop 停止生产者。
|
||||
func (p *Producer) Stop() {
|
||||
if p != nil && p.producer != nil {
|
||||
p.producer.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// Handler 处理一条消息;返回后由管理器 FIN。消息处理期间可调用 touch 延长 NSQ 消息超时。
|
||||
type Handler func(ctx context.Context, body []byte, touch func())
|
||||
|
||||
// ConsumerConfig 是消费者参数。
|
||||
type ConsumerConfig struct {
|
||||
NSQDAddress string
|
||||
NSQLookupdAddresses []string
|
||||
MsgTimeout time.Duration
|
||||
MaxAttempts uint16
|
||||
}
|
||||
|
||||
// ConsumerManager 按 topic 动态订阅与取消订阅;每个 topic 一个消费者,max_in_flight 控制本进程对该 topic 的并发。
|
||||
type ConsumerManager struct {
|
||||
cfg ConsumerConfig
|
||||
channel string
|
||||
handler Handler
|
||||
logger *zap.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
consumers map[string]*subscription
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
inflight sync.WaitGroup
|
||||
}
|
||||
|
||||
type subscription struct {
|
||||
consumer *nsq.Consumer
|
||||
maxInFlight int
|
||||
}
|
||||
|
||||
// NewConsumerManager 创建消费者管理器。
|
||||
func NewConsumerManager(cfg ConsumerConfig, channel string, handler Handler, logger *zap.Logger) *ConsumerManager {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &ConsumerManager{cfg: cfg, channel: channel, handler: handler, logger: logger,
|
||||
consumers: map[string]*subscription{}, ctx: ctx, cancel: cancel}
|
||||
}
|
||||
|
||||
// Sync 使订阅集合与 desired(topic → max_in_flight)一致:新增订阅、取消多余订阅、调整 max_in_flight。
|
||||
func (m *ConsumerManager) Sync(desired map[string]int) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
for topic, sub := range m.consumers {
|
||||
if _, keep := desired[topic]; !keep {
|
||||
sub.consumer.Stop()
|
||||
delete(m.consumers, topic)
|
||||
m.logger.Info("取消订阅卡任务 topic", zap.String("topic", topic))
|
||||
}
|
||||
}
|
||||
for topic, maxInFlight := range desired {
|
||||
if maxInFlight <= 0 {
|
||||
maxInFlight = 1
|
||||
}
|
||||
if sub, ok := m.consumers[topic]; ok {
|
||||
if sub.maxInFlight != maxInFlight {
|
||||
sub.consumer.ChangeMaxInFlight(maxInFlight)
|
||||
sub.maxInFlight = maxInFlight
|
||||
}
|
||||
continue
|
||||
}
|
||||
consumer, err := m.subscribe(topic, maxInFlight)
|
||||
if err != nil {
|
||||
m.logger.Warn("订阅卡任务 topic 失败,下一轮重试", zap.String("topic", topic), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
m.consumers[topic] = &subscription{consumer: consumer, maxInFlight: maxInFlight}
|
||||
m.logger.Info("订阅卡任务 topic", zap.String("topic", topic), zap.Int("max_in_flight", maxInFlight))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ConsumerManager) subscribe(topic string, maxInFlight int) (*nsq.Consumer, error) {
|
||||
cfg := nsq.NewConfig()
|
||||
cfg.MaxInFlight = maxInFlight
|
||||
if m.cfg.MsgTimeout > 0 {
|
||||
cfg.MsgTimeout = m.cfg.MsgTimeout
|
||||
}
|
||||
if m.cfg.MaxAttempts > 0 {
|
||||
cfg.MaxAttempts = m.cfg.MaxAttempts
|
||||
}
|
||||
consumer, err := nsq.NewConsumer(topic, m.channel, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
consumer.SetLogger(nsqLogger{logger: m.logger}, nsq.LogLevelWarning)
|
||||
handlerCtx := m.ctx // subscribe 在持锁时调用,捕获当前处理上下文
|
||||
consumer.AddConcurrentHandlers(nsq.HandlerFunc(func(message *nsq.Message) error {
|
||||
message.DisableAutoResponse()
|
||||
m.inflight.Add(1)
|
||||
defer m.inflight.Done()
|
||||
defer message.Finish()
|
||||
m.handler(handlerCtx, message.Body, message.Touch)
|
||||
return nil
|
||||
}), maxInFlight)
|
||||
if len(m.cfg.NSQLookupdAddresses) > 0 {
|
||||
err = consumer.ConnectToNSQLookupds(m.cfg.NSQLookupdAddresses)
|
||||
} else {
|
||||
err = consumer.ConnectToNSQD(m.cfg.NSQDAddress)
|
||||
}
|
||||
if err != nil {
|
||||
consumer.Stop()
|
||||
return nil, err
|
||||
}
|
||||
return consumer, nil
|
||||
}
|
||||
|
||||
// Topics 返回当前已订阅的 topic。
|
||||
func (m *ConsumerManager) Topics() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
topics := make([]string, 0, len(m.consumers))
|
||||
for topic := range m.consumers {
|
||||
topics = append(topics, topic)
|
||||
}
|
||||
return topics
|
||||
}
|
||||
|
||||
// StopAll 停止全部订阅;在宽限期内等待执行中的消息完成并 FIN,超过宽限期后取消处理上下文。
|
||||
func (m *ConsumerManager) StopAll(grace time.Duration) {
|
||||
m.mu.Lock()
|
||||
for topic, sub := range m.consumers {
|
||||
sub.consumer.Stop()
|
||||
delete(m.consumers, topic)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
m.inflight.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(grace):
|
||||
m.logger.Warn("优雅退出宽限期内仍有卡任务未完成,取消执行(由租约到期后重新领取)")
|
||||
}
|
||||
m.cancel()
|
||||
}
|
||||
|
||||
// Reset 在 StopAll 后重新允许订阅(切回 carrier 模式时使用)。
|
||||
func (m *ConsumerManager) Reset() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.ctx.Err() != nil {
|
||||
m.ctx, m.cancel = context.WithCancel(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
// SplitAddresses 解析逗号分隔的地址列表。
|
||||
func SplitAddresses(raw string) []string {
|
||||
var result []string
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
result = append(result, part)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type nsqLogger struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// Output 实现 go-nsq 的日志接口。
|
||||
func (l nsqLogger) Output(_ int, s string) error {
|
||||
if l.logger != nil {
|
||||
l.logger.Warn("NSQ", zap.String("message", s))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user