feat: 按运营商账户调度的分发器、卡任务执行器与模式切换

- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
邹进
2026-10-25 18:44:23 +09:00
parent e7f8c4bae9
commit 2fe7c1f8ac
23 changed files with 1899 additions and 72 deletions

View File

@@ -0,0 +1,299 @@
package task
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"sync/atomic"
"time"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/gateway"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/polling"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// CarrierCardExecutor 按运营商账户调度的卡任务执行器:一张卡一个卡任务,按配置步骤顺序串行执行已到期的检查类型。
type CarrierCardExecutor struct {
base *PollingBase
store *carrierqueue.Store
settings *carrierqueue.SettingsProvider
alerts *carrierqueue.AlertWriter
realname *PollingRealnameHandler
carddata *PollingCarddataHandler
cardStatus *PollingCardStatusHandler
pkg *PollingPackageHandler
cfg config.CarrierSchedulerConfig
instanceID string
}
// NewCarrierCardExecutor 创建卡任务执行器。
func NewCarrierCardExecutor(base *PollingBase, store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, alerts *carrierqueue.AlertWriter,
realname *PollingRealnameHandler, carddata *PollingCarddataHandler, cardStatus *PollingCardStatusHandler, pkg *PollingPackageHandler,
cfg config.CarrierSchedulerConfig, instanceID string) *CarrierCardExecutor {
return &CarrierCardExecutor{base: base, store: store, settings: settings, alerts: alerts, realname: realname, carddata: carddata,
cardStatus: cardStatus, pkg: pkg, cfg: cfg, instanceID: instanceID}
}
func (e *CarrierCardExecutor) renewInterval() time.Duration {
if e.cfg.RenewInterval > 0 {
return e.cfg.RenewInterval
}
return 30 * time.Second
}
func (e *CarrierCardExecutor) lease() time.Duration {
if e.cfg.Lease > 0 {
return e.cfg.Lease
}
return 5 * time.Minute
}
// Handle 消费一条卡任务消息;返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
func (e *CarrierCardExecutor) Handle(ctx context.Context, body []byte, touch func()) {
var msg carrierqueue.CardMessage
if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
e.base.logger.Warn("卡任务消息无效,丢弃", zap.ByteString("body", body))
return
}
execValue := msg.Token + "|" + e.instanceID
execTTL := e.renewInterval() * 2
got, err := e.store.AcquireExec(ctx, msg.CardID, msg.Token, execValue, execTTL)
if err != nil || got != 1 {
// 0:处理中标记已不是该 token(过期或重复投递);-1:该 token 已有执行者(NSQ 重投时原消费者仍在执行)。
return
}
settings := e.settings.Current(ctx)
taskCtx, cancel := context.WithTimeout(ctx, settings.TaskTimeout())
defer cancel()
var lost atomic.Bool
renewDone := make(chan struct{})
go func() {
ticker := time.NewTicker(e.renewInterval())
defer ticker.Stop()
for {
select {
case <-renewDone:
return
case <-taskCtx.Done():
return
case <-ticker.C:
ok, renewErr := e.store.Renew(context.WithoutCancel(ctx), msg.CardID, msg.CarrierID, msg.Token, execValue, e.lease(), execTTL)
if renewErr == nil && !ok {
// token 已不一致:立即停止后续步骤,不再发起新的网关调用,只写已产生的业务结果。
lost.Store(true)
cancel()
return
}
if touch != nil {
touch()
}
}
}
}()
set, del := e.execute(taskCtx, msg, settings, &lost)
close(renewDone)
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer finishCancel()
finished, finishErr := e.store.Finish(finishCtx, msg.CardID, msg.CarrierID, msg.Token, set, del)
if finishErr != nil {
e.base.logger.Warn("卡任务收尾失败,等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(finishErr))
return
}
if !finished {
e.base.logger.Warn("卡任务收尾时处理中标记已不一致,只写入业务结果", zap.Uint("card_id", msg.CardID),
zap.String("metric", "polling.carrier.finish_token_mismatch"))
}
}
// execute 串行执行到期检查步骤,返回收尾要写入与删除的到期字段。
func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.CardMessage, settings carrierqueue.Settings, lost *atomic.Bool) (map[string]string, []string) {
set, del := map[string]string{}, []string{}
card, err := e.base.getCardWithCache(ctx, msg.CardID)
if err != nil {
if isNotFound(err) {
e.removeCard(ctx, msg.CardID, msg.CarrierID)
}
return set, del
}
if !card.EnablePolling || (e.base.scopeChecker != nil && !e.base.scopeChecker.ShouldEnqueue(ctx, card)) {
e.removeCard(ctx, msg.CardID, msg.CarrierID)
return set, del
}
due, err := e.store.ReadDue(ctx, msg.CardID)
if err != nil {
return set, del
}
now := time.Now()
flowEvaluated := false
executed := map[string]StepOutcome{}
for _, checkType := range settings.StepOrder {
item, ok := due.Types[checkType]
if !ok || !item.HasDue || item.Due.After(now) {
continue
}
if lost.Load() || ctx.Err() != nil {
break
}
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow && flowEvaluated {
executed[checkType] = stepOK()
continue
}
outcome := e.runStep(ctx, msg, checkType, settings.StepTimeout(checkType))
if checkType == constants.CheckTypeCarddata && outcome.TriggeredEvaluation {
flowEvaluated = true
}
executed[checkType] = outcome
if outcome.Result == StepCardGone || outcome.Result == StepStopPolling {
e.removeCard(ctx, msg.CardID, msg.CarrierID)
return set, del
}
}
if len(executed) == 0 {
return set, del
}
// 收尾时按最新卡数据重新匹配间隔(步骤中卡状态、套餐可能已变化)。
e.base.invalidateCardCache(ctx, msg.CardID)
if fresh, freshErr := e.base.getCardWithCache(ctx, msg.CardID); freshErr == nil {
card = fresh
}
intervals := e.base.configMgr.MergedTaskIntervals(card)
now = time.Now()
for checkType, outcome := range executed {
e.writeBack(ctx, card, checkType, outcome, due.Types[checkType], intervals, settings, now, set, &del)
}
return set, del
}
// runStep 以单步超时、等待模式限流与 panic 恢复执行一个检查步骤。
func (e *CarrierCardExecutor) runStep(ctx context.Context, msg carrierqueue.CardMessage, checkType string, timeout time.Duration) (outcome StepOutcome) {
stepCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
stepCtx = gateway.WithRateLimitWait(stepCtx, timeout)
defer func() {
if r := recover(); r != nil {
e.base.logger.Error("卡任务步骤 panic,按失败处理", zap.Uint("card_id", msg.CardID), zap.String("check_type", checkType), zap.Any("panic", r))
outcome = StepOutcome{Result: StepFailed, Category: "panic", Message: fmt.Sprint(r)}
}
}()
switch checkType {
case constants.CheckTypeCardStatus:
return e.cardStatus.Step(stepCtx, msg.CardID)
case constants.CheckTypeCarddata:
return e.carddata.Step(stepCtx, msg.CardID)
case constants.CheckTypeRealname:
return e.realname.Step(stepCtx, msg.CardID)
case constants.CheckTypePackage:
stepCtx = withPollingWorkerAuditContext(stepCtx, constants.TaskTypePollingPackage, "套餐状态轮询任务", msg.Token)
return e.pkg.Step(stepCtx, msg.CardID)
default:
return stepSkip()
}
}
// writeBack 按步骤结果计算该检查类型下次到期、失败计数与延迟确认,并触发告警或恢复。
func (e *CarrierCardExecutor) writeBack(ctx context.Context, card *model.IotCard, checkType string, outcome StepOutcome, previous carrierqueue.TypeDue,
intervals map[string]polling.TaskTypeInterval, settings carrierqueue.Settings, now time.Time, set map[string]string, del *[]string) {
info, ok := intervals[constants.TaskTypeFromCheckType(checkType)]
if !ok || info.Interval <= 0 {
// 该类型已不在配置中(间隔为 0 或未配置):移除到期,不再执行。
*del = append(*del, carrierqueue.FieldDue(checkType), carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType),
carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
return
}
interval := time.Duration(info.Interval) * time.Second
normalNext := e.base.configMgr.NextCheckAt(ctx, card, constants.TaskTypeFromCheckType(checkType), info, now)
switch outcome.Result {
case StepSucceeded:
next := normalNext
set[carrierqueue.FieldOK(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
if previous.FailCount > 0 {
e.alerts.ReportRecovered(card.ID, checkType)
}
*del = append(*del, carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType), carrierqueue.FieldError(checkType))
// 延迟确认:结果已满足预期时取消,否则按序列取下一次确认 = min(正常间隔, 当前 + 确认延迟)。
if len(previous.Confirm) > 0 {
if previous.Expected != "" && outcome.Observed == previous.Expected {
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
} else {
if confirmAt := now.Add(previous.Confirm[0]); confirmAt.Before(next) {
next = confirmAt
}
if rest := previous.Confirm[1:]; len(rest) > 0 {
set[carrierqueue.FieldConfirm(checkType)] = joinSeconds(rest)
} else {
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
}
}
}
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(next.UnixMilli(), 10)
case StepFailed:
set[carrierqueue.FieldError(checkType)] = truncateRunes(outcome.Category+"|"+outcome.Message, 300)
if !outcome.CountsAsFailure() {
// 上游频率限制、已是目标状态:不计入连续失败,按首个退避间隔重试。
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, 1, interval)).UnixMilli(), 10)
return
}
failCount := previous.FailCount + 1
firstFailed := previous.FirstFailed
if firstFailed.IsZero() {
firstFailed = now
set[carrierqueue.FieldFirstFail(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
}
set[carrierqueue.FieldFail(checkType)] = strconv.Itoa(failCount)
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, failCount, interval)).UnixMilli(), 10)
if failCount >= settings.AlertThreshold(checkType) || now.Sub(firstFailed) >= settings.AlertFailDuration || settings.ImmediateAlert[outcome.Category] {
e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, CheckType: checkType,
Category: outcome.Category, Message: outcome.Message, FailCount: failCount, FirstFailed: firstFailed, At: now})
}
case StepRateLimited:
// 限流等待超时:请求未发出,按短延迟重排,不计失败。
delay := 5 * time.Second
if e.base.rateLimitedDelay != nil {
if configured := e.base.rateLimitedDelay(); configured > 0 {
delay = configured
}
}
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
case StepDeferred:
delay := outcome.DeferDelay
if delay <= 0 {
delay = 30 * time.Second
}
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
default:
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(normalNext.UnixMilli(), 10)
}
}
func (e *CarrierCardExecutor) removeCard(ctx context.Context, cardID, carrierID uint) {
removeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
defer cancel()
if err := e.store.Remove(removeCtx, cardID, carrierID); err != nil {
e.base.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
}
}
func joinSeconds(delays []time.Duration) string {
parts := make([]string, 0, len(delays))
for _, delay := range delays {
parts = append(parts, strconv.FormatInt(int64(delay/time.Second), 10))
}
return strings.Join(parts, ",")
}
func truncateRunes(s string, max int) string {
runes := []rune(s)
if len(runes) <= max {
return s
}
return string(runes[:max])
}