feat: 按运营商账户调度的分发器、卡任务执行器与模式切换
- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
299
internal/task/carrier_card_executor.go
Normal file
299
internal/task/carrier_card_executor.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// CarrierCardExecutor 按运营商账户调度的卡任务执行器:一张卡一个卡任务,按配置步骤顺序串行执行已到期的检查类型。
|
||||
type CarrierCardExecutor struct {
|
||||
base *PollingBase
|
||||
store *carrierqueue.Store
|
||||
settings *carrierqueue.SettingsProvider
|
||||
alerts *carrierqueue.AlertWriter
|
||||
realname *PollingRealnameHandler
|
||||
carddata *PollingCarddataHandler
|
||||
cardStatus *PollingCardStatusHandler
|
||||
pkg *PollingPackageHandler
|
||||
cfg config.CarrierSchedulerConfig
|
||||
instanceID string
|
||||
}
|
||||
|
||||
// NewCarrierCardExecutor 创建卡任务执行器。
|
||||
func NewCarrierCardExecutor(base *PollingBase, store *carrierqueue.Store, settings *carrierqueue.SettingsProvider, alerts *carrierqueue.AlertWriter,
|
||||
realname *PollingRealnameHandler, carddata *PollingCarddataHandler, cardStatus *PollingCardStatusHandler, pkg *PollingPackageHandler,
|
||||
cfg config.CarrierSchedulerConfig, instanceID string) *CarrierCardExecutor {
|
||||
return &CarrierCardExecutor{base: base, store: store, settings: settings, alerts: alerts, realname: realname, carddata: carddata,
|
||||
cardStatus: cardStatus, pkg: pkg, cfg: cfg, instanceID: instanceID}
|
||||
}
|
||||
|
||||
func (e *CarrierCardExecutor) renewInterval() time.Duration {
|
||||
if e.cfg.RenewInterval > 0 {
|
||||
return e.cfg.RenewInterval
|
||||
}
|
||||
return 30 * time.Second
|
||||
}
|
||||
|
||||
func (e *CarrierCardExecutor) lease() time.Duration {
|
||||
if e.cfg.Lease > 0 {
|
||||
return e.cfg.Lease
|
||||
}
|
||||
return 5 * time.Minute
|
||||
}
|
||||
|
||||
// Handle 消费一条卡任务消息;返回后由消费者 FIN。重试与退避全部由 Redis score 表达。
|
||||
func (e *CarrierCardExecutor) Handle(ctx context.Context, body []byte, touch func()) {
|
||||
var msg carrierqueue.CardMessage
|
||||
if err := json.Unmarshal(body, &msg); err != nil || msg.CardID == 0 || msg.Token == "" {
|
||||
e.base.logger.Warn("卡任务消息无效,丢弃", zap.ByteString("body", body))
|
||||
return
|
||||
}
|
||||
execValue := msg.Token + "|" + e.instanceID
|
||||
execTTL := e.renewInterval() * 2
|
||||
got, err := e.store.AcquireExec(ctx, msg.CardID, msg.Token, execValue, execTTL)
|
||||
if err != nil || got != 1 {
|
||||
// 0:处理中标记已不是该 token(过期或重复投递);-1:该 token 已有执行者(NSQ 重投时原消费者仍在执行)。
|
||||
return
|
||||
}
|
||||
settings := e.settings.Current(ctx)
|
||||
taskCtx, cancel := context.WithTimeout(ctx, settings.TaskTimeout())
|
||||
defer cancel()
|
||||
|
||||
var lost atomic.Bool
|
||||
renewDone := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(e.renewInterval())
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-renewDone:
|
||||
return
|
||||
case <-taskCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ok, renewErr := e.store.Renew(context.WithoutCancel(ctx), msg.CardID, msg.CarrierID, msg.Token, execValue, e.lease(), execTTL)
|
||||
if renewErr == nil && !ok {
|
||||
// token 已不一致:立即停止后续步骤,不再发起新的网关调用,只写已产生的业务结果。
|
||||
lost.Store(true)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
if touch != nil {
|
||||
touch()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
set, del := e.execute(taskCtx, msg, settings, &lost)
|
||||
close(renewDone)
|
||||
|
||||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer finishCancel()
|
||||
finished, finishErr := e.store.Finish(finishCtx, msg.CardID, msg.CarrierID, msg.Token, set, del)
|
||||
if finishErr != nil {
|
||||
e.base.logger.Warn("卡任务收尾失败,等待租约到期后重新领取", zap.Uint("card_id", msg.CardID), zap.Error(finishErr))
|
||||
return
|
||||
}
|
||||
if !finished {
|
||||
e.base.logger.Warn("卡任务收尾时处理中标记已不一致,只写入业务结果", zap.Uint("card_id", msg.CardID),
|
||||
zap.String("metric", "polling.carrier.finish_token_mismatch"))
|
||||
}
|
||||
}
|
||||
|
||||
// execute 串行执行到期检查步骤,返回收尾要写入与删除的到期字段。
|
||||
func (e *CarrierCardExecutor) execute(ctx context.Context, msg carrierqueue.CardMessage, settings carrierqueue.Settings, lost *atomic.Bool) (map[string]string, []string) {
|
||||
set, del := map[string]string{}, []string{}
|
||||
card, err := e.base.getCardWithCache(ctx, msg.CardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
}
|
||||
return set, del
|
||||
}
|
||||
if !card.EnablePolling || (e.base.scopeChecker != nil && !e.base.scopeChecker.ShouldEnqueue(ctx, card)) {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
return set, del
|
||||
}
|
||||
due, err := e.store.ReadDue(ctx, msg.CardID)
|
||||
if err != nil {
|
||||
return set, del
|
||||
}
|
||||
now := time.Now()
|
||||
flowEvaluated := false
|
||||
executed := map[string]StepOutcome{}
|
||||
for _, checkType := range settings.StepOrder {
|
||||
item, ok := due.Types[checkType]
|
||||
if !ok || !item.HasDue || item.Due.After(now) {
|
||||
continue
|
||||
}
|
||||
if lost.Load() || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if checkType == constants.CheckTypePackage && settings.SkipPackageAfterFlow && flowEvaluated {
|
||||
executed[checkType] = stepOK()
|
||||
continue
|
||||
}
|
||||
outcome := e.runStep(ctx, msg, checkType, settings.StepTimeout(checkType))
|
||||
if checkType == constants.CheckTypeCarddata && outcome.TriggeredEvaluation {
|
||||
flowEvaluated = true
|
||||
}
|
||||
executed[checkType] = outcome
|
||||
if outcome.Result == StepCardGone || outcome.Result == StepStopPolling {
|
||||
e.removeCard(ctx, msg.CardID, msg.CarrierID)
|
||||
return set, del
|
||||
}
|
||||
}
|
||||
if len(executed) == 0 {
|
||||
return set, del
|
||||
}
|
||||
// 收尾时按最新卡数据重新匹配间隔(步骤中卡状态、套餐可能已变化)。
|
||||
e.base.invalidateCardCache(ctx, msg.CardID)
|
||||
if fresh, freshErr := e.base.getCardWithCache(ctx, msg.CardID); freshErr == nil {
|
||||
card = fresh
|
||||
}
|
||||
intervals := e.base.configMgr.MergedTaskIntervals(card)
|
||||
now = time.Now()
|
||||
for checkType, outcome := range executed {
|
||||
e.writeBack(ctx, card, checkType, outcome, due.Types[checkType], intervals, settings, now, set, &del)
|
||||
}
|
||||
return set, del
|
||||
}
|
||||
|
||||
// runStep 以单步超时、等待模式限流与 panic 恢复执行一个检查步骤。
|
||||
func (e *CarrierCardExecutor) runStep(ctx context.Context, msg carrierqueue.CardMessage, checkType string, timeout time.Duration) (outcome StepOutcome) {
|
||||
stepCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
stepCtx = gateway.WithRateLimitWait(stepCtx, timeout)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
e.base.logger.Error("卡任务步骤 panic,按失败处理", zap.Uint("card_id", msg.CardID), zap.String("check_type", checkType), zap.Any("panic", r))
|
||||
outcome = StepOutcome{Result: StepFailed, Category: "panic", Message: fmt.Sprint(r)}
|
||||
}
|
||||
}()
|
||||
switch checkType {
|
||||
case constants.CheckTypeCardStatus:
|
||||
return e.cardStatus.Step(stepCtx, msg.CardID)
|
||||
case constants.CheckTypeCarddata:
|
||||
return e.carddata.Step(stepCtx, msg.CardID)
|
||||
case constants.CheckTypeRealname:
|
||||
return e.realname.Step(stepCtx, msg.CardID)
|
||||
case constants.CheckTypePackage:
|
||||
stepCtx = withPollingWorkerAuditContext(stepCtx, constants.TaskTypePollingPackage, "套餐状态轮询任务", msg.Token)
|
||||
return e.pkg.Step(stepCtx, msg.CardID)
|
||||
default:
|
||||
return stepSkip()
|
||||
}
|
||||
}
|
||||
|
||||
// writeBack 按步骤结果计算该检查类型下次到期、失败计数与延迟确认,并触发告警或恢复。
|
||||
func (e *CarrierCardExecutor) writeBack(ctx context.Context, card *model.IotCard, checkType string, outcome StepOutcome, previous carrierqueue.TypeDue,
|
||||
intervals map[string]polling.TaskTypeInterval, settings carrierqueue.Settings, now time.Time, set map[string]string, del *[]string) {
|
||||
info, ok := intervals[constants.TaskTypeFromCheckType(checkType)]
|
||||
if !ok || info.Interval <= 0 {
|
||||
// 该类型已不在配置中(间隔为 0 或未配置):移除到期,不再执行。
|
||||
*del = append(*del, carrierqueue.FieldDue(checkType), carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType),
|
||||
carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||
return
|
||||
}
|
||||
interval := time.Duration(info.Interval) * time.Second
|
||||
normalNext := e.base.configMgr.NextCheckAt(ctx, card, constants.TaskTypeFromCheckType(checkType), info, now)
|
||||
switch outcome.Result {
|
||||
case StepSucceeded:
|
||||
next := normalNext
|
||||
set[carrierqueue.FieldOK(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||
if previous.FailCount > 0 {
|
||||
e.alerts.ReportRecovered(card.ID, checkType)
|
||||
}
|
||||
*del = append(*del, carrierqueue.FieldFail(checkType), carrierqueue.FieldFirstFail(checkType), carrierqueue.FieldError(checkType))
|
||||
// 延迟确认:结果已满足预期时取消,否则按序列取下一次确认 = min(正常间隔, 当前 + 确认延迟)。
|
||||
if len(previous.Confirm) > 0 {
|
||||
if previous.Expected != "" && outcome.Observed == previous.Expected {
|
||||
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||
} else {
|
||||
if confirmAt := now.Add(previous.Confirm[0]); confirmAt.Before(next) {
|
||||
next = confirmAt
|
||||
}
|
||||
if rest := previous.Confirm[1:]; len(rest) > 0 {
|
||||
set[carrierqueue.FieldConfirm(checkType)] = joinSeconds(rest)
|
||||
} else {
|
||||
*del = append(*del, carrierqueue.FieldConfirm(checkType), carrierqueue.FieldExpected(checkType))
|
||||
}
|
||||
}
|
||||
}
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(next.UnixMilli(), 10)
|
||||
case StepFailed:
|
||||
set[carrierqueue.FieldError(checkType)] = truncateRunes(outcome.Category+"|"+outcome.Message, 300)
|
||||
if !outcome.CountsAsFailure() {
|
||||
// 上游频率限制、已是目标状态:不计入连续失败,按首个退避间隔重试。
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, 1, interval)).UnixMilli(), 10)
|
||||
return
|
||||
}
|
||||
failCount := previous.FailCount + 1
|
||||
firstFailed := previous.FirstFailed
|
||||
if firstFailed.IsZero() {
|
||||
firstFailed = now
|
||||
set[carrierqueue.FieldFirstFail(checkType)] = strconv.FormatInt(now.UnixMilli(), 10)
|
||||
}
|
||||
set[carrierqueue.FieldFail(checkType)] = strconv.Itoa(failCount)
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(settings.BackoffFor(checkType, failCount, interval)).UnixMilli(), 10)
|
||||
if failCount >= settings.AlertThreshold(checkType) || now.Sub(firstFailed) >= settings.AlertFailDuration || settings.ImmediateAlert[outcome.Category] {
|
||||
e.alerts.ReportFailure(carrierqueue.FailureEvent{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, CheckType: checkType,
|
||||
Category: outcome.Category, Message: outcome.Message, FailCount: failCount, FirstFailed: firstFailed, At: now})
|
||||
}
|
||||
case StepRateLimited:
|
||||
// 限流等待超时:请求未发出,按短延迟重排,不计失败。
|
||||
delay := 5 * time.Second
|
||||
if e.base.rateLimitedDelay != nil {
|
||||
if configured := e.base.rateLimitedDelay(); configured > 0 {
|
||||
delay = configured
|
||||
}
|
||||
}
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||
case StepDeferred:
|
||||
delay := outcome.DeferDelay
|
||||
if delay <= 0 {
|
||||
delay = 30 * time.Second
|
||||
}
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(now.Add(delay).UnixMilli(), 10)
|
||||
default:
|
||||
set[carrierqueue.FieldDue(checkType)] = strconv.FormatInt(normalNext.UnixMilli(), 10)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *CarrierCardExecutor) removeCard(ctx context.Context, cardID, carrierID uint) {
|
||||
removeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := e.store.Remove(removeCtx, cardID, carrierID); err != nil {
|
||||
e.base.logger.Warn("移出运营商账户轮询队列失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func joinSeconds(delays []time.Duration) string {
|
||||
parts := make([]string, 0, len(delays))
|
||||
for _, delay := range delays {
|
||||
parts = append(parts, strconv.FormatInt(int64(delay/time.Second), 10))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func truncateRunes(s string, max int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max])
|
||||
}
|
||||
@@ -71,6 +71,8 @@ type PollingBase struct {
|
||||
priorityAudit *audit.Writer
|
||||
scopeChecker *polling.PollingLifecycleService
|
||||
rateLimitedDelay func() time.Duration // 卡网关调用被运营商账户限流后的固定重入队延迟
|
||||
errorClassifier ErrorClassifier // 上游错误分类(按运营商账户调度的失败退避与告警使用)
|
||||
schedulerMode SchedulerModeReader // 当前调度模式;carrier 时旧 Asynq 轮询任务直接结束
|
||||
}
|
||||
|
||||
// SetRateLimitedDelay 设置被运营商账户限流后的重入队延迟来源(受控系统配置,即时生效)。
|
||||
|
||||
@@ -30,34 +30,42 @@ func NewPollingCarddataHandler(base *PollingBase, gw *gateway.Client, carrier *p
|
||||
return &PollingCarddataHandler{base: base, gateway: gw, carrier: carrier, observation: observation, integration: integration}
|
||||
}
|
||||
|
||||
// Handle 处理流量检查任务。
|
||||
// Handle 处理流量检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
startedAt := time.Now()
|
||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCarddata) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCarddata)
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCarddata, h.Step(ctx, cardID))
|
||||
}
|
||||
|
||||
// Step 执行一次流量检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingCarddataHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startedAt := time.Now()
|
||||
lockToken, locked, err := h.base.acquireCardTrafficSyncLock(ctx, cardID)
|
||||
if err != nil || !locked {
|
||||
h.base.logger.Warn("卡流量同步锁不可用,延迟重入队", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
return h.base.queueMgr.Requeue(ctx, cardID, constants.TaskTypePollingCarddata, time.Now().Add(5*time.Second))
|
||||
return stepDefer(5 * time.Second)
|
||||
}
|
||||
defer h.base.releaseCardTrafficSyncLock(ctx, cardID, lockToken)
|
||||
|
||||
// 优先轮询互斥接缝:放在流量同步锁获取成功之后,避免认领后拿不到锁导致优先项空占。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
if priorityErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCarddata))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
@@ -65,42 +73,42 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return stepSkip()
|
||||
}
|
||||
if h.gateway == nil {
|
||||
priority.FailFinal(ctx, "流量查询能力未配置")
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return stepSkip()
|
||||
}
|
||||
priority.MarkExecution(ctx)
|
||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayTraffic, constants.CardObservationSceneTrafficPolling)
|
||||
result, err := h.gateway.QueryFlow(gateway.WithCarrier(ctx, card.CarrierID), &gateway.FlowQueryReq{CardNo: card.ICCID})
|
||||
if gateway.IsRateLimited(err) {
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCarddata)
|
||||
return StepOutcome{Result: StepRateLimited}
|
||||
}
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询流量失败", priority)
|
||||
priority.FailRetryable(ctx, "查询流量失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询流量失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "查询流量失败", err)
|
||||
}
|
||||
if strings.TrimSpace(result.ICCID) == "" {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "流量查询响应缺少 ICCID", priority)
|
||||
priority.FailRetryable(ctx, "流量查询响应缺少 ICCID")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "流量查询响应缺少 ICCID", nil)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil || h.carrier == nil {
|
||||
priority.FailFinal(ctx, "卡流量观测能力未配置")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
decision, err := h.observation.ApplyTrafficObservation(ctx, carddomain.TrafficObservation{
|
||||
@@ -113,7 +121,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用流量观测失败", priority)
|
||||
priority.FailRetryable(ctx, "应用流量观测失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用流量观测失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "应用流量观测失败", err)
|
||||
}
|
||||
h.base.configMgr.RecordPreResetCheck(ctx, card, now)
|
||||
changed := decision.IncrementMB > 0 || decision.CrossMonth
|
||||
@@ -132,15 +140,15 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
h.base.logger.Info("流量轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||
priority.Complete(ctx)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, true, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return StepOutcome{Result: StepSucceeded, TriggeredEvaluation: changed}
|
||||
}
|
||||
|
||||
func (h *PollingCarddataHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
||||
func (h *PollingCarddataHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
h.base.logger.Warn(message, fields...)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCarddata, false, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCarddata)
|
||||
return h.base.stepFail(ctx, constants.GatewayOpQueryFlow, message, err)
|
||||
}
|
||||
|
||||
@@ -37,70 +37,85 @@ func NewPollingCardStatusHandler(base *PollingBase, gw *gateway.Client, observat
|
||||
return &PollingCardStatusHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||
}
|
||||
|
||||
// Handle 处理卡状态轮询任务。
|
||||
// Handle 处理卡状态轮询任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
startedAt := time.Now()
|
||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingCardStatus) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingCardStatus)
|
||||
outcome := h.Step(ctx, cardID)
|
||||
if outcome.Result == StepSucceeded {
|
||||
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
||||
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingCardStatus, outcome)
|
||||
}
|
||||
|
||||
// Step 执行一次卡状态检查步骤,返回结果分类与观测到的网络状态;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingCardStatusHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startedAt := time.Now()
|
||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
if priorityErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingCardStatus))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return stepSkip()
|
||||
}
|
||||
if h.gateway == nil {
|
||||
priority.FailFinal(ctx, "卡状态查询能力未配置")
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return stepSkip()
|
||||
}
|
||||
priority.MarkExecution(ctx)
|
||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayNetwork, constants.CardObservationSceneNetworkPolling)
|
||||
result, err := h.gateway.QueryCardStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||
if gateway.IsRateLimited(err) {
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingCardStatus)
|
||||
return StepOutcome{Result: StepRateLimited}
|
||||
}
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询卡状态失败", priority)
|
||||
priority.FailRetryable(ctx, "查询卡状态失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询卡状态失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "查询卡状态失败", err)
|
||||
}
|
||||
if strings.TrimSpace(result.ICCID) == "" {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "卡状态查询响应缺少 ICCID", priority)
|
||||
priority.FailRetryable(ctx, "卡状态查询响应缺少 ICCID")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡状态查询响应缺少 ICCID", nil)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil {
|
||||
priority.FailFinal(ctx, "卡网络观测能力未配置")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
decision, err := h.observation.ApplyNetworkObservation(ctx, carddomain.NetworkObservation{
|
||||
@@ -113,7 +128,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用网络状态观测失败", priority)
|
||||
priority.FailRetryable(ctx, "应用网络状态观测失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "应用网络状态观测失败", err)
|
||||
}
|
||||
changed := decision.StatusChanged || decision.StopReasonChanged || decision.StopPolling
|
||||
if changed {
|
||||
@@ -134,23 +149,24 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, true, time.Since(startedAt))
|
||||
if decision.StopPolling {
|
||||
h.base.logger.Info("卡命中风险停机或已销户,已关闭该卡轮询", zap.Uint("card_id", cardID), zap.String("gateway_extend", decision.GatewayExtend))
|
||||
return nil
|
||||
return StepOutcome{Result: StepStopPolling}
|
||||
}
|
||||
|
||||
// 卡状态任务仍正常但套餐任务丢失时,仅补入缺失项;不改写已有套餐任务的执行时间。
|
||||
h.base.invalidateCardCache(ctx, cardID)
|
||||
if err := h.base.ensureMissingTask(ctx, cardID, constants.TaskTypePollingPackage); err != nil {
|
||||
h.base.logger.Warn("卡状态轮询补齐套餐任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||
outcome := stepOK()
|
||||
if decision.StatusKnown {
|
||||
outcome.Observed = ObservedOffline
|
||||
if decision.AfterStatus == constants.NetworkStatusOnline {
|
||||
outcome.Observed = ObservedOnline
|
||||
}
|
||||
}
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return outcome
|
||||
}
|
||||
|
||||
func (h *PollingCardStatusHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
||||
func (h *PollingCardStatusHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
h.base.logger.Warn(message, fields...)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingCardStatus, false, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingCardStatus)
|
||||
return h.base.stepFail(ctx, constants.GatewayOpQueryStatus, message, err)
|
||||
}
|
||||
|
||||
@@ -37,14 +37,15 @@ func NewPollingPackageHandler(
|
||||
}
|
||||
}
|
||||
|
||||
// Handle 处理套餐检查任务
|
||||
// Handle 处理套餐检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
startTime := time.Now()
|
||||
|
||||
cardID, ok := parseTaskPayload(t.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
||||
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
||||
@@ -52,42 +53,48 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingPackage)
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingPackage, h.Step(ctx, cardID))
|
||||
}
|
||||
|
||||
// Step 执行一次套餐检查步骤(按规则重新判断停复机),返回结果分类;不负责重入队。
|
||||
// 调用方负责在 ctx 中设置 Worker 审计上下文。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingPackageHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startTime := time.Now()
|
||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
if priorityErr != nil {
|
||||
h.base.logger.Error("查询活动优先轮询项失败", zap.Uint("card_id", cardID), zap.Error(priorityErr))
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return h.base.stepFail(ctx, "", "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingPackage))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
card, cacheErr := h.base.getCardWithCache(ctx, cardID)
|
||||
if cacheErr != nil {
|
||||
if isNotFound(cacheErr) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
h.base.logger.Error("获取卡信息失败", zap.Uint("card_id", cardID), zap.Error(cacheErr))
|
||||
// 本地读卡失败属可恢复失败:与其余三个轮询 handler 统一按尝试上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, false, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return h.base.stepFail(ctx, "", "获取卡信息失败", cacheErr)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepSkip()
|
||||
}
|
||||
|
||||
if h.stopResumeSvc == nil {
|
||||
priority.FailFinal(ctx, "套餐评估能力未配置")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepSkip()
|
||||
}
|
||||
|
||||
// 停复机评估就是本任务的上游执行动作:先累加尝试次数再发起。
|
||||
@@ -98,7 +105,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
zap.Uint("card_id", cardID), zap.Error(loadErr))
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return h.base.stepFail(ctx, "", "获取卡信息失败", loadErr)
|
||||
}
|
||||
h.base.logger.Info("套餐检查:执行停复机评估",
|
||||
zap.Uint("card_id", cardID),
|
||||
@@ -119,10 +126,11 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
zap.Uint("card_id", cardID), zap.Error(evalErr))
|
||||
priority.FailRetryable(ctx, "停复机评估失败")
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
outcome := h.base.stepFail(ctx, constants.GatewayOpStart, "停复机评估失败", evalErr)
|
||||
return outcome
|
||||
}
|
||||
|
||||
priority.Complete(ctx)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingPackage, true, time.Since(startTime))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingPackage)
|
||||
return stepOK()
|
||||
}
|
||||
|
||||
@@ -28,81 +28,88 @@ func NewPollingRealnameHandler(base *PollingBase, gw *gateway.Client, observatio
|
||||
return &PollingRealnameHandler{base: base, gateway: gw, observation: observation, integration: integration}
|
||||
}
|
||||
|
||||
// Handle 处理实名检查任务。
|
||||
// Handle 处理实名检查任务(旧分片调度)。按运营商账户调度时直接结束。
|
||||
func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
startedAt := time.Now()
|
||||
cardID, ok := parseTaskPayload(task.Payload(), h.base.logger)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if h.base.carrierSchedulerActive(ctx) {
|
||||
return nil
|
||||
}
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingRealname) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
}
|
||||
defer h.base.releaseConcurrency(ctx, constants.TaskTypePollingRealname)
|
||||
return h.base.applyLegacyOutcome(ctx, cardID, constants.TaskTypePollingRealname, h.Step(ctx, cardID))
|
||||
}
|
||||
|
||||
// Step 执行一次实名检查步骤,返回结果分类;不负责重入队。旧分片调度与按运营商账户调度共用。
|
||||
func (h *PollingRealnameHandler) Step(ctx context.Context, cardID uint) StepOutcome {
|
||||
startedAt := time.Now()
|
||||
// 优先轮询互斥接缝:存在活动优先项时本轮即优先执行或被跳过;无活动项时行为与既有实现一致。
|
||||
priority, skip, priorityErr := h.base.beginPollingPriority(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
if priorityErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询活动优先轮询项失败", priorityErr)
|
||||
}
|
||||
if skip {
|
||||
h.base.logger.Debug("卡轮询优先项已被其它执行持有,本轮跳过并延后",
|
||||
zap.Uint("card_id", cardID), zap.String("task_type", constants.TaskTypePollingRealname))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepDefer(0)
|
||||
}
|
||||
|
||||
card, err := h.base.getCardWithCache(ctx, cardID)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
priority.FailFinal(ctx, "卡不存在")
|
||||
return nil
|
||||
return StepOutcome{Result: StepCardGone}
|
||||
}
|
||||
// 本地读卡失败属可恢复失败:先按真实执行累加尝试次数,再按上限收敛,避免加急事实被静默丢弃。
|
||||
priority.MarkExecution(ctx)
|
||||
priority.FailRetryable(ctx, "获取卡信息失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "获取卡信息失败", err)
|
||||
}
|
||||
// 执行前轮询范围校验:优先通道不得重启已停止轮询的卡。
|
||||
if !h.base.priorityInPollingScope(ctx, priority, card) {
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepSkip()
|
||||
}
|
||||
if h.gateway == nil {
|
||||
priority.FailFinal(ctx, "实名查询能力未配置")
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepSkip()
|
||||
}
|
||||
// 所属运营商账户未开启实名管控时实名结果不影响停复机,跳过查询并按正常间隔重入队(不再按行业卡跳过)。
|
||||
skippable, scopeErr := h.base.realnameQuerySkippable(ctx, card)
|
||||
if scopeErr != nil {
|
||||
priority.FailRetryable(ctx, "查询运营商账户实名管控失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
||||
return h.fail(ctx, cardID, startedAt, "查询运营商账户实名管控失败", scopeErr)
|
||||
}
|
||||
if skippable {
|
||||
priority.FailFinal(ctx, "所属运营商账户未开启实名管控,跳过实名查询")
|
||||
h.base.logger.Debug("所属运营商账户未开启实名管控,跳过实名查询", zap.Uint("card_id", cardID), zap.Uint("carrier_id", card.CarrierID))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepSkip()
|
||||
}
|
||||
priority.MarkExecution(ctx)
|
||||
attempt := newGatewayAttempt(cardID, constants.IntegrationOperationGatewayRealname, constants.CardObservationSceneRealnamePolling)
|
||||
result, err := h.gateway.QueryRealnameStatus(gateway.WithCarrier(ctx, card.CarrierID), &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||
if gateway.IsRateLimited(err) {
|
||||
priority.FailRetryable(ctx, "运营商账户网关限流,稍后重试")
|
||||
return h.base.requeueRateLimited(cardID, constants.TaskTypePollingRealname)
|
||||
return StepOutcome{Result: StepRateLimited}
|
||||
}
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultFailed, false, "查询实名状态失败", priority)
|
||||
priority.FailRetryable(ctx, "查询实名状态失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "查询实名状态失败", err)
|
||||
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, "查询实名状态失败", err)
|
||||
}
|
||||
if strings.TrimSpace(result.ICCID) == "" {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultInvalidPayload, false, "实名查询响应缺少 ICCID", priority)
|
||||
priority.FailRetryable(ctx, "实名查询响应缺少 ICCID")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "实名查询响应缺少 ICCID", nil)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil {
|
||||
priority.FailFinal(ctx, "卡实名观测能力未配置")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||
return h.fail(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||
}
|
||||
now := time.Now()
|
||||
decision, err := h.observation.ApplyCardObservation(ctx, carddomain.RealnameObservation{
|
||||
@@ -115,7 +122,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if err != nil {
|
||||
_ = recordPriorityGatewayAttempt(ctx, h.integration, attempt, constants.IntegrationResultUnknown, false, "应用实名观测失败", priority)
|
||||
priority.FailRetryable(ctx, "应用实名观测失败")
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "应用实名观测失败", err)
|
||||
return h.fail(ctx, cardID, startedAt, "应用实名观测失败", err)
|
||||
}
|
||||
changed := decision.StatusChanged
|
||||
if changed {
|
||||
@@ -133,15 +140,19 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
h.base.logger.Info("实名轮询观测完成", zap.Uint("card_id", cardID), zap.Bool("changed", changed), zap.String("metric", metric))
|
||||
priority.Complete(ctx)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, true, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return stepOK()
|
||||
}
|
||||
|
||||
func (h *PollingRealnameHandler) failAndRequeue(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) error {
|
||||
func (h *PollingRealnameHandler) fail(ctx context.Context, cardID uint, startedAt time.Time, message string, err error) StepOutcome {
|
||||
return h.failOp(ctx, cardID, startedAt, constants.GatewayOpQueryRealname, message, err)
|
||||
}
|
||||
|
||||
func (h *PollingRealnameHandler) failOp(ctx context.Context, cardID uint, startedAt time.Time, op, message string, err error) StepOutcome {
|
||||
fields := []zap.Field{zap.Uint("card_id", cardID)}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
h.base.logger.Warn(message, fields...)
|
||||
h.base.updateStats(ctx, constants.TaskTypePollingRealname, false, time.Since(startedAt))
|
||||
return h.base.requeueCard(ctx, cardID, constants.TaskTypePollingRealname)
|
||||
return h.base.stepFail(ctx, op, message, err)
|
||||
}
|
||||
|
||||
123
internal/task/polling_step.go
Normal file
123
internal/task/polling_step.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StepResult 是一个检查步骤的执行结果分类。
|
||||
type StepResult int
|
||||
|
||||
const (
|
||||
// StepSucceeded 检查成功(含观测入账、评估完成)。
|
||||
StepSucceeded StepResult = iota
|
||||
// StepFailed 检查失败,计入连续失败(是否计入还要看失败类别,见 countsAsFailure)。
|
||||
StepFailed
|
||||
// StepRateLimited 本账户限流拒绝或等待超时,请求未发出:短延迟重排,不计失败。
|
||||
StepRateLimited
|
||||
// StepDeferred 暂不可执行(流量同步锁被占、优先项由其他执行持有):短延迟或按间隔重排,不计失败。
|
||||
StepDeferred
|
||||
// StepSkipped 本轮无需执行(能力未配置、不在轮询范围、所属账户未开启实名管控):按正常间隔重排。
|
||||
StepSkipped
|
||||
// StepCardGone 卡不存在:不再重排。
|
||||
StepCardGone
|
||||
// StepStopPolling 观测后该卡已关闭轮询(风险停机、已销户):不再重排。
|
||||
StepStopPolling
|
||||
)
|
||||
|
||||
// StepOutcome 是一个检查步骤的结果。旧分片调度把它映射为原有的重入队动作;按运营商账户调度据此写回到期与失败计数。
|
||||
type StepOutcome struct {
|
||||
Result StepResult
|
||||
// DeferDelay 为 StepDeferred 时的建议短延迟;0 表示按正常间隔。
|
||||
DeferDelay time.Duration
|
||||
// Category 失败类别:网关错误类别(network / timeout / http / invalid_response),上游业务错误时为错误分类类别。
|
||||
Category string
|
||||
// Message 失败信息(已脱敏截断)。
|
||||
Message string
|
||||
// Observed 卡状态步骤观测到的网络状态(online / offline),用于延迟确认是否已满足预期。
|
||||
Observed string
|
||||
// TriggeredEvaluation 流量步骤本轮观测产生了流量变化(会触发停复机评估),可据此跳过套餐步骤。
|
||||
TriggeredEvaluation bool
|
||||
}
|
||||
|
||||
// 卡状态步骤的观测结果取值。
|
||||
const (
|
||||
ObservedOnline = "online"
|
||||
ObservedOffline = "offline"
|
||||
)
|
||||
|
||||
func stepOK() StepOutcome { return StepOutcome{Result: StepSucceeded} }
|
||||
|
||||
func stepSkip() StepOutcome { return StepOutcome{Result: StepSkipped} }
|
||||
|
||||
func stepDefer(delay time.Duration) StepOutcome {
|
||||
return StepOutcome{Result: StepDeferred, DeferDelay: delay}
|
||||
}
|
||||
|
||||
// stepFail 按错误构造失败结果:网关结构化错误取其类别,上游业务错误再按错误分类规则细分(用于判断是否计入连续失败与告警)。
|
||||
func (b *PollingBase) stepFail(ctx context.Context, op, message string, err error) StepOutcome {
|
||||
outcome := StepOutcome{Result: StepFailed, Message: message, Category: "local"}
|
||||
if gwErr, ok := gateway.AsGatewayError(err); ok {
|
||||
outcome.Category = gwErr.Category
|
||||
if gwErr.Category == constants.GatewayErrorUpstream {
|
||||
outcome.Message = message + ":" + gwErr.ProviderMessage
|
||||
if b.errorClassifier != nil {
|
||||
outcome.Category = b.errorClassifier(ctx, op, gwErr.ProviderMessage)
|
||||
}
|
||||
}
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
// CountsAsFailure 判断失败结果是否计入连续失败:上游频率限制与已是目标状态不计入。
|
||||
func (o StepOutcome) CountsAsFailure() bool {
|
||||
if o.Result != StepFailed {
|
||||
return false
|
||||
}
|
||||
switch o.Category {
|
||||
case constants.GatewayClassRateLimitedUpstream, constants.GatewayClassAlreadyTarget:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ErrorClassifier 对上游业务错误信息分类,返回错误分类类别。
|
||||
type ErrorClassifier func(ctx context.Context, op, message string) string
|
||||
|
||||
// SetErrorClassifier 注入上游错误分类器(按运营商账户调度的失败退避与告警使用)。
|
||||
func (b *PollingBase) SetErrorClassifier(classifier ErrorClassifier) {
|
||||
b.errorClassifier = classifier
|
||||
}
|
||||
|
||||
// SchedulerModeReader 返回当前卡轮询调度模式(受控系统配置 polling.scheduler.mode)。
|
||||
type SchedulerModeReader func(ctx context.Context) string
|
||||
|
||||
// SetSchedulerModeReader 注入调度模式读取;未注入时视为旧分片调度。
|
||||
func (b *PollingBase) SetSchedulerModeReader(reader SchedulerModeReader) {
|
||||
b.schedulerMode = reader
|
||||
}
|
||||
|
||||
// carrierSchedulerActive 判断当前是否已切到按运营商账户调度:是则旧 Asynq 轮询任务直接结束,不调网关、不重入旧队列。
|
||||
func (b *PollingBase) carrierSchedulerActive(ctx context.Context) bool {
|
||||
return b != nil && b.schedulerMode != nil && b.schedulerMode(ctx) == constants.PollingSchedulerModeCarrier
|
||||
}
|
||||
|
||||
// applyLegacyOutcome 把步骤结果映射为旧分片调度原有的重入队动作。
|
||||
func (b *PollingBase) applyLegacyOutcome(ctx context.Context, cardID uint, taskType string, outcome StepOutcome) error {
|
||||
switch outcome.Result {
|
||||
case StepCardGone, StepStopPolling:
|
||||
return nil
|
||||
case StepRateLimited:
|
||||
return b.requeueRateLimited(cardID, taskType)
|
||||
case StepDeferred:
|
||||
if outcome.DeferDelay > 0 {
|
||||
return b.queueMgr.Requeue(ctx, cardID, taskType, time.Now().Add(outcome.DeferDelay))
|
||||
}
|
||||
return b.requeueCard(ctx, cardID, taskType)
|
||||
default:
|
||||
return b.requeueCard(ctx, cardID, taskType)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user