feat: 按运营商账户调度的分发器、卡任务执行器与模式切换
- 四个轮询处理器拆为步骤函数 Step(返回结果分类),旧 Asynq Handle 只做并发控制与原有重入队映射,行为不变;carrier 模式下旧 handler 直接结束
- 网关限流增加等待模式(ctx 标记),NSQ 消费者等名额到单步超时,超时按限流处理不计失败;记录限流等待次数
- 卡任务执行器:核对 token 并抢执行权,按配置步骤顺序串行执行到期检查(单步超时、总超时、panic 恢复、流量已触发评估时跳过套餐),执行中续期并 Touch,续期失败立即停止;收尾写回到期、连续失败、退避、延迟确认与告警
- 分发器:每秒按账户抢分发锁领取(执行上限按消费能力收紧、优先占比、按限流组剩余 QPS 估算),发布到 polling.card.{账户},发布失败释放;消费旧优先提示与手动触发;心跳与处理中标记到期计数
- NSQ 生产者与按账户动态订阅的消费者管理器,优雅退出宽限期
- 失败告警异步写 tb_polling_failure_alert(同卡同类型一条未处理,成功转已恢复),账户窗口内告警数超阈值产生账户级告警
- 模式监听:切 carrier 时旧分片调度停止出队,加锁初始化新结构后开始领取并订阅;回 legacy 时停止消费、重建旧分片队列;生命周期(API 与 Worker)在 carrier 模式写账户队列
This commit is contained in:
177
internal/polling/carrierqueue/alerts.go
Normal file
177
internal/polling/carrierqueue/alerts.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package carrierqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// FailureEvent 是一次需要告警的检查失败。
|
||||
type FailureEvent struct {
|
||||
CardID uint
|
||||
ICCID string
|
||||
CarrierID uint
|
||||
CheckType string
|
||||
Category string
|
||||
Message string
|
||||
FailCount int
|
||||
FirstFailed time.Time
|
||||
At time.Time
|
||||
}
|
||||
|
||||
type alertOp struct {
|
||||
failure *FailureEvent
|
||||
recovered bool
|
||||
cardID uint
|
||||
checkType string
|
||||
}
|
||||
|
||||
// AlertWriter 异步写卡轮询失败告警:同一卡同一检查类型只保留一条未处理告警,后续失败更新计数;
|
||||
// 首次成功转为已恢复;同账户窗口内新增告警超阈值时产生账户级告警。写入失败只记日志,不影响轮询。
|
||||
type AlertWriter struct {
|
||||
db *gorm.DB
|
||||
redis *redis.Client
|
||||
settings *SettingsProvider
|
||||
logger *zap.Logger
|
||||
ch chan alertOp
|
||||
}
|
||||
|
||||
// NewAlertWriter 创建告警写入器;调用 Start 后开始消费。
|
||||
func NewAlertWriter(db *gorm.DB, redisClient *redis.Client, settings *SettingsProvider, logger *zap.Logger) *AlertWriter {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &AlertWriter{db: db, redis: redisClient, settings: settings, logger: logger, ch: make(chan alertOp, 2048)}
|
||||
}
|
||||
|
||||
// Start 启动后台写入协程,直到 ctx 结束。
|
||||
func (w *AlertWriter) Start(ctx context.Context) {
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case op := <-w.ch:
|
||||
w.apply(op)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// ReportFailure 提交一条失败告警(达到阈值后才调用)。
|
||||
func (w *AlertWriter) ReportFailure(event FailureEvent) {
|
||||
w.offer(alertOp{failure: &event})
|
||||
}
|
||||
|
||||
// ReportRecovered 提交一次恢复:该卡该检查类型的未处理告警转为已恢复。
|
||||
func (w *AlertWriter) ReportRecovered(cardID uint, checkType string) {
|
||||
w.offer(alertOp{recovered: true, cardID: cardID, checkType: checkType})
|
||||
}
|
||||
|
||||
func (w *AlertWriter) offer(op alertOp) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case w.ch <- op:
|
||||
default:
|
||||
w.logger.Warn("卡轮询失败告警队列已满,丢弃本次写入", zap.String("metric", "polling.failure_alert.dropped"))
|
||||
}
|
||||
}
|
||||
|
||||
func (w *AlertWriter) apply(op alertOp) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if op.recovered {
|
||||
err := w.db.WithContext(ctx).Exec(`UPDATE tb_polling_failure_alert SET status = ?, recovered_at = NOW(), updated_at = NOW()
|
||||
WHERE alert_level = ? AND card_id = ? AND check_type = ? AND status = ?`,
|
||||
constants.PollingAlertStatusRecovered, constants.PollingAlertLevelCard, op.cardID, op.checkType, constants.PollingAlertStatusOpen).Error
|
||||
if err != nil {
|
||||
w.logger.Warn("卡轮询告警转为已恢复失败", zap.Uint("card_id", op.cardID), zap.String("check_type", op.checkType), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
event := op.failure
|
||||
var inserted bool
|
||||
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||
(alert_level, card_id, iccid, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||
DO UPDATE SET fail_count = EXCLUDED.fail_count, error_category = EXCLUDED.error_category,
|
||||
last_failed_at = EXCLUDED.last_failed_at, last_error = EXCLUDED.last_error, updated_at = NOW()
|
||||
RETURNING (xmax = 0)`,
|
||||
constants.PollingAlertLevelCard, event.CardID, event.ICCID, event.CarrierID, event.CheckType, event.Category, event.FailCount,
|
||||
nullableTime(event.FirstFailed), event.At, truncate(event.Message, 500), constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||
if err != nil {
|
||||
w.logger.Warn("写卡轮询失败告警失败", zap.Uint("card_id", event.CardID), zap.String("check_type", event.CheckType), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if !inserted {
|
||||
return
|
||||
}
|
||||
w.logger.Error("卡轮询持续失败告警", zap.Uint("card_id", event.CardID), zap.String("iccid", event.ICCID),
|
||||
zap.Uint("carrier_id", event.CarrierID), zap.String("check_type", event.CheckType), zap.String("category", event.Category),
|
||||
zap.Int("fail_count", event.FailCount), zap.String("metric", "polling.failure_alert.opened"))
|
||||
w.checkAccountAlert(ctx, event.CarrierID)
|
||||
}
|
||||
|
||||
// checkAccountAlert 同账户窗口内新增卡级告警数超过阈值时,产生(或更新)一条账户级告警。
|
||||
func (w *AlertWriter) checkAccountAlert(ctx context.Context, carrierID uint) {
|
||||
settings := w.settings.Current(ctx)
|
||||
window, limit := settings.AccountAlertWindow, settings.AccountAlertLimit
|
||||
if window <= 0 || limit <= 0 || w.redis == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
key := constants.RedisCarrierAccountAlertKey(carrierID)
|
||||
member := strconv.FormatInt(now.UnixNano(), 36)
|
||||
pipe := w.redis.TxPipeline()
|
||||
pipe.ZAdd(ctx, key, redis.Z{Score: float64(now.UnixMilli()), Member: member})
|
||||
pipe.ZRemRangeByScore(ctx, key, "-inf", strconv.FormatInt(now.Add(-window).UnixMilli(), 10))
|
||||
count := pipe.ZCard(ctx, key)
|
||||
pipe.Expire(ctx, key, window*2)
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
if int(count.Val()) <= limit {
|
||||
return
|
||||
}
|
||||
var inserted bool
|
||||
err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
|
||||
(alert_level, card_id, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
|
||||
VALUES (?, 0, ?, ?, '', ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
|
||||
DO UPDATE SET fail_count = EXCLUDED.fail_count, last_failed_at = EXCLUDED.last_failed_at, updated_at = NOW()
|
||||
RETURNING (xmax = 0)`,
|
||||
constants.PollingAlertLevelCarrier, carrierID, constants.PollingAlertCheckTypeAll, count.Val(), now, now,
|
||||
"窗口内同账户新增失败告警数超过阈值,通常是账户额度、凭证或上游故障,请联系上游处理账户", constants.PollingAlertStatusOpen).Row().Scan(&inserted)
|
||||
if err != nil {
|
||||
w.logger.Warn("写账户级轮询告警失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if inserted {
|
||||
w.logger.Error("运营商账户轮询失败告警数超过阈值", zap.Uint("carrier_id", carrierID), zap.Int64("count", count.Val()),
|
||||
zap.Duration("window", window), zap.String("metric", "polling.failure_alert.carrier_opened"))
|
||||
}
|
||||
}
|
||||
|
||||
func nullableTime(t time.Time) any {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max])
|
||||
}
|
||||
Reference in New Issue
Block a user