package carrierqueue import ( "context" "strconv" "time" "github.com/redis/go-redis/v9" "go.uber.org/zap" "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/pkg/constants" ) // FailureEvent 是一次需要告警的检查失败。 type FailureEvent struct { CardID uint ICCID string CarrierID uint CheckType string Category string Message string FailCount int FirstFailed time.Time At time.Time } type alertOp struct { failure *FailureEvent recovered bool cardID uint checkType string } // AlertWriter 异步写卡轮询失败告警:同一卡同一检查类型只保留一条未处理告警,后续失败更新计数; // 首次成功转为已恢复;同账户窗口内新增告警超阈值时产生账户级告警。写入失败只记日志,不影响轮询。 type AlertWriter struct { db *gorm.DB redis *redis.Client settings *SettingsProvider logger *zap.Logger ch chan alertOp } // NewAlertWriter 创建告警写入器;调用 Start 后开始消费。 func NewAlertWriter(db *gorm.DB, redisClient *redis.Client, settings *SettingsProvider, logger *zap.Logger) *AlertWriter { if logger == nil { logger = zap.NewNop() } return &AlertWriter{db: db, redis: redisClient, settings: settings, logger: logger, ch: make(chan alertOp, 2048)} } // Start 启动后台写入协程,直到 ctx 结束。 func (w *AlertWriter) Start(ctx context.Context) { go func() { for { select { case <-ctx.Done(): return case op := <-w.ch: w.apply(op) } } }() } // ReportFailure 提交一条失败告警(达到阈值后才调用)。 func (w *AlertWriter) ReportFailure(event FailureEvent) { w.offer(alertOp{failure: &event}) } // ReportRecovered 提交一次恢复:该卡该检查类型的未处理告警转为已恢复。 func (w *AlertWriter) ReportRecovered(cardID uint, checkType string) { w.offer(alertOp{recovered: true, cardID: cardID, checkType: checkType}) } func (w *AlertWriter) offer(op alertOp) { if w == nil { return } select { case w.ch <- op: default: w.logger.Warn("卡轮询失败告警队列已满,丢弃本次写入", zap.String("metric", "polling.failure_alert.dropped")) } } func (w *AlertWriter) apply(op alertOp) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if op.recovered { err := w.db.WithContext(ctx).Exec(`UPDATE tb_polling_failure_alert SET status = ?, recovered_at = NOW(), updated_at = NOW() WHERE alert_level = ? AND card_id = ? AND check_type = ? AND status = ?`, constants.PollingAlertStatusRecovered, constants.PollingAlertLevelCard, op.cardID, op.checkType, constants.PollingAlertStatusOpen).Error if err != nil { w.logger.Warn("卡轮询告警转为已恢复失败", zap.Uint("card_id", op.cardID), zap.String("check_type", op.checkType), zap.Error(err)) } return } event := op.failure var inserted bool err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert (alert_level, card_id, iccid, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1 DO UPDATE SET fail_count = EXCLUDED.fail_count, error_category = EXCLUDED.error_category, last_failed_at = EXCLUDED.last_failed_at, last_error = EXCLUDED.last_error, updated_at = NOW() RETURNING (xmax = 0)`, constants.PollingAlertLevelCard, event.CardID, event.ICCID, event.CarrierID, event.CheckType, event.Category, event.FailCount, nullableTime(event.FirstFailed), event.At, truncate(event.Message, 500), constants.PollingAlertStatusOpen).Row().Scan(&inserted) if err != nil { w.logger.Warn("写卡轮询失败告警失败", zap.Uint("card_id", event.CardID), zap.String("check_type", event.CheckType), zap.Error(err)) return } if !inserted { return } w.logger.Error("卡轮询持续失败告警", zap.Uint("card_id", event.CardID), zap.String("iccid", event.ICCID), zap.Uint("carrier_id", event.CarrierID), zap.String("check_type", event.CheckType), zap.String("category", event.Category), zap.Int("fail_count", event.FailCount), zap.String("metric", "polling.failure_alert.opened")) w.checkAccountAlert(ctx, event.CarrierID) } // checkAccountAlert 同账户窗口内新增卡级告警数超过阈值时,产生(或更新)一条账户级告警。 func (w *AlertWriter) checkAccountAlert(ctx context.Context, carrierID uint) { settings := w.settings.Current(ctx) window, limit := settings.AccountAlertWindow, settings.AccountAlertLimit if window <= 0 || limit <= 0 || w.redis == nil { return } now := time.Now() key := constants.RedisCarrierAccountAlertKey(carrierID) member := strconv.FormatInt(now.UnixNano(), 36) pipe := w.redis.TxPipeline() pipe.ZAdd(ctx, key, redis.Z{Score: float64(now.UnixMilli()), Member: member}) pipe.ZRemRangeByScore(ctx, key, "-inf", strconv.FormatInt(now.Add(-window).UnixMilli(), 10)) count := pipe.ZCard(ctx, key) pipe.Expire(ctx, key, window*2) if _, err := pipe.Exec(ctx); err != nil { return } if int(count.Val()) <= limit { return } var inserted bool err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert (alert_level, card_id, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status) VALUES (?, 0, ?, ?, '', ?, ?, ?, ?, ?) ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1 DO UPDATE SET fail_count = EXCLUDED.fail_count, last_failed_at = EXCLUDED.last_failed_at, updated_at = NOW() RETURNING (xmax = 0)`, constants.PollingAlertLevelCarrier, carrierID, constants.PollingAlertCheckTypeAll, count.Val(), now, now, "窗口内同账户新增失败告警数超过阈值,通常是账户额度、凭证或上游故障,请联系上游处理账户", constants.PollingAlertStatusOpen).Row().Scan(&inserted) if err != nil { w.logger.Warn("写账户级轮询告警失败", zap.Uint("carrier_id", carrierID), zap.Error(err)) return } if inserted { w.logger.Error("运营商账户轮询失败告警数超过阈值", zap.Uint("carrier_id", carrierID), zap.Int64("count", count.Val()), zap.Duration("window", window), zap.String("metric", "polling.failure_alert.carrier_opened")) } } func nullableTime(t time.Time) any { if t.IsZero() { return nil } return t } func truncate(s string, max int) string { runes := []rune(s) if len(runes) <= max { return s } return string(runes[:max]) }