junhong_cmp_fiber/internal/polling/carrierqueue/alerts.go

package carrierqueue

import (
	"context"
	"strconv"
	"time"

	"github.com/redis/go-redis/v9"
	"go.uber.org/zap"
	"gorm.io/gorm"

	"github.com/break/junhong_cmp_fiber/pkg/constants"
)

// FailureEvent 是一次需要告警的检查失败。
type FailureEvent struct {
	CardID      uint
	ICCID       string
	CarrierID   uint
	CheckType   string
	Category    string
	Message     string
	FailCount   int
	FirstFailed time.Time
	At          time.Time
}

type alertOp struct {
	failure   *FailureEvent
	recovered bool
	cardID    uint
	checkType string
}

// AlertWriter 异步写卡轮询失败告警：同一卡同一检查类型只保留一条未处理告警，后续失败更新计数；
// 首次成功转为已恢复；同账户窗口内新增告警超阈值时产生账户级告警。写入失败只记日志，不影响轮询。
type AlertWriter struct {
	db       *gorm.DB
	redis    *redis.Client
	settings *SettingsProvider
	logger   *zap.Logger
	ch       chan alertOp
}

// NewAlertWriter 创建告警写入器；调用 Start 后开始消费。
func NewAlertWriter(db *gorm.DB, redisClient *redis.Client, settings *SettingsProvider, logger *zap.Logger) *AlertWriter {
	if logger == nil {
		logger = zap.NewNop()
	}
	return &AlertWriter{db: db, redis: redisClient, settings: settings, logger: logger, ch: make(chan alertOp, 2048)}
}

// Start 启动后台写入协程，直到 ctx 结束。
func (w *AlertWriter) Start(ctx context.Context) {
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case op := <-w.ch:
				w.apply(op)
			}
		}
	}()
}

// ReportFailure 提交一条失败告警（达到阈值后才调用）。
func (w *AlertWriter) ReportFailure(event FailureEvent) {
	w.offer(alertOp{failure: &event})
}

// ReportRecovered 提交一次恢复：该卡该检查类型的未处理告警转为已恢复。
func (w *AlertWriter) ReportRecovered(cardID uint, checkType string) {
	w.offer(alertOp{recovered: true, cardID: cardID, checkType: checkType})
}

func (w *AlertWriter) offer(op alertOp) {
	if w == nil {
		return
	}
	select {
	case w.ch <- op:
	default:
		w.logger.Warn("卡轮询失败告警队列已满，丢弃本次写入", zap.String("metric", "polling.failure_alert.dropped"))
	}
}

func (w *AlertWriter) apply(op alertOp) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if op.recovered {
		err := w.db.WithContext(ctx).Exec(`UPDATE tb_polling_failure_alert SET status = ?, recovered_at = NOW(), updated_at = NOW()
			WHERE alert_level = ? AND card_id = ? AND check_type = ? AND status = ?`,
			constants.PollingAlertStatusRecovered, constants.PollingAlertLevelCard, op.cardID, op.checkType, constants.PollingAlertStatusOpen).Error
		if err != nil {
			w.logger.Warn("卡轮询告警转为已恢复失败", zap.Uint("card_id", op.cardID), zap.String("check_type", op.checkType), zap.Error(err))
		}
		return
	}
	event := op.failure
	var inserted bool
	err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
			(alert_level, card_id, iccid, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
		ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
		DO UPDATE SET fail_count = EXCLUDED.fail_count, error_category = EXCLUDED.error_category,
			last_failed_at = EXCLUDED.last_failed_at, last_error = EXCLUDED.last_error, updated_at = NOW()
		RETURNING (xmax = 0)`,
		constants.PollingAlertLevelCard, event.CardID, event.ICCID, event.CarrierID, event.CheckType, event.Category, event.FailCount,
		nullableTime(event.FirstFailed), event.At, truncate(event.Message, 500), constants.PollingAlertStatusOpen).Row().Scan(&inserted)
	if err != nil {
		w.logger.Warn("写卡轮询失败告警失败", zap.Uint("card_id", event.CardID), zap.String("check_type", event.CheckType), zap.Error(err))
		return
	}
	if !inserted {
		return
	}
	w.logger.Error("卡轮询持续失败告警", zap.Uint("card_id", event.CardID), zap.String("iccid", event.ICCID),
		zap.Uint("carrier_id", event.CarrierID), zap.String("check_type", event.CheckType), zap.String("category", event.Category),
		zap.Int("fail_count", event.FailCount), zap.String("metric", "polling.failure_alert.opened"))
	w.checkAccountAlert(ctx, event.CarrierID)
}

// checkAccountAlert 同账户窗口内新增卡级告警数超过阈值时，产生（或更新）一条账户级告警。
func (w *AlertWriter) checkAccountAlert(ctx context.Context, carrierID uint) {
	settings := w.settings.Current(ctx)
	window, limit := settings.AccountAlertWindow, settings.AccountAlertLimit
	if window <= 0 || limit <= 0 || w.redis == nil {
		return
	}
	now := time.Now()
	key := constants.RedisCarrierAccountAlertKey(carrierID)
	member := strconv.FormatInt(now.UnixNano(), 36)
	pipe := w.redis.TxPipeline()
	pipe.ZAdd(ctx, key, redis.Z{Score: float64(now.UnixMilli()), Member: member})
	pipe.ZRemRangeByScore(ctx, key, "-inf", strconv.FormatInt(now.Add(-window).UnixMilli(), 10))
	count := pipe.ZCard(ctx, key)
	pipe.Expire(ctx, key, window*2)
	if _, err := pipe.Exec(ctx); err != nil {
		return
	}
	if int(count.Val()) <= limit {
		return
	}
	var inserted bool
	err := w.db.WithContext(ctx).Raw(`INSERT INTO tb_polling_failure_alert
			(alert_level, card_id, carrier_id, check_type, error_category, fail_count, first_failed_at, last_failed_at, last_error, status)
		VALUES (?, 0, ?, ?, '', ?, ?, ?, ?, ?)
		ON CONFLICT (alert_level, card_id, carrier_id, check_type) WHERE status = 1
		DO UPDATE SET fail_count = EXCLUDED.fail_count, last_failed_at = EXCLUDED.last_failed_at, updated_at = NOW()
		RETURNING (xmax = 0)`,
		constants.PollingAlertLevelCarrier, carrierID, constants.PollingAlertCheckTypeAll, count.Val(), now, now,
		"窗口内同账户新增失败告警数超过阈值，通常是账户额度、凭证或上游故障，请联系上游处理账户", constants.PollingAlertStatusOpen).Row().Scan(&inserted)
	if err != nil {
		w.logger.Warn("写账户级轮询告警失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
		return
	}
	if inserted {
		w.logger.Error("运营商账户轮询失败告警数超过阈值", zap.Uint("carrier_id", carrierID), zap.Int64("count", count.Val()),
			zap.Duration("window", window), zap.String("metric", "polling.failure_alert.carrier_opened"))
	}
}

func nullableTime(t time.Time) any {
	if t.IsZero() {
		return nil
	}
	return t
}

func truncate(s string, max int) string {
	runes := []rune(s)
	if len(runes) <= max {
		return s
	}
	return string(runes[:max])
}