实现七月迭代公共技术基础

internal/query/outbox/metrics.go

@@ -0,0 +1,143 @@
+// Package outbox 提供公共 Outbox 的只读运行状态投影。
+package outbox
+
+import (
+	"context"
+	"time"
+
+	"gorm.io/gorm"
+
+	"github.com/break/junhong_cmp_fiber/internal/model"
+	"github.com/break/junhong_cmp_fiber/pkg/constants"
+)
+
+// EventTypeBacklog 表示某事件类型的未投递积压。
+type EventTypeBacklog struct {
+	EventType string `json:"event_type"`
+	Count     int64  `json:"count"`
+}
+
+// RetryBucket 表示某重试次数上的事件数量。
+type RetryBucket struct {
+	RetryCount int   `json:"retry_count"`
+	Count      int64 `json:"count"`
+}
+
+// Metrics 是 Outbox 运维公开指标投影。
+type Metrics struct {
+	PendingCount         int64              `json:"pending_count"`
+	OldestPendingAgeSecs int64              `json:"oldest_pending_age_seconds"`
+	DeliveringCount      int64              `json:"delivering_count"`
+	ExpiredLeaseCount    int64              `json:"expired_lease_count"`
+	DeliveredInWindow    int64              `json:"delivered_in_window"`
+	FinalFailedCount     int64              `json:"final_failed_count"`
+	DeliverySuccessRate  float64            `json:"delivery_success_rate"`
+	RetryDistribution    []RetryBucket      `json:"retry_distribution"`
+	BacklogByEventType   []EventTypeBacklog `json:"backlog_by_event_type"`
+	ObservedAt           time.Time          `json:"observed_at"`
+}
+
+// Query 查询 PostgreSQL 中的 Outbox 运行事实。
+type Query struct {
+	db  *gorm.DB
+	now func() time.Time
+}
+
+// NewQuery 创建 Outbox 指标查询。
+func NewQuery(db *gorm.DB, now func() time.Time) *Query {
+	if now == nil {
+		now = time.Now
+	}
+	return &Query{db: db, now: now}
+}
+
+// GetMetrics 查询指定统计窗口的积压、成功率、重试和最终失败。
+func (q *Query) GetMetrics(ctx context.Context, window time.Duration) (Metrics, error) {
+	now := q.now().UTC()
+	if window <= 0 {
+		window = time.Hour
+	}
+	metrics := Metrics{ObservedAt: now, RetryDistribution: []RetryBucket{}, BacklogByEventType: []EventTypeBacklog{}}
+	base := func() *gorm.DB { return q.db.WithContext(ctx).Model(&model.OutboxEvent{}) }
+	if err := base().Where("status = ?", constants.OutboxStatusPending).Count(&metrics.PendingCount).Error; err != nil {
+		return metrics, err
+	}
+	if err := base().Where("status = ?", constants.OutboxStatusDelivering).Count(&metrics.DeliveringCount).Error; err != nil {
+		return metrics, err
+	}
+	if err := base().Where("status = ? AND lease_expires_at <= ?", constants.OutboxStatusDelivering, now).
+		Count(&metrics.ExpiredLeaseCount).Error; err != nil {
+		return metrics, err
+	}
+	if err := base().Where("status = ?", constants.OutboxStatusFailed).Count(&metrics.FinalFailedCount).Error; err != nil {
+		return metrics, err
+	}
+	var oldestEvent model.OutboxEvent
+	err := base().Select("created_at").Where("status = ?", constants.OutboxStatusPending).
+		Order("created_at ASC").Take(&oldestEvent).Error
+	if err != nil && err != gorm.ErrRecordNotFound {
+		return metrics, err
+	}
+	if err == nil && oldestEvent.CreatedAt.Before(now) {
+		metrics.OldestPendingAgeSecs = int64(now.Sub(oldestEvent.CreatedAt).Seconds())
+	}
+	windowStart := now.Add(-window)
+	if err := base().Where("status = ? AND delivered_at >= ?", constants.OutboxStatusDelivered, windowStart).
+		Count(&metrics.DeliveredInWindow).Error; err != nil {
+		return metrics, err
+	}
+	var failedInWindow int64
+	if err := base().Where("status = ? AND updated_at >= ?", constants.OutboxStatusFailed, windowStart).
+		Count(&failedInWindow).Error; err != nil {
+		return metrics, err
+	}
+	totalTerminal := metrics.DeliveredInWindow + failedInWindow
+	if totalTerminal > 0 {
+		metrics.DeliverySuccessRate = float64(metrics.DeliveredInWindow) / float64(totalTerminal)
+	}
+	if err := base().Select("retry_count, COUNT(*) AS count").Where("retry_count > 0").
+		Group("retry_count").Order("retry_count ASC").Scan(&metrics.RetryDistribution).Error; err != nil {
+		return metrics, err
+	}
+	if err := base().Select("event_type, COUNT(*) AS count").Where("status IN ?", []int{
+		constants.OutboxStatusPending, constants.OutboxStatusDelivering, constants.OutboxStatusFailed,
+	}).Group("event_type").Order("event_type ASC").Scan(&metrics.BacklogByEventType).Error; err != nil {
+		return metrics, err
+	}
+	return metrics, nil
+}
+
+// Thresholds 是不包含高基数字段的 Outbox 告警阈值。
+type Thresholds struct {
+	PendingCount      int64
+	OldestPendingAge  time.Duration
+	ExpiredLeaseCount int64
+	FinalFailedCount  int64
+}
+
+// Alert 是可交给现有告警通道的中文安全摘要。
+type Alert struct {
+	Code      string `json:"code"`
+	Component string `json:"component"`
+	Count     int64  `json:"count"`
+	Summary   string `json:"summary"`
+	Window    string `json:"window"`
+}
+
+// EvaluateAlerts 根据聚合指标生成低基数告警，不包含 payload 或敏感配置。
+func EvaluateAlerts(metrics Metrics, thresholds Thresholds) []Alert {
+	alerts := make([]Alert, 0, 4)
+	if thresholds.PendingCount > 0 && metrics.PendingCount >= thresholds.PendingCount {
+		alerts = append(alerts, Alert{Code: "OUTBOX_PENDING_HIGH", Component: "outbox", Count: metrics.PendingCount, Summary: "Outbox 待投递事件持续积压", Window: "当前快照"})
+	}
+	if thresholds.OldestPendingAge > 0 && metrics.OldestPendingAgeSecs >= int64(thresholds.OldestPendingAge.Seconds()) {
+		alerts = append(alerts, Alert{Code: "OUTBOX_OLDEST_PENDING_HIGH", Component: "outbox", Count: metrics.OldestPendingAgeSecs, Summary: "Outbox 最老待投递事件超过阈值", Window: "秒"})
+	}
+	if thresholds.ExpiredLeaseCount > 0 && metrics.ExpiredLeaseCount >= thresholds.ExpiredLeaseCount {
+		alerts = append(alerts, Alert{Code: "OUTBOX_EXPIRED_LEASE_HIGH", Component: "outbox", Count: metrics.ExpiredLeaseCount, Summary: "Outbox 过期租约数量超过阈值", Window: "当前快照"})
+	}
+	if thresholds.FinalFailedCount > 0 && metrics.FinalFailedCount >= thresholds.FinalFailedCount {
+		alerts = append(alerts, Alert{Code: "OUTBOX_FINAL_FAILED_HIGH", Component: "outbox", Count: metrics.FinalFailedCount, Summary: "Outbox 最终失败事件需要人工处理", Window: "当前快照"})
+	}
+	return alerts
+}

internal/query/outbox/metrics_integration_test.go

@@ -0,0 +1,73 @@
+package outbox_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox"
+	"github.com/break/junhong_cmp_fiber/internal/model"
+	outboxquery "github.com/break/junhong_cmp_fiber/internal/query/outbox"
+	"github.com/break/junhong_cmp_fiber/internal/testutil"
+	"github.com/break/junhong_cmp_fiber/pkg/constants"
+)
+
+func TestMetricsCoverBacklogRetriesFailuresAndThresholdAlerts(t *testing.T) {
+	db := testutil.NewPostgresTransaction(t)
+	testutil.CreateTemporaryOutboxTable(t, db)
+	repository := outbox.NewRepository()
+	now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC)
+
+	states := []struct {
+		id        string
+		eventType string
+		status    int
+		retries   int
+	}{
+		{id: "metrics-pending", eventType: "example.a", status: constants.OutboxStatusPending},
+		{id: "metrics-delivering", eventType: "example.a", status: constants.OutboxStatusDelivering, retries: 1},
+		{id: "metrics-delivered", eventType: "example.b", status: constants.OutboxStatusDelivered},
+		{id: "metrics-failed", eventType: "example.b", status: constants.OutboxStatusFailed, retries: 3},
+	}
+	for _, state := range states {
+		event, err := repository.Append(context.Background(), db, outbox.Envelope{
+			EventID: state.id, EventType: state.eventType, AggregateType: "example", AggregateID: state.id,
+			ResourceType: "example", ResourceID: state.id, Payload: struct {
+				Visible bool `json:"visible"`
+			}{Visible: true},
+		})
+		if err != nil {
+			t.Fatalf("准备指标事件失败：%v", err)
+		}
+		updates := map[string]any{"status": state.status, "retry_count": state.retries, "created_at": now.Add(-2 * time.Minute), "updated_at": now}
+		if state.status == constants.OutboxStatusDelivering {
+			updates["lease_owner"] = "dead-worker"
+			updates["lease_expires_at"] = now.Add(-time.Minute)
+		}
+		if state.status == constants.OutboxStatusDelivered {
+			updates["delivered_at"] = now.Add(-time.Minute)
+		}
+		if err := db.Model(&model.OutboxEvent{}).Where("id = ?", event.ID).Updates(updates).Error; err != nil {
+			t.Fatalf("设置指标事件状态失败：%v", err)
+		}
+	}
+
+	query := outboxquery.NewQuery(db, func() time.Time { return now })
+	metrics, err := query.GetMetrics(context.Background(), time.Hour)
+	if err != nil {
+		t.Fatalf("查询 Outbox 指标失败：%v", err)
+	}
+	if metrics.PendingCount != 1 || metrics.DeliveringCount != 1 || metrics.ExpiredLeaseCount != 1 ||
+		metrics.DeliveredInWindow != 1 || metrics.FinalFailedCount != 1 || metrics.OldestPendingAgeSecs != 120 {
+		t.Fatalf("Outbox 指标不完整：%+v", metrics)
+	}
+	if len(metrics.RetryDistribution) != 2 || len(metrics.BacklogByEventType) != 2 {
+		t.Fatalf("重试分布或事件类型积压不完整：%+v", metrics)
+	}
+	alerts := outboxquery.EvaluateAlerts(metrics, outboxquery.Thresholds{
+		PendingCount: 1, OldestPendingAge: time.Minute, ExpiredLeaseCount: 1, FinalFailedCount: 1,
+	})
+	if len(alerts) != 4 {
+		t.Fatalf("阈值告警数量错误：%+v", alerts)
+	}
+}