监控
This commit is contained in:
@@ -264,7 +264,7 @@ func initServices(s *stores, deps *Dependencies) *services {
|
|||||||
Recharge: rechargeSvc.New(deps.DB, s.AssetWallet, s.AssetWalletTransaction, s.IotCard, s.Device, s.ShopSeriesAllocation, s.PackageSeries, s.CommissionRecord, wechatConfig, paymentLoader, deps.Logger),
|
Recharge: rechargeSvc.New(deps.DB, s.AssetWallet, s.AssetWalletTransaction, s.IotCard, s.Device, s.ShopSeriesAllocation, s.PackageSeries, s.CommissionRecord, wechatConfig, paymentLoader, deps.Logger),
|
||||||
PollingConfig: pollingSvc.NewConfigService(s.PollingConfig, deps.Redis, deps.Logger),
|
PollingConfig: pollingSvc.NewConfigService(s.PollingConfig, deps.Redis, deps.Logger),
|
||||||
PollingConcurrency: pollingSvc.NewConcurrencyService(s.PollingConcurrencyConfig, deps.Redis),
|
PollingConcurrency: pollingSvc.NewConcurrencyService(s.PollingConcurrencyConfig, deps.Redis),
|
||||||
PollingMonitoring: pollingSvc.NewMonitoringService(deps.Redis),
|
PollingMonitoring: pollingSvc.NewMonitoringServiceWithQueueMgr(deps.Redis, pollingQueueMgr, deps.Logger),
|
||||||
PollingAlert: pollingSvc.NewAlertService(s.PollingAlertRule, s.PollingAlertHistory, deps.Redis, deps.Logger),
|
PollingAlert: pollingSvc.NewAlertService(s.PollingAlertRule, s.PollingAlertHistory, deps.Redis, deps.Logger),
|
||||||
PollingCleanup: pollingSvc.NewCleanupService(s.DataCleanupConfig, s.DataCleanupLog, deps.Logger),
|
PollingCleanup: pollingSvc.NewCleanupService(s.DataCleanupConfig, s.DataCleanupLog, deps.Logger),
|
||||||
PollingManualTrigger: pollingSvc.NewManualTriggerService(s.PollingManualTriggerLog, s.IotCard, deps.Redis, deps.Logger),
|
PollingManualTrigger: pollingSvc.NewManualTriggerService(s.PollingManualTriggerLog, s.IotCard, deps.Redis, deps.Logger),
|
||||||
|
|||||||
@@ -186,3 +186,66 @@ func (m *PollingQueueManager) GetTotalQueueDepth(ctx context.Context, taskType s
|
|||||||
}
|
}
|
||||||
return total, firstErr
|
return total, firstErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetTotalDueCount 获取指定任务类型所有分片中已到期的队列数量。
|
||||||
|
// 只统计 score <= now 的成员,用于后台监控展示真实待调度积压。
|
||||||
|
func (m *PollingQueueManager) GetTotalDueCount(ctx context.Context, taskType string, now time.Time) (int64, error) {
|
||||||
|
var total int64
|
||||||
|
var firstErr error
|
||||||
|
for i := 0; i < m.shardCount; i++ {
|
||||||
|
key := constants.RedisPollingShardQueueKey(i, taskType)
|
||||||
|
count, err := m.redis.ZCount(ctx, key, "-inf", fmt.Sprintf("%d", now.Unix())).Result()
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Warn("获取分片到期数量失败",
|
||||||
|
zap.Int("shard_id", i),
|
||||||
|
zap.String("task_type", taskType),
|
||||||
|
zap.Error(err))
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total += count
|
||||||
|
}
|
||||||
|
return total, firstErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAverageWaitTime 获取指定任务类型所有分片最早若干任务的平均等待秒数。
|
||||||
|
// 每个分片最多取 samplePerShard 条,避免监控接口扫描完整队列。
|
||||||
|
func (m *PollingQueueManager) GetAverageWaitTime(ctx context.Context, taskType string, now time.Time, samplePerShard int64) (float64, error) {
|
||||||
|
if samplePerShard <= 0 {
|
||||||
|
samplePerShard = 10
|
||||||
|
}
|
||||||
|
|
||||||
|
var totalWait float64
|
||||||
|
var sampleCount int64
|
||||||
|
var firstErr error
|
||||||
|
nowUnix := now.Unix()
|
||||||
|
|
||||||
|
for i := 0; i < m.shardCount; i++ {
|
||||||
|
key := constants.RedisPollingShardQueueKey(i, taskType)
|
||||||
|
earliest, err := m.redis.ZRangeWithScores(ctx, key, 0, samplePerShard-1).Result()
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Warn("获取分片最早任务失败",
|
||||||
|
zap.Int("shard_id", i),
|
||||||
|
zap.String("task_type", taskType),
|
||||||
|
zap.Error(err))
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, z := range earliest {
|
||||||
|
waitTime := float64(nowUnix) - z.Score
|
||||||
|
if waitTime > 0 {
|
||||||
|
totalWait += waitTime
|
||||||
|
}
|
||||||
|
sampleCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if sampleCount == 0 {
|
||||||
|
return 0, firstErr
|
||||||
|
}
|
||||||
|
return totalWait / float64(sampleCount), firstErr
|
||||||
|
}
|
||||||
|
|||||||
@@ -114,14 +114,10 @@ func (s *MonitoringService) GetOverview(ctx context.Context) (*OverviewStats, er
|
|||||||
|
|
||||||
// GetQueueStatuses 获取所有队列状态
|
// GetQueueStatuses 获取所有队列状态
|
||||||
func (s *MonitoringService) GetQueueStatuses(ctx context.Context) ([]*QueueStatus, error) {
|
func (s *MonitoringService) GetQueueStatuses(ctx context.Context) ([]*QueueStatus, error) {
|
||||||
taskTypes := []string{
|
taskTypes := pollingMonitorTaskTypes()
|
||||||
constants.TaskTypePollingRealname,
|
|
||||||
constants.TaskTypePollingCarddata,
|
|
||||||
constants.TaskTypePollingPackage,
|
|
||||||
}
|
|
||||||
|
|
||||||
result := make([]*QueueStatus, 0, len(taskTypes))
|
result := make([]*QueueStatus, 0, len(taskTypes))
|
||||||
now := time.Now().Unix()
|
now := time.Now()
|
||||||
|
|
||||||
for _, taskType := range taskTypes {
|
for _, taskType := range taskTypes {
|
||||||
status := &QueueStatus{
|
status := &QueueStatus{
|
||||||
@@ -142,43 +138,43 @@ func (s *MonitoringService) GetQueueStatuses(ctx context.Context) ([]*QueueStatu
|
|||||||
|
|
||||||
if s.queueMgr != nil {
|
if s.queueMgr != nil {
|
||||||
status.QueueSize, _ = s.queueMgr.GetTotalQueueDepth(ctx, taskType)
|
status.QueueSize, _ = s.queueMgr.GetTotalQueueDepth(ctx, taskType)
|
||||||
|
status.DueCount, _ = s.queueMgr.GetTotalDueCount(ctx, taskType, now)
|
||||||
|
status.AvgWaitTime, _ = s.queueMgr.GetAverageWaitTime(ctx, taskType, now, 10)
|
||||||
} else {
|
} else {
|
||||||
status.QueueSize, _ = s.redis.ZCard(ctx, queueKey).Result()
|
status.QueueSize, _ = s.redis.ZCard(ctx, queueKey).Result()
|
||||||
|
status.DueCount, _ = s.redis.ZCount(ctx, queueKey, "-inf", formatInt64(now.Unix())).Result()
|
||||||
|
status.AvgWaitTime = s.getLegacyAverageWaitTime(ctx, queueKey, now.Unix())
|
||||||
}
|
}
|
||||||
|
|
||||||
// 获取到期数量(score <= now)
|
|
||||||
status.DueCount, _ = s.redis.ZCount(ctx, queueKey, "-inf", formatInt64(now)).Result()
|
|
||||||
|
|
||||||
// 获取手动触发队列待处理数
|
// 获取手动触发队列待处理数
|
||||||
manualKey := constants.RedisPollingManualQueueKey(taskType)
|
manualKey := constants.RedisPollingManualQueueKey(taskType)
|
||||||
status.ManualPending, _ = s.redis.LLen(ctx, manualKey).Result()
|
status.ManualPending, _ = s.redis.LLen(ctx, manualKey).Result()
|
||||||
|
|
||||||
// 计算平均等待时间(取最早的10个任务的平均等待时间)
|
|
||||||
earliest, err := s.redis.ZRangeWithScores(ctx, queueKey, 0, 9).Result()
|
|
||||||
if err == nil && len(earliest) > 0 {
|
|
||||||
var totalWait float64
|
|
||||||
for _, z := range earliest {
|
|
||||||
waitTime := float64(now) - z.Score
|
|
||||||
if waitTime > 0 {
|
|
||||||
totalWait += waitTime
|
|
||||||
}
|
|
||||||
}
|
|
||||||
status.AvgWaitTime = totalWait / float64(len(earliest))
|
|
||||||
}
|
|
||||||
|
|
||||||
result = append(result, status)
|
result = append(result, status)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getLegacyAverageWaitTime 从旧版非分片队列计算平均等待时间。
|
||||||
|
func (s *MonitoringService) getLegacyAverageWaitTime(ctx context.Context, queueKey string, now int64) float64 {
|
||||||
|
earliest, err := s.redis.ZRangeWithScores(ctx, queueKey, 0, 9).Result()
|
||||||
|
if err != nil || len(earliest) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var totalWait float64
|
||||||
|
for _, z := range earliest {
|
||||||
|
waitTime := float64(now) - z.Score
|
||||||
|
if waitTime > 0 {
|
||||||
|
totalWait += waitTime
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return totalWait / float64(len(earliest))
|
||||||
|
}
|
||||||
|
|
||||||
// GetTaskStatuses 获取所有任务统计
|
// GetTaskStatuses 获取所有任务统计
|
||||||
func (s *MonitoringService) GetTaskStatuses(ctx context.Context) ([]*TaskStats, error) {
|
func (s *MonitoringService) GetTaskStatuses(ctx context.Context) ([]*TaskStats, error) {
|
||||||
taskTypes := []string{
|
taskTypes := pollingMonitorTaskTypes()
|
||||||
constants.TaskTypePollingRealname,
|
|
||||||
constants.TaskTypePollingCarddata,
|
|
||||||
constants.TaskTypePollingPackage,
|
|
||||||
}
|
|
||||||
|
|
||||||
result := make([]*TaskStats, 0, len(taskTypes))
|
result := make([]*TaskStats, 0, len(taskTypes))
|
||||||
|
|
||||||
@@ -269,11 +265,26 @@ func (s *MonitoringService) getTaskTypeName(taskType string) string {
|
|||||||
return "流量检查"
|
return "流量检查"
|
||||||
case constants.TaskTypePollingPackage:
|
case constants.TaskTypePollingPackage:
|
||||||
return "套餐检查"
|
return "套餐检查"
|
||||||
|
case constants.TaskTypePollingProtect:
|
||||||
|
return "保护期检查"
|
||||||
|
case constants.TaskTypePollingCardStatus:
|
||||||
|
return "卡状态检查"
|
||||||
default:
|
default:
|
||||||
return taskType
|
return taskType
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pollingMonitorTaskTypes 返回后台监控需要展示的全部轮询任务类型。
|
||||||
|
func pollingMonitorTaskTypes() []string {
|
||||||
|
return []string{
|
||||||
|
constants.TaskTypePollingRealname,
|
||||||
|
constants.TaskTypePollingCarddata,
|
||||||
|
constants.TaskTypePollingPackage,
|
||||||
|
constants.TaskTypePollingProtect,
|
||||||
|
constants.TaskTypePollingCardStatus,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// parseInt64 解析字符串为 int64
|
// parseInt64 解析字符串为 int64
|
||||||
func parseInt64(s string) int64 {
|
func parseInt64(s string) int64 {
|
||||||
var result int64
|
var result int64
|
||||||
|
|||||||
Reference in New Issue
Block a user