package polling import ( "context" "encoding/json" stderrors "errors" "strconv" "time" "github.com/redis/go-redis/v9" "gorm.io/datatypes" "gorm.io/gorm" auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit" "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/internal/model/dto" "github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue" "github.com/break/junhong_cmp_fiber/pkg/config" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" "github.com/break/junhong_cmp_fiber/pkg/middleware" "github.com/break/junhong_cmp_fiber/pkg/utils" ) // CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。 // 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。 type CarrierSchedulerService struct { db *gorm.DB redis *redis.Client auditWriter *auditinfra.Writer store *carrierqueue.Store settings *carrierqueue.SettingsProvider cfg config.CarrierSchedulerConfig } // NewCarrierSchedulerService 创建按运营商账户调度后台服务。 func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService { return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient), settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg} } func (s *CarrierSchedulerService) processes() int { if s.cfg.SubscriberProcesses > 0 { return s.cfg.SubscriberProcesses } return 1 } // ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。 func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } names, err := s.carrierNameMap(ctx) if err != nil { return nil, err } var rows []model.CarrierPollingParam if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败") } byCarrier := make(map[uint]model.CarrierPollingParam, len(rows)) for _, row := range rows { byCarrier[row.CarrierID] = row } defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg) resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(), DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}} for _, id := range sortedCarrierIDs(names) { row, configured := byCarrier[id] resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured)) } return resp, nil } func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem { merged := carrierqueue.MergeParams(row, configured, s.cfg) item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured, InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()} if configured { item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight } return item } // GetParam 查询单个运营商账户调度参数。 func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } carrier, err := s.loadCarrier(ctx, carrierID) if err != nil { return nil, err } row, configured, err := s.loadParam(ctx, carrierID) if err != nil { return nil, err } item := s.paramItem(carrierID, carrier.CarrierName, row, configured) return &item, nil } // SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。 func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) { operatorID, err := requireSuperAdminWriter(ctx) if err != nil { return nil, err } carrier, err := s.loadCarrier(ctx, req.CarrierID) if err != nil { return nil, err } row, configured, err := s.loadParam(ctx, req.CarrierID) if err != nil { return nil, err } before := paramState(row, configured) row.CarrierID = req.CarrierID row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay merged := carrierqueue.MergeParams(row, true, s.cfg) if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity { return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+ strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight") } if !configured { row.Creator = operatorID } row.Updater = operatorID err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error { if err := tx.WithContext(ctx).Save(&row).Error; err != nil { return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败") } return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{ ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数", ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID, ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID, IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio}, BeforeData: before, AfterData: paramState(row, true), }) }) if err != nil { return nil, err } item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true) return &item, nil } // DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。 func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error { operatorID, err := requireSuperAdminWriter(ctx) if err != nil { return err } row, configured, err := s.loadParam(ctx, carrierID) if err != nil { return err } if !configured { return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数") } return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error { if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil { return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败") } return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{ ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)", ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID, ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID, IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true), }) }) } func paramState(row model.CarrierPollingParam, configured bool) map[string]any { if !configured { return map[string]any{"configured": false} } return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight, "stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit, "priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval, "stop_resume_confirm_delay": row.StopResumeConfirmDelay} } func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) { var row model.CarrierPollingParam err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error if stderrors.Is(err, gorm.ErrRecordNotFound) { return model.CarrierPollingParam{}, false, nil } if err != nil { return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败") } return row, true, nil } func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) { var carrier model.Carrier if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil { if stderrors.Is(err, gorm.ErrRecordNotFound) { return nil, errors.New(errors.CodeNotFound, "运营商账户不存在") } return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败") } return &carrier, nil } func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) { var carriers []model.Carrier if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败") } names := make(map[uint]string, len(carriers)) for _, carrier := range carriers { names[carrier.ID] = carrier.CarrierName } return names, nil } // ListScenes 列出请求检查场景。 func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } var rows []model.PollingCheckScene if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败") } items := make([]dto.PollingCheckSceneItem, 0, len(rows)) for _, row := range rows { items = append(items, sceneItem(row)) } return items, nil } func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem { item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt, CheckTypes: []string{}, ConfirmDelays: []int{}} _ = json.Unmarshal(row.CheckTypes, &item.CheckTypes) _ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays) return item } // UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。 func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) { operatorID, err := requireSuperAdminWriter(ctx) if err != nil { return nil, err } for index := 1; index < len(req.ConfirmDelays); index++ { if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] { return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增") } } types := uniqueStrings(req.CheckTypes) var row model.PollingCheckScene if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil { if stderrors.Is(err, gorm.ErrRecordNotFound) { return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在") } return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败") } before := sceneItem(row) typesJSON, _ := json.Marshal(types) delays := req.ConfirmDelays if delays == nil { delays = []int{} } delaysJSON, _ := json.Marshal(delays) row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error { if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{ "check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(), }).Error; err != nil { return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败") } return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{ ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景", ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID, ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID, IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName}, BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown}, AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown}, }) }) if err != nil { return nil, err } item := sceneItem(row) return &item, nil } // ListAlerts 分页查询卡轮询失败告警。 func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } page, size := normalizePage(req.Page, req.PageSize) query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}) if req.CarrierID != nil { query = query.Where("carrier_id = ?", *req.CarrierID) } if req.CheckType != "" { query = query.Where("check_type = ?", req.CheckType) } if req.ErrorCategory != "" { query = query.Where("error_category = ?", req.ErrorCategory) } if req.Status != nil { query = query.Where("status = ?", *req.Status) } if req.AlertLevel != "" { query = query.Where("alert_level = ?", req.AlertLevel) } if req.ICCID != "" { query = query.Where("iccid = ?", req.ICCID) } start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime) if err != nil { return nil, err } if start != nil { query = query.Where("last_failed_at >= ?", *start) } if end != nil { query = query.Where("last_failed_at <= ?", *end) } var total int64 if err := query.Count(&total).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败") } var rows []model.PollingFailureAlert if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败") } names, err := s.carrierNameMap(ctx) if err != nil { return nil, err } result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))} for _, row := range rows { result.List = append(result.List, alertItem(row, names)) } return result, nil } func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem { return dto.PollingFailureAlertItem{ ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID, CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType), ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt, LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status), HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt, } } // AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。 func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } var rows []struct { CarrierID uint ErrorCategory string OpenCount int64 } if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}). Select("carrier_id, error_category, COUNT(*) AS open_count"). Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard). Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败") } names, err := s.carrierNameMap(ctx) if err != nil { return nil, err } items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows)) for _, row := range rows { items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID], ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount}) } return items, nil } // GetAlert 查询告警详情。 func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } row, err := s.loadAlert(ctx, id) if err != nil { return nil, err } names, err := s.carrierNameMap(ctx) if err != nil { return nil, err } item := alertItem(*row, names) return &item, nil } // HandleAlert 把未处理告警标记为已处理。 func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error { operatorID, err := requireSuperAdminWriterOrPlatform(ctx) if err != nil { return err } row, err := s.loadAlert(ctx, req.ID) if err != nil { return err } if row.Status != constants.PollingAlertStatusOpen { return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理") } now := time.Now() return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error { result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen). Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now}) if result.Error != nil { return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败") } if result.RowsAffected == 0 { return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试") } return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{ ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理", ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID, ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID, IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID, "carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status}, BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark}, }) }) } func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) { var row model.PollingFailureAlert if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil { if stderrors.Is(err, gorm.ErrRecordNotFound) { return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在") } return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败") } return &row, nil } // CardState 查询单卡轮询状态(只读)。 func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } var card model.IotCard if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil { if stderrors.Is(err, gorm.ErrRecordNotFound) { return nil, errors.New(errors.CodeNotFound, "IoT卡不存在") } return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败") } resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID, SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}} due, err := s.store.ReadDue(ctx, card.ID) if err != nil { return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败") } carrierID := due.CarrierID if carrierID == 0 { carrierID = card.CarrierID } member := strconv.FormatUint(uint64(card.ID), 10) if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil { resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score)) } else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil { resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score)) } if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 { resp.Running = true expire := time.Now().Add(ttl) resp.ClaimExpireAt = &expire } resp.PriorityMark = due.Priority for _, checkType := range constants.AllCheckTypes() { item, ok := due.Types[checkType] if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) { continue } state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType), FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}} if item.HasDue { state.NextDueAt = timePtr(item.Due) } if !item.LastOK.IsZero() { state.LastSuccessAt = timePtr(item.LastOK) } if !item.FirstFailed.IsZero() { state.FirstFailedAt = timePtr(item.FirstFailed) } for _, delay := range item.Confirm { state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second)) } resp.Types = append(resp.Types, state) } resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled command, err := s.store.ReadCommand(ctx, card.ID) if err != nil { return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败") } if !command.LastCallAt.IsZero() { resp.LastStopResumeCallAt = timePtr(command.LastCallAt) } if command.Exists { item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source, Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown} if item.Scenes == nil { item.Scenes = []string{} } if !command.WrittenAt.IsZero() { item.WrittenAt = timePtr(command.WrittenAt) } if !command.NextAt.IsZero() { item.NextAt = timePtr(command.NextAt) } resp.StopResumeCommand = item } return resp, nil } // CarrierStats 按运营商账户的轮询监控。 func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) { if err := requirePlatformReader(ctx); err != nil { return nil, err } names, err := s.carrierNameMap(ctx) if err != nil { return nil, err } var params []model.CarrierPollingParam if err := s.db.WithContext(ctx).Find(¶ms).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败") } paramByCarrier := map[uint]model.CarrierPollingParam{} for _, row := range params { paramByCarrier[row.CarrierID] = row } var alertRows []struct { CarrierID uint ErrorCategory string Count int64 } if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count"). Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard). Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败") } state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result() resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}} now := time.Now() nowMS := strconv.FormatInt(now.UnixMilli(), 10) lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504") for _, carrierID := range sortedCarrierIDs(names) { row, configured := paramByCarrier[carrierID] item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID], InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}} pipe := s.redis.Pipeline() priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID)) queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID)) priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS) queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS) running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf") expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID)) var statCmds []*redis.MapStringStringCmd for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} { statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute))) } _, _ = pipe.Exec(ctx) item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val() item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val() cutoff := now.Add(-time.Hour).Unix() / 60 for minute, value := range expired.Val() { if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff { count, _ := strconv.ParseInt(value, 10, 64) item.ExpiredClaims1h += count } } var attempts int64 for _, cmd := range statCmds { values := cmd.Val() attempts += atoi64(values["attempts"]) item.RateLimitWaits += atoi64(values["rate_wait"]) item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited]) } item.QPSLastMinute = float64(attempts) / 60 if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil { item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending } srCounts := s.store.StopResumeStatLastHour(ctx, carrierID) item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed] item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown] for _, alert := range alertRows { if alert.CarrierID == carrierID { item.OpenAlerts += alert.Count item.OpenAlertsByType[alert.ErrorCategory] += alert.Count } } resp.Items = append(resp.Items, item) } return resp, nil } // requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。 func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) { if err := requirePlatformReader(ctx); err != nil { return 0, err } operatorID := middleware.GetUserIDFromContext(ctx) if operatorID == 0 { return 0, errors.New(errors.CodeUnauthorized, "未授权访问") } return operatorID, nil } func sortedCarrierIDs(names map[uint]string) []uint { ids := make([]uint, 0, len(names)) for id := range names { ids = append(ids, id) } for i := 1; i < len(ids); i++ { for j := i; j > 0 && ids[j] < ids[j-1]; j-- { ids[j], ids[j-1] = ids[j-1], ids[j] } } return ids } func msTime(ms int64) *time.Time { t := time.UnixMilli(ms) return &t } func timePtr(t time.Time) *time.Time { return &t } func atoi64(s string) int64 { value, _ := strconv.ParseInt(s, 10, 64) return value }