feat: 按运营商账户调度的后台接口与重算到期

- /polling-carrier-params 列表(含默认标记与消费能力)、详情、保存(执行上限不得大于订阅进程 max_in_flight 之和)、删除;/polling-check-scenes 列表与修改(检查类型、递增的确认延迟、冷却);/polling-failure-alerts 列表、按账户与错误类别汇总、详情、标记已处理;GET /iot-cards/:id/polling-state 单卡轮询状态;GET /polling-stats/carrier-queues 按账户队列与执行监控
- 读取仅超级管理员与平台账号,修改仅超级管理员(告警处理平台账号可用),均写统一审计;新增审计动作与资源
- carrier 模式下重算到期时间写入卡到期信息与账户队列,只提前不推后
This commit is contained in:
邹进
2026-10-25 18:56:32 +09:00
parent 0d966249b8
commit c741f1751d
14 changed files with 1190 additions and 7 deletions

View File

@@ -0,0 +1,622 @@
package polling
import (
"context"
"encoding/json"
stderrors "errors"
"strconv"
"time"
"github.com/redis/go-redis/v9"
"gorm.io/datatypes"
"gorm.io/gorm"
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/model/dto"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
"github.com/break/junhong_cmp_fiber/pkg/utils"
)
// CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。
// 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。
type CarrierSchedulerService struct {
db *gorm.DB
redis *redis.Client
auditWriter *auditinfra.Writer
store *carrierqueue.Store
settings *carrierqueue.SettingsProvider
cfg config.CarrierSchedulerConfig
}
// NewCarrierSchedulerService 创建按运营商账户调度后台服务。
func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService {
return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient),
settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg}
}
func (s *CarrierSchedulerService) processes() int {
if s.cfg.SubscriberProcesses > 0 {
return s.cfg.SubscriberProcesses
}
return 1
}
// ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。
func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
var rows []model.CarrierPollingParam
if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
byCarrier := make(map[uint]model.CarrierPollingParam, len(rows))
for _, row := range rows {
byCarrier[row.CarrierID] = row
}
defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg)
resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(),
DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}}
for _, id := range sortedCarrierIDs(names) {
row, configured := byCarrier[id]
resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured))
}
return resp, nil
}
func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem {
merged := carrierqueue.MergeParams(row, configured, s.cfg)
item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured,
InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()}
if configured {
item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit
item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay
item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight
}
return item
}
// GetParam 查询单个运营商账户调度参数。
func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
carrier, err := s.loadCarrier(ctx, carrierID)
if err != nil {
return nil, err
}
row, configured, err := s.loadParam(ctx, carrierID)
if err != nil {
return nil, err
}
item := s.paramItem(carrierID, carrier.CarrierName, row, configured)
return &item, nil
}
// SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。
func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return nil, err
}
carrier, err := s.loadCarrier(ctx, req.CarrierID)
if err != nil {
return nil, err
}
row, configured, err := s.loadParam(ctx, req.CarrierID)
if err != nil {
return nil, err
}
before := paramState(row, configured)
row.CarrierID = req.CarrierID
row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight
row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio
row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay
merged := carrierqueue.MergeParams(row, true, s.cfg)
if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity {
return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+
strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight")
}
if !configured {
row.Creator = operatorID
}
row.Updater = operatorID
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Save(&row).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数",
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID,
ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit,
"card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio},
BeforeData: before, AfterData: paramState(row, true),
})
})
if err != nil {
return nil, err
}
item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true)
return &item, nil
}
// DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。
func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return err
}
row, configured, err := s.loadParam(ctx, carrierID)
if err != nil {
return err
}
if !configured {
return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数")
}
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)",
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID,
ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID,
IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true),
})
})
}
func paramState(row model.CarrierPollingParam, configured bool) map[string]any {
if !configured {
return map[string]any{"configured": false}
}
return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight,
"stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit,
"priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval,
"stop_resume_confirm_delay": row.StopResumeConfirmDelay}
}
func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) {
var row model.CarrierPollingParam
err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return model.CarrierPollingParam{}, false, nil
}
if err != nil {
return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
return row, true, nil
}
func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) {
var carrier model.Carrier
if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "运营商账户不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
}
return &carrier, nil
}
func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) {
var carriers []model.Carrier
if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
}
names := make(map[uint]string, len(carriers))
for _, carrier := range carriers {
names[carrier.ID] = carrier.CarrierName
}
return names, nil
}
// ListScenes 列出请求检查场景。
func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var rows []model.PollingCheckScene
if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
}
items := make([]dto.PollingCheckSceneItem, 0, len(rows))
for _, row := range rows {
items = append(items, sceneItem(row))
}
return items, nil
}
func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem {
item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt,
CheckTypes: []string{}, ConfirmDelays: []int{}}
_ = json.Unmarshal(row.CheckTypes, &item.CheckTypes)
_ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays)
return item
}
// UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。
func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return nil, err
}
for index := 1; index < len(req.ConfirmDelays); index++ {
if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] {
return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增")
}
}
types := uniqueStrings(req.CheckTypes)
var row model.PollingCheckScene
if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
}
before := sceneItem(row)
typesJSON, _ := json.Marshal(types)
delays := req.ConfirmDelays
if delays == nil {
delays = []int{}
}
delaysJSON, _ := json.Marshal(delays)
row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{
"check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(),
}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景",
ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID,
ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName},
BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown},
AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown},
})
})
if err != nil {
return nil, err
}
item := sceneItem(row)
return &item, nil
}
// ListAlerts 分页查询卡轮询失败告警。
func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
page, size := normalizePage(req.Page, req.PageSize)
query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{})
if req.CarrierID != nil {
query = query.Where("carrier_id = ?", *req.CarrierID)
}
if req.CheckType != "" {
query = query.Where("check_type = ?", req.CheckType)
}
if req.ErrorCategory != "" {
query = query.Where("error_category = ?", req.ErrorCategory)
}
if req.Status != nil {
query = query.Where("status = ?", *req.Status)
}
if req.AlertLevel != "" {
query = query.Where("alert_level = ?", req.AlertLevel)
}
if req.ICCID != "" {
query = query.Where("iccid = ?", req.ICCID)
}
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
if err != nil {
return nil, err
}
if start != nil {
query = query.Where("last_failed_at >= ?", *start)
}
if end != nil {
query = query.Where("last_failed_at <= ?", *end)
}
var total int64
if err := query.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
}
var rows []model.PollingFailureAlert
if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
for _, row := range rows {
result.List = append(result.List, alertItem(row, names))
}
return result, nil
}
func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem {
return dto.PollingFailureAlertItem{
ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType),
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt,
}
}
// AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。
func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var rows []struct {
CarrierID uint
ErrorCategory string
OpenCount int64
}
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).
Select("carrier_id, error_category, COUNT(*) AS open_count").
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败")
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows))
for _, row := range rows {
items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID],
ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount})
}
return items, nil
}
// GetAlert 查询告警详情。
func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
row, err := s.loadAlert(ctx, id)
if err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
item := alertItem(*row, names)
return &item, nil
}
// HandleAlert 把未处理告警标记为已处理。
func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error {
operatorID, err := requireSuperAdminWriterOrPlatform(ctx)
if err != nil {
return err
}
row, err := s.loadAlert(ctx, req.ID)
if err != nil {
return err
}
if row.Status != constants.PollingAlertStatusOpen {
return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理")
}
now := time.Now()
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen).
Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now})
if result.Error != nil {
return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败")
}
if result.RowsAffected == 0 {
return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理",
ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID,
ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID,
"carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status},
BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark},
})
})
}
func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) {
var row model.PollingFailureAlert
if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
}
return &row, nil
}
// CardState 查询单卡轮询状态(只读)。
func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var card model.IotCard
if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "IoT卡不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败")
}
resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}}
due, err := s.store.ReadDue(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败")
}
carrierID := due.CarrierID
if carrierID == 0 {
carrierID = card.CarrierID
}
member := strconv.FormatUint(uint64(card.ID), 10)
if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil {
resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score))
} else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil {
resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score))
}
if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 {
resp.Running = true
expire := time.Now().Add(ttl)
resp.ClaimExpireAt = &expire
}
resp.PriorityMark = due.Priority
for _, checkType := range constants.AllCheckTypes() {
item, ok := due.Types[checkType]
if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) {
continue
}
state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType),
FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}}
if item.HasDue {
state.NextDueAt = timePtr(item.Due)
}
if !item.LastOK.IsZero() {
state.LastSuccessAt = timePtr(item.LastOK)
}
if !item.FirstFailed.IsZero() {
state.FirstFailedAt = timePtr(item.FirstFailed)
}
for _, delay := range item.Confirm {
state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second))
}
resp.Types = append(resp.Types, state)
}
return resp, nil
}
// CarrierStats 按运营商账户的轮询监控。
func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
var params []model.CarrierPollingParam
if err := s.db.WithContext(ctx).Find(&params).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
paramByCarrier := map[uint]model.CarrierPollingParam{}
for _, row := range params {
paramByCarrier[row.CarrierID] = row
}
var alertRows []struct {
CarrierID uint
ErrorCategory string
Count int64
}
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count").
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
}
state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}}
now := time.Now()
nowMS := strconv.FormatInt(now.UnixMilli(), 10)
lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504")
for _, carrierID := range sortedCarrierIDs(names) {
row, configured := paramByCarrier[carrierID]
item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID],
InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}}
pipe := s.redis.Pipeline()
priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID))
queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID))
priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS)
queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS)
running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf")
expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID))
var statCmds []*redis.MapStringStringCmd
for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} {
statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute)))
}
_, _ = pipe.Exec(ctx)
item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val()
item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val()
cutoff := now.Add(-time.Hour).Unix() / 60
for minute, value := range expired.Val() {
if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff {
count, _ := strconv.ParseInt(value, 10, 64)
item.ExpiredClaims1h += count
}
}
var attempts int64
for _, cmd := range statCmds {
values := cmd.Val()
attempts += atoi64(values["attempts"])
item.RateLimitWaits += atoi64(values["rate_wait"])
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
}
item.QPSLastMinute = float64(attempts) / 60
for _, alert := range alertRows {
if alert.CarrierID == carrierID {
item.OpenAlerts += alert.Count
item.OpenAlertsByType[alert.ErrorCategory] += alert.Count
}
}
resp.Items = append(resp.Items, item)
}
return resp, nil
}
// requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。
func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) {
if err := requirePlatformReader(ctx); err != nil {
return 0, err
}
operatorID := middleware.GetUserIDFromContext(ctx)
if operatorID == 0 {
return 0, errors.New(errors.CodeUnauthorized, "未授权访问")
}
return operatorID, nil
}
func sortedCarrierIDs(names map[uint]string) []uint {
ids := make([]uint, 0, len(names))
for id := range names {
ids = append(ids, id)
}
for i := 1; i < len(ids); i++ {
for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
ids[j], ids[j-1] = ids[j-1], ids[j]
}
}
return ids
}
func msTime(ms int64) *time.Time {
t := time.UnixMilli(ms)
return &t
}
func timePtr(t time.Time) *time.Time {
return &t
}
func atoi64(s string) int64 {
value, _ := strconv.ParseInt(s, 10, 64)
return value
}