Files
junhong_cmp_fiber/internal/service/polling/carrier_scheduler_service.go
邹进 934a3a27a1
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
feat: 停复机指令队列与设备维度判断
自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭):
- 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机
- 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制
- 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认
- 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后
- 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估
- 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令
- 迁移 000255 插入开关与告警、退避参数

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 00:36:26 +09:00

652 lines
28 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package polling
import (
"context"
"encoding/json"
stderrors "errors"
"strconv"
"time"
"github.com/redis/go-redis/v9"
"gorm.io/datatypes"
"gorm.io/gorm"
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/model/dto"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
"github.com/break/junhong_cmp_fiber/pkg/utils"
)
// CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。
// 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。
type CarrierSchedulerService struct {
db *gorm.DB
redis *redis.Client
auditWriter *auditinfra.Writer
store *carrierqueue.Store
settings *carrierqueue.SettingsProvider
cfg config.CarrierSchedulerConfig
}
// NewCarrierSchedulerService 创建按运营商账户调度后台服务。
func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService {
return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient),
settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg}
}
func (s *CarrierSchedulerService) processes() int {
if s.cfg.SubscriberProcesses > 0 {
return s.cfg.SubscriberProcesses
}
return 1
}
// ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。
func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
var rows []model.CarrierPollingParam
if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
byCarrier := make(map[uint]model.CarrierPollingParam, len(rows))
for _, row := range rows {
byCarrier[row.CarrierID] = row
}
defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg)
resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(),
DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}}
for _, id := range sortedCarrierIDs(names) {
row, configured := byCarrier[id]
resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured))
}
return resp, nil
}
func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem {
merged := carrierqueue.MergeParams(row, configured, s.cfg)
item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured,
InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()}
if configured {
item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit
item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay
item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight
}
return item
}
// GetParam 查询单个运营商账户调度参数。
func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
carrier, err := s.loadCarrier(ctx, carrierID)
if err != nil {
return nil, err
}
row, configured, err := s.loadParam(ctx, carrierID)
if err != nil {
return nil, err
}
item := s.paramItem(carrierID, carrier.CarrierName, row, configured)
return &item, nil
}
// SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。
func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return nil, err
}
carrier, err := s.loadCarrier(ctx, req.CarrierID)
if err != nil {
return nil, err
}
row, configured, err := s.loadParam(ctx, req.CarrierID)
if err != nil {
return nil, err
}
before := paramState(row, configured)
row.CarrierID = req.CarrierID
row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight
row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio
row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay
merged := carrierqueue.MergeParams(row, true, s.cfg)
if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity {
return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+
strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight")
}
if !configured {
row.Creator = operatorID
}
row.Updater = operatorID
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Save(&row).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数",
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID,
ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit,
"card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio},
BeforeData: before, AfterData: paramState(row, true),
})
})
if err != nil {
return nil, err
}
item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true)
return &item, nil
}
// DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。
func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return err
}
row, configured, err := s.loadParam(ctx, carrierID)
if err != nil {
return err
}
if !configured {
return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数")
}
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)",
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID,
ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID,
IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true),
})
})
}
func paramState(row model.CarrierPollingParam, configured bool) map[string]any {
if !configured {
return map[string]any{"configured": false}
}
return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight,
"stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit,
"priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval,
"stop_resume_confirm_delay": row.StopResumeConfirmDelay}
}
func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) {
var row model.CarrierPollingParam
err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return model.CarrierPollingParam{}, false, nil
}
if err != nil {
return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
return row, true, nil
}
func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) {
var carrier model.Carrier
if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "运营商账户不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
}
return &carrier, nil
}
func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) {
var carriers []model.Carrier
if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
}
names := make(map[uint]string, len(carriers))
for _, carrier := range carriers {
names[carrier.ID] = carrier.CarrierName
}
return names, nil
}
// ListScenes 列出请求检查场景。
func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var rows []model.PollingCheckScene
if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
}
items := make([]dto.PollingCheckSceneItem, 0, len(rows))
for _, row := range rows {
items = append(items, sceneItem(row))
}
return items, nil
}
func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem {
item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt,
CheckTypes: []string{}, ConfirmDelays: []int{}}
_ = json.Unmarshal(row.CheckTypes, &item.CheckTypes)
_ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays)
return item
}
// UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。
func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return nil, err
}
for index := 1; index < len(req.ConfirmDelays); index++ {
if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] {
return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增")
}
}
types := uniqueStrings(req.CheckTypes)
var row model.PollingCheckScene
if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
}
before := sceneItem(row)
typesJSON, _ := json.Marshal(types)
delays := req.ConfirmDelays
if delays == nil {
delays = []int{}
}
delaysJSON, _ := json.Marshal(delays)
row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{
"check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(),
}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景",
ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID,
ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName},
BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown},
AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown},
})
})
if err != nil {
return nil, err
}
item := sceneItem(row)
return &item, nil
}
// ListAlerts 分页查询卡轮询失败告警。
func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
page, size := normalizePage(req.Page, req.PageSize)
query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{})
if req.CarrierID != nil {
query = query.Where("carrier_id = ?", *req.CarrierID)
}
if req.CheckType != "" {
query = query.Where("check_type = ?", req.CheckType)
}
if req.ErrorCategory != "" {
query = query.Where("error_category = ?", req.ErrorCategory)
}
if req.Status != nil {
query = query.Where("status = ?", *req.Status)
}
if req.AlertLevel != "" {
query = query.Where("alert_level = ?", req.AlertLevel)
}
if req.ICCID != "" {
query = query.Where("iccid = ?", req.ICCID)
}
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
if err != nil {
return nil, err
}
if start != nil {
query = query.Where("last_failed_at >= ?", *start)
}
if end != nil {
query = query.Where("last_failed_at <= ?", *end)
}
var total int64
if err := query.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
}
var rows []model.PollingFailureAlert
if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
for _, row := range rows {
result.List = append(result.List, alertItem(row, names))
}
return result, nil
}
func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem {
return dto.PollingFailureAlertItem{
ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType),
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt,
}
}
// AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。
func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var rows []struct {
CarrierID uint
ErrorCategory string
OpenCount int64
}
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).
Select("carrier_id, error_category, COUNT(*) AS open_count").
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败")
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows))
for _, row := range rows {
items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID],
ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount})
}
return items, nil
}
// GetAlert 查询告警详情。
func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
row, err := s.loadAlert(ctx, id)
if err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
item := alertItem(*row, names)
return &item, nil
}
// HandleAlert 把未处理告警标记为已处理。
func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error {
operatorID, err := requireSuperAdminWriterOrPlatform(ctx)
if err != nil {
return err
}
row, err := s.loadAlert(ctx, req.ID)
if err != nil {
return err
}
if row.Status != constants.PollingAlertStatusOpen {
return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理")
}
now := time.Now()
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen).
Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now})
if result.Error != nil {
return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败")
}
if result.RowsAffected == 0 {
return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理",
ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID,
ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID,
"carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status},
BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark},
})
})
}
func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) {
var row model.PollingFailureAlert
if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
}
return &row, nil
}
// CardState 查询单卡轮询状态(只读)。
func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var card model.IotCard
if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "IoT卡不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败")
}
resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}}
due, err := s.store.ReadDue(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败")
}
carrierID := due.CarrierID
if carrierID == 0 {
carrierID = card.CarrierID
}
member := strconv.FormatUint(uint64(card.ID), 10)
if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil {
resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score))
} else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil {
resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score))
}
if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 {
resp.Running = true
expire := time.Now().Add(ttl)
resp.ClaimExpireAt = &expire
}
resp.PriorityMark = due.Priority
for _, checkType := range constants.AllCheckTypes() {
item, ok := due.Types[checkType]
if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) {
continue
}
state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType),
FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}}
if item.HasDue {
state.NextDueAt = timePtr(item.Due)
}
if !item.LastOK.IsZero() {
state.LastSuccessAt = timePtr(item.LastOK)
}
if !item.FirstFailed.IsZero() {
state.FirstFailedAt = timePtr(item.FirstFailed)
}
for _, delay := range item.Confirm {
state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second))
}
resp.Types = append(resp.Types, state)
}
resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled
command, err := s.store.ReadCommand(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败")
}
if !command.LastCallAt.IsZero() {
resp.LastStopResumeCallAt = timePtr(command.LastCallAt)
}
if command.Exists {
item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source,
Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown}
if item.Scenes == nil {
item.Scenes = []string{}
}
if !command.WrittenAt.IsZero() {
item.WrittenAt = timePtr(command.WrittenAt)
}
if !command.NextAt.IsZero() {
item.NextAt = timePtr(command.NextAt)
}
resp.StopResumeCommand = item
}
return resp, nil
}
// CarrierStats 按运营商账户的轮询监控。
func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
var params []model.CarrierPollingParam
if err := s.db.WithContext(ctx).Find(&params).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
paramByCarrier := map[uint]model.CarrierPollingParam{}
for _, row := range params {
paramByCarrier[row.CarrierID] = row
}
var alertRows []struct {
CarrierID uint
ErrorCategory string
Count int64
}
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count").
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
}
state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}}
now := time.Now()
nowMS := strconv.FormatInt(now.UnixMilli(), 10)
lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504")
for _, carrierID := range sortedCarrierIDs(names) {
row, configured := paramByCarrier[carrierID]
item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID],
InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}}
pipe := s.redis.Pipeline()
priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID))
queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID))
priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS)
queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS)
running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf")
expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID))
var statCmds []*redis.MapStringStringCmd
for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} {
statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute)))
}
_, _ = pipe.Exec(ctx)
item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val()
item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val()
cutoff := now.Add(-time.Hour).Unix() / 60
for minute, value := range expired.Val() {
if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff {
count, _ := strconv.ParseInt(value, 10, 64)
item.ExpiredClaims1h += count
}
}
var attempts int64
for _, cmd := range statCmds {
values := cmd.Val()
attempts += atoi64(values["attempts"])
item.RateLimitWaits += atoi64(values["rate_wait"])
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
}
item.QPSLastMinute = float64(attempts) / 60
if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil {
item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due
item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending
}
srCounts := s.store.StopResumeStatLastHour(ctx, carrierID)
item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed]
item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown]
for _, alert := range alertRows {
if alert.CarrierID == carrierID {
item.OpenAlerts += alert.Count
item.OpenAlertsByType[alert.ErrorCategory] += alert.Count
}
}
resp.Items = append(resp.Items, item)
}
return resp, nil
}
// requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。
func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) {
if err := requirePlatformReader(ctx); err != nil {
return 0, err
}
operatorID := middleware.GetUserIDFromContext(ctx)
if operatorID == 0 {
return 0, errors.New(errors.CodeUnauthorized, "未授权访问")
}
return operatorID, nil
}
func sortedCarrierIDs(names map[uint]string) []uint {
ids := make([]uint, 0, len(names))
for id := range names {
ids = append(ids, id)
}
for i := 1; i < len(ids); i++ {
for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
ids[j], ids[j-1] = ids[j-1], ids[j]
}
}
return ids
}
func msTime(ms int64) *time.Time {
t := time.UnixMilli(ms)
return &t
}
func timePtr(t time.Time) *time.Time {
return &t
}
func atoi64(s string) int64 {
value, _ := strconv.ParseInt(s, 10, 64)
return value
}