Files
junhong_cmp_fiber/internal/service/polling/carrier_scheduler_service.go
邹进 7ddb9312e3
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m35s
feat: 不轮询清单与批量开关轮询
- 新增不轮询清单 tb_card_polling_pause(迁移 000256 补写存量),所有改 enable_polling 的入口同事务写清单
- 卡、设备勾选批量开关轮询(单次 500,逐项结果,开启跳过风险停机与已销户)与 CSV 文件导入
- 订单套餐生效、改有效期后自动恢复轮询并按规则评估复机
- 不轮询清单查询与导出;卡、设备列表与导出增加是否轮询
- 失败告警停机被拒视图、导出与批量关闭轮询;开关轮询影响说明

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 03:07:20 +09:00

698 lines
30 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package polling
import (
"context"
"encoding/json"
stderrors "errors"
"github.com/break/junhong_cmp_fiber/internal/query/pollingalert"
"strconv"
"time"
"github.com/redis/go-redis/v9"
"gorm.io/datatypes"
"gorm.io/gorm"
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/model/dto"
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
"github.com/break/junhong_cmp_fiber/pkg/utils"
)
// CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。
// 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。
type CarrierSchedulerService struct {
db *gorm.DB
redis *redis.Client
auditWriter *auditinfra.Writer
store *carrierqueue.Store
settings *carrierqueue.SettingsProvider
cfg config.CarrierSchedulerConfig
}
// NewCarrierSchedulerService 创建按运营商账户调度后台服务。
func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService {
return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient),
settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg}
}
func (s *CarrierSchedulerService) processes() int {
if s.cfg.SubscriberProcesses > 0 {
return s.cfg.SubscriberProcesses
}
return 1
}
// ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。
func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
var rows []model.CarrierPollingParam
if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
byCarrier := make(map[uint]model.CarrierPollingParam, len(rows))
for _, row := range rows {
byCarrier[row.CarrierID] = row
}
defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg)
resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(),
DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}}
for _, id := range sortedCarrierIDs(names) {
row, configured := byCarrier[id]
resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured))
}
return resp, nil
}
func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem {
merged := carrierqueue.MergeParams(row, configured, s.cfg)
item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured,
InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()}
if configured {
item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit
item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay
item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight
}
return item
}
// GetParam 查询单个运营商账户调度参数。
func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
carrier, err := s.loadCarrier(ctx, carrierID)
if err != nil {
return nil, err
}
row, configured, err := s.loadParam(ctx, carrierID)
if err != nil {
return nil, err
}
item := s.paramItem(carrierID, carrier.CarrierName, row, configured)
return &item, nil
}
// SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。
func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return nil, err
}
carrier, err := s.loadCarrier(ctx, req.CarrierID)
if err != nil {
return nil, err
}
row, configured, err := s.loadParam(ctx, req.CarrierID)
if err != nil {
return nil, err
}
before := paramState(row, configured)
row.CarrierID = req.CarrierID
row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight
row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio
row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay
merged := carrierqueue.MergeParams(row, true, s.cfg)
if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity {
return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+
strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight")
}
if !configured {
row.Creator = operatorID
}
row.Updater = operatorID
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Save(&row).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数",
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID,
ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit,
"card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio},
BeforeData: before, AfterData: paramState(row, true),
})
})
if err != nil {
return nil, err
}
item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true)
return &item, nil
}
// DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。
func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return err
}
row, configured, err := s.loadParam(ctx, carrierID)
if err != nil {
return err
}
if !configured {
return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数")
}
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)",
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID,
ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID,
IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true),
})
})
}
func paramState(row model.CarrierPollingParam, configured bool) map[string]any {
if !configured {
return map[string]any{"configured": false}
}
return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight,
"stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit,
"priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval,
"stop_resume_confirm_delay": row.StopResumeConfirmDelay}
}
func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) {
var row model.CarrierPollingParam
err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return model.CarrierPollingParam{}, false, nil
}
if err != nil {
return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
return row, true, nil
}
func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) {
var carrier model.Carrier
if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "运营商账户不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
}
return &carrier, nil
}
func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) {
var carriers []model.Carrier
if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
}
names := make(map[uint]string, len(carriers))
for _, carrier := range carriers {
names[carrier.ID] = carrier.CarrierName
}
return names, nil
}
// ListScenes 列出请求检查场景。
func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var rows []model.PollingCheckScene
if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
}
items := make([]dto.PollingCheckSceneItem, 0, len(rows))
for _, row := range rows {
items = append(items, sceneItem(row))
}
return items, nil
}
func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem {
item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt,
CheckTypes: []string{}, ConfirmDelays: []int{}}
_ = json.Unmarshal(row.CheckTypes, &item.CheckTypes)
_ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays)
return item
}
// UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。
func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) {
operatorID, err := requireSuperAdminWriter(ctx)
if err != nil {
return nil, err
}
for index := 1; index < len(req.ConfirmDelays); index++ {
if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] {
return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增")
}
}
types := uniqueStrings(req.CheckTypes)
var row model.PollingCheckScene
if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
}
before := sceneItem(row)
typesJSON, _ := json.Marshal(types)
delays := req.ConfirmDelays
if delays == nil {
delays = []int{}
}
delaysJSON, _ := json.Marshal(delays)
row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{
"check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(),
}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景",
ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID,
ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName},
BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown},
AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown},
})
})
if err != nil {
return nil, err
}
item := sceneItem(row)
return &item, nil
}
// ListAlerts 分页查询卡轮询失败告警。
func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
page, size := normalizePage(req.Page, req.PageSize)
if req.View == constants.PollingAlertViewStopRefused {
return s.listStopRefused(ctx, req, page, size)
}
query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{})
if req.CarrierID != nil {
query = query.Where("carrier_id = ?", *req.CarrierID)
}
if req.CheckType != "" {
query = query.Where("check_type = ?", req.CheckType)
}
if req.ErrorCategory != "" {
query = query.Where("error_category = ?", req.ErrorCategory)
}
if req.Status != nil {
query = query.Where("status = ?", *req.Status)
}
if req.AlertLevel != "" {
query = query.Where("alert_level = ?", req.AlertLevel)
}
if req.ICCID != "" {
query = query.Where("iccid = ?", req.ICCID)
}
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
if err != nil {
return nil, err
}
if start != nil {
query = query.Where("last_failed_at >= ?", *start)
}
if end != nil {
query = query.Where("last_failed_at <= ?", *end)
}
var total int64
if err := query.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
}
var rows []model.PollingFailureAlert
if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
for _, row := range rows {
result.List = append(result.List, alertItem(row, names))
}
return result, nil
}
func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem {
return dto.PollingFailureAlertItem{
ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType),
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt,
}
}
// AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。
func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var rows []struct {
CarrierID uint
ErrorCategory string
OpenCount int64
}
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).
Select("carrier_id, error_category, COUNT(*) AS open_count").
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败")
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows))
for _, row := range rows {
items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID],
ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount})
}
return items, nil
}
// GetAlert 查询告警详情。
func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
row, err := s.loadAlert(ctx, id)
if err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
item := alertItem(*row, names)
return &item, nil
}
// HandleAlert 把未处理告警标记为已处理。
func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error {
operatorID, err := requireSuperAdminWriterOrPlatform(ctx)
if err != nil {
return err
}
row, err := s.loadAlert(ctx, req.ID)
if err != nil {
return err
}
if row.Status != constants.PollingAlertStatusOpen {
return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理")
}
now := time.Now()
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen).
Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now})
if result.Error != nil {
return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败")
}
if result.RowsAffected == 0 {
return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试")
}
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理",
ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID,
ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID,
IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID,
"carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status},
BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark},
})
})
}
func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) {
var row model.PollingFailureAlert
if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
}
return &row, nil
}
// CardState 查询单卡轮询状态(只读)。
func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
var card model.IotCard
if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errors.CodeNotFound, "IoT卡不存在")
}
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败")
}
resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}}
due, err := s.store.ReadDue(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败")
}
carrierID := due.CarrierID
if carrierID == 0 {
carrierID = card.CarrierID
}
member := strconv.FormatUint(uint64(card.ID), 10)
if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil {
resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score))
} else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil {
resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score))
}
if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 {
resp.Running = true
expire := time.Now().Add(ttl)
resp.ClaimExpireAt = &expire
}
resp.PriorityMark = due.Priority
for _, checkType := range constants.AllCheckTypes() {
item, ok := due.Types[checkType]
if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) {
continue
}
state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType),
FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}}
if item.HasDue {
state.NextDueAt = timePtr(item.Due)
}
if !item.LastOK.IsZero() {
state.LastSuccessAt = timePtr(item.LastOK)
}
if !item.FirstFailed.IsZero() {
state.FirstFailedAt = timePtr(item.FirstFailed)
}
for _, delay := range item.Confirm {
state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second))
}
resp.Types = append(resp.Types, state)
}
resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled
command, err := s.store.ReadCommand(ctx, card.ID)
if err != nil {
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败")
}
if !command.LastCallAt.IsZero() {
resp.LastStopResumeCallAt = timePtr(command.LastCallAt)
}
if command.Exists {
item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source,
Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown}
if item.Scenes == nil {
item.Scenes = []string{}
}
if !command.WrittenAt.IsZero() {
item.WrittenAt = timePtr(command.WrittenAt)
}
if !command.NextAt.IsZero() {
item.NextAt = timePtr(command.NextAt)
}
resp.StopResumeCommand = item
}
return resp, nil
}
// CarrierStats 按运营商账户的轮询监控。
func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) {
if err := requirePlatformReader(ctx); err != nil {
return nil, err
}
names, err := s.carrierNameMap(ctx)
if err != nil {
return nil, err
}
var params []model.CarrierPollingParam
if err := s.db.WithContext(ctx).Find(&params).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
}
paramByCarrier := map[uint]model.CarrierPollingParam{}
for _, row := range params {
paramByCarrier[row.CarrierID] = row
}
var alertRows []struct {
CarrierID uint
ErrorCategory string
Count int64
}
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count").
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
}
state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}}
now := time.Now()
nowMS := strconv.FormatInt(now.UnixMilli(), 10)
lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504")
for _, carrierID := range sortedCarrierIDs(names) {
row, configured := paramByCarrier[carrierID]
item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID],
InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}}
pipe := s.redis.Pipeline()
priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID))
queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID))
priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS)
queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS)
running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf")
expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID))
var statCmds []*redis.MapStringStringCmd
for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} {
statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute)))
}
_, _ = pipe.Exec(ctx)
item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val()
item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val()
cutoff := now.Add(-time.Hour).Unix() / 60
for minute, value := range expired.Val() {
if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff {
count, _ := strconv.ParseInt(value, 10, 64)
item.ExpiredClaims1h += count
}
}
var attempts int64
for _, cmd := range statCmds {
values := cmd.Val()
attempts += atoi64(values["attempts"])
item.RateLimitWaits += atoi64(values["rate_wait"])
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
}
item.QPSLastMinute = float64(attempts) / 60
if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil {
item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due
item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending
}
srCounts := s.store.StopResumeStatLastHour(ctx, carrierID)
item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed]
item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown]
for _, alert := range alertRows {
if alert.CarrierID == carrierID {
item.OpenAlerts += alert.Count
item.OpenAlertsByType[alert.ErrorCategory] += alert.Count
}
}
resp.Items = append(resp.Items, item)
}
return resp, nil
}
// requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。
func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) {
if err := requirePlatformReader(ctx); err != nil {
return 0, err
}
operatorID := middleware.GetUserIDFromContext(ctx)
if operatorID == 0 {
return 0, errors.New(errors.CodeUnauthorized, "未授权访问")
}
return operatorID, nil
}
func sortedCarrierIDs(names map[uint]string) []uint {
ids := make([]uint, 0, len(names))
for id := range names {
ids = append(ids, id)
}
for i := 1; i < len(ids); i++ {
for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
ids[j], ids[j-1] = ids[j-1], ids[j]
}
}
return ids
}
func msTime(ms int64) *time.Time {
t := time.UnixMilli(ms)
return &t
}
func timePtr(t time.Time) *time.Time {
return &t
}
func atoi64(s string) int64 {
value, _ := strconv.ParseInt(s, 10, 64)
return value
}
// listStopRefused 停机被拒视图:停复机方向为停机,且错误类型为终态或业务错误、或停机失败次数达到阈值。
func (s *CarrierSchedulerService) listStopRefused(ctx context.Context, req *dto.PollingFailureAlertListRequest, page, size int) (*dto.PollingFailureAlertPageResult, error) {
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
if err != nil {
return nil, err
}
filter := pollingalert.StopRefusedFilter{CarrierID: req.CarrierID, Status: req.Status, ICCID: req.ICCID, StartTime: start, EndTime: end}
threshold := s.settings.Current(ctx).SRAlertFailCount
query := pollingalert.NewQuery(s.db)
total, err := query.Count(ctx, filter, threshold)
if err != nil {
return nil, err
}
rows, err := query.Page(ctx, filter, threshold, (page-1)*size, size)
if err != nil {
return nil, err
}
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
for index := range rows {
result.List = append(result.List, StopRefusedItem(&rows[index]))
}
return result, nil
}
// StopRefusedItem 把停机被拒查询行转换为告警返回项。
func StopRefusedItem(row *pollingalert.StopRefusedRow) dto.PollingFailureAlertItem {
network, polling, hasPackage := row.NetworkStatus, row.EnablePolling, row.HasPackage
item := dto.PollingFailureAlertItem{
ID: row.ID, AlertLevel: constants.PollingAlertLevelCard, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
CarrierName: row.CarrierName, CheckType: constants.CheckTypeStopResume, CheckTypeName: "停复机",
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark,
NetworkStatus: &network, StopReason: row.StopReason, EnablePolling: &polling, HasPackage: &hasPackage, HandledByName: row.HandledByName,
}
if row.DeviceID > 0 {
deviceID := row.DeviceID
item.DeviceID = &deviceID
}
return item
}