Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭): - 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机 - 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制 - 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认 - 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后 - 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估 - 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令 - 迁移 000255 插入开关与告警、退避参数 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
652 lines
28 KiB
Go
652 lines
28 KiB
Go
package polling
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
stderrors "errors"
|
||
"strconv"
|
||
"time"
|
||
|
||
"github.com/redis/go-redis/v9"
|
||
"gorm.io/datatypes"
|
||
"gorm.io/gorm"
|
||
|
||
auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||
"github.com/break/junhong_cmp_fiber/internal/model/dto"
|
||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||
"github.com/break/junhong_cmp_fiber/pkg/utils"
|
||
)
|
||
|
||
// CarrierSchedulerService 按运营商账户调度的后台维护与监控:调度参数、请求检查场景、失败告警、单卡轮询状态与按账户监控。
|
||
// 读取仅超级管理员与平台账号;修改仅超级管理员并写统一审计,下一轮调度生效。
|
||
type CarrierSchedulerService struct {
|
||
db *gorm.DB
|
||
redis *redis.Client
|
||
auditWriter *auditinfra.Writer
|
||
store *carrierqueue.Store
|
||
settings *carrierqueue.SettingsProvider
|
||
cfg config.CarrierSchedulerConfig
|
||
}
|
||
|
||
// NewCarrierSchedulerService 创建按运营商账户调度后台服务。
|
||
func NewCarrierSchedulerService(db *gorm.DB, redisClient *redis.Client, auditWriter *auditinfra.Writer, cfg config.CarrierSchedulerConfig) *CarrierSchedulerService {
|
||
return &CarrierSchedulerService{db: db, redis: redisClient, auditWriter: auditWriter, store: carrierqueue.NewStore(redisClient),
|
||
settings: carrierqueue.NewDefaultSettingsProvider(db, redisClient, nil), cfg: cfg}
|
||
}
|
||
|
||
func (s *CarrierSchedulerService) processes() int {
|
||
if s.cfg.SubscriberProcesses > 0 {
|
||
return s.cfg.SubscriberProcesses
|
||
}
|
||
return 1
|
||
}
|
||
|
||
// ListParams 列出全部运营商账户的调度参数(未配置的账户返回全局默认并标记默认)。
|
||
func (s *CarrierSchedulerService) ListParams(ctx context.Context) (*dto.CarrierPollingParamListResponse, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
names, err := s.carrierNameMap(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var rows []model.CarrierPollingParam
|
||
if err := s.db.WithContext(ctx).Find(&rows).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||
}
|
||
byCarrier := make(map[uint]model.CarrierPollingParam, len(rows))
|
||
for _, row := range rows {
|
||
byCarrier[row.CarrierID] = row
|
||
}
|
||
defaults := carrierqueue.MergeParams(model.CarrierPollingParam{}, false, s.cfg)
|
||
resp := &dto.CarrierPollingParamListResponse{SubscriberProcesses: s.processes(),
|
||
DefaultInflight: defaults.InflightLimit, DefaultMaxInFlight: defaults.CardMaxInFlight, Items: []dto.CarrierPollingParamItem{}}
|
||
for _, id := range sortedCarrierIDs(names) {
|
||
row, configured := byCarrier[id]
|
||
resp.Items = append(resp.Items, s.paramItem(id, names[id], row, configured))
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
func (s *CarrierSchedulerService) paramItem(carrierID uint, name string, row model.CarrierPollingParam, configured bool) dto.CarrierPollingParamItem {
|
||
merged := carrierqueue.MergeParams(row, configured, s.cfg)
|
||
item := dto.CarrierPollingParamItem{CarrierID: carrierID, CarrierName: name, IsDefault: !configured,
|
||
InflightLimit: merged.InflightLimit, CardMaxInFlight: merged.CardMaxInFlight, ConsumeCapacity: merged.CardMaxInFlight * s.processes()}
|
||
if configured {
|
||
item.PriorityMaxRatio, item.StopResumeMaxInFlight, item.SRInflightLimit = row.PriorityMaxRatio, row.StopResumeMaxInFlight, row.SRInflightLimit
|
||
item.StopResumeMinInterval, item.StopResumeConfirmDelay = row.StopResumeMinInterval, row.StopResumeConfirmDelay
|
||
item.ConfiguredInflight, item.ConfiguredMaxInFlight = row.InflightLimit, row.CardMaxInFlight
|
||
}
|
||
return item
|
||
}
|
||
|
||
// GetParam 查询单个运营商账户调度参数。
|
||
func (s *CarrierSchedulerService) GetParam(ctx context.Context, carrierID uint) (*dto.CarrierPollingParamItem, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
carrier, err := s.loadCarrier(ctx, carrierID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
row, configured, err := s.loadParam(ctx, carrierID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
item := s.paramItem(carrierID, carrier.CarrierName, row, configured)
|
||
return &item, nil
|
||
}
|
||
|
||
// SaveParam 保存运营商账户调度参数;执行上限不得大于当前订阅进程 max_in_flight 之和。
|
||
func (s *CarrierSchedulerService) SaveParam(ctx context.Context, req *dto.SaveCarrierPollingParamRequest) (*dto.CarrierPollingParamItem, error) {
|
||
operatorID, err := requireSuperAdminWriter(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
carrier, err := s.loadCarrier(ctx, req.CarrierID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
row, configured, err := s.loadParam(ctx, req.CarrierID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
before := paramState(row, configured)
|
||
row.CarrierID = req.CarrierID
|
||
row.InflightLimit, row.CardMaxInFlight, row.StopResumeMaxInFlight = req.InflightLimit, req.CardMaxInFlight, req.StopResumeMaxInFlight
|
||
row.SRInflightLimit, row.PriorityMaxRatio = req.SRInflightLimit, req.PriorityMaxRatio
|
||
row.StopResumeMinInterval, row.StopResumeConfirmDelay = req.StopResumeMinInterval, req.StopResumeConfirmDelay
|
||
merged := carrierqueue.MergeParams(row, true, s.cfg)
|
||
if capacity := merged.CardMaxInFlight * s.processes(); merged.InflightLimit > capacity {
|
||
return nil, errors.New(errors.CodeInvalidParam, "执行上限 "+strconv.Itoa(merged.InflightLimit)+" 大于当前订阅进程 max_in_flight 之和 "+
|
||
strconv.Itoa(capacity)+"("+strconv.Itoa(s.processes())+" 个进程 × "+strconv.Itoa(merged.CardMaxInFlight)+"),请调低执行上限或调高 max_in_flight")
|
||
}
|
||
if !configured {
|
||
row.Creator = operatorID
|
||
}
|
||
row.Updater = operatorID
|
||
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||
if err := tx.WithContext(ctx).Save(&row).Error; err != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "保存运营商账户调度参数失败")
|
||
}
|
||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||
ActionCode: constants.AuditActionCarrierPollingParamSaved, Summary: "保存运营商账户调度参数",
|
||
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: req.CarrierID,
|
||
ResourceKey: "carrier_polling_param:" + uintString(req.CarrierID), DisplayName: carrier.CarrierName, OperatorID: operatorID,
|
||
IdentitySnapshot: map[string]any{"carrier_id": req.CarrierID, "inflight_limit": row.InflightLimit,
|
||
"card_max_in_flight": row.CardMaxInFlight, "priority_max_ratio": row.PriorityMaxRatio},
|
||
BeforeData: before, AfterData: paramState(row, true),
|
||
})
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
item := s.paramItem(req.CarrierID, carrier.CarrierName, row, true)
|
||
return &item, nil
|
||
}
|
||
|
||
// DeleteParam 删除运营商账户调度参数,该账户恢复使用全局默认。
|
||
func (s *CarrierSchedulerService) DeleteParam(ctx context.Context, carrierID uint) error {
|
||
operatorID, err := requireSuperAdminWriter(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
row, configured, err := s.loadParam(ctx, carrierID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !configured {
|
||
return errors.New(errors.CodeNotFound, "该运营商账户未单独配置调度参数")
|
||
}
|
||
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||
if err := tx.WithContext(ctx).Delete(&model.CarrierPollingParam{}, row.ID).Error; err != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "删除运营商账户调度参数失败")
|
||
}
|
||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||
ActionCode: constants.AuditActionCarrierPollingParamDeleted, Summary: "删除运营商账户调度参数(恢复全局默认)",
|
||
ResourceType: constants.AuditResourceCarrierPollingParam, ResourceID: carrierID,
|
||
ResourceKey: "carrier_polling_param:" + uintString(carrierID), DisplayName: uintString(carrierID), OperatorID: operatorID,
|
||
IdentitySnapshot: map[string]any{"carrier_id": carrierID}, BeforeData: paramState(row, true),
|
||
})
|
||
})
|
||
}
|
||
|
||
func paramState(row model.CarrierPollingParam, configured bool) map[string]any {
|
||
if !configured {
|
||
return map[string]any{"configured": false}
|
||
}
|
||
return map[string]any{"configured": true, "inflight_limit": row.InflightLimit, "card_max_in_flight": row.CardMaxInFlight,
|
||
"stop_resume_max_in_flight": row.StopResumeMaxInFlight, "sr_inflight_limit": row.SRInflightLimit,
|
||
"priority_max_ratio": row.PriorityMaxRatio, "stop_resume_min_interval": row.StopResumeMinInterval,
|
||
"stop_resume_confirm_delay": row.StopResumeConfirmDelay}
|
||
}
|
||
|
||
func (s *CarrierSchedulerService) loadParam(ctx context.Context, carrierID uint) (model.CarrierPollingParam, bool, error) {
|
||
var row model.CarrierPollingParam
|
||
err := s.db.WithContext(ctx).Where("carrier_id = ?", carrierID).First(&row).Error
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return model.CarrierPollingParam{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return row, false, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||
}
|
||
return row, true, nil
|
||
}
|
||
|
||
func (s *CarrierSchedulerService) loadCarrier(ctx context.Context, carrierID uint) (*model.Carrier, error) {
|
||
var carrier model.Carrier
|
||
if err := s.db.WithContext(ctx).Select("id", "carrier_name").First(&carrier, carrierID).Error; err != nil {
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return nil, errors.New(errors.CodeNotFound, "运营商账户不存在")
|
||
}
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
|
||
}
|
||
return &carrier, nil
|
||
}
|
||
|
||
func (s *CarrierSchedulerService) carrierNameMap(ctx context.Context) (map[uint]string, error) {
|
||
var carriers []model.Carrier
|
||
if err := s.db.WithContext(ctx).Select("id", "carrier_name").Find(&carriers).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户失败")
|
||
}
|
||
names := make(map[uint]string, len(carriers))
|
||
for _, carrier := range carriers {
|
||
names[carrier.ID] = carrier.CarrierName
|
||
}
|
||
return names, nil
|
||
}
|
||
|
||
// ListScenes 列出请求检查场景。
|
||
func (s *CarrierSchedulerService) ListScenes(ctx context.Context) ([]dto.PollingCheckSceneItem, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
var rows []model.PollingCheckScene
|
||
if err := s.db.WithContext(ctx).Order("id").Find(&rows).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
|
||
}
|
||
items := make([]dto.PollingCheckSceneItem, 0, len(rows))
|
||
for _, row := range rows {
|
||
items = append(items, sceneItem(row))
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
func sceneItem(row model.PollingCheckScene) dto.PollingCheckSceneItem {
|
||
item := dto.PollingCheckSceneItem{Scene: row.Scene, SceneName: row.SceneName, Cooldown: row.Cooldown, UpdatedAt: row.UpdatedAt,
|
||
CheckTypes: []string{}, ConfirmDelays: []int{}}
|
||
_ = json.Unmarshal(row.CheckTypes, &item.CheckTypes)
|
||
_ = json.Unmarshal(row.ConfirmDelays, &item.ConfirmDelays)
|
||
return item
|
||
}
|
||
|
||
// UpdateScene 修改请求检查场景的检查类型、确认延迟与冷却;场景由代码预置,不可新增或删除。
|
||
func (s *CarrierSchedulerService) UpdateScene(ctx context.Context, req *dto.UpdatePollingCheckSceneRequest) (*dto.PollingCheckSceneItem, error) {
|
||
operatorID, err := requireSuperAdminWriter(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for index := 1; index < len(req.ConfirmDelays); index++ {
|
||
if req.ConfirmDelays[index] <= req.ConfirmDelays[index-1] {
|
||
return nil, errors.New(errors.CodeInvalidParam, "延迟确认序列必须递增")
|
||
}
|
||
}
|
||
types := uniqueStrings(req.CheckTypes)
|
||
var row model.PollingCheckScene
|
||
if err := s.db.WithContext(ctx).Where("scene = ?", req.Scene).First(&row).Error; err != nil {
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return nil, errors.New(errors.CodeNotFound, "请求检查场景不存在")
|
||
}
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询请求检查场景失败")
|
||
}
|
||
before := sceneItem(row)
|
||
typesJSON, _ := json.Marshal(types)
|
||
delays := req.ConfirmDelays
|
||
if delays == nil {
|
||
delays = []int{}
|
||
}
|
||
delaysJSON, _ := json.Marshal(delays)
|
||
row.CheckTypes, row.ConfirmDelays, row.Cooldown, row.Updater = datatypes.JSON(typesJSON), datatypes.JSON(delaysJSON), req.Cooldown, operatorID
|
||
err = runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||
if err := tx.WithContext(ctx).Model(&model.PollingCheckScene{}).Where("id = ?", row.ID).Updates(map[string]any{
|
||
"check_types": row.CheckTypes, "confirm_delays": row.ConfirmDelays, "cooldown": row.Cooldown, "updater": operatorID, "updated_at": time.Now(),
|
||
}).Error; err != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "修改请求检查场景失败")
|
||
}
|
||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||
ActionCode: constants.AuditActionPollingCheckSceneUpdated, Summary: "修改请求检查场景",
|
||
ResourceType: constants.AuditResourcePollingCheckScene, ResourceID: row.ID,
|
||
ResourceKey: "polling_check_scene:" + row.Scene, DisplayName: row.SceneName, OperatorID: operatorID,
|
||
IdentitySnapshot: map[string]any{"scene": row.Scene, "scene_name": row.SceneName},
|
||
BeforeData: map[string]any{"check_types": before.CheckTypes, "confirm_delays": before.ConfirmDelays, "cooldown": before.Cooldown},
|
||
AfterData: map[string]any{"check_types": types, "confirm_delays": delays, "cooldown": req.Cooldown},
|
||
})
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
item := sceneItem(row)
|
||
return &item, nil
|
||
}
|
||
|
||
// ListAlerts 分页查询卡轮询失败告警。
|
||
func (s *CarrierSchedulerService) ListAlerts(ctx context.Context, req *dto.PollingFailureAlertListRequest) (*dto.PollingFailureAlertPageResult, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
page, size := normalizePage(req.Page, req.PageSize)
|
||
query := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{})
|
||
if req.CarrierID != nil {
|
||
query = query.Where("carrier_id = ?", *req.CarrierID)
|
||
}
|
||
if req.CheckType != "" {
|
||
query = query.Where("check_type = ?", req.CheckType)
|
||
}
|
||
if req.ErrorCategory != "" {
|
||
query = query.Where("error_category = ?", req.ErrorCategory)
|
||
}
|
||
if req.Status != nil {
|
||
query = query.Where("status = ?", *req.Status)
|
||
}
|
||
if req.AlertLevel != "" {
|
||
query = query.Where("alert_level = ?", req.AlertLevel)
|
||
}
|
||
if req.ICCID != "" {
|
||
query = query.Where("iccid = ?", req.ICCID)
|
||
}
|
||
start, end, err := utils.ParseTimeRange(req.StartTime, req.EndTime)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if start != nil {
|
||
query = query.Where("last_failed_at >= ?", *start)
|
||
}
|
||
if end != nil {
|
||
query = query.Where("last_failed_at <= ?", *end)
|
||
}
|
||
var total int64
|
||
if err := query.Count(&total).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
|
||
}
|
||
var rows []model.PollingFailureAlert
|
||
if err := query.Order("last_failed_at DESC NULLS LAST, id DESC").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
|
||
}
|
||
names, err := s.carrierNameMap(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result := &dto.PollingFailureAlertPageResult{Total: total, Page: page, PageSize: size, List: make([]dto.PollingFailureAlertItem, 0, len(rows))}
|
||
for _, row := range rows {
|
||
result.List = append(result.List, alertItem(row, names))
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func alertItem(row model.PollingFailureAlert, names map[uint]string) dto.PollingFailureAlertItem {
|
||
return dto.PollingFailureAlertItem{
|
||
ID: row.ID, AlertLevel: row.AlertLevel, CardID: row.CardID, ICCID: row.ICCID, CarrierID: row.CarrierID,
|
||
CarrierName: names[row.CarrierID], CheckType: row.CheckType, CheckTypeName: constants.CheckTypeName(row.CheckType),
|
||
ErrorCategory: row.ErrorCategory, FailCount: row.FailCount, FirstFailedAt: row.FirstFailedAt, LastFailedAt: row.LastFailedAt,
|
||
LastError: row.LastError, Status: row.Status, StatusName: constants.PollingAlertStatusName(row.Status),
|
||
HandledBy: row.HandledBy, HandledAt: row.HandledAt, HandleRemark: row.HandleRemark, RecoveredAt: row.RecoveredAt, CreatedAt: row.CreatedAt,
|
||
}
|
||
}
|
||
|
||
// AlertSummary 未处理卡级告警按运营商账户 × 错误类别汇总。
|
||
func (s *CarrierSchedulerService) AlertSummary(ctx context.Context) ([]dto.PollingFailureAlertSummaryItem, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
var rows []struct {
|
||
CarrierID uint
|
||
ErrorCategory string
|
||
OpenCount int64
|
||
}
|
||
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).
|
||
Select("carrier_id, error_category, COUNT(*) AS open_count").
|
||
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
|
||
Group("carrier_id, error_category").Order("open_count DESC").Scan(&rows).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "汇总卡轮询失败告警失败")
|
||
}
|
||
names, err := s.carrierNameMap(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
items := make([]dto.PollingFailureAlertSummaryItem, 0, len(rows))
|
||
for _, row := range rows {
|
||
items = append(items, dto.PollingFailureAlertSummaryItem{CarrierID: row.CarrierID, CarrierName: names[row.CarrierID],
|
||
ErrorCategory: row.ErrorCategory, OpenCount: row.OpenCount})
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
// GetAlert 查询告警详情。
|
||
func (s *CarrierSchedulerService) GetAlert(ctx context.Context, id uint) (*dto.PollingFailureAlertItem, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
row, err := s.loadAlert(ctx, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
names, err := s.carrierNameMap(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
item := alertItem(*row, names)
|
||
return &item, nil
|
||
}
|
||
|
||
// HandleAlert 把未处理告警标记为已处理。
|
||
func (s *CarrierSchedulerService) HandleAlert(ctx context.Context, req *dto.HandlePollingFailureAlertRequest) error {
|
||
operatorID, err := requireSuperAdminWriterOrPlatform(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
row, err := s.loadAlert(ctx, req.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if row.Status != constants.PollingAlertStatusOpen {
|
||
return errors.New(errors.CodeInvalidStatus, "只有未处理的告警可以标记已处理")
|
||
}
|
||
now := time.Now()
|
||
return runPollingTransaction(ctx, s.db, s.auditWriter, func(tx *gorm.DB) error {
|
||
result := tx.WithContext(ctx).Model(&model.PollingFailureAlert{}).Where("id = ? AND status = ?", row.ID, constants.PollingAlertStatusOpen).
|
||
Updates(map[string]any{"status": req.Status, "handled_by": operatorID, "handled_at": now, "handle_remark": req.Remark, "updated_at": now})
|
||
if result.Error != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, result.Error, "处理卡轮询失败告警失败")
|
||
}
|
||
if result.RowsAffected == 0 {
|
||
return errors.New(errors.CodeConflict, "告警状态已变化,请刷新后重试")
|
||
}
|
||
return writePollingAudit(ctx, tx, s.auditWriter, auditinfra.PollingInput{
|
||
ActionCode: constants.AuditActionPollingFailureAlertHandled, Summary: "标记卡轮询失败告警已处理",
|
||
ResourceType: constants.AuditResourcePollingFailureAlert, ResourceID: row.ID,
|
||
ResourceKey: "polling_failure_alert:" + uintString(row.ID), DisplayName: row.ICCID, OperatorID: operatorID,
|
||
IdentitySnapshot: map[string]any{"id": row.ID, "alert_level": row.AlertLevel, "card_id": row.CardID,
|
||
"carrier_id": row.CarrierID, "check_type": row.CheckType, "status": row.Status},
|
||
BeforeData: map[string]any{"status": row.Status}, AfterData: map[string]any{"status": req.Status, "handle_remark": req.Remark},
|
||
})
|
||
})
|
||
}
|
||
|
||
func (s *CarrierSchedulerService) loadAlert(ctx context.Context, id uint) (*model.PollingFailureAlert, error) {
|
||
var row model.PollingFailureAlert
|
||
if err := s.db.WithContext(ctx).First(&row, id).Error; err != nil {
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return nil, errors.New(errors.CodeNotFound, "卡轮询失败告警不存在")
|
||
}
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询卡轮询失败告警失败")
|
||
}
|
||
return &row, nil
|
||
}
|
||
|
||
// CardState 查询单卡轮询状态(只读)。
|
||
func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*dto.CardPollingStateResponse, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
var card model.IotCard
|
||
if err := s.db.WithContext(ctx).Select("id", "iccid", "carrier_id", "enable_polling").First(&card, cardID).Error; err != nil {
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return nil, errors.New(errors.CodeNotFound, "IoT卡不存在")
|
||
}
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询IoT卡失败")
|
||
}
|
||
resp := &dto.CardPollingStateResponse{CardID: card.ID, ICCID: card.ICCID, CarrierID: card.CarrierID,
|
||
SchedulerMode: s.settings.Current(ctx).Mode, EnablePolling: card.EnablePolling, Queue: "none", Types: []dto.CardPollingTypeState{}}
|
||
due, err := s.store.ReadDue(ctx, card.ID)
|
||
if err != nil {
|
||
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡轮询到期信息失败")
|
||
}
|
||
carrierID := due.CarrierID
|
||
if carrierID == 0 {
|
||
carrierID = card.CarrierID
|
||
}
|
||
member := strconv.FormatUint(uint64(card.ID), 10)
|
||
if score, err := s.redis.ZScore(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), member).Result(); err == nil {
|
||
resp.Queue, resp.QueueScoreAt = "priority", msTime(int64(score))
|
||
} else if score, err := s.redis.ZScore(ctx, constants.RedisCarrierQueueKey(carrierID), member).Result(); err == nil {
|
||
resp.Queue, resp.QueueScoreAt = "queue", msTime(int64(score))
|
||
}
|
||
if ttl, err := s.redis.PTTL(ctx, constants.RedisCardClaimKey(card.ID)).Result(); err == nil && ttl > 0 {
|
||
resp.Running = true
|
||
expire := time.Now().Add(ttl)
|
||
resp.ClaimExpireAt = &expire
|
||
}
|
||
resp.PriorityMark = due.Priority
|
||
for _, checkType := range constants.AllCheckTypes() {
|
||
item, ok := due.Types[checkType]
|
||
if !ok || (!item.HasDue && item.FailCount == 0 && item.LastOK.IsZero()) {
|
||
continue
|
||
}
|
||
state := dto.CardPollingTypeState{CheckType: checkType, CheckTypeName: constants.CheckTypeName(checkType),
|
||
FailCount: item.FailCount, LastError: item.LastError, PendingConfirm: []int{}}
|
||
if item.HasDue {
|
||
state.NextDueAt = timePtr(item.Due)
|
||
}
|
||
if !item.LastOK.IsZero() {
|
||
state.LastSuccessAt = timePtr(item.LastOK)
|
||
}
|
||
if !item.FirstFailed.IsZero() {
|
||
state.FirstFailedAt = timePtr(item.FirstFailed)
|
||
}
|
||
for _, delay := range item.Confirm {
|
||
state.PendingConfirm = append(state.PendingConfirm, int(delay/time.Second))
|
||
}
|
||
resp.Types = append(resp.Types, state)
|
||
}
|
||
resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled
|
||
command, err := s.store.ReadCommand(ctx, card.ID)
|
||
if err != nil {
|
||
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败")
|
||
}
|
||
if !command.LastCallAt.IsZero() {
|
||
resp.LastStopResumeCallAt = timePtr(command.LastCallAt)
|
||
}
|
||
if command.Exists {
|
||
item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source,
|
||
Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown}
|
||
if item.Scenes == nil {
|
||
item.Scenes = []string{}
|
||
}
|
||
if !command.WrittenAt.IsZero() {
|
||
item.WrittenAt = timePtr(command.WrittenAt)
|
||
}
|
||
if !command.NextAt.IsZero() {
|
||
item.NextAt = timePtr(command.NextAt)
|
||
}
|
||
resp.StopResumeCommand = item
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
// CarrierStats 按运营商账户的轮询监控。
|
||
func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.CarrierPollingStatsResponse, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
names, err := s.carrierNameMap(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var params []model.CarrierPollingParam
|
||
if err := s.db.WithContext(ctx).Find(¶ms).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询运营商账户调度参数失败")
|
||
}
|
||
paramByCarrier := map[uint]model.CarrierPollingParam{}
|
||
for _, row := range params {
|
||
paramByCarrier[row.CarrierID] = row
|
||
}
|
||
var alertRows []struct {
|
||
CarrierID uint
|
||
ErrorCategory string
|
||
Count int64
|
||
}
|
||
if err := s.db.WithContext(ctx).Model(&model.PollingFailureAlert{}).Select("carrier_id, error_category, COUNT(*) AS count").
|
||
Where("status = ? AND alert_level = ?", constants.PollingAlertStatusOpen, constants.PollingAlertLevelCard).
|
||
Group("carrier_id, error_category").Scan(&alertRows).Error; err != nil {
|
||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计卡轮询失败告警失败")
|
||
}
|
||
state, _ := s.redis.Get(ctx, constants.RedisCarrierSchedulerInitStateKey).Result()
|
||
resp := &dto.CarrierPollingStatsResponse{SchedulerMode: s.settings.Current(ctx).Mode, InitState: state, Items: []dto.CarrierPollingStatsItem{}}
|
||
now := time.Now()
|
||
nowMS := strconv.FormatInt(now.UnixMilli(), 10)
|
||
lastMinute := now.Add(-time.Minute).In(time.FixedZone("CST", 8*3600)).Format("200601021504")
|
||
for _, carrierID := range sortedCarrierIDs(names) {
|
||
row, configured := paramByCarrier[carrierID]
|
||
item := dto.CarrierPollingStatsItem{CarrierID: carrierID, CarrierName: names[carrierID],
|
||
InflightLimit: carrierqueue.MergeParams(row, configured, s.cfg).InflightLimit, OpenAlertsByType: map[string]int64{}}
|
||
pipe := s.redis.Pipeline()
|
||
priorityLen := pipe.ZCard(ctx, constants.RedisCarrierPriorityQueueKey(carrierID))
|
||
queueLen := pipe.ZCard(ctx, constants.RedisCarrierQueueKey(carrierID))
|
||
priorityDue := pipe.ZCount(ctx, constants.RedisCarrierPriorityQueueKey(carrierID), "-inf", nowMS)
|
||
queueDue := pipe.ZCount(ctx, constants.RedisCarrierQueueKey(carrierID), "-inf", nowMS)
|
||
running := pipe.ZCount(ctx, constants.RedisCarrierRunningKey(carrierID), nowMS, "+inf")
|
||
expired := pipe.HGetAll(ctx, constants.RedisCarrierExpiredClaimsKey(carrierID))
|
||
var statCmds []*redis.MapStringStringCmd
|
||
for _, op := range []string{constants.GatewayOpQueryStatus, constants.GatewayOpQueryFlow, constants.GatewayOpQueryRealname} {
|
||
statCmds = append(statCmds, pipe.HGetAll(ctx, constants.RedisGatewayCallStatKey(carrierID, op, lastMinute)))
|
||
}
|
||
_, _ = pipe.Exec(ctx)
|
||
item.PriorityLength, item.QueueLength = priorityLen.Val(), queueLen.Val()
|
||
item.DueCount, item.Running = priorityDue.Val()+queueDue.Val(), running.Val()
|
||
cutoff := now.Add(-time.Hour).Unix() / 60
|
||
for minute, value := range expired.Val() {
|
||
if m, err := strconv.ParseInt(minute, 10, 64); err == nil && m >= cutoff {
|
||
count, _ := strconv.ParseInt(value, 10, 64)
|
||
item.ExpiredClaims1h += count
|
||
}
|
||
}
|
||
var attempts int64
|
||
for _, cmd := range statCmds {
|
||
values := cmd.Val()
|
||
attempts += atoi64(values["attempts"])
|
||
item.RateLimitWaits += atoi64(values["rate_wait"])
|
||
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
|
||
}
|
||
item.QPSLastMinute = float64(attempts) / 60
|
||
if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil {
|
||
item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due
|
||
item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending
|
||
}
|
||
srCounts := s.store.StopResumeStatLastHour(ctx, carrierID)
|
||
item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed]
|
||
item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown]
|
||
for _, alert := range alertRows {
|
||
if alert.CarrierID == carrierID {
|
||
item.OpenAlerts += alert.Count
|
||
item.OpenAlertsByType[alert.ErrorCategory] += alert.Count
|
||
}
|
||
}
|
||
resp.Items = append(resp.Items, item)
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
// requireSuperAdminWriterOrPlatform 校验告警处理权限:超级管理员与平台账号均可,返回操作者 ID。
|
||
func requireSuperAdminWriterOrPlatform(ctx context.Context) (uint, error) {
|
||
if err := requirePlatformReader(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
operatorID := middleware.GetUserIDFromContext(ctx)
|
||
if operatorID == 0 {
|
||
return 0, errors.New(errors.CodeUnauthorized, "未授权访问")
|
||
}
|
||
return operatorID, nil
|
||
}
|
||
|
||
func sortedCarrierIDs(names map[uint]string) []uint {
|
||
ids := make([]uint, 0, len(names))
|
||
for id := range names {
|
||
ids = append(ids, id)
|
||
}
|
||
for i := 1; i < len(ids); i++ {
|
||
for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
|
||
ids[j], ids[j-1] = ids[j-1], ids[j]
|
||
}
|
||
}
|
||
return ids
|
||
}
|
||
|
||
func msTime(ms int64) *time.Time {
|
||
t := time.UnixMilli(ms)
|
||
return &t
|
||
}
|
||
|
||
func timePtr(t time.Time) *time.Time {
|
||
return &t
|
||
}
|
||
|
||
func atoi64(s string) int64 {
|
||
value, _ := strconv.ParseInt(s, 10, 64)
|
||
return value
|
||
}
|