feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭): - 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机 - 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制 - 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认 - 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后 - 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估 - 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令 - 迁移 000255 插入开关与告警、退避参数 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ package carrier
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -26,6 +27,22 @@ type Service struct {
|
||||
audit systemconfigapp.AuditWriter
|
||||
// thresholdHits 是通道阈值命中记录只读查询;与运营商配置共用同一数据库连接。
|
||||
thresholdHits *carrierthresholdquery.Query
|
||||
// realnameEvaluator 实名管控修改后对受影响的卡批量请求停复机评估;未注入时不触发。
|
||||
realnameEvaluator RealnameControlEvaluator
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// RealnameControlEvaluator 是实名管控修改后批量请求停复机评估的能力(停复机出口开关关闭时不执行)。
|
||||
type RealnameControlEvaluator interface {
|
||||
RequestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) (int, error)
|
||||
}
|
||||
|
||||
// SetRealnameControlEvaluator 注入实名管控修改后的批量评估能力。
|
||||
func (s *Service) SetRealnameControlEvaluator(evaluator RealnameControlEvaluator, logger *zap.Logger) {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
s.realnameEvaluator, s.logger = evaluator, logger
|
||||
}
|
||||
|
||||
func New(carrierStore *postgres.CarrierStore, audit systemconfigapp.AuditWriter) *Service {
|
||||
@@ -210,10 +227,24 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateCarrierReq
|
||||
s.recordFailure(ctx, constants.AuditOperationCarrierUpdate, "更新运营商配置失败", carrier)
|
||||
return nil, errors.Wrap(errors.CodeInternalError, err, "更新运营商失败")
|
||||
}
|
||||
if before.RealnameControlEnabled != carrier.RealnameControlEnabled && s.realnameEvaluator != nil {
|
||||
s.requestRealnameControlEvaluation(ctx, carrier.ID, carrier.RealnameControlEnabled == constants.StatusEnabled)
|
||||
}
|
||||
|
||||
return s.toResponse(ctx, carrier), nil
|
||||
}
|
||||
|
||||
// requestRealnameControlEvaluation 在事务提交后异步批量请求评估(只读库与写共享缓存,不阻塞保存接口)。
|
||||
func (s *Service) requestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) {
|
||||
evalCtx := context.WithoutCancel(ctx)
|
||||
go func() {
|
||||
defer func() { _ = recover() }()
|
||||
if _, err := s.realnameEvaluator.RequestRealnameControlEvaluation(evalCtx, carrierID, enabled); err != nil {
|
||||
s.logger.Error("实名管控修改后批量请求停复机评估失败", zap.Uint("carrier_id", carrierID), zap.Error(err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Service) Delete(ctx context.Context, id uint) error {
|
||||
carrier, err := s.carrierStore.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
|
||||
@@ -2006,7 +2006,7 @@ func (s *Service) handleManualRealnameStatusChanged(ctx context.Context, oldStat
|
||||
}
|
||||
|
||||
if s.stopResumeService != nil {
|
||||
if err := s.stopResumeService.EvaluateAndAct(ctx, card); err != nil {
|
||||
if err := s.stopResumeService.EvaluateAndAct(WithStopResumeScene(ctx, constants.StopResumeSceneRealnameObserved), card); err != nil {
|
||||
s.logger.Warn("手动实名后触发停复机评估失败",
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.Error(err))
|
||||
|
||||
190
internal/service/iot_card/stop_resume_command.go
Normal file
190
internal/service/iot_card/stop_resume_command.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// 停复机指令执行器使用的服务能力(add-stop-resume-command-queue)。执行器只调用一次网关,不 sleep 重试。
|
||||
|
||||
// CommandDecision 是执行前按指令场景重新判断的结果。
|
||||
type CommandDecision struct {
|
||||
Card *model.IotCard
|
||||
Action string // 空表示条件不再满足或已是目标状态,结束指令
|
||||
Reason string
|
||||
Gone bool // 卡已不存在
|
||||
}
|
||||
|
||||
// DecideCommand 执行前按指令记录的场景重新判断该卡(只看本卡,不扩散到设备)。
|
||||
// 复机持通道阈值锁时视为不再满足。
|
||||
func (s *StopResumeService) DecideCommand(ctx context.Context, cardID uint, scenes []string) (CommandDecision, error) {
|
||||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||||
if err != nil {
|
||||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return CommandDecision{Gone: true}, nil
|
||||
}
|
||||
return CommandDecision{}, errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
|
||||
}
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return CommandDecision{Card: card}, err
|
||||
}
|
||||
if decision.Action == constants.StopResumeActionStart {
|
||||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||||
if blockErr != nil {
|
||||
return CommandDecision{Card: card}, blockErr
|
||||
}
|
||||
if blocked {
|
||||
return CommandDecision{Card: card}, nil
|
||||
}
|
||||
}
|
||||
return CommandDecision{Card: card, Action: decision.Action, Reason: decision.Reason}, nil
|
||||
}
|
||||
|
||||
// CommandCallResult 是一次停复机网关调用的结果。
|
||||
type CommandCallResult struct {
|
||||
// Unknown 为 true 表示结果未知:超时、连接失败,或上游已成功但本地回写失败。
|
||||
Unknown bool
|
||||
}
|
||||
|
||||
// ExecuteCommand 对卡调用一次停机或复机(限流等待模式由调用方 ctx 声明),成功后按现有回写更新卡状态、
|
||||
// 停机原因与网络状态序列,并写停复机记录。返回的 error 为网关错误或回写错误,由执行器按错误分类处理。
|
||||
func (s *StopResumeService) ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (CommandCallResult, error) {
|
||||
if card == nil {
|
||||
return CommandCallResult{}, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
if action == constants.StopResumeActionStop {
|
||||
outcome, err := s.stopCardAttempts(ctx, card, reason, 1)
|
||||
return CommandCallResult{Unknown: err != nil && outcome.Result == constants.AuditResultUnknown && !gateway.IsRateLimited(err)}, err
|
||||
}
|
||||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
|
||||
attempt, integrationID, err := s.resumeCardAttempts(ctx, card, actionCode, summary, 1)
|
||||
if err != nil {
|
||||
return CommandCallResult{Unknown: isUnknownGatewayError(err)}, err
|
||||
}
|
||||
if err := s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary); err != nil {
|
||||
return CommandCallResult{Unknown: true}, err
|
||||
}
|
||||
return CommandCallResult{}, nil
|
||||
}
|
||||
|
||||
// isUnknownGatewayError 判断网关错误是否为结果未知(超时、连接失败):请求可能已到达上游。
|
||||
func isUnknownGatewayError(err error) bool {
|
||||
gwErr, ok := gateway.AsGatewayError(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return gwErr.Category == constants.GatewayErrorTimeout || gwErr.Category == constants.GatewayErrorNetwork
|
||||
}
|
||||
|
||||
// QueryCommandCardStatus 结果未知后的确认查询:只查一次卡状态(受限流、不受单卡最小重试间隔约束)。
|
||||
// 查询失败时 known=false 并返回错误,调用方保持待确认、不发起停复机。
|
||||
func (s *StopResumeService) QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error) {
|
||||
if s.gatewayClient == nil {
|
||||
return 0, false, errors.New(errors.CodeInternalError, "Gateway 未配置,无法查询卡状态")
|
||||
}
|
||||
queryCtx := gateway.WithCarrier(ctx, card.CarrierID)
|
||||
response, err := s.gatewayClient.QueryCardStatus(queryCtx, &gateway.CardStatusReq{CardNo: card.ICCID})
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
status, known := carddomain.MapGatewayNetworkStatus(response.CardStatus, response.Extend)
|
||||
return status, known, nil
|
||||
}
|
||||
|
||||
// ConfirmCommandState 按已确认的上游结果(确认查询已是目标状态、上游返回已是目标状态、停机遇上游终态)回写卡状态。
|
||||
func (s *StopResumeService) ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error {
|
||||
if card == nil {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络(确认)"
|
||||
scene, expected := constants.CardObservationSceneBusinessResume, "online"
|
||||
fields := map[string]any{"network_status": constants.NetworkStatusOnline, "resumed_at": time.Now(), "stop_reason": ""}
|
||||
if action == constants.StopResumeActionStop {
|
||||
actionCode, summary = stopAuditAction(ctx, reason)
|
||||
summary += "(确认)"
|
||||
scene, expected = constants.CardObservationSceneBusinessStop, "offline"
|
||||
fields = map[string]any{"network_status": constants.NetworkStatusOffline, "stopped_at": time.Now(), "stop_reason": reason}
|
||||
}
|
||||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, fields, scene, expected, uuid.NewString(), actionCode, summary, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
s.reschedulePolling(ctx, card.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EvaluateDeviceCommands 设备维度判断:读取设备下各卡最新数据,按统一判断函数(各卡按所属运营商账户判断实名,
|
||||
// 暂停轮询按场景集合判断)为需要动作的卡生成指令,进入各卡所属账户的停复机队列。只读数据库与写共享缓存,不调网关。
|
||||
func (s *StopResumeService) EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error {
|
||||
if s.commandQueue == nil {
|
||||
return errors.New(errors.CodeInternalError, "停复机指令队列未配置")
|
||||
}
|
||||
cards, err := s.deviceCards(ctx, deviceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var failed []error
|
||||
for _, card := range cards {
|
||||
decision, decideErr := s.decideCard(ctx, card, scenes)
|
||||
if decideErr != nil {
|
||||
failed = append(failed, decideErr)
|
||||
continue
|
||||
}
|
||||
if decision.Action == "" {
|
||||
s.markNotRealnameIfBlocked(ctx, card, scenes)
|
||||
continue
|
||||
}
|
||||
decision.DeviceID = 0
|
||||
if writeErr := s.writeCardCommand(ctx, card, decision, scenes, "device_eval"); writeErr != nil {
|
||||
failed = append(failed, writeErr)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return errors.Wrap(errors.CodeInternalError, stderrors.Join(failed...), "设备判断存在未写入指令的卡")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// markNotRealnameIfBlocked 设备下停机卡套餐满足但实名不满足时,把停机原因更新为未实名(与旧设备复机路径一致)。
|
||||
func (s *StopResumeService) markNotRealnameIfBlocked(ctx context.Context, card *model.IotCard, scenes []string) {
|
||||
if card.NetworkStatus != constants.NetworkStatusOffline || card.StopReason == constants.StopReasonNotRealname {
|
||||
return
|
||||
}
|
||||
if suspended, err := s.autoControlSuspended(ctx, card); err != nil || suspended {
|
||||
return
|
||||
}
|
||||
hasPackage, err := s.hasValidPackage(ctx, card)
|
||||
if err != nil || !hasPackage {
|
||||
return
|
||||
}
|
||||
if exhausted, err := s.isTrafficExhausted(ctx, card); err != nil || exhausted {
|
||||
return
|
||||
}
|
||||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||||
if err != nil || realnameOK {
|
||||
return
|
||||
}
|
||||
if err := s.updateCardStopReasonWithAudit(ctx, card, constants.StopReasonNotRealname); err != nil {
|
||||
s.logger.Warn("更新未实名卡停机原因失败", zap.Uint("card_id", card.ID), zap.Strings("scenes", scenes), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// LegacyDeviceOpLocked 判断旧路径是否正在对该设备执行停复机(持设备锁);设备判断须推后,不并行执行。
|
||||
func (s *StopResumeService) LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool {
|
||||
if s.redis == nil {
|
||||
return false
|
||||
}
|
||||
exists, err := s.redis.Exists(ctx, constants.RedisPollingDeviceOpLockKey(deviceID)).Result()
|
||||
return err == nil && exists > 0
|
||||
}
|
||||
92
internal/service/iot_card/stop_resume_decide.go
Normal file
92
internal/service/iot_card/stop_resume_decide.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// WithStopResumeScene 声明本次自动停复机评估的触发场景;未声明时按套餐步骤(周期评估)处理。
|
||||
func WithStopResumeScene(ctx context.Context, scene string) context.Context {
|
||||
return stopresumedomain.WithScene(ctx, scene)
|
||||
}
|
||||
|
||||
// stopResumeScenes 返回 ctx 中声明的场景集合(单个场景)。
|
||||
func stopResumeScenes(ctx context.Context) []string {
|
||||
return []string{stopresumedomain.Scene(ctx)}
|
||||
}
|
||||
|
||||
// stopResumeDecision 是统一判断函数的结果。
|
||||
type stopResumeDecision struct {
|
||||
Action string // constants.StopResumeActionStop / StopResumeActionStart;空表示不动作
|
||||
Reason string // 停机原因
|
||||
Reasons []string // 全部命中的停机原因(按优先级)
|
||||
DeviceID uint // 设备维度动作的设备;0 表示单卡
|
||||
}
|
||||
|
||||
// decideCard 统一判断函数:暂停轮询(按场景) → 停机条件 → 复机条件。
|
||||
// 暂停轮询的卡只在套餐失效类场景(套餐到期、退款)下放行无套餐停机,其余自动停复机一律不做。
|
||||
// 设备维度原因(无套餐、流量耗尽)与绑定设备的复机返回 DeviceID,由调用方按设备处理。通道阈值锁由复机执行方判断。
|
||||
func (s *StopResumeService) decideCard(ctx context.Context, card *model.IotCard, scenes []string) (stopResumeDecision, error) {
|
||||
suspended, err := s.autoControlSuspended(ctx, card)
|
||||
if err != nil {
|
||||
return stopResumeDecision{}, err
|
||||
}
|
||||
allowSuspendedStop := constants.StopResumeSceneAllowsSuspendedStop(scenes)
|
||||
if suspended && !allowSuspendedStop {
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
switch card.NetworkStatus {
|
||||
case constants.NetworkStatusOnline:
|
||||
reasons, err := s.checkStopReasons(ctx, card)
|
||||
if err != nil || len(reasons) == 0 {
|
||||
return stopResumeDecision{}, err
|
||||
}
|
||||
primary := reasons[0]
|
||||
if suspended {
|
||||
// 暂停轮询:只有套餐失效(无可用主套餐)才停机。
|
||||
if !containsStopReason(reasons, constants.StopReasonNoPackage) {
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
primary = constants.StopReasonNoPackage
|
||||
}
|
||||
decision := stopResumeDecision{Action: constants.StopResumeActionStop, Reason: primary, Reasons: reasons}
|
||||
if !card.IsStandalone && isDeviceScopeReason(primary) {
|
||||
decision.DeviceID = s.boundDeviceID(ctx, card)
|
||||
}
|
||||
return decision, nil
|
||||
case constants.NetworkStatusOffline:
|
||||
if suspended {
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
// 不看停机原因,是否复机只由本系统规则决定。
|
||||
resume, err := s.shouldResume(ctx, card)
|
||||
if err != nil || !resume {
|
||||
return stopResumeDecision{}, err
|
||||
}
|
||||
decision := stopResumeDecision{Action: constants.StopResumeActionStart}
|
||||
if !card.IsStandalone {
|
||||
decision.DeviceID = s.boundDeviceID(ctx, card)
|
||||
}
|
||||
return decision, nil
|
||||
default:
|
||||
return stopResumeDecision{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// boundDeviceID 返回卡的有效绑定设备;查询失败时降级为单卡处理并记日志。
|
||||
func (s *StopResumeService) boundDeviceID(ctx context.Context, card *model.IotCard) uint {
|
||||
deviceID, bound, err := s.getCardDeviceID(ctx, card)
|
||||
if err != nil {
|
||||
s.logger.Error("查询设备绑定关系失败,降级为单卡处理", zap.Uint("card_id", card.ID), zap.Error(err))
|
||||
return 0
|
||||
}
|
||||
if !bound {
|
||||
return 0
|
||||
}
|
||||
return deviceID
|
||||
}
|
||||
70
internal/service/iot_card/stop_resume_realname_control.go
Normal file
70
internal/service/iot_card/stop_resume_realname_control.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// realnameControlBatchSize 是实名管控修改后分批读取受影响卡的批大小。
|
||||
const realnameControlBatchSize = 500
|
||||
|
||||
// RequestRealnameControlEvaluation 运营商账户实名管控修改后对受影响的卡批量请求评估(只在停复机出口开关开启时执行):
|
||||
// 0 → 1 时为该账户在线且未实名的卡,1 → 0 时为停机且未实名的卡;绑定设备的卡按设备请求设备判断。
|
||||
// 停复机按账户 QPS 由指令队列平滑执行。返回请求评估的卡与设备数。
|
||||
func (s *StopResumeService) RequestRealnameControlEvaluation(ctx context.Context, carrierID uint, enabled bool) (int, error) {
|
||||
if !s.commandQueueEnabled(ctx) || s.db == nil {
|
||||
return 0, nil
|
||||
}
|
||||
ctx = WithStopResumeScene(ctx, constants.StopResumeSceneRealnameControl)
|
||||
scenes := []string{constants.StopResumeSceneRealnameControl}
|
||||
networkStatus := constants.NetworkStatusOffline
|
||||
if enabled {
|
||||
networkStatus = constants.NetworkStatusOnline
|
||||
}
|
||||
requested := 0
|
||||
devices := map[uint]bool{}
|
||||
lastID := uint(0)
|
||||
for {
|
||||
var cards []*model.IotCard
|
||||
if err := s.db.WithContext(ctx).Where("carrier_id = ? AND network_status = ? AND real_name_status <> ? AND id > ?",
|
||||
carrierID, networkStatus, constants.RealNameStatusVerified, lastID).
|
||||
Order("id").Limit(realnameControlBatchSize).Find(&cards).Error; err != nil {
|
||||
return requested, errors.Wrap(errors.CodeDatabaseError, err, "查询实名管控受影响的卡失败")
|
||||
}
|
||||
if len(cards) == 0 {
|
||||
break
|
||||
}
|
||||
for _, card := range cards {
|
||||
lastID = card.ID
|
||||
if deviceID := s.boundDeviceID(ctx, card); deviceID > 0 {
|
||||
if !devices[deviceID] {
|
||||
devices[deviceID] = true
|
||||
if err := s.requestDeviceEval(ctx, deviceID, scenes, "realname_control"); err == nil {
|
||||
requested++
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
s.logger.Warn("实名管控修改后判断卡失败", zap.Uint("card_id", card.ID), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if decision.Action == "" {
|
||||
continue
|
||||
}
|
||||
decision.DeviceID = 0
|
||||
if err := s.writeCardCommand(ctx, card, decision, scenes, "realname_control"); err == nil {
|
||||
requested++
|
||||
}
|
||||
}
|
||||
}
|
||||
s.logger.Info("运营商账户实名管控修改后已批量请求评估", zap.Uint("carrier_id", carrierID), zap.Bool("enabled", enabled),
|
||||
zap.Int("requested", requested), zap.String("metric", "stop_resume.realname_control_requested"))
|
||||
return requested, nil
|
||||
}
|
||||
144
internal/service/iot_card/stop_resume_request.go
Normal file
144
internal/service/iot_card/stop_resume_request.go
Normal file
@@ -0,0 +1,144 @@
|
||||
package iot_card
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/polling/carrierqueue"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// StopResumeCommandQueue 是停复机指令队列的写入接缝(共享缓存)。
|
||||
type StopResumeCommandQueue interface {
|
||||
// Enabled 返回停复机出口开关 polling.stop_resume.queue_enabled。
|
||||
Enabled(ctx context.Context) bool
|
||||
WriteCommand(ctx context.Context, cmd carrierqueue.StopResumeCommand) (int64, error)
|
||||
RequestDeviceEval(ctx context.Context, deviceID uint, scenes []string) error
|
||||
}
|
||||
|
||||
// SetCommandQueue 注入停复机指令队列;未注入时始终当场执行。
|
||||
func (s *StopResumeService) SetCommandQueue(queue StopResumeCommandQueue) {
|
||||
s.commandQueue = queue
|
||||
}
|
||||
|
||||
// CommandQueueEnabled 返回停复机出口开关是否开启(供调用方决定是否还需要异步执行)。
|
||||
func (s *StopResumeService) CommandQueueEnabled(ctx context.Context) bool {
|
||||
return s.commandQueueEnabled(ctx)
|
||||
}
|
||||
|
||||
func (s *StopResumeService) commandQueueEnabled(ctx context.Context) bool {
|
||||
return s != nil && s.commandQueue != nil && s.commandQueue.Enabled(ctx)
|
||||
}
|
||||
|
||||
// RequestAssetEvaluation 请求评估门面:开关开启时单卡生成指令、设备请求设备判断;关闭时走旧路径当场执行。
|
||||
// assetType 为 constants.AssetTypeIotCard / AssetTypeDevice。
|
||||
func (s *StopResumeService) RequestAssetEvaluation(ctx context.Context, assetType string, assetID uint, scene, source string) error {
|
||||
ctx = WithStopResumeScene(ctx, scene)
|
||||
if !s.commandQueueEnabled(ctx) {
|
||||
return s.legacyAssetEvaluation(ctx, assetType, assetID)
|
||||
}
|
||||
if assetType == constants.AssetTypeDevice {
|
||||
return s.requestDeviceEval(ctx, assetID, []string{scene}, source)
|
||||
}
|
||||
card, err := s.iotCardStore.GetByID(ctx, assetID)
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
|
||||
}
|
||||
scenes := []string{scene}
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.submitDecisionWithSource(ctx, card, decision, scenes, source)
|
||||
}
|
||||
|
||||
// legacyAssetEvaluation 开关关闭时的当场执行:单卡按统一判断执行,设备按设备复机(套餐激活、重置、订单)或逐卡评估(其他场景)。
|
||||
func (s *StopResumeService) legacyAssetEvaluation(ctx context.Context, assetType string, assetID uint) error {
|
||||
if assetType == constants.AssetTypeDevice {
|
||||
cards, err := s.deviceCards(ctx, assetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, card := range cards {
|
||||
if err := s.EvaluateAndAct(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
card, err := s.iotCardStore.GetByID(ctx, assetID)
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
|
||||
}
|
||||
return s.EvaluateAndAct(ctx, card)
|
||||
}
|
||||
|
||||
// submitDecision 把统一判断结果写入指令队列(来源取场景)。
|
||||
func (s *StopResumeService) submitDecision(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string) error {
|
||||
return s.submitDecisionWithSource(ctx, card, decision, scenes, "")
|
||||
}
|
||||
|
||||
func (s *StopResumeService) submitDecisionWithSource(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string, source string) error {
|
||||
if decision.Action == "" {
|
||||
return nil
|
||||
}
|
||||
if decision.DeviceID > 0 {
|
||||
return s.requestDeviceEval(ctx, decision.DeviceID, scenes, source)
|
||||
}
|
||||
return s.writeCardCommand(ctx, card, decision, scenes, source)
|
||||
}
|
||||
|
||||
// writeCardCommand 生成单卡停复机指令;复机前检查通道阈值锁,持锁时记录拒绝事实不生成指令。
|
||||
func (s *StopResumeService) writeCardCommand(ctx context.Context, card *model.IotCard, decision stopResumeDecision, scenes []string, source string) error {
|
||||
if decision.Action == constants.StopResumeActionStart {
|
||||
blocked, err := s.channelThresholdBlocked(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if blocked {
|
||||
s.recordChannelThresholdDenial(ctx, card, constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if source == "" {
|
||||
source = constants.JoinStopResumeScenes(scenes)
|
||||
}
|
||||
version, err := s.commandQueue.WriteCommand(ctx, carrierqueue.StopResumeCommand{
|
||||
CardID: card.ID, CarrierID: card.CarrierID, Action: decision.Action, Reason: decision.Reason, Scenes: scenes, Source: source,
|
||||
})
|
||||
if err != nil {
|
||||
// 写入失败只记日志:由该卡下一次套餐步骤兜底。
|
||||
s.logger.Error("写入停复机指令失败", zap.Uint("card_id", card.ID), zap.String("action", decision.Action),
|
||||
zap.Strings("scenes", scenes), zap.String("metric", "stop_resume.request_failed"), zap.Error(err))
|
||||
return errors.Wrap(errors.CodeRedisError, err, "写入停复机指令失败")
|
||||
}
|
||||
s.logger.Info("已生成停复机指令", zap.Uint("card_id", card.ID), zap.Uint("carrier_id", card.CarrierID),
|
||||
zap.String("action", decision.Action), zap.String("reason", decision.Reason), zap.Strings("scenes", scenes), zap.Int64("version", version))
|
||||
return nil
|
||||
}
|
||||
|
||||
// requestDeviceEval 请求一次设备维度判断。
|
||||
func (s *StopResumeService) requestDeviceEval(ctx context.Context, deviceID uint, scenes []string, source string) error {
|
||||
if err := s.commandQueue.RequestDeviceEval(ctx, deviceID, scenes); err != nil {
|
||||
s.logger.Error("请求设备判断失败", zap.Uint("device_id", deviceID), zap.Strings("scenes", scenes), zap.String("source", source),
|
||||
zap.String("metric", "stop_resume.request_failed"), zap.Error(err))
|
||||
return errors.Wrap(errors.CodeRedisError, err, "请求设备判断失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deviceCards 返回设备下有效绑定的全部卡(在线与停机)。
|
||||
func (s *StopResumeService) deviceCards(ctx context.Context, deviceID uint) ([]*model.IotCard, error) {
|
||||
online, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOnline)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询设备在线卡失败")
|
||||
}
|
||||
offline, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOffline)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询设备停机卡失败")
|
||||
}
|
||||
return append(online, offline...), nil
|
||||
}
|
||||
@@ -60,6 +60,8 @@ type StopResumeService struct {
|
||||
stopResumeRecorder *gatewaylimit.StopResumeRecorder
|
||||
// terminalPolling 停复机命中"自动关闭轮询"终态规则时关闭该卡轮询;未注入时按普通失败处理。
|
||||
terminalPolling *gatewaylimit.TerminalPollingDisabler
|
||||
// commandQueue 停复机指令队列;开关开启时自动入口只请求评估。
|
||||
commandQueue StopResumeCommandQueue
|
||||
}
|
||||
|
||||
// SetStopResumeRecorder 注入停复机记录写入器。
|
||||
@@ -128,69 +130,36 @@ func (s *StopResumeService) SetPollingCallback(callback PollingCallback) {
|
||||
s.pollingCallback = callback
|
||||
}
|
||||
|
||||
// EvaluateAndAct 停复机统一入口
|
||||
// 根据卡的当前网络状态,自动判断并执行停机或复机操作
|
||||
// 设备/单卡维度通过 card.IsStandalone 推导(false 则为设备维度,停/复机覆盖设备下所有卡)
|
||||
// EvaluateAndAct 停复机统一入口:按统一判断函数决定停机或复机。
|
||||
// 停复机出口开关开启时只请求评估(生成指令或请求设备判断),不当场调用网关;关闭时当场执行。
|
||||
// 设备/单卡维度通过 card.IsStandalone 推导(false 则为设备维度,停/复机覆盖设备下所有卡)。触发场景见 WithStopResumeScene。
|
||||
func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error {
|
||||
// 总开关开启时,卡或所在设备暂停轮询的不自动停复机。
|
||||
suspended, err := s.autoControlSuspended(ctx, card)
|
||||
scenes := stopResumeScenes(ctx)
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if suspended {
|
||||
return nil
|
||||
// 无有效套餐且存在待生效套餐使用记录:仅由普通套餐轮询来源追加优先轮询请求。
|
||||
if containsStopReason(decision.Reasons, constants.StopReasonNoPackage) {
|
||||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch card.NetworkStatus {
|
||||
case constants.NetworkStatusOnline:
|
||||
// 卡在线,检查是否需要停机
|
||||
reasons, err := s.checkStopReasons(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
if s.commandQueueEnabled(ctx) {
|
||||
return s.submitDecision(ctx, card, decision, scenes)
|
||||
}
|
||||
switch decision.Action {
|
||||
case constants.StopResumeActionStop:
|
||||
if decision.DeviceID > 0 {
|
||||
return s.stopDeviceCards(ctx, decision.DeviceID, decision.Reason)
|
||||
}
|
||||
if len(reasons) == 0 {
|
||||
return nil
|
||||
}
|
||||
// 无有效套餐且存在待生效套餐使用记录:仅由普通套餐轮询来源追加优先轮询请求。
|
||||
if containsStopReason(reasons, constants.StopReasonNoPackage) {
|
||||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// 取最高优先级原因(第一个)
|
||||
primaryReason := reasons[0]
|
||||
if !card.IsStandalone && isDeviceScopeReason(primaryReason) {
|
||||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||||
if lookupErr != nil {
|
||||
s.logger.Error("查询设备绑定关系失败,降级为单卡停机",
|
||||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||||
// 降级:查不到绑定时按单卡处理
|
||||
} else if bound {
|
||||
return s.stopDeviceCards(ctx, deviceID, primaryReason)
|
||||
}
|
||||
}
|
||||
_, stopErr := s.stopCardWithRetry(ctx, card, primaryReason)
|
||||
_, stopErr := s.stopCardWithRetry(ctx, card, decision.Reason)
|
||||
return stopErr
|
||||
|
||||
case constants.NetworkStatusOffline:
|
||||
// 卡停机,检查是否可以复机:不看停机原因,是否复机只由本系统规则决定
|
||||
shouldResume, err := s.shouldResume(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !shouldResume {
|
||||
return nil
|
||||
}
|
||||
if !card.IsStandalone {
|
||||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||||
if lookupErr != nil {
|
||||
s.logger.Error("查询设备绑定关系失败,降级为单卡复机",
|
||||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||||
} else if bound {
|
||||
return s.resumeDeviceCards(ctx, deviceID)
|
||||
}
|
||||
case constants.StopResumeActionStart:
|
||||
if decision.DeviceID > 0 {
|
||||
return s.resumeDeviceCards(ctx, decision.DeviceID)
|
||||
}
|
||||
return s.resumeSingleCard(ctx, card.ID)
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
@@ -414,11 +383,11 @@ func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint,
|
||||
|
||||
var cardErrors []error
|
||||
for _, card := range cards {
|
||||
// 总开关开启时,暂停轮询的卡不随设备自动停机。
|
||||
// 暂停轮询的卡不随设备自动停机;套餐失效类场景(套餐到期、退款)照常停机。
|
||||
if suspended, suspendErr := s.autoControlSuspended(ctx, card); suspendErr != nil {
|
||||
cardErrors = append(cardErrors, suspendErr)
|
||||
continue
|
||||
} else if suspended {
|
||||
} else if suspended && !(stopReason == constants.StopReasonNoPackage && constants.StopResumeSceneAllowsSuspendedStop(stopResumeScenes(ctx))) {
|
||||
continue
|
||||
}
|
||||
if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
|
||||
@@ -503,6 +472,9 @@ func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) e
|
||||
// ResumeCardIfStopped 套餐激活或流量重置后自动复机入口
|
||||
// 支持 iot_card 和 device 两种载体类型
|
||||
func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error {
|
||||
if s.commandQueueEnabled(ctx) {
|
||||
return s.RequestAssetEvaluation(ctx, carrierType, carrierID, stopResumeScenes(ctx)[0], "resume_if_stopped")
|
||||
}
|
||||
switch carrierType {
|
||||
case "iot_card":
|
||||
return s.resumeSingleCard(ctx, carrierID)
|
||||
@@ -582,7 +554,12 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
zap.Error(err))
|
||||
return err
|
||||
}
|
||||
return s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary)
|
||||
}
|
||||
|
||||
// completeAutoResume 自动复机网关成功后回写在线状态、观测序列与统一审计,并重排轮询。
|
||||
// 回写失败时返回错误,调用方按结果未知处理。
|
||||
func (s *StopResumeService) completeAutoResume(ctx context.Context, card *model.IotCard, attempt *gatewayAttempt, integrationID, actionCode, summary string) error {
|
||||
now := time.Now()
|
||||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||||
"network_status": constants.NetworkStatusOnline,
|
||||
@@ -593,7 +570,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||||
}
|
||||
s.logger.Error("复机 Gateway 成功但 DB 更新失败",
|
||||
zap.Uint("card_id", cardID),
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.String("iccid", card.ICCID),
|
||||
zap.Error(err))
|
||||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||||
@@ -608,7 +585,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
}
|
||||
|
||||
s.logger.Info("卡已自动复机",
|
||||
zap.Uint("card_id", cardID),
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.String("iccid", card.ICCID))
|
||||
|
||||
return nil
|
||||
@@ -619,6 +596,11 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
// 返回的 outcome 始终携带可回填可靠任务的结果分类(成功/失败/未知)与最后一次尝试的
|
||||
// Integration Log 标识;error 与既有语义一致,表示本次停机未成功。
|
||||
func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) {
|
||||
return s.stopCardAttempts(ctx, card, stopReason, s.maxRetries)
|
||||
}
|
||||
|
||||
// stopCardAttempts 调用运营商停机接口最多 attempts 次(停复机指令执行器只调用一次),成功后回写停机状态。
|
||||
func (s *StopResumeService) stopCardAttempts(ctx context.Context, card *model.IotCard, stopReason string, attempts int) (carrierthresholddomain.CommandOutcome, error) {
|
||||
actionCode, summary := stopAuditAction(ctx, stopReason)
|
||||
if s.gatewayClient == nil {
|
||||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||||
@@ -641,7 +623,7 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
|
||||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||||
var lastErr error
|
||||
lastIntegrationID := ""
|
||||
for i := 0; i < s.maxRetries; i++ {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
s.logger.Debug("重试调用停机接口",
|
||||
zap.Int("attempt", i+1),
|
||||
@@ -732,6 +714,11 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
|
||||
// resumeCardWithRetry 调用运营商复机接口(带重试机制)。
|
||||
// 第二个返回值是最后一次尝试的 Integration Log 标识(无论成功或失败),供调用方回填可靠任务状态。
|
||||
func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) {
|
||||
return s.resumeCardAttempts(ctx, card, actionCode, summary, s.maxRetries)
|
||||
}
|
||||
|
||||
// resumeCardAttempts 调用运营商复机接口最多 attempts 次,不回写卡状态。
|
||||
func (s *StopResumeService) resumeCardAttempts(ctx context.Context, card *model.IotCard, actionCode, summary string, attempts int) (*gatewayAttempt, string, error) {
|
||||
if s.gatewayClient == nil {
|
||||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||||
@@ -751,7 +738,7 @@ func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model
|
||||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||||
var lastErr error
|
||||
lastIntegrationID := ""
|
||||
for i := 0; i < s.maxRetries; i++ {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
s.logger.Debug("重试调用复机接口",
|
||||
zap.Int("attempt", i+1),
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -2202,8 +2203,9 @@ func (s *Service) tryResumeAfterPayment(ctx context.Context, order *model.Order)
|
||||
return
|
||||
}
|
||||
|
||||
resumeCtx := context.WithoutCancel(ctx)
|
||||
go func(ct string, cid uint) {
|
||||
resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneOrder)
|
||||
ct, cid := resumeCarrierType, resumeCarrierID
|
||||
stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() {
|
||||
if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, ct, cid); err != nil {
|
||||
s.logger.Error("支付后自动复机失败",
|
||||
zap.String("carrier_type", ct),
|
||||
@@ -2211,7 +2213,7 @@ func (s *Service) tryResumeAfterPayment(ctx context.Context, order *model.Order)
|
||||
zap.Uint("order_id", order.ID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(resumeCarrierType, resumeCarrierID)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) activatePackage(ctx context.Context, tx *gorm.DB, order *model.Order) error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package packagepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -778,15 +779,15 @@ func (s *ActivationService) resumeAfterActivation(ctx context.Context, carrierTy
|
||||
if s.resumeCallback == nil {
|
||||
return
|
||||
}
|
||||
resumeCtx := context.WithoutCancel(ctx)
|
||||
go func() {
|
||||
resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneActivation)
|
||||
stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() {
|
||||
if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, carrierType, carrierID); err != nil {
|
||||
s.logger.Error(errorMessage,
|
||||
zap.String("carrier_type", carrierType),
|
||||
zap.Uint("carrier_id", carrierID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (s *ActivationService) appendActivationObservation(ctx context.Context, tx *gorm.DB, usage *model.PackageUsage, carrierType string, carrierID uint, occurredAt time.Time) error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package packagepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||||
@@ -381,7 +382,7 @@ func (s *ResetService) triggerResumeForPackages(ctx context.Context, packages []
|
||||
return
|
||||
}
|
||||
|
||||
resumeCtx := context.WithoutCancel(ctx)
|
||||
resumeCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneReset)
|
||||
for _, pkg := range packages {
|
||||
var carrierType string
|
||||
var carrierID uint
|
||||
@@ -396,13 +397,14 @@ func (s *ResetService) triggerResumeForPackages(ctx context.Context, packages []
|
||||
continue
|
||||
}
|
||||
|
||||
go func(ct string, cid uint) {
|
||||
ct, cid := carrierType, carrierID
|
||||
stopresumedomain.RunAfterCommit(resumeCtx, s.resumeCallback, func() {
|
||||
if err := s.resumeCallback.ResumeCardIfStopped(resumeCtx, ct, cid); err != nil {
|
||||
s.logger.Warn("流量重置后自动复机失败",
|
||||
zap.String("carrier_type", ct),
|
||||
zap.Uint("carrier_id", cid),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(carrierType, carrierID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package packagepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
stopresumedomain "github.com/break/junhong_cmp_fiber/internal/domain/stopresume"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -449,15 +450,15 @@ func (s *UsageService) triggerSuspensionAfterCommit(ctx context.Context, carrier
|
||||
return
|
||||
}
|
||||
|
||||
stopCtx := context.WithoutCancel(ctx)
|
||||
stopCtx := stopresumedomain.WithScene(context.WithoutCancel(ctx), constants.StopResumeSceneUsage)
|
||||
if carrierType == constants.AssetTypeIotCard {
|
||||
go func() {
|
||||
stopresumedomain.RunAfterCommit(stopCtx, s.stopResumeCallback, func() {
|
||||
if err := s.stopResumeCallback.CheckAndStopCard(stopCtx, carrierID); err != nil {
|
||||
s.logger.Error("调用停机服务失败",
|
||||
zap.Uint("card_id", carrierID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}()
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -479,14 +480,14 @@ func (s *UsageService) triggerSuspensionAfterCommit(ctx context.Context, carrier
|
||||
}
|
||||
|
||||
for _, b := range bindings {
|
||||
cardID := b.IotCardID
|
||||
go func(cID uint) {
|
||||
cID := b.IotCardID
|
||||
stopresumedomain.RunAfterCommit(stopCtx, s.stopResumeCallback, func() {
|
||||
if err := s.stopResumeCallback.CheckAndStopCard(stopCtx, cID); err != nil {
|
||||
s.logger.Error("调用停机服务失败",
|
||||
zap.Uint("card_id", cID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}(cardID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -503,6 +503,28 @@ func (s *CarrierSchedulerService) CardState(ctx context.Context, cardID uint) (*
|
||||
}
|
||||
resp.Types = append(resp.Types, state)
|
||||
}
|
||||
resp.StopResumeQueueEnabled = s.settings.Current(ctx).StopResumeQueueEnabled
|
||||
command, err := s.store.ReadCommand(ctx, card.ID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(errors.CodeRedisError, err, "读取卡停复机指令失败")
|
||||
}
|
||||
if !command.LastCallAt.IsZero() {
|
||||
resp.LastStopResumeCallAt = timePtr(command.LastCallAt)
|
||||
}
|
||||
if command.Exists {
|
||||
item := &dto.CardStopResumeCommandState{Action: command.Action, Reason: command.Reason, Scenes: command.Scenes, Source: command.Source,
|
||||
Version: command.Version, FailCount: command.FailCount, LastError: command.LastError, Pending: command.Pending, Unknown: command.Unknown}
|
||||
if item.Scenes == nil {
|
||||
item.Scenes = []string{}
|
||||
}
|
||||
if !command.WrittenAt.IsZero() {
|
||||
item.WrittenAt = timePtr(command.WrittenAt)
|
||||
}
|
||||
if !command.NextAt.IsZero() {
|
||||
item.NextAt = timePtr(command.NextAt)
|
||||
}
|
||||
resp.StopResumeCommand = item
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
@@ -571,6 +593,13 @@ func (s *CarrierSchedulerService) CarrierStats(ctx context.Context) (*dto.Carrie
|
||||
item.RateLimited += atoi64(values[constants.GatewayErrorRateLimited])
|
||||
}
|
||||
item.QPSLastMinute = float64(attempts) / 60
|
||||
if srStats, srErr := s.store.StopResumeStats(ctx, carrierID); srErr == nil {
|
||||
item.StopResumeQueueLength, item.StopResumeDue = srStats.Queued, srStats.Due
|
||||
item.StopResumeRunning, item.StopResumePending = srStats.Running, srStats.Pending
|
||||
}
|
||||
srCounts := s.store.StopResumeStatLastHour(ctx, carrierID)
|
||||
item.StopResumeSuccess1h, item.StopResumeFailed1h = srCounts[constants.StopResumeStatSuccess], srCounts[constants.StopResumeStatFailed]
|
||||
item.StopResumeRetry1h, item.StopResumeUnknown1h = srCounts[constants.StopResumeStatRetry], srCounts[constants.StopResumeStatUnknown]
|
||||
for _, alert := range alertRows {
|
||||
if alert.CarrierID == carrierID {
|
||||
item.OpenAlerts += alert.Count
|
||||
|
||||
@@ -1243,6 +1243,20 @@ func (s *Service) handleRefundAssetProcessing(ctx context.Context, refundID uint
|
||||
func (s *Service) stopAsset(ctx context.Context, assetType string, assetID uint) bool {
|
||||
logger := s.logger
|
||||
|
||||
// 停复机出口开关开启:只请求评估("退款"场景,暂停轮询的资产同样停机),写入成功即视为资产后处理完成;
|
||||
// 停机失败由指令队列退避与告警处理,不再触发 Outbox 重投与补偿扫描。
|
||||
if s.stopResumeService != nil && s.stopResumeService.CommandQueueEnabled(ctx) {
|
||||
evalType := constants.AssetTypeIotCard
|
||||
if assetType == constants.AssetTypeDevice {
|
||||
evalType = constants.AssetTypeDevice
|
||||
}
|
||||
if err := s.stopResumeService.RequestAssetEvaluation(ctx, evalType, assetID, constants.StopResumeSceneRefund, "refund"); err != nil {
|
||||
logger.Error("退款资产处理:请求停机评估失败", zap.String("asset_type", assetType), zap.Uint("asset_id", assetID), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
switch assetType {
|
||||
case "iot_card":
|
||||
// 单卡停机:需要先查卡获取 iccid
|
||||
|
||||
Reference in New Issue
Block a user