All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m29s
- 停复机成功后保护窗口内(polling.stop_resume.status_grace_seconds,默认 300 秒)查卡状态返回操作前的状态视为上游滞后,不回写网络状态 - 套餐到期、退款场景不看库里的开停状态,资产无可用主套餐即停机,上游已停时按已是目标状态结束 - 迁移 000257 插入保护窗口配置 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1028 lines
42 KiB
Go
1028 lines
42 KiB
Go
package iot_card
|
||
|
||
import (
|
||
"context"
|
||
stderrors "errors"
|
||
"strings"
|
||
"time"
|
||
|
||
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||
priorityapp "github.com/break/junhong_cmp_fiber/internal/application/prioritypolling"
|
||
carrierthresholddomain "github.com/break/junhong_cmp_fiber/internal/domain/carrierthreshold"
|
||
"github.com/google/uuid"
|
||
"github.com/redis/go-redis/v9"
|
||
"go.uber.org/zap"
|
||
"gorm.io/gorm"
|
||
|
||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/gatewaylimit"
|
||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/integrationlog"
|
||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||
packagepkg "github.com/break/junhong_cmp_fiber/internal/service/package"
|
||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||
"github.com/break/junhong_cmp_fiber/pkg/outboxid"
|
||
)
|
||
|
||
// StopResumeServiceInterface 停复机服务接口
|
||
// 供轮询 Task Handler 通过接口注入,避免循环依赖
|
||
type StopResumeServiceInterface interface {
|
||
// EvaluateAndAct 停复机统一入口,根据卡的当前状态自动判断并执行停机或复机
|
||
EvaluateAndAct(ctx context.Context, card *model.IotCard) error
|
||
}
|
||
|
||
// 编译时验证 StopResumeService 实现了 StopResumeServiceInterface
|
||
var _ StopResumeServiceInterface = (*StopResumeService)(nil)
|
||
|
||
// StopResumeService 停复机服务
|
||
// 处理 IoT 卡的自动停机、复机和手动停复机逻辑
|
||
type StopResumeService struct {
|
||
db *gorm.DB
|
||
redis *redis.Client
|
||
iotCardStore *postgres.IotCardStore
|
||
packageUsageStore *postgres.PackageUsageStore
|
||
deviceSimBindingStore *postgres.DeviceSimBindingStore
|
||
gatewayClient *gateway.Client
|
||
logger *zap.Logger
|
||
pollingCallback PollingCallback
|
||
observationSeriesEvents cardObservationApp.SeriesEventWriter
|
||
priorityEvents priorityapp.PriorityEventWriter
|
||
auditWriter *audit.Writer
|
||
integration *integrationlog.Repository
|
||
|
||
maxRetries int
|
||
retryInterval time.Duration
|
||
// channelThresholdGuard 是可选的通道阈值持锁判定能力;未注入时不做持锁拒绝。
|
||
channelThresholdGuard ChannelThresholdLockGuard
|
||
// stopResumeRecorder 写每次上游停复机调用的停复机记录;未注入时不写。
|
||
stopResumeRecorder *gatewaylimit.StopResumeRecorder
|
||
// terminalPolling 停复机命中"自动关闭轮询"终态规则时关闭该卡轮询;未注入时按普通失败处理。
|
||
terminalPolling *gatewaylimit.TerminalPollingDisabler
|
||
// commandQueue 停复机指令队列;开关开启时自动入口只请求评估。
|
||
commandQueue StopResumeCommandQueue
|
||
}
|
||
|
||
// SetStopResumeRecorder 注入停复机记录写入器。
|
||
func (s *StopResumeService) SetStopResumeRecorder(recorder *gatewaylimit.StopResumeRecorder) {
|
||
s.stopResumeRecorder = recorder
|
||
}
|
||
|
||
// SetTerminalPollingDisabler 注入终态关闭轮询处理器。
|
||
func (s *StopResumeService) SetTerminalPollingDisabler(disabler *gatewaylimit.TerminalPollingDisabler) {
|
||
s.terminalPolling = disabler
|
||
}
|
||
|
||
// disablePollingOnTerminal 停复机上游失败命中"自动关闭轮询"规则时关闭该卡轮询并通知调度器;返回 true 表示不再重试。
|
||
func (s *StopResumeService) disablePollingOnTerminal(ctx context.Context, card *model.IotCard, action string, callErr error) bool {
|
||
if !s.terminalPolling.HandleCommandError(ctx, card, action, callErr) {
|
||
return false
|
||
}
|
||
card.EnablePolling = false
|
||
if s.pollingCallback != nil {
|
||
s.pollingCallback.OnCardDisabled(ctx, card.ID)
|
||
}
|
||
return true
|
||
}
|
||
|
||
// SetObservationSeriesEventWriter 注入停复机成功观测序列 Outbox Writer。
|
||
func (s *StopResumeService) SetObservationSeriesEventWriter(db *gorm.DB, writer cardObservationApp.SeriesEventWriter) {
|
||
s.db = db
|
||
s.observationSeriesEvents = writer
|
||
}
|
||
|
||
// SetUnifiedAudit 注入停复机统一审计和外部交互日志接缝。
|
||
func (s *StopResumeService) SetUnifiedAudit(writer *audit.Writer, integration *integrationlog.Repository) {
|
||
s.auditWriter = writer
|
||
s.integration = integration
|
||
}
|
||
|
||
// SetPriorityEventWriter 注入「资产无有效套餐」优先轮询请求 Outbox Writer。
|
||
func (s *StopResumeService) SetPriorityEventWriter(writer priorityapp.PriorityEventWriter) {
|
||
s.priorityEvents = writer
|
||
}
|
||
|
||
// NewStopResumeService 创建停复机服务
|
||
func NewStopResumeService(
|
||
redis *redis.Client,
|
||
iotCardStore *postgres.IotCardStore,
|
||
packageUsageStore *postgres.PackageUsageStore,
|
||
deviceSimBindingStore *postgres.DeviceSimBindingStore,
|
||
gatewayClient *gateway.Client,
|
||
logger *zap.Logger,
|
||
) *StopResumeService {
|
||
return &StopResumeService{
|
||
redis: redis,
|
||
iotCardStore: iotCardStore,
|
||
packageUsageStore: packageUsageStore,
|
||
deviceSimBindingStore: deviceSimBindingStore,
|
||
gatewayClient: gatewayClient,
|
||
logger: logger,
|
||
maxRetries: 3,
|
||
retryInterval: 2 * time.Second,
|
||
}
|
||
}
|
||
|
||
// SetPollingCallback 设置轮询生命周期回调。
|
||
// 停复机成功改变卡状态后,需要立即按新状态重排轮询队列,避免继续使用旧 score。
|
||
func (s *StopResumeService) SetPollingCallback(callback PollingCallback) {
|
||
s.pollingCallback = callback
|
||
}
|
||
|
||
// EvaluateAndAct 停复机统一入口:按统一判断函数决定停机或复机。
|
||
// 停复机出口开关开启时只请求评估(生成指令或请求设备判断),不当场调用网关;关闭时当场执行。
|
||
// 设备/单卡维度通过 card.IsStandalone 推导(false 则为设备维度,停/复机覆盖设备下所有卡)。触发场景见 WithStopResumeScene。
|
||
func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error {
|
||
scenes := stopResumeScenes(ctx)
|
||
decision, err := s.decideCard(ctx, card, scenes)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// 无有效套餐且存在待生效套餐使用记录:仅由普通套餐轮询来源追加优先轮询请求。
|
||
if containsStopReason(decision.Reasons, constants.StopReasonNoPackage) {
|
||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if s.commandQueueEnabled(ctx) {
|
||
return s.submitDecision(ctx, card, decision, scenes)
|
||
}
|
||
switch decision.Action {
|
||
case constants.StopResumeActionStop:
|
||
if decision.DeviceID > 0 {
|
||
return s.stopDeviceCards(ctx, decision.DeviceID, decision.Reason)
|
||
}
|
||
_, stopErr := s.stopCardWithRetry(ctx, card, decision.Reason)
|
||
return stopErr
|
||
case constants.StopResumeActionStart:
|
||
if decision.DeviceID > 0 {
|
||
return s.resumeDeviceCards(ctx, decision.DeviceID)
|
||
}
|
||
return s.resumeSingleCard(ctx, card.ID)
|
||
default:
|
||
return nil
|
||
}
|
||
}
|
||
|
||
// isRiskGatewayExtend 判断网关扩展状态是否为风险状态(风险停机或已销户)
|
||
// 独立卡命中此状态时禁止发起复机
|
||
func isRiskGatewayExtend(extend string) bool {
|
||
trimmed := strings.TrimSpace(extend)
|
||
return trimmed == constants.GatewayCardExtendRiskStop ||
|
||
trimmed == constants.GatewayCardExtendCancelled
|
||
}
|
||
|
||
// containsStopReason 判断停机原因集合是否包含指定原因。
|
||
func containsStopReason(reasons []string, target string) bool {
|
||
for _, reason := range reasons {
|
||
if reason == target {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// appendNoValidPackagePriority 在套餐轮询来源下,为无有效套餐且存在待生效套餐使用记录的载体追加优先轮询请求。
|
||
//
|
||
// 三个前置条件缺一不可:
|
||
// 1. 已注入优先轮询 Writer;
|
||
// 2. 本次评估来自普通套餐轮询(观测消费者、手动实名等调用链不打标,一律不写);
|
||
// 3. 该载体存在待生效的套餐使用记录,否则每个普通套餐周期都重新入队会形成自激循环。
|
||
//
|
||
// 事件资源为该载体对应的资产(独立卡为自身,绑定设备为设备),卡快照由 Writer 在同一事务内冻结。
|
||
func (s *StopResumeService) appendNoValidPackagePriority(ctx context.Context, card *model.IotCard) error {
|
||
if s.priorityEvents == nil || card == nil || s.db == nil || !priorityapp.IsPollingPriorityTrigger(ctx) {
|
||
return nil
|
||
}
|
||
carrierType, carrierID, err := s.resolvePackageCarrier(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
now := time.Now()
|
||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
hasPending, pendingErr := packagepkg.HasPendingPackageUsage(ctx, tx, carrierType, carrierID)
|
||
if pendingErr != nil {
|
||
return pendingErr
|
||
}
|
||
if !hasPending {
|
||
return nil
|
||
}
|
||
return packagepkg.AppendNoValidPackagePriorityRequested(ctx, tx, s.priorityEvents, carrierType, carrierID, now)
|
||
})
|
||
}
|
||
|
||
// isDeviceScopeReason 判断停机原因是否应扩散到设备维度
|
||
// 套餐/流量是设备共享资源,需整体停机;实名是单卡属性,只停该卡
|
||
func isDeviceScopeReason(reason string) bool {
|
||
return reason == constants.StopReasonNoPackage ||
|
||
reason == constants.StopReasonTrafficExhausted
|
||
}
|
||
|
||
// getCardDeviceID 获取卡所属设备ID(非独立卡时有效)
|
||
// 返回:deviceID, isDeviceBound, error
|
||
func (s *StopResumeService) getCardDeviceID(ctx context.Context, card *model.IotCard) (uint, bool, error) {
|
||
if card.IsStandalone {
|
||
return 0, false, nil
|
||
}
|
||
binding, err := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
|
||
if err != nil {
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return 0, false, nil
|
||
}
|
||
return 0, false, err
|
||
}
|
||
if binding == nil {
|
||
return 0, false, nil
|
||
}
|
||
return binding.DeviceID, true, nil
|
||
}
|
||
|
||
// resolvePackageCarrier 解析卡对应的套餐载体:非独立卡优先取绑定设备,其余取卡自身。
|
||
// 与 hasValidPackage/isTrafficExhausted 的既有口径一致,供优先轮询触发与停复机判定共用。
|
||
func (s *StopResumeService) resolvePackageCarrier(ctx context.Context, card *model.IotCard) (string, uint, error) {
|
||
if !card.IsStandalone {
|
||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||
if lookupErr != nil {
|
||
return "", 0, lookupErr
|
||
}
|
||
if bound {
|
||
return constants.AssetTypeDevice, deviceID, nil
|
||
}
|
||
}
|
||
return constants.AssetTypeIotCard, card.ID, nil
|
||
}
|
||
|
||
// hasValidPackage 检查卡是否有有效套餐(status IN 0,1)
|
||
// 修复Bug1:绑定设备的卡查 device_id,独立卡查 iot_card_id
|
||
func (s *StopResumeService) hasValidPackage(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
carrierType, carrierID, err := s.resolvePackageCarrier(ctx, card)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
return s.packageUsageStore.HasValidByCarrier(ctx, carrierType, carrierID)
|
||
}
|
||
|
||
// isTrafficExhausted 检查卡的流量是否已耗尽
|
||
// 条件:活跃套餐 status=2(已用完),或 data_usage_mb >= data_limit_mb(data_limit_mb > 0)
|
||
// 业务约定:data_limit_mb=0 表示无限流量套餐,永远不判定为耗尽
|
||
func (s *StopResumeService) isTrafficExhausted(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
carrierType := "iot_card"
|
||
carrierID := card.ID
|
||
|
||
if !card.IsStandalone {
|
||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||
if lookupErr != nil {
|
||
return false, lookupErr
|
||
}
|
||
if bound {
|
||
carrierType = "device"
|
||
carrierID = deviceID
|
||
}
|
||
}
|
||
|
||
packages, err := s.packageUsageStore.ListActiveOrDepletedByCarrier(ctx, carrierType, carrierID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if len(packages) == 0 {
|
||
return false, nil
|
||
}
|
||
|
||
for _, pkg := range packages {
|
||
if pkg == nil {
|
||
continue
|
||
}
|
||
if !pkg.IsTrafficDepleted() {
|
||
return false, nil
|
||
}
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// checkStopReasons 检查卡的停机原因列表,按优先级排序
|
||
// 优先级:no_package > traffic_exhausted > not_realname
|
||
func (s *StopResumeService) checkStopReasons(ctx context.Context, card *model.IotCard) ([]string, error) {
|
||
var reasons []string
|
||
|
||
// 条件A:流量耗尽
|
||
exhausted, err := s.isTrafficExhausted(ctx, card)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if exhausted {
|
||
reasons = append(reasons, constants.StopReasonTrafficExhausted)
|
||
}
|
||
|
||
// 条件B:无有效套餐(仅在未命中流量耗尽时判断)
|
||
hasPackage, err := s.hasValidPackage(ctx, card)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !hasPackage && !exhausted {
|
||
reasons = append(reasons, constants.StopReasonNoPackage)
|
||
}
|
||
|
||
// 条件C:不满足实际生效的实名要求
|
||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !realnameOK {
|
||
reasons = append(reasons, constants.StopReasonNotRealname)
|
||
}
|
||
|
||
return reasons, nil
|
||
}
|
||
|
||
// shouldResume 检查停机卡是否满足复机条件
|
||
// 须同时满足:有有效套餐 + 流量未耗尽 + 实名OK
|
||
func (s *StopResumeService) shouldResume(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
hasPackage, err := s.hasValidPackage(ctx, card)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if !hasPackage {
|
||
return false, nil
|
||
}
|
||
|
||
exhausted, err := s.isTrafficExhausted(ctx, card)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if exhausted {
|
||
return false, nil
|
||
}
|
||
|
||
return s.isRealnameOK(ctx, card)
|
||
}
|
||
|
||
// stopDeviceCards 停机设备下所有在线卡(含设备维度幂等锁)
|
||
// 防止设备下多张卡并发触发 EvaluateAndAct 导致重复 Gateway 停机调用
|
||
func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint, stopReason string) error {
|
||
// 设备维度幂等锁:防止多协程同时对同一设备停机
|
||
lockKey := constants.RedisPollingDeviceOpLockKey(deviceID)
|
||
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
|
||
if lockErr != nil {
|
||
return errors.Wrap(errors.CodeRedisError, lockErr, "获取设备停机锁失败")
|
||
}
|
||
if !locked {
|
||
s.logger.Debug("设备停机操作已在进行中,跳过重复调用",
|
||
zap.Uint("device_id", deviceID))
|
||
return nil
|
||
}
|
||
defer s.redis.Del(ctx, lockKey)
|
||
|
||
// 查询设备下在线卡;套餐失效类场景(套餐到期、退款)不看库里的开停状态(可能因上游延迟是旧值),设备下全部卡都停。
|
||
var cards []*model.IotCard
|
||
var err error
|
||
if stopReason == constants.StopReasonNoPackage && constants.StopResumeSceneAllowsSuspendedStop(stopResumeScenes(ctx)) {
|
||
cards, err = s.deviceCards(ctx, deviceID)
|
||
} else {
|
||
cards, err = s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOnline)
|
||
}
|
||
if err != nil {
|
||
s.logger.Error("查询设备在线卡失败",
|
||
zap.Uint("device_id", deviceID), zap.Error(err))
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "查询设备在线卡失败")
|
||
}
|
||
|
||
var cardErrors []error
|
||
for _, card := range cards {
|
||
// 暂停轮询的卡不随设备自动停机;套餐失效类场景(套餐到期、退款)照常停机。
|
||
if suspended, suspendErr := s.autoControlSuspended(ctx, card); suspendErr != nil {
|
||
cardErrors = append(cardErrors, suspendErr)
|
||
continue
|
||
} else if suspended && !(stopReason == constants.StopReasonNoPackage && constants.StopResumeSceneAllowsSuspendedStop(stopResumeScenes(ctx))) {
|
||
continue
|
||
}
|
||
if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
|
||
cardErrors = append(cardErrors, stopErr)
|
||
s.logger.Warn("设备卡停机失败,继续处理其他卡",
|
||
zap.Uint("device_id", deviceID),
|
||
zap.Uint("card_id", card.ID),
|
||
zap.Error(stopErr))
|
||
}
|
||
}
|
||
if len(cardErrors) > 0 {
|
||
return errors.Wrap(errors.CodeInternalError, stderrors.Join(cardErrors...), "设备停机存在未成功处理的卡")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// resumeDeviceCards 复机设备下满足条件的停机卡(含设备维度幂等锁)
|
||
// 遍历时对每张卡检查有效实名策略:不满足要求的卡更新 stop_reason='not_realname' 后跳过。
|
||
func (s *StopResumeService) resumeDeviceCards(ctx context.Context, deviceID uint) error {
|
||
// 设备维度幂等锁:与 stopDeviceCards 共用,防止停/复机并发
|
||
lockKey := constants.RedisPollingDeviceOpLockKey(deviceID)
|
||
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
|
||
if lockErr != nil {
|
||
return errors.Wrap(errors.CodeRedisError, lockErr, "获取设备复机锁失败")
|
||
}
|
||
if !locked {
|
||
s.logger.Debug("设备复机操作已在进行中,跳过重复调用",
|
||
zap.Uint("device_id", deviceID))
|
||
return nil
|
||
}
|
||
defer s.redis.Del(ctx, lockKey)
|
||
|
||
// 查询设备下全部停机卡(不看停机原因)
|
||
cards, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOffline)
|
||
if err != nil {
|
||
s.logger.Error("查询设备停机卡失败",
|
||
zap.Uint("device_id", deviceID), zap.Error(err))
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "查询设备停机卡失败")
|
||
}
|
||
|
||
var cardErrors []error
|
||
for _, card := range cards {
|
||
realnameOK, realnameErr := s.isRealnameOK(ctx, card)
|
||
if realnameErr != nil {
|
||
cardErrors = append(cardErrors, realnameErr)
|
||
s.logger.Warn("检查设备卡实名策略失败,继续处理其他卡",
|
||
zap.Uint("device_id", deviceID), zap.Uint("card_id", card.ID), zap.Error(realnameErr))
|
||
continue
|
||
}
|
||
if !realnameOK {
|
||
if updateErr := s.updateCardStopReasonWithAudit(ctx, card, constants.StopReasonNotRealname); updateErr != nil {
|
||
cardErrors = append(cardErrors, updateErr)
|
||
s.logger.Warn("更新未实名卡停机原因失败",
|
||
zap.Uint("card_id", card.ID), zap.Error(updateErr))
|
||
}
|
||
continue
|
||
}
|
||
if resumeErr := s.resumeSingleCard(ctx, card.ID); resumeErr != nil {
|
||
cardErrors = append(cardErrors, resumeErr)
|
||
s.logger.Warn("设备卡复机失败,继续处理其他卡",
|
||
zap.Uint("device_id", deviceID),
|
||
zap.Uint("card_id", card.ID),
|
||
zap.Error(resumeErr))
|
||
}
|
||
}
|
||
if len(cardErrors) > 0 {
|
||
return errors.Wrap(errors.CodeInternalError, stderrors.Join(cardErrors...), "设备复机存在未成功处理的卡")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// CheckAndStopCard 检查流量耗尽并停机(旧入口,保留兼容性)
|
||
// 新代码应优先使用 EvaluateAndAct
|
||
func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) error {
|
||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return s.EvaluateAndAct(ctx, card)
|
||
}
|
||
|
||
// ResumeCardIfStopped 套餐激活或流量重置后自动复机入口
|
||
// 支持 iot_card 和 device 两种载体类型
|
||
func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error {
|
||
if s.commandQueueEnabled(ctx) {
|
||
return s.RequestAssetEvaluation(ctx, carrierType, carrierID, stopResumeScenes(ctx)[0], "resume_if_stopped")
|
||
}
|
||
switch carrierType {
|
||
case "iot_card":
|
||
return s.resumeSingleCard(ctx, carrierID)
|
||
case "device":
|
||
return s.resumeDeviceCards(ctx, carrierID)
|
||
default:
|
||
return nil
|
||
}
|
||
}
|
||
|
||
// resumeSingleCard 对单张卡执行复机逻辑
|
||
// 依次检查:持通道阈值锁则拒绝 → 已开机则跳过 → 非轮询停机原因则跳过 → 不满足复机条件则跳过 → 加锁 → 调 Gateway → 更新 DB
|
||
func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) error {
|
||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
|
||
|
||
// 总开关开启时,卡或所在设备暂停轮询的不自动复机(含套餐激活、重置、充值等非轮询触发)。
|
||
suspended, suspendErr := s.autoControlSuspended(ctx, card)
|
||
if suspendErr != nil {
|
||
return suspendErr
|
||
}
|
||
if suspended {
|
||
return nil
|
||
}
|
||
|
||
// 持通道阈值锁:周期内拒绝一切复机,记录拒绝事实并保留锁。
|
||
// 该入口被轮询评估、套餐激活/重置与设备维度复机复用,返回 nil 表示本次不动作,
|
||
// 避免让自动链路按失败重试;用户可见入口由 ManualStartCard 返回拒绝错误。
|
||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||
if blockErr != nil {
|
||
return blockErr
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
s.logger.Info("卡持通道阈值锁,跳过自动复机",
|
||
zap.Uint("card_id", cardID), zap.String("stop_reason", card.StopReason))
|
||
return nil
|
||
}
|
||
|
||
if card.NetworkStatus == constants.NetworkStatusOnline {
|
||
return nil
|
||
}
|
||
|
||
// 检查复机条件
|
||
resume, err := s.shouldResume(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !resume {
|
||
return nil
|
||
}
|
||
|
||
// 分布式锁防止同一张卡并发触发多次 Gateway 调用,TTL 30s
|
||
lockKey := constants.RedisCardResumeLockKey(cardID)
|
||
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
|
||
if lockErr != nil {
|
||
s.logger.Warn("获取复机分布式锁失败,跳过本次复机",
|
||
zap.Uint("card_id", cardID),
|
||
zap.Error(lockErr))
|
||
return nil
|
||
}
|
||
if !locked {
|
||
s.logger.Debug("复机操作进行中,跳过", zap.Uint("card_id", cardID))
|
||
return nil
|
||
}
|
||
defer s.redis.Del(ctx, lockKey)
|
||
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
s.logger.Error("调用运营商复机接口失败",
|
||
zap.Uint("card_id", cardID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(err))
|
||
return err
|
||
}
|
||
return s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary)
|
||
}
|
||
|
||
// completeAutoResume 自动复机网关成功后回写在线状态、观测序列与统一审计,并重排轮询。
|
||
// 回写失败时返回错误,调用方按结果未知处理。
|
||
func (s *StopResumeService) completeAutoResume(ctx context.Context, card *model.IotCard, attempt *gatewayAttempt, integrationID, actionCode, summary string) error {
|
||
now := time.Now()
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": now,
|
||
"stop_reason": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
s.logger.Error("复机 Gateway 成功但 DB 更新失败",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(err))
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOnline}, err)
|
||
return err
|
||
}
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, logErr, "终结复机 Integration Log 失败")
|
||
}
|
||
|
||
s.logger.Info("卡已自动复机",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
|
||
return nil
|
||
}
|
||
|
||
// stopCardWithRetry 调用运营商停机接口(带重试机制),并更新 DB 停机原因。
|
||
//
|
||
// 返回的 outcome 始终携带可回填可靠任务的结果分类(成功/失败/未知)与最后一次尝试的
|
||
// Integration Log 标识;error 与既有语义一致,表示本次停机未成功。
|
||
func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) {
|
||
return s.stopCardAttempts(ctx, card, stopReason, s.maxRetries)
|
||
}
|
||
|
||
// stopCardAttempts 调用运营商停机接口最多 attempts 次(停复机指令执行器只调用一次),成功后回写停机状态。
|
||
func (s *StopResumeService) stopCardAttempts(ctx context.Context, card *model.IotCard, stopReason string, attempts int) (carrierthresholddomain.CommandOutcome, error) {
|
||
actionCode, summary := stopAuditAction(ctx, stopReason)
|
||
if s.gatewayClient == nil {
|
||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "",
|
||
cardSnapshot(card), nil, failErr)
|
||
return carrierthresholddomain.CommandOutcome{Result: constants.AuditResultFailed}, failErr
|
||
}
|
||
|
||
s.logger.Info("调用网关停机",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.String("stop_reason", stopReason))
|
||
|
||
seriesKey := cardCommandSeriesKey(ctx)
|
||
attemptObserver := &cardCommandAttemptObserver{
|
||
service: s, card: card, operation: constants.IntegrationOperationGatewayStopCard,
|
||
scene: constants.CardObservationSceneBusinessStop, seriesKey: seriesKey,
|
||
action: constants.StopResumeActionStop, reason: stopReason, legacy: legacyCommandLogging(ctx),
|
||
}
|
||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||
var lastErr error
|
||
lastIntegrationID := ""
|
||
for i := 0; i < attempts; i++ {
|
||
if i > 0 {
|
||
s.logger.Debug("重试调用停机接口",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID))
|
||
time.Sleep(s.retryInterval)
|
||
}
|
||
|
||
callErr := s.gatewayClient.StopCard(gatewayCtx, &gateway.CardOperationReq{CardNo: card.ICCID})
|
||
if gateway.IsRateLimited(callErr) {
|
||
// 被运营商账户限流拒绝时请求并未发出:不重试、不写失败审计,由调用方的既有重试或兜底路径稍后处理。
|
||
s.logger.Warn("停机被运营商账户网关限流,本次不调用上游", zap.Uint("card_id", card.ID), zap.String("iccid", card.ICCID))
|
||
return carrierthresholddomain.CommandOutcome{Result: constants.AuditResultFailed}, callErr
|
||
}
|
||
lastIntegrationID = attemptObserver.lastIntegrationID
|
||
if attemptObserver.recordingErr != nil {
|
||
s.logger.Error("记录停机 Integration Log 失败", zap.String("integration_id", lastIntegrationID), zap.Error(attemptObserver.recordingErr))
|
||
lastErr = attemptObserver.lastCallErr
|
||
if lastErr == nil {
|
||
lastErr = attemptObserver.recordingErr
|
||
}
|
||
// Integration Log 无法终结时仍必须留下本次停机的审计事实(与重试耗尽路径同一写入)。
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, lastErr)
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: attemptObserver.auditResult(lastErr),
|
||
}, lastErr
|
||
}
|
||
if callErr == nil {
|
||
attempt := attemptObserver.successful
|
||
s.logger.Info("网关停机成功",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
|
||
now := time.Now()
|
||
if updateErr := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOffline,
|
||
"stopped_at": now,
|
||
"stop_reason": stopReason,
|
||
}, constants.CardObservationSceneBusinessStop, "offline", uuid.NewString(), actionCode, summary, lastIntegrationID); updateErr != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结停机 Integration Log 失败", zap.String("integration_id", lastIntegrationID), zap.Error(logErr))
|
||
}
|
||
s.logger.Error("停机 Gateway 成功但 DB 更新失败",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(updateErr))
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, updateErr)
|
||
// 运营商已成功但本地回写失败:按结果未知交恢复扫描查询确认。
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: constants.AuditResultUnknown,
|
||
}, updateErr
|
||
}
|
||
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: constants.AuditResultSuccess,
|
||
}, errors.Wrap(errors.CodeDatabaseError, logErr, "终结停机 Integration Log 失败")
|
||
}
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: constants.AuditResultSuccess,
|
||
}, nil
|
||
}
|
||
|
||
lastErr = callErr
|
||
if s.disablePollingOnTerminal(ctx, card, constants.StopResumeActionStop, callErr) {
|
||
break
|
||
}
|
||
s.logger.Warn("调用停机接口失败,准备重试",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(callErr))
|
||
}
|
||
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, lastErr)
|
||
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: attemptObserver.auditResult(lastErr),
|
||
}, lastErr
|
||
}
|
||
|
||
// resumeCardWithRetry 调用运营商复机接口(带重试机制)。
|
||
// 第二个返回值是最后一次尝试的 Integration Log 标识(无论成功或失败),供调用方回填可靠任务状态。
|
||
func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) {
|
||
return s.resumeCardAttempts(ctx, card, actionCode, summary, s.maxRetries)
|
||
}
|
||
|
||
// resumeCardAttempts 调用运营商复机接口最多 attempts 次,不回写卡状态。
|
||
func (s *StopResumeService) resumeCardAttempts(ctx context.Context, card *model.IotCard, actionCode, summary string, attempts int) (*gatewayAttempt, string, error) {
|
||
if s.gatewayClient == nil {
|
||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||
return nil, "", failErr
|
||
}
|
||
|
||
s.logger.Info("调用网关复机",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
|
||
seriesKey := cardCommandSeriesKey(ctx)
|
||
attemptObserver := &cardCommandAttemptObserver{
|
||
service: s, card: card, operation: constants.IntegrationOperationGatewayStartCard,
|
||
scene: constants.CardObservationSceneBusinessResume, seriesKey: seriesKey,
|
||
action: constants.StopResumeActionStart, legacy: legacyCommandLogging(ctx),
|
||
}
|
||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||
var lastErr error
|
||
lastIntegrationID := ""
|
||
for i := 0; i < attempts; i++ {
|
||
if i > 0 {
|
||
s.logger.Debug("重试调用复机接口",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID))
|
||
time.Sleep(s.retryInterval)
|
||
}
|
||
|
||
req := &gateway.CardOperationReq{CardNo: card.ICCID}
|
||
if strings.TrimSpace(card.GatewayExtend) == constants.GatewayCardExtendMachineSeparated {
|
||
req.Extend = constants.GatewayCardStartExtendMachineSeparated
|
||
}
|
||
callErr := s.gatewayClient.StartCard(gatewayCtx, req)
|
||
if gateway.IsRateLimited(callErr) {
|
||
// 被运营商账户限流拒绝时请求并未发出:不重试、不写失败审计,由调用方的既有重试或兜底路径稍后处理。
|
||
s.logger.Warn("复机被运营商账户网关限流,本次不调用上游", zap.Uint("card_id", card.ID), zap.String("iccid", card.ICCID))
|
||
return nil, "", callErr
|
||
}
|
||
lastIntegrationID = attemptObserver.lastIntegrationID
|
||
if attemptObserver.recordingErr != nil {
|
||
s.logger.Error("记录复机 Integration Log 失败", zap.String("integration_id", lastIntegrationID), zap.Error(attemptObserver.recordingErr))
|
||
lastErr = attemptObserver.lastCallErr
|
||
if lastErr == nil {
|
||
lastErr = attemptObserver.recordingErr
|
||
}
|
||
// Integration Log 无法终结时仍必须留下本次复机的审计事实(与重试耗尽路径同一写入)。
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOnline}, lastErr)
|
||
return nil, lastIntegrationID, lastErr
|
||
}
|
||
if callErr == nil {
|
||
s.logger.Info("网关复机成功",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
return attemptObserver.successful, lastIntegrationID, nil
|
||
}
|
||
|
||
lastErr = callErr
|
||
if s.disablePollingOnTerminal(ctx, card, constants.StopResumeActionStart, callErr) {
|
||
break
|
||
}
|
||
s.logger.Warn("调用复机接口失败,准备重试",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(callErr))
|
||
}
|
||
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOnline}, lastErr)
|
||
return nil, lastIntegrationID, lastErr
|
||
}
|
||
|
||
// StartMachineSeparatedCard 对机卡分离停机卡执行复机
|
||
// 调用上游复机前检查本地落库的 Gateway extend,只有“机卡分离停机”才允许复机。
|
||
func (s *StopResumeService) StartMachineSeparatedCard(ctx context.Context, card *model.IotCard) error {
|
||
if card == nil {
|
||
return errors.New(errors.CodeInvalidParam)
|
||
}
|
||
actionCode, summary := constants.AuditActionIotCardOpenAPIStarted, "OpenAPI 恢复 IoT 卡网络"
|
||
|
||
// 持通道阈值锁:周期内拒绝一切复机,记录拒绝事实并保留锁。
|
||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||
if blockErr != nil {
|
||
return blockErr
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
return channelThresholdDenyError()
|
||
}
|
||
|
||
if s.gatewayClient == nil {
|
||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||
return failErr
|
||
}
|
||
|
||
gatewayExtend := strings.TrimSpace(card.GatewayExtend)
|
||
|
||
// 独立卡处于风险停机或已销户状态时,优先返回明确的拒绝原因
|
||
if card.IsStandalone && isRiskGatewayExtend(gatewayExtend) {
|
||
var denyMsg string
|
||
if gatewayExtend == constants.GatewayCardExtendRiskStop {
|
||
denyMsg = "该卡已被运营商风险停机,不允许复机"
|
||
} else {
|
||
denyMsg = "该卡已被运营商销户,不允许复机"
|
||
}
|
||
denyErr := errors.New(errors.CodeForbidden, denyMsg)
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
if gatewayExtend != constants.GatewayCardExtendMachineSeparated {
|
||
denyErr := errors.New(errors.CodeForbidden, constants.AgentOpenAPIResumeOnlyMachineSeparatedMessage)
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "",
|
||
cardSnapshot(card), map[string]any{"gateway_extend": gatewayExtend}, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
wrapErr := errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败,请稍后重试")
|
||
return wrapErr
|
||
}
|
||
|
||
now := time.Now()
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": now,
|
||
"stop_reason": "",
|
||
"gateway_extend": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
wrapErr := errors.Wrap(errors.CodeDatabaseError, err, "更新卡状态失败")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID, cardSnapshot(card), map[string]any{"requested_network_status": constants.NetworkStatusOnline}, wrapErr)
|
||
return wrapErr
|
||
}
|
||
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, logErr, "终结 OpenAPI 复机 Integration Log 失败")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ManualStopCard 手动停机单张卡(通过ICCID)
|
||
func (s *StopResumeService) ManualStopCard(ctx context.Context, iccid string) error {
|
||
card, err := s.iotCardStore.GetByICCID(ctx, iccid)
|
||
if err != nil {
|
||
return errors.New(errors.CodeNotFound, "卡不存在")
|
||
}
|
||
actionCode, summary := stopAuditAction(ctx, constants.StopReasonManual)
|
||
|
||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||
if err != nil {
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, err)
|
||
return err
|
||
}
|
||
if !realnameOK {
|
||
denyErr := errors.New(errors.CodeForbidden, "卡未实名,无法操作")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
if _, err := s.stopCardWithRetry(ctx, card, constants.StopReasonManual); err != nil {
|
||
return err
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// ManualStartCard 手动复机单张卡(通过ICCID)
|
||
func (s *StopResumeService) ManualStartCard(ctx context.Context, iccid string) error {
|
||
card, err := s.iotCardStore.GetByICCID(ctx, iccid)
|
||
if err != nil {
|
||
return errors.New(errors.CodeNotFound, "卡不存在")
|
||
}
|
||
actionCode, summary := constants.AuditActionIotCardManualStarted, "人工恢复 IoT 卡网络"
|
||
|
||
// 持通道阈值锁:周期内拒绝一切复机,记录拒绝事实并保留锁。
|
||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||
if blockErr != nil {
|
||
return blockErr
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
return channelThresholdDenyError()
|
||
}
|
||
|
||
// 独立卡处于风险停机或已销户状态时,拒绝复机
|
||
if card.IsStandalone && isRiskGatewayExtend(card.GatewayExtend) {
|
||
var denyMsg string
|
||
if strings.TrimSpace(card.GatewayExtend) == constants.GatewayCardExtendRiskStop {
|
||
denyMsg = "该卡已被运营商风险停机,不允许复机"
|
||
} else {
|
||
denyMsg = "该卡已被运营商销户,不允许复机"
|
||
}
|
||
denyErr := errors.New(errors.CodeForbidden, denyMsg)
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||
if err != nil {
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, err)
|
||
return err
|
||
}
|
||
if !realnameOK {
|
||
denyErr := errors.New(errors.CodeForbidden, "卡未实名,无法操作")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
wrapErr := errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败,请稍后重试")
|
||
return wrapErr
|
||
}
|
||
|
||
now := time.Now()
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": now,
|
||
"stop_reason": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
wrapErr := errors.Wrap(errors.CodeDatabaseError, err, "更新卡状态失败")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID, cardSnapshot(card), map[string]any{"requested_network_status": constants.NetworkStatusOnline}, wrapErr)
|
||
return wrapErr
|
||
}
|
||
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, logErr, "终结人工复机 Integration Log 失败")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *StopResumeService) updateCardAndAppendNetworkSeries(
|
||
ctx context.Context,
|
||
card *model.IotCard,
|
||
fields map[string]any,
|
||
scene, expected, operationID, actionCode, summary, integrationID string,
|
||
) error {
|
||
if s.db == nil || s.observationSeriesEvents == nil || card == nil || card.ID == 0 {
|
||
return errors.New(errors.CodeInternalError, "停复机观测 Outbox 能力未配置")
|
||
}
|
||
requestID := requestIDFromContext(ctx)
|
||
if requestID == "" {
|
||
requestID = operationID
|
||
}
|
||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
if err := tx.Model(&model.IotCard{}).Where("id = ?", card.ID).Updates(fields).Error; err != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "更新卡停复机状态失败")
|
||
}
|
||
if err := s.appendCardCommandAudit(ctx, tx, card, actionCode, summary, constants.AuditResultSuccess,
|
||
integrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
|
||
fields, nil); err != nil {
|
||
return err
|
||
}
|
||
if cardObservationApp.IsSeriesTriggerSuppressed(ctx) {
|
||
return nil
|
||
}
|
||
return s.observationSeriesEvents.AppendSeriesRequested(ctx, tx, cardObservationApp.SeriesRequestedEvent{
|
||
EventID: outboxid.Stable("card-observation:network-command:", operationID),
|
||
Scene: scene, ResourceType: constants.CardObservationResourceTypeCard, ResourceID: card.ID,
|
||
SyncTypes: []string{constants.CardObservationSyncTypeNetwork}, ExpectedValue: expected,
|
||
Source: constants.CardObservationSourceBusinessEvent, OccurredAt: time.Now().UTC(),
|
||
RequestID: requestID, CorrelationID: requestID,
|
||
})
|
||
})
|
||
}
|
||
|
||
// invalidatePollingCache 删除轮询卡缓存,下次轮询自动从 DB 重建
|
||
func (s *StopResumeService) invalidatePollingCache(ctx context.Context, cardID uint) {
|
||
if s.redis == nil {
|
||
return
|
||
}
|
||
key := constants.RedisPollingCardInfoKey(cardID)
|
||
if err := s.redis.Del(ctx, key).Err(); err != nil {
|
||
s.logger.Warn("删除轮询卡缓存失败", zap.Uint("card_id", cardID))
|
||
}
|
||
}
|
||
|
||
// reschedulePolling 按卡最新状态重排轮询队列。
|
||
// 未注入生命周期回调时退化为只清缓存,保持原有行为。
|
||
func (s *StopResumeService) reschedulePolling(ctx context.Context, cardID uint) {
|
||
if s.pollingCallback == nil {
|
||
s.invalidatePollingCache(ctx, cardID)
|
||
return
|
||
}
|
||
s.pollingCallback.OnCardStatusChanged(ctx, cardID)
|
||
}
|