All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 14m13s
新增 000228 成对迁移 tb_polling_priority_item:卡、任务类型、状态、触发类型、来源订单/套餐使用记录、
触发次数与来源集合、尝试次数、失败原因、人工原因与操作者、店铺快照与各时间列;以活动项部分唯一索引
uq_polling_priority_item_active(仅 deleted_at IS NULL AND status IN ('pending','processing') 占键位)
表达「同卡同任务类型至多一条活动项」,另有状态/时间索引与全列注释;down 守卫在存在活动项或未终态行时
拒绝回滚并给出中文原因。
新增优先轮询请求可靠事件 polling.priority.requested(载荷版本 v1、事件键前缀 prio:)与消费者:只在原
业务事务内追加、幂等键稳定;消费者按卡 × 纳入任务类型(realname/carddata/card_status/package)逐条
建项并在提交后下发执行提示,重复投递只合并触发次数、来源集合与最近触发时间,不新建行也不重复调用。
触发点为四类自动场景 purchase_activated / renewal_activated(按同载体更早套餐使用记录判定)/
queue_activated / addon_activated 与「无有效套餐」no_valid_package(仅在普通套餐轮询来源且存在待生效
套餐使用记录时追加;事件通道显式拒绝 manual_trigger);入队对象恒为卡,绑定设备资产在触发事务内冻结
在用卡快照逐卡建项,不使用设备当前卡槽口径。
轮询共享基类新增认领接缝:四个 Handler(realname/carddata/card_status/package)在并发信号量之后、调用
上游之前探测活动项——待执行条件认领、执行中且 90 秒租约未到期则跳过并延后、无活动项时行为与既有完全
等价;超租约允许相邻执行接管,尝试次数只在真正发起执行后累加,未达上限(3)回到活动态按既有间隔重排,
达上限或业务校验类失败进入失败终态并保留可安全展示原因;执行前校验卡自身与绑定设备的轮询开关。未引入
通用卡级锁与 Redis 活动标记,分片队列的出队、入队与移除路径未改动。
提示通道按任务类型独立键(polling:priority:{taskType}),与既有手动触发队列分离;调度器在同一周期内先
排空优先提示、再排空手动触发队列,提示排空不受分片背压跳过影响;未新建调度设施或异步任务类型。
新增人工优先入队与只读查询三条路由 POST /api/admin/polling-priority-items、
GET /api/admin/polling-priority-items、GET /api/admin/polling-priority-items/:id:人工入队复用既有轮询
权限判定(抽取为同包共享函数),原因必填,不受每日 500 次上限与 24 小时去重约束,重复抑制由活动项合并
承担;读侧按店铺快照下推数据范围,越权与不存在不可区分,不提供优先级分级、有效期或人工重触发入口。
新增 7 个审计动作(enqueue/claim/fail/retry/complete/dequeue/manual_denied)与资源
polling_priority_item,并按(操作者类型,来源)注册,人工侧与 Worker 侧均通过来源校验。
同步 OpenAPI 文档装配三处与路由注册;归档 Change 至
openspec/changes/archive/2026-09-17-add-priority-polling-queue/ 并同步主 Spec(新增
priority-polling-queue、polling-operations 追加单次执行互斥 Requirement 与三条路由索引)与上下文健康
证据(requirement-evidence 150 行、入口矩阵 http 403 / async 56)。
本机验证:junhong_cmp_test 与隔离 Redis DB 15,未连生产、未启动 Worker/API、未调用运营商上游;迁移
up/down/up 与 down 守卫实测(含 dirty=true 记账口径与 force 恢复),A–F 批 94 PASS、接缝 63 PASS、
提示通道 12 PASS、清理零残留 20 PASS。成功路径 Complete、真并发互斥、尝试上限第 3 次判定、HTTP 层权限
矩阵、通道阈值持锁复机边界与三类生效触发点生产集成留待测试部署验证(见
docs/verification/add-priority-polling-queue-verification.md 第 4 节)。自动化测试按项目决策为 N/A,
未新增 *_test.go。
1098 lines
44 KiB
Go
1098 lines
44 KiB
Go
package iot_card
|
||
|
||
import (
|
||
"context"
|
||
stderrors "errors"
|
||
"strings"
|
||
"time"
|
||
|
||
cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||
priorityapp "github.com/break/junhong_cmp_fiber/internal/application/prioritypolling"
|
||
carrierthresholddomain "github.com/break/junhong_cmp_fiber/internal/domain/carrierthreshold"
|
||
"github.com/google/uuid"
|
||
"github.com/redis/go-redis/v9"
|
||
"go.uber.org/zap"
|
||
"gorm.io/gorm"
|
||
|
||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/integrationlog"
|
||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||
packagepkg "github.com/break/junhong_cmp_fiber/internal/service/package"
|
||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||
"github.com/break/junhong_cmp_fiber/pkg/outboxid"
|
||
)
|
||
|
||
// StopResumeServiceInterface 停复机服务接口
|
||
// 供 4 个 Task Handler 通过接口注入,避免循环依赖
|
||
type StopResumeServiceInterface interface {
|
||
// EvaluateAndAct 停复机统一入口,根据卡的当前状态自动判断并执行停机或复机
|
||
EvaluateAndAct(ctx context.Context, card *model.IotCard) error
|
||
// ForceStopCard 强制停机单张卡,不执行正常停机条件判断。
|
||
ForceStopCard(ctx context.Context, card *model.IotCard, stopReason string) error
|
||
// ForceStartCard 强制复机单张卡,不执行正常复机条件判断。
|
||
ForceStartCard(ctx context.Context, card *model.IotCard) error
|
||
}
|
||
|
||
// 编译时验证 StopResumeService 实现了 StopResumeServiceInterface
|
||
var _ StopResumeServiceInterface = (*StopResumeService)(nil)
|
||
|
||
// StopResumeService 停复机服务
|
||
// 处理 IoT 卡的自动停机、复机和手动停复机逻辑
|
||
type StopResumeService struct {
|
||
db *gorm.DB
|
||
redis *redis.Client
|
||
iotCardStore *postgres.IotCardStore
|
||
packageUsageStore *postgres.PackageUsageStore
|
||
deviceSimBindingStore *postgres.DeviceSimBindingStore
|
||
gatewayClient *gateway.Client
|
||
logger *zap.Logger
|
||
pollingCallback PollingCallback
|
||
observationSeriesEvents cardObservationApp.SeriesEventWriter
|
||
priorityEvents priorityapp.PriorityEventWriter
|
||
auditWriter *audit.Writer
|
||
integration *integrationlog.Repository
|
||
|
||
maxRetries int
|
||
retryInterval time.Duration
|
||
// channelThresholdGuard 是可选的通道阈值持锁判定能力;未注入时不做持锁拒绝。
|
||
channelThresholdGuard ChannelThresholdLockGuard
|
||
}
|
||
|
||
// SetObservationSeriesEventWriter 注入停复机成功观测序列 Outbox Writer。
|
||
func (s *StopResumeService) SetObservationSeriesEventWriter(db *gorm.DB, writer cardObservationApp.SeriesEventWriter) {
|
||
s.db = db
|
||
s.observationSeriesEvents = writer
|
||
}
|
||
|
||
// SetUnifiedAudit 注入停复机统一审计和外部交互日志接缝。
|
||
func (s *StopResumeService) SetUnifiedAudit(writer *audit.Writer, integration *integrationlog.Repository) {
|
||
s.auditWriter = writer
|
||
s.integration = integration
|
||
}
|
||
|
||
// SetPriorityEventWriter 注入「资产无有效套餐」优先轮询请求 Outbox Writer。
|
||
func (s *StopResumeService) SetPriorityEventWriter(writer priorityapp.PriorityEventWriter) {
|
||
s.priorityEvents = writer
|
||
}
|
||
|
||
// NewStopResumeService 创建停复机服务
|
||
func NewStopResumeService(
|
||
redis *redis.Client,
|
||
iotCardStore *postgres.IotCardStore,
|
||
packageUsageStore *postgres.PackageUsageStore,
|
||
deviceSimBindingStore *postgres.DeviceSimBindingStore,
|
||
gatewayClient *gateway.Client,
|
||
logger *zap.Logger,
|
||
) *StopResumeService {
|
||
return &StopResumeService{
|
||
redis: redis,
|
||
iotCardStore: iotCardStore,
|
||
packageUsageStore: packageUsageStore,
|
||
deviceSimBindingStore: deviceSimBindingStore,
|
||
gatewayClient: gatewayClient,
|
||
logger: logger,
|
||
maxRetries: 3,
|
||
retryInterval: 2 * time.Second,
|
||
}
|
||
}
|
||
|
||
// SetPollingCallback 设置轮询生命周期回调。
|
||
// 停复机成功改变卡状态后,需要立即按新状态重排轮询队列,避免继续使用旧 score。
|
||
func (s *StopResumeService) SetPollingCallback(callback PollingCallback) {
|
||
s.pollingCallback = callback
|
||
}
|
||
|
||
// EvaluateAndAct 停复机统一入口
|
||
// 根据卡的当前网络状态,自动判断并执行停机或复机操作
|
||
// 设备/单卡维度通过 card.IsStandalone 推导(false 则为设备维度,停/复机覆盖设备下所有卡)
|
||
func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error {
|
||
switch card.NetworkStatus {
|
||
case constants.NetworkStatusOnline:
|
||
// 卡在线,检查是否需要停机
|
||
reasons, err := s.checkStopReasons(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(reasons) == 0 {
|
||
return nil
|
||
}
|
||
// 无有效套餐且存在待生效套餐使用记录:仅由普通套餐轮询来源追加优先轮询请求。
|
||
if containsStopReason(reasons, constants.StopReasonNoPackage) {
|
||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
// 取最高优先级原因(第一个)
|
||
primaryReason := reasons[0]
|
||
if !card.IsStandalone && isDeviceScopeReason(primaryReason) {
|
||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||
if lookupErr != nil {
|
||
s.logger.Error("查询设备绑定关系失败,降级为单卡停机",
|
||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||
// 降级:查不到绑定时按单卡处理
|
||
} else if bound {
|
||
return s.stopDeviceCards(ctx, deviceID, primaryReason)
|
||
}
|
||
}
|
||
_, stopErr := s.stopCardWithRetry(ctx, card, primaryReason)
|
||
return stopErr
|
||
|
||
case constants.NetworkStatusOffline:
|
||
// 卡停机,检查是否可以复机
|
||
// 只处理轮询系统引起的停机原因(手动停机等不自动复机)
|
||
if !isPollingStopReason(card.StopReason) {
|
||
return nil
|
||
}
|
||
shouldResume, err := s.shouldResume(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !shouldResume {
|
||
return nil
|
||
}
|
||
if !card.IsStandalone {
|
||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||
if lookupErr != nil {
|
||
s.logger.Error("查询设备绑定关系失败,降级为单卡复机",
|
||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||
} else if bound {
|
||
return s.resumeDeviceCards(ctx, deviceID)
|
||
}
|
||
}
|
||
return s.resumeSingleCard(ctx, card.ID)
|
||
|
||
default:
|
||
return nil
|
||
}
|
||
}
|
||
|
||
// isRiskGatewayExtend 判断网关扩展状态是否为风险状态(风险停机或已销户)
|
||
// 独立卡命中此状态时禁止发起复机
|
||
func isRiskGatewayExtend(extend string) bool {
|
||
trimmed := strings.TrimSpace(extend)
|
||
return trimmed == constants.GatewayCardExtendRiskStop ||
|
||
trimmed == constants.GatewayCardExtendCancelled
|
||
}
|
||
|
||
// isPollingStopReason 判断停机原因是否为轮询系统引起(可自动复机)
|
||
func isPollingStopReason(reason string) bool {
|
||
switch reason {
|
||
case constants.StopReasonTrafficExhausted,
|
||
constants.StopReasonNoPackage,
|
||
constants.StopReasonNotRealname:
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// containsStopReason 判断停机原因集合是否包含指定原因。
|
||
func containsStopReason(reasons []string, target string) bool {
|
||
for _, reason := range reasons {
|
||
if reason == target {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// appendNoValidPackagePriority 在套餐轮询来源下,为无有效套餐且存在待生效套餐使用记录的载体追加优先轮询请求。
|
||
//
|
||
// 三个前置条件缺一不可:
|
||
// 1. 已注入优先轮询 Writer;
|
||
// 2. 本次评估来自普通套餐轮询(观测消费者、手动实名、保护期轮询等调用链不打标,一律不写);
|
||
// 3. 该载体存在待生效的套餐使用记录,否则每个普通套餐周期都重新入队会形成自激循环。
|
||
//
|
||
// 事件资源为该载体对应的资产(独立卡为自身,绑定设备为设备),卡快照由 Writer 在同一事务内冻结。
|
||
func (s *StopResumeService) appendNoValidPackagePriority(ctx context.Context, card *model.IotCard) error {
|
||
if s.priorityEvents == nil || card == nil || s.db == nil || !priorityapp.IsPollingPriorityTrigger(ctx) {
|
||
return nil
|
||
}
|
||
carrierType, carrierID, err := s.resolvePackageCarrier(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
now := time.Now()
|
||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
hasPending, pendingErr := packagepkg.HasPendingPackageUsage(ctx, tx, carrierType, carrierID)
|
||
if pendingErr != nil {
|
||
return pendingErr
|
||
}
|
||
if !hasPending {
|
||
return nil
|
||
}
|
||
return packagepkg.AppendNoValidPackagePriorityRequested(ctx, tx, s.priorityEvents, carrierType, carrierID, now)
|
||
})
|
||
}
|
||
|
||
// isDeviceScopeReason 判断停机原因是否应扩散到设备维度
|
||
// 套餐/流量是设备共享资源,需整体停机;实名是单卡属性,只停该卡
|
||
func isDeviceScopeReason(reason string) bool {
|
||
return reason == constants.StopReasonNoPackage ||
|
||
reason == constants.StopReasonTrafficExhausted
|
||
}
|
||
|
||
// getCardDeviceID 获取卡所属设备ID(非独立卡时有效)
|
||
// 返回:deviceID, isDeviceBound, error
|
||
func (s *StopResumeService) getCardDeviceID(ctx context.Context, card *model.IotCard) (uint, bool, error) {
|
||
if card.IsStandalone {
|
||
return 0, false, nil
|
||
}
|
||
binding, err := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
|
||
if err != nil {
|
||
if stderrors.Is(err, gorm.ErrRecordNotFound) {
|
||
return 0, false, nil
|
||
}
|
||
return 0, false, err
|
||
}
|
||
if binding == nil {
|
||
return 0, false, nil
|
||
}
|
||
return binding.DeviceID, true, nil
|
||
}
|
||
|
||
// resolvePackageCarrier 解析卡对应的套餐载体:非独立卡优先取绑定设备,其余取卡自身。
|
||
// 与 hasValidPackage/isTrafficExhausted 的既有口径一致,供优先轮询触发与停复机判定共用。
|
||
func (s *StopResumeService) resolvePackageCarrier(ctx context.Context, card *model.IotCard) (string, uint, error) {
|
||
if !card.IsStandalone {
|
||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||
if lookupErr != nil {
|
||
return "", 0, lookupErr
|
||
}
|
||
if bound {
|
||
return constants.AssetTypeDevice, deviceID, nil
|
||
}
|
||
}
|
||
return constants.AssetTypeIotCard, card.ID, nil
|
||
}
|
||
|
||
// hasValidPackage 检查卡是否有有效套餐(status IN 0,1)
|
||
// 修复Bug1:绑定设备的卡查 device_id,独立卡查 iot_card_id
|
||
func (s *StopResumeService) hasValidPackage(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
carrierType, carrierID, err := s.resolvePackageCarrier(ctx, card)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
return s.packageUsageStore.HasValidByCarrier(ctx, carrierType, carrierID)
|
||
}
|
||
|
||
// isTrafficExhausted 检查卡的流量是否已耗尽
|
||
// 条件:活跃套餐 status=2(已用完),或 data_usage_mb >= data_limit_mb(data_limit_mb > 0)
|
||
// 业务约定:data_limit_mb=0 表示无限流量套餐,永远不判定为耗尽
|
||
func (s *StopResumeService) isTrafficExhausted(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
carrierType := "iot_card"
|
||
carrierID := card.ID
|
||
|
||
if !card.IsStandalone {
|
||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||
if lookupErr != nil {
|
||
return false, lookupErr
|
||
}
|
||
if bound {
|
||
carrierType = "device"
|
||
carrierID = deviceID
|
||
}
|
||
}
|
||
|
||
packages, err := s.packageUsageStore.ListActiveOrDepletedByCarrier(ctx, carrierType, carrierID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if len(packages) == 0 {
|
||
return false, nil
|
||
}
|
||
|
||
for _, pkg := range packages {
|
||
if pkg == nil {
|
||
continue
|
||
}
|
||
if !pkg.IsTrafficDepleted() {
|
||
return false, nil
|
||
}
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// isRealnameOK 检查卡是否满足实名要求
|
||
// 行业卡、已实名卡直接放行;其他卡按有效设备或自身的无需实名策略判断。
|
||
func (s *StopResumeService) isRealnameOK(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
if card.CardCategory == constants.CardCategoryIndustry || card.RealNameStatus == constants.RealNameStatusVerified {
|
||
return true, nil
|
||
}
|
||
|
||
deviceID, bound, err := s.getCardDeviceID(ctx, card)
|
||
if err != nil {
|
||
return false, errors.Wrap(errors.CodeDatabaseError, err, "查询卡实名策略绑定关系失败")
|
||
}
|
||
if !bound {
|
||
return card.RealnamePolicy == constants.RealnamePolicyNone, nil
|
||
}
|
||
|
||
var device model.Device
|
||
if err := s.db.WithContext(ctx).Select("realname_policy").First(&device, deviceID).Error; err != nil {
|
||
return false, errors.Wrap(errors.CodeDatabaseError, err, "查询设备实名策略失败")
|
||
}
|
||
return device.RealnamePolicy == constants.RealnamePolicyNone, nil
|
||
}
|
||
|
||
// checkStopReasons 检查卡的停机原因列表,按优先级排序
|
||
// 优先级:no_package > traffic_exhausted > not_realname
|
||
func (s *StopResumeService) checkStopReasons(ctx context.Context, card *model.IotCard) ([]string, error) {
|
||
var reasons []string
|
||
|
||
// 条件A:流量耗尽
|
||
exhausted, err := s.isTrafficExhausted(ctx, card)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if exhausted {
|
||
reasons = append(reasons, constants.StopReasonTrafficExhausted)
|
||
}
|
||
|
||
// 条件B:无有效套餐(仅在未命中流量耗尽时判断)
|
||
hasPackage, err := s.hasValidPackage(ctx, card)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !hasPackage && !exhausted {
|
||
reasons = append(reasons, constants.StopReasonNoPackage)
|
||
}
|
||
|
||
// 条件C:不满足实际生效的实名要求
|
||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !realnameOK {
|
||
reasons = append(reasons, constants.StopReasonNotRealname)
|
||
}
|
||
|
||
return reasons, nil
|
||
}
|
||
|
||
// shouldResume 检查停机卡是否满足复机条件
|
||
// 须同时满足:有有效套餐 + 流量未耗尽 + 实名OK
|
||
func (s *StopResumeService) shouldResume(ctx context.Context, card *model.IotCard) (bool, error) {
|
||
hasPackage, err := s.hasValidPackage(ctx, card)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if !hasPackage {
|
||
return false, nil
|
||
}
|
||
|
||
exhausted, err := s.isTrafficExhausted(ctx, card)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if exhausted {
|
||
return false, nil
|
||
}
|
||
|
||
return s.isRealnameOK(ctx, card)
|
||
}
|
||
|
||
// stopDeviceCards 停机设备下所有在线卡(含设备维度幂等锁)
|
||
// 防止设备下多张卡并发触发 EvaluateAndAct 导致重复 Gateway 停机调用
|
||
func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint, stopReason string) error {
|
||
// 设备维度幂等锁:防止多协程同时对同一设备停机
|
||
lockKey := constants.RedisPollingDeviceOpLockKey(deviceID)
|
||
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
|
||
if lockErr != nil {
|
||
return errors.Wrap(errors.CodeRedisError, lockErr, "获取设备停机锁失败")
|
||
}
|
||
if !locked {
|
||
s.logger.Debug("设备停机操作已在进行中,跳过重复调用",
|
||
zap.Uint("device_id", deviceID))
|
||
return nil
|
||
}
|
||
defer s.redis.Del(ctx, lockKey)
|
||
|
||
// 查询设备下所有在线卡
|
||
cards, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOnline)
|
||
if err != nil {
|
||
s.logger.Error("查询设备在线卡失败",
|
||
zap.Uint("device_id", deviceID), zap.Error(err))
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "查询设备在线卡失败")
|
||
}
|
||
|
||
var cardErrors []error
|
||
for _, card := range cards {
|
||
if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
|
||
cardErrors = append(cardErrors, stopErr)
|
||
s.logger.Warn("设备卡停机失败,继续处理其他卡",
|
||
zap.Uint("device_id", deviceID),
|
||
zap.Uint("card_id", card.ID),
|
||
zap.Error(stopErr))
|
||
}
|
||
}
|
||
if len(cardErrors) > 0 {
|
||
return errors.Wrap(errors.CodeInternalError, stderrors.Join(cardErrors...), "设备停机存在未成功处理的卡")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// resumeDeviceCards 复机设备下满足条件的停机卡(含设备维度幂等锁)
|
||
// 遍历时对每张卡检查有效实名策略:不满足要求的卡更新 stop_reason='not_realname' 后跳过。
|
||
func (s *StopResumeService) resumeDeviceCards(ctx context.Context, deviceID uint) error {
|
||
// 设备维度幂等锁:与 stopDeviceCards 共用,防止停/复机并发
|
||
lockKey := constants.RedisPollingDeviceOpLockKey(deviceID)
|
||
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
|
||
if lockErr != nil {
|
||
return errors.Wrap(errors.CodeRedisError, lockErr, "获取设备复机锁失败")
|
||
}
|
||
if !locked {
|
||
s.logger.Debug("设备复机操作已在进行中,跳过重复调用",
|
||
zap.Uint("device_id", deviceID))
|
||
return nil
|
||
}
|
||
defer s.redis.Del(ctx, lockKey)
|
||
|
||
// 查询设备下因轮询原因停机的卡
|
||
cards, err := s.iotCardStore.ListByDeviceIDAndPollingStopReasons(ctx, deviceID)
|
||
if err != nil {
|
||
s.logger.Error("查询设备停机卡失败",
|
||
zap.Uint("device_id", deviceID), zap.Error(err))
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "查询设备停机卡失败")
|
||
}
|
||
|
||
var cardErrors []error
|
||
for _, card := range cards {
|
||
realnameOK, realnameErr := s.isRealnameOK(ctx, card)
|
||
if realnameErr != nil {
|
||
cardErrors = append(cardErrors, realnameErr)
|
||
s.logger.Warn("检查设备卡实名策略失败,继续处理其他卡",
|
||
zap.Uint("device_id", deviceID), zap.Uint("card_id", card.ID), zap.Error(realnameErr))
|
||
continue
|
||
}
|
||
if !realnameOK {
|
||
if updateErr := s.updateCardStopReasonWithAudit(ctx, card, constants.StopReasonNotRealname); updateErr != nil {
|
||
cardErrors = append(cardErrors, updateErr)
|
||
s.logger.Warn("更新未实名卡停机原因失败",
|
||
zap.Uint("card_id", card.ID), zap.Error(updateErr))
|
||
}
|
||
continue
|
||
}
|
||
if resumeErr := s.resumeSingleCard(ctx, card.ID); resumeErr != nil {
|
||
cardErrors = append(cardErrors, resumeErr)
|
||
s.logger.Warn("设备卡复机失败,继续处理其他卡",
|
||
zap.Uint("device_id", deviceID),
|
||
zap.Uint("card_id", card.ID),
|
||
zap.Error(resumeErr))
|
||
}
|
||
}
|
||
if len(cardErrors) > 0 {
|
||
return errors.Wrap(errors.CodeInternalError, stderrors.Join(cardErrors...), "设备复机存在未成功处理的卡")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// CheckAndStopCard 检查流量耗尽并停机(旧入口,保留兼容性)
|
||
// 新代码应优先使用 EvaluateAndAct
|
||
func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) error {
|
||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return s.EvaluateAndAct(ctx, card)
|
||
}
|
||
|
||
// ResumeCardIfStopped 套餐激活或流量重置后自动复机入口
|
||
// 支持 iot_card 和 device 两种载体类型
|
||
func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error {
|
||
switch carrierType {
|
||
case "iot_card":
|
||
return s.resumeSingleCard(ctx, carrierID)
|
||
case "device":
|
||
return s.resumeDeviceCards(ctx, carrierID)
|
||
default:
|
||
return nil
|
||
}
|
||
}
|
||
|
||
// ForceStopCard 强制停机单张卡,不执行正常停机条件判断。
|
||
func (s *StopResumeService) ForceStopCard(ctx context.Context, card *model.IotCard, stopReason string) error {
|
||
if card == nil || card.ID == 0 {
|
||
return errors.New(errors.CodeInvalidParam)
|
||
}
|
||
_, err := s.stopCardWithRetry(ctx, card, stopReason)
|
||
return err
|
||
}
|
||
|
||
// ForceStartCard 强制复机单张卡,不执行正常复机条件判断。
|
||
// 持通道阈值锁的卡必须先拒绝,再走既有保护期一致性检查,避免强行复机锁定期内的卡。
|
||
func (s *StopResumeService) ForceStartCard(ctx context.Context, card *model.IotCard) error {
|
||
if card == nil || card.ID == 0 {
|
||
return errors.New(errors.CodeInvalidParam)
|
||
}
|
||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
|
||
blocked, err := s.channelThresholdBlocked(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
return channelThresholdDenyError()
|
||
}
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": time.Now(),
|
||
"stop_reason": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结保护期复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID, cardSnapshot(card), map[string]any{"requested_network_status": constants.NetworkStatusOnline}, err)
|
||
return err
|
||
}
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if err := s.completeCardCommandAttempt(ctx, attempt, nil, true); err != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "终结保护期复机 Integration Log 失败")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// resumeSingleCard 对单张卡执行复机逻辑
|
||
// 依次检查:持通道阈值锁则拒绝 → 已开机则跳过 → 非轮询停机原因则跳过 → 不满足复机条件则跳过 → 加锁 → 调 Gateway → 更新 DB
|
||
func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) error {
|
||
card, err := s.iotCardStore.GetByID(ctx, cardID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
|
||
|
||
// 持通道阈值锁:周期内拒绝一切复机,记录拒绝事实并保留锁。
|
||
// 该入口被轮询评估、套餐激活/重置与设备维度复机复用,返回 nil 表示本次不动作,
|
||
// 避免让自动链路按失败重试;用户可见入口由 ManualStartCard/ForceStartCard 返回拒绝错误。
|
||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||
if blockErr != nil {
|
||
return blockErr
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
s.logger.Info("卡持通道阈值锁,跳过自动复机",
|
||
zap.Uint("card_id", cardID), zap.String("stop_reason", card.StopReason))
|
||
return nil
|
||
}
|
||
|
||
if card.NetworkStatus == constants.NetworkStatusOnline {
|
||
return nil
|
||
}
|
||
|
||
// 只处理轮询系统引起的停机
|
||
if !isPollingStopReason(card.StopReason) {
|
||
s.logger.Debug("卡非轮询停机原因,不自动复机",
|
||
zap.Uint("card_id", cardID),
|
||
zap.String("stop_reason", card.StopReason))
|
||
return nil
|
||
}
|
||
|
||
// 检查复机条件
|
||
resume, err := s.shouldResume(ctx, card)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !resume {
|
||
return nil
|
||
}
|
||
|
||
// 分布式锁防止同一张卡并发触发多次 Gateway 调用,TTL 30s
|
||
lockKey := constants.RedisCardResumeLockKey(cardID)
|
||
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
|
||
if lockErr != nil {
|
||
s.logger.Warn("获取复机分布式锁失败,跳过本次复机",
|
||
zap.Uint("card_id", cardID),
|
||
zap.Error(lockErr))
|
||
return nil
|
||
}
|
||
if !locked {
|
||
s.logger.Debug("复机操作进行中,跳过", zap.Uint("card_id", cardID))
|
||
return nil
|
||
}
|
||
defer s.redis.Del(ctx, lockKey)
|
||
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
s.logger.Error("调用运营商复机接口失败",
|
||
zap.Uint("card_id", cardID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(err))
|
||
return err
|
||
}
|
||
|
||
now := time.Now()
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": now,
|
||
"stop_reason": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
s.logger.Error("复机 Gateway 成功但 DB 更新失败",
|
||
zap.Uint("card_id", cardID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(err))
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOnline}, err)
|
||
return err
|
||
}
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, logErr, "终结复机 Integration Log 失败")
|
||
}
|
||
|
||
s.logger.Info("卡已自动复机",
|
||
zap.Uint("card_id", cardID),
|
||
zap.String("iccid", card.ICCID))
|
||
|
||
return nil
|
||
}
|
||
|
||
// stopCardWithRetry 调用运营商停机接口(带重试机制),并更新 DB 停机原因。
|
||
//
|
||
// 返回的 outcome 始终携带可回填可靠任务的结果分类(成功/失败/未知)与最后一次尝试的
|
||
// Integration Log 标识;error 与既有语义一致,表示本次停机未成功。
|
||
func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) {
|
||
actionCode, summary := stopAuditAction(ctx, stopReason)
|
||
if s.gatewayClient == nil {
|
||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "",
|
||
cardSnapshot(card), nil, failErr)
|
||
return carrierthresholddomain.CommandOutcome{Result: constants.AuditResultFailed}, failErr
|
||
}
|
||
|
||
s.logger.Info("调用网关停机",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.String("stop_reason", stopReason))
|
||
|
||
seriesKey := cardCommandSeriesKey(ctx)
|
||
attemptObserver := &cardCommandAttemptObserver{
|
||
service: s, card: card, operation: constants.IntegrationOperationGatewayStopCard,
|
||
scene: constants.CardObservationSceneBusinessStop, seriesKey: seriesKey,
|
||
}
|
||
gatewayCtx := gateway.WithAttemptObserver(ctx, attemptObserver)
|
||
var lastErr error
|
||
lastIntegrationID := ""
|
||
for i := 0; i < s.maxRetries; i++ {
|
||
if i > 0 {
|
||
s.logger.Debug("重试调用停机接口",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID))
|
||
time.Sleep(s.retryInterval)
|
||
}
|
||
|
||
callErr := s.gatewayClient.StopCard(gatewayCtx, &gateway.CardOperationReq{CardNo: card.ICCID})
|
||
lastIntegrationID = attemptObserver.lastIntegrationID
|
||
if attemptObserver.recordingErr != nil {
|
||
s.logger.Error("记录停机 Integration Log 失败", zap.String("integration_id", lastIntegrationID), zap.Error(attemptObserver.recordingErr))
|
||
lastErr = attemptObserver.lastCallErr
|
||
if lastErr == nil {
|
||
lastErr = attemptObserver.recordingErr
|
||
}
|
||
// Integration Log 无法终结时仍必须留下本次停机的审计事实(与重试耗尽路径同一写入)。
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, lastErr)
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: attemptObserver.auditResult(lastErr),
|
||
}, lastErr
|
||
}
|
||
if callErr == nil {
|
||
attempt := attemptObserver.successful
|
||
s.logger.Info("网关停机成功",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
|
||
now := time.Now()
|
||
if updateErr := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOffline,
|
||
"stopped_at": now,
|
||
"stop_reason": stopReason,
|
||
}, constants.CardObservationSceneBusinessStop, "offline", uuid.NewString(), actionCode, summary, lastIntegrationID); updateErr != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结停机 Integration Log 失败", zap.String("integration_id", lastIntegrationID), zap.Error(logErr))
|
||
}
|
||
s.logger.Error("停机 Gateway 成功但 DB 更新失败",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(updateErr))
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, updateErr)
|
||
// 运营商已成功但本地回写失败:按结果未知交恢复扫描查询确认。
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: constants.AuditResultUnknown,
|
||
}, updateErr
|
||
}
|
||
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: constants.AuditResultSuccess,
|
||
}, errors.Wrap(errors.CodeDatabaseError, logErr, "终结停机 Integration Log 失败")
|
||
}
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: constants.AuditResultSuccess,
|
||
}, nil
|
||
}
|
||
|
||
lastErr = callErr
|
||
s.logger.Warn("调用停机接口失败,准备重试",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(callErr))
|
||
}
|
||
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, lastErr)
|
||
|
||
return carrierthresholddomain.CommandOutcome{
|
||
IntegrationID: lastIntegrationID, Result: attemptObserver.auditResult(lastErr),
|
||
}, lastErr
|
||
}
|
||
|
||
// resumeCardWithRetry 调用运营商复机接口(带重试机制)。
|
||
// 第二个返回值是最后一次尝试的 Integration Log 标识(无论成功或失败),供调用方回填可靠任务状态。
|
||
func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) {
|
||
if s.gatewayClient == nil {
|
||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||
return nil, "", failErr
|
||
}
|
||
|
||
s.logger.Info("调用网关复机",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
|
||
seriesKey := cardCommandSeriesKey(ctx)
|
||
attemptObserver := &cardCommandAttemptObserver{
|
||
service: s, card: card, operation: constants.IntegrationOperationGatewayStartCard,
|
||
scene: constants.CardObservationSceneBusinessResume, seriesKey: seriesKey,
|
||
}
|
||
gatewayCtx := gateway.WithAttemptObserver(ctx, attemptObserver)
|
||
var lastErr error
|
||
lastIntegrationID := ""
|
||
for i := 0; i < s.maxRetries; i++ {
|
||
if i > 0 {
|
||
s.logger.Debug("重试调用复机接口",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID))
|
||
time.Sleep(s.retryInterval)
|
||
}
|
||
|
||
req := &gateway.CardOperationReq{CardNo: card.ICCID}
|
||
if strings.TrimSpace(card.GatewayExtend) == constants.GatewayCardExtendMachineSeparated {
|
||
req.Extend = constants.GatewayCardStartExtendMachineSeparated
|
||
}
|
||
callErr := s.gatewayClient.StartCard(gatewayCtx, req)
|
||
lastIntegrationID = attemptObserver.lastIntegrationID
|
||
if attemptObserver.recordingErr != nil {
|
||
s.logger.Error("记录复机 Integration Log 失败", zap.String("integration_id", lastIntegrationID), zap.Error(attemptObserver.recordingErr))
|
||
lastErr = attemptObserver.lastCallErr
|
||
if lastErr == nil {
|
||
lastErr = attemptObserver.recordingErr
|
||
}
|
||
// Integration Log 无法终结时仍必须留下本次复机的审计事实(与重试耗尽路径同一写入)。
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOnline}, lastErr)
|
||
return nil, lastIntegrationID, lastErr
|
||
}
|
||
if callErr == nil {
|
||
s.logger.Info("网关复机成功",
|
||
zap.Uint("card_id", card.ID),
|
||
zap.String("iccid", card.ICCID))
|
||
return attemptObserver.successful, lastIntegrationID, nil
|
||
}
|
||
|
||
lastErr = callErr
|
||
s.logger.Warn("调用复机接口失败,准备重试",
|
||
zap.Int("attempt", i+1),
|
||
zap.String("iccid", card.ICCID),
|
||
zap.Error(callErr))
|
||
}
|
||
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
|
||
map[string]any{"requested_network_status": constants.NetworkStatusOnline}, lastErr)
|
||
return nil, lastIntegrationID, lastErr
|
||
}
|
||
|
||
// StartMachineSeparatedCard 对机卡分离停机卡执行复机
|
||
// 调用上游复机前检查本地落库的 Gateway extend,只有“机卡分离停机”才允许复机。
|
||
func (s *StopResumeService) StartMachineSeparatedCard(ctx context.Context, card *model.IotCard) error {
|
||
if card == nil {
|
||
return errors.New(errors.CodeInvalidParam)
|
||
}
|
||
actionCode, summary := constants.AuditActionIotCardOpenAPIStarted, "OpenAPI 恢复 IoT 卡网络"
|
||
|
||
// 持通道阈值锁:周期内拒绝一切复机,记录拒绝事实并保留锁。
|
||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||
if blockErr != nil {
|
||
return blockErr
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
return channelThresholdDenyError()
|
||
}
|
||
|
||
if s.gatewayClient == nil {
|
||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||
return failErr
|
||
}
|
||
|
||
gatewayExtend := strings.TrimSpace(card.GatewayExtend)
|
||
|
||
// 独立卡处于风险停机或已销户状态时,优先返回明确的拒绝原因
|
||
if card.IsStandalone && isRiskGatewayExtend(gatewayExtend) {
|
||
var denyMsg string
|
||
if gatewayExtend == constants.GatewayCardExtendRiskStop {
|
||
denyMsg = "该卡已被运营商风险停机,不允许复机"
|
||
} else {
|
||
denyMsg = "该卡已被运营商销户,不允许复机"
|
||
}
|
||
denyErr := errors.New(errors.CodeForbidden, denyMsg)
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
if gatewayExtend != constants.GatewayCardExtendMachineSeparated {
|
||
denyErr := errors.New(errors.CodeForbidden, constants.AgentOpenAPIResumeOnlyMachineSeparatedMessage)
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "",
|
||
cardSnapshot(card), map[string]any{"gateway_extend": gatewayExtend}, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
wrapErr := errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败,请稍后重试")
|
||
return wrapErr
|
||
}
|
||
|
||
now := time.Now()
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": now,
|
||
"stop_reason": "",
|
||
"gateway_extend": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
wrapErr := errors.Wrap(errors.CodeDatabaseError, err, "更新卡状态失败")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID, cardSnapshot(card), map[string]any{"requested_network_status": constants.NetworkStatusOnline}, wrapErr)
|
||
return wrapErr
|
||
}
|
||
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, logErr, "终结 OpenAPI 复机 Integration Log 失败")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ManualStopCard 手动停机单张卡(通过ICCID)
|
||
func (s *StopResumeService) ManualStopCard(ctx context.Context, iccid string) error {
|
||
card, err := s.iotCardStore.GetByICCID(ctx, iccid)
|
||
if err != nil {
|
||
return errors.New(errors.CodeNotFound, "卡不存在")
|
||
}
|
||
actionCode, summary := stopAuditAction(ctx, constants.StopReasonManual)
|
||
|
||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||
if err != nil {
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, err)
|
||
return err
|
||
}
|
||
if !realnameOK {
|
||
denyErr := errors.New(errors.CodeForbidden, "卡未实名,无法操作")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
// 检查绑定设备是否在复机保护期
|
||
if s.deviceSimBindingStore != nil && s.redis != nil {
|
||
binding, bindErr := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
|
||
if bindErr == nil && binding != nil {
|
||
exists, _ := s.redis.Exists(ctx, constants.RedisDeviceProtectKey(binding.DeviceID, "start")).Result()
|
||
if exists > 0 {
|
||
denyErr := errors.New(errors.CodeForbidden, "设备复机保护期内,禁止停机")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "",
|
||
cardSnapshot(card), map[string]any{"device_id": binding.DeviceID}, denyErr)
|
||
return denyErr
|
||
}
|
||
} else if bindErr != nil && !stderrors.Is(bindErr, gorm.ErrRecordNotFound) {
|
||
wrapErr := errors.Wrap(errors.CodeInternalError, bindErr, "查询卡绑定关系失败")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, wrapErr)
|
||
return wrapErr
|
||
}
|
||
}
|
||
|
||
if _, err := s.stopCardWithRetry(ctx, card, constants.StopReasonManual); err != nil {
|
||
return err
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// ManualStartCard 手动复机单张卡(通过ICCID)
|
||
func (s *StopResumeService) ManualStartCard(ctx context.Context, iccid string) error {
|
||
card, err := s.iotCardStore.GetByICCID(ctx, iccid)
|
||
if err != nil {
|
||
return errors.New(errors.CodeNotFound, "卡不存在")
|
||
}
|
||
actionCode, summary := constants.AuditActionIotCardManualStarted, "人工恢复 IoT 卡网络"
|
||
|
||
// 持通道阈值锁:周期内拒绝一切复机,记录拒绝事实并保留锁。
|
||
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
|
||
if blockErr != nil {
|
||
return blockErr
|
||
}
|
||
if blocked {
|
||
s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
|
||
return channelThresholdDenyError()
|
||
}
|
||
|
||
// 独立卡处于风险停机或已销户状态时,拒绝复机
|
||
if card.IsStandalone && isRiskGatewayExtend(card.GatewayExtend) {
|
||
var denyMsg string
|
||
if strings.TrimSpace(card.GatewayExtend) == constants.GatewayCardExtendRiskStop {
|
||
denyMsg = "该卡已被运营商风险停机,不允许复机"
|
||
} else {
|
||
denyMsg = "该卡已被运营商销户,不允许复机"
|
||
}
|
||
denyErr := errors.New(errors.CodeForbidden, denyMsg)
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
realnameOK, err := s.isRealnameOK(ctx, card)
|
||
if err != nil {
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, err)
|
||
return err
|
||
}
|
||
if !realnameOK {
|
||
denyErr := errors.New(errors.CodeForbidden, "卡未实名,无法操作")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "", cardSnapshot(card), nil, denyErr)
|
||
return denyErr
|
||
}
|
||
|
||
// 检查绑定设备是否在停机保护期
|
||
if s.deviceSimBindingStore != nil && s.redis != nil {
|
||
binding, bindErr := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
|
||
if bindErr == nil && binding != nil {
|
||
exists, _ := s.redis.Exists(ctx, constants.RedisDeviceProtectKey(binding.DeviceID, "stop")).Result()
|
||
if exists > 0 {
|
||
denyErr := errors.New(errors.CodeForbidden, "设备停机保护期内,禁止复机")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"被拒绝", constants.AuditResultDenied, "",
|
||
cardSnapshot(card), map[string]any{"device_id": binding.DeviceID}, denyErr)
|
||
return denyErr
|
||
}
|
||
} else if bindErr != nil && !stderrors.Is(bindErr, gorm.ErrRecordNotFound) {
|
||
wrapErr := errors.Wrap(errors.CodeInternalError, bindErr, "查询卡绑定关系失败")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, wrapErr)
|
||
return wrapErr
|
||
}
|
||
}
|
||
|
||
attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
|
||
if err != nil {
|
||
wrapErr := errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败,请稍后重试")
|
||
return wrapErr
|
||
}
|
||
|
||
now := time.Now()
|
||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||
"network_status": constants.NetworkStatusOnline,
|
||
"resumed_at": now,
|
||
"stop_reason": "",
|
||
}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
|
||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||
}
|
||
wrapErr := errors.Wrap(errors.CodeDatabaseError, err, "更新卡状态失败")
|
||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||
attempt.log.IntegrationID, cardSnapshot(card), map[string]any{"requested_network_status": constants.NetworkStatusOnline}, wrapErr)
|
||
return wrapErr
|
||
}
|
||
|
||
s.reschedulePolling(ctx, card.ID)
|
||
if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, logErr, "终结人工复机 Integration Log 失败")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *StopResumeService) updateCardAndAppendNetworkSeries(
|
||
ctx context.Context,
|
||
card *model.IotCard,
|
||
fields map[string]any,
|
||
scene, expected, operationID, actionCode, summary, integrationID string,
|
||
) error {
|
||
if s.db == nil || s.observationSeriesEvents == nil || card == nil || card.ID == 0 {
|
||
return errors.New(errors.CodeInternalError, "停复机观测 Outbox 能力未配置")
|
||
}
|
||
requestID := requestIDFromContext(ctx)
|
||
if requestID == "" {
|
||
requestID = operationID
|
||
}
|
||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
if err := tx.Model(&model.IotCard{}).Where("id = ?", card.ID).Updates(fields).Error; err != nil {
|
||
return errors.Wrap(errors.CodeDatabaseError, err, "更新卡停复机状态失败")
|
||
}
|
||
if err := s.appendCardCommandAudit(ctx, tx, card, actionCode, summary, constants.AuditResultSuccess,
|
||
integrationID,
|
||
map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
|
||
fields, nil); err != nil {
|
||
return err
|
||
}
|
||
if cardObservationApp.IsSeriesTriggerSuppressed(ctx) {
|
||
return nil
|
||
}
|
||
return s.observationSeriesEvents.AppendSeriesRequested(ctx, tx, cardObservationApp.SeriesRequestedEvent{
|
||
EventID: outboxid.Stable("card-observation:network-command:", operationID),
|
||
Scene: scene, ResourceType: constants.CardObservationResourceTypeCard, ResourceID: card.ID,
|
||
SyncTypes: []string{constants.CardObservationSyncTypeNetwork}, ExpectedValue: expected,
|
||
Source: constants.CardObservationSourceBusinessEvent, OccurredAt: time.Now().UTC(),
|
||
RequestID: requestID, CorrelationID: requestID,
|
||
})
|
||
})
|
||
}
|
||
|
||
// invalidatePollingCache 删除轮询卡缓存,下次轮询自动从 DB 重建
|
||
func (s *StopResumeService) invalidatePollingCache(ctx context.Context, cardID uint) {
|
||
if s.redis == nil {
|
||
return
|
||
}
|
||
key := constants.RedisPollingCardInfoKey(cardID)
|
||
if err := s.redis.Del(ctx, key).Err(); err != nil {
|
||
s.logger.Warn("删除轮询卡缓存失败", zap.Uint("card_id", cardID))
|
||
}
|
||
}
|
||
|
||
// reschedulePolling 按卡最新状态重排轮询队列。
|
||
// 未注入生命周期回调时退化为只清缓存,保持原有行为。
|
||
func (s *StopResumeService) reschedulePolling(ctx context.Context, cardID uint) {
|
||
if s.pollingCallback == nil {
|
||
s.invalidatePollingCache(ctx, cardID)
|
||
return
|
||
}
|
||
s.pollingCallback.OnCardStatusChanged(ctx, cardID)
|
||
}
|