feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled

自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭):
- 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机
- 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制
- 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认
- 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后
- 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估
- 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令
- 迁移 000255 插入开关与告警、退避参数

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-10-26 00:36:26 +09:00
parent 1369fed642
commit 934a3a27a1
31 changed files with 2275 additions and 117 deletions

View File

@@ -0,0 +1,190 @@
package iot_card
import (
"context"
stderrors "errors"
"time"
"github.com/google/uuid"
"go.uber.org/zap"
"gorm.io/gorm"
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/gateway"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// 停复机指令执行器使用的服务能力(add-stop-resume-command-queue)。执行器只调用一次网关,不 sleep 重试。
// CommandDecision 是执行前按指令场景重新判断的结果。
type CommandDecision struct {
Card *model.IotCard
Action string // 空表示条件不再满足或已是目标状态,结束指令
Reason string
Gone bool // 卡已不存在
}
// DecideCommand 执行前按指令记录的场景重新判断该卡(只看本卡,不扩散到设备)。
// 复机持通道阈值锁时视为不再满足。
func (s *StopResumeService) DecideCommand(ctx context.Context, cardID uint, scenes []string) (CommandDecision, error) {
card, err := s.iotCardStore.GetByID(ctx, cardID)
if err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return CommandDecision{Gone: true}, nil
}
return CommandDecision{}, errors.Wrap(errors.CodeDatabaseError, err, "读取卡失败")
}
decision, err := s.decideCard(ctx, card, scenes)
if err != nil {
return CommandDecision{Card: card}, err
}
if decision.Action == constants.StopResumeActionStart {
blocked, blockErr := s.channelThresholdBlocked(ctx, card)
if blockErr != nil {
return CommandDecision{Card: card}, blockErr
}
if blocked {
return CommandDecision{Card: card}, nil
}
}
return CommandDecision{Card: card, Action: decision.Action, Reason: decision.Reason}, nil
}
// CommandCallResult 是一次停复机网关调用的结果。
type CommandCallResult struct {
// Unknown 为 true 表示结果未知:超时、连接失败,或上游已成功但本地回写失败。
Unknown bool
}
// ExecuteCommand 对卡调用一次停机或复机(限流等待模式由调用方 ctx 声明),成功后按现有回写更新卡状态、
// 停机原因与网络状态序列,并写停复机记录。返回的 error 为网关错误或回写错误,由执行器按错误分类处理。
func (s *StopResumeService) ExecuteCommand(ctx context.Context, card *model.IotCard, action, reason string) (CommandCallResult, error) {
if card == nil {
return CommandCallResult{}, errors.New(errors.CodeInvalidParam)
}
if action == constants.StopResumeActionStop {
outcome, err := s.stopCardAttempts(ctx, card, reason, 1)
return CommandCallResult{Unknown: err != nil && outcome.Result == constants.AuditResultUnknown && !gateway.IsRateLimited(err)}, err
}
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
attempt, integrationID, err := s.resumeCardAttempts(ctx, card, actionCode, summary, 1)
if err != nil {
return CommandCallResult{Unknown: isUnknownGatewayError(err)}, err
}
if err := s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary); err != nil {
return CommandCallResult{Unknown: true}, err
}
return CommandCallResult{}, nil
}
// isUnknownGatewayError 判断网关错误是否为结果未知(超时、连接失败):请求可能已到达上游。
func isUnknownGatewayError(err error) bool {
gwErr, ok := gateway.AsGatewayError(err)
if !ok {
return false
}
return gwErr.Category == constants.GatewayErrorTimeout || gwErr.Category == constants.GatewayErrorNetwork
}
// QueryCommandCardStatus 结果未知后的确认查询:只查一次卡状态(受限流、不受单卡最小重试间隔约束)。
// 查询失败时 known=false 并返回错误,调用方保持待确认、不发起停复机。
func (s *StopResumeService) QueryCommandCardStatus(ctx context.Context, card *model.IotCard) (int, bool, error) {
if s.gatewayClient == nil {
return 0, false, errors.New(errors.CodeInternalError, "Gateway 未配置,无法查询卡状态")
}
queryCtx := gateway.WithCarrier(ctx, card.CarrierID)
response, err := s.gatewayClient.QueryCardStatus(queryCtx, &gateway.CardStatusReq{CardNo: card.ICCID})
if err != nil {
return 0, false, err
}
status, known := carddomain.MapGatewayNetworkStatus(response.CardStatus, response.Extend)
return status, known, nil
}
// ConfirmCommandState 按已确认的上游结果(确认查询已是目标状态、上游返回已是目标状态、停机遇上游终态)回写卡状态。
func (s *StopResumeService) ConfirmCommandState(ctx context.Context, card *model.IotCard, action, reason string) error {
if card == nil {
return errors.New(errors.CodeInvalidParam)
}
actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络(确认)"
scene, expected := constants.CardObservationSceneBusinessResume, "online"
fields := map[string]any{"network_status": constants.NetworkStatusOnline, "resumed_at": time.Now(), "stop_reason": ""}
if action == constants.StopResumeActionStop {
actionCode, summary = stopAuditAction(ctx, reason)
summary += "(确认)"
scene, expected = constants.CardObservationSceneBusinessStop, "offline"
fields = map[string]any{"network_status": constants.NetworkStatusOffline, "stopped_at": time.Now(), "stop_reason": reason}
}
if err := s.updateCardAndAppendNetworkSeries(ctx, card, fields, scene, expected, uuid.NewString(), actionCode, summary, ""); err != nil {
return err
}
s.reschedulePolling(ctx, card.ID)
return nil
}
// EvaluateDeviceCommands 设备维度判断:读取设备下各卡最新数据,按统一判断函数(各卡按所属运营商账户判断实名,
// 暂停轮询按场景集合判断)为需要动作的卡生成指令,进入各卡所属账户的停复机队列。只读数据库与写共享缓存,不调网关。
func (s *StopResumeService) EvaluateDeviceCommands(ctx context.Context, deviceID uint, scenes []string) error {
if s.commandQueue == nil {
return errors.New(errors.CodeInternalError, "停复机指令队列未配置")
}
cards, err := s.deviceCards(ctx, deviceID)
if err != nil {
return err
}
var failed []error
for _, card := range cards {
decision, decideErr := s.decideCard(ctx, card, scenes)
if decideErr != nil {
failed = append(failed, decideErr)
continue
}
if decision.Action == "" {
s.markNotRealnameIfBlocked(ctx, card, scenes)
continue
}
decision.DeviceID = 0
if writeErr := s.writeCardCommand(ctx, card, decision, scenes, "device_eval"); writeErr != nil {
failed = append(failed, writeErr)
}
}
if len(failed) > 0 {
return errors.Wrap(errors.CodeInternalError, stderrors.Join(failed...), "设备判断存在未写入指令的卡")
}
return nil
}
// markNotRealnameIfBlocked 设备下停机卡套餐满足但实名不满足时,把停机原因更新为未实名(与旧设备复机路径一致)。
func (s *StopResumeService) markNotRealnameIfBlocked(ctx context.Context, card *model.IotCard, scenes []string) {
if card.NetworkStatus != constants.NetworkStatusOffline || card.StopReason == constants.StopReasonNotRealname {
return
}
if suspended, err := s.autoControlSuspended(ctx, card); err != nil || suspended {
return
}
hasPackage, err := s.hasValidPackage(ctx, card)
if err != nil || !hasPackage {
return
}
if exhausted, err := s.isTrafficExhausted(ctx, card); err != nil || exhausted {
return
}
realnameOK, err := s.isRealnameOK(ctx, card)
if err != nil || realnameOK {
return
}
if err := s.updateCardStopReasonWithAudit(ctx, card, constants.StopReasonNotRealname); err != nil {
s.logger.Warn("更新未实名卡停机原因失败", zap.Uint("card_id", card.ID), zap.Strings("scenes", scenes), zap.Error(err))
}
}
// LegacyDeviceOpLocked 判断旧路径是否正在对该设备执行停复机(持设备锁);设备判断须推后,不并行执行。
func (s *StopResumeService) LegacyDeviceOpLocked(ctx context.Context, deviceID uint) bool {
if s.redis == nil {
return false
}
exists, err := s.redis.Exists(ctx, constants.RedisPollingDeviceOpLockKey(deviceID)).Result()
return err == nil && exists > 0
}