feat: 停复机指令队列与设备维度判断
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
Some checks failed
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Has been cancelled
自动停复机统一经停复机指令队列执行(全局开关 polling.stop_resume.queue_enabled,默认关闭): - 统一判断函数 decideCard,新旧路径共用;暂停轮询的卡在套餐到期、退款场景照常停机 - 同卡一条指令、版本号覆盖;领取校验单卡最小重试间隔;先于卡任务领取,不受 scheduler_mode 限制 - 执行器只调用一次网关,按错误分类退避与告警,结果未知在队列内确认 - 设备维度判断合并请求与场景,版本变化再判断,旧路径持设备锁时推后 - 套餐激活、重置、订单、流量扣减、套餐到期、退款、实名管控修改改为请求评估 - 监控增加停复机队列与近 1 小时执行计数,单卡轮询状态返回当前指令 - 迁移 000255 插入开关与告警、退避参数 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -60,6 +60,8 @@ type StopResumeService struct {
|
||||
stopResumeRecorder *gatewaylimit.StopResumeRecorder
|
||||
// terminalPolling 停复机命中"自动关闭轮询"终态规则时关闭该卡轮询;未注入时按普通失败处理。
|
||||
terminalPolling *gatewaylimit.TerminalPollingDisabler
|
||||
// commandQueue 停复机指令队列;开关开启时自动入口只请求评估。
|
||||
commandQueue StopResumeCommandQueue
|
||||
}
|
||||
|
||||
// SetStopResumeRecorder 注入停复机记录写入器。
|
||||
@@ -128,69 +130,36 @@ func (s *StopResumeService) SetPollingCallback(callback PollingCallback) {
|
||||
s.pollingCallback = callback
|
||||
}
|
||||
|
||||
// EvaluateAndAct 停复机统一入口
|
||||
// 根据卡的当前网络状态,自动判断并执行停机或复机操作
|
||||
// 设备/单卡维度通过 card.IsStandalone 推导(false 则为设备维度,停/复机覆盖设备下所有卡)
|
||||
// EvaluateAndAct 停复机统一入口:按统一判断函数决定停机或复机。
|
||||
// 停复机出口开关开启时只请求评估(生成指令或请求设备判断),不当场调用网关;关闭时当场执行。
|
||||
// 设备/单卡维度通过 card.IsStandalone 推导(false 则为设备维度,停/复机覆盖设备下所有卡)。触发场景见 WithStopResumeScene。
|
||||
func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error {
|
||||
// 总开关开启时,卡或所在设备暂停轮询的不自动停复机。
|
||||
suspended, err := s.autoControlSuspended(ctx, card)
|
||||
scenes := stopResumeScenes(ctx)
|
||||
decision, err := s.decideCard(ctx, card, scenes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if suspended {
|
||||
return nil
|
||||
// 无有效套餐且存在待生效套餐使用记录:仅由普通套餐轮询来源追加优先轮询请求。
|
||||
if containsStopReason(decision.Reasons, constants.StopReasonNoPackage) {
|
||||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch card.NetworkStatus {
|
||||
case constants.NetworkStatusOnline:
|
||||
// 卡在线,检查是否需要停机
|
||||
reasons, err := s.checkStopReasons(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
if s.commandQueueEnabled(ctx) {
|
||||
return s.submitDecision(ctx, card, decision, scenes)
|
||||
}
|
||||
switch decision.Action {
|
||||
case constants.StopResumeActionStop:
|
||||
if decision.DeviceID > 0 {
|
||||
return s.stopDeviceCards(ctx, decision.DeviceID, decision.Reason)
|
||||
}
|
||||
if len(reasons) == 0 {
|
||||
return nil
|
||||
}
|
||||
// 无有效套餐且存在待生效套餐使用记录:仅由普通套餐轮询来源追加优先轮询请求。
|
||||
if containsStopReason(reasons, constants.StopReasonNoPackage) {
|
||||
if err := s.appendNoValidPackagePriority(ctx, card); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// 取最高优先级原因(第一个)
|
||||
primaryReason := reasons[0]
|
||||
if !card.IsStandalone && isDeviceScopeReason(primaryReason) {
|
||||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||||
if lookupErr != nil {
|
||||
s.logger.Error("查询设备绑定关系失败,降级为单卡停机",
|
||||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||||
// 降级:查不到绑定时按单卡处理
|
||||
} else if bound {
|
||||
return s.stopDeviceCards(ctx, deviceID, primaryReason)
|
||||
}
|
||||
}
|
||||
_, stopErr := s.stopCardWithRetry(ctx, card, primaryReason)
|
||||
_, stopErr := s.stopCardWithRetry(ctx, card, decision.Reason)
|
||||
return stopErr
|
||||
|
||||
case constants.NetworkStatusOffline:
|
||||
// 卡停机,检查是否可以复机:不看停机原因,是否复机只由本系统规则决定
|
||||
shouldResume, err := s.shouldResume(ctx, card)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !shouldResume {
|
||||
return nil
|
||||
}
|
||||
if !card.IsStandalone {
|
||||
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
|
||||
if lookupErr != nil {
|
||||
s.logger.Error("查询设备绑定关系失败,降级为单卡复机",
|
||||
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
|
||||
} else if bound {
|
||||
return s.resumeDeviceCards(ctx, deviceID)
|
||||
}
|
||||
case constants.StopResumeActionStart:
|
||||
if decision.DeviceID > 0 {
|
||||
return s.resumeDeviceCards(ctx, decision.DeviceID)
|
||||
}
|
||||
return s.resumeSingleCard(ctx, card.ID)
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
@@ -414,11 +383,11 @@ func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint,
|
||||
|
||||
var cardErrors []error
|
||||
for _, card := range cards {
|
||||
// 总开关开启时,暂停轮询的卡不随设备自动停机。
|
||||
// 暂停轮询的卡不随设备自动停机;套餐失效类场景(套餐到期、退款)照常停机。
|
||||
if suspended, suspendErr := s.autoControlSuspended(ctx, card); suspendErr != nil {
|
||||
cardErrors = append(cardErrors, suspendErr)
|
||||
continue
|
||||
} else if suspended {
|
||||
} else if suspended && !(stopReason == constants.StopReasonNoPackage && constants.StopResumeSceneAllowsSuspendedStop(stopResumeScenes(ctx))) {
|
||||
continue
|
||||
}
|
||||
if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
|
||||
@@ -503,6 +472,9 @@ func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) e
|
||||
// ResumeCardIfStopped 套餐激活或流量重置后自动复机入口
|
||||
// 支持 iot_card 和 device 两种载体类型
|
||||
func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error {
|
||||
if s.commandQueueEnabled(ctx) {
|
||||
return s.RequestAssetEvaluation(ctx, carrierType, carrierID, stopResumeScenes(ctx)[0], "resume_if_stopped")
|
||||
}
|
||||
switch carrierType {
|
||||
case "iot_card":
|
||||
return s.resumeSingleCard(ctx, carrierID)
|
||||
@@ -582,7 +554,12 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
zap.Error(err))
|
||||
return err
|
||||
}
|
||||
return s.completeAutoResume(ctx, card, attempt, integrationID, actionCode, summary)
|
||||
}
|
||||
|
||||
// completeAutoResume 自动复机网关成功后回写在线状态、观测序列与统一审计,并重排轮询。
|
||||
// 回写失败时返回错误,调用方按结果未知处理。
|
||||
func (s *StopResumeService) completeAutoResume(ctx context.Context, card *model.IotCard, attempt *gatewayAttempt, integrationID, actionCode, summary string) error {
|
||||
now := time.Now()
|
||||
if err := s.updateCardAndAppendNetworkSeries(ctx, card, map[string]any{
|
||||
"network_status": constants.NetworkStatusOnline,
|
||||
@@ -593,7 +570,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
|
||||
}
|
||||
s.logger.Error("复机 Gateway 成功但 DB 更新失败",
|
||||
zap.Uint("card_id", cardID),
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.String("iccid", card.ICCID),
|
||||
zap.Error(err))
|
||||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"结果待核对", constants.AuditResultUnknown,
|
||||
@@ -608,7 +585,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
}
|
||||
|
||||
s.logger.Info("卡已自动复机",
|
||||
zap.Uint("card_id", cardID),
|
||||
zap.Uint("card_id", card.ID),
|
||||
zap.String("iccid", card.ICCID))
|
||||
|
||||
return nil
|
||||
@@ -619,6 +596,11 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
|
||||
// 返回的 outcome 始终携带可回填可靠任务的结果分类(成功/失败/未知)与最后一次尝试的
|
||||
// Integration Log 标识;error 与既有语义一致,表示本次停机未成功。
|
||||
func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) {
|
||||
return s.stopCardAttempts(ctx, card, stopReason, s.maxRetries)
|
||||
}
|
||||
|
||||
// stopCardAttempts 调用运营商停机接口最多 attempts 次(停复机指令执行器只调用一次),成功后回写停机状态。
|
||||
func (s *StopResumeService) stopCardAttempts(ctx context.Context, card *model.IotCard, stopReason string, attempts int) (carrierthresholddomain.CommandOutcome, error) {
|
||||
actionCode, summary := stopAuditAction(ctx, stopReason)
|
||||
if s.gatewayClient == nil {
|
||||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||||
@@ -641,7 +623,7 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
|
||||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||||
var lastErr error
|
||||
lastIntegrationID := ""
|
||||
for i := 0; i < s.maxRetries; i++ {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
s.logger.Debug("重试调用停机接口",
|
||||
zap.Int("attempt", i+1),
|
||||
@@ -732,6 +714,11 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
|
||||
// resumeCardWithRetry 调用运营商复机接口(带重试机制)。
|
||||
// 第二个返回值是最后一次尝试的 Integration Log 标识(无论成功或失败),供调用方回填可靠任务状态。
|
||||
func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) {
|
||||
return s.resumeCardAttempts(ctx, card, actionCode, summary, s.maxRetries)
|
||||
}
|
||||
|
||||
// resumeCardAttempts 调用运营商复机接口最多 attempts 次,不回写卡状态。
|
||||
func (s *StopResumeService) resumeCardAttempts(ctx context.Context, card *model.IotCard, actionCode, summary string, attempts int) (*gatewayAttempt, string, error) {
|
||||
if s.gatewayClient == nil {
|
||||
failErr := errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
|
||||
s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
|
||||
@@ -751,7 +738,7 @@ func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model
|
||||
gatewayCtx := gateway.WithCarrier(gateway.WithAttemptObserver(ctx, attemptObserver), card.CarrierID)
|
||||
var lastErr error
|
||||
lastIntegrationID := ""
|
||||
for i := 0; i < s.maxRetries; i++ {
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
s.logger.Debug("重试调用复机接口",
|
||||
zap.Int("attempt", i+1),
|
||||
|
||||
Reference in New Issue
Block a user