feat(通道流量阈值): AUG26-011 运营商通道流量阈值达量停机与周期复机

2026-09-16 15:54:49 +08:00
parent 41722760b1
commit 59b3df868a
36 changed files with 2431 additions and 93 deletions
--- a/internal/service/iot_card/stop_resume_service.go
+++ b/internal/service/iot_card/stop_resume_service.go
@@ -7,6 +7,7 @@ import (
 	"time"

 	cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
+	carrierthresholddomain "github.com/break/junhong_cmp_fiber/internal/domain/carrierthreshold"
 	"github.com/google/uuid"
 	"github.com/redis/go-redis/v9"
 	"go.uber.org/zap"
@@ -53,6 +54,8 @@ type StopResumeService struct {

 	maxRetries    int
 	retryInterval time.Duration
+	// channelThresholdGuard 是可选的通道阈值持锁判定能力；未注入时不做持锁拒绝。
+	channelThresholdGuard ChannelThresholdLockGuard
 }

 // SetObservationSeriesEventWriter 注入停复机成功观测序列 Outbox Writer。
@@ -120,7 +123,8 @@ func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotC
 				return s.stopDeviceCards(ctx, deviceID, primaryReason)
 			}
 		}
-		return s.stopCardWithRetry(ctx, card, primaryReason)
+		_, stopErr := s.stopCardWithRetry(ctx, card, primaryReason)
+		return stopErr

 	case constants.NetworkStatusOffline:
 		// 卡停机，检查是否可以复机
@@ -353,7 +357,7 @@ func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint,

 	var cardErrors []error
 	for _, card := range cards {
-		if stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
+		if _, stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
 			cardErrors = append(cardErrors, stopErr)
 			s.logger.Warn("设备卡停机失败，继续处理其他卡",
 				zap.Uint("device_id", deviceID),
@@ -450,16 +454,26 @@ func (s *StopResumeService) ForceStopCard(ctx context.Context, card *model.IotCa
 	if card == nil || card.ID == 0 {
 		return errors.New(errors.CodeInvalidParam)
 	}
-	return s.stopCardWithRetry(ctx, card, stopReason)
+	_, err := s.stopCardWithRetry(ctx, card, stopReason)
+	return err
 }

 // ForceStartCard 强制复机单张卡，不执行正常复机条件判断。
+// 持通道阈值锁的卡必须先拒绝，再走既有保护期一致性检查，避免强行复机锁定期内的卡。
 func (s *StopResumeService) ForceStartCard(ctx context.Context, card *model.IotCard) error {
 	if card == nil || card.ID == 0 {
 		return errors.New(errors.CodeInvalidParam)
 	}
 	actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
-	attempt, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
+	blocked, err := s.channelThresholdBlocked(ctx, card)
+	if err != nil {
+		return err
+	}
+	if blocked {
+		s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
+		return channelThresholdDenyError()
+	}
+	attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
 	if err != nil {
 		return err
 	}
@@ -467,7 +481,7 @@ func (s *StopResumeService) ForceStartCard(ctx context.Context, card *model.IotC
 		"network_status": constants.NetworkStatusOnline,
 		"resumed_at":     time.Now(),
 		"stop_reason":    "",
-	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, attempt.log.IntegrationID); err != nil {
+	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
 		if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
 			s.logger.Error("终结保护期复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
 		}
@@ -483,13 +497,29 @@ func (s *StopResumeService) ForceStartCard(ctx context.Context, card *model.IotC
 }

 // resumeSingleCard 对单张卡执行复机逻辑
-// 依次检查：已开机则跳过 → 非轮询停机原因则跳过 → 不满足复机条件则跳过 → 加锁 → 调 Gateway → 更新 DB
+// 依次检查：持通道阈值锁则拒绝 → 已开机则跳过 → 非轮询停机原因则跳过 → 不满足复机条件则跳过 → 加锁 → 调 Gateway → 更新 DB
 func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) error {
 	card, err := s.iotCardStore.GetByID(ctx, cardID)
 	if err != nil {
 		return err
 	}

+	actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
+
+	// 持通道阈值锁：周期内拒绝一切复机，记录拒绝事实并保留锁。
+	// 该入口被轮询评估、套餐激活/重置与设备维度复机复用，返回 nil 表示本次不动作，
+	// 避免让自动链路按失败重试；用户可见入口由 ManualStartCard/ForceStartCard 返回拒绝错误。
+	blocked, blockErr := s.channelThresholdBlocked(ctx, card)
+	if blockErr != nil {
+		return blockErr
+	}
+	if blocked {
+		s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
+		s.logger.Info("卡持通道阈值锁，跳过自动复机",
+			zap.Uint("card_id", cardID), zap.String("stop_reason", card.StopReason))
+		return nil
+	}
+
 	if card.NetworkStatus == constants.NetworkStatusOnline {
 		return nil
 	}
@@ -526,8 +556,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
 	}
 	defer s.redis.Del(ctx, lockKey)

-	actionCode, summary := constants.AuditActionIotCardAutoStarted, "自动恢复 IoT 卡网络"
-	attempt, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
+	attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
 	if err != nil {
 		s.logger.Error("调用运营商复机接口失败",
 			zap.Uint("card_id", cardID),
@@ -541,7 +570,7 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
 		"network_status": constants.NetworkStatusOnline,
 		"resumed_at":     now,
 		"stop_reason":    "",
-	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, attempt.log.IntegrationID); err != nil {
+	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
 		if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
 			s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
 		}
@@ -567,14 +596,17 @@ func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) e
 	return nil
 }

-// stopCardWithRetry 调用运营商停机接口（带重试机制），并更新 DB 停机原因
-func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) error {
+// stopCardWithRetry 调用运营商停机接口（带重试机制），并更新 DB 停机原因。
+//
+// 返回的 outcome 始终携带可回填可靠任务的结果分类（成功/失败/未知）与最后一次尝试的
+// Integration Log 标识；error 与既有语义一致，表示本次停机未成功。
+func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) (carrierthresholddomain.CommandOutcome, error) {
 	actionCode, summary := stopAuditAction(ctx, stopReason)
 	if s.gatewayClient == nil {
 		failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用")
 		s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "",
 			cardSnapshot(card), nil, failErr)
-		return failErr
+		return carrierthresholddomain.CommandOutcome{Result: constants.AuditResultFailed}, failErr
 	}

 	s.logger.Info("调用网关停机",
@@ -606,7 +638,9 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
 			if lastErr == nil {
 				lastErr = attemptObserver.recordingErr
 			}
-			break
+			return carrierthresholddomain.CommandOutcome{
+				IntegrationID: lastIntegrationID, Result: attemptObserver.auditResult(lastErr),
+			}, lastErr
 		}
 		if callErr == nil {
 			attempt := attemptObserver.successful
@@ -631,14 +665,21 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
 					lastIntegrationID,
 					map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
 					map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, updateErr)
-				return updateErr
+				// 运营商已成功但本地回写失败：按结果未知交恢复扫描查询确认。
+				return carrierthresholddomain.CommandOutcome{
+					IntegrationID: lastIntegrationID, Result: constants.AuditResultUnknown,
+				}, updateErr
 			}

 			s.reschedulePolling(ctx, card.ID)
 			if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, true); logErr != nil {
-				return errors.Wrap(errors.CodeDatabaseError, logErr, "终结停机 Integration Log 失败")
+				return carrierthresholddomain.CommandOutcome{
+					IntegrationID: lastIntegrationID, Result: constants.AuditResultSuccess,
+				}, errors.Wrap(errors.CodeDatabaseError, logErr, "终结停机 Integration Log 失败")
 			}
-			return nil
+			return carrierthresholddomain.CommandOutcome{
+				IntegrationID: lastIntegrationID, Result: constants.AuditResultSuccess,
+			}, nil
 		}

 		lastErr = callErr
@@ -652,15 +693,18 @@ func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.I
 		map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason},
 		map[string]any{"requested_network_status": constants.NetworkStatusOffline, "stop_reason": stopReason}, lastErr)

-	return lastErr
+	return carrierthresholddomain.CommandOutcome{
+		IntegrationID: lastIntegrationID, Result: attemptObserver.auditResult(lastErr),
+	}, lastErr
 }

 // resumeCardWithRetry 调用运营商复机接口（带重试机制）。
-func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, error) {
+// 第二个返回值是最后一次尝试的 Integration Log 标识（无论成功或失败），供调用方回填可靠任务状态。
+func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard, actionCode, summary string) (*gatewayAttempt, string, error) {
 	if s.gatewayClient == nil {
 		failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用")
 		s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
-		return nil, failErr
+		return nil, "", failErr
 	}

 	s.logger.Info("调用网关复机",
@@ -695,13 +739,13 @@ func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model
 			if lastErr == nil {
 				lastErr = attemptObserver.recordingErr
 			}
-			break
+			return nil, lastIntegrationID, lastErr
 		}
 		if callErr == nil {
 			s.logger.Info("网关复机成功",
 				zap.Uint("card_id", card.ID),
 				zap.String("iccid", card.ICCID))
-			return attemptObserver.successful, nil
+			return attemptObserver.successful, lastIntegrationID, nil
 		}

 		lastErr = callErr
@@ -714,7 +758,7 @@ func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model
 	s.recordCardCommandAudit(ctx, card, actionCode, summary+"未完成", attemptObserver.auditResult(lastErr), lastIntegrationID,
 		map[string]any{"network_status": card.NetworkStatus, "stop_reason": card.StopReason, "gateway_extend": card.GatewayExtend},
 		map[string]any{"requested_network_status": constants.NetworkStatusOnline}, lastErr)
-	return nil, lastErr
+	return nil, lastIntegrationID, lastErr
 }

 // StartMachineSeparatedCard 对机卡分离停机卡执行复机
@@ -724,6 +768,17 @@ func (s *StopResumeService) StartMachineSeparatedCard(ctx context.Context, card
 		return errors.New(errors.CodeInvalidParam)
 	}
 	actionCode, summary := constants.AuditActionIotCardOpenAPIStarted, "OpenAPI 恢复 IoT 卡网络"
+
+	// 持通道阈值锁：周期内拒绝一切复机，记录拒绝事实并保留锁。
+	blocked, blockErr := s.channelThresholdBlocked(ctx, card)
+	if blockErr != nil {
+		return blockErr
+	}
+	if blocked {
+		s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
+		return channelThresholdDenyError()
+	}
+
 	if s.gatewayClient == nil {
 		failErr := errors.New(errors.CodeInternalError, "Gateway 未配置，停复机操作不可用")
 		s.recordCardCommandAudit(ctx, card, actionCode, summary+"失败", constants.AuditResultFailed, "", cardSnapshot(card), nil, failErr)
@@ -752,7 +807,7 @@ func (s *StopResumeService) StartMachineSeparatedCard(ctx context.Context, card
 		return denyErr
 	}

-	attempt, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
+	attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
 	if err != nil {
 		wrapErr := errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败，请稍后重试")
 		return wrapErr
@@ -764,7 +819,7 @@ func (s *StopResumeService) StartMachineSeparatedCard(ctx context.Context, card
 		"resumed_at":     now,
 		"stop_reason":    "",
 		"gateway_extend": "",
-	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, attempt.log.IntegrationID); err != nil {
+	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
 		if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
 			s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
 		}
@@ -818,7 +873,7 @@ func (s *StopResumeService) ManualStopCard(ctx context.Context, iccid string) er
 		}
 	}

-	if err := s.stopCardWithRetry(ctx, card, constants.StopReasonManual); err != nil {
+	if _, err := s.stopCardWithRetry(ctx, card, constants.StopReasonManual); err != nil {
 		return err
 	}

@@ -833,6 +888,16 @@ func (s *StopResumeService) ManualStartCard(ctx context.Context, iccid string) e
 	}
 	actionCode, summary := constants.AuditActionIotCardManualStarted, "人工恢复 IoT 卡网络"

+	// 持通道阈值锁：周期内拒绝一切复机，记录拒绝事实并保留锁。
+	blocked, blockErr := s.channelThresholdBlocked(ctx, card)
+	if blockErr != nil {
+		return blockErr
+	}
+	if blocked {
+		s.recordChannelThresholdDenial(ctx, card, actionCode, summary)
+		return channelThresholdDenyError()
+	}
+
 	// 独立卡处于风险停机或已销户状态时，拒绝复机
 	if card.IsStandalone && isRiskGatewayExtend(card.GatewayExtend) {
 		var denyMsg string
@@ -875,7 +940,7 @@ func (s *StopResumeService) ManualStartCard(ctx context.Context, iccid string) e
 		}
 	}

-	attempt, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
+	attempt, integrationID, err := s.resumeCardWithRetry(ctx, card, actionCode, summary)
 	if err != nil {
 		wrapErr := errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败，请稍后重试")
 		return wrapErr
@@ -886,7 +951,7 @@ func (s *StopResumeService) ManualStartCard(ctx context.Context, iccid string) e
 		"network_status": constants.NetworkStatusOnline,
 		"resumed_at":     now,
 		"stop_reason":    "",
-	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, attempt.log.IntegrationID); err != nil {
+	}, constants.CardObservationSceneBusinessResume, "online", uuid.NewString(), actionCode, summary, integrationID); err != nil {
 		if logErr := s.completeCardCommandAttempt(ctx, attempt, nil, false); logErr != nil {
 			s.logger.Error("终结复机 Integration Log 失败", zap.String("integration_id", attempt.log.IntegrationID), zap.Error(logErr))
 		}