fix(通道流量阈值): AUG26-011 修复失败/未知结果收敛、锁定 carrier 缺失出路与审计回归
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 13m29s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 13m29s
This commit is contained in:
@@ -18,38 +18,42 @@ const cycleBatchSize = 200
|
||||
|
||||
// CycleStats 是一次周期处理的可观察结果。
|
||||
//
|
||||
// Scanned 为扫到的跨期持锁数;Unlocked 为本次认领解锁成功的锁数;Resumed 为同时写出复机事件的锁数;
|
||||
// Skipped 为本地事实缺失、被并发推进或判定失败而未改动的锁数。
|
||||
type CycleStats struct{ Scanned, Unlocked, Resumed, Skipped int }
|
||||
// Scanned 为扫到的待处理持锁数;Unlocked 为本次认领解锁成功的锁数;Resumed 为同时写出复机事件的锁数;
|
||||
// Anomaly 为因运营商配置缺失或重置日非法而解锁并转人工的锁数;Skipped 为本地事实缺失、被并发推进
|
||||
// 或判定失败而未改动的锁数。
|
||||
type CycleStats struct{ Scanned, Unlocked, Resumed, Anomaly, Skipped int }
|
||||
|
||||
// ProcessDueLocks 扫描已跨期的持锁锁行:认领解锁,并在新周期条件满足时同事务写复机事件。
|
||||
// ProcessDueLocks 扫描待处理的持锁锁行:认领解锁,并在新周期条件满足时同事务写复机事件。
|
||||
//
|
||||
// 过期判断按锁行自身运营商的 data_reset_day(支持换运营商后旧锁仍按其归属处理)。解锁与复机事件
|
||||
// 在同一事务提交:事务失败则解锁一起回滚,下一分钟重新处理,不会出现「已解锁但没有复机任务」的中间态。
|
||||
// 任一复机条件不满足时只解锁、不调运营商,并把可观察原因写入锁行。
|
||||
//
|
||||
// 锁行引用的运营商已不存在或重置日非法时无法计算周期归属:这类行按「新周期对仍持锁卡解除通道锁」
|
||||
// 的语义解锁并标记异常转人工,绝不写复机事件、不调运营商,也绝不静默跳过(否则持锁卡会永久禁止复机)。
|
||||
func (s *Service) ProcessDueLocks(ctx context.Context, now time.Time) (CycleStats, error) {
|
||||
stats := CycleStats{}
|
||||
if s == nil || s.db == nil || s.repository == nil {
|
||||
return stats, errors.New(errors.CodeServiceUnavailable, "通道流量阈值周期处理能力未配置")
|
||||
}
|
||||
locks, err := s.ScanExpiredLocks(ctx, now, cycleBatchSize)
|
||||
due, err := s.ScanDueLocks(ctx, now, cycleBatchSize)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Scanned = len(locks)
|
||||
if len(locks) == 0 {
|
||||
stats.Scanned = len(due)
|
||||
if len(due) == 0 {
|
||||
return stats, nil
|
||||
}
|
||||
if s.commander == nil {
|
||||
return stats, errors.New(errors.CodeServiceUnavailable, "通道流量阈值停复机执行端口未配置")
|
||||
}
|
||||
var firstErr error
|
||||
for index := range locks {
|
||||
lock := locks[index]
|
||||
if err := s.processDueLock(ctx, &lock, &stats); err != nil {
|
||||
for index := range due {
|
||||
item := due[index]
|
||||
if err := s.processDueLock(ctx, &item, &stats); err != nil {
|
||||
stats.Skipped++
|
||||
s.logger.Warn("通道阈值跨期处理单条失败",
|
||||
zap.Uint("lock_id", lock.ID), zap.Uint("card_id", lock.CardID), zap.Error(err))
|
||||
zap.Uint("lock_id", item.Lock.ID), zap.Uint("card_id", item.Lock.CardID), zap.Error(err))
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
@@ -58,8 +62,12 @@ func (s *Service) ProcessDueLocks(ctx context.Context, now time.Time) (CycleStat
|
||||
return stats, firstErr
|
||||
}
|
||||
|
||||
// processDueLock 处理单条跨期锁:认领解锁并在条件满足时同事务写复机事件。
|
||||
func (s *Service) processDueLock(ctx context.Context, lock *model.CarrierTrafficThresholdLock, stats *CycleStats) error {
|
||||
// processDueLock 处理单条待处理锁行:周期归属不可判定时解锁并转人工,已跨期时认领解锁并条件复机。
|
||||
func (s *Service) processDueLock(ctx context.Context, item *DueLock, stats *CycleStats) error {
|
||||
if item.Kind == DueLockUnresolvable {
|
||||
return s.processUnresolvableLock(ctx, item, stats)
|
||||
}
|
||||
lock := &item.Lock
|
||||
card, err := s.loadCard(ctx, lock.CardID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -111,6 +119,42 @@ func (s *Service) processDueLock(ctx context.Context, lock *model.CarrierTraffic
|
||||
return nil
|
||||
}
|
||||
|
||||
// processUnresolvableLock 处理周期归属不可判定的锁行:同事务认领解锁并标记异常转人工。
|
||||
//
|
||||
// 只解锁不写复机事件、不调运营商:配置缺失时无法判断是否已跨期,解除通道锁交由既有复机链路
|
||||
// 与人工决定;anomaly_flag 与失败原因使运维可见并转人工核对。解锁与异常标记都是条件更新,
|
||||
// 重复执行不会产生第二次副作用。
|
||||
func (s *Service) processUnresolvableLock(ctx context.Context, item *DueLock, stats *CycleStats) error {
|
||||
lock := &item.Lock
|
||||
s.logger.Warn("通道阈值锁行周期归属不可判定,解锁并转人工核对",
|
||||
zap.Uint("lock_id", lock.ID), zap.Uint("card_id", lock.CardID), zap.Uint("carrier_id", lock.CarrierID),
|
||||
zap.String("reason", item.Reason))
|
||||
unlocked := false
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
claimed, unlockErr := s.unlockInTx(ctx, tx, lock.ID, item.Reason)
|
||||
if unlockErr != nil {
|
||||
return unlockErr
|
||||
}
|
||||
if !claimed {
|
||||
return nil
|
||||
}
|
||||
unlocked = true
|
||||
_, anomalyErr := s.markAnomaly(ctx, lock.ID, item.Reason)
|
||||
return anomalyErr
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !unlocked {
|
||||
stats.Skipped++
|
||||
s.logger.Info("通道阈值不可判定锁已被并发处理,跳过", zap.Uint("lock_id", lock.ID))
|
||||
return nil
|
||||
}
|
||||
stats.Unlocked++
|
||||
stats.Anomaly++
|
||||
return nil
|
||||
}
|
||||
|
||||
// recoveryBatchSize 是单次恢复扫描的锁行上限。
|
||||
const recoveryBatchSize = 200
|
||||
|
||||
@@ -158,8 +202,11 @@ func (s *Service) RecoverSubmitted(ctx context.Context, now time.Time) (Recovery
|
||||
}
|
||||
|
||||
// recoverSubmittedLock 处理单条待确认锁:只查询状态回填,不发起任何停复机调用。
|
||||
//
|
||||
// 入口只处理未决子任务(submitted/unknown/failed):调用失败或结果未知同样可能已在运营商侧生效,
|
||||
// 必须继续收敛;confirmed 是终态,pending 表示从未对外调用,都不在本扫描范围。
|
||||
func (s *Service) recoverSubmittedLock(ctx context.Context, lock *model.CarrierTrafficThresholdLock, now time.Time, stats *RecoveryStats) error {
|
||||
if lock.StopStatus != domain.TaskStatusSubmitted && lock.ResumeStatus != domain.TaskStatusSubmitted {
|
||||
if !domain.IsUnresolvedTaskStatus(lock.StopStatus) && !domain.IsUnresolvedTaskStatus(lock.ResumeStatus) {
|
||||
return nil
|
||||
}
|
||||
card, err := s.loadCard(ctx, lock.CardID)
|
||||
@@ -178,7 +225,7 @@ func (s *Service) recoverSubmittedLock(ctx context.Context, lock *model.CarrierT
|
||||
}
|
||||
confirmed := 0
|
||||
unconfirmed := 0
|
||||
if lock.StopStatus == domain.TaskStatusSubmitted {
|
||||
if domain.IsUnresolvedTaskStatus(lock.StopStatus) {
|
||||
switch {
|
||||
case known && status == constants.NetworkStatusOffline:
|
||||
confirmed++
|
||||
@@ -189,7 +236,7 @@ func (s *Service) recoverSubmittedLock(ctx context.Context, lock *model.CarrierT
|
||||
unconfirmed++
|
||||
}
|
||||
}
|
||||
if lock.ResumeStatus == domain.TaskStatusSubmitted {
|
||||
if domain.IsUnresolvedTaskStatus(lock.ResumeStatus) {
|
||||
switch {
|
||||
case known && status == constants.NetworkStatusOnline:
|
||||
confirmed++
|
||||
|
||||
Reference in New Issue
Block a user