feat: 退款资产后处理逐次尝试记录与重试上限,停复机指令调用前结束写跳过记录

- 新增 tb_refund_post_process_attempt,每次尝试一条(步骤、触发方式、结果、错误码与信息、耗时)
- 退款单增加 asset_post_status/attempts/last_error/next_at;失败按 2^(n-1) 分钟退避(最长 1 小时),10 次后转人工
- 中间失败只写应用日志,成功或转人工写一条结论审计(带总尝试次数、首次失败时间、最后错误码)
- 补偿扫描跳过需人工处理与未到重试时间的退款单,Outbox 不再无限重投
- 新增 POST /refunds/:id/asset-post-processing/retry 人工重试
- 停复机指令执行前结束(已是目标状态、条件不再满足、需相反动作)写 skipped 停复机记录
- 迁移 000271 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-05 18:33:32 +09:00
parent b7a7430cc3
commit 8eb7d403c4
15 changed files with 488 additions and 77 deletions

View File

@@ -290,16 +290,4 @@ func (s *Service) ProcessCommissionDeduction(ctx context.Context, refundID uint)
return s.clawbackCommission(ctx, refundID)
}
func (s *Service) ProcessAssetPostProcessing(ctx context.Context, refundID uint) error {
s.handleRefundAssetProcessing(ctx, refundID)
var refund model.RefundRequest
if err := s.db.WithContext(ctx).Select("asset_reset").First(&refund, refundID).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "复核退款资产后处理状态失败")
}
if !refund.AssetReset {
return errors.New(errors.CodeServiceUnavailable, "退款资产后处理尚未完成")
}
return nil
}
var _ approvalapp.BusinessDecisionHandler = (*Service)(nil)

View File

@@ -0,0 +1,236 @@
package refund
import (
"context"
stderrors "errors"
"strconv"
"time"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/model/dto"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
)
// assetPostErrorMessageMax 是尝试记录与退款单上错误摘要的最大长度(字符)。
const assetPostErrorMessageMax = 500
// ProcessAssetPostProcessing 是 Outbox 投递(含补偿扫描重投)的退款资产后处理入口。
// 后处理失败只更新重试状态并返回 nil,由补偿扫描按下次重试时间拉起,不靠 Outbox 自身重投;
// 只有读写退款单本身失败时返回错误,交给 Outbox 重试。
func (s *Service) ProcessAssetPostProcessing(ctx context.Context, refundID uint) error {
return s.runAssetPostProcessing(ctx, refundID, constants.RefundPostTriggerOutbox)
}
// RetryAssetPostProcessingByAdmin 是后台人工重试接口:仅平台账号,按详情可见范围定位后重试并返回最新详情。
func (s *Service) RetryAssetPostProcessingByAdmin(ctx context.Context, refundID uint) (*dto.RefundResponse, error) {
userType := middleware.GetUserTypeFromContext(ctx)
if userType != constants.UserTypeSuperAdmin && userType != constants.UserTypePlatform {
return nil, errors.New(errors.CodeForbidden, "只有平台账号可以重试退款资产后处理")
}
if _, err := s.refundStore.GetByID(ctx, refundID); err != nil {
return nil, errors.New(errors.CodeNotFound, "退款申请不存在")
}
if err := s.RetryAssetPostProcessing(ctx, refundID); err != nil {
return nil, err
}
return s.GetByID(ctx, refundID)
}
// RetryAssetPostProcessing 是人工重试入口,只允许对"需人工处理"的退款单发起。
func (s *Service) RetryAssetPostProcessing(ctx context.Context, refundID uint) error {
var refund model.RefundRequest
if err := s.db.WithContext(ctx).Select("id", "status", "asset_reset", "asset_post_status").First(&refund, refundID).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return errors.New(errors.CodeNotFound, "退款单不存在")
}
return errors.Wrap(errors.CodeDatabaseError, err, "查询退款单失败")
}
if refund.AssetReset || refund.AssetPostStatus != model.RefundAssetPostManualRequired {
return errors.New(errors.CodeInvalidStatus, "只有需人工处理的退款单可以重试资产后处理")
}
if err := s.runAssetPostProcessing(ctx, refundID, constants.RefundPostTriggerManual); err != nil {
return err
}
if err := s.db.WithContext(ctx).Select("asset_reset", "asset_post_last_error").First(&refund, refundID).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "复核退款资产后处理状态失败")
}
if !refund.AssetReset {
return errors.New(errors.CodeServiceUnavailable, "资产后处理仍未完成:"+refund.AssetPostLastError)
}
return nil
}
func (s *Service) runAssetPostProcessing(ctx context.Context, refundID uint, trigger string) error {
var refund model.RefundRequest
if err := s.db.WithContext(ctx).Where("id = ?", refundID).First(&refund).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "查询退款单失败")
}
if refund.AssetReset {
return nil
}
now := time.Now()
if trigger == constants.RefundPostTriggerOutbox {
if refund.AssetPostStatus == model.RefundAssetPostManualRequired {
return nil
}
if refund.AssetPostNextAt != nil && refund.AssetPostNextAt.After(now) {
return nil
}
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.AuditActorIDRefundAssetPostProcessing,
ActorName: "退款资产自动后处理任务", Source: constants.AuditSourceWorker, CorrelationID: refund.RefundNo,
})
}
// 先占用本次尝试序号,使完成审计能读到总尝试次数。
attempt := refund.AssetPostAttempts + 1
if err := s.db.WithContext(ctx).Model(&model.RefundRequest{}).Where("id = ?", refundID).
Update("asset_post_attempts", attempt).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新退款资产后处理次数失败")
}
refund.AssetPostAttempts = attempt
outcome := s.handleRefundAssetProcessing(ctx, &refund)
record := s.recordAssetPostAttempt(ctx, &refund, outcome, attempt, trigger, time.Since(now))
if outcome.Err == nil {
return s.completeAssetPostProcessing(ctx, refundID)
}
return s.failAssetPostProcessing(ctx, &refund, outcome, attempt, trigger, record)
}
func (s *Service) completeAssetPostProcessing(ctx context.Context, refundID uint) error {
if err := s.db.WithContext(ctx).Model(&model.RefundRequest{}).Where("id = ?", refundID).Updates(map[string]any{
"asset_post_status": model.RefundAssetPostCompleted, "asset_post_last_error": "", "asset_post_next_at": nil,
}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新退款资产后处理状态失败")
}
return nil
}
func (s *Service) failAssetPostProcessing(
ctx context.Context, refund *model.RefundRequest, outcome refundAssetOutcome, attempt int, trigger string, record *model.RefundPostProcessAttempt,
) error {
message := truncateRunes(outcome.Err.Error(), assetPostErrorMessageMax)
updates := map[string]any{"asset_post_last_error": message}
final := attempt >= constants.RefundPostMaxAttempts
if final {
updates["asset_post_status"] = model.RefundAssetPostManualRequired
updates["asset_post_next_at"] = nil
} else {
updates["asset_post_status"] = model.RefundAssetPostRetryPending
updates["asset_post_next_at"] = time.Now().Add(assetPostBackoff(attempt))
}
if err := s.db.WithContext(ctx).Model(&model.RefundRequest{}).Where("id = ?", refund.ID).Updates(updates).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新退款资产后处理重试状态失败")
}
s.recordAssetPostFailureAudit(ctx, refund, outcome, attempt, record, final || trigger == constants.RefundPostTriggerManual)
return nil
}
// recordAssetPostFailureAudit 写后处理失败:中间失败只写应用日志,转人工或人工重试失败写审计结论。
func (s *Service) recordAssetPostFailureAudit(
ctx context.Context, refund *model.RefundRequest, outcome refundAssetOutcome, attempt int, record *model.RefundPostProcessAttempt, final bool,
) {
if s.auditWriter == nil || s.db == nil {
return
}
primary := audit.RefundResource(refund, constants.AuditResourceRelationPrimary, constants.AuditResourceRoleRefundTarget)
primary.SubjectVisibility = constants.AuditSubjectInternalOnly
input := audit.AppendInput{
EventID: "refund:" + strconv.FormatUint(uint64(refund.ID), 10) + ":asset-processed:failed:" + strconv.Itoa(attempt),
ActionCode: constants.AuditActionRefundAssetProcessed, Summary: "退款资产后处理失败(" + outcome.Step + ")",
ScopeType: constants.AuditScopePlatform, CorrelationID: refund.RefundNo, Resources: []audit.ResourceInput{primary},
Final: final, Attempt: attempt,
Metadata: map[string]any{"asset_post_attempts": attempt, "step": outcome.Step, "manual_required": attempt >= constants.RefundPostMaxAttempts},
}
if final {
input.Summary = "退款资产后处理重试用尽,转人工处理(" + outcome.Step + ")"
for key, value := range assetPostOutcomeMetadata(ctx, s.db, refund) {
input.Metadata[key] = value
}
}
if record != nil && record.ID > 0 {
input.AttemptRecordID = strconv.FormatUint(uint64(record.ID), 10)
}
s.auditWriter.RecordFailure(ctx, s.db, input, outcome.Err)
}
// recordAssetPostAttempt 写一条尝试记录;写入失败只告警,不影响后处理结果。
func (s *Service) recordAssetPostAttempt(
ctx context.Context, refund *model.RefundRequest, outcome refundAssetOutcome, attempt int, trigger string, duration time.Duration,
) *model.RefundPostProcessAttempt {
record := &model.RefundPostProcessAttempt{
RefundID: refund.ID, RefundNo: refund.RefundNo, Step: outcome.Step, AssetType: outcome.AssetType, AssetID: outcome.AssetID,
Attempt: attempt, Trigger: trigger, Result: constants.RefundPostResultSuccess, DurationMS: int(duration.Milliseconds()),
}
if outcome.AssetType == constants.AssetTypeIotCard && outcome.AssetID > 0 {
var card model.IotCard
if err := s.db.WithContext(ctx).Select("iccid").First(&card, outcome.AssetID).Error; err == nil {
record.ICCID = card.ICCID
}
}
if outcome.Err != nil {
record.Result = constants.RefundPostResultFailed
record.ErrorCode, record.ErrorMessage = assetPostErrorDetail(outcome.Err)
}
if err := s.db.WithContext(ctx).Create(record).Error; err != nil {
s.logger.Warn("写退款资产后处理尝试记录失败", zap.Uint("refund_id", refund.ID), zap.Int("attempt", attempt), zap.Error(err))
return nil
}
return record
}
// assetPostOutcomeMetadata 返回结论审计需要的总尝试次数、首次失败时间与最后错误。
func assetPostOutcomeMetadata(ctx context.Context, db *gorm.DB, refund *model.RefundRequest) map[string]any {
metadata := map[string]any{"asset_post_attempts": refund.AssetPostAttempts}
var first model.RefundPostProcessAttempt
if err := db.WithContext(ctx).Where("refund_id = ? AND result = ?", refund.ID, constants.RefundPostResultFailed).
Order("created_at ASC, id ASC").Limit(1).Find(&first).Error; err == nil && first.ID > 0 {
metadata["first_failed_at"] = first.CreatedAt
}
var last model.RefundPostProcessAttempt
if err := db.WithContext(ctx).Where("refund_id = ? AND result = ?", refund.ID, constants.RefundPostResultFailed).
Order("created_at DESC, id DESC").Limit(1).Find(&last).Error; err == nil && last.ID > 0 {
metadata["last_error_code"] = last.ErrorCode
}
return metadata
}
// assetPostBackoff 返回第 attempt 次失败后的重试间隔:2^(attempt-1) 分钟,最长 1 小时。
func assetPostBackoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
if attempt > 7 {
return constants.RefundPostMaxBackoff
}
delay := time.Duration(1<<(attempt-1)) * time.Minute
if delay > constants.RefundPostMaxBackoff {
return constants.RefundPostMaxBackoff
}
return delay
}
func assetPostErrorDetail(err error) (string, string) {
code := strconv.Itoa(errors.CodeInternalError)
var appErr *errors.AppError
if stderrors.As(err, &appErr) && appErr != nil {
code = strconv.Itoa(appErr.Code)
}
return code, truncateRunes(err.Error(), assetPostErrorMessageMax)
}
func truncateRunes(value string, limit int) string {
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}

View File

@@ -71,15 +71,20 @@ func (s *Service) appendRefundAudit(
resources = append(resources, chain...)
}
metadata := map[string]any{
"requested_refund_amount": refund.RequestedRefundAmount,
"approved_refund_amount": refund.ApprovedRefundAmount,
}
if actionCode == constants.AuditActionRefundAssetProcessed {
for key, value := range assetPostOutcomeMetadata(ctx, tx, &refund) {
metadata[key] = value
}
}
// 退款审计都是业务结论;对"只写结论"策略的动作(退款资产后处理)标记为最终结论。
return s.auditWriter.Append(ctx, tx, audit.AppendInput{
EventID: eventID, ActionCode: actionCode, Summary: summary,
ScopeType: constants.AuditScopePlatform, Result: constants.AuditResultSuccess,
CorrelationID: refund.RefundNo,
Metadata: map[string]any{
"requested_refund_amount": refund.RequestedRefundAmount,
"approved_refund_amount": refund.ApprovedRefundAmount,
},
Resources: resources,
CorrelationID: refund.RefundNo, Metadata: metadata, Resources: resources, Final: true,
})
}

View File

@@ -1125,34 +1125,26 @@ func (s *Service) appendWithdrawalRejectAudit(
return nil
}
// refundAssetOutcome 是一次退款资产后处理的结果:失败时 Step 为失败步骤,Err 非空。
type refundAssetOutcome struct {
Step string
AssetType string
AssetID uint
Err error
}
// handleRefundAssetProcessing 幂等处理退款后的资产状态。
// 包括退款套餐精准失效、尝试接续待生效主套餐和必要时停机;全部完成后才设置完成标记。
func (s *Service) handleRefundAssetProcessing(ctx context.Context, refundID uint) {
// 失败不在这里写审计,由 ProcessAssetPostProcessing 写尝试记录、更新重试状态并按"只写结论"写审计。
func (s *Service) handleRefundAssetProcessing(ctx context.Context, refund *model.RefundRequest) refundAssetOutcome {
logger := s.logger
// 查询退款单
var refund model.RefundRequest
if err := s.db.Where("id = ?", refundID).First(&refund).Error; err != nil {
logger.Error("退款资产处理:查询退款单失败", zap.Uint("refund_id", refundID), zap.Error(err))
return
}
if refund.AssetReset {
return
}
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.AuditActorIDRefundAssetPostProcessing,
ActorName: "退款资产自动后处理任务", Source: constants.AuditSourceWorker, CorrelationID: refund.RefundNo,
})
refundID := refund.ID
// 查询关联订单
var order model.Order
if err := s.db.Where("id = ?", refund.OrderID).First(&order).Error; err != nil {
s.recordRefundFailure(ctx, constants.AuditActionRefundAssetProcessed, "完成退款资产后处理失败", &refund, nil, err)
if err := s.db.WithContext(ctx).Where("id = ?", refund.OrderID).First(&order).Error; err != nil {
logger.Error("退款资产处理:查询订单失败", zap.Uint("refund_id", refundID), zap.Uint("order_id", refund.OrderID), zap.Error(err))
return
}
recordFailure := func(err error) {
s.recordRefundFailure(ctx, constants.AuditActionRefundAssetProcessed, "完成退款资产后处理失败", &refund, &order, err)
return refundAssetOutcome{Step: constants.RefundPostStepLoad, Err: errors.Wrap(errors.CodeDatabaseError, err, "查询退款关联订单失败")}
}
// 确定资产类型和 ID
@@ -1161,37 +1153,32 @@ func (s *Service) handleRefundAssetProcessing(ctx context.Context, refundID uint
switch order.OrderType {
case model.OrderTypeSingleCard:
if order.IotCardID == nil {
recordFailure(errors.New(errors.CodeInternalError, "退款单卡订单缺少资产ID"))
logger.Error("退款资产处理:单卡订单缺少 iot_card_id", zap.Uint("order_id", order.ID))
return
return refundAssetOutcome{Step: constants.RefundPostStepLoad, Err: errors.New(errors.CodeInternalError, "退款单卡订单缺少资产ID")}
}
assetType = "iot_card"
assetID = *order.IotCardID
case model.OrderTypeDevice:
if order.DeviceID == nil {
recordFailure(errors.New(errors.CodeInternalError, "退款设备订单缺少资产ID"))
logger.Error("退款资产处理:设备订单缺少 device_id", zap.Uint("order_id", order.ID))
return
return refundAssetOutcome{Step: constants.RefundPostStepLoad, Err: errors.New(errors.CodeInternalError, "退款设备订单缺少资产ID")}
}
assetType = "device"
assetID = *order.DeviceID
default:
recordFailure(errors.New(errors.CodeInvalidParam, "退款订单资产类型无效"))
logger.Error("退款资产处理:未知订单类型", zap.String("order_type", order.OrderType))
return
return refundAssetOutcome{Step: constants.RefundPostStepLoad, Err: errors.New(errors.CodeInvalidParam, "退款订单资产类型无效")}
}
fail := func(step string, err error) refundAssetOutcome {
return refundAssetOutcome{Step: step, AssetType: assetType, AssetID: assetID, Err: err}
}
// 1. 按退款单精准失效套餐(仅处理本次退款订单关联套餐)
if s.packageActivationService == nil {
businessErr := errors.New(errors.CodeServiceUnavailable, "退款套餐处理能力未配置")
recordFailure(businessErr)
logger.Error("退款资产处理:套餐激活服务未注入",
zap.Uint("refund_id", refund.ID),
zap.Uint("order_id", order.ID))
return
logger.Error("退款资产处理:套餐激活服务未注入", zap.Uint("refund_id", refund.ID), zap.Uint("order_id", order.ID))
return fail(constants.RefundPostStepInvalidate, errors.New(errors.CodeServiceUnavailable, "退款套餐处理能力未配置"))
}
if err := s.packageActivationService.InvalidatePackagesForRefund(ctx, assetType, assetID, order.ID, refund.ID, refund.RefundNo, refund.PackageUsageID); err != nil {
recordFailure(err)
fields := []zap.Field{
zap.String("asset_type", assetType),
zap.Uint("asset_id", assetID),
@@ -1204,43 +1191,35 @@ func (s *Service) handleRefundAssetProcessing(ctx context.Context, refundID uint
fields = append(fields, zap.Uint("package_usage_id", *refund.PackageUsageID))
}
logger.Error("退款资产处理:退款套餐精准失效失败", fields...)
return
return fail(constants.RefundPostStepInvalidate, err)
}
// 2. 尝试按购买顺序接续待生效主套餐
if _, err := s.packageActivationService.ActivateNextPendingMainPackage(ctx, assetType, assetID); err != nil {
recordFailure(err)
logger.Error("退款资产处理:接续激活待生效套餐失败",
zap.String("asset_type", assetType),
zap.Uint("asset_id", assetID),
zap.Uint("refund_id", refund.ID),
zap.Error(err))
return
zap.String("asset_type", assetType), zap.Uint("asset_id", assetID), zap.Uint("refund_id", refund.ID), zap.Error(err))
return fail(constants.RefundPostStepActivate, err)
}
hasActiveMain, err := s.packageActivationService.HasActiveMainPackage(ctx, assetType, assetID)
if err != nil {
recordFailure(err)
logger.Error("退款资产处理:查询生效主套餐失败",
zap.String("asset_type", assetType),
zap.Uint("asset_id", assetID),
zap.Uint("refund_id", refund.ID),
zap.Error(err))
return
zap.String("asset_type", assetType), zap.Uint("asset_id", assetID), zap.Uint("refund_id", refund.ID), zap.Error(err))
return fail(constants.RefundPostStepCheckMain, err)
}
if !hasActiveMain {
// 3. 无可用主套餐时才停机;退款不再重置世代或重建钱包。
if !s.stopAsset(ctx, assetType, assetID) {
recordFailure(errors.New(errors.CodeServiceUnavailable, "退款资产停机处理失败"))
return
return fail(constants.RefundPostStepStop, errors.New(errors.CodeServiceUnavailable, "退款资产停机处理失败"))
}
}
// 4. 标记退款后资产处理已完成
if err := s.markRefundAssetProcessed(ctx, refundID); err != nil {
recordFailure(err)
logger.Error("退款资产处理:更新处理标记失败", zap.Uint("refund_id", refundID), zap.Error(err))
return fail(constants.RefundPostStepMark, err)
}
return refundAssetOutcome{Step: constants.RefundPostStepDone, AssetType: assetType, AssetID: assetID}
}
// stopAsset 根据资产类型执行停机操作