feat: 退款资产后处理逐次尝试记录与重试上限,停复机指令调用前结束写跳过记录

- 新增 tb_refund_post_process_attempt,每次尝试一条(步骤、触发方式、结果、错误码与信息、耗时)
- 退款单增加 asset_post_status/attempts/last_error/next_at;失败按 2^(n-1) 分钟退避(最长 1 小时),10 次后转人工
- 中间失败只写应用日志,成功或转人工写一条结论审计(带总尝试次数、首次失败时间、最后错误码)
- 补偿扫描跳过需人工处理与未到重试时间的退款单,Outbox 不再无限重投
- 新增 POST /refunds/:id/asset-post-processing/retry 人工重试
- 停复机指令执行前结束(已是目标状态、条件不再满足、需相反动作)写 skipped 停复机记录
- 迁移 000271 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-05 18:33:32 +09:00
parent b7a7430cc3
commit 8eb7d403c4
15 changed files with 488 additions and 77 deletions

View File

@@ -0,0 +1,236 @@
package refund
import (
"context"
stderrors "errors"
"strconv"
"time"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/model/dto"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
)
// assetPostErrorMessageMax 是尝试记录与退款单上错误摘要的最大长度(字符)。
const assetPostErrorMessageMax = 500
// ProcessAssetPostProcessing 是 Outbox 投递(含补偿扫描重投)的退款资产后处理入口。
// 后处理失败只更新重试状态并返回 nil,由补偿扫描按下次重试时间拉起,不靠 Outbox 自身重投;
// 只有读写退款单本身失败时返回错误,交给 Outbox 重试。
func (s *Service) ProcessAssetPostProcessing(ctx context.Context, refundID uint) error {
return s.runAssetPostProcessing(ctx, refundID, constants.RefundPostTriggerOutbox)
}
// RetryAssetPostProcessingByAdmin 是后台人工重试接口:仅平台账号,按详情可见范围定位后重试并返回最新详情。
func (s *Service) RetryAssetPostProcessingByAdmin(ctx context.Context, refundID uint) (*dto.RefundResponse, error) {
userType := middleware.GetUserTypeFromContext(ctx)
if userType != constants.UserTypeSuperAdmin && userType != constants.UserTypePlatform {
return nil, errors.New(errors.CodeForbidden, "只有平台账号可以重试退款资产后处理")
}
if _, err := s.refundStore.GetByID(ctx, refundID); err != nil {
return nil, errors.New(errors.CodeNotFound, "退款申请不存在")
}
if err := s.RetryAssetPostProcessing(ctx, refundID); err != nil {
return nil, err
}
return s.GetByID(ctx, refundID)
}
// RetryAssetPostProcessing 是人工重试入口,只允许对"需人工处理"的退款单发起。
func (s *Service) RetryAssetPostProcessing(ctx context.Context, refundID uint) error {
var refund model.RefundRequest
if err := s.db.WithContext(ctx).Select("id", "status", "asset_reset", "asset_post_status").First(&refund, refundID).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return errors.New(errors.CodeNotFound, "退款单不存在")
}
return errors.Wrap(errors.CodeDatabaseError, err, "查询退款单失败")
}
if refund.AssetReset || refund.AssetPostStatus != model.RefundAssetPostManualRequired {
return errors.New(errors.CodeInvalidStatus, "只有需人工处理的退款单可以重试资产后处理")
}
if err := s.runAssetPostProcessing(ctx, refundID, constants.RefundPostTriggerManual); err != nil {
return err
}
if err := s.db.WithContext(ctx).Select("asset_reset", "asset_post_last_error").First(&refund, refundID).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "复核退款资产后处理状态失败")
}
if !refund.AssetReset {
return errors.New(errors.CodeServiceUnavailable, "资产后处理仍未完成:"+refund.AssetPostLastError)
}
return nil
}
func (s *Service) runAssetPostProcessing(ctx context.Context, refundID uint, trigger string) error {
var refund model.RefundRequest
if err := s.db.WithContext(ctx).Where("id = ?", refundID).First(&refund).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "查询退款单失败")
}
if refund.AssetReset {
return nil
}
now := time.Now()
if trigger == constants.RefundPostTriggerOutbox {
if refund.AssetPostStatus == model.RefundAssetPostManualRequired {
return nil
}
if refund.AssetPostNextAt != nil && refund.AssetPostNextAt.After(now) {
return nil
}
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.AuditActorIDRefundAssetPostProcessing,
ActorName: "退款资产自动后处理任务", Source: constants.AuditSourceWorker, CorrelationID: refund.RefundNo,
})
}
// 先占用本次尝试序号,使完成审计能读到总尝试次数。
attempt := refund.AssetPostAttempts + 1
if err := s.db.WithContext(ctx).Model(&model.RefundRequest{}).Where("id = ?", refundID).
Update("asset_post_attempts", attempt).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新退款资产后处理次数失败")
}
refund.AssetPostAttempts = attempt
outcome := s.handleRefundAssetProcessing(ctx, &refund)
record := s.recordAssetPostAttempt(ctx, &refund, outcome, attempt, trigger, time.Since(now))
if outcome.Err == nil {
return s.completeAssetPostProcessing(ctx, refundID)
}
return s.failAssetPostProcessing(ctx, &refund, outcome, attempt, trigger, record)
}
func (s *Service) completeAssetPostProcessing(ctx context.Context, refundID uint) error {
if err := s.db.WithContext(ctx).Model(&model.RefundRequest{}).Where("id = ?", refundID).Updates(map[string]any{
"asset_post_status": model.RefundAssetPostCompleted, "asset_post_last_error": "", "asset_post_next_at": nil,
}).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新退款资产后处理状态失败")
}
return nil
}
func (s *Service) failAssetPostProcessing(
ctx context.Context, refund *model.RefundRequest, outcome refundAssetOutcome, attempt int, trigger string, record *model.RefundPostProcessAttempt,
) error {
message := truncateRunes(outcome.Err.Error(), assetPostErrorMessageMax)
updates := map[string]any{"asset_post_last_error": message}
final := attempt >= constants.RefundPostMaxAttempts
if final {
updates["asset_post_status"] = model.RefundAssetPostManualRequired
updates["asset_post_next_at"] = nil
} else {
updates["asset_post_status"] = model.RefundAssetPostRetryPending
updates["asset_post_next_at"] = time.Now().Add(assetPostBackoff(attempt))
}
if err := s.db.WithContext(ctx).Model(&model.RefundRequest{}).Where("id = ?", refund.ID).Updates(updates).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新退款资产后处理重试状态失败")
}
s.recordAssetPostFailureAudit(ctx, refund, outcome, attempt, record, final || trigger == constants.RefundPostTriggerManual)
return nil
}
// recordAssetPostFailureAudit 写后处理失败:中间失败只写应用日志,转人工或人工重试失败写审计结论。
func (s *Service) recordAssetPostFailureAudit(
ctx context.Context, refund *model.RefundRequest, outcome refundAssetOutcome, attempt int, record *model.RefundPostProcessAttempt, final bool,
) {
if s.auditWriter == nil || s.db == nil {
return
}
primary := audit.RefundResource(refund, constants.AuditResourceRelationPrimary, constants.AuditResourceRoleRefundTarget)
primary.SubjectVisibility = constants.AuditSubjectInternalOnly
input := audit.AppendInput{
EventID: "refund:" + strconv.FormatUint(uint64(refund.ID), 10) + ":asset-processed:failed:" + strconv.Itoa(attempt),
ActionCode: constants.AuditActionRefundAssetProcessed, Summary: "退款资产后处理失败(" + outcome.Step + ")",
ScopeType: constants.AuditScopePlatform, CorrelationID: refund.RefundNo, Resources: []audit.ResourceInput{primary},
Final: final, Attempt: attempt,
Metadata: map[string]any{"asset_post_attempts": attempt, "step": outcome.Step, "manual_required": attempt >= constants.RefundPostMaxAttempts},
}
if final {
input.Summary = "退款资产后处理重试用尽,转人工处理(" + outcome.Step + ")"
for key, value := range assetPostOutcomeMetadata(ctx, s.db, refund) {
input.Metadata[key] = value
}
}
if record != nil && record.ID > 0 {
input.AttemptRecordID = strconv.FormatUint(uint64(record.ID), 10)
}
s.auditWriter.RecordFailure(ctx, s.db, input, outcome.Err)
}
// recordAssetPostAttempt 写一条尝试记录;写入失败只告警,不影响后处理结果。
func (s *Service) recordAssetPostAttempt(
ctx context.Context, refund *model.RefundRequest, outcome refundAssetOutcome, attempt int, trigger string, duration time.Duration,
) *model.RefundPostProcessAttempt {
record := &model.RefundPostProcessAttempt{
RefundID: refund.ID, RefundNo: refund.RefundNo, Step: outcome.Step, AssetType: outcome.AssetType, AssetID: outcome.AssetID,
Attempt: attempt, Trigger: trigger, Result: constants.RefundPostResultSuccess, DurationMS: int(duration.Milliseconds()),
}
if outcome.AssetType == constants.AssetTypeIotCard && outcome.AssetID > 0 {
var card model.IotCard
if err := s.db.WithContext(ctx).Select("iccid").First(&card, outcome.AssetID).Error; err == nil {
record.ICCID = card.ICCID
}
}
if outcome.Err != nil {
record.Result = constants.RefundPostResultFailed
record.ErrorCode, record.ErrorMessage = assetPostErrorDetail(outcome.Err)
}
if err := s.db.WithContext(ctx).Create(record).Error; err != nil {
s.logger.Warn("写退款资产后处理尝试记录失败", zap.Uint("refund_id", refund.ID), zap.Int("attempt", attempt), zap.Error(err))
return nil
}
return record
}
// assetPostOutcomeMetadata 返回结论审计需要的总尝试次数、首次失败时间与最后错误。
func assetPostOutcomeMetadata(ctx context.Context, db *gorm.DB, refund *model.RefundRequest) map[string]any {
metadata := map[string]any{"asset_post_attempts": refund.AssetPostAttempts}
var first model.RefundPostProcessAttempt
if err := db.WithContext(ctx).Where("refund_id = ? AND result = ?", refund.ID, constants.RefundPostResultFailed).
Order("created_at ASC, id ASC").Limit(1).Find(&first).Error; err == nil && first.ID > 0 {
metadata["first_failed_at"] = first.CreatedAt
}
var last model.RefundPostProcessAttempt
if err := db.WithContext(ctx).Where("refund_id = ? AND result = ?", refund.ID, constants.RefundPostResultFailed).
Order("created_at DESC, id DESC").Limit(1).Find(&last).Error; err == nil && last.ID > 0 {
metadata["last_error_code"] = last.ErrorCode
}
return metadata
}
// assetPostBackoff 返回第 attempt 次失败后的重试间隔:2^(attempt-1) 分钟,最长 1 小时。
func assetPostBackoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
if attempt > 7 {
return constants.RefundPostMaxBackoff
}
delay := time.Duration(1<<(attempt-1)) * time.Minute
if delay > constants.RefundPostMaxBackoff {
return constants.RefundPostMaxBackoff
}
return delay
}
func assetPostErrorDetail(err error) (string, string) {
code := strconv.Itoa(errors.CodeInternalError)
var appErr *errors.AppError
if stderrors.As(err, &appErr) && appErr != nil {
code = strconv.Itoa(appErr.Code)
}
return code, truncateRunes(err.Error(), assetPostErrorMessageMax)
}
func truncateRunes(value string, limit int) string {
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}