每日清理

2026-08-20 12:03:17 +08:00
parent 143df60485
commit 22b95db2f9
23 changed files with 614 additions and 581 deletions
--- a/cmd/audit-retention-simulate/main.go
+++ b/cmd/audit-retention-simulate/main.go
@@ -1,4 +1,4 @@
-// Command audit-retention-simulate 在测试环境演练完整自然月归档与清理边界。
+// Command audit-retention-simulate 在测试环境演练逐日归档与清理边界。
 package main

 import (
@@ -6,22 +6,21 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"

 	"github.com/bytedance/sonic"
-	"github.com/hibiken/asynq"
 	"go.uber.org/zap"
 	"gorm.io/datatypes"
 	"gorm.io/gorm"

 	auditarchive "github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
-	auditinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
 	"github.com/break/junhong_cmp_fiber/internal/model"
-	taskapp "github.com/break/junhong_cmp_fiber/internal/task"
 	"github.com/break/junhong_cmp_fiber/pkg/config"
 	"github.com/break/junhong_cmp_fiber/pkg/constants"
 	"github.com/break/junhong_cmp_fiber/pkg/database"
+	logpkg "github.com/break/junhong_cmp_fiber/pkg/logger"
 	"github.com/break/junhong_cmp_fiber/pkg/storage"
 )

@@ -67,6 +66,11 @@ func run(ctx context.Context) error {
 		return fmt.Errorf("仅允许显式确认的测试数据库，当前数据库为 %q", cfg.Database.DBName)
 	}
 	logger := zap.NewNop()
+	if err := os.MkdirAll(filepath.Dir(cfg.Logging.RetentionLog.Filename), 0755); err != nil {
+		return err
+	}
+	retentionLogger, syncRetentionLogger := logpkg.NewRetentionLogger(cfg.Logging.Level, logpkg.LogRotationConfig{Filename: cfg.Logging.RetentionLog.Filename, MaxSize: cfg.Logging.RetentionLog.MaxSize, MaxBackups: cfg.Logging.RetentionLog.MaxBackups, MaxAge: cfg.Logging.RetentionLog.MaxAge, Compress: cfg.Logging.RetentionLog.Compress})
+	defer func() { _ = syncRetentionLogger() }()
 	db, err := database.InitPostgreSQL(&cfg.Database, logger)
 	if err != nil {
 		return err
@@ -81,8 +85,7 @@ func run(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	auditWriter := auditinfra.NewWriter(auditinfra.NewRegistry(), nil)
-	service, err := auditarchive.NewService(db, provider, simulationInstance, auditWriter)
+	service, err := auditarchive.NewService(db, provider, simulationInstance)
 	if err != nil {
 		return err
 	}
@@ -108,14 +111,17 @@ func run(ctx context.Context) error {
 		}
 	}

-	payload, _ := sonic.Marshal(taskapp.AuditMonthlyRetentionPayload{ArchiveMonth: simulationMonth})
-	dryRunTask := asynq.NewTask(constants.TaskTypeAuditMonthlyRetention, payload)
-	if err := taskapp.NewAuditMonthlyRetentionHandler(service, logger, false).Handle(ctx, dryRunTask); err != nil {
-		return fmt.Errorf("月度只读演练失败: %w", err)
-	}
-	dryRun, err := service.ValidateMonth(ctx, monthStart)
-	if err != nil {
-		return err
+	var dryRun auditarchive.RetentionResult
+	for date := monthStart; date.Before(monthEnd); date = date.AddDate(0, 0, 1) {
+		result, retainErr := service.RetainDate(ctx, date, false)
+		if retainErr != nil {
+			return fmt.Errorf("逐日只读演练失败: %w", retainErr)
+		}
+		retentionLogger.Info("日留存仿真只读校验通过", zap.String("archive_date", result.ArchiveDate))
+		dryRun.EventCount += result.EventCount
+		dryRun.ResourceCount += result.ResourceCount
+		dryRun.IntegrationCount += result.IntegrationCount
+		dryRun.EstimatedBatches += result.EstimatedBatches
 	}
 	summary, err := collectBeforeCleanup(ctx, db, monthStart, monthEnd, dryRun)
 	if err != nil {
@@ -125,16 +131,24 @@ func run(ctx context.Context) error {
 		return fmt.Errorf("只归档模式写入了 %d 个清理断点", summary.CleanupMarkersBefore)
 	}

-	cleanupTask := asynq.NewTask(constants.TaskTypeAuditMonthlyRetention, payload)
-	if err := taskapp.NewAuditMonthlyRetentionHandler(service, logger, true).Handle(ctx, cleanupTask); err != nil {
-		return fmt.Errorf("隔离测试库物理清理演练失败: %w", err)
+	for date := monthStart; date.Before(monthEnd); date = date.AddDate(0, 0, 1) {
+		result, retainErr := service.RetainDate(ctx, date, true)
+		if retainErr != nil {
+			return fmt.Errorf("隔离测试库逐日物理清理演练失败: %w", retainErr)
+		}
+		retentionLogger.Info("日留存仿真物理清理完成", zap.String("archive_date", result.ArchiveDate))
 	}
 	if err := collectAfterCleanup(ctx, db, monthStart, monthEnd, &summary); err != nil {
 		return err
 	}
-	if summary.TargetRowsAfterCleanup != 0 || summary.BoundaryRowsAfterCleanup != 6 || summary.CleanupMarkersAfter != int64(summary.Days*2) || !summary.RetentionAuditRecorded {
-		return fmt.Errorf("清理范围复核失败: target=%d boundary=%d markers=%d audit=%t",
-			summary.TargetRowsAfterCleanup, summary.BoundaryRowsAfterCleanup, summary.CleanupMarkersAfter, summary.RetentionAuditRecorded)
+	if summary.TargetRowsAfterCleanup != 0 || summary.BoundaryRowsAfterCleanup != 6 || summary.CleanupMarkersAfter != int64(summary.Days*2) {
+		return fmt.Errorf("清理范围复核失败: target=%d boundary=%d markers=%d", summary.TargetRowsAfterCleanup, summary.BoundaryRowsAfterCleanup, summary.CleanupMarkersAfter)
+	}
+	if err := syncRetentionLogger(); err != nil {
+		return fmt.Errorf("刷新日留存仿真日志失败: %w", err)
+	}
+	if _, err := os.Stat(cfg.Logging.RetentionLog.Filename); err != nil {
+		return fmt.Errorf("独立日留存日志未生成: %w", err)
 	}
 	encoded, _ := sonic.MarshalIndent(summary, "", "  ")
 	fmt.Println(string(encoded))
@@ -261,7 +275,12 @@ func archiveMonth(ctx context.Context, db *gorm.DB, service *auditarchive.Servic
 		Updates(map[string]any{"result": constants.IntegrationResultSuccess, "updated_at": time.Now()}).Error; err != nil {
 		return err
 	}
-	return service.FinalizeIntegrationMonth(ctx, start)
+	for date := start; date.Before(end); date = date.AddDate(0, 0, 1) {
+		if err := service.FinalizeIntegrationDate(ctx, date); err != nil {
+			return err
+		}
+	}
+	return nil
 }

 func collectBeforeCleanup(ctx context.Context, db *gorm.DB, start, end time.Time, dryRun auditarchive.RetentionResult) (simulationSummary, error) {
@@ -311,10 +330,5 @@ func collectAfterCleanup(ctx context.Context, db *gorm.DB, start, end time.Time,
 		Count(&summary.CleanupMarkersAfter).Error; err != nil {
 		return err
 	}
-	var auditCount int64
-	if err := db.WithContext(ctx).Model(&model.AuditEvent{}).Where("event_id = ?", "evt_retention_"+strings.ReplaceAll(simulationMonth, "-", "_")).Count(&auditCount).Error; err != nil {
-		return err
-	}
-	summary.RetentionAuditRecorded = auditCount == 1
 	return nil
 }
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -122,6 +122,11 @@ func runWorker(cfg *config.Config) {
 	defer func() {
 		_ = logger.Sync() // 忽略 sync 错误
 	}()
+	retentionLogger, syncRetentionLogger := logger.NewRetentionLogger(cfg.Logging.Level, logger.LogRotationConfig{
+		Filename: cfg.Logging.RetentionLog.Filename, MaxSize: cfg.Logging.RetentionLog.MaxSize,
+		MaxBackups: cfg.Logging.RetentionLog.MaxBackups, MaxAge: cfg.Logging.RetentionLog.MaxAge, Compress: cfg.Logging.RetentionLog.Compress,
+	})
+	defer func() { _ = syncRetentionLogger() }()

 	appLogger := logger.GetAppLogger()
 	ctx, cancel := context.WithCancel(context.Background())
@@ -148,7 +153,7 @@ func runWorker(cfg *config.Config) {
 	taskHandler.RegisterHandlers()
 	registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
 	registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
-	registerAuditArchiveTask(taskHandler.GetMux(), runtime, cfg.Worker.AuditRetentionCleanupEnabled, appLogger)
+	registerAuditArchiveTask(taskHandler.GetMux(), runtime, cfg.Worker.AuditRetentionCleanupEnabled, appLogger, retentionLogger)
 	outboxHandler := outbox.NewHandler(runtime.outboxConsumers)
 	taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle)
 	startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger)
@@ -794,43 +799,32 @@ func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error {
 	)); err != nil {
 		return fmt.Errorf("注册 Integration Log 每日冷归档定时任务失败: %w", err)
 	}
-	if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 5 1 * *", asynq.NewTask(
-		constants.TaskTypeIntegrationMonthlyFinalize,
-		nil,
-		asynq.MaxRetry(10),
-		asynq.Timeout(6*time.Hour),
-		asynq.Unique(27*24*time.Hour),
-		asynq.Queue(constants.QueueForTaskType(constants.TaskTypeIntegrationMonthlyFinalize)),
-	)); err != nil {
-		return fmt.Errorf("注册 Integration Log 月度最终版本复核任务失败: %w", err)
-	}
-	if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 6 1 * *", asynq.NewTask(
-		constants.TaskTypeAuditMonthlyRetention,
+	if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 5 * * *", asynq.NewTask(
+		constants.TaskTypeAuditDailyRetention,
 		nil,
 		asynq.MaxRetry(10),
 		asynq.Timeout(12*time.Hour),
-		asynq.Unique(27*24*time.Hour),
-		asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAuditMonthlyRetention)),
+		asynq.Unique(23*time.Hour),
+		asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAuditDailyRetention)),
 	)); err != nil {
-		return fmt.Errorf("注册月度日志留存演练或清理任务失败: %w", err)
+		return fmt.Errorf("注册日志日留存演练或清理任务失败: %w", err)
 	}
 	return nil
 }

 // registerAuditArchiveTask 注册 Audit 与 Integration 冷归档任务处理器。
-func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, cleanupEnabled bool, appLogger *zap.Logger) {
+func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, cleanupEnabled bool, appLogger, retentionLogger *zap.Logger) {
 	if runtime.storageSvc == nil {
 		appLogger.Warn("对象存储未配置，审计归档任务将在执行时重试")
 		mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(nil, appLogger).Handle)
 		integrationHandler := task.NewIntegrationArchiveHandler(nil, appLogger)
 		mux.HandleFunc(constants.TaskTypeIntegrationDailyArchive, integrationHandler.HandleDaily)
-		mux.HandleFunc(constants.TaskTypeIntegrationMonthlyFinalize, integrationHandler.HandleMonthlyFinalize)
-		mux.HandleFunc(constants.TaskTypeAuditMonthlyRetention, task.NewAuditMonthlyRetentionHandler(nil, appLogger, cleanupEnabled).Handle)
+		mux.HandleFunc(constants.TaskTypeAuditDailyRetention, task.NewAuditRetentionHandler(nil, retentionLogger, cleanupEnabled).Handle)
 		return
 	}
 	auditWriter, ok := runtime.workerResult.Services.PaymentAudit.(*auditInfra.Writer)
 	if !ok || auditWriter == nil {
-		appLogger.Fatal("初始化月度日志留存清理失败：统一审计 Writer 未配置")
+		appLogger.Fatal("初始化日志日留存清理失败：统一审计 Writer 未配置")
 	}
 	service, err := auditArchiveApp.NewService(runtime.db, runtime.storageSvc.Provider(), constants.AuditArchiveInstanceID, auditWriter)
 	if err != nil {
@@ -839,13 +833,11 @@ func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, clean
 	mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(service, appLogger).Handle)
 	integrationHandler := task.NewIntegrationArchiveHandler(service, appLogger)
 	mux.HandleFunc(constants.TaskTypeIntegrationDailyArchive, integrationHandler.HandleDaily)
-	mux.HandleFunc(constants.TaskTypeIntegrationMonthlyFinalize, integrationHandler.HandleMonthlyFinalize)
-	mux.HandleFunc(constants.TaskTypeAuditMonthlyRetention, task.NewAuditMonthlyRetentionHandler(service, appLogger, cleanupEnabled).Handle)
+	mux.HandleFunc(constants.TaskTypeAuditDailyRetention, task.NewAuditRetentionHandler(service, retentionLogger, cleanupEnabled).Handle)
 	appLogger.Info("注册审计归档任务处理器",
 		zap.String("audit_task_type", constants.TaskTypeAuditDailyArchive),
 		zap.String("integration_daily_task_type", constants.TaskTypeIntegrationDailyArchive),
-		zap.String("integration_monthly_task_type", constants.TaskTypeIntegrationMonthlyFinalize),
-		zap.String("retention_task_type", constants.TaskTypeAuditMonthlyRetention),
+		zap.String("retention_task_type", constants.TaskTypeAuditDailyRetention),
 		zap.Bool("retention_cleanup_enabled", cleanupEnabled))
 }