每日清理
This commit is contained in:
@@ -2,101 +2,47 @@ package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
|
||||
)
|
||||
|
||||
// AuditMonthlyRetentionPayload 是人工补跑月度清理时可选的任务载荷。
|
||||
type AuditMonthlyRetentionPayload struct {
|
||||
ArchiveMonth string `json:"archive_month"`
|
||||
}
|
||||
|
||||
// AuditMonthlyRetentionHandler 处理归档完整性门禁与上月在线日志物理清理。
|
||||
type AuditMonthlyRetentionHandler struct {
|
||||
// AuditRetentionHandler 连续校验并按日物理清理已结束的在线日志。
|
||||
type AuditRetentionHandler struct {
|
||||
service *auditarchive.Service
|
||||
logger *zap.Logger
|
||||
cleanupEnabled bool
|
||||
}
|
||||
|
||||
// NewAuditMonthlyRetentionHandler 创建月度日志留存清理处理器。
|
||||
func NewAuditMonthlyRetentionHandler(service *auditarchive.Service, logger *zap.Logger, cleanupEnabled bool) *AuditMonthlyRetentionHandler {
|
||||
return &AuditMonthlyRetentionHandler{service: service, logger: logger, cleanupEnabled: cleanupEnabled}
|
||||
// NewAuditRetentionHandler 创建日留存处理器。
|
||||
func NewAuditRetentionHandler(service *auditarchive.Service, logger *zap.Logger, cleanupEnabled bool) *AuditRetentionHandler {
|
||||
return &AuditRetentionHandler{service: service, logger: logger, cleanupEnabled: cleanupEnabled}
|
||||
}
|
||||
|
||||
// Handle 校验整月归档后按固定顺序分批物理删除 PostgreSQL 在线日志。
|
||||
func (h *AuditMonthlyRetentionHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
if !h.cleanupEnabled {
|
||||
return h.handleDryRun(ctx, task)
|
||||
}
|
||||
// Handle 在关闭清理开关时只读校验，开启后从最早未完成日连续删除。
|
||||
func (h *AuditRetentionHandler) Handle(ctx context.Context, _ *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("月度日志留存清理服务未配置")
|
||||
return fmt.Errorf("日志日留存服务未配置")
|
||||
}
|
||||
startedAt := time.Now()
|
||||
var result auditarchive.RetentionResult
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
result, err = h.service.CleanupPreviousMonth(ctx)
|
||||
} else {
|
||||
var payload AuditMonthlyRetentionPayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析月度日志留存清理任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
result, err = h.service.CleanupMonth(ctx, month)
|
||||
}
|
||||
fields := []zap.Field{
|
||||
zap.String("archive_month", result.Month), zap.Int64("audit_event_count", result.EventCount),
|
||||
zap.Int64("event_resource_count", result.ResourceCount), zap.Int64("integration_log_count", result.IntegrationCount),
|
||||
zap.Duration("duration", time.Since(startedAt)), zap.Int("manifest_count", len(result.ManifestKeys)),
|
||||
results, err := h.service.RetainPendingDays(ctx, h.cleanupEnabled)
|
||||
for _, result := range results {
|
||||
h.logger.Info("日志日留存处理完成",
|
||||
zap.String("archive_date", result.ArchiveDate), zap.Bool("cleanup_enabled", h.cleanupEnabled),
|
||||
zap.Int64("audit_event_count", result.EventCount), zap.Int64("event_resource_count", result.ResourceCount),
|
||||
zap.Int64("integration_log_count", result.IntegrationCount), zap.Duration("duration", result.Duration))
|
||||
}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.String("severity", "critical"), zap.Error(err))
|
||||
h.logger.Error("月度日志留存清理失败，PostgreSQL 整月清理已阻断或等待断点续跑", fields...)
|
||||
var blocked *auditarchive.RetentionBlockedError
|
||||
if stderrors.As(err, &blocked) {
|
||||
h.logger.Error("日志日留存日期推进已阻断", zap.String("archive_date", blocked.ArchiveDate), zap.String("source", blocked.Source), zap.String("failure_category", "archive_or_validation"), zap.Error(err))
|
||||
} else {
|
||||
h.logger.Error("日志日留存日期推进已阻断", zap.String("source", "retention"), zap.String("failure_category", "internal"), zap.Error(err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
h.logger.Info("月度日志留存清理完成", fields...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *AuditMonthlyRetentionHandler) handleDryRun(ctx context.Context, task *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("月度日志留存演练服务未配置")
|
||||
}
|
||||
var result auditarchive.RetentionResult
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
result, err = h.service.ValidatePreviousMonth(ctx)
|
||||
} else {
|
||||
var payload AuditMonthlyRetentionPayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析月度日志留存演练任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
result, err = h.service.ValidateMonth(ctx, month)
|
||||
}
|
||||
fields := []zap.Field{
|
||||
zap.Bool("cleanup_enabled", false), zap.String("archive_month", result.Month),
|
||||
zap.Int64("audit_event_count", result.EventCount), zap.Int64("event_resource_count", result.ResourceCount),
|
||||
zap.Int64("integration_log_count", result.IntegrationCount), zap.Int64("estimated_cleanup_batches", result.EstimatedBatches),
|
||||
zap.Duration("validation_duration", result.Duration), zap.Int("manifest_count", len(result.ManifestKeys)),
|
||||
}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.String("severity", "critical"), zap.Error(err))
|
||||
h.logger.Error("审计日志月度只读演练失败，物理清理保持关闭", fields...)
|
||||
return err
|
||||
}
|
||||
h.logger.Info("审计日志月度只读演练通过，物理清理保持关闭", fields...)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -18,12 +18,7 @@ type IntegrationDailyArchivePayload struct {
|
||||
ArchiveDate string `json:"archive_date"`
|
||||
}
|
||||
|
||||
// IntegrationMonthlyFinalizePayload 是人工月度复核时可选的任务载荷。
|
||||
type IntegrationMonthlyFinalizePayload struct {
|
||||
ArchiveMonth string `json:"archive_month"`
|
||||
}
|
||||
|
||||
// IntegrationArchiveHandler 处理 Integration Log 每日归档与月度最终复核。
|
||||
// IntegrationArchiveHandler 处理 Integration Log 每日归档。
|
||||
type IntegrationArchiveHandler struct {
|
||||
service *auditarchive.Service
|
||||
logger *zap.Logger
|
||||
@@ -61,33 +56,6 @@ func (h *IntegrationArchiveHandler) HandleDaily(ctx context.Context, task *asynq
|
||||
return nil
|
||||
}
|
||||
|
||||
// HandleMonthlyFinalize 执行上一个完整自然月复核，或按任务载荷复核指定月份。
|
||||
func (h *IntegrationArchiveHandler) HandleMonthlyFinalize(ctx context.Context, task *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("Integration Log 归档服务未配置")
|
||||
}
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
err = h.service.FinalizePreviousIntegrationMonth(ctx)
|
||||
} else {
|
||||
var payload IntegrationMonthlyFinalizePayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析 Integration Log 月度复核任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
err = h.service.FinalizeIntegrationMonth(ctx, month)
|
||||
}
|
||||
if err != nil {
|
||||
h.logger.Error("Integration Log 月度最终版本复核失败，后续清理必须阻止", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
h.logger.Info("Integration Log 月度最终版本复核完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseArchiveDate(value string) (time.Time, error) {
|
||||
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if err != nil {
|
||||
@@ -99,15 +67,3 @@ func parseArchiveDate(value string) (time.Time, error) {
|
||||
}
|
||||
return date, nil
|
||||
}
|
||||
|
||||
func parseArchiveMonth(value string) (time.Time, error) {
|
||||
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err)
|
||||
}
|
||||
month, err := time.ParseInLocation("2006-01", value, location)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("解析 Integration Log 归档月份失败: %w", err)
|
||||
}
|
||||
return month, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user