436 lines
18 KiB
Go
436 lines
18 KiB
Go
package auditarchive
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/bytedance/sonic"
|
|
|
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
|
)
|
|
|
|
const maxManifestBytes = 1024 * 1024
|
|
|
|
// RetentionAudit 保留系统审计 Writer 的输入兼容类型。
|
|
type RetentionAudit struct {
|
|
EventID, Month, Summary, Result, ErrorSummary string
|
|
RangeStart, RangeEnd time.Time
|
|
EventCount, ResourceCount, IntegrationCount int64
|
|
ManifestKeys []string
|
|
DurationMS int64
|
|
}
|
|
|
|
// RetentionResult 是单日留存处理结果。
|
|
type RetentionResult struct {
|
|
ArchiveDate string
|
|
EventCount int64
|
|
ResourceCount int64
|
|
IntegrationCount int64
|
|
EstimatedBatches int64
|
|
ManifestKeys []string
|
|
Duration time.Duration
|
|
}
|
|
|
|
// RetentionBlockedError 描述阻断日期推进的安全上下文。
|
|
type RetentionBlockedError struct {
|
|
ArchiveDate string
|
|
Source string
|
|
Err error
|
|
}
|
|
|
|
func (e *RetentionBlockedError) Error() string {
|
|
return e.ArchiveDate + " " + e.Source + ": " + e.Err.Error()
|
|
}
|
|
func (e *RetentionBlockedError) Unwrap() error { return e.Err }
|
|
|
|
// RetainPendingDays 每次最多处理一个最早待处理的上海自然日，避免单任务跨历史日期长时间占用数据库。
|
|
func (s *Service) RetainPendingDays(ctx context.Context, cleanup bool) ([]RetentionResult, error) {
|
|
if s.db == nil || s.store == nil {
|
|
return nil, fmt.Errorf("日志日留存数据库或对象存储未配置")
|
|
}
|
|
start, end, err := s.pendingRetentionRange(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !start.Before(end) {
|
|
return nil, nil
|
|
}
|
|
result, err := s.retainDate(ctx, start, cleanup)
|
|
if err != nil {
|
|
return nil, &RetentionBlockedError{ArchiveDate: start.Format(time.DateOnly), Source: constants.AuditArchiveSource, Err: err}
|
|
}
|
|
return []RetentionResult{result}, nil
|
|
}
|
|
|
|
// RetainDate 校验并按需清理指定已结束自然日，供受控演练使用。
|
|
func (s *Service) RetainDate(ctx context.Context, date time.Time, cleanup bool) (RetentionResult, error) {
|
|
return s.retainDate(ctx, date, cleanup)
|
|
}
|
|
|
|
func (s *Service) pendingRetentionRange(ctx context.Context) (time.Time, time.Time, error) {
|
|
today := time.Now().In(s.location)
|
|
end := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)
|
|
var earliest *time.Time
|
|
for _, item := range []struct{ table, column, condition string }{
|
|
{"tb_audit_event", "created_at", ""},
|
|
{"tb_integration_log", "created_at", "result <> 'pending'"},
|
|
} {
|
|
query := s.db.WithContext(ctx).Table(item.table).Select("MIN(" + item.column + ")")
|
|
if item.condition != "" {
|
|
query = query.Where(item.condition)
|
|
}
|
|
var value *time.Time
|
|
if err := query.Scan(&value).Error; err != nil {
|
|
return time.Time{}, time.Time{}, fmt.Errorf("查询日留存起点失败: %w", err)
|
|
}
|
|
if value != nil && (earliest == nil || value.Before(*earliest)) {
|
|
local := value.In(s.location)
|
|
earliest = &local
|
|
}
|
|
}
|
|
if earliest == nil {
|
|
return end, end, nil
|
|
}
|
|
start := time.Date(earliest.Year(), earliest.Month(), earliest.Day(), 0, 0, 0, 0, s.location)
|
|
return start, end, nil
|
|
}
|
|
|
|
func (s *Service) retainDate(ctx context.Context, date time.Time, cleanup bool) (RetentionResult, error) {
|
|
startedAt := time.Now()
|
|
auditResult, err := s.retainAuditDate(ctx, date, cleanup)
|
|
if err != nil {
|
|
return RetentionResult{}, err
|
|
}
|
|
integrationResult, err := s.retainIntegrationDate(ctx, date, cleanup)
|
|
if err != nil {
|
|
return RetentionResult{}, err
|
|
}
|
|
result := RetentionResult{ArchiveDate: date.Format(time.DateOnly), EventCount: auditResult.EventCount, ResourceCount: auditResult.ResourceCount, IntegrationCount: integrationResult.IntegrationCount, ManifestKeys: append(auditResult.ManifestKeys, integrationResult.ManifestKeys...)}
|
|
result.EstimatedBatches = estimatedRetentionBatches(result)
|
|
result.Duration = time.Since(startedAt)
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Service) retainAuditDate(ctx context.Context, date time.Time, cleanup bool) (RetentionResult, error) {
|
|
run, err := s.retentionRun(ctx, date, constants.AuditArchiveSource)
|
|
if err != nil {
|
|
return RetentionResult{}, err
|
|
}
|
|
if run == nil || run.Status != constants.ArchiveStatusSuccess {
|
|
if err := s.ArchiveDate(ctx, date); err != nil {
|
|
return RetentionResult{}, fmt.Errorf("Audit 归档失败: %w", err)
|
|
}
|
|
run, err = s.retentionRun(ctx, date, constants.AuditArchiveSource)
|
|
if err != nil {
|
|
return RetentionResult{}, err
|
|
}
|
|
}
|
|
if err := s.validateAuditRetentionDay(ctx, date, run); err != nil {
|
|
return RetentionResult{}, fmt.Errorf("Audit 完整性校验失败: %w", err)
|
|
}
|
|
result := RetentionResult{ArchiveDate: date.Format(time.DateOnly), EventCount: run.EventCount, ResourceCount: run.ResourceCount, ManifestKeys: []string{run.ManifestKey}}
|
|
if cleanup {
|
|
if err := s.cleanupAuditDate(ctx, date, run); err != nil {
|
|
return result, err
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Service) retainIntegrationDate(ctx context.Context, date time.Time, cleanup bool) (RetentionResult, error) {
|
|
run, err := s.retentionRun(ctx, date, constants.IntegrationArchiveSource)
|
|
if err != nil {
|
|
return RetentionResult{}, err
|
|
}
|
|
if run == nil || run.Status != constants.ArchiveStatusSuccess {
|
|
if err := s.ArchiveIntegrationDate(ctx, date); err != nil {
|
|
return RetentionResult{}, fmt.Errorf("Integration Log 归档失败: %w", err)
|
|
}
|
|
}
|
|
if err := s.FinalizeIntegrationDate(ctx, date); err != nil {
|
|
return RetentionResult{}, fmt.Errorf("Integration Log 最终归档失败: %w", err)
|
|
}
|
|
run, err = s.retentionRun(ctx, date, constants.IntegrationArchiveSource)
|
|
if err != nil {
|
|
return RetentionResult{}, err
|
|
}
|
|
if err := s.validateIntegrationRetentionDay(ctx, date, run); err != nil {
|
|
return RetentionResult{}, fmt.Errorf("Integration Log 完整性校验失败: %w", err)
|
|
}
|
|
result := RetentionResult{ArchiveDate: date.Format(time.DateOnly), IntegrationCount: run.RecordCount, ManifestKeys: []string{run.ManifestKey}}
|
|
if cleanup {
|
|
if err := s.cleanupIntegrationDate(ctx, date, run); err != nil {
|
|
return result, err
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Service) retentionRun(ctx context.Context, date time.Time, source string) (*model.LogArchiveRun, error) {
|
|
var runs []model.LogArchiveRun
|
|
if err := s.db.WithContext(ctx).Where("source = ? AND archive_date = ? AND instance_id = ?", source, date.Format(time.DateOnly), s.instanceID).Find(&runs).Error; err != nil {
|
|
return nil, fmt.Errorf("读取日归档账本失败: %w", err)
|
|
}
|
|
if len(runs) == 0 {
|
|
return nil, nil
|
|
}
|
|
return &runs[0], nil
|
|
}
|
|
|
|
func (s *Service) validateAuditRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
|
|
if err := validateRunBase(run, date, constants.AuditArchiveSchemaVersion, false); err != nil {
|
|
return err
|
|
}
|
|
if err := s.validateAuditManifest(ctx, run); err != nil {
|
|
return err
|
|
}
|
|
events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return validateRemainingCounts(run, events, resources)
|
|
}
|
|
|
|
func (s *Service) validateIntegrationRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
|
|
if err := validateRunBase(run, date, constants.IntegrationArchiveSchemaVersion, true); err != nil {
|
|
return err
|
|
}
|
|
if err := s.validateIntegrationManifest(ctx, run); err != nil {
|
|
return err
|
|
}
|
|
count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
terminalCount, err := s.integrationTerminalCount(ctx, run.RangeStart, run.RangeEnd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if run.CleanedAt != nil && terminalCount != 0 {
|
|
return fmt.Errorf("已标记清理完成但数据库仍有 %d 条终态记录", terminalCount)
|
|
}
|
|
if run.CleanupStartedAt != nil && count > run.RecordCount {
|
|
return fmt.Errorf("续跑窗口记录数超过最终归档数量")
|
|
}
|
|
if terminalCount > 0 && run.CleanupStartedAt == nil {
|
|
file, err := s.buildIntegrationArchiveFile(ctx, run.RangeStart, run.RangeEnd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.Remove(file.path)
|
|
if file.recordCount != run.RecordCount || file.sha256 != run.SHA256 {
|
|
return fmt.Errorf("数据库当前 Integration 内容与最终 revision 不一致")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRunBase(run *model.LogArchiveRun, date time.Time, schema string, final bool) error {
|
|
if run.Status != constants.ArchiveStatusSuccess || run.SchemaVersion != schema {
|
|
return fmt.Errorf("归档状态或 schema version 不符合清理要求")
|
|
}
|
|
if run.ArchiveDate.Format(time.DateOnly) != date.Format(time.DateOnly) || !run.RangeStart.Equal(date) || !run.RangeEnd.Equal(date.AddDate(0, 0, 1)) {
|
|
return fmt.Errorf("归档日期或半开时间范围不一致")
|
|
}
|
|
if final && !run.IsFinal {
|
|
return fmt.Errorf("Integration 最终 revision 尚未形成")
|
|
}
|
|
if run.ObjectKey == "" || run.ManifestKey == "" || run.SHA256 == "" {
|
|
return fmt.Errorf("归档对象、manifest 或 SHA-256 缺失")
|
|
}
|
|
return nil
|
|
}
|
|
func validateRemainingCounts(run *model.LogArchiveRun, events, resources int64) error {
|
|
if run.CleanedAt != nil && (events != 0 || resources != 0) {
|
|
return fmt.Errorf("已标记清理完成但数据库仍有事件或资源")
|
|
}
|
|
if run.CleanupStartedAt != nil && (events > run.EventCount || resources > run.ResourceCount) {
|
|
return fmt.Errorf("续跑窗口数量超过已归档数量")
|
|
}
|
|
if run.CleanupStartedAt == nil && (events != run.EventCount || resources != run.ResourceCount) {
|
|
return fmt.Errorf("数据库事件或资源数量与 manifest 不一致")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) validateAuditManifest(ctx context.Context, run *model.LogArchiveRun) error {
|
|
var manifest Manifest
|
|
if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil {
|
|
return err
|
|
}
|
|
if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone || manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) || manifest.EventCount != run.EventCount || manifest.ResourceCount != run.ResourceCount || manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision {
|
|
return fmt.Errorf("Audit manifest 与 ledger 不一致")
|
|
}
|
|
if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision)}); err != nil {
|
|
return err
|
|
}
|
|
return s.verifyRetentionObject(ctx, run, false)
|
|
}
|
|
func (s *Service) validateIntegrationManifest(ctx context.Context, run *model.LogArchiveRun) error {
|
|
var manifest integrationArchiveManifest
|
|
if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil {
|
|
return err
|
|
}
|
|
if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || !manifest.Final || manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone || manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) || manifest.RecordCount != run.RecordCount || manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision {
|
|
return fmt.Errorf("Integration manifest 与最终 ledger 不一致")
|
|
}
|
|
if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{"source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision), "final": "true"}); err != nil {
|
|
return err
|
|
}
|
|
return s.verifyRetentionObject(ctx, run, true)
|
|
}
|
|
func (s *Service) readManifest(ctx context.Context, key string, target any) error {
|
|
object, err := s.store.Stat(ctx, key)
|
|
if err != nil {
|
|
return fmt.Errorf("读取 manifest metadata 失败: %w", err)
|
|
}
|
|
reader, err := s.store.Download(ctx, key)
|
|
if err != nil {
|
|
return fmt.Errorf("下载 manifest 失败: %w", err)
|
|
}
|
|
data, readErr := io.ReadAll(io.LimitReader(reader, maxManifestBytes+1))
|
|
closeErr := reader.Close()
|
|
if readErr != nil {
|
|
return fmt.Errorf("读取 manifest 失败: %w", readErr)
|
|
}
|
|
if closeErr != nil {
|
|
return fmt.Errorf("关闭 manifest 对象失败: %w", closeErr)
|
|
}
|
|
if len(data) > maxManifestBytes || int64(len(data)) != object.Size {
|
|
return fmt.Errorf("manifest 大小非法或不完整")
|
|
}
|
|
if err := sonic.Unmarshal(data, target); err != nil {
|
|
return fmt.Errorf("解析 manifest 失败: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Service) verifyRetentionObject(ctx context.Context, run *model.LogArchiveRun, final bool) error {
|
|
metadata := map[string]string{"schema-version": run.SchemaVersion, "source": run.Source, "archive-date": run.RangeStart.Format(time.DateOnly), "timezone": constants.AuditArchiveTimezone, "sha256": run.SHA256, "revision": strconv.Itoa(run.Revision)}
|
|
if run.Source == constants.AuditArchiveSource {
|
|
metadata["event-count"] = strconv.FormatInt(run.EventCount, 10)
|
|
metadata["resource-count"] = strconv.FormatInt(run.ResourceCount, 10)
|
|
} else {
|
|
metadata["record-count"] = strconv.FormatInt(run.RecordCount, 10)
|
|
metadata["final"] = strconv.FormatBool(final)
|
|
}
|
|
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
|
|
return err
|
|
}
|
|
reader, err := s.store.Download(ctx, run.ObjectKey)
|
|
if err != nil {
|
|
return fmt.Errorf("下载归档对象复核 SHA-256 失败: %w", err)
|
|
}
|
|
hasher := sha256.New()
|
|
written, copyErr := io.Copy(hasher, reader)
|
|
closeErr := reader.Close()
|
|
if copyErr != nil {
|
|
return fmt.Errorf("读取归档对象复核 SHA-256 失败: %w", copyErr)
|
|
}
|
|
if closeErr != nil {
|
|
return fmt.Errorf("关闭归档对象失败: %w", closeErr)
|
|
}
|
|
if written != run.CompressedBytes || fmt.Sprintf("%x", hasher.Sum(nil)) != run.SHA256 {
|
|
return fmt.Errorf("归档对象大小或 SHA-256 复核失败")
|
|
}
|
|
return nil
|
|
}
|
|
func estimatedRetentionBatches(result RetentionResult) int64 {
|
|
batch := int64(constants.AuditRetentionDeleteBatchSize)
|
|
return (result.EventCount+batch-1)/batch + (result.ResourceCount+batch-1)/batch + (result.IntegrationCount+batch-1)/batch
|
|
}
|
|
func (s *Service) cleanupAuditDate(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
|
|
if run.CleanedAt != nil {
|
|
return nil
|
|
}
|
|
if err := s.markCleanupStarted(ctx, constants.AuditArchiveSource, date); err != nil {
|
|
return err
|
|
}
|
|
if err := s.deleteAuditResources(ctx, date, date.AddDate(0, 0, 1)); err != nil {
|
|
return err
|
|
}
|
|
if err := s.deleteAuditEvents(ctx, date, date.AddDate(0, 0, 1)); err != nil {
|
|
return err
|
|
}
|
|
return s.markCleaned(ctx, constants.AuditArchiveSource, date)
|
|
}
|
|
func (s *Service) cleanupIntegrationDate(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
|
|
terminalCount, err := s.integrationTerminalCount(ctx, date, date.AddDate(0, 0, 1))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if run.CleanedAt != nil && terminalCount == 0 {
|
|
return nil
|
|
}
|
|
if err := s.markCleanupStarted(ctx, constants.IntegrationArchiveSource, date); err != nil {
|
|
return err
|
|
}
|
|
end := date.AddDate(0, 0, 1)
|
|
for {
|
|
subquery := s.db.Model(&model.IntegrationLog{}).Select("id").Where("created_at >= ? AND created_at < ? AND result <> ?", date, end, constants.IntegrationResultPending).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
|
|
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.IntegrationLog{})
|
|
if deleted.Error != nil {
|
|
return fmt.Errorf("分批物理删除 Integration Log 失败: %w", deleted.Error)
|
|
}
|
|
if deleted.RowsAffected == 0 {
|
|
break
|
|
}
|
|
}
|
|
return s.markCleaned(ctx, constants.IntegrationArchiveSource, date)
|
|
}
|
|
func (s *Service) integrationTerminalCount(ctx context.Context, start, end time.Time) (int64, error) {
|
|
var count int64
|
|
if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}).
|
|
Where("created_at >= ? AND created_at < ? AND result <> ?", start, end, constants.IntegrationResultPending).
|
|
Count(&count).Error; err != nil {
|
|
return 0, fmt.Errorf("统计终态 Integration Log 失败: %w", err)
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
func (s *Service) deleteAuditResources(ctx context.Context, start, end time.Time) error {
|
|
for {
|
|
subquery := s.db.Model(&model.AuditEventResource{}).Select("tb_audit_event_resource.id").Joins("JOIN tb_audit_event ON tb_audit_event.id = tb_audit_event_resource.audit_event_id").Where("tb_audit_event.created_at >= ? AND tb_audit_event.created_at < ?", start, end).Order("tb_audit_event_resource.id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
|
|
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEventResource{})
|
|
if deleted.Error != nil {
|
|
return fmt.Errorf("分批物理删除 Audit Event Resource 失败: %w", deleted.Error)
|
|
}
|
|
if deleted.RowsAffected == 0 {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
func (s *Service) deleteAuditEvents(ctx context.Context, start, end time.Time) error {
|
|
for {
|
|
subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
|
|
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEvent{})
|
|
if deleted.Error != nil {
|
|
return fmt.Errorf("分批物理删除 Audit Event 失败: %w", deleted.Error)
|
|
}
|
|
if deleted.RowsAffected == 0 {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
func (s *Service) markCleanupStarted(ctx context.Context, source string, date time.Time) error {
|
|
now := time.Now()
|
|
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("source = ? AND archive_date = ? AND instance_id = ? AND cleanup_started_at IS NULL", source, date.Format(time.DateOnly), s.instanceID).Updates(map[string]any{"cleanup_started_at": now, "updated_at": now})
|
|
if result.Error != nil {
|
|
return fmt.Errorf("记录日清理开始断点失败: %w", result.Error)
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Service) markCleaned(ctx context.Context, source string, date time.Time) error {
|
|
now := time.Now()
|
|
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("source = ? AND archive_date = ? AND instance_id = ? AND cleanup_started_at IS NOT NULL", source, date.Format(time.DateOnly), s.instanceID).Updates(map[string]any{"cleaned_at": now, "updated_at": now})
|
|
if result.Error != nil {
|
|
return fmt.Errorf("记录日清理完成断点失败: %w", result.Error)
|
|
}
|
|
return nil
|
|
}
|