package auditarchive import ( "context" "crypto/sha256" "fmt" "io" "os" "strconv" "strings" "time" "github.com/bytedance/sonic" "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/pkg/constants" ) const maxManifestBytes = 1024 * 1024 // RetentionAudit 保留系统审计 Writer 的输入兼容类型。 type RetentionAudit struct { EventID, Month, Summary, Result, ErrorSummary string RangeStart, RangeEnd time.Time EventCount, ResourceCount, IntegrationCount int64 ManifestKeys []string DurationMS int64 } // RetentionResult 是单日留存处理结果。 type RetentionResult struct { ArchiveDate string EventCount int64 ResourceCount int64 IntegrationCount int64 EstimatedBatches int64 ManifestKeys []string Duration time.Duration } // RetentionBlockedError 描述阻断日期推进的安全上下文。 type RetentionBlockedError struct { ArchiveDate string Source string Err error } func (e *RetentionBlockedError) Error() string { return e.ArchiveDate + " " + e.Source + ": " + e.Err.Error() } func (e *RetentionBlockedError) Unwrap() error { return e.Err } type retentionRuns struct { audit *model.LogArchiveRun integration *model.LogArchiveRun } // RetainPendingDays 从最早仍在线的日期连续处理至昨天。cleanup 为 false 时只读校验。 func (s *Service) RetainPendingDays(ctx context.Context, cleanup bool) ([]RetentionResult, error) { if s.db == nil || s.store == nil { return nil, fmt.Errorf("日志日留存数据库或对象存储未配置") } start, end, err := s.pendingRetentionRange(ctx) if err != nil { return nil, err } results := make([]RetentionResult, 0) for date := start; date.Before(end); date = date.AddDate(0, 0, 1) { result, err := s.retainDate(ctx, date, cleanup) if err != nil { source := "retention" if strings.HasPrefix(err.Error(), "Audit") { source = constants.AuditArchiveSource } if strings.HasPrefix(err.Error(), "Integration") { source = constants.IntegrationArchiveSource } return results, &RetentionBlockedError{ArchiveDate: date.Format(time.DateOnly), Source: source, Err: err} } results = append(results, result) } return results, nil } // RetainDate 校验并按需清理指定已结束自然日,供受控演练使用。 func (s *Service) RetainDate(ctx context.Context, date time.Time, cleanup bool) (RetentionResult, error) { return s.retainDate(ctx, date, cleanup) } func (s *Service) pendingRetentionRange(ctx context.Context) (time.Time, time.Time, error) { today := time.Now().In(s.location) end := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location) var earliest *time.Time for _, item := range []struct{ table, column string }{{"tb_audit_event", "created_at"}, {"tb_integration_log", "created_at"}} { var value *time.Time if err := s.db.WithContext(ctx).Table(item.table).Select("MIN(" + item.column + ")").Scan(&value).Error; err != nil { return time.Time{}, time.Time{}, fmt.Errorf("查询日留存起点失败: %w", err) } if value != nil && (earliest == nil || value.Before(*earliest)) { local := value.In(s.location) earliest = &local } } if earliest == nil { return end, end, nil } start := time.Date(earliest.Year(), earliest.Month(), earliest.Day(), 0, 0, 0, 0, s.location) return start, end, nil } func (s *Service) retainDate(ctx context.Context, date time.Time, cleanup bool) (RetentionResult, error) { startedAt := time.Now() var existing []model.LogArchiveRun if err := s.db.WithContext(ctx).Where("source IN ? AND archive_date = ? AND instance_id = ?", []string{constants.AuditArchiveSource, constants.IntegrationArchiveSource}, date.Format(time.DateOnly), s.instanceID).Find(&existing).Error; err != nil { return RetentionResult{}, fmt.Errorf("读取日归档账本失败: %w", err) } statuses := map[string]string{} for _, run := range existing { statuses[run.Source] = run.Status } if statuses[constants.AuditArchiveSource] != constants.ArchiveStatusSuccess { if err := s.ArchiveDate(ctx, date); err != nil { return RetentionResult{}, fmt.Errorf("Audit 归档失败: %w", err) } } if statuses[constants.IntegrationArchiveSource] != constants.ArchiveStatusSuccess { if err := s.ArchiveIntegrationDate(ctx, date); err != nil { return RetentionResult{}, fmt.Errorf("Integration Log 归档失败: %w", err) } } if err := s.FinalizeIntegrationDate(ctx, date); err != nil { return RetentionResult{}, fmt.Errorf("Integration Log 最终归档失败: %w", err) } runs, err := s.loadRetentionRuns(ctx, date) if err != nil { return RetentionResult{}, err } if err := s.validateAuditRetentionDay(ctx, date, runs.audit); err != nil { return RetentionResult{}, fmt.Errorf("Audit 完整性校验失败: %w", err) } if err := s.validateIntegrationRetentionDay(ctx, date, runs.integration); err != nil { return RetentionResult{}, fmt.Errorf("Integration Log 完整性校验失败: %w", err) } result := RetentionResult{ArchiveDate: date.Format(time.DateOnly), EventCount: runs.audit.EventCount, ResourceCount: runs.audit.ResourceCount, IntegrationCount: runs.integration.RecordCount, ManifestKeys: []string{runs.audit.ManifestKey, runs.integration.ManifestKey}} result.EstimatedBatches = estimatedRetentionBatches(result) if cleanup { if err := s.cleanupAuditDate(ctx, date, runs.audit); err != nil { return result, err } if err := s.cleanupIntegrationDate(ctx, date, runs.integration); err != nil { return result, err } } result.Duration = time.Since(startedAt) return result, nil } func (s *Service) loadRetentionRuns(ctx context.Context, date time.Time) (retentionRuns, error) { var rows []model.LogArchiveRun if err := s.db.WithContext(ctx).Where("source IN ? AND archive_date = ? AND instance_id = ?", []string{constants.AuditArchiveSource, constants.IntegrationArchiveSource}, date.Format(time.DateOnly), s.instanceID).Find(&rows).Error; err != nil { return retentionRuns{}, fmt.Errorf("读取日归档账本失败: %w", err) } runs := retentionRuns{} for index := range rows { switch rows[index].Source { case constants.AuditArchiveSource: runs.audit = &rows[index] case constants.IntegrationArchiveSource: runs.integration = &rows[index] } } if runs.audit == nil || runs.integration == nil { return retentionRuns{}, fmt.Errorf("Audit 或 Integration Log 归档账本缺失") } return runs, nil } func (s *Service) validateAuditRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error { if err := validateRunBase(run, date, constants.AuditArchiveSchemaVersion, false); err != nil { return err } if err := s.validateAuditManifest(ctx, run); err != nil { return err } events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd) if err != nil { return err } return validateRemainingCounts(run, events, resources) } func (s *Service) validateIntegrationRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error { if err := validateRunBase(run, date, constants.IntegrationArchiveSchemaVersion, true); err != nil { return err } if err := s.validateIntegrationManifest(ctx, run); err != nil { return err } count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd) if err != nil { return err } if run.CleanedAt != nil && count != 0 { return fmt.Errorf("已标记清理完成但数据库仍有 %d 条记录", count) } if run.CleanupStartedAt != nil && count > run.RecordCount { return fmt.Errorf("续跑窗口记录数超过最终归档数量") } if run.CleanupStartedAt == nil { file, err := s.buildIntegrationArchiveFile(ctx, run.RangeStart, run.RangeEnd) if err != nil { return err } defer os.Remove(file.path) if file.recordCount != run.RecordCount || file.sha256 != run.SHA256 { return fmt.Errorf("数据库当前 Integration 内容与最终 revision 不一致") } } return nil } func validateRunBase(run *model.LogArchiveRun, date time.Time, schema string, final bool) error { if run.Status != constants.ArchiveStatusSuccess || run.SchemaVersion != schema { return fmt.Errorf("归档状态或 schema version 不符合清理要求") } if run.ArchiveDate.Format(time.DateOnly) != date.Format(time.DateOnly) || !run.RangeStart.Equal(date) || !run.RangeEnd.Equal(date.AddDate(0, 0, 1)) { return fmt.Errorf("归档日期或半开时间范围不一致") } if final && !run.IsFinal { return fmt.Errorf("Integration 最终 revision 尚未形成") } if run.ObjectKey == "" || run.ManifestKey == "" || run.SHA256 == "" { return fmt.Errorf("归档对象、manifest 或 SHA-256 缺失") } return nil } func validateRemainingCounts(run *model.LogArchiveRun, events, resources int64) error { if run.CleanedAt != nil && (events != 0 || resources != 0) { return fmt.Errorf("已标记清理完成但数据库仍有事件或资源") } if run.CleanupStartedAt != nil && (events > run.EventCount || resources > run.ResourceCount) { return fmt.Errorf("续跑窗口数量超过已归档数量") } if run.CleanupStartedAt == nil && (events != run.EventCount || resources != run.ResourceCount) { return fmt.Errorf("数据库事件或资源数量与 manifest 不一致") } return nil } func (s *Service) validateAuditManifest(ctx context.Context, run *model.LogArchiveRun) error { var manifest Manifest if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil { return err } if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone || manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) || manifest.EventCount != run.EventCount || manifest.ResourceCount != run.ResourceCount || manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision { return fmt.Errorf("Audit manifest 与 ledger 不一致") } if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision)}); err != nil { return err } return s.verifyRetentionObject(ctx, run, false) } func (s *Service) validateIntegrationManifest(ctx context.Context, run *model.LogArchiveRun) error { var manifest integrationArchiveManifest if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil { return err } if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || !manifest.Final || manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone || manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) || manifest.RecordCount != run.RecordCount || manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision { return fmt.Errorf("Integration manifest 与最终 ledger 不一致") } if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{"source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision), "final": "true"}); err != nil { return err } return s.verifyRetentionObject(ctx, run, true) } func (s *Service) readManifest(ctx context.Context, key string, target any) error { object, err := s.store.Stat(ctx, key) if err != nil { return fmt.Errorf("读取 manifest metadata 失败: %w", err) } reader, err := s.store.Download(ctx, key) if err != nil { return fmt.Errorf("下载 manifest 失败: %w", err) } data, readErr := io.ReadAll(io.LimitReader(reader, maxManifestBytes+1)) closeErr := reader.Close() if readErr != nil { return fmt.Errorf("读取 manifest 失败: %w", readErr) } if closeErr != nil { return fmt.Errorf("关闭 manifest 对象失败: %w", closeErr) } if len(data) > maxManifestBytes || int64(len(data)) != object.Size { return fmt.Errorf("manifest 大小非法或不完整") } if err := sonic.Unmarshal(data, target); err != nil { return fmt.Errorf("解析 manifest 失败: %w", err) } return nil } func (s *Service) verifyRetentionObject(ctx context.Context, run *model.LogArchiveRun, final bool) error { metadata := map[string]string{"schema-version": run.SchemaVersion, "source": run.Source, "archive-date": run.RangeStart.Format(time.DateOnly), "timezone": constants.AuditArchiveTimezone, "sha256": run.SHA256, "revision": strconv.Itoa(run.Revision)} if run.Source == constants.AuditArchiveSource { metadata["event-count"] = strconv.FormatInt(run.EventCount, 10) metadata["resource-count"] = strconv.FormatInt(run.ResourceCount, 10) } else { metadata["record-count"] = strconv.FormatInt(run.RecordCount, 10) metadata["final"] = strconv.FormatBool(final) } if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil { return err } reader, err := s.store.Download(ctx, run.ObjectKey) if err != nil { return fmt.Errorf("下载归档对象复核 SHA-256 失败: %w", err) } hasher := sha256.New() written, copyErr := io.Copy(hasher, reader) closeErr := reader.Close() if copyErr != nil { return fmt.Errorf("读取归档对象复核 SHA-256 失败: %w", copyErr) } if closeErr != nil { return fmt.Errorf("关闭归档对象失败: %w", closeErr) } if written != run.CompressedBytes || fmt.Sprintf("%x", hasher.Sum(nil)) != run.SHA256 { return fmt.Errorf("归档对象大小或 SHA-256 复核失败") } return nil } func estimatedRetentionBatches(result RetentionResult) int64 { batch := int64(constants.AuditRetentionDeleteBatchSize) return (result.EventCount+batch-1)/batch + (result.ResourceCount+batch-1)/batch + (result.IntegrationCount+batch-1)/batch } func (s *Service) cleanupAuditDate(ctx context.Context, date time.Time, run *model.LogArchiveRun) error { if run.CleanedAt != nil { return nil } if err := s.markCleanupStarted(ctx, constants.AuditArchiveSource, date); err != nil { return err } if err := s.deleteAuditResources(ctx, date, date.AddDate(0, 0, 1)); err != nil { return err } if err := s.deleteAuditEvents(ctx, date, date.AddDate(0, 0, 1)); err != nil { return err } return s.markCleaned(ctx, constants.AuditArchiveSource, date) } func (s *Service) cleanupIntegrationDate(ctx context.Context, date time.Time, run *model.LogArchiveRun) error { if run.CleanedAt != nil { return nil } if err := s.markCleanupStarted(ctx, constants.IntegrationArchiveSource, date); err != nil { return err } end := date.AddDate(0, 0, 1) for { subquery := s.db.Model(&model.IntegrationLog{}).Select("id").Where("created_at >= ? AND created_at < ?", date, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize) deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.IntegrationLog{}) if deleted.Error != nil { return fmt.Errorf("分批物理删除 Integration Log 失败: %w", deleted.Error) } if deleted.RowsAffected == 0 { break } } return s.markCleaned(ctx, constants.IntegrationArchiveSource, date) } func (s *Service) deleteAuditResources(ctx context.Context, start, end time.Time) error { for { subquery := s.db.Model(&model.AuditEventResource{}).Select("tb_audit_event_resource.id").Joins("JOIN tb_audit_event ON tb_audit_event.id = tb_audit_event_resource.audit_event_id").Where("tb_audit_event.created_at >= ? AND tb_audit_event.created_at < ?", start, end).Order("tb_audit_event_resource.id ASC").Limit(constants.AuditRetentionDeleteBatchSize) deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEventResource{}) if deleted.Error != nil { return fmt.Errorf("分批物理删除 Audit Event Resource 失败: %w", deleted.Error) } if deleted.RowsAffected == 0 { return nil } } } func (s *Service) deleteAuditEvents(ctx context.Context, start, end time.Time) error { for { subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize) deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEvent{}) if deleted.Error != nil { return fmt.Errorf("分批物理删除 Audit Event 失败: %w", deleted.Error) } if deleted.RowsAffected == 0 { return nil } } } func (s *Service) markCleanupStarted(ctx context.Context, source string, date time.Time) error { now := time.Now() result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("source = ? AND archive_date = ? AND instance_id = ? AND cleanup_started_at IS NULL", source, date.Format(time.DateOnly), s.instanceID).Updates(map[string]any{"cleanup_started_at": now, "updated_at": now}) if result.Error != nil { return fmt.Errorf("记录日清理开始断点失败: %w", result.Error) } return nil } func (s *Service) markCleaned(ctx context.Context, source string, date time.Time) error { now := time.Now() result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("source = ? AND archive_date = ? AND instance_id = ? AND cleanup_started_at IS NOT NULL", source, date.Format(time.DateOnly), s.instanceID).Updates(map[string]any{"cleaned_at": now, "updated_at": now}) if result.Error != nil { return fmt.Errorf("记录日清理完成断点失败: %w", result.Error) } return nil }