// Package auditarchive 实现统一审计每日冷归档用例。 package auditarchive import ( "compress/gzip" "context" "crypto/sha256" "fmt" "io" "os" "strconv" "strings" "time" "github.com/bytedance/sonic" "gorm.io/gorm" "gorm.io/gorm/clause" "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/storage" ) const archivePageSize = 1000 // ObjectStore 是每日归档需要的最小对象存储能力。 type ObjectStore interface { UploadWithMetadata(context.Context, string, io.Reader, string, map[string]string) error Stat(context.Context, string) (*storage.ObjectMetadata, error) Download(context.Context, string) (io.ReadCloser, error) } // RetentionAuditWriter 记录月度留存清理的系统审计事实。 type RetentionAuditWriter interface { WriteRetentionCleanup(context.Context, *gorm.DB, RetentionAudit) error } // Service 编排审计归档生成、上传、复核和幂等账本更新。 type Service struct { db *gorm.DB store ObjectStore audit RetentionAuditWriter instanceID string location *time.Location } // Manifest 是归档对象的完整性清单。 type Manifest struct { SchemaVersion string `json:"schema_version"` Source string `json:"source"` ArchiveDate string `json:"archive_date"` Timezone string `json:"timezone"` RangeStart time.Time `json:"range_start"` RangeEnd time.Time `json:"range_end"` InstanceID string `json:"instance_id"` EventCount int64 `json:"event_count"` ResourceCount int64 `json:"resource_count"` UncompressedBytes int64 `json:"uncompressed_bytes"` CompressedBytes int64 `json:"compressed_bytes"` ObjectKey string `json:"object_key"` SHA256 string `json:"sha256"` Revision int `json:"revision"` GeneratedAt time.Time `json:"generated_at"` Status string `json:"status"` } type archiveLine struct { Event model.AuditEvent `json:"event"` Resources []model.AuditEventResource `json:"resources"` } type archiveFile struct { path string eventCount int64 resourceCount int64 uncompressedBytes int64 compressedBytes int64 sha256 string } // NewService 创建统一审计每日冷归档服务。 func NewService(db *gorm.DB, store ObjectStore, instanceID string, audit ...RetentionAuditWriter) (*Service, error) { location, err := time.LoadLocation(constants.AuditArchiveTimezone) if err != nil { return nil, fmt.Errorf("加载审计归档时区失败: %w", err) } if strings.TrimSpace(instanceID) == "" { instanceID = "audit-archive" } var auditWriter RetentionAuditWriter if len(audit) > 0 { auditWriter = audit[0] } return &Service{db: db, store: store, audit: auditWriter, instanceID: instanceID, location: location}, nil } // ArchivePreviousDay 归档 Asia/Shanghai 前一完整自然日。 func (s *Service) ArchivePreviousDay(ctx context.Context) error { now := time.Now().In(s.location) return s.ArchiveDate(ctx, now.AddDate(0, 0, -1)) } // ArchiveDate 归档指定 Asia/Shanghai 自然日。 func (s *Service) ArchiveDate(ctx context.Context, archiveDate time.Time) error { if s.db == nil || s.store == nil { return fmt.Errorf("审计归档数据库或对象存储未配置") } start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location) end := start.AddDate(0, 0, 1) today := time.Now().In(s.location) todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location) if end.After(todayStart) { return fmt.Errorf("统一审计只能归档已经结束的完整自然日") } run, err := s.ensureRun(ctx, start, end) if err != nil { return err } if run.Status == constants.ArchiveStatusSuccess { valid, validateErr := s.validateSuccessfulRun(ctx, run) if validateErr == nil && valid { return nil } } acquired, err := s.acquireRun(ctx, run) if err != nil || !acquired { return err } if err := s.execute(ctx, run); err != nil { s.markFailed(ctx, run.ID, err) return err } return nil } func (s *Service) ensureRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) { run := model.LogArchiveRun{ Source: constants.AuditArchiveSource, ArchiveDate: start, InstanceID: s.instanceID, SchemaVersion: constants.AuditArchiveSchemaVersion, Revision: 1, Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end, } result := s.db.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}}, DoNothing: true, }).Create(&run) if result.Error != nil { return nil, fmt.Errorf("创建审计归档账本失败: %w", result.Error) } if result.RowsAffected == 0 { if err := s.db.WithContext(ctx).Where( "source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?", constants.AuditArchiveSource, start, s.instanceID, constants.AuditArchiveSchemaVersion, ).First(&run).Error; err != nil { return nil, fmt.Errorf("读取审计归档账本失败: %w", err) } } return &run, nil } func (s *Service) acquireRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) { revision := run.Revision if run.Status == constants.ArchiveStatusFailed || run.Status == constants.ArchiveStatusSuccess || run.Status == constants.ArchiveStatusRunning { revision++ } now := time.Now() staleBefore := now.Add(-3 * time.Hour) result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, staleBefore). Updates(map[string]any{ "status": constants.ArchiveStatusRunning, "revision": revision, "attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "", "completed_at": nil, "updated_at": now, }) if result.Error != nil { return false, fmt.Errorf("锁定审计归档任务失败: %w", result.Error) } if result.RowsAffected == 0 { return false, nil } run.Revision = revision run.Status = constants.ArchiveStatusRunning return true, nil } func (s *Service) execute(ctx context.Context, run *model.LogArchiveRun) error { file, err := s.buildArchiveFile(ctx, run.RangeStart, run.RangeEnd) if err != nil { return err } defer os.Remove(file.path) dbEvents, dbResources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd) if err != nil { return err } if dbEvents != file.eventCount || dbResources != file.resourceCount { return fmt.Errorf("审计归档生成期间数据数量发生变化") } objectKey, manifestKey := objectKeys(run.RangeStart, run.Revision) metadata := archiveMetadata(file, run) reader, err := os.Open(file.path) if err != nil { return fmt.Errorf("打开审计归档临时文件失败: %w", err) } uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata) closeErr := reader.Close() if uploadErr != nil { return fmt.Errorf("上传审计归档对象失败: %w", uploadErr) } if closeErr != nil { return fmt.Errorf("关闭审计归档临时文件失败: %w", closeErr) } if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil { return err } generatedAt := time.Now().In(s.location) manifest := Manifest{ SchemaVersion: constants.AuditArchiveSchemaVersion, Source: constants.AuditArchiveSource, ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone, RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID, EventCount: file.eventCount, ResourceCount: file.resourceCount, UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes, ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision, GeneratedAt: generatedAt, Status: constants.ArchiveStatusSuccess, } manifestBytes, err := sonic.Marshal(manifest) if err != nil { return fmt.Errorf("序列化审计归档清单失败: %w", err) } manifestMetadata := map[string]string{"source": constants.AuditArchiveSource, "data-sha256": file.sha256, "revision": strconv.Itoa(run.Revision)} if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil { return fmt.Errorf("上传审计归档清单失败: %w", err) } if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil { return err } completedAt := time.Now() updates := map[string]any{ "status": constants.ArchiveStatusSuccess, "object_key": objectKey, "manifest_key": manifestKey, "event_count": file.eventCount, "resource_count": file.resourceCount, "uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes, "sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt, "updated_at": completedAt, } if err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(updates).Error; err != nil { return fmt.Errorf("更新审计归档成功账本失败: %w", err) } return nil } func (s *Service) buildArchiveFile(ctx context.Context, start, end time.Time) (*archiveFile, error) { temp, err := os.CreateTemp("", "audit-events-*.jsonl.gz") if err != nil { return nil, fmt.Errorf("创建审计归档临时文件失败: %w", err) } path := temp.Name() failed := true defer func() { _ = temp.Close() if failed { _ = os.Remove(path) } }() hasher := sha256.New() gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher)) result := &archiveFile{path: path} var lastID uint for { var events []model.AuditEvent if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID). Order("id ASC").Limit(archivePageSize).Find(&events).Error; err != nil { return nil, fmt.Errorf("读取审计归档事件失败: %w", err) } if len(events) == 0 { break } ids := make([]uint, 0, len(events)) for i := range events { ids = append(ids, events[i].ID) } var resources []model.AuditEventResource if err := s.db.WithContext(ctx).Where("audit_event_id IN ?", ids). Order("audit_event_id ASC, sort_order ASC, id ASC").Find(&resources).Error; err != nil { return nil, fmt.Errorf("读取审计归档资源失败: %w", err) } grouped := make(map[uint][]model.AuditEventResource, len(events)) for i := range resources { resource := resources[i] grouped[resource.AuditEventID] = append(grouped[resource.AuditEventID], resource) } for i := range events { eventResources := grouped[events[i].ID] if eventResources == nil { eventResources = []model.AuditEventResource{} } line, marshalErr := sonic.Marshal(archiveLine{Event: events[i], Resources: eventResources}) if marshalErr != nil { return nil, fmt.Errorf("序列化审计归档事件失败: %w", marshalErr) } line = append(line, '\n') if _, writeErr := gzipWriter.Write(line); writeErr != nil { return nil, fmt.Errorf("写入审计归档压缩流失败: %w", writeErr) } result.eventCount++ result.resourceCount += int64(len(grouped[events[i].ID])) result.uncompressedBytes += int64(len(line)) } lastID = events[len(events)-1].ID } if err := gzipWriter.Close(); err != nil { return nil, fmt.Errorf("关闭审计归档压缩流失败: %w", err) } if err := temp.Close(); err != nil { return nil, fmt.Errorf("关闭审计归档临时文件失败: %w", err) } info, err := os.Stat(path) if err != nil { return nil, fmt.Errorf("读取审计归档临时文件信息失败: %w", err) } result.compressedBytes = info.Size() result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil)) failed = false return result, nil } func (s *Service) databaseCounts(ctx context.Context, start, end time.Time) (int64, int64, error) { var eventCount int64 if err := s.db.WithContext(ctx).Model(&model.AuditEvent{}). Where("created_at >= ? AND created_at < ?", start, end).Count(&eventCount).Error; err != nil { return 0, 0, fmt.Errorf("统计审计归档事件失败: %w", err) } var resourceCount int64 subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end) if err := s.db.WithContext(ctx).Model(&model.AuditEventResource{}). Where("audit_event_id IN (?)", subquery).Count(&resourceCount).Error; err != nil { return 0, 0, fmt.Errorf("统计审计归档资源失败: %w", err) } return eventCount, resourceCount, nil } func (s *Service) validateSuccessfulRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) { events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd) if err != nil || events != run.EventCount || resources != run.ResourceCount { return false, err } metadata := map[string]string{ "sha256": run.SHA256, "event-count": strconv.FormatInt(run.EventCount, 10), "resource-count": strconv.FormatInt(run.ResourceCount, 10), "revision": strconv.Itoa(run.Revision), } if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil { return false, nil } manifestMetadata := map[string]string{ "source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision), } if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil { return false, nil } return true, nil } func (s *Service) verifyObject(ctx context.Context, key string, expectedSize int64, expectedMetadata map[string]string) error { object, err := s.store.Stat(ctx, key) if err != nil { return fmt.Errorf("复核归档对象 metadata 失败: %w", err) } if expectedSize >= 0 && object.Size != expectedSize { return fmt.Errorf("归档对象大小复核失败") } for name, value := range expectedMetadata { if object.Metadata[strings.ToLower(name)] != value { return fmt.Errorf("归档对象 metadata 字段 %s 复核失败", name) } } return nil } func (s *Service) markFailed(ctx context.Context, runID uint, archiveErr error) { summary := []rune(archiveErr.Error()) if len(summary) > 500 { summary = summary[:500] } failedCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() _ = s.db.WithContext(failedCtx).Model(&model.LogArchiveRun{}).Where("id = ?", runID).Updates(map[string]any{ "status": constants.ArchiveStatusFailed, "error_summary": string(summary), "updated_at": time.Now(), }).Error } func objectKeys(date time.Time, revision int) (string, string) { prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day()) name := fmt.Sprintf("audit-events-%s-r%d", date.Format(time.DateOnly), revision) return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json" } func archiveMetadata(file *archiveFile, run *model.LogArchiveRun) map[string]string { return map[string]string{ "schema-version": constants.AuditArchiveSchemaVersion, "source": constants.AuditArchiveSource, "archive-date": run.RangeStart.Format(time.DateOnly), "timezone": constants.AuditArchiveTimezone, "event-count": strconv.FormatInt(file.eventCount, 10), "resource-count": strconv.FormatInt(file.resourceCount, 10), "sha256": file.sha256, "revision": strconv.Itoa(run.Revision), } }