暂存

2026-08-06 09:35:00 +08:00
parent 8659dfc658
commit 88cc5e96ec
75 changed files with 6520 additions and 513 deletions
--- a/internal/application/auditarchive/service.go
+++ b/internal/application/auditarchive/service.go
@@ -0,0 +1,412 @@
+// Package auditarchive 实现统一审计每日冷归档用例。
+package auditarchive
+
+import (
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/bytedance/sonic"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+
+	"github.com/break/junhong_cmp_fiber/internal/model"
+	"github.com/break/junhong_cmp_fiber/pkg/constants"
+	"github.com/break/junhong_cmp_fiber/pkg/storage"
+)
+
+const archivePageSize = 1000
+
+// ObjectStore 是每日归档需要的最小对象存储能力。
+type ObjectStore interface {
+	UploadWithMetadata(context.Context, string, io.Reader, string, map[string]string) error
+	Stat(context.Context, string) (*storage.ObjectMetadata, error)
+	Download(context.Context, string) (io.ReadCloser, error)
+}
+
+// RetentionAuditWriter 记录月度留存清理的系统审计事实。
+type RetentionAuditWriter interface {
+	WriteRetentionCleanup(context.Context, *gorm.DB, RetentionAudit) error
+}
+
+// Service 编排审计归档生成、上传、复核和幂等账本更新。
+type Service struct {
+	db         *gorm.DB
+	store      ObjectStore
+	audit      RetentionAuditWriter
+	instanceID string
+	location   *time.Location
+}
+
+// Manifest 是归档对象的完整性清单。
+type Manifest struct {
+	SchemaVersion     string    `json:"schema_version"`
+	Source            string    `json:"source"`
+	ArchiveDate       string    `json:"archive_date"`
+	Timezone          string    `json:"timezone"`
+	RangeStart        time.Time `json:"range_start"`
+	RangeEnd          time.Time `json:"range_end"`
+	InstanceID        string    `json:"instance_id"`
+	EventCount        int64     `json:"event_count"`
+	ResourceCount     int64     `json:"resource_count"`
+	UncompressedBytes int64     `json:"uncompressed_bytes"`
+	CompressedBytes   int64     `json:"compressed_bytes"`
+	ObjectKey         string    `json:"object_key"`
+	SHA256            string    `json:"sha256"`
+	Revision          int       `json:"revision"`
+	GeneratedAt       time.Time `json:"generated_at"`
+	Status            string    `json:"status"`
+}
+
+type archiveLine struct {
+	Event     model.AuditEvent           `json:"event"`
+	Resources []model.AuditEventResource `json:"resources"`
+}
+
+type archiveFile struct {
+	path              string
+	eventCount        int64
+	resourceCount     int64
+	uncompressedBytes int64
+	compressedBytes   int64
+	sha256            string
+}
+
+// NewService 创建统一审计每日冷归档服务。
+func NewService(db *gorm.DB, store ObjectStore, instanceID string, audit ...RetentionAuditWriter) (*Service, error) {
+	location, err := time.LoadLocation(constants.AuditArchiveTimezone)
+	if err != nil {
+		return nil, fmt.Errorf("加载审计归档时区失败: %w", err)
+	}
+	if strings.TrimSpace(instanceID) == "" {
+		instanceID = "audit-archive"
+	}
+	var auditWriter RetentionAuditWriter
+	if len(audit) > 0 {
+		auditWriter = audit[0]
+	}
+	return &Service{db: db, store: store, audit: auditWriter, instanceID: instanceID, location: location}, nil
+}
+
+// ArchivePreviousDay 归档 Asia/Shanghai 前一完整自然日。
+func (s *Service) ArchivePreviousDay(ctx context.Context) error {
+	now := time.Now().In(s.location)
+	return s.ArchiveDate(ctx, now.AddDate(0, 0, -1))
+}
+
+// ArchiveDate 归档指定 Asia/Shanghai 自然日。
+func (s *Service) ArchiveDate(ctx context.Context, archiveDate time.Time) error {
+	if s.db == nil || s.store == nil {
+		return fmt.Errorf("审计归档数据库或对象存储未配置")
+	}
+	start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location)
+	end := start.AddDate(0, 0, 1)
+	today := time.Now().In(s.location)
+	todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)
+	if end.After(todayStart) {
+		return fmt.Errorf("统一审计只能归档已经结束的完整自然日")
+	}
+	run, err := s.ensureRun(ctx, start, end)
+	if err != nil {
+		return err
+	}
+	if run.Status == constants.ArchiveStatusSuccess {
+		valid, validateErr := s.validateSuccessfulRun(ctx, run)
+		if validateErr == nil && valid {
+			return nil
+		}
+	}
+
+	acquired, err := s.acquireRun(ctx, run)
+	if err != nil || !acquired {
+		return err
+	}
+	if err := s.execute(ctx, run); err != nil {
+		s.markFailed(ctx, run.ID, err)
+		return err
+	}
+	return nil
+}
+
+func (s *Service) ensureRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) {
+	run := model.LogArchiveRun{
+		Source: constants.AuditArchiveSource, ArchiveDate: start, InstanceID: s.instanceID,
+		SchemaVersion: constants.AuditArchiveSchemaVersion, Revision: 1,
+		Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end,
+	}
+	result := s.db.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}},
+		DoNothing: true,
+	}).Create(&run)
+	if result.Error != nil {
+		return nil, fmt.Errorf("创建审计归档账本失败: %w", result.Error)
+	}
+	if result.RowsAffected == 0 {
+		if err := s.db.WithContext(ctx).Where(
+			"source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?",
+			constants.AuditArchiveSource, start, s.instanceID, constants.AuditArchiveSchemaVersion,
+		).First(&run).Error; err != nil {
+			return nil, fmt.Errorf("读取审计归档账本失败: %w", err)
+		}
+	}
+	return &run, nil
+}
+
+func (s *Service) acquireRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
+	revision := run.Revision
+	if run.Status == constants.ArchiveStatusFailed || run.Status == constants.ArchiveStatusSuccess || run.Status == constants.ArchiveStatusRunning {
+		revision++
+	}
+	now := time.Now()
+	staleBefore := now.Add(-3 * time.Hour)
+	result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
+		Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, staleBefore).
+		Updates(map[string]any{
+			"status": constants.ArchiveStatusRunning, "revision": revision,
+			"attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "",
+			"completed_at": nil, "updated_at": now,
+		})
+	if result.Error != nil {
+		return false, fmt.Errorf("锁定审计归档任务失败: %w", result.Error)
+	}
+	if result.RowsAffected == 0 {
+		return false, nil
+	}
+	run.Revision = revision
+	run.Status = constants.ArchiveStatusRunning
+	return true, nil
+}
+
+func (s *Service) execute(ctx context.Context, run *model.LogArchiveRun) error {
+	file, err := s.buildArchiveFile(ctx, run.RangeStart, run.RangeEnd)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(file.path)
+
+	dbEvents, dbResources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
+	if err != nil {
+		return err
+	}
+	if dbEvents != file.eventCount || dbResources != file.resourceCount {
+		return fmt.Errorf("审计归档生成期间数据数量发生变化")
+	}
+
+	objectKey, manifestKey := objectKeys(run.RangeStart, run.Revision)
+	metadata := archiveMetadata(file, run)
+	reader, err := os.Open(file.path)
+	if err != nil {
+		return fmt.Errorf("打开审计归档临时文件失败: %w", err)
+	}
+	uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata)
+	closeErr := reader.Close()
+	if uploadErr != nil {
+		return fmt.Errorf("上传审计归档对象失败: %w", uploadErr)
+	}
+	if closeErr != nil {
+		return fmt.Errorf("关闭审计归档临时文件失败: %w", closeErr)
+	}
+	if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil {
+		return err
+	}
+
+	generatedAt := time.Now().In(s.location)
+	manifest := Manifest{
+		SchemaVersion: constants.AuditArchiveSchemaVersion, Source: constants.AuditArchiveSource,
+		ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone,
+		RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID,
+		EventCount: file.eventCount, ResourceCount: file.resourceCount,
+		UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes,
+		ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision,
+		GeneratedAt: generatedAt, Status: constants.ArchiveStatusSuccess,
+	}
+	manifestBytes, err := sonic.Marshal(manifest)
+	if err != nil {
+		return fmt.Errorf("序列化审计归档清单失败: %w", err)
+	}
+	manifestMetadata := map[string]string{"source": constants.AuditArchiveSource, "data-sha256": file.sha256, "revision": strconv.Itoa(run.Revision)}
+	if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil {
+		return fmt.Errorf("上传审计归档清单失败: %w", err)
+	}
+	if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil {
+		return err
+	}
+
+	completedAt := time.Now()
+	updates := map[string]any{
+		"status": constants.ArchiveStatusSuccess, "object_key": objectKey, "manifest_key": manifestKey,
+		"event_count": file.eventCount, "resource_count": file.resourceCount,
+		"uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes,
+		"sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt,
+		"updated_at": completedAt,
+	}
+	if err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(updates).Error; err != nil {
+		return fmt.Errorf("更新审计归档成功账本失败: %w", err)
+	}
+	return nil
+}
+
+func (s *Service) buildArchiveFile(ctx context.Context, start, end time.Time) (*archiveFile, error) {
+	temp, err := os.CreateTemp("", "audit-events-*.jsonl.gz")
+	if err != nil {
+		return nil, fmt.Errorf("创建审计归档临时文件失败: %w", err)
+	}
+	path := temp.Name()
+	failed := true
+	defer func() {
+		_ = temp.Close()
+		if failed {
+			_ = os.Remove(path)
+		}
+	}()
+
+	hasher := sha256.New()
+	gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher))
+	result := &archiveFile{path: path}
+	var lastID uint
+	for {
+		var events []model.AuditEvent
+		if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID).
+			Order("id ASC").Limit(archivePageSize).Find(&events).Error; err != nil {
+			return nil, fmt.Errorf("读取审计归档事件失败: %w", err)
+		}
+		if len(events) == 0 {
+			break
+		}
+		ids := make([]uint, 0, len(events))
+		for i := range events {
+			ids = append(ids, events[i].ID)
+		}
+		var resources []model.AuditEventResource
+		if err := s.db.WithContext(ctx).Where("audit_event_id IN ?", ids).
+			Order("audit_event_id ASC, sort_order ASC, id ASC").Find(&resources).Error; err != nil {
+			return nil, fmt.Errorf("读取审计归档资源失败: %w", err)
+		}
+		grouped := make(map[uint][]model.AuditEventResource, len(events))
+		for i := range resources {
+			resource := resources[i]
+			grouped[resource.AuditEventID] = append(grouped[resource.AuditEventID], resource)
+		}
+		for i := range events {
+			eventResources := grouped[events[i].ID]
+			if eventResources == nil {
+				eventResources = []model.AuditEventResource{}
+			}
+			line, marshalErr := sonic.Marshal(archiveLine{Event: events[i], Resources: eventResources})
+			if marshalErr != nil {
+				return nil, fmt.Errorf("序列化审计归档事件失败: %w", marshalErr)
+			}
+			line = append(line, '\n')
+			if _, writeErr := gzipWriter.Write(line); writeErr != nil {
+				return nil, fmt.Errorf("写入审计归档压缩流失败: %w", writeErr)
+			}
+			result.eventCount++
+			result.resourceCount += int64(len(grouped[events[i].ID]))
+			result.uncompressedBytes += int64(len(line))
+		}
+		lastID = events[len(events)-1].ID
+	}
+	if err := gzipWriter.Close(); err != nil {
+		return nil, fmt.Errorf("关闭审计归档压缩流失败: %w", err)
+	}
+	if err := temp.Close(); err != nil {
+		return nil, fmt.Errorf("关闭审计归档临时文件失败: %w", err)
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return nil, fmt.Errorf("读取审计归档临时文件信息失败: %w", err)
+	}
+	result.compressedBytes = info.Size()
+	result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil))
+	failed = false
+	return result, nil
+}
+
+func (s *Service) databaseCounts(ctx context.Context, start, end time.Time) (int64, int64, error) {
+	var eventCount int64
+	if err := s.db.WithContext(ctx).Model(&model.AuditEvent{}).
+		Where("created_at >= ? AND created_at < ?", start, end).Count(&eventCount).Error; err != nil {
+		return 0, 0, fmt.Errorf("统计审计归档事件失败: %w", err)
+	}
+	var resourceCount int64
+	subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end)
+	if err := s.db.WithContext(ctx).Model(&model.AuditEventResource{}).
+		Where("audit_event_id IN (?)", subquery).Count(&resourceCount).Error; err != nil {
+		return 0, 0, fmt.Errorf("统计审计归档资源失败: %w", err)
+	}
+	return eventCount, resourceCount, nil
+}
+
+func (s *Service) validateSuccessfulRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
+	events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
+	if err != nil || events != run.EventCount || resources != run.ResourceCount {
+		return false, err
+	}
+	metadata := map[string]string{
+		"sha256": run.SHA256, "event-count": strconv.FormatInt(run.EventCount, 10),
+		"resource-count": strconv.FormatInt(run.ResourceCount, 10), "revision": strconv.Itoa(run.Revision),
+	}
+	if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
+		return false, nil
+	}
+	manifestMetadata := map[string]string{
+		"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
+	}
+	if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (s *Service) verifyObject(ctx context.Context, key string, expectedSize int64, expectedMetadata map[string]string) error {
+	object, err := s.store.Stat(ctx, key)
+	if err != nil {
+		return fmt.Errorf("复核归档对象 metadata 失败: %w", err)
+	}
+	if expectedSize >= 0 && object.Size != expectedSize {
+		return fmt.Errorf("归档对象大小复核失败")
+	}
+	for name, value := range expectedMetadata {
+		if object.Metadata[strings.ToLower(name)] != value {
+			return fmt.Errorf("归档对象 metadata 字段 %s 复核失败", name)
+		}
+	}
+	return nil
+}
+
+func (s *Service) markFailed(ctx context.Context, runID uint, archiveErr error) {
+	summary := []rune(archiveErr.Error())
+	if len(summary) > 500 {
+		summary = summary[:500]
+	}
+	failedCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+	defer cancel()
+	_ = s.db.WithContext(failedCtx).Model(&model.LogArchiveRun{}).Where("id = ?", runID).Updates(map[string]any{
+		"status": constants.ArchiveStatusFailed, "error_summary": string(summary), "updated_at": time.Now(),
+	}).Error
+}
+
+func objectKeys(date time.Time, revision int) (string, string) {
+	prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day())
+	name := fmt.Sprintf("audit-events-%s-r%d", date.Format(time.DateOnly), revision)
+	return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json"
+}
+
+func archiveMetadata(file *archiveFile, run *model.LogArchiveRun) map[string]string {
+	return map[string]string{
+		"schema-version": constants.AuditArchiveSchemaVersion,
+		"source":         constants.AuditArchiveSource,
+		"archive-date":   run.RangeStart.Format(time.DateOnly),
+		"timezone":       constants.AuditArchiveTimezone,
+		"event-count":    strconv.FormatInt(file.eventCount, 10),
+		"resource-count": strconv.FormatInt(file.resourceCount, 10),
+		"sha256":         file.sha256,
+		"revision":       strconv.Itoa(run.Revision),
+	}
+}