暂存
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 8m32s

This commit is contained in:
2026-08-06 09:35:00 +08:00
parent 8659dfc658
commit 88cc5e96ec
75 changed files with 6520 additions and 513 deletions

View File

@@ -0,0 +1,350 @@
package auditarchive
import (
"compress/gzip"
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/bytedance/sonic"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
type integrationArchiveFile struct {
path string
recordCount int64
uncompressedBytes int64
compressedBytes int64
sha256 string
}
type integrationArchiveManifest struct {
SchemaVersion string `json:"schema_version"`
Source string `json:"source"`
ArchiveDate string `json:"archive_date"`
Timezone string `json:"timezone"`
RangeStart time.Time `json:"range_start"`
RangeEnd time.Time `json:"range_end"`
InstanceID string `json:"instance_id"`
RecordCount int64 `json:"record_count"`
UncompressedBytes int64 `json:"uncompressed_bytes"`
CompressedBytes int64 `json:"compressed_bytes"`
ObjectKey string `json:"object_key"`
SHA256 string `json:"sha256"`
Revision int `json:"revision"`
GeneratedAt time.Time `json:"generated_at"`
Status string `json:"status"`
Final bool `json:"final"`
}
// ArchivePreviousIntegrationDay 归档 Asia/Shanghai 前一完整自然日的 Integration Log 创建日快照。
func (s *Service) ArchivePreviousIntegrationDay(ctx context.Context) error {
now := time.Now().In(s.location)
return s.ArchiveIntegrationDate(ctx, now.AddDate(0, 0, -1))
}
// ArchiveIntegrationDate 归档指定 Asia/Shanghai 自然日的 Integration Log 创建日快照。
func (s *Service) ArchiveIntegrationDate(ctx context.Context, archiveDate time.Time) error {
return s.archiveIntegrationDate(ctx, archiveDate, false)
}
// FinalizePreviousIntegrationMonth 复核并终结上一个完整自然月的 Integration Log 归档。
func (s *Service) FinalizePreviousIntegrationMonth(ctx context.Context) error {
now := time.Now().In(s.location)
return s.FinalizeIntegrationMonth(ctx, now.AddDate(0, -1, 0))
}
// FinalizeIntegrationMonth 逐日复核指定完整自然月,并为变化内容创建最终 revision。
func (s *Service) FinalizeIntegrationMonth(ctx context.Context, month time.Time) error {
monthStart := time.Date(month.In(s.location).Year(), month.In(s.location).Month(), 1, 0, 0, 0, 0, s.location)
currentMonth := time.Now().In(s.location)
currentMonthStart := time.Date(currentMonth.Year(), currentMonth.Month(), 1, 0, 0, 0, 0, s.location)
if !monthStart.Before(currentMonthStart) {
return fmt.Errorf("只能终结已经结束的 Integration Log 完整自然月")
}
for date := monthStart; date.Before(monthStart.AddDate(0, 1, 0)); date = date.AddDate(0, 0, 1) {
if err := s.archiveIntegrationDate(ctx, date, true); err != nil {
return fmt.Errorf("终结 %s Integration Log 归档失败: %w", date.Format(time.DateOnly), err)
}
}
return nil
}
func (s *Service) archiveIntegrationDate(ctx context.Context, archiveDate time.Time, final bool) error {
if s.db == nil || s.store == nil {
return fmt.Errorf("Integration Log 归档数据库或对象存储未配置")
}
start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location)
end := start.AddDate(0, 0, 1)
today := time.Now().In(s.location)
if end.After(time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)) {
return fmt.Errorf("Integration Log 只能归档已经结束的完整自然日")
}
run, err := s.ensureIntegrationRun(ctx, start, end)
if err != nil {
return err
}
file, err := s.buildIntegrationArchiveFile(ctx, start, end)
if err != nil {
return err
}
defer os.Remove(file.path)
if final {
pending, pendingErr := s.integrationPendingCount(ctx, start, end)
if pendingErr != nil {
return pendingErr
}
if pending > 0 {
_ = s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).
Updates(map[string]any{"is_final": false, "error_summary": "存在 pending Integration Log无法形成最终归档", "updated_at": time.Now()}).Error
return fmt.Errorf("仍有 %d 条 pending Integration Log无法形成最终归档", pending)
}
}
if run.Status == constants.ArchiveStatusSuccess && run.RecordCount == file.recordCount && run.SHA256 == file.sha256 {
valid, validateErr := s.validateIntegrationRun(ctx, run)
if validateErr == nil && valid && (!final || run.IsFinal) {
return nil
}
}
acquired, err := s.acquireIntegrationRun(ctx, run)
if err != nil {
return err
}
if !acquired {
return fmt.Errorf("Integration Log 归档任务正在执行")
}
if err := s.uploadIntegrationArchive(ctx, run, file, final); err != nil {
s.markFailed(ctx, run.ID, err)
return err
}
return nil
}
func (s *Service) ensureIntegrationRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) {
run := model.LogArchiveRun{
Source: constants.IntegrationArchiveSource, ArchiveDate: start, InstanceID: s.instanceID,
SchemaVersion: constants.IntegrationArchiveSchemaVersion, Revision: 1,
Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end,
}
result := s.db.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}},
DoNothing: true,
}).Create(&run)
if result.Error != nil {
return nil, fmt.Errorf("创建 Integration Log 归档账本失败: %w", result.Error)
}
if result.RowsAffected == 0 {
if err := s.db.WithContext(ctx).Where(
"source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?",
constants.IntegrationArchiveSource, start, s.instanceID, constants.IntegrationArchiveSchemaVersion,
).First(&run).Error; err != nil {
return nil, fmt.Errorf("读取 Integration Log 归档账本失败: %w", err)
}
}
return &run, nil
}
func (s *Service) acquireIntegrationRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
revision := run.Revision
if run.Status != constants.ArchiveStatusPending {
revision++
}
now := time.Now()
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, now.Add(-3*time.Hour)).
Updates(map[string]any{
"status": constants.ArchiveStatusRunning, "revision": revision, "is_final": false,
"attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "", "completed_at": nil, "updated_at": now,
})
if result.Error != nil {
return false, fmt.Errorf("锁定 Integration Log 归档任务失败: %w", result.Error)
}
if result.RowsAffected == 0 {
return false, nil
}
run.Revision = revision
return true, nil
}
func (s *Service) buildIntegrationArchiveFile(ctx context.Context, start, end time.Time) (*integrationArchiveFile, error) {
temp, err := os.CreateTemp("", "integration-logs-*.jsonl.gz")
if err != nil {
return nil, fmt.Errorf("创建 Integration Log 归档临时文件失败: %w", err)
}
path := temp.Name()
failed := true
defer func() {
_ = temp.Close()
if failed {
_ = os.Remove(path)
}
}()
hasher := sha256.New()
gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher))
result := &integrationArchiveFile{path: path}
var lastID uint
for {
var logs []model.IntegrationLog
if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID).
Order("id ASC").Limit(archivePageSize).Find(&logs).Error; err != nil {
return nil, fmt.Errorf("读取 Integration Log 归档记录失败: %w", err)
}
if len(logs) == 0 {
break
}
for i := range logs {
line, marshalErr := sonic.Marshal(logs[i])
if marshalErr != nil {
return nil, fmt.Errorf("序列化 Integration Log 归档记录失败: %w", marshalErr)
}
line = append(line, '\n')
if _, writeErr := gzipWriter.Write(line); writeErr != nil {
return nil, fmt.Errorf("写入 Integration Log 归档压缩流失败: %w", writeErr)
}
result.recordCount++
result.uncompressedBytes += int64(len(line))
}
lastID = logs[len(logs)-1].ID
}
if err := gzipWriter.Close(); err != nil {
return nil, fmt.Errorf("关闭 Integration Log 归档压缩流失败: %w", err)
}
if err := temp.Close(); err != nil {
return nil, fmt.Errorf("关闭 Integration Log 归档临时文件失败: %w", err)
}
info, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("读取 Integration Log 归档临时文件信息失败: %w", err)
}
result.compressedBytes = info.Size()
result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil))
failed = false
return result, nil
}
func (s *Service) uploadIntegrationArchive(ctx context.Context, run *model.LogArchiveRun, file *integrationArchiveFile, final bool) error {
count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd)
if err != nil {
return err
}
if count != file.recordCount {
return fmt.Errorf("Integration Log 归档生成期间记录数量发生变化")
}
objectKey, manifestKey := integrationObjectKeys(run.RangeStart, run.Revision)
metadata := integrationArchiveMetadata(file, run, final)
reader, err := os.Open(file.path)
if err != nil {
return fmt.Errorf("打开 Integration Log 归档临时文件失败: %w", err)
}
uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata)
closeErr := reader.Close()
if uploadErr != nil {
return fmt.Errorf("上传 Integration Log 归档对象失败: %w", uploadErr)
}
if closeErr != nil {
return fmt.Errorf("关闭 Integration Log 归档临时文件失败: %w", closeErr)
}
if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil {
return err
}
generatedAt := time.Now().In(s.location)
manifest := integrationArchiveManifest{
SchemaVersion: constants.IntegrationArchiveSchemaVersion, Source: constants.IntegrationArchiveSource,
ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone,
RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID,
RecordCount: file.recordCount, UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes,
ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision, GeneratedAt: generatedAt,
Status: constants.ArchiveStatusSuccess, Final: final,
}
manifestBytes, err := sonic.Marshal(manifest)
if err != nil {
return fmt.Errorf("序列化 Integration Log 归档清单失败: %w", err)
}
manifestMetadata := map[string]string{
"source": constants.IntegrationArchiveSource, "data-sha256": file.sha256,
"revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(final),
}
if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil {
return fmt.Errorf("上传 Integration Log 归档清单失败: %w", err)
}
if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil {
return err
}
completedAt := time.Now()
return s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(map[string]any{
"status": constants.ArchiveStatusSuccess, "is_final": final,
"object_key": objectKey, "manifest_key": manifestKey, "record_count": file.recordCount,
"uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes,
"sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt, "updated_at": completedAt,
}).Error
}
func (s *Service) validateIntegrationRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
metadata := map[string]string{
"sha256": run.SHA256, "record-count": strconv.FormatInt(run.RecordCount, 10),
"revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(run.IsFinal),
}
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
return false, nil
}
manifestMetadata := map[string]string{
"source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256,
"revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(run.IsFinal),
}
if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil {
return false, nil
}
return true, nil
}
func (s *Service) integrationRecordCount(ctx context.Context, start, end time.Time) (int64, error) {
var count int64
if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}).
Where("created_at >= ? AND created_at < ?", start, end).Count(&count).Error; err != nil {
return 0, fmt.Errorf("统计 Integration Log 归档记录失败: %w", err)
}
return count, nil
}
func (s *Service) integrationPendingCount(ctx context.Context, start, end time.Time) (int64, error) {
var count int64
if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}).
Where("created_at >= ? AND created_at < ? AND result = ?", start, end, constants.IntegrationResultPending).
Count(&count).Error; err != nil {
return 0, fmt.Errorf("统计 pending Integration Log 失败: %w", err)
}
return count, nil
}
func integrationObjectKeys(date time.Time, revision int) (string, string) {
prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day())
name := fmt.Sprintf("integration-logs-%s-r%d", date.Format(time.DateOnly), revision)
return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json"
}
func integrationArchiveMetadata(file *integrationArchiveFile, run *model.LogArchiveRun, final bool) map[string]string {
return map[string]string{
"schema-version": constants.IntegrationArchiveSchemaVersion,
"source": constants.IntegrationArchiveSource,
"archive-date": run.RangeStart.Format(time.DateOnly),
"timezone": constants.AuditArchiveTimezone,
"record-count": strconv.FormatInt(file.recordCount, 10),
"sha256": file.sha256,
"revision": strconv.Itoa(run.Revision),
"final": strconv.FormatBool(final),
}
}

View File

@@ -0,0 +1,542 @@
package auditarchive
import (
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/bytedance/sonic"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
const maxManifestBytes = 1024 * 1024
// RetentionAudit 描述月度物理清理的统一审计事实。
type RetentionAudit struct {
EventID string
Month string
Summary string
Result string
ErrorSummary string
RangeStart time.Time
RangeEnd time.Time
EventCount int64
ResourceCount int64
IntegrationCount int64
ManifestKeys []string
DurationMS int64
}
// RetentionResult 是月度留存清理的结构化执行结果。
type RetentionResult struct {
Month string
EventCount int64
ResourceCount int64
IntegrationCount int64
ManifestKeys []string
Duration time.Duration
}
type retentionRuns struct {
audit []*model.LogArchiveRun
integration []*model.LogArchiveRun
}
// CleanupPreviousMonth 校验并物理清理上一个完整自然月的在线审计日志。
func (s *Service) CleanupPreviousMonth(ctx context.Context) (RetentionResult, error) {
now := time.Now().In(s.location)
return s.CleanupMonth(ctx, now.AddDate(0, -1, 0))
}
// CleanupMonth 校验归档硬门禁后按固定顺序物理清理指定完整自然月。
func (s *Service) CleanupMonth(ctx context.Context, month time.Time) (result RetentionResult, cleanupErr error) {
if s.db == nil || s.store == nil || s.audit == nil {
return result, fmt.Errorf("日志留存清理数据库、对象存储或审计 Writer 未配置")
}
start, end, err := s.retentionMonthRange(month)
if err != nil {
return result, err
}
startedAt := time.Now()
result.Month = start.Format("2006-01")
cleanupErr = s.executeRetention(ctx, start, end, &result)
result.Duration = time.Since(startedAt)
if auditErr := s.recordRetentionAudit(ctx, start, end, result, cleanupErr); auditErr != nil {
if cleanupErr != nil {
return result, fmt.Errorf("%w记录留存清理失败审计失败: %v", cleanupErr, auditErr)
}
return result, fmt.Errorf("记录留存清理成功审计失败: %w", auditErr)
}
return result, cleanupErr
}
func (s *Service) retentionMonthRange(month time.Time) (time.Time, time.Time, error) {
start := time.Date(month.In(s.location).Year(), month.In(s.location).Month(), 1, 0, 0, 0, 0, s.location)
end := start.AddDate(0, 1, 0)
now := time.Now().In(s.location)
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, s.location)
if !end.Before(currentMonth) && !end.Equal(currentMonth) {
return time.Time{}, time.Time{}, fmt.Errorf("只能清理已经结束的完整自然月")
}
return start, end, nil
}
func (s *Service) executeRetention(ctx context.Context, start, end time.Time, result *RetentionResult) error {
started, err := s.retentionCleanupStarted(ctx, start, end)
if err != nil {
return err
}
if !started {
lastDay := end.AddDate(0, 0, -1)
if err := s.ArchiveDate(ctx, lastDay); err != nil {
return fmt.Errorf("完成上月最后一天 Audit 归档失败: %w", err)
}
if err := s.ArchiveIntegrationDate(ctx, lastDay); err != nil {
return fmt.Errorf("完成上月最后一天 Integration Log 归档失败: %w", err)
}
if err := s.FinalizeIntegrationMonth(ctx, start); err != nil {
return err
}
}
runs, err := s.loadRetentionRuns(ctx, start, end)
if err != nil {
return err
}
if err := s.validateRetentionRuns(ctx, start, end, runs); err != nil {
return err
}
summarizeRetentionRuns(runs, result)
if err := s.cleanupAuditMonth(ctx, start, end, runs.audit); err != nil {
return err
}
return s.cleanupIntegrationMonth(ctx, start, end, runs.integration)
}
func (s *Service) retentionCleanupStarted(ctx context.Context, start, end time.Time) (bool, error) {
var count int64
err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NOT NULL",
start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
Count(&count).Error
if err != nil {
return false, fmt.Errorf("读取月度清理断点失败: %w", err)
}
return count > 0, nil
}
func (s *Service) loadRetentionRuns(ctx context.Context, start, end time.Time) (retentionRuns, error) {
var rows []model.LogArchiveRun
err := s.db.WithContext(ctx).Where(
"source IN ? AND archive_date >= ? AND archive_date < ? AND instance_id = ?",
[]string{constants.AuditArchiveSource, constants.IntegrationArchiveSource}, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID,
).Order("archive_date ASC, source ASC").Find(&rows).Error
if err != nil {
return retentionRuns{}, fmt.Errorf("读取月度归档账本失败: %w", err)
}
days := int(end.Sub(start).Hours() / 24)
if len(rows) != days*2 {
return retentionRuns{}, fmt.Errorf("月度归档账本缺日:期望 %d 条,实际 %d 条", days*2, len(rows))
}
runs := retentionRuns{audit: make([]*model.LogArchiveRun, 0, days), integration: make([]*model.LogArchiveRun, 0, days)}
for index := range rows {
run := &rows[index]
switch run.Source {
case constants.AuditArchiveSource:
runs.audit = append(runs.audit, run)
case constants.IntegrationArchiveSource:
runs.integration = append(runs.integration, run)
}
}
if len(runs.audit) != days || len(runs.integration) != days {
return retentionRuns{}, fmt.Errorf("月度 Audit 或 Integration 归档账本不完整")
}
return runs, nil
}
func (s *Service) validateRetentionRuns(ctx context.Context, start, end time.Time, runs retentionRuns) error {
if err := validateCleanupLedgerState(runs.audit); err != nil {
return fmt.Errorf("Audit 清理断点非法: %w", err)
}
if err := validateCleanupLedgerState(runs.integration); err != nil {
return fmt.Errorf("Integration 清理断点非法: %w", err)
}
for index := range runs.audit {
date := start.AddDate(0, 0, index)
if err := s.validateAuditRetentionDay(ctx, date, runs.audit[index]); err != nil {
return fmt.Errorf("%s Audit 清理门禁失败: %w", date.Format(time.DateOnly), err)
}
if err := s.validateIntegrationRetentionDay(ctx, date, runs.integration[index]); err != nil {
return fmt.Errorf("%s Integration 清理门禁失败: %w", date.Format(time.DateOnly), err)
}
}
return nil
}
func validateCleanupLedgerState(runs []*model.LogArchiveRun) error {
started, cleaned := 0, 0
for _, run := range runs {
if run.CleanupStartedAt != nil {
started++
}
if run.CleanedAt != nil {
cleaned++
}
}
if started != 0 && started != len(runs) {
return fmt.Errorf("清理开始断点不是整月原子状态")
}
if cleaned != 0 && cleaned != len(runs) {
return fmt.Errorf("清理完成断点不是整月原子状态")
}
if cleaned > 0 && started == 0 {
return fmt.Errorf("清理完成但缺少开始断点")
}
return nil
}
func (s *Service) validateAuditRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
if err := validateRunBase(run, date, constants.AuditArchiveSchemaVersion, false); err != nil {
return err
}
if err := s.validateAuditManifest(ctx, run); err != nil {
return err
}
events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
if err != nil {
return err
}
return validateRemainingCounts(run, events, resources)
}
func (s *Service) validateIntegrationRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
if err := validateRunBase(run, date, constants.IntegrationArchiveSchemaVersion, true); err != nil {
return err
}
if err := s.validateIntegrationManifest(ctx, run); err != nil {
return err
}
count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd)
if err != nil {
return err
}
if run.CleanedAt != nil {
if count != 0 {
return fmt.Errorf("已标记清理完成但数据库仍有 %d 条记录", count)
}
return nil
}
if run.CleanupStartedAt != nil {
if count > run.RecordCount {
return fmt.Errorf("续跑窗口记录数超过最终归档数量")
}
return nil
}
file, err := s.buildIntegrationArchiveFile(ctx, run.RangeStart, run.RangeEnd)
if err != nil {
return err
}
defer os.Remove(file.path)
if file.recordCount != run.RecordCount || file.sha256 != run.SHA256 {
return fmt.Errorf("数据库当前 Integration 内容与最终 revision 不一致")
}
return nil
}
func validateRunBase(run *model.LogArchiveRun, date time.Time, schema string, final bool) error {
if run.Status != constants.ArchiveStatusSuccess || run.SchemaVersion != schema {
return fmt.Errorf("归档状态或 schema version 不符合清理要求")
}
if run.ArchiveDate.Format(time.DateOnly) != date.Format(time.DateOnly) ||
!run.RangeStart.Equal(date) || !run.RangeEnd.Equal(date.AddDate(0, 0, 1)) {
return fmt.Errorf("归档日期或半开时间范围不一致")
}
if final && !run.IsFinal {
return fmt.Errorf("Integration 最终 revision 尚未形成")
}
if run.ObjectKey == "" || run.ManifestKey == "" || run.SHA256 == "" {
return fmt.Errorf("归档对象、manifest 或 SHA-256 缺失")
}
return nil
}
func validateRemainingCounts(run *model.LogArchiveRun, events, resources int64) error {
if run.CleanedAt != nil {
if events != 0 || resources != 0 {
return fmt.Errorf("已标记清理完成但数据库仍有事件或资源")
}
return nil
}
if run.CleanupStartedAt != nil {
if events > run.EventCount || resources > run.ResourceCount {
return fmt.Errorf("续跑窗口数量超过已归档数量")
}
return nil
}
if events != run.EventCount || resources != run.ResourceCount {
return fmt.Errorf("数据库事件或资源数量与 manifest 不一致")
}
return nil
}
func (s *Service) validateAuditManifest(ctx context.Context, run *model.LogArchiveRun) error {
var manifest Manifest
if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil {
return err
}
if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess ||
manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone ||
manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) ||
manifest.EventCount != run.EventCount || manifest.ResourceCount != run.ResourceCount ||
manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey ||
manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision {
return fmt.Errorf("Audit manifest 与 ledger 不一致")
}
if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{
"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
}); err != nil {
return err
}
return s.verifyRetentionObject(ctx, run, false)
}
func (s *Service) validateIntegrationManifest(ctx context.Context, run *model.LogArchiveRun) error {
var manifest integrationArchiveManifest
if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil {
return err
}
if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || !manifest.Final ||
manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone ||
manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) ||
manifest.RecordCount != run.RecordCount || manifest.CompressedBytes != run.CompressedBytes ||
manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision {
return fmt.Errorf("Integration manifest 与最终 ledger 不一致")
}
if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{
"source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256,
"revision": strconv.Itoa(run.Revision), "final": "true",
}); err != nil {
return err
}
return s.verifyRetentionObject(ctx, run, true)
}
func (s *Service) readManifest(ctx context.Context, key string, target any) error {
object, err := s.store.Stat(ctx, key)
if err != nil {
return fmt.Errorf("读取 manifest metadata 失败: %w", err)
}
reader, err := s.store.Download(ctx, key)
if err != nil {
return fmt.Errorf("下载 manifest 失败: %w", err)
}
data, readErr := io.ReadAll(io.LimitReader(reader, maxManifestBytes+1))
closeErr := reader.Close()
if readErr != nil {
return fmt.Errorf("读取 manifest 失败: %w", readErr)
}
if closeErr != nil {
return fmt.Errorf("关闭 manifest 对象失败: %w", closeErr)
}
if len(data) > maxManifestBytes || int64(len(data)) != object.Size {
return fmt.Errorf("manifest 大小非法或不完整")
}
if err := sonic.Unmarshal(data, target); err != nil {
return fmt.Errorf("解析 manifest 失败: %w", err)
}
return nil
}
func (s *Service) verifyRetentionObject(ctx context.Context, run *model.LogArchiveRun, final bool) error {
metadata := map[string]string{
"schema-version": run.SchemaVersion, "source": run.Source,
"archive-date": run.RangeStart.Format(time.DateOnly), "timezone": constants.AuditArchiveTimezone,
"sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
}
if run.Source == constants.AuditArchiveSource {
metadata["event-count"] = strconv.FormatInt(run.EventCount, 10)
metadata["resource-count"] = strconv.FormatInt(run.ResourceCount, 10)
} else {
metadata["record-count"] = strconv.FormatInt(run.RecordCount, 10)
metadata["final"] = strconv.FormatBool(final)
}
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
return err
}
reader, err := s.store.Download(ctx, run.ObjectKey)
if err != nil {
return fmt.Errorf("下载归档对象复核 SHA-256 失败: %w", err)
}
hasher := sha256.New()
written, copyErr := io.Copy(hasher, reader)
closeErr := reader.Close()
if copyErr != nil {
return fmt.Errorf("读取归档对象复核 SHA-256 失败: %w", copyErr)
}
if closeErr != nil {
return fmt.Errorf("关闭归档对象失败: %w", closeErr)
}
if written != run.CompressedBytes || fmt.Sprintf("%x", hasher.Sum(nil)) != run.SHA256 {
return fmt.Errorf("归档对象大小或 SHA-256 复核失败")
}
return nil
}
func summarizeRetentionRuns(runs retentionRuns, result *RetentionResult) {
result.ManifestKeys = make([]string, 0, len(runs.audit)+len(runs.integration))
for _, run := range runs.audit {
result.EventCount += run.EventCount
result.ResourceCount += run.ResourceCount
result.ManifestKeys = append(result.ManifestKeys, run.ManifestKey)
}
for _, run := range runs.integration {
result.IntegrationCount += run.RecordCount
result.ManifestKeys = append(result.ManifestKeys, run.ManifestKey)
}
}
func (s *Service) cleanupAuditMonth(ctx context.Context, start, end time.Time, runs []*model.LogArchiveRun) error {
if allRunsCleaned(runs) {
return nil
}
if err := s.markCleanupStarted(ctx, constants.AuditArchiveSource, start, end); err != nil {
return err
}
if err := s.deleteAuditResources(ctx, start, end); err != nil {
return err
}
if err := s.deleteAuditEvents(ctx, start, end); err != nil {
return err
}
return s.markCleaned(ctx, constants.AuditArchiveSource, start, end)
}
func (s *Service) cleanupIntegrationMonth(ctx context.Context, start, end time.Time, runs []*model.LogArchiveRun) error {
if allRunsCleaned(runs) {
return nil
}
if err := s.markCleanupStarted(ctx, constants.IntegrationArchiveSource, start, end); err != nil {
return err
}
for {
subquery := s.db.Model(&model.IntegrationLog{}).Select("id").
Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.IntegrationLog{})
if deleted.Error != nil {
return fmt.Errorf("分批物理删除 Integration Log 失败: %w", deleted.Error)
}
if deleted.RowsAffected == 0 {
break
}
}
return s.markCleaned(ctx, constants.IntegrationArchiveSource, start, end)
}
func (s *Service) deleteAuditResources(ctx context.Context, start, end time.Time) error {
for {
subquery := s.db.Model(&model.AuditEventResource{}).Select("tb_audit_event_resource.id").
Joins("JOIN tb_audit_event ON tb_audit_event.id = tb_audit_event_resource.audit_event_id").
Where("tb_audit_event.created_at >= ? AND tb_audit_event.created_at < ?", start, end).
Order("tb_audit_event_resource.id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEventResource{})
if deleted.Error != nil {
return fmt.Errorf("分批物理删除 Audit Event Resource 失败: %w", deleted.Error)
}
if deleted.RowsAffected == 0 {
return nil
}
}
}
func (s *Service) deleteAuditEvents(ctx context.Context, start, end time.Time) error {
for {
subquery := s.db.Model(&model.AuditEvent{}).Select("id").
Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEvent{})
if deleted.Error != nil {
return fmt.Errorf("分批物理删除 Audit Event 失败: %w", deleted.Error)
}
if deleted.RowsAffected == 0 {
return nil
}
}
}
func (s *Service) markCleanupStarted(ctx context.Context, source string, start, end time.Time) error {
now := time.Now()
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NULL",
source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
Updates(map[string]any{"cleanup_started_at": now, "updated_at": now})
if result.Error != nil {
return fmt.Errorf("记录月度清理开始断点失败: %w", result.Error)
}
return s.validateCleanupMarkerCount(ctx, source, start, end, "cleanup_started_at IS NOT NULL", "开始")
}
func (s *Service) markCleaned(ctx context.Context, source string, start, end time.Time) error {
now := time.Now()
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NOT NULL",
source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
Updates(map[string]any{"cleaned_at": now, "updated_at": now})
if result.Error != nil {
return fmt.Errorf("记录月度清理完成断点失败: %w", result.Error)
}
return s.validateCleanupMarkerCount(ctx, source, start, end, "cleaned_at IS NOT NULL", "完成")
}
func (s *Service) validateCleanupMarkerCount(ctx context.Context, source string, start, end time.Time, marker, label string) error {
var count int64
err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND "+marker,
source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
Count(&count).Error
if err != nil {
return fmt.Errorf("复核月度清理%s断点失败: %w", label, err)
}
expected := int64(end.Sub(start).Hours() / 24)
if count != expected {
return fmt.Errorf("月度清理%s断点不完整期望 %d 条,实际 %d 条", label, expected, count)
}
return nil
}
func allRunsCleaned(runs []*model.LogArchiveRun) bool {
return len(runs) > 0 && runs[0].CleanedAt != nil
}
func (s *Service) recordRetentionAudit(ctx context.Context, start, end time.Time, result RetentionResult, cleanupErr error) error {
audit := RetentionAudit{
Month: result.Month, RangeStart: start, RangeEnd: end,
EventCount: result.EventCount, ResourceCount: result.ResourceCount,
IntegrationCount: result.IntegrationCount, ManifestKeys: result.ManifestKeys,
DurationMS: result.Duration.Milliseconds(), Result: constants.AuditResultSuccess,
Summary: "完成已归档在线日志月度物理清理",
EventID: "evt_retention_" + strings.ReplaceAll(result.Month, "-", "_"),
}
if cleanupErr != nil {
audit.EventID = ""
audit.Result = constants.AuditResultFailed
audit.Summary = "已归档在线日志月度物理清理失败"
audit.ErrorSummary = truncateRetentionError(cleanupErr)
}
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
return s.audit.WriteRetentionCleanup(ctx, tx, audit)
})
}
func truncateRetentionError(err error) string {
value := []rune(err.Error())
if len(value) > 500 {
value = value[:500]
}
return string(value)
}

View File

@@ -0,0 +1,412 @@
// Package auditarchive 实现统一审计每日冷归档用例。
package auditarchive
import (
"compress/gzip"
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/bytedance/sonic"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/storage"
)
const archivePageSize = 1000
// ObjectStore 是每日归档需要的最小对象存储能力。
type ObjectStore interface {
UploadWithMetadata(context.Context, string, io.Reader, string, map[string]string) error
Stat(context.Context, string) (*storage.ObjectMetadata, error)
Download(context.Context, string) (io.ReadCloser, error)
}
// RetentionAuditWriter 记录月度留存清理的系统审计事实。
type RetentionAuditWriter interface {
WriteRetentionCleanup(context.Context, *gorm.DB, RetentionAudit) error
}
// Service 编排审计归档生成、上传、复核和幂等账本更新。
type Service struct {
db *gorm.DB
store ObjectStore
audit RetentionAuditWriter
instanceID string
location *time.Location
}
// Manifest 是归档对象的完整性清单。
type Manifest struct {
SchemaVersion string `json:"schema_version"`
Source string `json:"source"`
ArchiveDate string `json:"archive_date"`
Timezone string `json:"timezone"`
RangeStart time.Time `json:"range_start"`
RangeEnd time.Time `json:"range_end"`
InstanceID string `json:"instance_id"`
EventCount int64 `json:"event_count"`
ResourceCount int64 `json:"resource_count"`
UncompressedBytes int64 `json:"uncompressed_bytes"`
CompressedBytes int64 `json:"compressed_bytes"`
ObjectKey string `json:"object_key"`
SHA256 string `json:"sha256"`
Revision int `json:"revision"`
GeneratedAt time.Time `json:"generated_at"`
Status string `json:"status"`
}
type archiveLine struct {
Event model.AuditEvent `json:"event"`
Resources []model.AuditEventResource `json:"resources"`
}
type archiveFile struct {
path string
eventCount int64
resourceCount int64
uncompressedBytes int64
compressedBytes int64
sha256 string
}
// NewService 创建统一审计每日冷归档服务。
func NewService(db *gorm.DB, store ObjectStore, instanceID string, audit ...RetentionAuditWriter) (*Service, error) {
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
if err != nil {
return nil, fmt.Errorf("加载审计归档时区失败: %w", err)
}
if strings.TrimSpace(instanceID) == "" {
instanceID = "audit-archive"
}
var auditWriter RetentionAuditWriter
if len(audit) > 0 {
auditWriter = audit[0]
}
return &Service{db: db, store: store, audit: auditWriter, instanceID: instanceID, location: location}, nil
}
// ArchivePreviousDay 归档 Asia/Shanghai 前一完整自然日。
func (s *Service) ArchivePreviousDay(ctx context.Context) error {
now := time.Now().In(s.location)
return s.ArchiveDate(ctx, now.AddDate(0, 0, -1))
}
// ArchiveDate 归档指定 Asia/Shanghai 自然日。
func (s *Service) ArchiveDate(ctx context.Context, archiveDate time.Time) error {
if s.db == nil || s.store == nil {
return fmt.Errorf("审计归档数据库或对象存储未配置")
}
start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location)
end := start.AddDate(0, 0, 1)
today := time.Now().In(s.location)
todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)
if end.After(todayStart) {
return fmt.Errorf("统一审计只能归档已经结束的完整自然日")
}
run, err := s.ensureRun(ctx, start, end)
if err != nil {
return err
}
if run.Status == constants.ArchiveStatusSuccess {
valid, validateErr := s.validateSuccessfulRun(ctx, run)
if validateErr == nil && valid {
return nil
}
}
acquired, err := s.acquireRun(ctx, run)
if err != nil || !acquired {
return err
}
if err := s.execute(ctx, run); err != nil {
s.markFailed(ctx, run.ID, err)
return err
}
return nil
}
func (s *Service) ensureRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) {
run := model.LogArchiveRun{
Source: constants.AuditArchiveSource, ArchiveDate: start, InstanceID: s.instanceID,
SchemaVersion: constants.AuditArchiveSchemaVersion, Revision: 1,
Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end,
}
result := s.db.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}},
DoNothing: true,
}).Create(&run)
if result.Error != nil {
return nil, fmt.Errorf("创建审计归档账本失败: %w", result.Error)
}
if result.RowsAffected == 0 {
if err := s.db.WithContext(ctx).Where(
"source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?",
constants.AuditArchiveSource, start, s.instanceID, constants.AuditArchiveSchemaVersion,
).First(&run).Error; err != nil {
return nil, fmt.Errorf("读取审计归档账本失败: %w", err)
}
}
return &run, nil
}
func (s *Service) acquireRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
revision := run.Revision
if run.Status == constants.ArchiveStatusFailed || run.Status == constants.ArchiveStatusSuccess || run.Status == constants.ArchiveStatusRunning {
revision++
}
now := time.Now()
staleBefore := now.Add(-3 * time.Hour)
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, staleBefore).
Updates(map[string]any{
"status": constants.ArchiveStatusRunning, "revision": revision,
"attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "",
"completed_at": nil, "updated_at": now,
})
if result.Error != nil {
return false, fmt.Errorf("锁定审计归档任务失败: %w", result.Error)
}
if result.RowsAffected == 0 {
return false, nil
}
run.Revision = revision
run.Status = constants.ArchiveStatusRunning
return true, nil
}
func (s *Service) execute(ctx context.Context, run *model.LogArchiveRun) error {
file, err := s.buildArchiveFile(ctx, run.RangeStart, run.RangeEnd)
if err != nil {
return err
}
defer os.Remove(file.path)
dbEvents, dbResources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
if err != nil {
return err
}
if dbEvents != file.eventCount || dbResources != file.resourceCount {
return fmt.Errorf("审计归档生成期间数据数量发生变化")
}
objectKey, manifestKey := objectKeys(run.RangeStart, run.Revision)
metadata := archiveMetadata(file, run)
reader, err := os.Open(file.path)
if err != nil {
return fmt.Errorf("打开审计归档临时文件失败: %w", err)
}
uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata)
closeErr := reader.Close()
if uploadErr != nil {
return fmt.Errorf("上传审计归档对象失败: %w", uploadErr)
}
if closeErr != nil {
return fmt.Errorf("关闭审计归档临时文件失败: %w", closeErr)
}
if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil {
return err
}
generatedAt := time.Now().In(s.location)
manifest := Manifest{
SchemaVersion: constants.AuditArchiveSchemaVersion, Source: constants.AuditArchiveSource,
ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone,
RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID,
EventCount: file.eventCount, ResourceCount: file.resourceCount,
UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes,
ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision,
GeneratedAt: generatedAt, Status: constants.ArchiveStatusSuccess,
}
manifestBytes, err := sonic.Marshal(manifest)
if err != nil {
return fmt.Errorf("序列化审计归档清单失败: %w", err)
}
manifestMetadata := map[string]string{"source": constants.AuditArchiveSource, "data-sha256": file.sha256, "revision": strconv.Itoa(run.Revision)}
if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil {
return fmt.Errorf("上传审计归档清单失败: %w", err)
}
if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil {
return err
}
completedAt := time.Now()
updates := map[string]any{
"status": constants.ArchiveStatusSuccess, "object_key": objectKey, "manifest_key": manifestKey,
"event_count": file.eventCount, "resource_count": file.resourceCount,
"uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes,
"sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt,
"updated_at": completedAt,
}
if err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(updates).Error; err != nil {
return fmt.Errorf("更新审计归档成功账本失败: %w", err)
}
return nil
}
func (s *Service) buildArchiveFile(ctx context.Context, start, end time.Time) (*archiveFile, error) {
temp, err := os.CreateTemp("", "audit-events-*.jsonl.gz")
if err != nil {
return nil, fmt.Errorf("创建审计归档临时文件失败: %w", err)
}
path := temp.Name()
failed := true
defer func() {
_ = temp.Close()
if failed {
_ = os.Remove(path)
}
}()
hasher := sha256.New()
gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher))
result := &archiveFile{path: path}
var lastID uint
for {
var events []model.AuditEvent
if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID).
Order("id ASC").Limit(archivePageSize).Find(&events).Error; err != nil {
return nil, fmt.Errorf("读取审计归档事件失败: %w", err)
}
if len(events) == 0 {
break
}
ids := make([]uint, 0, len(events))
for i := range events {
ids = append(ids, events[i].ID)
}
var resources []model.AuditEventResource
if err := s.db.WithContext(ctx).Where("audit_event_id IN ?", ids).
Order("audit_event_id ASC, sort_order ASC, id ASC").Find(&resources).Error; err != nil {
return nil, fmt.Errorf("读取审计归档资源失败: %w", err)
}
grouped := make(map[uint][]model.AuditEventResource, len(events))
for i := range resources {
resource := resources[i]
grouped[resource.AuditEventID] = append(grouped[resource.AuditEventID], resource)
}
for i := range events {
eventResources := grouped[events[i].ID]
if eventResources == nil {
eventResources = []model.AuditEventResource{}
}
line, marshalErr := sonic.Marshal(archiveLine{Event: events[i], Resources: eventResources})
if marshalErr != nil {
return nil, fmt.Errorf("序列化审计归档事件失败: %w", marshalErr)
}
line = append(line, '\n')
if _, writeErr := gzipWriter.Write(line); writeErr != nil {
return nil, fmt.Errorf("写入审计归档压缩流失败: %w", writeErr)
}
result.eventCount++
result.resourceCount += int64(len(grouped[events[i].ID]))
result.uncompressedBytes += int64(len(line))
}
lastID = events[len(events)-1].ID
}
if err := gzipWriter.Close(); err != nil {
return nil, fmt.Errorf("关闭审计归档压缩流失败: %w", err)
}
if err := temp.Close(); err != nil {
return nil, fmt.Errorf("关闭审计归档临时文件失败: %w", err)
}
info, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("读取审计归档临时文件信息失败: %w", err)
}
result.compressedBytes = info.Size()
result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil))
failed = false
return result, nil
}
func (s *Service) databaseCounts(ctx context.Context, start, end time.Time) (int64, int64, error) {
var eventCount int64
if err := s.db.WithContext(ctx).Model(&model.AuditEvent{}).
Where("created_at >= ? AND created_at < ?", start, end).Count(&eventCount).Error; err != nil {
return 0, 0, fmt.Errorf("统计审计归档事件失败: %w", err)
}
var resourceCount int64
subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end)
if err := s.db.WithContext(ctx).Model(&model.AuditEventResource{}).
Where("audit_event_id IN (?)", subquery).Count(&resourceCount).Error; err != nil {
return 0, 0, fmt.Errorf("统计审计归档资源失败: %w", err)
}
return eventCount, resourceCount, nil
}
func (s *Service) validateSuccessfulRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
if err != nil || events != run.EventCount || resources != run.ResourceCount {
return false, err
}
metadata := map[string]string{
"sha256": run.SHA256, "event-count": strconv.FormatInt(run.EventCount, 10),
"resource-count": strconv.FormatInt(run.ResourceCount, 10), "revision": strconv.Itoa(run.Revision),
}
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
return false, nil
}
manifestMetadata := map[string]string{
"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
}
if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil {
return false, nil
}
return true, nil
}
func (s *Service) verifyObject(ctx context.Context, key string, expectedSize int64, expectedMetadata map[string]string) error {
object, err := s.store.Stat(ctx, key)
if err != nil {
return fmt.Errorf("复核归档对象 metadata 失败: %w", err)
}
if expectedSize >= 0 && object.Size != expectedSize {
return fmt.Errorf("归档对象大小复核失败")
}
for name, value := range expectedMetadata {
if object.Metadata[strings.ToLower(name)] != value {
return fmt.Errorf("归档对象 metadata 字段 %s 复核失败", name)
}
}
return nil
}
func (s *Service) markFailed(ctx context.Context, runID uint, archiveErr error) {
summary := []rune(archiveErr.Error())
if len(summary) > 500 {
summary = summary[:500]
}
failedCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
_ = s.db.WithContext(failedCtx).Model(&model.LogArchiveRun{}).Where("id = ?", runID).Updates(map[string]any{
"status": constants.ArchiveStatusFailed, "error_summary": string(summary), "updated_at": time.Now(),
}).Error
}
func objectKeys(date time.Time, revision int) (string, string) {
prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day())
name := fmt.Sprintf("audit-events-%s-r%d", date.Format(time.DateOnly), revision)
return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json"
}
func archiveMetadata(file *archiveFile, run *model.LogArchiveRun) map[string]string {
return map[string]string{
"schema-version": constants.AuditArchiveSchemaVersion,
"source": constants.AuditArchiveSource,
"archive-date": run.RangeStart.Format(time.DateOnly),
"timezone": constants.AuditArchiveTimezone,
"event-count": strconv.FormatInt(file.eventCount, 10),
"resource-count": strconv.FormatInt(file.resourceCount, 10),
"sha256": file.sha256,
"revision": strconv.Itoa(run.Revision),
}
}

View File

@@ -84,7 +84,7 @@ func (s *Service) RecordCarrierCallbackFailure(ctx context.Context, card *model.
}
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardRealnameCallbackSynced,
Summary: "运营商回调同步 IoT 卡实名状态失败", Card: card, IntegrationID: integrationID,
Summary: "运营商回调同步 IoT 卡实名状态失败", Card: card, IntegrationID: integrationID,
}, businessErr)
}
@@ -94,6 +94,7 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
return domain.RealnameDecision{}, errors.New(errors.CodeInternalError, "卡实名观测能力未完整配置")
}
var decision domain.RealnameDecision
var auditedCard *model.IotCard
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var card model.IotCard
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
@@ -102,6 +103,7 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
}
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡失败")
}
auditedCard = &card
nextDecision, decisionErr := domain.ApplyRealname(domain.CardRealnameSnapshot{
CardID: card.ID, Status: card.RealNameStatus, FirstRealnameAt: card.FirstRealnameAt,
ReversalCount: card.RealnameReversalCount, ReversalStartedAt: card.RealnameReversalStartedAt,
@@ -185,11 +187,26 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: constants.AuditActionIotCardRealnameCallbackSynced,
Summary: "运营商回调同步 IoT 卡实名状态",
Card: &card,
ActionCode: constants.AuditActionIotCardRealnameCallbackSynced,
Summary: "运营商回调同步 IoT 卡实名状态",
Card: &card,
IntegrationID: observation.Metadata.ObservationID,
BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt},
BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt},
AfterData: map[string]any{
"real_name_status": decision.AfterStatus, "first_realname_at": firstRealnameAfter(card.FirstRealnameAt, observation.Metadata.ObservedAt, decision.FirstVerified),
},
}); err != nil {
return err
}
} else if workerObservationAudited(ctx) && decision.StatusChanged {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerRealnameSynced,
Summary: "Worker 同步 IoT 卡实名事实", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt},
AfterData: map[string]any{
"real_name_status": decision.AfterStatus, "first_realname_at": firstRealnameAfter(card.FirstRealnameAt, observation.Metadata.ObservedAt, decision.FirstVerified),
},
@@ -200,6 +217,13 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
return nil
})
if err != nil {
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerRealnameSynced,
Summary: "Worker 同步 IoT 卡实名事实失败", Card: auditedCard,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
return domain.RealnameDecision{}, err
}
if s.cache != nil {
@@ -226,6 +250,11 @@ func manualRefreshAuditAction(ctx context.Context) (string, bool) {
}
}
func workerObservationAudited(ctx context.Context) bool {
linkage := auditcontext.From(ctx)
return linkage.ActorKind == constants.AuditActorSystemTask && linkage.Source == constants.AuditSourceWorker
}
func realnameChangedEventID(cardID uint, observationID string) string {
prefix := "card-realname:"
digest := sha256.Sum256([]byte(strconv.FormatUint(uint64(cardID), 10) + ":" + observationID + ":changed"))

View File

@@ -33,6 +33,7 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai
return domain.NetworkDecision{}, errors.New(errors.CodeInternalError, "卡网络观测能力未完整配置")
}
var decision domain.NetworkDecision
var auditedCard *model.IotCard
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var card model.IotCard
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
@@ -41,6 +42,7 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai
}
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡网络事实失败")
}
auditedCard = &card
nextDecision, decisionErr := domain.ApplyNetwork(domain.CardNetworkSnapshot{
CardID: card.ID, NetworkStatus: card.NetworkStatus, StopReason: card.StopReason,
IsStandalone: card.IsStandalone, EnablePolling: card.EnablePolling,
@@ -117,10 +119,46 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai
}); err != nil {
return err
}
} else if workerObservationAudited(ctx) && stateChanged {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
enablePolling := card.EnablePolling
if decision.StopPolling {
enablePolling = false
}
gatewayIMEI := card.GatewayCardIMEI
if decision.UpdateIMEI {
gatewayIMEI = decision.GatewayIMEI
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerNetworkSynced,
Summary: "Worker 同步 IoT 卡网络事实", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
BeforeData: map[string]any{
"network_status": card.NetworkStatus, "stop_reason": card.StopReason,
"gateway_extend": card.GatewayExtend, "gateway_card_imei": card.GatewayCardIMEI,
"enable_polling": card.EnablePolling,
},
AfterData: map[string]any{
"network_status": decision.AfterStatus, "stop_reason": decision.StopReason,
"gateway_extend": decision.GatewayExtend, "gateway_card_imei": gatewayIMEI,
"enable_polling": enablePolling,
},
}); err != nil {
return err
}
}
return nil
})
if err != nil {
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerNetworkSynced,
Summary: "Worker 同步 IoT 卡网络事实失败", Card: auditedCard,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
return domain.NetworkDecision{}, err
}
if s.cache != nil {

View File

@@ -7,6 +7,7 @@ import (
"github.com/google/uuid"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
@@ -21,6 +22,7 @@ type SeriesRequest struct {
Source string `json:"source"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
ParentEventID string `json:"parent_event_id,omitempty"`
}
// DeviceCardsSeriesRequest 描述需要在后台展开设备有效绑定卡的观测请求。
@@ -59,6 +61,7 @@ type SeriesTaskPayload struct {
Source string `json:"source"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
ParentEventID string `json:"parent_event_id,omitempty"`
}
// RunResult 描述一次实际 Gateway 请求及公共观测应用结果。
@@ -135,7 +138,7 @@ func (s *SeriesTrigger) trigger(ctx context.Context, request SeriesRequest, cand
if s == nil || s.coordinator == nil || s.scheduler == nil || s.logger == nil {
return "", false, errors.New(errors.CodeInternalError, "卡观测序列触发能力未完整配置")
}
request = normalizeSeriesTrace(request)
request = normalizeSeriesTrace(ctx, request)
if err := validateSeriesRequest(request); err != nil {
return "", false, err
}
@@ -171,6 +174,7 @@ func (s *SeriesTrigger) trigger(ctx context.Context, request SeriesRequest, cand
Scene: originalRequest.Scene, ResourceType: originalRequest.ResourceType, ResourceID: originalRequest.ResourceID,
SyncType: originalRequest.SyncType, ExpectedValue: originalRequest.ExpectedValue, Source: originalRequest.Source,
RequestID: originalRequest.RequestID, CorrelationID: originalRequest.CorrelationID,
ParentEventID: originalRequest.ParentEventID,
}
if err := s.scheduler.Enqueue(ctx, payload); err != nil {
s.coordinator.ReleaseSchedule(ctx, seriesID, attempt)
@@ -303,12 +307,24 @@ func validateSeriesPayload(payload SeriesTaskPayload) error {
return validateSeriesRequest(SeriesRequest{
Scene: payload.Scene, ResourceType: payload.ResourceType, ResourceID: payload.ResourceID,
SyncType: payload.SyncType, Source: payload.Source, RequestID: payload.RequestID, CorrelationID: payload.CorrelationID,
ParentEventID: payload.ParentEventID,
})
}
func normalizeSeriesTrace(request SeriesRequest) SeriesRequest {
func normalizeSeriesTrace(ctx context.Context, request SeriesRequest) SeriesRequest {
linkage := auditcontext.From(ctx)
request.RequestID = strings.TrimSpace(request.RequestID)
request.CorrelationID = strings.TrimSpace(request.CorrelationID)
request.ParentEventID = strings.TrimSpace(request.ParentEventID)
if request.RequestID == "" {
request.RequestID = strings.TrimSpace(linkage.RequestID)
}
if request.CorrelationID == "" {
request.CorrelationID = strings.TrimSpace(linkage.CorrelationID)
}
if request.ParentEventID == "" {
request.ParentEventID = strings.TrimSpace(linkage.ParentEventID)
}
if request.RequestID == "" && request.CorrelationID == "" {
traceID := uuid.NewString()
request.RequestID = traceID

View File

@@ -32,6 +32,7 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
return domain.TrafficDecision{}, errors.New(errors.CodeInternalError, "卡流量观测能力未完整配置")
}
var decision domain.TrafficDecision
var auditedCard *model.IotCard
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var card model.IotCard
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
@@ -40,6 +41,7 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
}
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡流量事实失败")
}
auditedCard = &card
nextDecision, decisionErr := domain.ApplyTraffic(domain.CardTrafficSnapshot{
CardID: card.ID, DataUsageMB: card.DataUsageMB, CurrentMonthUsageMB: card.CurrentMonthUsageMB,
CurrentMonthStartDate: card.CurrentMonthStartDate, LastMonthTotalMB: card.LastMonthTotalMB,
@@ -99,10 +101,38 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
}); err != nil {
return err
}
} else if workerObservationAudited(ctx) && stateChanged {
if s.auditWriter == nil {
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
}
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实", Card: &card,
IntegrationID: observation.Metadata.ObservationID,
BeforeData: map[string]any{
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
"last_gateway_reading_mb": card.LastGatewayReadingMB,
},
AfterData: map[string]any{
"data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB,
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
"last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB,
},
}); err != nil {
return err
}
}
return nil
})
if err != nil {
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
Summary: "Worker 同步 IoT 卡流量事实失败", Card: auditedCard,
IntegrationID: observation.Metadata.ObservationID,
}, err)
}
return domain.TrafficDecision{}, err
}
if s.cache != nil {

View File

@@ -191,7 +191,7 @@ func initServices(s *stores, deps *Dependencies) *services {
)
cardObservationSeries := cardObservationApp.NewSeriesAttemptService(
seriesCoordinator,
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration),
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration, auditWriter),
cardObservationInfra.NewSeriesAttemptLogger(seriesIntegration),
)
observationSeries := cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier)

View File

@@ -102,11 +102,18 @@ func initWorkerServices(stores *queue.WorkerStores, deps *WorkerDependencies) *q
cardObservationInfra.NewEventWriter(cardObservationOutbox),
cardObservationInfra.NewCacheInvalidator(deps.Redis, deps.Logger),
)
iotCardAuditService := iotCardSvc.New(
deps.DB, stores.IotCard, stores.Shop, stores.AssetAllocationRecord,
stores.ShopPackageAllocation, stores.ShopSeriesAllocation, stores.PackageSeries,
deps.GatewayClient, deps.Logger, assetAudit,
)
iotCardAuditService.SetAccessAudit(auditWriter)
cardObservationService.SetStateAuditWriter(iotCardAuditService)
cardObservationIntegration := integrationlog.NewRepository(deps.DB)
cardObservationSeriesCoordinator := cardObservationInfra.NewSeriesCoordinator(deps.Redis)
cardObservationSeriesService := cardObservationApp.NewSeriesAttemptService(
cardObservationSeriesCoordinator,
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, cardObservationIntegration),
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, cardObservationIntegration, auditWriter),
cardObservationInfra.NewSeriesAttemptLogger(cardObservationIntegration),
)

View File

@@ -121,15 +121,115 @@ func (h *AuditHandler) ResourceTimeline(c *fiber.Ctx) error {
return response.Success(c, result)
}
// RequestTimeline 查询指定 HTTP 请求关联的跨事实时间线。
// GET /api/admin/audit/requests/:request_id/timeline
func (h *AuditHandler) RequestTimeline(c *fiber.Ctx) error {
requestID := c.Params("request_id")
if requestID == "" {
return errors.New(errors.CodeInvalidParam)
}
result, err := h.auditQuery.RequestTimeline(c.UserContext(), requestID)
if err != nil {
return err
}
return response.Success(c, result)
}
// CorrelationTimeline 查询跨请求业务关联时间线。
// GET /api/admin/audit/correlations/:correlation_id/timeline
func (h *AuditHandler) CorrelationTimeline(c *fiber.Ctx) error {
correlationID := c.Params("correlation_id")
if correlationID == "" {
return errors.New(errors.CodeInvalidParam)
}
result, err := h.auditQuery.CorrelationTimeline(c.UserContext(), correlationID)
if err != nil {
return err
}
return response.Success(c, result)
}
// FinanceTimeline 查询资金审计与业务账本的组合时间线。
// GET /api/admin/audit/finance/timeline
func (h *AuditHandler) FinanceTimeline(c *fiber.Ctx) error {
var request dto.AuditFinanceTimelineRequest
if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) {
return errors.New(errors.CodeInvalidParam)
}
from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo)
if err != nil {
return errors.New(errors.CodeInvalidParam)
}
result, err := h.auditQuery.FinanceTimeline(c.UserContext(), auditquery.FinanceFilter{
ShopID: request.ShopID, WalletID: request.WalletID, OrderID: request.OrderID, OrderNo: request.OrderNo,
PaymentID: request.PaymentID, PaymentNo: request.PaymentNo, RefundID: request.RefundID, RefundNo: request.RefundNo,
RechargeID: request.RechargeID, RechargeNo: request.RechargeNo, ApprovalInstanceID: request.ApprovalInstanceID,
ThirdPartyTradeNo: request.ThirdPartyTradeNo, ActorKind: request.ActorKind, ActorID: request.ActorID,
CorrelationID: request.CorrelationID, CreatedFrom: from, CreatedTo: to, Page: request.Page, PageSize: request.PageSize,
})
if err != nil {
return err
}
return response.Success(c, result)
}
// RiskOverview 查询固定风险信号总览。
// GET /api/admin/audit/risks/overview
func (h *AuditHandler) RiskOverview(c *fiber.Ctx) error {
var request dto.AuditRiskOverviewRequest
if err := c.QueryParser(&request); err != nil {
return errors.New(errors.CodeInvalidParam)
}
filter, err := riskFilter(request.AuditRiskFilterRequest, 0, 0)
if err != nil {
return err
}
result, err := h.auditQuery.RiskOverview(c.UserContext(), filter)
if err != nil {
return err
}
return response.Success(c, result)
}
// RiskEvents 查询固定风险集合的事件明细。
// GET /api/admin/audit/risks/events
func (h *AuditHandler) RiskEvents(c *fiber.Ctx) error {
var request dto.AuditRiskEventsRequest
if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) {
return errors.New(errors.CodeInvalidParam)
}
filter, err := riskFilter(request.AuditRiskFilterRequest, request.Page, request.PageSize)
if err != nil {
return err
}
result, err := h.auditQuery.RiskEvents(c.UserContext(), filter)
if err != nil {
return err
}
return response.Success(c, result)
}
func riskFilter(request dto.AuditRiskFilterRequest, page, pageSize int) (auditquery.RiskFilter, error) {
from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo)
if err != nil {
return auditquery.RiskFilter{}, errors.New(errors.CodeInvalidParam)
}
return auditquery.RiskFilter{
CreatedFrom: from, CreatedTo: to, Risk: request.Risk, Result: request.Result,
Action: request.Action, Source: request.Source, Page: page, PageSize: pageSize,
}, nil
}
// AgentResourceActivities 查询代理范围内的安全资源活动。
// GET /api/admin/agent/resource-activities/:resource_type/:identifier
func (h *AuditHandler) AgentResourceActivities(c *fiber.Ctx) error {
request, err := subjectActivityRequest(c)
request, from, to, err := subjectActivityRequest(c)
if err != nil {
return err
}
result, err := h.auditQuery.AgentResourceActivities(c.UserContext(), auditquery.SubjectActivityFilter{
ResourceType: request.ResourceType, Identifier: request.Identifier,
CreatedFrom: from, CreatedTo: to,
Page: request.Page, PageSize: request.PageSize,
})
if err != nil {
@@ -141,12 +241,13 @@ func (h *AuditHandler) AgentResourceActivities(c *fiber.Ctx) error {
// EnterpriseResourceActivities 查询企业当前有效授权资产的安全资源活动。
// GET /api/admin/enterprise/resource-activities/:resource_type/:identifier
func (h *AuditHandler) EnterpriseResourceActivities(c *fiber.Ctx) error {
request, err := subjectActivityRequest(c)
request, from, to, err := subjectActivityRequest(c)
if err != nil {
return err
}
result, err := h.auditQuery.EnterpriseResourceActivities(c.UserContext(), auditquery.SubjectActivityFilter{
ResourceType: request.ResourceType, Identifier: request.Identifier,
CreatedFrom: from, CreatedTo: to,
Page: request.Page, PageSize: request.PageSize,
})
if err != nil {
@@ -155,17 +256,21 @@ func (h *AuditHandler) EnterpriseResourceActivities(c *fiber.Ctx) error {
return response.Success(c, result)
}
func subjectActivityRequest(c *fiber.Ctx) (dto.SubjectResourceActivityRequest, error) {
func subjectActivityRequest(c *fiber.Ctx) (dto.SubjectResourceActivityRequest, *time.Time, *time.Time, error) {
var request dto.SubjectResourceActivityRequest
if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) {
return request, errors.New(errors.CodeInvalidParam)
return request, nil, nil, errors.New(errors.CodeInvalidParam)
}
request.ResourceType = c.Params("resource_type")
request.Identifier = c.Params("identifier")
if request.ResourceType == "" || request.Identifier == "" {
return request, errors.New(errors.CodeInvalidParam)
return request, nil, nil, errors.New(errors.CodeInvalidParam)
}
return request, nil
from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo)
if err != nil {
return request, nil, nil, errors.New(errors.CodeInvalidParam)
}
return request, from, to, nil
}
// IntegrationOverview 查询外部集成交互总览。

View File

@@ -109,7 +109,7 @@ func (h *CMCCRealnameHandler) process(ctx context.Context, body []byte, contentT
},
})
if err != nil {
h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err)
h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err)
return h.fail(ctx, log.IntegrationID, err)
}
if h.series != nil {

View File

@@ -135,7 +135,7 @@ func (h *CTCCRealnameHandler) process(ctx context.Context, body []byte, contentT
},
})
if err != nil {
h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err)
h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err)
return h.failPending(ctx, log.IntegrationID, err)
}
if h.series != nil {

View File

@@ -108,7 +108,7 @@ func (h *CUCCRealnameHandler) process(ctx context.Context, body []byte, contentT
},
})
if err != nil {
h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err)
h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err)
return h.fail(ctx, log.IntegrationID, err)
}
if h.series != nil {

View File

@@ -303,7 +303,7 @@ func (h *PaymentHandler) confirmAgentRechargePayment(ctx context.Context, callba
}
if log.Result == constants.IntegrationResultPending {
resolvedResourceID := strconv.FormatUint(uint64(result.PaymentID), 10)
_, err = h.integration.Complete(ctx, log.IntegrationID, integrationlog.Completion{
_, err := h.integration.Complete(ctx, log.IntegrationID, integrationlog.Completion{
Result: constants.IntegrationResultSuccess, ProviderCode: "SUCCESS",
ResponseSummary: map[string]any{"confirmed": true, "already_confirmed": result.AlreadyConfirmed},
StateChanged: !result.AlreadyConfirmed, ResourceID: &resolvedResourceID,

View File

@@ -105,6 +105,9 @@ func NewRegistry() *Registry {
iotCardRealnameCallbackSynced := iotCardAction(constants.AuditActionIotCardRealnameCallbackSynced, "运营商回调同步 IoT 卡实名状态", constants.AuditActorExternalSystem, constants.AuditSourceCallback)
iotCardManualRefreshed := iotCardAction(constants.AuditActionIotCardManualRefreshed, "人工刷新 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
iotCardPersonalRefreshed := iotCardAction(constants.AuditActionIotCardPersonalRefreshed, "个人客户刷新 IoT 卡", constants.AuditActorPersonalCustomer, constants.AuditSourcePersonalAPI)
iotCardWorkerRealnameSynced := iotCardAction(constants.AuditActionIotCardWorkerRealnameSynced, "Worker 同步 IoT 卡实名事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
iotCardWorkerTrafficSynced := iotCardAction(constants.AuditActionIotCardWorkerTrafficSynced, "Worker 同步 IoT 卡流量事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
iotCardWorkerNetworkSynced := iotCardAction(constants.AuditActionIotCardWorkerNetworkSynced, "Worker 同步 IoT 卡网络事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
iotCardManualStopped := iotCardAction(constants.AuditActionIotCardManualStopped, "人工停用 IoT 卡网络", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
iotCardManualStarted := iotCardAction(constants.AuditActionIotCardManualStarted, "人工恢复 IoT 卡网络", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
iotCardAutoStopped := iotCardAction(constants.AuditActionIotCardAutoStopped, "自动停用 IoT 卡网络", constants.AuditActorSystemTask, constants.AuditSourceWorker)
@@ -131,6 +134,7 @@ func NewRegistry() *Registry {
deviceCardBound := deviceAction(constants.AuditActionDeviceCardBound, "设备绑定 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
deviceCardUnbound := deviceAction(constants.AuditActionDeviceCardUnbound, "设备解绑 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
deviceCurrentCardSwitched := deviceExternalAction(constants.AuditActionDeviceCurrentCardSwitched, "切换设备当前卡", true)
deviceWorkerObservationSynced := deviceAction(constants.AuditActionDeviceWorkerObservationSynced, "Worker 同步设备观测事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
cardExchangeCreated := cardExchangeAction(constants.AuditActionCardExchangeCreated, "创建卡换货单", constants.AuditRiskNormal, false)
cardExchangeShippingInfoSubmitted := cardExchangeAction(constants.AuditActionCardExchangeShippingInfoSubmitted, "提交卡换货收货信息", constants.AuditRiskHigh, true)
cardExchangeShipped := cardExchangeAction(constants.AuditActionCardExchangeShipped, "卡换货发货", constants.AuditRiskNormal, false)
@@ -205,6 +209,14 @@ func NewRegistry() *Registry {
notificationReadAll.AllowedOrigins = []ActionOrigin{{Actor: constants.AuditActorPersonalCustomer, Source: constants.AuditSourcePersonalAPI}}
notificationCleanup := notificationAction(constants.AuditActionNotificationCleanup, "清理过期通知", constants.AuditResourceNotificationCleanupBatch, constants.AuditActorSystemTask, constants.AuditSourceWorker)
notificationCleanupItem := notificationAction(constants.AuditActionNotificationCleanupItem, "清理单条过期通知", constants.AuditResourceNotification, constants.AuditActorSystemTask, constants.AuditSourceWorker)
retentionCleanup := ActionDefinition{
Code: constants.AuditActionLogRetentionCleanup, Name: "清理已归档在线日志",
Category: constants.AuditCategoryReliability, Risk: constants.AuditRiskHigh,
PrimaryResource: constants.AuditResourceLogArchiveMonth, AllowedActor: constants.AuditActorSystemTask,
Source: constants.AuditSourceWorker, RequireTransaction: true,
DefaultVisibility: constants.AuditSubjectInternalOnly,
AllowedVisibility: []string{constants.AuditSubjectInternalOnly},
}
pollingConfigCreated := pollingAction(constants.AuditActionPollingConfigCreated, "创建轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
pollingConfigUpdated := pollingAction(constants.AuditActionPollingConfigUpdated, "更新轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
pollingConfigDeleted := pollingAction(constants.AuditActionPollingConfigDeleted, "删除轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
@@ -407,6 +419,9 @@ func NewRegistry() *Registry {
constants.AuditActionIotCardRealnameCallbackSynced: iotCardRealnameCallbackSynced,
constants.AuditActionIotCardManualRefreshed: iotCardManualRefreshed,
constants.AuditActionIotCardPersonalRefreshed: iotCardPersonalRefreshed,
constants.AuditActionIotCardWorkerRealnameSynced: iotCardWorkerRealnameSynced,
constants.AuditActionIotCardWorkerTrafficSynced: iotCardWorkerTrafficSynced,
constants.AuditActionIotCardWorkerNetworkSynced: iotCardWorkerNetworkSynced,
constants.AuditActionIotCardManualStopped: iotCardManualStopped,
constants.AuditActionIotCardManualStarted: iotCardManualStarted,
constants.AuditActionIotCardAutoStopped: iotCardAutoStopped,
@@ -432,6 +447,7 @@ func NewRegistry() *Registry {
constants.AuditActionDeviceCardBound: deviceCardBound,
constants.AuditActionDeviceCardUnbound: deviceCardUnbound,
constants.AuditActionDeviceCurrentCardSwitched: deviceCurrentCardSwitched,
constants.AuditActionDeviceWorkerObservationSynced: deviceWorkerObservationSynced,
constants.AuditActionCardExchangeCreated: cardExchangeCreated,
constants.AuditActionCardExchangeShippingInfoSubmitted: cardExchangeShippingInfoSubmitted,
constants.AuditActionCardExchangeShipped: cardExchangeShipped,
@@ -478,6 +494,7 @@ func NewRegistry() *Registry {
constants.AuditActionNotificationReadAll: notificationReadAll,
constants.AuditActionNotificationCleanup: notificationCleanup,
constants.AuditActionNotificationCleanupItem: notificationCleanupItem,
constants.AuditActionLogRetentionCleanup: retentionCleanup,
constants.AuditActionPollingConfigCreated: pollingConfigCreated,
constants.AuditActionPollingConfigUpdated: pollingConfigUpdated,
constants.AuditActionPollingConfigDeleted: pollingConfigDeleted,
@@ -608,6 +625,10 @@ func NewRegistry() *Registry {
"resource_type", "resource_id", "resource_key", "correlation_id",
},
},
constants.AuditResourceLogArchiveMonth: {
Type: constants.AuditResourceLogArchiveMonth, Name: "日志归档自然月",
IdentityFields: []string{"month", "timezone", "range_start", "range_end"},
},
constants.AuditResourceDeviceBatchTask: {
Type: constants.AuditResourceDeviceBatchTask, Name: "设备批量分配任务",
IdentityFields: []string{"task_no", "operation_type"},

View File

@@ -0,0 +1,38 @@
package audit
import (
"context"
"gorm.io/gorm"
auditarchive "github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// WriteRetentionCleanup 将月度物理清理结果写入当前在线月份的统一审计事件。
func (w *Writer) WriteRetentionCleanup(ctx context.Context, tx *gorm.DB, input auditarchive.RetentionAudit) error {
return w.Append(ctx, tx, AppendInput{
EventID: input.EventID, ActionCode: constants.AuditActionLogRetentionCleanup,
Summary: input.Summary,
Actor: ActorInput{
Kind: constants.AuditActorSystemTask, ID: constants.AuditActorIDRetentionWorker, Name: "日志留存清理任务",
},
Source: constants.AuditSourceWorker, ScopeType: constants.AuditScopePlatform,
Result: input.Result, ErrorSummary: input.ErrorSummary,
CorrelationID: "retention:" + input.Month,
Metadata: map[string]any{
"event_count": input.EventCount, "resource_count": input.ResourceCount,
"integration_count": input.IntegrationCount, "manifest_keys": input.ManifestKeys,
"duration_ms": input.DurationMS,
},
Resources: []ResourceInput{{
Type: constants.AuditResourceLogArchiveMonth, Key: input.Month, DisplayName: input.Month,
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleRetentionMonth,
IdentitySnapshot: map[string]any{
"month": input.Month, "timezone": constants.AuditArchiveTimezone,
"range_start": input.RangeStart, "range_end": input.RangeEnd,
},
SubjectVisibility: constants.AuditSubjectInternalOnly,
}},
})
}

View File

@@ -12,9 +12,11 @@ import (
cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/gateway"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/integrationlog"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
apperrors "github.com/break/junhong_cmp_fiber/pkg/errors"
)
@@ -77,11 +79,15 @@ type SeriesRunner struct {
observation *cardapp.Service
integration *integrationlog.Repository
carrier *postgres.CarrierStore
auditWriter *audit.Writer
}
// NewSeriesRunner 创建卡观测序列 Gateway 执行器。
func NewSeriesRunner(db *gorm.DB, gatewayClient *gateway.Client, observation *cardapp.Service, integration *integrationlog.Repository) *SeriesRunner {
return &SeriesRunner{db: db, gateway: gatewayClient, observation: observation, integration: integration, carrier: postgres.NewCarrierStore(db)}
func NewSeriesRunner(db *gorm.DB, gatewayClient *gateway.Client, observation *cardapp.Service, integration *integrationlog.Repository, auditWriter *audit.Writer) *SeriesRunner {
return &SeriesRunner{
db: db, gateway: gatewayClient, observation: observation, integration: integration,
carrier: postgres.NewCarrierStore(db), auditWriter: auditWriter,
}
}
// Provider 返回用于请求互斥的运营商接入标识。
@@ -188,7 +194,7 @@ func (r *SeriesRunner) runDeviceInfo(ctx context.Context, payload cardapp.Series
response, runErr := r.gateway.SyncDeviceInfo(ctx, &gateway.SyncDeviceInfoReq{CardNo: deviceGatewayIdentifier(device)})
result := cardapp.RunResult{}
if runErr == nil {
result.StateChanged, runErr = r.applyDeviceInfo(ctx, device, response)
result.StateChanged, runErr = r.applyDeviceInfo(ctx, device, response, attempt.IntegrationID)
}
completionResult := constants.IntegrationResultSuccess
if runErr != nil {
@@ -222,7 +228,7 @@ func (r *SeriesRunner) startDeviceAttempt(ctx context.Context, payload cardapp.S
})
}
func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device, response *gateway.SyncDeviceInfoResp) (bool, error) {
func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device, response *gateway.SyncDeviceInfoResp, integrationID string) (bool, error) {
if response == nil {
return false, apperrors.New(apperrors.CodeGatewayError, "Gateway 设备信息响应为空")
}
@@ -236,21 +242,19 @@ func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device
if lastOnlineTime := parseGatewayTime(response.LastOnlineTime); lastOnlineTime != nil {
updates["last_online_time"] = lastOnlineTime
}
var currentBinding struct {
SlotPosition int
}
currentSlotErr := r.db.WithContext(ctx).Model(&model.DeviceSimBinding{}).
Select("slot_position").
Where("device_id = ? AND bind_status = ? AND is_current = ?", device.ID, constants.BindStatusBound, true).
Take(&currentBinding).Error
if currentSlotErr != nil && currentSlotErr != gorm.ErrRecordNotFound {
return false, apperrors.Wrap(apperrors.CodeDatabaseError, currentSlotErr, "读取设备当前槽位失败")
}
changed := device.OnlineStatus != int(response.OnlineStatus) ||
deviceChanged := device.OnlineStatus != int(response.OnlineStatus) ||
device.SoftwareVersion != string(response.SoftwareVersion) ||
device.SwitchMode != string(response.SwitchMode) ||
currentBinding.SlotPosition != int(response.CurrentSlotNo)
device.SwitchMode != string(response.SwitchMode)
changed := deviceChanged
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
state, err := loadDeviceObservationState(ctx, tx, device.ID, int(response.CurrentSlotNo))
if err != nil {
return err
}
beforeSlot := bindingSlot(state.current)
afterSlot := bindingSlot(state.target)
changed = deviceChanged || beforeSlot != int(response.CurrentSlotNo)
auditChanged := deviceChanged || beforeSlot != afterSlot
if err := tx.Model(&model.Device{}).Where("id = ?", device.ID).Updates(updates).Error; err != nil {
return err
}
@@ -259,19 +263,192 @@ func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device
Update("is_current", false).Error; err != nil {
return err
}
if int(response.CurrentSlotNo) <= 0 {
if int(response.CurrentSlotNo) > 0 {
if err := tx.Model(&model.DeviceSimBinding{}).
Where("device_id = ? AND slot_position = ? AND bind_status = ?", device.ID, int(response.CurrentSlotNo), constants.BindStatusBound).
Update("is_current", true).Error; err != nil {
return err
}
}
if !auditChanged {
return nil
}
return tx.Model(&model.DeviceSimBinding{}).
Where("device_id = ? AND slot_position = ? AND bind_status = ?", device.ID, int(response.CurrentSlotNo), constants.BindStatusBound).
Update("is_current", true).Error
return r.appendDeviceObservationAudit(ctx, tx, device, state, beforeSlot, afterSlot, integrationID, map[string]any{
"online_status": device.OnlineStatus, "software_version": device.SoftwareVersion,
"switch_mode": device.SwitchMode, "current_slot": beforeSlot,
}, map[string]any{
"online_status": int(response.OnlineStatus), "software_version": string(response.SoftwareVersion),
"switch_mode": string(response.SwitchMode), "current_slot": afterSlot,
})
})
if err != nil {
return false, apperrors.Wrap(apperrors.CodeDatabaseError, err, "回写设备 Gateway 信息失败")
wrapped := apperrors.Wrap(apperrors.CodeDatabaseError, err, "回写设备 Gateway 信息失败")
r.recordDeviceObservationFailure(ctx, device, integrationID, wrapped)
return false, wrapped
}
return changed, nil
}
type deviceObservationState struct {
bindings []model.DeviceSimBinding
cards map[uint]*model.IotCard
current *model.DeviceSimBinding
target *model.DeviceSimBinding
}
func loadDeviceObservationState(ctx context.Context, tx *gorm.DB, deviceID uint, targetSlot int) (*deviceObservationState, error) {
state := &deviceObservationState{cards: make(map[uint]*model.IotCard)}
if err := tx.WithContext(ctx).Where("device_id = ? AND bind_status = ?", deviceID, constants.BindStatusBound).
Order("slot_position ASC").Find(&state.bindings).Error; err != nil {
return nil, apperrors.Wrap(apperrors.CodeDatabaseError, err, "读取设备有效卡槽失败")
}
cardIDs := make([]uint, 0, len(state.bindings))
for index := range state.bindings {
binding := &state.bindings[index]
cardIDs = append(cardIDs, binding.IotCardID)
if binding.IsCurrent && state.current == nil {
state.current = binding
}
if targetSlot > 0 && binding.SlotPosition == targetSlot {
state.target = binding
}
}
if len(cardIDs) == 0 {
return state, nil
}
var cards []*model.IotCard
if err := tx.WithContext(ctx).Where("id IN ?", cardIDs).Find(&cards).Error; err != nil {
return nil, apperrors.Wrap(apperrors.CodeDatabaseError, err, "读取设备卡槽关联 IoT 卡失败")
}
for _, card := range cards {
state.cards[card.ID] = card
}
return state, nil
}
func bindingSlot(binding *model.DeviceSimBinding) int {
if binding == nil {
return 0
}
return binding.SlotPosition
}
func (r *SeriesRunner) appendDeviceObservationAudit(
ctx context.Context,
tx *gorm.DB,
device *model.Device,
state *deviceObservationState,
beforeSlot, afterSlot int,
integrationID string,
beforeData, afterData map[string]any,
) error {
if r.auditWriter == nil {
return apperrors.New(apperrors.CodeInternalError, "设备观测统一审计能力未配置")
}
deviceID := strconv.FormatUint(uint64(device.ID), 10)
resources := []audit.ResourceInput{{
Type: constants.AuditResourceDevice, ID: &deviceID,
Key: audit.DeviceResourceKey(device), DisplayName: device.VirtualNo,
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleDeviceTarget,
IdentitySnapshot: audit.DeviceIdentitySnapshot(device), BeforeData: beforeData, AfterData: afterData,
SubjectVisibility: constants.AuditSubjectResult, SubjectSummary: "Worker 同步设备观测事实",
}}
if beforeSlot != afterSlot {
resources = appendDeviceObservationBindingResources(resources, device, state.current, state.cards[stateCardID(state.current)], false,
constants.AuditResourceRoleDeviceOldCurrentCard, constants.AuditResourceRoleDeviceOldCurrentBinding)
resources = appendDeviceObservationBindingResources(resources, device, state.target, state.cards[stateCardID(state.target)], true,
constants.AuditResourceRoleDeviceNewCurrentCard, constants.AuditResourceRoleDeviceNewCurrentBinding)
}
resources = append(resources, deviceObservationIntegrationResource(ctx, device, integrationID))
return r.auditWriter.Append(ctx, tx, audit.AppendInput{
ActionCode: constants.AuditActionDeviceWorkerObservationSynced,
Summary: "Worker 同步设备观测事实", ScopeType: constants.AuditScopePlatform,
Result: constants.AuditResultSuccess, Resources: resources,
})
}
func appendDeviceObservationBindingResources(
resources []audit.ResourceInput,
device *model.Device,
binding *model.DeviceSimBinding,
card *model.IotCard,
afterCurrent bool,
cardRole, bindingRole string,
) []audit.ResourceInput {
if binding == nil {
return resources
}
if card != nil {
cardID := strconv.FormatUint(uint64(card.ID), 10)
resources = append(resources, audit.ResourceInput{
Type: constants.AuditResourceIotCard, ID: &cardID,
Key: audit.IotCardResourceKey(card), DisplayName: card.ICCID,
Relation: constants.AuditResourceRelationAffected, Role: cardRole,
IdentitySnapshot: audit.IotCardIdentitySnapshot(card),
BeforeData: map[string]any{"is_current": binding.IsCurrent}, AfterData: map[string]any{"is_current": afterCurrent},
SubjectVisibility: constants.AuditSubjectResult, SubjectSummary: "设备当前卡槽已同步",
})
}
bindingID := strconv.FormatUint(uint64(binding.ID), 10)
identity := map[string]any{
"id": binding.ID, "device_id": binding.DeviceID, "device_virtual_no": device.VirtualNo,
"slot_position": binding.SlotPosition, "iot_card_id": binding.IotCardID, "is_current": afterCurrent,
}
if card != nil {
identity["iccid"] = card.ICCID
identity["virtual_no"] = card.VirtualNo
}
return append(resources, audit.ResourceInput{
Type: constants.AuditResourceDeviceSIMBinding, ID: &bindingID,
Key: bindingID, DisplayName: device.VirtualNo,
Relation: constants.AuditResourceRelationAffected, Role: bindingRole,
IdentitySnapshot: identity,
BeforeData: map[string]any{"is_current": binding.IsCurrent}, AfterData: map[string]any{"is_current": afterCurrent},
SubjectVisibility: constants.AuditSubjectInternalOnly,
})
}
func stateCardID(binding *model.DeviceSimBinding) uint {
if binding == nil {
return 0
}
return binding.IotCardID
}
func deviceObservationIntegrationResource(ctx context.Context, device *model.Device, integrationID string) audit.ResourceInput {
deviceID := strconv.FormatUint(uint64(device.ID), 10)
return audit.ResourceInput{
Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: integrationID,
Relation: constants.AuditResourceRelationReference, Role: constants.AuditResourceRoleWorkerIntegration,
IdentitySnapshot: map[string]any{
"integration_id": integrationID, "provider": constants.IntegrationProviderGateway,
"direction": constants.IntegrationDirectionOutbound, "operation": constants.IntegrationOperationGatewayDeviceInfo,
"resource_type": constants.CardObservationResourceTypeDevice, "resource_id": deviceID,
"resource_key": "device:" + deviceID, "correlation_id": auditcontext.From(ctx).CorrelationID,
},
SubjectVisibility: constants.AuditSubjectInternalOnly,
}
}
func (r *SeriesRunner) recordDeviceObservationFailure(ctx context.Context, device *model.Device, integrationID string, businessErr error) {
deviceID := strconv.FormatUint(uint64(device.ID), 10)
r.auditWriter.RecordFailure(ctx, r.db, audit.AppendInput{
ActionCode: constants.AuditActionDeviceWorkerObservationSynced,
Summary: "Worker 同步设备观测事实失败", ScopeType: constants.AuditScopePlatform,
Resources: []audit.ResourceInput{{
Type: constants.AuditResourceDevice, ID: &deviceID,
Key: audit.DeviceResourceKey(device), DisplayName: device.VirtualNo,
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleDeviceTarget,
IdentitySnapshot: audit.DeviceIdentitySnapshot(device),
BeforeData: map[string]any{
"online_status": device.OnlineStatus, "software_version": device.SoftwareVersion,
"switch_mode": device.SwitchMode,
},
SubjectVisibility: constants.AuditSubjectResult,
}, deviceObservationIntegrationResource(ctx, device, integrationID)},
}, businessErr)
}
func (r *SeriesRunner) loadDevice(ctx context.Context, payload cardapp.SeriesTaskPayload) (*model.Device, error) {
if payload.ResourceType != constants.CardObservationResourceTypeDevice {
return nil, apperrors.New(apperrors.CodeInvalidParam, "设备信息观测资源类型无效")

View File

@@ -62,18 +62,75 @@ type AuditResourceTimelineRequest struct {
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量默认20最大100"`
}
// AuditRequestTimelineParams 是请求链路时间线的路径参数。
type AuditRequestTimelineParams struct {
RequestID string `json:"request_id" path:"request_id" required:"true" description:"HTTP请求关联ID来自审计或外部集成节点也可从Access Log粘贴"`
}
// AuditCorrelationTimelineParams 是业务关联时间线的路径参数。
type AuditCorrelationTimelineParams struct {
CorrelationID string `json:"correlation_id" path:"correlation_id" required:"true" description:"跨请求、异步任务和外部交互的稳定业务链路ID"`
}
// AuditFinanceTimelineRequest 是资金调查时间线的稳定业务筛选参数。
type AuditFinanceTimelineRequest struct {
ShopID uint `json:"shop_id" query:"shop_id" description:"店铺ID"`
WalletID uint `json:"wallet_id" query:"wallet_id" description:"代理或资产钱包ID"`
OrderID uint `json:"order_id" query:"order_id" description:"订单ID"`
OrderNo string `json:"order_no" query:"order_no" description:"订单编号"`
PaymentID uint `json:"payment_id" query:"payment_id" description:"支付记录ID"`
PaymentNo string `json:"payment_no" query:"payment_no" description:"支付单号"`
RefundID uint `json:"refund_id" query:"refund_id" description:"退款单ID"`
RefundNo string `json:"refund_no" query:"refund_no" description:"退款单号"`
RechargeID uint `json:"recharge_id" query:"recharge_id" description:"代理充值或个人资产充值ID"`
RechargeNo string `json:"recharge_no" query:"recharge_no" description:"充值单号"`
ApprovalInstanceID uint `json:"approval_instance_id" query:"approval_instance_id" description:"审批实例ID"`
ThirdPartyTradeNo string `json:"third_party_trade_no" query:"third_party_trade_no" description:"第三方交易号"`
ActorKind string `json:"actor_kind" query:"actor_kind" description:"操作者类型与actor_id同时提供"`
ActorID string `json:"actor_id" query:"actor_id" description:"操作者稳定ID与actor_kind同时提供"`
CorrelationID string `json:"correlation_id" query:"correlation_id" description:"跨步骤业务链路ID"`
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间RFC3339含时区"`
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间RFC3339含时区不包含该时刻"`
Page int `json:"page" query:"page" minimum:"1" description:"页码默认1"`
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量默认20最大100"`
}
// AuditRiskFilterRequest 是风险总览和明细共用的受控筛选参数。
type AuditRiskFilterRequest struct {
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间RFC3339含时区默认从在线窗口开始"`
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间RFC3339含时区不包含该时刻最长31天默认当前时间"`
Risk string `json:"risk" query:"risk" description:"风险等级 (low:低, normal:普通, high:高, critical:严重)"`
Result string `json:"result" query:"result" description:"结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知)"`
Action string `json:"action" query:"action" description:"稳定动作编码"`
Source string `json:"source" query:"source" description:"来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调)"`
}
// AuditRiskOverviewRequest 是风险总览请求参数。
type AuditRiskOverviewRequest struct {
AuditRiskFilterRequest
}
// AuditRiskEventsRequest 是风险事件明细请求参数。
type AuditRiskEventsRequest struct {
AuditRiskFilterRequest
Page int `json:"page" query:"page" minimum:"1" description:"页码默认1"`
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量默认20最大100"`
}
// SubjectResourceActivityRequest 是代理和企业安全资源活动的路径及分页参数。
type SubjectResourceActivityRequest struct {
ResourceType string `json:"resource_type" path:"resource_type" required:"true" description:"资源类型 (iot_card:IoT卡, device:设备, asset_allocation_record:资产分配记录, exchange_order:换货单, shop:店铺, enterprise:企业)"`
Identifier string `json:"identifier" path:"identifier" required:"true" description:"业务稳定标识卡使用ICCID设备使用VirtualNo其他资源使用对应业务编号"`
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间RFC3339含时区默认从在线窗口开始"`
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间RFC3339含时区不包含该时刻默认当前时间"`
Page int `json:"page" query:"page" minimum:"1" description:"页码默认1"`
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量默认20最大100"`
}
// IntegrationFilterRequest 是外部集成调查的公共受控筛选参数。
type IntegrationFilterRequest struct {
CreatedFrom string `json:"created_from" query:"created_from" required:"true" description:"开始时间RFC3339含时区,必填"`
CreatedTo string `json:"created_to" query:"created_to" required:"true" description:"结束时间RFC3339含时区不包含该时刻,必填"`
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间RFC3339含时区;默认从在线窗口开始"`
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间RFC3339含时区不包含该时刻;默认当前时间"`
IntegrationID string `json:"integration_id" query:"integration_id" description:"稳定外部集成记录ID"`
Provider string `json:"provider" query:"provider" description:"外部服务提供方稳定编码"`
Direction string `json:"direction" query:"direction" description:"交互方向 (inbound:入站, outbound:出站)"`

View File

@@ -0,0 +1,38 @@
package model
import "time"
// LogArchiveRun 是日志冷归档运行账本,不保存日志正文。
type LogArchiveRun struct {
ID uint `gorm:"column:id;primaryKey;autoIncrement" json:"id"`
Source string `gorm:"column:source;type:varchar(32);not null" json:"source"`
ArchiveDate time.Time `gorm:"column:archive_date;type:date;not null" json:"archive_date"`
InstanceID string `gorm:"column:instance_id;type:varchar(100);not null" json:"instance_id"`
SchemaVersion string `gorm:"column:schema_version;type:varchar(32);not null" json:"schema_version"`
Revision int `gorm:"column:revision;not null;default:1" json:"revision"`
Status string `gorm:"column:status;type:varchar(16);not null" json:"status"`
IsFinal bool `gorm:"column:is_final;not null;default:false" json:"is_final"`
RangeStart time.Time `gorm:"column:range_start;type:timestamptz;not null" json:"range_start"`
RangeEnd time.Time `gorm:"column:range_end;type:timestamptz;not null" json:"range_end"`
ObjectKey string `gorm:"column:object_key;type:varchar(500);not null;default:''" json:"object_key"`
ManifestKey string `gorm:"column:manifest_key;type:varchar(500);not null;default:''" json:"manifest_key"`
EventCount int64 `gorm:"column:event_count;not null;default:0" json:"event_count"`
ResourceCount int64 `gorm:"column:resource_count;not null;default:0" json:"resource_count"`
RecordCount int64 `gorm:"column:record_count;not null;default:0" json:"record_count"`
UncompressedBytes int64 `gorm:"column:uncompressed_bytes;not null;default:0" json:"uncompressed_bytes"`
CompressedBytes int64 `gorm:"column:compressed_bytes;not null;default:0" json:"compressed_bytes"`
SHA256 string `gorm:"column:sha256;type:varchar(64);not null;default:''" json:"sha256"`
AttemptCount int `gorm:"column:attempt_count;not null;default:0" json:"attempt_count"`
ErrorSummary string `gorm:"column:error_summary;type:varchar(500);not null;default:''" json:"error_summary"`
GeneratedAt *time.Time `gorm:"column:generated_at" json:"generated_at,omitempty"`
CompletedAt *time.Time `gorm:"column:completed_at" json:"completed_at,omitempty"`
CleanupStartedAt *time.Time `gorm:"column:cleanup_started_at" json:"cleanup_started_at,omitempty"`
CleanedAt *time.Time `gorm:"column:cleaned_at" json:"cleaned_at,omitempty"`
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`
}
// TableName 返回日志冷归档运行账本表名。
func (LogArchiveRun) TableName() string {
return "tb_log_archive_run"
}

View File

@@ -41,6 +41,13 @@ type PackageActivationHandler struct {
logger *zap.Logger
}
func workerCorrelation(correlationID string, task *asynq.Task) string {
if correlationID != "" {
return correlationID
}
return task.ResultWriter().TaskID()
}
// PackageActivationPayload 套餐激活任务载荷
type PackageActivationPayload struct {
PackageUsageID uint `json:"package_usage_id"`
@@ -48,6 +55,9 @@ type PackageActivationPayload struct {
CarrierID uint `json:"carrier_id"`
ActivationType string `json:"activation_type"` // "queue" 或 "realname"
Timestamp int64 `json:"timestamp"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
ParentEventID string `json:"parent_event_id,omitempty"`
}
// NewPackageActivationHandler 创建套餐激活检查处理器
@@ -397,10 +407,11 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
if h.stopResumeCallback == nil {
return
}
detachedCtx := context.WithoutCancel(ctx)
if carrierType == "iot_card" {
go func() {
if err := h.stopResumeCallback.CheckAndStopCard(context.Background(), carrierID); err != nil {
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, carrierID); err != nil {
h.logger.Error("套餐过期后停机失败",
zap.Uint("card_id", carrierID),
zap.Error(err))
@@ -420,7 +431,7 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
for _, b := range bindings {
cardID := b.IotCardID
go func(cID uint) {
if err := h.stopResumeCallback.CheckAndStopCard(context.Background(), cID); err != nil {
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, cID); err != nil {
h.logger.Error("套餐过期后停机失败",
zap.Uint("card_id", cID),
zap.Error(err))
@@ -432,12 +443,16 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
// enqueueActivationTask 提交套餐激活任务到 Asynq
func (h *PackageActivationHandler) enqueueActivationTask(ctx context.Context, packageUsageID uint, carrierType string, carrierID uint, activationType string) error {
linkage := auditcontext.From(ctx)
payload := PackageActivationPayload{
PackageUsageID: packageUsageID,
CarrierType: carrierType,
CarrierID: carrierID,
ActivationType: activationType,
Timestamp: time.Now().Unix(),
RequestID: linkage.RequestID,
CorrelationID: linkage.CorrelationID,
ParentEventID: linkage.ParentEventID,
}
payloadBytes, err := sonic.Marshal(payload)
@@ -477,6 +492,7 @@ func (h *PackageActivationHandler) HandlePackageQueueActivation(ctx context.Cont
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypePackageQueueActivation,
ActorName: "套餐排队激活任务", Source: constants.AuditSourceWorker,
RequestID: payload.RequestID, CorrelationID: workerCorrelation(payload.CorrelationID, t), ParentEventID: payload.ParentEventID,
})
h.logger.Info("开始执行套餐激活",
@@ -534,6 +550,7 @@ func (h *PackageActivationHandler) HandlePackageFirstActivation(ctx context.Cont
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypePackageFirstActivation,
ActorName: "套餐首次实名激活任务", Source: constants.AuditSourceWorker,
RequestID: payload.RequestID, CorrelationID: workerCorrelation(payload.CorrelationID, t), ParentEventID: payload.ParentEventID,
})
if payload.CarrierType == "" || payload.CarrierID == 0 {

View File

@@ -10,6 +10,7 @@ import (
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
@@ -39,10 +40,17 @@ type EventFilter struct {
// EventPage 是平台全局事件稳定分页结果。
type EventPage struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []EventView `json:"items"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []EventView `json:"items"`
Retention retentionquery.Info `json:"retention"`
}
// EventDetail 是单个审计事件及在线留存边界。
type EventDetail struct {
EventView
Retention retentionquery.Info `json:"retention"`
}
// EventView 是不暴露 GORM Model 的审计事件投影。
@@ -147,6 +155,14 @@ func (q *Query) List(ctx context.Context, filter EventFilter) (*EventPage, error
if err := q.authorize(ctx); err != nil {
return nil, err
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo)
if err != nil {
return nil, err
}
if !validEventFilter(filter) {
return nil, errors.New(errors.CodeInvalidParam)
}
@@ -156,16 +172,33 @@ func (q *Query) List(ctx context.Context, filter EventFilter) (*EventPage, error
if err := query.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计审计事件失败")
}
rows := make([]model.AuditEvent, 0, filter.PageSize)
if err := query.Order("occurred_at DESC, id DESC").
Offset((filter.Page - 1) * filter.PageSize).Limit(filter.PageSize).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询审计事件失败")
rows, err := q.loadEventPage(ctx, query, filter.Page, filter.PageSize)
if err != nil {
return nil, err
}
items, err := q.project(ctx, rows)
if err != nil {
return nil, err
}
return &EventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
return &EventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
}
// loadEventPage 先分页主键,再批量读取事件宽行,避免排序阶段加载 JSON 字段。
func (q *Query) loadEventPage(ctx context.Context, query *gorm.DB, page, pageSize int) ([]model.AuditEvent, error) {
ids := make([]uint, 0, pageSize)
if err := query.Select("id").Order("occurred_at DESC, id DESC").
Offset((page-1)*pageSize).Limit(pageSize).Pluck("id", &ids).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询审计事件分页ID失败")
}
rows := make([]model.AuditEvent, 0, len(ids))
if len(ids) == 0 {
return rows, nil
}
if err := q.db.WithContext(ctx).Where("id IN ?", ids).
Order("occurred_at DESC, id DESC").Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "批量投影审计事件失败")
}
return rows, nil
}
func validEventFilter(filter EventFilter) bool {
@@ -192,15 +225,19 @@ func validOptionalValue(value string, allowed ...string) bool {
}
// Get 查询平台范围的单个稳定审计事件详情。
func (q *Query) Get(ctx context.Context, eventID string) (*EventView, error) {
func (q *Query) Get(ctx context.Context, eventID string) (*EventDetail, error) {
if err := q.authorize(ctx); err != nil {
return nil, err
}
if eventID == "" {
return nil, errors.New(errors.CodeInvalidParam)
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
var row model.AuditEvent
if err := q.db.WithContext(ctx).Where("event_id = ?", eventID).First(&row).Error; err != nil {
if err := q.db.WithContext(ctx).Where("event_id = ? AND occurred_at >= ?", eventID, retention.OnlineFrom.UTC()).First(&row).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil, errors.New(errors.CodeNotFound, "审计事件不存在")
}
@@ -210,7 +247,7 @@ func (q *Query) Get(ctx context.Context, eventID string) (*EventView, error) {
if err != nil {
return nil, err
}
return &items[0], nil
return &EventDetail{EventView: items[0], Retention: retention}, nil
}
func (q *Query) authorize(ctx context.Context) error {

File diff suppressed because it is too large Load Diff

View File

@@ -9,6 +9,7 @@ import (
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
@@ -17,16 +18,18 @@ import (
type ResourceSearchFilter struct {
ResourceType string
Keyword string
OnlineFrom time.Time
Page int
PageSize int
}
// ResourceSearchPage 是资源候选稳定分页结果。
type ResourceSearchPage struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []ResourceCandidate `json:"items"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []ResourceCandidate `json:"items"`
Retention retentionquery.Info `json:"retention"`
}
// ResourceCandidate 是当前业务表或历史事件快照解析出的稳定资源候选。
@@ -59,6 +62,11 @@ func (q *Query) SearchResources(ctx context.Context, filter ResourceSearchFilter
if filter.Keyword == "" || !searchableResourceType(filter.ResourceType) {
return nil, errors.New(errors.CodeInvalidParam)
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
filter.OnlineFrom = retention.OnlineFrom
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
items, total, err := q.searchCurrent(ctx, filter)
if err != nil {
@@ -70,7 +78,7 @@ func (q *Query) SearchResources(ctx context.Context, filter ResourceSearchFilter
return nil, err
}
}
return &ResourceSearchPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
return &ResourceSearchPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
}
// ResourceTimeline 查询注册资源作为任意关系参与的统一事件时间线。
@@ -196,7 +204,7 @@ func (q *Query) searchHistorical(ctx context.Context, filter ResourceSearchFilte
func (q *Query) historicalIdentifierQuery(ctx context.Context, filter ResourceSearchFilter) *gorm.DB {
query := q.db.WithContext(ctx).Model(&model.AuditEventResource{}).
Where("resource_type = ? AND resource_id IS NOT NULL", filter.ResourceType)
Where("resource_type = ? AND resource_id IS NOT NULL AND created_at >= ?", filter.ResourceType, filter.OnlineFrom.UTC())
switch filter.ResourceType {
case constants.AuditResourceIotCard:
return query.Where("resource_key = ? OR identity_snapshot ->> 'iccid' = ? OR identity_snapshot ->> 'iccid_19' = ? OR identity_snapshot ->> 'iccid_20' = ? OR identity_snapshot ->> 'virtual_no' = ?", filter.Keyword, filter.Keyword, filter.Keyword, filter.Keyword, filter.Keyword)

View File

@@ -0,0 +1,299 @@
package audit
import (
"context"
"time"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// RiskFilter 定义固定风险调查视角的时间范围与筛选条件。
type RiskFilter struct {
CreatedFrom *time.Time
CreatedTo *time.Time
Risk string
Result string
Action string
Source string
Page int
PageSize int
}
// RiskOverview 是风险信号、固定维度与时间趋势的只读聚合。
type RiskOverview struct {
Total int64 `json:"total"`
Bucket string `json:"bucket"`
Signals []RiskNamedCount `json:"signals"`
Risks []RiskNamedCount `json:"risks"`
Results []RiskNamedCount `json:"results"`
Actions []RiskNamedCount `json:"actions"`
Sources []RiskNamedCount `json:"sources"`
Trend []RiskTrendPoint `json:"trend"`
Retention retentionquery.Info `json:"retention"`
}
// RiskNamedCount 是风险聚合维度的稳定编码、中文名称和数量。
type RiskNamedCount struct {
Code string `json:"code"`
Name string `json:"name"`
Count int64 `json:"count"`
}
// RiskTrendPoint 是固定时间桶内的风险信号趋势。
type RiskTrendPoint struct {
BucketAt time.Time `json:"bucket_at"`
Total int64 `json:"total"`
HighRisk int64 `json:"high_risk"`
Finance int64 `json:"finance"`
Security int64 `json:"security"`
Failed int64 `json:"failed"`
Denied int64 `json:"denied"`
Partial int64 `json:"partial"`
Unknown int64 `json:"unknown"`
}
// RiskEventPage 是风险事件的稳定分页结果。
type RiskEventPage struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []EventView `json:"items"`
Retention retentionquery.Info `json:"retention"`
}
// RiskOverview 查询指定时间范围内的固定风险调查总览。
func (q *Query) RiskOverview(ctx context.Context, filter RiskFilter) (*RiskOverview, error) {
if err := q.authorize(ctx); err != nil {
return nil, err
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.AuditRiskQueryMaxRange)
if err != nil {
return nil, err
}
if !validRiskFilter(filter) {
return nil, errors.New(errors.CodeInvalidParam)
}
base := q.applyRiskFilters(q.db.WithContext(ctx).Model(&model.AuditEvent{}), filter)
result := &RiskOverview{
Bucket: riskTrendBucket(*filter.CreatedFrom, *filter.CreatedTo),
Signals: []RiskNamedCount{}, Risks: []RiskNamedCount{}, Results: []RiskNamedCount{},
Actions: []RiskNamedCount{}, Sources: []RiskNamedCount{}, Trend: []RiskTrendPoint{},
Retention: retention,
}
if err := loadRiskSignals(base, result); err != nil {
return nil, err
}
if err := loadRiskDimension(base, "risk_level", result); err != nil {
return nil, err
}
if err := loadRiskDimension(base, "result", result); err != nil {
return nil, err
}
if err := loadRiskDimension(base, "action_code", result); err != nil {
return nil, err
}
if err := loadRiskDimension(base, "source", result); err != nil {
return nil, err
}
if err := loadRiskTrend(base, result); err != nil {
return nil, err
}
return result, nil
}
// RiskEvents 查询指定时间范围内的风险事件明细。
func (q *Query) RiskEvents(ctx context.Context, filter RiskFilter) (*RiskEventPage, error) {
if err := q.authorize(ctx); err != nil {
return nil, err
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.AuditRiskQueryMaxRange)
if err != nil {
return nil, err
}
if !validRiskFilter(filter) {
return nil, errors.New(errors.CodeInvalidParam)
}
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
query := q.applyRiskFilters(q.db.WithContext(ctx).Model(&model.AuditEvent{}), filter)
var total int64
if err := query.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计风险事件失败")
}
rows, err := q.loadEventPage(ctx, query, filter.Page, filter.PageSize)
if err != nil {
return nil, err
}
items, err := q.project(ctx, rows)
if err != nil {
return nil, err
}
return &RiskEventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
}
func validRiskFilter(filter RiskFilter) bool {
if filter.CreatedFrom == nil || filter.CreatedTo == nil || !filter.CreatedFrom.Before(*filter.CreatedTo) ||
filter.CreatedTo.Sub(*filter.CreatedFrom) > constants.AuditRiskQueryMaxRange {
return false
}
return validEventFilter(EventFilter{
Risk: filter.Risk, Result: filter.Result, Action: filter.Action, Source: filter.Source,
Page: filter.Page, PageSize: filter.PageSize,
})
}
func (q *Query) applyRiskFilters(query *gorm.DB, filter RiskFilter) *gorm.DB {
query = q.applyFilters(query, EventFilter{
CreatedFrom: filter.CreatedFrom, CreatedTo: filter.CreatedTo,
Risk: filter.Risk, Result: filter.Result, Action: filter.Action, Source: filter.Source,
})
return query.Where(riskScopeSQL(), riskLevels(), constants.AuditCategorySecurity, abnormalResults(), financeResourceTypes())
}
func loadRiskSignals(query *gorm.DB, result *RiskOverview) error {
var row struct {
Total, HighRisk, Finance, Security, Failed, Denied, Partial, Unknown int64
}
err := query.Select(`COUNT(*) AS total,
COUNT(*) FILTER (WHERE risk_level IN ?) AS high_risk,
COUNT(*) FILTER (WHERE EXISTS (SELECT 1 FROM tb_audit_event_resource aer WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?)) AS finance,
COUNT(*) FILTER (WHERE category = ?) AS security,
COUNT(*) FILTER (WHERE result = ?) AS failed,
COUNT(*) FILTER (WHERE result = ?) AS denied,
COUNT(*) FILTER (WHERE result = ?) AS partial,
COUNT(*) FILTER (WHERE result = ?) AS unknown`, riskLevels(), financeResourceTypes(), constants.AuditCategorySecurity,
constants.AuditResultFailed, constants.AuditResultDenied, constants.AuditResultPartial, constants.AuditResultUnknown).Scan(&row).Error
if err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险信号失败")
}
result.Total = row.Total
result.Signals = []RiskNamedCount{
{Code: constants.AuditRiskSignalHighRisk, Name: "高风险", Count: row.HighRisk},
{Code: constants.AuditRiskSignalFinance, Name: "资金", Count: row.Finance},
{Code: constants.AuditRiskSignalSecurity, Name: "安全", Count: row.Security},
{Code: constants.AuditRiskSignalFailed, Name: "失败", Count: row.Failed},
{Code: constants.AuditRiskSignalDenied, Name: "拒绝", Count: row.Denied},
{Code: constants.AuditRiskSignalPartial, Name: "部分成功", Count: row.Partial},
{Code: constants.AuditRiskSignalUnknown, Name: "结果未知", Count: row.Unknown},
}
return nil
}
func loadRiskDimension(query *gorm.DB, column string, result *RiskOverview) error {
var rows []struct {
Code string
Name string
Count int64
}
selectClause := column + " AS code, '' AS name, COUNT(*) AS count"
groupClause := column
if column == "action_code" {
selectClause = "action_code AS code, action_name AS name, COUNT(*) AS count"
groupClause = "action_code, action_name"
}
if err := query.Select(selectClause).Group(groupClause).Order(column + " ASC").Scan(&rows).Error; err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险维度失败")
}
items := make([]RiskNamedCount, 0, len(rows))
for _, row := range rows {
name := row.Name
if name == "" {
name = riskDimensionName(column, row.Code)
}
items = append(items, RiskNamedCount{Code: row.Code, Name: name, Count: row.Count})
}
switch column {
case "risk_level":
result.Risks = items
case "result":
result.Results = items
case "action_code":
result.Actions = items
case "source":
result.Sources = items
}
return nil
}
func loadRiskTrend(query *gorm.DB, result *RiskOverview) error {
err := query.Select(`date_trunc(?, occurred_at) AS bucket_at, COUNT(*) AS total,
COUNT(*) FILTER (WHERE risk_level IN ?) AS high_risk,
COUNT(*) FILTER (WHERE EXISTS (SELECT 1 FROM tb_audit_event_resource aer WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?)) AS finance,
COUNT(*) FILTER (WHERE category = ?) AS security,
COUNT(*) FILTER (WHERE result = ?) AS failed,
COUNT(*) FILTER (WHERE result = ?) AS denied,
COUNT(*) FILTER (WHERE result = ?) AS partial,
COUNT(*) FILTER (WHERE result = ?) AS unknown`, result.Bucket, riskLevels(), financeResourceTypes(),
constants.AuditCategorySecurity, constants.AuditResultFailed, constants.AuditResultDenied,
constants.AuditResultPartial, constants.AuditResultUnknown).
Group("bucket_at").Order("bucket_at ASC").Scan(&result.Trend).Error
if err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险趋势失败")
}
return nil
}
func riskScopeSQL() string {
return `(risk_level IN ? OR category = ? OR result IN ? OR EXISTS (
SELECT 1 FROM tb_audit_event_resource aer
WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?
))`
}
func riskLevels() []string {
return []string{constants.AuditRiskHigh, constants.AuditRiskCritical}
}
func abnormalResults() []string {
return []string{constants.AuditResultFailed, constants.AuditResultDenied, constants.AuditResultPartial, constants.AuditResultUnknown}
}
func financeResourceTypes() []string {
return []string{
constants.AuditResourceOrder, constants.AuditResourceRefund, constants.AuditResourceAgentRecharge,
constants.AuditResourceRechargeOrder, constants.AuditResourceAssetWallet, constants.AuditResourceAssetWalletTransaction,
constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction,
constants.AuditResourceAgentWalletReservation, constants.AuditResourcePayment,
constants.AuditResourceCommissionRecord, constants.AuditResourceCommissionWithdrawal,
}
}
func riskTrendBucket(from, to time.Time) string {
if to.Sub(from) <= constants.AuditRiskHourlyTrendMaxRange {
return "hour"
}
return "day"
}
func riskDimensionName(column, code string) string {
names := map[string]map[string]string{
"risk_level": {
constants.AuditRiskLow: "低", constants.AuditRiskNormal: "普通",
constants.AuditRiskHigh: "高", constants.AuditRiskCritical: "严重",
},
"result": {
constants.AuditResultSuccess: "成功", constants.AuditResultFailed: "失败",
constants.AuditResultDenied: "拒绝", constants.AuditResultPartial: "部分成功",
constants.AuditResultUnknown: "结果未知",
},
"source": {
constants.AuditSourceAdminAPI: "后台管理 API", constants.AuditSourcePersonalAPI: "个人客户 API",
constants.AuditSourceOpenAPI: "代理 OpenAPI", constants.AuditSourceWorker: "异步 Worker",
constants.AuditSourceScheduler: "计划任务", constants.AuditSourceCallback: "外部系统回调",
},
}
return names[column][code]
}

View File

@@ -9,6 +9,7 @@ import (
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
@@ -18,17 +19,20 @@ import (
type SubjectActivityFilter struct {
ResourceType string
Identifier string
CreatedFrom *time.Time
CreatedTo *time.Time
Page int
PageSize int
}
// SubjectActivityPage 是不包含平台调查字段的代理资源活动分页结果。
type SubjectActivityPage struct {
Resource SubjectResourceSummary `json:"resource"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []SubjectActivity `json:"items"`
Resource SubjectResourceSummary `json:"resource"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []SubjectActivity `json:"items"`
Retention retentionquery.Info `json:"retention"`
}
// SubjectActivity 是写入时已生成的主体安全活动投影。
@@ -78,11 +82,19 @@ func (q *Query) AgentResourceActivities(ctx context.Context, filter SubjectActiv
return nil, errors.New(errors.CodeInvalidParam)
}
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo)
if err != nil {
return nil, err
}
target, err := q.resolveAgentTarget(ctx, filter.ResourceType, filter.Identifier, shopIDs)
if err != nil {
return nil, err
}
return q.subjectActivitiesForTarget(ctx, filter, target, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
return q.subjectActivitiesForTarget(ctx, filter, target, retention, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
return q.agentAllowedResourceIDs(ctx, resources, shopIDs)
})
}
@@ -100,21 +112,30 @@ func (q *Query) EnterpriseResourceActivities(ctx context.Context, filter Subject
return nil, errors.New(errors.CodeInvalidParam)
}
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
if err != nil {
return nil, err
}
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo)
if err != nil {
return nil, err
}
target, err := q.resolveEnterpriseTarget(ctx, filter.ResourceType, filter.Identifier, enterpriseID)
if err != nil {
return nil, err
}
return q.subjectActivitiesForTarget(ctx, filter, target, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
return q.subjectActivitiesForTarget(ctx, filter, target, retention, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
return q.enterpriseAllowedResourceIDs(ctx, resources, enterpriseID)
})
}
func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectActivityFilter, target subjectTarget, authorize subjectResourceAuthorizer) (*SubjectActivityPage, error) {
func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectActivityFilter, target subjectTarget, retention retentionquery.Info, authorize subjectResourceAuthorizer) (*SubjectActivityPage, error) {
resourceMatch := q.db.Table("tb_audit_event_resource AS target").Select("1").
Where("target.audit_event_id = tb_audit_event.id AND target.resource_type = ? AND target.resource_id = ?", filter.ResourceType, target.id).
Where("target.subject_visibility IN ?", []string{constants.AuditSubjectResult, constants.AuditSubjectDetail})
base := q.db.WithContext(ctx).Model(&model.AuditEvent{}).Where("EXISTS (?)", resourceMatch)
base := q.db.WithContext(ctx).Model(&model.AuditEvent{}).
Where("occurred_at >= ? AND occurred_at < ?", filter.CreatedFrom.UTC(), filter.CreatedTo.UTC()).Where("EXISTS (?)", resourceMatch)
var total int64
if err := base.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计代理资源活动失败")
@@ -131,7 +152,7 @@ func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectAc
if err != nil {
return nil, err
}
return &SubjectActivityPage{Resource: target.summary, Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
return &SubjectActivityPage{Resource: target.summary, Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
}
func (q *Query) resolveEnterpriseTarget(ctx context.Context, resourceType, identifier string, enterpriseID uint) (subjectTarget, error) {

View File

@@ -0,0 +1,317 @@
package audit
import (
"context"
"fmt"
"sort"
"strconv"
"time"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// LinkTimeline 是 request 或 correlation 的跨事实只读时间线。
type LinkTimeline struct {
RequestID *string `json:"request_id"`
CorrelationID *string `json:"correlation_id"`
AccessLogLookupRequestID *string `json:"access_log_lookup_request_id"`
Nodes []LinkTimelineNode `json:"nodes"`
Retention retentionquery.Info `json:"retention"`
}
// LinkTimelineNode 是保留各事实源权威边界的时间线节点。
type LinkTimelineNode struct {
RecordSource string `json:"record_source"`
NodeID string `json:"node_id"`
OccurredAt time.Time `json:"occurred_at"`
Code string `json:"code"`
Title string `json:"title"`
Result string `json:"result"`
ResultName string `json:"result_name"`
Summary string `json:"summary"`
ReferenceOnly bool `json:"reference_only"`
RequestID *string `json:"request_id"`
CorrelationID *string `json:"correlation_id"`
ParentEventID *string `json:"parent_event_id"`
Resources []InvestigationResourceRef `json:"resources"`
InvestigationRefs InvestigationRefs `json:"investigation_refs"`
Fidelity LinkageFidelity `json:"fidelity"`
}
// LinkageFidelity 明确节点已有的稳定关联能力,不补猜历史缺失字段。
type LinkageFidelity struct {
RequestAvailable bool `json:"request_available"`
CorrelationAvailable bool `json:"correlation_available"`
ParentEventAvailable bool `json:"parent_event_available"`
DirectAuditLinkAvailable bool `json:"direct_audit_link_available"`
StableResourceAvailable bool `json:"stable_resource_available"`
}
// RequestTimeline 按精确 request ID 组合已持久化事实,不扫描 Access Log。
func (q *Query) RequestTimeline(ctx context.Context, requestID string) (*LinkTimeline, error) {
return q.linkTimeline(ctx, "request_id", requestID)
}
// CorrelationTimeline 按精确 correlation ID 组合跨请求业务链路。
func (q *Query) CorrelationTimeline(ctx context.Context, correlationID string) (*LinkTimeline, error) {
return q.linkTimeline(ctx, "correlation_id", correlationID)
}
func (q *Query) linkTimeline(ctx context.Context, column, value string) (*LinkTimeline, error) {
if err := q.authorize(ctx); err != nil {
return nil, err
}
if value == "" || (column != "request_id" && column != "correlation_id") {
return nil, errors.New(errors.CodeInvalidParam)
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit, retentionquery.SourceIntegration)
if err != nil {
return nil, err
}
auditRows, integrationRows, outboxRows, err := q.loadLinkRows(ctx, column, value, retention.OnlineFrom)
if err != nil {
return nil, err
}
events, err := q.project(ctx, auditRows)
if err != nil {
return nil, err
}
nodes := make([]LinkTimelineNode, 0, len(events)+len(integrationRows)+len(outboxRows))
integrationByAudit := integrationRefsByAuditID(integrationRows)
for index, event := range events {
refs := event.InvestigationRefs
refs.IntegrationRefs = append(refs.IntegrationRefs, integrationByAudit[auditRows[index].ID]...)
refs.IntegrationRefs = append(refs.IntegrationRefs, integrationResourceRefs(event.Resources)...)
refs.IntegrationRefs = uniqueIntegrationRefs(refs.IntegrationRefs)
nodes = append(nodes, auditTimelineNode(event, refs))
nodes = append(nodes, resourceReferenceNodes(event, refs)...)
}
for _, row := range integrationRows {
nodes = append(nodes, integrationTimelineNode(row))
}
for _, row := range outboxRows {
nodes = append(nodes, outboxTimelineNode(row))
}
sort.Slice(nodes, func(i, j int) bool {
if nodes[i].OccurredAt.Equal(nodes[j].OccurredAt) {
if nodes[i].RecordSource == nodes[j].RecordSource {
return nodes[i].NodeID < nodes[j].NodeID
}
return nodes[i].RecordSource < nodes[j].RecordSource
}
return nodes[i].OccurredAt.Before(nodes[j].OccurredAt)
})
timeline := &LinkTimeline{Nodes: nodes, Retention: retention}
if timeline.Nodes == nil {
timeline.Nodes = []LinkTimelineNode{}
}
if column == "request_id" {
timeline.RequestID = stringPointer(value)
timeline.AccessLogLookupRequestID = stringPointer(value)
} else {
timeline.CorrelationID = stringPointer(value)
}
return timeline, nil
}
func (q *Query) loadLinkRows(ctx context.Context, column, value string, onlineFrom time.Time) ([]model.AuditEvent, []model.IntegrationLog, []model.OutboxEvent, error) {
auditRows := []model.AuditEvent{}
if err := q.db.WithContext(ctx).Where(column+" = ? AND occurred_at >= ?", value, onlineFrom.UTC()).Find(&auditRows).Error; err != nil {
return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路审计事件失败")
}
integrationRows := []model.IntegrationLog{}
if err := q.db.WithContext(ctx).Where(column+" = ? AND created_at >= ?", value, onlineFrom.UTC()).Find(&integrationRows).Error; err != nil {
return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路外部交互失败")
}
outboxRows := []model.OutboxEvent{}
if err := q.db.WithContext(ctx).Where(column+" = ? AND created_at >= ?", value, onlineFrom.UTC()).Find(&outboxRows).Error; err != nil {
return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路可靠事件失败")
}
return auditRows, integrationRows, outboxRows, nil
}
func auditTimelineNode(event EventView, refs InvestigationRefs) LinkTimelineNode {
return LinkTimelineNode{
RecordSource: constants.AuditRecordSourceAuditEvent, NodeID: event.EventID,
OccurredAt: event.OccurredAt, Code: event.ActionCode, Title: event.ActionName,
Result: event.Result, Summary: event.Summary,
RequestID: stringPointer(event.RequestID), CorrelationID: stringPointer(event.CorrelationID),
ParentEventID: stringPointer(event.ParentEventID), Resources: refs.ResourceRefs, InvestigationRefs: refs,
Fidelity: linkageFidelity(event.RequestID, event.CorrelationID, event.ParentEventID, true, len(refs.ResourceRefs) > 0),
}
}
func integrationTimelineNode(row model.IntegrationLog) LinkTimelineNode {
resource := integrationResourceRef(row)
resources := make([]InvestigationResourceRef, 0, 1)
if resource != nil {
resources = append(resources, *resource)
}
refs := InvestigationRefs{
ResourceRefs: resources, RequestID: row.RequestID, CorrelationID: row.CorrelationID,
IntegrationRefs: []IntegrationRef{{IntegrationID: row.IntegrationID}},
}
return LinkTimelineNode{
RecordSource: constants.AuditRecordSourceIntegrationLog, NodeID: row.IntegrationID,
OccurredAt: row.CreatedAt, Code: row.Operation,
Title: constants.IntegrationProviderName(row.Provider) + " · " + constants.IntegrationOperationName(row.Operation),
Result: row.Result, ResultName: constants.IntegrationResultName(row.Result), Summary: "外部交互事实",
RequestID: row.RequestID, CorrelationID: row.CorrelationID, Resources: resources, InvestigationRefs: refs,
Fidelity: linkageFidelity(pointerValue(row.RequestID), pointerValue(row.CorrelationID), "", row.AuditEventID != nil, resource != nil),
}
}
func outboxTimelineNode(row model.OutboxEvent) LinkTimelineNode {
resourceType := row.ResourceType
if resourceType == "" {
resourceType = row.AggregateType
}
resourceID := row.ResourceID
if resourceID == "" {
resourceID = row.AggregateID
}
resourceKey := row.BusinessKey
if resourceKey == "" {
resourceKey = row.AggregateID
}
resource := InvestigationResourceRef{ResourceType: resourceType, ResourceID: stringPointer(resourceID), ResourceKey: resourceKey, DisplayName: resourceKey}
refs := InvestigationRefs{
ResourceRefs: []InvestigationResourceRef{resource}, RequestID: stringPointer(row.RequestID),
CorrelationID: stringPointer(row.CorrelationID), IntegrationRefs: []IntegrationRef{},
}
return LinkTimelineNode{
RecordSource: constants.AuditRecordSourceOutboxEvent, NodeID: row.EventID,
OccurredAt: row.CreatedAt, Code: row.EventType, Title: "可靠事件:" + row.EventType,
Result: strconv.Itoa(row.Status), ResultName: constants.GetOutboxStatusName(row.Status),
Summary: fmt.Sprintf("%s/%s重试 %d 次", row.AggregateType, row.AggregateID, row.RetryCount),
RequestID: stringPointer(row.RequestID), CorrelationID: stringPointer(row.CorrelationID),
ParentEventID: stringPointer(row.ParentEventID), Resources: refs.ResourceRefs, InvestigationRefs: refs,
Fidelity: linkageFidelity(row.RequestID, row.CorrelationID, row.ParentEventID, row.ParentEventID != "", resourceType != "" && resourceID != ""),
}
}
func resourceReferenceNodes(event EventView, refs InvestigationRefs) []LinkTimelineNode {
nodes := make([]LinkTimelineNode, 0, len(event.Resources))
for _, resource := range event.Resources {
recordSource := ""
titlePrefix := ""
summary := ""
switch {
case isAsynqTaskResource(resource.ResourceType):
recordSource = constants.AuditRecordSourceAsynqTask
titlePrefix = "异步任务:"
summary = "持久化任务资源摘要;不读取或推断 Redis 队列历史"
case isDomainLedgerResource(resource.ResourceType):
recordSource = constants.AuditRecordSourceDomainLedgerRef
titlePrefix = "业务账本引用:"
summary = "状态、金额及业务结论以对应业务表为准"
default:
continue
}
resourceRef := InvestigationResourceRef{ResourceType: resource.ResourceType, ResourceID: resource.ResourceID, ResourceKey: resource.ResourceKey, DisplayName: resource.DisplayName}
nodeRefs := refs
nodeRefs.ResourceRefs = []InvestigationResourceRef{resourceRef}
nodes = append(nodes, LinkTimelineNode{
RecordSource: recordSource,
NodeID: fmt.Sprintf("%s:%s:%s:%s:%s", event.EventID, resource.ResourceType, pointerValue(resource.ResourceID), resource.ResourceKey, resource.Role),
OccurredAt: event.OccurredAt, Code: resource.ResourceType, Title: titlePrefix + resource.DisplayName,
Result: event.Result, Summary: summary, ReferenceOnly: true,
RequestID: stringPointer(event.RequestID), CorrelationID: stringPointer(event.CorrelationID),
ParentEventID: stringPointer(event.ParentEventID), Resources: nodeRefs.ResourceRefs, InvestigationRefs: nodeRefs,
Fidelity: linkageFidelity(event.RequestID, event.CorrelationID, event.ParentEventID, true, resource.ResourceID != nil || resource.ResourceKey != ""),
})
}
return nodes
}
func integrationRefsByAuditID(rows []model.IntegrationLog) map[uint][]IntegrationRef {
refs := make(map[uint][]IntegrationRef)
for _, row := range rows {
if row.AuditEventID != nil {
refs[*row.AuditEventID] = append(refs[*row.AuditEventID], IntegrationRef{IntegrationID: row.IntegrationID})
}
}
return refs
}
func integrationResourceRefs(resources []ResourceView) []IntegrationRef {
refs := make([]IntegrationRef, 0)
for _, resource := range resources {
if resource.ResourceType == constants.AuditResourceIntegrationLog && resource.ResourceKey != "" {
refs = append(refs, IntegrationRef{IntegrationID: resource.ResourceKey})
}
}
return refs
}
func uniqueIntegrationRefs(refs []IntegrationRef) []IntegrationRef {
unique := make([]IntegrationRef, 0, len(refs))
seen := make(map[string]bool, len(refs))
for _, ref := range refs {
if ref.IntegrationID == "" || seen[ref.IntegrationID] {
continue
}
seen[ref.IntegrationID] = true
unique = append(unique, ref)
}
return unique
}
func integrationResourceRef(row model.IntegrationLog) *InvestigationResourceRef {
if row.ResourceType == nil || *row.ResourceType == "" {
return nil
}
ref := InvestigationResourceRef{ResourceType: *row.ResourceType, ResourceID: row.ResourceID}
if row.ResourceKey != nil {
ref.ResourceKey = *row.ResourceKey
ref.DisplayName = *row.ResourceKey
}
return &ref
}
func linkageFidelity(requestID, correlationID, parentEventID string, directAuditLink, stableResource bool) LinkageFidelity {
return LinkageFidelity{
RequestAvailable: requestID != "", CorrelationAvailable: correlationID != "",
ParentEventAvailable: parentEventID != "", DirectAuditLinkAvailable: directAuditLink,
StableResourceAvailable: stableResource,
}
}
func pointerValue(value *string) string {
if value == nil {
return ""
}
return *value
}
func isAsynqTaskResource(resourceType string) bool {
switch resourceType {
case constants.AuditResourceDeviceBatchTask, constants.AuditResourceIotCardImportTask,
constants.AuditResourceDeviceImportTask, constants.AuditResourceAssetPackageBatchOrderTask,
constants.AuditResourceOrderPackageInvalidateTask, constants.AuditResourceExportTask:
return true
default:
return false
}
}
func isDomainLedgerResource(resourceType string) bool {
switch resourceType {
case constants.AuditResourceOrder, constants.AuditResourcePayment, constants.AuditResourceRefund,
constants.AuditResourceAgentRecharge, constants.AuditResourceRechargeOrder,
constants.AuditResourceAssetWallet, constants.AuditResourceAssetWalletTransaction,
constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction,
constants.AuditResourceAgentWalletReservation, constants.AuditResourcePackageUsage,
constants.AuditResourceApprovalInstance, constants.AuditResourceCommissionRecord,
constants.AuditResourceCommissionWithdrawal:
return true
default:
return false
}
}

View File

@@ -12,6 +12,7 @@ import (
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
@@ -35,10 +36,11 @@ type ListFilter struct {
// ListPage 是按创建时间和主键稳定倒序的分页结果。
type ListPage struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []ListItem `json:"items"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []ListItem `json:"items"`
Retention retentionquery.Info `json:"retention"`
}
// ListItem 是 Integration Log 列表投影。
@@ -81,6 +83,12 @@ type Detail struct {
Fidelity FidelityView `json:"fidelity"`
}
// DetailResponse 是外部交互详情及在线留存边界。
type DetailResponse struct {
Detail
Retention retentionquery.Info `json:"retention"`
}
// AttemptView 是显式 trigger_series 下的单次技术尝试。
type AttemptView struct {
IntegrationID string `json:"integration_id"`
@@ -169,6 +177,10 @@ func (q *Query) List(ctx context.Context, filter ListFilter) (*ListPage, error)
if err := q.authorize(ctx); err != nil {
return nil, err
}
filter, retention, err := q.normalizeOnlineFilter(ctx, filter)
if err != nil {
return nil, err
}
if !validFilter(filter) {
return nil, errors.New(errors.CodeInvalidParam)
}
@@ -178,27 +190,52 @@ func (q *Query) List(ctx context.Context, filter ListFilter) (*ListPage, error)
if err := query.Count(&total).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计外部交互日志失败")
}
rows := make([]model.IntegrationLog, 0, filter.PageSize)
if err := query.Order("created_at DESC, id DESC").Offset((filter.Page - 1) * filter.PageSize).Limit(filter.PageSize).Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互日志失败")
rows, err := q.loadListPage(ctx, query, filter.Page, filter.PageSize)
if err != nil {
return nil, err
}
items := make([]ListItem, len(rows))
for i, row := range rows {
items[i] = projectListItem(row)
}
return &ListPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
return &ListPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
}
// loadListPage 先分页主键,再批量读取列表字段,避免加载正文摘要 JSON。
func (q *Query) loadListPage(ctx context.Context, query *gorm.DB, page, pageSize int) ([]model.IntegrationLog, error) {
ids := make([]uint, 0, pageSize)
if err := query.Select("id").Order("created_at DESC, id DESC").
Offset((page-1)*pageSize).Limit(pageSize).Pluck("id", &ids).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互日志分页ID失败")
}
rows := make([]model.IntegrationLog, 0, len(ids))
if len(ids) == 0 {
return rows, nil
}
if err := q.db.WithContext(ctx).Select(
"id", "integration_id", "provider", "direction", "operation",
"resource_type", "resource_id", "resource_key", "result", "duration_ms",
"state_changed", "request_id", "correlation_id", "created_at",
).Where("id IN ?", ids).Order("created_at DESC, id DESC").Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "批量投影外部交互日志失败")
}
return rows, nil
}
// Get 使用稳定 integration_id 查询结构化详情。
func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error) {
func (q *Query) Get(ctx context.Context, integrationID string) (*DetailResponse, error) {
if err := q.authorize(ctx); err != nil {
return nil, err
}
if integrationID == "" {
return nil, errors.New(errors.CodeInvalidParam)
}
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceIntegration)
if err != nil {
return nil, err
}
var row model.IntegrationLog
if err := q.db.WithContext(ctx).Where("integration_id = ?", integrationID).First(&row).Error; err != nil {
if err := q.db.WithContext(ctx).Where("integration_id = ? AND created_at >= ?", integrationID, retention.OnlineFrom.UTC()).First(&row).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil, errors.New(errors.CodeNotFound, "外部交互日志不存在")
}
@@ -216,12 +253,12 @@ func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error)
if err != nil {
return nil, err
}
attempts, err := q.loadAttempts(ctx, row)
attempts, err := q.loadAttempts(ctx, row, retention.OnlineFrom)
if err != nil {
return nil, err
}
providerMessage, providerMessageFidelity := safeProviderMessage(row.ProviderMessage)
return &Detail{
return &DetailResponse{Detail: Detail{
Identity: IdentityView{IntegrationID: row.IntegrationID, Provider: row.Provider, ProviderName: constants.IntegrationProviderName(row.Provider), Direction: row.Direction, DirectionName: constants.IntegrationDirectionName(row.Direction), Operation: row.Operation, OperationName: constants.IntegrationOperationName(row.Operation), ExternalID: row.ExternalID},
Resource: resourceView(row), Trigger: TriggerView{Source: row.TriggerSource, Scene: row.TriggerScene, Series: row.TriggerSeries, Attempt: row.Attempt},
Result: ResultView{Code: row.Result, Name: constants.IntegrationResultName(row.Result), Category: constants.IntegrationResultCategory(row.Result), HTTPStatus: row.HTTPStatus, ProviderCode: row.ProviderCode, ProviderMessage: providerMessage, DurationMS: row.DurationMS, StateChanged: row.StateChanged, RecoveryStrategy: row.RecoveryStrategy},
@@ -235,13 +272,13 @@ func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error)
ResourceIDAvailable: row.ResourceID != nil && *row.ResourceID != "",
ProviderMessageFidelity: providerMessageFidelity,
},
}, nil
}, Retention: retention}, nil
}
func (q *Query) loadAttempts(ctx context.Context, current model.IntegrationLog) ([]AttemptView, error) {
func (q *Query) loadAttempts(ctx context.Context, current model.IntegrationLog, onlineFrom time.Time) ([]AttemptView, error) {
rows := []model.IntegrationLog{current}
if current.TriggerSeries != nil && *current.TriggerSeries != "" {
if err := q.db.WithContext(ctx).Where("trigger_series = ?", *current.TriggerSeries).
if err := q.db.WithContext(ctx).Where("trigger_series = ? AND created_at >= ?", *current.TriggerSeries, onlineFrom.UTC()).
Order("attempt ASC, created_at ASC, id ASC").Find(&rows).Error; err != nil {
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互尝试序列失败")
}
@@ -270,6 +307,15 @@ func (q *Query) authorize(ctx context.Context) error {
return nil
}
func (q *Query) normalizeOnlineFilter(ctx context.Context, filter ListFilter) (ListFilter, retentionquery.Info, error) {
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceIntegration)
if err != nil {
return filter, retention, err
}
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.IntegrationQueryMaxRange)
return filter, retention, err
}
func validFilter(filter ListFilter) bool {
if filter.CreatedFrom == nil || filter.CreatedTo == nil || !filter.CreatedFrom.Before(*filter.CreatedTo) || filter.CreatedTo.Sub(*filter.CreatedFrom) > constants.IntegrationQueryMaxRange {
return false

View File

@@ -7,6 +7,7 @@ import (
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
@@ -19,17 +20,18 @@ type OverviewFilter struct {
// Overview 是外部交互固定维度聚合结果。
type Overview struct {
Total int64 `json:"total"`
AnomalyCount int64 `json:"anomaly_count"`
UnknownCount int64 `json:"unknown_count"`
StalePendingCount int64 `json:"stale_pending_count"`
StateChangedCount int64 `json:"state_changed_count"`
AverageDurationMS float64 `json:"average_duration_ms"`
P95DurationMS float64 `json:"p95_duration_ms"`
Results []ResultCount `json:"results"`
Providers []NamedCount `json:"providers"`
Directions []NamedCount `json:"directions"`
Trend []TrendPoint `json:"trend"`
Total int64 `json:"total"`
AnomalyCount int64 `json:"anomaly_count"`
UnknownCount int64 `json:"unknown_count"`
StalePendingCount int64 `json:"stale_pending_count"`
StateChangedCount int64 `json:"state_changed_count"`
AverageDurationMS float64 `json:"average_duration_ms"`
P95DurationMS float64 `json:"p95_duration_ms"`
Results []ResultCount `json:"results"`
Providers []NamedCount `json:"providers"`
Directions []NamedCount `json:"directions"`
Trend []TrendPoint `json:"trend"`
Retention retentionquery.Info `json:"retention"`
}
// ResultCount 是原始结果及其派生类别计数。
@@ -66,11 +68,17 @@ func (q *Query) Overview(ctx context.Context, filter OverviewFilter) (*Overview,
if filter.Bucket == "" {
filter.Bucket = "hour"
}
var retention retentionquery.Info
var err error
filter.ListFilter, retention, err = q.normalizeOnlineFilter(ctx, filter.ListFilter)
if err != nil {
return nil, err
}
if !validFilter(filter.ListFilter) || (filter.Bucket != "hour" && filter.Bucket != "day") {
return nil, errors.New(errors.CodeInvalidParam)
}
base := applyFilters(q.db.WithContext(ctx).Model(&model.IntegrationLog{}), filter.ListFilter)
result := &Overview{Results: []ResultCount{}, Providers: []NamedCount{}, Directions: []NamedCount{}, Trend: []TrendPoint{}}
result := &Overview{Results: []ResultCount{}, Providers: []NamedCount{}, Directions: []NamedCount{}, Trend: []TrendPoint{}, Retention: retention}
if err := loadOverviewMetrics(base, result); err != nil {
return nil, err
}

View File

@@ -0,0 +1,115 @@
// Package retention 提供在线审计查询的统一留存边界。
package retention
import (
"context"
"time"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// Source 表示受在线留存边界约束的数据源。
type Source string
const (
// SourceAudit 表示统一审计事件。
SourceAudit Source = constants.AuditArchiveSource
// SourceIntegration 表示外部交互日志。
SourceIntegration Source = constants.IntegrationArchiveSource
)
// Info 是查询响应公开的在线留存边界。
type Info struct {
OnlineFrom time.Time `json:"online_from" description:"当前可在线查询的最早时间"`
ArchivedBefore *time.Time `json:"archived_before" description:"早于该时间的数据已归档;尚未清理时为空"`
Timezone string `json:"timezone" description:"留存自然日时区"`
}
// Load 从归档账本读取已完成物理清理的数据边界。
func Load(ctx context.Context, db *gorm.DB, sources ...Source) (Info, error) {
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
if err != nil {
return Info{}, errors.Wrap(errors.CodeInternalError, err, "加载审计留存时区失败")
}
now := time.Now().In(location)
info := Info{OnlineFrom: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location), Timezone: constants.AuditArchiveTimezone}
for _, source := range sources {
boundary, cleaned, err := sourceBoundary(ctx, db, source, location)
if err != nil {
return Info{}, err
}
if boundary.Before(info.OnlineFrom) && info.ArchivedBefore == nil {
info.OnlineFrom = boundary
}
if cleaned && (info.ArchivedBefore == nil || boundary.After(*info.ArchivedBefore)) {
value := boundary
info.ArchivedBefore = &value
info.OnlineFrom = boundary
}
}
return info, nil
}
func sourceBoundary(ctx context.Context, db *gorm.DB, source Source, location *time.Location) (time.Time, bool, error) {
var cleanedEnd *time.Time
if err := db.WithContext(ctx).Model(&model.LogArchiveRun{}).
Where("source = ? AND cleaned_at IS NOT NULL", source).
Select("MAX(range_end)").Scan(&cleanedEnd).Error; err != nil {
return time.Time{}, false, errors.Wrap(errors.CodeDatabaseError, err, "查询审计留存清理边界失败")
}
if cleanedEnd != nil {
return cleanedEnd.In(location), true, nil
}
var earliest *time.Time
table, column := "tb_audit_event", "occurred_at"
if source == SourceIntegration {
table, column = "tb_integration_log", "created_at"
}
if err := db.WithContext(ctx).Table(table).Select("MIN(" + column + ")").Scan(&earliest).Error; err != nil {
return time.Time{}, false, errors.Wrap(errors.CodeDatabaseError, err, "查询审计在线数据边界失败")
}
if earliest != nil {
return earliest.In(location), false, nil
}
now := time.Now().In(location)
return time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location), false, nil
}
// NormalizeRange 将缺省范围收敛到在线窗口,并拒绝归档或跨边界查询。
func NormalizeRange(info Info, from, to *time.Time, maxRange ...time.Duration) (*time.Time, *time.Time, error) {
explicitFrom := from != nil
if from != nil && info.ArchivedBefore != nil && from.Before(info.OnlineFrom) {
return nil, nil, archivedError(info)
}
if to != nil && info.ArchivedBefore != nil && !to.After(info.OnlineFrom) {
return nil, nil, archivedError(info)
}
if from == nil {
value := info.OnlineFrom
from = &value
}
if to == nil {
value := time.Now()
to = &value
}
if len(maxRange) > 0 && maxRange[0] > 0 && to.Sub(*from) > maxRange[0] {
if explicitFrom {
return nil, nil, errors.New(errors.CodeInvalidParam)
}
value := to.Add(-maxRange[0])
from = &value
}
if !from.Before(*to) {
return nil, nil, errors.New(errors.CodeInvalidParam)
}
return from, to, nil
}
func archivedError(info Info) error {
return errors.NewWithData(errors.CodeAuditDataArchived, map[string]any{"retention": info})
}

View File

@@ -15,13 +15,13 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc *
agent := router.Group("/agent/resource-activities")
Register(agent, doc, basePath+"/agent/resource-activities", "GET", "/:resource_type/:identifier", handler.AgentResourceActivities, RouteSpec{
Summary: "查询代理资源活动",
Description: "resource_type/identifier 来自代理当前业务页面稳定字段;店铺范围只读取认证上下文。仅返回写入时生成的安全业务结论和白名单详情,越权与不存在同错。",
Description: "resource_type/identifier 来自代理当前业务页面稳定字段;店铺范围只读取认证上下文。仅查询 retention 标明的在线窗口,归档范围不从对象存储读取。",
Tags: []string{"资源活动"}, Input: new(dto.SubjectResourceActivityRequest), Output: new(auditquery.SubjectActivityPage), Auth: true,
})
enterprise := router.Group("/enterprise/resource-activities")
Register(enterprise, doc, basePath+"/enterprise/resource-activities", "GET", "/:resource_type/:identifier", handler.EnterpriseResourceActivities, RouteSpec{
Summary: "查询企业资源活动",
Description: "仅支持企业当前有效授权的卡和设备;企业身份只读取认证上下文,授权撤销后立即不可读取。响应不包含平台操作者、风险、内部前后值或外部交互内容。",
Description: "仅支持企业当前有效授权的卡和设备;企业身份只读取认证上下文。仅查询 retention 标明的在线窗口,响应不包含平台内部调查字段。",
Tags: []string{"资源活动"}, Input: new(dto.SubjectResourceActivityRequest), Output: new(auditquery.SubjectActivityPage), Auth: true,
})
@@ -30,13 +30,13 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc *
Register(audit, doc, groupPath, "GET", "/events", handler.ListEvents, RouteSpec{
Summary: "查询全局审计事件",
Description: "筛选值来自调查人员输入或其他调查节点的稳定引用;身份范围只读取认证上下文。固定按发生时间和事件ID倒序,不提供导出、修改或删除。",
Description: "筛选值来自调查人员输入或其他调查节点的稳定引用;缺省只查 retention 标明的在线窗口,归档范围返回稳定错误。固定倒序分页,不提供导出、修改或删除。",
Tags: []string{"审计调查"}, Input: new(dto.AuditEventListRequest), Output: new(auditquery.EventPage), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/events/:event_id", handler.GetEvent, RouteSpec{
Summary: "查询审计事件详情",
Description: "event_id 来自事件、资源、操作者或链路节点的 investigation_refs返回全部资源快照和各资源 before/after。",
Tags: []string{"审计调查"}, Input: new(dto.AuditEventIDParams), Output: new(auditquery.EventView), Auth: true,
Description: "event_id 来自 investigation_refs只查询在线 PostgreSQL未命中仍返回资源不存在不扫描对象存储。返回全部资源快照和各资源 before/after。",
Tags: []string{"审计调查"}, Input: new(dto.AuditEventIDParams), Output: new(auditquery.EventDetail), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/actors/:kind/:id/events", handler.ListActorEvents, RouteSpec{
Summary: "查询操作者行为时间线",
@@ -54,20 +54,45 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc *
Description: "resource_type/resource_id 必须来自业务页面稳定字段、资源搜索结果或 investigation_refs。事件在资源作为 primary、affected 或 reference 时均会返回。",
Tags: []string{"审计调查"}, Input: new(dto.AuditResourceTimelineRequest), Output: new(auditquery.EventPage), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/requests/:request_id/timeline", handler.RequestTimeline, RouteSpec{
Summary: "查询请求关联时间线",
Description: "request_id 来自审计或外部集成节点,也可从 Access Log 粘贴。只组合 retention 在线窗口内的持久化事实,不扫描 Access Log 或对象存储。",
Tags: []string{"审计调查"}, Input: new(dto.AuditRequestTimelineParams), Output: new(auditquery.LinkTimeline), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/correlations/:correlation_id/timeline", handler.CorrelationTimeline, RouteSpec{
Summary: "查询业务关联时间线",
Description: "correlation_id 来自稳定调查引用。只组合 retention 在线窗口内的持久化事实;相同 correlation 不用于猜测技术重试。",
Tags: []string{"审计调查"}, Input: new(dto.AuditCorrelationTimelineParams), Output: new(auditquery.LinkTimeline), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/finance/timeline", handler.FinanceTimeline, RouteSpec{
Summary: "查询资金调查时间线",
Description: "可使用任一稳定资金条件进入;缺省只查 retention 在线窗口,归档范围不返回部分结果。关联事实由服务端解析,金额以业务账本为权威。",
Tags: []string{"审计调查"}, Input: new(dto.AuditFinanceTimelineRequest), Output: new(auditquery.FinanceTimelinePage), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/risks/overview", handler.RiskOverview, RouteSpec{
Summary: "查询风险调查总览",
Description: "时间范围最长31天缺省时使用当前在线窗口只聚合高风险、资金、安全、失败、拒绝、部分成功和结果未知事件。",
Tags: []string{"审计调查"}, Input: new(dto.AuditRiskOverviewRequest), Output: new(auditquery.RiskOverview), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/risks/events", handler.RiskEvents, RouteSpec{
Summary: "查询风险事件明细",
Description: "筛选条件来自风险总览分桶或调查人员输入,缺省只查 retention 在线窗口;明细返回 investigation_refs不提供处置或封禁能力。",
Tags: []string{"审计调查"}, Input: new(dto.AuditRiskEventsRequest), Output: new(auditquery.RiskEventPage), Auth: true,
})
// Integration 总览静态路径必须先于动态详情路径,避免 overview 被当作 integration_id。
Register(audit, doc, groupPath, "GET", "/integrations/overview", handler.IntegrationOverview, RouteSpec{
Summary: "查询外部集成交互总览",
Description: "筛选和时间范围来自调查输入或关联视角跳转,身份只来自认证上下文。总览区分成功、处理中、结果不确定、失败和未发送终态。",
Description: "筛选和时间范围来自调查输入或关联视角跳转;缺省只查 retention 在线窗口,归档范围返回稳定错误。总览区分五类结果。",
Tags: []string{"审计调查"}, Input: new(dto.IntegrationOverviewRequest), Output: new(integrationquery.Overview), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/integrations", handler.ListIntegrations, RouteSpec{
Summary: "查询外部集成交互列表",
Description: "组合筛选来自调查输入或关联视角稳定引用固定按创建时间和记录ID倒序分页,不提供任意摘要搜索。",
Description: "组合筛选来自调查输入或稳定引用;缺省只查 retention 在线窗口,归档范围不返回空页或部分结果。固定倒序分页,不提供任意摘要搜索。",
Tags: []string{"审计调查"}, Input: new(dto.IntegrationListRequest), Output: new(integrationquery.ListPage), Auth: true,
})
Register(audit, doc, groupPath, "GET", "/integrations/:integration_id", handler.GetIntegration, RouteSpec{
Summary: "查询外部集成交互详情",
Description: "integration_id 来自列表、通知目标 target_key 或调查节点稳定引用;只展示结构化详情和显式尝试序列,不提供重试、补偿、确认、绑定、恢复、修改、删除或导出。",
Tags: []string{"审计调查"}, Input: new(dto.IntegrationIDParams), Output: new(integrationquery.Detail), Auth: true,
Description: "integration_id 来自稳定引用;只查询在线 PostgreSQL未命中仍返回资源不存在。展示结构化详情和在线尝试序列,不提供归档读取、恢复、修改、删除或导出。",
Tags: []string{"审计调查"}, Input: new(dto.IntegrationIDParams), Output: new(integrationquery.DetailResponse), Auth: true,
})
}

View File

@@ -83,10 +83,6 @@ func (s *Service) Get(ctx context.Context, id uint) (*dto.CarrierResponse, error
}
return nil, errors.Wrap(errors.CodeInternalError, err, "获取运营商失败")
}
if s.audit == nil {
return nil, errors.New(errors.CodeInvalidStatus, "运营商配置审计接缝未配置")
}
before := *carrier
return s.toResponse(carrier), nil
}
@@ -103,6 +99,7 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateCarrierReq
}
return nil, errors.Wrap(errors.CodeInternalError, err, "获取运营商失败")
}
before := *carrier
if req.CarrierName != nil {
carrier.CarrierName = *req.CarrierName

View File

@@ -65,12 +65,12 @@ func (s *Service) AllocateDevices(ctx context.Context, enterpriseID uint, req *d
return nil, errors.New(errors.CodeInvalidStatus, "企业设备授权审计接缝未配置")
}
if err := validateEnterpriseDeviceActor(ctx); err != nil {
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, err)
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, err)
return nil, err
}
if err := middleware.CanManageEnterprise(ctx, enterpriseID, s.enterpriseStore); err != nil {
permissionErr := errors.New(errors.CodeForbidden, "无权限操作该资源或资源不存在")
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, permissionErr)
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, permissionErr)
return nil, permissionErr
}
enterprise, err := s.enterpriseStore.GetByID(ctx, enterpriseID)
@@ -417,12 +417,12 @@ func (s *Service) RecallDevices(ctx context.Context, enterpriseID uint, req *dto
return nil, errors.New(errors.CodeInvalidStatus, "企业设备授权审计接缝未配置")
}
if err := validateEnterpriseDeviceActor(ctx); err != nil {
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, err)
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, err)
return nil, err
}
if err := middleware.CanManageEnterprise(ctx, enterpriseID, s.enterpriseStore); err != nil {
permissionErr := errors.New(errors.CodeForbidden, "无权限操作该资源或资源不存在")
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, permissionErr)
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, permissionErr)
return nil, permissionErr
}
enterprise, err := s.enterpriseStore.GetByID(ctx, enterpriseID)

View File

@@ -23,38 +23,46 @@ func (s *Service) SetAccessAudit(writer *audit.Writer) {
s.auditWriter = writer
}
// WriteCardStateAudit 将卡观测事务中的人工状态操作写入统一 Audit Event。
// WriteCardStateAudit 将卡观测事务中的人工、回调或 Worker 状态变化写入统一 Audit Event。
func (s *Service) WriteCardStateAudit(ctx context.Context, tx *gorm.DB, input cardapp.StateAudit) error {
extraResources := make([]audit.ResourceInput, 0, 1)
if input.IntegrationID != "" {
extraResources = append(extraResources, callbackIntegrationAuditResource(ctx, input.IntegrationID))
extraResources = append(extraResources, cardStateIntegrationAuditResource(ctx, input.IntegrationID))
}
return s.appendCardLifecycleAudit(ctx, tx, input.ActionCode, input.Summary, constants.AuditResultSuccess,
input.Card, input.BeforeData, input.AfterData, nil, extraResources...)
}
// WriteCardStateFailure 使用独立短事务记录已解析卡资源后的回调失败。
// WriteCardStateFailure 使用独立短事务记录已解析卡资源后的回调或 Worker 失败。
func (s *Service) WriteCardStateFailure(ctx context.Context, input cardapp.StateAudit, businessErr error) {
extraResources := make([]audit.ResourceInput, 0, 1)
if input.IntegrationID != "" {
extraResources = append(extraResources, callbackIntegrationAuditResource(ctx, input.IntegrationID))
extraResources = append(extraResources, cardStateIntegrationAuditResource(ctx, input.IntegrationID))
}
s.recordCardLifecycleFailure(ctx, input.ActionCode, input.Summary, constants.AuditResultFailed,
input.Card, input.Card.ID, businessErr, extraResources...)
}
func callbackIntegrationAuditResource(ctx context.Context, integrationID string) audit.ResourceInput {
func cardStateIntegrationAuditResource(ctx context.Context, integrationID string) audit.ResourceInput {
linkage := auditcontext.From(ctx)
correlationID := linkage.CorrelationID
if correlationID == "" {
correlationID = linkage.RequestID
}
direction := constants.IntegrationDirectionInbound
role := constants.AuditResourceRoleCallbackIntegration
provider := linkage.ActorID
if linkage.Source == constants.AuditSourceWorker {
direction = constants.IntegrationDirectionOutbound
role = constants.AuditResourceRoleWorkerIntegration
provider = constants.IntegrationProviderGateway
}
return audit.ResourceInput{
Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: integrationID,
Relation: constants.AuditResourceRelationReference, Role: constants.AuditResourceRoleCallbackIntegration,
Relation: constants.AuditResourceRelationReference, Role: role,
IdentitySnapshot: map[string]any{
"integration_id": integrationID, "provider": linkage.ActorID,
"direction": constants.IntegrationDirectionInbound, "correlation_id": correlationID,
"integration_id": integrationID, "provider": provider,
"direction": direction, "correlation_id": correlationID,
},
SubjectVisibility: constants.AuditSubjectInternalOnly,
}

View File

@@ -2709,8 +2709,12 @@ func (s *Service) enqueueCommissionCalculation(ctx context.Context, orderID uint
return
}
linkage := auditcontext.From(ctx)
// 直接传 map由 EnqueueTask 内部统一序列化一次(传 []byte 会导致 sonic.Marshal 二次 base64 编码)
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeCommission, map[string]any{"order_id": orderID}); err != nil {
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeCommission, map[string]any{
"order_id": orderID, "request_id": linkage.RequestID, "correlation_id": linkage.CorrelationID,
"parent_event_id": linkage.ParentEventID,
}); err != nil {
s.logger.Error("佣金计算任务入队失败",
zap.Uint("order_id", orderID),
zap.Error(err),

View File

@@ -14,6 +14,7 @@ import (
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// ResumeCallback 复机回调接口

View File

@@ -9,6 +9,7 @@ import (
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/internal/task"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/queue"
@@ -227,7 +228,11 @@ func (s *Service) HandlePaymentCallback(ctx context.Context, paymentNo string, p
linkedIDs := rechargeOrder.LinkedPackageIDs
if len(linkedIDs) > 0 {
taskPayload := task.AutoPurchasePayload{RechargeOrderID: rechargeOrder.ID}
linkage := auditcontext.From(ctx)
taskPayload := task.AutoPurchasePayload{
RechargeOrderID: rechargeOrder.ID, RequestID: linkage.RequestID,
CorrelationID: linkage.CorrelationID, ParentEventID: linkage.ParentEventID,
}
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeAutoPurchaseAfterRecharge, taskPayload,
asynq.MaxRetry(3),
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAutoPurchaseAfterRecharge)),

View File

@@ -132,12 +132,12 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateRoleReques
if req.RoleName != nil && *req.RoleName != role.RoleName {
exists, err := s.roleStore.ExistsByName(ctx, *req.RoleName, id)
if err != nil {
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, beforeData, nil, err)
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, nil, beforeData, err)
return nil, errors.Wrap(errors.CodeInternalError, err, "检查角色名失败")
}
if exists {
appErr := errors.New(errors.CodeRoleNameExists)
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "拒绝更新重复角色名", constants.AuditResultDenied, role, beforeData, nil, appErr)
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "拒绝更新重复角色名", constants.AuditResultDenied, role, nil, beforeData, appErr)
return nil, appErr
}
role.RoleName = *req.RoleName
@@ -162,7 +162,7 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateRoleReques
OperatorID: currentUserID, Role: role, BeforeData: beforeData, AfterData: roleAuditData(role),
})
}); err != nil {
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, beforeData, nil, err)
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, nil, beforeData, err)
return nil, errors.Wrap(errors.CodeInternalError, err, "更新角色失败")
}
@@ -181,19 +181,19 @@ func (s *Service) Delete(ctx context.Context, id uint) error {
accountCount, err := s.accountRoleStore.CountByRoleID(ctx, id)
if err != nil {
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
}
shopCount, err := s.shopRoleStore.CountByRoleID(ctx, id)
if err != nil {
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
}
if accountCount > 0 || shopCount > 0 {
appErr := errors.New(errors.CodeRoleInUse, fmt.Sprintf("该角色已分配给 %d 个账号、%d 个店铺,请先移除相关分配后再删除", accountCount, shopCount))
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "拒绝删除使用中的角色", constants.AuditResultDenied, role, roleAuditData(role), nil, appErr)
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "拒绝删除使用中的角色", constants.AuditResultDenied, role, nil, roleAuditData(role), appErr)
return appErr
}
@@ -208,7 +208,7 @@ func (s *Service) Delete(ctx context.Context, id uint) error {
OperatorID: operatorID, Role: role, BeforeData: beforeData, AfterData: map[string]any{"deleted": true},
})
}); err != nil {
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, beforeData, nil, err)
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, beforeData, err)
return errors.Wrap(errors.CodeInternalError, err, "删除角色失败")
}
@@ -510,19 +510,19 @@ func (s *Service) UpdateStatus(ctx context.Context, id uint, status int) error {
if status == constants.StatusDisabled {
accountCount, err := s.accountRoleStore.CountByRoleID(ctx, id)
if err != nil {
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
}
shopCount, err := s.shopRoleStore.CountByRoleID(ctx, id)
if err != nil {
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
}
if accountCount > 0 || shopCount > 0 {
appErr := errors.New(errors.CodeRoleInUse, fmt.Sprintf("该角色已分配给 %d 个账号、%d 个店铺,请先移除相关分配后再禁用", accountCount, shopCount))
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "拒绝禁用使用中的角色", constants.AuditResultDenied, role, roleAuditData(role), nil, appErr)
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "拒绝禁用使用中的角色", constants.AuditResultDenied, role, nil, roleAuditData(role), appErr)
return appErr
}
}
@@ -540,7 +540,7 @@ func (s *Service) UpdateStatus(ctx context.Context, id uint, status int) error {
OperatorID: currentUserID, Role: role, BeforeData: beforeData, AfterData: roleAuditData(role),
})
}); err != nil {
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, beforeData, nil, err)
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, beforeData, err)
return errors.Wrap(errors.CodeInternalError, err, "更新角色状态失败")
}

View File

@@ -0,0 +1,61 @@
package task
import (
"context"
"fmt"
"time"
"github.com/bytedance/sonic"
"github.com/hibiken/asynq"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// AuditDailyArchivePayload 是人工补档时可选的任务载荷。
type AuditDailyArchivePayload struct {
ArchiveDate string `json:"archive_date"`
}
// AuditDailyArchiveHandler 处理统一审计每日冷归档任务。
type AuditDailyArchiveHandler struct {
service *auditarchive.Service
logger *zap.Logger
}
// NewAuditDailyArchiveHandler 创建统一审计每日冷归档任务处理器。
func NewAuditDailyArchiveHandler(service *auditarchive.Service, logger *zap.Logger) *AuditDailyArchiveHandler {
return &AuditDailyArchiveHandler{service: service, logger: logger}
}
// Handle 执行前一完整自然日归档,或按任务载荷补档指定自然日。
func (h *AuditDailyArchiveHandler) Handle(ctx context.Context, task *asynq.Task) error {
if h.service == nil {
return fmt.Errorf("统一审计归档服务未配置")
}
var err error
if len(task.Payload()) == 0 {
err = h.service.ArchivePreviousDay(ctx)
} else {
var payload AuditDailyArchivePayload
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
return fmt.Errorf("解析统一审计归档任务载荷失败: %w", unmarshalErr)
}
location, locationErr := time.LoadLocation(constants.AuditArchiveTimezone)
if locationErr != nil {
return fmt.Errorf("加载统一审计归档时区失败: %w", locationErr)
}
archiveDate, parseErr := time.ParseInLocation(time.DateOnly, payload.ArchiveDate, location)
if parseErr != nil {
return fmt.Errorf("解析统一审计归档日期失败: %w", parseErr)
}
err = h.service.ArchiveDate(ctx, archiveDate)
}
if err != nil {
h.logger.Error("统一审计每日冷归档失败", zap.Error(err))
return err
}
h.logger.Info("统一审计每日冷归档完成")
return nil
}

View File

@@ -0,0 +1,64 @@
package task
import (
"context"
"fmt"
"time"
"github.com/bytedance/sonic"
"github.com/hibiken/asynq"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
)
// AuditMonthlyRetentionPayload 是人工补跑月度清理时可选的任务载荷。
type AuditMonthlyRetentionPayload struct {
ArchiveMonth string `json:"archive_month"`
}
// AuditMonthlyRetentionHandler 处理归档完整性门禁与上月在线日志物理清理。
type AuditMonthlyRetentionHandler struct {
service *auditarchive.Service
logger *zap.Logger
}
// NewAuditMonthlyRetentionHandler 创建月度日志留存清理处理器。
func NewAuditMonthlyRetentionHandler(service *auditarchive.Service, logger *zap.Logger) *AuditMonthlyRetentionHandler {
return &AuditMonthlyRetentionHandler{service: service, logger: logger}
}
// Handle 校验整月归档后按固定顺序分批物理删除 PostgreSQL 在线日志。
func (h *AuditMonthlyRetentionHandler) Handle(ctx context.Context, task *asynq.Task) error {
if h.service == nil {
return fmt.Errorf("月度日志留存清理服务未配置")
}
startedAt := time.Now()
var result auditarchive.RetentionResult
var err error
if len(task.Payload()) == 0 {
result, err = h.service.CleanupPreviousMonth(ctx)
} else {
var payload AuditMonthlyRetentionPayload
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
return fmt.Errorf("解析月度日志留存清理任务载荷失败: %w", unmarshalErr)
}
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
if parseErr != nil {
return parseErr
}
result, err = h.service.CleanupMonth(ctx, month)
}
fields := []zap.Field{
zap.String("archive_month", result.Month), zap.Int64("audit_event_count", result.EventCount),
zap.Int64("event_resource_count", result.ResourceCount), zap.Int64("integration_log_count", result.IntegrationCount),
zap.Duration("duration", time.Since(startedAt)), zap.Int("manifest_count", len(result.ManifestKeys)),
}
if err != nil {
fields = append(fields, zap.String("severity", "critical"), zap.Error(err))
h.logger.Error("月度日志留存清理失败PostgreSQL 整月清理已阻断或等待断点续跑", fields...)
return err
}
h.logger.Info("月度日志留存清理完成", fields...)
return nil
}

View File

@@ -27,7 +27,10 @@ import (
// AutoPurchasePayload 充值后自动购包任务载荷
type AutoPurchasePayload struct {
RechargeOrderID uint `json:"recharge_order_id"`
RechargeOrderID uint `json:"recharge_order_id"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
ParentEventID string `json:"parent_event_id,omitempty"`
}
// AutoPurchaseHandler 充值后自动购包任务处理器
@@ -123,10 +126,14 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task)
h.logger.Error("查询充值订单失败", zap.Uint("recharge_order_id", payload.RechargeOrderID), zap.Error(err))
return err
}
correlationID := payload.CorrelationID
if correlationID == "" {
correlationID = rechargeOrder.RechargeOrderNo
}
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeAutoPurchaseAfterRecharge,
ActorName: "充值后自动购包任务", Source: constants.AuditSourceWorker,
CorrelationID: rechargeOrder.RechargeOrderNo,
RequestID: payload.RequestID, CorrelationID: correlationID, ParentEventID: payload.ParentEventID,
})
if rechargeOrder.AutoPurchaseStatus == constants.AutoPurchaseStatusSuccess {
@@ -301,7 +308,11 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task)
// 事务提交成功后触发佣金计算(不在事务内,防止任务提交后事务回滚的数据一致性问题)
if h.asynqClient != nil && createdOrderID > 0 {
payloadBytes, marshalErr := sonic.Marshal(map[string]any{"order_id": createdOrderID})
linkage := auditcontext.From(ctx)
payloadBytes, marshalErr := sonic.Marshal(CommissionCalculationPayload{
OrderID: createdOrderID, RequestID: linkage.RequestID,
CorrelationID: linkage.CorrelationID, ParentEventID: linkage.ParentEventID,
})
if marshalErr != nil {
h.logger.Warn("佣金任务载荷序列化失败",
zap.Uint("order_id", createdOrderID),

View File

@@ -8,6 +8,8 @@ import (
"go.uber.org/zap"
cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
@@ -32,6 +34,11 @@ func (h *CardObservationSeriesHandler) Handle(ctx context.Context, task *asynq.T
h.logger.Error("解析卡观测序列任务载荷失败", zap.Error(err))
return errors.Wrap(errors.CodeInvalidParam, err, "卡观测序列任务载荷无法解析")
}
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeCardObservationSeries,
ActorName: "卡观测序列任务", Source: constants.AuditSourceWorker,
RequestID: payload.RequestID, CorrelationID: payload.CorrelationID, ParentEventID: payload.ParentEventID,
})
if err := h.service.Execute(ctx, payload); err != nil {
h.logger.Warn("卡观测序列当前尝试失败",
zap.String("series_id", payload.SeriesID), zap.Int("attempt", payload.Attempt), zap.Error(err))

View File

@@ -18,7 +18,10 @@ const (
)
type CommissionCalculationPayload struct {
OrderID uint `json:"order_id"`
OrderID uint `json:"order_id"`
RequestID string `json:"request_id,omitempty"`
CorrelationID string `json:"correlation_id,omitempty"`
ParentEventID string `json:"parent_event_id,omitempty"`
}
type CommissionCalculationHandler struct {
@@ -48,9 +51,15 @@ func (h *CommissionCalculationHandler) HandleCommissionCalculation(ctx context.C
)
return asynq.SkipRetry
}
correlationID := payload.CorrelationID
if correlationID == "" {
correlationID = task.ResultWriter().TaskID()
}
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.AuditActorIDCommissionCalculationWorker,
ActorName: "订单佣金计算任务", Source: constants.AuditSourceWorker,
RequestID: payload.RequestID, CorrelationID: correlationID,
ParentEventID: payload.ParentEventID,
})
if err := h.service.CalculateCommission(ctx, payload.OrderID); err != nil {

View File

@@ -0,0 +1,113 @@
package task
import (
"context"
"fmt"
"time"
"github.com/bytedance/sonic"
"github.com/hibiken/asynq"
"go.uber.org/zap"
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// IntegrationDailyArchivePayload 是人工补档时可选的任务载荷。
type IntegrationDailyArchivePayload struct {
ArchiveDate string `json:"archive_date"`
}
// IntegrationMonthlyFinalizePayload 是人工月度复核时可选的任务载荷。
type IntegrationMonthlyFinalizePayload struct {
ArchiveMonth string `json:"archive_month"`
}
// IntegrationArchiveHandler 处理 Integration Log 每日归档与月度最终复核。
type IntegrationArchiveHandler struct {
service *auditarchive.Service
logger *zap.Logger
}
// NewIntegrationArchiveHandler 创建 Integration Log 归档任务处理器。
func NewIntegrationArchiveHandler(service *auditarchive.Service, logger *zap.Logger) *IntegrationArchiveHandler {
return &IntegrationArchiveHandler{service: service, logger: logger}
}
// HandleDaily 执行前一完整自然日归档,或按任务载荷补档指定自然日。
func (h *IntegrationArchiveHandler) HandleDaily(ctx context.Context, task *asynq.Task) error {
if h.service == nil {
return fmt.Errorf("Integration Log 归档服务未配置")
}
var err error
if len(task.Payload()) == 0 {
err = h.service.ArchivePreviousIntegrationDay(ctx)
} else {
var payload IntegrationDailyArchivePayload
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
return fmt.Errorf("解析 Integration Log 每日归档任务载荷失败: %w", unmarshalErr)
}
date, parseErr := parseArchiveDate(payload.ArchiveDate)
if parseErr != nil {
return parseErr
}
err = h.service.ArchiveIntegrationDate(ctx, date)
}
if err != nil {
h.logger.Error("Integration Log 每日冷归档失败", zap.Error(err))
return err
}
h.logger.Info("Integration Log 每日冷归档完成")
return nil
}
// HandleMonthlyFinalize 执行上一个完整自然月复核,或按任务载荷复核指定月份。
func (h *IntegrationArchiveHandler) HandleMonthlyFinalize(ctx context.Context, task *asynq.Task) error {
if h.service == nil {
return fmt.Errorf("Integration Log 归档服务未配置")
}
var err error
if len(task.Payload()) == 0 {
err = h.service.FinalizePreviousIntegrationMonth(ctx)
} else {
var payload IntegrationMonthlyFinalizePayload
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
return fmt.Errorf("解析 Integration Log 月度复核任务载荷失败: %w", unmarshalErr)
}
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
if parseErr != nil {
return parseErr
}
err = h.service.FinalizeIntegrationMonth(ctx, month)
}
if err != nil {
h.logger.Error("Integration Log 月度最终版本复核失败,后续清理必须阻止", zap.Error(err))
return err
}
h.logger.Info("Integration Log 月度最终版本复核完成")
return nil
}
func parseArchiveDate(value string) (time.Time, error) {
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
if err != nil {
return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err)
}
date, err := time.ParseInLocation(time.DateOnly, value, location)
if err != nil {
return time.Time{}, fmt.Errorf("解析 Integration Log 归档日期失败: %w", err)
}
return date, nil
}
func parseArchiveMonth(value string) (time.Time, error) {
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
if err != nil {
return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err)
}
month, err := time.ParseInLocation("2006-01", value, location)
if err != nil {
return time.Time{}, fmt.Errorf("解析 Integration Log 归档月份失败: %w", err)
}
return month, nil
}

View File

@@ -23,10 +23,11 @@ func NewNotificationCleanupHandler(service *notificationinfra.CleanupService, lo
}
// Handle 执行有界、可重入的通知分批清理。
func (h *NotificationCleanupHandler) Handle(ctx context.Context, _ *asynq.Task) error {
func (h *NotificationCleanupHandler) Handle(ctx context.Context, task *asynq.Task) error {
ctx = auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeNotificationCleanup,
ActorName: "站内通知清理任务", Source: constants.AuditSourceWorker,
CorrelationID: task.ResultWriter().TaskID(),
})
h.logger.Info("开始执行站内通知保留清理")
if err := h.service.Run(ctx); err != nil {

View File

@@ -77,6 +77,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil {
return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 流量 Integration Log 失败", logErr)
}
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
if h.observation == nil || h.carrier == nil {
return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
}

View File

@@ -76,6 +76,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil {
return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 网络 Integration Log 失败", logErr)
}
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
if h.observation == nil {
return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
}

View File

@@ -44,6 +44,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
if !ok {
return nil
}
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))

View File

@@ -52,6 +52,7 @@ func (h *PollingProtectHandler) Handle(ctx context.Context, t *asynq.Task) error
if !ok {
return nil
}
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingProtect, "保护期一致性轮询任务", t.ResultWriter().TaskID())
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingProtect) {
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))

View File

@@ -68,6 +68,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil {
return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 实名 Integration Log 失败", logErr)
}
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
if h.observation == nil {
return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
}

View File

@@ -1,14 +1,29 @@
package task
import (
"context"
"strconv"
"time"
"github.com/bytedance/sonic"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// withPollingWorkerAuditContext 为实际改变业务事实的轮询任务补充真实系统操作者与链路。
func withPollingWorkerAuditContext(ctx context.Context, taskType, taskName, correlationID string) context.Context {
if correlationID == "" {
correlationID = taskType
}
return auditcontext.With(ctx, auditcontext.Context{
ActorKind: constants.AuditActorSystemTask, ActorID: taskType,
ActorName: taskName, Source: constants.AuditSourceWorker, CorrelationID: correlationID,
})
}
// shortTaskType 从完整任务类型中提取简短名称(如 polling:carddata → carddata
func shortTaskType(fullTaskType string) string {
for i := len(fullTaskType) - 1; i >= 0; i-- {