This commit is contained in:
350
internal/application/auditarchive/integration.go
Normal file
350
internal/application/auditarchive/integration.go
Normal file
@@ -0,0 +1,350 @@
|
||||
package auditarchive
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
type integrationArchiveFile struct {
|
||||
path string
|
||||
recordCount int64
|
||||
uncompressedBytes int64
|
||||
compressedBytes int64
|
||||
sha256 string
|
||||
}
|
||||
|
||||
type integrationArchiveManifest struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
Source string `json:"source"`
|
||||
ArchiveDate string `json:"archive_date"`
|
||||
Timezone string `json:"timezone"`
|
||||
RangeStart time.Time `json:"range_start"`
|
||||
RangeEnd time.Time `json:"range_end"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
RecordCount int64 `json:"record_count"`
|
||||
UncompressedBytes int64 `json:"uncompressed_bytes"`
|
||||
CompressedBytes int64 `json:"compressed_bytes"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Revision int `json:"revision"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Status string `json:"status"`
|
||||
Final bool `json:"final"`
|
||||
}
|
||||
|
||||
// ArchivePreviousIntegrationDay 归档 Asia/Shanghai 前一完整自然日的 Integration Log 创建日快照。
|
||||
func (s *Service) ArchivePreviousIntegrationDay(ctx context.Context) error {
|
||||
now := time.Now().In(s.location)
|
||||
return s.ArchiveIntegrationDate(ctx, now.AddDate(0, 0, -1))
|
||||
}
|
||||
|
||||
// ArchiveIntegrationDate 归档指定 Asia/Shanghai 自然日的 Integration Log 创建日快照。
|
||||
func (s *Service) ArchiveIntegrationDate(ctx context.Context, archiveDate time.Time) error {
|
||||
return s.archiveIntegrationDate(ctx, archiveDate, false)
|
||||
}
|
||||
|
||||
// FinalizePreviousIntegrationMonth 复核并终结上一个完整自然月的 Integration Log 归档。
|
||||
func (s *Service) FinalizePreviousIntegrationMonth(ctx context.Context) error {
|
||||
now := time.Now().In(s.location)
|
||||
return s.FinalizeIntegrationMonth(ctx, now.AddDate(0, -1, 0))
|
||||
}
|
||||
|
||||
// FinalizeIntegrationMonth 逐日复核指定完整自然月,并为变化内容创建最终 revision。
|
||||
func (s *Service) FinalizeIntegrationMonth(ctx context.Context, month time.Time) error {
|
||||
monthStart := time.Date(month.In(s.location).Year(), month.In(s.location).Month(), 1, 0, 0, 0, 0, s.location)
|
||||
currentMonth := time.Now().In(s.location)
|
||||
currentMonthStart := time.Date(currentMonth.Year(), currentMonth.Month(), 1, 0, 0, 0, 0, s.location)
|
||||
if !monthStart.Before(currentMonthStart) {
|
||||
return fmt.Errorf("只能终结已经结束的 Integration Log 完整自然月")
|
||||
}
|
||||
for date := monthStart; date.Before(monthStart.AddDate(0, 1, 0)); date = date.AddDate(0, 0, 1) {
|
||||
if err := s.archiveIntegrationDate(ctx, date, true); err != nil {
|
||||
return fmt.Errorf("终结 %s Integration Log 归档失败: %w", date.Format(time.DateOnly), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) archiveIntegrationDate(ctx context.Context, archiveDate time.Time, final bool) error {
|
||||
if s.db == nil || s.store == nil {
|
||||
return fmt.Errorf("Integration Log 归档数据库或对象存储未配置")
|
||||
}
|
||||
start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location)
|
||||
end := start.AddDate(0, 0, 1)
|
||||
today := time.Now().In(s.location)
|
||||
if end.After(time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)) {
|
||||
return fmt.Errorf("Integration Log 只能归档已经结束的完整自然日")
|
||||
}
|
||||
|
||||
run, err := s.ensureIntegrationRun(ctx, start, end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := s.buildIntegrationArchiveFile(ctx, start, end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.path)
|
||||
if final {
|
||||
pending, pendingErr := s.integrationPendingCount(ctx, start, end)
|
||||
if pendingErr != nil {
|
||||
return pendingErr
|
||||
}
|
||||
if pending > 0 {
|
||||
_ = s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).
|
||||
Updates(map[string]any{"is_final": false, "error_summary": "存在 pending Integration Log,无法形成最终归档", "updated_at": time.Now()}).Error
|
||||
return fmt.Errorf("仍有 %d 条 pending Integration Log,无法形成最终归档", pending)
|
||||
}
|
||||
}
|
||||
if run.Status == constants.ArchiveStatusSuccess && run.RecordCount == file.recordCount && run.SHA256 == file.sha256 {
|
||||
valid, validateErr := s.validateIntegrationRun(ctx, run)
|
||||
if validateErr == nil && valid && (!final || run.IsFinal) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
acquired, err := s.acquireIntegrationRun(ctx, run)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !acquired {
|
||||
return fmt.Errorf("Integration Log 归档任务正在执行")
|
||||
}
|
||||
if err := s.uploadIntegrationArchive(ctx, run, file, final); err != nil {
|
||||
s.markFailed(ctx, run.ID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) ensureIntegrationRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) {
|
||||
run := model.LogArchiveRun{
|
||||
Source: constants.IntegrationArchiveSource, ArchiveDate: start, InstanceID: s.instanceID,
|
||||
SchemaVersion: constants.IntegrationArchiveSchemaVersion, Revision: 1,
|
||||
Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end,
|
||||
}
|
||||
result := s.db.WithContext(ctx).Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}},
|
||||
DoNothing: true,
|
||||
}).Create(&run)
|
||||
if result.Error != nil {
|
||||
return nil, fmt.Errorf("创建 Integration Log 归档账本失败: %w", result.Error)
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
if err := s.db.WithContext(ctx).Where(
|
||||
"source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?",
|
||||
constants.IntegrationArchiveSource, start, s.instanceID, constants.IntegrationArchiveSchemaVersion,
|
||||
).First(&run).Error; err != nil {
|
||||
return nil, fmt.Errorf("读取 Integration Log 归档账本失败: %w", err)
|
||||
}
|
||||
}
|
||||
return &run, nil
|
||||
}
|
||||
|
||||
func (s *Service) acquireIntegrationRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
|
||||
revision := run.Revision
|
||||
if run.Status != constants.ArchiveStatusPending {
|
||||
revision++
|
||||
}
|
||||
now := time.Now()
|
||||
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, now.Add(-3*time.Hour)).
|
||||
Updates(map[string]any{
|
||||
"status": constants.ArchiveStatusRunning, "revision": revision, "is_final": false,
|
||||
"attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "", "completed_at": nil, "updated_at": now,
|
||||
})
|
||||
if result.Error != nil {
|
||||
return false, fmt.Errorf("锁定 Integration Log 归档任务失败: %w", result.Error)
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return false, nil
|
||||
}
|
||||
run.Revision = revision
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Service) buildIntegrationArchiveFile(ctx context.Context, start, end time.Time) (*integrationArchiveFile, error) {
|
||||
temp, err := os.CreateTemp("", "integration-logs-*.jsonl.gz")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建 Integration Log 归档临时文件失败: %w", err)
|
||||
}
|
||||
path := temp.Name()
|
||||
failed := true
|
||||
defer func() {
|
||||
_ = temp.Close()
|
||||
if failed {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher))
|
||||
result := &integrationArchiveFile{path: path}
|
||||
var lastID uint
|
||||
for {
|
||||
var logs []model.IntegrationLog
|
||||
if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID).
|
||||
Order("id ASC").Limit(archivePageSize).Find(&logs).Error; err != nil {
|
||||
return nil, fmt.Errorf("读取 Integration Log 归档记录失败: %w", err)
|
||||
}
|
||||
if len(logs) == 0 {
|
||||
break
|
||||
}
|
||||
for i := range logs {
|
||||
line, marshalErr := sonic.Marshal(logs[i])
|
||||
if marshalErr != nil {
|
||||
return nil, fmt.Errorf("序列化 Integration Log 归档记录失败: %w", marshalErr)
|
||||
}
|
||||
line = append(line, '\n')
|
||||
if _, writeErr := gzipWriter.Write(line); writeErr != nil {
|
||||
return nil, fmt.Errorf("写入 Integration Log 归档压缩流失败: %w", writeErr)
|
||||
}
|
||||
result.recordCount++
|
||||
result.uncompressedBytes += int64(len(line))
|
||||
}
|
||||
lastID = logs[len(logs)-1].ID
|
||||
}
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
return nil, fmt.Errorf("关闭 Integration Log 归档压缩流失败: %w", err)
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return nil, fmt.Errorf("关闭 Integration Log 归档临时文件失败: %w", err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取 Integration Log 归档临时文件信息失败: %w", err)
|
||||
}
|
||||
result.compressedBytes = info.Size()
|
||||
result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
failed = false
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) uploadIntegrationArchive(ctx context.Context, run *model.LogArchiveRun, file *integrationArchiveFile, final bool) error {
|
||||
count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count != file.recordCount {
|
||||
return fmt.Errorf("Integration Log 归档生成期间记录数量发生变化")
|
||||
}
|
||||
objectKey, manifestKey := integrationObjectKeys(run.RangeStart, run.Revision)
|
||||
metadata := integrationArchiveMetadata(file, run, final)
|
||||
reader, err := os.Open(file.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("打开 Integration Log 归档临时文件失败: %w", err)
|
||||
}
|
||||
uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata)
|
||||
closeErr := reader.Close()
|
||||
if uploadErr != nil {
|
||||
return fmt.Errorf("上传 Integration Log 归档对象失败: %w", uploadErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("关闭 Integration Log 归档临时文件失败: %w", closeErr)
|
||||
}
|
||||
if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
generatedAt := time.Now().In(s.location)
|
||||
manifest := integrationArchiveManifest{
|
||||
SchemaVersion: constants.IntegrationArchiveSchemaVersion, Source: constants.IntegrationArchiveSource,
|
||||
ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone,
|
||||
RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID,
|
||||
RecordCount: file.recordCount, UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes,
|
||||
ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision, GeneratedAt: generatedAt,
|
||||
Status: constants.ArchiveStatusSuccess, Final: final,
|
||||
}
|
||||
manifestBytes, err := sonic.Marshal(manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("序列化 Integration Log 归档清单失败: %w", err)
|
||||
}
|
||||
manifestMetadata := map[string]string{
|
||||
"source": constants.IntegrationArchiveSource, "data-sha256": file.sha256,
|
||||
"revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(final),
|
||||
}
|
||||
if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil {
|
||||
return fmt.Errorf("上传 Integration Log 归档清单失败: %w", err)
|
||||
}
|
||||
if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
completedAt := time.Now()
|
||||
return s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(map[string]any{
|
||||
"status": constants.ArchiveStatusSuccess, "is_final": final,
|
||||
"object_key": objectKey, "manifest_key": manifestKey, "record_count": file.recordCount,
|
||||
"uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes,
|
||||
"sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt, "updated_at": completedAt,
|
||||
}).Error
|
||||
}
|
||||
|
||||
func (s *Service) validateIntegrationRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
|
||||
metadata := map[string]string{
|
||||
"sha256": run.SHA256, "record-count": strconv.FormatInt(run.RecordCount, 10),
|
||||
"revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(run.IsFinal),
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
manifestMetadata := map[string]string{
|
||||
"source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256,
|
||||
"revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(run.IsFinal),
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Service) integrationRecordCount(ctx context.Context, start, end time.Time) (int64, error) {
|
||||
var count int64
|
||||
if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}).
|
||||
Where("created_at >= ? AND created_at < ?", start, end).Count(&count).Error; err != nil {
|
||||
return 0, fmt.Errorf("统计 Integration Log 归档记录失败: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *Service) integrationPendingCount(ctx context.Context, start, end time.Time) (int64, error) {
|
||||
var count int64
|
||||
if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}).
|
||||
Where("created_at >= ? AND created_at < ? AND result = ?", start, end, constants.IntegrationResultPending).
|
||||
Count(&count).Error; err != nil {
|
||||
return 0, fmt.Errorf("统计 pending Integration Log 失败: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func integrationObjectKeys(date time.Time, revision int) (string, string) {
|
||||
prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day())
|
||||
name := fmt.Sprintf("integration-logs-%s-r%d", date.Format(time.DateOnly), revision)
|
||||
return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json"
|
||||
}
|
||||
|
||||
func integrationArchiveMetadata(file *integrationArchiveFile, run *model.LogArchiveRun, final bool) map[string]string {
|
||||
return map[string]string{
|
||||
"schema-version": constants.IntegrationArchiveSchemaVersion,
|
||||
"source": constants.IntegrationArchiveSource,
|
||||
"archive-date": run.RangeStart.Format(time.DateOnly),
|
||||
"timezone": constants.AuditArchiveTimezone,
|
||||
"record-count": strconv.FormatInt(file.recordCount, 10),
|
||||
"sha256": file.sha256,
|
||||
"revision": strconv.Itoa(run.Revision),
|
||||
"final": strconv.FormatBool(final),
|
||||
}
|
||||
}
|
||||
542
internal/application/auditarchive/retention.go
Normal file
542
internal/application/auditarchive/retention.go
Normal file
@@ -0,0 +1,542 @@
|
||||
package auditarchive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
const maxManifestBytes = 1024 * 1024
|
||||
|
||||
// RetentionAudit 描述月度物理清理的统一审计事实。
|
||||
type RetentionAudit struct {
|
||||
EventID string
|
||||
Month string
|
||||
Summary string
|
||||
Result string
|
||||
ErrorSummary string
|
||||
RangeStart time.Time
|
||||
RangeEnd time.Time
|
||||
EventCount int64
|
||||
ResourceCount int64
|
||||
IntegrationCount int64
|
||||
ManifestKeys []string
|
||||
DurationMS int64
|
||||
}
|
||||
|
||||
// RetentionResult 是月度留存清理的结构化执行结果。
|
||||
type RetentionResult struct {
|
||||
Month string
|
||||
EventCount int64
|
||||
ResourceCount int64
|
||||
IntegrationCount int64
|
||||
ManifestKeys []string
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
type retentionRuns struct {
|
||||
audit []*model.LogArchiveRun
|
||||
integration []*model.LogArchiveRun
|
||||
}
|
||||
|
||||
// CleanupPreviousMonth 校验并物理清理上一个完整自然月的在线审计日志。
|
||||
func (s *Service) CleanupPreviousMonth(ctx context.Context) (RetentionResult, error) {
|
||||
now := time.Now().In(s.location)
|
||||
return s.CleanupMonth(ctx, now.AddDate(0, -1, 0))
|
||||
}
|
||||
|
||||
// CleanupMonth 校验归档硬门禁后按固定顺序物理清理指定完整自然月。
|
||||
func (s *Service) CleanupMonth(ctx context.Context, month time.Time) (result RetentionResult, cleanupErr error) {
|
||||
if s.db == nil || s.store == nil || s.audit == nil {
|
||||
return result, fmt.Errorf("日志留存清理数据库、对象存储或审计 Writer 未配置")
|
||||
}
|
||||
start, end, err := s.retentionMonthRange(month)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
startedAt := time.Now()
|
||||
result.Month = start.Format("2006-01")
|
||||
cleanupErr = s.executeRetention(ctx, start, end, &result)
|
||||
result.Duration = time.Since(startedAt)
|
||||
if auditErr := s.recordRetentionAudit(ctx, start, end, result, cleanupErr); auditErr != nil {
|
||||
if cleanupErr != nil {
|
||||
return result, fmt.Errorf("%w;记录留存清理失败审计失败: %v", cleanupErr, auditErr)
|
||||
}
|
||||
return result, fmt.Errorf("记录留存清理成功审计失败: %w", auditErr)
|
||||
}
|
||||
return result, cleanupErr
|
||||
}
|
||||
|
||||
func (s *Service) retentionMonthRange(month time.Time) (time.Time, time.Time, error) {
|
||||
start := time.Date(month.In(s.location).Year(), month.In(s.location).Month(), 1, 0, 0, 0, 0, s.location)
|
||||
end := start.AddDate(0, 1, 0)
|
||||
now := time.Now().In(s.location)
|
||||
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, s.location)
|
||||
if !end.Before(currentMonth) && !end.Equal(currentMonth) {
|
||||
return time.Time{}, time.Time{}, fmt.Errorf("只能清理已经结束的完整自然月")
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
func (s *Service) executeRetention(ctx context.Context, start, end time.Time, result *RetentionResult) error {
|
||||
started, err := s.retentionCleanupStarted(ctx, start, end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !started {
|
||||
lastDay := end.AddDate(0, 0, -1)
|
||||
if err := s.ArchiveDate(ctx, lastDay); err != nil {
|
||||
return fmt.Errorf("完成上月最后一天 Audit 归档失败: %w", err)
|
||||
}
|
||||
if err := s.ArchiveIntegrationDate(ctx, lastDay); err != nil {
|
||||
return fmt.Errorf("完成上月最后一天 Integration Log 归档失败: %w", err)
|
||||
}
|
||||
if err := s.FinalizeIntegrationMonth(ctx, start); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
runs, err := s.loadRetentionRuns(ctx, start, end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.validateRetentionRuns(ctx, start, end, runs); err != nil {
|
||||
return err
|
||||
}
|
||||
summarizeRetentionRuns(runs, result)
|
||||
if err := s.cleanupAuditMonth(ctx, start, end, runs.audit); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.cleanupIntegrationMonth(ctx, start, end, runs.integration)
|
||||
}
|
||||
|
||||
func (s *Service) retentionCleanupStarted(ctx context.Context, start, end time.Time) (bool, error) {
|
||||
var count int64
|
||||
err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NOT NULL",
|
||||
start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
|
||||
Count(&count).Error
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("读取月度清理断点失败: %w", err)
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func (s *Service) loadRetentionRuns(ctx context.Context, start, end time.Time) (retentionRuns, error) {
|
||||
var rows []model.LogArchiveRun
|
||||
err := s.db.WithContext(ctx).Where(
|
||||
"source IN ? AND archive_date >= ? AND archive_date < ? AND instance_id = ?",
|
||||
[]string{constants.AuditArchiveSource, constants.IntegrationArchiveSource}, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID,
|
||||
).Order("archive_date ASC, source ASC").Find(&rows).Error
|
||||
if err != nil {
|
||||
return retentionRuns{}, fmt.Errorf("读取月度归档账本失败: %w", err)
|
||||
}
|
||||
days := int(end.Sub(start).Hours() / 24)
|
||||
if len(rows) != days*2 {
|
||||
return retentionRuns{}, fmt.Errorf("月度归档账本缺日:期望 %d 条,实际 %d 条", days*2, len(rows))
|
||||
}
|
||||
runs := retentionRuns{audit: make([]*model.LogArchiveRun, 0, days), integration: make([]*model.LogArchiveRun, 0, days)}
|
||||
for index := range rows {
|
||||
run := &rows[index]
|
||||
switch run.Source {
|
||||
case constants.AuditArchiveSource:
|
||||
runs.audit = append(runs.audit, run)
|
||||
case constants.IntegrationArchiveSource:
|
||||
runs.integration = append(runs.integration, run)
|
||||
}
|
||||
}
|
||||
if len(runs.audit) != days || len(runs.integration) != days {
|
||||
return retentionRuns{}, fmt.Errorf("月度 Audit 或 Integration 归档账本不完整")
|
||||
}
|
||||
return runs, nil
|
||||
}
|
||||
|
||||
func (s *Service) validateRetentionRuns(ctx context.Context, start, end time.Time, runs retentionRuns) error {
|
||||
if err := validateCleanupLedgerState(runs.audit); err != nil {
|
||||
return fmt.Errorf("Audit 清理断点非法: %w", err)
|
||||
}
|
||||
if err := validateCleanupLedgerState(runs.integration); err != nil {
|
||||
return fmt.Errorf("Integration 清理断点非法: %w", err)
|
||||
}
|
||||
for index := range runs.audit {
|
||||
date := start.AddDate(0, 0, index)
|
||||
if err := s.validateAuditRetentionDay(ctx, date, runs.audit[index]); err != nil {
|
||||
return fmt.Errorf("%s Audit 清理门禁失败: %w", date.Format(time.DateOnly), err)
|
||||
}
|
||||
if err := s.validateIntegrationRetentionDay(ctx, date, runs.integration[index]); err != nil {
|
||||
return fmt.Errorf("%s Integration 清理门禁失败: %w", date.Format(time.DateOnly), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCleanupLedgerState(runs []*model.LogArchiveRun) error {
|
||||
started, cleaned := 0, 0
|
||||
for _, run := range runs {
|
||||
if run.CleanupStartedAt != nil {
|
||||
started++
|
||||
}
|
||||
if run.CleanedAt != nil {
|
||||
cleaned++
|
||||
}
|
||||
}
|
||||
if started != 0 && started != len(runs) {
|
||||
return fmt.Errorf("清理开始断点不是整月原子状态")
|
||||
}
|
||||
if cleaned != 0 && cleaned != len(runs) {
|
||||
return fmt.Errorf("清理完成断点不是整月原子状态")
|
||||
}
|
||||
if cleaned > 0 && started == 0 {
|
||||
return fmt.Errorf("清理完成但缺少开始断点")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) validateAuditRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
|
||||
if err := validateRunBase(run, date, constants.AuditArchiveSchemaVersion, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.validateAuditManifest(ctx, run); err != nil {
|
||||
return err
|
||||
}
|
||||
events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return validateRemainingCounts(run, events, resources)
|
||||
}
|
||||
|
||||
func (s *Service) validateIntegrationRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error {
|
||||
if err := validateRunBase(run, date, constants.IntegrationArchiveSchemaVersion, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.validateIntegrationManifest(ctx, run); err != nil {
|
||||
return err
|
||||
}
|
||||
count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if run.CleanedAt != nil {
|
||||
if count != 0 {
|
||||
return fmt.Errorf("已标记清理完成但数据库仍有 %d 条记录", count)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if run.CleanupStartedAt != nil {
|
||||
if count > run.RecordCount {
|
||||
return fmt.Errorf("续跑窗口记录数超过最终归档数量")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
file, err := s.buildIntegrationArchiveFile(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.path)
|
||||
if file.recordCount != run.RecordCount || file.sha256 != run.SHA256 {
|
||||
return fmt.Errorf("数据库当前 Integration 内容与最终 revision 不一致")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRunBase(run *model.LogArchiveRun, date time.Time, schema string, final bool) error {
|
||||
if run.Status != constants.ArchiveStatusSuccess || run.SchemaVersion != schema {
|
||||
return fmt.Errorf("归档状态或 schema version 不符合清理要求")
|
||||
}
|
||||
if run.ArchiveDate.Format(time.DateOnly) != date.Format(time.DateOnly) ||
|
||||
!run.RangeStart.Equal(date) || !run.RangeEnd.Equal(date.AddDate(0, 0, 1)) {
|
||||
return fmt.Errorf("归档日期或半开时间范围不一致")
|
||||
}
|
||||
if final && !run.IsFinal {
|
||||
return fmt.Errorf("Integration 最终 revision 尚未形成")
|
||||
}
|
||||
if run.ObjectKey == "" || run.ManifestKey == "" || run.SHA256 == "" {
|
||||
return fmt.Errorf("归档对象、manifest 或 SHA-256 缺失")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRemainingCounts(run *model.LogArchiveRun, events, resources int64) error {
|
||||
if run.CleanedAt != nil {
|
||||
if events != 0 || resources != 0 {
|
||||
return fmt.Errorf("已标记清理完成但数据库仍有事件或资源")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if run.CleanupStartedAt != nil {
|
||||
if events > run.EventCount || resources > run.ResourceCount {
|
||||
return fmt.Errorf("续跑窗口数量超过已归档数量")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if events != run.EventCount || resources != run.ResourceCount {
|
||||
return fmt.Errorf("数据库事件或资源数量与 manifest 不一致")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) validateAuditManifest(ctx context.Context, run *model.LogArchiveRun) error {
|
||||
var manifest Manifest
|
||||
if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess ||
|
||||
manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone ||
|
||||
manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) ||
|
||||
manifest.EventCount != run.EventCount || manifest.ResourceCount != run.ResourceCount ||
|
||||
manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey ||
|
||||
manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision {
|
||||
return fmt.Errorf("Audit manifest 与 ledger 不一致")
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{
|
||||
"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.verifyRetentionObject(ctx, run, false)
|
||||
}
|
||||
|
||||
func (s *Service) validateIntegrationManifest(ctx context.Context, run *model.LogArchiveRun) error {
|
||||
var manifest integrationArchiveManifest
|
||||
if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || !manifest.Final ||
|
||||
manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone ||
|
||||
manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) ||
|
||||
manifest.RecordCount != run.RecordCount || manifest.CompressedBytes != run.CompressedBytes ||
|
||||
manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision {
|
||||
return fmt.Errorf("Integration manifest 与最终 ledger 不一致")
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{
|
||||
"source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256,
|
||||
"revision": strconv.Itoa(run.Revision), "final": "true",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.verifyRetentionObject(ctx, run, true)
|
||||
}
|
||||
|
||||
func (s *Service) readManifest(ctx context.Context, key string, target any) error {
|
||||
object, err := s.store.Stat(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("读取 manifest metadata 失败: %w", err)
|
||||
}
|
||||
reader, err := s.store.Download(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("下载 manifest 失败: %w", err)
|
||||
}
|
||||
data, readErr := io.ReadAll(io.LimitReader(reader, maxManifestBytes+1))
|
||||
closeErr := reader.Close()
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("读取 manifest 失败: %w", readErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("关闭 manifest 对象失败: %w", closeErr)
|
||||
}
|
||||
if len(data) > maxManifestBytes || int64(len(data)) != object.Size {
|
||||
return fmt.Errorf("manifest 大小非法或不完整")
|
||||
}
|
||||
if err := sonic.Unmarshal(data, target); err != nil {
|
||||
return fmt.Errorf("解析 manifest 失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) verifyRetentionObject(ctx context.Context, run *model.LogArchiveRun, final bool) error {
|
||||
metadata := map[string]string{
|
||||
"schema-version": run.SchemaVersion, "source": run.Source,
|
||||
"archive-date": run.RangeStart.Format(time.DateOnly), "timezone": constants.AuditArchiveTimezone,
|
||||
"sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
|
||||
}
|
||||
if run.Source == constants.AuditArchiveSource {
|
||||
metadata["event-count"] = strconv.FormatInt(run.EventCount, 10)
|
||||
metadata["resource-count"] = strconv.FormatInt(run.ResourceCount, 10)
|
||||
} else {
|
||||
metadata["record-count"] = strconv.FormatInt(run.RecordCount, 10)
|
||||
metadata["final"] = strconv.FormatBool(final)
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
reader, err := s.store.Download(ctx, run.ObjectKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("下载归档对象复核 SHA-256 失败: %w", err)
|
||||
}
|
||||
hasher := sha256.New()
|
||||
written, copyErr := io.Copy(hasher, reader)
|
||||
closeErr := reader.Close()
|
||||
if copyErr != nil {
|
||||
return fmt.Errorf("读取归档对象复核 SHA-256 失败: %w", copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("关闭归档对象失败: %w", closeErr)
|
||||
}
|
||||
if written != run.CompressedBytes || fmt.Sprintf("%x", hasher.Sum(nil)) != run.SHA256 {
|
||||
return fmt.Errorf("归档对象大小或 SHA-256 复核失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func summarizeRetentionRuns(runs retentionRuns, result *RetentionResult) {
|
||||
result.ManifestKeys = make([]string, 0, len(runs.audit)+len(runs.integration))
|
||||
for _, run := range runs.audit {
|
||||
result.EventCount += run.EventCount
|
||||
result.ResourceCount += run.ResourceCount
|
||||
result.ManifestKeys = append(result.ManifestKeys, run.ManifestKey)
|
||||
}
|
||||
for _, run := range runs.integration {
|
||||
result.IntegrationCount += run.RecordCount
|
||||
result.ManifestKeys = append(result.ManifestKeys, run.ManifestKey)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) cleanupAuditMonth(ctx context.Context, start, end time.Time, runs []*model.LogArchiveRun) error {
|
||||
if allRunsCleaned(runs) {
|
||||
return nil
|
||||
}
|
||||
if err := s.markCleanupStarted(ctx, constants.AuditArchiveSource, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.deleteAuditResources(ctx, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.deleteAuditEvents(ctx, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.markCleaned(ctx, constants.AuditArchiveSource, start, end)
|
||||
}
|
||||
|
||||
func (s *Service) cleanupIntegrationMonth(ctx context.Context, start, end time.Time, runs []*model.LogArchiveRun) error {
|
||||
if allRunsCleaned(runs) {
|
||||
return nil
|
||||
}
|
||||
if err := s.markCleanupStarted(ctx, constants.IntegrationArchiveSource, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
subquery := s.db.Model(&model.IntegrationLog{}).Select("id").
|
||||
Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
|
||||
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.IntegrationLog{})
|
||||
if deleted.Error != nil {
|
||||
return fmt.Errorf("分批物理删除 Integration Log 失败: %w", deleted.Error)
|
||||
}
|
||||
if deleted.RowsAffected == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return s.markCleaned(ctx, constants.IntegrationArchiveSource, start, end)
|
||||
}
|
||||
|
||||
func (s *Service) deleteAuditResources(ctx context.Context, start, end time.Time) error {
|
||||
for {
|
||||
subquery := s.db.Model(&model.AuditEventResource{}).Select("tb_audit_event_resource.id").
|
||||
Joins("JOIN tb_audit_event ON tb_audit_event.id = tb_audit_event_resource.audit_event_id").
|
||||
Where("tb_audit_event.created_at >= ? AND tb_audit_event.created_at < ?", start, end).
|
||||
Order("tb_audit_event_resource.id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
|
||||
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEventResource{})
|
||||
if deleted.Error != nil {
|
||||
return fmt.Errorf("分批物理删除 Audit Event Resource 失败: %w", deleted.Error)
|
||||
}
|
||||
if deleted.RowsAffected == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) deleteAuditEvents(ctx context.Context, start, end time.Time) error {
|
||||
for {
|
||||
subquery := s.db.Model(&model.AuditEvent{}).Select("id").
|
||||
Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize)
|
||||
deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEvent{})
|
||||
if deleted.Error != nil {
|
||||
return fmt.Errorf("分批物理删除 Audit Event 失败: %w", deleted.Error)
|
||||
}
|
||||
if deleted.RowsAffected == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) markCleanupStarted(ctx context.Context, source string, start, end time.Time) error {
|
||||
now := time.Now()
|
||||
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NULL",
|
||||
source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
|
||||
Updates(map[string]any{"cleanup_started_at": now, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("记录月度清理开始断点失败: %w", result.Error)
|
||||
}
|
||||
return s.validateCleanupMarkerCount(ctx, source, start, end, "cleanup_started_at IS NOT NULL", "开始")
|
||||
}
|
||||
|
||||
func (s *Service) markCleaned(ctx context.Context, source string, start, end time.Time) error {
|
||||
now := time.Now()
|
||||
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NOT NULL",
|
||||
source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
|
||||
Updates(map[string]any{"cleaned_at": now, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("记录月度清理完成断点失败: %w", result.Error)
|
||||
}
|
||||
return s.validateCleanupMarkerCount(ctx, source, start, end, "cleaned_at IS NOT NULL", "完成")
|
||||
}
|
||||
|
||||
func (s *Service) validateCleanupMarkerCount(ctx context.Context, source string, start, end time.Time, marker, label string) error {
|
||||
var count int64
|
||||
err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND "+marker,
|
||||
source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID).
|
||||
Count(&count).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("复核月度清理%s断点失败: %w", label, err)
|
||||
}
|
||||
expected := int64(end.Sub(start).Hours() / 24)
|
||||
if count != expected {
|
||||
return fmt.Errorf("月度清理%s断点不完整:期望 %d 条,实际 %d 条", label, expected, count)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func allRunsCleaned(runs []*model.LogArchiveRun) bool {
|
||||
return len(runs) > 0 && runs[0].CleanedAt != nil
|
||||
}
|
||||
|
||||
func (s *Service) recordRetentionAudit(ctx context.Context, start, end time.Time, result RetentionResult, cleanupErr error) error {
|
||||
audit := RetentionAudit{
|
||||
Month: result.Month, RangeStart: start, RangeEnd: end,
|
||||
EventCount: result.EventCount, ResourceCount: result.ResourceCount,
|
||||
IntegrationCount: result.IntegrationCount, ManifestKeys: result.ManifestKeys,
|
||||
DurationMS: result.Duration.Milliseconds(), Result: constants.AuditResultSuccess,
|
||||
Summary: "完成已归档在线日志月度物理清理",
|
||||
EventID: "evt_retention_" + strings.ReplaceAll(result.Month, "-", "_"),
|
||||
}
|
||||
if cleanupErr != nil {
|
||||
audit.EventID = ""
|
||||
audit.Result = constants.AuditResultFailed
|
||||
audit.Summary = "已归档在线日志月度物理清理失败"
|
||||
audit.ErrorSummary = truncateRetentionError(cleanupErr)
|
||||
}
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
return s.audit.WriteRetentionCleanup(ctx, tx, audit)
|
||||
})
|
||||
}
|
||||
|
||||
func truncateRetentionError(err error) string {
|
||||
value := []rune(err.Error())
|
||||
if len(value) > 500 {
|
||||
value = value[:500]
|
||||
}
|
||||
return string(value)
|
||||
}
|
||||
412
internal/application/auditarchive/service.go
Normal file
412
internal/application/auditarchive/service.go
Normal file
@@ -0,0 +1,412 @@
|
||||
// Package auditarchive 实现统一审计每日冷归档用例。
|
||||
package auditarchive
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/storage"
|
||||
)
|
||||
|
||||
const archivePageSize = 1000
|
||||
|
||||
// ObjectStore 是每日归档需要的最小对象存储能力。
|
||||
type ObjectStore interface {
|
||||
UploadWithMetadata(context.Context, string, io.Reader, string, map[string]string) error
|
||||
Stat(context.Context, string) (*storage.ObjectMetadata, error)
|
||||
Download(context.Context, string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// RetentionAuditWriter 记录月度留存清理的系统审计事实。
|
||||
type RetentionAuditWriter interface {
|
||||
WriteRetentionCleanup(context.Context, *gorm.DB, RetentionAudit) error
|
||||
}
|
||||
|
||||
// Service 编排审计归档生成、上传、复核和幂等账本更新。
|
||||
type Service struct {
|
||||
db *gorm.DB
|
||||
store ObjectStore
|
||||
audit RetentionAuditWriter
|
||||
instanceID string
|
||||
location *time.Location
|
||||
}
|
||||
|
||||
// Manifest 是归档对象的完整性清单。
|
||||
type Manifest struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
Source string `json:"source"`
|
||||
ArchiveDate string `json:"archive_date"`
|
||||
Timezone string `json:"timezone"`
|
||||
RangeStart time.Time `json:"range_start"`
|
||||
RangeEnd time.Time `json:"range_end"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
EventCount int64 `json:"event_count"`
|
||||
ResourceCount int64 `json:"resource_count"`
|
||||
UncompressedBytes int64 `json:"uncompressed_bytes"`
|
||||
CompressedBytes int64 `json:"compressed_bytes"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Revision int `json:"revision"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type archiveLine struct {
|
||||
Event model.AuditEvent `json:"event"`
|
||||
Resources []model.AuditEventResource `json:"resources"`
|
||||
}
|
||||
|
||||
type archiveFile struct {
|
||||
path string
|
||||
eventCount int64
|
||||
resourceCount int64
|
||||
uncompressedBytes int64
|
||||
compressedBytes int64
|
||||
sha256 string
|
||||
}
|
||||
|
||||
// NewService 创建统一审计每日冷归档服务。
|
||||
func NewService(db *gorm.DB, store ObjectStore, instanceID string, audit ...RetentionAuditWriter) (*Service, error) {
|
||||
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("加载审计归档时区失败: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(instanceID) == "" {
|
||||
instanceID = "audit-archive"
|
||||
}
|
||||
var auditWriter RetentionAuditWriter
|
||||
if len(audit) > 0 {
|
||||
auditWriter = audit[0]
|
||||
}
|
||||
return &Service{db: db, store: store, audit: auditWriter, instanceID: instanceID, location: location}, nil
|
||||
}
|
||||
|
||||
// ArchivePreviousDay 归档 Asia/Shanghai 前一完整自然日。
|
||||
func (s *Service) ArchivePreviousDay(ctx context.Context) error {
|
||||
now := time.Now().In(s.location)
|
||||
return s.ArchiveDate(ctx, now.AddDate(0, 0, -1))
|
||||
}
|
||||
|
||||
// ArchiveDate 归档指定 Asia/Shanghai 自然日。
|
||||
func (s *Service) ArchiveDate(ctx context.Context, archiveDate time.Time) error {
|
||||
if s.db == nil || s.store == nil {
|
||||
return fmt.Errorf("审计归档数据库或对象存储未配置")
|
||||
}
|
||||
start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location)
|
||||
end := start.AddDate(0, 0, 1)
|
||||
today := time.Now().In(s.location)
|
||||
todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)
|
||||
if end.After(todayStart) {
|
||||
return fmt.Errorf("统一审计只能归档已经结束的完整自然日")
|
||||
}
|
||||
run, err := s.ensureRun(ctx, start, end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if run.Status == constants.ArchiveStatusSuccess {
|
||||
valid, validateErr := s.validateSuccessfulRun(ctx, run)
|
||||
if validateErr == nil && valid {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
acquired, err := s.acquireRun(ctx, run)
|
||||
if err != nil || !acquired {
|
||||
return err
|
||||
}
|
||||
if err := s.execute(ctx, run); err != nil {
|
||||
s.markFailed(ctx, run.ID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) ensureRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) {
|
||||
run := model.LogArchiveRun{
|
||||
Source: constants.AuditArchiveSource, ArchiveDate: start, InstanceID: s.instanceID,
|
||||
SchemaVersion: constants.AuditArchiveSchemaVersion, Revision: 1,
|
||||
Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end,
|
||||
}
|
||||
result := s.db.WithContext(ctx).Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}},
|
||||
DoNothing: true,
|
||||
}).Create(&run)
|
||||
if result.Error != nil {
|
||||
return nil, fmt.Errorf("创建审计归档账本失败: %w", result.Error)
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
if err := s.db.WithContext(ctx).Where(
|
||||
"source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?",
|
||||
constants.AuditArchiveSource, start, s.instanceID, constants.AuditArchiveSchemaVersion,
|
||||
).First(&run).Error; err != nil {
|
||||
return nil, fmt.Errorf("读取审计归档账本失败: %w", err)
|
||||
}
|
||||
}
|
||||
return &run, nil
|
||||
}
|
||||
|
||||
func (s *Service) acquireRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
|
||||
revision := run.Revision
|
||||
if run.Status == constants.ArchiveStatusFailed || run.Status == constants.ArchiveStatusSuccess || run.Status == constants.ArchiveStatusRunning {
|
||||
revision++
|
||||
}
|
||||
now := time.Now()
|
||||
staleBefore := now.Add(-3 * time.Hour)
|
||||
result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, staleBefore).
|
||||
Updates(map[string]any{
|
||||
"status": constants.ArchiveStatusRunning, "revision": revision,
|
||||
"attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "",
|
||||
"completed_at": nil, "updated_at": now,
|
||||
})
|
||||
if result.Error != nil {
|
||||
return false, fmt.Errorf("锁定审计归档任务失败: %w", result.Error)
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return false, nil
|
||||
}
|
||||
run.Revision = revision
|
||||
run.Status = constants.ArchiveStatusRunning
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Service) execute(ctx context.Context, run *model.LogArchiveRun) error {
|
||||
file, err := s.buildArchiveFile(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.path)
|
||||
|
||||
dbEvents, dbResources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dbEvents != file.eventCount || dbResources != file.resourceCount {
|
||||
return fmt.Errorf("审计归档生成期间数据数量发生变化")
|
||||
}
|
||||
|
||||
objectKey, manifestKey := objectKeys(run.RangeStart, run.Revision)
|
||||
metadata := archiveMetadata(file, run)
|
||||
reader, err := os.Open(file.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("打开审计归档临时文件失败: %w", err)
|
||||
}
|
||||
uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata)
|
||||
closeErr := reader.Close()
|
||||
if uploadErr != nil {
|
||||
return fmt.Errorf("上传审计归档对象失败: %w", uploadErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("关闭审计归档临时文件失败: %w", closeErr)
|
||||
}
|
||||
if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
generatedAt := time.Now().In(s.location)
|
||||
manifest := Manifest{
|
||||
SchemaVersion: constants.AuditArchiveSchemaVersion, Source: constants.AuditArchiveSource,
|
||||
ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone,
|
||||
RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID,
|
||||
EventCount: file.eventCount, ResourceCount: file.resourceCount,
|
||||
UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes,
|
||||
ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision,
|
||||
GeneratedAt: generatedAt, Status: constants.ArchiveStatusSuccess,
|
||||
}
|
||||
manifestBytes, err := sonic.Marshal(manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("序列化审计归档清单失败: %w", err)
|
||||
}
|
||||
manifestMetadata := map[string]string{"source": constants.AuditArchiveSource, "data-sha256": file.sha256, "revision": strconv.Itoa(run.Revision)}
|
||||
if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil {
|
||||
return fmt.Errorf("上传审计归档清单失败: %w", err)
|
||||
}
|
||||
if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
completedAt := time.Now()
|
||||
updates := map[string]any{
|
||||
"status": constants.ArchiveStatusSuccess, "object_key": objectKey, "manifest_key": manifestKey,
|
||||
"event_count": file.eventCount, "resource_count": file.resourceCount,
|
||||
"uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes,
|
||||
"sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt,
|
||||
"updated_at": completedAt,
|
||||
}
|
||||
if err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(updates).Error; err != nil {
|
||||
return fmt.Errorf("更新审计归档成功账本失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) buildArchiveFile(ctx context.Context, start, end time.Time) (*archiveFile, error) {
|
||||
temp, err := os.CreateTemp("", "audit-events-*.jsonl.gz")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建审计归档临时文件失败: %w", err)
|
||||
}
|
||||
path := temp.Name()
|
||||
failed := true
|
||||
defer func() {
|
||||
_ = temp.Close()
|
||||
if failed {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher))
|
||||
result := &archiveFile{path: path}
|
||||
var lastID uint
|
||||
for {
|
||||
var events []model.AuditEvent
|
||||
if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID).
|
||||
Order("id ASC").Limit(archivePageSize).Find(&events).Error; err != nil {
|
||||
return nil, fmt.Errorf("读取审计归档事件失败: %w", err)
|
||||
}
|
||||
if len(events) == 0 {
|
||||
break
|
||||
}
|
||||
ids := make([]uint, 0, len(events))
|
||||
for i := range events {
|
||||
ids = append(ids, events[i].ID)
|
||||
}
|
||||
var resources []model.AuditEventResource
|
||||
if err := s.db.WithContext(ctx).Where("audit_event_id IN ?", ids).
|
||||
Order("audit_event_id ASC, sort_order ASC, id ASC").Find(&resources).Error; err != nil {
|
||||
return nil, fmt.Errorf("读取审计归档资源失败: %w", err)
|
||||
}
|
||||
grouped := make(map[uint][]model.AuditEventResource, len(events))
|
||||
for i := range resources {
|
||||
resource := resources[i]
|
||||
grouped[resource.AuditEventID] = append(grouped[resource.AuditEventID], resource)
|
||||
}
|
||||
for i := range events {
|
||||
eventResources := grouped[events[i].ID]
|
||||
if eventResources == nil {
|
||||
eventResources = []model.AuditEventResource{}
|
||||
}
|
||||
line, marshalErr := sonic.Marshal(archiveLine{Event: events[i], Resources: eventResources})
|
||||
if marshalErr != nil {
|
||||
return nil, fmt.Errorf("序列化审计归档事件失败: %w", marshalErr)
|
||||
}
|
||||
line = append(line, '\n')
|
||||
if _, writeErr := gzipWriter.Write(line); writeErr != nil {
|
||||
return nil, fmt.Errorf("写入审计归档压缩流失败: %w", writeErr)
|
||||
}
|
||||
result.eventCount++
|
||||
result.resourceCount += int64(len(grouped[events[i].ID]))
|
||||
result.uncompressedBytes += int64(len(line))
|
||||
}
|
||||
lastID = events[len(events)-1].ID
|
||||
}
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
return nil, fmt.Errorf("关闭审计归档压缩流失败: %w", err)
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return nil, fmt.Errorf("关闭审计归档临时文件失败: %w", err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取审计归档临时文件信息失败: %w", err)
|
||||
}
|
||||
result.compressedBytes = info.Size()
|
||||
result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
failed = false
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) databaseCounts(ctx context.Context, start, end time.Time) (int64, int64, error) {
|
||||
var eventCount int64
|
||||
if err := s.db.WithContext(ctx).Model(&model.AuditEvent{}).
|
||||
Where("created_at >= ? AND created_at < ?", start, end).Count(&eventCount).Error; err != nil {
|
||||
return 0, 0, fmt.Errorf("统计审计归档事件失败: %w", err)
|
||||
}
|
||||
var resourceCount int64
|
||||
subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end)
|
||||
if err := s.db.WithContext(ctx).Model(&model.AuditEventResource{}).
|
||||
Where("audit_event_id IN (?)", subquery).Count(&resourceCount).Error; err != nil {
|
||||
return 0, 0, fmt.Errorf("统计审计归档资源失败: %w", err)
|
||||
}
|
||||
return eventCount, resourceCount, nil
|
||||
}
|
||||
|
||||
func (s *Service) validateSuccessfulRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) {
|
||||
events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd)
|
||||
if err != nil || events != run.EventCount || resources != run.ResourceCount {
|
||||
return false, err
|
||||
}
|
||||
metadata := map[string]string{
|
||||
"sha256": run.SHA256, "event-count": strconv.FormatInt(run.EventCount, 10),
|
||||
"resource-count": strconv.FormatInt(run.ResourceCount, 10), "revision": strconv.Itoa(run.Revision),
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
manifestMetadata := map[string]string{
|
||||
"source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision),
|
||||
}
|
||||
if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Service) verifyObject(ctx context.Context, key string, expectedSize int64, expectedMetadata map[string]string) error {
|
||||
object, err := s.store.Stat(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("复核归档对象 metadata 失败: %w", err)
|
||||
}
|
||||
if expectedSize >= 0 && object.Size != expectedSize {
|
||||
return fmt.Errorf("归档对象大小复核失败")
|
||||
}
|
||||
for name, value := range expectedMetadata {
|
||||
if object.Metadata[strings.ToLower(name)] != value {
|
||||
return fmt.Errorf("归档对象 metadata 字段 %s 复核失败", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) markFailed(ctx context.Context, runID uint, archiveErr error) {
|
||||
summary := []rune(archiveErr.Error())
|
||||
if len(summary) > 500 {
|
||||
summary = summary[:500]
|
||||
}
|
||||
failedCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = s.db.WithContext(failedCtx).Model(&model.LogArchiveRun{}).Where("id = ?", runID).Updates(map[string]any{
|
||||
"status": constants.ArchiveStatusFailed, "error_summary": string(summary), "updated_at": time.Now(),
|
||||
}).Error
|
||||
}
|
||||
|
||||
func objectKeys(date time.Time, revision int) (string, string) {
|
||||
prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day())
|
||||
name := fmt.Sprintf("audit-events-%s-r%d", date.Format(time.DateOnly), revision)
|
||||
return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json"
|
||||
}
|
||||
|
||||
func archiveMetadata(file *archiveFile, run *model.LogArchiveRun) map[string]string {
|
||||
return map[string]string{
|
||||
"schema-version": constants.AuditArchiveSchemaVersion,
|
||||
"source": constants.AuditArchiveSource,
|
||||
"archive-date": run.RangeStart.Format(time.DateOnly),
|
||||
"timezone": constants.AuditArchiveTimezone,
|
||||
"event-count": strconv.FormatInt(file.eventCount, 10),
|
||||
"resource-count": strconv.FormatInt(file.resourceCount, 10),
|
||||
"sha256": file.sha256,
|
||||
"revision": strconv.Itoa(run.Revision),
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func (s *Service) RecordCarrierCallbackFailure(ctx context.Context, card *model.
|
||||
}
|
||||
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardRealnameCallbackSynced,
|
||||
Summary: "运营商回调同步 IoT 卡实名状态失败", Card: card, IntegrationID: integrationID,
|
||||
Summary: "运营商回调同步 IoT 卡实名状态失败", Card: card, IntegrationID: integrationID,
|
||||
}, businessErr)
|
||||
}
|
||||
|
||||
@@ -94,6 +94,7 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
|
||||
return domain.RealnameDecision{}, errors.New(errors.CodeInternalError, "卡实名观测能力未完整配置")
|
||||
}
|
||||
var decision domain.RealnameDecision
|
||||
var auditedCard *model.IotCard
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var card model.IotCard
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
|
||||
@@ -102,6 +103,7 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
|
||||
}
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡失败")
|
||||
}
|
||||
auditedCard = &card
|
||||
nextDecision, decisionErr := domain.ApplyRealname(domain.CardRealnameSnapshot{
|
||||
CardID: card.ID, Status: card.RealNameStatus, FirstRealnameAt: card.FirstRealnameAt,
|
||||
ReversalCount: card.RealnameReversalCount, ReversalStartedAt: card.RealnameReversalStartedAt,
|
||||
@@ -185,11 +187,26 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
|
||||
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
|
||||
}
|
||||
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardRealnameCallbackSynced,
|
||||
Summary: "运营商回调同步 IoT 卡实名状态",
|
||||
Card: &card,
|
||||
ActionCode: constants.AuditActionIotCardRealnameCallbackSynced,
|
||||
Summary: "运营商回调同步 IoT 卡实名状态",
|
||||
Card: &card,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt},
|
||||
BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt},
|
||||
AfterData: map[string]any{
|
||||
"real_name_status": decision.AfterStatus, "first_realname_at": firstRealnameAfter(card.FirstRealnameAt, observation.Metadata.ObservedAt, decision.FirstVerified),
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if workerObservationAudited(ctx) && decision.StatusChanged {
|
||||
if s.auditWriter == nil {
|
||||
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
|
||||
}
|
||||
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardWorkerRealnameSynced,
|
||||
Summary: "Worker 同步 IoT 卡实名事实", Card: &card,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt},
|
||||
AfterData: map[string]any{
|
||||
"real_name_status": decision.AfterStatus, "first_realname_at": firstRealnameAfter(card.FirstRealnameAt, observation.Metadata.ObservedAt, decision.FirstVerified),
|
||||
},
|
||||
@@ -200,6 +217,13 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
|
||||
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardWorkerRealnameSynced,
|
||||
Summary: "Worker 同步 IoT 卡实名事实失败", Card: auditedCard,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
}, err)
|
||||
}
|
||||
return domain.RealnameDecision{}, err
|
||||
}
|
||||
if s.cache != nil {
|
||||
@@ -226,6 +250,11 @@ func manualRefreshAuditAction(ctx context.Context) (string, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func workerObservationAudited(ctx context.Context) bool {
|
||||
linkage := auditcontext.From(ctx)
|
||||
return linkage.ActorKind == constants.AuditActorSystemTask && linkage.Source == constants.AuditSourceWorker
|
||||
}
|
||||
|
||||
func realnameChangedEventID(cardID uint, observationID string) string {
|
||||
prefix := "card-realname:"
|
||||
digest := sha256.Sum256([]byte(strconv.FormatUint(uint64(cardID), 10) + ":" + observationID + ":changed"))
|
||||
|
||||
@@ -33,6 +33,7 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai
|
||||
return domain.NetworkDecision{}, errors.New(errors.CodeInternalError, "卡网络观测能力未完整配置")
|
||||
}
|
||||
var decision domain.NetworkDecision
|
||||
var auditedCard *model.IotCard
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var card model.IotCard
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
|
||||
@@ -41,6 +42,7 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai
|
||||
}
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡网络事实失败")
|
||||
}
|
||||
auditedCard = &card
|
||||
nextDecision, decisionErr := domain.ApplyNetwork(domain.CardNetworkSnapshot{
|
||||
CardID: card.ID, NetworkStatus: card.NetworkStatus, StopReason: card.StopReason,
|
||||
IsStandalone: card.IsStandalone, EnablePolling: card.EnablePolling,
|
||||
@@ -117,10 +119,46 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if workerObservationAudited(ctx) && stateChanged {
|
||||
if s.auditWriter == nil {
|
||||
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
|
||||
}
|
||||
enablePolling := card.EnablePolling
|
||||
if decision.StopPolling {
|
||||
enablePolling = false
|
||||
}
|
||||
gatewayIMEI := card.GatewayCardIMEI
|
||||
if decision.UpdateIMEI {
|
||||
gatewayIMEI = decision.GatewayIMEI
|
||||
}
|
||||
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardWorkerNetworkSynced,
|
||||
Summary: "Worker 同步 IoT 卡网络事实", Card: &card,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
BeforeData: map[string]any{
|
||||
"network_status": card.NetworkStatus, "stop_reason": card.StopReason,
|
||||
"gateway_extend": card.GatewayExtend, "gateway_card_imei": card.GatewayCardIMEI,
|
||||
"enable_polling": card.EnablePolling,
|
||||
},
|
||||
AfterData: map[string]any{
|
||||
"network_status": decision.AfterStatus, "stop_reason": decision.StopReason,
|
||||
"gateway_extend": decision.GatewayExtend, "gateway_card_imei": gatewayIMEI,
|
||||
"enable_polling": enablePolling,
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
|
||||
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardWorkerNetworkSynced,
|
||||
Summary: "Worker 同步 IoT 卡网络事实失败", Card: auditedCard,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
}, err)
|
||||
}
|
||||
return domain.NetworkDecision{}, err
|
||||
}
|
||||
if s.cache != nil {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
@@ -21,6 +22,7 @@ type SeriesRequest struct {
|
||||
Source string `json:"source"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
ParentEventID string `json:"parent_event_id,omitempty"`
|
||||
}
|
||||
|
||||
// DeviceCardsSeriesRequest 描述需要在后台展开设备有效绑定卡的观测请求。
|
||||
@@ -59,6 +61,7 @@ type SeriesTaskPayload struct {
|
||||
Source string `json:"source"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
ParentEventID string `json:"parent_event_id,omitempty"`
|
||||
}
|
||||
|
||||
// RunResult 描述一次实际 Gateway 请求及公共观测应用结果。
|
||||
@@ -135,7 +138,7 @@ func (s *SeriesTrigger) trigger(ctx context.Context, request SeriesRequest, cand
|
||||
if s == nil || s.coordinator == nil || s.scheduler == nil || s.logger == nil {
|
||||
return "", false, errors.New(errors.CodeInternalError, "卡观测序列触发能力未完整配置")
|
||||
}
|
||||
request = normalizeSeriesTrace(request)
|
||||
request = normalizeSeriesTrace(ctx, request)
|
||||
if err := validateSeriesRequest(request); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
@@ -171,6 +174,7 @@ func (s *SeriesTrigger) trigger(ctx context.Context, request SeriesRequest, cand
|
||||
Scene: originalRequest.Scene, ResourceType: originalRequest.ResourceType, ResourceID: originalRequest.ResourceID,
|
||||
SyncType: originalRequest.SyncType, ExpectedValue: originalRequest.ExpectedValue, Source: originalRequest.Source,
|
||||
RequestID: originalRequest.RequestID, CorrelationID: originalRequest.CorrelationID,
|
||||
ParentEventID: originalRequest.ParentEventID,
|
||||
}
|
||||
if err := s.scheduler.Enqueue(ctx, payload); err != nil {
|
||||
s.coordinator.ReleaseSchedule(ctx, seriesID, attempt)
|
||||
@@ -303,12 +307,24 @@ func validateSeriesPayload(payload SeriesTaskPayload) error {
|
||||
return validateSeriesRequest(SeriesRequest{
|
||||
Scene: payload.Scene, ResourceType: payload.ResourceType, ResourceID: payload.ResourceID,
|
||||
SyncType: payload.SyncType, Source: payload.Source, RequestID: payload.RequestID, CorrelationID: payload.CorrelationID,
|
||||
ParentEventID: payload.ParentEventID,
|
||||
})
|
||||
}
|
||||
|
||||
func normalizeSeriesTrace(request SeriesRequest) SeriesRequest {
|
||||
func normalizeSeriesTrace(ctx context.Context, request SeriesRequest) SeriesRequest {
|
||||
linkage := auditcontext.From(ctx)
|
||||
request.RequestID = strings.TrimSpace(request.RequestID)
|
||||
request.CorrelationID = strings.TrimSpace(request.CorrelationID)
|
||||
request.ParentEventID = strings.TrimSpace(request.ParentEventID)
|
||||
if request.RequestID == "" {
|
||||
request.RequestID = strings.TrimSpace(linkage.RequestID)
|
||||
}
|
||||
if request.CorrelationID == "" {
|
||||
request.CorrelationID = strings.TrimSpace(linkage.CorrelationID)
|
||||
}
|
||||
if request.ParentEventID == "" {
|
||||
request.ParentEventID = strings.TrimSpace(linkage.ParentEventID)
|
||||
}
|
||||
if request.RequestID == "" && request.CorrelationID == "" {
|
||||
traceID := uuid.NewString()
|
||||
request.RequestID = traceID
|
||||
|
||||
@@ -32,6 +32,7 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
|
||||
return domain.TrafficDecision{}, errors.New(errors.CodeInternalError, "卡流量观测能力未完整配置")
|
||||
}
|
||||
var decision domain.TrafficDecision
|
||||
var auditedCard *model.IotCard
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var card model.IotCard
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil {
|
||||
@@ -40,6 +41,7 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
|
||||
}
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡流量事实失败")
|
||||
}
|
||||
auditedCard = &card
|
||||
nextDecision, decisionErr := domain.ApplyTraffic(domain.CardTrafficSnapshot{
|
||||
CardID: card.ID, DataUsageMB: card.DataUsageMB, CurrentMonthUsageMB: card.CurrentMonthUsageMB,
|
||||
CurrentMonthStartDate: card.CurrentMonthStartDate, LastMonthTotalMB: card.LastMonthTotalMB,
|
||||
@@ -99,10 +101,38 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if workerObservationAudited(ctx) && stateChanged {
|
||||
if s.auditWriter == nil {
|
||||
return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置")
|
||||
}
|
||||
if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
|
||||
Summary: "Worker 同步 IoT 卡流量事实", Card: &card,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
BeforeData: map[string]any{
|
||||
"data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB,
|
||||
"current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB,
|
||||
"last_gateway_reading_mb": card.LastGatewayReadingMB,
|
||||
},
|
||||
AfterData: map[string]any{
|
||||
"data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB,
|
||||
"current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB,
|
||||
"last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB,
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil {
|
||||
s.auditWriter.WriteCardStateFailure(ctx, StateAudit{
|
||||
ActionCode: constants.AuditActionIotCardWorkerTrafficSynced,
|
||||
Summary: "Worker 同步 IoT 卡流量事实失败", Card: auditedCard,
|
||||
IntegrationID: observation.Metadata.ObservationID,
|
||||
}, err)
|
||||
}
|
||||
return domain.TrafficDecision{}, err
|
||||
}
|
||||
if s.cache != nil {
|
||||
|
||||
@@ -191,7 +191,7 @@ func initServices(s *stores, deps *Dependencies) *services {
|
||||
)
|
||||
cardObservationSeries := cardObservationApp.NewSeriesAttemptService(
|
||||
seriesCoordinator,
|
||||
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration),
|
||||
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration, auditWriter),
|
||||
cardObservationInfra.NewSeriesAttemptLogger(seriesIntegration),
|
||||
)
|
||||
observationSeries := cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier)
|
||||
|
||||
@@ -102,11 +102,18 @@ func initWorkerServices(stores *queue.WorkerStores, deps *WorkerDependencies) *q
|
||||
cardObservationInfra.NewEventWriter(cardObservationOutbox),
|
||||
cardObservationInfra.NewCacheInvalidator(deps.Redis, deps.Logger),
|
||||
)
|
||||
iotCardAuditService := iotCardSvc.New(
|
||||
deps.DB, stores.IotCard, stores.Shop, stores.AssetAllocationRecord,
|
||||
stores.ShopPackageAllocation, stores.ShopSeriesAllocation, stores.PackageSeries,
|
||||
deps.GatewayClient, deps.Logger, assetAudit,
|
||||
)
|
||||
iotCardAuditService.SetAccessAudit(auditWriter)
|
||||
cardObservationService.SetStateAuditWriter(iotCardAuditService)
|
||||
cardObservationIntegration := integrationlog.NewRepository(deps.DB)
|
||||
cardObservationSeriesCoordinator := cardObservationInfra.NewSeriesCoordinator(deps.Redis)
|
||||
cardObservationSeriesService := cardObservationApp.NewSeriesAttemptService(
|
||||
cardObservationSeriesCoordinator,
|
||||
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, cardObservationIntegration),
|
||||
cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, cardObservationIntegration, auditWriter),
|
||||
cardObservationInfra.NewSeriesAttemptLogger(cardObservationIntegration),
|
||||
)
|
||||
|
||||
|
||||
@@ -121,15 +121,115 @@ func (h *AuditHandler) ResourceTimeline(c *fiber.Ctx) error {
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// RequestTimeline 查询指定 HTTP 请求关联的跨事实时间线。
|
||||
// GET /api/admin/audit/requests/:request_id/timeline
|
||||
func (h *AuditHandler) RequestTimeline(c *fiber.Ctx) error {
|
||||
requestID := c.Params("request_id")
|
||||
if requestID == "" {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
result, err := h.auditQuery.RequestTimeline(c.UserContext(), requestID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// CorrelationTimeline 查询跨请求业务关联时间线。
|
||||
// GET /api/admin/audit/correlations/:correlation_id/timeline
|
||||
func (h *AuditHandler) CorrelationTimeline(c *fiber.Ctx) error {
|
||||
correlationID := c.Params("correlation_id")
|
||||
if correlationID == "" {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
result, err := h.auditQuery.CorrelationTimeline(c.UserContext(), correlationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// FinanceTimeline 查询资金审计与业务账本的组合时间线。
|
||||
// GET /api/admin/audit/finance/timeline
|
||||
func (h *AuditHandler) FinanceTimeline(c *fiber.Ctx) error {
|
||||
var request dto.AuditFinanceTimelineRequest
|
||||
if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo)
|
||||
if err != nil {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
result, err := h.auditQuery.FinanceTimeline(c.UserContext(), auditquery.FinanceFilter{
|
||||
ShopID: request.ShopID, WalletID: request.WalletID, OrderID: request.OrderID, OrderNo: request.OrderNo,
|
||||
PaymentID: request.PaymentID, PaymentNo: request.PaymentNo, RefundID: request.RefundID, RefundNo: request.RefundNo,
|
||||
RechargeID: request.RechargeID, RechargeNo: request.RechargeNo, ApprovalInstanceID: request.ApprovalInstanceID,
|
||||
ThirdPartyTradeNo: request.ThirdPartyTradeNo, ActorKind: request.ActorKind, ActorID: request.ActorID,
|
||||
CorrelationID: request.CorrelationID, CreatedFrom: from, CreatedTo: to, Page: request.Page, PageSize: request.PageSize,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// RiskOverview 查询固定风险信号总览。
|
||||
// GET /api/admin/audit/risks/overview
|
||||
func (h *AuditHandler) RiskOverview(c *fiber.Ctx) error {
|
||||
var request dto.AuditRiskOverviewRequest
|
||||
if err := c.QueryParser(&request); err != nil {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
filter, err := riskFilter(request.AuditRiskFilterRequest, 0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.auditQuery.RiskOverview(c.UserContext(), filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
// RiskEvents 查询固定风险集合的事件明细。
|
||||
// GET /api/admin/audit/risks/events
|
||||
func (h *AuditHandler) RiskEvents(c *fiber.Ctx) error {
|
||||
var request dto.AuditRiskEventsRequest
|
||||
if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) {
|
||||
return errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
filter, err := riskFilter(request.AuditRiskFilterRequest, request.Page, request.PageSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.auditQuery.RiskEvents(c.UserContext(), filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
func riskFilter(request dto.AuditRiskFilterRequest, page, pageSize int) (auditquery.RiskFilter, error) {
|
||||
from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo)
|
||||
if err != nil {
|
||||
return auditquery.RiskFilter{}, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
return auditquery.RiskFilter{
|
||||
CreatedFrom: from, CreatedTo: to, Risk: request.Risk, Result: request.Result,
|
||||
Action: request.Action, Source: request.Source, Page: page, PageSize: pageSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AgentResourceActivities 查询代理范围内的安全资源活动。
|
||||
// GET /api/admin/agent/resource-activities/:resource_type/:identifier
|
||||
func (h *AuditHandler) AgentResourceActivities(c *fiber.Ctx) error {
|
||||
request, err := subjectActivityRequest(c)
|
||||
request, from, to, err := subjectActivityRequest(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.auditQuery.AgentResourceActivities(c.UserContext(), auditquery.SubjectActivityFilter{
|
||||
ResourceType: request.ResourceType, Identifier: request.Identifier,
|
||||
CreatedFrom: from, CreatedTo: to,
|
||||
Page: request.Page, PageSize: request.PageSize,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -141,12 +241,13 @@ func (h *AuditHandler) AgentResourceActivities(c *fiber.Ctx) error {
|
||||
// EnterpriseResourceActivities 查询企业当前有效授权资产的安全资源活动。
|
||||
// GET /api/admin/enterprise/resource-activities/:resource_type/:identifier
|
||||
func (h *AuditHandler) EnterpriseResourceActivities(c *fiber.Ctx) error {
|
||||
request, err := subjectActivityRequest(c)
|
||||
request, from, to, err := subjectActivityRequest(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := h.auditQuery.EnterpriseResourceActivities(c.UserContext(), auditquery.SubjectActivityFilter{
|
||||
ResourceType: request.ResourceType, Identifier: request.Identifier,
|
||||
CreatedFrom: from, CreatedTo: to,
|
||||
Page: request.Page, PageSize: request.PageSize,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -155,17 +256,21 @@ func (h *AuditHandler) EnterpriseResourceActivities(c *fiber.Ctx) error {
|
||||
return response.Success(c, result)
|
||||
}
|
||||
|
||||
func subjectActivityRequest(c *fiber.Ctx) (dto.SubjectResourceActivityRequest, error) {
|
||||
func subjectActivityRequest(c *fiber.Ctx) (dto.SubjectResourceActivityRequest, *time.Time, *time.Time, error) {
|
||||
var request dto.SubjectResourceActivityRequest
|
||||
if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) {
|
||||
return request, errors.New(errors.CodeInvalidParam)
|
||||
return request, nil, nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
request.ResourceType = c.Params("resource_type")
|
||||
request.Identifier = c.Params("identifier")
|
||||
if request.ResourceType == "" || request.Identifier == "" {
|
||||
return request, errors.New(errors.CodeInvalidParam)
|
||||
return request, nil, nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
return request, nil
|
||||
from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo)
|
||||
if err != nil {
|
||||
return request, nil, nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
return request, from, to, nil
|
||||
}
|
||||
|
||||
// IntegrationOverview 查询外部集成交互总览。
|
||||
|
||||
@@ -109,7 +109,7 @@ func (h *CMCCRealnameHandler) process(ctx context.Context, body []byte, contentT
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err)
|
||||
h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err)
|
||||
return h.fail(ctx, log.IntegrationID, err)
|
||||
}
|
||||
if h.series != nil {
|
||||
|
||||
@@ -135,7 +135,7 @@ func (h *CTCCRealnameHandler) process(ctx context.Context, body []byte, contentT
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err)
|
||||
h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err)
|
||||
return h.failPending(ctx, log.IntegrationID, err)
|
||||
}
|
||||
if h.series != nil {
|
||||
|
||||
@@ -108,7 +108,7 @@ func (h *CUCCRealnameHandler) process(ctx context.Context, body []byte, contentT
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err)
|
||||
h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err)
|
||||
return h.fail(ctx, log.IntegrationID, err)
|
||||
}
|
||||
if h.series != nil {
|
||||
|
||||
@@ -303,7 +303,7 @@ func (h *PaymentHandler) confirmAgentRechargePayment(ctx context.Context, callba
|
||||
}
|
||||
if log.Result == constants.IntegrationResultPending {
|
||||
resolvedResourceID := strconv.FormatUint(uint64(result.PaymentID), 10)
|
||||
_, err = h.integration.Complete(ctx, log.IntegrationID, integrationlog.Completion{
|
||||
_, err := h.integration.Complete(ctx, log.IntegrationID, integrationlog.Completion{
|
||||
Result: constants.IntegrationResultSuccess, ProviderCode: "SUCCESS",
|
||||
ResponseSummary: map[string]any{"confirmed": true, "already_confirmed": result.AlreadyConfirmed},
|
||||
StateChanged: !result.AlreadyConfirmed, ResourceID: &resolvedResourceID,
|
||||
|
||||
@@ -105,6 +105,9 @@ func NewRegistry() *Registry {
|
||||
iotCardRealnameCallbackSynced := iotCardAction(constants.AuditActionIotCardRealnameCallbackSynced, "运营商回调同步 IoT 卡实名状态", constants.AuditActorExternalSystem, constants.AuditSourceCallback)
|
||||
iotCardManualRefreshed := iotCardAction(constants.AuditActionIotCardManualRefreshed, "人工刷新 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
|
||||
iotCardPersonalRefreshed := iotCardAction(constants.AuditActionIotCardPersonalRefreshed, "个人客户刷新 IoT 卡", constants.AuditActorPersonalCustomer, constants.AuditSourcePersonalAPI)
|
||||
iotCardWorkerRealnameSynced := iotCardAction(constants.AuditActionIotCardWorkerRealnameSynced, "Worker 同步 IoT 卡实名事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
iotCardWorkerTrafficSynced := iotCardAction(constants.AuditActionIotCardWorkerTrafficSynced, "Worker 同步 IoT 卡流量事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
iotCardWorkerNetworkSynced := iotCardAction(constants.AuditActionIotCardWorkerNetworkSynced, "Worker 同步 IoT 卡网络事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
iotCardManualStopped := iotCardAction(constants.AuditActionIotCardManualStopped, "人工停用 IoT 卡网络", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
|
||||
iotCardManualStarted := iotCardAction(constants.AuditActionIotCardManualStarted, "人工恢复 IoT 卡网络", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
|
||||
iotCardAutoStopped := iotCardAction(constants.AuditActionIotCardAutoStopped, "自动停用 IoT 卡网络", constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
@@ -131,6 +134,7 @@ func NewRegistry() *Registry {
|
||||
deviceCardBound := deviceAction(constants.AuditActionDeviceCardBound, "设备绑定 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
|
||||
deviceCardUnbound := deviceAction(constants.AuditActionDeviceCardUnbound, "设备解绑 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI)
|
||||
deviceCurrentCardSwitched := deviceExternalAction(constants.AuditActionDeviceCurrentCardSwitched, "切换设备当前卡", true)
|
||||
deviceWorkerObservationSynced := deviceAction(constants.AuditActionDeviceWorkerObservationSynced, "Worker 同步设备观测事实", constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
cardExchangeCreated := cardExchangeAction(constants.AuditActionCardExchangeCreated, "创建卡换货单", constants.AuditRiskNormal, false)
|
||||
cardExchangeShippingInfoSubmitted := cardExchangeAction(constants.AuditActionCardExchangeShippingInfoSubmitted, "提交卡换货收货信息", constants.AuditRiskHigh, true)
|
||||
cardExchangeShipped := cardExchangeAction(constants.AuditActionCardExchangeShipped, "卡换货发货", constants.AuditRiskNormal, false)
|
||||
@@ -205,6 +209,14 @@ func NewRegistry() *Registry {
|
||||
notificationReadAll.AllowedOrigins = []ActionOrigin{{Actor: constants.AuditActorPersonalCustomer, Source: constants.AuditSourcePersonalAPI}}
|
||||
notificationCleanup := notificationAction(constants.AuditActionNotificationCleanup, "清理过期通知", constants.AuditResourceNotificationCleanupBatch, constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
notificationCleanupItem := notificationAction(constants.AuditActionNotificationCleanupItem, "清理单条过期通知", constants.AuditResourceNotification, constants.AuditActorSystemTask, constants.AuditSourceWorker)
|
||||
retentionCleanup := ActionDefinition{
|
||||
Code: constants.AuditActionLogRetentionCleanup, Name: "清理已归档在线日志",
|
||||
Category: constants.AuditCategoryReliability, Risk: constants.AuditRiskHigh,
|
||||
PrimaryResource: constants.AuditResourceLogArchiveMonth, AllowedActor: constants.AuditActorSystemTask,
|
||||
Source: constants.AuditSourceWorker, RequireTransaction: true,
|
||||
DefaultVisibility: constants.AuditSubjectInternalOnly,
|
||||
AllowedVisibility: []string{constants.AuditSubjectInternalOnly},
|
||||
}
|
||||
pollingConfigCreated := pollingAction(constants.AuditActionPollingConfigCreated, "创建轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
|
||||
pollingConfigUpdated := pollingAction(constants.AuditActionPollingConfigUpdated, "更新轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
|
||||
pollingConfigDeleted := pollingAction(constants.AuditActionPollingConfigDeleted, "删除轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
|
||||
@@ -407,6 +419,9 @@ func NewRegistry() *Registry {
|
||||
constants.AuditActionIotCardRealnameCallbackSynced: iotCardRealnameCallbackSynced,
|
||||
constants.AuditActionIotCardManualRefreshed: iotCardManualRefreshed,
|
||||
constants.AuditActionIotCardPersonalRefreshed: iotCardPersonalRefreshed,
|
||||
constants.AuditActionIotCardWorkerRealnameSynced: iotCardWorkerRealnameSynced,
|
||||
constants.AuditActionIotCardWorkerTrafficSynced: iotCardWorkerTrafficSynced,
|
||||
constants.AuditActionIotCardWorkerNetworkSynced: iotCardWorkerNetworkSynced,
|
||||
constants.AuditActionIotCardManualStopped: iotCardManualStopped,
|
||||
constants.AuditActionIotCardManualStarted: iotCardManualStarted,
|
||||
constants.AuditActionIotCardAutoStopped: iotCardAutoStopped,
|
||||
@@ -432,6 +447,7 @@ func NewRegistry() *Registry {
|
||||
constants.AuditActionDeviceCardBound: deviceCardBound,
|
||||
constants.AuditActionDeviceCardUnbound: deviceCardUnbound,
|
||||
constants.AuditActionDeviceCurrentCardSwitched: deviceCurrentCardSwitched,
|
||||
constants.AuditActionDeviceWorkerObservationSynced: deviceWorkerObservationSynced,
|
||||
constants.AuditActionCardExchangeCreated: cardExchangeCreated,
|
||||
constants.AuditActionCardExchangeShippingInfoSubmitted: cardExchangeShippingInfoSubmitted,
|
||||
constants.AuditActionCardExchangeShipped: cardExchangeShipped,
|
||||
@@ -478,6 +494,7 @@ func NewRegistry() *Registry {
|
||||
constants.AuditActionNotificationReadAll: notificationReadAll,
|
||||
constants.AuditActionNotificationCleanup: notificationCleanup,
|
||||
constants.AuditActionNotificationCleanupItem: notificationCleanupItem,
|
||||
constants.AuditActionLogRetentionCleanup: retentionCleanup,
|
||||
constants.AuditActionPollingConfigCreated: pollingConfigCreated,
|
||||
constants.AuditActionPollingConfigUpdated: pollingConfigUpdated,
|
||||
constants.AuditActionPollingConfigDeleted: pollingConfigDeleted,
|
||||
@@ -608,6 +625,10 @@ func NewRegistry() *Registry {
|
||||
"resource_type", "resource_id", "resource_key", "correlation_id",
|
||||
},
|
||||
},
|
||||
constants.AuditResourceLogArchiveMonth: {
|
||||
Type: constants.AuditResourceLogArchiveMonth, Name: "日志归档自然月",
|
||||
IdentityFields: []string{"month", "timezone", "range_start", "range_end"},
|
||||
},
|
||||
constants.AuditResourceDeviceBatchTask: {
|
||||
Type: constants.AuditResourceDeviceBatchTask, Name: "设备批量分配任务",
|
||||
IdentityFields: []string{"task_no", "operation_type"},
|
||||
|
||||
38
internal/infrastructure/audit/retention.go
Normal file
38
internal/infrastructure/audit/retention.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
auditarchive "github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// WriteRetentionCleanup 将月度物理清理结果写入当前在线月份的统一审计事件。
|
||||
func (w *Writer) WriteRetentionCleanup(ctx context.Context, tx *gorm.DB, input auditarchive.RetentionAudit) error {
|
||||
return w.Append(ctx, tx, AppendInput{
|
||||
EventID: input.EventID, ActionCode: constants.AuditActionLogRetentionCleanup,
|
||||
Summary: input.Summary,
|
||||
Actor: ActorInput{
|
||||
Kind: constants.AuditActorSystemTask, ID: constants.AuditActorIDRetentionWorker, Name: "日志留存清理任务",
|
||||
},
|
||||
Source: constants.AuditSourceWorker, ScopeType: constants.AuditScopePlatform,
|
||||
Result: input.Result, ErrorSummary: input.ErrorSummary,
|
||||
CorrelationID: "retention:" + input.Month,
|
||||
Metadata: map[string]any{
|
||||
"event_count": input.EventCount, "resource_count": input.ResourceCount,
|
||||
"integration_count": input.IntegrationCount, "manifest_keys": input.ManifestKeys,
|
||||
"duration_ms": input.DurationMS,
|
||||
},
|
||||
Resources: []ResourceInput{{
|
||||
Type: constants.AuditResourceLogArchiveMonth, Key: input.Month, DisplayName: input.Month,
|
||||
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleRetentionMonth,
|
||||
IdentitySnapshot: map[string]any{
|
||||
"month": input.Month, "timezone": constants.AuditArchiveTimezone,
|
||||
"range_start": input.RangeStart, "range_end": input.RangeEnd,
|
||||
},
|
||||
SubjectVisibility: constants.AuditSubjectInternalOnly,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -12,9 +12,11 @@ import (
|
||||
cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||
carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/integrationlog"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
apperrors "github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
@@ -77,11 +79,15 @@ type SeriesRunner struct {
|
||||
observation *cardapp.Service
|
||||
integration *integrationlog.Repository
|
||||
carrier *postgres.CarrierStore
|
||||
auditWriter *audit.Writer
|
||||
}
|
||||
|
||||
// NewSeriesRunner 创建卡观测序列 Gateway 执行器。
|
||||
func NewSeriesRunner(db *gorm.DB, gatewayClient *gateway.Client, observation *cardapp.Service, integration *integrationlog.Repository) *SeriesRunner {
|
||||
return &SeriesRunner{db: db, gateway: gatewayClient, observation: observation, integration: integration, carrier: postgres.NewCarrierStore(db)}
|
||||
func NewSeriesRunner(db *gorm.DB, gatewayClient *gateway.Client, observation *cardapp.Service, integration *integrationlog.Repository, auditWriter *audit.Writer) *SeriesRunner {
|
||||
return &SeriesRunner{
|
||||
db: db, gateway: gatewayClient, observation: observation, integration: integration,
|
||||
carrier: postgres.NewCarrierStore(db), auditWriter: auditWriter,
|
||||
}
|
||||
}
|
||||
|
||||
// Provider 返回用于请求互斥的运营商接入标识。
|
||||
@@ -188,7 +194,7 @@ func (r *SeriesRunner) runDeviceInfo(ctx context.Context, payload cardapp.Series
|
||||
response, runErr := r.gateway.SyncDeviceInfo(ctx, &gateway.SyncDeviceInfoReq{CardNo: deviceGatewayIdentifier(device)})
|
||||
result := cardapp.RunResult{}
|
||||
if runErr == nil {
|
||||
result.StateChanged, runErr = r.applyDeviceInfo(ctx, device, response)
|
||||
result.StateChanged, runErr = r.applyDeviceInfo(ctx, device, response, attempt.IntegrationID)
|
||||
}
|
||||
completionResult := constants.IntegrationResultSuccess
|
||||
if runErr != nil {
|
||||
@@ -222,7 +228,7 @@ func (r *SeriesRunner) startDeviceAttempt(ctx context.Context, payload cardapp.S
|
||||
})
|
||||
}
|
||||
|
||||
func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device, response *gateway.SyncDeviceInfoResp) (bool, error) {
|
||||
func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device, response *gateway.SyncDeviceInfoResp, integrationID string) (bool, error) {
|
||||
if response == nil {
|
||||
return false, apperrors.New(apperrors.CodeGatewayError, "Gateway 设备信息响应为空")
|
||||
}
|
||||
@@ -236,21 +242,19 @@ func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device
|
||||
if lastOnlineTime := parseGatewayTime(response.LastOnlineTime); lastOnlineTime != nil {
|
||||
updates["last_online_time"] = lastOnlineTime
|
||||
}
|
||||
var currentBinding struct {
|
||||
SlotPosition int
|
||||
}
|
||||
currentSlotErr := r.db.WithContext(ctx).Model(&model.DeviceSimBinding{}).
|
||||
Select("slot_position").
|
||||
Where("device_id = ? AND bind_status = ? AND is_current = ?", device.ID, constants.BindStatusBound, true).
|
||||
Take(¤tBinding).Error
|
||||
if currentSlotErr != nil && currentSlotErr != gorm.ErrRecordNotFound {
|
||||
return false, apperrors.Wrap(apperrors.CodeDatabaseError, currentSlotErr, "读取设备当前槽位失败")
|
||||
}
|
||||
changed := device.OnlineStatus != int(response.OnlineStatus) ||
|
||||
deviceChanged := device.OnlineStatus != int(response.OnlineStatus) ||
|
||||
device.SoftwareVersion != string(response.SoftwareVersion) ||
|
||||
device.SwitchMode != string(response.SwitchMode) ||
|
||||
currentBinding.SlotPosition != int(response.CurrentSlotNo)
|
||||
device.SwitchMode != string(response.SwitchMode)
|
||||
changed := deviceChanged
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
state, err := loadDeviceObservationState(ctx, tx, device.ID, int(response.CurrentSlotNo))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
beforeSlot := bindingSlot(state.current)
|
||||
afterSlot := bindingSlot(state.target)
|
||||
changed = deviceChanged || beforeSlot != int(response.CurrentSlotNo)
|
||||
auditChanged := deviceChanged || beforeSlot != afterSlot
|
||||
if err := tx.Model(&model.Device{}).Where("id = ?", device.ID).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -259,19 +263,192 @@ func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device
|
||||
Update("is_current", false).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if int(response.CurrentSlotNo) <= 0 {
|
||||
if int(response.CurrentSlotNo) > 0 {
|
||||
if err := tx.Model(&model.DeviceSimBinding{}).
|
||||
Where("device_id = ? AND slot_position = ? AND bind_status = ?", device.ID, int(response.CurrentSlotNo), constants.BindStatusBound).
|
||||
Update("is_current", true).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !auditChanged {
|
||||
return nil
|
||||
}
|
||||
return tx.Model(&model.DeviceSimBinding{}).
|
||||
Where("device_id = ? AND slot_position = ? AND bind_status = ?", device.ID, int(response.CurrentSlotNo), constants.BindStatusBound).
|
||||
Update("is_current", true).Error
|
||||
return r.appendDeviceObservationAudit(ctx, tx, device, state, beforeSlot, afterSlot, integrationID, map[string]any{
|
||||
"online_status": device.OnlineStatus, "software_version": device.SoftwareVersion,
|
||||
"switch_mode": device.SwitchMode, "current_slot": beforeSlot,
|
||||
}, map[string]any{
|
||||
"online_status": int(response.OnlineStatus), "software_version": string(response.SoftwareVersion),
|
||||
"switch_mode": string(response.SwitchMode), "current_slot": afterSlot,
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return false, apperrors.Wrap(apperrors.CodeDatabaseError, err, "回写设备 Gateway 信息失败")
|
||||
wrapped := apperrors.Wrap(apperrors.CodeDatabaseError, err, "回写设备 Gateway 信息失败")
|
||||
r.recordDeviceObservationFailure(ctx, device, integrationID, wrapped)
|
||||
return false, wrapped
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
type deviceObservationState struct {
|
||||
bindings []model.DeviceSimBinding
|
||||
cards map[uint]*model.IotCard
|
||||
current *model.DeviceSimBinding
|
||||
target *model.DeviceSimBinding
|
||||
}
|
||||
|
||||
func loadDeviceObservationState(ctx context.Context, tx *gorm.DB, deviceID uint, targetSlot int) (*deviceObservationState, error) {
|
||||
state := &deviceObservationState{cards: make(map[uint]*model.IotCard)}
|
||||
if err := tx.WithContext(ctx).Where("device_id = ? AND bind_status = ?", deviceID, constants.BindStatusBound).
|
||||
Order("slot_position ASC").Find(&state.bindings).Error; err != nil {
|
||||
return nil, apperrors.Wrap(apperrors.CodeDatabaseError, err, "读取设备有效卡槽失败")
|
||||
}
|
||||
cardIDs := make([]uint, 0, len(state.bindings))
|
||||
for index := range state.bindings {
|
||||
binding := &state.bindings[index]
|
||||
cardIDs = append(cardIDs, binding.IotCardID)
|
||||
if binding.IsCurrent && state.current == nil {
|
||||
state.current = binding
|
||||
}
|
||||
if targetSlot > 0 && binding.SlotPosition == targetSlot {
|
||||
state.target = binding
|
||||
}
|
||||
}
|
||||
if len(cardIDs) == 0 {
|
||||
return state, nil
|
||||
}
|
||||
var cards []*model.IotCard
|
||||
if err := tx.WithContext(ctx).Where("id IN ?", cardIDs).Find(&cards).Error; err != nil {
|
||||
return nil, apperrors.Wrap(apperrors.CodeDatabaseError, err, "读取设备卡槽关联 IoT 卡失败")
|
||||
}
|
||||
for _, card := range cards {
|
||||
state.cards[card.ID] = card
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func bindingSlot(binding *model.DeviceSimBinding) int {
|
||||
if binding == nil {
|
||||
return 0
|
||||
}
|
||||
return binding.SlotPosition
|
||||
}
|
||||
|
||||
func (r *SeriesRunner) appendDeviceObservationAudit(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
device *model.Device,
|
||||
state *deviceObservationState,
|
||||
beforeSlot, afterSlot int,
|
||||
integrationID string,
|
||||
beforeData, afterData map[string]any,
|
||||
) error {
|
||||
if r.auditWriter == nil {
|
||||
return apperrors.New(apperrors.CodeInternalError, "设备观测统一审计能力未配置")
|
||||
}
|
||||
deviceID := strconv.FormatUint(uint64(device.ID), 10)
|
||||
resources := []audit.ResourceInput{{
|
||||
Type: constants.AuditResourceDevice, ID: &deviceID,
|
||||
Key: audit.DeviceResourceKey(device), DisplayName: device.VirtualNo,
|
||||
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleDeviceTarget,
|
||||
IdentitySnapshot: audit.DeviceIdentitySnapshot(device), BeforeData: beforeData, AfterData: afterData,
|
||||
SubjectVisibility: constants.AuditSubjectResult, SubjectSummary: "Worker 同步设备观测事实",
|
||||
}}
|
||||
if beforeSlot != afterSlot {
|
||||
resources = appendDeviceObservationBindingResources(resources, device, state.current, state.cards[stateCardID(state.current)], false,
|
||||
constants.AuditResourceRoleDeviceOldCurrentCard, constants.AuditResourceRoleDeviceOldCurrentBinding)
|
||||
resources = appendDeviceObservationBindingResources(resources, device, state.target, state.cards[stateCardID(state.target)], true,
|
||||
constants.AuditResourceRoleDeviceNewCurrentCard, constants.AuditResourceRoleDeviceNewCurrentBinding)
|
||||
}
|
||||
resources = append(resources, deviceObservationIntegrationResource(ctx, device, integrationID))
|
||||
return r.auditWriter.Append(ctx, tx, audit.AppendInput{
|
||||
ActionCode: constants.AuditActionDeviceWorkerObservationSynced,
|
||||
Summary: "Worker 同步设备观测事实", ScopeType: constants.AuditScopePlatform,
|
||||
Result: constants.AuditResultSuccess, Resources: resources,
|
||||
})
|
||||
}
|
||||
|
||||
func appendDeviceObservationBindingResources(
|
||||
resources []audit.ResourceInput,
|
||||
device *model.Device,
|
||||
binding *model.DeviceSimBinding,
|
||||
card *model.IotCard,
|
||||
afterCurrent bool,
|
||||
cardRole, bindingRole string,
|
||||
) []audit.ResourceInput {
|
||||
if binding == nil {
|
||||
return resources
|
||||
}
|
||||
if card != nil {
|
||||
cardID := strconv.FormatUint(uint64(card.ID), 10)
|
||||
resources = append(resources, audit.ResourceInput{
|
||||
Type: constants.AuditResourceIotCard, ID: &cardID,
|
||||
Key: audit.IotCardResourceKey(card), DisplayName: card.ICCID,
|
||||
Relation: constants.AuditResourceRelationAffected, Role: cardRole,
|
||||
IdentitySnapshot: audit.IotCardIdentitySnapshot(card),
|
||||
BeforeData: map[string]any{"is_current": binding.IsCurrent}, AfterData: map[string]any{"is_current": afterCurrent},
|
||||
SubjectVisibility: constants.AuditSubjectResult, SubjectSummary: "设备当前卡槽已同步",
|
||||
})
|
||||
}
|
||||
bindingID := strconv.FormatUint(uint64(binding.ID), 10)
|
||||
identity := map[string]any{
|
||||
"id": binding.ID, "device_id": binding.DeviceID, "device_virtual_no": device.VirtualNo,
|
||||
"slot_position": binding.SlotPosition, "iot_card_id": binding.IotCardID, "is_current": afterCurrent,
|
||||
}
|
||||
if card != nil {
|
||||
identity["iccid"] = card.ICCID
|
||||
identity["virtual_no"] = card.VirtualNo
|
||||
}
|
||||
return append(resources, audit.ResourceInput{
|
||||
Type: constants.AuditResourceDeviceSIMBinding, ID: &bindingID,
|
||||
Key: bindingID, DisplayName: device.VirtualNo,
|
||||
Relation: constants.AuditResourceRelationAffected, Role: bindingRole,
|
||||
IdentitySnapshot: identity,
|
||||
BeforeData: map[string]any{"is_current": binding.IsCurrent}, AfterData: map[string]any{"is_current": afterCurrent},
|
||||
SubjectVisibility: constants.AuditSubjectInternalOnly,
|
||||
})
|
||||
}
|
||||
|
||||
func stateCardID(binding *model.DeviceSimBinding) uint {
|
||||
if binding == nil {
|
||||
return 0
|
||||
}
|
||||
return binding.IotCardID
|
||||
}
|
||||
|
||||
func deviceObservationIntegrationResource(ctx context.Context, device *model.Device, integrationID string) audit.ResourceInput {
|
||||
deviceID := strconv.FormatUint(uint64(device.ID), 10)
|
||||
return audit.ResourceInput{
|
||||
Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: integrationID,
|
||||
Relation: constants.AuditResourceRelationReference, Role: constants.AuditResourceRoleWorkerIntegration,
|
||||
IdentitySnapshot: map[string]any{
|
||||
"integration_id": integrationID, "provider": constants.IntegrationProviderGateway,
|
||||
"direction": constants.IntegrationDirectionOutbound, "operation": constants.IntegrationOperationGatewayDeviceInfo,
|
||||
"resource_type": constants.CardObservationResourceTypeDevice, "resource_id": deviceID,
|
||||
"resource_key": "device:" + deviceID, "correlation_id": auditcontext.From(ctx).CorrelationID,
|
||||
},
|
||||
SubjectVisibility: constants.AuditSubjectInternalOnly,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *SeriesRunner) recordDeviceObservationFailure(ctx context.Context, device *model.Device, integrationID string, businessErr error) {
|
||||
deviceID := strconv.FormatUint(uint64(device.ID), 10)
|
||||
r.auditWriter.RecordFailure(ctx, r.db, audit.AppendInput{
|
||||
ActionCode: constants.AuditActionDeviceWorkerObservationSynced,
|
||||
Summary: "Worker 同步设备观测事实失败", ScopeType: constants.AuditScopePlatform,
|
||||
Resources: []audit.ResourceInput{{
|
||||
Type: constants.AuditResourceDevice, ID: &deviceID,
|
||||
Key: audit.DeviceResourceKey(device), DisplayName: device.VirtualNo,
|
||||
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleDeviceTarget,
|
||||
IdentitySnapshot: audit.DeviceIdentitySnapshot(device),
|
||||
BeforeData: map[string]any{
|
||||
"online_status": device.OnlineStatus, "software_version": device.SoftwareVersion,
|
||||
"switch_mode": device.SwitchMode,
|
||||
},
|
||||
SubjectVisibility: constants.AuditSubjectResult,
|
||||
}, deviceObservationIntegrationResource(ctx, device, integrationID)},
|
||||
}, businessErr)
|
||||
}
|
||||
|
||||
func (r *SeriesRunner) loadDevice(ctx context.Context, payload cardapp.SeriesTaskPayload) (*model.Device, error) {
|
||||
if payload.ResourceType != constants.CardObservationResourceTypeDevice {
|
||||
return nil, apperrors.New(apperrors.CodeInvalidParam, "设备信息观测资源类型无效")
|
||||
|
||||
@@ -62,18 +62,75 @@ type AuditResourceTimelineRequest struct {
|
||||
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量,默认20,最大100"`
|
||||
}
|
||||
|
||||
// AuditRequestTimelineParams 是请求链路时间线的路径参数。
|
||||
type AuditRequestTimelineParams struct {
|
||||
RequestID string `json:"request_id" path:"request_id" required:"true" description:"HTTP请求关联ID,来自审计或外部集成节点,也可从Access Log粘贴"`
|
||||
}
|
||||
|
||||
// AuditCorrelationTimelineParams 是业务关联时间线的路径参数。
|
||||
type AuditCorrelationTimelineParams struct {
|
||||
CorrelationID string `json:"correlation_id" path:"correlation_id" required:"true" description:"跨请求、异步任务和外部交互的稳定业务链路ID"`
|
||||
}
|
||||
|
||||
// AuditFinanceTimelineRequest 是资金调查时间线的稳定业务筛选参数。
|
||||
type AuditFinanceTimelineRequest struct {
|
||||
ShopID uint `json:"shop_id" query:"shop_id" description:"店铺ID"`
|
||||
WalletID uint `json:"wallet_id" query:"wallet_id" description:"代理或资产钱包ID"`
|
||||
OrderID uint `json:"order_id" query:"order_id" description:"订单ID"`
|
||||
OrderNo string `json:"order_no" query:"order_no" description:"订单编号"`
|
||||
PaymentID uint `json:"payment_id" query:"payment_id" description:"支付记录ID"`
|
||||
PaymentNo string `json:"payment_no" query:"payment_no" description:"支付单号"`
|
||||
RefundID uint `json:"refund_id" query:"refund_id" description:"退款单ID"`
|
||||
RefundNo string `json:"refund_no" query:"refund_no" description:"退款单号"`
|
||||
RechargeID uint `json:"recharge_id" query:"recharge_id" description:"代理充值或个人资产充值ID"`
|
||||
RechargeNo string `json:"recharge_no" query:"recharge_no" description:"充值单号"`
|
||||
ApprovalInstanceID uint `json:"approval_instance_id" query:"approval_instance_id" description:"审批实例ID"`
|
||||
ThirdPartyTradeNo string `json:"third_party_trade_no" query:"third_party_trade_no" description:"第三方交易号"`
|
||||
ActorKind string `json:"actor_kind" query:"actor_kind" description:"操作者类型;与actor_id同时提供"`
|
||||
ActorID string `json:"actor_id" query:"actor_id" description:"操作者稳定ID;与actor_kind同时提供"`
|
||||
CorrelationID string `json:"correlation_id" query:"correlation_id" description:"跨步骤业务链路ID"`
|
||||
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间(RFC3339,含时区)"`
|
||||
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间(RFC3339,含时区,不包含该时刻)"`
|
||||
Page int `json:"page" query:"page" minimum:"1" description:"页码,默认1"`
|
||||
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量,默认20,最大100"`
|
||||
}
|
||||
|
||||
// AuditRiskFilterRequest 是风险总览和明细共用的受控筛选参数。
|
||||
type AuditRiskFilterRequest struct {
|
||||
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间(RFC3339,含时区);默认从在线窗口开始"`
|
||||
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间(RFC3339,含时区,不包含该时刻,最长31天);默认当前时间"`
|
||||
Risk string `json:"risk" query:"risk" description:"风险等级 (low:低, normal:普通, high:高, critical:严重)"`
|
||||
Result string `json:"result" query:"result" description:"结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知)"`
|
||||
Action string `json:"action" query:"action" description:"稳定动作编码"`
|
||||
Source string `json:"source" query:"source" description:"来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调)"`
|
||||
}
|
||||
|
||||
// AuditRiskOverviewRequest 是风险总览请求参数。
|
||||
type AuditRiskOverviewRequest struct {
|
||||
AuditRiskFilterRequest
|
||||
}
|
||||
|
||||
// AuditRiskEventsRequest 是风险事件明细请求参数。
|
||||
type AuditRiskEventsRequest struct {
|
||||
AuditRiskFilterRequest
|
||||
Page int `json:"page" query:"page" minimum:"1" description:"页码,默认1"`
|
||||
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量,默认20,最大100"`
|
||||
}
|
||||
|
||||
// SubjectResourceActivityRequest 是代理和企业安全资源活动的路径及分页参数。
|
||||
type SubjectResourceActivityRequest struct {
|
||||
ResourceType string `json:"resource_type" path:"resource_type" required:"true" description:"资源类型 (iot_card:IoT卡, device:设备, asset_allocation_record:资产分配记录, exchange_order:换货单, shop:店铺, enterprise:企业)"`
|
||||
Identifier string `json:"identifier" path:"identifier" required:"true" description:"业务稳定标识;卡使用ICCID,设备使用VirtualNo,其他资源使用对应业务编号"`
|
||||
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间(RFC3339,含时区);默认从在线窗口开始"`
|
||||
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间(RFC3339,含时区,不包含该时刻);默认当前时间"`
|
||||
Page int `json:"page" query:"page" minimum:"1" description:"页码,默认1"`
|
||||
PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量,默认20,最大100"`
|
||||
}
|
||||
|
||||
// IntegrationFilterRequest 是外部集成调查的公共受控筛选参数。
|
||||
type IntegrationFilterRequest struct {
|
||||
CreatedFrom string `json:"created_from" query:"created_from" required:"true" description:"开始时间(RFC3339,含时区,必填)"`
|
||||
CreatedTo string `json:"created_to" query:"created_to" required:"true" description:"结束时间(RFC3339,含时区,不包含该时刻,必填)"`
|
||||
CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间(RFC3339,含时区);默认从在线窗口开始"`
|
||||
CreatedTo string `json:"created_to" query:"created_to" description:"结束时间(RFC3339,含时区,不包含该时刻);默认当前时间"`
|
||||
IntegrationID string `json:"integration_id" query:"integration_id" description:"稳定外部集成记录ID"`
|
||||
Provider string `json:"provider" query:"provider" description:"外部服务提供方稳定编码"`
|
||||
Direction string `json:"direction" query:"direction" description:"交互方向 (inbound:入站, outbound:出站)"`
|
||||
|
||||
38
internal/model/log_archive_run.go
Normal file
38
internal/model/log_archive_run.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// LogArchiveRun 是日志冷归档运行账本,不保存日志正文。
|
||||
type LogArchiveRun struct {
|
||||
ID uint `gorm:"column:id;primaryKey;autoIncrement" json:"id"`
|
||||
Source string `gorm:"column:source;type:varchar(32);not null" json:"source"`
|
||||
ArchiveDate time.Time `gorm:"column:archive_date;type:date;not null" json:"archive_date"`
|
||||
InstanceID string `gorm:"column:instance_id;type:varchar(100);not null" json:"instance_id"`
|
||||
SchemaVersion string `gorm:"column:schema_version;type:varchar(32);not null" json:"schema_version"`
|
||||
Revision int `gorm:"column:revision;not null;default:1" json:"revision"`
|
||||
Status string `gorm:"column:status;type:varchar(16);not null" json:"status"`
|
||||
IsFinal bool `gorm:"column:is_final;not null;default:false" json:"is_final"`
|
||||
RangeStart time.Time `gorm:"column:range_start;type:timestamptz;not null" json:"range_start"`
|
||||
RangeEnd time.Time `gorm:"column:range_end;type:timestamptz;not null" json:"range_end"`
|
||||
ObjectKey string `gorm:"column:object_key;type:varchar(500);not null;default:''" json:"object_key"`
|
||||
ManifestKey string `gorm:"column:manifest_key;type:varchar(500);not null;default:''" json:"manifest_key"`
|
||||
EventCount int64 `gorm:"column:event_count;not null;default:0" json:"event_count"`
|
||||
ResourceCount int64 `gorm:"column:resource_count;not null;default:0" json:"resource_count"`
|
||||
RecordCount int64 `gorm:"column:record_count;not null;default:0" json:"record_count"`
|
||||
UncompressedBytes int64 `gorm:"column:uncompressed_bytes;not null;default:0" json:"uncompressed_bytes"`
|
||||
CompressedBytes int64 `gorm:"column:compressed_bytes;not null;default:0" json:"compressed_bytes"`
|
||||
SHA256 string `gorm:"column:sha256;type:varchar(64);not null;default:''" json:"sha256"`
|
||||
AttemptCount int `gorm:"column:attempt_count;not null;default:0" json:"attempt_count"`
|
||||
ErrorSummary string `gorm:"column:error_summary;type:varchar(500);not null;default:''" json:"error_summary"`
|
||||
GeneratedAt *time.Time `gorm:"column:generated_at" json:"generated_at,omitempty"`
|
||||
CompletedAt *time.Time `gorm:"column:completed_at" json:"completed_at,omitempty"`
|
||||
CleanupStartedAt *time.Time `gorm:"column:cleanup_started_at" json:"cleanup_started_at,omitempty"`
|
||||
CleanedAt *time.Time `gorm:"column:cleaned_at" json:"cleaned_at,omitempty"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`
|
||||
}
|
||||
|
||||
// TableName 返回日志冷归档运行账本表名。
|
||||
func (LogArchiveRun) TableName() string {
|
||||
return "tb_log_archive_run"
|
||||
}
|
||||
@@ -41,6 +41,13 @@ type PackageActivationHandler struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
func workerCorrelation(correlationID string, task *asynq.Task) string {
|
||||
if correlationID != "" {
|
||||
return correlationID
|
||||
}
|
||||
return task.ResultWriter().TaskID()
|
||||
}
|
||||
|
||||
// PackageActivationPayload 套餐激活任务载荷
|
||||
type PackageActivationPayload struct {
|
||||
PackageUsageID uint `json:"package_usage_id"`
|
||||
@@ -48,6 +55,9 @@ type PackageActivationPayload struct {
|
||||
CarrierID uint `json:"carrier_id"`
|
||||
ActivationType string `json:"activation_type"` // "queue" 或 "realname"
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
ParentEventID string `json:"parent_event_id,omitempty"`
|
||||
}
|
||||
|
||||
// NewPackageActivationHandler 创建套餐激活检查处理器
|
||||
@@ -397,10 +407,11 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
if h.stopResumeCallback == nil {
|
||||
return
|
||||
}
|
||||
detachedCtx := context.WithoutCancel(ctx)
|
||||
|
||||
if carrierType == "iot_card" {
|
||||
go func() {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(context.Background(), carrierID); err != nil {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, carrierID); err != nil {
|
||||
h.logger.Error("套餐过期后停机失败",
|
||||
zap.Uint("card_id", carrierID),
|
||||
zap.Error(err))
|
||||
@@ -420,7 +431,7 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
for _, b := range bindings {
|
||||
cardID := b.IotCardID
|
||||
go func(cID uint) {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(context.Background(), cID); err != nil {
|
||||
if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, cID); err != nil {
|
||||
h.logger.Error("套餐过期后停机失败",
|
||||
zap.Uint("card_id", cID),
|
||||
zap.Error(err))
|
||||
@@ -432,12 +443,16 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c
|
||||
|
||||
// enqueueActivationTask 提交套餐激活任务到 Asynq
|
||||
func (h *PackageActivationHandler) enqueueActivationTask(ctx context.Context, packageUsageID uint, carrierType string, carrierID uint, activationType string) error {
|
||||
linkage := auditcontext.From(ctx)
|
||||
payload := PackageActivationPayload{
|
||||
PackageUsageID: packageUsageID,
|
||||
CarrierType: carrierType,
|
||||
CarrierID: carrierID,
|
||||
ActivationType: activationType,
|
||||
Timestamp: time.Now().Unix(),
|
||||
RequestID: linkage.RequestID,
|
||||
CorrelationID: linkage.CorrelationID,
|
||||
ParentEventID: linkage.ParentEventID,
|
||||
}
|
||||
|
||||
payloadBytes, err := sonic.Marshal(payload)
|
||||
@@ -477,6 +492,7 @@ func (h *PackageActivationHandler) HandlePackageQueueActivation(ctx context.Cont
|
||||
ctx = auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypePackageQueueActivation,
|
||||
ActorName: "套餐排队激活任务", Source: constants.AuditSourceWorker,
|
||||
RequestID: payload.RequestID, CorrelationID: workerCorrelation(payload.CorrelationID, t), ParentEventID: payload.ParentEventID,
|
||||
})
|
||||
|
||||
h.logger.Info("开始执行套餐激活",
|
||||
@@ -534,6 +550,7 @@ func (h *PackageActivationHandler) HandlePackageFirstActivation(ctx context.Cont
|
||||
ctx = auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypePackageFirstActivation,
|
||||
ActorName: "套餐首次实名激活任务", Source: constants.AuditSourceWorker,
|
||||
RequestID: payload.RequestID, CorrelationID: workerCorrelation(payload.CorrelationID, t), ParentEventID: payload.ParentEventID,
|
||||
})
|
||||
|
||||
if payload.CarrierType == "" || payload.CarrierID == 0 {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||
@@ -39,10 +40,17 @@ type EventFilter struct {
|
||||
|
||||
// EventPage 是平台全局事件稳定分页结果。
|
||||
type EventPage struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []EventView `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []EventView `json:"items"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// EventDetail 是单个审计事件及在线留存边界。
|
||||
type EventDetail struct {
|
||||
EventView
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// EventView 是不暴露 GORM Model 的审计事件投影。
|
||||
@@ -147,6 +155,14 @@ func (q *Query) List(ctx context.Context, filter EventFilter) (*EventPage, error
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validEventFilter(filter) {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
@@ -156,16 +172,33 @@ func (q *Query) List(ctx context.Context, filter EventFilter) (*EventPage, error
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计审计事件失败")
|
||||
}
|
||||
rows := make([]model.AuditEvent, 0, filter.PageSize)
|
||||
if err := query.Order("occurred_at DESC, id DESC").
|
||||
Offset((filter.Page - 1) * filter.PageSize).Limit(filter.PageSize).Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询审计事件失败")
|
||||
rows, err := q.loadEventPage(ctx, query, filter.Page, filter.PageSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := q.project(ctx, rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
|
||||
return &EventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
|
||||
}
|
||||
|
||||
// loadEventPage 先分页主键,再批量读取事件宽行,避免排序阶段加载 JSON 字段。
|
||||
func (q *Query) loadEventPage(ctx context.Context, query *gorm.DB, page, pageSize int) ([]model.AuditEvent, error) {
|
||||
ids := make([]uint, 0, pageSize)
|
||||
if err := query.Select("id").Order("occurred_at DESC, id DESC").
|
||||
Offset((page-1)*pageSize).Limit(pageSize).Pluck("id", &ids).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询审计事件分页ID失败")
|
||||
}
|
||||
rows := make([]model.AuditEvent, 0, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return rows, nil
|
||||
}
|
||||
if err := q.db.WithContext(ctx).Where("id IN ?", ids).
|
||||
Order("occurred_at DESC, id DESC").Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "批量投影审计事件失败")
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func validEventFilter(filter EventFilter) bool {
|
||||
@@ -192,15 +225,19 @@ func validOptionalValue(value string, allowed ...string) bool {
|
||||
}
|
||||
|
||||
// Get 查询平台范围的单个稳定审计事件详情。
|
||||
func (q *Query) Get(ctx context.Context, eventID string) (*EventView, error) {
|
||||
func (q *Query) Get(ctx context.Context, eventID string) (*EventDetail, error) {
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if eventID == "" {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var row model.AuditEvent
|
||||
if err := q.db.WithContext(ctx).Where("event_id = ?", eventID).First(&row).Error; err != nil {
|
||||
if err := q.db.WithContext(ctx).Where("event_id = ? AND occurred_at >= ?", eventID, retention.OnlineFrom.UTC()).First(&row).Error; err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, errors.New(errors.CodeNotFound, "审计事件不存在")
|
||||
}
|
||||
@@ -210,7 +247,7 @@ func (q *Query) Get(ctx context.Context, eventID string) (*EventView, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &items[0], nil
|
||||
return &EventDetail{EventView: items[0], Retention: retention}, nil
|
||||
}
|
||||
|
||||
func (q *Query) authorize(ctx context.Context) error {
|
||||
|
||||
1519
internal/query/audit/finance.go
Normal file
1519
internal/query/audit/finance.go
Normal file
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
@@ -17,16 +18,18 @@ import (
|
||||
type ResourceSearchFilter struct {
|
||||
ResourceType string
|
||||
Keyword string
|
||||
OnlineFrom time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// ResourceSearchPage 是资源候选稳定分页结果。
|
||||
type ResourceSearchPage struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []ResourceCandidate `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []ResourceCandidate `json:"items"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// ResourceCandidate 是当前业务表或历史事件快照解析出的稳定资源候选。
|
||||
@@ -59,6 +62,11 @@ func (q *Query) SearchResources(ctx context.Context, filter ResourceSearchFilter
|
||||
if filter.Keyword == "" || !searchableResourceType(filter.ResourceType) {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter.OnlineFrom = retention.OnlineFrom
|
||||
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
|
||||
items, total, err := q.searchCurrent(ctx, filter)
|
||||
if err != nil {
|
||||
@@ -70,7 +78,7 @@ func (q *Query) SearchResources(ctx context.Context, filter ResourceSearchFilter
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &ResourceSearchPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
|
||||
return &ResourceSearchPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
|
||||
}
|
||||
|
||||
// ResourceTimeline 查询注册资源作为任意关系参与的统一事件时间线。
|
||||
@@ -196,7 +204,7 @@ func (q *Query) searchHistorical(ctx context.Context, filter ResourceSearchFilte
|
||||
|
||||
func (q *Query) historicalIdentifierQuery(ctx context.Context, filter ResourceSearchFilter) *gorm.DB {
|
||||
query := q.db.WithContext(ctx).Model(&model.AuditEventResource{}).
|
||||
Where("resource_type = ? AND resource_id IS NOT NULL", filter.ResourceType)
|
||||
Where("resource_type = ? AND resource_id IS NOT NULL AND created_at >= ?", filter.ResourceType, filter.OnlineFrom.UTC())
|
||||
switch filter.ResourceType {
|
||||
case constants.AuditResourceIotCard:
|
||||
return query.Where("resource_key = ? OR identity_snapshot ->> 'iccid' = ? OR identity_snapshot ->> 'iccid_19' = ? OR identity_snapshot ->> 'iccid_20' = ? OR identity_snapshot ->> 'virtual_no' = ?", filter.Keyword, filter.Keyword, filter.Keyword, filter.Keyword, filter.Keyword)
|
||||
|
||||
299
internal/query/audit/risks.go
Normal file
299
internal/query/audit/risks.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// RiskFilter 定义固定风险调查视角的时间范围与筛选条件。
|
||||
type RiskFilter struct {
|
||||
CreatedFrom *time.Time
|
||||
CreatedTo *time.Time
|
||||
Risk string
|
||||
Result string
|
||||
Action string
|
||||
Source string
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// RiskOverview 是风险信号、固定维度与时间趋势的只读聚合。
|
||||
type RiskOverview struct {
|
||||
Total int64 `json:"total"`
|
||||
Bucket string `json:"bucket"`
|
||||
Signals []RiskNamedCount `json:"signals"`
|
||||
Risks []RiskNamedCount `json:"risks"`
|
||||
Results []RiskNamedCount `json:"results"`
|
||||
Actions []RiskNamedCount `json:"actions"`
|
||||
Sources []RiskNamedCount `json:"sources"`
|
||||
Trend []RiskTrendPoint `json:"trend"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// RiskNamedCount 是风险聚合维度的稳定编码、中文名称和数量。
|
||||
type RiskNamedCount struct {
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
// RiskTrendPoint 是固定时间桶内的风险信号趋势。
|
||||
type RiskTrendPoint struct {
|
||||
BucketAt time.Time `json:"bucket_at"`
|
||||
Total int64 `json:"total"`
|
||||
HighRisk int64 `json:"high_risk"`
|
||||
Finance int64 `json:"finance"`
|
||||
Security int64 `json:"security"`
|
||||
Failed int64 `json:"failed"`
|
||||
Denied int64 `json:"denied"`
|
||||
Partial int64 `json:"partial"`
|
||||
Unknown int64 `json:"unknown"`
|
||||
}
|
||||
|
||||
// RiskEventPage 是风险事件的稳定分页结果。
|
||||
type RiskEventPage struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []EventView `json:"items"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// RiskOverview 查询指定时间范围内的固定风险调查总览。
|
||||
func (q *Query) RiskOverview(ctx context.Context, filter RiskFilter) (*RiskOverview, error) {
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.AuditRiskQueryMaxRange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validRiskFilter(filter) {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
|
||||
base := q.applyRiskFilters(q.db.WithContext(ctx).Model(&model.AuditEvent{}), filter)
|
||||
result := &RiskOverview{
|
||||
Bucket: riskTrendBucket(*filter.CreatedFrom, *filter.CreatedTo),
|
||||
Signals: []RiskNamedCount{}, Risks: []RiskNamedCount{}, Results: []RiskNamedCount{},
|
||||
Actions: []RiskNamedCount{}, Sources: []RiskNamedCount{}, Trend: []RiskTrendPoint{},
|
||||
Retention: retention,
|
||||
}
|
||||
if err := loadRiskSignals(base, result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadRiskDimension(base, "risk_level", result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadRiskDimension(base, "result", result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadRiskDimension(base, "action_code", result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadRiskDimension(base, "source", result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadRiskTrend(base, result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RiskEvents 查询指定时间范围内的风险事件明细。
|
||||
func (q *Query) RiskEvents(ctx context.Context, filter RiskFilter) (*RiskEventPage, error) {
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.AuditRiskQueryMaxRange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validRiskFilter(filter) {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
|
||||
query := q.applyRiskFilters(q.db.WithContext(ctx).Model(&model.AuditEvent{}), filter)
|
||||
var total int64
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计风险事件失败")
|
||||
}
|
||||
rows, err := q.loadEventPage(ctx, query, filter.Page, filter.PageSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := q.project(ctx, rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &RiskEventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
|
||||
}
|
||||
|
||||
func validRiskFilter(filter RiskFilter) bool {
|
||||
if filter.CreatedFrom == nil || filter.CreatedTo == nil || !filter.CreatedFrom.Before(*filter.CreatedTo) ||
|
||||
filter.CreatedTo.Sub(*filter.CreatedFrom) > constants.AuditRiskQueryMaxRange {
|
||||
return false
|
||||
}
|
||||
return validEventFilter(EventFilter{
|
||||
Risk: filter.Risk, Result: filter.Result, Action: filter.Action, Source: filter.Source,
|
||||
Page: filter.Page, PageSize: filter.PageSize,
|
||||
})
|
||||
}
|
||||
|
||||
func (q *Query) applyRiskFilters(query *gorm.DB, filter RiskFilter) *gorm.DB {
|
||||
query = q.applyFilters(query, EventFilter{
|
||||
CreatedFrom: filter.CreatedFrom, CreatedTo: filter.CreatedTo,
|
||||
Risk: filter.Risk, Result: filter.Result, Action: filter.Action, Source: filter.Source,
|
||||
})
|
||||
return query.Where(riskScopeSQL(), riskLevels(), constants.AuditCategorySecurity, abnormalResults(), financeResourceTypes())
|
||||
}
|
||||
|
||||
func loadRiskSignals(query *gorm.DB, result *RiskOverview) error {
|
||||
var row struct {
|
||||
Total, HighRisk, Finance, Security, Failed, Denied, Partial, Unknown int64
|
||||
}
|
||||
err := query.Select(`COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE risk_level IN ?) AS high_risk,
|
||||
COUNT(*) FILTER (WHERE EXISTS (SELECT 1 FROM tb_audit_event_resource aer WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?)) AS finance,
|
||||
COUNT(*) FILTER (WHERE category = ?) AS security,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS failed,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS denied,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS partial,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS unknown`, riskLevels(), financeResourceTypes(), constants.AuditCategorySecurity,
|
||||
constants.AuditResultFailed, constants.AuditResultDenied, constants.AuditResultPartial, constants.AuditResultUnknown).Scan(&row).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险信号失败")
|
||||
}
|
||||
result.Total = row.Total
|
||||
result.Signals = []RiskNamedCount{
|
||||
{Code: constants.AuditRiskSignalHighRisk, Name: "高风险", Count: row.HighRisk},
|
||||
{Code: constants.AuditRiskSignalFinance, Name: "资金", Count: row.Finance},
|
||||
{Code: constants.AuditRiskSignalSecurity, Name: "安全", Count: row.Security},
|
||||
{Code: constants.AuditRiskSignalFailed, Name: "失败", Count: row.Failed},
|
||||
{Code: constants.AuditRiskSignalDenied, Name: "拒绝", Count: row.Denied},
|
||||
{Code: constants.AuditRiskSignalPartial, Name: "部分成功", Count: row.Partial},
|
||||
{Code: constants.AuditRiskSignalUnknown, Name: "结果未知", Count: row.Unknown},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadRiskDimension(query *gorm.DB, column string, result *RiskOverview) error {
|
||||
var rows []struct {
|
||||
Code string
|
||||
Name string
|
||||
Count int64
|
||||
}
|
||||
selectClause := column + " AS code, '' AS name, COUNT(*) AS count"
|
||||
groupClause := column
|
||||
if column == "action_code" {
|
||||
selectClause = "action_code AS code, action_name AS name, COUNT(*) AS count"
|
||||
groupClause = "action_code, action_name"
|
||||
}
|
||||
if err := query.Select(selectClause).Group(groupClause).Order(column + " ASC").Scan(&rows).Error; err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险维度失败")
|
||||
}
|
||||
items := make([]RiskNamedCount, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
name := row.Name
|
||||
if name == "" {
|
||||
name = riskDimensionName(column, row.Code)
|
||||
}
|
||||
items = append(items, RiskNamedCount{Code: row.Code, Name: name, Count: row.Count})
|
||||
}
|
||||
switch column {
|
||||
case "risk_level":
|
||||
result.Risks = items
|
||||
case "result":
|
||||
result.Results = items
|
||||
case "action_code":
|
||||
result.Actions = items
|
||||
case "source":
|
||||
result.Sources = items
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadRiskTrend(query *gorm.DB, result *RiskOverview) error {
|
||||
err := query.Select(`date_trunc(?, occurred_at) AS bucket_at, COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE risk_level IN ?) AS high_risk,
|
||||
COUNT(*) FILTER (WHERE EXISTS (SELECT 1 FROM tb_audit_event_resource aer WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?)) AS finance,
|
||||
COUNT(*) FILTER (WHERE category = ?) AS security,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS failed,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS denied,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS partial,
|
||||
COUNT(*) FILTER (WHERE result = ?) AS unknown`, result.Bucket, riskLevels(), financeResourceTypes(),
|
||||
constants.AuditCategorySecurity, constants.AuditResultFailed, constants.AuditResultDenied,
|
||||
constants.AuditResultPartial, constants.AuditResultUnknown).
|
||||
Group("bucket_at").Order("bucket_at ASC").Scan(&result.Trend).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险趋势失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func riskScopeSQL() string {
|
||||
return `(risk_level IN ? OR category = ? OR result IN ? OR EXISTS (
|
||||
SELECT 1 FROM tb_audit_event_resource aer
|
||||
WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?
|
||||
))`
|
||||
}
|
||||
|
||||
func riskLevels() []string {
|
||||
return []string{constants.AuditRiskHigh, constants.AuditRiskCritical}
|
||||
}
|
||||
|
||||
func abnormalResults() []string {
|
||||
return []string{constants.AuditResultFailed, constants.AuditResultDenied, constants.AuditResultPartial, constants.AuditResultUnknown}
|
||||
}
|
||||
|
||||
func financeResourceTypes() []string {
|
||||
return []string{
|
||||
constants.AuditResourceOrder, constants.AuditResourceRefund, constants.AuditResourceAgentRecharge,
|
||||
constants.AuditResourceRechargeOrder, constants.AuditResourceAssetWallet, constants.AuditResourceAssetWalletTransaction,
|
||||
constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction,
|
||||
constants.AuditResourceAgentWalletReservation, constants.AuditResourcePayment,
|
||||
constants.AuditResourceCommissionRecord, constants.AuditResourceCommissionWithdrawal,
|
||||
}
|
||||
}
|
||||
|
||||
func riskTrendBucket(from, to time.Time) string {
|
||||
if to.Sub(from) <= constants.AuditRiskHourlyTrendMaxRange {
|
||||
return "hour"
|
||||
}
|
||||
return "day"
|
||||
}
|
||||
|
||||
func riskDimensionName(column, code string) string {
|
||||
names := map[string]map[string]string{
|
||||
"risk_level": {
|
||||
constants.AuditRiskLow: "低", constants.AuditRiskNormal: "普通",
|
||||
constants.AuditRiskHigh: "高", constants.AuditRiskCritical: "严重",
|
||||
},
|
||||
"result": {
|
||||
constants.AuditResultSuccess: "成功", constants.AuditResultFailed: "失败",
|
||||
constants.AuditResultDenied: "拒绝", constants.AuditResultPartial: "部分成功",
|
||||
constants.AuditResultUnknown: "结果未知",
|
||||
},
|
||||
"source": {
|
||||
constants.AuditSourceAdminAPI: "后台管理 API", constants.AuditSourcePersonalAPI: "个人客户 API",
|
||||
constants.AuditSourceOpenAPI: "代理 OpenAPI", constants.AuditSourceWorker: "异步 Worker",
|
||||
constants.AuditSourceScheduler: "计划任务", constants.AuditSourceCallback: "外部系统回调",
|
||||
},
|
||||
}
|
||||
return names[column][code]
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||
@@ -18,17 +19,20 @@ import (
|
||||
type SubjectActivityFilter struct {
|
||||
ResourceType string
|
||||
Identifier string
|
||||
CreatedFrom *time.Time
|
||||
CreatedTo *time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// SubjectActivityPage 是不包含平台调查字段的代理资源活动分页结果。
|
||||
type SubjectActivityPage struct {
|
||||
Resource SubjectResourceSummary `json:"resource"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []SubjectActivity `json:"items"`
|
||||
Resource SubjectResourceSummary `json:"resource"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []SubjectActivity `json:"items"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// SubjectActivity 是写入时已生成的主体安全活动投影。
|
||||
@@ -78,11 +82,19 @@ func (q *Query) AgentResourceActivities(ctx context.Context, filter SubjectActiv
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target, err := q.resolveAgentTarget(ctx, filter.ResourceType, filter.Identifier, shopIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q.subjectActivitiesForTarget(ctx, filter, target, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
|
||||
return q.subjectActivitiesForTarget(ctx, filter, target, retention, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
|
||||
return q.agentAllowedResourceIDs(ctx, resources, shopIDs)
|
||||
})
|
||||
}
|
||||
@@ -100,21 +112,30 @@ func (q *Query) EnterpriseResourceActivities(ctx context.Context, filter Subject
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize)
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target, err := q.resolveEnterpriseTarget(ctx, filter.ResourceType, filter.Identifier, enterpriseID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q.subjectActivitiesForTarget(ctx, filter, target, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
|
||||
return q.subjectActivitiesForTarget(ctx, filter, target, retention, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) {
|
||||
return q.enterpriseAllowedResourceIDs(ctx, resources, enterpriseID)
|
||||
})
|
||||
}
|
||||
|
||||
func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectActivityFilter, target subjectTarget, authorize subjectResourceAuthorizer) (*SubjectActivityPage, error) {
|
||||
func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectActivityFilter, target subjectTarget, retention retentionquery.Info, authorize subjectResourceAuthorizer) (*SubjectActivityPage, error) {
|
||||
|
||||
resourceMatch := q.db.Table("tb_audit_event_resource AS target").Select("1").
|
||||
Where("target.audit_event_id = tb_audit_event.id AND target.resource_type = ? AND target.resource_id = ?", filter.ResourceType, target.id).
|
||||
Where("target.subject_visibility IN ?", []string{constants.AuditSubjectResult, constants.AuditSubjectDetail})
|
||||
base := q.db.WithContext(ctx).Model(&model.AuditEvent{}).Where("EXISTS (?)", resourceMatch)
|
||||
base := q.db.WithContext(ctx).Model(&model.AuditEvent{}).
|
||||
Where("occurred_at >= ? AND occurred_at < ?", filter.CreatedFrom.UTC(), filter.CreatedTo.UTC()).Where("EXISTS (?)", resourceMatch)
|
||||
var total int64
|
||||
if err := base.Count(&total).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计代理资源活动失败")
|
||||
@@ -131,7 +152,7 @@ func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectAc
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &SubjectActivityPage{Resource: target.summary, Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
|
||||
return &SubjectActivityPage{Resource: target.summary, Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
|
||||
}
|
||||
|
||||
func (q *Query) resolveEnterpriseTarget(ctx context.Context, resourceType, identifier string, enterpriseID uint) (subjectTarget, error) {
|
||||
|
||||
317
internal/query/audit/timeline.go
Normal file
317
internal/query/audit/timeline.go
Normal file
@@ -0,0 +1,317 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// LinkTimeline 是 request 或 correlation 的跨事实只读时间线。
|
||||
type LinkTimeline struct {
|
||||
RequestID *string `json:"request_id"`
|
||||
CorrelationID *string `json:"correlation_id"`
|
||||
AccessLogLookupRequestID *string `json:"access_log_lookup_request_id"`
|
||||
Nodes []LinkTimelineNode `json:"nodes"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// LinkTimelineNode 是保留各事实源权威边界的时间线节点。
|
||||
type LinkTimelineNode struct {
|
||||
RecordSource string `json:"record_source"`
|
||||
NodeID string `json:"node_id"`
|
||||
OccurredAt time.Time `json:"occurred_at"`
|
||||
Code string `json:"code"`
|
||||
Title string `json:"title"`
|
||||
Result string `json:"result"`
|
||||
ResultName string `json:"result_name"`
|
||||
Summary string `json:"summary"`
|
||||
ReferenceOnly bool `json:"reference_only"`
|
||||
RequestID *string `json:"request_id"`
|
||||
CorrelationID *string `json:"correlation_id"`
|
||||
ParentEventID *string `json:"parent_event_id"`
|
||||
Resources []InvestigationResourceRef `json:"resources"`
|
||||
InvestigationRefs InvestigationRefs `json:"investigation_refs"`
|
||||
Fidelity LinkageFidelity `json:"fidelity"`
|
||||
}
|
||||
|
||||
// LinkageFidelity 明确节点已有的稳定关联能力,不补猜历史缺失字段。
|
||||
type LinkageFidelity struct {
|
||||
RequestAvailable bool `json:"request_available"`
|
||||
CorrelationAvailable bool `json:"correlation_available"`
|
||||
ParentEventAvailable bool `json:"parent_event_available"`
|
||||
DirectAuditLinkAvailable bool `json:"direct_audit_link_available"`
|
||||
StableResourceAvailable bool `json:"stable_resource_available"`
|
||||
}
|
||||
|
||||
// RequestTimeline 按精确 request ID 组合已持久化事实,不扫描 Access Log。
|
||||
func (q *Query) RequestTimeline(ctx context.Context, requestID string) (*LinkTimeline, error) {
|
||||
return q.linkTimeline(ctx, "request_id", requestID)
|
||||
}
|
||||
|
||||
// CorrelationTimeline 按精确 correlation ID 组合跨请求业务链路。
|
||||
func (q *Query) CorrelationTimeline(ctx context.Context, correlationID string) (*LinkTimeline, error) {
|
||||
return q.linkTimeline(ctx, "correlation_id", correlationID)
|
||||
}
|
||||
|
||||
func (q *Query) linkTimeline(ctx context.Context, column, value string) (*LinkTimeline, error) {
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if value == "" || (column != "request_id" && column != "correlation_id") {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit, retentionquery.SourceIntegration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
auditRows, integrationRows, outboxRows, err := q.loadLinkRows(ctx, column, value, retention.OnlineFrom)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events, err := q.project(ctx, auditRows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes := make([]LinkTimelineNode, 0, len(events)+len(integrationRows)+len(outboxRows))
|
||||
integrationByAudit := integrationRefsByAuditID(integrationRows)
|
||||
for index, event := range events {
|
||||
refs := event.InvestigationRefs
|
||||
refs.IntegrationRefs = append(refs.IntegrationRefs, integrationByAudit[auditRows[index].ID]...)
|
||||
refs.IntegrationRefs = append(refs.IntegrationRefs, integrationResourceRefs(event.Resources)...)
|
||||
refs.IntegrationRefs = uniqueIntegrationRefs(refs.IntegrationRefs)
|
||||
nodes = append(nodes, auditTimelineNode(event, refs))
|
||||
nodes = append(nodes, resourceReferenceNodes(event, refs)...)
|
||||
}
|
||||
for _, row := range integrationRows {
|
||||
nodes = append(nodes, integrationTimelineNode(row))
|
||||
}
|
||||
for _, row := range outboxRows {
|
||||
nodes = append(nodes, outboxTimelineNode(row))
|
||||
}
|
||||
sort.Slice(nodes, func(i, j int) bool {
|
||||
if nodes[i].OccurredAt.Equal(nodes[j].OccurredAt) {
|
||||
if nodes[i].RecordSource == nodes[j].RecordSource {
|
||||
return nodes[i].NodeID < nodes[j].NodeID
|
||||
}
|
||||
return nodes[i].RecordSource < nodes[j].RecordSource
|
||||
}
|
||||
return nodes[i].OccurredAt.Before(nodes[j].OccurredAt)
|
||||
})
|
||||
|
||||
timeline := &LinkTimeline{Nodes: nodes, Retention: retention}
|
||||
if timeline.Nodes == nil {
|
||||
timeline.Nodes = []LinkTimelineNode{}
|
||||
}
|
||||
if column == "request_id" {
|
||||
timeline.RequestID = stringPointer(value)
|
||||
timeline.AccessLogLookupRequestID = stringPointer(value)
|
||||
} else {
|
||||
timeline.CorrelationID = stringPointer(value)
|
||||
}
|
||||
return timeline, nil
|
||||
}
|
||||
|
||||
func (q *Query) loadLinkRows(ctx context.Context, column, value string, onlineFrom time.Time) ([]model.AuditEvent, []model.IntegrationLog, []model.OutboxEvent, error) {
|
||||
auditRows := []model.AuditEvent{}
|
||||
if err := q.db.WithContext(ctx).Where(column+" = ? AND occurred_at >= ?", value, onlineFrom.UTC()).Find(&auditRows).Error; err != nil {
|
||||
return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路审计事件失败")
|
||||
}
|
||||
integrationRows := []model.IntegrationLog{}
|
||||
if err := q.db.WithContext(ctx).Where(column+" = ? AND created_at >= ?", value, onlineFrom.UTC()).Find(&integrationRows).Error; err != nil {
|
||||
return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路外部交互失败")
|
||||
}
|
||||
outboxRows := []model.OutboxEvent{}
|
||||
if err := q.db.WithContext(ctx).Where(column+" = ? AND created_at >= ?", value, onlineFrom.UTC()).Find(&outboxRows).Error; err != nil {
|
||||
return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路可靠事件失败")
|
||||
}
|
||||
return auditRows, integrationRows, outboxRows, nil
|
||||
}
|
||||
|
||||
func auditTimelineNode(event EventView, refs InvestigationRefs) LinkTimelineNode {
|
||||
return LinkTimelineNode{
|
||||
RecordSource: constants.AuditRecordSourceAuditEvent, NodeID: event.EventID,
|
||||
OccurredAt: event.OccurredAt, Code: event.ActionCode, Title: event.ActionName,
|
||||
Result: event.Result, Summary: event.Summary,
|
||||
RequestID: stringPointer(event.RequestID), CorrelationID: stringPointer(event.CorrelationID),
|
||||
ParentEventID: stringPointer(event.ParentEventID), Resources: refs.ResourceRefs, InvestigationRefs: refs,
|
||||
Fidelity: linkageFidelity(event.RequestID, event.CorrelationID, event.ParentEventID, true, len(refs.ResourceRefs) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
func integrationTimelineNode(row model.IntegrationLog) LinkTimelineNode {
|
||||
resource := integrationResourceRef(row)
|
||||
resources := make([]InvestigationResourceRef, 0, 1)
|
||||
if resource != nil {
|
||||
resources = append(resources, *resource)
|
||||
}
|
||||
refs := InvestigationRefs{
|
||||
ResourceRefs: resources, RequestID: row.RequestID, CorrelationID: row.CorrelationID,
|
||||
IntegrationRefs: []IntegrationRef{{IntegrationID: row.IntegrationID}},
|
||||
}
|
||||
return LinkTimelineNode{
|
||||
RecordSource: constants.AuditRecordSourceIntegrationLog, NodeID: row.IntegrationID,
|
||||
OccurredAt: row.CreatedAt, Code: row.Operation,
|
||||
Title: constants.IntegrationProviderName(row.Provider) + " · " + constants.IntegrationOperationName(row.Operation),
|
||||
Result: row.Result, ResultName: constants.IntegrationResultName(row.Result), Summary: "外部交互事实",
|
||||
RequestID: row.RequestID, CorrelationID: row.CorrelationID, Resources: resources, InvestigationRefs: refs,
|
||||
Fidelity: linkageFidelity(pointerValue(row.RequestID), pointerValue(row.CorrelationID), "", row.AuditEventID != nil, resource != nil),
|
||||
}
|
||||
}
|
||||
|
||||
func outboxTimelineNode(row model.OutboxEvent) LinkTimelineNode {
|
||||
resourceType := row.ResourceType
|
||||
if resourceType == "" {
|
||||
resourceType = row.AggregateType
|
||||
}
|
||||
resourceID := row.ResourceID
|
||||
if resourceID == "" {
|
||||
resourceID = row.AggregateID
|
||||
}
|
||||
resourceKey := row.BusinessKey
|
||||
if resourceKey == "" {
|
||||
resourceKey = row.AggregateID
|
||||
}
|
||||
resource := InvestigationResourceRef{ResourceType: resourceType, ResourceID: stringPointer(resourceID), ResourceKey: resourceKey, DisplayName: resourceKey}
|
||||
refs := InvestigationRefs{
|
||||
ResourceRefs: []InvestigationResourceRef{resource}, RequestID: stringPointer(row.RequestID),
|
||||
CorrelationID: stringPointer(row.CorrelationID), IntegrationRefs: []IntegrationRef{},
|
||||
}
|
||||
return LinkTimelineNode{
|
||||
RecordSource: constants.AuditRecordSourceOutboxEvent, NodeID: row.EventID,
|
||||
OccurredAt: row.CreatedAt, Code: row.EventType, Title: "可靠事件:" + row.EventType,
|
||||
Result: strconv.Itoa(row.Status), ResultName: constants.GetOutboxStatusName(row.Status),
|
||||
Summary: fmt.Sprintf("%s/%s,重试 %d 次", row.AggregateType, row.AggregateID, row.RetryCount),
|
||||
RequestID: stringPointer(row.RequestID), CorrelationID: stringPointer(row.CorrelationID),
|
||||
ParentEventID: stringPointer(row.ParentEventID), Resources: refs.ResourceRefs, InvestigationRefs: refs,
|
||||
Fidelity: linkageFidelity(row.RequestID, row.CorrelationID, row.ParentEventID, row.ParentEventID != "", resourceType != "" && resourceID != ""),
|
||||
}
|
||||
}
|
||||
|
||||
func resourceReferenceNodes(event EventView, refs InvestigationRefs) []LinkTimelineNode {
|
||||
nodes := make([]LinkTimelineNode, 0, len(event.Resources))
|
||||
for _, resource := range event.Resources {
|
||||
recordSource := ""
|
||||
titlePrefix := ""
|
||||
summary := ""
|
||||
switch {
|
||||
case isAsynqTaskResource(resource.ResourceType):
|
||||
recordSource = constants.AuditRecordSourceAsynqTask
|
||||
titlePrefix = "异步任务:"
|
||||
summary = "持久化任务资源摘要;不读取或推断 Redis 队列历史"
|
||||
case isDomainLedgerResource(resource.ResourceType):
|
||||
recordSource = constants.AuditRecordSourceDomainLedgerRef
|
||||
titlePrefix = "业务账本引用:"
|
||||
summary = "状态、金额及业务结论以对应业务表为准"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
resourceRef := InvestigationResourceRef{ResourceType: resource.ResourceType, ResourceID: resource.ResourceID, ResourceKey: resource.ResourceKey, DisplayName: resource.DisplayName}
|
||||
nodeRefs := refs
|
||||
nodeRefs.ResourceRefs = []InvestigationResourceRef{resourceRef}
|
||||
nodes = append(nodes, LinkTimelineNode{
|
||||
RecordSource: recordSource,
|
||||
NodeID: fmt.Sprintf("%s:%s:%s:%s:%s", event.EventID, resource.ResourceType, pointerValue(resource.ResourceID), resource.ResourceKey, resource.Role),
|
||||
OccurredAt: event.OccurredAt, Code: resource.ResourceType, Title: titlePrefix + resource.DisplayName,
|
||||
Result: event.Result, Summary: summary, ReferenceOnly: true,
|
||||
RequestID: stringPointer(event.RequestID), CorrelationID: stringPointer(event.CorrelationID),
|
||||
ParentEventID: stringPointer(event.ParentEventID), Resources: nodeRefs.ResourceRefs, InvestigationRefs: nodeRefs,
|
||||
Fidelity: linkageFidelity(event.RequestID, event.CorrelationID, event.ParentEventID, true, resource.ResourceID != nil || resource.ResourceKey != ""),
|
||||
})
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func integrationRefsByAuditID(rows []model.IntegrationLog) map[uint][]IntegrationRef {
|
||||
refs := make(map[uint][]IntegrationRef)
|
||||
for _, row := range rows {
|
||||
if row.AuditEventID != nil {
|
||||
refs[*row.AuditEventID] = append(refs[*row.AuditEventID], IntegrationRef{IntegrationID: row.IntegrationID})
|
||||
}
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
func integrationResourceRefs(resources []ResourceView) []IntegrationRef {
|
||||
refs := make([]IntegrationRef, 0)
|
||||
for _, resource := range resources {
|
||||
if resource.ResourceType == constants.AuditResourceIntegrationLog && resource.ResourceKey != "" {
|
||||
refs = append(refs, IntegrationRef{IntegrationID: resource.ResourceKey})
|
||||
}
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
func uniqueIntegrationRefs(refs []IntegrationRef) []IntegrationRef {
|
||||
unique := make([]IntegrationRef, 0, len(refs))
|
||||
seen := make(map[string]bool, len(refs))
|
||||
for _, ref := range refs {
|
||||
if ref.IntegrationID == "" || seen[ref.IntegrationID] {
|
||||
continue
|
||||
}
|
||||
seen[ref.IntegrationID] = true
|
||||
unique = append(unique, ref)
|
||||
}
|
||||
return unique
|
||||
}
|
||||
|
||||
func integrationResourceRef(row model.IntegrationLog) *InvestigationResourceRef {
|
||||
if row.ResourceType == nil || *row.ResourceType == "" {
|
||||
return nil
|
||||
}
|
||||
ref := InvestigationResourceRef{ResourceType: *row.ResourceType, ResourceID: row.ResourceID}
|
||||
if row.ResourceKey != nil {
|
||||
ref.ResourceKey = *row.ResourceKey
|
||||
ref.DisplayName = *row.ResourceKey
|
||||
}
|
||||
return &ref
|
||||
}
|
||||
|
||||
func linkageFidelity(requestID, correlationID, parentEventID string, directAuditLink, stableResource bool) LinkageFidelity {
|
||||
return LinkageFidelity{
|
||||
RequestAvailable: requestID != "", CorrelationAvailable: correlationID != "",
|
||||
ParentEventAvailable: parentEventID != "", DirectAuditLinkAvailable: directAuditLink,
|
||||
StableResourceAvailable: stableResource,
|
||||
}
|
||||
}
|
||||
|
||||
func pointerValue(value *string) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func isAsynqTaskResource(resourceType string) bool {
|
||||
switch resourceType {
|
||||
case constants.AuditResourceDeviceBatchTask, constants.AuditResourceIotCardImportTask,
|
||||
constants.AuditResourceDeviceImportTask, constants.AuditResourceAssetPackageBatchOrderTask,
|
||||
constants.AuditResourceOrderPackageInvalidateTask, constants.AuditResourceExportTask:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isDomainLedgerResource(resourceType string) bool {
|
||||
switch resourceType {
|
||||
case constants.AuditResourceOrder, constants.AuditResourcePayment, constants.AuditResourceRefund,
|
||||
constants.AuditResourceAgentRecharge, constants.AuditResourceRechargeOrder,
|
||||
constants.AuditResourceAssetWallet, constants.AuditResourceAssetWalletTransaction,
|
||||
constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction,
|
||||
constants.AuditResourceAgentWalletReservation, constants.AuditResourcePackageUsage,
|
||||
constants.AuditResourceApprovalInstance, constants.AuditResourceCommissionRecord,
|
||||
constants.AuditResourceCommissionWithdrawal:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||
@@ -35,10 +36,11 @@ type ListFilter struct {
|
||||
|
||||
// ListPage 是按创建时间和主键稳定倒序的分页结果。
|
||||
type ListPage struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []ListItem `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []ListItem `json:"items"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// ListItem 是 Integration Log 列表投影。
|
||||
@@ -81,6 +83,12 @@ type Detail struct {
|
||||
Fidelity FidelityView `json:"fidelity"`
|
||||
}
|
||||
|
||||
// DetailResponse 是外部交互详情及在线留存边界。
|
||||
type DetailResponse struct {
|
||||
Detail
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// AttemptView 是显式 trigger_series 下的单次技术尝试。
|
||||
type AttemptView struct {
|
||||
IntegrationID string `json:"integration_id"`
|
||||
@@ -169,6 +177,10 @@ func (q *Query) List(ctx context.Context, filter ListFilter) (*ListPage, error)
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter, retention, err := q.normalizeOnlineFilter(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validFilter(filter) {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
@@ -178,27 +190,52 @@ func (q *Query) List(ctx context.Context, filter ListFilter) (*ListPage, error)
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计外部交互日志失败")
|
||||
}
|
||||
rows := make([]model.IntegrationLog, 0, filter.PageSize)
|
||||
if err := query.Order("created_at DESC, id DESC").Offset((filter.Page - 1) * filter.PageSize).Limit(filter.PageSize).Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互日志失败")
|
||||
rows, err := q.loadListPage(ctx, query, filter.Page, filter.PageSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]ListItem, len(rows))
|
||||
for i, row := range rows {
|
||||
items[i] = projectListItem(row)
|
||||
}
|
||||
return &ListPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil
|
||||
return &ListPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil
|
||||
}
|
||||
|
||||
// loadListPage 先分页主键,再批量读取列表字段,避免加载正文摘要 JSON。
|
||||
func (q *Query) loadListPage(ctx context.Context, query *gorm.DB, page, pageSize int) ([]model.IntegrationLog, error) {
|
||||
ids := make([]uint, 0, pageSize)
|
||||
if err := query.Select("id").Order("created_at DESC, id DESC").
|
||||
Offset((page-1)*pageSize).Limit(pageSize).Pluck("id", &ids).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互日志分页ID失败")
|
||||
}
|
||||
rows := make([]model.IntegrationLog, 0, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return rows, nil
|
||||
}
|
||||
if err := q.db.WithContext(ctx).Select(
|
||||
"id", "integration_id", "provider", "direction", "operation",
|
||||
"resource_type", "resource_id", "resource_key", "result", "duration_ms",
|
||||
"state_changed", "request_id", "correlation_id", "created_at",
|
||||
).Where("id IN ?", ids).Order("created_at DESC, id DESC").Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "批量投影外部交互日志失败")
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// Get 使用稳定 integration_id 查询结构化详情。
|
||||
func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error) {
|
||||
func (q *Query) Get(ctx context.Context, integrationID string) (*DetailResponse, error) {
|
||||
if err := q.authorize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if integrationID == "" {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceIntegration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var row model.IntegrationLog
|
||||
if err := q.db.WithContext(ctx).Where("integration_id = ?", integrationID).First(&row).Error; err != nil {
|
||||
if err := q.db.WithContext(ctx).Where("integration_id = ? AND created_at >= ?", integrationID, retention.OnlineFrom.UTC()).First(&row).Error; err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, errors.New(errors.CodeNotFound, "外部交互日志不存在")
|
||||
}
|
||||
@@ -216,12 +253,12 @@ func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
attempts, err := q.loadAttempts(ctx, row)
|
||||
attempts, err := q.loadAttempts(ctx, row, retention.OnlineFrom)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
providerMessage, providerMessageFidelity := safeProviderMessage(row.ProviderMessage)
|
||||
return &Detail{
|
||||
return &DetailResponse{Detail: Detail{
|
||||
Identity: IdentityView{IntegrationID: row.IntegrationID, Provider: row.Provider, ProviderName: constants.IntegrationProviderName(row.Provider), Direction: row.Direction, DirectionName: constants.IntegrationDirectionName(row.Direction), Operation: row.Operation, OperationName: constants.IntegrationOperationName(row.Operation), ExternalID: row.ExternalID},
|
||||
Resource: resourceView(row), Trigger: TriggerView{Source: row.TriggerSource, Scene: row.TriggerScene, Series: row.TriggerSeries, Attempt: row.Attempt},
|
||||
Result: ResultView{Code: row.Result, Name: constants.IntegrationResultName(row.Result), Category: constants.IntegrationResultCategory(row.Result), HTTPStatus: row.HTTPStatus, ProviderCode: row.ProviderCode, ProviderMessage: providerMessage, DurationMS: row.DurationMS, StateChanged: row.StateChanged, RecoveryStrategy: row.RecoveryStrategy},
|
||||
@@ -235,13 +272,13 @@ func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error)
|
||||
ResourceIDAvailable: row.ResourceID != nil && *row.ResourceID != "",
|
||||
ProviderMessageFidelity: providerMessageFidelity,
|
||||
},
|
||||
}, nil
|
||||
}, Retention: retention}, nil
|
||||
}
|
||||
|
||||
func (q *Query) loadAttempts(ctx context.Context, current model.IntegrationLog) ([]AttemptView, error) {
|
||||
func (q *Query) loadAttempts(ctx context.Context, current model.IntegrationLog, onlineFrom time.Time) ([]AttemptView, error) {
|
||||
rows := []model.IntegrationLog{current}
|
||||
if current.TriggerSeries != nil && *current.TriggerSeries != "" {
|
||||
if err := q.db.WithContext(ctx).Where("trigger_series = ?", *current.TriggerSeries).
|
||||
if err := q.db.WithContext(ctx).Where("trigger_series = ? AND created_at >= ?", *current.TriggerSeries, onlineFrom.UTC()).
|
||||
Order("attempt ASC, created_at ASC, id ASC").Find(&rows).Error; err != nil {
|
||||
return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互尝试序列失败")
|
||||
}
|
||||
@@ -270,6 +307,15 @@ func (q *Query) authorize(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *Query) normalizeOnlineFilter(ctx context.Context, filter ListFilter) (ListFilter, retentionquery.Info, error) {
|
||||
retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceIntegration)
|
||||
if err != nil {
|
||||
return filter, retention, err
|
||||
}
|
||||
filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.IntegrationQueryMaxRange)
|
||||
return filter, retention, err
|
||||
}
|
||||
|
||||
func validFilter(filter ListFilter) bool {
|
||||
if filter.CreatedFrom == nil || filter.CreatedTo == nil || !filter.CreatedFrom.Before(*filter.CreatedTo) || filter.CreatedTo.Sub(*filter.CreatedFrom) > constants.IntegrationQueryMaxRange {
|
||||
return false
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
@@ -19,17 +20,18 @@ type OverviewFilter struct {
|
||||
|
||||
// Overview 是外部交互固定维度聚合结果。
|
||||
type Overview struct {
|
||||
Total int64 `json:"total"`
|
||||
AnomalyCount int64 `json:"anomaly_count"`
|
||||
UnknownCount int64 `json:"unknown_count"`
|
||||
StalePendingCount int64 `json:"stale_pending_count"`
|
||||
StateChangedCount int64 `json:"state_changed_count"`
|
||||
AverageDurationMS float64 `json:"average_duration_ms"`
|
||||
P95DurationMS float64 `json:"p95_duration_ms"`
|
||||
Results []ResultCount `json:"results"`
|
||||
Providers []NamedCount `json:"providers"`
|
||||
Directions []NamedCount `json:"directions"`
|
||||
Trend []TrendPoint `json:"trend"`
|
||||
Total int64 `json:"total"`
|
||||
AnomalyCount int64 `json:"anomaly_count"`
|
||||
UnknownCount int64 `json:"unknown_count"`
|
||||
StalePendingCount int64 `json:"stale_pending_count"`
|
||||
StateChangedCount int64 `json:"state_changed_count"`
|
||||
AverageDurationMS float64 `json:"average_duration_ms"`
|
||||
P95DurationMS float64 `json:"p95_duration_ms"`
|
||||
Results []ResultCount `json:"results"`
|
||||
Providers []NamedCount `json:"providers"`
|
||||
Directions []NamedCount `json:"directions"`
|
||||
Trend []TrendPoint `json:"trend"`
|
||||
Retention retentionquery.Info `json:"retention"`
|
||||
}
|
||||
|
||||
// ResultCount 是原始结果及其派生类别计数。
|
||||
@@ -66,11 +68,17 @@ func (q *Query) Overview(ctx context.Context, filter OverviewFilter) (*Overview,
|
||||
if filter.Bucket == "" {
|
||||
filter.Bucket = "hour"
|
||||
}
|
||||
var retention retentionquery.Info
|
||||
var err error
|
||||
filter.ListFilter, retention, err = q.normalizeOnlineFilter(ctx, filter.ListFilter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validFilter(filter.ListFilter) || (filter.Bucket != "hour" && filter.Bucket != "day") {
|
||||
return nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
base := applyFilters(q.db.WithContext(ctx).Model(&model.IntegrationLog{}), filter.ListFilter)
|
||||
result := &Overview{Results: []ResultCount{}, Providers: []NamedCount{}, Directions: []NamedCount{}, Trend: []TrendPoint{}}
|
||||
result := &Overview{Results: []ResultCount{}, Providers: []NamedCount{}, Directions: []NamedCount{}, Trend: []TrendPoint{}, Retention: retention}
|
||||
if err := loadOverviewMetrics(base, result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
115
internal/query/retention/retention.go
Normal file
115
internal/query/retention/retention.go
Normal file
@@ -0,0 +1,115 @@
|
||||
// Package retention 提供在线审计查询的统一留存边界。
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
// Source 表示受在线留存边界约束的数据源。
|
||||
type Source string
|
||||
|
||||
const (
|
||||
// SourceAudit 表示统一审计事件。
|
||||
SourceAudit Source = constants.AuditArchiveSource
|
||||
// SourceIntegration 表示外部交互日志。
|
||||
SourceIntegration Source = constants.IntegrationArchiveSource
|
||||
)
|
||||
|
||||
// Info 是查询响应公开的在线留存边界。
|
||||
type Info struct {
|
||||
OnlineFrom time.Time `json:"online_from" description:"当前可在线查询的最早时间"`
|
||||
ArchivedBefore *time.Time `json:"archived_before" description:"早于该时间的数据已归档;尚未清理时为空"`
|
||||
Timezone string `json:"timezone" description:"留存自然日时区"`
|
||||
}
|
||||
|
||||
// Load 从归档账本读取已完成物理清理的数据边界。
|
||||
func Load(ctx context.Context, db *gorm.DB, sources ...Source) (Info, error) {
|
||||
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if err != nil {
|
||||
return Info{}, errors.Wrap(errors.CodeInternalError, err, "加载审计留存时区失败")
|
||||
}
|
||||
now := time.Now().In(location)
|
||||
info := Info{OnlineFrom: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location), Timezone: constants.AuditArchiveTimezone}
|
||||
for _, source := range sources {
|
||||
boundary, cleaned, err := sourceBoundary(ctx, db, source, location)
|
||||
if err != nil {
|
||||
return Info{}, err
|
||||
}
|
||||
if boundary.Before(info.OnlineFrom) && info.ArchivedBefore == nil {
|
||||
info.OnlineFrom = boundary
|
||||
}
|
||||
if cleaned && (info.ArchivedBefore == nil || boundary.After(*info.ArchivedBefore)) {
|
||||
value := boundary
|
||||
info.ArchivedBefore = &value
|
||||
info.OnlineFrom = boundary
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func sourceBoundary(ctx context.Context, db *gorm.DB, source Source, location *time.Location) (time.Time, bool, error) {
|
||||
var cleanedEnd *time.Time
|
||||
if err := db.WithContext(ctx).Model(&model.LogArchiveRun{}).
|
||||
Where("source = ? AND cleaned_at IS NOT NULL", source).
|
||||
Select("MAX(range_end)").Scan(&cleanedEnd).Error; err != nil {
|
||||
return time.Time{}, false, errors.Wrap(errors.CodeDatabaseError, err, "查询审计留存清理边界失败")
|
||||
}
|
||||
if cleanedEnd != nil {
|
||||
return cleanedEnd.In(location), true, nil
|
||||
}
|
||||
|
||||
var earliest *time.Time
|
||||
table, column := "tb_audit_event", "occurred_at"
|
||||
if source == SourceIntegration {
|
||||
table, column = "tb_integration_log", "created_at"
|
||||
}
|
||||
if err := db.WithContext(ctx).Table(table).Select("MIN(" + column + ")").Scan(&earliest).Error; err != nil {
|
||||
return time.Time{}, false, errors.Wrap(errors.CodeDatabaseError, err, "查询审计在线数据边界失败")
|
||||
}
|
||||
if earliest != nil {
|
||||
return earliest.In(location), false, nil
|
||||
}
|
||||
now := time.Now().In(location)
|
||||
return time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location), false, nil
|
||||
}
|
||||
|
||||
// NormalizeRange 将缺省范围收敛到在线窗口,并拒绝归档或跨边界查询。
|
||||
func NormalizeRange(info Info, from, to *time.Time, maxRange ...time.Duration) (*time.Time, *time.Time, error) {
|
||||
explicitFrom := from != nil
|
||||
if from != nil && info.ArchivedBefore != nil && from.Before(info.OnlineFrom) {
|
||||
return nil, nil, archivedError(info)
|
||||
}
|
||||
if to != nil && info.ArchivedBefore != nil && !to.After(info.OnlineFrom) {
|
||||
return nil, nil, archivedError(info)
|
||||
}
|
||||
if from == nil {
|
||||
value := info.OnlineFrom
|
||||
from = &value
|
||||
}
|
||||
if to == nil {
|
||||
value := time.Now()
|
||||
to = &value
|
||||
}
|
||||
if len(maxRange) > 0 && maxRange[0] > 0 && to.Sub(*from) > maxRange[0] {
|
||||
if explicitFrom {
|
||||
return nil, nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
value := to.Add(-maxRange[0])
|
||||
from = &value
|
||||
}
|
||||
if !from.Before(*to) {
|
||||
return nil, nil, errors.New(errors.CodeInvalidParam)
|
||||
}
|
||||
return from, to, nil
|
||||
}
|
||||
|
||||
func archivedError(info Info) error {
|
||||
return errors.NewWithData(errors.CodeAuditDataArchived, map[string]any{"retention": info})
|
||||
}
|
||||
@@ -15,13 +15,13 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc *
|
||||
agent := router.Group("/agent/resource-activities")
|
||||
Register(agent, doc, basePath+"/agent/resource-activities", "GET", "/:resource_type/:identifier", handler.AgentResourceActivities, RouteSpec{
|
||||
Summary: "查询代理资源活动",
|
||||
Description: "resource_type/identifier 来自代理当前业务页面稳定字段;店铺范围只读取认证上下文。仅返回写入时生成的安全业务结论和白名单详情,越权与不存在同错。",
|
||||
Description: "resource_type/identifier 来自代理当前业务页面稳定字段;店铺范围只读取认证上下文。仅查询 retention 标明的在线窗口,归档范围不从对象存储读取。",
|
||||
Tags: []string{"资源活动"}, Input: new(dto.SubjectResourceActivityRequest), Output: new(auditquery.SubjectActivityPage), Auth: true,
|
||||
})
|
||||
enterprise := router.Group("/enterprise/resource-activities")
|
||||
Register(enterprise, doc, basePath+"/enterprise/resource-activities", "GET", "/:resource_type/:identifier", handler.EnterpriseResourceActivities, RouteSpec{
|
||||
Summary: "查询企业资源活动",
|
||||
Description: "仅支持企业当前有效授权的卡和设备;企业身份只读取认证上下文,授权撤销后立即不可读取。响应不包含平台操作者、风险、内部前后值或外部交互内容。",
|
||||
Description: "仅支持企业当前有效授权的卡和设备;企业身份只读取认证上下文。仅查询 retention 标明的在线窗口,响应不包含平台内部调查字段。",
|
||||
Tags: []string{"资源活动"}, Input: new(dto.SubjectResourceActivityRequest), Output: new(auditquery.SubjectActivityPage), Auth: true,
|
||||
})
|
||||
|
||||
@@ -30,13 +30,13 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc *
|
||||
|
||||
Register(audit, doc, groupPath, "GET", "/events", handler.ListEvents, RouteSpec{
|
||||
Summary: "查询全局审计事件",
|
||||
Description: "筛选值来自调查人员输入或其他调查节点的稳定引用;身份范围只读取认证上下文。固定按发生时间和事件ID倒序,不提供导出、修改或删除。",
|
||||
Description: "筛选值来自调查人员输入或其他调查节点的稳定引用;缺省只查 retention 标明的在线窗口,归档范围返回稳定错误。固定倒序分页,不提供导出、修改或删除。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditEventListRequest), Output: new(auditquery.EventPage), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/events/:event_id", handler.GetEvent, RouteSpec{
|
||||
Summary: "查询审计事件详情",
|
||||
Description: "event_id 来自事件、资源、操作者或链路节点的 investigation_refs;返回全部资源快照和各资源 before/after。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditEventIDParams), Output: new(auditquery.EventView), Auth: true,
|
||||
Description: "event_id 来自 investigation_refs;只查询在线 PostgreSQL,未命中仍返回资源不存在,不扫描对象存储。返回全部资源快照和各资源 before/after。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditEventIDParams), Output: new(auditquery.EventDetail), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/actors/:kind/:id/events", handler.ListActorEvents, RouteSpec{
|
||||
Summary: "查询操作者行为时间线",
|
||||
@@ -54,20 +54,45 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc *
|
||||
Description: "resource_type/resource_id 必须来自业务页面稳定字段、资源搜索结果或 investigation_refs。事件在资源作为 primary、affected 或 reference 时均会返回。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditResourceTimelineRequest), Output: new(auditquery.EventPage), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/requests/:request_id/timeline", handler.RequestTimeline, RouteSpec{
|
||||
Summary: "查询请求关联时间线",
|
||||
Description: "request_id 来自审计或外部集成节点,也可从 Access Log 粘贴。只组合 retention 在线窗口内的持久化事实,不扫描 Access Log 或对象存储。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditRequestTimelineParams), Output: new(auditquery.LinkTimeline), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/correlations/:correlation_id/timeline", handler.CorrelationTimeline, RouteSpec{
|
||||
Summary: "查询业务关联时间线",
|
||||
Description: "correlation_id 来自稳定调查引用。只组合 retention 在线窗口内的持久化事实;相同 correlation 不用于猜测技术重试。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditCorrelationTimelineParams), Output: new(auditquery.LinkTimeline), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/finance/timeline", handler.FinanceTimeline, RouteSpec{
|
||||
Summary: "查询资金调查时间线",
|
||||
Description: "可使用任一稳定资金条件进入;缺省只查 retention 在线窗口,归档范围不返回部分结果。关联事实由服务端解析,金额以业务账本为权威。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditFinanceTimelineRequest), Output: new(auditquery.FinanceTimelinePage), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/risks/overview", handler.RiskOverview, RouteSpec{
|
||||
Summary: "查询风险调查总览",
|
||||
Description: "时间范围最长31天,缺省时使用当前在线窗口;只聚合高风险、资金、安全、失败、拒绝、部分成功和结果未知事件。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditRiskOverviewRequest), Output: new(auditquery.RiskOverview), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/risks/events", handler.RiskEvents, RouteSpec{
|
||||
Summary: "查询风险事件明细",
|
||||
Description: "筛选条件来自风险总览分桶或调查人员输入,缺省只查 retention 在线窗口;明细返回 investigation_refs,不提供处置或封禁能力。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.AuditRiskEventsRequest), Output: new(auditquery.RiskEventPage), Auth: true,
|
||||
})
|
||||
// Integration 总览静态路径必须先于动态详情路径,避免 overview 被当作 integration_id。
|
||||
Register(audit, doc, groupPath, "GET", "/integrations/overview", handler.IntegrationOverview, RouteSpec{
|
||||
Summary: "查询外部集成交互总览",
|
||||
Description: "筛选和时间范围来自调查输入或关联视角跳转,身份只来自认证上下文。总览区分成功、处理中、结果不确定、失败和未发送终态。",
|
||||
Description: "筛选和时间范围来自调查输入或关联视角跳转;缺省只查 retention 在线窗口,归档范围返回稳定错误。总览区分五类结果。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.IntegrationOverviewRequest), Output: new(integrationquery.Overview), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/integrations", handler.ListIntegrations, RouteSpec{
|
||||
Summary: "查询外部集成交互列表",
|
||||
Description: "组合筛选来自调查输入或关联视角稳定引用,固定按创建时间和记录ID倒序分页,不提供任意摘要搜索。",
|
||||
Description: "组合筛选来自调查输入或稳定引用;缺省只查 retention 在线窗口,归档范围不返回空页或部分结果。固定倒序分页,不提供任意摘要搜索。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.IntegrationListRequest), Output: new(integrationquery.ListPage), Auth: true,
|
||||
})
|
||||
Register(audit, doc, groupPath, "GET", "/integrations/:integration_id", handler.GetIntegration, RouteSpec{
|
||||
Summary: "查询外部集成交互详情",
|
||||
Description: "integration_id 来自列表、通知目标 target_key 或调查节点稳定引用;只展示结构化详情和显式尝试序列,不提供重试、补偿、确认、绑定、恢复、修改、删除或导出。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.IntegrationIDParams), Output: new(integrationquery.Detail), Auth: true,
|
||||
Description: "integration_id 来自稳定引用;只查询在线 PostgreSQL,未命中仍返回资源不存在。展示结构化详情和在线尝试序列,不提供归档读取、恢复、修改、删除或导出。",
|
||||
Tags: []string{"审计调查"}, Input: new(dto.IntegrationIDParams), Output: new(integrationquery.DetailResponse), Auth: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -83,10 +83,6 @@ func (s *Service) Get(ctx context.Context, id uint) (*dto.CarrierResponse, error
|
||||
}
|
||||
return nil, errors.Wrap(errors.CodeInternalError, err, "获取运营商失败")
|
||||
}
|
||||
if s.audit == nil {
|
||||
return nil, errors.New(errors.CodeInvalidStatus, "运营商配置审计接缝未配置")
|
||||
}
|
||||
before := *carrier
|
||||
return s.toResponse(carrier), nil
|
||||
}
|
||||
|
||||
@@ -103,6 +99,7 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateCarrierReq
|
||||
}
|
||||
return nil, errors.Wrap(errors.CodeInternalError, err, "获取运营商失败")
|
||||
}
|
||||
before := *carrier
|
||||
|
||||
if req.CarrierName != nil {
|
||||
carrier.CarrierName = *req.CarrierName
|
||||
|
||||
@@ -65,12 +65,12 @@ func (s *Service) AllocateDevices(ctx context.Context, enterpriseID uint, req *d
|
||||
return nil, errors.New(errors.CodeInvalidStatus, "企业设备授权审计接缝未配置")
|
||||
}
|
||||
if err := validateEnterpriseDeviceActor(ctx); err != nil {
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, err)
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, err)
|
||||
return nil, err
|
||||
}
|
||||
if err := middleware.CanManageEnterprise(ctx, enterpriseID, s.enterpriseStore); err != nil {
|
||||
permissionErr := errors.New(errors.CodeForbidden, "无权限操作该资源或资源不存在")
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, permissionErr)
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, permissionErr)
|
||||
return nil, permissionErr
|
||||
}
|
||||
enterprise, err := s.enterpriseStore.GetByID(ctx, enterpriseID)
|
||||
@@ -417,12 +417,12 @@ func (s *Service) RecallDevices(ctx context.Context, enterpriseID uint, req *dto
|
||||
return nil, errors.New(errors.CodeInvalidStatus, "企业设备授权审计接缝未配置")
|
||||
}
|
||||
if err := validateEnterpriseDeviceActor(ctx); err != nil {
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, err)
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, err)
|
||||
return nil, err
|
||||
}
|
||||
if err := middleware.CanManageEnterprise(ctx, enterpriseID, s.enterpriseStore); err != nil {
|
||||
permissionErr := errors.New(errors.CodeForbidden, "无权限操作该资源或资源不存在")
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, permissionErr)
|
||||
s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, permissionErr)
|
||||
return nil, permissionErr
|
||||
}
|
||||
enterprise, err := s.enterpriseStore.GetByID(ctx, enterpriseID)
|
||||
|
||||
@@ -23,38 +23,46 @@ func (s *Service) SetAccessAudit(writer *audit.Writer) {
|
||||
s.auditWriter = writer
|
||||
}
|
||||
|
||||
// WriteCardStateAudit 将卡观测事务中的人工状态操作写入统一 Audit Event。
|
||||
// WriteCardStateAudit 将卡观测事务中的人工、回调或 Worker 状态变化写入统一 Audit Event。
|
||||
func (s *Service) WriteCardStateAudit(ctx context.Context, tx *gorm.DB, input cardapp.StateAudit) error {
|
||||
extraResources := make([]audit.ResourceInput, 0, 1)
|
||||
if input.IntegrationID != "" {
|
||||
extraResources = append(extraResources, callbackIntegrationAuditResource(ctx, input.IntegrationID))
|
||||
extraResources = append(extraResources, cardStateIntegrationAuditResource(ctx, input.IntegrationID))
|
||||
}
|
||||
return s.appendCardLifecycleAudit(ctx, tx, input.ActionCode, input.Summary, constants.AuditResultSuccess,
|
||||
input.Card, input.BeforeData, input.AfterData, nil, extraResources...)
|
||||
}
|
||||
|
||||
// WriteCardStateFailure 使用独立短事务记录已解析卡资源后的回调失败。
|
||||
// WriteCardStateFailure 使用独立短事务记录已解析卡资源后的回调或 Worker 失败。
|
||||
func (s *Service) WriteCardStateFailure(ctx context.Context, input cardapp.StateAudit, businessErr error) {
|
||||
extraResources := make([]audit.ResourceInput, 0, 1)
|
||||
if input.IntegrationID != "" {
|
||||
extraResources = append(extraResources, callbackIntegrationAuditResource(ctx, input.IntegrationID))
|
||||
extraResources = append(extraResources, cardStateIntegrationAuditResource(ctx, input.IntegrationID))
|
||||
}
|
||||
s.recordCardLifecycleFailure(ctx, input.ActionCode, input.Summary, constants.AuditResultFailed,
|
||||
input.Card, input.Card.ID, businessErr, extraResources...)
|
||||
}
|
||||
|
||||
func callbackIntegrationAuditResource(ctx context.Context, integrationID string) audit.ResourceInput {
|
||||
func cardStateIntegrationAuditResource(ctx context.Context, integrationID string) audit.ResourceInput {
|
||||
linkage := auditcontext.From(ctx)
|
||||
correlationID := linkage.CorrelationID
|
||||
if correlationID == "" {
|
||||
correlationID = linkage.RequestID
|
||||
}
|
||||
direction := constants.IntegrationDirectionInbound
|
||||
role := constants.AuditResourceRoleCallbackIntegration
|
||||
provider := linkage.ActorID
|
||||
if linkage.Source == constants.AuditSourceWorker {
|
||||
direction = constants.IntegrationDirectionOutbound
|
||||
role = constants.AuditResourceRoleWorkerIntegration
|
||||
provider = constants.IntegrationProviderGateway
|
||||
}
|
||||
return audit.ResourceInput{
|
||||
Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: integrationID,
|
||||
Relation: constants.AuditResourceRelationReference, Role: constants.AuditResourceRoleCallbackIntegration,
|
||||
Relation: constants.AuditResourceRelationReference, Role: role,
|
||||
IdentitySnapshot: map[string]any{
|
||||
"integration_id": integrationID, "provider": linkage.ActorID,
|
||||
"direction": constants.IntegrationDirectionInbound, "correlation_id": correlationID,
|
||||
"integration_id": integrationID, "provider": provider,
|
||||
"direction": direction, "correlation_id": correlationID,
|
||||
},
|
||||
SubjectVisibility: constants.AuditSubjectInternalOnly,
|
||||
}
|
||||
|
||||
@@ -2709,8 +2709,12 @@ func (s *Service) enqueueCommissionCalculation(ctx context.Context, orderID uint
|
||||
return
|
||||
}
|
||||
|
||||
linkage := auditcontext.From(ctx)
|
||||
// 直接传 map,由 EnqueueTask 内部统一序列化一次(传 []byte 会导致 sonic.Marshal 二次 base64 编码)
|
||||
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeCommission, map[string]any{"order_id": orderID}); err != nil {
|
||||
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeCommission, map[string]any{
|
||||
"order_id": orderID, "request_id": linkage.RequestID, "correlation_id": linkage.CorrelationID,
|
||||
"parent_event_id": linkage.ParentEventID,
|
||||
}); err != nil {
|
||||
s.logger.Error("佣金计算任务入队失败",
|
||||
zap.Uint("order_id", orderID),
|
||||
zap.Error(err),
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// ResumeCallback 复机回调接口
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
"github.com/break/junhong_cmp_fiber/internal/task"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||
@@ -227,7 +228,11 @@ func (s *Service) HandlePaymentCallback(ctx context.Context, paymentNo string, p
|
||||
|
||||
linkedIDs := rechargeOrder.LinkedPackageIDs
|
||||
if len(linkedIDs) > 0 {
|
||||
taskPayload := task.AutoPurchasePayload{RechargeOrderID: rechargeOrder.ID}
|
||||
linkage := auditcontext.From(ctx)
|
||||
taskPayload := task.AutoPurchasePayload{
|
||||
RechargeOrderID: rechargeOrder.ID, RequestID: linkage.RequestID,
|
||||
CorrelationID: linkage.CorrelationID, ParentEventID: linkage.ParentEventID,
|
||||
}
|
||||
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeAutoPurchaseAfterRecharge, taskPayload,
|
||||
asynq.MaxRetry(3),
|
||||
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAutoPurchaseAfterRecharge)),
|
||||
|
||||
@@ -132,12 +132,12 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateRoleReques
|
||||
if req.RoleName != nil && *req.RoleName != role.RoleName {
|
||||
exists, err := s.roleStore.ExistsByName(ctx, *req.RoleName, id)
|
||||
if err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, beforeData, nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, nil, beforeData, err)
|
||||
return nil, errors.Wrap(errors.CodeInternalError, err, "检查角色名失败")
|
||||
}
|
||||
if exists {
|
||||
appErr := errors.New(errors.CodeRoleNameExists)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "拒绝更新重复角色名", constants.AuditResultDenied, role, beforeData, nil, appErr)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "拒绝更新重复角色名", constants.AuditResultDenied, role, nil, beforeData, appErr)
|
||||
return nil, appErr
|
||||
}
|
||||
role.RoleName = *req.RoleName
|
||||
@@ -162,7 +162,7 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateRoleReques
|
||||
OperatorID: currentUserID, Role: role, BeforeData: beforeData, AfterData: roleAuditData(role),
|
||||
})
|
||||
}); err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, beforeData, nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, nil, beforeData, err)
|
||||
return nil, errors.Wrap(errors.CodeInternalError, err, "更新角色失败")
|
||||
}
|
||||
|
||||
@@ -181,19 +181,19 @@ func (s *Service) Delete(ctx context.Context, id uint) error {
|
||||
|
||||
accountCount, err := s.accountRoleStore.CountByRoleID(ctx, id)
|
||||
if err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
|
||||
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
|
||||
}
|
||||
|
||||
shopCount, err := s.shopRoleStore.CountByRoleID(ctx, id)
|
||||
if err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
|
||||
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
|
||||
}
|
||||
|
||||
if accountCount > 0 || shopCount > 0 {
|
||||
appErr := errors.New(errors.CodeRoleInUse, fmt.Sprintf("该角色已分配给 %d 个账号、%d 个店铺,请先移除相关分配后再删除", accountCount, shopCount))
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "拒绝删除使用中的角色", constants.AuditResultDenied, role, roleAuditData(role), nil, appErr)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "拒绝删除使用中的角色", constants.AuditResultDenied, role, nil, roleAuditData(role), appErr)
|
||||
return appErr
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ func (s *Service) Delete(ctx context.Context, id uint) error {
|
||||
OperatorID: operatorID, Role: role, BeforeData: beforeData, AfterData: map[string]any{"deleted": true},
|
||||
})
|
||||
}); err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, beforeData, nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, beforeData, err)
|
||||
return errors.Wrap(errors.CodeInternalError, err, "删除角色失败")
|
||||
}
|
||||
|
||||
@@ -510,19 +510,19 @@ func (s *Service) UpdateStatus(ctx context.Context, id uint, status int) error {
|
||||
if status == constants.StatusDisabled {
|
||||
accountCount, err := s.accountRoleStore.CountByRoleID(ctx, id)
|
||||
if err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
|
||||
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
|
||||
}
|
||||
|
||||
shopCount, err := s.shopRoleStore.CountByRoleID(ctx, id)
|
||||
if err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err)
|
||||
return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败")
|
||||
}
|
||||
|
||||
if accountCount > 0 || shopCount > 0 {
|
||||
appErr := errors.New(errors.CodeRoleInUse, fmt.Sprintf("该角色已分配给 %d 个账号、%d 个店铺,请先移除相关分配后再禁用", accountCount, shopCount))
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "拒绝禁用使用中的角色", constants.AuditResultDenied, role, roleAuditData(role), nil, appErr)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "拒绝禁用使用中的角色", constants.AuditResultDenied, role, nil, roleAuditData(role), appErr)
|
||||
return appErr
|
||||
}
|
||||
}
|
||||
@@ -540,7 +540,7 @@ func (s *Service) UpdateStatus(ctx context.Context, id uint, status int) error {
|
||||
OperatorID: currentUserID, Role: role, BeforeData: beforeData, AfterData: roleAuditData(role),
|
||||
})
|
||||
}); err != nil {
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, beforeData, nil, err)
|
||||
s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, beforeData, err)
|
||||
return errors.Wrap(errors.CodeInternalError, err, "更新角色状态失败")
|
||||
}
|
||||
|
||||
|
||||
61
internal/task/audit_daily_archive.go
Normal file
61
internal/task/audit_daily_archive.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// AuditDailyArchivePayload 是人工补档时可选的任务载荷。
|
||||
type AuditDailyArchivePayload struct {
|
||||
ArchiveDate string `json:"archive_date"`
|
||||
}
|
||||
|
||||
// AuditDailyArchiveHandler 处理统一审计每日冷归档任务。
|
||||
type AuditDailyArchiveHandler struct {
|
||||
service *auditarchive.Service
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewAuditDailyArchiveHandler 创建统一审计每日冷归档任务处理器。
|
||||
func NewAuditDailyArchiveHandler(service *auditarchive.Service, logger *zap.Logger) *AuditDailyArchiveHandler {
|
||||
return &AuditDailyArchiveHandler{service: service, logger: logger}
|
||||
}
|
||||
|
||||
// Handle 执行前一完整自然日归档,或按任务载荷补档指定自然日。
|
||||
func (h *AuditDailyArchiveHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("统一审计归档服务未配置")
|
||||
}
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
err = h.service.ArchivePreviousDay(ctx)
|
||||
} else {
|
||||
var payload AuditDailyArchivePayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析统一审计归档任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
location, locationErr := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if locationErr != nil {
|
||||
return fmt.Errorf("加载统一审计归档时区失败: %w", locationErr)
|
||||
}
|
||||
archiveDate, parseErr := time.ParseInLocation(time.DateOnly, payload.ArchiveDate, location)
|
||||
if parseErr != nil {
|
||||
return fmt.Errorf("解析统一审计归档日期失败: %w", parseErr)
|
||||
}
|
||||
err = h.service.ArchiveDate(ctx, archiveDate)
|
||||
}
|
||||
if err != nil {
|
||||
h.logger.Error("统一审计每日冷归档失败", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
h.logger.Info("统一审计每日冷归档完成")
|
||||
return nil
|
||||
}
|
||||
64
internal/task/audit_monthly_retention.go
Normal file
64
internal/task/audit_monthly_retention.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
|
||||
)
|
||||
|
||||
// AuditMonthlyRetentionPayload 是人工补跑月度清理时可选的任务载荷。
|
||||
type AuditMonthlyRetentionPayload struct {
|
||||
ArchiveMonth string `json:"archive_month"`
|
||||
}
|
||||
|
||||
// AuditMonthlyRetentionHandler 处理归档完整性门禁与上月在线日志物理清理。
|
||||
type AuditMonthlyRetentionHandler struct {
|
||||
service *auditarchive.Service
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewAuditMonthlyRetentionHandler 创建月度日志留存清理处理器。
|
||||
func NewAuditMonthlyRetentionHandler(service *auditarchive.Service, logger *zap.Logger) *AuditMonthlyRetentionHandler {
|
||||
return &AuditMonthlyRetentionHandler{service: service, logger: logger}
|
||||
}
|
||||
|
||||
// Handle 校验整月归档后按固定顺序分批物理删除 PostgreSQL 在线日志。
|
||||
func (h *AuditMonthlyRetentionHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("月度日志留存清理服务未配置")
|
||||
}
|
||||
startedAt := time.Now()
|
||||
var result auditarchive.RetentionResult
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
result, err = h.service.CleanupPreviousMonth(ctx)
|
||||
} else {
|
||||
var payload AuditMonthlyRetentionPayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析月度日志留存清理任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
result, err = h.service.CleanupMonth(ctx, month)
|
||||
}
|
||||
fields := []zap.Field{
|
||||
zap.String("archive_month", result.Month), zap.Int64("audit_event_count", result.EventCount),
|
||||
zap.Int64("event_resource_count", result.ResourceCount), zap.Int64("integration_log_count", result.IntegrationCount),
|
||||
zap.Duration("duration", time.Since(startedAt)), zap.Int("manifest_count", len(result.ManifestKeys)),
|
||||
}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.String("severity", "critical"), zap.Error(err))
|
||||
h.logger.Error("月度日志留存清理失败,PostgreSQL 整月清理已阻断或等待断点续跑", fields...)
|
||||
return err
|
||||
}
|
||||
h.logger.Info("月度日志留存清理完成", fields...)
|
||||
return nil
|
||||
}
|
||||
@@ -27,7 +27,10 @@ import (
|
||||
|
||||
// AutoPurchasePayload 充值后自动购包任务载荷
|
||||
type AutoPurchasePayload struct {
|
||||
RechargeOrderID uint `json:"recharge_order_id"`
|
||||
RechargeOrderID uint `json:"recharge_order_id"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
ParentEventID string `json:"parent_event_id,omitempty"`
|
||||
}
|
||||
|
||||
// AutoPurchaseHandler 充值后自动购包任务处理器
|
||||
@@ -123,10 +126,14 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task)
|
||||
h.logger.Error("查询充值订单失败", zap.Uint("recharge_order_id", payload.RechargeOrderID), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
correlationID := payload.CorrelationID
|
||||
if correlationID == "" {
|
||||
correlationID = rechargeOrder.RechargeOrderNo
|
||||
}
|
||||
ctx = auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeAutoPurchaseAfterRecharge,
|
||||
ActorName: "充值后自动购包任务", Source: constants.AuditSourceWorker,
|
||||
CorrelationID: rechargeOrder.RechargeOrderNo,
|
||||
RequestID: payload.RequestID, CorrelationID: correlationID, ParentEventID: payload.ParentEventID,
|
||||
})
|
||||
|
||||
if rechargeOrder.AutoPurchaseStatus == constants.AutoPurchaseStatusSuccess {
|
||||
@@ -301,7 +308,11 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task)
|
||||
|
||||
// 事务提交成功后触发佣金计算(不在事务内,防止任务提交后事务回滚的数据一致性问题)
|
||||
if h.asynqClient != nil && createdOrderID > 0 {
|
||||
payloadBytes, marshalErr := sonic.Marshal(map[string]any{"order_id": createdOrderID})
|
||||
linkage := auditcontext.From(ctx)
|
||||
payloadBytes, marshalErr := sonic.Marshal(CommissionCalculationPayload{
|
||||
OrderID: createdOrderID, RequestID: linkage.RequestID,
|
||||
CorrelationID: linkage.CorrelationID, ParentEventID: linkage.ParentEventID,
|
||||
})
|
||||
if marshalErr != nil {
|
||||
h.logger.Warn("佣金任务载荷序列化失败",
|
||||
zap.Uint("order_id", createdOrderID),
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"go.uber.org/zap"
|
||||
|
||||
cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
@@ -32,6 +34,11 @@ func (h *CardObservationSeriesHandler) Handle(ctx context.Context, task *asynq.T
|
||||
h.logger.Error("解析卡观测序列任务载荷失败", zap.Error(err))
|
||||
return errors.Wrap(errors.CodeInvalidParam, err, "卡观测序列任务载荷无法解析")
|
||||
}
|
||||
ctx = auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeCardObservationSeries,
|
||||
ActorName: "卡观测序列任务", Source: constants.AuditSourceWorker,
|
||||
RequestID: payload.RequestID, CorrelationID: payload.CorrelationID, ParentEventID: payload.ParentEventID,
|
||||
})
|
||||
if err := h.service.Execute(ctx, payload); err != nil {
|
||||
h.logger.Warn("卡观测序列当前尝试失败",
|
||||
zap.String("series_id", payload.SeriesID), zap.Int("attempt", payload.Attempt), zap.Error(err))
|
||||
|
||||
@@ -18,7 +18,10 @@ const (
|
||||
)
|
||||
|
||||
type CommissionCalculationPayload struct {
|
||||
OrderID uint `json:"order_id"`
|
||||
OrderID uint `json:"order_id"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
ParentEventID string `json:"parent_event_id,omitempty"`
|
||||
}
|
||||
|
||||
type CommissionCalculationHandler struct {
|
||||
@@ -48,9 +51,15 @@ func (h *CommissionCalculationHandler) HandleCommissionCalculation(ctx context.C
|
||||
)
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
correlationID := payload.CorrelationID
|
||||
if correlationID == "" {
|
||||
correlationID = task.ResultWriter().TaskID()
|
||||
}
|
||||
ctx = auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: constants.AuditActorIDCommissionCalculationWorker,
|
||||
ActorName: "订单佣金计算任务", Source: constants.AuditSourceWorker,
|
||||
RequestID: payload.RequestID, CorrelationID: correlationID,
|
||||
ParentEventID: payload.ParentEventID,
|
||||
})
|
||||
|
||||
if err := h.service.CalculateCommission(ctx, payload.OrderID); err != nil {
|
||||
|
||||
113
internal/task/integration_archive.go
Normal file
113
internal/task/integration_archive.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/application/auditarchive"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// IntegrationDailyArchivePayload 是人工补档时可选的任务载荷。
|
||||
type IntegrationDailyArchivePayload struct {
|
||||
ArchiveDate string `json:"archive_date"`
|
||||
}
|
||||
|
||||
// IntegrationMonthlyFinalizePayload 是人工月度复核时可选的任务载荷。
|
||||
type IntegrationMonthlyFinalizePayload struct {
|
||||
ArchiveMonth string `json:"archive_month"`
|
||||
}
|
||||
|
||||
// IntegrationArchiveHandler 处理 Integration Log 每日归档与月度最终复核。
|
||||
type IntegrationArchiveHandler struct {
|
||||
service *auditarchive.Service
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewIntegrationArchiveHandler 创建 Integration Log 归档任务处理器。
|
||||
func NewIntegrationArchiveHandler(service *auditarchive.Service, logger *zap.Logger) *IntegrationArchiveHandler {
|
||||
return &IntegrationArchiveHandler{service: service, logger: logger}
|
||||
}
|
||||
|
||||
// HandleDaily 执行前一完整自然日归档,或按任务载荷补档指定自然日。
|
||||
func (h *IntegrationArchiveHandler) HandleDaily(ctx context.Context, task *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("Integration Log 归档服务未配置")
|
||||
}
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
err = h.service.ArchivePreviousIntegrationDay(ctx)
|
||||
} else {
|
||||
var payload IntegrationDailyArchivePayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析 Integration Log 每日归档任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
date, parseErr := parseArchiveDate(payload.ArchiveDate)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
err = h.service.ArchiveIntegrationDate(ctx, date)
|
||||
}
|
||||
if err != nil {
|
||||
h.logger.Error("Integration Log 每日冷归档失败", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
h.logger.Info("Integration Log 每日冷归档完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// HandleMonthlyFinalize 执行上一个完整自然月复核,或按任务载荷复核指定月份。
|
||||
func (h *IntegrationArchiveHandler) HandleMonthlyFinalize(ctx context.Context, task *asynq.Task) error {
|
||||
if h.service == nil {
|
||||
return fmt.Errorf("Integration Log 归档服务未配置")
|
||||
}
|
||||
var err error
|
||||
if len(task.Payload()) == 0 {
|
||||
err = h.service.FinalizePreviousIntegrationMonth(ctx)
|
||||
} else {
|
||||
var payload IntegrationMonthlyFinalizePayload
|
||||
if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil {
|
||||
return fmt.Errorf("解析 Integration Log 月度复核任务载荷失败: %w", unmarshalErr)
|
||||
}
|
||||
month, parseErr := parseArchiveMonth(payload.ArchiveMonth)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
err = h.service.FinalizeIntegrationMonth(ctx, month)
|
||||
}
|
||||
if err != nil {
|
||||
h.logger.Error("Integration Log 月度最终版本复核失败,后续清理必须阻止", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
h.logger.Info("Integration Log 月度最终版本复核完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseArchiveDate(value string) (time.Time, error) {
|
||||
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err)
|
||||
}
|
||||
date, err := time.ParseInLocation(time.DateOnly, value, location)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("解析 Integration Log 归档日期失败: %w", err)
|
||||
}
|
||||
return date, nil
|
||||
}
|
||||
|
||||
func parseArchiveMonth(value string) (time.Time, error) {
|
||||
location, err := time.LoadLocation(constants.AuditArchiveTimezone)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err)
|
||||
}
|
||||
month, err := time.ParseInLocation("2006-01", value, location)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("解析 Integration Log 归档月份失败: %w", err)
|
||||
}
|
||||
return month, nil
|
||||
}
|
||||
@@ -23,10 +23,11 @@ func NewNotificationCleanupHandler(service *notificationinfra.CleanupService, lo
|
||||
}
|
||||
|
||||
// Handle 执行有界、可重入的通知分批清理。
|
||||
func (h *NotificationCleanupHandler) Handle(ctx context.Context, _ *asynq.Task) error {
|
||||
func (h *NotificationCleanupHandler) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
ctx = auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeNotificationCleanup,
|
||||
ActorName: "站内通知清理任务", Source: constants.AuditSourceWorker,
|
||||
CorrelationID: task.ResultWriter().TaskID(),
|
||||
})
|
||||
h.logger.Info("开始执行站内通知保留清理")
|
||||
if err := h.service.Run(ctx); err != nil {
|
||||
|
||||
@@ -77,6 +77,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 流量 Integration Log 失败", logErr)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil || h.carrier == nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil)
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task)
|
||||
if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 网络 Integration Log 失败", logErr)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil)
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID())
|
||||
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
|
||||
@@ -52,6 +52,7 @@ func (h *PollingProtectHandler) Handle(ctx context.Context, t *asynq.Task) error
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingProtect, "保护期一致性轮询任务", t.ResultWriter().TaskID())
|
||||
|
||||
if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingProtect) {
|
||||
h.base.logger.Debug("并发已满,重新入队", zap.Uint("card_id", cardID))
|
||||
|
||||
@@ -68,6 +68,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e
|
||||
if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 实名 Integration Log 失败", logErr)
|
||||
}
|
||||
ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID)
|
||||
if h.observation == nil {
|
||||
return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil)
|
||||
}
|
||||
|
||||
@@ -1,14 +1,29 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// withPollingWorkerAuditContext 为实际改变业务事实的轮询任务补充真实系统操作者与链路。
|
||||
func withPollingWorkerAuditContext(ctx context.Context, taskType, taskName, correlationID string) context.Context {
|
||||
if correlationID == "" {
|
||||
correlationID = taskType
|
||||
}
|
||||
return auditcontext.With(ctx, auditcontext.Context{
|
||||
ActorKind: constants.AuditActorSystemTask, ActorID: taskType,
|
||||
ActorName: taskName, Source: constants.AuditSourceWorker, CorrelationID: correlationID,
|
||||
})
|
||||
}
|
||||
|
||||
// shortTaskType 从完整任务类型中提取简短名称(如 polling:carddata → carddata)
|
||||
func shortTaskType(fullTaskType string) string {
|
||||
for i := len(fullTaskType) - 1; i >= 0; i-- {
|
||||
|
||||
Reference in New Issue
Block a user