Files
junhong_cmp_fiber/internal/infrastructure/integrationlog/repository.go
邹进 b7a7430cc3
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 12m8s
feat: 审计动作按落库策略分级写入,系统过程动作改写应用日志
- 审计注册表为可由系统任务、定时任务发起的动作声明落库策略(always/on_success/outcome/log_only/none),未声明按 log_only
- 人工、开放接口、个人客户、外部系统发起的动作始终写审计表
- log_only 写所在服务 app.log(logger=audit),不含快照
- 对接日志不再补写 integration.attempt_started,无上游审计时 audit_event_id 为空
- 新增 audit.persist_overrides 按动作覆盖策略
- 审计覆盖扫描要求机器动作必须声明策略
- 新增 OpenSpec 提案 refactor-audit-log-persistence 与技术方案文档

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 18:27:24 +09:00

492 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package integrationlog 提供外部交互尝试的可靠持久化能力。
package integrationlog
import (
"context"
"crypto/sha256"
"encoding/hex"
"strings"
"time"
"unicode/utf8"
"github.com/google/uuid"
"go.uber.org/zap"
"gorm.io/datatypes"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/break/junhong_cmp_fiber/internal/infrastructure/audit"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/pkg/auditcontext"
"github.com/break/junhong_cmp_fiber/pkg/constants"
pkgerrors "github.com/break/junhong_cmp_fiber/pkg/errors"
"github.com/break/junhong_cmp_fiber/pkg/sanitizer"
)
// Attempt 描述一次外部调用前必须持久化的稳定事实。
type Attempt struct {
IntegrationID string
Provider string
Direction string
Operation string
ExternalID *string
ResourceType string
ResourceID *string
ResourceKey *string
TriggerSource *string
TriggerScene *string
TriggerSeries *string
ScheduledAt *time.Time
StartedAt *time.Time
Attempt int
RequestSummary any
Metadata any
RequestID *string
CorrelationID *string
AuditEventID *uint
InitialResult string
StateChanged bool
RecoveryStrategy *string
}
// Completion 描述外部尝试从待处理状态进入终态的结果。
type Completion struct {
Result string
HTTPStatus int
ProviderCode string
ProviderMessage string
SafeProviderMessage string
ResponseSummary any
DurationMS int64
StateChanged bool
ResourceID *string
ResourceKey *string
RecoveryStrategy string
}
// InboundAttempt 描述业务处理前必须保存的入站回调安全事实。
type InboundAttempt struct {
IntegrationID string
IdempotencyKey string
Provider string
Operation string
ExternalID string
ResourceType string
ResourceID *string
ResourceKey *string
RawPayload []byte
ContentType string
RequestID *string
CorrelationID *string
AuditEventID *uint
}
// Repository 负责创建稳定尝试及受控地进入终态。
type Repository struct {
db *gorm.DB
now func() time.Time
audit *audit.Writer
}
// NewRepository 创建 Integration Log Repository。
func NewRepository(db *gorm.DB) *Repository {
return &Repository{db: db, now: time.Now, audit: audit.NewWriter(audit.NewRegistry(), nil)}
}
// Start 在实际调用外部系统前持久化尝试事实。
func (r *Repository) Start(ctx context.Context, input Attempt) (*model.IntegrationLog, error) {
if r == nil || r.db == nil {
return nil, pkgerrors.New(pkgerrors.CodeInvalidStatus, "Integration Log 数据库未配置")
}
if err := validateAttempt(input); err != nil {
return nil, err
}
requestSummary, err := marshalSummary(input.RequestSummary)
if err != nil {
return nil, pkgerrors.Wrap(pkgerrors.CodeInvalidParam, err, "Integration Log 请求摘要无效")
}
metadata, err := marshalSummary(input.Metadata)
if err != nil {
return nil, pkgerrors.Wrap(pkgerrors.CodeInvalidParam, err, "Integration Log 元数据无效")
}
if input.IntegrationID == "" {
input.IntegrationID = uuid.NewString()
}
if input.AuditEventID == nil && input.TriggerSeries != nil {
input.AuditEventID = r.auditEventIDForSeries(ctx, *input.TriggerSeries)
}
// 没有上游审计事件时 audit_event_id 留空,不再补写 integration.attempt_started(与本表内容重复)。
autoAttempt := input.Attempt <= 0 && input.TriggerSeries != nil
if input.Attempt <= 0 {
input.Attempt = 1
}
if input.StartedAt == nil {
startedAt := r.now().UTC()
input.StartedAt = &startedAt
}
result := input.InitialResult
if result == "" {
result = constants.IntegrationResultPending
}
resourceType := optionalString(input.ResourceType)
log := &model.IntegrationLog{
IntegrationID: input.IntegrationID, Provider: input.Provider, Direction: input.Direction,
Operation: input.Operation, ExternalID: sanitizedOptionalText(input.ExternalID), ResourceType: resourceType,
ResourceID: input.ResourceID, ResourceKey: sanitizedOptionalText(input.ResourceKey), TriggerSource: input.TriggerSource,
TriggerScene: sanitizedOptionalText(input.TriggerScene), TriggerSeries: input.TriggerSeries, ScheduledAt: input.ScheduledAt,
StartedAt: input.StartedAt, Attempt: input.Attempt, Result: result, StateChanged: input.StateChanged,
RequestSummary: requestSummary, Metadata: metadata, RequestID: input.RequestID,
CorrelationID: input.CorrelationID, AuditEventID: input.AuditEventID,
RecoveryStrategy: sanitizedOptionalText(input.RecoveryStrategy),
}
createAttempt := func(tx *gorm.DB) error {
if !autoAttempt {
return tx.Create(log).Error
}
if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtext(?))", *input.TriggerSeries).Error; err != nil {
return err
}
if err := tx.Model(&model.IntegrationLog{}).Select("COALESCE(MAX(attempt), 0) + 1").
Where("trigger_series = ?", *input.TriggerSeries).Scan(&log.Attempt).Error; err != nil {
return err
}
return tx.Create(log).Error
}
var createErr error
if autoAttempt {
createErr = r.db.WithContext(ctx).Transaction(createAttempt)
} else {
createErr = createAttempt(r.db.WithContext(ctx))
}
if createErr != nil {
return nil, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, createErr, "写入 Integration Log 失败")
}
return log, nil
}
// Complete 仅允许把待处理尝试条件更新为一个公开终态。
func (r *Repository) Complete(ctx context.Context, integrationID string, completion Completion) (*model.IntegrationLog, error) {
if r == nil || r.db == nil {
return nil, pkgerrors.New(pkgerrors.CodeInvalidStatus, "Integration Log 数据库未配置")
}
if !validRequiredString(integrationID, constants.IntegrationIDMaxLength) ||
!validOptionalString(completion.ResourceID, constants.IntegrationResourceIDMaxLength) ||
!validOptionalString(completion.ResourceKey, constants.IntegrationResourceKeyMaxLength) ||
!isTerminalResult(completion.Result) {
return nil, pkgerrors.New(pkgerrors.CodeInvalidParam, "Integration Log 终态参数无效")
}
if completion.Result == constants.IntegrationResultUnknown && strings.TrimSpace(completion.RecoveryStrategy) == "" {
return nil, pkgerrors.New(pkgerrors.CodeInvalidParam, "结果未知必须记录明确恢复策略")
}
safeProviderMessage := sanitizer.SanitizeText(strings.TrimSpace(completion.SafeProviderMessage))
if safeProviderMessage != "" && utf8.RuneCountInString(constants.IntegrationSafeMessagePrefix+safeProviderMessage) > constants.IntegrationProviderMessageMaxLength {
return nil, pkgerrors.New(pkgerrors.CodeInvalidParam, "Integration Log 安全结果摘要过长")
}
responseSummary, err := marshalSummary(completion.ResponseSummary)
if err != nil {
return nil, pkgerrors.Wrap(pkgerrors.CodeInvalidParam, err, "Integration Log 响应摘要无效")
}
updates := map[string]any{
"result": completion.Result, "duration_ms": completion.DurationMS,
"state_changed": completion.StateChanged, "response_summary": responseSummary,
"updated_at": r.now().UTC(),
}
if completion.HTTPStatus != 0 {
updates["http_status"] = completion.HTTPStatus
}
if completion.ProviderCode != "" {
updates["provider_code"] = completion.ProviderCode
}
if safeProviderMessage != "" {
updates["provider_message"] = constants.IntegrationSafeMessagePrefix + safeProviderMessage
} else if completion.ProviderMessage != "" {
updates["provider_message"] = sanitizer.TextSummary(completion.ProviderMessage)
}
if completion.ResourceID != nil {
updates["resource_id"] = completion.ResourceID
}
if completion.ResourceKey != nil {
updates["resource_key"] = sanitizedOptionalText(completion.ResourceKey)
}
if completion.RecoveryStrategy != "" {
updates["recovery_strategy"] = sanitizer.SanitizeText(completion.RecoveryStrategy)
}
result := r.db.WithContext(ctx).Model(&model.IntegrationLog{}).
Where("integration_id = ? AND result = ?", integrationID, constants.IntegrationResultPending).
Updates(updates)
if result.Error != nil {
return nil, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, result.Error, "终结 Integration Log 失败")
}
if result.RowsAffected != 1 {
return nil, pkgerrors.New(pkgerrors.CodeConflict, "Integration Log 已进入终态或不存在")
}
var saved model.IntegrationLog
if err := r.db.WithContext(ctx).Where("integration_id = ?", integrationID).First(&saved).Error; err != nil {
return nil, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, err, "读取 Integration Log 终态失败")
}
return &saved, nil
}
// RecordInbound 在业务处理前幂等保存入站回调的安全摘要。
func (r *Repository) RecordInbound(ctx context.Context, input InboundAttempt) (*model.IntegrationLog, bool, error) {
if r == nil || r.db == nil {
return nil, false, pkgerrors.New(pkgerrors.CodeInvalidStatus, "Integration Log 数据库未配置")
}
if input.Provider == "" || input.Operation == "" || input.IdempotencyKey == "" ||
!validGeneratedString(input.IntegrationID, constants.IntegrationIDMaxLength) ||
!validOptionalString(input.ResourceID, constants.IntegrationResourceIDMaxLength) ||
!validOptionalString(input.ResourceKey, constants.IntegrationResourceKeyMaxLength) ||
!validOptionalString(input.CorrelationID, constants.IntegrationCorrelationIDMaxLength) {
return nil, false, pkgerrors.New(pkgerrors.CodeInvalidParam, "入站 Integration Log 参数无效")
}
if input.IntegrationID == "" {
input.IntegrationID = uuid.NewString()
}
if input.AuditEventID == nil {
input.AuditEventID = r.recordAuditEvent(ctx, constants.AuditActionIntegrationInboundReceived, input.IntegrationID, input.Provider, constants.IntegrationDirectionInbound, input.Operation, input.ResourceType, input.ResourceID, input.ResourceKey, input.CorrelationID)
}
triggerSeries := input.IntegrationID
hash := sha256.Sum256(input.RawPayload)
summary, err := marshalSummary(map[string]any{
"content_type": input.ContentType,
"payload_bytes": len(input.RawPayload),
"content_hash": hex.EncodeToString(hash[:]),
})
if err != nil {
return nil, false, pkgerrors.Wrap(pkgerrors.CodeInvalidParam, err, "入站 Integration Log 摘要无效")
}
now := r.now().UTC()
log := &model.IntegrationLog{
IntegrationID: input.IntegrationID, IdempotencyKey: &input.IdempotencyKey,
Provider: input.Provider, Direction: constants.IntegrationDirectionInbound, Operation: input.Operation,
ExternalID: sanitizedOptionalText(optionalString(input.ExternalID)), ResourceType: optionalString(input.ResourceType),
ResourceID: input.ResourceID, ResourceKey: sanitizedOptionalText(input.ResourceKey), StartedAt: &now, Attempt: 1,
TriggerSeries: &triggerSeries,
Result: constants.IntegrationResultPending, RequestSummary: summary,
ContentHash: hex.EncodeToString(hash[:]), RequestID: input.RequestID, CorrelationID: input.CorrelationID,
AuditEventID: input.AuditEventID,
}
result := r.db.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "provider"}, {Name: "operation"}, {Name: "idempotency_key"}},
DoNothing: true,
}).Create(log)
if result.Error != nil {
return nil, false, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, result.Error, "写入入站 Integration Log 失败")
}
if result.RowsAffected == 1 {
return log, true, nil
}
var existing model.IntegrationLog
if err := r.db.WithContext(ctx).Where(
"provider = ? AND operation = ? AND idempotency_key = ?", input.Provider, input.Operation, input.IdempotencyKey,
).First(&existing).Error; err != nil {
return nil, false, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, err, "读取重复入站 Integration Log 失败")
}
if existing.ContentHash != hex.EncodeToString(hash[:]) {
return nil, false, pkgerrors.New(pkgerrors.CodeConflict, "入站幂等标识对应的载荷不一致")
}
return &existing, false, nil
}
// recordAuditEvent 为新的外部交互建立稳定审计关联;审计失败不丢失外部事实。
func (r *Repository) recordAuditEvent(ctx context.Context, actionCode, integrationID, provider, direction, operation, resourceType string, resourceID, resourceKey, correlationID *string) *uint {
if r == nil || r.db == nil || r.audit == nil {
zap.L().Warn("Integration Log 缺少审计关联", zap.String("integration_id", integrationID), zap.String("reason", "审计 Writer 未配置"))
return nil
}
value := auditcontext.From(ctx)
if !validAuditOrigin(value.ActorKind, value.ActorID, value.Source) {
value.ActorKind = constants.AuditActorSystemTask
value.ActorID = "integration_log"
value.Source = constants.AuditSourceWorker
}
event, err := r.audit.AppendAndGet(ctx, r.db, audit.AppendInput{
EventID: "integration:" + integrationID,
ActionCode: actionCode, Summary: "记录外部交互审计关联",
Actor: audit.ActorInput{Kind: value.ActorKind, ID: value.ActorID, Name: value.ActorName, ShopID: value.ActorShopID, EnterpriseID: value.ActorEnterpriseID},
Source: value.Source, ScopeType: constants.AuditScopePlatform,
Result: constants.AuditResultSuccess, RequestID: textValue(correlationID), CorrelationID: textValue(correlationID),
Resources: []audit.ResourceInput{{
Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: operation,
Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleWorkerIntegration,
IdentitySnapshot: map[string]any{
"integration_id": integrationID, "provider": provider, "direction": direction, "operation": operation,
"resource_type": resourceType, "resource_id": textValue(resourceID), "resource_key": textValue(resourceKey), "correlation_id": textValue(correlationID),
},
}},
})
if err != nil {
zap.L().Warn("Integration Log 缺少审计关联", zap.String("integration_id", integrationID), zap.Error(err))
return nil
}
if event == nil {
// 落库策略决定不写审计表(只写应用日志或不写)时没有事件可关联。
return nil
}
return &event.ID
}
func (r *Repository) auditEventIDForSeries(ctx context.Context, triggerSeries string) *uint {
if r == nil || r.db == nil || triggerSeries == "" {
return nil
}
var log model.IntegrationLog
if err := r.db.WithContext(ctx).Where("trigger_series = ? AND audit_event_id IS NOT NULL", triggerSeries).Order("attempt DESC").First(&log).Error; err != nil {
return nil
}
return log.AuditEventID
}
func validAuditOrigin(actorKind, actorID, source string) bool {
if actorID == "" {
return false
}
switch source {
case constants.AuditSourceAdminAPI:
return actorKind == constants.AuditActorAccount
case constants.AuditSourcePersonalAPI:
return actorKind == constants.AuditActorPersonalCustomer
case constants.AuditSourceOpenAPI:
return actorKind == constants.AuditActorOpenAPI
case constants.AuditSourceCallback:
return actorKind == constants.AuditActorExternalSystem
case constants.AuditSourceScheduler:
return actorKind == constants.AuditActorScheduledJob
case constants.AuditSourceWorker:
return actorKind == constants.AuditActorSystemTask
default:
return false
}
}
func textValue(value *string) string {
if value == nil {
return ""
}
return *value
}
// ClaimExpiredInboundPending 原子认领已超过处理租约的入站 pending 记录。
func (r *Repository) ClaimExpiredInboundPending(ctx context.Context, integrationID string, lease time.Duration) (bool, error) {
if r == nil || r.db == nil {
return false, pkgerrors.New(pkgerrors.CodeInvalidStatus, "Integration Log 数据库未配置")
}
if strings.TrimSpace(integrationID) == "" || lease <= 0 {
return false, pkgerrors.New(pkgerrors.CodeInvalidParam, "入站 Integration Log 恢复参数无效")
}
now := r.now().UTC()
result := r.db.WithContext(ctx).Model(&model.IntegrationLog{}).
Where("integration_id = ? AND direction = ? AND result = ? AND (started_at IS NULL OR started_at <= ?)",
integrationID, constants.IntegrationDirectionInbound, constants.IntegrationResultPending, now.Add(-lease)).
Updates(map[string]any{
"started_at": now,
"attempt": gorm.Expr("attempt + 1"),
"updated_at": now,
})
if result.Error != nil {
return false, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, result.Error, "认领待恢复入站 Integration Log 失败")
}
return result.RowsAffected == 1, nil
}
// ClaimFailedInbound 原子认领失败的入站回调并恢复为待处理状态。
func (r *Repository) ClaimFailedInbound(ctx context.Context, integrationID string) (bool, error) {
if r == nil || r.db == nil {
return false, pkgerrors.New(pkgerrors.CodeInvalidStatus, "Integration Log 数据库未配置")
}
if strings.TrimSpace(integrationID) == "" {
return false, pkgerrors.New(pkgerrors.CodeInvalidParam, "入站 Integration Log 恢复参数无效")
}
now := r.now().UTC()
result := r.db.WithContext(ctx).Model(&model.IntegrationLog{}).
Where("integration_id = ? AND direction = ? AND result = ?", integrationID, constants.IntegrationDirectionInbound, constants.IntegrationResultFailed).
Updates(map[string]any{
"result": constants.IntegrationResultPending, "started_at": now,
"attempt": gorm.Expr("attempt + 1"), "updated_at": now,
"http_status": nil, "provider_code": nil, "provider_message": nil,
"response_summary": nil, "duration_ms": 0, "state_changed": false, "recovery_strategy": nil,
})
if result.Error != nil {
return false, pkgerrors.Wrap(pkgerrors.CodeDatabaseError, result.Error, "认领失败入站 Integration Log 失败")
}
return result.RowsAffected == 1, nil
}
func validateAttempt(input Attempt) error {
if input.Provider == "" || input.Operation == "" {
return pkgerrors.New(pkgerrors.CodeInvalidParam, "Integration Log 提供方和操作不能为空")
}
if input.Direction != constants.IntegrationDirectionInbound && input.Direction != constants.IntegrationDirectionOutbound {
return pkgerrors.New(pkgerrors.CodeInvalidParam, "Integration Log 方向无效")
}
if input.InitialResult != "" && input.InitialResult != constants.IntegrationResultPending && !isTerminalResult(input.InitialResult) {
return pkgerrors.New(pkgerrors.CodeInvalidParam, "Integration Log 初始结果无效")
}
if !validGeneratedString(input.IntegrationID, constants.IntegrationIDMaxLength) ||
!validOptionalString(input.TriggerSeries, constants.IntegrationTriggerSeriesMaxLength) ||
!validOptionalString(input.CorrelationID, constants.IntegrationCorrelationIDMaxLength) ||
!validOptionalString(input.ResourceID, constants.IntegrationResourceIDMaxLength) ||
!validOptionalString(input.ResourceKey, constants.IntegrationResourceKeyMaxLength) {
return pkgerrors.New(pkgerrors.CodeInvalidParam, "Integration Log 链路或资源标识无效")
}
return nil
}
func validGeneratedString(value string, maxLength int) bool {
return value == "" || validRequiredString(value, maxLength)
}
func validRequiredString(value string, maxLength int) bool {
return strings.TrimSpace(value) == value && value != "" && utf8.RuneCountInString(value) <= maxLength
}
func validOptionalString(value *string, maxLength int) bool {
return value == nil || validRequiredString(*value, maxLength)
}
func isTerminalResult(result string) bool {
switch result {
case constants.IntegrationResultSuccess, constants.IntegrationResultFailed, constants.IntegrationResultUnknown,
constants.IntegrationResultNotFound, constants.IntegrationResultInvalidPayload, constants.IntegrationResultIgnored,
constants.IntegrationResultConflict,
constants.IntegrationResultMerged, constants.IntegrationResultRateLimited, constants.IntegrationResultCompleted,
constants.IntegrationResultCancelled:
return true
default:
return false
}
}
func isUnsentResult(result string) bool {
switch result {
case constants.IntegrationResultIgnored, constants.IntegrationResultMerged, constants.IntegrationResultRateLimited,
constants.IntegrationResultCompleted, constants.IntegrationResultCancelled:
return true
default:
return false
}
}
func marshalSummary(value any) (datatypes.JSON, error) {
if value == nil {
return nil, nil
}
encoded, err := sanitizer.MarshalSummary(value)
return datatypes.JSON(encoded), err
}
func optionalString(value string) *string {
if value == "" {
return nil
}
return &value
}
func sanitizedOptionalText(value *string) *string {
if value == nil {
return nil
}
sanitized := sanitizer.SanitizeText(*value)
return &sanitized
}