实现七月迭代公共技术基础
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m20s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m20s
This commit is contained in:
103
pkg/asynctask/contract.go
Normal file
103
pkg/asynctask/contract.go
Normal file
@@ -0,0 +1,103 @@
|
||||
// Package asynctask 定义业务异步任务的统一公开契约。
|
||||
package asynctask
|
||||
|
||||
import (
|
||||
stderrors "errors"
|
||||
"reflect"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// StatusPending 表示任务待处理。
|
||||
StatusPending = 1
|
||||
// StatusProcessing 表示任务处理中。
|
||||
StatusProcessing = 2
|
||||
// StatusCompleted 表示任务已到达业务处理终点。
|
||||
StatusCompleted = 3
|
||||
// StatusFailed 表示任务整体无法执行到业务终点。
|
||||
StatusFailed = 4
|
||||
// StatusCancelled 表示业务明确支持且已经取消任务。
|
||||
StatusCancelled = 5
|
||||
)
|
||||
|
||||
var statusNames = map[int]string{
|
||||
StatusPending: "待处理",
|
||||
StatusProcessing: "处理中",
|
||||
StatusCompleted: "已完成",
|
||||
StatusFailed: "已失败",
|
||||
StatusCancelled: "已取消",
|
||||
}
|
||||
|
||||
// Projection 是跨业务统一的任务公开投影。
|
||||
type Projection struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status int `json:"status"`
|
||||
StatusName string `json:"status_name"`
|
||||
TotalCount int `json:"total_count"`
|
||||
SuccessCount int `json:"success_count"`
|
||||
FailedCount int `json:"failed_count"`
|
||||
Progress int `json:"progress"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
ErrorSummary string `json:"error_summary,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// TerminalResult 是构造终态投影所需的业务结果。
|
||||
type TerminalResult struct {
|
||||
TaskID string
|
||||
Status int
|
||||
TotalCount int
|
||||
SuccessCount int
|
||||
FailedCount int
|
||||
ErrorCode string
|
||||
ErrorSummary string
|
||||
StartedAt *time.Time
|
||||
CompletedAt *time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// StatusName 返回固定任务状态的中文名称。
|
||||
func StatusName(status int) string {
|
||||
return statusNames[status]
|
||||
}
|
||||
|
||||
// IsTerminal 判断状态是否为统一终态。
|
||||
func IsTerminal(status int) bool {
|
||||
return status == StatusCompleted || status == StatusFailed || status == StatusCancelled
|
||||
}
|
||||
|
||||
// NewTerminalProjection 校验终态守恒并构造公开投影。
|
||||
func NewTerminalProjection(result TerminalResult) (Projection, error) {
|
||||
if !IsTerminal(result.Status) {
|
||||
return Projection{}, stderrors.New("任务状态不是终态")
|
||||
}
|
||||
if result.TotalCount < 0 || result.SuccessCount < 0 || result.FailedCount < 0 {
|
||||
return Projection{}, stderrors.New("任务结果计数不能为负数")
|
||||
}
|
||||
if result.Status == StatusCompleted && result.TotalCount != result.SuccessCount+result.FailedCount {
|
||||
return Projection{}, stderrors.New("已完成任务的结果计数不守恒")
|
||||
}
|
||||
return Projection{
|
||||
TaskID: result.TaskID, Status: result.Status, StatusName: StatusName(result.Status),
|
||||
TotalCount: result.TotalCount, SuccessCount: result.SuccessCount, FailedCount: result.FailedCount,
|
||||
Progress: 100, ErrorCode: result.ErrorCode, ErrorSummary: result.ErrorSummary,
|
||||
StartedAt: result.StartedAt, CompletedAt: result.CompletedAt, UpdatedAt: result.UpdatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValidatePayload 确保统一队列客户端接收 struct 或 map,而非预序列化字节。
|
||||
func ValidatePayload(payload any) error {
|
||||
if payload == nil {
|
||||
return stderrors.New("任务载荷不能为空")
|
||||
}
|
||||
kind := reflect.TypeOf(payload).Kind()
|
||||
if kind != reflect.Struct && kind != reflect.Map && kind != reflect.Ptr {
|
||||
return stderrors.New("任务载荷必须是结构体或 map")
|
||||
}
|
||||
if kind == reflect.Ptr && reflect.TypeOf(payload).Elem().Kind() != reflect.Struct {
|
||||
return stderrors.New("任务载荷指针必须指向结构体")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
105
pkg/asynctask/contract_test.go
Normal file
105
pkg/asynctask/contract_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package asynctask_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/asynctask"
|
||||
)
|
||||
|
||||
func TestCompletedTaskUsesCountsForPartialAndAllItemFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
success int
|
||||
failed int
|
||||
}{
|
||||
{name: "部分成功", success: 7, failed: 3},
|
||||
{name: "业务项全部失败", success: 0, failed: 10},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
projection, err := asynctask.NewTerminalProjection(asynctask.TerminalResult{
|
||||
TaskID: "task-1", Status: asynctask.StatusCompleted,
|
||||
TotalCount: 10, SuccessCount: tc.success, FailedCount: tc.failed,
|
||||
StartedAt: &now, CompletedAt: &now, UpdatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("构造任务投影失败:%v", err)
|
||||
}
|
||||
if projection.Status != asynctask.StatusCompleted || projection.StatusName != "已完成" {
|
||||
t.Fatalf("业务项处理到终点必须为已完成:%+v", projection)
|
||||
}
|
||||
if projection.Progress != 100 {
|
||||
t.Fatalf("终态进度必须为 100,得到:%d", projection.Progress)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminalProjectionRejectsBrokenCountInvariant(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := asynctask.NewTerminalProjection(asynctask.TerminalResult{
|
||||
TaskID: "task-2", Status: asynctask.StatusCompleted,
|
||||
TotalCount: 10, SuccessCount: 5, FailedCount: 4,
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("终态计数不守恒时应拒绝构造公开投影")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredPayloadRejectsBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if err := asynctask.ValidatePayload([]byte(`{"task_id":1}`)); err == nil {
|
||||
t.Fatal("预序列化 []byte 载荷必须被拒绝")
|
||||
}
|
||||
if err := asynctask.ValidatePayload(struct {
|
||||
TaskID uint `json:"task_id"`
|
||||
}{TaskID: 1}); err != nil {
|
||||
t.Fatalf("结构化载荷应通过校验:%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimRecoversOnlyExpiredProcessingTask(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC)
|
||||
activeExpiry := now.Add(time.Minute)
|
||||
active, err := asynctask.Claim(asynctask.LeaseState{
|
||||
Status: asynctask.StatusProcessing, LeaseOwner: "worker-a", LeaseExpiresAt: &activeExpiry,
|
||||
}, "worker-b", now, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("检查有效租约失败:%v", err)
|
||||
}
|
||||
if active.Claimed {
|
||||
t.Fatal("其他 Worker 不得抢占有效租约")
|
||||
}
|
||||
|
||||
expiredAt := now.Add(-time.Second)
|
||||
recovered, err := asynctask.Claim(asynctask.LeaseState{
|
||||
Status: asynctask.StatusProcessing, LeaseOwner: "worker-a", LeaseExpiresAt: &expiredAt,
|
||||
}, "worker-b", now, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("恢复过期任务失败:%v", err)
|
||||
}
|
||||
if !recovered.Claimed || recovered.State.LeaseOwner != "worker-b" {
|
||||
t.Fatalf("过期任务应由新 Worker 恢复:%+v", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminalTaskCannotBeClaimedOrCancelledAgain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result, err := asynctask.Claim(asynctask.LeaseState{Status: asynctask.StatusCompleted}, "worker-a", time.Now(), time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("检查终态任务失败:%v", err)
|
||||
}
|
||||
if result.Claimed || asynctask.CanCancel(asynctask.StatusCompleted) {
|
||||
t.Fatal("终态任务不得被重复消费或取消")
|
||||
}
|
||||
}
|
||||
53
pkg/asynctask/state.go
Normal file
53
pkg/asynctask/state.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package asynctask
|
||||
|
||||
import (
|
||||
stderrors "errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LeaseState 是业务任务持久化层应保存的最小状态与租约事实。
|
||||
type LeaseState struct {
|
||||
Status int
|
||||
LeaseOwner string
|
||||
LeaseExpiresAt *time.Time
|
||||
}
|
||||
|
||||
// ClaimResult 表示条件领取后的新事实。
|
||||
type ClaimResult struct {
|
||||
State LeaseState
|
||||
Claimed bool
|
||||
}
|
||||
|
||||
// Claim 按统一语义判断待处理或租约过期任务能否被领取。
|
||||
//
|
||||
// 业务持久化 Adapter 必须把同样的条件放进 PostgreSQL UPDATE 的 WHERE 中;
|
||||
// 本函数用于跨业务共享状态判定,不能替代数据库条件更新。
|
||||
func Claim(current LeaseState, owner string, now time.Time, leaseDuration time.Duration) (ClaimResult, error) {
|
||||
if owner == "" || leaseDuration <= 0 {
|
||||
return ClaimResult{}, stderrors.New("任务租约所有者和时长不能为空")
|
||||
}
|
||||
claimable := current.Status == StatusPending
|
||||
if current.Status == StatusProcessing && current.LeaseExpiresAt != nil && !current.LeaseExpiresAt.After(now) {
|
||||
claimable = true
|
||||
}
|
||||
if !claimable {
|
||||
return ClaimResult{State: current}, nil
|
||||
}
|
||||
expiresAt := now.Add(leaseDuration)
|
||||
return ClaimResult{
|
||||
State: LeaseState{Status: StatusProcessing, LeaseOwner: owner, LeaseExpiresAt: &expiresAt},
|
||||
Claimed: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CanFinish 判断当前租约所有者能否从处理中进入终态。
|
||||
func CanFinish(current LeaseState, owner string, now time.Time) bool {
|
||||
return current.Status == StatusProcessing &&
|
||||
current.LeaseOwner == owner &&
|
||||
current.LeaseExpiresAt != nil && current.LeaseExpiresAt.After(now)
|
||||
}
|
||||
|
||||
// CanCancel 判断业务支持取消时是否可用预期状态条件更新。
|
||||
func CanCancel(status int) bool {
|
||||
return status == StatusPending || status == StatusProcessing
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
package constants
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/asynctask"
|
||||
)
|
||||
|
||||
// Fiber Locals 的上下文键
|
||||
const (
|
||||
@@ -79,6 +83,7 @@ const (
|
||||
TaskTypeAlertCheck = "alert:check" // 告警检查
|
||||
TaskTypeDataCleanup = "data:cleanup" // 数据清理
|
||||
TaskTypeDailyTrafficFlush = "traffic:daily:flush" // 每日流量落盘
|
||||
TaskTypeOutboxDeliver = "outbox:deliver" // 公共 Outbox 事件投递
|
||||
)
|
||||
|
||||
// 用户状态常量
|
||||
@@ -219,6 +224,7 @@ const (
|
||||
QueueAlertCheck = TaskTypeAlertCheck // 告警检查任务队列
|
||||
QueueDataCleanup = TaskTypeDataCleanup // 数据清理任务队列
|
||||
QueueDailyTrafficFlush = TaskTypeDailyTrafficFlush // 每日流量落盘任务队列
|
||||
QueueOutboxDeliver = TaskTypeOutboxDeliver // 公共 Outbox 投递队列
|
||||
|
||||
DefaultRetryMax = 5 // 默认任务最大重试次数
|
||||
DefaultTimeout = 10 * time.Minute // 默认任务超时时间
|
||||
@@ -276,6 +282,8 @@ func QueueForTaskType(taskType string) string {
|
||||
return QueueDataCleanup
|
||||
case TaskTypeDailyTrafficFlush:
|
||||
return QueueDailyTrafficFlush
|
||||
case TaskTypeOutboxDeliver:
|
||||
return QueueOutboxDeliver
|
||||
default:
|
||||
return QueueDefault
|
||||
}
|
||||
@@ -309,6 +317,7 @@ func DefaultTaskQueueWeights() map[string]int {
|
||||
QueuePackageDataReset: 1,
|
||||
QueueDataCleanup: 1,
|
||||
QueueDailyTrafficFlush: 1,
|
||||
QueueOutboxDeliver: 4,
|
||||
QueueDefault: 1,
|
||||
QueueLow: 1,
|
||||
}
|
||||
@@ -329,11 +338,11 @@ const (
|
||||
|
||||
// 导出主任务状态常量
|
||||
const (
|
||||
ExportTaskStatusPending = 1 // 待处理
|
||||
ExportTaskStatusProcessing = 2 // 处理中
|
||||
ExportTaskStatusCompleted = 3 // 已完成
|
||||
ExportTaskStatusFailed = 4 // 已失败
|
||||
ExportTaskStatusCancelled = 5 // 已取消
|
||||
ExportTaskStatusPending = asynctask.StatusPending // 待处理
|
||||
ExportTaskStatusProcessing = asynctask.StatusProcessing // 处理中
|
||||
ExportTaskStatusCompleted = asynctask.StatusCompleted // 已完成
|
||||
ExportTaskStatusFailed = asynctask.StatusFailed // 已失败
|
||||
ExportTaskStatusCancelled = asynctask.StatusCancelled // 已取消
|
||||
)
|
||||
|
||||
// 导出分片任务状态常量
|
||||
@@ -450,20 +459,11 @@ func GetShelfStatusName(status int) string {
|
||||
|
||||
// GetExportTaskStatusName 获取导出主任务状态名称
|
||||
func GetExportTaskStatusName(status int) string {
|
||||
switch status {
|
||||
case ExportTaskStatusPending:
|
||||
return "待处理"
|
||||
case ExportTaskStatusProcessing:
|
||||
return "处理中"
|
||||
case ExportTaskStatusCompleted:
|
||||
return "已完成"
|
||||
case ExportTaskStatusFailed:
|
||||
return "已失败"
|
||||
case ExportTaskStatusCancelled:
|
||||
return "已取消"
|
||||
default:
|
||||
name := asynctask.StatusName(status)
|
||||
if name == "" {
|
||||
return "未知"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// GetExportShardStatusName 获取导出分片任务状态名称
|
||||
|
||||
45
pkg/constants/outbox.go
Normal file
45
pkg/constants/outbox.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package constants
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
// OutboxStatusPending 表示事件待投递。
|
||||
OutboxStatusPending = 1
|
||||
// OutboxStatusDelivering 表示事件已被 Relay 租约领取。
|
||||
OutboxStatusDelivering = 2
|
||||
// OutboxStatusDelivered 表示事件已成功提交到队列。
|
||||
OutboxStatusDelivered = 3
|
||||
// OutboxStatusFailed 表示事件达到最终失败或等待人工重放。
|
||||
OutboxStatusFailed = 4
|
||||
)
|
||||
|
||||
const (
|
||||
// OutboxDefaultMaxRetries 是公共 Outbox 默认最大重试次数。
|
||||
OutboxDefaultMaxRetries = 10
|
||||
// OutboxDefaultLeaseDuration 是 Relay 默认租约时长。
|
||||
OutboxDefaultLeaseDuration = time.Minute
|
||||
// OutboxDefaultBatchSize 是 Relay 默认单批领取数。
|
||||
OutboxDefaultBatchSize = 100
|
||||
// OutboxBaseRetryDelay 是瞬时失败指数退避的基础时长。
|
||||
OutboxBaseRetryDelay = 5 * time.Second
|
||||
// OutboxMaxRetryDelay 是瞬时失败指数退避的上限。
|
||||
OutboxMaxRetryDelay = 30 * time.Minute
|
||||
// OutboxRelayPollInterval 是 Worker 扫描到期事件的默认间隔。
|
||||
OutboxRelayPollInterval = time.Second
|
||||
)
|
||||
|
||||
// GetOutboxStatusName 返回 Outbox 内部状态中文名称。
|
||||
func GetOutboxStatusName(status int) string {
|
||||
switch status {
|
||||
case OutboxStatusPending:
|
||||
return "待投递"
|
||||
case OutboxStatusDelivering:
|
||||
return "投递中"
|
||||
case OutboxStatusDelivered:
|
||||
return "已投递"
|
||||
case OutboxStatusFailed:
|
||||
return "投递失败"
|
||||
default:
|
||||
return "未知"
|
||||
}
|
||||
}
|
||||
24
pkg/constants/system_config.go
Normal file
24
pkg/constants/system_config.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package constants
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// SystemConfigTypeString 表示字符串配置。
|
||||
SystemConfigTypeString = "string"
|
||||
// SystemConfigTypeInt 表示整数配置。
|
||||
SystemConfigTypeInt = "int"
|
||||
// SystemConfigTypeBool 表示布尔配置。
|
||||
SystemConfigTypeBool = "bool"
|
||||
// SystemConfigTypeJSON 表示 JSON 配置。
|
||||
SystemConfigTypeJSON = "json"
|
||||
// SystemConfigCacheTTL 是系统配置默认缓存时长。
|
||||
SystemConfigCacheTTL = 5 * time.Minute
|
||||
)
|
||||
|
||||
// RedisSystemConfigKey 生成单个系统配置的 Redis 缓存键。
|
||||
func RedisSystemConfigKey(configKey string) string {
|
||||
return fmt.Sprintf("system_config:value:%s", configKey)
|
||||
}
|
||||
124
pkg/idempotency/fingerprint.go
Normal file
124
pkg/idempotency/fingerprint.go
Normal file
@@ -0,0 +1,124 @@
|
||||
// Package idempotency 提供创建类命令的幂等身份与请求指纹公共契约。
|
||||
package idempotency
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
)
|
||||
|
||||
const (
|
||||
// FingerprintAlgorithmV1 是请求指纹算法的首个稳定版本。
|
||||
FingerprintAlgorithmV1 = "sha256-canonical-json-v1"
|
||||
)
|
||||
|
||||
// Scope 表示请求 ID 的幂等作用域。
|
||||
type Scope struct {
|
||||
Subject string `json:"subject"`
|
||||
Operation string `json:"operation"`
|
||||
}
|
||||
|
||||
// FingerprintValue 表示带算法版本的请求指纹。
|
||||
type FingerprintValue struct {
|
||||
Algorithm string `json:"algorithm"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// Record 表示业务模块自行持久化的幂等事实最小投影。
|
||||
type Record struct {
|
||||
Scope Scope `json:"scope"`
|
||||
RequestID string `json:"request_id"`
|
||||
Fingerprint FingerprintValue `json:"fingerprint"`
|
||||
}
|
||||
|
||||
// ReplayKind 表示已有事实与本次命令的关系。
|
||||
type ReplayKind string
|
||||
|
||||
const (
|
||||
// ReplaySame 表示相同命令重放,应返回原业务结果。
|
||||
ReplaySame ReplayKind = "same"
|
||||
// ReplayConflict 表示同作用域、同请求 ID 携带了不同业务内容。
|
||||
ReplayConflict ReplayKind = "conflict"
|
||||
// ReplayUnrelated 表示不是同一幂等作用域内的命令。
|
||||
ReplayUnrelated ReplayKind = "unrelated"
|
||||
)
|
||||
|
||||
var volatileFieldNames = map[string]struct{}{
|
||||
"authorization": {},
|
||||
"cookie": {},
|
||||
"nonce": {},
|
||||
"sign": {},
|
||||
"signature": {},
|
||||
"timestamp": {},
|
||||
"token": {},
|
||||
}
|
||||
|
||||
// Fingerprint 对影响业务结果的规范化字段计算稳定指纹。
|
||||
//
|
||||
// 调用方仍应优先传入专用命令 DTO;本函数会递归排除约定的易变传输字段,
|
||||
// 并去除字符串首尾空白,避免签名、令牌或对象字段顺序污染业务身份。
|
||||
func Fingerprint(command any) (FingerprintValue, error) {
|
||||
raw, err := sonic.ConfigStd.Marshal(command)
|
||||
if err != nil {
|
||||
return FingerprintValue{}, err
|
||||
}
|
||||
var value any
|
||||
if err := sonic.Unmarshal(raw, &value); err != nil {
|
||||
return FingerprintValue{}, err
|
||||
}
|
||||
normalized := normalize(value)
|
||||
canonical, err := sonic.ConfigStd.Marshal(normalized)
|
||||
if err != nil {
|
||||
return FingerprintValue{}, err
|
||||
}
|
||||
sum := sha256.Sum256(canonical)
|
||||
return FingerprintValue{
|
||||
Algorithm: FingerprintAlgorithmV1,
|
||||
Value: hex.EncodeToString(sum[:]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValidateScope 校验幂等作用域和稳定请求 ID 是否完整。
|
||||
func ValidateScope(scope Scope, requestID string) bool {
|
||||
return strings.TrimSpace(scope.Subject) != "" &&
|
||||
strings.TrimSpace(scope.Operation) != "" &&
|
||||
strings.TrimSpace(requestID) != ""
|
||||
}
|
||||
|
||||
// Classify 比较已有幂等事实与本次命令。
|
||||
func Classify(existing Record, scope Scope, requestID string, fingerprint FingerprintValue) ReplayKind {
|
||||
if existing.Scope != scope || existing.RequestID != requestID {
|
||||
return ReplayUnrelated
|
||||
}
|
||||
if reflect.DeepEqual(existing.Fingerprint, fingerprint) {
|
||||
return ReplaySame
|
||||
}
|
||||
return ReplayConflict
|
||||
}
|
||||
|
||||
func normalize(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
result := make(map[string]any, len(typed))
|
||||
for key, item := range typed {
|
||||
if _, volatile := volatileFieldNames[strings.ToLower(strings.TrimSpace(key))]; volatile {
|
||||
continue
|
||||
}
|
||||
result[key] = normalize(item)
|
||||
}
|
||||
return result
|
||||
case []any:
|
||||
result := make([]any, len(typed))
|
||||
for index, item := range typed {
|
||||
result[index] = normalize(item)
|
||||
}
|
||||
return result
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
58
pkg/idempotency/fingerprint_test.go
Normal file
58
pkg/idempotency/fingerprint_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package idempotency_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/idempotency"
|
||||
)
|
||||
|
||||
func TestFingerprintIgnoresTransportFieldsAndNormalizesObjectOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
left, err := idempotency.Fingerprint(map[string]any{
|
||||
"amount": 100,
|
||||
"remark": " 月度充值 ",
|
||||
"token": "token-a",
|
||||
"nested": map[string]any{"enabled": true, "count": 2},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("计算第一个请求指纹失败:%v", err)
|
||||
}
|
||||
right, err := idempotency.Fingerprint(map[string]any{
|
||||
"nested": map[string]any{"count": 2, "enabled": true},
|
||||
"timestamp": 1720000000,
|
||||
"remark": "月度充值",
|
||||
"amount": 100,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("计算第二个请求指纹失败:%v", err)
|
||||
}
|
||||
|
||||
if left.Algorithm != idempotency.FingerprintAlgorithmV1 {
|
||||
t.Fatalf("算法版本不正确:%s", left.Algorithm)
|
||||
}
|
||||
if left.Value != right.Value {
|
||||
t.Fatalf("仅传输字段或对象顺序变化不应改变指纹:%s != %s", left.Value, right.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyReplayDistinguishesScopeAndConflict(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
original := idempotency.Record{
|
||||
Scope: idempotency.Scope{Subject: "account:7", Operation: "agent-recharge.create"},
|
||||
RequestID: "request-1",
|
||||
Fingerprint: idempotency.FingerprintValue{Algorithm: idempotency.FingerprintAlgorithmV1, Value: "abc"},
|
||||
}
|
||||
|
||||
if got := idempotency.Classify(original, original.Scope, "request-1", original.Fingerprint); got != idempotency.ReplaySame {
|
||||
t.Fatalf("相同命令应识别为重放,得到:%s", got)
|
||||
}
|
||||
if got := idempotency.Classify(original, original.Scope, "request-1", idempotency.FingerprintValue{Algorithm: idempotency.FingerprintAlgorithmV1, Value: "def"}); got != idempotency.ReplayConflict {
|
||||
t.Fatalf("同作用域同请求 ID 的不同内容应冲突,得到:%s", got)
|
||||
}
|
||||
otherScope := idempotency.Scope{Subject: "account:8", Operation: original.Scope.Operation}
|
||||
if got := idempotency.Classify(original, otherScope, "request-1", original.Fingerprint); got != idempotency.ReplayUnrelated {
|
||||
t.Fatalf("不同主体不应相互污染,得到:%s", got)
|
||||
}
|
||||
}
|
||||
86
pkg/idempotency/postgres_integration_test.go
Normal file
86
pkg/idempotency/postgres_integration_test.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package idempotency_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/testutil"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/idempotency"
|
||||
)
|
||||
|
||||
func TestPostgresUniqueConstraintArbitratesConcurrentFirstSubmission(t *testing.T) {
|
||||
db := testutil.NewPostgresTransaction(t)
|
||||
if err := db.Exec(`
|
||||
CREATE TEMP TABLE test_command_idempotency (
|
||||
subject varchar(100) NOT NULL,
|
||||
operation varchar(100) NOT NULL,
|
||||
request_id varchar(100) NOT NULL,
|
||||
fingerprint varchar(128) NOT NULL,
|
||||
result_id bigint NOT NULL,
|
||||
UNIQUE (subject, operation, request_id)
|
||||
) ON COMMIT DROP`).Error; err != nil {
|
||||
t.Fatalf("创建幂等契约测试表失败:%v", err)
|
||||
}
|
||||
|
||||
fingerprint, err := idempotency.Fingerprint(map[string]any{"amount": 100})
|
||||
if err != nil {
|
||||
t.Fatalf("计算指纹失败:%v", err)
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
results := make(chan int64, 2)
|
||||
errs := make(chan error, 2)
|
||||
var wait sync.WaitGroup
|
||||
for _, resultID := range []int64{101, 102} {
|
||||
wait.Add(1)
|
||||
go func(id int64) {
|
||||
defer wait.Done()
|
||||
<-start
|
||||
result := db.WithContext(context.Background()).Exec(`
|
||||
INSERT INTO test_command_idempotency
|
||||
(subject, operation, request_id, fingerprint, result_id)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT (subject, operation, request_id) DO NOTHING`,
|
||||
"account:7", "agent-recharge.create", "request-1", fingerprint.Value, id,
|
||||
)
|
||||
results <- result.RowsAffected
|
||||
errs <- result.Error
|
||||
}(resultID)
|
||||
}
|
||||
close(start)
|
||||
wait.Wait()
|
||||
close(results)
|
||||
close(errs)
|
||||
|
||||
var inserted int64
|
||||
for resultErr := range errs {
|
||||
if resultErr != nil {
|
||||
t.Fatalf("并发写入失败:%v", resultErr)
|
||||
}
|
||||
}
|
||||
for affected := range results {
|
||||
inserted += affected
|
||||
}
|
||||
if inserted != 1 {
|
||||
t.Fatalf("并发首写必须只有一个数据库事实,实际插入:%d", inserted)
|
||||
}
|
||||
|
||||
var count int64
|
||||
if err := db.Table("test_command_idempotency").Count(&count).Error; err != nil {
|
||||
t.Fatalf("统计幂等事实失败:%v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("幂等事实数量错误:%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdempotencyScopeDoesNotDependOnRedis(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
scope := idempotency.Scope{Subject: "account:7", Operation: "example.create"}
|
||||
if !idempotency.ValidateScope(scope, "request-1") {
|
||||
t.Fatal("完整作用域和请求 ID 应通过校验")
|
||||
}
|
||||
// 公共幂等契约没有 Redis 参数;Redis 仅可由业务作为削峰优化,不能参与最终裁决。
|
||||
}
|
||||
130
pkg/logger/access_policy.go
Normal file
130
pkg/logger/access_policy.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
)
|
||||
|
||||
// AccessPolicy 是敏感路由的安全摘要策略。
|
||||
type AccessPolicy struct {
|
||||
Name string
|
||||
Sensitive bool
|
||||
SafeFields map[string]struct{}
|
||||
FileFields map[string]struct{}
|
||||
}
|
||||
|
||||
var (
|
||||
loginPolicy = AccessPolicy{
|
||||
Name: "login_token", Sensitive: true,
|
||||
SafeFields: fieldSet("username", "user_id", "account_id", "success", "result_code"),
|
||||
}
|
||||
paymentPolicy = AccessPolicy{
|
||||
Name: "payment", Sensitive: true,
|
||||
SafeFields: fieldSet("order_no", "payment_no", "channel", "payment_method", "result_code", "amount", "status"),
|
||||
}
|
||||
wecomPolicy = AccessPolicy{
|
||||
Name: "wecom_callback", Sensitive: true,
|
||||
SafeFields: fieldSet("event_type", "resource_type", "resource_id", "result_code", "status"),
|
||||
}
|
||||
filePolicy = AccessPolicy{
|
||||
Name: "file_export", Sensitive: true,
|
||||
SafeFields: fieldSet("content_type", "file_size", "size", "count", "task_id", "status", "result_code"),
|
||||
FileFields: fieldSet("file_name", "filename", "name"),
|
||||
}
|
||||
defaultPolicy = AccessPolicy{Name: "default_json"}
|
||||
)
|
||||
|
||||
func fieldSet(fields ...string) map[string]struct{} {
|
||||
result := make(map[string]struct{}, len(fields))
|
||||
for _, field := range fields {
|
||||
result[field] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func policyForPath(path string) AccessPolicy {
|
||||
normalized := strings.ToLower(path)
|
||||
switch {
|
||||
case strings.Contains(normalized, "/login"), strings.Contains(normalized, "token"):
|
||||
return loginPolicy
|
||||
case strings.Contains(normalized, "/callback") && (strings.Contains(normalized, "wecom") || strings.Contains(normalized, "wework")):
|
||||
return wecomPolicy
|
||||
case strings.Contains(normalized, "payment"), strings.Contains(normalized, "wechat-pay"),
|
||||
strings.Contains(normalized, "alipay"), strings.Contains(normalized, "fuiou-pay"):
|
||||
return paymentPolicy
|
||||
case strings.Contains(normalized, "/storage"), strings.Contains(normalized, "/upload"),
|
||||
strings.Contains(normalized, "/download"), strings.Contains(normalized, "/export"):
|
||||
return filePolicy
|
||||
default:
|
||||
return defaultPolicy
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeWithPolicy(raw []byte, contentType string, policy AccessPolicy) SanitizedContent {
|
||||
if len(raw) == 0 {
|
||||
return SanitizedContent{}
|
||||
}
|
||||
if !policy.Sensitive {
|
||||
return sanitizeBody(raw, BodyPolicyJSON)
|
||||
}
|
||||
summary := make(map[string]any)
|
||||
normalizedContentType := strings.ToLower(contentType)
|
||||
switch {
|
||||
case strings.Contains(normalizedContentType, "json"):
|
||||
var payload any
|
||||
if sonic.Unmarshal(raw, &payload) == nil {
|
||||
collectSafeFields(payload, policy, summary)
|
||||
}
|
||||
case strings.Contains(normalizedContentType, "x-www-form-urlencoded"):
|
||||
if values, err := url.ParseQuery(string(raw)); err == nil {
|
||||
for key, items := range values {
|
||||
collectSafeScalar(key, strings.Join(items, ","), policy, summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
content := "[仅记录安全摘要]"
|
||||
if len(summary) > 0 {
|
||||
if encoded, err := sonic.Marshal(summary); err == nil {
|
||||
content, _ = truncateBody(encoded, MaxBodyLogSize)
|
||||
}
|
||||
}
|
||||
return SanitizedContent{Content: content, Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize}
|
||||
}
|
||||
|
||||
func collectSafeFields(value any, policy AccessPolicy, output map[string]any) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for key, item := range typed {
|
||||
switch scalar := item.(type) {
|
||||
case string:
|
||||
collectSafeScalar(key, scalar, policy, output)
|
||||
case float64, bool:
|
||||
if _, allowed := policy.SafeFields[strings.ToLower(key)]; allowed {
|
||||
output[key] = scalar
|
||||
}
|
||||
default:
|
||||
collectSafeFields(item, policy, output)
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
collectSafeFields(item, policy, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectSafeScalar(key, value string, policy AccessPolicy, output map[string]any) {
|
||||
normalized := strings.ToLower(key)
|
||||
if _, isFileName := policy.FileFields[normalized]; isFileName {
|
||||
base := filepath.Base(value)
|
||||
hash := digest([]byte(base))
|
||||
output[key] = "sha256:" + hash[:16]
|
||||
return
|
||||
}
|
||||
if _, allowed := policy.SafeFields[normalized]; allowed {
|
||||
output[key] = value
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package logger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -15,20 +17,38 @@ import (
|
||||
const (
|
||||
// MaxBodyLogSize 限制记录的请求/响应 body 大小为 50KB
|
||||
MaxBodyLogSize = 50 * 1024
|
||||
redactedValue = "[已脱敏]"
|
||||
)
|
||||
|
||||
// BodyPolicy 表示访问日志正文记录策略。
|
||||
type BodyPolicy int
|
||||
|
||||
const (
|
||||
// BodyPolicyJSON 表示只允许记录脱敏后的 JSON 正文。
|
||||
BodyPolicyJSON BodyPolicy = iota + 1
|
||||
// BodyPolicySummary 表示只记录不可逆安全摘要。
|
||||
BodyPolicySummary
|
||||
)
|
||||
|
||||
// SanitizedContent 表示访问日志可安全记录的正文或查询摘要。
|
||||
type SanitizedContent struct {
|
||||
Content string
|
||||
Size int
|
||||
SHA256 string
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
// truncateBody 截断 body 到指定大小
|
||||
func truncateBody(body []byte, maxSize int) string {
|
||||
func truncateBody(body []byte, maxSize int) (string, bool) {
|
||||
if len(body) == 0 {
|
||||
return ""
|
||||
return "", false
|
||||
}
|
||||
|
||||
if len(body) <= maxSize {
|
||||
return string(body)
|
||||
return string(body), false
|
||||
}
|
||||
|
||||
// 超过限制,截断并添加提示
|
||||
return string(body[:maxSize]) + "... (truncated)"
|
||||
return string(body[:maxSize]), true
|
||||
}
|
||||
|
||||
// maskSensitiveValue 按字段名判断并脱敏访问日志中的敏感值
|
||||
@@ -36,25 +56,20 @@ func maskSensitiveValue(key, value string) string {
|
||||
if value == "" {
|
||||
return value
|
||||
}
|
||||
normalized := strings.ToLower(key)
|
||||
if strings.Contains(normalized, "password") ||
|
||||
strings.Contains(normalized, "sign") ||
|
||||
strings.Contains(normalized, "nonce") ||
|
||||
strings.Contains(normalized, "token") ||
|
||||
strings.Contains(normalized, "secret") {
|
||||
return "***"
|
||||
if shouldMaskField(key) {
|
||||
return redactedValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// sanitizeQuery 脱敏 query 中的密码、签名、nonce、token 等字段
|
||||
func sanitizeQuery(rawQuery string) string {
|
||||
func sanitizeQuery(rawQuery string) SanitizedContent {
|
||||
if rawQuery == "" {
|
||||
return ""
|
||||
return SanitizedContent{}
|
||||
}
|
||||
values, err := url.ParseQuery(rawQuery)
|
||||
if err != nil {
|
||||
return rawQuery
|
||||
return summarize([]byte(rawQuery))
|
||||
}
|
||||
for key, items := range values {
|
||||
for index, item := range items {
|
||||
@@ -62,24 +77,38 @@ func sanitizeQuery(rawQuery string) string {
|
||||
}
|
||||
values[key] = items
|
||||
}
|
||||
return values.Encode()
|
||||
content, truncated := truncateBody([]byte(values.Encode()), MaxBodyLogSize)
|
||||
return SanitizedContent{Content: content, Size: len(rawQuery), SHA256: digest([]byte(rawQuery)), Truncated: truncated}
|
||||
}
|
||||
|
||||
// sanitizeBody 脱敏 JSON 请求体后再写入访问日志
|
||||
func sanitizeBody(rawBody []byte) string {
|
||||
// sanitizeBody 按策略脱敏正文后再写入访问日志。
|
||||
func sanitizeBody(rawBody []byte, policy BodyPolicy) SanitizedContent {
|
||||
if len(rawBody) == 0 {
|
||||
return ""
|
||||
return SanitizedContent{}
|
||||
}
|
||||
if policy == BodyPolicySummary {
|
||||
return summarize(rawBody)
|
||||
}
|
||||
var payload any
|
||||
if err := sonic.Unmarshal(rawBody, &payload); err != nil {
|
||||
return truncateBody(rawBody, MaxBodyLogSize)
|
||||
return summarize(rawBody)
|
||||
}
|
||||
sanitizeJSONValue(payload)
|
||||
data, err := sonic.Marshal(payload)
|
||||
if err != nil {
|
||||
return truncateBody(rawBody, MaxBodyLogSize)
|
||||
return summarize(rawBody)
|
||||
}
|
||||
return truncateBody(data, MaxBodyLogSize)
|
||||
content, truncated := truncateBody(data, MaxBodyLogSize)
|
||||
return SanitizedContent{Content: content, Size: len(rawBody), SHA256: digest(rawBody), Truncated: truncated}
|
||||
}
|
||||
|
||||
func summarize(raw []byte) SanitizedContent {
|
||||
return SanitizedContent{Content: "[仅记录安全摘要]", Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize}
|
||||
}
|
||||
|
||||
func digest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// sanitizeJSONValue 递归脱敏 JSON 对象中的敏感字段
|
||||
@@ -92,7 +121,7 @@ func sanitizeJSONValue(value any) {
|
||||
continue
|
||||
}
|
||||
if shouldMaskField(key) {
|
||||
typed[key] = "***"
|
||||
typed[key] = redactedValue
|
||||
continue
|
||||
}
|
||||
sanitizeJSONValue(item)
|
||||
@@ -108,6 +137,14 @@ func sanitizeJSONValue(value any) {
|
||||
func shouldMaskField(key string) bool {
|
||||
normalized := strings.ToLower(key)
|
||||
return strings.Contains(normalized, "password") ||
|
||||
strings.Contains(normalized, "passwd") ||
|
||||
strings.Contains(normalized, "credential") ||
|
||||
strings.Contains(normalized, "authorization") ||
|
||||
strings.Contains(normalized, "cookie") ||
|
||||
strings.Contains(normalized, "key") ||
|
||||
strings.Contains(normalized, "url") ||
|
||||
strings.Contains(normalized, "qr_content") ||
|
||||
strings.Contains(normalized, "verification_code") ||
|
||||
strings.Contains(normalized, "sign") ||
|
||||
strings.Contains(normalized, "nonce") ||
|
||||
strings.Contains(normalized, "token") ||
|
||||
@@ -117,6 +154,12 @@ func shouldMaskField(key string) bool {
|
||||
// Middleware 创建 Fiber 日志中间件
|
||||
// 记录所有 HTTP 请求到访问日志(包括请求和响应 body)
|
||||
func Middleware() fiber.Handler {
|
||||
return MiddlewareWithLogger(GetAccessLogger())
|
||||
}
|
||||
|
||||
// MiddlewareWithLogger 创建可注入访问日志器的 Fiber 中间件。
|
||||
// 生产环境使用 Middleware;该入口让集成测试捕获最终 JSON 日志而无需修改全局状态。
|
||||
func MiddlewareWithLogger(accessLogger *zap.Logger) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
// 记录请求开始时间
|
||||
startTime := time.Now()
|
||||
@@ -136,10 +179,14 @@ func Middleware() fiber.Handler {
|
||||
c.SetUserContext(ctx)
|
||||
|
||||
// 获取请求 body(在 c.Next() 之前读取)
|
||||
requestBody := sanitizeBody(c.Body())
|
||||
policy := policyForPath(c.Path())
|
||||
requestBody := sanitizeWithPolicy(c.Body(), c.Get("Content-Type"), policy)
|
||||
|
||||
// 获取 query 参数
|
||||
queryParams := sanitizeQuery(string(c.Request().URI().QueryString()))
|
||||
if policy.Sensitive && queryParams.Size > 0 {
|
||||
queryParams = summarize([]byte(c.Request().URI().QueryString()))
|
||||
}
|
||||
|
||||
// 处理请求
|
||||
err := c.Next()
|
||||
@@ -162,22 +209,29 @@ func Middleware() fiber.Handler {
|
||||
}
|
||||
|
||||
// 获取响应 body
|
||||
responseBody := truncateBody(c.Response().Body(), MaxBodyLogSize)
|
||||
responseBody := sanitizeWithPolicy(c.Response().Body(), string(c.Response().Header.ContentType()), policy)
|
||||
|
||||
// 记录访问日志
|
||||
accessLogger := GetAccessLogger()
|
||||
accessLogger.Info("",
|
||||
zap.String("method", c.Method()),
|
||||
zap.String("path", c.Path()),
|
||||
zap.String("query", queryParams),
|
||||
zap.String("body_policy", policy.Name),
|
||||
zap.String("query", queryParams.Content),
|
||||
zap.Bool("query_truncated", queryParams.Truncated),
|
||||
zap.Int("status", c.Response().StatusCode()),
|
||||
zap.Float64("duration_ms", float64(duration.Microseconds())/1000.0),
|
||||
zap.String("request_id", requestID),
|
||||
zap.String("ip", c.IP()),
|
||||
zap.String("user_agent", c.Get("User-Agent")),
|
||||
zap.Uint("user_id", userID),
|
||||
zap.String("request_body", requestBody),
|
||||
zap.String("response_body", responseBody),
|
||||
zap.String("request_body", requestBody.Content),
|
||||
zap.Int("request_body_size", requestBody.Size),
|
||||
zap.String("request_body_sha256", requestBody.SHA256),
|
||||
zap.Bool("request_body_truncated", requestBody.Truncated),
|
||||
zap.String("response_body", responseBody.Content),
|
||||
zap.Int("response_body_size", responseBody.Size),
|
||||
zap.String("response_body_sha256", responseBody.SHA256),
|
||||
zap.Bool("response_body_truncated", responseBody.Truncated),
|
||||
)
|
||||
|
||||
return err
|
||||
|
||||
154
pkg/logger/middleware_test.go
Normal file
154
pkg/logger/middleware_test.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
func TestSanitizeJSONRecursivelyMasksRequestAndResponseSecrets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
raw := []byte(`{"Password":"p@ss","items":[{"access_token":"token-value","count":3}],"profile":{"private_url":"https://secret.example/a"}}`)
|
||||
result := sanitizeBody(raw, BodyPolicyJSON)
|
||||
if strings.Contains(result.Content, "p@ss") || strings.Contains(result.Content, "token-value") || strings.Contains(result.Content, "secret.example") {
|
||||
t.Fatalf("访问日志仍包含敏感值:%s", result.Content)
|
||||
}
|
||||
if !strings.Contains(result.Content, redactedValue) {
|
||||
t.Fatalf("访问日志未输出统一脱敏占位:%s", result.Content)
|
||||
}
|
||||
if result.Truncated {
|
||||
t.Fatal("短载荷不应标记为截断")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFiberAccessLogSanitizesFinalRequestAndResponseJSON(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
encoderConfig := zap.NewProductionEncoderConfig()
|
||||
log := zap.New(zapcore.NewCore(
|
||||
zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(&output), zapcore.InfoLevel,
|
||||
))
|
||||
app := fiber.New()
|
||||
app.Use(func(c *fiber.Ctx) error {
|
||||
c.Locals("requestid", "request-test-1")
|
||||
return c.Next()
|
||||
})
|
||||
app.Use(MiddlewareWithLogger(log))
|
||||
app.Post("/tokens", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"access_token": "response-token", "data": fiber.Map{"ok": true}})
|
||||
})
|
||||
|
||||
request := httptest.NewRequest("POST", "/tokens?Authorization=query-secret&page=1", strings.NewReader(`{"password":"request-secret"}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response, err := app.Test(request)
|
||||
if err != nil {
|
||||
t.Fatalf("执行 Fiber 请求失败:%v", err)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, response.Body)
|
||||
_ = response.Body.Close()
|
||||
|
||||
logged := output.String()
|
||||
for _, secret := range []string{"request-secret", "response-token", "query-secret"} {
|
||||
if strings.Contains(logged, secret) {
|
||||
t.Fatalf("最终 JSON 访问日志泄露测试凭证 %q:%s", secret, logged)
|
||||
}
|
||||
}
|
||||
for _, field := range []string{"request_id", "status", "duration_ms", "request_body_truncated", "response_body_truncated"} {
|
||||
if !strings.Contains(logged, field) {
|
||||
t.Fatalf("最终 JSON 访问日志缺少字段 %q:%s", field, logged)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSensitiveRouteMatrixNeverLogsRawPayload(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
path string
|
||||
contentType string
|
||||
body string
|
||||
secret string
|
||||
}{
|
||||
{name: "登录JSON", path: "/api/auth/login", contentType: "application/json", body: `{"username":"safe-user","password":"json-secret"}`, secret: "json-secret"},
|
||||
{name: "企微XML", path: "/api/callback/wecom", contentType: "application/xml", body: `<xml><Encrypt>xml-secret</Encrypt></xml>`, secret: "xml-secret"},
|
||||
{name: "支付表单", path: "/api/callback/alipay", contentType: "application/x-www-form-urlencoded", body: "order_no=SAFE-1&sign=form-secret", secret: "form-secret"},
|
||||
{name: "文件multipart", path: "/api/admin/storage/upload", contentType: "multipart/form-data; boundary=x", body: "--x\r\nContent-Disposition: form-data; name=\"file\"; filename=\"secret.bin\"\r\nContent-Type: application/octet-stream\r\n\r\nfile-bytes-secret\r\n--x--\r\n", secret: "file-bytes-secret"},
|
||||
{name: "导出二进制", path: "/api/admin/export-tasks/1/download", contentType: "application/octet-stream", body: "binary-secret", secret: "binary-secret"},
|
||||
{name: "支付不可解析", path: "/api/callback/wechat-pay", contentType: "application/json", body: `{"sign":"broken-secret"`, secret: "broken-secret"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel))
|
||||
app := fiber.New()
|
||||
app.Use(MiddlewareWithLogger(log))
|
||||
app.Post(tc.path, func(c *fiber.Ctx) error {
|
||||
return c.Status(fiber.StatusOK).Type("json").SendString(`{"access_token":"response-secret","status":"ok"}`)
|
||||
})
|
||||
request := httptest.NewRequest("POST", tc.path+"?signature=query-secret", strings.NewReader(tc.body))
|
||||
request.Header.Set("Content-Type", tc.contentType)
|
||||
response, err := app.Test(request)
|
||||
if err != nil {
|
||||
t.Fatalf("执行敏感路由请求失败:%v", err)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, response.Body)
|
||||
_ = response.Body.Close()
|
||||
logged := output.String()
|
||||
for _, secret := range []string{tc.secret, "query-secret", "response-secret"} {
|
||||
if strings.Contains(logged, secret) {
|
||||
t.Fatalf("敏感路由日志泄露 %q:%s", secret, logged)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(logged, "body_policy") || !strings.Contains(logged, "sha256") {
|
||||
t.Fatalf("敏感路由日志缺少策略或摘要:%s", logged)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSensitiveRouteLongBodyRecordsTruncationWithoutRawSecret(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel))
|
||||
app := fiber.New()
|
||||
app.Use(MiddlewareWithLogger(log))
|
||||
app.Post("/api/admin/storage/upload", func(c *fiber.Ctx) error { return c.SendStatus(fiber.StatusOK) })
|
||||
secret := strings.Repeat("long-file-secret", MaxBodyLogSize)
|
||||
request := httptest.NewRequest("POST", "/api/admin/storage/upload", strings.NewReader(secret))
|
||||
request.Header.Set("Content-Type", "application/octet-stream")
|
||||
response, err := app.Test(request, 10000)
|
||||
if err != nil {
|
||||
t.Fatalf("执行超长敏感请求失败:%v", err)
|
||||
}
|
||||
_ = response.Body.Close()
|
||||
logged := output.String()
|
||||
if strings.Contains(logged, "long-file-secret") || !strings.Contains(logged, `"request_body_truncated":true`) {
|
||||
t.Fatalf("超长敏感载荷未安全摘要或未标记截断:%s", logged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeInvalidJSONNeverFallsBackToRawBody(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
raw := []byte(`{"password":"should-never-appear"`)
|
||||
result := sanitizeBody(raw, BodyPolicyJSON)
|
||||
if strings.Contains(result.Content, "should-never-appear") {
|
||||
t.Fatalf("解析失败时不得回退原文:%s", result.Content)
|
||||
}
|
||||
if result.SHA256 == "" || result.Size != len(raw) {
|
||||
t.Fatalf("解析失败摘要缺少大小或哈希:%+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeQueryUsesSameSensitiveFieldRules(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
query := sanitizeQuery("page=1&Authorization=Bearer-secret&nonce=abc")
|
||||
if strings.Contains(query.Content, "Bearer-secret") || strings.Contains(query.Content, "abc") {
|
||||
t.Fatalf("query 敏感值未脱敏:%s", query.Content)
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,7 @@ func BuildDocHandlers() *bootstrap.Handlers {
|
||||
OrderPackageInvalidate: admin.NewOrderPackageInvalidateHandler(nil),
|
||||
ClientWechat: app.NewClientWechatHandler(nil, nil, nil),
|
||||
SuperAdmin: admin.NewSuperAdminHandler(nil),
|
||||
SystemConfig: admin.NewSystemConfigHandler(nil, nil),
|
||||
AgentOpenAPI: openapiHandler.NewHandler(nil, nil),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/asynctask"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/bytedance/sonic"
|
||||
@@ -39,10 +40,10 @@ func NewClient(redisClient *redis.Client, logger *zap.Logger) *Client {
|
||||
// payload 必须传入 struct 或 map,禁止传入 []byte。
|
||||
// 传入 []byte 会导致 sonic.Marshal 将其 base64 编码,handler 解析时类型不匹配。
|
||||
func (c *Client) EnqueueTask(ctx context.Context, taskType string, payload interface{}, opts ...asynq.Option) error {
|
||||
if _, ok := payload.([]byte); ok {
|
||||
if err := asynctask.ValidatePayload(payload); err != nil {
|
||||
c.logger.Error("任务载荷类型错误,禁止传入 []byte",
|
||||
zap.String("task_type", taskType))
|
||||
return fmt.Errorf("task payload must be struct or map, got []byte")
|
||||
zap.String("task_type", taskType), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
// 内部统一序列化,调用方无需预先 Marshal
|
||||
|
||||
17
pkg/queue/client_test.go
Normal file
17
pkg/queue/client_test.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func TestEnqueueTaskRejectsPreSerializedBytesBeforeQueueAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := &Client{logger: zap.NewNop()}
|
||||
if err := client.EnqueueTask(context.Background(), "test:bytes", []byte(`{"task_id":1}`)); err == nil {
|
||||
t.Fatal("预序列化 []byte 载荷必须在访问 Asynq 前被拒绝")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user