实现七月迭代公共技术基础
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m20s

This commit is contained in:
2026-07-23 17:52:48 +09:00
parent f7c42252c0
commit 17782d5f8e
76 changed files with 5246 additions and 158 deletions

103
pkg/asynctask/contract.go Normal file
View File

@@ -0,0 +1,103 @@
// Package asynctask 定义业务异步任务的统一公开契约。
package asynctask
import (
stderrors "errors"
"reflect"
"time"
)
const (
// StatusPending 表示任务待处理。
StatusPending = 1
// StatusProcessing 表示任务处理中。
StatusProcessing = 2
// StatusCompleted 表示任务已到达业务处理终点。
StatusCompleted = 3
// StatusFailed 表示任务整体无法执行到业务终点。
StatusFailed = 4
// StatusCancelled 表示业务明确支持且已经取消任务。
StatusCancelled = 5
)
var statusNames = map[int]string{
StatusPending: "待处理",
StatusProcessing: "处理中",
StatusCompleted: "已完成",
StatusFailed: "已失败",
StatusCancelled: "已取消",
}
// Projection 是跨业务统一的任务公开投影。
type Projection struct {
TaskID string `json:"task_id"`
Status int `json:"status"`
StatusName string `json:"status_name"`
TotalCount int `json:"total_count"`
SuccessCount int `json:"success_count"`
FailedCount int `json:"failed_count"`
Progress int `json:"progress"`
ErrorCode string `json:"error_code,omitempty"`
ErrorSummary string `json:"error_summary,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
// TerminalResult 是构造终态投影所需的业务结果。
type TerminalResult struct {
TaskID string
Status int
TotalCount int
SuccessCount int
FailedCount int
ErrorCode string
ErrorSummary string
StartedAt *time.Time
CompletedAt *time.Time
UpdatedAt time.Time
}
// StatusName 返回固定任务状态的中文名称。
func StatusName(status int) string {
return statusNames[status]
}
// IsTerminal 判断状态是否为统一终态。
func IsTerminal(status int) bool {
return status == StatusCompleted || status == StatusFailed || status == StatusCancelled
}
// NewTerminalProjection 校验终态守恒并构造公开投影。
func NewTerminalProjection(result TerminalResult) (Projection, error) {
if !IsTerminal(result.Status) {
return Projection{}, stderrors.New("任务状态不是终态")
}
if result.TotalCount < 0 || result.SuccessCount < 0 || result.FailedCount < 0 {
return Projection{}, stderrors.New("任务结果计数不能为负数")
}
if result.Status == StatusCompleted && result.TotalCount != result.SuccessCount+result.FailedCount {
return Projection{}, stderrors.New("已完成任务的结果计数不守恒")
}
return Projection{
TaskID: result.TaskID, Status: result.Status, StatusName: StatusName(result.Status),
TotalCount: result.TotalCount, SuccessCount: result.SuccessCount, FailedCount: result.FailedCount,
Progress: 100, ErrorCode: result.ErrorCode, ErrorSummary: result.ErrorSummary,
StartedAt: result.StartedAt, CompletedAt: result.CompletedAt, UpdatedAt: result.UpdatedAt,
}, nil
}
// ValidatePayload 确保统一队列客户端接收 struct 或 map而非预序列化字节。
func ValidatePayload(payload any) error {
if payload == nil {
return stderrors.New("任务载荷不能为空")
}
kind := reflect.TypeOf(payload).Kind()
if kind != reflect.Struct && kind != reflect.Map && kind != reflect.Ptr {
return stderrors.New("任务载荷必须是结构体或 map")
}
if kind == reflect.Ptr && reflect.TypeOf(payload).Elem().Kind() != reflect.Struct {
return stderrors.New("任务载荷指针必须指向结构体")
}
return nil
}

View File

@@ -0,0 +1,105 @@
package asynctask_test
import (
"testing"
"time"
"github.com/break/junhong_cmp_fiber/pkg/asynctask"
)
func TestCompletedTaskUsesCountsForPartialAndAllItemFailures(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC)
cases := []struct {
name string
success int
failed int
}{
{name: "部分成功", success: 7, failed: 3},
{name: "业务项全部失败", success: 0, failed: 10},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
projection, err := asynctask.NewTerminalProjection(asynctask.TerminalResult{
TaskID: "task-1", Status: asynctask.StatusCompleted,
TotalCount: 10, SuccessCount: tc.success, FailedCount: tc.failed,
StartedAt: &now, CompletedAt: &now, UpdatedAt: now,
})
if err != nil {
t.Fatalf("构造任务投影失败:%v", err)
}
if projection.Status != asynctask.StatusCompleted || projection.StatusName != "已完成" {
t.Fatalf("业务项处理到终点必须为已完成:%+v", projection)
}
if projection.Progress != 100 {
t.Fatalf("终态进度必须为 100得到%d", projection.Progress)
}
})
}
}
func TestTerminalProjectionRejectsBrokenCountInvariant(t *testing.T) {
t.Parallel()
_, err := asynctask.NewTerminalProjection(asynctask.TerminalResult{
TaskID: "task-2", Status: asynctask.StatusCompleted,
TotalCount: 10, SuccessCount: 5, FailedCount: 4,
UpdatedAt: time.Now(),
})
if err == nil {
t.Fatal("终态计数不守恒时应拒绝构造公开投影")
}
}
func TestStructuredPayloadRejectsBytes(t *testing.T) {
t.Parallel()
if err := asynctask.ValidatePayload([]byte(`{"task_id":1}`)); err == nil {
t.Fatal("预序列化 []byte 载荷必须被拒绝")
}
if err := asynctask.ValidatePayload(struct {
TaskID uint `json:"task_id"`
}{TaskID: 1}); err != nil {
t.Fatalf("结构化载荷应通过校验:%v", err)
}
}
func TestClaimRecoversOnlyExpiredProcessingTask(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC)
activeExpiry := now.Add(time.Minute)
active, err := asynctask.Claim(asynctask.LeaseState{
Status: asynctask.StatusProcessing, LeaseOwner: "worker-a", LeaseExpiresAt: &activeExpiry,
}, "worker-b", now, time.Minute)
if err != nil {
t.Fatalf("检查有效租约失败:%v", err)
}
if active.Claimed {
t.Fatal("其他 Worker 不得抢占有效租约")
}
expiredAt := now.Add(-time.Second)
recovered, err := asynctask.Claim(asynctask.LeaseState{
Status: asynctask.StatusProcessing, LeaseOwner: "worker-a", LeaseExpiresAt: &expiredAt,
}, "worker-b", now, time.Minute)
if err != nil {
t.Fatalf("恢复过期任务失败:%v", err)
}
if !recovered.Claimed || recovered.State.LeaseOwner != "worker-b" {
t.Fatalf("过期任务应由新 Worker 恢复:%+v", recovered)
}
}
func TestTerminalTaskCannotBeClaimedOrCancelledAgain(t *testing.T) {
t.Parallel()
result, err := asynctask.Claim(asynctask.LeaseState{Status: asynctask.StatusCompleted}, "worker-a", time.Now(), time.Minute)
if err != nil {
t.Fatalf("检查终态任务失败:%v", err)
}
if result.Claimed || asynctask.CanCancel(asynctask.StatusCompleted) {
t.Fatal("终态任务不得被重复消费或取消")
}
}

53
pkg/asynctask/state.go Normal file
View File

@@ -0,0 +1,53 @@
package asynctask
import (
stderrors "errors"
"time"
)
// LeaseState 是业务任务持久化层应保存的最小状态与租约事实。
type LeaseState struct {
Status int
LeaseOwner string
LeaseExpiresAt *time.Time
}
// ClaimResult 表示条件领取后的新事实。
type ClaimResult struct {
State LeaseState
Claimed bool
}
// Claim 按统一语义判断待处理或租约过期任务能否被领取。
//
// 业务持久化 Adapter 必须把同样的条件放进 PostgreSQL UPDATE 的 WHERE 中;
// 本函数用于跨业务共享状态判定,不能替代数据库条件更新。
func Claim(current LeaseState, owner string, now time.Time, leaseDuration time.Duration) (ClaimResult, error) {
if owner == "" || leaseDuration <= 0 {
return ClaimResult{}, stderrors.New("任务租约所有者和时长不能为空")
}
claimable := current.Status == StatusPending
if current.Status == StatusProcessing && current.LeaseExpiresAt != nil && !current.LeaseExpiresAt.After(now) {
claimable = true
}
if !claimable {
return ClaimResult{State: current}, nil
}
expiresAt := now.Add(leaseDuration)
return ClaimResult{
State: LeaseState{Status: StatusProcessing, LeaseOwner: owner, LeaseExpiresAt: &expiresAt},
Claimed: true,
}, nil
}
// CanFinish 判断当前租约所有者能否从处理中进入终态。
func CanFinish(current LeaseState, owner string, now time.Time) bool {
return current.Status == StatusProcessing &&
current.LeaseOwner == owner &&
current.LeaseExpiresAt != nil && current.LeaseExpiresAt.After(now)
}
// CanCancel 判断业务支持取消时是否可用预期状态条件更新。
func CanCancel(status int) bool {
return status == StatusPending || status == StatusProcessing
}

View File

@@ -1,6 +1,10 @@
package constants
import "time"
import (
"time"
"github.com/break/junhong_cmp_fiber/pkg/asynctask"
)
// Fiber Locals 的上下文键
const (
@@ -79,6 +83,7 @@ const (
TaskTypeAlertCheck = "alert:check" // 告警检查
TaskTypeDataCleanup = "data:cleanup" // 数据清理
TaskTypeDailyTrafficFlush = "traffic:daily:flush" // 每日流量落盘
TaskTypeOutboxDeliver = "outbox:deliver" // 公共 Outbox 事件投递
)
// 用户状态常量
@@ -219,6 +224,7 @@ const (
QueueAlertCheck = TaskTypeAlertCheck // 告警检查任务队列
QueueDataCleanup = TaskTypeDataCleanup // 数据清理任务队列
QueueDailyTrafficFlush = TaskTypeDailyTrafficFlush // 每日流量落盘任务队列
QueueOutboxDeliver = TaskTypeOutboxDeliver // 公共 Outbox 投递队列
DefaultRetryMax = 5 // 默认任务最大重试次数
DefaultTimeout = 10 * time.Minute // 默认任务超时时间
@@ -276,6 +282,8 @@ func QueueForTaskType(taskType string) string {
return QueueDataCleanup
case TaskTypeDailyTrafficFlush:
return QueueDailyTrafficFlush
case TaskTypeOutboxDeliver:
return QueueOutboxDeliver
default:
return QueueDefault
}
@@ -309,6 +317,7 @@ func DefaultTaskQueueWeights() map[string]int {
QueuePackageDataReset: 1,
QueueDataCleanup: 1,
QueueDailyTrafficFlush: 1,
QueueOutboxDeliver: 4,
QueueDefault: 1,
QueueLow: 1,
}
@@ -329,11 +338,11 @@ const (
// 导出主任务状态常量
const (
ExportTaskStatusPending = 1 // 待处理
ExportTaskStatusProcessing = 2 // 处理中
ExportTaskStatusCompleted = 3 // 已完成
ExportTaskStatusFailed = 4 // 已失败
ExportTaskStatusCancelled = 5 // 已取消
ExportTaskStatusPending = asynctask.StatusPending // 待处理
ExportTaskStatusProcessing = asynctask.StatusProcessing // 处理中
ExportTaskStatusCompleted = asynctask.StatusCompleted // 已完成
ExportTaskStatusFailed = asynctask.StatusFailed // 已失败
ExportTaskStatusCancelled = asynctask.StatusCancelled // 已取消
)
// 导出分片任务状态常量
@@ -450,20 +459,11 @@ func GetShelfStatusName(status int) string {
// GetExportTaskStatusName 获取导出主任务状态名称
func GetExportTaskStatusName(status int) string {
switch status {
case ExportTaskStatusPending:
return "待处理"
case ExportTaskStatusProcessing:
return "处理中"
case ExportTaskStatusCompleted:
return "已完成"
case ExportTaskStatusFailed:
return "已失败"
case ExportTaskStatusCancelled:
return "已取消"
default:
name := asynctask.StatusName(status)
if name == "" {
return "未知"
}
return name
}
// GetExportShardStatusName 获取导出分片任务状态名称

45
pkg/constants/outbox.go Normal file
View File

@@ -0,0 +1,45 @@
package constants
import "time"
const (
// OutboxStatusPending 表示事件待投递。
OutboxStatusPending = 1
// OutboxStatusDelivering 表示事件已被 Relay 租约领取。
OutboxStatusDelivering = 2
// OutboxStatusDelivered 表示事件已成功提交到队列。
OutboxStatusDelivered = 3
// OutboxStatusFailed 表示事件达到最终失败或等待人工重放。
OutboxStatusFailed = 4
)
const (
// OutboxDefaultMaxRetries 是公共 Outbox 默认最大重试次数。
OutboxDefaultMaxRetries = 10
// OutboxDefaultLeaseDuration 是 Relay 默认租约时长。
OutboxDefaultLeaseDuration = time.Minute
// OutboxDefaultBatchSize 是 Relay 默认单批领取数。
OutboxDefaultBatchSize = 100
// OutboxBaseRetryDelay 是瞬时失败指数退避的基础时长。
OutboxBaseRetryDelay = 5 * time.Second
// OutboxMaxRetryDelay 是瞬时失败指数退避的上限。
OutboxMaxRetryDelay = 30 * time.Minute
// OutboxRelayPollInterval 是 Worker 扫描到期事件的默认间隔。
OutboxRelayPollInterval = time.Second
)
// GetOutboxStatusName 返回 Outbox 内部状态中文名称。
func GetOutboxStatusName(status int) string {
switch status {
case OutboxStatusPending:
return "待投递"
case OutboxStatusDelivering:
return "投递中"
case OutboxStatusDelivered:
return "已投递"
case OutboxStatusFailed:
return "投递失败"
default:
return "未知"
}
}

View File

@@ -0,0 +1,24 @@
package constants
import (
"fmt"
"time"
)
const (
// SystemConfigTypeString 表示字符串配置。
SystemConfigTypeString = "string"
// SystemConfigTypeInt 表示整数配置。
SystemConfigTypeInt = "int"
// SystemConfigTypeBool 表示布尔配置。
SystemConfigTypeBool = "bool"
// SystemConfigTypeJSON 表示 JSON 配置。
SystemConfigTypeJSON = "json"
// SystemConfigCacheTTL 是系统配置默认缓存时长。
SystemConfigCacheTTL = 5 * time.Minute
)
// RedisSystemConfigKey 生成单个系统配置的 Redis 缓存键。
func RedisSystemConfigKey(configKey string) string {
return fmt.Sprintf("system_config:value:%s", configKey)
}

View File

@@ -0,0 +1,124 @@
// Package idempotency 提供创建类命令的幂等身份与请求指纹公共契约。
package idempotency
import (
"crypto/sha256"
"encoding/hex"
"reflect"
"strings"
"github.com/bytedance/sonic"
)
const (
// FingerprintAlgorithmV1 是请求指纹算法的首个稳定版本。
FingerprintAlgorithmV1 = "sha256-canonical-json-v1"
)
// Scope 表示请求 ID 的幂等作用域。
type Scope struct {
Subject string `json:"subject"`
Operation string `json:"operation"`
}
// FingerprintValue 表示带算法版本的请求指纹。
type FingerprintValue struct {
Algorithm string `json:"algorithm"`
Value string `json:"value"`
}
// Record 表示业务模块自行持久化的幂等事实最小投影。
type Record struct {
Scope Scope `json:"scope"`
RequestID string `json:"request_id"`
Fingerprint FingerprintValue `json:"fingerprint"`
}
// ReplayKind 表示已有事实与本次命令的关系。
type ReplayKind string
const (
// ReplaySame 表示相同命令重放,应返回原业务结果。
ReplaySame ReplayKind = "same"
// ReplayConflict 表示同作用域、同请求 ID 携带了不同业务内容。
ReplayConflict ReplayKind = "conflict"
// ReplayUnrelated 表示不是同一幂等作用域内的命令。
ReplayUnrelated ReplayKind = "unrelated"
)
var volatileFieldNames = map[string]struct{}{
"authorization": {},
"cookie": {},
"nonce": {},
"sign": {},
"signature": {},
"timestamp": {},
"token": {},
}
// Fingerprint 对影响业务结果的规范化字段计算稳定指纹。
//
// 调用方仍应优先传入专用命令 DTO本函数会递归排除约定的易变传输字段
// 并去除字符串首尾空白,避免签名、令牌或对象字段顺序污染业务身份。
func Fingerprint(command any) (FingerprintValue, error) {
raw, err := sonic.ConfigStd.Marshal(command)
if err != nil {
return FingerprintValue{}, err
}
var value any
if err := sonic.Unmarshal(raw, &value); err != nil {
return FingerprintValue{}, err
}
normalized := normalize(value)
canonical, err := sonic.ConfigStd.Marshal(normalized)
if err != nil {
return FingerprintValue{}, err
}
sum := sha256.Sum256(canonical)
return FingerprintValue{
Algorithm: FingerprintAlgorithmV1,
Value: hex.EncodeToString(sum[:]),
}, nil
}
// ValidateScope 校验幂等作用域和稳定请求 ID 是否完整。
func ValidateScope(scope Scope, requestID string) bool {
return strings.TrimSpace(scope.Subject) != "" &&
strings.TrimSpace(scope.Operation) != "" &&
strings.TrimSpace(requestID) != ""
}
// Classify 比较已有幂等事实与本次命令。
func Classify(existing Record, scope Scope, requestID string, fingerprint FingerprintValue) ReplayKind {
if existing.Scope != scope || existing.RequestID != requestID {
return ReplayUnrelated
}
if reflect.DeepEqual(existing.Fingerprint, fingerprint) {
return ReplaySame
}
return ReplayConflict
}
func normalize(value any) any {
switch typed := value.(type) {
case map[string]any:
result := make(map[string]any, len(typed))
for key, item := range typed {
if _, volatile := volatileFieldNames[strings.ToLower(strings.TrimSpace(key))]; volatile {
continue
}
result[key] = normalize(item)
}
return result
case []any:
result := make([]any, len(typed))
for index, item := range typed {
result[index] = normalize(item)
}
return result
case string:
return strings.TrimSpace(typed)
default:
return value
}
}

View File

@@ -0,0 +1,58 @@
package idempotency_test
import (
"testing"
"github.com/break/junhong_cmp_fiber/pkg/idempotency"
)
func TestFingerprintIgnoresTransportFieldsAndNormalizesObjectOrder(t *testing.T) {
t.Parallel()
left, err := idempotency.Fingerprint(map[string]any{
"amount": 100,
"remark": " 月度充值 ",
"token": "token-a",
"nested": map[string]any{"enabled": true, "count": 2},
})
if err != nil {
t.Fatalf("计算第一个请求指纹失败:%v", err)
}
right, err := idempotency.Fingerprint(map[string]any{
"nested": map[string]any{"count": 2, "enabled": true},
"timestamp": 1720000000,
"remark": "月度充值",
"amount": 100,
})
if err != nil {
t.Fatalf("计算第二个请求指纹失败:%v", err)
}
if left.Algorithm != idempotency.FingerprintAlgorithmV1 {
t.Fatalf("算法版本不正确:%s", left.Algorithm)
}
if left.Value != right.Value {
t.Fatalf("仅传输字段或对象顺序变化不应改变指纹:%s != %s", left.Value, right.Value)
}
}
func TestClassifyReplayDistinguishesScopeAndConflict(t *testing.T) {
t.Parallel()
original := idempotency.Record{
Scope: idempotency.Scope{Subject: "account:7", Operation: "agent-recharge.create"},
RequestID: "request-1",
Fingerprint: idempotency.FingerprintValue{Algorithm: idempotency.FingerprintAlgorithmV1, Value: "abc"},
}
if got := idempotency.Classify(original, original.Scope, "request-1", original.Fingerprint); got != idempotency.ReplaySame {
t.Fatalf("相同命令应识别为重放,得到:%s", got)
}
if got := idempotency.Classify(original, original.Scope, "request-1", idempotency.FingerprintValue{Algorithm: idempotency.FingerprintAlgorithmV1, Value: "def"}); got != idempotency.ReplayConflict {
t.Fatalf("同作用域同请求 ID 的不同内容应冲突,得到:%s", got)
}
otherScope := idempotency.Scope{Subject: "account:8", Operation: original.Scope.Operation}
if got := idempotency.Classify(original, otherScope, "request-1", original.Fingerprint); got != idempotency.ReplayUnrelated {
t.Fatalf("不同主体不应相互污染,得到:%s", got)
}
}

View File

@@ -0,0 +1,86 @@
package idempotency_test
import (
"context"
"sync"
"testing"
"github.com/break/junhong_cmp_fiber/internal/testutil"
"github.com/break/junhong_cmp_fiber/pkg/idempotency"
)
func TestPostgresUniqueConstraintArbitratesConcurrentFirstSubmission(t *testing.T) {
db := testutil.NewPostgresTransaction(t)
if err := db.Exec(`
CREATE TEMP TABLE test_command_idempotency (
subject varchar(100) NOT NULL,
operation varchar(100) NOT NULL,
request_id varchar(100) NOT NULL,
fingerprint varchar(128) NOT NULL,
result_id bigint NOT NULL,
UNIQUE (subject, operation, request_id)
) ON COMMIT DROP`).Error; err != nil {
t.Fatalf("创建幂等契约测试表失败:%v", err)
}
fingerprint, err := idempotency.Fingerprint(map[string]any{"amount": 100})
if err != nil {
t.Fatalf("计算指纹失败:%v", err)
}
start := make(chan struct{})
results := make(chan int64, 2)
errs := make(chan error, 2)
var wait sync.WaitGroup
for _, resultID := range []int64{101, 102} {
wait.Add(1)
go func(id int64) {
defer wait.Done()
<-start
result := db.WithContext(context.Background()).Exec(`
INSERT INTO test_command_idempotency
(subject, operation, request_id, fingerprint, result_id)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT (subject, operation, request_id) DO NOTHING`,
"account:7", "agent-recharge.create", "request-1", fingerprint.Value, id,
)
results <- result.RowsAffected
errs <- result.Error
}(resultID)
}
close(start)
wait.Wait()
close(results)
close(errs)
var inserted int64
for resultErr := range errs {
if resultErr != nil {
t.Fatalf("并发写入失败:%v", resultErr)
}
}
for affected := range results {
inserted += affected
}
if inserted != 1 {
t.Fatalf("并发首写必须只有一个数据库事实,实际插入:%d", inserted)
}
var count int64
if err := db.Table("test_command_idempotency").Count(&count).Error; err != nil {
t.Fatalf("统计幂等事实失败:%v", err)
}
if count != 1 {
t.Fatalf("幂等事实数量错误:%d", count)
}
}
func TestIdempotencyScopeDoesNotDependOnRedis(t *testing.T) {
t.Parallel()
scope := idempotency.Scope{Subject: "account:7", Operation: "example.create"}
if !idempotency.ValidateScope(scope, "request-1") {
t.Fatal("完整作用域和请求 ID 应通过校验")
}
// 公共幂等契约没有 Redis 参数Redis 仅可由业务作为削峰优化,不能参与最终裁决。
}

130
pkg/logger/access_policy.go Normal file
View File

@@ -0,0 +1,130 @@
package logger
import (
"net/url"
"path/filepath"
"strings"
"github.com/bytedance/sonic"
)
// AccessPolicy 是敏感路由的安全摘要策略。
type AccessPolicy struct {
Name string
Sensitive bool
SafeFields map[string]struct{}
FileFields map[string]struct{}
}
var (
loginPolicy = AccessPolicy{
Name: "login_token", Sensitive: true,
SafeFields: fieldSet("username", "user_id", "account_id", "success", "result_code"),
}
paymentPolicy = AccessPolicy{
Name: "payment", Sensitive: true,
SafeFields: fieldSet("order_no", "payment_no", "channel", "payment_method", "result_code", "amount", "status"),
}
wecomPolicy = AccessPolicy{
Name: "wecom_callback", Sensitive: true,
SafeFields: fieldSet("event_type", "resource_type", "resource_id", "result_code", "status"),
}
filePolicy = AccessPolicy{
Name: "file_export", Sensitive: true,
SafeFields: fieldSet("content_type", "file_size", "size", "count", "task_id", "status", "result_code"),
FileFields: fieldSet("file_name", "filename", "name"),
}
defaultPolicy = AccessPolicy{Name: "default_json"}
)
func fieldSet(fields ...string) map[string]struct{} {
result := make(map[string]struct{}, len(fields))
for _, field := range fields {
result[field] = struct{}{}
}
return result
}
func policyForPath(path string) AccessPolicy {
normalized := strings.ToLower(path)
switch {
case strings.Contains(normalized, "/login"), strings.Contains(normalized, "token"):
return loginPolicy
case strings.Contains(normalized, "/callback") && (strings.Contains(normalized, "wecom") || strings.Contains(normalized, "wework")):
return wecomPolicy
case strings.Contains(normalized, "payment"), strings.Contains(normalized, "wechat-pay"),
strings.Contains(normalized, "alipay"), strings.Contains(normalized, "fuiou-pay"):
return paymentPolicy
case strings.Contains(normalized, "/storage"), strings.Contains(normalized, "/upload"),
strings.Contains(normalized, "/download"), strings.Contains(normalized, "/export"):
return filePolicy
default:
return defaultPolicy
}
}
func sanitizeWithPolicy(raw []byte, contentType string, policy AccessPolicy) SanitizedContent {
if len(raw) == 0 {
return SanitizedContent{}
}
if !policy.Sensitive {
return sanitizeBody(raw, BodyPolicyJSON)
}
summary := make(map[string]any)
normalizedContentType := strings.ToLower(contentType)
switch {
case strings.Contains(normalizedContentType, "json"):
var payload any
if sonic.Unmarshal(raw, &payload) == nil {
collectSafeFields(payload, policy, summary)
}
case strings.Contains(normalizedContentType, "x-www-form-urlencoded"):
if values, err := url.ParseQuery(string(raw)); err == nil {
for key, items := range values {
collectSafeScalar(key, strings.Join(items, ","), policy, summary)
}
}
}
content := "[仅记录安全摘要]"
if len(summary) > 0 {
if encoded, err := sonic.Marshal(summary); err == nil {
content, _ = truncateBody(encoded, MaxBodyLogSize)
}
}
return SanitizedContent{Content: content, Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize}
}
func collectSafeFields(value any, policy AccessPolicy, output map[string]any) {
switch typed := value.(type) {
case map[string]any:
for key, item := range typed {
switch scalar := item.(type) {
case string:
collectSafeScalar(key, scalar, policy, output)
case float64, bool:
if _, allowed := policy.SafeFields[strings.ToLower(key)]; allowed {
output[key] = scalar
}
default:
collectSafeFields(item, policy, output)
}
}
case []any:
for _, item := range typed {
collectSafeFields(item, policy, output)
}
}
}
func collectSafeScalar(key, value string, policy AccessPolicy, output map[string]any) {
normalized := strings.ToLower(key)
if _, isFileName := policy.FileFields[normalized]; isFileName {
base := filepath.Base(value)
hash := digest([]byte(base))
output[key] = "sha256:" + hash[:16]
return
}
if _, allowed := policy.SafeFields[normalized]; allowed {
output[key] = value
}
}

View File

@@ -2,6 +2,8 @@ package logger
import (
"context"
"crypto/sha256"
"encoding/hex"
"net/url"
"strings"
"time"
@@ -15,20 +17,38 @@ import (
const (
// MaxBodyLogSize 限制记录的请求/响应 body 大小为 50KB
MaxBodyLogSize = 50 * 1024
redactedValue = "[已脱敏]"
)
// BodyPolicy 表示访问日志正文记录策略。
type BodyPolicy int
const (
// BodyPolicyJSON 表示只允许记录脱敏后的 JSON 正文。
BodyPolicyJSON BodyPolicy = iota + 1
// BodyPolicySummary 表示只记录不可逆安全摘要。
BodyPolicySummary
)
// SanitizedContent 表示访问日志可安全记录的正文或查询摘要。
type SanitizedContent struct {
Content string
Size int
SHA256 string
Truncated bool
}
// truncateBody 截断 body 到指定大小
func truncateBody(body []byte, maxSize int) string {
func truncateBody(body []byte, maxSize int) (string, bool) {
if len(body) == 0 {
return ""
return "", false
}
if len(body) <= maxSize {
return string(body)
return string(body), false
}
// 超过限制,截断并添加提示
return string(body[:maxSize]) + "... (truncated)"
return string(body[:maxSize]), true
}
// maskSensitiveValue 按字段名判断并脱敏访问日志中的敏感值
@@ -36,25 +56,20 @@ func maskSensitiveValue(key, value string) string {
if value == "" {
return value
}
normalized := strings.ToLower(key)
if strings.Contains(normalized, "password") ||
strings.Contains(normalized, "sign") ||
strings.Contains(normalized, "nonce") ||
strings.Contains(normalized, "token") ||
strings.Contains(normalized, "secret") {
return "***"
if shouldMaskField(key) {
return redactedValue
}
return value
}
// sanitizeQuery 脱敏 query 中的密码、签名、nonce、token 等字段
func sanitizeQuery(rawQuery string) string {
func sanitizeQuery(rawQuery string) SanitizedContent {
if rawQuery == "" {
return ""
return SanitizedContent{}
}
values, err := url.ParseQuery(rawQuery)
if err != nil {
return rawQuery
return summarize([]byte(rawQuery))
}
for key, items := range values {
for index, item := range items {
@@ -62,24 +77,38 @@ func sanitizeQuery(rawQuery string) string {
}
values[key] = items
}
return values.Encode()
content, truncated := truncateBody([]byte(values.Encode()), MaxBodyLogSize)
return SanitizedContent{Content: content, Size: len(rawQuery), SHA256: digest([]byte(rawQuery)), Truncated: truncated}
}
// sanitizeBody 脱敏 JSON 请求体后再写入访问日志
func sanitizeBody(rawBody []byte) string {
// sanitizeBody 按策略脱敏正文后再写入访问日志
func sanitizeBody(rawBody []byte, policy BodyPolicy) SanitizedContent {
if len(rawBody) == 0 {
return ""
return SanitizedContent{}
}
if policy == BodyPolicySummary {
return summarize(rawBody)
}
var payload any
if err := sonic.Unmarshal(rawBody, &payload); err != nil {
return truncateBody(rawBody, MaxBodyLogSize)
return summarize(rawBody)
}
sanitizeJSONValue(payload)
data, err := sonic.Marshal(payload)
if err != nil {
return truncateBody(rawBody, MaxBodyLogSize)
return summarize(rawBody)
}
return truncateBody(data, MaxBodyLogSize)
content, truncated := truncateBody(data, MaxBodyLogSize)
return SanitizedContent{Content: content, Size: len(rawBody), SHA256: digest(rawBody), Truncated: truncated}
}
func summarize(raw []byte) SanitizedContent {
return SanitizedContent{Content: "[仅记录安全摘要]", Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize}
}
func digest(raw []byte) string {
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// sanitizeJSONValue 递归脱敏 JSON 对象中的敏感字段
@@ -92,7 +121,7 @@ func sanitizeJSONValue(value any) {
continue
}
if shouldMaskField(key) {
typed[key] = "***"
typed[key] = redactedValue
continue
}
sanitizeJSONValue(item)
@@ -108,6 +137,14 @@ func sanitizeJSONValue(value any) {
func shouldMaskField(key string) bool {
normalized := strings.ToLower(key)
return strings.Contains(normalized, "password") ||
strings.Contains(normalized, "passwd") ||
strings.Contains(normalized, "credential") ||
strings.Contains(normalized, "authorization") ||
strings.Contains(normalized, "cookie") ||
strings.Contains(normalized, "key") ||
strings.Contains(normalized, "url") ||
strings.Contains(normalized, "qr_content") ||
strings.Contains(normalized, "verification_code") ||
strings.Contains(normalized, "sign") ||
strings.Contains(normalized, "nonce") ||
strings.Contains(normalized, "token") ||
@@ -117,6 +154,12 @@ func shouldMaskField(key string) bool {
// Middleware 创建 Fiber 日志中间件
// 记录所有 HTTP 请求到访问日志(包括请求和响应 body
func Middleware() fiber.Handler {
return MiddlewareWithLogger(GetAccessLogger())
}
// MiddlewareWithLogger 创建可注入访问日志器的 Fiber 中间件。
// 生产环境使用 Middleware该入口让集成测试捕获最终 JSON 日志而无需修改全局状态。
func MiddlewareWithLogger(accessLogger *zap.Logger) fiber.Handler {
return func(c *fiber.Ctx) error {
// 记录请求开始时间
startTime := time.Now()
@@ -136,10 +179,14 @@ func Middleware() fiber.Handler {
c.SetUserContext(ctx)
// 获取请求 body在 c.Next() 之前读取)
requestBody := sanitizeBody(c.Body())
policy := policyForPath(c.Path())
requestBody := sanitizeWithPolicy(c.Body(), c.Get("Content-Type"), policy)
// 获取 query 参数
queryParams := sanitizeQuery(string(c.Request().URI().QueryString()))
if policy.Sensitive && queryParams.Size > 0 {
queryParams = summarize([]byte(c.Request().URI().QueryString()))
}
// 处理请求
err := c.Next()
@@ -162,22 +209,29 @@ func Middleware() fiber.Handler {
}
// 获取响应 body
responseBody := truncateBody(c.Response().Body(), MaxBodyLogSize)
responseBody := sanitizeWithPolicy(c.Response().Body(), string(c.Response().Header.ContentType()), policy)
// 记录访问日志
accessLogger := GetAccessLogger()
accessLogger.Info("",
zap.String("method", c.Method()),
zap.String("path", c.Path()),
zap.String("query", queryParams),
zap.String("body_policy", policy.Name),
zap.String("query", queryParams.Content),
zap.Bool("query_truncated", queryParams.Truncated),
zap.Int("status", c.Response().StatusCode()),
zap.Float64("duration_ms", float64(duration.Microseconds())/1000.0),
zap.String("request_id", requestID),
zap.String("ip", c.IP()),
zap.String("user_agent", c.Get("User-Agent")),
zap.Uint("user_id", userID),
zap.String("request_body", requestBody),
zap.String("response_body", responseBody),
zap.String("request_body", requestBody.Content),
zap.Int("request_body_size", requestBody.Size),
zap.String("request_body_sha256", requestBody.SHA256),
zap.Bool("request_body_truncated", requestBody.Truncated),
zap.String("response_body", responseBody.Content),
zap.Int("response_body_size", responseBody.Size),
zap.String("response_body_sha256", responseBody.SHA256),
zap.Bool("response_body_truncated", responseBody.Truncated),
)
return err

View File

@@ -0,0 +1,154 @@
package logger
import (
"bytes"
"io"
"net/http/httptest"
"strings"
"testing"
"github.com/gofiber/fiber/v2"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
func TestSanitizeJSONRecursivelyMasksRequestAndResponseSecrets(t *testing.T) {
t.Parallel()
raw := []byte(`{"Password":"p@ss","items":[{"access_token":"token-value","count":3}],"profile":{"private_url":"https://secret.example/a"}}`)
result := sanitizeBody(raw, BodyPolicyJSON)
if strings.Contains(result.Content, "p@ss") || strings.Contains(result.Content, "token-value") || strings.Contains(result.Content, "secret.example") {
t.Fatalf("访问日志仍包含敏感值:%s", result.Content)
}
if !strings.Contains(result.Content, redactedValue) {
t.Fatalf("访问日志未输出统一脱敏占位:%s", result.Content)
}
if result.Truncated {
t.Fatal("短载荷不应标记为截断")
}
}
func TestFiberAccessLogSanitizesFinalRequestAndResponseJSON(t *testing.T) {
var output bytes.Buffer
encoderConfig := zap.NewProductionEncoderConfig()
log := zap.New(zapcore.NewCore(
zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(&output), zapcore.InfoLevel,
))
app := fiber.New()
app.Use(func(c *fiber.Ctx) error {
c.Locals("requestid", "request-test-1")
return c.Next()
})
app.Use(MiddlewareWithLogger(log))
app.Post("/tokens", func(c *fiber.Ctx) error {
return c.JSON(fiber.Map{"access_token": "response-token", "data": fiber.Map{"ok": true}})
})
request := httptest.NewRequest("POST", "/tokens?Authorization=query-secret&page=1", strings.NewReader(`{"password":"request-secret"}`))
request.Header.Set("Content-Type", "application/json")
response, err := app.Test(request)
if err != nil {
t.Fatalf("执行 Fiber 请求失败:%v", err)
}
_, _ = io.Copy(io.Discard, response.Body)
_ = response.Body.Close()
logged := output.String()
for _, secret := range []string{"request-secret", "response-token", "query-secret"} {
if strings.Contains(logged, secret) {
t.Fatalf("最终 JSON 访问日志泄露测试凭证 %q%s", secret, logged)
}
}
for _, field := range []string{"request_id", "status", "duration_ms", "request_body_truncated", "response_body_truncated"} {
if !strings.Contains(logged, field) {
t.Fatalf("最终 JSON 访问日志缺少字段 %q%s", field, logged)
}
}
}
func TestSensitiveRouteMatrixNeverLogsRawPayload(t *testing.T) {
cases := []struct {
name string
path string
contentType string
body string
secret string
}{
{name: "登录JSON", path: "/api/auth/login", contentType: "application/json", body: `{"username":"safe-user","password":"json-secret"}`, secret: "json-secret"},
{name: "企微XML", path: "/api/callback/wecom", contentType: "application/xml", body: `<xml><Encrypt>xml-secret</Encrypt></xml>`, secret: "xml-secret"},
{name: "支付表单", path: "/api/callback/alipay", contentType: "application/x-www-form-urlencoded", body: "order_no=SAFE-1&sign=form-secret", secret: "form-secret"},
{name: "文件multipart", path: "/api/admin/storage/upload", contentType: "multipart/form-data; boundary=x", body: "--x\r\nContent-Disposition: form-data; name=\"file\"; filename=\"secret.bin\"\r\nContent-Type: application/octet-stream\r\n\r\nfile-bytes-secret\r\n--x--\r\n", secret: "file-bytes-secret"},
{name: "导出二进制", path: "/api/admin/export-tasks/1/download", contentType: "application/octet-stream", body: "binary-secret", secret: "binary-secret"},
{name: "支付不可解析", path: "/api/callback/wechat-pay", contentType: "application/json", body: `{"sign":"broken-secret"`, secret: "broken-secret"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var output bytes.Buffer
log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel))
app := fiber.New()
app.Use(MiddlewareWithLogger(log))
app.Post(tc.path, func(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).Type("json").SendString(`{"access_token":"response-secret","status":"ok"}`)
})
request := httptest.NewRequest("POST", tc.path+"?signature=query-secret", strings.NewReader(tc.body))
request.Header.Set("Content-Type", tc.contentType)
response, err := app.Test(request)
if err != nil {
t.Fatalf("执行敏感路由请求失败:%v", err)
}
_, _ = io.Copy(io.Discard, response.Body)
_ = response.Body.Close()
logged := output.String()
for _, secret := range []string{tc.secret, "query-secret", "response-secret"} {
if strings.Contains(logged, secret) {
t.Fatalf("敏感路由日志泄露 %q%s", secret, logged)
}
}
if !strings.Contains(logged, "body_policy") || !strings.Contains(logged, "sha256") {
t.Fatalf("敏感路由日志缺少策略或摘要:%s", logged)
}
})
}
}
func TestSensitiveRouteLongBodyRecordsTruncationWithoutRawSecret(t *testing.T) {
var output bytes.Buffer
log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel))
app := fiber.New()
app.Use(MiddlewareWithLogger(log))
app.Post("/api/admin/storage/upload", func(c *fiber.Ctx) error { return c.SendStatus(fiber.StatusOK) })
secret := strings.Repeat("long-file-secret", MaxBodyLogSize)
request := httptest.NewRequest("POST", "/api/admin/storage/upload", strings.NewReader(secret))
request.Header.Set("Content-Type", "application/octet-stream")
response, err := app.Test(request, 10000)
if err != nil {
t.Fatalf("执行超长敏感请求失败:%v", err)
}
_ = response.Body.Close()
logged := output.String()
if strings.Contains(logged, "long-file-secret") || !strings.Contains(logged, `"request_body_truncated":true`) {
t.Fatalf("超长敏感载荷未安全摘要或未标记截断:%s", logged)
}
}
func TestSanitizeInvalidJSONNeverFallsBackToRawBody(t *testing.T) {
t.Parallel()
raw := []byte(`{"password":"should-never-appear"`)
result := sanitizeBody(raw, BodyPolicyJSON)
if strings.Contains(result.Content, "should-never-appear") {
t.Fatalf("解析失败时不得回退原文:%s", result.Content)
}
if result.SHA256 == "" || result.Size != len(raw) {
t.Fatalf("解析失败摘要缺少大小或哈希:%+v", result)
}
}
func TestSanitizeQueryUsesSameSensitiveFieldRules(t *testing.T) {
t.Parallel()
query := sanitizeQuery("page=1&Authorization=Bearer-secret&nonce=abc")
if strings.Contains(query.Content, "Bearer-secret") || strings.Contains(query.Content, "abc") {
t.Fatalf("query 敏感值未脱敏:%s", query.Content)
}
}

View File

@@ -67,6 +67,7 @@ func BuildDocHandlers() *bootstrap.Handlers {
OrderPackageInvalidate: admin.NewOrderPackageInvalidateHandler(nil),
ClientWechat: app.NewClientWechatHandler(nil, nil, nil),
SuperAdmin: admin.NewSuperAdminHandler(nil),
SystemConfig: admin.NewSystemConfigHandler(nil, nil),
AgentOpenAPI: openapiHandler.NewHandler(nil, nil),
}
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/break/junhong_cmp_fiber/pkg/asynctask"
"github.com/break/junhong_cmp_fiber/pkg/config"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/bytedance/sonic"
@@ -39,10 +40,10 @@ func NewClient(redisClient *redis.Client, logger *zap.Logger) *Client {
// payload 必须传入 struct 或 map禁止传入 []byte。
// 传入 []byte 会导致 sonic.Marshal 将其 base64 编码handler 解析时类型不匹配。
func (c *Client) EnqueueTask(ctx context.Context, taskType string, payload interface{}, opts ...asynq.Option) error {
if _, ok := payload.([]byte); ok {
if err := asynctask.ValidatePayload(payload); err != nil {
c.logger.Error("任务载荷类型错误,禁止传入 []byte",
zap.String("task_type", taskType))
return fmt.Errorf("task payload must be struct or map, got []byte")
zap.String("task_type", taskType), zap.Error(err))
return err
}
// 内部统一序列化,调用方无需预先 Marshal

17
pkg/queue/client_test.go Normal file
View File

@@ -0,0 +1,17 @@
package queue
import (
"context"
"testing"
"go.uber.org/zap"
)
func TestEnqueueTaskRejectsPreSerializedBytesBeforeQueueAccess(t *testing.T) {
t.Parallel()
client := &Client{logger: zap.NewNop()}
if err := client.EnqueueTask(context.Background(), "test:bytes", []byte(`{"task_id":1}`)); err == nil {
t.Fatal("预序列化 []byte 载荷必须在访问 Asynq 前被拒绝")
}
}