csxj2026/junhong_cmp_fiber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

实现七月迭代公共技术基础
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 9m20s
Details

Browse Source
This commit is contained in:
break
2026-07-23 17:52:48 +09:00
parent f7c42252c0
commit 17782d5f8e
76 changed files with 5246 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
pkg/logger/access_policy.go Normal file
View File

@@ -0,0 +1,130 @@
package logger
import (
"net/url"
"path/filepath"
"strings"
"github.com/bytedance/sonic"
)
// AccessPolicy 是敏感路由的安全摘要策略。
type AccessPolicy struct {
Name string
Sensitive bool
SafeFields map[string]struct{}
FileFields map[string]struct{}
}
var (
loginPolicy = AccessPolicy{
Name: "login_token", Sensitive: true,
SafeFields: fieldSet("username", "user_id", "account_id", "success", "result_code"),
}
paymentPolicy = AccessPolicy{
Name: "payment", Sensitive: true,
SafeFields: fieldSet("order_no", "payment_no", "channel", "payment_method", "result_code", "amount", "status"),
}
wecomPolicy = AccessPolicy{
Name: "wecom_callback", Sensitive: true,
SafeFields: fieldSet("event_type", "resource_type", "resource_id", "result_code", "status"),
}
filePolicy = AccessPolicy{
Name: "file_export", Sensitive: true,
SafeFields: fieldSet("content_type", "file_size", "size", "count", "task_id", "status", "result_code"),
FileFields: fieldSet("file_name", "filename", "name"),
}
defaultPolicy = AccessPolicy{Name: "default_json"}
)
func fieldSet(fields ...string) map[string]struct{} {
result := make(map[string]struct{}, len(fields))
for _, field := range fields {
result[field] = struct{}{}
}
return result
}
func policyForPath(path string) AccessPolicy {
normalized := strings.ToLower(path)
switch {
case strings.Contains(normalized, "/login"), strings.Contains(normalized, "token"):
return loginPolicy
case strings.Contains(normalized, "/callback") && (strings.Contains(normalized, "wecom") || strings.Contains(normalized, "wework")):
return wecomPolicy
case strings.Contains(normalized, "payment"), strings.Contains(normalized, "wechat-pay"),
strings.Contains(normalized, "alipay"), strings.Contains(normalized, "fuiou-pay"):
return paymentPolicy
case strings.Contains(normalized, "/storage"), strings.Contains(normalized, "/upload"),
strings.Contains(normalized, "/download"), strings.Contains(normalized, "/export"):
return filePolicy
default:
return defaultPolicy
}
}
func sanitizeWithPolicy(raw []byte, contentType string, policy AccessPolicy) SanitizedContent {
if len(raw) == 0 {
return SanitizedContent{}
}
if !policy.Sensitive {
return sanitizeBody(raw, BodyPolicyJSON)
}
summary := make(map[string]any)
normalizedContentType := strings.ToLower(contentType)
switch {
case strings.Contains(normalizedContentType, "json"):
var payload any
if sonic.Unmarshal(raw, &payload) == nil {
collectSafeFields(payload, policy, summary)
}
case strings.Contains(normalizedContentType, "x-www-form-urlencoded"):
if values, err := url.ParseQuery(string(raw)); err == nil {
for key, items := range values {
collectSafeScalar(key, strings.Join(items, ","), policy, summary)
}
}
}
content := "[仅记录安全摘要]"
if len(summary) > 0 {
if encoded, err := sonic.Marshal(summary); err == nil {
content, _ = truncateBody(encoded, MaxBodyLogSize)
}
}
return SanitizedContent{Content: content, Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize}
}
func collectSafeFields(value any, policy AccessPolicy, output map[string]any) {
switch typed := value.(type) {
case map[string]any:
for key, item := range typed {
switch scalar := item.(type) {
case string:
collectSafeScalar(key, scalar, policy, output)
case float64, bool:
if _, allowed := policy.SafeFields[strings.ToLower(key)]; allowed {
output[key] = scalar
}
default:
collectSafeFields(item, policy, output)
}
}
case []any:
for _, item := range typed {
collectSafeFields(item, policy, output)
}
}
}
func collectSafeScalar(key, value string, policy AccessPolicy, output map[string]any) {
normalized := strings.ToLower(key)
if _, isFileName := policy.FileFields[normalized]; isFileName {
base := filepath.Base(value)
hash := digest([]byte(base))
output[key] = "sha256:" + hash[:16]
return
}
if _, allowed := policy.SafeFields[normalized]; allowed {
output[key] = value
}
}

112
pkg/logger/middleware.go
View File

@@ -2,6 +2,8 @@ package logger
import (
"context"
"crypto/sha256"
"encoding/hex"
"net/url"
"strings"
"time"
@@ -15,20 +17,38 @@ import (
const (
// MaxBodyLogSize 限制记录的请求/响应 body 大小为 50KB
MaxBodyLogSize = 50 * 1024
redactedValue = "[已脱敏]"
)
// BodyPolicy 表示访问日志正文记录策略。
type BodyPolicy int
const (
// BodyPolicyJSON 表示只允许记录脱敏后的 JSON 正文。
BodyPolicyJSON BodyPolicy = iota + 1
// BodyPolicySummary 表示只记录不可逆安全摘要。
BodyPolicySummary
)
// SanitizedContent 表示访问日志可安全记录的正文或查询摘要。
type SanitizedContent struct {
Content string
Size int
SHA256 string
Truncated bool
}
// truncateBody 截断 body 到指定大小
func truncateBody(body []byte, maxSize int) string {
func truncateBody(body []byte, maxSize int) (string, bool) {
if len(body) == 0 {
return ""
return "", false
}
if len(body) <= maxSize {
return string(body)
return string(body), false
}
// 超过限制，截断并添加提示
return string(body[:maxSize]) + "... (truncated)"
return string(body[:maxSize]), true
}
// maskSensitiveValue 按字段名判断并脱敏访问日志中的敏感值
@@ -36,25 +56,20 @@ func maskSensitiveValue(key, value string) string {
if value == "" {
return value
}
normalized := strings.ToLower(key)
if strings.Contains(normalized, "password") ||
strings.Contains(normalized, "sign") ||
strings.Contains(normalized, "nonce") ||
strings.Contains(normalized, "token") ||
strings.Contains(normalized, "secret") {
return "***"
if shouldMaskField(key) {
return redactedValue
}
return value
}
// sanitizeQuery 脱敏 query 中的密码、签名、nonce、token 等字段
func sanitizeQuery(rawQuery string) string {
func sanitizeQuery(rawQuery string) SanitizedContent {
if rawQuery == "" {
return ""
return SanitizedContent{}
}
values, err := url.ParseQuery(rawQuery)
if err != nil {
return rawQuery
return summarize([]byte(rawQuery))
}
for key, items := range values {
for index, item := range items {
@@ -62,24 +77,38 @@ func sanitizeQuery(rawQuery string) string {
}
values[key] = items
}
return values.Encode()
content, truncated := truncateBody([]byte(values.Encode()), MaxBodyLogSize)
return SanitizedContent{Content: content, Size: len(rawQuery), SHA256: digest([]byte(rawQuery)), Truncated: truncated}
}
// sanitizeBody 脱敏 JSON 请求体后再写入访问日志
func sanitizeBody(rawBody []byte) string {
// sanitizeBody 按策略脱敏正文后再写入访问日志。
func sanitizeBody(rawBody []byte, policy BodyPolicy) SanitizedContent {
if len(rawBody) == 0 {
return ""
return SanitizedContent{}
}
if policy == BodyPolicySummary {
return summarize(rawBody)
}
var payload any
if err := sonic.Unmarshal(rawBody, &payload); err != nil {
return truncateBody(rawBody, MaxBodyLogSize)
return summarize(rawBody)
}
sanitizeJSONValue(payload)
data, err := sonic.Marshal(payload)
if err != nil {
return truncateBody(rawBody, MaxBodyLogSize)
return summarize(rawBody)
}
return truncateBody(data, MaxBodyLogSize)
content, truncated := truncateBody(data, MaxBodyLogSize)
return SanitizedContent{Content: content, Size: len(rawBody), SHA256: digest(rawBody), Truncated: truncated}
}
func summarize(raw []byte) SanitizedContent {
return SanitizedContent{Content: "[仅记录安全摘要]", Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize}
}
func digest(raw []byte) string {
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// sanitizeJSONValue 递归脱敏 JSON 对象中的敏感字段
@@ -92,7 +121,7 @@ func sanitizeJSONValue(value any) {
continue
}
if shouldMaskField(key) {
typed[key] = "***"
typed[key] = redactedValue
continue
}
sanitizeJSONValue(item)
@@ -108,6 +137,14 @@ func sanitizeJSONValue(value any) {
func shouldMaskField(key string) bool {
normalized := strings.ToLower(key)
return strings.Contains(normalized, "password") ||
strings.Contains(normalized, "passwd") ||
strings.Contains(normalized, "credential") ||
strings.Contains(normalized, "authorization") ||
strings.Contains(normalized, "cookie") ||
strings.Contains(normalized, "key") ||
strings.Contains(normalized, "url") ||
strings.Contains(normalized, "qr_content") ||
strings.Contains(normalized, "verification_code") ||
strings.Contains(normalized, "sign") ||
strings.Contains(normalized, "nonce") ||
strings.Contains(normalized, "token") ||
@@ -117,6 +154,12 @@ func shouldMaskField(key string) bool {
// Middleware 创建 Fiber 日志中间件
// 记录所有 HTTP 请求到访问日志（包括请求和响应 body）
func Middleware() fiber.Handler {
return MiddlewareWithLogger(GetAccessLogger())
}
// MiddlewareWithLogger 创建可注入访问日志器的 Fiber 中间件。
// 生产环境使用 Middleware；该入口让集成测试捕获最终 JSON 日志而无需修改全局状态。
func MiddlewareWithLogger(accessLogger *zap.Logger) fiber.Handler {
return func(c *fiber.Ctx) error {
// 记录请求开始时间
startTime := time.Now()
@@ -136,10 +179,14 @@ func Middleware() fiber.Handler {
c.SetUserContext(ctx)
// 获取请求 body（在 c.Next() 之前读取）
requestBody := sanitizeBody(c.Body())
policy := policyForPath(c.Path())
requestBody := sanitizeWithPolicy(c.Body(), c.Get("Content-Type"), policy)
// 获取 query 参数
queryParams := sanitizeQuery(string(c.Request().URI().QueryString()))
if policy.Sensitive && queryParams.Size > 0 {
queryParams = summarize([]byte(c.Request().URI().QueryString()))
}
// 处理请求
err := c.Next()
@@ -162,22 +209,29 @@ func Middleware() fiber.Handler {
}
// 获取响应 body
responseBody := truncateBody(c.Response().Body(), MaxBodyLogSize)
responseBody := sanitizeWithPolicy(c.Response().Body(), string(c.Response().Header.ContentType()), policy)
// 记录访问日志
accessLogger := GetAccessLogger()
accessLogger.Info("",
zap.String("method", c.Method()),
zap.String("path", c.Path()),
zap.String("query", queryParams),
zap.String("body_policy", policy.Name),
zap.String("query", queryParams.Content),
zap.Bool("query_truncated", queryParams.Truncated),
zap.Int("status", c.Response().StatusCode()),
zap.Float64("duration_ms", float64(duration.Microseconds())/1000.0),
zap.String("request_id", requestID),
zap.String("ip", c.IP()),
zap.String("user_agent", c.Get("User-Agent")),
zap.Uint("user_id", userID),
zap.String("request_body", requestBody),
zap.String("response_body", responseBody),
zap.String("request_body", requestBody.Content),
zap.Int("request_body_size", requestBody.Size),
zap.String("request_body_sha256", requestBody.SHA256),
zap.Bool("request_body_truncated", requestBody.Truncated),
zap.String("response_body", responseBody.Content),
zap.Int("response_body_size", responseBody.Size),
zap.String("response_body_sha256", responseBody.SHA256),
zap.Bool("response_body_truncated", responseBody.Truncated),
)
return err

154
pkg/logger/middleware_test.go Normal file
View File

@@ -0,0 +1,154 @@
package logger
import (
"bytes"
"io"
"net/http/httptest"
"strings"
"testing"
"github.com/gofiber/fiber/v2"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
func TestSanitizeJSONRecursivelyMasksRequestAndResponseSecrets(t *testing.T) {
t.Parallel()
raw := []byte(`{"Password":"p@ss","items":[{"access_token":"token-value","count":3}],"profile":{"private_url":"https://secret.example/a"}}`)
result := sanitizeBody(raw, BodyPolicyJSON)
if strings.Contains(result.Content, "p@ss") || strings.Contains(result.Content, "token-value") || strings.Contains(result.Content, "secret.example") {
t.Fatalf("访问日志仍包含敏感值：%s", result.Content)
}
if !strings.Contains(result.Content, redactedValue) {
t.Fatalf("访问日志未输出统一脱敏占位：%s", result.Content)
}
if result.Truncated {
t.Fatal("短载荷不应标记为截断")
}
}
func TestFiberAccessLogSanitizesFinalRequestAndResponseJSON(t *testing.T) {
var output bytes.Buffer
encoderConfig := zap.NewProductionEncoderConfig()
log := zap.New(zapcore.NewCore(
zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(&output), zapcore.InfoLevel,
))
app := fiber.New()
app.Use(func(c *fiber.Ctx) error {
c.Locals("requestid", "request-test-1")
return c.Next()
})
app.Use(MiddlewareWithLogger(log))
app.Post("/tokens", func(c *fiber.Ctx) error {
return c.JSON(fiber.Map{"access_token": "response-token", "data": fiber.Map{"ok": true}})
})
request := httptest.NewRequest("POST", "/tokens?Authorization=query-secret&page=1", strings.NewReader(`{"password":"request-secret"}`))
request.Header.Set("Content-Type", "application/json")
response, err := app.Test(request)
if err != nil {
t.Fatalf("执行 Fiber 请求失败：%v", err)
}
_, _ = io.Copy(io.Discard, response.Body)
_ = response.Body.Close()
logged := output.String()
for _, secret := range []string{"request-secret", "response-token", "query-secret"} {
if strings.Contains(logged, secret) {
t.Fatalf("最终 JSON 访问日志泄露测试凭证 %q：%s", secret, logged)
}
}
for _, field := range []string{"request_id", "status", "duration_ms", "request_body_truncated", "response_body_truncated"} {
if !strings.Contains(logged, field) {
t.Fatalf("最终 JSON 访问日志缺少字段 %q：%s", field, logged)
}
}
}
func TestSensitiveRouteMatrixNeverLogsRawPayload(t *testing.T) {
cases := []struct {
name string
path string
contentType string
body string
secret string
}{
{name: "登录JSON", path: "/api/auth/login", contentType: "application/json", body: `{"username":"safe-user","password":"json-secret"}`, secret: "json-secret"},
{name: "企微XML", path: "/api/callback/wecom", contentType: "application/xml", body: `<xml><Encrypt>xml-secret</Encrypt></xml>`, secret: "xml-secret"},
{name: "支付表单", path: "/api/callback/alipay", contentType: "application/x-www-form-urlencoded", body: "order_no=SAFE-1&sign=form-secret", secret: "form-secret"},
{name: "文件multipart", path: "/api/admin/storage/upload", contentType: "multipart/form-data; boundary=x", body: "--x\r\nContent-Disposition: form-data; name=\"file\"; filename=\"secret.bin\"\r\nContent-Type: application/octet-stream\r\n\r\nfile-bytes-secret\r\n--x--\r\n", secret: "file-bytes-secret"},
{name: "导出二进制", path: "/api/admin/export-tasks/1/download", contentType: "application/octet-stream", body: "binary-secret", secret: "binary-secret"},
{name: "支付不可解析", path: "/api/callback/wechat-pay", contentType: "application/json", body: `{"sign":"broken-secret"`, secret: "broken-secret"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var output bytes.Buffer
log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel))
app := fiber.New()
app.Use(MiddlewareWithLogger(log))
app.Post(tc.path, func(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).Type("json").SendString(`{"access_token":"response-secret","status":"ok"}`)
})
request := httptest.NewRequest("POST", tc.path+"?signature=query-secret", strings.NewReader(tc.body))
request.Header.Set("Content-Type", tc.contentType)
response, err := app.Test(request)
if err != nil {
t.Fatalf("执行敏感路由请求失败：%v", err)
}
_, _ = io.Copy(io.Discard, response.Body)
_ = response.Body.Close()
logged := output.String()
for _, secret := range []string{tc.secret, "query-secret", "response-secret"} {
if strings.Contains(logged, secret) {
t.Fatalf("敏感路由日志泄露 %q：%s", secret, logged)
}
}
if !strings.Contains(logged, "body_policy") || !strings.Contains(logged, "sha256") {
t.Fatalf("敏感路由日志缺少策略或摘要：%s", logged)
}
})
}
}
func TestSensitiveRouteLongBodyRecordsTruncationWithoutRawSecret(t *testing.T) {
var output bytes.Buffer
log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel))
app := fiber.New()
app.Use(MiddlewareWithLogger(log))
app.Post("/api/admin/storage/upload", func(c *fiber.Ctx) error { return c.SendStatus(fiber.StatusOK) })
secret := strings.Repeat("long-file-secret", MaxBodyLogSize)
request := httptest.NewRequest("POST", "/api/admin/storage/upload", strings.NewReader(secret))
request.Header.Set("Content-Type", "application/octet-stream")
response, err := app.Test(request, 10000)
if err != nil {
t.Fatalf("执行超长敏感请求失败：%v", err)
}
_ = response.Body.Close()
logged := output.String()
if strings.Contains(logged, "long-file-secret") || !strings.Contains(logged, `"request_body_truncated":true`) {
t.Fatalf("超长敏感载荷未安全摘要或未标记截断：%s", logged)
}
}
func TestSanitizeInvalidJSONNeverFallsBackToRawBody(t *testing.T) {
t.Parallel()
raw := []byte(`{"password":"should-never-appear"`)
result := sanitizeBody(raw, BodyPolicyJSON)
if strings.Contains(result.Content, "should-never-appear") {
t.Fatalf("解析失败时不得回退原文：%s", result.Content)
}
if result.SHA256 == "" || result.Size != len(raw) {
t.Fatalf("解析失败摘要缺少大小或哈希：%+v", result)
}
}
func TestSanitizeQueryUsesSameSensitiveFieldRules(t *testing.T) {
t.Parallel()
query := sanitizeQuery("page=1&Authorization=Bearer-secret&nonce=abc")
if strings.Contains(query.Content, "Bearer-secret") || strings.Contains(query.Content, "abc") {
t.Fatalf("query 敏感值未脱敏：%s", query.Content)
}
}