feat: 卡网关按运营商账户限流并记录停复机调用
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m33s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m33s
- 新增网关限流:按本系统运营商账户(卡 carrier_id)→ 限流组 → 操作限制每秒请求数,每次 HTTP 尝试计量,默认限流、仅后台人工点击/通道阈值/自动续费/手动刷新豁免;支持只统计与拦截两种模式,默认只统计 - Asynq 遇限流不计失败、固定短延迟重试;轮询任务与停复机业务重试遇限流短延迟重排 - 停机、复机不再使用网关内部重试,只发一次请求 - 网关返回结构化错误;新增上游错误分类规则表与测试接口 - 新增按月分区的停复机记录表:系统自动停复机只写记录,不再写统一审计与集成日志;普通轮询查询不再写集成日志 - 新增限流组、错误分类、停复机记录、按账户调用统计与枚举后台接口 - 迁移 000243~000246;同步 OpenSpec 提案(含后续 6 个 change)与技术方案、工程约束
This commit is contained in:
@@ -5,7 +5,9 @@ import (
|
||||
"context"
|
||||
|
||||
assetAutoRenewalApp "github.com/break/junhong_cmp_fiber/internal/application/assetautorenewal"
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
iotCardSvc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
@@ -35,7 +37,7 @@ func (c *CardCommander) ResumeAssetForAutoRenewal(ctx context.Context, assetType
|
||||
if c == nil || c.service == nil {
|
||||
return assetAutoRenewalApp.ResumeOutcome{}, errCommanderUnavailable()
|
||||
}
|
||||
outcome, err := c.service.ResumeAssetForAutoRenewal(ctx, assetType, assetID)
|
||||
outcome, err := c.service.ResumeAssetForAutoRenewal(gateway.WithRateLimitExempt(ctx, constants.RateLimitExemptAutoRenewal), assetType, assetID)
|
||||
return assetAutoRenewalApp.ResumeOutcome{
|
||||
Applied: outcome.Applied, IntegrationID: outcome.IntegrationID,
|
||||
Result: outcome.Result, SafeReason: outcome.SafeReason,
|
||||
@@ -47,7 +49,7 @@ func (c *CardCommander) QueryAutoRenewalResumeState(ctx context.Context, assetTy
|
||||
if c == nil || c.service == nil {
|
||||
return false, false, "", errCommanderUnavailable()
|
||||
}
|
||||
return c.service.QueryAutoRenewalResumeState(ctx, assetType, assetID)
|
||||
return c.service.QueryAutoRenewalResumeState(gateway.WithRateLimitExempt(ctx, constants.RateLimitExemptAutoRenewal), assetType, assetID)
|
||||
}
|
||||
|
||||
// errCommanderUnavailable 返回复机执行端口未配置的稳定错误。
|
||||
|
||||
@@ -259,6 +259,14 @@ func NewRegistry() *Registry {
|
||||
pollingConfigUpdated := pollingAction(constants.AuditActionPollingConfigUpdated, "更新轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
|
||||
pollingConfigDeleted := pollingAction(constants.AuditActionPollingConfigDeleted, "删除轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
|
||||
pollingConfigStatusUpdated := pollingAction(constants.AuditActionPollingConfigStatusUpdated, "更新轮询配置状态", constants.AuditResourcePollingConfig, constants.AuditRiskHigh)
|
||||
rateLimitGroupCreated := pollingAction(constants.AuditActionRateLimitGroupCreated, "创建网关限流组", constants.AuditResourceRateLimitGroup, constants.AuditRiskHigh)
|
||||
rateLimitGroupUpdated := pollingAction(constants.AuditActionRateLimitGroupUpdated, "更新网关限流组", constants.AuditResourceRateLimitGroup, constants.AuditRiskHigh)
|
||||
rateLimitGroupStatusUpdated := pollingAction(constants.AuditActionRateLimitGroupStatusUpdated, "更新网关限流组状态", constants.AuditResourceRateLimitGroup, constants.AuditRiskHigh)
|
||||
rateLimitGroupDeleted := pollingAction(constants.AuditActionRateLimitGroupDeleted, "删除网关限流组", constants.AuditResourceRateLimitGroup, constants.AuditRiskHigh)
|
||||
gatewayErrorRuleCreated := pollingAction(constants.AuditActionGatewayErrorRuleCreated, "创建上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
gatewayErrorRuleUpdated := pollingAction(constants.AuditActionGatewayErrorRuleUpdated, "更新上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
gatewayErrorRuleStatusUpdated := pollingAction(constants.AuditActionGatewayErrorRuleStatusUpdated, "更新上游错误分类规则状态", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
gatewayErrorRuleDeleted := pollingAction(constants.AuditActionGatewayErrorRuleDeleted, "删除上游错误分类规则", constants.AuditResourceGatewayErrorRule, constants.AuditRiskNormal)
|
||||
pollingConcurrencyUpdated := pollingAction(constants.AuditActionPollingConcurrencyUpdated, "更新轮询并发配置", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
||||
pollingConcurrencyReset := pollingAction(constants.AuditActionPollingConcurrencyReset, "重置轮询并发计数", constants.AuditResourcePollingConcurrencyConfig, constants.AuditRiskNormal)
|
||||
pollingAlertRuleCreated := pollingAction(constants.AuditActionPollingAlertRuleCreated, "创建轮询告警规则", constants.AuditResourcePollingAlertRule, constants.AuditRiskNormal)
|
||||
@@ -650,6 +658,14 @@ func NewRegistry() *Registry {
|
||||
constants.AuditActionPollingConfigUpdated: pollingConfigUpdated,
|
||||
constants.AuditActionPollingConfigDeleted: pollingConfigDeleted,
|
||||
constants.AuditActionPollingConfigStatusUpdated: pollingConfigStatusUpdated,
|
||||
constants.AuditActionRateLimitGroupCreated: rateLimitGroupCreated,
|
||||
constants.AuditActionRateLimitGroupUpdated: rateLimitGroupUpdated,
|
||||
constants.AuditActionRateLimitGroupStatusUpdated: rateLimitGroupStatusUpdated,
|
||||
constants.AuditActionRateLimitGroupDeleted: rateLimitGroupDeleted,
|
||||
constants.AuditActionGatewayErrorRuleCreated: gatewayErrorRuleCreated,
|
||||
constants.AuditActionGatewayErrorRuleUpdated: gatewayErrorRuleUpdated,
|
||||
constants.AuditActionGatewayErrorRuleStatusUpdated: gatewayErrorRuleStatusUpdated,
|
||||
constants.AuditActionGatewayErrorRuleDeleted: gatewayErrorRuleDeleted,
|
||||
constants.AuditActionPollingConcurrencyUpdated: pollingConcurrencyUpdated,
|
||||
constants.AuditActionPollingConcurrencyReset: pollingConcurrencyReset,
|
||||
constants.AuditActionPollingAlertRuleCreated: pollingAlertRuleCreated,
|
||||
@@ -930,6 +946,14 @@ func NewRegistry() *Registry {
|
||||
Type: constants.AuditResourcePollingConcurrencyConfig, Name: "轮询并发配置",
|
||||
IdentityFields: []string{"id", "task_type", "max_concurrency"},
|
||||
},
|
||||
constants.AuditResourceRateLimitGroup: {
|
||||
Type: constants.AuditResourceRateLimitGroup, Name: "网关限流组",
|
||||
IdentityFields: []string{"id", "carrier_id", "group_code", "group_name", "max_qps", "reserved_qps", "status"},
|
||||
},
|
||||
constants.AuditResourceGatewayErrorRule: {
|
||||
Type: constants.AuditResourceGatewayErrorRule, Name: "上游错误分类规则",
|
||||
IdentityFields: []string{"id", "keyword", "match_type", "category", "priority", "status"},
|
||||
},
|
||||
constants.AuditResourcePollingAlertRule: {
|
||||
Type: constants.AuditResourcePollingAlertRule, Name: "轮询告警规则",
|
||||
IdentityFields: []string{"id", "rule_name", "task_type", "metric_type", "operator", "threshold", "alert_level", "status"},
|
||||
|
||||
@@ -9,7 +9,9 @@ import (
|
||||
|
||||
carrierthresholdapp "github.com/break/junhong_cmp_fiber/internal/application/carrierthreshold"
|
||||
carrierthresholddomain "github.com/break/junhong_cmp_fiber/internal/domain/carrierthreshold"
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
iotcardsvc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||
)
|
||||
|
||||
@@ -34,7 +36,7 @@ func (c *CardCommander) StopCardForThreshold(ctx context.Context, cardID uint) (
|
||||
if c == nil || c.service == nil {
|
||||
return carrierthresholddomain.CommandOutcome{}, errors.New(errors.CodeServiceUnavailable, "通道流量阈值停机执行能力未配置")
|
||||
}
|
||||
return c.service.StopCardForChannelThreshold(ctx, cardID)
|
||||
return c.service.StopCardForChannelThreshold(gateway.WithRateLimitExempt(ctx, constants.RateLimitExemptChannelThreshold), cardID)
|
||||
}
|
||||
|
||||
// ResumeCardForThreshold 执行通道阈值新周期复机。
|
||||
@@ -42,7 +44,7 @@ func (c *CardCommander) ResumeCardForThreshold(ctx context.Context, cardID uint)
|
||||
if c == nil || c.service == nil {
|
||||
return carrierthresholddomain.CommandOutcome{}, errors.New(errors.CodeServiceUnavailable, "通道流量阈值复机执行能力未配置")
|
||||
}
|
||||
return c.service.ResumeCardForChannelThreshold(ctx, cardID)
|
||||
return c.service.ResumeCardForChannelThreshold(gateway.WithRateLimitExempt(ctx, constants.RateLimitExemptChannelThreshold), cardID)
|
||||
}
|
||||
|
||||
// ResumeReady 判断解锁后的卡是否满足自动复机条件。
|
||||
@@ -58,7 +60,7 @@ func (c *CardCommander) CardNetworkStatus(ctx context.Context, cardID uint) (int
|
||||
if c == nil || c.service == nil {
|
||||
return 0, false, "", errors.New(errors.CodeServiceUnavailable, "通道流量阈值状态查询能力未配置")
|
||||
}
|
||||
return c.service.QueryChannelThresholdCardStatus(ctx, cardID)
|
||||
return c.service.QueryChannelThresholdCardStatus(gateway.WithRateLimitExempt(ctx, constants.RateLimitExemptChannelThreshold), cardID)
|
||||
}
|
||||
|
||||
// ConfirmCardState 按已确认的运营商结果补写卡状态。
|
||||
|
||||
139
internal/infrastructure/gatewaylimit/classifier.go
Normal file
139
internal/infrastructure/gatewaylimit/classifier.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// ClassificationLoader 读取全部启用的错误分类规则(按优先级排序)。
|
||||
type ClassificationLoader interface {
|
||||
ListEnabled(ctx context.Context) ([]model.GatewayErrorClassification, error)
|
||||
}
|
||||
|
||||
// Classification 是一次上游错误的分类结果。
|
||||
type Classification struct {
|
||||
Category string // 见 constants.GatewayClass*
|
||||
RuleID uint // 命中的规则 ID,未命中为 0
|
||||
Keyword string // 命中的关键字
|
||||
MaybeAccepted bool // 上游可能已受理
|
||||
}
|
||||
|
||||
type compiledRule struct {
|
||||
row model.GatewayErrorClassification
|
||||
ops map[string]bool
|
||||
regex *regexp.Regexp
|
||||
}
|
||||
|
||||
// Classifier 按后台维护的关键字规则对上游错误分类;规则在进程内缓存,超过 TTL 重新加载。
|
||||
type Classifier struct {
|
||||
loader ClassificationLoader
|
||||
logger *zap.Logger
|
||||
mu sync.Mutex
|
||||
rules []compiledRule
|
||||
loaded time.Time
|
||||
}
|
||||
|
||||
// NewClassifier 创建错误分类器;loader 为 nil 时一律归为业务错误。
|
||||
func NewClassifier(loader ClassificationLoader, logger *zap.Logger) *Classifier {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &Classifier{loader: loader, logger: logger}
|
||||
}
|
||||
|
||||
// Invalidate 使规则缓存立即失效,后台保存规则后调用。
|
||||
func (c *Classifier) Invalidate() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.loaded = time.Time{}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Classify 对某操作的上游返回信息分类;未命中任何启用规则时归为业务错误。
|
||||
func (c *Classifier) Classify(ctx context.Context, op, message string) Classification {
|
||||
if c == nil {
|
||||
return Classification{Category: constants.GatewayClassBusiness}
|
||||
}
|
||||
return MatchRules(c.current(ctx), op, message)
|
||||
}
|
||||
|
||||
// ClassifyWith 用给定规则列表分类,供测试接口在不写数据的前提下验证规则。
|
||||
func ClassifyWith(rows []model.GatewayErrorClassification, op, message string) Classification {
|
||||
return MatchRules(compileRules(rows, zap.NewNop()), op, message)
|
||||
}
|
||||
|
||||
// MatchRules 按优先级顺序返回第一个命中的规则分类。
|
||||
func MatchRules(rules []compiledRule, op, message string) Classification {
|
||||
for _, rule := range rules {
|
||||
if len(rule.ops) > 0 && !rule.ops[op] {
|
||||
continue
|
||||
}
|
||||
if !ruleMatches(rule, message) {
|
||||
continue
|
||||
}
|
||||
return Classification{Category: rule.row.Category, RuleID: rule.row.ID, Keyword: rule.row.Keyword, MaybeAccepted: rule.row.MaybeAccepted}
|
||||
}
|
||||
return Classification{Category: constants.GatewayClassBusiness}
|
||||
}
|
||||
|
||||
func ruleMatches(rule compiledRule, message string) bool {
|
||||
switch rule.row.MatchType {
|
||||
case constants.GatewayMatchPrefix:
|
||||
return strings.HasPrefix(message, rule.row.Keyword)
|
||||
case constants.GatewayMatchRegex:
|
||||
return rule.regex != nil && rule.regex.MatchString(message)
|
||||
default:
|
||||
return strings.Contains(message, rule.row.Keyword)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Classifier) current(ctx context.Context) []compiledRule {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.loader == nil || time.Since(c.loaded) < constants.GatewayRateLimitConfigTTL {
|
||||
return c.rules
|
||||
}
|
||||
rows, err := c.loader.ListEnabled(ctx)
|
||||
c.loaded = time.Now()
|
||||
if err != nil {
|
||||
c.logger.Warn("加载上游错误分类规则失败,沿用上一次规则", zap.Error(err))
|
||||
return c.rules
|
||||
}
|
||||
c.rules = compileRules(rows, c.logger)
|
||||
return c.rules
|
||||
}
|
||||
|
||||
func compileRules(rows []model.GatewayErrorClassification, logger *zap.Logger) []compiledRule {
|
||||
rules := make([]compiledRule, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
rule := compiledRule{row: row, ops: map[string]bool{}}
|
||||
ops, err := DecodeOperations(row.Operations)
|
||||
if err != nil {
|
||||
logger.Warn("错误分类规则适用操作非法,已跳过", zap.Uint("rule_id", row.ID))
|
||||
continue
|
||||
}
|
||||
for _, op := range ops {
|
||||
rule.ops[op] = true
|
||||
}
|
||||
if row.MatchType == constants.GatewayMatchRegex {
|
||||
re, err := regexp.Compile(row.Keyword)
|
||||
if err != nil {
|
||||
logger.Warn("错误分类规则正则非法,已跳过", zap.Uint("rule_id", row.ID))
|
||||
continue
|
||||
}
|
||||
rule.regex = re
|
||||
}
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
return rules
|
||||
}
|
||||
130
internal/infrastructure/gatewaylimit/groups.go
Normal file
130
internal/infrastructure/gatewaylimit/groups.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// defaultGroupCode 是未被任何启用限流组覆盖时使用的默认组标识(每个账户每个操作独立窗口)。
|
||||
const defaultGroupCode = "default"
|
||||
|
||||
// GroupRule 是一个限流组对某个操作生效的规则快照。
|
||||
type GroupRule struct {
|
||||
GroupCode string
|
||||
MaxQPS int
|
||||
ReservedQPS int
|
||||
HighPriority bool // 该操作在组内是否为高优先级
|
||||
}
|
||||
|
||||
// groupIndex 是启用限流组按"账户 → 操作"建立的索引。
|
||||
type groupIndex struct {
|
||||
byCarrier map[uint]map[string]GroupRule
|
||||
global map[string]GroupRule
|
||||
}
|
||||
|
||||
// GroupLoader 读取全部启用的限流组。
|
||||
type GroupLoader interface {
|
||||
ListEnabled(ctx context.Context) ([]model.CarrierRateLimitGroup, error)
|
||||
}
|
||||
|
||||
// GroupProvider 在进程内缓存启用限流组,超过 TTL 重新加载,后台修改数秒内生效。
|
||||
type GroupProvider struct {
|
||||
loader GroupLoader
|
||||
logger *zap.Logger
|
||||
mu sync.Mutex
|
||||
index groupIndex
|
||||
loaded time.Time
|
||||
}
|
||||
|
||||
// NewGroupProvider 创建限流组提供者;loader 为 nil 时所有操作使用默认 QPS。
|
||||
func NewGroupProvider(loader GroupLoader, logger *zap.Logger) *GroupProvider {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &GroupProvider{loader: loader, logger: logger, index: groupIndex{byCarrier: map[uint]map[string]GroupRule{}, global: map[string]GroupRule{}}}
|
||||
}
|
||||
|
||||
// Invalidate 使缓存立即失效,后台保存限流组后调用。
|
||||
func (p *GroupProvider) Invalidate() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.loaded = time.Time{}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// Resolve 返回某账户某操作的账户组规则(未覆盖时 ok=false)与全局组规则(未配置时 ok=false)。
|
||||
func (p *GroupProvider) Resolve(ctx context.Context, carrierID uint, op string) (account GroupRule, accountOK bool, global GroupRule, globalOK bool) {
|
||||
index := p.current(ctx)
|
||||
if rules, ok := index.byCarrier[carrierID]; ok {
|
||||
account, accountOK = rules[op]
|
||||
}
|
||||
global, globalOK = index.global[op]
|
||||
return
|
||||
}
|
||||
|
||||
func (p *GroupProvider) current(ctx context.Context) groupIndex {
|
||||
if p == nil {
|
||||
return groupIndex{}
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.loader == nil || time.Since(p.loaded) < constants.GatewayRateLimitConfigTTL {
|
||||
return p.index
|
||||
}
|
||||
rows, err := p.loader.ListEnabled(ctx)
|
||||
p.loaded = time.Now()
|
||||
if err != nil {
|
||||
p.logger.Warn("加载网关限流组失败,沿用上一次配置", zap.Error(err))
|
||||
return p.index
|
||||
}
|
||||
p.index = buildGroupIndex(rows, p.logger)
|
||||
return p.index
|
||||
}
|
||||
|
||||
func buildGroupIndex(rows []model.CarrierRateLimitGroup, logger *zap.Logger) groupIndex {
|
||||
index := groupIndex{byCarrier: map[uint]map[string]GroupRule{}, global: map[string]GroupRule{}}
|
||||
for _, row := range rows {
|
||||
ops, err := DecodeOperations(row.Operations)
|
||||
if err != nil {
|
||||
logger.Warn("限流组操作列表非法,已跳过该组", zap.Uint("group_id", row.ID), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
highOps, _ := DecodeOperations(row.HighPriorityOperations)
|
||||
high := map[string]bool{}
|
||||
for _, op := range highOps {
|
||||
high[op] = true
|
||||
}
|
||||
target := index.global
|
||||
if row.CarrierID != nil {
|
||||
if index.byCarrier[*row.CarrierID] == nil {
|
||||
index.byCarrier[*row.CarrierID] = map[string]GroupRule{}
|
||||
}
|
||||
target = index.byCarrier[*row.CarrierID]
|
||||
}
|
||||
for _, op := range ops {
|
||||
target[op] = GroupRule{GroupCode: row.GroupCode, MaxQPS: row.MaxQPS, ReservedQPS: row.ReservedQPS, HighPriority: high[op]}
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
// DecodeOperations 解析 JSON 字符串数组形式的操作列表。
|
||||
func DecodeOperations(raw []byte) ([]string, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var ops []string
|
||||
if err := sonic.Unmarshal(raw, &ops); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ops, nil
|
||||
}
|
||||
169
internal/infrastructure/gatewaylimit/guard.go
Normal file
169
internal/infrastructure/gatewaylimit/guard.go
Normal file
@@ -0,0 +1,169 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// 分钟计数 Hash 的字段名。
|
||||
const (
|
||||
statFieldWouldLimit = "would_limit"
|
||||
statFieldDurationSum = "duration_ms_sum"
|
||||
statFieldAttempts = "attempts"
|
||||
statPrefixExempt = "exempt:"
|
||||
statPrefixUpstream = "upstream:"
|
||||
statPrefixDuration = "duration_le:"
|
||||
)
|
||||
|
||||
// durationBuckets 是耗时分桶上界(毫秒),用于近似计算 p95。
|
||||
var durationBuckets = []int64{100, 300, 500, 1000, 3000, 10000, 60000}
|
||||
|
||||
// Guard 实现 gateway.CallGuard:每次卡网关 HTTP 尝试前按运营商账户限流,结束后写分钟计数与卡最近错误。
|
||||
type Guard struct {
|
||||
limiter *Limiter
|
||||
resolver *CarrierResolver
|
||||
classifier *Classifier
|
||||
redis *redis.Client
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewGuard 创建卡网关调用守卫。
|
||||
func NewGuard(limiter *Limiter, resolver *CarrierResolver, classifier *Classifier, redisClient *redis.Client, logger *zap.Logger) *Guard {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &Guard{limiter: limiter, resolver: resolver, classifier: classifier, redis: redisClient, logger: logger}
|
||||
}
|
||||
|
||||
var _ gateway.CallGuard = (*Guard)(nil)
|
||||
|
||||
// BeforeAttempt 取限流名额;豁免调用直接放行。拦截模式下拿不到名额返回限流错误,本次方法调用随即结束。
|
||||
func (g *Guard) BeforeAttempt(ctx context.Context, call gateway.CallInfo) error {
|
||||
if g == nil || g.limiter == nil || call.ExemptSource != "" {
|
||||
return nil
|
||||
}
|
||||
carrierID := g.carrierOf(ctx, call)
|
||||
decision, err := g.limiter.Check(ctx, carrierID, call.Operation)
|
||||
if err != nil {
|
||||
// 限流器故障时失败开放,避免 Redis 抖动阻断全部网关调用。
|
||||
g.logger.Warn("网关限流判断失败,本次放行", zap.Uint("carrier_id", carrierID), zap.String("operation", call.Operation), zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
if decision.Exceed {
|
||||
g.incr(ctx, carrierID, call.Operation, statFieldWouldLimit)
|
||||
}
|
||||
if !decision.Allowed {
|
||||
g.incr(ctx, carrierID, call.Operation, constants.GatewayErrorRateLimited)
|
||||
return gateway.NewRateLimitedError()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AfterAttempt 写分钟计数(按结果类别与耗时分桶);查询失败时写卡该操作最近一次错误。
|
||||
func (g *Guard) AfterAttempt(ctx context.Context, call gateway.CallInfo, callErr error, duration time.Duration) {
|
||||
if g == nil || g.redis == nil {
|
||||
return
|
||||
}
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
carrierID := g.carrierOf(ctx, call)
|
||||
field := constants.GatewayResultSuccess
|
||||
var classification Classification
|
||||
gwErr, isGatewayErr := gateway.AsGatewayError(callErr)
|
||||
if callErr != nil {
|
||||
field = constants.GatewayErrorNetwork
|
||||
if isGatewayErr {
|
||||
field = gwErr.Category
|
||||
if gwErr.Category == constants.GatewayErrorUpstream {
|
||||
classification = g.classifier.Classify(ctx, call.Operation, gwErr.ProviderMessage)
|
||||
field = statPrefixUpstream + classification.Category
|
||||
}
|
||||
}
|
||||
}
|
||||
key := constants.RedisGatewayCallStatKey(carrierID, call.Operation, statMinute(time.Now()))
|
||||
durationMS := duration.Milliseconds()
|
||||
pipe := g.redis.Pipeline()
|
||||
pipe.HIncrBy(ctx, key, field, 1)
|
||||
pipe.HIncrBy(ctx, key, statFieldAttempts, 1)
|
||||
pipe.HIncrBy(ctx, key, statFieldDurationSum, durationMS)
|
||||
pipe.HIncrBy(ctx, key, statPrefixDuration+durationBucket(durationMS), 1)
|
||||
if call.ExemptSource != "" {
|
||||
pipe.HIncrBy(ctx, key, statPrefixExempt+call.ExemptSource, 1)
|
||||
}
|
||||
pipe.Expire(ctx, key, constants.GatewayCallStatTTL)
|
||||
if callErr != nil && isQueryOperation(call.Operation) && call.CardNo != "" {
|
||||
errKey := constants.RedisGatewayCardLastErrorKey(call.CardNo, call.Operation)
|
||||
values := map[string]any{"category": field, "at": time.Now().Format(time.RFC3339)}
|
||||
if isGatewayErr {
|
||||
values["provider_code"] = strconv.Itoa(gwErr.ProviderCode)
|
||||
values["provider_message"] = gwErr.ProviderMessage
|
||||
}
|
||||
pipe.HSet(ctx, errKey, values)
|
||||
pipe.Expire(ctx, errKey, constants.GatewayCardLastErrorTTL)
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
g.logger.Debug("写网关调用统计失败", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// Classify 暴露分类器供停复机记录等调用方对上游错误分类。
|
||||
func (g *Guard) Classify(ctx context.Context, op, message string) Classification {
|
||||
if g == nil {
|
||||
return Classification{Category: constants.GatewayClassBusiness}
|
||||
}
|
||||
return g.classifier.Classify(ctx, op, message)
|
||||
}
|
||||
|
||||
func (g *Guard) carrierOf(ctx context.Context, call gateway.CallInfo) uint {
|
||||
if call.CarrierID != 0 || g.resolver == nil {
|
||||
return call.CarrierID
|
||||
}
|
||||
carrierID, err := g.resolver.Resolve(ctx, call.CardNo)
|
||||
if err != nil {
|
||||
g.logger.Debug("解析卡所属运营商账户失败,按未知账户计量", zap.String("iccid", call.CardNo), zap.Error(err))
|
||||
}
|
||||
return carrierID
|
||||
}
|
||||
|
||||
func (g *Guard) incr(ctx context.Context, carrierID uint, op, field string) {
|
||||
if g.redis == nil {
|
||||
return
|
||||
}
|
||||
key := constants.RedisGatewayCallStatKey(carrierID, op, statMinute(time.Now()))
|
||||
pipe := g.redis.Pipeline()
|
||||
pipe.HIncrBy(ctx, key, field, 1)
|
||||
pipe.Expire(ctx, key, constants.GatewayCallStatTTL)
|
||||
_, _ = pipe.Exec(context.WithoutCancel(ctx))
|
||||
}
|
||||
|
||||
func isQueryOperation(op string) bool {
|
||||
return op == constants.GatewayOpQueryFlow || op == constants.GatewayOpQueryStatus || op == constants.GatewayOpQueryRealname
|
||||
}
|
||||
|
||||
// statMinute 返回上海时区的分钟标识(yyyyMMddHHmm)。
|
||||
func statMinute(t time.Time) string {
|
||||
return t.In(shanghai()).Format("200601021504")
|
||||
}
|
||||
|
||||
func durationBucket(ms int64) string {
|
||||
for _, upper := range durationBuckets {
|
||||
if ms <= upper {
|
||||
return strconv.FormatInt(upper, 10)
|
||||
}
|
||||
}
|
||||
return "inf"
|
||||
}
|
||||
|
||||
func shanghai() *time.Location {
|
||||
location, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
return time.FixedZone("CST", 8*3600)
|
||||
}
|
||||
return location
|
||||
}
|
||||
191
internal/infrastructure/gatewaylimit/limiter.go
Normal file
191
internal/infrastructure/gatewaylimit/limiter.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// slidingWindowScript 在一次原子执行内判断账户组窗口与可选的全局组窗口。
|
||||
//
|
||||
// KEYS[1] 账户组窗口 ZSET;KEYS[2] 账户组高优先级等待计数;KEYS[3] 全局组窗口 ZSET(无全局组时传空串占位)。
|
||||
// ARGV:now_ms、window_ms、max_qps、reserved_qps、is_high(0/1)、global_max(0 表示无全局组)、member、dry_run(0/1)。
|
||||
// 返回 {allowed(1/0), wait_ms}:dry_run=1(只统计)时总是记录本次请求,allowed=0 表示"本应被限流"。
|
||||
const slidingWindowLua = `
|
||||
local now = tonumber(ARGV[1])
|
||||
local window = tonumber(ARGV[2])
|
||||
local maxQPS = tonumber(ARGV[3])
|
||||
local reserved = tonumber(ARGV[4])
|
||||
local isHigh = tonumber(ARGV[5])
|
||||
local globalMax = tonumber(ARGV[6])
|
||||
local member = ARGV[7]
|
||||
local dryRun = tonumber(ARGV[8])
|
||||
local useGlobal = globalMax > 0 and KEYS[3] ~= ''
|
||||
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now - window)
|
||||
if useGlobal then
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now - window)
|
||||
end
|
||||
|
||||
local limit = maxQPS
|
||||
if isHigh == 0 then
|
||||
limit = maxQPS - reserved
|
||||
if reserved > 0 then
|
||||
local waiting = tonumber(redis.call('GET', KEYS[2]) or '0')
|
||||
if waiting > 0 then
|
||||
limit = 0
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local allowed = 1
|
||||
local wait = 0
|
||||
local count = redis.call('ZCARD', KEYS[1])
|
||||
if count >= limit then
|
||||
allowed = 0
|
||||
local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
|
||||
if oldest[2] then
|
||||
wait = tonumber(oldest[2]) + window - now
|
||||
else
|
||||
wait = window
|
||||
end
|
||||
end
|
||||
if allowed == 1 and useGlobal then
|
||||
local gcount = redis.call('ZCARD', KEYS[3])
|
||||
if gcount >= globalMax then
|
||||
allowed = 0
|
||||
local goldest = redis.call('ZRANGE', KEYS[3], 0, 0, 'WITHSCORES')
|
||||
if goldest[2] then
|
||||
wait = tonumber(goldest[2]) + window - now
|
||||
else
|
||||
wait = window
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if allowed == 1 or dryRun == 1 then
|
||||
redis.call('ZADD', KEYS[1], now, member)
|
||||
redis.call('PEXPIRE', KEYS[1], window * 2)
|
||||
if useGlobal then
|
||||
redis.call('ZADD', KEYS[3], now, member)
|
||||
redis.call('PEXPIRE', KEYS[3], window * 2)
|
||||
end
|
||||
end
|
||||
if wait < 1 and allowed == 0 then
|
||||
wait = 1
|
||||
end
|
||||
return {allowed, wait}
|
||||
`
|
||||
|
||||
var slidingWindowScript = redis.NewScript(slidingWindowLua)
|
||||
|
||||
// Decision 是一次限流判断的结果。
|
||||
type Decision struct {
|
||||
Allowed bool // 是否放行(只统计模式下恒为 true)
|
||||
Exceed bool // 是否超过上限(只统计模式下表示"本应被限流")
|
||||
Wait time.Duration // 拦截时建议的等待时长
|
||||
}
|
||||
|
||||
// Limiter 是基于 Redis 滑动窗口的运营商账户限流器,跨全部进程共享。
|
||||
type Limiter struct {
|
||||
redis *redis.Client
|
||||
groups *GroupProvider
|
||||
settings *SettingsProvider
|
||||
}
|
||||
|
||||
// NewLimiter 创建限流器。
|
||||
func NewLimiter(redisClient *redis.Client, groups *GroupProvider, settings *SettingsProvider) *Limiter {
|
||||
return &Limiter{redis: redisClient, groups: groups, settings: settings}
|
||||
}
|
||||
|
||||
// Check 立即判断一次请求能否放行,不等待。拦截模式下超过上限返回 Allowed=false;
|
||||
// 只统计模式下总是放行,Exceed 表示本应被限流。Redis 故障时放行(失败开放),避免限流故障阻断业务。
|
||||
func (l *Limiter) Check(ctx context.Context, carrierID uint, op string) (Decision, error) {
|
||||
settings := l.settings.Current(ctx)
|
||||
account, accountOK, global, globalOK := l.groups.Resolve(ctx, carrierID, op)
|
||||
if !accountOK {
|
||||
account = GroupRule{GroupCode: defaultGroupCode + ":" + op, MaxQPS: settings.DefaultQPS}
|
||||
}
|
||||
globalKey, globalMax := "", 0
|
||||
if globalOK {
|
||||
globalKey, globalMax = constants.RedisGatewayGlobalQPSKey(global.GroupCode), global.MaxQPS
|
||||
}
|
||||
isHigh, dryRun := 0, 1
|
||||
if account.HighPriority {
|
||||
isHigh = 1
|
||||
}
|
||||
if settings.Enforce() {
|
||||
dryRun = 0
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
result, err := slidingWindowScript.Run(ctx, l.redis,
|
||||
[]string{constants.RedisGatewayQPSKey(carrierID, account.GroupCode), constants.RedisGatewayQPSHighWaitingKey(carrierID, account.GroupCode), globalKey},
|
||||
now, constants.GatewayRateLimitWindow.Milliseconds(), account.MaxQPS, account.ReservedQPS, isHigh, globalMax, windowMember(now), dryRun,
|
||||
).Int64Slice()
|
||||
if err != nil || len(result) != 2 {
|
||||
return Decision{Allowed: true}, err
|
||||
}
|
||||
exceed := result[0] == 0
|
||||
decision := Decision{Allowed: !exceed || dryRun == 1, Exceed: exceed}
|
||||
if exceed && dryRun == 0 {
|
||||
decision.Wait = time.Duration(result[1]) * time.Millisecond
|
||||
}
|
||||
return decision, nil
|
||||
}
|
||||
|
||||
// Wait 在拦截模式下按建议时长等待名额,总等待不超过 maxWait;超时返回 false。
|
||||
// 高优先级操作等待期间登记等待计数,使同组普通操作拿不到新名额。本 Change 中 Asynq 任务不使用该方法。
|
||||
func (l *Limiter) Wait(ctx context.Context, carrierID uint, op string, maxWait time.Duration) (bool, error) {
|
||||
deadline := time.Now().Add(maxWait)
|
||||
registered := false
|
||||
var waitingKey string
|
||||
defer func() {
|
||||
if registered {
|
||||
l.redis.Decr(context.WithoutCancel(ctx), waitingKey)
|
||||
}
|
||||
}()
|
||||
for {
|
||||
decision, err := l.Check(ctx, carrierID, op)
|
||||
if err != nil || decision.Allowed {
|
||||
return true, err
|
||||
}
|
||||
if !registered {
|
||||
account, accountOK, _, _ := l.groups.Resolve(ctx, carrierID, op)
|
||||
if accountOK && account.HighPriority && account.ReservedQPS > 0 {
|
||||
waitingKey = constants.RedisGatewayQPSHighWaitingKey(carrierID, account.GroupCode)
|
||||
pipe := l.redis.TxPipeline()
|
||||
pipe.Incr(ctx, waitingKey)
|
||||
pipe.Expire(ctx, waitingKey, 10*time.Second)
|
||||
if _, err := pipe.Exec(ctx); err == nil {
|
||||
registered = true
|
||||
}
|
||||
}
|
||||
}
|
||||
sleep := decision.Wait
|
||||
if remaining := time.Until(deadline); remaining <= 0 {
|
||||
return false, nil
|
||||
} else if sleep > remaining {
|
||||
sleep = remaining
|
||||
}
|
||||
timer := time.NewTimer(sleep)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return false, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// windowMember 生成窗口内唯一成员,避免同毫秒请求互相覆盖。
|
||||
func windowMember(nowMS int64) string {
|
||||
buf := make([]byte, 6)
|
||||
_, _ = rand.Read(buf)
|
||||
return strconv.FormatInt(nowMS, 10) + "-" + hex.EncodeToString(buf)
|
||||
}
|
||||
61
internal/infrastructure/gatewaylimit/partition.go
Normal file
61
internal/infrastructure/gatewaylimit/partition.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
)
|
||||
|
||||
// partitionDropLookbackMonths 是每次清理向前检查的过期月数,覆盖维护任务停摆一段时间后的补删。
|
||||
const partitionDropLookbackMonths = 12
|
||||
|
||||
// PartitionMaintainer 维护停复机记录按月分区:提前创建未来分区,按保留期整区删除过期分区。
|
||||
type PartitionMaintainer struct {
|
||||
store *postgres.CardStopResumeRecordStore
|
||||
settings *SettingsProvider
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewPartitionMaintainer 创建分区维护任务。
|
||||
func NewPartitionMaintainer(store *postgres.CardStopResumeRecordStore, settings *SettingsProvider, logger *zap.Logger) *PartitionMaintainer {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &PartitionMaintainer{store: store, settings: settings, logger: logger}
|
||||
}
|
||||
|
||||
// Handle 是 Asynq 定时任务入口;任一分区操作失败时写错误告警并返回错误,由 Asynq 按既有规则重试。
|
||||
func (m *PartitionMaintainer) Handle(ctx context.Context, _ *asynq.Task) error {
|
||||
if m == nil || m.store == nil {
|
||||
return stderrors.New("停复机记录分区维护未配置")
|
||||
}
|
||||
settings := m.settings.Current(ctx)
|
||||
monthStart := currentMonthStart(time.Now())
|
||||
var failed error
|
||||
for i := 0; i <= settings.RecordPremakeMonth; i++ {
|
||||
month := monthStart.AddDate(0, i, 0)
|
||||
if err := m.store.EnsureMonthPartition(ctx, month); err != nil {
|
||||
m.logger.Error("创建停复机记录分区失败,记录将落入默认分区", zap.String("month", month.Format("200601")), zap.Error(err))
|
||||
failed = err
|
||||
}
|
||||
}
|
||||
for k := 0; k < partitionDropLookbackMonths; k++ {
|
||||
month := monthStart.AddDate(0, -(settings.RecordRetention + k), 0)
|
||||
if err := m.store.DropMonthPartition(ctx, month); err != nil {
|
||||
m.logger.Error("删除过期停复机记录分区失败", zap.String("month", month.Format("200601")), zap.Error(err))
|
||||
failed = err
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// currentMonthStart 返回上海时区当月 1 日 0 点。
|
||||
func currentMonthStart(now time.Time) time.Time {
|
||||
local := now.In(shanghai())
|
||||
return time.Date(local.Year(), local.Month(), 1, 0, 0, 0, 0, shanghai())
|
||||
}
|
||||
53
internal/infrastructure/gatewaylimit/resolver.go
Normal file
53
internal/infrastructure/gatewaylimit/resolver.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// CarrierResolver 按 ICCID 解析卡所属的本系统运营商账户(tb_iot_card.carrier_id),结果缓存在 Redis。
|
||||
type CarrierResolver struct {
|
||||
db *gorm.DB
|
||||
redis *redis.Client
|
||||
}
|
||||
|
||||
// NewCarrierResolver 创建运营商账户解析器。
|
||||
func NewCarrierResolver(db *gorm.DB, redisClient *redis.Client) *CarrierResolver {
|
||||
return &CarrierResolver{db: db, redis: redisClient}
|
||||
}
|
||||
|
||||
// Resolve 返回 ICCID 对应卡的运营商账户 ID;卡不存在时返回 0、nil,由调用方按未知账户处理。
|
||||
func (r *CarrierResolver) Resolve(ctx context.Context, iccid string) (uint, error) {
|
||||
if r == nil || iccid == "" {
|
||||
return 0, nil
|
||||
}
|
||||
cacheKey := constants.RedisGatewayCardCarrierKey(iccid)
|
||||
if r.redis != nil {
|
||||
if raw, err := r.redis.Get(ctx, cacheKey).Result(); err == nil {
|
||||
if value, parseErr := strconv.ParseUint(raw, 10, 64); parseErr == nil {
|
||||
return uint(value), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.db == nil {
|
||||
return 0, nil
|
||||
}
|
||||
var carrierIDs []uint
|
||||
err := r.db.WithContext(ctx).Table("tb_iot_card").
|
||||
Where("iccid = ? AND deleted_at IS NULL", iccid).Limit(1).Pluck("carrier_id", &carrierIDs).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(carrierIDs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if r.redis != nil {
|
||||
_ = r.redis.Set(ctx, cacheKey, strconv.FormatUint(uint64(carrierIDs[0]), 10), constants.GatewayCardCarrierCacheTTL).Err()
|
||||
}
|
||||
return carrierIDs[0], nil
|
||||
}
|
||||
79
internal/infrastructure/gatewaylimit/runtime.go
Normal file
79
internal/infrastructure/gatewaylimit/runtime.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// Runtime 汇总一个进程内的网关限流组件,API 与 Worker 各自创建一份,限流窗口经 Redis 跨进程共享。
|
||||
type Runtime struct {
|
||||
Settings *SettingsProvider
|
||||
Groups *GroupProvider
|
||||
Classifier *Classifier
|
||||
Limiter *Limiter
|
||||
Guard *Guard
|
||||
Recorder *StopResumeRecorder
|
||||
|
||||
redisClient *redis.Client
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewRuntime 创建网关限流运行组件;配置读取使用独立注册表,与后台系统配置共享同一份数据库与缓存数据。
|
||||
func NewRuntime(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *Runtime {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
registry := systemconfig.NewRegistry()
|
||||
RegisterConfigDefinitions(registry, logger)
|
||||
var cache systemconfig.Cache
|
||||
if redisClient != nil {
|
||||
cache = systemconfig.NewRedisCache(redisClient)
|
||||
}
|
||||
reader := systemconfig.NewReader(db, registry, cache, systemconfig.NewLogAlertSink(logger))
|
||||
settings := NewSettingsProvider(reader, logger)
|
||||
groups := NewGroupProvider(postgres.NewCarrierRateLimitGroupStore(db), logger)
|
||||
classifier := NewClassifier(postgres.NewGatewayErrorClassificationStore(db), logger)
|
||||
limiter := NewLimiter(redisClient, groups, settings)
|
||||
guard := NewGuard(limiter, NewCarrierResolver(db, redisClient), classifier, redisClient, logger)
|
||||
recorder := NewStopResumeRecorder(postgres.NewCardStopResumeRecordStore(db), classifier, logger)
|
||||
return &Runtime{Settings: settings, Groups: groups, Classifier: classifier, Limiter: limiter, Guard: guard, Recorder: recorder, redisClient: redisClient, logger: logger}
|
||||
}
|
||||
|
||||
// TrackDeferral 记录 Asynq 任务因限流被推迟;同一任务首次被限流起累计推迟超过阈值时写一次错误告警,任务继续重试。
|
||||
// 任务以"类型 + 载荷摘要"标识,记录 1 天后自动过期。
|
||||
func (r *Runtime) TrackDeferral(task *asynq.Task) {
|
||||
if r == nil || r.redisClient == nil || task == nil {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
sum := sha1.Sum(task.Payload())
|
||||
taskKey := task.Type() + ":" + hex.EncodeToString(sum[:])
|
||||
key := constants.RedisRateLimitDeferStartKey(taskKey)
|
||||
now := time.Now()
|
||||
if ok, err := r.redisClient.SetNX(ctx, key, now.Unix(), 24*time.Hour).Result(); err != nil || ok {
|
||||
return
|
||||
}
|
||||
startedUnix, err := r.redisClient.Get(ctx, key).Int64()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
deferred := now.Sub(time.Unix(startedUnix, 0))
|
||||
if deferred < r.Settings.Current(ctx).DeferAlert {
|
||||
return
|
||||
}
|
||||
if alerted, _ := r.redisClient.SetNX(ctx, key+":alerted", 1, 24*time.Hour).Result(); alerted {
|
||||
r.logger.Error("异步任务因运营商账户网关限流累计推迟超过告警阈值,任务继续重试",
|
||||
zap.String("task_type", task.Type()), zap.String("task_key", taskKey), zap.Duration("deferred", deferred))
|
||||
}
|
||||
}
|
||||
136
internal/infrastructure/gatewaylimit/settings.go
Normal file
136
internal/infrastructure/gatewaylimit/settings.go
Normal file
@@ -0,0 +1,136 @@
|
||||
// Package gatewaylimit 实现按运营商账户(本系统 tb_carrier)的卡网关调用限流、错误分类与调用统计。
|
||||
//
|
||||
// 账户口径:限流单位是本系统的运营商账户(卡上的 carrier_id),与网关客户端自身的 appId 无关。
|
||||
// 调用方已知卡所属账户时用 gateway.WithCarrier 传入;否则守卫按 ICCID 查本系统卡表解析。
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// RegisterConfigDefinitions 向受控系统配置注册表注册网关限流相关配置;已注册的 Key 跳过。
|
||||
func RegisterConfigDefinitions(registry *systemconfig.Registry, logger *zap.Logger) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
minOne, minZero := int64(1), int64(0)
|
||||
maxQPS, maxDelay, maxAlert, maxMonths := int64(10000), int64(3600), int64(86400*7), int64(120)
|
||||
definitions := []systemconfig.Definition{
|
||||
{Key: constants.SystemConfigPollingRateLimitMode, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeString, DefaultValue: constants.RateLimitModeObserve, Description: "网关限流模式:observe 只统计不拦截,enforce 拦截", EnumValues: []string{constants.RateLimitModeObserve, constants.RateLimitModeEnforce}},
|
||||
{Key: constants.SystemConfigPollingGatewayDefaultQPS, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, DefaultValue: "10", Description: "未被启用限流组覆盖的卡网关操作的默认每秒请求上限", Min: &minOne, Max: &maxQPS},
|
||||
{Key: constants.SystemConfigPollingRateLimitedRetryDelay, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, DefaultValue: "5", Description: "Asynq 任务遇限流后的固定重试延迟(秒)", Min: &minOne, Max: &maxDelay},
|
||||
{Key: constants.SystemConfigPollingRateLimitDeferAlert, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, DefaultValue: "600", Description: "同一任务因限流累计推迟超过该时长(秒)时写告警", Min: &minOne, Max: &maxAlert},
|
||||
{Key: constants.SystemConfigPollingStopResumeRecordRetention, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, DefaultValue: "6", Description: "停复机记录保留月数,到期整区删除", Min: &minOne, Max: &maxMonths},
|
||||
{Key: constants.SystemConfigPollingStopResumeRecordPremake, Module: constants.SystemConfigModulePolling, ValueType: constants.SystemConfigTypeInt, DefaultValue: "3", Description: "停复机记录提前创建未来分区的月数", Min: &minZero, Max: &maxMonths},
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if _, exists := registry.Get(definition.Key); exists {
|
||||
continue
|
||||
}
|
||||
if err := registry.Register(definition); err != nil && logger != nil {
|
||||
logger.Error("注册网关限流系统配置失败,将使用代码默认值", zap.String("config_key", definition.Key), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ConfigReader 读取已注册的受控系统配置。
|
||||
type ConfigReader interface {
|
||||
Get(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
// Settings 是网关限流运行参数快照。
|
||||
type Settings struct {
|
||||
Mode string
|
||||
DefaultQPS int
|
||||
RetryDelay time.Duration
|
||||
DeferAlert time.Duration
|
||||
RecordRetention int
|
||||
RecordPremakeMonth int
|
||||
}
|
||||
|
||||
// Enforce 判断当前是否为拦截模式。
|
||||
func (s Settings) Enforce() bool {
|
||||
return s.Mode == constants.RateLimitModeEnforce
|
||||
}
|
||||
|
||||
// defaultSettings 是配置读取失败时的安全默认值:只统计、不拦截。
|
||||
func defaultSettings() Settings {
|
||||
return Settings{Mode: constants.RateLimitModeObserve, DefaultQPS: 10, RetryDelay: 5 * time.Second, DeferAlert: 10 * time.Minute, RecordRetention: 6, RecordPremakeMonth: 3}
|
||||
}
|
||||
|
||||
// SettingsProvider 在进程内缓存限流参数,超过 TTL 重新读取,后台修改数秒内生效。
|
||||
type SettingsProvider struct {
|
||||
reader ConfigReader
|
||||
logger *zap.Logger
|
||||
mu sync.Mutex
|
||||
cached Settings
|
||||
loaded time.Time
|
||||
}
|
||||
|
||||
// NewSettingsProvider 创建限流参数提供者;reader 为 nil 时始终返回安全默认值。
|
||||
func NewSettingsProvider(reader ConfigReader, logger *zap.Logger) *SettingsProvider {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &SettingsProvider{reader: reader, logger: logger, cached: defaultSettings()}
|
||||
}
|
||||
|
||||
// Current 返回当前限流参数;读取失败时保留上一次成功值。
|
||||
func (p *SettingsProvider) Current(ctx context.Context) Settings {
|
||||
if p == nil {
|
||||
return defaultSettings()
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.reader == nil || time.Since(p.loaded) < constants.GatewayRateLimitConfigTTL {
|
||||
return p.cached
|
||||
}
|
||||
next, err := p.load(ctx)
|
||||
p.loaded = time.Now()
|
||||
if err != nil {
|
||||
p.logger.Warn("读取网关限流配置失败,沿用上一次配置", zap.Error(err))
|
||||
return p.cached
|
||||
}
|
||||
p.cached = next
|
||||
return next
|
||||
}
|
||||
|
||||
func (p *SettingsProvider) load(ctx context.Context) (Settings, error) {
|
||||
settings := defaultSettings()
|
||||
mode, err := p.reader.Get(ctx, constants.SystemConfigPollingRateLimitMode)
|
||||
if err != nil {
|
||||
return settings, err
|
||||
}
|
||||
settings.Mode = mode
|
||||
ints := []struct {
|
||||
key string
|
||||
set func(int)
|
||||
}{
|
||||
{constants.SystemConfigPollingGatewayDefaultQPS, func(v int) { settings.DefaultQPS = v }},
|
||||
{constants.SystemConfigPollingRateLimitedRetryDelay, func(v int) { settings.RetryDelay = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingRateLimitDeferAlert, func(v int) { settings.DeferAlert = time.Duration(v) * time.Second }},
|
||||
{constants.SystemConfigPollingStopResumeRecordRetention, func(v int) { settings.RecordRetention = v }},
|
||||
{constants.SystemConfigPollingStopResumeRecordPremake, func(v int) { settings.RecordPremakeMonth = v }},
|
||||
}
|
||||
for _, item := range ints {
|
||||
raw, err := p.reader.Get(ctx, item.key)
|
||||
if err != nil {
|
||||
return settings, err
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return settings, stderrors.New("网关限流整数配置非法")
|
||||
}
|
||||
item.set(value)
|
||||
}
|
||||
return settings, nil
|
||||
}
|
||||
86
internal/infrastructure/gatewaylimit/stop_resume_record.go
Normal file
86
internal/infrastructure/gatewaylimit/stop_resume_record.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package gatewaylimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// StopResumeRecordInput 描述一次实际调用上游停机或复机的结果。
|
||||
type StopResumeRecordInput struct {
|
||||
CardID uint
|
||||
ICCID string
|
||||
CarrierID uint
|
||||
Action string // constants.StopResumeActionStop / StopResumeActionStart
|
||||
Source string // 来源:豁免来源或审计来源
|
||||
Scene string
|
||||
Reason string // 停机原因
|
||||
CommandID string // 同一次停复机命令的多次尝试共用
|
||||
Attempt int
|
||||
CallErr error
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// StopResumeRecorder 把每次停复机上游调用写入停复机记录表;写入失败只记日志与计数,不影响业务回写。
|
||||
type StopResumeRecorder struct {
|
||||
store *postgres.CardStopResumeRecordStore
|
||||
classifier *Classifier
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewStopResumeRecorder 创建停复机记录写入器。
|
||||
func NewStopResumeRecorder(store *postgres.CardStopResumeRecordStore, classifier *Classifier, logger *zap.Logger) *StopResumeRecorder {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
return &StopResumeRecorder{store: store, classifier: classifier, logger: logger}
|
||||
}
|
||||
|
||||
// Record 写入一条停复机记录;限流拒绝(请求未发出)不写。
|
||||
func (r *StopResumeRecorder) Record(ctx context.Context, input StopResumeRecordInput) {
|
||||
if r == nil || r.store == nil || gateway.IsRateLimited(input.CallErr) {
|
||||
return
|
||||
}
|
||||
row := &model.CardStopResumeRecord{
|
||||
CardID: input.CardID, ICCID: input.ICCID, CarrierID: input.CarrierID, Action: input.Action,
|
||||
Source: input.Source, Scene: input.Scene, Reason: input.Reason, CommandID: input.CommandID,
|
||||
Attempt: input.Attempt, Result: constants.StopResumeRecordSuccess,
|
||||
DurationMS: int(input.Duration.Milliseconds()), CreatedAt: time.Now(),
|
||||
}
|
||||
if input.CallErr != nil {
|
||||
row.Result = constants.StopResumeRecordFailed
|
||||
if gwErr, ok := gateway.AsGatewayError(input.CallErr); ok {
|
||||
row.ErrorCategory = gwErr.Category
|
||||
switch gwErr.Category {
|
||||
case constants.GatewayErrorTimeout, constants.GatewayErrorNetwork:
|
||||
// 超时或连接失败时上游可能已受理,按结果未知记录。
|
||||
row.Result = constants.StopResumeRecordUnknown
|
||||
case constants.GatewayErrorUpstream:
|
||||
op := constants.GatewayOpStop
|
||||
if input.Action == constants.StopResumeActionStart {
|
||||
op = constants.GatewayOpStart
|
||||
}
|
||||
classification := r.classifier.Classify(ctx, op, gwErr.ProviderMessage)
|
||||
row.ErrorCategory = classification.Category
|
||||
if classification.Category == constants.GatewayClassAlreadyTarget {
|
||||
row.Result = constants.StopResumeRecordAlreadyTarget
|
||||
}
|
||||
row.ProviderCode = strconv.Itoa(gwErr.ProviderCode)
|
||||
row.ProviderMessage = gwErr.ProviderMessage
|
||||
}
|
||||
}
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.Create(writeCtx, row); err != nil {
|
||||
r.logger.Warn("写停复机记录失败,不影响停复机结果回写",
|
||||
zap.Uint("card_id", input.CardID), zap.String("action", input.Action), zap.String("metric", "stop_resume_record.write_failed"), zap.Error(err))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user