feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投 - 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引 - 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存 - 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -204,6 +204,51 @@ type PollingConfig struct {
|
||||
VerboseLog bool `mapstructure:"verbose_log"`
|
||||
// Carrier 按运营商账户调度(add-carrier-polling-scheduler)的部署参数;调度模式本身放受控系统配置 polling.scheduler.mode。
|
||||
Carrier CarrierSchedulerConfig `mapstructure:"carrier"`
|
||||
// Ingest 轮询观测入库消费者参数(add-polling-async-ingest);入库模式本身放受控系统配置 polling.ingest.mode。
|
||||
Ingest PollingIngestConfig `mapstructure:"ingest"`
|
||||
// Collection 流量采集记录批量写入参数。
|
||||
Collection PollingCollectionConfig `mapstructure:"collection"`
|
||||
}
|
||||
|
||||
// PollingIngestConfig 轮询观测入库消费者的启动参数。
|
||||
type PollingIngestConfig struct {
|
||||
// Concurrency 每进程入库消费者并发(card.ingest 的 max_in_flight),默认 30。
|
||||
// 环境变量:JUNHONG_POLLING_INGEST_CONCURRENCY
|
||||
Concurrency int `mapstructure:"concurrency"`
|
||||
// Timeout 单条入库消息的处理超时,从消费者开始处理时计时,默认 30s。
|
||||
// 环境变量:JUNHONG_POLLING_INGEST_TIMEOUT
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
// PollingCollectionConfig 流量采集记录批量写入参数。
|
||||
type PollingCollectionConfig struct {
|
||||
// BatchSize 每批写入的采集记录条数上限,默认 200;不足时每 1 秒写一次。
|
||||
// 环境变量:JUNHONG_POLLING_COLLECTION_BATCH_SIZE
|
||||
BatchSize int `mapstructure:"batch_size"`
|
||||
}
|
||||
|
||||
// EffectiveConcurrency 返回入库消费者并发,未配置时取默认值。
|
||||
func (c PollingIngestConfig) EffectiveConcurrency() int {
|
||||
if c.Concurrency > 0 {
|
||||
return c.Concurrency
|
||||
}
|
||||
return constants.PollingIngestDefaultConcurrency
|
||||
}
|
||||
|
||||
// EffectiveTimeout 返回单条入库超时,未配置时取默认值。
|
||||
func (c PollingIngestConfig) EffectiveTimeout() time.Duration {
|
||||
if c.Timeout > 0 {
|
||||
return c.Timeout
|
||||
}
|
||||
return constants.PollingIngestDefaultTimeout
|
||||
}
|
||||
|
||||
// EffectiveBatchSize 返回采集记录每批写入条数,未配置时取默认值。
|
||||
func (c PollingCollectionConfig) EffectiveBatchSize() int {
|
||||
if c.BatchSize > 0 {
|
||||
return c.BatchSize
|
||||
}
|
||||
return constants.PollingCollectionDefaultBatchSize
|
||||
}
|
||||
|
||||
// CarrierSchedulerConfig 按运营商账户调度的 yaml 参数。
|
||||
@@ -430,6 +475,13 @@ func (c *Config) Validate() error {
|
||||
return fmt.Errorf("invalid configuration: worker.polling_total_max_concurrency: must be 1-%d (current value: %d)", constants.PollingMaxConcurrencyLimit, c.Worker.PollingTotalMaxConcurrency)
|
||||
}
|
||||
|
||||
if c.Polling.Ingest.Concurrency < 0 || c.Polling.Ingest.Concurrency > constants.PollingMaxConcurrencyLimit {
|
||||
return fmt.Errorf("invalid configuration: polling.ingest.concurrency: must be 0-%d (current value: %d)", constants.PollingMaxConcurrencyLimit, c.Polling.Ingest.Concurrency)
|
||||
}
|
||||
if c.Polling.Collection.BatchSize < 0 || c.Polling.Collection.BatchSize > constants.PollingCollectionMaxInFlight {
|
||||
return fmt.Errorf("invalid configuration: polling.collection.batch_size: must be 0-%d (current value: %d)", constants.PollingCollectionMaxInFlight, c.Polling.Collection.BatchSize)
|
||||
}
|
||||
|
||||
if !validWorkerRoles[c.Worker.Role] {
|
||||
return fmt.Errorf(
|
||||
"invalid configuration: worker.role: invalid worker role (current value: %s, expected: %s, %s, %s)",
|
||||
|
||||
@@ -146,6 +146,12 @@ polling:
|
||||
default_inflight_limit: 20 # 未配置账户的执行上限(跨实例)
|
||||
default_card_max_in_flight: 20 # 未配置账户的每进程 max_in_flight
|
||||
subscriber_processes: 1 # 订阅卡任务的 Worker 进程数(校验执行上限用)
|
||||
# 轮询观测入库(入库模式在受控系统配置 polling.ingest.mode 切换:sync / async)
|
||||
ingest:
|
||||
concurrency: 30 # 每进程入库消费者并发(环境变量:JUNHONG_POLLING_INGEST_CONCURRENCY)
|
||||
timeout: 30s # 单条入库超时(环境变量:JUNHONG_POLLING_INGEST_TIMEOUT)
|
||||
collection:
|
||||
batch_size: 200 # 采集记录每批写入条数(环境变量:JUNHONG_POLLING_COLLECTION_BATCH_SIZE)
|
||||
|
||||
# 轮询自动触发配置
|
||||
polling_auto_trigger:
|
||||
|
||||
@@ -135,6 +135,9 @@ func bindEnvVariables(v *viper.Viper) {
|
||||
"polling.carrier.nsqd_address",
|
||||
"polling.carrier.nsqlookupd_addresses",
|
||||
"polling.carrier.subscriber_processes",
|
||||
"polling.ingest.concurrency",
|
||||
"polling.ingest.timeout",
|
||||
"polling.collection.batch_size",
|
||||
"polling_auto_trigger.enable_auto_trigger",
|
||||
"polling_auto_trigger.auto_trigger_system_user_id",
|
||||
"worker.role",
|
||||
|
||||
89
pkg/constants/polling_ingest.go
Normal file
89
pkg/constants/polling_ingest.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package constants
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 轮询观测入库异步化(add-polling-async-ingest):调网关与入库分离,入库单独发消息、单独消费。
|
||||
|
||||
// 入库 NSQ topic。
|
||||
const (
|
||||
// NSQCardIngestTopic 轮询观测入库消息(卡状态、流量、实名)。
|
||||
NSQCardIngestTopic = "card.ingest"
|
||||
// NSQCardCollectionTopic 流量采集记录消息(成功表与失败表),由批量消费者攒批写入。
|
||||
NSQCardCollectionTopic = "card.collection"
|
||||
)
|
||||
|
||||
// 入库模式(受控系统配置 polling.ingest.mode)。
|
||||
const (
|
||||
PollingIngestModeSync = "sync" // 轮询步骤在卡任务内同步入库(回退模式)
|
||||
PollingIngestModeAsync = "async" // 轮询步骤只调网关并发布入库消息
|
||||
)
|
||||
|
||||
// 入库相关受控系统配置键(module = polling)。
|
||||
const (
|
||||
SystemConfigPollingIngestMode = "polling.ingest.mode"
|
||||
SystemConfigPollingIngestAlertDepth = "polling.ingest.alert_depth"
|
||||
SystemConfigPollingIngestPauseDepth = "polling.ingest.pause_depth"
|
||||
)
|
||||
|
||||
// 入库运行默认值。
|
||||
const (
|
||||
PollingIngestDefaultConcurrency = 30
|
||||
PollingIngestDefaultTimeout = 30 * time.Second
|
||||
PollingCollectionDefaultBatchSize = 200
|
||||
PollingCollectionFlushInterval = time.Second
|
||||
PollingCollectionMaxInFlight = 500
|
||||
PollingIngestDefaultAlertDepth = 200000
|
||||
PollingIngestDefaultPauseDepth = 500000
|
||||
PollingIngestDoneTTL = 48 * time.Hour
|
||||
PollingIngestErrorLogFromAttempt = 10
|
||||
PollingIngestLagAlert = 10 * time.Minute
|
||||
PollingIngestMonitorInterval = time.Minute
|
||||
PollingIngestConnReserve = 20
|
||||
PollingIngestCollectionWriters = 1
|
||||
PollingFlowEvaluatedFallbackTTL = 30 * time.Minute
|
||||
PollingPriorityIndexRebuildEvery = 30 * time.Second
|
||||
PollingPriorityIndexRecentTTL = 2 * time.Minute
|
||||
PollingPriorityIndexReadyTTL = 2 * time.Minute
|
||||
PollingScopeCacheTTL = 30 * time.Second
|
||||
PollingCarrierResetDayCacheTTL = time.Minute
|
||||
PollingTierQueryTimeout = 2 * time.Second
|
||||
PollingIngestSyncCollectionTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// PollingIngestRetryDelays 是入库可重试错误的延迟重投序列;超过序列长度后取最后一项,不设次数上限。
|
||||
var PollingIngestRetryDelays = []time.Duration{
|
||||
5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute,
|
||||
}
|
||||
|
||||
// RedisPollingIngestDoneKey 入库已处理标记(String,TTL 48h),按采集 ID 去重。
|
||||
func RedisPollingIngestDoneKey(observationID string) string {
|
||||
return "polling:ingest:done:" + observationID
|
||||
}
|
||||
|
||||
// RedisPollingFlowEvaluatedKey 流量入库产生增量或跨月后的评估标记(String,TTL=套餐检查间隔),套餐步骤据此跳过。
|
||||
func RedisPollingFlowEvaluatedKey(cardID uint) string {
|
||||
return fmt.Sprintf("polling:flow_evaluated:%d", cardID)
|
||||
}
|
||||
|
||||
// RedisPollingIngestPausedKey 入库积压超过反压阈值时置位(String),分发器暂停领取新卡任务。
|
||||
const RedisPollingIngestPausedKey = "polling:ingest:paused"
|
||||
|
||||
// 有活动优先轮询项的卡索引:member = "card_id:task_type"。
|
||||
const (
|
||||
// RedisPollingPriorityActiveKey 活动优先项索引(SET)。
|
||||
RedisPollingPriorityActiveKey = "polling:priority:active"
|
||||
// RedisPollingPriorityRecentKey 最近入队的优先项(SET,短 TTL),重建索引时并入,避免重建期间漏掉新项。
|
||||
RedisPollingPriorityRecentKey = "polling:priority:recent"
|
||||
// RedisPollingPriorityReadyKey 索引已由重建任务建立(String,TTL);不存在时轮询步骤回退查库。
|
||||
RedisPollingPriorityReadyKey = "polling:priority:ready"
|
||||
// RedisPollingPriorityRebuildKey 重建过程使用的临时 key。
|
||||
RedisPollingPriorityRebuildKey = "polling:priority:active:rebuild"
|
||||
)
|
||||
|
||||
// PollingPriorityIndexMember 返回活动优先项索引成员。
|
||||
func PollingPriorityIndexMember(cardID uint, taskType string) string {
|
||||
return fmt.Sprintf("%d:%s", cardID, taskType)
|
||||
}
|
||||
Reference in New Issue
Block a user