feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s

- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投
- 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引
- 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存
- 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-06 16:33:29 +09:00
parent 7e0e37933e
commit 1214181e78
41 changed files with 2007 additions and 46 deletions

View File

@@ -204,6 +204,51 @@ type PollingConfig struct {
VerboseLog bool `mapstructure:"verbose_log"`
// Carrier 按运营商账户调度(add-carrier-polling-scheduler)的部署参数;调度模式本身放受控系统配置 polling.scheduler.mode。
Carrier CarrierSchedulerConfig `mapstructure:"carrier"`
// Ingest 轮询观测入库消费者参数(add-polling-async-ingest);入库模式本身放受控系统配置 polling.ingest.mode。
Ingest PollingIngestConfig `mapstructure:"ingest"`
// Collection 流量采集记录批量写入参数。
Collection PollingCollectionConfig `mapstructure:"collection"`
}
// PollingIngestConfig 轮询观测入库消费者的启动参数。
type PollingIngestConfig struct {
// Concurrency 每进程入库消费者并发(card.ingest 的 max_in_flight),默认 30。
// 环境变量:JUNHONG_POLLING_INGEST_CONCURRENCY
Concurrency int `mapstructure:"concurrency"`
// Timeout 单条入库消息的处理超时,从消费者开始处理时计时,默认 30s。
// 环境变量:JUNHONG_POLLING_INGEST_TIMEOUT
Timeout time.Duration `mapstructure:"timeout"`
}
// PollingCollectionConfig 流量采集记录批量写入参数。
type PollingCollectionConfig struct {
// BatchSize 每批写入的采集记录条数上限,默认 200;不足时每 1 秒写一次。
// 环境变量:JUNHONG_POLLING_COLLECTION_BATCH_SIZE
BatchSize int `mapstructure:"batch_size"`
}
// EffectiveConcurrency 返回入库消费者并发,未配置时取默认值。
func (c PollingIngestConfig) EffectiveConcurrency() int {
if c.Concurrency > 0 {
return c.Concurrency
}
return constants.PollingIngestDefaultConcurrency
}
// EffectiveTimeout 返回单条入库超时,未配置时取默认值。
func (c PollingIngestConfig) EffectiveTimeout() time.Duration {
if c.Timeout > 0 {
return c.Timeout
}
return constants.PollingIngestDefaultTimeout
}
// EffectiveBatchSize 返回采集记录每批写入条数,未配置时取默认值。
func (c PollingCollectionConfig) EffectiveBatchSize() int {
if c.BatchSize > 0 {
return c.BatchSize
}
return constants.PollingCollectionDefaultBatchSize
}
// CarrierSchedulerConfig 按运营商账户调度的 yaml 参数。
@@ -430,6 +475,13 @@ func (c *Config) Validate() error {
return fmt.Errorf("invalid configuration: worker.polling_total_max_concurrency: must be 1-%d (current value: %d)", constants.PollingMaxConcurrencyLimit, c.Worker.PollingTotalMaxConcurrency)
}
if c.Polling.Ingest.Concurrency < 0 || c.Polling.Ingest.Concurrency > constants.PollingMaxConcurrencyLimit {
return fmt.Errorf("invalid configuration: polling.ingest.concurrency: must be 0-%d (current value: %d)", constants.PollingMaxConcurrencyLimit, c.Polling.Ingest.Concurrency)
}
if c.Polling.Collection.BatchSize < 0 || c.Polling.Collection.BatchSize > constants.PollingCollectionMaxInFlight {
return fmt.Errorf("invalid configuration: polling.collection.batch_size: must be 0-%d (current value: %d)", constants.PollingCollectionMaxInFlight, c.Polling.Collection.BatchSize)
}
if !validWorkerRoles[c.Worker.Role] {
return fmt.Errorf(
"invalid configuration: worker.role: invalid worker role (current value: %s, expected: %s, %s, %s)",