Files
junhong_cmp_fiber/internal/infrastructure/nsqbus/stats.go
邹进 1214181e78
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投
- 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引
- 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存
- 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 16:33:29 +09:00

61 lines
1.6 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package nsqbus
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"time"
)
type nsqdStats struct {
Topics []struct {
TopicName string `json:"topic_name"`
Depth int64 `json:"depth"`
Channels []struct {
Depth int64 `json:"depth"`
DeferredCount int64 `json:"deferred_count"`
} `json:"channels"`
} `json:"topics"`
}
// TopicBacklog 读取 nsqd 上某 topic 的积压量:topic 未分发量 + 各 channel 待消费量与延迟重投量(不含处理中)。
// topic 不存在时返回 0。
func TopicBacklog(ctx context.Context, nsqdTCPAddress, topic string) (int64, error) {
httpAddress := nsqdHTTPAddress(nsqdTCPAddress)
if httpAddress == "" {
return 0, fmt.Errorf("nsqd 地址无效:%s", nsqdTCPAddress)
}
requestCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
endpoint := "http://" + httpAddress + "/stats?format=json&topic=" + url.QueryEscape(topic)
request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, endpoint, nil)
if err != nil {
return 0, err
}
response, err := http.DefaultClient.Do(request)
if err != nil {
return 0, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("读取 nsqd 统计失败,HTTP %d", response.StatusCode)
}
var stats nsqdStats
if err := json.NewDecoder(response.Body).Decode(&stats); err != nil {
return 0, err
}
var backlog int64
for _, item := range stats.Topics {
if item.TopicName != topic {
continue
}
backlog += item.Depth
for _, channel := range item.Channels {
backlog += channel.Depth + channel.DeferredCount
}
}
return backlog, nil
}