Files
junhong_cmp_fiber/internal/infrastructure/nsqbus/nsqbus.go
邹进 1214181e78
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 16m2s
feat: 轮询观测入库异步化,调网关与入库分离,采集记录经消息攒批写入
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息,入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投
- 采集记录经 card.collection 攒批写入,失败整批重投,新增 (observation_id, observed_at) 唯一索引
- 网络、实名观测增加乱序保护;轮询读查询走缓存,套餐状态查询各自计时且失败不缓存
- 入库积压告警与反压;系统配置 polling.ingest.mode 默认 sync 可回退;测试环境 nsqd 消息全部落盘

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 16:33:29 +09:00

282 lines
8.5 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package nsqbus 封装卡轮询使用的 NSQ 生产者与按 topic 动态订阅的消费者管理器。
// NSQ 只承担执行分发与消费者并发控制,业务状态与重试全部在 Redis(add-carrier-polling-scheduler)。
package nsqbus
import (
"context"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/nsqio/go-nsq"
"go.uber.org/zap"
)
// Producer 是 nsqd 生产者。
type Producer struct {
producer *nsq.Producer
}
// NewProducer 连接 nsqd TCP 地址创建生产者。
func NewProducer(address string, logger *zap.Logger) (*Producer, error) {
cfg := nsq.NewConfig()
producer, err := nsq.NewProducer(address, cfg)
if err != nil {
return nil, err
}
producer.SetLogger(nsqLogger{logger: logger}, nsq.LogLevelWarning)
return &Producer{producer: producer}, nil
}
// Publish 同步发布一条消息;topic 不存在时由 nsqd 自动创建。
func (p *Producer) Publish(topic string, body []byte) error {
return p.producer.Publish(topic, body)
}
// Stop 停止生产者。
func (p *Producer) Stop() {
if p != nil && p.producer != nil {
p.producer.Stop()
}
}
// Handler 处理一条消息;返回后由管理器 FIN。消息处理期间可调用 touch 延长 NSQ 消息超时。
type Handler func(ctx context.Context, body []byte, touch func())
// RetryHandler 处理一条消息并决定是否重投:返回 0 表示 FIN,返回正数表示按该延迟重投(REQ,不计 NSQ 退避)。
// attempts 是 NSQ 记录的投递次数(含本次,从 1 起)。
type RetryHandler func(ctx context.Context, body []byte, attempts uint16, touch func()) time.Duration
// ConsumerConfig 是消费者参数。
type ConsumerConfig struct {
NSQDAddress string
NSQLookupdAddresses []string
MsgTimeout time.Duration
MaxAttempts uint16
// LookupdPollInterval 消费者向 nsqlookupd 查询 nsqd 的间隔;0 表示 15 秒。
LookupdPollInterval time.Duration
// UnlimitedAttempts 为 true 时不限投递次数(忽略 MaxAttempts),用于不允许丢弃的入库消息。
UnlimitedAttempts bool
}
// ConsumerManager 按 topic 动态订阅与取消订阅;每个 topic 一个消费者,max_in_flight 控制本进程对该 topic 的并发。
type ConsumerManager struct {
cfg ConsumerConfig
channel string
handler Handler
retryHandler RetryHandler
logger *zap.Logger
mu sync.Mutex
consumers map[string]*subscription
ctx context.Context
cancel context.CancelFunc
inflight sync.WaitGroup
}
type subscription struct {
consumer *nsq.Consumer
maxInFlight int
}
// NewConsumerManager 创建消费者管理器。
func NewConsumerManager(cfg ConsumerConfig, channel string, handler Handler, logger *zap.Logger) *ConsumerManager {
if logger == nil {
logger = zap.NewNop()
}
ctx, cancel := context.WithCancel(context.Background())
return &ConsumerManager{cfg: cfg, channel: channel, handler: handler, logger: logger,
consumers: map[string]*subscription{}, ctx: ctx, cancel: cancel}
}
// NewRetryConsumerManager 创建由处理结果决定 FIN 或延迟重投的消费者管理器。
func NewRetryConsumerManager(cfg ConsumerConfig, channel string, handler RetryHandler, logger *zap.Logger) *ConsumerManager {
manager := NewConsumerManager(cfg, channel, nil, logger)
manager.retryHandler = handler
return manager
}
// Sync 使订阅集合与 desired(topic → max_in_flight)一致:新增订阅、取消多余订阅、调整 max_in_flight。
func (m *ConsumerManager) Sync(desired map[string]int) {
m.mu.Lock()
defer m.mu.Unlock()
if m.ctx.Err() != nil {
return
}
for topic, sub := range m.consumers {
if _, keep := desired[topic]; !keep {
sub.consumer.Stop()
delete(m.consumers, topic)
m.logger.Info("取消订阅 NSQ topic", zap.String("topic", topic))
}
}
for topic, maxInFlight := range desired {
if maxInFlight <= 0 {
maxInFlight = 1
}
if sub, ok := m.consumers[topic]; ok {
if sub.maxInFlight != maxInFlight {
sub.consumer.ChangeMaxInFlight(maxInFlight)
sub.maxInFlight = maxInFlight
}
continue
}
consumer, err := m.subscribe(topic, maxInFlight)
if err != nil {
m.logger.Warn("订阅 NSQ topic 失败,下一轮重试", zap.String("topic", topic), zap.Error(err))
continue
}
m.consumers[topic] = &subscription{consumer: consumer, maxInFlight: maxInFlight}
m.logger.Info("订阅 NSQ topic", zap.String("topic", topic), zap.Int("max_in_flight", maxInFlight))
}
}
func (m *ConsumerManager) subscribe(topic string, maxInFlight int) (*nsq.Consumer, error) {
cfg := nsq.NewConfig()
cfg.MaxInFlight = maxInFlight
if m.cfg.MsgTimeout > 0 {
cfg.MsgTimeout = m.cfg.MsgTimeout
}
if m.cfg.MaxAttempts > 0 {
cfg.MaxAttempts = m.cfg.MaxAttempts
}
if m.cfg.UnlimitedAttempts {
cfg.MaxAttempts = 0
}
cfg.LookupdPollInterval = 15 * time.Second
if m.cfg.LookupdPollInterval > 0 {
cfg.LookupdPollInterval = m.cfg.LookupdPollInterval
}
// 先在 nsqd 上创建 topic:否则首次发布前 nsqlookupd 查不到该 topic,消费者要等下一轮轮询才能发现 nsqd。
m.ensureTopic(topic)
consumer, err := nsq.NewConsumer(topic, m.channel, cfg)
if err != nil {
return nil, err
}
consumer.SetLogger(nsqLogger{logger: m.logger}, nsq.LogLevelWarning)
handlerCtx := m.ctx // subscribe 在持锁时调用,捕获当前处理上下文
consumer.AddConcurrentHandlers(nsq.HandlerFunc(func(message *nsq.Message) error {
message.DisableAutoResponse()
m.inflight.Add(1)
defer m.inflight.Done()
if m.retryHandler != nil {
if delay := m.retryHandler(handlerCtx, message.Body, message.Attempts, message.Touch); delay > 0 {
message.RequeueWithoutBackoff(delay)
return nil
}
message.Finish()
return nil
}
defer message.Finish()
m.handler(handlerCtx, message.Body, message.Touch)
return nil
}), maxInFlight)
if len(m.cfg.NSQLookupdAddresses) > 0 {
err = consumer.ConnectToNSQLookupds(m.cfg.NSQLookupdAddresses)
} else {
err = consumer.ConnectToNSQD(m.cfg.NSQDAddress)
}
if err != nil {
consumer.Stop()
return nil, err
}
return consumer, nil
}
// Topics 返回当前已订阅的 topic。
func (m *ConsumerManager) Topics() []string {
m.mu.Lock()
defer m.mu.Unlock()
topics := make([]string, 0, len(m.consumers))
for topic := range m.consumers {
topics = append(topics, topic)
}
return topics
}
// StopAll 停止全部订阅;在宽限期内等待执行中的消息完成并 FIN,超过宽限期后取消处理上下文。
func (m *ConsumerManager) StopAll(grace time.Duration) {
m.mu.Lock()
for topic, sub := range m.consumers {
sub.consumer.Stop()
delete(m.consumers, topic)
}
m.mu.Unlock()
done := make(chan struct{})
go func() {
m.inflight.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(grace):
m.logger.Warn("优雅退出宽限期内仍有卡任务未完成,取消执行(由租约到期后重新领取)")
}
m.cancel()
}
// Reset 在 StopAll 后重新允许订阅(切回 carrier 模式时使用)。
func (m *ConsumerManager) Reset() {
m.mu.Lock()
defer m.mu.Unlock()
if m.ctx.Err() != nil {
m.ctx, m.cancel = context.WithCancel(context.Background())
}
}
// SplitAddresses 解析逗号分隔的地址列表。
func SplitAddresses(raw string) []string {
var result []string
for _, part := range strings.Split(raw, ",") {
if part = strings.TrimSpace(part); part != "" {
result = append(result, part)
}
}
return result
}
type nsqLogger struct {
logger *zap.Logger
}
// Output 实现 go-nsq 的日志接口。
func (l nsqLogger) Output(_ int, s string) error {
if l.logger != nil {
l.logger.Warn("NSQ", zap.String("message", s))
}
return nil
}
// ensureTopic 通过 nsqd HTTP 接口创建 topic(已存在时幂等);失败只记日志,消费者仍会在后续轮询中发现。
func (m *ConsumerManager) ensureTopic(topic string) {
httpAddress := nsqdHTTPAddress(m.cfg.NSQDAddress)
if httpAddress == "" {
return
}
client := http.Client{Timeout: 3 * time.Second}
resp, err := client.Post("http://"+httpAddress+"/topic/create?topic="+url.QueryEscape(topic), "text/plain", nil)
if err != nil {
m.logger.Debug("创建 NSQ topic 失败", zap.String("topic", topic), zap.Error(err))
return
}
_ = resp.Body.Close()
}
// nsqdHTTPAddress 由 nsqd TCP 地址推导 HTTP 地址(默认端口 4150 → 4151)。
func nsqdHTTPAddress(tcpAddress string) string {
host, port, err := net.SplitHostPort(strings.TrimSpace(tcpAddress))
if err != nil || host == "" {
return ""
}
if port == "4150" {
port = "4151"
} else if value, convErr := strconv.Atoi(port); convErr == nil {
port = strconv.Itoa(value + 1)
}
return net.JoinHostPort(host, port)
}