1214181e78
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 16m2s
- 轮询步骤异步模式下只调网关并发布 card.ingest 入库消息，入库消费者固定并发 30、按采集 ID 去重、可重试错误不限次数退避重投 - 采集记录经 card.collection 攒批写入，失败整批重投，新增 (observation_id, observed_at) 唯一索引 - 网络、实名观测增加乱序保护；轮询读查询走缓存，套餐状态查询各自计时且失败不缓存 - 入库积压告警与反压；系统配置 polling.ingest.mode 默认 sync 可回退；测试环境 nsqd 消息全部落盘 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
2.7 KiB
Go
74 lines
2.7 KiB
Go
package task
|
||
|
||
import (
|
||
"context"
|
||
|
||
"github.com/redis/go-redis/v9"
|
||
"go.uber.org/zap"
|
||
|
||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||
)
|
||
|
||
// priorityIndexAbsent 判断活动优先项索引能否确认该卡该任务类型没有活动优先项。
|
||
// 索引未建立、读取失败或成员存在时返回 false，调用方照常查库。
|
||
func (b *PollingBase) priorityIndexAbsent(ctx context.Context, cardID uint, taskType string) bool {
|
||
if b == nil || b.redis == nil {
|
||
return false
|
||
}
|
||
pipe := b.redis.Pipeline()
|
||
ready := pipe.Exists(ctx, constants.RedisPollingPriorityReadyKey)
|
||
member := pipe.SIsMember(ctx, constants.RedisPollingPriorityActiveKey, constants.PollingPriorityIndexMember(cardID, taskType))
|
||
if _, err := pipe.Exec(ctx); err != nil {
|
||
return false
|
||
}
|
||
return ready.Val() > 0 && !member.Val()
|
||
}
|
||
|
||
// PriorityIndexRebuilder 定期按库内活动优先项重建索引（并入最近入队的项），使已终态的项退出索引。
|
||
type PriorityIndexRebuilder struct {
|
||
store *postgres.PollingPriorityItemStore
|
||
redis *redis.Client
|
||
logger *zap.Logger
|
||
}
|
||
|
||
// NewPriorityIndexRebuilder 创建活动优先项索引重建器。
|
||
func NewPriorityIndexRebuilder(store *postgres.PollingPriorityItemStore, redisClient *redis.Client, logger *zap.Logger) *PriorityIndexRebuilder {
|
||
if logger == nil {
|
||
logger = zap.NewNop()
|
||
}
|
||
return &PriorityIndexRebuilder{store: store, redis: redisClient, logger: logger}
|
||
}
|
||
|
||
// Rebuild 重建一次索引；失败时清除就绪标记，轮询步骤回退为查库。
|
||
func (r *PriorityIndexRebuilder) Rebuild(ctx context.Context) {
|
||
if r == nil || r.store == nil || r.redis == nil {
|
||
return
|
||
}
|
||
keys, err := r.store.ListActiveKeys(ctx)
|
||
if err != nil {
|
||
r.logger.Warn("重建活动优先项索引失败，轮询步骤回退查库", zap.Error(err))
|
||
r.redis.Del(ctx, constants.RedisPollingPriorityReadyKey)
|
||
return
|
||
}
|
||
members := make([]any, 0, len(keys))
|
||
for _, key := range keys {
|
||
members = append(members, constants.PollingPriorityIndexMember(key.CardID, key.TaskType))
|
||
}
|
||
tmp := constants.RedisPollingPriorityRebuildKey
|
||
pipe := r.redis.TxPipeline()
|
||
pipe.Del(ctx, tmp)
|
||
if len(members) > 0 {
|
||
pipe.SAdd(ctx, tmp, members...)
|
||
}
|
||
pipe.SUnionStore(ctx, tmp, tmp, constants.RedisPollingPriorityRecentKey)
|
||
pipe.Del(ctx, constants.RedisPollingPriorityActiveKey)
|
||
pipe.SUnionStore(ctx, constants.RedisPollingPriorityActiveKey, tmp)
|
||
pipe.Del(ctx, tmp)
|
||
pipe.Set(ctx, constants.RedisPollingPriorityReadyKey, "1", constants.PollingPriorityIndexReadyTTL)
|
||
if _, err := pipe.Exec(ctx); err != nil {
|
||
r.logger.Warn("写入活动优先项索引失败，轮询步骤回退查库", zap.Error(err))
|
||
r.redis.Del(ctx, constants.RedisPollingPriorityReadyKey)
|
||
}
|
||
}
|