Files
junhong_cmp_fiber/internal/service/iot_card/stop_resume_service.go
huang 434a8b0349
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m46s
feat: 轮询系统重构(分片队列 + 停复机统一 + Handler 拆分)
【核心变更】

1. 停复机逻辑统一(StopResumeService)
   - 新增 EvaluateAndAct 统一入口,封装三条件停复机判断
   - 停机条件:无套餐(no_package) / 流量耗尽(traffic_exhausted) / 未实名(not_realname)
   - 复机条件:stop_reason 合规 + 有套餐且未耗尽 + 已实名或行业卡
   - 修复设备套餐 Bug:hasValidPackage 按 device_id 查套餐,而非仅 iot_card_id
   - 设备维度停复机加幂等锁(Redis SetNX,TTL 30s),防止多卡并发重复调 Gateway

2. Redis 分片队列(PollingQueueManager)
   - 新建 queue_manager.go,封装所有轮询 Redis 操作
   - 16 分片 Sorted Set,Key 格式:polling:shard:{shardID}:queue:{taskType}
   - Lua 脚本原子出队(ZRANGEBYSCORE + 分批 ZREM),消除竞态窗口
   - 新增背压检测:队列深度超 50 万时 Scheduler 跳过该分片
   - RemoveFromAllQueues 覆盖 4 种任务类型(含 protect)

3. Handler 拆分(polling_handler.go 1360行 → 5个专注文件)
   - polling_base.go:共享基类(并发控制/卡缓存/重入队)
   - polling_realname_handler.go:实名采集,实名 0→1 时立即触发复机
   - polling_carddata_handler.go:流量采集,保留跨月边界检测逻辑
   - polling_package_handler.go:套餐采集,委托 EvaluateAndAct 决策
   - polling_protect_handler.go:保护期一致性检查,保护期内强制修正

4. 配置管理(PollingConfigManager)
   - 新建 config_manager.go,从 scheduler.go 提取配置职责
   - 内存缓存 + 5 分钟定时刷新,刷新失败保留原缓存
   - 修复 getCardCondition:停机卡返回 suspended,不再错配 activated 配置

5. 渐进式初始化(CardInitializer)
   - 新建 initializer.go,分批加载(每批 10 万),批次间 sleep 500ms
   - 过滤 enable_polling=false 的卡,初始化完成前 Scheduler 不出队

6. 卡生命周期服务(PollingLifecycleService)
   - 新建 lifecycle_service.go,替代已删除的 callbacks.go 和 api_callback.go
   - OnCardCreated/OnCardEnabled/OnCardStatusChanged 入队前检查 enable_polling

7. Scheduler 精简(1000+行 → 227行)
   - 保留纯调度循环:scheduleLoop + processShardSchedule + enqueueBatch
   - 保留每 10 秒触发套餐过期检测和流量重置
   - 移除所有 DB 操作、配置加载、卡初始化逻辑

8. 轮询管控 API(enable_polling)
   - 新增 PUT /api/admin/assets/:id/polling-status 接口
   - 支持对设备/卡维度开关轮询,关闭后从所有分片队列移除

9. 数据库迁移
   - 000103:tb_device 新增 enable_polling 字段(boolean, NOT NULL, DEFAULT true)
   - 000104:新增 suspended 轮询配置,为 activated 配置补全 protect_check_interval

【文件统计】
- 新增:19 个文件(handler × 5、polling 组件 × 4、迁移 × 3 等)
- 修改:20 个文件(bootstrap 注入、store 接口、monitoring 适配分片等)
- 删除:3 个文件(polling_handler.go、callbacks.go、api_callback.go)

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-04-07 12:27:04 +08:00

604 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package iot_card
import (
"context"
"time"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
"gorm.io/gorm"
stderrors "errors"
"github.com/break/junhong_cmp_fiber/internal/gateway"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
"github.com/break/junhong_cmp_fiber/pkg/errors"
)
// StopResumeServiceInterface 停复机服务接口
// 供 4 个 Task Handler 通过接口注入,避免循环依赖
type StopResumeServiceInterface interface {
// EvaluateAndAct 停复机统一入口,根据卡的当前状态自动判断并执行停机或复机
EvaluateAndAct(ctx context.Context, card *model.IotCard) error
}
// 编译时验证 StopResumeService 实现了 StopResumeServiceInterface
var _ StopResumeServiceInterface = (*StopResumeService)(nil)
// StopResumeService 停复机服务
// 处理 IoT 卡的自动停机、复机和手动停复机逻辑
type StopResumeService struct {
redis *redis.Client
iotCardStore *postgres.IotCardStore
packageUsageStore *postgres.PackageUsageStore
deviceSimBindingStore *postgres.DeviceSimBindingStore
gatewayClient *gateway.Client
logger *zap.Logger
maxRetries int
retryInterval time.Duration
}
// NewStopResumeService 创建停复机服务
func NewStopResumeService(
redis *redis.Client,
iotCardStore *postgres.IotCardStore,
packageUsageStore *postgres.PackageUsageStore,
deviceSimBindingStore *postgres.DeviceSimBindingStore,
gatewayClient *gateway.Client,
logger *zap.Logger,
) *StopResumeService {
return &StopResumeService{
redis: redis,
iotCardStore: iotCardStore,
packageUsageStore: packageUsageStore,
deviceSimBindingStore: deviceSimBindingStore,
gatewayClient: gatewayClient,
logger: logger,
maxRetries: 3,
retryInterval: 2 * time.Second,
}
}
// EvaluateAndAct 停复机统一入口
// 根据卡的当前网络状态,自动判断并执行停机或复机操作
// 设备/单卡维度通过 card.IsStandalone 推导false 则为设备维度,停/复机覆盖设备下所有卡)
func (s *StopResumeService) EvaluateAndAct(ctx context.Context, card *model.IotCard) error {
switch card.NetworkStatus {
case constants.NetworkStatusOnline:
// 卡在线,检查是否需要停机
reasons, err := s.checkStopReasons(ctx, card)
if err != nil {
return err
}
if len(reasons) == 0 {
return nil
}
// 取最高优先级原因(第一个)
primaryReason := reasons[0]
if !card.IsStandalone {
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
if lookupErr != nil {
s.logger.Error("查询设备绑定关系失败,降级为单卡停机",
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
// 降级:查不到绑定时按单卡处理
} else if bound {
s.stopDeviceCards(ctx, deviceID, primaryReason)
return nil
}
}
return s.stopCardWithRetry(ctx, card, primaryReason)
case constants.NetworkStatusOffline:
// 卡停机,检查是否可以复机
// 只处理轮询系统引起的停机原因(手动停机等不自动复机)
if !isPollingStopReason(card.StopReason) {
return nil
}
shouldResume, err := s.shouldResume(ctx, card)
if err != nil {
return err
}
if !shouldResume {
return nil
}
if !card.IsStandalone {
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
if lookupErr != nil {
s.logger.Error("查询设备绑定关系失败,降级为单卡复机",
zap.Uint("card_id", card.ID), zap.Error(lookupErr))
} else if bound {
s.resumeDeviceCards(ctx, deviceID)
return nil
}
}
return s.resumeSingleCard(ctx, card.ID)
default:
return nil
}
}
// isPollingStopReason 判断停机原因是否为轮询系统引起(可自动复机)
func isPollingStopReason(reason string) bool {
switch reason {
case constants.StopReasonTrafficExhausted,
constants.StopReasonNoPackage,
constants.StopReasonNotRealname:
return true
}
return false
}
// getCardDeviceID 获取卡所属设备ID非独立卡时有效
// 返回deviceID, isDeviceBound, error
func (s *StopResumeService) getCardDeviceID(ctx context.Context, card *model.IotCard) (uint, bool, error) {
if card.IsStandalone {
return 0, false, nil
}
binding, err := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
if err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return 0, false, nil
}
return 0, false, err
}
if binding == nil {
return 0, false, nil
}
return binding.DeviceID, true, nil
}
// hasValidPackage 检查卡是否有有效套餐status IN 0,1
// 修复Bug1绑定设备的卡查 device_id独立卡查 iot_card_id
func (s *StopResumeService) hasValidPackage(ctx context.Context, card *model.IotCard) (bool, error) {
if !card.IsStandalone {
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
if lookupErr != nil {
return false, lookupErr
}
if bound {
return s.packageUsageStore.HasValidByCarrier(ctx, "device", deviceID)
}
}
return s.packageUsageStore.HasValidByCarrier(ctx, "iot_card", card.ID)
}
// isTrafficExhausted 检查卡的流量是否已耗尽
// 条件:活跃套餐 status=2已用完或 data_usage_mb >= data_limit_mbdata_limit_mb > 0
// 业务约定data_limit_mb=0 表示无限流量套餐,永远不判定为耗尽
func (s *StopResumeService) isTrafficExhausted(ctx context.Context, card *model.IotCard) (bool, error) {
carrierType := "iot_card"
carrierID := card.ID
if !card.IsStandalone {
deviceID, bound, lookupErr := s.getCardDeviceID(ctx, card)
if lookupErr != nil {
return false, lookupErr
}
if bound {
carrierType = "device"
carrierID = deviceID
}
}
pkg, err := s.packageUsageStore.GetActiveOrDepletedByCarrier(ctx, carrierType, carrierID)
if err != nil {
if stderrors.Is(err, gorm.ErrRecordNotFound) {
return false, nil
}
return false, err
}
if pkg.Status == constants.PackageUsageStatusDepleted {
return true, nil
}
return pkg.DataLimitMB > 0 && pkg.DataUsageMB >= pkg.DataLimitMB, nil
}
// isRealnameOK 检查卡是否满足实名要求
// 行业卡card_category='industry')无需实名;其他卡需 real_name_status=1
func (s *StopResumeService) isRealnameOK(card *model.IotCard) bool {
return card.CardCategory == constants.CardCategoryIndustry ||
card.RealNameStatus == constants.RealNameStatusVerified
}
// checkStopReasons 检查卡的停机原因列表,按优先级排序
// 优先级no_package > traffic_exhausted > not_realname
func (s *StopResumeService) checkStopReasons(ctx context.Context, card *model.IotCard) ([]string, error) {
var reasons []string
// 条件A无有效套餐优先级最高
hasPackage, err := s.hasValidPackage(ctx, card)
if err != nil {
return nil, err
}
if !hasPackage {
reasons = append(reasons, constants.StopReasonNoPackage)
}
// 条件B虚流量耗尽
exhausted, err := s.isTrafficExhausted(ctx, card)
if err != nil {
return nil, err
}
if exhausted {
reasons = append(reasons, constants.StopReasonTrafficExhausted)
}
// 条件C非行业卡且未实名
if !s.isRealnameOK(card) {
reasons = append(reasons, constants.StopReasonNotRealname)
}
return reasons, nil
}
// shouldResume 检查停机卡是否满足复机条件
// 须同时满足:有有效套餐 + 流量未耗尽 + 实名OK
func (s *StopResumeService) shouldResume(ctx context.Context, card *model.IotCard) (bool, error) {
hasPackage, err := s.hasValidPackage(ctx, card)
if err != nil {
return false, err
}
if !hasPackage {
return false, nil
}
exhausted, err := s.isTrafficExhausted(ctx, card)
if err != nil {
return false, err
}
if exhausted {
return false, nil
}
return s.isRealnameOK(card), nil
}
// stopDeviceCards 停机设备下所有在线卡(含设备维度幂等锁)
// 防止设备下多张卡并发触发 EvaluateAndAct 导致重复 Gateway 停机调用
func (s *StopResumeService) stopDeviceCards(ctx context.Context, deviceID uint, stopReason string) {
// 设备维度幂等锁:防止多协程同时对同一设备停机
lockKey := constants.RedisPollingDeviceOpLockKey(deviceID)
locked, _ := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
if !locked {
s.logger.Debug("设备停机操作已在进行中,跳过重复调用",
zap.Uint("device_id", deviceID))
return
}
defer s.redis.Del(ctx, lockKey)
// 查询设备下所有在线卡
cards, err := s.iotCardStore.ListByDeviceIDAndNetworkStatus(ctx, deviceID, constants.NetworkStatusOnline)
if err != nil {
s.logger.Error("查询设备在线卡失败",
zap.Uint("device_id", deviceID), zap.Error(err))
return
}
for _, card := range cards {
if stopErr := s.stopCardWithRetry(ctx, card, stopReason); stopErr != nil {
s.logger.Warn("设备卡停机失败,继续处理其他卡",
zap.Uint("device_id", deviceID),
zap.Uint("card_id", card.ID),
zap.Error(stopErr))
}
}
}
// resumeDeviceCards 复机设备下满足条件的停机卡(含设备维度幂等锁)
// 遍历时对每张卡检查实名状态:未实名普通卡更新 stop_reason='not_realname' 后跳过
func (s *StopResumeService) resumeDeviceCards(ctx context.Context, deviceID uint) {
// 设备维度幂等锁:与 stopDeviceCards 共用,防止停/复机并发
lockKey := constants.RedisPollingDeviceOpLockKey(deviceID)
locked, _ := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
if !locked {
s.logger.Debug("设备复机操作已在进行中,跳过重复调用",
zap.Uint("device_id", deviceID))
return
}
defer s.redis.Del(ctx, lockKey)
// 查询设备下因轮询原因停机的卡
cards, err := s.iotCardStore.ListByDeviceIDAndPollingStopReasons(ctx, deviceID)
if err != nil {
s.logger.Error("查询设备停机卡失败",
zap.Uint("device_id", deviceID), zap.Error(err))
return
}
for _, card := range cards {
if !s.isRealnameOK(card) {
if updateErr := s.iotCardStore.UpdateStopReason(ctx, card.ID, constants.StopReasonNotRealname); updateErr != nil {
s.logger.Warn("更新未实名卡停机原因失败",
zap.Uint("card_id", card.ID), zap.Error(updateErr))
}
continue
}
if resumeErr := s.resumeSingleCard(ctx, card.ID); resumeErr != nil {
s.logger.Warn("设备卡复机失败,继续处理其他卡",
zap.Uint("device_id", deviceID),
zap.Uint("card_id", card.ID),
zap.Error(resumeErr))
}
}
}
// CheckAndStopCard 检查流量耗尽并停机(旧入口,保留兼容性)
// 新代码应优先使用 EvaluateAndAct
func (s *StopResumeService) CheckAndStopCard(ctx context.Context, cardID uint) error {
card, err := s.iotCardStore.GetByID(ctx, cardID)
if err != nil {
return err
}
return s.EvaluateAndAct(ctx, card)
}
// ResumeCardIfStopped 套餐激活或流量重置后自动复机入口
// 支持 iot_card 和 device 两种载体类型
func (s *StopResumeService) ResumeCardIfStopped(ctx context.Context, carrierType string, carrierID uint) error {
switch carrierType {
case "iot_card":
return s.resumeSingleCard(ctx, carrierID)
case "device":
s.resumeDeviceCards(ctx, carrierID)
return nil
default:
return nil
}
}
// resumeSingleCard 对单张卡执行复机逻辑
// 依次检查:已开机则跳过 → 非轮询停机原因则跳过 → 不满足复机条件则跳过 → 加锁 → 调 Gateway → 更新 DB
func (s *StopResumeService) resumeSingleCard(ctx context.Context, cardID uint) error {
card, err := s.iotCardStore.GetByID(ctx, cardID)
if err != nil {
return err
}
if card.NetworkStatus == constants.NetworkStatusOnline {
return nil
}
// 只处理轮询系统引起的停机
if !isPollingStopReason(card.StopReason) {
s.logger.Debug("卡非轮询停机原因,不自动复机",
zap.Uint("card_id", cardID),
zap.String("stop_reason", card.StopReason))
return nil
}
// 检查复机条件
resume, err := s.shouldResume(ctx, card)
if err != nil {
return err
}
if !resume {
return nil
}
// 分布式锁防止同一张卡并发触发多次 Gateway 调用TTL 30s
lockKey := constants.RedisCardResumeLockKey(cardID)
locked, lockErr := s.redis.SetNX(ctx, lockKey, "1", 30*time.Second).Result()
if lockErr != nil {
s.logger.Warn("获取复机分布式锁失败,跳过本次复机",
zap.Uint("card_id", cardID),
zap.Error(lockErr))
return nil
}
if !locked {
s.logger.Debug("复机操作进行中,跳过", zap.Uint("card_id", cardID))
return nil
}
defer s.redis.Del(ctx, lockKey)
if err := s.resumeCardWithRetry(ctx, card); err != nil {
s.logger.Error("调用运营商复机接口失败",
zap.Uint("card_id", cardID),
zap.String("iccid", card.ICCID),
zap.Error(err))
return err
}
now := time.Now()
if err := s.iotCardStore.UpdateFields(ctx, cardID, map[string]any{
"network_status": constants.NetworkStatusOnline,
"resumed_at": now,
"stop_reason": "",
}); err != nil {
s.logger.Error("复机 Gateway 成功但 DB 更新失败",
zap.Uint("card_id", cardID),
zap.String("iccid", card.ICCID),
zap.Error(err))
return nil
}
s.invalidatePollingCache(ctx, card.ID)
s.logger.Info("卡已自动复机",
zap.Uint("card_id", cardID),
zap.String("iccid", card.ICCID))
return nil
}
// stopCardWithRetry 调用运营商停机接口(带重试机制),并更新 DB 停机原因
func (s *StopResumeService) stopCardWithRetry(ctx context.Context, card *model.IotCard, stopReason string) error {
if s.gatewayClient == nil {
return errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
}
s.logger.Info("调用网关停机",
zap.Uint("card_id", card.ID),
zap.String("iccid", card.ICCID),
zap.String("stop_reason", stopReason))
var lastErr error
for i := 0; i < s.maxRetries; i++ {
if i > 0 {
s.logger.Debug("重试调用停机接口",
zap.Int("attempt", i+1),
zap.String("iccid", card.ICCID))
time.Sleep(s.retryInterval)
}
err := s.gatewayClient.StopCard(ctx, &gateway.CardOperationReq{
CardNo: card.ICCID,
})
if err == nil {
s.logger.Info("网关停机成功",
zap.Uint("card_id", card.ID),
zap.String("iccid", card.ICCID))
now := time.Now()
if updateErr := s.iotCardStore.UpdateFields(ctx, card.ID, map[string]any{
"network_status": constants.NetworkStatusOffline,
"stopped_at": now,
"stop_reason": stopReason,
}); updateErr != nil {
s.logger.Error("停机 Gateway 成功但 DB 更新失败",
zap.Uint("card_id", card.ID),
zap.String("iccid", card.ICCID),
zap.Error(updateErr))
}
s.invalidatePollingCache(ctx, card.ID)
return nil
}
lastErr = err
s.logger.Warn("调用停机接口失败,准备重试",
zap.Int("attempt", i+1),
zap.String("iccid", card.ICCID),
zap.Error(err))
}
return lastErr
}
// resumeCardWithRetry 调用运营商复机接口(带重试机制)
func (s *StopResumeService) resumeCardWithRetry(ctx context.Context, card *model.IotCard) error {
if s.gatewayClient == nil {
return errors.New(errors.CodeInternalError, "Gateway 未配置,停复机操作不可用")
}
s.logger.Info("调用网关复机",
zap.Uint("card_id", card.ID),
zap.String("iccid", card.ICCID))
var lastErr error
for i := 0; i < s.maxRetries; i++ {
if i > 0 {
s.logger.Debug("重试调用复机接口",
zap.Int("attempt", i+1),
zap.String("iccid", card.ICCID))
time.Sleep(s.retryInterval)
}
err := s.gatewayClient.StartCard(ctx, &gateway.CardOperationReq{
CardNo: card.ICCID,
})
if err == nil {
s.logger.Info("网关复机成功",
zap.Uint("card_id", card.ID),
zap.String("iccid", card.ICCID))
return nil
}
lastErr = err
s.logger.Warn("调用复机接口失败,准备重试",
zap.Int("attempt", i+1),
zap.String("iccid", card.ICCID),
zap.Error(err))
}
return lastErr
}
// ManualStopCard 手动停机单张卡通过ICCID
func (s *StopResumeService) ManualStopCard(ctx context.Context, iccid string) error {
card, err := s.iotCardStore.GetByICCID(ctx, iccid)
if err != nil {
return errors.New(errors.CodeNotFound, "卡不存在")
}
if card.RealNameStatus != constants.RealNameStatusVerified {
return errors.New(errors.CodeForbidden, "卡未实名,无法操作")
}
// 检查绑定设备是否在复机保护期
if s.deviceSimBindingStore != nil && s.redis != nil {
binding, bindErr := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
if bindErr == nil && binding != nil {
exists, _ := s.redis.Exists(ctx, constants.RedisDeviceProtectKey(binding.DeviceID, "start")).Result()
if exists > 0 {
return errors.New(errors.CodeForbidden, "设备复机保护期内,禁止停机")
}
} else if bindErr != nil && !stderrors.Is(bindErr, gorm.ErrRecordNotFound) {
return errors.Wrap(errors.CodeInternalError, bindErr, "查询卡绑定关系失败")
}
}
if err := s.stopCardWithRetry(ctx, card, constants.StopReasonManual); err != nil {
return errors.Wrap(errors.CodeGatewayError, err, "调用运营商停机失败,请稍后重试")
}
return nil
}
// ManualStartCard 手动复机单张卡通过ICCID
func (s *StopResumeService) ManualStartCard(ctx context.Context, iccid string) error {
card, err := s.iotCardStore.GetByICCID(ctx, iccid)
if err != nil {
return errors.New(errors.CodeNotFound, "卡不存在")
}
if card.RealNameStatus != constants.RealNameStatusVerified {
return errors.New(errors.CodeForbidden, "卡未实名,无法操作")
}
// 检查绑定设备是否在停机保护期
if s.deviceSimBindingStore != nil && s.redis != nil {
binding, bindErr := s.deviceSimBindingStore.GetActiveBindingByCardID(ctx, card.ID)
if bindErr == nil && binding != nil {
exists, _ := s.redis.Exists(ctx, constants.RedisDeviceProtectKey(binding.DeviceID, "stop")).Result()
if exists > 0 {
return errors.New(errors.CodeForbidden, "设备停机保护期内,禁止复机")
}
} else if bindErr != nil && !stderrors.Is(bindErr, gorm.ErrRecordNotFound) {
return errors.Wrap(errors.CodeInternalError, bindErr, "查询卡绑定关系失败")
}
}
if err := s.resumeCardWithRetry(ctx, card); err != nil {
return errors.Wrap(errors.CodeGatewayError, err, "调用运营商复机失败,请稍后重试")
}
now := time.Now()
if err := s.iotCardStore.UpdateFields(ctx, card.ID, map[string]any{
"network_status": constants.NetworkStatusOnline,
"resumed_at": now,
"stop_reason": "",
}); err != nil {
return errors.Wrap(errors.CodeDatabaseError, err, "更新卡状态失败")
}
s.invalidatePollingCache(ctx, card.ID)
return nil
}
// invalidatePollingCache 删除轮询卡缓存,下次轮询自动从 DB 重建
func (s *StopResumeService) invalidatePollingCache(ctx context.Context, cardID uint) {
if s.redis == nil {
return
}
key := constants.RedisPollingCardInfoKey(cardID)
if err := s.redis.Del(ctx, key).Err(); err != nil {
s.logger.Warn("删除轮询卡缓存失败", zap.Uint("card_id", cardID))
}
}