package main import ( "context" "fmt" "os" "os/signal" "strconv" "syscall" "time" "github.com/bytedance/sonic" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" "go.uber.org/zap" "github.com/break/junhong_cmp_fiber/internal/bootstrap" "github.com/break/junhong_cmp_fiber/internal/gateway" "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/internal/polling" iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card" "github.com/break/junhong_cmp_fiber/internal/store/postgres" "github.com/break/junhong_cmp_fiber/internal/task" pkgBootstrap "github.com/break/junhong_cmp_fiber/pkg/bootstrap" "github.com/break/junhong_cmp_fiber/pkg/config" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/database" "github.com/break/junhong_cmp_fiber/pkg/logger" "github.com/break/junhong_cmp_fiber/pkg/queue" "github.com/break/junhong_cmp_fiber/pkg/storage" "gorm.io/gorm" ) const ( workerModuleQueueServer = "queue_server" workerModulePollingInitializer = "polling_initializer" workerModulePollingScheduler = "polling_scheduler" workerModuleAsynqScheduler = "asynq_scheduler" importRescueLimit = 500 // 启动补偿单次扫描的最大导入任务数 ) // workerModuleStatus 描述当前 Worker 角色下的模块启停计划。 type workerModuleStatus struct { enabled []string disabled []string } // workerRuntime 保存所有角色共享的 Worker 运行时依赖。 type workerRuntime struct { redisAddr string redisClient *redis.Client db *gorm.DB storageSvc *storage.Service gatewayClient *gateway.Client asynqClient *asynq.Client workerResult *bootstrap.WorkerBootstrapResult workerServer *queue.Server pollingConfigMgr *polling.PollingConfigManager pollingQueueMgr *polling.PollingQueueManager pollingIotCardStore *postgres.IotCardStore pollingBase *task.PollingBase lifecycleSvc *polling.PollingLifecycleService } func main() { cfg, err := config.Load() if err != nil { panic("加载配置失败: " + err.Error()) } runWorker(cfg) } // runWorker 编排 Worker 进程启动、模块启停和优雅关闭流程。 func runWorker(cfg *config.Config) { if _, err := pkgBootstrap.EnsureDirectories(cfg, nil); err != nil { panic("初始化目录失败: " + err.Error()) } if err := logger.InitLoggers( cfg.Logging.Level, cfg.Logging.Development, logger.LogRotationConfig{ Filename: cfg.Logging.AppLog.Filename, MaxSize: cfg.Logging.AppLog.MaxSize, MaxBackups: cfg.Logging.AppLog.MaxBackups, MaxAge: cfg.Logging.AppLog.MaxAge, Compress: cfg.Logging.AppLog.Compress, }, logger.LogRotationConfig{ Filename: cfg.Logging.AccessLog.Filename, MaxSize: cfg.Logging.AccessLog.MaxSize, MaxBackups: cfg.Logging.AccessLog.MaxBackups, MaxAge: cfg.Logging.AccessLog.MaxAge, Compress: cfg.Logging.AccessLog.Compress, }, ); err != nil { panic("初始化日志失败: " + err.Error()) } defer func() { _ = logger.Sync() // 忽略 sync 错误 }() appLogger := logger.GetAppLogger() ctx, cancel := context.WithCancel(context.Background()) defer cancel() moduleStatus := buildWorkerModuleStatus(cfg.Worker.Role) appLogger.Info("Worker 服务启动中...") logWorkerRole(appLogger, cfg.Worker.Role, cfg.Worker.InstanceName, moduleStatus) runtime := initWorkerRuntime(ctx, cfg, appLogger) defer runtime.close(appLogger) var pollingInitializer *polling.PollingInitializer var pollingScheduler *polling.Scheduler var asynqScheduler *asynq.Scheduler if runsSingletonModules(cfg.Worker.Role) { pollingInitializer = startPollingInitializer(ctx, runtime, appLogger) pollingScheduler = startPollingScheduler(ctx, runtime, pollingInitializer, appLogger) asynqScheduler = startAsynqScheduler(cfg, runtime.redisAddr, appLogger) } taskHandler := createTaskHandler(runtime, appLogger) taskHandler.RegisterHandlers() rescuePendingImportTasks(ctx, runtime, appLogger) appLogger.Info("Worker 服务器配置完成", zap.Int("concurrency", cfg.Queue.Concurrency), zap.Any("queues", cfg.Queue.Queues)) quit := make(chan os.Signal, 1) signal.Notify(quit, os.Interrupt, syscall.SIGTERM) defer signal.Stop(quit) go func() { if err := runtime.workerServer.Run(taskHandler.GetMux()); err != nil { appLogger.Fatal("Worker 服务器运行失败", zap.Error(err)) } }() appLogger.Info("Worker 服务器已启动") <-quit shutdownWorker(cancel, appLogger, runtime.workerServer, pollingInitializer, pollingScheduler, asynqScheduler) } // buildWorkerModuleStatus 根据角色生成启用/禁用模块列表,供启动日志和人工验收使用。 func buildWorkerModuleStatus(role string) workerModuleStatus { status := workerModuleStatus{ enabled: []string{workerModuleQueueServer}, } if runsSingletonModules(role) { status.enabled = append( status.enabled, workerModulePollingInitializer, workerModulePollingScheduler, workerModuleAsynqScheduler, ) return status } status.disabled = []string{ workerModulePollingInitializer, workerModulePollingScheduler, workerModuleAsynqScheduler, } return status } // logWorkerRole 输出当前实例角色、实例名以及模块启停计划。 func logWorkerRole(appLogger *zap.Logger, role string, instanceName string, moduleStatus workerModuleStatus) { appLogger.Info("Worker 角色已确认", zap.String("role", role), zap.String("instance_name", instanceName)) appLogger.Info("Worker 模块启停计划", zap.Strings("enabled_modules", moduleStatus.enabled), zap.Strings("disabled_modules", moduleStatus.disabled)) } // runsSingletonModules 判断当前角色是否承担主动调度和初始化职责。 func runsSingletonModules(role string) bool { return role == constants.WorkerRoleAll || role == constants.WorkerRoleLeader } // initWorkerRuntime 初始化所有角色共享的外部依赖和轮询运行时对象。 func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.Logger) *workerRuntime { redisAddr := cfg.Redis.Address + ":" + strconv.Itoa(cfg.Redis.Port) redisClient := redis.NewClient(&redis.Options{ Addr: redisAddr, Password: cfg.Redis.Password, DB: cfg.Redis.DB, PoolSize: cfg.Redis.PoolSize, MinIdleConns: cfg.Redis.MinIdleConns, DialTimeout: cfg.Redis.DialTimeout, ReadTimeout: cfg.Redis.ReadTimeout, WriteTimeout: cfg.Redis.WriteTimeout, }) if err := redisClient.Ping(ctx).Err(); err != nil { appLogger.Fatal("连接 Redis 失败", zap.Error(err)) } appLogger.Info("Redis 已连接", zap.String("address", redisAddr)) db, err := database.InitPostgreSQL(&cfg.Database, appLogger) if err != nil { appLogger.Fatal("初始化 PostgreSQL 失败", zap.Error(err)) } storageSvc := initStorage(cfg, appLogger) gatewayClient := initGateway(cfg, appLogger) asynqClient := asynq.NewClient(asynq.RedisClientOpt{ Addr: redisAddr, Password: cfg.Redis.Password, DB: cfg.Redis.DB, }) workerDeps := &bootstrap.WorkerDependencies{ DB: db, Redis: redisClient, Logger: appLogger, AsynqClient: asynqClient, StorageService: storageSvc, GatewayClient: gatewayClient, } workerResult, err := bootstrap.BootstrapWorker(workerDeps) if err != nil { appLogger.Fatal("Worker Bootstrap 失败", zap.Error(err)) } workerServer := queue.NewServer(redisClient, &cfg.Queue, appLogger) pollingConfigStore := postgres.NewPollingConfigStore(db) pollingConfigMgr := polling.NewPollingConfigManager(pollingConfigStore, redisClient, appLogger) if err := pollingConfigMgr.Load(ctx); err != nil { appLogger.Warn("加载轮询配置失败,使用空配置继续", zap.Error(err)) } pollingConfigMgr.Start(ctx) pollingQueueMgr := polling.NewPollingQueueManager(redisClient, constants.PollingShardCount, appLogger) pollingIotCardStore := postgres.NewIotCardStore(db, redisClient) pollingBase := task.NewPollingBase( redisClient, pollingQueueMgr, pollingConfigMgr, pollingIotCardStore, appLogger, cfg.Polling.VerboseLog, ) pollingDeviceSimBindingStore := postgres.NewDeviceSimBindingStore(db, redisClient) pollingDeviceStore := postgres.NewDeviceStore(db, redisClient) lifecycleSvc := polling.NewPollingLifecycleService( pollingQueueMgr, pollingConfigMgr, pollingIotCardStore, pollingDeviceSimBindingStore, pollingDeviceStore, appLogger, ) if stopResumeSvc, ok := workerResult.Services.StopResumeService.(*iot_card_svc.StopResumeService); ok { stopResumeSvc.SetPollingCallback(lifecycleSvc) } return &workerRuntime{ redisAddr: redisAddr, redisClient: redisClient, db: db, storageSvc: storageSvc, gatewayClient: gatewayClient, asynqClient: asynqClient, workerResult: workerResult, workerServer: workerServer, pollingConfigMgr: pollingConfigMgr, pollingQueueMgr: pollingQueueMgr, pollingIotCardStore: pollingIotCardStore, pollingBase: pollingBase, lifecycleSvc: lifecycleSvc, } } // close 在 Worker 退出时关闭共享客户端与数据库连接。 func (r *workerRuntime) close(appLogger *zap.Logger) { if r.asynqClient != nil { if err := r.asynqClient.Close(); err != nil { appLogger.Error("关闭 Asynq 客户端失败", zap.Error(err)) } } if r.db != nil { sqlDB, _ := r.db.DB() if sqlDB != nil { if err := sqlDB.Close(); err != nil { appLogger.Error("关闭 PostgreSQL 连接失败", zap.Error(err)) } } } if r.redisClient != nil { if err := r.redisClient.Close(); err != nil { appLogger.Error("关闭 Redis 客户端失败", zap.Error(err)) } } } // startPollingInitializer 为 leader/all 启动渐进式初始化器,并保留配置变更重启逻辑。 func startPollingInitializer(ctx context.Context, runtime *workerRuntime, appLogger *zap.Logger) *polling.PollingInitializer { pollingInitializer := polling.NewPollingInitializer( runtime.pollingIotCardStore, runtime.redisClient, runtime.pollingConfigMgr, runtime.pollingQueueMgr, appLogger, ) pollingInitializer.StartBackground(ctx) runtime.pollingConfigMgr.WatchChanges(ctx, func(hadConfigs, hasConfigs bool) { if hasConfigs { appLogger.Info("轮询配置已变更,触发队列重新初始化", zap.Bool("had_configs", hadConfigs)) pollingInitializer.Restart(ctx) return } appLogger.Info("轮询配置已清空,跳过队列重新初始化") }) return pollingInitializer } // startPollingScheduler 为 leader/all 启动轮询调度器。 func startPollingScheduler( ctx context.Context, runtime *workerRuntime, pollingInitializer *polling.PollingInitializer, appLogger *zap.Logger, ) *polling.Scheduler { dataResetHandler := polling.NewDataResetHandler(runtime.workerResult.Services.ResetService, appLogger) activationHandler := polling.NewPackageActivationHandler( runtime.db, runtime.redisClient, runtime.asynqClient, runtime.workerResult.Services.ActivationService, runtime.workerResult.Services.StopResumeService, appLogger, ) // 注入套餐失效前流量同步器:最小化 iot_card.Service(只需 gateway + store + redis + 流量扣减) trafficSyncer := iot_card_svc.New( runtime.db, runtime.pollingIotCardStore, nil, nil, nil, nil, nil, runtime.gatewayClient, appLogger, nil, ) trafficSyncer.SetRedisClient(runtime.redisClient) trafficSyncer.SetDataDeductor(runtime.workerResult.Services.UsageService) activationHandler.SetTrafficSyncer(trafficSyncer) pollingScheduler := polling.NewScheduler( runtime.redisClient, runtime.asynqClient, runtime.pollingQueueMgr, runtime.pollingConfigMgr, appLogger, activationHandler, dataResetHandler, ) pollingScheduler.SetInitializer(pollingInitializer) if err := pollingScheduler.Start(ctx); err != nil { appLogger.Error("启动轮询调度器失败", zap.Error(err)) return nil } return pollingScheduler } // startAsynqScheduler 为 leader/all 创建并启动 Asynq Scheduler。 func startAsynqScheduler(cfg *config.Config, redisAddr string, appLogger *zap.Logger) *asynq.Scheduler { asynqScheduler := asynq.NewScheduler( asynq.RedisClientOpt{ Addr: redisAddr, Password: cfg.Redis.Password, DB: cfg.Redis.DB, }, &asynq.SchedulerOpts{Location: time.Local}, ) if err := registerAsynqScheduleTasks(asynqScheduler); err != nil { appLogger.Fatal("注册 Asynq 定时任务失败", zap.Error(err)) } go func() { if err := asynqScheduler.Run(); err != nil { appLogger.Fatal("Asynq Scheduler 启动失败", zap.Error(err)) } }() appLogger.Info("Asynq Scheduler 已启动(订单超时: @every 1m, 告警检查: @every 1m, 数据清理: 0 2 * * *, 每日流量落盘: 0 2 * * *)") return asynqScheduler } // registerAsynqScheduleTasks 注册 Worker 入口需要的全部定时任务。 func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error { if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask( constants.TaskTypeOrderExpire, nil, asynq.Queue(constants.QueueForTaskType(constants.TaskTypeOrderExpire)), )); err != nil { return fmt.Errorf("注册订单超时定时任务失败: %w", err) } if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask( constants.TaskTypeAlertCheck, nil, asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAlertCheck)), )); err != nil { return fmt.Errorf("注册告警检查定时任务失败: %w", err) } if _, err := asynqScheduler.Register("0 2 * * *", asynq.NewTask( constants.TaskTypeDataCleanup, nil, asynq.Queue(constants.QueueForTaskType(constants.TaskTypeDataCleanup)), )); err != nil { return fmt.Errorf("注册数据清理定时任务失败: %w", err) } if _, err := asynqScheduler.Register( "0 2 * * *", asynq.NewTask( constants.TaskTypeDailyTrafficFlush, nil, asynq.MaxRetry(3), asynq.Timeout(5*time.Minute), asynq.Queue(constants.QueueForTaskType(constants.TaskTypeDailyTrafficFlush)), ), ); err != nil { return fmt.Errorf("注册每日流量落盘定时任务失败: %w", err) } return nil } // createTaskHandler 创建并返回包含全部任务处理器的 Asynq Handler。 func createTaskHandler(runtime *workerRuntime, appLogger *zap.Logger) *queue.Handler { stopResumeSvc, _ := runtime.workerResult.Services.StopResumeService.(iot_card_svc.StopResumeServiceInterface) return queue.NewHandler( runtime.db, runtime.redisClient, runtime.storageSvc, runtime.gatewayClient, runtime.lifecycleSvc, runtime.workerResult, runtime.asynqClient, appLogger, runtime.pollingBase, stopResumeSvc, ) } // rescuePendingImportTasks 将历史遗留的待处理导入任务重新投递到独立导入队列。 func rescuePendingImportTasks(ctx context.Context, runtime *workerRuntime, appLogger *zap.Logger) { rescuePendingIotCardImportTasks(ctx, runtime.db, runtime.asynqClient, appLogger) rescuePendingDeviceImportTasks(ctx, runtime.db, runtime.asynqClient, appLogger) } // rescuePendingIotCardImportTasks 补偿仍停留在待处理状态的 IoT 卡导入任务。 func rescuePendingIotCardImportTasks(ctx context.Context, db *gorm.DB, asynqClient *asynq.Client, appLogger *zap.Logger) { var importTasks []model.IotCardImportTask if err := db.WithContext(ctx). Where("status = ?", model.ImportTaskStatusPending). Limit(importRescueLimit). Find(&importTasks).Error; err != nil { appLogger.Warn("扫描待补偿 IoT 卡导入任务失败", zap.Error(err)) return } for _, importTask := range importTasks { payload := task.IotCardImportPayload{TaskID: importTask.ID} enqueueImportRescueTask(ctx, asynqClient, constants.TaskTypeIotCardImport, payload, importTask.ID, appLogger) } } // rescuePendingDeviceImportTasks 补偿仍停留在待处理状态的设备导入任务。 func rescuePendingDeviceImportTasks(ctx context.Context, db *gorm.DB, asynqClient *asynq.Client, appLogger *zap.Logger) { var importTasks []model.DeviceImportTask if err := db.WithContext(ctx). Where("status = ?", model.ImportTaskStatusPending). Limit(importRescueLimit). Find(&importTasks).Error; err != nil { appLogger.Warn("扫描待补偿设备导入任务失败", zap.Error(err)) return } for _, importTask := range importTasks { payload := task.DeviceImportPayload{TaskID: importTask.ID} enqueueImportRescueTask(ctx, asynqClient, constants.TaskTypeDeviceImport, payload, importTask.ID, appLogger) } } // enqueueImportRescueTask 将补偿任务提交到任务类型对应的独立队列。 func enqueueImportRescueTask(ctx context.Context, asynqClient *asynq.Client, taskType string, payload any, taskID uint, appLogger *zap.Logger) { payloadBytes, err := sonic.Marshal(payload) if err != nil { appLogger.Warn("序列化导入补偿任务载荷失败", zap.String("task_type", taskType), zap.Uint("task_id", taskID), zap.Error(err)) return } queueName := constants.QueueForTaskType(taskType) taskMessage := asynq.NewTask( taskType, payloadBytes, asynq.Queue(queueName), asynq.TaskID(fmt.Sprintf("import-rescue:%s:%d", taskType, taskID)), asynq.Unique(30*time.Minute), ) if _, err := asynqClient.EnqueueContext(ctx, taskMessage); err != nil { appLogger.Warn("提交导入补偿任务失败", zap.String("task_type", taskType), zap.String("queue", queueName), zap.Uint("task_id", taskID), zap.Error(err)) return } appLogger.Info("导入补偿任务已提交", zap.String("task_type", taskType), zap.String("queue", queueName), zap.Uint("task_id", taskID)) } // shutdownWorker 按当前实例实际启动过的模块执行优雅关闭。 func shutdownWorker( cancel context.CancelFunc, appLogger *zap.Logger, workerServer *queue.Server, pollingInitializer *polling.PollingInitializer, pollingScheduler *polling.Scheduler, asynqScheduler *asynq.Scheduler, ) { appLogger.Info("正在关闭 Worker 服务器...") if asynqScheduler != nil { asynqScheduler.Shutdown() } if pollingScheduler != nil { pollingScheduler.Stop() } cancel() if pollingInitializer != nil { pollingInitializer.Stop() } workerServer.Shutdown() appLogger.Info("Worker 服务器已停止") } func initStorage(cfg *config.Config, appLogger *zap.Logger) *storage.Service { if cfg.Storage.Provider == "" || cfg.Storage.S3.Endpoint == "" { appLogger.Info("对象存储未配置,跳过初始化") return nil } provider, err := storage.NewS3Provider(&cfg.Storage) if err != nil { appLogger.Warn("初始化对象存储失败,功能将不可用", zap.Error(err)) return nil } appLogger.Info("对象存储已初始化", zap.String("provider", cfg.Storage.Provider), zap.String("bucket", cfg.Storage.S3.Bucket), ) return storage.NewService(provider, &cfg.Storage) } // initGateway 初始化 Gateway 客户端 func initGateway(cfg *config.Config, appLogger *zap.Logger) *gateway.Client { if cfg.Gateway.BaseURL == "" { appLogger.Info("Gateway 未配置,跳过初始化(轮询任务将无法查询真实数据)") return nil } client := gateway.NewClient( cfg.Gateway.BaseURL, cfg.Gateway.AppID, cfg.Gateway.AppSecret, appLogger, ).WithTimeout(time.Duration(cfg.Gateway.Timeout) * time.Second) appLogger.Info("Gateway 客户端初始化成功", zap.String("base_url", cfg.Gateway.BaseURL), zap.String("app_id", cfg.Gateway.AppID)) return client }