csxj2026/junhong_cmp_fiber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

缓解io压力

Browse Source
This commit is contained in:
break
2026-09-02 16:14:27 +08:00
parent 395e5fb47c
commit dbfeeee253
17 changed files with 181 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
cmd/worker/main.go
View File

@@ -153,7 +153,7 @@ func runWorker(cfg *config.Config) {
taskHandler.RegisterHandlers()
registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger)
registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger)
registerAuditArchiveTask(taskHandler.GetMux(), runtime, cfg.Worker.AuditRetentionCleanupEnabled, appLogger, retentionLogger)
registerAuditArchiveTask(taskHandler.GetMux(), runtime, cfg.Worker.AuditRetentionCleanupEnabled, cfg.Worker.AuditArchiveTasksEnabled, appLogger, retentionLogger)
outboxHandler := outbox.NewHandler(runtime.outboxConsumers)
taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle)
startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger)
@@ -286,6 +286,7 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L
pollingIotCardStore,
appLogger,
cfg.Polling.VerboseLog,
cfg.Worker.PollingTotalMaxConcurrency,
)
pollingDeviceSimBindingStore := postgres.NewDeviceSimBindingStore(db, redisClient)
@@ -693,7 +694,7 @@ func startAsynqScheduler(cfg *config.Config, redisAddr string, appLogger *zap.Lo
&asynq.SchedulerOpts{Location: time.Local},
)
if err := registerAsynqScheduleTasks(asynqScheduler); err != nil {
if err := registerAsynqScheduleTasks(asynqScheduler, cfg.Worker.AuditArchiveTasksEnabled); err != nil {
appLogger.Fatal("注册 Asynq 定时任务失败", zap.Error(err))
}
@@ -704,12 +705,14 @@ func startAsynqScheduler(cfg *config.Config, redisAddr string, appLogger *zap.Lo
}()
appLogger.Info("Asynq Scheduler 已启动",
zap.Bool("audit_retention_cleanup_enabled", cfg.Worker.AuditRetentionCleanupEnabled))
zap.Bool("audit_retention_cleanup_enabled", cfg.Worker.AuditRetentionCleanupEnabled),
zap.Bool("audit_archive_tasks_enabled", cfg.Worker.AuditArchiveTasksEnabled),
zap.Int("polling_total_max_concurrency", cfg.Worker.PollingTotalMaxConcurrency))
return asynqScheduler
}
// registerAsynqScheduleTasks 注册 Worker 入口需要的全部定时任务。
func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error {
func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler, auditArchiveEnabled bool) error {
if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask(
constants.TaskTypeAgentRechargeRecovery,
nil,
@@ -779,6 +782,9 @@ func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error {
); err != nil {
return fmt.Errorf("注册每日流量落盘定时任务失败: %w", err)
}
if !auditArchiveEnabled {
return nil
}
if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 4 * * *", asynq.NewTask(
constants.TaskTypeAuditDailyArchive,
nil,
@@ -813,13 +819,13 @@ func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error {
}
// registerAuditArchiveTask 注册 Audit 与 Integration 冷归档任务处理器。
func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, cleanupEnabled bool, appLogger, retentionLogger *zap.Logger) {
func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, cleanupEnabled, enabled bool, appLogger, retentionLogger *zap.Logger) {
if runtime.storageSvc == nil {
appLogger.Warn("对象存储未配置，审计归档任务将在执行时重试")
mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(nil, appLogger).Handle)
integrationHandler := task.NewIntegrationArchiveHandler(nil, appLogger)
mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(nil, appLogger, enabled).Handle)
integrationHandler := task.NewIntegrationArchiveHandler(nil, appLogger, enabled)
mux.HandleFunc(constants.TaskTypeIntegrationDailyArchive, integrationHandler.HandleDaily)
mux.HandleFunc(constants.TaskTypeAuditDailyRetention, task.NewAuditRetentionHandler(nil, retentionLogger, cleanupEnabled).Handle)
mux.HandleFunc(constants.TaskTypeAuditDailyRetention, task.NewAuditRetentionHandler(nil, retentionLogger, cleanupEnabled, enabled).Handle)
return
}
auditWriter, ok := runtime.workerResult.Services.PaymentAudit.(*auditInfra.Writer)
@@ -830,15 +836,15 @@ func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, clean
if err != nil {
appLogger.Fatal("初始化统一审计归档服务失败", zap.Error(err))
}
mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(service, appLogger).Handle)
integrationHandler := task.NewIntegrationArchiveHandler(service, appLogger)
mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(service, appLogger, enabled).Handle)
integrationHandler := task.NewIntegrationArchiveHandler(service, appLogger, enabled)
mux.HandleFunc(constants.TaskTypeIntegrationDailyArchive, integrationHandler.HandleDaily)
mux.HandleFunc(constants.TaskTypeAuditDailyRetention, task.NewAuditRetentionHandler(service, retentionLogger, cleanupEnabled).Handle)
mux.HandleFunc(constants.TaskTypeAuditDailyRetention, task.NewAuditRetentionHandler(service, retentionLogger, cleanupEnabled, enabled).Handle)
appLogger.Info("注册审计归档任务处理器",
zap.String("audit_task_type", constants.TaskTypeAuditDailyArchive),
zap.String("integration_daily_task_type", constants.TaskTypeIntegrationDailyArchive),
zap.String("retention_task_type", constants.TaskTypeAuditDailyRetention),
zap.Bool("retention_cleanup_enabled", cleanupEnabled))
zap.Bool("retention_cleanup_enabled", cleanupEnabled), zap.Bool("archive_tasks_enabled", enabled))
}
// createTaskHandler 创建并返回包含全部任务处理器的 Asynq Handler。