修复并发,暂时的,不够完整,后续还是需要重新设计,这个太乱了,傻逼AI
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m22s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m22s
This commit is contained in:
@@ -9,12 +9,14 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/bytedance/sonic"
|
||||||
"github.com/hibiken/asynq"
|
"github.com/hibiken/asynq"
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
||||||
"github.com/break/junhong_cmp_fiber/internal/bootstrap"
|
"github.com/break/junhong_cmp_fiber/internal/bootstrap"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
"github.com/break/junhong_cmp_fiber/internal/gateway"
|
||||||
|
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/polling"
|
"github.com/break/junhong_cmp_fiber/internal/polling"
|
||||||
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card"
|
||||||
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
|
||||||
@@ -34,6 +36,7 @@ const (
|
|||||||
workerModulePollingInitializer = "polling_initializer"
|
workerModulePollingInitializer = "polling_initializer"
|
||||||
workerModulePollingScheduler = "polling_scheduler"
|
workerModulePollingScheduler = "polling_scheduler"
|
||||||
workerModuleAsynqScheduler = "asynq_scheduler"
|
workerModuleAsynqScheduler = "asynq_scheduler"
|
||||||
|
importRescueLimit = 500 // 启动补偿单次扫描的最大导入任务数
|
||||||
)
|
)
|
||||||
|
|
||||||
// workerModuleStatus 描述当前 Worker 角色下的模块启停计划。
|
// workerModuleStatus 描述当前 Worker 角色下的模块启停计划。
|
||||||
@@ -121,6 +124,7 @@ func runWorker(cfg *config.Config) {
|
|||||||
|
|
||||||
taskHandler := createTaskHandler(runtime, appLogger)
|
taskHandler := createTaskHandler(runtime, appLogger)
|
||||||
taskHandler.RegisterHandlers()
|
taskHandler.RegisterHandlers()
|
||||||
|
rescuePendingImportTasks(ctx, runtime, appLogger)
|
||||||
|
|
||||||
appLogger.Info("Worker 服务器配置完成",
|
appLogger.Info("Worker 服务器配置完成",
|
||||||
zap.Int("concurrency", cfg.Queue.Concurrency),
|
zap.Int("concurrency", cfg.Queue.Concurrency),
|
||||||
@@ -382,18 +386,36 @@ func startAsynqScheduler(cfg *config.Config, redisAddr string, appLogger *zap.Lo
|
|||||||
|
|
||||||
// registerAsynqScheduleTasks 注册 Worker 入口需要的全部定时任务。
|
// registerAsynqScheduleTasks 注册 Worker 入口需要的全部定时任务。
|
||||||
func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error {
|
func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error {
|
||||||
if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask(constants.TaskTypeOrderExpire, nil)); err != nil {
|
if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask(
|
||||||
|
constants.TaskTypeOrderExpire,
|
||||||
|
nil,
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeOrderExpire)),
|
||||||
|
)); err != nil {
|
||||||
return fmt.Errorf("注册订单超时定时任务失败: %w", err)
|
return fmt.Errorf("注册订单超时定时任务失败: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask(constants.TaskTypeAlertCheck, nil)); err != nil {
|
if _, err := asynqScheduler.Register("@every 1m", asynq.NewTask(
|
||||||
|
constants.TaskTypeAlertCheck,
|
||||||
|
nil,
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAlertCheck)),
|
||||||
|
)); err != nil {
|
||||||
return fmt.Errorf("注册告警检查定时任务失败: %w", err)
|
return fmt.Errorf("注册告警检查定时任务失败: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := asynqScheduler.Register("0 2 * * *", asynq.NewTask(constants.TaskTypeDataCleanup, nil)); err != nil {
|
if _, err := asynqScheduler.Register("0 2 * * *", asynq.NewTask(
|
||||||
|
constants.TaskTypeDataCleanup,
|
||||||
|
nil,
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeDataCleanup)),
|
||||||
|
)); err != nil {
|
||||||
return fmt.Errorf("注册数据清理定时任务失败: %w", err)
|
return fmt.Errorf("注册数据清理定时任务失败: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := asynqScheduler.Register(
|
if _, err := asynqScheduler.Register(
|
||||||
"0 2 * * *",
|
"0 2 * * *",
|
||||||
asynq.NewTask(constants.TaskTypeDailyTrafficFlush, nil, asynq.MaxRetry(3), asynq.Timeout(5*time.Minute)),
|
asynq.NewTask(
|
||||||
|
constants.TaskTypeDailyTrafficFlush,
|
||||||
|
nil,
|
||||||
|
asynq.MaxRetry(3),
|
||||||
|
asynq.Timeout(5*time.Minute),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeDailyTrafficFlush)),
|
||||||
|
),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return fmt.Errorf("注册每日流量落盘定时任务失败: %w", err)
|
return fmt.Errorf("注册每日流量落盘定时任务失败: %w", err)
|
||||||
}
|
}
|
||||||
@@ -417,6 +439,80 @@ func createTaskHandler(runtime *workerRuntime, appLogger *zap.Logger) *queue.Han
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rescuePendingImportTasks 将历史遗留的待处理导入任务重新投递到独立导入队列。
|
||||||
|
func rescuePendingImportTasks(ctx context.Context, runtime *workerRuntime, appLogger *zap.Logger) {
|
||||||
|
rescuePendingIotCardImportTasks(ctx, runtime.db, runtime.asynqClient, appLogger)
|
||||||
|
rescuePendingDeviceImportTasks(ctx, runtime.db, runtime.asynqClient, appLogger)
|
||||||
|
}
|
||||||
|
|
||||||
|
// rescuePendingIotCardImportTasks 补偿仍停留在待处理状态的 IoT 卡导入任务。
|
||||||
|
func rescuePendingIotCardImportTasks(ctx context.Context, db *gorm.DB, asynqClient *asynq.Client, appLogger *zap.Logger) {
|
||||||
|
var importTasks []model.IotCardImportTask
|
||||||
|
if err := db.WithContext(ctx).
|
||||||
|
Where("status = ?", model.ImportTaskStatusPending).
|
||||||
|
Limit(importRescueLimit).
|
||||||
|
Find(&importTasks).Error; err != nil {
|
||||||
|
appLogger.Warn("扫描待补偿 IoT 卡导入任务失败", zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, importTask := range importTasks {
|
||||||
|
payload := task.IotCardImportPayload{TaskID: importTask.ID}
|
||||||
|
enqueueImportRescueTask(ctx, asynqClient, constants.TaskTypeIotCardImport, payload, importTask.ID, appLogger)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rescuePendingDeviceImportTasks 补偿仍停留在待处理状态的设备导入任务。
|
||||||
|
func rescuePendingDeviceImportTasks(ctx context.Context, db *gorm.DB, asynqClient *asynq.Client, appLogger *zap.Logger) {
|
||||||
|
var importTasks []model.DeviceImportTask
|
||||||
|
if err := db.WithContext(ctx).
|
||||||
|
Where("status = ?", model.ImportTaskStatusPending).
|
||||||
|
Limit(importRescueLimit).
|
||||||
|
Find(&importTasks).Error; err != nil {
|
||||||
|
appLogger.Warn("扫描待补偿设备导入任务失败", zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, importTask := range importTasks {
|
||||||
|
payload := task.DeviceImportPayload{TaskID: importTask.ID}
|
||||||
|
enqueueImportRescueTask(ctx, asynqClient, constants.TaskTypeDeviceImport, payload, importTask.ID, appLogger)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// enqueueImportRescueTask 将补偿任务提交到任务类型对应的独立队列。
|
||||||
|
func enqueueImportRescueTask(ctx context.Context, asynqClient *asynq.Client, taskType string, payload any, taskID uint, appLogger *zap.Logger) {
|
||||||
|
payloadBytes, err := sonic.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
appLogger.Warn("序列化导入补偿任务载荷失败",
|
||||||
|
zap.String("task_type", taskType),
|
||||||
|
zap.Uint("task_id", taskID),
|
||||||
|
zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
queueName := constants.QueueForTaskType(taskType)
|
||||||
|
taskMessage := asynq.NewTask(
|
||||||
|
taskType,
|
||||||
|
payloadBytes,
|
||||||
|
asynq.Queue(queueName),
|
||||||
|
asynq.TaskID(fmt.Sprintf("import-rescue:%s:%d", taskType, taskID)),
|
||||||
|
asynq.Unique(30*time.Minute),
|
||||||
|
)
|
||||||
|
if _, err := asynqClient.EnqueueContext(ctx, taskMessage); err != nil {
|
||||||
|
appLogger.Warn("提交导入补偿任务失败",
|
||||||
|
zap.String("task_type", taskType),
|
||||||
|
zap.String("queue", queueName),
|
||||||
|
zap.Uint("task_id", taskID),
|
||||||
|
zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
appLogger.Info("导入补偿任务已提交",
|
||||||
|
zap.String("task_type", taskType),
|
||||||
|
zap.String("queue", queueName),
|
||||||
|
zap.Uint("task_id", taskID))
|
||||||
|
}
|
||||||
|
|
||||||
// shutdownWorker 按当前实例实际启动过的模块执行优雅关闭。
|
// shutdownWorker 按当前实例实际启动过的模块执行优雅关闭。
|
||||||
func shutdownWorker(
|
func shutdownWorker(
|
||||||
cancel context.CancelFunc,
|
cancel context.CancelFunc,
|
||||||
|
|||||||
@@ -484,7 +484,7 @@ func (h *PackageActivationHandler) enqueueActivationTask(ctx context.Context, pa
|
|||||||
task := asynq.NewTask(constants.TaskTypePackageQueueActivation, payloadBytes,
|
task := asynq.NewTask(constants.TaskTypePackageQueueActivation, payloadBytes,
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
asynq.Timeout(30*time.Second),
|
asynq.Timeout(30*time.Second),
|
||||||
asynq.Queue(constants.QueueDefault),
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypePackageQueueActivation)),
|
||||||
)
|
)
|
||||||
|
|
||||||
_, err = h.queueClient.Enqueue(task)
|
_, err = h.queueClient.Enqueue(task)
|
||||||
|
|||||||
@@ -263,7 +263,7 @@ func (s *Scheduler) enqueueBatch(ctx context.Context, taskType string, cardIDs [
|
|||||||
task := asynq.NewTask(taskType, payloadBytes,
|
task := asynq.NewTask(taskType, payloadBytes,
|
||||||
asynq.MaxRetry(0),
|
asynq.MaxRetry(0),
|
||||||
asynq.Timeout(30*time.Second),
|
asynq.Timeout(30*time.Second),
|
||||||
asynq.Queue(constants.QueueDefault),
|
asynq.Queue(constants.QueueForTaskType(taskType)),
|
||||||
)
|
)
|
||||||
if _, err := s.queueClient.Enqueue(task); err != nil {
|
if _, err := s.queueClient.Enqueue(task); err != nil {
|
||||||
s.logger.Error("提交任务失败,回退至分片队列防止卡永久丢失",
|
s.logger.Error("提交任务失败,回退至分片队列防止卡永久丢失",
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||||
|
"github.com/hibiken/asynq"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -70,7 +71,12 @@ func (s *Service) CreateImportTask(ctx context.Context, req *dto.ImportDeviceReq
|
|||||||
}
|
}
|
||||||
|
|
||||||
payload := DeviceImportPayload{TaskID: task.ID}
|
payload := DeviceImportPayload{TaskID: task.ID}
|
||||||
err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeDeviceImport, payload)
|
err := s.queueClient.EnqueueTask(
|
||||||
|
ctx,
|
||||||
|
constants.TaskTypeDeviceImport,
|
||||||
|
payload,
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeDeviceImport)),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.importTaskStore.UpdateStatus(ctx, task.ID, model.ImportTaskStatusFailed, "任务入队失败: "+err.Error())
|
s.importTaskStore.UpdateStatus(ctx, task.ID, model.ImportTaskStatusFailed, "任务入队失败: "+err.Error())
|
||||||
appErr := errors.Wrap(errors.CodeInternalError, err, "任务入队失败")
|
appErr := errors.Wrap(errors.CodeInternalError, err, "任务入队失败")
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||||
"github.com/bytedance/sonic"
|
|
||||||
"github.com/hibiken/asynq"
|
"github.com/hibiken/asynq"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
@@ -39,21 +38,12 @@ func (s *Service) SendWelcomeEmail(ctx context.Context, userID uint, email strin
|
|||||||
Body: "感谢您注册我们的服务!我们很高兴为您提供服务。",
|
Body: "感谢您注册我们的服务!我们很高兴为您提供服务。",
|
||||||
}
|
}
|
||||||
|
|
||||||
payloadBytes, err := sonic.Marshal(payload)
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Error("序列化邮件任务载荷失败",
|
|
||||||
zap.Uint("user_id", userID),
|
|
||||||
zap.String("email", email),
|
|
||||||
zap.Error(err))
|
|
||||||
return errors.Wrap(errors.CodeInternalError, err, "序列化邮件任务载荷失败")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 提交任务到队列
|
// 提交任务到队列
|
||||||
err = s.queueClient.EnqueueTask(
|
err := s.queueClient.EnqueueTask(
|
||||||
ctx,
|
ctx,
|
||||||
constants.TaskTypeEmailSend,
|
constants.TaskTypeEmailSend,
|
||||||
payloadBytes,
|
payload,
|
||||||
asynq.Queue(constants.QueueDefault),
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeEmailSend)),
|
||||||
asynq.MaxRetry(constants.DefaultRetryMax),
|
asynq.MaxRetry(constants.DefaultRetryMax),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -82,20 +72,12 @@ func (s *Service) SendPasswordResetEmail(ctx context.Context, email string, rese
|
|||||||
Body: fmt.Sprintf("您的密码重置令牌是: %s\n此令牌将在 1 小时后过期。", resetToken),
|
Body: fmt.Sprintf("您的密码重置令牌是: %s\n此令牌将在 1 小时后过期。", resetToken),
|
||||||
}
|
}
|
||||||
|
|
||||||
payloadBytes, err := sonic.Marshal(payload)
|
// 提交任务到独立邮件队列
|
||||||
if err != nil {
|
err := s.queueClient.EnqueueTask(
|
||||||
s.logger.Error("序列化密码重置邮件任务载荷失败",
|
|
||||||
zap.String("email", email),
|
|
||||||
zap.Error(err))
|
|
||||||
return errors.Wrap(errors.CodeInternalError, err, "序列化密码重置邮件任务载荷失败")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 提交任务到队列(高优先级)
|
|
||||||
err = s.queueClient.EnqueueTask(
|
|
||||||
ctx,
|
ctx,
|
||||||
constants.TaskTypeEmailSend,
|
constants.TaskTypeEmailSend,
|
||||||
payloadBytes,
|
payload,
|
||||||
asynq.Queue(constants.QueueCritical), // 密码重置使用高优先级队列
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeEmailSend)),
|
||||||
asynq.MaxRetry(constants.DefaultRetryMax),
|
asynq.MaxRetry(constants.DefaultRetryMax),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -122,20 +104,12 @@ func (s *Service) SendNotificationEmail(ctx context.Context, to string, subject
|
|||||||
Body: body,
|
Body: body,
|
||||||
}
|
}
|
||||||
|
|
||||||
payloadBytes, err := sonic.Marshal(payload)
|
// 提交任务到独立邮件队列
|
||||||
if err != nil {
|
err := s.queueClient.EnqueueTask(
|
||||||
s.logger.Error("序列化通知邮件任务载荷失败",
|
|
||||||
zap.String("to", to),
|
|
||||||
zap.Error(err))
|
|
||||||
return errors.Wrap(errors.CodeInternalError, err, "序列化通知邮件任务载荷失败")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 提交任务到队列(低优先级)
|
|
||||||
err = s.queueClient.EnqueueTask(
|
|
||||||
ctx,
|
ctx,
|
||||||
constants.TaskTypeEmailSend,
|
constants.TaskTypeEmailSend,
|
||||||
payloadBytes,
|
payload,
|
||||||
asynq.Queue(constants.QueueLow), // 通知邮件使用低优先级队列
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeEmailSend)),
|
||||||
asynq.MaxRetry(constants.DefaultRetryMax),
|
asynq.MaxRetry(constants.DefaultRetryMax),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -126,9 +126,9 @@ func (s *Service) CreateTask(ctx context.Context, req *dto.CreateExportTaskReque
|
|||||||
ctx,
|
ctx,
|
||||||
constants.TaskTypeExportDispatch,
|
constants.TaskTypeExportDispatch,
|
||||||
dispatchPayload{TaskID: task.ID},
|
dispatchPayload{TaskID: task.ID},
|
||||||
asynq.Queue(constants.QueueDefault),
|
|
||||||
asynq.MaxRetry(constants.ExportDispatchRetryMax),
|
asynq.MaxRetry(constants.ExportDispatchRetryMax),
|
||||||
asynq.Timeout(constants.ExportDispatchTaskTimeout),
|
asynq.Timeout(constants.ExportDispatchTaskTimeout),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeExportDispatch)),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
_ = s.taskStore.MarkFailed(ctx, task.ID, userID, "导出任务入队失败")
|
_ = s.taskStore.MarkFailed(ctx, task.ID, userID, "导出任务入队失败")
|
||||||
return nil, errors.Wrap(errors.CodeTaskQueueError, err, "导出任务入队失败")
|
return nil, errors.Wrap(errors.CodeTaskQueueError, err, "导出任务入队失败")
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
"github.com/break/junhong_cmp_fiber/pkg/errors"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
"github.com/break/junhong_cmp_fiber/pkg/middleware"
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
"github.com/break/junhong_cmp_fiber/pkg/queue"
|
||||||
|
"github.com/hibiken/asynq"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -108,7 +109,12 @@ func (s *Service) CreateImportTask(ctx context.Context, req *dto.ImportIotCardRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
payload := IotCardImportPayload{TaskID: task.ID}
|
payload := IotCardImportPayload{TaskID: task.ID}
|
||||||
err = s.queueClient.EnqueueTask(ctx, constants.TaskTypeIotCardImport, payload)
|
err = s.queueClient.EnqueueTask(
|
||||||
|
ctx,
|
||||||
|
constants.TaskTypeIotCardImport,
|
||||||
|
payload,
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeIotCardImport)),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.importTaskStore.UpdateStatus(ctx, task.ID, model.ImportTaskStatusFailed, "任务入队失败: "+err.Error())
|
s.importTaskStore.UpdateStatus(ctx, task.ID, model.ImportTaskStatusFailed, "任务入队失败: "+err.Error())
|
||||||
appErr := errors.Wrap(errors.CodeInternalError, err, "任务入队失败")
|
appErr := errors.Wrap(errors.CodeInternalError, err, "任务入队失败")
|
||||||
|
|||||||
@@ -169,8 +169,8 @@ func (s *Service) HandlePaymentCallback(ctx context.Context, paymentNo string, p
|
|||||||
if len(linkedIDs) > 0 {
|
if len(linkedIDs) > 0 {
|
||||||
taskPayload := task.AutoPurchasePayload{RechargeOrderID: rechargeOrder.ID}
|
taskPayload := task.AutoPurchasePayload{RechargeOrderID: rechargeOrder.ID}
|
||||||
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeAutoPurchaseAfterRecharge, taskPayload,
|
if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeAutoPurchaseAfterRecharge, taskPayload,
|
||||||
asynq.Queue(constants.QueueDefault),
|
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAutoPurchaseAfterRecharge)),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
s.logger.Error("自动购包任务入队失败",
|
s.logger.Error("自动购包任务入队失败",
|
||||||
zap.Uint("recharge_order_id", rechargeOrder.ID),
|
zap.Uint("recharge_order_id", rechargeOrder.ID),
|
||||||
|
|||||||
@@ -271,8 +271,8 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task)
|
|||||||
zap.Error(marshalErr))
|
zap.Error(marshalErr))
|
||||||
} else {
|
} else {
|
||||||
commissionTask := asynq.NewTask(constants.TaskTypeCommission, payloadBytes,
|
commissionTask := asynq.NewTask(constants.TaskTypeCommission, payloadBytes,
|
||||||
asynq.Queue(constants.QueueLow),
|
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeCommission)),
|
||||||
)
|
)
|
||||||
if _, enqueueErr := h.asynqClient.EnqueueContext(ctx, commissionTask); enqueueErr != nil {
|
if _, enqueueErr := h.asynqClient.EnqueueContext(ctx, commissionTask); enqueueErr != nil {
|
||||||
h.logger.Warn("自动购包后提交佣金任务失败",
|
h.logger.Warn("自动购包后提交佣金任务失败",
|
||||||
@@ -296,7 +296,7 @@ func NewAutoPurchaseTask(rechargeOrderID uint) (*asynq.Task, error) {
|
|||||||
return asynq.NewTask(constants.TaskTypeAutoPurchaseAfterRecharge, payloadBytes,
|
return asynq.NewTask(constants.TaskTypeAutoPurchaseAfterRecharge, payloadBytes,
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
asynq.Timeout(2*time.Minute),
|
asynq.Timeout(2*time.Minute),
|
||||||
asynq.Queue(constants.QueueDefault),
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAutoPurchaseAfterRecharge)),
|
||||||
), nil
|
), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ func enqueueTask(ctx context.Context, client *asynq.Client, taskType string, pay
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts = append(opts, asynq.Queue(constants.QueueForTaskType(taskType)))
|
||||||
task := asynq.NewTask(taskType, bytes, opts...)
|
task := asynq.NewTask(taskType, bytes, opts...)
|
||||||
_, err = client.EnqueueContext(ctx, task)
|
_, err = client.EnqueueContext(ctx, task)
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -215,9 +215,9 @@ func (h *ExportDispatchHandler) enqueueShardTasks(ctx context.Context, task *mod
|
|||||||
h.asynqClient,
|
h.asynqClient,
|
||||||
constants.TaskTypeExportShard,
|
constants.TaskTypeExportShard,
|
||||||
payload,
|
payload,
|
||||||
asynq.Queue(constants.QueueLow),
|
|
||||||
asynq.MaxRetry(constants.ExportShardRetryMax),
|
asynq.MaxRetry(constants.ExportShardRetryMax),
|
||||||
asynq.Timeout(constants.ExportShardTaskTimeout),
|
asynq.Timeout(constants.ExportShardTaskTimeout),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeExportShard)),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("导出分片任务入队失败",
|
h.logger.Error("导出分片任务入队失败",
|
||||||
@@ -237,10 +237,10 @@ func (h *ExportDispatchHandler) enqueueFinalizeTask(ctx context.Context, taskID
|
|||||||
h.asynqClient,
|
h.asynqClient,
|
||||||
constants.TaskTypeExportFinalize,
|
constants.TaskTypeExportFinalize,
|
||||||
ExportFinalizePayload{TaskID: taskID},
|
ExportFinalizePayload{TaskID: taskID},
|
||||||
asynq.Queue(constants.QueueDefault),
|
|
||||||
asynq.MaxRetry(constants.ExportFinalizeRetryMax),
|
asynq.MaxRetry(constants.ExportFinalizeRetryMax),
|
||||||
asynq.Timeout(constants.ExportFinalizeTaskTimeout),
|
asynq.Timeout(constants.ExportFinalizeTaskTimeout),
|
||||||
asynq.ProcessIn(5*time.Second),
|
asynq.ProcessIn(5*time.Second),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeExportFinalize)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -175,9 +175,9 @@ func (h *ExportShardHandler) enqueueFinalize(ctx context.Context, taskID uint) e
|
|||||||
h.asynqClient,
|
h.asynqClient,
|
||||||
constants.TaskTypeExportFinalize,
|
constants.TaskTypeExportFinalize,
|
||||||
ExportFinalizePayload{TaskID: taskID},
|
ExportFinalizePayload{TaskID: taskID},
|
||||||
asynq.Queue(constants.QueueDefault),
|
|
||||||
asynq.MaxRetry(constants.ExportFinalizeRetryMax),
|
asynq.MaxRetry(constants.ExportFinalizeRetryMax),
|
||||||
asynq.Timeout(constants.ExportFinalizeTaskTimeout),
|
asynq.Timeout(constants.ExportFinalizeTaskTimeout),
|
||||||
asynq.ProcessIn(2*time.Second),
|
asynq.ProcessIn(2*time.Second),
|
||||||
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeExportFinalize)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -206,7 +206,7 @@ func (h *PollingRealnameHandler) triggerFirstRealnameActivation(ctx context.Cont
|
|||||||
task := asynq.NewTask(constants.TaskTypePackageFirstActivation, payloadBytes,
|
task := asynq.NewTask(constants.TaskTypePackageFirstActivation, payloadBytes,
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
asynq.Timeout(30*time.Second),
|
asynq.Timeout(30*time.Second),
|
||||||
asynq.Queue(constants.QueueDefault),
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypePackageFirstActivation)),
|
||||||
)
|
)
|
||||||
if _, err := h.asynqClient.EnqueueContext(ctx, task); err != nil {
|
if _, err := h.asynqClient.EnqueueContext(ctx, task); err != nil {
|
||||||
h.base.logger.Warn("提交首次实名激活任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
h.base.logger.Warn("提交首次实名激活任务失败", zap.Uint("card_id", cardID), zap.Error(err))
|
||||||
@@ -244,7 +244,7 @@ func (h *PollingRealnameHandler) triggerDeviceRealnameActivation(ctx context.Con
|
|||||||
task := asynq.NewTask(constants.TaskTypePackageFirstActivation, payloadBytes,
|
task := asynq.NewTask(constants.TaskTypePackageFirstActivation, payloadBytes,
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
asynq.Timeout(30*time.Second),
|
asynq.Timeout(30*time.Second),
|
||||||
asynq.Queue(constants.QueueDefault),
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypePackageFirstActivation)),
|
||||||
)
|
)
|
||||||
if _, err := h.asynqClient.EnqueueContext(ctx, task); err != nil {
|
if _, err := h.asynqClient.EnqueueContext(ctx, task); err != nil {
|
||||||
h.base.logger.Warn("提交设备实名激活任务失败",
|
h.base.logger.Warn("提交设备实名激活任务失败",
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ type DatabaseConfig struct {
|
|||||||
// QueueConfig 任务队列配置
|
// QueueConfig 任务队列配置
|
||||||
type QueueConfig struct {
|
type QueueConfig struct {
|
||||||
Concurrency int `mapstructure:"concurrency"` // Worker 并发数(默认:10)
|
Concurrency int `mapstructure:"concurrency"` // Worker 并发数(默认:10)
|
||||||
Queues map[string]int `mapstructure:"queues"` // 队列优先级配置(队列名 -> 权重)
|
Queues map[string]int `mapstructure:"queues"` // Asynq 队列取任务权重(不是业务并发上限)
|
||||||
RetryMax int `mapstructure:"retry_max"` // 最大重试次数(默认:5)
|
RetryMax int `mapstructure:"retry_max"` // 最大重试次数(默认:5)
|
||||||
Timeout time.Duration `mapstructure:"timeout"` // 任务超时时间(默认:10m)
|
Timeout time.Duration `mapstructure:"timeout"` // 任务超时时间(默认:10m)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -172,14 +172,125 @@ const (
|
|||||||
|
|
||||||
// 队列配置常量
|
// 队列配置常量
|
||||||
const (
|
const (
|
||||||
QueueCritical = "critical" // 关键任务队列
|
QueueCritical = "critical" // 旧版关键任务队列(兼容历史任务)
|
||||||
QueueDefault = "default" // 默认队列
|
QueueDefault = "default" // 旧版默认队列(兼容历史任务)
|
||||||
QueueLow = "low" // 低优先级队列
|
QueueLow = "low" // 旧版低优先级队列(兼容历史任务)
|
||||||
DefaultRetryMax = 5
|
|
||||||
DefaultTimeout = 10 * time.Minute
|
QueueEmailSend = TaskTypeEmailSend // 发送邮件任务队列
|
||||||
DefaultConcurrency = 10
|
QueueCommission = TaskTypeCommission // 分佣计算任务队列
|
||||||
|
QueueIotCardImport = TaskTypeIotCardImport // IoT 卡导入任务队列
|
||||||
|
QueueDeviceImport = TaskTypeDeviceImport // 设备导入任务队列
|
||||||
|
QueueExportDispatch = TaskTypeExportDispatch // 导出分发任务队列
|
||||||
|
QueueExportShard = TaskTypeExportShard // 导出分片任务队列
|
||||||
|
QueueExportFinalize = TaskTypeExportFinalize // 导出收尾任务队列
|
||||||
|
QueueCommissionStatsUpdate = TaskTypeCommissionStatsUpdate // 佣金统计更新任务队列
|
||||||
|
QueueCommissionStatsSync = TaskTypeCommissionStatsSync // 佣金统计同步任务队列
|
||||||
|
QueueCommissionStatsArchive = TaskTypeCommissionStatsArchive // 佣金统计归档任务队列
|
||||||
|
QueuePollingRealname = TaskTypePollingRealname // 实名轮询任务队列
|
||||||
|
QueuePollingCarddata = TaskTypePollingCarddata // 流量轮询任务队列
|
||||||
|
QueuePollingPackage = TaskTypePollingPackage // 套餐轮询任务队列
|
||||||
|
QueuePollingProtect = TaskTypePollingProtect // 保护期轮询任务队列
|
||||||
|
QueuePollingCardStatus = TaskTypePollingCardStatus // 卡状态轮询任务队列
|
||||||
|
QueuePackageFirstActivation = TaskTypePackageFirstActivation // 首次实名激活任务队列
|
||||||
|
QueuePackageQueueActivation = TaskTypePackageQueueActivation // 主套餐排队激活任务队列
|
||||||
|
QueuePackageDataReset = TaskTypePackageDataReset // 套餐流量重置任务队列
|
||||||
|
QueueOrderExpire = TaskTypeOrderExpire // 订单超时取消任务队列
|
||||||
|
QueueAutoPurchase = TaskTypeAutoPurchaseAfterRecharge // 充值后自动购包任务队列
|
||||||
|
QueueAlertCheck = TaskTypeAlertCheck // 告警检查任务队列
|
||||||
|
QueueDataCleanup = TaskTypeDataCleanup // 数据清理任务队列
|
||||||
|
QueueDailyTrafficFlush = TaskTypeDailyTrafficFlush // 每日流量落盘任务队列
|
||||||
|
|
||||||
|
DefaultRetryMax = 5 // 默认任务最大重试次数
|
||||||
|
DefaultTimeout = 10 * time.Minute // 默认任务超时时间
|
||||||
|
DefaultConcurrency = 10 // 默认 Worker 并发数
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// QueueForTaskType 根据任务类型返回独立队列名。
|
||||||
|
func QueueForTaskType(taskType string) string {
|
||||||
|
switch taskType {
|
||||||
|
case TaskTypeEmailSend:
|
||||||
|
return QueueEmailSend
|
||||||
|
case TaskTypeCommission:
|
||||||
|
return QueueCommission
|
||||||
|
case TaskTypeIotCardImport:
|
||||||
|
return QueueIotCardImport
|
||||||
|
case TaskTypeDeviceImport:
|
||||||
|
return QueueDeviceImport
|
||||||
|
case TaskTypeExportDispatch:
|
||||||
|
return QueueExportDispatch
|
||||||
|
case TaskTypeExportShard:
|
||||||
|
return QueueExportShard
|
||||||
|
case TaskTypeExportFinalize:
|
||||||
|
return QueueExportFinalize
|
||||||
|
case TaskTypeCommissionStatsUpdate:
|
||||||
|
return QueueCommissionStatsUpdate
|
||||||
|
case TaskTypeCommissionStatsSync:
|
||||||
|
return QueueCommissionStatsSync
|
||||||
|
case TaskTypeCommissionStatsArchive:
|
||||||
|
return QueueCommissionStatsArchive
|
||||||
|
case TaskTypePollingRealname:
|
||||||
|
return QueuePollingRealname
|
||||||
|
case TaskTypePollingCarddata:
|
||||||
|
return QueuePollingCarddata
|
||||||
|
case TaskTypePollingPackage:
|
||||||
|
return QueuePollingPackage
|
||||||
|
case TaskTypePollingProtect:
|
||||||
|
return QueuePollingProtect
|
||||||
|
case TaskTypePollingCardStatus:
|
||||||
|
return QueuePollingCardStatus
|
||||||
|
case TaskTypePackageFirstActivation:
|
||||||
|
return QueuePackageFirstActivation
|
||||||
|
case TaskTypePackageQueueActivation:
|
||||||
|
return QueuePackageQueueActivation
|
||||||
|
case TaskTypePackageDataReset:
|
||||||
|
return QueuePackageDataReset
|
||||||
|
case TaskTypeOrderExpire:
|
||||||
|
return QueueOrderExpire
|
||||||
|
case TaskTypeAutoPurchaseAfterRecharge:
|
||||||
|
return QueueAutoPurchase
|
||||||
|
case TaskTypeAlertCheck:
|
||||||
|
return QueueAlertCheck
|
||||||
|
case TaskTypeDataCleanup:
|
||||||
|
return QueueDataCleanup
|
||||||
|
case TaskTypeDailyTrafficFlush:
|
||||||
|
return QueueDailyTrafficFlush
|
||||||
|
default:
|
||||||
|
return QueueDefault
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultTaskQueueWeights 返回 Worker 默认监听的全部任务队列及其权重。
|
||||||
|
func DefaultTaskQueueWeights() map[string]int {
|
||||||
|
return map[string]int{
|
||||||
|
QueueCritical: 6,
|
||||||
|
QueueIotCardImport: 6,
|
||||||
|
QueueDeviceImport: 6,
|
||||||
|
QueueAutoPurchase: 5,
|
||||||
|
QueuePackageFirstActivation: 5,
|
||||||
|
QueuePackageQueueActivation: 5,
|
||||||
|
QueueOrderExpire: 4,
|
||||||
|
QueueEmailSend: 4,
|
||||||
|
QueueExportDispatch: 4,
|
||||||
|
QueueExportFinalize: 4,
|
||||||
|
QueueCommission: 3,
|
||||||
|
QueueAlertCheck: 3,
|
||||||
|
QueueExportShard: 2,
|
||||||
|
QueueCommissionStatsUpdate: 2,
|
||||||
|
QueueCommissionStatsSync: 2,
|
||||||
|
QueueCommissionStatsArchive: 2,
|
||||||
|
QueuePollingRealname: 1,
|
||||||
|
QueuePollingCarddata: 1,
|
||||||
|
QueuePollingPackage: 1,
|
||||||
|
QueuePollingProtect: 1,
|
||||||
|
QueuePollingCardStatus: 1,
|
||||||
|
QueuePackageDataReset: 1,
|
||||||
|
QueueDataCleanup: 1,
|
||||||
|
QueueDailyTrafficFlush: 1,
|
||||||
|
QueueDefault: 1,
|
||||||
|
QueueLow: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 导出任务相关常量
|
// 导出任务相关常量
|
||||||
const (
|
const (
|
||||||
ExportTaskSceneDevice = "device" // 导出场景:设备
|
ExportTaskSceneDevice = "device" // 导出场景:设备
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||||
|
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||||
"github.com/bytedance/sonic"
|
"github.com/bytedance/sonic"
|
||||||
"github.com/hibiken/asynq"
|
"github.com/hibiken/asynq"
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
@@ -38,6 +39,12 @@ func NewClient(redisClient *redis.Client, logger *zap.Logger) *Client {
|
|||||||
// payload 必须传入 struct 或 map,禁止传入 []byte。
|
// payload 必须传入 struct 或 map,禁止传入 []byte。
|
||||||
// 传入 []byte 会导致 sonic.Marshal 将其 base64 编码,handler 解析时类型不匹配。
|
// 传入 []byte 会导致 sonic.Marshal 将其 base64 编码,handler 解析时类型不匹配。
|
||||||
func (c *Client) EnqueueTask(ctx context.Context, taskType string, payload interface{}, opts ...asynq.Option) error {
|
func (c *Client) EnqueueTask(ctx context.Context, taskType string, payload interface{}, opts ...asynq.Option) error {
|
||||||
|
if _, ok := payload.([]byte); ok {
|
||||||
|
c.logger.Error("任务载荷类型错误,禁止传入 []byte",
|
||||||
|
zap.String("task_type", taskType))
|
||||||
|
return fmt.Errorf("task payload must be struct or map, got []byte")
|
||||||
|
}
|
||||||
|
|
||||||
// 内部统一序列化,调用方无需预先 Marshal
|
// 内部统一序列化,调用方无需预先 Marshal
|
||||||
payloadBytes, err := sonic.Marshal(payload)
|
payloadBytes, err := sonic.Marshal(payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -47,6 +54,8 @@ func (c *Client) EnqueueTask(ctx context.Context, taskType string, payload inter
|
|||||||
return fmt.Errorf("failed to marshal task payload: %w", err)
|
return fmt.Errorf("failed to marshal task payload: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts = append(opts, asynq.Queue(constants.QueueForTaskType(taskType)))
|
||||||
|
|
||||||
// 创建任务
|
// 创建任务
|
||||||
task := asynq.NewTask(taskType, payloadBytes, opts...)
|
task := asynq.NewTask(taskType, payloadBytes, opts...)
|
||||||
|
|
||||||
@@ -78,13 +87,12 @@ func (c *Client) Close() error {
|
|||||||
|
|
||||||
// ParseQueueConfig 解析队列配置为 Asynq 格式
|
// ParseQueueConfig 解析队列配置为 Asynq 格式
|
||||||
func ParseQueueConfig(cfg *config.QueueConfig) map[string]int {
|
func ParseQueueConfig(cfg *config.QueueConfig) map[string]int {
|
||||||
|
queues := constants.DefaultTaskQueueWeights()
|
||||||
if cfg.Queues != nil && len(cfg.Queues) > 0 {
|
if cfg.Queues != nil && len(cfg.Queues) > 0 {
|
||||||
return cfg.Queues
|
// 配置只覆盖权重;任务队列全集由代码兜底,避免漏监听新队列。
|
||||||
}
|
for queueName, weight := range cfg.Queues {
|
||||||
// 默认队列优先级
|
queues[queueName] = weight
|
||||||
return map[string]int{
|
}
|
||||||
"critical": 6,
|
|
||||||
"default": 3,
|
|
||||||
"low": 1,
|
|
||||||
}
|
}
|
||||||
|
return queues
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user