fix: 企业微信审批同步任务在已停用应用下无限堆积
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m21s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 9m21s
恢复任务每 2 分钟为未终态审批投递详情同步,已停用应用的审批同步必然失败并重试 25 次, 且同一审批单每轮重复投递,测试环境 wecom:approval 队列积压约 15 万任务,在用应用的审批结果迟迟不生效。 - 轮询与结果未知恢复只取启用中企业微信应用的审批 - 轮询投递的详情同步按审批单加 10 分钟唯一性锁,最多重试 3 次 - 应用不存在或已停用时同步任务直接结束不重试 - 恢复定时任务加唯一性,队列拥堵时不再堆积重复恢复任务 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1045,6 +1045,8 @@ func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler, auditArchiveEna
|
|||||||
nil,
|
nil,
|
||||||
asynq.MaxRetry(3),
|
asynq.MaxRetry(3),
|
||||||
asynq.Timeout(2*time.Minute),
|
asynq.Timeout(2*time.Minute),
|
||||||
|
// 同一时刻只保留一个恢复任务:队列拥堵时不再堆积重复的恢复任务。
|
||||||
|
asynq.Unique(2*time.Minute),
|
||||||
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeWeComApprovalRecovery)),
|
asynq.Queue(constants.QueueForTaskType(constants.TaskTypeWeComApprovalRecovery)),
|
||||||
)); err != nil {
|
)); err != nil {
|
||||||
return fmt.Errorf("注册企业微信审批主动恢复定时任务失败: %w", err)
|
return fmt.Errorf("注册企业微信审批主动恢复定时任务失败: %w", err)
|
||||||
|
|||||||
@@ -159,6 +159,8 @@ func (r *ApprovalContextRepository) ListUnknownRecovery(ctx context.Context, cut
|
|||||||
err := r.db.WithContext(ctx).Table("tb_wecom_approval_context AS wc").
|
err := r.db.WithContext(ctx).Table("tb_wecom_approval_context AS wc").
|
||||||
Select(`wc.approval_instance_id AS instance_id, wc.application_id, wc.template_id, wc.creator_userid,
|
Select(`wc.approval_instance_id AS instance_id, wc.application_id, wc.template_id, wc.creator_userid,
|
||||||
wc.sp_no, wc.submission_status, COALESCE(wc.submission_attempted_at, wc.updated_at) AS submission_attempted_at`).
|
wc.sp_no, wc.submission_status, COALESCE(wc.submission_attempted_at, wc.updated_at) AS submission_attempted_at`).
|
||||||
|
// 只恢复启用中企业微信应用的审批:已停用或已删除的应用调用必然失败,反复投递会堆积任务。
|
||||||
|
Joins("JOIN tb_wecom_application AS wa ON wa.id = wc.application_id AND wa.status = ? AND wa.deleted_at IS NULL", constants.StatusEnabled).
|
||||||
Where("wc.submission_status = ? AND wc.sp_no = '' AND (wc.last_recovery_at IS NULL OR wc.last_recovery_at <= ?)",
|
Where("wc.submission_status = ? AND wc.sp_no = '' AND (wc.last_recovery_at IS NULL OR wc.last_recovery_at <= ?)",
|
||||||
constants.WeComSubmissionStatusUnknown, cutoff.UTC()).
|
constants.WeComSubmissionStatusUnknown, cutoff.UTC()).
|
||||||
Order("wc.id ASC").Limit(limit).Scan(&records).Error
|
Order("wc.id ASC").Limit(limit).Scan(&records).Error
|
||||||
@@ -191,6 +193,8 @@ func (r *ApprovalContextRepository) ListPendingSync(ctx context.Context, cutoff
|
|||||||
Select(`wc.approval_instance_id AS instance_id, wc.application_id, wc.template_id, wc.creator_userid,
|
Select(`wc.approval_instance_id AS instance_id, wc.application_id, wc.template_id, wc.creator_userid,
|
||||||
wc.sp_no, wc.submission_status, COALESCE(wc.submission_attempted_at, wc.updated_at) AS submission_attempted_at`).
|
wc.sp_no, wc.submission_status, COALESCE(wc.submission_attempted_at, wc.updated_at) AS submission_attempted_at`).
|
||||||
Joins("JOIN tb_approval_instance AS ai ON ai.id = wc.approval_instance_id").
|
Joins("JOIN tb_approval_instance AS ai ON ai.id = wc.approval_instance_id").
|
||||||
|
// 只轮询启用中企业微信应用的审批:已停用或已删除的应用调用必然失败,反复投递会堆积任务。
|
||||||
|
Joins("JOIN tb_wecom_application AS wa ON wa.id = wc.application_id AND wa.status = ? AND wa.deleted_at IS NULL", constants.StatusEnabled).
|
||||||
Where("wc.submission_status = ? AND wc.sp_no <> '' AND ai.status = ? AND (wc.last_synced_at IS NULL OR wc.last_synced_at <= ?)",
|
Where("wc.submission_status = ? AND wc.sp_no <> '' AND ai.status = ? AND (wc.last_synced_at IS NULL OR wc.last_synced_at <= ?)",
|
||||||
constants.WeComSubmissionStatusSubmitted, constants.ApprovalStatusPending, cutoff.UTC()).
|
constants.WeComSubmissionStatusSubmitted, constants.ApprovalStatusPending, cutoff.UTC()).
|
||||||
Order("COALESCE(wc.last_synced_at, wc.created_at) ASC, wc.id ASC").Limit(limit).Scan(&records).Error
|
Order("COALESCE(wc.last_synced_at, wc.created_at) ASC, wc.id ASC").Limit(limit).Scan(&records).Error
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package wecom
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
stderrors "errors"
|
||||||
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -53,6 +55,11 @@ func (h *ApprovalDetailTaskHandler) Handle(ctx context.Context, task *asynq.Task
|
|||||||
}
|
}
|
||||||
detail, err := h.details.Get(ctx, payload.ApplicationID, payload.SPNo, resourceID, correlationID)
|
detail, err := h.details.Get(ctx, payload.ApplicationID, payload.SPNo, resourceID, correlationID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// 企业微信应用已停用或不存在:重试必然失败,直接结束不重试(启用后由轮询重新投递)。
|
||||||
|
var appErr *errors.AppError
|
||||||
|
if stderrors.As(err, &appErr) && appErr.Code == errors.CodeWeComApplicationNotFound {
|
||||||
|
return fmt.Errorf("%w: %v", asynq.SkipRetry, err)
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if record == nil {
|
if record == nil {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package wecom
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
stderrors "errors"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -176,8 +177,14 @@ func (h *ApprovalRecoveryTaskHandler) findUniqueSPNo(ctx context.Context, record
|
|||||||
return unbound[0], integrationIDs, nil
|
return unbound[0], integrationIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// enqueueDetailSync 投递轮询详情同步:同一应用同一审批单在唯一性锁有效期内只保留一个任务(锁在任务成功或到期后释放,
|
||||||
|
// 归档的失败任务不会永久占用),轮询每轮都会再次投递,失败只少量重试。
|
||||||
func (h *ApprovalRecoveryTaskHandler) enqueueDetailSync(ctx context.Context, applicationID uint, spNo string) error {
|
func (h *ApprovalRecoveryTaskHandler) enqueueDetailSync(ctx context.Context, applicationID uint, spNo string) error {
|
||||||
return h.queue.EnqueueTask(ctx, constants.TaskTypeWeComApprovalSync, ApprovalDetailSyncTask{
|
err := h.queue.EnqueueTask(ctx, constants.TaskTypeWeComApprovalSync, ApprovalDetailSyncTask{
|
||||||
ApplicationID: applicationID, SPNo: spNo, Source: constants.ApprovalSyncSourcePolling,
|
ApplicationID: applicationID, SPNo: spNo, Source: constants.ApprovalSyncSourcePolling,
|
||||||
})
|
}, asynq.Unique(constants.WeComApprovalPollingSyncUnique), asynq.MaxRetry(constants.WeComApprovalPollingSyncMaxRetry))
|
||||||
|
if stderrors.Is(err, asynq.ErrTaskIDConflict) || stderrors.Is(err, asynq.ErrDuplicateTask) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -84,15 +84,23 @@ const (
|
|||||||
TaskTypeAutoPurchaseAfterRecharge = "task:auto_purchase_after_recharge" // 充值后自动购包
|
TaskTypeAutoPurchaseAfterRecharge = "task:auto_purchase_after_recharge" // 充值后自动购包
|
||||||
|
|
||||||
// 定时任务类型(由 Asynq Scheduler 调度)
|
// 定时任务类型(由 Asynq Scheduler 调度)
|
||||||
TaskTypeAlertCheck = "alert:check" // 告警检查
|
TaskTypeAlertCheck = "alert:check" // 告警检查
|
||||||
TaskTypeDataCleanup = "data:cleanup" // 数据清理
|
TaskTypeDataCleanup = "data:cleanup" // 数据清理
|
||||||
TaskTypeNotificationCleanup = "notification:cleanup" // 站内通知保留清理
|
TaskTypeNotificationCleanup = "notification:cleanup" // 站内通知保留清理
|
||||||
TaskTypePackageExpiryReminder = "package:expiry:reminder" // 每日套餐临期提醒扫描
|
TaskTypePackageExpiryReminder = "package:expiry:reminder" // 每日套餐临期提醒扫描
|
||||||
TaskTypeDailyTrafficFlush = "traffic:daily:flush" // 每日流量落盘
|
TaskTypeDailyTrafficFlush = "traffic:daily:flush" // 每日流量落盘
|
||||||
TaskTypeOutboxDeliver = "outbox:deliver" // 公共 Outbox 事件投递
|
TaskTypeOutboxDeliver = "outbox:deliver" // 公共 Outbox 事件投递
|
||||||
TaskTypeCardObservationSeries = "card_observation:series" // 卡观测事件序列尝试
|
TaskTypeCardObservationSeries = "card_observation:series" // 卡观测事件序列尝试
|
||||||
TaskTypeWeComApprovalSync = "wecom:approval:sync" // 企业微信审批详情异步同步
|
TaskTypeWeComApprovalSync = "wecom:approval:sync" // 企业微信审批详情异步同步
|
||||||
TaskTypeWeComApprovalRecovery = "wecom:approval:recovery" // 企业微信审批主动恢复与轮询
|
TaskTypeWeComApprovalRecovery = "wecom:approval:recovery" // 企业微信审批主动恢复与轮询
|
||||||
|
// WeComApprovalPollingSyncMaxRetry 轮询投递的审批详情同步最大重试次数:轮询每轮都会再次投递,无需长重试。
|
||||||
|
WeComApprovalPollingSyncMaxRetry = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
// WeComApprovalPollingSyncUnique 轮询投递的审批详情同步唯一性锁有效期:期间同一审批单不重复投递。
|
||||||
|
const WeComApprovalPollingSyncUnique = 10 * time.Minute
|
||||||
|
|
||||||
|
const (
|
||||||
TaskTypeAgentRechargeRecovery = "agent_recharge:payment:recovery" // 代理在线充值支付恢复与查单
|
TaskTypeAgentRechargeRecovery = "agent_recharge:payment:recovery" // 代理在线充值支付恢复与查单
|
||||||
TaskTypeRefundChannelRecovery = "refund:channel:recovery" // 渠道原路退款结果恢复与查询
|
TaskTypeRefundChannelRecovery = "refund:channel:recovery" // 渠道原路退款结果恢复与查询
|
||||||
TaskTypeRefundCommissionRecovery = "refund:commission:recovery" // 退款佣金回溯后处理补偿
|
TaskTypeRefundCommissionRecovery = "refund:commission:recovery" // 退款佣金回溯后处理补偿
|
||||||
|
|||||||
Reference in New Issue
Block a user