缓解io压力

This commit is contained in:
2026-09-02 16:14:27 +08:00
parent 395e5fb47c
commit dbfeeee253
17 changed files with 181 additions and 144 deletions

View File

@@ -49,7 +49,7 @@ func (e *RetentionBlockedError) Error() string {
}
func (e *RetentionBlockedError) Unwrap() error { return e.Err }
// RetainPendingDays 从最早仍在线的日期连续处理至昨天。cleanup 为 false 时只读校验
// RetainPendingDays 每次最多处理一个最早待处理的上海自然日,避免单任务跨历史日期长时间占用数据库
func (s *Service) RetainPendingDays(ctx context.Context, cleanup bool) ([]RetentionResult, error) {
if s.db == nil || s.store == nil {
return nil, fmt.Errorf("日志日留存数据库或对象存储未配置")
@@ -58,42 +58,14 @@ func (s *Service) RetainPendingDays(ctx context.Context, cleanup bool) ([]Retent
if err != nil {
return nil, err
}
results := make([]RetentionResult, 0)
var auditBlocked, integrationBlocked *RetentionBlockedError
for date := start; date.Before(end); date = date.AddDate(0, 0, 1) {
startedAt := time.Now()
result := RetentionResult{ArchiveDate: date.Format(time.DateOnly)}
if auditBlocked == nil {
auditResult, auditErr := s.retainAuditDate(ctx, date, cleanup)
if auditErr != nil {
auditBlocked = &RetentionBlockedError{ArchiveDate: result.ArchiveDate, Source: constants.AuditArchiveSource, Err: auditErr}
} else {
result.EventCount, result.ResourceCount = auditResult.EventCount, auditResult.ResourceCount
result.ManifestKeys = append(result.ManifestKeys, auditResult.ManifestKeys...)
}
}
if integrationBlocked == nil {
integrationResult, integrationErr := s.retainIntegrationDate(ctx, date, cleanup)
if integrationErr != nil {
integrationBlocked = &RetentionBlockedError{ArchiveDate: result.ArchiveDate, Source: constants.IntegrationArchiveSource, Err: integrationErr}
} else {
result.IntegrationCount = integrationResult.IntegrationCount
result.ManifestKeys = append(result.ManifestKeys, integrationResult.ManifestKeys...)
}
}
result.EstimatedBatches = estimatedRetentionBatches(result)
result.Duration = time.Since(startedAt)
if auditBlocked == nil || integrationBlocked == nil {
results = append(results, result)
}
if !start.Before(end) {
return nil, nil
}
if auditBlocked != nil {
return results, auditBlocked
result, err := s.retainDate(ctx, start, cleanup)
if err != nil {
return nil, &RetentionBlockedError{ArchiveDate: start.Format(time.DateOnly), Source: constants.AuditArchiveSource, Err: err}
}
if integrationBlocked != nil {
return results, integrationBlocked
}
return results, nil
return []RetentionResult{result}, nil
}
// RetainDate 校验并按需清理指定已结束自然日,供受控演练使用。