csxj2026/junhong_cmp_fiber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

导出系统
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 7m50s
Details

Browse Source
This commit is contained in:
Break
2026-06-15 15:28:29 +08:00
parent 2f0b24ce88
commit 7ec84fbc0f
17 changed files with 939 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
internal/task/export_dispatch.go
View File

@@ -119,14 +119,22 @@ func (h *ExportDispatchHandler) HandleExportDispatch(ctx context.Context, task *
return asynq.SkipRetry
}
shards, totalRows, err := h.buildShards(ctx, exportTask, strategy, updater)
params := exporter.ParseExportParams(exportTask)
headers, err := strategy.Headers(ctx, params)
if err != nil {
_ = h.taskStore.MarkFailed(ctx, exportTask.ID, updater, "解析导出表头失败")
h.logger.Error("解析导出表头失败", zap.Uint("task_id", exportTask.ID), zap.Error(err))
return err
}
shards, totalRows, err := h.buildShards(ctx, strategy, params, exportTask.CancelRequested, exportTask.ID, updater)
if err != nil {
_ = h.taskStore.MarkFailed(ctx, exportTask.ID, updater, "构建导出分片失败")
h.logger.Error("构建导出分片失败", zap.Uint("task_id", exportTask.ID), zap.Error(err))
return err
}
if err := h.persistShardsAndPlan(ctx, exportTask.ID, updater, shards, totalRows); err != nil {
if err := h.persistShardsAndPlan(ctx, exportTask.ID, updater, shards, totalRows, headers); err != nil {
_ = h.taskStore.MarkFailed(ctx, exportTask.ID, updater, "保存导出分片失败")
return err
}
@@ -141,58 +149,53 @@ func (h *ExportDispatchHandler) HandleExportDispatch(ctx context.Context, task *
h.logger.Info("导出 dispatch 完成",
zap.Uint("task_id", exportTask.ID),
zap.Uint("total_rows", totalRows),
zap.Int("total_rows", totalRows),
zap.Int("total_shards", len(shards)),
)
return nil
}
func (h *ExportDispatchHandler) buildShards(ctx context.Context, task *model.ExportTask, strategy exporter.SceneStrategy, updater uint) ([]*model.ExportShardTask, uint, error) {
func (h *ExportDispatchHandler) buildShards(ctx context.Context, source exporter.DataSource, params exporter.ExportParams, cancelRequested bool, taskID uint, updater uint) ([]*model.ExportShardTask, int, error) {
shards := make([]*model.ExportShardTask, 0)
totalRows := uint(0)
afterID := uint64(0)
if cancelRequested {
return shards, 0, nil
}
totalRows, err := source.Count(ctx, params)
if err != nil {
return nil, 0, err
}
if totalRows <= 0 {
return shards, 0, nil
}
shardNo := 1
for {
if task.CancelRequested {
break
for offset := 0; offset < totalRows; offset += constants.ExportDefaultShardSize {
limit := constants.ExportDefaultShardSize
if remaining := totalRows - offset; remaining < limit {
limit = remaining
}
boundary, err := strategy.NextShardBoundary(ctx, task, afterID, constants.ExportDefaultShardSize)
if err != nil {
return nil, 0, err
}
if boundary == nil {
break
}
rowCount := boundary.RowCount
if rowCount < 0 {
rowCount = 0
}
totalRows += uint(rowCount)
shard := &model.ExportShardTask{
TaskID: task.ID,
TaskID: taskID,
ShardNo: shardNo,
Status: constants.ExportShardStatusPending,
CursorStart: boundary.StartID,
CursorEnd: boundary.EndID,
RowCount: rowCount,
ShardOffset: offset,
ShardLimit: limit,
RowCount: limit,
}
shard.Creator = updater
shard.Updater = updater
shards = append(shards, shard)
afterID = boundary.EndID
shardNo++
}
return shards, totalRows, nil
}
func (h *ExportDispatchHandler) persistShardsAndPlan(ctx context.Context, taskID uint, updater uint, shards []*model.ExportShardTask, totalRows uint) error {
func (h *ExportDispatchHandler) persistShardsAndPlan(ctx context.Context, taskID uint, updater uint, shards []*model.ExportShardTask, totalRows int, headers []string) error {
return h.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
txShardStore := postgres.NewExportShardTaskStore(tx, h.redis)
txTaskStore := postgres.NewExportTaskStore(tx, h.redis)
@@ -200,7 +203,10 @@ func (h *ExportDispatchHandler) persistShardsAndPlan(ctx context.Context, taskID
if err := txShardStore.CreateBatch(ctx, shards); err != nil {
return err
}
if err := txTaskStore.UpdateDispatchPlan(ctx, taskID, totalRows, uint(len(shards)), updater); err != nil {
if err := txTaskStore.UpdateResolvedHeaders(ctx, taskID, headers, updater); err != nil {
return err
}
if err := txTaskStore.UpdateDispatchPlan(ctx, taskID, totalRows, len(shards), updater); err != nil {
return err
}
return nil