feat(import): 用 Excel 格式替换 CSV 导入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 6m33s

- 删除 CSV 解析代码,新增 Excel 解析器 (excelize)

- 更新 IoT 卡和设备导入任务处理器

- 更新 API 路由文档和前端接入指南

- 归档变更到 openspec/changes/archive/

- 同步 delta specs 到 main specs
This commit is contained in:
2026-01-31 14:13:02 +08:00
parent 62708892ec
commit d309951493
24 changed files with 2279 additions and 589 deletions

View File

@@ -2,11 +2,9 @@ package task
import (
"context"
"encoding/csv"
stderrors "errors"
"io"
"os"
"strconv"
"fmt"
"path/filepath"
"strings"
"time"
@@ -21,6 +19,7 @@ import (
"github.com/break/junhong_cmp_fiber/pkg/constants"
pkggorm "github.com/break/junhong_cmp_fiber/pkg/gorm"
"github.com/break/junhong_cmp_fiber/pkg/storage"
"github.com/break/junhong_cmp_fiber/pkg/utils"
)
const deviceBatchSize = 100
@@ -99,9 +98,9 @@ func (h *DeviceImportHandler) HandleDeviceImport(ctx context.Context, task *asyn
zap.String("storage_key", importTask.StorageKey),
)
rows, totalCount, err := h.downloadAndParseCSV(ctx, importTask)
rows, totalCount, err := h.downloadAndParse(ctx, importTask)
if err != nil {
h.logger.Error("下载或解析 CSV 失败",
h.logger.Error("下载或解析 Excel 失败",
zap.Uint("task_id", importTask.ID),
zap.Error(err),
)
@@ -129,18 +128,7 @@ func (h *DeviceImportHandler) HandleDeviceImport(ctx context.Context, task *asyn
return nil
}
type deviceRow struct {
Line int
DeviceNo string
DeviceName string
DeviceModel string
DeviceType string
MaxSimSlots int
Manufacturer string
ICCIDs []string
}
func (h *DeviceImportHandler) downloadAndParseCSV(ctx context.Context, task *model.DeviceImportTask) ([]deviceRow, int, error) {
func (h *DeviceImportHandler) downloadAndParse(ctx context.Context, task *model.DeviceImportTask) ([]utils.DeviceRow, int, error) {
if h.storageService == nil {
return nil, 0, ErrStorageNotConfigured
}
@@ -155,113 +143,12 @@ func (h *DeviceImportHandler) downloadAndParseCSV(ctx context.Context, task *mod
}
defer cleanup()
f, err := os.Open(localPath)
if err != nil {
return nil, 0, err
}
defer f.Close()
return h.parseDeviceCSV(f)
}
func (h *DeviceImportHandler) parseDeviceCSV(r io.Reader) ([]deviceRow, int, error) {
reader := csv.NewReader(r)
reader.FieldsPerRecord = -1
reader.TrimLeadingSpace = true
header, err := reader.Read()
if err != nil {
return nil, 0, err
if !strings.HasSuffix(strings.ToLower(task.FileName), ".xlsx") {
ext := filepath.Ext(task.FileName)
return nil, 0, fmt.Errorf("不支持的文件格式 %s,请上传Excel文件(.xlsx)", ext)
}
colIndex := h.buildColumnIndex(header)
if colIndex["device_no"] == -1 {
return nil, 0, ErrMissingDeviceNoColumn
}
var rows []deviceRow
lineNum := 1
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
continue
}
lineNum++
row := deviceRow{Line: lineNum}
if idx := colIndex["device_no"]; idx >= 0 && idx < len(record) {
row.DeviceNo = strings.TrimSpace(record[idx])
}
if idx := colIndex["device_name"]; idx >= 0 && idx < len(record) {
row.DeviceName = strings.TrimSpace(record[idx])
}
if idx := colIndex["device_model"]; idx >= 0 && idx < len(record) {
row.DeviceModel = strings.TrimSpace(record[idx])
}
if idx := colIndex["device_type"]; idx >= 0 && idx < len(record) {
row.DeviceType = strings.TrimSpace(record[idx])
}
if idx := colIndex["max_sim_slots"]; idx >= 0 && idx < len(record) {
if n, err := strconv.Atoi(strings.TrimSpace(record[idx])); err == nil {
row.MaxSimSlots = n
}
}
if idx := colIndex["manufacturer"]; idx >= 0 && idx < len(record) {
row.Manufacturer = strings.TrimSpace(record[idx])
}
row.ICCIDs = make([]string, 0, 4)
for i := 1; i <= 4; i++ {
colName := "iccid_" + strconv.Itoa(i)
if idx := colIndex[colName]; idx >= 0 && idx < len(record) {
iccid := strings.TrimSpace(record[idx])
if iccid != "" {
row.ICCIDs = append(row.ICCIDs, iccid)
}
}
}
if row.DeviceNo == "" {
continue
}
if row.MaxSimSlots == 0 {
row.MaxSimSlots = 4
}
rows = append(rows, row)
}
return rows, len(rows), nil
}
func (h *DeviceImportHandler) buildColumnIndex(header []string) map[string]int {
index := map[string]int{
"device_no": -1,
"device_name": -1,
"device_model": -1,
"device_type": -1,
"max_sim_slots": -1,
"manufacturer": -1,
"iccid_1": -1,
"iccid_2": -1,
"iccid_3": -1,
"iccid_4": -1,
}
for i, col := range header {
col = strings.ToLower(strings.TrimSpace(col))
if _, exists := index[col]; exists {
index[col] = i
}
}
return index
return utils.ParseDeviceExcel(localPath)
}
type deviceImportResult struct {
@@ -272,7 +159,7 @@ type deviceImportResult struct {
failedItems model.ImportResultItems
}
func (h *DeviceImportHandler) processImport(ctx context.Context, task *model.DeviceImportTask, rows []deviceRow, totalCount int) *deviceImportResult {
func (h *DeviceImportHandler) processImport(ctx context.Context, task *model.DeviceImportTask, rows []utils.DeviceRow, totalCount int) *deviceImportResult {
result := &deviceImportResult{
skippedItems: make(model.ImportResultItems, 0),
failedItems: make(model.ImportResultItems, 0),
@@ -291,7 +178,7 @@ func (h *DeviceImportHandler) processImport(ctx context.Context, task *model.Dev
return result
}
func (h *DeviceImportHandler) processBatch(ctx context.Context, task *model.DeviceImportTask, batch []deviceRow, result *deviceImportResult) {
func (h *DeviceImportHandler) processBatch(ctx context.Context, task *model.DeviceImportTask, batch []utils.DeviceRow, result *deviceImportResult) {
deviceNos := make([]string, 0, len(batch))
allICCIDs := make([]string, 0)

View File

@@ -6,6 +6,7 @@ import (
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
@@ -39,7 +40,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
}
task.Creator = 1
batch := []deviceRow{
batch := []utils.DeviceRow{
{Line: 2, DeviceNo: "DEV-OWNER-001", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001001"}},
}
result := &deviceImportResult{
@@ -59,7 +60,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
}
task.Creator = 1
batch := []deviceRow{
batch := []utils.DeviceRow{
{Line: 3, DeviceNo: "DEV-OWNER-002", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001003", "89860012345670001002"}},
}
result := &deviceImportResult{
@@ -81,7 +82,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
}
task.Creator = 1
batch := []deviceRow{
batch := []utils.DeviceRow{
{Line: 4, DeviceNo: "DEV-OWNER-003", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001002", "89860012345670009999"}},
}
result := &deviceImportResult{
@@ -103,7 +104,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
}
task.Creator = 1
batch := []deviceRow{
batch := []utils.DeviceRow{
{Line: 5, DeviceNo: "DEV-OWNER-004", MaxSimSlots: 4, ICCIDs: []string{}},
}
result := &deviceImportResult{
@@ -128,7 +129,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
}
task.Creator = 1
batch := []deviceRow{
batch := []utils.DeviceRow{
{Line: 6, DeviceNo: "DEV-OWNER-005", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001010", "89860012345670001011"}},
}
result := &deviceImportResult{
@@ -170,7 +171,7 @@ func TestDeviceImportHandler_ProcessImport_AllOrNothing(t *testing.T) {
}
task.Creator = 1
rows := []deviceRow{
rows := []utils.DeviceRow{
{Line: 2, DeviceNo: "DEV-PI-001", MaxSimSlots: 4, ICCIDs: []string{"89860012345680001001"}},
{Line: 3, DeviceNo: "DEV-PI-002", MaxSimSlots: 4, ICCIDs: []string{"89860012345680001002", "89860012345680001003"}},
{Line: 4, DeviceNo: "DEV-PI-003", MaxSimSlots: 4, ICCIDs: []string{"89860012345680001003", "89860012345680009999"}},

View File

@@ -3,7 +3,9 @@ package task
import (
"context"
"errors"
"os"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/bytedance/sonic"
@@ -96,9 +98,9 @@ func (h *IotCardImportHandler) HandleIotCardImport(ctx context.Context, task *as
zap.String("storage_key", importTask.StorageKey),
)
cards, totalCount, err := h.downloadAndParseCSV(ctx, importTask)
cards, totalCount, err := h.downloadAndParse(ctx, importTask)
if err != nil {
h.logger.Error("下载或解析 CSV 失败",
h.logger.Error("下载或解析 Excel 失败",
zap.Uint("task_id", importTask.ID),
zap.Error(err),
)
@@ -130,7 +132,7 @@ func (h *IotCardImportHandler) HandleIotCardImport(ctx context.Context, task *as
return nil
}
func (h *IotCardImportHandler) downloadAndParseCSV(ctx context.Context, task *model.IotCardImportTask) (model.CardListJSON, int, error) {
func (h *IotCardImportHandler) downloadAndParse(ctx context.Context, task *model.IotCardImportTask) (model.CardListJSON, int, error) {
if h.storageService == nil {
return nil, 0, ErrStorageNotConfigured
}
@@ -145,13 +147,12 @@ func (h *IotCardImportHandler) downloadAndParseCSV(ctx context.Context, task *mo
}
defer cleanup()
f, err := os.Open(localPath)
if err != nil {
return nil, 0, err
if !strings.HasSuffix(strings.ToLower(task.FileName), ".xlsx") {
ext := filepath.Ext(task.FileName)
return nil, 0, fmt.Errorf("不支持的文件格式 %s,请上传Excel文件(.xlsx)", ext)
}
defer f.Close()
parseResult, err := utils.ParseCardCSV(f)
parseResult, err := utils.ParseCardExcel(localPath)
if err != nil {
return nil, 0, err
}