feat(import): 用 Excel 格式替换 CSV 导入

- 删除 CSV 解析代码，新增 Excel 解析器 (excelize) - 更新 IoT 卡和设备导入任务处理器 - 更新 API 路由文档和前端接入指南 - 归档变更到 openspec/changes/archive/ - 同步 delta specs 到 main specs
2026-01-31 14:13:02 +08:00
parent 62708892ec
commit d309951493
24 changed files with 2279 additions and 589 deletions
--- a/internal/routes/device.go
+++ b/internal/routes/device.go
@@ -91,24 +91,26 @@ func registerDeviceRoutes(router fiber.Router, handler *admin.DeviceHandler, imp

 	Register(devices, doc, groupPath, "POST", "/import", importHandler.Import, RouteSpec{
 		Summary: "批量导入设备",
-		Description: `仅平台用户可操作。
+		Description: `仅平台用户可操作。文件格式已从 CSV 升级为 Excel (.xlsx)。

 ### 完整导入流程

 1. **获取上传 URL**: 调用 ` + "`POST /api/admin/storage/upload-url`" + `
-2. **上传 CSV 文件**: 使用预签名 URL 上传文件到对象存储
+2. **上传 Excel 文件**: 使用预签名 URL 上传文件到对象存储
 3. **调用本接口**: 使用返回的 ` + "`file_key`" + ` 提交导入任务

-### CSV 文件格式
+### Excel 文件格式

-必须包含列（首行为表头）:
- ` + "`device_no`" + `: 设备号（必填，唯一）
- ` + "`device_name`" + `: 设备名称
- ` + "`device_model`" + `: 设备型号
- ` + "`device_type`" + `: 设备类型
- ` + "`max_sim_slots`" + `: 最大插槽数（默认4）
- ` + "`manufacturer`" + `: 制造商
- ` + "`iccid_1`" + ` ~ ` + "`iccid_4`" + `: 绑定的卡 ICCID（卡必须已存在且未绑定）`,
+- 文件格式：仅支持 .xlsx (Excel 2007+)
+- 必须包含列（首行为表头）:
+  - ` + "`device_no`" + `: 设备号（必填，唯一）
+  - ` + "`device_name`" + `: 设备名称
+  - ` + "`device_model`" + `: 设备型号
+  - ` + "`device_type`" + `: 设备类型
+  - ` + "`max_sim_slots`" + `: 最大插槽数（默认4）
+  - ` + "`manufacturer`" + `: 制造商
+  - ` + "`iccid_1`" + ` ~ ` + "`iccid_4`" + `: 绑定的卡 ICCID（卡必须已存在且未绑定）
+- 列格式：设置为文本格式(避免长数字被转为科学记数法)`,
 		Tags:   []string{"设备管理"},
 		Input:  new(dto.ImportDeviceRequest),
 		Output: new(dto.ImportDeviceResponse),
--- a/internal/routes/iot_card.go
+++ b/internal/routes/iot_card.go
@@ -33,11 +33,12 @@ func registerIotCardRoutes(router fiber.Router, handler *admin.IotCardHandler, i
 		Description: `## ⚠️ 接口变更说明（BREAKING CHANGE）

 本接口已从 ` + "`multipart/form-data`" + ` 改为 ` + "`application/json`" + `。
+文件格式从 CSV 升级为 Excel (.xlsx),解决长数字被转为科学记数法的问题。

 ### 完整导入流程

 1. **获取上传 URL**: 调用 ` + "`POST /api/admin/storage/upload-url`" + `
-2. **上传 CSV 文件**: 使用预签名 URL 上传文件到对象存储
+2. **上传 Excel 文件**: 使用预签名 URL 上传文件到对象存储
 3. **调用本接口**: 使用返回的 ` + "`file_key`" + ` 提交导入任务

 ### 请求示例
@@ -46,15 +47,16 @@ func registerIotCardRoutes(router fiber.Router, handler *admin.IotCardHandler, i
 {
  "carrier_id": 1,
  "batch_no": "BATCH-2025-01",
-  "file_key": "imports/2025/01/24/abc123.csv"
+  "file_key": "imports/2025/01/24/abc123.xlsx"
 }
 ` + "```" + `

-### CSV 文件格式
+### Excel 文件格式

- 必须包含两列：` + "`iccid`" + `, ` + "`msisdn`" + `
- 首行为表头
- 编码：UTF-8`,
+- 文件格式：仅支持 .xlsx (Excel 2007+)
+- 必须包含两列：` + "`ICCID`" + `, ` + "`MSISDN`" + `
+- 首行为表头(可选,但建议包含)
+- 列格式：设置为文本格式(避免长数字被转为科学记数法)`,
 		Tags:   []string{"IoT卡管理"},
 		Input:  new(dto.ImportIotCardRequest),
 		Output: new(dto.ImportIotCardResponse),
--- a/internal/routes/storage.go
+++ b/internal/routes/storage.go
@@ -29,15 +29,15 @@ func registerStorageRoutes(router fiber.Router, handler *admin.StorageHandler, d
 ` + "```" + `javascript
 // 1. 获取预签名 URL
 const { data } = await api.post('/storage/upload-url', {
-  file_name: 'cards.csv',
-  content_type: 'text/csv',
+  file_name: 'cards.xlsx',
+  content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
  purpose: 'iot_import'
 });

 // 2. 上传文件到对象存储
 await fetch(data.upload_url, {
  method: 'PUT',
-  headers: { 'Content-Type': 'text/csv' },
+  headers: { 'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' },
  body: file
 });

@@ -53,7 +53,7 @@ await api.post('/iot-cards/import', {

 | 值 | 说明 | 生成路径格式 |
 |---|------|-------------|
-| iot_import | ICCID 导入 | imports/YYYY/MM/DD/uuid.csv |
+| iot_import | ICCID/设备导入 (Excel) | imports/YYYY/MM/DD/uuid.xlsx |
 | export | 数据导出 | exports/YYYY/MM/DD/uuid.xlsx |
 | attachment | 附件上传 | attachments/YYYY/MM/DD/uuid.ext |

--- a/internal/task/device_import.go
+++ b/internal/task/device_import.go
@@ -2,11 +2,9 @@ package task

 import (
 	"context"
-	"encoding/csv"
 	stderrors "errors"
-	"io"
-	"os"
-	"strconv"
+	"fmt"
+	"path/filepath"
 	"strings"
 	"time"

@@ -21,6 +19,7 @@ import (
 	"github.com/break/junhong_cmp_fiber/pkg/constants"
 	pkggorm "github.com/break/junhong_cmp_fiber/pkg/gorm"
 	"github.com/break/junhong_cmp_fiber/pkg/storage"
+	"github.com/break/junhong_cmp_fiber/pkg/utils"
 )

 const deviceBatchSize = 100
@@ -99,9 +98,9 @@ func (h *DeviceImportHandler) HandleDeviceImport(ctx context.Context, task *asyn
 		zap.String("storage_key", importTask.StorageKey),
 	)

-	rows, totalCount, err := h.downloadAndParseCSV(ctx, importTask)
+	rows, totalCount, err := h.downloadAndParse(ctx, importTask)
 	if err != nil {
-		h.logger.Error("下载或解析 CSV 失败",
+		h.logger.Error("下载或解析 Excel 失败",
 			zap.Uint("task_id", importTask.ID),
 			zap.Error(err),
 		)
@@ -129,18 +128,7 @@ func (h *DeviceImportHandler) HandleDeviceImport(ctx context.Context, task *asyn
 	return nil
 }

-type deviceRow struct {
-	Line         int
-	DeviceNo     string
-	DeviceName   string
-	DeviceModel  string
-	DeviceType   string
-	MaxSimSlots  int
-	Manufacturer string
-	ICCIDs       []string
-}
-
-func (h *DeviceImportHandler) downloadAndParseCSV(ctx context.Context, task *model.DeviceImportTask) ([]deviceRow, int, error) {
+func (h *DeviceImportHandler) downloadAndParse(ctx context.Context, task *model.DeviceImportTask) ([]utils.DeviceRow, int, error) {
 	if h.storageService == nil {
 		return nil, 0, ErrStorageNotConfigured
 	}
@@ -155,113 +143,12 @@ func (h *DeviceImportHandler) downloadAndParseCSV(ctx context.Context, task *mod
 	}
 	defer cleanup()

-	f, err := os.Open(localPath)
-	if err != nil {
-		return nil, 0, err
-	}
-	defer f.Close()
-
-	return h.parseDeviceCSV(f)
-}
-
-func (h *DeviceImportHandler) parseDeviceCSV(r io.Reader) ([]deviceRow, int, error) {
-	reader := csv.NewReader(r)
-	reader.FieldsPerRecord = -1
-	reader.TrimLeadingSpace = true
-
-	header, err := reader.Read()
-	if err != nil {
-		return nil, 0, err
+	if !strings.HasSuffix(strings.ToLower(task.FileName), ".xlsx") {
+		ext := filepath.Ext(task.FileName)
+		return nil, 0, fmt.Errorf("不支持的文件格式 %s,请上传Excel文件(.xlsx)", ext)
 	}

-	colIndex := h.buildColumnIndex(header)
-	if colIndex["device_no"] == -1 {
-		return nil, 0, ErrMissingDeviceNoColumn
-	}
-
-	var rows []deviceRow
-	lineNum := 1
-
-	for {
-		record, err := reader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			continue
-		}
-		lineNum++
-
-		row := deviceRow{Line: lineNum}
-
-		if idx := colIndex["device_no"]; idx >= 0 && idx < len(record) {
-			row.DeviceNo = strings.TrimSpace(record[idx])
-		}
-		if idx := colIndex["device_name"]; idx >= 0 && idx < len(record) {
-			row.DeviceName = strings.TrimSpace(record[idx])
-		}
-		if idx := colIndex["device_model"]; idx >= 0 && idx < len(record) {
-			row.DeviceModel = strings.TrimSpace(record[idx])
-		}
-		if idx := colIndex["device_type"]; idx >= 0 && idx < len(record) {
-			row.DeviceType = strings.TrimSpace(record[idx])
-		}
-		if idx := colIndex["max_sim_slots"]; idx >= 0 && idx < len(record) {
-			if n, err := strconv.Atoi(strings.TrimSpace(record[idx])); err == nil {
-				row.MaxSimSlots = n
-			}
-		}
-		if idx := colIndex["manufacturer"]; idx >= 0 && idx < len(record) {
-			row.Manufacturer = strings.TrimSpace(record[idx])
-		}
-
-		row.ICCIDs = make([]string, 0, 4)
-		for i := 1; i <= 4; i++ {
-			colName := "iccid_" + strconv.Itoa(i)
-			if idx := colIndex[colName]; idx >= 0 && idx < len(record) {
-				iccid := strings.TrimSpace(record[idx])
-				if iccid != "" {
-					row.ICCIDs = append(row.ICCIDs, iccid)
-				}
-			}
-		}
-
-		if row.DeviceNo == "" {
-			continue
-		}
-
-		if row.MaxSimSlots == 0 {
-			row.MaxSimSlots = 4
-		}
-
-		rows = append(rows, row)
-	}
-
-	return rows, len(rows), nil
-}
-
-func (h *DeviceImportHandler) buildColumnIndex(header []string) map[string]int {
-	index := map[string]int{
-		"device_no":     -1,
-		"device_name":   -1,
-		"device_model":  -1,
-		"device_type":   -1,
-		"max_sim_slots": -1,
-		"manufacturer":  -1,
-		"iccid_1":       -1,
-		"iccid_2":       -1,
-		"iccid_3":       -1,
-		"iccid_4":       -1,
-	}
-
-	for i, col := range header {
-		col = strings.ToLower(strings.TrimSpace(col))
-		if _, exists := index[col]; exists {
-			index[col] = i
-		}
-	}
-
-	return index
+	return utils.ParseDeviceExcel(localPath)
 }

 type deviceImportResult struct {
@@ -272,7 +159,7 @@ type deviceImportResult struct {
 	failedItems  model.ImportResultItems
 }

-func (h *DeviceImportHandler) processImport(ctx context.Context, task *model.DeviceImportTask, rows []deviceRow, totalCount int) *deviceImportResult {
+func (h *DeviceImportHandler) processImport(ctx context.Context, task *model.DeviceImportTask, rows []utils.DeviceRow, totalCount int) *deviceImportResult {
 	result := &deviceImportResult{
 		skippedItems: make(model.ImportResultItems, 0),
 		failedItems:  make(model.ImportResultItems, 0),
@@ -291,7 +178,7 @@ func (h *DeviceImportHandler) processImport(ctx context.Context, task *model.Dev
 	return result
 }

-func (h *DeviceImportHandler) processBatch(ctx context.Context, task *model.DeviceImportTask, batch []deviceRow, result *deviceImportResult) {
+func (h *DeviceImportHandler) processBatch(ctx context.Context, task *model.DeviceImportTask, batch []utils.DeviceRow, result *deviceImportResult) {
 	deviceNos := make([]string, 0, len(batch))
 	allICCIDs := make([]string, 0)

--- a/internal/task/device_import_test.go
+++ b/internal/task/device_import_test.go
@@ -6,6 +6,7 @@ import (

 	"github.com/break/junhong_cmp_fiber/internal/model"
 	"github.com/break/junhong_cmp_fiber/internal/store/postgres"
+	"github.com/break/junhong_cmp_fiber/pkg/utils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
@@ -39,7 +40,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
 		}
 		task.Creator = 1

-		batch := []deviceRow{
+		batch := []utils.DeviceRow{
 			{Line: 2, DeviceNo: "DEV-OWNER-001", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001001"}},
 		}
 		result := &deviceImportResult{
@@ -59,7 +60,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
 		}
 		task.Creator = 1

-		batch := []deviceRow{
+		batch := []utils.DeviceRow{
 			{Line: 3, DeviceNo: "DEV-OWNER-002", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001003", "89860012345670001002"}},
 		}
 		result := &deviceImportResult{
@@ -81,7 +82,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
 		}
 		task.Creator = 1

-		batch := []deviceRow{
+		batch := []utils.DeviceRow{
 			{Line: 4, DeviceNo: "DEV-OWNER-003", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001002", "89860012345670009999"}},
 		}
 		result := &deviceImportResult{
@@ -103,7 +104,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
 		}
 		task.Creator = 1

-		batch := []deviceRow{
+		batch := []utils.DeviceRow{
 			{Line: 5, DeviceNo: "DEV-OWNER-004", MaxSimSlots: 4, ICCIDs: []string{}},
 		}
 		result := &deviceImportResult{
@@ -128,7 +129,7 @@ func TestDeviceImportHandler_ProcessBatch_AllOrNothingValidation(t *testing.T) {
 		}
 		task.Creator = 1

-		batch := []deviceRow{
+		batch := []utils.DeviceRow{
 			{Line: 6, DeviceNo: "DEV-OWNER-005", MaxSimSlots: 4, ICCIDs: []string{"89860012345670001010", "89860012345670001011"}},
 		}
 		result := &deviceImportResult{
@@ -170,7 +171,7 @@ func TestDeviceImportHandler_ProcessImport_AllOrNothing(t *testing.T) {
 	}
 	task.Creator = 1

-	rows := []deviceRow{
+	rows := []utils.DeviceRow{
 		{Line: 2, DeviceNo: "DEV-PI-001", MaxSimSlots: 4, ICCIDs: []string{"89860012345680001001"}},
 		{Line: 3, DeviceNo: "DEV-PI-002", MaxSimSlots: 4, ICCIDs: []string{"89860012345680001002", "89860012345680001003"}},
 		{Line: 4, DeviceNo: "DEV-PI-003", MaxSimSlots: 4, ICCIDs: []string{"89860012345680001003", "89860012345680009999"}},
--- a/internal/task/iot_card_import.go
+++ b/internal/task/iot_card_import.go
@@ -3,7 +3,9 @@ package task
 import (
 	"context"
 	"errors"
-	"os"
+	"fmt"
+	"path/filepath"
+	"strings"
 	"time"

 	"github.com/bytedance/sonic"
@@ -96,9 +98,9 @@ func (h *IotCardImportHandler) HandleIotCardImport(ctx context.Context, task *as
 		zap.String("storage_key", importTask.StorageKey),
 	)

-	cards, totalCount, err := h.downloadAndParseCSV(ctx, importTask)
+	cards, totalCount, err := h.downloadAndParse(ctx, importTask)
 	if err != nil {
-		h.logger.Error("下载或解析 CSV 失败",
+		h.logger.Error("下载或解析 Excel 失败",
 			zap.Uint("task_id", importTask.ID),
 			zap.Error(err),
 		)
@@ -130,7 +132,7 @@ func (h *IotCardImportHandler) HandleIotCardImport(ctx context.Context, task *as
 	return nil
 }

-func (h *IotCardImportHandler) downloadAndParseCSV(ctx context.Context, task *model.IotCardImportTask) (model.CardListJSON, int, error) {
+func (h *IotCardImportHandler) downloadAndParse(ctx context.Context, task *model.IotCardImportTask) (model.CardListJSON, int, error) {
 	if h.storageService == nil {
 		return nil, 0, ErrStorageNotConfigured
 	}
@@ -145,13 +147,12 @@ func (h *IotCardImportHandler) downloadAndParseCSV(ctx context.Context, task *mo
 	}
 	defer cleanup()

-	f, err := os.Open(localPath)
-	if err != nil {
-		return nil, 0, err
+	if !strings.HasSuffix(strings.ToLower(task.FileName), ".xlsx") {
+		ext := filepath.Ext(task.FileName)
+		return nil, 0, fmt.Errorf("不支持的文件格式 %s,请上传Excel文件(.xlsx)", ext)
 	}
-	defer f.Close()

-	parseResult, err := utils.ParseCardCSV(f)
+	parseResult, err := utils.ParseCardExcel(localPath)
 	if err != nil {
 		return nil, 0, err
 	}