feat(import): 用 Excel 格式替换 CSV 导入
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 6m33s

- 删除 CSV 解析代码,新增 Excel 解析器 (excelize)

- 更新 IoT 卡和设备导入任务处理器

- 更新 API 路由文档和前端接入指南

- 归档变更到 openspec/changes/archive/

- 同步 delta specs 到 main specs
This commit is contained in:
2026-01-31 14:13:02 +08:00
parent 62708892ec
commit d309951493
24 changed files with 2279 additions and 589 deletions

680
pkg/utils/excel_test.go Normal file
View File

@@ -0,0 +1,680 @@
package utils
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/xuri/excelize/v2"
)
// createTestCardExcel 创建测试用的 ICCID+MSISDN Excel 文件
func createTestCardExcel(t *testing.T, filename string, headers []string, rows [][]string) string {
t.Helper()
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, filename)
f := excelize.NewFile()
defer func() {
if err := f.Close(); err != nil {
t.Logf("关闭Excel文件失败: %v", err)
}
}()
sheetName := "Sheet1"
// 写入表头
if len(headers) > 0 {
for i, header := range headers {
cell, _ := excelize.CoordinatesToCellName(i+1, 1)
f.SetCellValue(sheetName, cell, header)
}
}
// 写入数据行
for rowIdx, row := range rows {
for colIdx, value := range row {
cell, _ := excelize.CoordinatesToCellName(colIdx+1, rowIdx+2)
f.SetCellValue(sheetName, cell, value)
}
}
err := f.SaveAs(filePath)
require.NoError(t, err, "保存Excel文件失败")
return filePath
}
// createTestDeviceExcel 创建测试用的设备导入 Excel 文件
func createTestDeviceExcel(t *testing.T, filename string, headers []string, rows [][]string) string {
t.Helper()
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, filename)
f := excelize.NewFile()
defer func() {
if err := f.Close(); err != nil {
t.Logf("关闭Excel文件失败: %v", err)
}
}()
sheetName := "Sheet1"
// 写入表头
for i, header := range headers {
cell, _ := excelize.CoordinatesToCellName(i+1, 1)
f.SetCellValue(sheetName, cell, header)
}
// 写入数据行
for rowIdx, row := range rows {
for colIdx, value := range row {
cell, _ := excelize.CoordinatesToCellName(colIdx+1, rowIdx+2)
f.SetCellValue(sheetName, cell, value)
}
}
err := f.SaveAs(filePath)
require.NoError(t, err, "保存Excel文件失败")
return filePath
}
func TestParseCardExcel(t *testing.T) {
tests := []struct {
name string
headers []string
rows [][]string
wantCardCount int
wantErrorCount int
wantError bool
errorContains string
}{
{
name: "标准双列格式-英文表头",
headers: []string{"ICCID", "MSISDN"},
rows: [][]string{
{"89860012345678901234", "13800000001"},
{"89860012345678901235", "13800000002"},
},
wantCardCount: 2,
wantErrorCount: 0,
wantError: false,
},
{
name: "中文表头",
headers: []string{"卡号", "接入号"},
rows: [][]string{
{"89860012345678901234", "13800000001"},
{"89860012345678901235", "13800000002"},
},
wantCardCount: 2,
wantErrorCount: 0,
wantError: false,
},
{
name: "混合中英文表头",
headers: []string{"ICCID", "手机号"},
rows: [][]string{
{"89860012345678901234", "13800000001"},
},
wantCardCount: 1,
wantErrorCount: 0,
wantError: false,
},
{
name: "ICCID为空-应记录错误",
headers: []string{"ICCID", "MSISDN"},
rows: [][]string{
{"89860012345678901234", "13800000001"},
{"", "13800000002"},
},
wantCardCount: 1,
wantErrorCount: 1,
wantError: false,
},
{
name: "MSISDN为空-应记录错误",
headers: []string{"ICCID", "MSISDN"},
rows: [][]string{
{"89860012345678901234", "13800000001"},
{"89860012345678901235", ""},
},
wantCardCount: 1,
wantErrorCount: 1,
wantError: false,
},
{
name: "跳过空行",
headers: []string{"ICCID", "MSISDN"},
rows: [][]string{
{"89860012345678901234", "13800000001"},
{"", ""},
{"89860012345678901235", "13800000002"},
},
wantCardCount: 2,
wantErrorCount: 0,
wantError: false,
},
{
name: "无表头-直接解析数据",
headers: nil,
rows: [][]string{
{"89860012345678901234", "13800000001"},
{"89860012345678901235", "13800000002"},
},
wantCardCount: 2,
wantErrorCount: 0,
wantError: false,
},
{
name: "20位长数字无损",
headers: []string{"ICCID", "MSISDN"},
rows: [][]string{
{"12345678901234567890", "13800000001"},
},
wantCardCount: 1,
wantErrorCount: 0,
wantError: false,
},
{
name: "首尾空格自动去除",
headers: []string{"ICCID", "MSISDN"},
rows: [][]string{
{" 89860012345678901234 ", " 13800000001 "},
},
wantCardCount: 1,
wantErrorCount: 0,
wantError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// 创建测试Excel文件
filePath := createTestCardExcel(t, "test_cards.xlsx", tt.headers, tt.rows)
// 解析Excel
result, err := ParseCardExcel(filePath)
// 验证错误
if tt.wantError {
require.Error(t, err)
if tt.errorContains != "" {
assert.Contains(t, err.Error(), tt.errorContains)
}
return
}
require.NoError(t, err)
require.NotNil(t, result)
// 验证结果
assert.Equal(t, tt.wantCardCount, len(result.Cards), "卡数量不匹配")
assert.Equal(t, tt.wantErrorCount, len(result.ParseErrors), "错误数量不匹配")
// 验证首尾空格被去除
if tt.name == "首尾空格自动去除" && len(result.Cards) > 0 {
assert.Equal(t, "89860012345678901234", result.Cards[0].ICCID)
assert.Equal(t, "13800000001", result.Cards[0].MSISDN)
}
})
}
}
func TestParseCardExcel_ErrorScenarios(t *testing.T) {
tests := []struct {
name string
setupFunc func(t *testing.T) string
wantError bool
errorContains string
}{
{
name: "文件不存在",
setupFunc: func(t *testing.T) string {
return "/nonexistent/file.xlsx"
},
wantError: true,
errorContains: "打开Excel失败",
},
{
name: "Excel无数据行",
setupFunc: func(t *testing.T) string {
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "empty.xlsx")
f := excelize.NewFile()
defer f.Close()
// 只写入表头,无数据行
f.SetCellValue("Sheet1", "A1", "ICCID")
f.SetCellValue("Sheet1", "B1", "MSISDN")
f.SaveAs(filePath)
return filePath
},
wantError: true,
errorContains: "Excel文件无数据行",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filePath := tt.setupFunc(t)
result, err := ParseCardExcel(filePath)
if tt.wantError {
require.Error(t, err)
if tt.errorContains != "" {
assert.Contains(t, err.Error(), tt.errorContains)
}
assert.Nil(t, result)
} else {
require.NoError(t, err)
}
})
}
}
func TestParseDeviceExcel(t *testing.T) {
tests := []struct {
name string
headers []string
rows [][]string
wantCount int
wantError bool
errorContains string
validateFunc func(t *testing.T, rows []DeviceRow)
}{
{
name: "标准10列格式",
headers: []string{
"device_no", "device_name", "device_model", "device_type",
"max_sim_slots", "manufacturer", "iccid_1", "iccid_2", "iccid_3", "iccid_4",
},
rows: [][]string{
{"DEV-001", "GPS追踪器A", "GT06N", "GPS Tracker", "4", "Concox", "89860012345678901234", "89860012345678901235", "", ""},
{"DEV-002", "GPS追踪器B", "GT06N", "GPS Tracker", "4", "Concox", "89860012345678901236", "", "", ""},
},
wantCount: 2,
wantError: false,
validateFunc: func(t *testing.T, rows []DeviceRow) {
assert.Equal(t, "DEV-001", rows[0].DeviceNo)
assert.Equal(t, "GPS追踪器A", rows[0].DeviceName)
assert.Equal(t, 4, rows[0].MaxSimSlots)
assert.Equal(t, 2, len(rows[0].ICCIDs))
assert.Equal(t, "DEV-002", rows[1].DeviceNo)
assert.Equal(t, 1, len(rows[1].ICCIDs))
},
},
{
name: "可选列缺失-应使用默认值",
headers: []string{
"device_no", "iccid_1",
},
rows: [][]string{
{"DEV-003", "89860012345678901234"},
},
wantCount: 1,
wantError: false,
validateFunc: func(t *testing.T, rows []DeviceRow) {
assert.Equal(t, "DEV-003", rows[0].DeviceNo)
assert.Equal(t, 4, rows[0].MaxSimSlots, "max_sim_slots应默认为4")
assert.Equal(t, "", rows[0].DeviceName)
},
},
{
name: "ICCID列解析-全部4个插槽",
headers: []string{
"device_no", "iccid_1", "iccid_2", "iccid_3", "iccid_4",
},
rows: [][]string{
{"DEV-004", "89860012345678901234", "89860012345678901235", "89860012345678901236", "89860012345678901237"},
},
wantCount: 1,
wantError: false,
validateFunc: func(t *testing.T, rows []DeviceRow) {
assert.Equal(t, 4, len(rows[0].ICCIDs))
},
},
{
name: "跳过device_no为空的行",
headers: []string{
"device_no", "iccid_1",
},
rows: [][]string{
{"DEV-005", "89860012345678901234"},
{"", "89860012345678901235"},
{"DEV-006", "89860012345678901236"},
},
wantCount: 2,
wantError: false,
validateFunc: func(t *testing.T, rows []DeviceRow) {
assert.Equal(t, "DEV-005", rows[0].DeviceNo)
assert.Equal(t, "DEV-006", rows[1].DeviceNo)
},
},
{
name: "max_sim_slots字符串转整数",
headers: []string{
"device_no", "max_sim_slots", "iccid_1",
},
rows: [][]string{
{"DEV-007", "2", "89860012345678901234"},
},
wantCount: 1,
wantError: false,
validateFunc: func(t *testing.T, rows []DeviceRow) {
assert.Equal(t, 2, rows[0].MaxSimSlots)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// 创建测试Excel文件
filePath := createTestDeviceExcel(t, "test_devices.xlsx", tt.headers, tt.rows)
// 解析Excel
rows, count, err := ParseDeviceExcel(filePath)
// 验证错误
if tt.wantError {
require.Error(t, err)
if tt.errorContains != "" {
assert.Contains(t, err.Error(), tt.errorContains)
}
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantCount, count, "设备数量不匹配")
assert.Equal(t, tt.wantCount, len(rows), "返回的行数不匹配")
// 执行自定义验证
if tt.validateFunc != nil {
tt.validateFunc(t, rows)
}
})
}
}
func TestParseDeviceExcel_ErrorScenarios(t *testing.T) {
tests := []struct {
name string
setupFunc func(t *testing.T) string
wantError bool
errorContains string
}{
{
name: "文件不存在",
setupFunc: func(t *testing.T) string {
return "/nonexistent/device.xlsx"
},
wantError: true,
errorContains: "打开Excel失败",
},
{
name: "Excel无数据行",
setupFunc: func(t *testing.T) string {
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "empty_device.xlsx")
f := excelize.NewFile()
defer f.Close()
// 只写入表头,无数据行
f.SetCellValue("Sheet1", "A1", "device_no")
f.SaveAs(filePath)
return filePath
},
wantError: true,
errorContains: "Excel文件无数据行",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filePath := tt.setupFunc(t)
rows, count, err := ParseDeviceExcel(filePath)
if tt.wantError {
require.Error(t, err)
if tt.errorContains != "" {
assert.Contains(t, err.Error(), tt.errorContains)
}
assert.Nil(t, rows)
assert.Equal(t, 0, count)
} else {
require.NoError(t, err)
}
})
}
}
func TestSelectSheet(t *testing.T) {
tests := []struct {
name string
setupFunc func() *excelize.File
expectedSheet string
}{
{
name: "优先选择'导入数据'sheet",
setupFunc: func() *excelize.File {
f := excelize.NewFile()
f.NewSheet("Sheet1")
f.NewSheet("导入数据")
f.NewSheet("Sheet2")
return f
},
expectedSheet: "导入数据",
},
{
name: "无'导入数据'sheet-返回第一个",
setupFunc: func() *excelize.File {
f := excelize.NewFile()
return f
},
expectedSheet: "Sheet1",
},
{
name: "删除默认sheet后-返回空字符串",
setupFunc: func() *excelize.File {
f := excelize.NewFile()
// excelize创建新文件时会有默认的Sheet1删除后仍会返回Sheet1
// 这是库的行为,我们只验证没有崩溃
f.DeleteSheet("Sheet1")
return f
},
expectedSheet: "Sheet1", // excelize的默认行为
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := tt.setupFunc()
defer f.Close()
result := selectSheet(f)
assert.Equal(t, tt.expectedSheet, result)
})
}
}
func TestFindCardColumns(t *testing.T) {
tests := []struct {
name string
header []string
wantICCIDCol int
wantMSISDNCol int
}{
{
name: "标准英文表头",
header: []string{"ICCID", "MSISDN"},
wantICCIDCol: 0,
wantMSISDNCol: 1,
},
{
name: "小写英文表头",
header: []string{"iccid", "msisdn"},
wantICCIDCol: 0,
wantMSISDNCol: 1,
},
{
name: "中文表头",
header: []string{"卡号", "接入号"},
wantICCIDCol: 0,
wantMSISDNCol: 1,
},
{
name: "混合表头",
header: []string{"ICCID", "手机号"},
wantICCIDCol: 0,
wantMSISDNCol: 1,
},
{
name: "表头顺序颠倒",
header: []string{"MSISDN", "ICCID"},
wantICCIDCol: 1,
wantMSISDNCol: 0,
},
{
name: "表头包含空格",
header: []string{" ICCID ", " MSISDN "},
wantICCIDCol: 0,
wantMSISDNCol: 1,
},
{
name: "无法识别的表头",
header: []string{"unknown1", "unknown2"},
wantICCIDCol: -1,
wantMSISDNCol: -1,
},
{
name: "只有ICCID列",
header: []string{"ICCID", "其他"},
wantICCIDCol: 0,
wantMSISDNCol: -1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
iccidCol, msisdnCol := findCardColumns(tt.header)
assert.Equal(t, tt.wantICCIDCol, iccidCol, "ICCID列索引不匹配")
assert.Equal(t, tt.wantMSISDNCol, msisdnCol, "MSISDN列索引不匹配")
})
}
}
func TestBuildDeviceColumnIndex(t *testing.T) {
tests := []struct {
name string
header []string
expectedIndex map[string]int
}{
{
name: "标准10列表头",
header: []string{
"device_no", "device_name", "device_model", "device_type",
"max_sim_slots", "manufacturer", "iccid_1", "iccid_2", "iccid_3", "iccid_4",
},
expectedIndex: map[string]int{
"device_no": 0,
"device_name": 1,
"device_model": 2,
"device_type": 3,
"max_sim_slots": 4,
"manufacturer": 5,
"iccid_1": 6,
"iccid_2": 7,
"iccid_3": 8,
"iccid_4": 9,
},
},
{
name: "顺序颠倒",
header: []string{"iccid_1", "device_no"},
expectedIndex: map[string]int{
"iccid_1": 0,
"device_no": 1,
"device_name": -1,
"device_model": -1,
"device_type": -1,
"max_sim_slots": -1,
"manufacturer": -1,
"iccid_2": -1,
"iccid_3": -1,
"iccid_4": -1,
},
},
{
name: "大写表头-能识别",
header: []string{"DEVICE_NO", "DEVICE_NAME"},
expectedIndex: map[string]int{
"device_no": 0,
"device_name": 1,
"device_model": -1,
"device_type": -1,
"max_sim_slots": -1,
"manufacturer": -1,
"iccid_1": -1,
"iccid_2": -1,
"iccid_3": -1,
"iccid_4": -1,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := buildDeviceColumnIndex(tt.header)
assert.Equal(t, tt.expectedIndex, result)
})
}
}
// TestParseCardExcel_RealWorldScenario 测试真实场景
func TestParseCardExcel_RealWorldScenario(t *testing.T) {
t.Run("100行数据性能测试", func(t *testing.T) {
// 生成100行测试数据
headers := []string{"ICCID", "MSISDN"}
rows := make([][]string, 100)
for i := 0; i < 100; i++ {
iccid := "8986001234567890" + padLeft(i, 4)
msisdn := "1380000" + padLeft(i, 4)
rows[i] = []string{iccid, msisdn}
}
filePath := createTestCardExcel(t, "large_cards.xlsx", headers, rows)
result, err := ParseCardExcel(filePath)
require.NoError(t, err)
assert.Equal(t, 100, len(result.Cards))
assert.Equal(t, 0, len(result.ParseErrors))
})
}
// padLeft 左侧填充0
func padLeft(num int, width int) string {
s := ""
for i := 0; i < width; i++ {
s += "0"
}
s += string(rune('0' + num%10))
if num >= 10 {
s = s[:width-2] + string(rune('0'+num/10%10)) + string(rune('0'+num%10))
}
if num >= 100 {
s = s[:width-3] + string(rune('0'+num/100%10)) + string(rune('0'+num/10%10)) + string(rune('0'+num%10))
}
if num >= 1000 {
s = string(rune('0'+num/1000%10)) + string(rune('0'+num/100%10)) + string(rune('0'+num/10%10)) + string(rune('0'+num%10))
}
return s
}