csxj2026/junhong_cmp_fiber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: 添加环境变量管理工具和部署配置改版
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 5m33s
Details

Browse Source 
主要改动：
- 新增交互式环境配置脚本 (scripts/setup-env.sh)
- 新增本地启动快捷脚本 (scripts/run-local.sh)
- 新增环境变量模板文件 (.env.example)
- 部署模式改版：使用嵌入式配置 + 环境变量覆盖
- 添加对象存储功能支持
- 改进 IoT 卡片导入任务
- 优化 OpenAPI 文档生成
- 删除旧的配置文件，改用嵌入式默认配置
This commit is contained in:
huang
2026-01-26 10:28:29 +08:00
parent 194078674a
commit 45aa7deb87
94 changed files with 6532 additions and 1967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
internal/task/iot_card_import.go
View File

@@ -2,6 +2,8 @@ package task
import (
"context"
"errors"
"os"
"time"
"github.com/bytedance/sonic"
@@ -14,9 +16,16 @@ import (
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
pkggorm "github.com/break/junhong_cmp_fiber/pkg/gorm"
"github.com/break/junhong_cmp_fiber/pkg/storage"
"github.com/break/junhong_cmp_fiber/pkg/utils"
"github.com/break/junhong_cmp_fiber/pkg/validator"
)
var (
ErrStorageNotConfigured = errors.New("对象存储服务未配置")
ErrStorageKeyEmpty = errors.New("文件存储路径为空")
)
const batchSize = 1000
type IotCardImportPayload struct {
@@ -28,15 +37,24 @@ type IotCardImportHandler struct {
redis *redis.Client
importTaskStore *postgres.IotCardImportTaskStore
iotCardStore *postgres.IotCardStore
storageService *storage.Service
logger *zap.Logger
}
func NewIotCardImportHandler(db *gorm.DB, redis *redis.Client, importTaskStore *postgres.IotCardImportTaskStore, iotCardStore *postgres.IotCardStore, logger *zap.Logger) *IotCardImportHandler {
func NewIotCardImportHandler(
db *gorm.DB,
redis *redis.Client,
importTaskStore *postgres.IotCardImportTaskStore,
iotCardStore *postgres.IotCardStore,
storageSvc *storage.Service,
logger *zap.Logger,
) *IotCardImportHandler {
return &IotCardImportHandler{
db: db,
redis: redis,
importTaskStore: importTaskStore,
iotCardStore: iotCardStore,
storageService: storageSvc,
logger: logger,
}
}
@@ -75,9 +93,23 @@ func (h *IotCardImportHandler) HandleIotCardImport(ctx context.Context, task *as
h.logger.Info("开始处理 IoT 卡导入任务",
zap.Uint("task_id", importTask.ID),
zap.String("task_no", importTask.TaskNo),
zap.Int("total_count", importTask.TotalCount),
zap.String("storage_key", importTask.StorageKey),
)
cards, totalCount, err := h.downloadAndParseCSV(ctx, importTask)
if err != nil {
h.logger.Error("下载或解析 CSV 失败",
zap.Uint("task_id", importTask.ID),
zap.Error(err),
)
h.importTaskStore.UpdateStatus(ctx, importTask.ID, model.ImportTaskStatusFailed, err.Error())
return asynq.SkipRetry
}
importTask.CardList = cards
importTask.TotalCount = totalCount
h.importTaskStore.UpdateCardList(ctx, importTask.ID, cards, totalCount)
result := h.processImport(ctx, importTask)
h.importTaskStore.UpdateResult(ctx, importTask.ID, result.successCount, result.skipCount, result.failCount, result.skippedItems, result.failedItems)
@@ -98,6 +130,43 @@ func (h *IotCardImportHandler) HandleIotCardImport(ctx context.Context, task *as
return nil
}
func (h *IotCardImportHandler) downloadAndParseCSV(ctx context.Context, task *model.IotCardImportTask) (model.CardListJSON, int, error) {
if h.storageService == nil {
return nil, 0, ErrStorageNotConfigured
}
if task.StorageKey == "" {
return nil, 0, ErrStorageKeyEmpty
}
localPath, cleanup, err := h.storageService.DownloadToTemp(ctx, task.StorageKey)
if err != nil {
return nil, 0, err
}
defer cleanup()
f, err := os.Open(localPath)
if err != nil {
return nil, 0, err
}
defer f.Close()
parseResult, err := utils.ParseCardCSV(f)
if err != nil {
return nil, 0, err
}
cards := make(model.CardListJSON, 0, len(parseResult.Cards))
for _, card := range parseResult.Cards {
cards = append(cards, model.CardItem{
ICCID: card.ICCID,
MSISDN: card.MSISDN,
})
}
return cards, parseResult.TotalCount, nil
}
type importResult struct {
successCount int
skipCount int
@@ -112,59 +181,72 @@ func (h *IotCardImportHandler) processImport(ctx context.Context, task *model.Io
failedItems: make(model.ImportResultItems, 0),
}
iccids := h.getICCIDsFromTask(task)
if len(iccids) == 0 {
cards := h.getCardsFromTask(task)
if len(cards) == 0 {
return result
}
for i := 0; i < len(iccids); i += batchSize {
end := min(i+batchSize, len(iccids))
batch := iccids[i:end]
for i := 0; i < len(cards); i += batchSize {
end := min(i+batchSize, len(cards))
batch := cards[i:end]
h.processBatch(ctx, task, batch, i+1, result)
}
return result
}
func (h *IotCardImportHandler) getICCIDsFromTask(task *model.IotCardImportTask) []string {
return []string(task.ICCIDList)
// getCardsFromTask 从任务中获取待导入的卡列表
func (h *IotCardImportHandler) getCardsFromTask(task *model.IotCardImportTask) []model.CardItem {
return []model.CardItem(task.CardList)
}
func (h *IotCardImportHandler) processBatch(ctx context.Context, task *model.IotCardImportTask, batch []string, startLine int, result *importResult) {
validICCIDs := make([]string, 0)
lineMap := make(map[string]int)
func (h *IotCardImportHandler) processBatch(ctx context.Context, task *model.IotCardImportTask, batch []model.CardItem, startLine int, result *importResult) {
type cardMeta struct {
line int
msisdn string
}
validCards := make([]model.CardItem, 0)
cardMetaMap := make(map[string]cardMeta)
for i, iccid := range batch {
for i, card := range batch {
line := startLine + i
lineMap[iccid] = line
cardMetaMap[card.ICCID] = cardMeta{line: line, msisdn: card.MSISDN}
validationResult := validator.ValidateICCID(iccid, task.CarrierType)
validationResult := validator.ValidateICCID(card.ICCID, task.CarrierType)
if !validationResult.Valid {
result.failedItems = append(result.failedItems, model.ImportResultItem{
Line: line,
ICCID: iccid,
ICCID: card.ICCID,
MSISDN: card.MSISDN,
Reason: validationResult.Message,
})
result.failCount++
continue
}
validICCIDs = append(validICCIDs, iccid)
validCards = append(validCards, card)
}
if len(validICCIDs) == 0 {
if len(validCards) == 0 {
return
}
validICCIDs := make([]string, len(validCards))
for i, card := range validCards {
validICCIDs[i] = card.ICCID
}
existingMap, err := h.iotCardStore.ExistsByICCIDBatch(ctx, validICCIDs)
if err != nil {
h.logger.Error("批量检查 ICCID 是否存在失败",
zap.Error(err),
zap.Int("batch_size", len(validICCIDs)),
)
for _, iccid := range validICCIDs {
for _, card := range validCards {
meta := cardMetaMap[card.ICCID]
result.failedItems = append(result.failedItems, model.ImportResultItem{
Line: lineMap[iccid],
ICCID: iccid,
Line: meta.line,
ICCID: card.ICCID,
MSISDN: meta.msisdn,
Reason: "数据库查询失败",
})
result.failCount++
@@ -172,29 +254,32 @@ func (h *IotCardImportHandler) processBatch(ctx context.Context, task *model.Iot
return
}
newICCIDs := make([]string, 0)
for _, iccid := range validICCIDs {
if existingMap[iccid] {
newCards := make([]model.CardItem, 0)
for _, card := range validCards {
meta := cardMetaMap[card.ICCID]
if existingMap[card.ICCID] {
result.skippedItems = append(result.skippedItems, model.ImportResultItem{
Line: lineMap[iccid],
ICCID: iccid,
Line: meta.line,
ICCID: card.ICCID,
MSISDN: meta.msisdn,
Reason: "ICCID 已存在",
})
result.skipCount++
} else {
newICCIDs = append(newICCIDs, iccid)
newCards = append(newCards, card)
}
}
if len(newICCIDs) == 0 {
if len(newCards) == 0 {
return
}
cards := make([]*model.IotCard, 0, len(newICCIDs))
iotCards := make([]*model.IotCard, 0, len(newCards))
now := time.Now()
for _, iccid := range newICCIDs {
card := &model.IotCard{
ICCID: iccid,
for _, card := range newCards {
iotCard := &model.IotCard{
ICCID: card.ICCID,
MSISDN: card.MSISDN,
CarrierID: task.CarrierID,
BatchNo: task.BatchNo,
Status: constants.IotCardStatusInStock,
@@ -203,22 +288,24 @@ func (h *IotCardImportHandler) processBatch(ctx context.Context, task *model.Iot
RealNameStatus: constants.RealNameStatusNotVerified,
NetworkStatus: constants.NetworkStatusOffline,
}
card.BaseModel.Creator = task.Creator
card.BaseModel.Updater = task.Creator
card.CreatedAt = now
card.UpdatedAt = now
cards = append(cards, card)
iotCard.BaseModel.Creator = task.Creator
iotCard.BaseModel.Updater = task.Creator
iotCard.CreatedAt = now
iotCard.UpdatedAt = now
iotCards = append(iotCards, iotCard)
}
if err := h.iotCardStore.CreateBatch(ctx, cards); err != nil {
if err := h.iotCardStore.CreateBatch(ctx, iotCards); err != nil {
h.logger.Error("批量创建 IoT 卡失败",
zap.Error(err),
zap.Int("batch_size", len(cards)),
zap.Int("batch_size", len(iotCards)),
)
for _, iccid := range newICCIDs {
for _, card := range newCards {
meta := cardMetaMap[card.ICCID]
result.failedItems = append(result.failedItems, model.ImportResultItem{
Line: lineMap[iccid],
ICCID: iccid,
Line: meta.line,
ICCID: card.ICCID,
MSISDN: meta.msisdn,
Reason: "数据库写入失败",
})
result.failCount++
@@ -226,5 +313,5 @@ func (h *IotCardImportHandler) processBatch(ctx context.Context, task *model.Iot
return
}
result.successCount += len(newICCIDs)
result.successCount += len(newCards)
}

55
internal/task/iot_card_import_test.go
View File

@@ -22,7 +22,7 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
importTaskStore := postgres.NewIotCardImportTaskStore(tx, rdb)
iotCardStore := postgres.NewIotCardStore(tx, rdb)
handler := NewIotCardImportHandler(tx, rdb, importTaskStore, iotCardStore, logger)
handler := NewIotCardImportHandler(tx, rdb, importTaskStore, iotCardStore, nil, logger)
ctx := context.Background()
t.Run("成功导入新ICCID", func(t *testing.T) {
@@ -30,8 +30,12 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
CarrierID: 1,
CarrierType: constants.CarrierCodeCMCC,
BatchNo: "TEST_BATCH_001",
ICCIDList: model.ICCIDListJSON{"89860012345678905001", "89860012345678905002", "89860012345678905003"},
TotalCount: 3,
CardList: model.CardListJSON{
{ICCID: "89860012345678905001", MSISDN: "13800000001"},
{ICCID: "89860012345678905002", MSISDN: "13800000002"},
{ICCID: "89860012345678905003", MSISDN: "13800000003"},
},
TotalCount: 3,
}
task.Creator = 1
@@ -43,6 +47,9 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
exists, _ := iotCardStore.ExistsByICCID(ctx, "89860012345678905001")
assert.True(t, exists)
card, _ := iotCardStore.GetByICCID(ctx, "89860012345678905001")
assert.Equal(t, "13800000001", card.MSISDN)
})
t.Run("跳过已存在的ICCID", func(t *testing.T) {
@@ -58,8 +65,11 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
CarrierID: 1,
CarrierType: constants.CarrierCodeCMCC,
BatchNo: "TEST_BATCH_002",
ICCIDList: model.ICCIDListJSON{"89860012345678906001", "89860012345678906002"},
TotalCount: 2,
CardList: model.CardListJSON{
{ICCID: "89860012345678906001", MSISDN: "13800000011"},
{ICCID: "89860012345678906002", MSISDN: "13800000012"},
},
TotalCount: 2,
}
task.Creator = 1
@@ -70,6 +80,7 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
assert.Equal(t, 0, result.failCount)
assert.Len(t, result.skippedItems, 1)
assert.Equal(t, "89860012345678906001", result.skippedItems[0].ICCID)
assert.Equal(t, "13800000011", result.skippedItems[0].MSISDN)
assert.Equal(t, "ICCID 已存在", result.skippedItems[0].Reason)
})
@@ -78,8 +89,11 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
CarrierID: 1,
CarrierType: constants.CarrierCodeCTCC,
BatchNo: "TEST_BATCH_003",
ICCIDList: model.ICCIDListJSON{"89860312345678907001", "898603123456789070"},
TotalCount: 2,
CardList: model.CardListJSON{
{ICCID: "89860312345678907001", MSISDN: "13900000001"},
{ICCID: "898603123456789070", MSISDN: "13900000002"},
},
TotalCount: 2,
}
task.Creator = 1
@@ -89,6 +103,7 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
assert.Equal(t, 0, result.skipCount)
assert.Equal(t, 2, result.failCount)
assert.Len(t, result.failedItems, 2)
assert.Equal(t, "13900000001", result.failedItems[0].MSISDN)
})
t.Run("混合场景-成功跳过和失败", func(t *testing.T) {
@@ -104,10 +119,10 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
CarrierID: 1,
CarrierType: constants.CarrierCodeCMCC,
BatchNo: "TEST_BATCH_004",
ICCIDList: model.ICCIDListJSON{
"89860012345678908001",
"89860012345678908002",
"invalid!iccid",
CardList: model.CardListJSON{
{ICCID: "89860012345678908001", MSISDN: "13800000021"},
{ICCID: "89860012345678908002", MSISDN: "13800000022"},
{ICCID: "invalid!iccid", MSISDN: "13800000023"},
},
TotalCount: 3,
}
@@ -120,12 +135,12 @@ func TestIotCardImportHandler_ProcessImport(t *testing.T) {
assert.Equal(t, 1, result.failCount)
})
t.Run("空ICCID列表", func(t *testing.T) {
t.Run("空卡列表", func(t *testing.T) {
task := &model.IotCardImportTask{
CarrierID: 1,
CarrierType: constants.CarrierCodeCMCC,
BatchNo: "TEST_BATCH_005",
ICCIDList: model.ICCIDListJSON{},
CardList: model.CardListJSON{},
TotalCount: 0,
}
@@ -146,10 +161,10 @@ func TestIotCardImportHandler_ProcessBatch(t *testing.T) {
importTaskStore := postgres.NewIotCardImportTaskStore(tx, rdb)
iotCardStore := postgres.NewIotCardStore(tx, rdb)
handler := NewIotCardImportHandler(tx, rdb, importTaskStore, iotCardStore, logger)
handler := NewIotCardImportHandler(tx, rdb, importTaskStore, iotCardStore, nil, logger)
ctx := context.Background()
t.Run("验证行号正确记录", func(t *testing.T) {
t.Run("验证行号和MSISDN正确记录", func(t *testing.T) {
existingCard := &model.IotCard{
ICCID: "89860012345678909002",
CardType: "data_card",
@@ -165,10 +180,10 @@ func TestIotCardImportHandler_ProcessBatch(t *testing.T) {
}
task.Creator = 1
batch := []string{
"89860012345678909001",
"89860012345678909002",
"invalid",
batch := []model.CardItem{
{ICCID: "89860012345678909001", MSISDN: "13800000031"},
{ICCID: "89860012345678909002", MSISDN: "13800000032"},
{ICCID: "invalid", MSISDN: "13800000033"},
}
result := &importResult{
skippedItems: make(model.ImportResultItems, 0),
@@ -182,6 +197,8 @@ func TestIotCardImportHandler_ProcessBatch(t *testing.T) {
assert.Equal(t, 1, result.failCount)
assert.Equal(t, 101, result.skippedItems[0].Line)
assert.Equal(t, "13800000032", result.skippedItems[0].MSISDN)
assert.Equal(t, 102, result.failedItems[0].Line)
assert.Equal(t, "13800000033", result.failedItems[0].MSISDN)
})
}