feat: 实现 IoT 卡轮询系统（支持千万级卡规模）

实现功能： - 实名状态检查轮询（可配置间隔） - 卡流量检查轮询（支持跨月流量追踪） - 套餐检查与超额自动停机 - 分布式并发控制（Redis 信号量） - 手动触发轮询（单卡/批量/条件筛选） - 数据清理配置与执行 - 告警规则与历史记录 - 实时监控统计（队列/性能/并发）性能优化： - Redis 缓存卡信息，减少 DB 查询 - Pipeline 批量写入 Redis - 异步流量记录写入 - 渐进式初始化（10万卡/批）压测工具（scripts/benchmark/）： - Mock Gateway 模拟上游服务 - 测试卡生成器 - 配置初始化脚本 - 实时监控脚本 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-05 17:32:44 +08:00
parent b11edde720
commit 931e140e8e
104 changed files with 16883 additions and 87 deletions
--- a/internal/handler/admin/polling_cleanup.go
+++ b/internal/handler/admin/polling_cleanup.go
@@ -0,0 +1,351 @@
+package admin
+
+import (
+	"github.com/gofiber/fiber/v2"
+
+	"github.com/break/junhong_cmp_fiber/pkg/middleware"
+	"github.com/break/junhong_cmp_fiber/internal/model"
+	"github.com/break/junhong_cmp_fiber/internal/model/dto"
+	"github.com/break/junhong_cmp_fiber/internal/service/polling"
+	"github.com/break/junhong_cmp_fiber/pkg/errors"
+	"github.com/break/junhong_cmp_fiber/pkg/response"
+)
+
+// PollingCleanupHandler 轮询数据清理处理器
+type PollingCleanupHandler struct {
+	service *polling.CleanupService
+}
+
+// NewPollingCleanupHandler 创建轮询数据清理处理器
+func NewPollingCleanupHandler(service *polling.CleanupService) *PollingCleanupHandler {
+	return &PollingCleanupHandler{service: service}
+}
+
+// CreateConfig 创建清理配置
+// @Summary 创建数据清理配置
+// @Description 创建新的数据清理配置
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Param request body dto.CreateDataCleanupConfigReq true "创建请求"
+// @Success 200 {object} response.Response{data=dto.DataCleanupConfigResp}
+// @Router /api/admin/data-cleanup-configs [post]
+func (h *PollingCleanupHandler) CreateConfig(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+
+	var req dto.CreateDataCleanupConfigReq
+	if err := c.BodyParser(&req); err != nil {
+		return errors.New(errors.CodeInvalidParam)
+	}
+
+	config := &model.DataCleanupConfig{
+		TargetTable:   req.TargetTable,
+		RetentionDays: req.RetentionDays,
+		BatchSize:     req.BatchSize,
+		Description:   req.Description,
+	}
+
+	if err := h.service.CreateConfig(ctx, config); err != nil {
+		return err
+	}
+
+	return response.Success(c, h.toConfigResp(config))
+}
+
+// ListConfigs 获取清理配置列表
+// @Summary 获取数据清理配置列表
+// @Description 获取所有数据清理配置
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Success 200 {object} response.Response{data=dto.DataCleanupConfigListResp}
+// @Router /api/admin/data-cleanup-configs [get]
+func (h *PollingCleanupHandler) ListConfigs(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+
+	configs, err := h.service.ListConfigs(ctx)
+	if err != nil {
+		return err
+	}
+
+	items := make([]*dto.DataCleanupConfigResp, 0, len(configs))
+	for _, config := range configs {
+		items = append(items, h.toConfigResp(config))
+	}
+
+	return response.Success(c, &dto.DataCleanupConfigListResp{Items: items})
+}
+
+// GetConfig 获取清理配置详情
+// @Summary 获取数据清理配置详情
+// @Description 获取指定数据清理配置的详细信息
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Param id path int true "配置ID"
+// @Success 200 {object} response.Response{data=dto.DataCleanupConfigResp}
+// @Router /api/admin/data-cleanup-configs/{id} [get]
+func (h *PollingCleanupHandler) GetConfig(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+	id, err := c.ParamsInt("id")
+	if err != nil || id <= 0 {
+		return errors.New(errors.CodeInvalidParam, "无效的配置ID")
+	}
+
+	config, err := h.service.GetConfig(ctx, uint(id))
+	if err != nil {
+		return err
+	}
+
+	return response.Success(c, h.toConfigResp(config))
+}
+
+// UpdateConfig 更新清理配置
+// @Summary 更新数据清理配置
+// @Description 更新指定数据清理配置
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Param id path int true "配置ID"
+// @Param request body dto.UpdateDataCleanupConfigReq true "更新请求"
+// @Success 200 {object} response.Response
+// @Router /api/admin/data-cleanup-configs/{id} [put]
+func (h *PollingCleanupHandler) UpdateConfig(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+	id, err := c.ParamsInt("id")
+	if err != nil || id <= 0 {
+		return errors.New(errors.CodeInvalidParam, "无效的配置ID")
+	}
+
+	var req dto.UpdateDataCleanupConfigReq
+	if err := c.BodyParser(&req); err != nil {
+		return errors.New(errors.CodeInvalidParam)
+	}
+
+	updates := make(map[string]any)
+	if req.RetentionDays != nil {
+		updates["retention_days"] = *req.RetentionDays
+	}
+	if req.BatchSize != nil {
+		updates["batch_size"] = *req.BatchSize
+	}
+	if req.Enabled != nil {
+		updates["enabled"] = *req.Enabled
+	}
+	if req.Description != nil {
+		updates["description"] = *req.Description
+	}
+
+	// 获取当前用户ID
+	userID := middleware.GetUserIDFromContext(ctx)
+	if userID > 0 {
+		updates["updated_by"] = userID
+	}
+
+	if err := h.service.UpdateConfig(ctx, uint(id), updates); err != nil {
+		return err
+	}
+
+	return response.Success(c, nil)
+}
+
+// DeleteConfig 删除清理配置
+// @Summary 删除数据清理配置
+// @Description 删除指定数据清理配置
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Param id path int true "配置ID"
+// @Success 200 {object} response.Response
+// @Router /api/admin/data-cleanup-configs/{id} [delete]
+func (h *PollingCleanupHandler) DeleteConfig(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+	id, err := c.ParamsInt("id")
+	if err != nil || id <= 0 {
+		return errors.New(errors.CodeInvalidParam, "无效的配置ID")
+	}
+
+	if err := h.service.DeleteConfig(ctx, uint(id)); err != nil {
+		return err
+	}
+
+	return response.Success(c, nil)
+}
+
+// ListLogs 获取清理日志列表
+// @Summary 获取数据清理日志列表
+// @Description 获取数据清理日志记录
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Param table_name query string false "表名筛选"
+// @Param page query int false "页码"
+// @Param page_size query int false "每页数量"
+// @Success 200 {object} response.Response{data=dto.DataCleanupLogListResp}
+// @Router /api/admin/data-cleanup-logs [get]
+func (h *PollingCleanupHandler) ListLogs(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+
+	var req dto.ListDataCleanupLogReq
+	if err := c.QueryParser(&req); err != nil {
+		return errors.New(errors.CodeInvalidParam)
+	}
+
+	if req.Page < 1 {
+		req.Page = 1
+	}
+	if req.PageSize < 1 || req.PageSize > 100 {
+		req.PageSize = 20
+	}
+
+	logs, total, err := h.service.ListLogs(ctx, req.Page, req.PageSize, req.TableName)
+	if err != nil {
+		return err
+	}
+
+	items := make([]*dto.DataCleanupLogResp, 0, len(logs))
+	for _, log := range logs {
+		items = append(items, h.toLogResp(log))
+	}
+
+	totalPages := int(total) / req.PageSize
+	if int(total)%req.PageSize > 0 {
+		totalPages++
+	}
+
+	return response.Success(c, &dto.DataCleanupLogListResp{
+		Items:      items,
+		Total:      total,
+		Page:       req.Page,
+		PageSize:   req.PageSize,
+		TotalPages: totalPages,
+	})
+}
+
+// Preview 预览待清理数据
+// @Summary 预览待清理数据
+// @Description 预览各表待清理的数据量
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Success 200 {object} response.Response{data=dto.DataCleanupPreviewResp}
+// @Router /api/admin/data-cleanup/preview [get]
+func (h *PollingCleanupHandler) Preview(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+
+	previews, err := h.service.Preview(ctx)
+	if err != nil {
+		return err
+	}
+
+	items := make([]*dto.DataCleanupPreviewItem, 0, len(previews))
+	for _, p := range previews {
+		items = append(items, &dto.DataCleanupPreviewItem{
+			TableName:     p.TableName,
+			RetentionDays: p.RetentionDays,
+			RecordCount:   p.RecordCount,
+			Description:   p.Description,
+		})
+	}
+
+	return response.Success(c, &dto.DataCleanupPreviewResp{Items: items})
+}
+
+// GetProgress 获取清理进度
+// @Summary 获取数据清理进度
+// @Description 获取当前数据清理任务的进度
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Success 200 {object} response.Response{data=dto.DataCleanupProgressResp}
+// @Router /api/admin/data-cleanup/progress [get]
+func (h *PollingCleanupHandler) GetProgress(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+
+	progress, err := h.service.GetProgress(ctx)
+	if err != nil {
+		return err
+	}
+
+	resp := &dto.DataCleanupProgressResp{
+		IsRunning:       progress.IsRunning,
+		CurrentTable:    progress.CurrentTable,
+		TotalTables:     progress.TotalTables,
+		ProcessedTables: progress.ProcessedTables,
+		TotalDeleted:    progress.TotalDeleted,
+		StartedAt:       progress.StartedAt,
+	}
+
+	if progress.LastLog != nil {
+		resp.LastLog = h.toLogResp(progress.LastLog)
+	}
+
+	return response.Success(c, resp)
+}
+
+// TriggerCleanup 手动触发清理
+// @Summary 手动触发数据清理
+// @Description 手动触发数据清理任务
+// @Tags 轮询管理-数据清理
+// @Accept json
+// @Produce json
+// @Security BearerAuth
+// @Param request body dto.TriggerDataCleanupReq true "触发请求"
+// @Success 200 {object} response.Response
+// @Router /api/admin/data-cleanup/trigger [post]
+func (h *PollingCleanupHandler) TriggerCleanup(c *fiber.Ctx) error {
+	ctx := c.UserContext()
+
+	var req dto.TriggerDataCleanupReq
+	if err := c.BodyParser(&req); err != nil {
+		return errors.New(errors.CodeInvalidParam)
+	}
+
+	// 获取当前用户ID
+	userID := middleware.GetUserIDFromContext(ctx)
+
+	// 异步执行清理
+	go func() {
+		_ = h.service.TriggerCleanup(ctx, req.TableName, userID)
+	}()
+
+	return response.Success(c, nil)
+}
+
+func (h *PollingCleanupHandler) toConfigResp(config *model.DataCleanupConfig) *dto.DataCleanupConfigResp {
+	return &dto.DataCleanupConfigResp{
+		ID:            config.ID,
+		TargetTable:   config.TargetTable,
+		RetentionDays: config.RetentionDays,
+		BatchSize:     config.BatchSize,
+		Enabled:       int(config.Enabled),
+		Description:   config.Description,
+		CreatedAt:     config.CreatedAt,
+		UpdatedAt:     config.UpdatedAt,
+		UpdatedBy:     config.UpdatedBy,
+	}
+}
+
+func (h *PollingCleanupHandler) toLogResp(log *model.DataCleanupLog) *dto.DataCleanupLogResp {
+	return &dto.DataCleanupLogResp{
+		ID:            log.ID,
+		TargetTable:   log.TargetTable,
+		CleanupType:   log.CleanupType,
+		RetentionDays: log.RetentionDays,
+		DeletedCount:  log.DeletedCount,
+		DurationMs:    log.DurationMs,
+		Status:        log.Status,
+		ErrorMessage:  log.ErrorMessage,
+		StartedAt:     log.StartedAt,
+		CompletedAt:   log.CompletedAt,
+		TriggeredBy:   log.TriggeredBy,
+	}
+}