feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口

- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id
- tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情
- 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月
- 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选
- 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选
- 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选
- 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录
- 迁移 000272、000273 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-05 18:47:00 +09:00
parent 8eb7d403c4
commit 94cbf2adab
39 changed files with 1794 additions and 140 deletions

View File

@@ -2,19 +2,12 @@ package postgres
import (
"context"
"fmt"
"strings"
"time"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
"github.com/break/junhong_cmp_fiber/internal/model"
)
// collectionPartitionPrefix 是采集记录月分区表名前缀,后接 YYYYMM。
const collectionPartitionPrefix = "tb_card_traffic_collection_"
// CardTrafficCollectionStore 卡流量采集记录存储。
type CardTrafficCollectionStore struct {
db *gorm.DB
@@ -30,68 +23,17 @@ func (s *CardTrafficCollectionStore) Create(ctx context.Context, record *model.C
return s.db.WithContext(ctx).Create(record).Error
}
// CardTrafficCollectionQuery 是采集记录查询条件。
type CardTrafficCollectionQuery struct {
CardID uint
From, To time.Time
Page int
PageSize int
// CardTrafficCollectionFailureStore 卡流量采集失败记录存储。
type CardTrafficCollectionFailureStore struct {
db *gorm.DB
}
// List 按卡与采集时间范围分页查询采集记录,按采集时间倒序。
func (s *CardTrafficCollectionStore) List(ctx context.Context, query CardTrafficCollectionQuery) ([]model.CardTrafficCollection, int64, error) {
db := s.db.WithContext(ctx).Model(&model.CardTrafficCollection{}).
Where("iot_card_id = ? AND observed_at >= ? AND observed_at < ?", query.CardID, query.From, query.To)
var total int64
if err := db.Count(&total).Error; err != nil {
return nil, 0, err
}
var rows []model.CardTrafficCollection
err := db.Order("observed_at DESC, id DESC").Offset((query.Page - 1) * query.PageSize).Limit(query.PageSize).Find(&rows).Error
return rows, total, err
// NewCardTrafficCollectionFailureStore 创建采集失败记录存储。
func NewCardTrafficCollectionFailureStore(db *gorm.DB) *CardTrafficCollectionFailureStore {
return &CardTrafficCollectionFailureStore{db: db}
}
// FindByObservationID 按采集 ID 查询采集记录。
func (s *CardTrafficCollectionStore) FindByObservationID(ctx context.Context, observationID string) ([]model.CardTrafficCollection, error) {
var rows []model.CardTrafficCollection
err := s.db.WithContext(ctx).Where("observation_id = ?", observationID).Order("observed_at DESC").Find(&rows).Error
return rows, err
}
// MaintainPartitions 预建本月起 aheadMonths 个月的分区,并删除整月都早于保留期的分区。
func (s *CardTrafficCollectionStore) MaintainPartitions(ctx context.Context, now time.Time, aheadMonths int, retentionDays int) error {
monthStart := cardflow.MonthStart(now)
for i := 0; i <= aheadMonths; i++ {
from := monthStart.AddDate(0, i, 0)
to := from.AddDate(0, 1, 0)
sql := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s%s PARTITION OF tb_card_traffic_collection FOR VALUES FROM ('%s') TO ('%s')`,
collectionPartitionPrefix, from.Format("200601"), from.Format(time.RFC3339), to.Format(time.RFC3339))
if err := s.db.WithContext(ctx).Exec(sql).Error; err != nil {
return err
}
}
cutoff := cardflow.ShanghaiDate(now).AddDate(0, 0, -retentionDays)
var partitions []string
if err := s.db.WithContext(ctx).Raw(`
SELECT c.relname FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
WHERE p.relname = 'tb_card_traffic_collection'`).Scan(&partitions).Error; err != nil {
return err
}
for _, name := range partitions {
if len(name) != len(collectionPartitionPrefix)+6 || !strings.HasPrefix(name, collectionPartitionPrefix) {
continue
}
start, err := time.ParseInLocation("200601", name[len(collectionPartitionPrefix):], cutoff.Location())
if err != nil {
continue
}
if !start.AddDate(0, 1, 0).After(cutoff) {
if err := s.db.WithContext(ctx).Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", name)).Error; err != nil {
return err
}
}
}
return nil
// Create 写入一条采集失败记录。
func (s *CardTrafficCollectionFailureStore) Create(ctx context.Context, record *model.CardTrafficCollectionFailure) error {
return s.db.WithContext(ctx).Create(record).Error
}

View File

@@ -186,7 +186,11 @@ func NewCardStopResumeRecordStore(db *gorm.DB) *CardStopResumeRecordStore {
// StopResumeRecordFilter 是停复机记录列表筛选条件;时间范围由调用方保证必填且有上限。
type StopResumeRecordFilter struct {
CardID *uint
CardIDs []uint // 资产标识解析出的卡;非 nil 时按其过滤(空切片表示匹配不到任何卡)
ICCID string
Scene string
Reason string
CommandID string
CarrierID *uint
Action string
Source string
@@ -210,9 +214,24 @@ func (s *CardStopResumeRecordStore) List(ctx context.Context, filter StopResumeR
if filter.CardID != nil {
query = query.Where("card_id = ?", *filter.CardID)
}
if filter.CardIDs != nil {
if len(filter.CardIDs) == 0 {
return []model.CardStopResumeRecord{}, 0, nil
}
query = query.Where("card_id IN ?", filter.CardIDs)
}
if filter.ICCID != "" {
query = query.Where("iccid = ?", filter.ICCID)
}
if filter.Scene != "" {
query = query.Where("scene = ?", filter.Scene)
}
if filter.Reason != "" {
query = query.Where("reason = ?", filter.Reason)
}
if filter.CommandID != "" {
query = query.Where("command_id = ?", filter.CommandID)
}
if filter.CarrierID != nil {
query = query.Where("carrier_id = ?", *filter.CarrierID)
}

View File

@@ -16,6 +16,7 @@ type RefundListFilters struct {
OrderID *uint `json:"order_id"`
ShopID *uint `json:"shop_id"`
AssetIdentifier string `json:"asset_identifier"`
AssetPostStatus *int `json:"asset_post_status"`
}
// RefundStore 退款申请数据访问层
@@ -100,6 +101,9 @@ func (s *RefundStore) List(ctx context.Context, opts *store.QueryOptions, filter
if filters.AssetIdentifier != "" {
query = query.Where("tb_refund_request.asset_identifier = ?", filters.AssetIdentifier)
}
if filters.AssetPostStatus != nil {
query = query.Where("tb_refund_request.asset_post_status = ?", *filters.AssetPostStatus)
}
}
if err := query.Count(&total).Error; err != nil {