feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口
- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id - tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情 - 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月 - 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选 - 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选 - 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选 - 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录 - 迁移 000272、000273 需维护者在测试库执行后再部署 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,19 +2,12 @@ package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
|
||||
"github.com/break/junhong_cmp_fiber/internal/model"
|
||||
)
|
||||
|
||||
// collectionPartitionPrefix 是采集记录月分区表名前缀,后接 YYYYMM。
|
||||
const collectionPartitionPrefix = "tb_card_traffic_collection_"
|
||||
|
||||
// CardTrafficCollectionStore 卡流量采集记录存储。
|
||||
type CardTrafficCollectionStore struct {
|
||||
db *gorm.DB
|
||||
@@ -30,68 +23,17 @@ func (s *CardTrafficCollectionStore) Create(ctx context.Context, record *model.C
|
||||
return s.db.WithContext(ctx).Create(record).Error
|
||||
}
|
||||
|
||||
// CardTrafficCollectionQuery 是采集记录查询条件。
|
||||
type CardTrafficCollectionQuery struct {
|
||||
CardID uint
|
||||
From, To time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
// CardTrafficCollectionFailureStore 卡流量采集失败记录存储。
|
||||
type CardTrafficCollectionFailureStore struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// List 按卡与采集时间范围分页查询采集记录,按采集时间倒序。
|
||||
func (s *CardTrafficCollectionStore) List(ctx context.Context, query CardTrafficCollectionQuery) ([]model.CardTrafficCollection, int64, error) {
|
||||
db := s.db.WithContext(ctx).Model(&model.CardTrafficCollection{}).
|
||||
Where("iot_card_id = ? AND observed_at >= ? AND observed_at < ?", query.CardID, query.From, query.To)
|
||||
var total int64
|
||||
if err := db.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var rows []model.CardTrafficCollection
|
||||
err := db.Order("observed_at DESC, id DESC").Offset((query.Page - 1) * query.PageSize).Limit(query.PageSize).Find(&rows).Error
|
||||
return rows, total, err
|
||||
// NewCardTrafficCollectionFailureStore 创建采集失败记录存储。
|
||||
func NewCardTrafficCollectionFailureStore(db *gorm.DB) *CardTrafficCollectionFailureStore {
|
||||
return &CardTrafficCollectionFailureStore{db: db}
|
||||
}
|
||||
|
||||
// FindByObservationID 按采集 ID 查询采集记录。
|
||||
func (s *CardTrafficCollectionStore) FindByObservationID(ctx context.Context, observationID string) ([]model.CardTrafficCollection, error) {
|
||||
var rows []model.CardTrafficCollection
|
||||
err := s.db.WithContext(ctx).Where("observation_id = ?", observationID).Order("observed_at DESC").Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// MaintainPartitions 预建本月起 aheadMonths 个月的分区,并删除整月都早于保留期的分区。
|
||||
func (s *CardTrafficCollectionStore) MaintainPartitions(ctx context.Context, now time.Time, aheadMonths int, retentionDays int) error {
|
||||
monthStart := cardflow.MonthStart(now)
|
||||
for i := 0; i <= aheadMonths; i++ {
|
||||
from := monthStart.AddDate(0, i, 0)
|
||||
to := from.AddDate(0, 1, 0)
|
||||
sql := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s%s PARTITION OF tb_card_traffic_collection FOR VALUES FROM ('%s') TO ('%s')`,
|
||||
collectionPartitionPrefix, from.Format("200601"), from.Format(time.RFC3339), to.Format(time.RFC3339))
|
||||
if err := s.db.WithContext(ctx).Exec(sql).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
cutoff := cardflow.ShanghaiDate(now).AddDate(0, 0, -retentionDays)
|
||||
var partitions []string
|
||||
if err := s.db.WithContext(ctx).Raw(`
|
||||
SELECT c.relname FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_class p ON p.oid = i.inhparent
|
||||
WHERE p.relname = 'tb_card_traffic_collection'`).Scan(&partitions).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range partitions {
|
||||
if len(name) != len(collectionPartitionPrefix)+6 || !strings.HasPrefix(name, collectionPartitionPrefix) {
|
||||
continue
|
||||
}
|
||||
start, err := time.ParseInLocation("200601", name[len(collectionPartitionPrefix):], cutoff.Location())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !start.AddDate(0, 1, 0).After(cutoff) {
|
||||
if err := s.db.WithContext(ctx).Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", name)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
// Create 写入一条采集失败记录。
|
||||
func (s *CardTrafficCollectionFailureStore) Create(ctx context.Context, record *model.CardTrafficCollectionFailure) error {
|
||||
return s.db.WithContext(ctx).Create(record).Error
|
||||
}
|
||||
|
||||
@@ -186,7 +186,11 @@ func NewCardStopResumeRecordStore(db *gorm.DB) *CardStopResumeRecordStore {
|
||||
// StopResumeRecordFilter 是停复机记录列表筛选条件;时间范围由调用方保证必填且有上限。
|
||||
type StopResumeRecordFilter struct {
|
||||
CardID *uint
|
||||
CardIDs []uint // 资产标识解析出的卡;非 nil 时按其过滤(空切片表示匹配不到任何卡)
|
||||
ICCID string
|
||||
Scene string
|
||||
Reason string
|
||||
CommandID string
|
||||
CarrierID *uint
|
||||
Action string
|
||||
Source string
|
||||
@@ -210,9 +214,24 @@ func (s *CardStopResumeRecordStore) List(ctx context.Context, filter StopResumeR
|
||||
if filter.CardID != nil {
|
||||
query = query.Where("card_id = ?", *filter.CardID)
|
||||
}
|
||||
if filter.CardIDs != nil {
|
||||
if len(filter.CardIDs) == 0 {
|
||||
return []model.CardStopResumeRecord{}, 0, nil
|
||||
}
|
||||
query = query.Where("card_id IN ?", filter.CardIDs)
|
||||
}
|
||||
if filter.ICCID != "" {
|
||||
query = query.Where("iccid = ?", filter.ICCID)
|
||||
}
|
||||
if filter.Scene != "" {
|
||||
query = query.Where("scene = ?", filter.Scene)
|
||||
}
|
||||
if filter.Reason != "" {
|
||||
query = query.Where("reason = ?", filter.Reason)
|
||||
}
|
||||
if filter.CommandID != "" {
|
||||
query = query.Where("command_id = ?", filter.CommandID)
|
||||
}
|
||||
if filter.CarrierID != nil {
|
||||
query = query.Where("carrier_id = ?", *filter.CarrierID)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ type RefundListFilters struct {
|
||||
OrderID *uint `json:"order_id"`
|
||||
ShopID *uint `json:"shop_id"`
|
||||
AssetIdentifier string `json:"asset_identifier"`
|
||||
AssetPostStatus *int `json:"asset_post_status"`
|
||||
}
|
||||
|
||||
// RefundStore 退款申请数据访问层
|
||||
@@ -100,6 +101,9 @@ func (s *RefundStore) List(ctx context.Context, opts *store.QueryOptions, filter
|
||||
if filters.AssetIdentifier != "" {
|
||||
query = query.Where("tb_refund_request.asset_identifier = ?", filters.AssetIdentifier)
|
||||
}
|
||||
if filters.AssetPostStatus != nil {
|
||||
query = query.Where("tb_refund_request.asset_post_status = ?", *filters.AssetPostStatus)
|
||||
}
|
||||
}
|
||||
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
|
||||
Reference in New Issue
Block a user