feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口

- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id
- tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情
- 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月
- 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选
- 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选
- 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选
- 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录
- 迁移 000272、000273 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-05 18:47:00 +09:00
parent 8eb7d403c4
commit 94cbf2adab
39 changed files with 1794 additions and 140 deletions

View File

@@ -0,0 +1,313 @@
package cardobservation
import (
"compress/gzip"
"context"
"crypto/sha256"
"database/sql"
"encoding/csv"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5/stdlib"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// collectionArchiveTables 是按月分区、到期归档的采集记录父表。
var collectionArchiveTables = []string{"tb_card_traffic_collection", "tb_card_traffic_collection_failure"}
// CollectionArchiver 维护采集记录月分区:预建未来分区;到期分区按"摘出 → 导出 csv.gz → 校验 → 删除"归档。
// 导出或校验失败时保留已摘出的表,下一轮重试,不会出现数据已删但没有压缩包的情况。
type CollectionArchiver struct {
db *gorm.DB
dir string
keepFiles int
aheadMonth int
logger *zap.Logger
}
// NewCollectionArchiver 创建采集记录归档器;dir 为压缩包目录,keepFiles 为每张表本地保留的压缩包个数。
func NewCollectionArchiver(db *gorm.DB, dir string, keepFiles int, logger *zap.Logger) *CollectionArchiver {
if logger == nil {
logger = zap.NewNop()
}
if keepFiles <= 0 {
keepFiles = constants.TrafficCollectionArchiveKeepFiles
}
return &CollectionArchiver{db: db, dir: dir, keepFiles: keepFiles, aheadMonth: 2, logger: logger}
}
// archiveManifest 是压缩包同名说明文件内容。
type archiveManifest struct {
Table string `json:"table"`
Partition string `json:"partition"`
Month string `json:"month"`
Rows int64 `json:"rows"`
FirstObservedAt *string `json:"first_observed_at"`
LastObservedAt *string `json:"last_observed_at"`
File string `json:"file"`
FileBytes int64 `json:"file_bytes"`
SHA256 string `json:"sha256"`
ExportedAt time.Time `json:"exported_at"`
}
// Maintain 对全部采集记录表执行一轮分区维护;单张表失败不影响其他表,返回第一个错误。
func (a *CollectionArchiver) Maintain(ctx context.Context, now time.Time) error {
var firstErr error
for _, table := range collectionArchiveTables {
if err := a.maintainTable(ctx, table, now); err != nil {
a.logger.Error("维护采集记录分区失败", zap.String("table", table), zap.Error(err))
if firstErr == nil {
firstErr = err
}
}
}
return firstErr
}
func (a *CollectionArchiver) maintainTable(ctx context.Context, table string, now time.Time) error {
monthStart := cardflow.MonthStart(now)
for i := 0; i <= a.aheadMonth; i++ {
from := monthStart.AddDate(0, i, 0)
sqlText := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`,
table, from.Format("200601"), table, from.Format(time.RFC3339), from.AddDate(0, 1, 0).Format(time.RFC3339))
if err := a.db.WithContext(ctx).Exec(sqlText).Error; err != nil {
return fmt.Errorf("预建分区失败: %w", err)
}
}
// 在线保留当月加最近 N 个整月,更早的月分区到期。
cutoff := monthStart.AddDate(0, -constants.TrafficCollectionRetentionMonths, 0)
partitions, err := a.monthPartitions(ctx, table)
if err != nil {
return err
}
for _, partition := range partitions {
if !partition.month.Before(cutoff) {
continue
}
if err := a.archivePartition(ctx, table, partition); err != nil {
return fmt.Errorf("归档分区 %s 失败: %w", partition.name, err)
}
}
a.cleanupLocalFiles(table)
return nil
}
type monthPartition struct {
name string
month time.Time
attached bool
}
// monthPartitions 返回该表全部按月命名的分区(含上一轮已摘出未删除的表),按月份升序。
func (a *CollectionArchiver) monthPartitions(ctx context.Context, table string) ([]monthPartition, error) {
var rows []struct {
Relname string
Attached bool
}
err := a.db.WithContext(ctx).Raw(`
SELECT c.relname, EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent
WHERE i.inhrelid = c.oid AND p.relname = ?) AS attached
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND c.relname ~ ?`,
table, "^"+table+`_[0-9]{6}$`).Scan(&rows).Error
if err != nil {
return nil, fmt.Errorf("查询分区失败: %w", err)
}
partitions := make([]monthPartition, 0, len(rows))
for _, row := range rows {
month, err := time.ParseInLocation("200601", row.Relname[len(table)+1:], cardflow.MonthStart(time.Now()).Location())
if err != nil {
continue
}
partitions = append(partitions, monthPartition{name: row.Relname, month: month, attached: row.Attached})
}
sort.Slice(partitions, func(i, j int) bool { return partitions[i].month.Before(partitions[j].month) })
return partitions, nil
}
func (a *CollectionArchiver) archivePartition(ctx context.Context, table string, partition monthPartition) error {
if partition.attached {
if err := a.db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", table, partition.name)).Error; err != nil {
return fmt.Errorf("摘出分区失败: %w", err)
}
}
if err := os.MkdirAll(a.dir, 0o755); err != nil {
return fmt.Errorf("创建归档目录失败: %w", err)
}
finalPath := filepath.Join(a.dir, partition.name+".csv.gz")
tmpPath := finalPath + ".tmp"
if err := a.exportCSV(ctx, partition.name, tmpPath); err != nil {
_ = os.Remove(tmpPath)
return err
}
manifest, err := a.verify(ctx, table, partition, tmpPath)
if err != nil {
_ = os.Remove(tmpPath)
return err
}
manifest.File = filepath.Base(finalPath)
if err := writeManifest(finalPath+".manifest.json", manifest); err != nil {
_ = os.Remove(tmpPath)
return err
}
if err := os.Rename(tmpPath, finalPath); err != nil {
return fmt.Errorf("压缩包改名失败: %w", err)
}
if err := a.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + partition.name).Error; err != nil {
return fmt.Errorf("删除已归档分区失败: %w", err)
}
a.logger.Info("采集记录分区已归档", zap.String("partition", partition.name), zap.Int64("rows", manifest.Rows),
zap.String("file", finalPath), zap.Int64("file_bytes", manifest.FileBytes))
return nil
}
// exportCSV 用 COPY 把分区按采集时间与 ID 排序流式导出为带表头的 CSV,并边写边 gzip。
func (a *CollectionArchiver) exportCSV(ctx context.Context, partition, path string) error {
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("创建压缩包失败: %w", err)
}
defer file.Close()
gz := gzip.NewWriter(file)
sqlDB, err := a.db.DB()
if err != nil {
return fmt.Errorf("获取数据库连接失败: %w", err)
}
conn, err := sqlDB.Conn(ctx)
if err != nil {
return fmt.Errorf("获取数据库连接失败: %w", err)
}
defer conn.Close()
copySQL := fmt.Sprintf("COPY (SELECT * FROM %s ORDER BY observed_at, id) TO STDOUT WITH (FORMAT csv, HEADER)", partition)
if err := rawCopyTo(ctx, conn, gz, copySQL); err != nil {
return fmt.Errorf("导出分区失败: %w", err)
}
if err := gz.Close(); err != nil {
return fmt.Errorf("写压缩包失败: %w", err)
}
if err := file.Sync(); err != nil {
return fmt.Errorf("落盘压缩包失败: %w", err)
}
return nil
}
func rawCopyTo(ctx context.Context, conn *sql.Conn, w io.Writer, copySQL string) error {
return conn.Raw(func(driverConn any) error {
stdConn, ok := driverConn.(*stdlib.Conn)
if !ok {
return fmt.Errorf("数据库驱动不支持 COPY 导出")
}
_, err := stdConn.Conn().PgConn().CopyTo(ctx, w, copySQL)
return err
})
}
// verify 重新读取压缩包统计行数并与分区行数比对,生成说明文件内容。
func (a *CollectionArchiver) verify(ctx context.Context, table string, partition monthPartition, path string) (archiveManifest, error) {
var stats struct {
Rows int64
First *time.Time
Last *time.Time
}
if err := a.db.WithContext(ctx).Raw(fmt.Sprintf(
"SELECT count(*) AS rows, min(observed_at) AS first, max(observed_at) AS last FROM %s", partition.name)).Scan(&stats).Error; err != nil {
return archiveManifest{}, fmt.Errorf("统计分区行数失败: %w", err)
}
fileRows, sum, size, err := countArchiveRows(path)
if err != nil {
return archiveManifest{}, err
}
if fileRows != stats.Rows {
return archiveManifest{}, fmt.Errorf("压缩包行数 %d 与分区行数 %d 不一致", fileRows, stats.Rows)
}
manifest := archiveManifest{
Table: table, Partition: partition.name, Month: partition.month.Format("2006-01"), Rows: stats.Rows,
FileBytes: size, SHA256: sum, ExportedAt: time.Now(),
}
if stats.First != nil {
first, last := stats.First.Format(time.RFC3339), stats.Last.Format(time.RFC3339)
manifest.FirstObservedAt, manifest.LastObservedAt = &first, &last
}
return manifest, nil
}
// countArchiveRows 解压并按 CSV 解析压缩包,返回数据行数(不含表头)、文件 SHA-256 与文件大小。
func countArchiveRows(path string) (int64, string, int64, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", 0, fmt.Errorf("打开压缩包失败: %w", err)
}
defer file.Close()
hash := sha256.New()
gz, err := gzip.NewReader(io.TeeReader(file, hash))
if err != nil {
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
}
reader := csv.NewReader(gz)
reader.FieldsPerRecord = -1
reader.ReuseRecord = true
var records int64
for {
if _, err := reader.Read(); err == io.EOF {
break
} else if err != nil {
return 0, "", 0, fmt.Errorf("解析压缩包失败: %w", err)
}
records++
}
if _, err := io.Copy(io.Discard, gz); err != nil {
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
}
info, err := file.Stat()
if err != nil {
return 0, "", 0, fmt.Errorf("读取压缩包大小失败: %w", err)
}
if records > 0 {
records-- // 表头
}
return records, hex.EncodeToString(hash.Sum(nil)), info.Size(), nil
}
func writeManifest(path string, manifest archiveManifest) error {
data, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("生成说明文件失败: %w", err)
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil {
return fmt.Errorf("写说明文件失败: %w", err)
}
if err := os.Rename(tmp, path); err != nil {
return fmt.Errorf("说明文件改名失败: %w", err)
}
return nil
}
// cleanupLocalFiles 每张表只保留最近 keepFiles 个月的压缩包与说明文件。
func (a *CollectionArchiver) cleanupLocalFiles(table string) {
matches, err := filepath.Glob(filepath.Join(a.dir, table+"_[0-9][0-9][0-9][0-9][0-9][0-9].csv.gz"))
if err != nil || len(matches) <= a.keepFiles {
return
}
sort.Strings(matches)
for _, path := range matches[:len(matches)-a.keepFiles] {
if err := os.Remove(path); err != nil {
a.logger.Warn("删除过期采集记录压缩包失败", zap.String("file", path), zap.Error(err))
continue
}
_ = os.Remove(path + ".manifest.json")
a.logger.Info("已删除过期采集记录压缩包", zap.String("file", strings.TrimPrefix(path, a.dir+"/")))
}
}