feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口
- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id - tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情 - 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月 - 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选 - 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选 - 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选 - 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录 - 迁移 000272、000273 需维护者在测试库执行后再部署 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
313
internal/infrastructure/cardobservation/collection_archive.go
Normal file
313
internal/infrastructure/cardobservation/collection_archive.go
Normal file
@@ -0,0 +1,313 @@
|
||||
package cardobservation
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
|
||||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||||
)
|
||||
|
||||
// collectionArchiveTables 是按月分区、到期归档的采集记录父表。
|
||||
var collectionArchiveTables = []string{"tb_card_traffic_collection", "tb_card_traffic_collection_failure"}
|
||||
|
||||
// CollectionArchiver 维护采集记录月分区:预建未来分区;到期分区按"摘出 → 导出 csv.gz → 校验 → 删除"归档。
|
||||
// 导出或校验失败时保留已摘出的表,下一轮重试,不会出现数据已删但没有压缩包的情况。
|
||||
type CollectionArchiver struct {
|
||||
db *gorm.DB
|
||||
dir string
|
||||
keepFiles int
|
||||
aheadMonth int
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewCollectionArchiver 创建采集记录归档器;dir 为压缩包目录,keepFiles 为每张表本地保留的压缩包个数。
|
||||
func NewCollectionArchiver(db *gorm.DB, dir string, keepFiles int, logger *zap.Logger) *CollectionArchiver {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
if keepFiles <= 0 {
|
||||
keepFiles = constants.TrafficCollectionArchiveKeepFiles
|
||||
}
|
||||
return &CollectionArchiver{db: db, dir: dir, keepFiles: keepFiles, aheadMonth: 2, logger: logger}
|
||||
}
|
||||
|
||||
// archiveManifest 是压缩包同名说明文件内容。
|
||||
type archiveManifest struct {
|
||||
Table string `json:"table"`
|
||||
Partition string `json:"partition"`
|
||||
Month string `json:"month"`
|
||||
Rows int64 `json:"rows"`
|
||||
FirstObservedAt *string `json:"first_observed_at"`
|
||||
LastObservedAt *string `json:"last_observed_at"`
|
||||
File string `json:"file"`
|
||||
FileBytes int64 `json:"file_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ExportedAt time.Time `json:"exported_at"`
|
||||
}
|
||||
|
||||
// Maintain 对全部采集记录表执行一轮分区维护;单张表失败不影响其他表,返回第一个错误。
|
||||
func (a *CollectionArchiver) Maintain(ctx context.Context, now time.Time) error {
|
||||
var firstErr error
|
||||
for _, table := range collectionArchiveTables {
|
||||
if err := a.maintainTable(ctx, table, now); err != nil {
|
||||
a.logger.Error("维护采集记录分区失败", zap.String("table", table), zap.Error(err))
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (a *CollectionArchiver) maintainTable(ctx context.Context, table string, now time.Time) error {
|
||||
monthStart := cardflow.MonthStart(now)
|
||||
for i := 0; i <= a.aheadMonth; i++ {
|
||||
from := monthStart.AddDate(0, i, 0)
|
||||
sqlText := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`,
|
||||
table, from.Format("200601"), table, from.Format(time.RFC3339), from.AddDate(0, 1, 0).Format(time.RFC3339))
|
||||
if err := a.db.WithContext(ctx).Exec(sqlText).Error; err != nil {
|
||||
return fmt.Errorf("预建分区失败: %w", err)
|
||||
}
|
||||
}
|
||||
// 在线保留当月加最近 N 个整月,更早的月分区到期。
|
||||
cutoff := monthStart.AddDate(0, -constants.TrafficCollectionRetentionMonths, 0)
|
||||
partitions, err := a.monthPartitions(ctx, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, partition := range partitions {
|
||||
if !partition.month.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
if err := a.archivePartition(ctx, table, partition); err != nil {
|
||||
return fmt.Errorf("归档分区 %s 失败: %w", partition.name, err)
|
||||
}
|
||||
}
|
||||
a.cleanupLocalFiles(table)
|
||||
return nil
|
||||
}
|
||||
|
||||
type monthPartition struct {
|
||||
name string
|
||||
month time.Time
|
||||
attached bool
|
||||
}
|
||||
|
||||
// monthPartitions 返回该表全部按月命名的分区(含上一轮已摘出未删除的表),按月份升序。
|
||||
func (a *CollectionArchiver) monthPartitions(ctx context.Context, table string) ([]monthPartition, error) {
|
||||
var rows []struct {
|
||||
Relname string
|
||||
Attached bool
|
||||
}
|
||||
err := a.db.WithContext(ctx).Raw(`
|
||||
SELECT c.relname, EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent
|
||||
WHERE i.inhrelid = c.oid AND p.relname = ?) AS attached
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND c.relname ~ ?`,
|
||||
table, "^"+table+`_[0-9]{6}$`).Scan(&rows).Error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询分区失败: %w", err)
|
||||
}
|
||||
partitions := make([]monthPartition, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
month, err := time.ParseInLocation("200601", row.Relname[len(table)+1:], cardflow.MonthStart(time.Now()).Location())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
partitions = append(partitions, monthPartition{name: row.Relname, month: month, attached: row.Attached})
|
||||
}
|
||||
sort.Slice(partitions, func(i, j int) bool { return partitions[i].month.Before(partitions[j].month) })
|
||||
return partitions, nil
|
||||
}
|
||||
|
||||
func (a *CollectionArchiver) archivePartition(ctx context.Context, table string, partition monthPartition) error {
|
||||
if partition.attached {
|
||||
if err := a.db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", table, partition.name)).Error; err != nil {
|
||||
return fmt.Errorf("摘出分区失败: %w", err)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(a.dir, 0o755); err != nil {
|
||||
return fmt.Errorf("创建归档目录失败: %w", err)
|
||||
}
|
||||
finalPath := filepath.Join(a.dir, partition.name+".csv.gz")
|
||||
tmpPath := finalPath + ".tmp"
|
||||
if err := a.exportCSV(ctx, partition.name, tmpPath); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return err
|
||||
}
|
||||
manifest, err := a.verify(ctx, table, partition, tmpPath)
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return err
|
||||
}
|
||||
manifest.File = filepath.Base(finalPath)
|
||||
if err := writeManifest(finalPath+".manifest.json", manifest); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpPath, finalPath); err != nil {
|
||||
return fmt.Errorf("压缩包改名失败: %w", err)
|
||||
}
|
||||
if err := a.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + partition.name).Error; err != nil {
|
||||
return fmt.Errorf("删除已归档分区失败: %w", err)
|
||||
}
|
||||
a.logger.Info("采集记录分区已归档", zap.String("partition", partition.name), zap.Int64("rows", manifest.Rows),
|
||||
zap.String("file", finalPath), zap.Int64("file_bytes", manifest.FileBytes))
|
||||
return nil
|
||||
}
|
||||
|
||||
// exportCSV 用 COPY 把分区按采集时间与 ID 排序流式导出为带表头的 CSV,并边写边 gzip。
|
||||
func (a *CollectionArchiver) exportCSV(ctx context.Context, partition, path string) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建压缩包失败: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
gz := gzip.NewWriter(file)
|
||||
sqlDB, err := a.db.DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("获取数据库连接失败: %w", err)
|
||||
}
|
||||
conn, err := sqlDB.Conn(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("获取数据库连接失败: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
copySQL := fmt.Sprintf("COPY (SELECT * FROM %s ORDER BY observed_at, id) TO STDOUT WITH (FORMAT csv, HEADER)", partition)
|
||||
if err := rawCopyTo(ctx, conn, gz, copySQL); err != nil {
|
||||
return fmt.Errorf("导出分区失败: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return fmt.Errorf("写压缩包失败: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return fmt.Errorf("落盘压缩包失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawCopyTo(ctx context.Context, conn *sql.Conn, w io.Writer, copySQL string) error {
|
||||
return conn.Raw(func(driverConn any) error {
|
||||
stdConn, ok := driverConn.(*stdlib.Conn)
|
||||
if !ok {
|
||||
return fmt.Errorf("数据库驱动不支持 COPY 导出")
|
||||
}
|
||||
_, err := stdConn.Conn().PgConn().CopyTo(ctx, w, copySQL)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// verify 重新读取压缩包统计行数并与分区行数比对,生成说明文件内容。
|
||||
func (a *CollectionArchiver) verify(ctx context.Context, table string, partition monthPartition, path string) (archiveManifest, error) {
|
||||
var stats struct {
|
||||
Rows int64
|
||||
First *time.Time
|
||||
Last *time.Time
|
||||
}
|
||||
if err := a.db.WithContext(ctx).Raw(fmt.Sprintf(
|
||||
"SELECT count(*) AS rows, min(observed_at) AS first, max(observed_at) AS last FROM %s", partition.name)).Scan(&stats).Error; err != nil {
|
||||
return archiveManifest{}, fmt.Errorf("统计分区行数失败: %w", err)
|
||||
}
|
||||
fileRows, sum, size, err := countArchiveRows(path)
|
||||
if err != nil {
|
||||
return archiveManifest{}, err
|
||||
}
|
||||
if fileRows != stats.Rows {
|
||||
return archiveManifest{}, fmt.Errorf("压缩包行数 %d 与分区行数 %d 不一致", fileRows, stats.Rows)
|
||||
}
|
||||
manifest := archiveManifest{
|
||||
Table: table, Partition: partition.name, Month: partition.month.Format("2006-01"), Rows: stats.Rows,
|
||||
FileBytes: size, SHA256: sum, ExportedAt: time.Now(),
|
||||
}
|
||||
if stats.First != nil {
|
||||
first, last := stats.First.Format(time.RFC3339), stats.Last.Format(time.RFC3339)
|
||||
manifest.FirstObservedAt, manifest.LastObservedAt = &first, &last
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
// countArchiveRows 解压并按 CSV 解析压缩包,返回数据行数(不含表头)、文件 SHA-256 与文件大小。
|
||||
func countArchiveRows(path string) (int64, string, int64, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, "", 0, fmt.Errorf("打开压缩包失败: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
gz, err := gzip.NewReader(io.TeeReader(file, hash))
|
||||
if err != nil {
|
||||
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
|
||||
}
|
||||
reader := csv.NewReader(gz)
|
||||
reader.FieldsPerRecord = -1
|
||||
reader.ReuseRecord = true
|
||||
var records int64
|
||||
for {
|
||||
if _, err := reader.Read(); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return 0, "", 0, fmt.Errorf("解析压缩包失败: %w", err)
|
||||
}
|
||||
records++
|
||||
}
|
||||
if _, err := io.Copy(io.Discard, gz); err != nil {
|
||||
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return 0, "", 0, fmt.Errorf("读取压缩包大小失败: %w", err)
|
||||
}
|
||||
if records > 0 {
|
||||
records-- // 表头
|
||||
}
|
||||
return records, hex.EncodeToString(hash.Sum(nil)), info.Size(), nil
|
||||
}
|
||||
|
||||
func writeManifest(path string, manifest archiveManifest) error {
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("生成说明文件失败: %w", err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil {
|
||||
return fmt.Errorf("写说明文件失败: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("说明文件改名失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupLocalFiles 每张表只保留最近 keepFiles 个月的压缩包与说明文件。
|
||||
func (a *CollectionArchiver) cleanupLocalFiles(table string) {
|
||||
matches, err := filepath.Glob(filepath.Join(a.dir, table+"_[0-9][0-9][0-9][0-9][0-9][0-9].csv.gz"))
|
||||
if err != nil || len(matches) <= a.keepFiles {
|
||||
return
|
||||
}
|
||||
sort.Strings(matches)
|
||||
for _, path := range matches[:len(matches)-a.keepFiles] {
|
||||
if err := os.Remove(path); err != nil {
|
||||
a.logger.Warn("删除过期采集记录压缩包失败", zap.String("file", path), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(path + ".manifest.json")
|
||||
a.logger.Info("已删除过期采集记录压缩包", zap.String("file", strings.TrimPrefix(path, a.dir+"/")))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user