Files
junhong_cmp_fiber/internal/infrastructure/cardobservation/collection_archive.go
邹进 94cbf2adab feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口
- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id
- tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情
- 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月
- 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选
- 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选
- 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选
- 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录
- 迁移 000272、000273 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 18:47:00 +09:00

314 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package cardobservation
import (
"compress/gzip"
"context"
"crypto/sha256"
"database/sql"
"encoding/csv"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5/stdlib"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// collectionArchiveTables 是按月分区、到期归档的采集记录父表。
var collectionArchiveTables = []string{"tb_card_traffic_collection", "tb_card_traffic_collection_failure"}
// CollectionArchiver 维护采集记录月分区:预建未来分区;到期分区按"摘出 → 导出 csv.gz → 校验 → 删除"归档。
// 导出或校验失败时保留已摘出的表,下一轮重试,不会出现数据已删但没有压缩包的情况。
type CollectionArchiver struct {
db *gorm.DB
dir string
keepFiles int
aheadMonth int
logger *zap.Logger
}
// NewCollectionArchiver 创建采集记录归档器;dir 为压缩包目录,keepFiles 为每张表本地保留的压缩包个数。
func NewCollectionArchiver(db *gorm.DB, dir string, keepFiles int, logger *zap.Logger) *CollectionArchiver {
if logger == nil {
logger = zap.NewNop()
}
if keepFiles <= 0 {
keepFiles = constants.TrafficCollectionArchiveKeepFiles
}
return &CollectionArchiver{db: db, dir: dir, keepFiles: keepFiles, aheadMonth: 2, logger: logger}
}
// archiveManifest 是压缩包同名说明文件内容。
type archiveManifest struct {
Table string `json:"table"`
Partition string `json:"partition"`
Month string `json:"month"`
Rows int64 `json:"rows"`
FirstObservedAt *string `json:"first_observed_at"`
LastObservedAt *string `json:"last_observed_at"`
File string `json:"file"`
FileBytes int64 `json:"file_bytes"`
SHA256 string `json:"sha256"`
ExportedAt time.Time `json:"exported_at"`
}
// Maintain 对全部采集记录表执行一轮分区维护;单张表失败不影响其他表,返回第一个错误。
func (a *CollectionArchiver) Maintain(ctx context.Context, now time.Time) error {
var firstErr error
for _, table := range collectionArchiveTables {
if err := a.maintainTable(ctx, table, now); err != nil {
a.logger.Error("维护采集记录分区失败", zap.String("table", table), zap.Error(err))
if firstErr == nil {
firstErr = err
}
}
}
return firstErr
}
func (a *CollectionArchiver) maintainTable(ctx context.Context, table string, now time.Time) error {
monthStart := cardflow.MonthStart(now)
for i := 0; i <= a.aheadMonth; i++ {
from := monthStart.AddDate(0, i, 0)
sqlText := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`,
table, from.Format("200601"), table, from.Format(time.RFC3339), from.AddDate(0, 1, 0).Format(time.RFC3339))
if err := a.db.WithContext(ctx).Exec(sqlText).Error; err != nil {
return fmt.Errorf("预建分区失败: %w", err)
}
}
// 在线保留当月加最近 N 个整月,更早的月分区到期。
cutoff := monthStart.AddDate(0, -constants.TrafficCollectionRetentionMonths, 0)
partitions, err := a.monthPartitions(ctx, table)
if err != nil {
return err
}
for _, partition := range partitions {
if !partition.month.Before(cutoff) {
continue
}
if err := a.archivePartition(ctx, table, partition); err != nil {
return fmt.Errorf("归档分区 %s 失败: %w", partition.name, err)
}
}
a.cleanupLocalFiles(table)
return nil
}
type monthPartition struct {
name string
month time.Time
attached bool
}
// monthPartitions 返回该表全部按月命名的分区(含上一轮已摘出未删除的表),按月份升序。
func (a *CollectionArchiver) monthPartitions(ctx context.Context, table string) ([]monthPartition, error) {
var rows []struct {
Relname string
Attached bool
}
err := a.db.WithContext(ctx).Raw(`
SELECT c.relname, EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent
WHERE i.inhrelid = c.oid AND p.relname = ?) AS attached
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND c.relname ~ ?`,
table, "^"+table+`_[0-9]{6}$`).Scan(&rows).Error
if err != nil {
return nil, fmt.Errorf("查询分区失败: %w", err)
}
partitions := make([]monthPartition, 0, len(rows))
for _, row := range rows {
month, err := time.ParseInLocation("200601", row.Relname[len(table)+1:], cardflow.MonthStart(time.Now()).Location())
if err != nil {
continue
}
partitions = append(partitions, monthPartition{name: row.Relname, month: month, attached: row.Attached})
}
sort.Slice(partitions, func(i, j int) bool { return partitions[i].month.Before(partitions[j].month) })
return partitions, nil
}
func (a *CollectionArchiver) archivePartition(ctx context.Context, table string, partition monthPartition) error {
if partition.attached {
if err := a.db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", table, partition.name)).Error; err != nil {
return fmt.Errorf("摘出分区失败: %w", err)
}
}
if err := os.MkdirAll(a.dir, 0o755); err != nil {
return fmt.Errorf("创建归档目录失败: %w", err)
}
finalPath := filepath.Join(a.dir, partition.name+".csv.gz")
tmpPath := finalPath + ".tmp"
if err := a.exportCSV(ctx, partition.name, tmpPath); err != nil {
_ = os.Remove(tmpPath)
return err
}
manifest, err := a.verify(ctx, table, partition, tmpPath)
if err != nil {
_ = os.Remove(tmpPath)
return err
}
manifest.File = filepath.Base(finalPath)
if err := writeManifest(finalPath+".manifest.json", manifest); err != nil {
_ = os.Remove(tmpPath)
return err
}
if err := os.Rename(tmpPath, finalPath); err != nil {
return fmt.Errorf("压缩包改名失败: %w", err)
}
if err := a.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + partition.name).Error; err != nil {
return fmt.Errorf("删除已归档分区失败: %w", err)
}
a.logger.Info("采集记录分区已归档", zap.String("partition", partition.name), zap.Int64("rows", manifest.Rows),
zap.String("file", finalPath), zap.Int64("file_bytes", manifest.FileBytes))
return nil
}
// exportCSV 用 COPY 把分区按采集时间与 ID 排序流式导出为带表头的 CSV,并边写边 gzip。
func (a *CollectionArchiver) exportCSV(ctx context.Context, partition, path string) error {
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("创建压缩包失败: %w", err)
}
defer file.Close()
gz := gzip.NewWriter(file)
sqlDB, err := a.db.DB()
if err != nil {
return fmt.Errorf("获取数据库连接失败: %w", err)
}
conn, err := sqlDB.Conn(ctx)
if err != nil {
return fmt.Errorf("获取数据库连接失败: %w", err)
}
defer conn.Close()
copySQL := fmt.Sprintf("COPY (SELECT * FROM %s ORDER BY observed_at, id) TO STDOUT WITH (FORMAT csv, HEADER)", partition)
if err := rawCopyTo(ctx, conn, gz, copySQL); err != nil {
return fmt.Errorf("导出分区失败: %w", err)
}
if err := gz.Close(); err != nil {
return fmt.Errorf("写压缩包失败: %w", err)
}
if err := file.Sync(); err != nil {
return fmt.Errorf("落盘压缩包失败: %w", err)
}
return nil
}
func rawCopyTo(ctx context.Context, conn *sql.Conn, w io.Writer, copySQL string) error {
return conn.Raw(func(driverConn any) error {
stdConn, ok := driverConn.(*stdlib.Conn)
if !ok {
return fmt.Errorf("数据库驱动不支持 COPY 导出")
}
_, err := stdConn.Conn().PgConn().CopyTo(ctx, w, copySQL)
return err
})
}
// verify 重新读取压缩包统计行数并与分区行数比对,生成说明文件内容。
func (a *CollectionArchiver) verify(ctx context.Context, table string, partition monthPartition, path string) (archiveManifest, error) {
var stats struct {
Rows int64
First *time.Time
Last *time.Time
}
if err := a.db.WithContext(ctx).Raw(fmt.Sprintf(
"SELECT count(*) AS rows, min(observed_at) AS first, max(observed_at) AS last FROM %s", partition.name)).Scan(&stats).Error; err != nil {
return archiveManifest{}, fmt.Errorf("统计分区行数失败: %w", err)
}
fileRows, sum, size, err := countArchiveRows(path)
if err != nil {
return archiveManifest{}, err
}
if fileRows != stats.Rows {
return archiveManifest{}, fmt.Errorf("压缩包行数 %d 与分区行数 %d 不一致", fileRows, stats.Rows)
}
manifest := archiveManifest{
Table: table, Partition: partition.name, Month: partition.month.Format("2006-01"), Rows: stats.Rows,
FileBytes: size, SHA256: sum, ExportedAt: time.Now(),
}
if stats.First != nil {
first, last := stats.First.Format(time.RFC3339), stats.Last.Format(time.RFC3339)
manifest.FirstObservedAt, manifest.LastObservedAt = &first, &last
}
return manifest, nil
}
// countArchiveRows 解压并按 CSV 解析压缩包,返回数据行数(不含表头)、文件 SHA-256 与文件大小。
func countArchiveRows(path string) (int64, string, int64, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", 0, fmt.Errorf("打开压缩包失败: %w", err)
}
defer file.Close()
hash := sha256.New()
gz, err := gzip.NewReader(io.TeeReader(file, hash))
if err != nil {
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
}
reader := csv.NewReader(gz)
reader.FieldsPerRecord = -1
reader.ReuseRecord = true
var records int64
for {
if _, err := reader.Read(); err == io.EOF {
break
} else if err != nil {
return 0, "", 0, fmt.Errorf("解析压缩包失败: %w", err)
}
records++
}
if _, err := io.Copy(io.Discard, gz); err != nil {
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
}
info, err := file.Stat()
if err != nil {
return 0, "", 0, fmt.Errorf("读取压缩包大小失败: %w", err)
}
if records > 0 {
records-- // 表头
}
return records, hex.EncodeToString(hash.Sum(nil)), info.Size(), nil
}
func writeManifest(path string, manifest archiveManifest) error {
data, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("生成说明文件失败: %w", err)
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil {
return fmt.Errorf("写说明文件失败: %w", err)
}
if err := os.Rename(tmp, path); err != nil {
return fmt.Errorf("说明文件改名失败: %w", err)
}
return nil
}
// cleanupLocalFiles 每张表只保留最近 keepFiles 个月的压缩包与说明文件。
func (a *CollectionArchiver) cleanupLocalFiles(table string) {
matches, err := filepath.Glob(filepath.Join(a.dir, table+"_[0-9][0-9][0-9][0-9][0-9][0-9].csv.gz"))
if err != nil || len(matches) <= a.keepFiles {
return
}
sort.Strings(matches)
for _, path := range matches[:len(matches)-a.keepFiles] {
if err := os.Remove(path); err != nil {
a.logger.Warn("删除过期采集记录压缩包失败", zap.String("file", path), zap.Error(err))
continue
}
_ = os.Remove(path + ".manifest.json")
a.logger.Info("已删除过期采集记录压缩包", zap.String("file", strings.TrimPrefix(path, a.dir+"/")))
}
}