- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id - tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情 - 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月 - 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选 - 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选 - 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选 - 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录 - 迁移 000272、000273 需维护者在测试库执行后再部署 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
314 lines
11 KiB
Go
314 lines
11 KiB
Go
package cardobservation
|
||
|
||
import (
|
||
"compress/gzip"
|
||
"context"
|
||
"crypto/sha256"
|
||
"database/sql"
|
||
"encoding/csv"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/stdlib"
|
||
"go.uber.org/zap"
|
||
"gorm.io/gorm"
|
||
|
||
"github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
|
||
"github.com/break/junhong_cmp_fiber/pkg/constants"
|
||
)
|
||
|
||
// collectionArchiveTables 是按月分区、到期归档的采集记录父表。
|
||
var collectionArchiveTables = []string{"tb_card_traffic_collection", "tb_card_traffic_collection_failure"}
|
||
|
||
// CollectionArchiver 维护采集记录月分区:预建未来分区;到期分区按"摘出 → 导出 csv.gz → 校验 → 删除"归档。
|
||
// 导出或校验失败时保留已摘出的表,下一轮重试,不会出现数据已删但没有压缩包的情况。
|
||
type CollectionArchiver struct {
|
||
db *gorm.DB
|
||
dir string
|
||
keepFiles int
|
||
aheadMonth int
|
||
logger *zap.Logger
|
||
}
|
||
|
||
// NewCollectionArchiver 创建采集记录归档器;dir 为压缩包目录,keepFiles 为每张表本地保留的压缩包个数。
|
||
func NewCollectionArchiver(db *gorm.DB, dir string, keepFiles int, logger *zap.Logger) *CollectionArchiver {
|
||
if logger == nil {
|
||
logger = zap.NewNop()
|
||
}
|
||
if keepFiles <= 0 {
|
||
keepFiles = constants.TrafficCollectionArchiveKeepFiles
|
||
}
|
||
return &CollectionArchiver{db: db, dir: dir, keepFiles: keepFiles, aheadMonth: 2, logger: logger}
|
||
}
|
||
|
||
// archiveManifest 是压缩包同名说明文件内容。
|
||
type archiveManifest struct {
|
||
Table string `json:"table"`
|
||
Partition string `json:"partition"`
|
||
Month string `json:"month"`
|
||
Rows int64 `json:"rows"`
|
||
FirstObservedAt *string `json:"first_observed_at"`
|
||
LastObservedAt *string `json:"last_observed_at"`
|
||
File string `json:"file"`
|
||
FileBytes int64 `json:"file_bytes"`
|
||
SHA256 string `json:"sha256"`
|
||
ExportedAt time.Time `json:"exported_at"`
|
||
}
|
||
|
||
// Maintain 对全部采集记录表执行一轮分区维护;单张表失败不影响其他表,返回第一个错误。
|
||
func (a *CollectionArchiver) Maintain(ctx context.Context, now time.Time) error {
|
||
var firstErr error
|
||
for _, table := range collectionArchiveTables {
|
||
if err := a.maintainTable(ctx, table, now); err != nil {
|
||
a.logger.Error("维护采集记录分区失败", zap.String("table", table), zap.Error(err))
|
||
if firstErr == nil {
|
||
firstErr = err
|
||
}
|
||
}
|
||
}
|
||
return firstErr
|
||
}
|
||
|
||
func (a *CollectionArchiver) maintainTable(ctx context.Context, table string, now time.Time) error {
|
||
monthStart := cardflow.MonthStart(now)
|
||
for i := 0; i <= a.aheadMonth; i++ {
|
||
from := monthStart.AddDate(0, i, 0)
|
||
sqlText := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`,
|
||
table, from.Format("200601"), table, from.Format(time.RFC3339), from.AddDate(0, 1, 0).Format(time.RFC3339))
|
||
if err := a.db.WithContext(ctx).Exec(sqlText).Error; err != nil {
|
||
return fmt.Errorf("预建分区失败: %w", err)
|
||
}
|
||
}
|
||
// 在线保留当月加最近 N 个整月,更早的月分区到期。
|
||
cutoff := monthStart.AddDate(0, -constants.TrafficCollectionRetentionMonths, 0)
|
||
partitions, err := a.monthPartitions(ctx, table)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, partition := range partitions {
|
||
if !partition.month.Before(cutoff) {
|
||
continue
|
||
}
|
||
if err := a.archivePartition(ctx, table, partition); err != nil {
|
||
return fmt.Errorf("归档分区 %s 失败: %w", partition.name, err)
|
||
}
|
||
}
|
||
a.cleanupLocalFiles(table)
|
||
return nil
|
||
}
|
||
|
||
type monthPartition struct {
|
||
name string
|
||
month time.Time
|
||
attached bool
|
||
}
|
||
|
||
// monthPartitions 返回该表全部按月命名的分区(含上一轮已摘出未删除的表),按月份升序。
|
||
func (a *CollectionArchiver) monthPartitions(ctx context.Context, table string) ([]monthPartition, error) {
|
||
var rows []struct {
|
||
Relname string
|
||
Attached bool
|
||
}
|
||
err := a.db.WithContext(ctx).Raw(`
|
||
SELECT c.relname, EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent
|
||
WHERE i.inhrelid = c.oid AND p.relname = ?) AS attached
|
||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||
WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND c.relname ~ ?`,
|
||
table, "^"+table+`_[0-9]{6}$`).Scan(&rows).Error
|
||
if err != nil {
|
||
return nil, fmt.Errorf("查询分区失败: %w", err)
|
||
}
|
||
partitions := make([]monthPartition, 0, len(rows))
|
||
for _, row := range rows {
|
||
month, err := time.ParseInLocation("200601", row.Relname[len(table)+1:], cardflow.MonthStart(time.Now()).Location())
|
||
if err != nil {
|
||
continue
|
||
}
|
||
partitions = append(partitions, monthPartition{name: row.Relname, month: month, attached: row.Attached})
|
||
}
|
||
sort.Slice(partitions, func(i, j int) bool { return partitions[i].month.Before(partitions[j].month) })
|
||
return partitions, nil
|
||
}
|
||
|
||
func (a *CollectionArchiver) archivePartition(ctx context.Context, table string, partition monthPartition) error {
|
||
if partition.attached {
|
||
if err := a.db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", table, partition.name)).Error; err != nil {
|
||
return fmt.Errorf("摘出分区失败: %w", err)
|
||
}
|
||
}
|
||
if err := os.MkdirAll(a.dir, 0o755); err != nil {
|
||
return fmt.Errorf("创建归档目录失败: %w", err)
|
||
}
|
||
finalPath := filepath.Join(a.dir, partition.name+".csv.gz")
|
||
tmpPath := finalPath + ".tmp"
|
||
if err := a.exportCSV(ctx, partition.name, tmpPath); err != nil {
|
||
_ = os.Remove(tmpPath)
|
||
return err
|
||
}
|
||
manifest, err := a.verify(ctx, table, partition, tmpPath)
|
||
if err != nil {
|
||
_ = os.Remove(tmpPath)
|
||
return err
|
||
}
|
||
manifest.File = filepath.Base(finalPath)
|
||
if err := writeManifest(finalPath+".manifest.json", manifest); err != nil {
|
||
_ = os.Remove(tmpPath)
|
||
return err
|
||
}
|
||
if err := os.Rename(tmpPath, finalPath); err != nil {
|
||
return fmt.Errorf("压缩包改名失败: %w", err)
|
||
}
|
||
if err := a.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + partition.name).Error; err != nil {
|
||
return fmt.Errorf("删除已归档分区失败: %w", err)
|
||
}
|
||
a.logger.Info("采集记录分区已归档", zap.String("partition", partition.name), zap.Int64("rows", manifest.Rows),
|
||
zap.String("file", finalPath), zap.Int64("file_bytes", manifest.FileBytes))
|
||
return nil
|
||
}
|
||
|
||
// exportCSV 用 COPY 把分区按采集时间与 ID 排序流式导出为带表头的 CSV,并边写边 gzip。
|
||
func (a *CollectionArchiver) exportCSV(ctx context.Context, partition, path string) error {
|
||
file, err := os.Create(path)
|
||
if err != nil {
|
||
return fmt.Errorf("创建压缩包失败: %w", err)
|
||
}
|
||
defer file.Close()
|
||
gz := gzip.NewWriter(file)
|
||
sqlDB, err := a.db.DB()
|
||
if err != nil {
|
||
return fmt.Errorf("获取数据库连接失败: %w", err)
|
||
}
|
||
conn, err := sqlDB.Conn(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("获取数据库连接失败: %w", err)
|
||
}
|
||
defer conn.Close()
|
||
copySQL := fmt.Sprintf("COPY (SELECT * FROM %s ORDER BY observed_at, id) TO STDOUT WITH (FORMAT csv, HEADER)", partition)
|
||
if err := rawCopyTo(ctx, conn, gz, copySQL); err != nil {
|
||
return fmt.Errorf("导出分区失败: %w", err)
|
||
}
|
||
if err := gz.Close(); err != nil {
|
||
return fmt.Errorf("写压缩包失败: %w", err)
|
||
}
|
||
if err := file.Sync(); err != nil {
|
||
return fmt.Errorf("落盘压缩包失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func rawCopyTo(ctx context.Context, conn *sql.Conn, w io.Writer, copySQL string) error {
|
||
return conn.Raw(func(driverConn any) error {
|
||
stdConn, ok := driverConn.(*stdlib.Conn)
|
||
if !ok {
|
||
return fmt.Errorf("数据库驱动不支持 COPY 导出")
|
||
}
|
||
_, err := stdConn.Conn().PgConn().CopyTo(ctx, w, copySQL)
|
||
return err
|
||
})
|
||
}
|
||
|
||
// verify 重新读取压缩包统计行数并与分区行数比对,生成说明文件内容。
|
||
func (a *CollectionArchiver) verify(ctx context.Context, table string, partition monthPartition, path string) (archiveManifest, error) {
|
||
var stats struct {
|
||
Rows int64
|
||
First *time.Time
|
||
Last *time.Time
|
||
}
|
||
if err := a.db.WithContext(ctx).Raw(fmt.Sprintf(
|
||
"SELECT count(*) AS rows, min(observed_at) AS first, max(observed_at) AS last FROM %s", partition.name)).Scan(&stats).Error; err != nil {
|
||
return archiveManifest{}, fmt.Errorf("统计分区行数失败: %w", err)
|
||
}
|
||
fileRows, sum, size, err := countArchiveRows(path)
|
||
if err != nil {
|
||
return archiveManifest{}, err
|
||
}
|
||
if fileRows != stats.Rows {
|
||
return archiveManifest{}, fmt.Errorf("压缩包行数 %d 与分区行数 %d 不一致", fileRows, stats.Rows)
|
||
}
|
||
manifest := archiveManifest{
|
||
Table: table, Partition: partition.name, Month: partition.month.Format("2006-01"), Rows: stats.Rows,
|
||
FileBytes: size, SHA256: sum, ExportedAt: time.Now(),
|
||
}
|
||
if stats.First != nil {
|
||
first, last := stats.First.Format(time.RFC3339), stats.Last.Format(time.RFC3339)
|
||
manifest.FirstObservedAt, manifest.LastObservedAt = &first, &last
|
||
}
|
||
return manifest, nil
|
||
}
|
||
|
||
// countArchiveRows 解压并按 CSV 解析压缩包,返回数据行数(不含表头)、文件 SHA-256 与文件大小。
|
||
func countArchiveRows(path string) (int64, string, int64, error) {
|
||
file, err := os.Open(path)
|
||
if err != nil {
|
||
return 0, "", 0, fmt.Errorf("打开压缩包失败: %w", err)
|
||
}
|
||
defer file.Close()
|
||
hash := sha256.New()
|
||
gz, err := gzip.NewReader(io.TeeReader(file, hash))
|
||
if err != nil {
|
||
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
|
||
}
|
||
reader := csv.NewReader(gz)
|
||
reader.FieldsPerRecord = -1
|
||
reader.ReuseRecord = true
|
||
var records int64
|
||
for {
|
||
if _, err := reader.Read(); err == io.EOF {
|
||
break
|
||
} else if err != nil {
|
||
return 0, "", 0, fmt.Errorf("解析压缩包失败: %w", err)
|
||
}
|
||
records++
|
||
}
|
||
if _, err := io.Copy(io.Discard, gz); err != nil {
|
||
return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err)
|
||
}
|
||
info, err := file.Stat()
|
||
if err != nil {
|
||
return 0, "", 0, fmt.Errorf("读取压缩包大小失败: %w", err)
|
||
}
|
||
if records > 0 {
|
||
records-- // 表头
|
||
}
|
||
return records, hex.EncodeToString(hash.Sum(nil)), info.Size(), nil
|
||
}
|
||
|
||
func writeManifest(path string, manifest archiveManifest) error {
|
||
data, err := json.MarshalIndent(manifest, "", " ")
|
||
if err != nil {
|
||
return fmt.Errorf("生成说明文件失败: %w", err)
|
||
}
|
||
tmp := path + ".tmp"
|
||
if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil {
|
||
return fmt.Errorf("写说明文件失败: %w", err)
|
||
}
|
||
if err := os.Rename(tmp, path); err != nil {
|
||
return fmt.Errorf("说明文件改名失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// cleanupLocalFiles 每张表只保留最近 keepFiles 个月的压缩包与说明文件。
|
||
func (a *CollectionArchiver) cleanupLocalFiles(table string) {
|
||
matches, err := filepath.Glob(filepath.Join(a.dir, table+"_[0-9][0-9][0-9][0-9][0-9][0-9].csv.gz"))
|
||
if err != nil || len(matches) <= a.keepFiles {
|
||
return
|
||
}
|
||
sort.Strings(matches)
|
||
for _, path := range matches[:len(matches)-a.keepFiles] {
|
||
if err := os.Remove(path); err != nil {
|
||
a.logger.Warn("删除过期采集记录压缩包失败", zap.String("file", path), zap.Error(err))
|
||
continue
|
||
}
|
||
_ = os.Remove(path + ".manifest.json")
|
||
a.logger.Info("已删除过期采集记录压缩包", zap.String("file", strings.TrimPrefix(path, a.dir+"/")))
|
||
}
|
||
}
|