package cardobservation import ( "compress/gzip" "context" "crypto/sha256" "database/sql" "encoding/csv" "encoding/hex" "encoding/json" "fmt" "io" "os" "path/filepath" "sort" "strings" "time" "github.com/jackc/pgx/v5/stdlib" "go.uber.org/zap" "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/internal/domain/cardflow" "github.com/break/junhong_cmp_fiber/pkg/constants" ) // collectionArchiveTables 是按月分区、到期归档的采集记录父表。 var collectionArchiveTables = []string{"tb_card_traffic_collection", "tb_card_traffic_collection_failure"} // CollectionArchiver 维护采集记录月分区:预建未来分区;到期分区按"摘出 → 导出 csv.gz → 校验 → 删除"归档。 // 导出或校验失败时保留已摘出的表,下一轮重试,不会出现数据已删但没有压缩包的情况。 type CollectionArchiver struct { db *gorm.DB dir string keepFiles int aheadMonth int logger *zap.Logger } // NewCollectionArchiver 创建采集记录归档器;dir 为压缩包目录,keepFiles 为每张表本地保留的压缩包个数。 func NewCollectionArchiver(db *gorm.DB, dir string, keepFiles int, logger *zap.Logger) *CollectionArchiver { if logger == nil { logger = zap.NewNop() } if keepFiles <= 0 { keepFiles = constants.TrafficCollectionArchiveKeepFiles } return &CollectionArchiver{db: db, dir: dir, keepFiles: keepFiles, aheadMonth: 2, logger: logger} } // archiveManifest 是压缩包同名说明文件内容。 type archiveManifest struct { Table string `json:"table"` Partition string `json:"partition"` Month string `json:"month"` Rows int64 `json:"rows"` FirstObservedAt *string `json:"first_observed_at"` LastObservedAt *string `json:"last_observed_at"` File string `json:"file"` FileBytes int64 `json:"file_bytes"` SHA256 string `json:"sha256"` ExportedAt time.Time `json:"exported_at"` } // Maintain 对全部采集记录表执行一轮分区维护;单张表失败不影响其他表,返回第一个错误。 func (a *CollectionArchiver) Maintain(ctx context.Context, now time.Time) error { var firstErr error for _, table := range collectionArchiveTables { if err := a.maintainTable(ctx, table, now); err != nil { a.logger.Error("维护采集记录分区失败", zap.String("table", table), zap.Error(err)) if firstErr == nil { firstErr = err } } } return firstErr } func (a *CollectionArchiver) maintainTable(ctx context.Context, table string, now time.Time) error { monthStart := cardflow.MonthStart(now) for i := 0; i <= a.aheadMonth; i++ { from := monthStart.AddDate(0, i, 0) sqlText := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`, table, from.Format("200601"), table, from.Format(time.RFC3339), from.AddDate(0, 1, 0).Format(time.RFC3339)) if err := a.db.WithContext(ctx).Exec(sqlText).Error; err != nil { return fmt.Errorf("预建分区失败: %w", err) } } // 在线保留当月加最近 N 个整月,更早的月分区到期。 cutoff := monthStart.AddDate(0, -constants.TrafficCollectionRetentionMonths, 0) partitions, err := a.monthPartitions(ctx, table) if err != nil { return err } for _, partition := range partitions { if !partition.month.Before(cutoff) { continue } if err := a.archivePartition(ctx, table, partition); err != nil { return fmt.Errorf("归档分区 %s 失败: %w", partition.name, err) } } a.cleanupLocalFiles(table) return nil } type monthPartition struct { name string month time.Time attached bool } // monthPartitions 返回该表全部按月命名的分区(含上一轮已摘出未删除的表),按月份升序。 func (a *CollectionArchiver) monthPartitions(ctx context.Context, table string) ([]monthPartition, error) { var rows []struct { Relname string Attached bool } err := a.db.WithContext(ctx).Raw(` SELECT c.relname, EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent WHERE i.inhrelid = c.oid AND p.relname = ?) AS attached FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND c.relname ~ ?`, table, "^"+table+`_[0-9]{6}$`).Scan(&rows).Error if err != nil { return nil, fmt.Errorf("查询分区失败: %w", err) } partitions := make([]monthPartition, 0, len(rows)) for _, row := range rows { month, err := time.ParseInLocation("200601", row.Relname[len(table)+1:], cardflow.MonthStart(time.Now()).Location()) if err != nil { continue } partitions = append(partitions, monthPartition{name: row.Relname, month: month, attached: row.Attached}) } sort.Slice(partitions, func(i, j int) bool { return partitions[i].month.Before(partitions[j].month) }) return partitions, nil } func (a *CollectionArchiver) archivePartition(ctx context.Context, table string, partition monthPartition) error { if partition.attached { if err := a.db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s", table, partition.name)).Error; err != nil { return fmt.Errorf("摘出分区失败: %w", err) } } if err := os.MkdirAll(a.dir, 0o755); err != nil { return fmt.Errorf("创建归档目录失败: %w", err) } finalPath := filepath.Join(a.dir, partition.name+".csv.gz") tmpPath := finalPath + ".tmp" if err := a.exportCSV(ctx, partition.name, tmpPath); err != nil { _ = os.Remove(tmpPath) return err } manifest, err := a.verify(ctx, table, partition, tmpPath) if err != nil { _ = os.Remove(tmpPath) return err } manifest.File = filepath.Base(finalPath) if err := writeManifest(finalPath+".manifest.json", manifest); err != nil { _ = os.Remove(tmpPath) return err } if err := os.Rename(tmpPath, finalPath); err != nil { return fmt.Errorf("压缩包改名失败: %w", err) } if err := a.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + partition.name).Error; err != nil { return fmt.Errorf("删除已归档分区失败: %w", err) } a.logger.Info("采集记录分区已归档", zap.String("partition", partition.name), zap.Int64("rows", manifest.Rows), zap.String("file", finalPath), zap.Int64("file_bytes", manifest.FileBytes)) return nil } // exportCSV 用 COPY 把分区按采集时间与 ID 排序流式导出为带表头的 CSV,并边写边 gzip。 func (a *CollectionArchiver) exportCSV(ctx context.Context, partition, path string) error { file, err := os.Create(path) if err != nil { return fmt.Errorf("创建压缩包失败: %w", err) } defer file.Close() gz := gzip.NewWriter(file) sqlDB, err := a.db.DB() if err != nil { return fmt.Errorf("获取数据库连接失败: %w", err) } conn, err := sqlDB.Conn(ctx) if err != nil { return fmt.Errorf("获取数据库连接失败: %w", err) } defer conn.Close() copySQL := fmt.Sprintf("COPY (SELECT * FROM %s ORDER BY observed_at, id) TO STDOUT WITH (FORMAT csv, HEADER)", partition) if err := rawCopyTo(ctx, conn, gz, copySQL); err != nil { return fmt.Errorf("导出分区失败: %w", err) } if err := gz.Close(); err != nil { return fmt.Errorf("写压缩包失败: %w", err) } if err := file.Sync(); err != nil { return fmt.Errorf("落盘压缩包失败: %w", err) } return nil } func rawCopyTo(ctx context.Context, conn *sql.Conn, w io.Writer, copySQL string) error { return conn.Raw(func(driverConn any) error { stdConn, ok := driverConn.(*stdlib.Conn) if !ok { return fmt.Errorf("数据库驱动不支持 COPY 导出") } _, err := stdConn.Conn().PgConn().CopyTo(ctx, w, copySQL) return err }) } // verify 重新读取压缩包统计行数并与分区行数比对,生成说明文件内容。 func (a *CollectionArchiver) verify(ctx context.Context, table string, partition monthPartition, path string) (archiveManifest, error) { var stats struct { Rows int64 First *time.Time Last *time.Time } if err := a.db.WithContext(ctx).Raw(fmt.Sprintf( "SELECT count(*) AS rows, min(observed_at) AS first, max(observed_at) AS last FROM %s", partition.name)).Scan(&stats).Error; err != nil { return archiveManifest{}, fmt.Errorf("统计分区行数失败: %w", err) } fileRows, sum, size, err := countArchiveRows(path) if err != nil { return archiveManifest{}, err } if fileRows != stats.Rows { return archiveManifest{}, fmt.Errorf("压缩包行数 %d 与分区行数 %d 不一致", fileRows, stats.Rows) } manifest := archiveManifest{ Table: table, Partition: partition.name, Month: partition.month.Format("2006-01"), Rows: stats.Rows, FileBytes: size, SHA256: sum, ExportedAt: time.Now(), } if stats.First != nil { first, last := stats.First.Format(time.RFC3339), stats.Last.Format(time.RFC3339) manifest.FirstObservedAt, manifest.LastObservedAt = &first, &last } return manifest, nil } // countArchiveRows 解压并按 CSV 解析压缩包,返回数据行数(不含表头)、文件 SHA-256 与文件大小。 func countArchiveRows(path string) (int64, string, int64, error) { file, err := os.Open(path) if err != nil { return 0, "", 0, fmt.Errorf("打开压缩包失败: %w", err) } defer file.Close() hash := sha256.New() gz, err := gzip.NewReader(io.TeeReader(file, hash)) if err != nil { return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err) } reader := csv.NewReader(gz) reader.FieldsPerRecord = -1 reader.ReuseRecord = true var records int64 for { if _, err := reader.Read(); err == io.EOF { break } else if err != nil { return 0, "", 0, fmt.Errorf("解析压缩包失败: %w", err) } records++ } if _, err := io.Copy(io.Discard, gz); err != nil { return 0, "", 0, fmt.Errorf("读取压缩包失败: %w", err) } info, err := file.Stat() if err != nil { return 0, "", 0, fmt.Errorf("读取压缩包大小失败: %w", err) } if records > 0 { records-- // 表头 } return records, hex.EncodeToString(hash.Sum(nil)), info.Size(), nil } func writeManifest(path string, manifest archiveManifest) error { data, err := json.MarshalIndent(manifest, "", " ") if err != nil { return fmt.Errorf("生成说明文件失败: %w", err) } tmp := path + ".tmp" if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil { return fmt.Errorf("写说明文件失败: %w", err) } if err := os.Rename(tmp, path); err != nil { return fmt.Errorf("说明文件改名失败: %w", err) } return nil } // cleanupLocalFiles 每张表只保留最近 keepFiles 个月的压缩包与说明文件。 func (a *CollectionArchiver) cleanupLocalFiles(table string) { matches, err := filepath.Glob(filepath.Join(a.dir, table+"_[0-9][0-9][0-9][0-9][0-9][0-9].csv.gz")) if err != nil || len(matches) <= a.keepFiles { return } sort.Strings(matches) for _, path := range matches[:len(matches)-a.keepFiles] { if err := os.Remove(path); err != nil { a.logger.Warn("删除过期采集记录压缩包失败", zap.String("file", path), zap.Error(err)) continue } _ = os.Remove(path + ".manifest.json") a.logger.Info("已删除过期采集记录压缩包", zap.String("file", strings.TrimPrefix(path, a.dir+"/"))) } }