Files
junhong_cmp_fiber/scripts/cleanup/main.go
huang 65e461eff7
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 7m6s
fix: 代码质量清理 — 统一C端支付枚举、修复静默错误、注入Worker回调、移除废弃同步代码
- C端订单支付状态直接使用管理端统一枚举(1/2/3/4),移除冗余映射函数
- 资产查询绑定卡/套餐查询失败时记录Warn日志而非静默忽略
- Worker bootstrap注入停复机回调(流量耗尽停机、套餐激活复机)
- 删除废弃的SIM状态同步服务和任务处理器
- 新增开发环境数据清理脚本(full/soft/table三种模式)
2026-03-28 15:27:59 +08:00

637 lines
18 KiB
Go
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//go:build ignore
// 开发环境数据清理脚本
//
// ⚠️ 警告:此脚本会删除数据,请确认连接的是开发/测试数据库!
//
// 三种清理模式:
// full — 完全清理:只保留系统配置 + 超管账号 + 权限 + 运营商
// soft — 轻量清理:只清流水/日志数据,保留组织架构、卡、设备、套餐
// table — 指定清理:按表名清理指定的表
//
// 用法:
// source .env.local
// go run ./scripts/cleanup/main.go -mode full # 完全清理
// go run ./scripts/cleanup/main.go -mode soft # 轻量清理
// go run ./scripts/cleanup/main.go -mode table -tables "tb_order,tb_order_item"
// go run ./scripts/cleanup/main.go -mode full -yes # 跳过确认
// go run ./scripts/cleanup/main.go -mode full -dry-run # 只预览不执行
package main
import (
"bufio"
"flag"
"fmt"
"log"
"os"
"strings"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// ==================== 表分类定义 ====================
// 系统配置表 — 任何模式都不清理
var systemTables = []string{
"schema_migrations",
"tb_permission",
"tb_carrier",
"tb_polling_config",
"tb_polling_concurrency_config",
"tb_polling_alert_rule",
"tb_data_cleanup_config",
"tb_wechat_config",
"tb_payment_merchant_setting",
"tb_commission_withdrawal_setting",
"tb_dev_capability_config",
}
// 事务/流水/日志表 — soft 和 full 模式都清理TRUNCATE
var transactionalTables = []string{
// 流量记录(最大的表,优先清理释放空间)
"tb_data_usage_record",
"tb_card_daily_usage",
// 订单
"tb_order_item",
"tb_order",
// 套餐使用
"tb_package_usage_daily_record",
"tb_package_usage",
// 资产钱包
"tb_asset_wallet_transaction",
"tb_asset_recharge_record",
"tb_asset_allocation_record",
"tb_asset_wallet",
// 代理钱包
"tb_agent_wallet_transaction",
"tb_agent_recharge_record",
"tb_agent_wallet",
// 佣金
"tb_commission_record",
"tb_commission_withdrawal_request",
"tb_shop_series_commission_stats",
// 换卡/换货
"tb_card_replacement_record_legacy",
"tb_card_replacement_request",
"tb_exchange_order",
// 个人客户(先删关联表再删主表)
"tb_personal_customer_device",
"tb_personal_customer_iccid",
"tb_personal_customer_phone",
"tb_personal_customer_openid",
"tb_personal_customer",
// 遗留个人客户表(无 tb_ 前缀)
"personal_customer_device",
"personal_customer_iccid",
"personal_customer_phone",
// 操作日志
"tb_account_operation_log",
"tb_data_cleanup_log",
"tb_polling_alert_history",
"tb_polling_manual_trigger_log",
}
// 业务数据表 — 仅 full 模式清理TRUNCATE
var businessTables = []string{
// 授权
"tb_enterprise_card_authorization",
"tb_enterprise_device_authorization",
// 设备
"tb_device_sim_binding",
"tb_device",
"tb_device_import_task",
// IoT 卡
"tb_iot_card",
"tb_iot_card_import_task",
"tb_number_card",
// 套餐分配
"tb_shop_package_allocation_price_history",
"tb_shop_package_allocation",
"tb_shop_series_allocation",
// 套餐定义
"tb_package",
"tb_package_series",
// 标签
"tb_resource_tag",
"tb_tag",
// 组织架构
"tb_shop_role",
"tb_enterprise",
"tb_shop",
}
// 部分清理规则 — full 模式下保留关键行DELETE WHERE
// 按执行顺序排列:先删关联数据,再删主数据
type partialCleanup struct {
table string // 表名
keepWhere string // 保留满足此条件的行
desc string // 保留说明
}
var partialCleanupRules = []partialCleanup{
{
table: "tb_account_role",
keepWhere: "account_id IN (SELECT id FROM tb_account WHERE user_type = 1)",
desc: "超管角色绑定",
},
{
table: "tb_role_permission",
keepWhere: "role_id IN (SELECT id FROM tb_role WHERE role_name = '超级管理员')",
desc: "超管权限映射",
},
{
table: "tb_role",
keepWhere: "role_name = '超级管理员'",
desc: "超级管理员角色",
},
{
table: "tb_account",
keepWhere: "user_type = 1",
desc: "超级管理员账号",
},
}
// ==================== 主入口 ====================
var (
flagMode = flag.String("mode", "", "清理模式: full / soft / table")
flagTables = flag.String("tables", "", "table 模式: 指定表名(逗号分隔)")
flagYes = flag.Bool("yes", false, "跳过确认直接执行")
flagDryRun = flag.Bool("dry-run", false, "预览模式,只显示将要清理的内容")
)
func main() {
flag.Parse()
if *flagMode == "" {
showUsage()
os.Exit(0)
}
db := connectDB()
switch *flagMode {
case "full":
doFull(db)
case "soft":
doSoft(db)
case "table":
if *flagTables == "" {
log.Fatal("❌ table 模式必须指定 -tables 参数,例如: -tables \"tb_order,tb_order_item\"")
}
doTable(db)
default:
log.Fatalf("❌ 未知模式: %s支持: full / soft / table", *flagMode)
}
}
// ==================== 清理模式实现 ====================
// doFull 完全清理:只保留系统配置 + 超管
func doFull(db *gorm.DB) {
fmt.Println("\n🔴 模式: full — 完全清理")
fmt.Println(" 保留: 系统配置表 + 超管账号 + 超管角色/权限 + 运营商")
fmt.Println(" 清理: 所有业务数据 + 组织架构 + 卡/设备 + 套餐 + 流水/日志")
// 收集所有要清理的表
truncateList := append([]string{}, transactionalTables...)
truncateList = append(truncateList, businessTables...)
// 显示影响概览
fmt.Println("\n📊 数据影响预览:")
showTableStats(db, "🗑️ TRUNCATE完全清空", truncateList)
showPartialStats(db, "✂️ DELETE部分保留", partialCleanupRules)
showTableStats(db, "🔒 保留(不动)", systemTables)
if *flagDryRun {
fmt.Println("\n⏸ dry-run 模式,未执行任何操作")
return
}
if !confirmAction("full 清理") {
return
}
start := time.Now()
// 第一步TRUNCATE 事务表 + 业务表
fmt.Println("\n⏳ 步骤 1/3: TRUNCATE 事务和业务数据表...")
existingTables := filterExistingTables(db, truncateList)
if len(existingTables) > 0 {
truncated := truncateTables(db, existingTables)
fmt.Printf(" ✅ 清空了 %d 张表\n", truncated)
}
// 第二步DELETE 部分保留表
fmt.Println("\n⏳ 步骤 2/3: 清理部分保留表...")
for _, rule := range partialCleanupRules {
deleted := deletePartial(db, rule)
fmt.Printf(" ✅ %s: 删除 %d 行(保留: %s\n", rule.table, deleted, rule.desc)
}
// 第三步:重置序列
fmt.Println("\n⏳ 步骤 3/3: 重置部分保留表的序列...")
resetPartialSequences(db)
elapsed := time.Since(start)
fmt.Printf("\n🎉 full 清理完成!耗时: %v\n", elapsed.Round(time.Millisecond))
showPostCleanupTips()
}
// doSoft 轻量清理:只清流水数据
func doSoft(db *gorm.DB) {
fmt.Println("\n🟡 模式: soft — 轻量清理")
fmt.Println(" 保留: 系统配置 + 所有账号 + 组织架构 + 卡/设备 + 套餐")
fmt.Println(" 清理: 订单 + 钱包流水 + 佣金记录 + 操作日志 + 流量记录")
fmt.Println("\n📊 数据影响预览:")
showTableStats(db, "🗑️ TRUNCATE完全清空", transactionalTables)
kept := append([]string{}, businessTables...)
kept = append(kept, systemTables...)
for _, rule := range partialCleanupRules {
kept = append(kept, rule.table)
}
showTableStats(db, "🔒 保留(不动)", kept)
if *flagDryRun {
fmt.Println("\n⏸ dry-run 模式,未执行任何操作")
return
}
if !confirmAction("soft 清理") {
return
}
start := time.Now()
fmt.Println("\n⏳ TRUNCATE 事务/流水表...")
existingTables := filterExistingTables(db, transactionalTables)
if len(existingTables) > 0 {
truncated := truncateTables(db, existingTables)
fmt.Printf(" ✅ 清空了 %d 张表\n", truncated)
}
elapsed := time.Since(start)
fmt.Printf("\n🎉 soft 清理完成!耗时: %v\n", elapsed.Round(time.Millisecond))
showPostCleanupTips()
}
// doTable 按表名清理
func doTable(db *gorm.DB) {
tableNames := parseTableNames(*flagTables)
if len(tableNames) == 0 {
log.Fatal("❌ 未指定有效的表名")
}
fmt.Println("\n🔵 模式: table — 指定表清理")
fmt.Printf(" 目标表: %s\n", strings.Join(tableNames, ", "))
// 检查是否包含系统表
systemSet := toSet(systemTables)
var warnings []string
var safeTables []string
for _, t := range tableNames {
if _, ok := systemSet[t]; ok {
warnings = append(warnings, t)
} else {
safeTables = append(safeTables, t)
}
}
if len(warnings) > 0 {
fmt.Printf("\n ⚠️ 以下系统表已跳过(不允许清理): %s\n", strings.Join(warnings, ", "))
}
if len(safeTables) == 0 {
fmt.Println("\n 没有可清理的表")
return
}
// 检查表是否存在
existingTables := filterExistingTables(db, safeTables)
if len(existingTables) == 0 {
fmt.Println("\n 指定的表在数据库中均不存在")
return
}
if len(existingTables) < len(safeTables) {
missing := diff(safeTables, existingTables)
fmt.Printf(" ⚠️ 以下表不存在,已跳过: %s\n", strings.Join(missing, ", "))
}
fmt.Println("\n📊 数据影响预览:")
showTableStats(db, "🗑️ TRUNCATE完全清空", existingTables)
if *flagDryRun {
fmt.Println("\n⏸ dry-run 模式,未执行任何操作")
return
}
if !confirmAction("table 清理") {
return
}
start := time.Now()
fmt.Println("\n⏳ TRUNCATE 指定表...")
truncated := truncateTables(db, existingTables)
fmt.Printf(" ✅ 清空了 %d 张表\n", truncated)
elapsed := time.Since(start)
fmt.Printf("\n🎉 table 清理完成!耗时: %v\n", elapsed.Round(time.Millisecond))
showPostCleanupTips()
}
// ==================== 执行辅助函数 ====================
// truncateTables 批量 TRUNCATE 多张表(重置序列,级联)
func truncateTables(db *gorm.DB, tables []string) int {
if len(tables) == 0 {
return 0
}
// PostgreSQL 支持单条 TRUNCATE 多张表,原子操作且更快
sql := fmt.Sprintf("TRUNCATE TABLE %s RESTART IDENTITY CASCADE", strings.Join(tables, ", "))
result := db.Exec(sql)
if result.Error != nil {
// 如果批量失败,尝试逐表清理
fmt.Printf(" ⚠️ 批量 TRUNCATE 失败: %v改为逐表清理...\n", result.Error)
count := 0
for _, t := range tables {
err := db.Exec(fmt.Sprintf("TRUNCATE TABLE %s RESTART IDENTITY CASCADE", t)).Error
if err != nil {
fmt.Printf(" ⚠️ 跳过 %s: %v\n", t, err)
} else {
count++
}
}
return count
}
return len(tables)
}
// deletePartial 删除表中不满足保留条件的行
func deletePartial(db *gorm.DB, rule partialCleanup) int64 {
sql := fmt.Sprintf("DELETE FROM %s WHERE NOT (%s)", rule.table, rule.keepWhere)
result := db.Exec(sql)
if result.Error != nil {
fmt.Printf(" ⚠️ %s 清理失败: %v\n", rule.table, result.Error)
return 0
}
return result.RowsAffected
}
// resetPartialSequences 重置部分保留表的序列到当前最大 ID + 1
func resetPartialSequences(db *gorm.DB) {
for _, rule := range partialCleanupRules {
// 查找表对应的序列名
var seqName string
db.Raw(`
SELECT pg_get_serial_sequence(?, 'id')
`, rule.table).Scan(&seqName)
if seqName == "" {
continue
}
// 获取当前最大 ID
var maxID *int64
db.Raw(fmt.Sprintf("SELECT MAX(id) FROM %s", rule.table)).Scan(&maxID)
if maxID == nil || *maxID == 0 {
// 表为空,重置到 1
db.Exec(fmt.Sprintf("ALTER SEQUENCE %s RESTART WITH 1", seqName))
} else {
// 重置到 max + 1
db.Exec(fmt.Sprintf("SELECT setval('%s', %d)", seqName, *maxID))
}
fmt.Printf(" ✅ %s 序列已重置\n", rule.table)
}
}
// filterExistingTables 过滤出数据库中实际存在的表
func filterExistingTables(db *gorm.DB, tables []string) []string {
var existing []string
for _, t := range tables {
var count int64
db.Raw("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = ?", t).Scan(&count)
if count > 0 {
existing = append(existing, t)
}
}
return existing
}
// ==================== 显示辅助函数 ====================
// showTableStats 显示一组表的行数统计
func showTableStats(db *gorm.DB, title string, tables []string) {
fmt.Printf("\n %s:\n", title)
totalRows := int64(0)
for _, t := range tables {
rows := getEstimatedRows(db, t)
totalRows += rows
if rows > 0 {
fmt.Printf(" %-50s %s 行\n", t, formatNumber(rows))
} else {
fmt.Printf(" %-50s (空)\n", t)
}
}
if totalRows > 0 {
fmt.Printf(" %s\n", strings.Repeat("─", 65))
fmt.Printf(" %-50s %s 行\n", "合计", formatNumber(totalRows))
}
}
// showPartialStats 显示部分清理表的统计
func showPartialStats(db *gorm.DB, title string, rules []partialCleanup) {
fmt.Printf("\n %s:\n", title)
for _, rule := range rules {
total := getEstimatedRows(db, rule.table)
// 查询将被保留的行数
var keepCount int64
db.Raw(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s", rule.table, rule.keepWhere)).Scan(&keepCount)
deleteCount := total - keepCount
if deleteCount < 0 {
deleteCount = 0
}
fmt.Printf(" %-50s 删除 %s / 保留 %s%s\n",
rule.table, formatNumber(deleteCount), formatNumber(keepCount), rule.desc)
}
}
// showUsage 显示使用说明
func showUsage() {
fmt.Println(`
📋 开发环境数据清理脚本
用法:
source .env.local
go run ./scripts/cleanup/main.go [选项]
模式:
-mode full 完全清理 — 只保留系统配置 + 超管账号
适用场景:重新开始,从零开始测试全流程
-mode soft 轻量清理 — 只清流水数据
适用场景:保留卡/设备/套餐,重新测试订单/支付流程
-mode table 指定清理 — 按表名清理
配合 -tables "tb_order,tb_order_item" 使用
选项:
-yes 跳过确认,直接执行
-dry-run 预览模式,只显示将要清理的内容,不实际执行
-tables table 模式下指定表名(逗号分隔)
示例:
go run ./scripts/cleanup/main.go -mode full # 完全清理
go run ./scripts/cleanup/main.go -mode soft -yes # 轻量清理(跳过确认)
go run ./scripts/cleanup/main.go -mode full -dry-run # 预览 full 清理
go run ./scripts/cleanup/main.go -mode table -tables "tb_order,tb_order_item,tb_data_usage_record"`)
}
// showPostCleanupTips 清理后提示
func showPostCleanupTips() {
fmt.Println("\n💡 后续建议:")
fmt.Println(" 1. 执行 VACUUM ANALYZE 回收磁盘空间(大表清理后建议执行):")
fmt.Println(" psql -c 'VACUUM ANALYZE;'")
fmt.Println(" 2. 清理 Redis 缓存(店铺层级、权限缓存等):")
fmt.Println(" redis-cli FLUSHDB")
fmt.Println(" 3. 重启 API/Worker 服务以刷新内存缓存")
}
// ==================== 数据库连接 ====================
func connectDB() *gorm.DB {
dsn := buildDSN()
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
Logger: logger.Default.LogMode(logger.Warn),
})
if err != nil {
log.Fatalf("❌ 连接数据库失败: %v", err)
}
sqlDB, _ := db.DB()
sqlDB.SetMaxOpenConns(5)
sqlDB.SetMaxIdleConns(3)
// 获取数据库名用于安全确认
var dbName string
db.Raw("SELECT current_database()").Scan(&dbName)
fmt.Printf("✅ 已连接数据库: %s\n", dbName)
return db
}
func buildDSN() string {
host := getEnv("JUNHONG_DATABASE_HOST", "")
port := getEnv("JUNHONG_DATABASE_PORT", "5432")
user := getEnv("JUNHONG_DATABASE_USER", "")
password := getEnv("JUNHONG_DATABASE_PASSWORD", "")
dbname := getEnv("JUNHONG_DATABASE_DBNAME", "")
if host == "" || user == "" || dbname == "" {
log.Fatal("❌ 缺少数据库配置,请设置环境变量:\n" +
" export JUNHONG_DATABASE_HOST=...\n" +
" export JUNHONG_DATABASE_USER=...\n" +
" export JUNHONG_DATABASE_PASSWORD=...\n" +
" export JUNHONG_DATABASE_DBNAME=...\n" +
" 或执行: source .env.local")
}
return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable TimeZone=Asia/Shanghai",
host, port, user, password, dbname)
}
// ==================== 工具函数 ====================
// confirmAction 确认操作(-yes 跳过)
func confirmAction(action string) bool {
if *flagYes {
return true
}
var dbName string
fmt.Printf("\n⚠ 即将对数据库执行 [%s],此操作不可撤销!\n", action)
fmt.Print(" 输入 yes 确认执行: ")
scanner := bufio.NewScanner(os.Stdin)
if scanner.Scan() {
dbName = strings.TrimSpace(scanner.Text())
}
if dbName != "yes" {
fmt.Println(" ❌ 已取消")
return false
}
return true
}
// getEstimatedRows 获取表的估算行数(从 pg_stat 读取,非常快)
func getEstimatedRows(db *gorm.DB, tableName string) int64 {
var rows int64
db.Raw("SELECT COALESCE(n_live_tup, 0) FROM pg_stat_user_tables WHERE relname = ?", tableName).Scan(&rows)
return rows
}
// formatNumber 千分位格式化数字
func formatNumber(n int64) string {
if n < 1000 {
return fmt.Sprintf("%d", n)
}
s := fmt.Sprintf("%d", n)
result := make([]byte, 0, len(s)+(len(s)-1)/3)
for i, c := range s {
if i > 0 && (len(s)-i)%3 == 0 {
result = append(result, ',')
}
result = append(result, byte(c))
}
return string(result)
}
// parseTableNames 解析逗号分隔的表名
func parseTableNames(input string) []string {
parts := strings.Split(input, ",")
var result []string
for _, p := range parts {
t := strings.TrimSpace(p)
if t != "" {
result = append(result, t)
}
}
return result
}
// toSet 将字符串切片转为 map
func toSet(items []string) map[string]struct{} {
m := make(map[string]struct{}, len(items))
for _, item := range items {
m[item] = struct{}{}
}
return m
}
// diff 返回在 a 中但不在 b 中的元素
func diff(a, b []string) []string {
bSet := toSet(b)
var result []string
for _, item := range a {
if _, ok := bSet[item]; !ok {
result = append(result, item)
}
}
return result
}
// getEnv 获取环境变量,支持默认值
func getEnv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}