break
31232ea899
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 53s
83 lines
3.2 KiB
Python
83 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""奇成数据迁移脚本1:资产导入。
|
|
|
|
工作流:
|
|
1. 读 mapping.yaml(必须先跑过 scan_legacy.py)
|
|
2. 读 cards.csv / devices.csv
|
|
3. 连奇成查每张卡的元数据(category/agent_id/msisdn/virtual_no/当前生效套餐)
|
|
4. 应用 mapping 把 legacy_* 翻译成 target_*,自动决定归属
|
|
5. 生成 output/step1_*.sql + errors.csv + summary.txt
|
|
|
|
约束:
|
|
- 对奇成只读;所有 INSERT 用 ON CONFLICT 保证幂等
|
|
- mapping 中任意 target_* 缺失会让对应卡进 errors.csv 跳过
|
|
|
|
用法:
|
|
python3 migrate_assets.py [--config-dir config] [--resources-dir resources] [--output-dir output]
|
|
[--batch-size 1000]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from lib import csv_loader, legacy_query, mapping_loader, sql_builder # noqa: E402
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="奇成数据迁移脚本1:资产导入")
|
|
parser.add_argument("--config-dir", default="config", help="yaml 配置目录")
|
|
parser.add_argument("--resources-dir", default="resources", help="CSV 资源目录")
|
|
parser.add_argument("--output-dir", default="output", help="SQL/CSV 输出目录")
|
|
parser.add_argument("--batch-size", type=int, default=legacy_query.BATCH_SIZE, help="老库批量查询大小")
|
|
args = parser.parse_args()
|
|
|
|
base = Path(__file__).resolve().parent
|
|
config_dir = (base / args.config_dir).resolve()
|
|
resources_dir = (base / args.resources_dir).resolve()
|
|
output_dir = (base / args.output_dir).resolve()
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
batch_size = args.batch_size
|
|
|
|
t0 = time.perf_counter()
|
|
mapping = mapping_loader.load_mapping(config_dir)
|
|
cards, devices, errors = csv_loader.load_assets(resources_dir, mapping)
|
|
print(f"[migrate] 加载 CSV: cards={len(cards)} devices={len(devices)} elapsed={time.perf_counter()-t0:.1f}s")
|
|
|
|
# 连奇成查卡元数据(运营商分类、agent_id、msisdn、virtual_no、当前生效套餐)
|
|
iccid_pairs = [(c.iccid_19, c.iccid_20, c.allow_iccid_19_lookup) for c in cards]
|
|
|
|
t1 = time.perf_counter()
|
|
dsn = mapping_loader.load_legacy_dsn(config_dir)
|
|
with legacy_query.connect_readonly(dsn) as conn:
|
|
print(f"[migrate] 连接老库: elapsed={time.perf_counter()-t1:.1f}s")
|
|
|
|
t2 = time.perf_counter()
|
|
card_metas, dup_iccids = legacy_query.fetch_card_meta(conn, iccid_pairs, batch_size=batch_size)
|
|
print(f"[migrate] fetch_card_meta: cards={len(card_metas)} duplicates={len(dup_iccids)} elapsed={time.perf_counter()-t2:.1f}s")
|
|
|
|
t3 = time.perf_counter()
|
|
sql_builder.write_step1(
|
|
output_dir,
|
|
cards=cards,
|
|
devices=devices,
|
|
mapping=mapping,
|
|
card_metas=card_metas,
|
|
duplicate_iccids=dup_iccids,
|
|
errors=errors,
|
|
)
|
|
print(f"[migrate] SQL 生成: elapsed={time.perf_counter()-t3:.1f}s")
|
|
|
|
total = time.perf_counter() - t0
|
|
print(f"完成:卡 {len(cards)} 张,设备 {len(devices)} 台,异常 {len(errors)} 行,总耗时 {total:.1f}s")
|
|
print(f"SQL 输出: {output_dir}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|