优化迁移脚本速度

scripts/migration/migrate_assets.py

@@ -4,7 +4,7 @@
 工作流:
     1. 读 mapping.yaml(必须先跑过 scan_legacy.py)
     2. 读 cards.csv / devices.csv
-    3. 连奇成查每张卡的元数据(category/agent_id/msisdn/virtual_no)
+    3. 连奇成查每张卡的元数据(category/agent_id/msisdn/virtual_no/当前生效套餐)
     4. 应用 mapping 把 legacy_* 翻译成 target_*,自动决定归属
     5. 生成 output/step1_*.sql + errors.csv + summary.txt
 
@@ -14,11 +14,13 @@
 
 用法:
     python3 migrate_assets.py [--config-dir config] [--resources-dir resources] [--output-dir output]
+                              [--batch-size 1000]
 """
 from __future__ import annotations
 
 import argparse
 import sys
+import time
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -31,6 +33,7 @@ def main() -> int:
     parser.add_argument("--config-dir", default="config", help="yaml 配置目录")
     parser.add_argument("--resources-dir", default="resources", help="CSV 资源目录")
     parser.add_argument("--output-dir", default="output", help="SQL/CSV 输出目录")
+    parser.add_argument("--batch-size", type=int, default=legacy_query.BATCH_SIZE, help="老库批量查询大小")
     args = parser.parse_args()
 
     base = Path(__file__).resolve().parent
@@ -38,16 +41,26 @@ def main() -> int:
     resources_dir = (base / args.resources_dir).resolve()
     output_dir = (base / args.output_dir).resolve()
     output_dir.mkdir(parents=True, exist_ok=True)
+    batch_size = args.batch_size
 
+    t0 = time.perf_counter()
     mapping = mapping_loader.load_mapping(config_dir)
     cards, devices, errors = csv_loader.load_assets(resources_dir, mapping)
+    print(f"[migrate] 加载 CSV: cards={len(cards)} devices={len(devices)} elapsed={time.perf_counter()-t0:.1f}s")
 
     # 连奇成查卡元数据(运营商分类、agent_id、msisdn、virtual_no、当前生效套餐)
     iccid_pairs = [(c.iccid_19, c.iccid_20, c.allow_iccid_19_lookup) for c in cards]
+
+    t1 = time.perf_counter()
     dsn = mapping_loader.load_legacy_dsn(config_dir)
     with legacy_query.connect_readonly(dsn) as conn:
-        card_metas, dup_iccids = legacy_query.fetch_card_meta(conn, iccid_pairs)
+        print(f"[migrate] 连接老库: elapsed={time.perf_counter()-t1:.1f}s")
 
+        t2 = time.perf_counter()
+        card_metas, dup_iccids = legacy_query.fetch_card_meta(conn, iccid_pairs, batch_size=batch_size)
+        print(f"[migrate] fetch_card_meta: cards={len(card_metas)} duplicates={len(dup_iccids)} elapsed={time.perf_counter()-t2:.1f}s")
+
+    t3 = time.perf_counter()
     sql_builder.write_step1(
         output_dir,
         cards=cards,
@@ -57,8 +70,10 @@ def main() -> int:
         duplicate_iccids=dup_iccids,
         errors=errors,
     )
+    print(f"[migrate] SQL 生成: elapsed={time.perf_counter()-t3:.1f}s")
 
-    print(f"完成:卡 {len(cards)} 张,设备 {len(devices)} 台,异常 {len(errors)} 行")
+    total = time.perf_counter() - t0
+    print(f"完成:卡 {len(cards)} 张,设备 {len(devices)} 台,异常 {len(errors)} 行,总耗时 {total:.1f}s")
     print(f"SQL 输出: {output_dir}")
     return 0