数据迁移脚本

scripts/migration/lib/csv_loader.py

@@ -0,0 +1,272 @@
+"""cards.csv / devices.csv 加载与跨文件一致性校验。
+
+新设计:CSV 只承载"业务方决策"——
+    - cards.csv 只读 iccid + 可选覆盖列(target_shop_code、is_industry)
+    - devices.csv 读设备元数据 + sim_iccid_*
+    - 卡的运营商、套餐、msisdn、agent_id 等元数据全部从奇成查(见 legacy_query.fetch_card_meta)
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+CARD_HEADERS_REQUIRED = ("iccid_19",)
+# 设备列至少包含 imei 或 virtual_no 之一(运行时校验,允许只填一列)
+DEVICE_HEADERS_AT_LEAST_ONE = ("imei", "virtual_no")
+
+
+@dataclass
+class CardRow:
+    """cards.csv 单行的解析结果(只含业务方决策字段)。
+
+    业务方必须同时提供 iccid_19 和 iccid_20(20 位运营商):
+        - iccid_19:前 19 位,所有卡必填,作为跨文件稳定 ID
+        - iccid_20:20 位完整 ICCID,CMCC/CUCC/CBN 必填、CTCC 必须留空
+        - 运营商类型与 iccid_20 的一致性在查完奇成后由 sql_builder 校验
+    新系统 iccid 唯一键 = iccid_20 优先,否则 iccid_19。
+    """
+
+    line_no: int                # 原始 CSV 行号(从 1 开始,含表头)
+    iccid_19: str
+    iccid_20: str = ""          # 留空表示 CTCC(查完奇成会再校验)
+    is_industry: bool = False   # 默认 False(普通卡)
+
+    # 跨文件校验后填充
+    bound_device_virtual_no: str = ""
+
+    @property
+    def iccid_full(self) -> str:
+        """完整 ICCID:有 iccid_20 用 iccid_20,否则用 iccid_19。"""
+        return self.iccid_20 or self.iccid_19
+
+
+@dataclass
+class DeviceRow:
+    """devices.csv 单行的解析结果。
+
+    简化为业务方核心关心的列:imei / virtual_no / sim_iccid_*。
+    店铺归属由"主卡 sim_iccid_1 的 agent_id → mapping.agents.target_shop_code"推导。
+    其他字段(device_name/device_model/manufacturer 等)直接用默认值,不暴露给业务方。
+    """
+
+    line_no: int
+    imei: str
+    virtual_no: str                 # 留空时用 imei 兜底,见 _load_devices
+    sim_iccids: dict[int, str]      # slot → iccid(只保留非空)
+    max_sim_slots: int = 4          # 默认 4 槽位
+    current_slot: int = 1           # 默认 slot 1 为当前生效
+
+
+@dataclass
+class ErrorRow:
+    """异常行汇总(写入 output/errors.csv)。"""
+
+    source_file: str
+    line_no: int
+    field: str
+    value: str
+    error_code: str
+    message: str
+
+    def to_csv_row(self) -> list[str]:
+        return [self.source_file, str(self.line_no), self.field, self.value, self.error_code, self.message]
+
+
+@dataclass
+class LoadResult:
+    cards: list[CardRow] = field(default_factory=list)
+    devices: list[DeviceRow] = field(default_factory=list)
+    errors: list[ErrorRow] = field(default_factory=list)
+
+
+# ---------------- 字段清洗 ----------------
+
+
+def _normalize_cell(raw: str) -> str:
+    if raw is None:
+        return ""
+    s = raw.strip()
+    if s.startswith(""):
+        s = s[1:]
+    if s.startswith("'"):
+        s = s[1:]
+    s = s.replace(" ", "")
+    return s.strip()
+
+
+def _parse_bool(raw: str) -> Optional[bool]:
+    v = _normalize_cell(raw).lower()
+    if v == "":
+        return False
+    if v in ("true", "1", "yes", "y", "t", "是", "行业卡"):
+        return True
+    if v in ("false", "0", "no", "n", "f", "否", "普通卡"):
+        return False
+    return None
+
+
+def _parse_int(raw: str, default: int = 0) -> int:
+    v = _normalize_cell(raw)
+    if not v:
+        return default
+    try:
+        return int(v)
+    except ValueError:
+        return default
+
+
+# ---------------- 主入口 ----------------
+
+
+def load_assets(resources_dir: Path) -> tuple[list[CardRow], list[DeviceRow], list[ErrorRow]]:
+    """读 cards.csv + devices.csv,做格式校验与跨文件一致性校验。
+
+    返回:(合法卡列表, 合法设备列表, 异常行列表)。
+    元数据(carrier/agent_id/msisdn 等)由调用方再用 legacy_query 补充。
+    """
+    result = LoadResult()
+
+    cards_path = resources_dir / "cards.csv"
+    if not cards_path.exists():
+        raise FileNotFoundError(f"缺少 {cards_path},请按 resources/README.md 准备数据")
+    _load_cards(cards_path, result)
+
+    devices_path = resources_dir / "devices.csv"
+    if devices_path.exists():
+        _load_devices(devices_path, result)
+
+    _cross_check(result)
+
+    return result.cards, result.devices, result.errors
+
+
+def _load_cards(path: Path, result: LoadResult) -> None:
+    import csv
+
+    from . import iccid_utils
+
+    with path.open("r", encoding="utf-8-sig", newline="") as f:
+        reader = csv.DictReader(f)
+        header = [_normalize_cell(h) for h in (reader.fieldnames or [])]
+        missing = [k for k in CARD_HEADERS_REQUIRED if k not in header]
+        if missing:
+            raise ValueError(f"{path} 缺少必备列: {missing}")
+        if "iccid_20" not in header:
+            raise ValueError(f"{path} 必须包含 iccid_20 列(20 位运营商卡填写,CTCC 留空)")
+
+        seen_iccid_19: dict[str, int] = {}
+        for idx, raw in enumerate(reader, start=2):
+            iccid_19 = _normalize_cell(raw.get("iccid_19", ""))
+            iccid_20 = _normalize_cell(raw.get("iccid_20", ""))
+
+            err = iccid_utils.validate_iccid_19(iccid_19)
+            if err is not None:
+                result.errors.append(ErrorRow("cards.csv", idx, "iccid_19", iccid_19, err.code, err.message))
+                continue
+
+            err = iccid_utils.validate_iccid_20(iccid_20, iccid_19)
+            if err is not None:
+                result.errors.append(ErrorRow("cards.csv", idx, "iccid_20", iccid_20, err.code, err.message))
+                continue
+
+            if iccid_19 in seen_iccid_19:
+                result.errors.append(
+                    ErrorRow("cards.csv", idx, "iccid_19", iccid_19, "duplicate",
+                             f"与第 {seen_iccid_19[iccid_19]} 行 iccid_19 重复")
+                )
+                continue
+            seen_iccid_19[iccid_19] = idx
+
+            is_industry = _parse_bool(raw.get("is_industry", ""))
+            if is_industry is None:
+                result.errors.append(
+                    ErrorRow("cards.csv", idx, "is_industry", raw.get("is_industry", ""), "invalid_bool", "is_industry 必须是 true/false")
+                )
+                continue
+
+            result.cards.append(
+                CardRow(line_no=idx, iccid_19=iccid_19, iccid_20=iccid_20, is_industry=is_industry)
+            )
+
+
+def _load_devices(path: Path, result: LoadResult) -> None:
+    import csv
+
+    with path.open("r", encoding="utf-8-sig", newline="") as f:
+        reader = csv.DictReader(f)
+        header = [_normalize_cell(h) for h in (reader.fieldnames or [])]
+        if not any(k in header for k in DEVICE_HEADERS_AT_LEAST_ONE):
+            raise ValueError(f"{path} 必须至少包含 imei 或 virtual_no 列")
+
+        seen_virtual_no: dict[str, int] = {}
+        for idx, raw in enumerate(reader, start=2):
+            imei = _normalize_cell(raw.get("imei", ""))
+            virtual_no = _normalize_cell(raw.get("virtual_no", ""))
+            # virtual_no 留空时用 imei 兜底(业务方说"虚拟号一般会提供,没有就用 imei")
+            effective_vno = virtual_no or imei
+            if not effective_vno:
+                result.errors.append(
+                    ErrorRow("devices.csv", idx, "virtual_no", "", "missing_identity",
+                             "imei 和 virtual_no 不能同时为空")
+                )
+                continue
+            if effective_vno in seen_virtual_no:
+                result.errors.append(
+                    ErrorRow("devices.csv", idx, "virtual_no", effective_vno, "duplicate",
+                             f"与第 {seen_virtual_no[effective_vno]} 行的设备号重复")
+                )
+                continue
+            seen_virtual_no[effective_vno] = idx
+
+            sim_iccids: dict[int, str] = {}
+            for slot in (1, 2, 3, 4):
+                v = _normalize_cell(raw.get(f"sim_iccid_{slot}", ""))
+                if v:
+                    sim_iccids[slot] = v
+
+            result.devices.append(
+                DeviceRow(
+                    line_no=idx,
+                    imei=imei,
+                    virtual_no=effective_vno,
+                    sim_iccids=sim_iccids,
+                )
+            )
+
+
+def _cross_check(result: LoadResult) -> None:
+    """校验设备与卡之间的一致性,并把 bound_device_virtual_no 回填到 CardRow。
+
+    devices.csv 里 sim_iccid_X 业务方可能填 19 位也可能填 20 位,
+    统一截取前 19 位回查 cards.iccid_19。
+    """
+    from . import iccid_utils
+
+    cards_by_iccid_19: dict[str, CardRow] = {c.iccid_19: c for c in result.cards}
+    device_card_refs: dict[str, tuple[str, int]] = {}
+
+    for device in result.devices:
+        for slot, raw_iccid in device.sim_iccids.items():
+            key19 = iccid_utils.prefix_19(raw_iccid)
+            if key19 not in cards_by_iccid_19:
+                result.errors.append(
+                    ErrorRow(
+                        "devices.csv", device.line_no, f"sim_iccid_{slot}", raw_iccid,
+                        "card_not_found", "引用的 ICCID(前 19 位)在 cards.csv 中不存在",
+                    )
+                )
+                continue
+            if key19 in device_card_refs:
+                prev_vno, prev_line = device_card_refs[key19]
+                result.errors.append(
+                    ErrorRow(
+                        "devices.csv", device.line_no, f"sim_iccid_{slot}", raw_iccid,
+                        "card_used_twice", f"该 ICCID 已被设备 {prev_vno}(第 {prev_line} 行)使用",
+                    )
+                )
+                continue
+            device_card_refs[key19] = (device.virtual_no, device.line_no)
+            cards_by_iccid_19[key19].bound_device_virtual_no = device.virtual_no
+            # 标准化:slot 里的值统一改写为 iccid_19,sql_builder 直接用它查 cards / card_metas
+            device.sim_iccids[slot] = key19