#!/usr/bin/env python3 """扫描奇成,自动生成/增量补全 config/mapping.yaml。 工作流: 1. 读 resources/cards.csv 拿到 iccid 列表 2. 连奇成只读查询每张卡的 account/category / agent_id / 当前生效套餐 / 套餐系列 3. 加载已有的 config/mapping.yaml(若没有则创建空的) 4. 把扫描到的 legacy_* 增量补到 mapping 里(已有项的 target_* 字段保留,新项的 target_* 为 null) 5. 写回 config/mapping.yaml 6. 输出"待人工填写"清单到 console + config/mapping_todo.txt 用法: python3 scan_legacy.py [--config-dir config] [--resources-dir resources] """ from __future__ import annotations import argparse import csv import sys from dataclasses import dataclass from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import legacy_query, mapping_loader # noqa: E402 from lib.mapping_loader import ( # noqa: E402 AgentMapping, CarrierMapping, Mapping, PackageMapping, SeriesMapping, ) @dataclass(frozen=True) class ICCIDInput: """业务方提供的单行 ICCID 输入。""" line_no: int iccid_19: str iccid_20: str allow_iccid_19_lookup: bool @property def card_key(self) -> str: """奇成扫描结果使用的业务方完整键。""" return self.iccid_20 or self.iccid_19 def _read_iccid_pairs(cards_csv: Path) -> list[ICCIDInput]: """从 cards.csv 读 iccid_19 + iccid_20 列(只关心这两列,其他列忽略)。 返回带行号的 ICCID 输入列表。 只填 iccid_20 时,iccid_19 从前 19 位派生,但不用于奇成主表兜底查询。 """ if not cards_csv.exists(): raise FileNotFoundError(f"找不到 {cards_csv},请按 resources/README.md 准备") pairs: dict[str, ICCIDInput] = {} first_seen_line: dict[str, int] = {} errors: list[str] = [] with cards_csv.open("r", encoding="utf-8-sig", newline="") as f: reader = csv.DictReader(f) headers = reader.fieldnames or [] if "iccid_19" not in headers: raise ValueError(f"{cards_csv} 必须包含 iccid_19 列") if "iccid_20" not in headers: raise ValueError(f"{cards_csv} 必须包含 iccid_20 列(CTCC 卡留空)") for line_no, row in enumerate(reader, start=2): i19 = (row.get("iccid_19") or "").strip() i20 = (row.get("iccid_20") or "").strip() allow_i19_lookup = bool(i19) if not i19 and not i20 and not any((v or "").strip() for v in row.values()): continue if i20 and len(i20) != 20: errors.append(f"第 {line_no} 行 iccid_20 必须是 20 位: {i20!r}") continue if not i19: if not i20: errors.append(f"第 {line_no} 行 iccid_19 和 iccid_20 不能同时为空") continue i19 = i20[:19] if len(i19) != 19: errors.append(f"第 {line_no} 行 iccid_19 必须是 19 位: {i19!r}") continue if i20 and i20[:19] != i19: errors.append(f"第 {line_no} 行 iccid_20 前 19 位必须等于 iccid_19: {i20!r}") continue # 有 iccid_20 的卡用 iccid_20 做唯一键，避免同批次卡派生出相同 iccid_19 误报重复 dedup_key = i20 if i20 else i19 if dedup_key in pairs: errors.append(f"第 {line_no} 行 {'iccid_20' if i20 else 'iccid_19'} 与第 {first_seen_line[dedup_key]} 行重复: {dedup_key!r}") continue first_seen_line[dedup_key] = line_no pairs[dedup_key] = ICCIDInput( line_no=line_no, iccid_19=i19, iccid_20=i20, allow_iccid_19_lookup=allow_i19_lookup, ) if errors: preview = "\n".join(f" - {msg}" for msg in errors[:20]) if len(errors) > 20: preview += f"\n - 其余 {len(errors) - 20} 个错误已省略" raise ValueError(f"{cards_csv} 存在无效 ICCID,请先修正后再扫描:\n{preview}") return [v for _, v in sorted(pairs.items())] def main() -> int: parser = argparse.ArgumentParser(description="奇成元数据预扫描,自动生成 mapping.yaml") parser.add_argument("--config-dir", default="config", help="yaml 配置目录") parser.add_argument("--resources-dir", default="resources", help="CSV 资源目录") args = parser.parse_args() base = Path(__file__).resolve().parent config_dir = (base / args.config_dir).resolve() resources_dir = (base / args.resources_dir).resolve() try: iccid_pairs = _read_iccid_pairs(resources_dir / "cards.csv") except (FileNotFoundError, ValueError) as exc: print(str(exc), file=sys.stderr) return 1 if not iccid_pairs: print("cards.csv 中没有有效 ICCID,无需扫描", file=sys.stderr) return 1 print(f"读到 {len(iccid_pairs)} 个 ICCID,连接奇成扫描元数据...") dsn = mapping_loader.load_legacy_dsn(config_dir) with legacy_query.connect_readonly(dsn) as conn: card_metas, dup_iccids = legacy_query.fetch_card_meta( conn, [(x.iccid_19, x.iccid_20, x.allow_iccid_19_lookup) for x in iccid_pairs], ) meal_ids = {m.current_meal_id for m in card_metas.values() if m.current_meal_id} meal_metas = legacy_query.fetch_meal_meta(conn, meal_ids) if dup_iccids: print(f"警告:{len(dup_iccids)} 张卡的 ICCID 在奇成 tbl_card 命中多条记录:") for x in sorted(dup_iccids): print(f" - {x}") print("这些卡 migrate_assets 阶段会写 errors.csv 阻断,请业务方人工确认。") existing = mapping_loader.load_mapping(config_dir) merged = _merge(existing, card_metas, meal_metas) out_path = mapping_loader.merge_and_save(config_dir, merged) todo = _collect_todo(merged) unmatched = _collect_unmatched_iccids(iccid_pairs, card_metas, dup_iccids) _write_todo(config_dir / "mapping_todo.txt", todo) unmatched_path = config_dir / "unmatched_iccids.csv" _write_unmatched_iccids(unmatched_path, unmatched) _print_summary(out_path, card_metas, meal_metas, todo, unmatched_path, len(unmatched)) return 0 def _merge( existing: Mapping, card_metas: dict[str, "legacy_query.LegacyCardMeta"], meal_metas: dict[str, "legacy_query.LegacyMealMeta"], ) -> Mapping: """把扫描到的 legacy_* 增量合并到现有 mapping(保留已填的 target_*)。""" merged = Mapping( migration_user_id=existing.migration_user_id, migration_batch_no=existing.migration_batch_no, ownership_rules=existing.ownership_rules, package_rules=existing.package_rules, overrides=existing.overrides, carriers=dict(existing.carriers), packages=dict(existing.packages), series=dict(existing.series), agents=dict(existing.agents), ) for meta in card_metas.values(): # carrier:按开卡公司/运营商账号映射,category 仅作为辅助信息。 if meta.account_id and meta.account_id not in merged.carriers: merged.carriers[meta.account_id] = CarrierMapping( legacy_account_id=meta.account_id, legacy_account_name=meta.account_name, legacy_category=meta.category, legacy_category_name=meta.category_name, target_carrier_id=None, target_carrier_type=None, target_carrier_name=None, ) elif meta.account_id in merged.carriers and ( not merged.carriers[meta.account_id].legacy_account_name or not merged.carriers[meta.account_id].legacy_category_name ): # 补全 legacy_* 快照(如果之前是空的) old = merged.carriers[meta.account_id] merged.carriers[meta.account_id] = CarrierMapping( legacy_account_id=old.legacy_account_id, legacy_account_name=meta.account_name or old.legacy_account_name, legacy_category=old.legacy_category, legacy_category_name=meta.category_name or old.legacy_category_name, target_carrier_id=old.target_carrier_id, target_carrier_type=old.target_carrier_type, target_carrier_name=old.target_carrier_name, ) # agent if meta.agent_id and meta.agent_id not in merged.agents: merged.agents[meta.agent_id] = AgentMapping( legacy_agent_id=meta.agent_id, legacy_agent_name=meta.agent_name, target_shop_code=None, ) for mm in meal_metas.values(): if mm.meal_id not in merged.packages: merged.packages[mm.meal_id] = PackageMapping( legacy_meal_id=mm.meal_id, legacy_meal_name=mm.meal_name, target_package_id=None, ) if mm.series_id and mm.series_id not in merged.series: merged.series[mm.series_id] = SeriesMapping( legacy_series_id=mm.series_id, legacy_series_name=mm.series_name, target_series_id=None, ) return merged def _collect_todo(mapping: Mapping) -> dict[str, list[str]]: """收集所有待填写的项,按类别分组。""" todo: dict[str, list[str]] = {"carriers": [], "packages": [], "agents": []} for c in sorted(mapping.carriers.values(), key=lambda x: (x.legacy_account_name, x.legacy_account_id)): if not c.target_carrier_id or not c.target_carrier_type or not c.target_carrier_name: todo["carriers"].append( f"account_id={c.legacy_account_id} ({c.legacy_account_name}, " f"category={c.legacy_category}/{c.legacy_category_name})" ) for p in sorted(mapping.packages.values(), key=lambda x: x.legacy_meal_id): if not p.target_package_id: todo["packages"].append(f"meal_id={p.legacy_meal_id} ({p.legacy_meal_name})") for a in sorted(mapping.agents.values(), key=lambda x: x.legacy_agent_id): if not (a.target_shop_code or "").strip(): todo["agents"].append(f"agent_id={a.legacy_agent_id} ({a.legacy_agent_name})") return todo def _collect_unmatched_iccids( iccid_pairs: list[ICCIDInput], card_metas: dict[str, "legacy_query.LegacyCardMeta"], duplicate_iccids: set[str], ) -> list[ICCIDInput]: """收集未在奇成 tbl_card 命中的 ICCID。 重复命中的卡属于脏数据异常,不是未命中,因此从未命中清单中排除。 """ resolved_keys = set(card_metas) | set(duplicate_iccids) return [item for item in iccid_pairs if item.card_key not in resolved_keys] def _write_todo(path: Path, todo: dict[str, list[str]]) -> None: lines = ["奇成迁移映射待填写清单", "=" * 40, ""] for kind in ("carriers", "packages", "agents"): lines.append(f"## {kind} (待填: {len(todo[kind])})") for item in todo[kind]: lines.append(f" - {item}") lines.append("") path.write_text("\n".join(lines), encoding="utf-8") def _write_unmatched_iccids(path: Path, unmatched: list[ICCIDInput]) -> None: """把未命中的 ICCID 写成 CSV,便于转给业务方核对。""" path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8-sig", newline="") as f: writer = csv.DictWriter( f, fieldnames=["line_no", "iccid_19", "iccid_20", "lookup_keys", "note"], ) writer.writeheader() for item in unmatched: lookup_keys = [item.iccid_20 or item.iccid_19] if item.iccid_20 and item.allow_iccid_19_lookup: lookup_keys.append(item.iccid_19) writer.writerow( { "line_no": item.line_no, "iccid_19": item.iccid_19, "iccid_20": item.iccid_20, "lookup_keys": ";".join(lookup_keys), "note": "奇成 tbl_card 未命中", } ) def _print_summary( out_path: Path, card_metas, meal_metas, todo: dict[str, list[str]], unmatched_path: Path, unmatched_count: int, ) -> None: print(f"扫描完成,mapping 写入: {out_path}") print(f" 奇成卡命中 : {len(card_metas)}") print(f" 奇成卡未命中 : {unmatched_count}") print(f" 奇成套餐命中 : {len(meal_metas)}") print(f" 待填 carriers : {len(todo['carriers'])}") print(f" 待填 packages : {len(todo['packages'])}") print(f" 待填 agents : {len(todo['agents'])}") if unmatched_count: print(f"未命中清单: {unmatched_path}") if any(todo.values()): print(f"详见: {out_path.parent / 'mapping_todo.txt'}") if __name__ == "__main__": sys.exit(main())