csxj2026/junhong_cmp_fiber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

迁移脚本的一些修复

Browse Source
This commit is contained in:
Break
2026-06-09 11:29:40 +08:00
parent 6a70713b30
commit 283e3e3eb3
8 changed files with 183 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
scripts/migration/lib/sql_builder.py
View File

@@ -144,8 +144,8 @@ def _device_package_source_iccid(device: DeviceRow) -> str:
)
def _target_series_id_expr_for_iccid_19(
iccid_19: str,
def _target_series_id_expr_for_iccid(
iccid: str,
card_metas: dict[str, LegacyCardMeta],
mapping: Mapping,
) -> str:
@@ -153,9 +153,9 @@ def _target_series_id_expr_for_iccid_19(
系列归属以新系统目标套餐为准，避免奇成套餐系列与新库系列二次映射不一致。
"""
if not iccid_19:
if not iccid:
return "NULL"
meta = card_metas.get(iccid_19)
meta = card_metas.get(iccid)
if meta is None or not meta.current_meal_id:
return "NULL"
pkg_map = mapping.lookup_package(meta.current_meal_id)
@@ -200,7 +200,7 @@ def _runtime_asset_for_card(card: CardRow, devices_by_virtual_no: dict[str, Devi
device = devices_by_virtual_no.get(card.bound_device_virtual_no)
if device is None:
return None
if card.iccid_19 != _device_package_source_iccid(device):
if card.iccid_full != _device_package_source_iccid(device):
return None
return "device", device.virtual_no
@@ -217,7 +217,7 @@ def write_step1(
devices: list[DeviceRow],
mapping: Mapping,
card_metas: dict[str, LegacyCardMeta],
duplicate_iccid_19s: set[str],
duplicate_iccids: set[str],
errors: list[ErrorRow],
) -> None:
"""生成 step1_* SQL + errors.csv + summary.txt。
@@ -225,21 +225,21 @@ def write_step1(
会就地修改 errors 列表,把映射缺失等问题追加进去。
Args:
card_metas: {iccid_19: LegacyCardMeta}
duplicate_iccid_19s: 同一 iccid_19 在奇成命中多条 tbl_card 记录的集合,直接报错阻断
card_metas: {完整 ICCID: LegacyCardMeta}
duplicate_iccids: 同一完整 ICCID 在奇成命中多条 tbl_card 记录的集合,直接报错阻断
"""
user_id = mapping.migration_user_id
# 预检 carrier 映射 + 运营商一致性 + 重复命中,把异常卡剔除
valid_cards = _filter_cards_by_carrier(cards, card_metas, duplicate_iccid_19s, mapping, errors)
valid_cards = _filter_cards_by_carrier(cards, card_metas, duplicate_iccids, mapping, errors)
cards_by_iccid_19 = {c.iccid_19: c for c in valid_cards}
cards_by_iccid = {c.iccid_full: c for c in valid_cards}
_write_iot_cards(output_dir / "step1_01_iot_cards.sql", valid_cards, devices, card_metas, mapping, user_id)
_write_devices(output_dir / "step1_02_devices.sql", devices, card_metas, mapping, user_id)
_write_asset_identifiers(output_dir / "step1_03_asset_identifiers.sql", valid_cards, devices, card_metas)
_write_asset_wallets(output_dir / "step1_04_asset_wallets.sql", valid_cards, devices)
_write_sim_bindings(output_dir / "step1_05_sim_bindings.sql", devices, cards_by_iccid_19)
_write_sim_bindings(output_dir / "step1_05_sim_bindings.sql", devices, cards_by_iccid)
_write_shop_allocations(
output_dir / "step1_06_shop_allocations.sql", valid_cards, devices, card_metas, mapping, user_id
)
@@ -251,29 +251,29 @@ def write_step1(
def _filter_cards_by_carrier(
cards: list[CardRow],
card_metas: dict[str, LegacyCardMeta],
duplicate_iccid_19s: set[str],
duplicate_iccids: set[str],
mapping: Mapping,
errors: list[ErrorRow],
) -> list[CardRow]:
"""筛掉:奇成查不到 / 命中多条 / carrier 映射缺失 / target_* 未填 / 运营商与 iccid 长度不一致 的卡。"""
valid: list[CardRow] = []
for c in cards:
if c.iccid_19 in duplicate_iccid_19s:
if c.iccid_full in duplicate_iccids:
errors.append(ErrorRow(
"cards.csv", c.line_no, "iccid_19", c.iccid_19,
"legacy_duplicate", "奇成 tbl_card 中同一 iccid_19 命中多条记录,请业务方人工确认",
"cards.csv", c.line_no, "iccid", c.iccid_full,
"legacy_duplicate", "奇成 tbl_card 中同一 ICCID 命中多条记录,请业务方人工确认",
))
continue
meta = card_metas.get(c.iccid_19)
meta = card_metas.get(c.iccid_full)
if meta is None:
errors.append(ErrorRow(
"cards.csv", c.line_no, "iccid_19", c.iccid_19,
"cards.csv", c.line_no, "iccid", c.iccid_full,
"not_in_legacy", "奇成 tbl_card 中找不到该 ICCID(iccid_19/iccid_20 都没命中)",
))
continue
if not meta.account_id:
errors.append(ErrorRow(
"cards.csv", c.line_no, "iccid_19", c.iccid_19,
"cards.csv", c.line_no, "iccid", c.iccid_full,
"no_carrier_account", "奇成 tbl_card.account_id 为空,无法定位具体运营商账号映射",
))
continue
@@ -320,7 +320,7 @@ def _write_iot_cards(
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step1_01 卡资产导入 (tb_iot_card)"))
for c in cards:
meta = card_metas[c.iccid_19]
meta = card_metas[c.iccid_full]
carrier = mapping.lookup_carrier(meta.account_id)
assert carrier is not None # 已被 _filter_cards_by_carrier 保证
@@ -332,7 +332,7 @@ def _write_iot_cards(
if c.bound_device_virtual_no:
device = devices_by_virtual_no.get(c.bound_device_virtual_no)
shop_code = _resolve_device_shop_code(device, card_metas, mapping) if device else ""
series_id_expr = _target_series_id_expr_for_iccid_19(c.iccid_19, card_metas, mapping)
series_id_expr = _target_series_id_expr_for_iccid(c.iccid_full, card_metas, mapping)
card_virtual_no = _card_virtual_no_for_import(meta, device_virtual_nos)
is_standalone = "FALSE" if c.bound_device_virtual_no else "TRUE"
device_virtual_no = c.bound_device_virtual_no or ""
@@ -408,7 +408,7 @@ def _write_devices(
device_status = 2 if shop_code else 1
shop_id_expr = _shop_id_sub(shop_code)
source_iccid = _device_package_source_iccid(d)
series_id_expr = _target_series_id_expr_for_iccid_19(source_iccid, card_metas, mapping)
series_id_expr = _target_series_id_expr_for_iccid(source_iccid, card_metas, mapping)
f.write(
"INSERT INTO tb_device (\n"
" virtual_no, max_sim_slots, imei,\n"
@@ -480,7 +480,7 @@ def _write_asset_identifiers(
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step1_03 资产标识符注册 (tb_asset_identifier)"))
for c in cards:
meta = card_metas[c.iccid_19]
meta = card_metas[c.iccid_full]
card_virtual_no = _card_virtual_no_for_import(meta, device_virtual_nos)
f.write(
"INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n"
@@ -563,13 +563,13 @@ def _write_asset_wallets(path: Path, cards: list[CardRow], devices: list[DeviceR
def _write_sim_bindings(
path: Path,
devices: list[DeviceRow],
cards_by_iccid_19: dict[str, CardRow],
cards_by_iccid: dict[str, CardRow],
) -> None:
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step1_05 设备-卡绑定 (tb_device_sim_binding)"))
for d in devices:
for slot, iccid_19 in sorted(d.sim_iccids.items()):
card = cards_by_iccid_19.get(iccid_19)
for slot, iccid in sorted(d.sim_iccids.items()):
card = cards_by_iccid.get(iccid)
if card is None:
continue # 该卡可能被 carrier 校验剔除,跳过此 slot 绑定
is_current = "TRUE" if slot == d.current_slot else "FALSE"
@@ -632,7 +632,7 @@ def _write_shop_allocations(
for c in cards:
if c.bound_device_virtual_no:
continue
meta = card_metas[c.iccid_19]
meta = card_metas[c.iccid_full]
shop_code = _resolve_card_shop_code(c, meta, mapping)
if not shop_code:
continue
@@ -685,6 +685,11 @@ def _write_errors(path: Path, errors: list[ErrorRow]) -> None:
writer.writerow(e.to_csv_row())
def write_runtime_input_errors(output_dir: Path, errors: list[ErrorRow]) -> None:
"""写运行态迁移的输入错误。"""
_write_errors(output_dir / "errors.csv", errors)
def _write_summary_step1(path: Path, cards: list[CardRow], devices: list[DeviceRow], errors: list[ErrorRow]) -> None:
bound = sum(1 for c in cards if c.bound_device_virtual_no)
lines = [
@@ -723,32 +728,32 @@ def write_step2(
warnings: list[tuple[str, str, str]] = []
devices_by_virtual_no = _device_by_virtual_no(devices)
# 卡 → package(从 card_meta.current_meal_id 经 mapping.packages 查 target)
resolved_packages: dict[str, tuple[int, str]] = {} # iccid_19 → (target_package_id, expire_date)
resolved_packages: dict[str, tuple[int, str]] = {} # 完整 ICCID → (target_package_id, expire_date)
for c in cards:
meta = card_metas.get(c.iccid_19)
meta = card_metas.get(c.iccid_full)
if meta is None or not meta.current_meal_id:
continue
pkg_map = mapping.lookup_package(meta.current_meal_id)
if pkg_map is None:
warnings.append(("package_not_in_mapping", c.iccid_19,
warnings.append(("package_not_in_mapping", c.iccid_full,
f"奇成 meal_id={meta.current_meal_id} ({meta.current_meal_name}) 未在 mapping.yaml.packages"))
continue
try:
pkg_map.require_target()
except MissingTargetError as exc:
warnings.append(("package_target_missing", c.iccid_19, str(exc)))
warnings.append(("package_target_missing", c.iccid_full, str(exc)))
continue
resolved_packages[c.iccid_19] = (pkg_map.target_package_id, meta.current_expire_date or "")
resolved_packages[c.iccid_full] = (pkg_map.target_package_id, meta.current_expire_date or "")
# 用量预检:虚用量 > 0 但没找到当前生效套餐的卡,无法精确反算真用量
# (按 flow_add_discount=0 处理,即真=虚,可能高估真用量;不阻断)
for c in cards:
snap = usage_snapshots.get(c.iccid_19)
snap = usage_snapshots.get(c.iccid_full)
if snap is None or snap.virtual_used_mb <= 0:
continue
if not snap.has_active_package:
warnings.append((
"usage_without_active_package", c.iccid_19,
"usage_without_active_package", c.iccid_full,
f"奇成虚用量 {snap.virtual_used_mb}MB,但 tbl_card_life 中没有当前生效套餐"
"(status=1),无法精确反算真用量,按 flow_add_discount=0 处理(真=虚)",
))
@@ -805,7 +810,7 @@ def _write_migration_orders(
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step2_01 迁移伪订单 (tb_order)"))
for c in cards:
if c.iccid_19 not in resolved_packages:
if c.iccid_full not in resolved_packages:
continue
asset = _runtime_asset_for_card(c, devices_by_virtual_no)
if asset is None:
@@ -873,16 +878,16 @@ def _write_package_usages(
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step2_02 套餐使用记录 (tb_package_usage)"))
for c in cards:
if c.iccid_19 not in resolved_packages:
if c.iccid_full not in resolved_packages:
continue
asset = _runtime_asset_for_card(c, devices_by_virtual_no)
if asset is None:
continue
asset_type, _asset_identifier = asset
target_pkg_id, expire = resolved_packages[c.iccid_19]
target_pkg_id, expire = resolved_packages[c.iccid_full]
order_no = _make_card_order_no(c.iccid_full)
expires_expr = f"{_sql_str(expire)}::timestamp" if expire else "NULL"
snap = usage_snapshots.get(c.iccid_19)
snap = usage_snapshots.get(c.iccid_full)
used_mb = snap.real_used_mb_floor if snap is not None else 0
effective_limit_expr = (
"CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 "
@@ -935,7 +940,7 @@ def _write_card_data_usage(path: Path, cards: list[CardRow], usage_snapshots: di
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step2_03 卡累计流量更新 (tb_iot_card 流量基线)"))
for c in cards:
snap = usage_snapshots.get(c.iccid_19)
snap = usage_snapshots.get(c.iccid_full)
if snap is None or snap.real_used_mb <= 0:
continue
real_decimal = _sql_decimal(snap.real_used_mb)
@@ -967,9 +972,9 @@ def _write_asset_series_update(
with path.open("w", encoding="utf-8") as f:
f.write(_file_header("step2_06 资产套餐系列回填 (tb_iot_card / tb_device)"))
for c in cards:
if c.iccid_19 not in resolved_packages:
if c.iccid_full not in resolved_packages:
continue
target_pkg_id, _expire = resolved_packages[c.iccid_19]
target_pkg_id, _expire = resolved_packages[c.iccid_full]
f.write(
"UPDATE tb_iot_card c\n"
"SET series_id = p.series_id,\n"
@@ -1124,7 +1129,7 @@ def _write_summary_step2(path: Path, *, cards, resolved, devices_by_virtual_no,
generated_count = sum(
1
for c in cards
if c.iccid_19 in resolved and _runtime_asset_for_card(c, devices_by_virtual_no) is not None
if c.iccid_full in resolved and _runtime_asset_for_card(c, devices_by_virtual_no) is not None
)
lines = [
"奇成迁移 - 脚本2 关联数据导入摘要",