迁移脚本的一些修复
This commit is contained in:
Break
@@ -11,7 +11,7 @@ from decimal import Decimal
|
||||
from typing import Any, Iterable, Iterator, Optional
|
||||
|
||||
|
||||
BATCH_SIZE = 500
|
||||
BATCH_SIZE = 200
|
||||
|
||||
|
||||
def _normalize_iccid_pairs(iccid_pairs: Iterable[tuple]) -> list[tuple[str, str, bool]]:
|
||||
@@ -67,7 +67,7 @@ class LegacyCardUsage:
|
||||
奇成没有按套餐分段的用量明细,无更好做法。
|
||||
"""
|
||||
|
||||
iccid: str # 业务方视角的 iccid_19
|
||||
iccid: str # 业务方视角的完整 ICCID
|
||||
virtual_used_mb: Decimal # 奇成 total_bytes_cnt 原始值(虚 MB)
|
||||
flow_add_discount_pct: Decimal # 当前生效套餐虚比百分比(0~100)
|
||||
has_active_package: bool # 是否在 tbl_card_life 中找到当前生效套餐
|
||||
@@ -273,28 +273,32 @@ def fetch_card_usage(conn, iccid_pairs: Iterable[tuple]) -> dict[str, LegacyCard
|
||||
iccid_pairs: (iccid_19, iccid_20, allow_iccid_19_lookup) 列表。
|
||||
|
||||
Returns:
|
||||
{iccid_19: LegacyCardUsage}。
|
||||
{完整 ICCID: LegacyCardUsage}。
|
||||
"""
|
||||
pairs = _normalize_iccid_pairs(iccid_pairs)
|
||||
out: dict[str, LegacyCardUsage] = {}
|
||||
if not pairs:
|
||||
return out
|
||||
|
||||
full_to_19: dict[str, tuple[str, int]] = {}
|
||||
related_keys_by_19: dict[str, set[str]] = {}
|
||||
query_to_full: dict[str, tuple[str, int]] = {}
|
||||
related_keys_by_full: dict[str, set[str]] = {}
|
||||
for i19, i20, allow_i19_lookup in pairs:
|
||||
card_key = i20 or i19
|
||||
# tbl_card 主表同时查优先键和兜底键,结果选择时按 rank:
|
||||
# 0=业务方完整 ICCID 精确命中,1=20 位卡在奇成主表只存 19 位时兜底。
|
||||
full_to_19[i20 or i19] = (i19, 0)
|
||||
query_to_full[card_key] = (card_key, 0)
|
||||
if i20 and allow_i19_lookup:
|
||||
full_to_19[i19] = (i19, 1)
|
||||
related_keys_by_19.setdefault(i19, set()).add(i19)
|
||||
query_to_full[i19] = (card_key, 1)
|
||||
if not i20 or allow_i19_lookup:
|
||||
related_keys_by_full.setdefault(card_key, set()).add(i19)
|
||||
else:
|
||||
related_keys_by_full.setdefault(card_key, set())
|
||||
if i20:
|
||||
related_keys_by_19[i19].add(i20)
|
||||
full_set = sorted(full_to_19.keys())
|
||||
related_keys_by_full[card_key].add(i20)
|
||||
full_set = sorted(query_to_full.keys())
|
||||
|
||||
# tbl_card 主表用 iccid_mark IN 全长精确匹配;tbl_card_life 从表用
|
||||
# LEFT(iccid_mark, 19) 兜底,兼容奇成两张表 iccid 长度不一致的脏数据。
|
||||
# tbl_card 主表用 iccid_mark IN 全长精确匹配;从表只在主表本身是 19 位时
|
||||
# 按前缀兜底,避免同前缀 20 位卡互相串套餐。
|
||||
sql = """
|
||||
SELECT
|
||||
c.iccid_mark,
|
||||
@@ -314,30 +318,33 @@ def fetch_card_usage(conn, iccid_pairs: Iterable[tuple]) -> dict[str, LegacyCard
|
||||
ON latest.iccid_mark = cl1.iccid_mark
|
||||
AND latest.max_expire = cl1.expire_date
|
||||
WHERE cl1.status = 1
|
||||
) cl ON LEFT(cl.iccid_mark, 19) = LEFT(c.iccid_mark, 19)
|
||||
) cl ON (
|
||||
cl.iccid_mark = c.iccid_mark
|
||||
OR (CHAR_LENGTH(c.iccid_mark) = 19 AND LEFT(cl.iccid_mark, 19) = c.iccid_mark)
|
||||
)
|
||||
LEFT JOIN tbl_set_meal sm ON sm.id = cl.meal_id
|
||||
WHERE c.iccid_mark IN %s
|
||||
"""
|
||||
hits: dict[str, list[tuple[int, dict]]] = {}
|
||||
with conn.cursor() as cur:
|
||||
for batch in _chunks(full_set):
|
||||
batch_i19s = {full_to_19[x][0] for x in batch}
|
||||
related_batch = sorted({x for i19 in batch_i19s for x in related_keys_by_19[i19]})
|
||||
batch_keys = {query_to_full[x][0] for x in batch}
|
||||
related_batch = sorted({x for key in batch_keys for x in related_keys_by_full[key]})
|
||||
cur.execute(sql, (tuple(related_batch), tuple(batch)))
|
||||
for row in cur.fetchall():
|
||||
legacy_iccid = row["iccid_mark"]
|
||||
if not legacy_iccid:
|
||||
continue
|
||||
matched = full_to_19.get(legacy_iccid)
|
||||
matched = query_to_full.get(legacy_iccid)
|
||||
if matched is None:
|
||||
continue
|
||||
i19, rank = matched
|
||||
hits.setdefault(i19, []).append((rank, row))
|
||||
for i19, rows in hits.items():
|
||||
card_key, rank = matched
|
||||
hits.setdefault(card_key, []).append((rank, row))
|
||||
for card_key, rows in hits.items():
|
||||
best_rank = min(rank for rank, _row in rows)
|
||||
row = next(row for rank, row in rows if rank == best_rank)
|
||||
out[i19] = LegacyCardUsage(
|
||||
iccid=i19,
|
||||
out[card_key] = LegacyCardUsage(
|
||||
iccid=card_key,
|
||||
virtual_used_mb=_to_mb_decimal(row.get("total_bytes_cnt")),
|
||||
flow_add_discount_pct=_to_mb_decimal(row.get("flow_add_discount")),
|
||||
has_active_package=bool(row.get("has_active_package")),
|
||||
@@ -388,31 +395,35 @@ def fetch_card_meta(
|
||||
- tbl_card 主表同时查优先键和显式允许的兜底键:有 iccid_20 时优先取
|
||||
20 位精确命中;只有业务方也显式填了 iccid_19 时,才允许退回 19 位。
|
||||
只填 iccid_20 的固定 20 位卡绝不拿 19 位前缀查主表。
|
||||
- tbl_card_life / tbl_virtual_number 从表用 LEFT(iccid_mark, 19) 兜底,
|
||||
应对奇成两张表 iccid 长度不一致的脏数据
|
||||
- tbl_card_life / tbl_virtual_number 从表在主表本身是 19 位时才用
|
||||
LEFT(iccid_mark, 19) 兜底,避免同前缀 20 位卡互相串数据
|
||||
|
||||
Args:
|
||||
iccid_pairs: (iccid_19, iccid_20, allow_iccid_19_lookup) 列表。
|
||||
|
||||
Returns:
|
||||
(metas, duplicate_iccid_19):
|
||||
- metas: {iccid_19: LegacyCardMeta} 单匹结果
|
||||
- duplicate_iccid_19: 一份业务方 iccid_19 命中奇成 tbl_card 多条记录的 set,
|
||||
(metas, duplicate_iccids):
|
||||
- metas: {完整 ICCID: LegacyCardMeta} 单匹结果
|
||||
- duplicate_iccids: 一份业务方完整 ICCID 命中奇成 tbl_card 多条记录的 set,
|
||||
这些卡不进 metas,sql_builder 会写 errors.csv 阻断
|
||||
"""
|
||||
pairs = _normalize_iccid_pairs(iccid_pairs)
|
||||
if not pairs:
|
||||
return {}, set()
|
||||
|
||||
by_full: dict[str, tuple[str, int]] = {} # tbl_card 查询键 → (iccid_19, rank)
|
||||
related_keys_by_19: dict[str, set[str]] = {}
|
||||
by_full: dict[str, tuple[str, int]] = {} # tbl_card 查询键 → (完整 ICCID, rank)
|
||||
related_keys_by_full: dict[str, set[str]] = {}
|
||||
for i19, i20, allow_i19_lookup in pairs:
|
||||
by_full[i20 or i19] = (i19, 0)
|
||||
card_key = i20 or i19
|
||||
by_full[card_key] = (card_key, 0)
|
||||
if i20 and allow_i19_lookup:
|
||||
by_full[i19] = (i19, 1)
|
||||
related_keys_by_19.setdefault(i19, set()).add(i19)
|
||||
by_full[i19] = (card_key, 1)
|
||||
if not i20 or allow_i19_lookup:
|
||||
related_keys_by_full.setdefault(card_key, set()).add(i19)
|
||||
else:
|
||||
related_keys_by_full.setdefault(card_key, set())
|
||||
if i20:
|
||||
related_keys_by_19[i19].add(i20)
|
||||
related_keys_by_full[card_key].add(i20)
|
||||
full_set = sorted(by_full.keys())
|
||||
|
||||
sql = """
|
||||
@@ -432,7 +443,10 @@ def fetch_card_meta(
|
||||
cl.expire_date AS expire_date
|
||||
FROM tbl_card c
|
||||
LEFT JOIN tbl_vendor_category vc ON vc.category = c.category
|
||||
LEFT JOIN tbl_virtual_number vn ON LEFT(vn.iccid_mark, 19) = LEFT(c.iccid_mark, 19)
|
||||
LEFT JOIN tbl_virtual_number vn ON (
|
||||
vn.iccid_mark = c.iccid_mark
|
||||
OR (CHAR_LENGTH(c.iccid_mark) = 19 AND LEFT(vn.iccid_mark, 19) = c.iccid_mark)
|
||||
)
|
||||
LEFT JOIN (
|
||||
SELECT cl1.iccid_mark, cl1.meal_id, cl1.meal_name, cl1.expire_date
|
||||
FROM tbl_card_life cl1
|
||||
@@ -445,16 +459,19 @@ def fetch_card_meta(
|
||||
ON latest.iccid_mark = cl1.iccid_mark
|
||||
AND latest.max_expire = cl1.expire_date
|
||||
WHERE cl1.status = 1
|
||||
) cl ON LEFT(cl.iccid_mark, 19) = LEFT(c.iccid_mark, 19)
|
||||
) cl ON (
|
||||
cl.iccid_mark = c.iccid_mark
|
||||
OR (CHAR_LENGTH(c.iccid_mark) = 19 AND LEFT(cl.iccid_mark, 19) = c.iccid_mark)
|
||||
)
|
||||
WHERE c.iccid_mark IN %s
|
||||
"""
|
||||
|
||||
# 收集每个 iccid_19 命中的奇成行(可能多条 = 脏数据)
|
||||
# 收集每个完整 ICCID 命中的奇成行(可能多条 = 脏数据)
|
||||
hits: dict[str, list[tuple[int, dict]]] = {}
|
||||
with conn.cursor() as cur:
|
||||
for batch in _chunks(full_set):
|
||||
batch_i19s = {by_full[x][0] for x in batch}
|
||||
related_batch = sorted({x for i19 in batch_i19s for x in related_keys_by_19[i19]})
|
||||
batch_keys = {by_full[x][0] for x in batch}
|
||||
related_batch = sorted({x for key in batch_keys for x in related_keys_by_full[key]})
|
||||
cur.execute(sql, (tuple(related_batch), tuple(batch)))
|
||||
for row in cur.fetchall():
|
||||
legacy_iccid = row["iccid"]
|
||||
@@ -463,12 +480,12 @@ def fetch_card_meta(
|
||||
matched = by_full.get(legacy_iccid)
|
||||
if matched is None:
|
||||
continue
|
||||
i19, rank = matched
|
||||
hits.setdefault(i19, []).append((rank, row))
|
||||
card_key, rank = matched
|
||||
hits.setdefault(card_key, []).append((rank, row))
|
||||
|
||||
metas: dict[str, LegacyCardMeta] = {}
|
||||
duplicates: set[str] = set()
|
||||
for i19, rows in hits.items():
|
||||
for card_key, rows in hits.items():
|
||||
best_rank = min(rank for rank, _row in rows)
|
||||
best_rows = [row for rank, row in rows if rank == best_rank]
|
||||
# 同一张 tbl_card 可能因 card_life / virtual_number 前缀兜底 JOIN 出多行,
|
||||
@@ -477,12 +494,12 @@ def fetch_card_meta(
|
||||
for row in best_rows:
|
||||
rows_by_card_id[str(row.get("card_id") or row.get("iccid") or "")] = row
|
||||
if len(rows_by_card_id) > 1:
|
||||
duplicates.add(i19)
|
||||
duplicates.add(card_key)
|
||||
continue
|
||||
row = next(iter(rows_by_card_id.values()))
|
||||
expire = row.get("expire_date")
|
||||
expire_str = expire.strftime("%Y-%m-%d %H:%M:%S") if expire else None
|
||||
metas[i19] = LegacyCardMeta(
|
||||
metas[card_key] = LegacyCardMeta(
|
||||
legacy_raw_iccid=row["iccid"],
|
||||
account_id=row.get("account_id") or "",
|
||||
account_name=row.get("account_name") or "",
|
||||
|
||||
Reference in New Issue
Block a user