This commit is contained in:
845
scripts/migration/lib/sql_builder.py
Normal file
845
scripts/migration/lib/sql_builder.py
Normal file
@@ -0,0 +1,845 @@
|
||||
"""SQL 输出构造。
|
||||
|
||||
输出文件:
|
||||
step1_01_iot_cards.sql - 卡资产
|
||||
step1_02_devices.sql - 设备资产
|
||||
step1_03_asset_identifiers.sql- 全局标识符(iccid / virtual_no)
|
||||
step1_04_asset_wallets.sql - 资产钱包
|
||||
step1_05_sim_bindings.sql - 设备-卡绑定
|
||||
step1_06_shop_allocations.sql - 店铺归属的分配记录
|
||||
|
||||
step2_01_migration_orders.sql - 迁移伪订单
|
||||
step2_02_package_usages.sql - 套餐使用记录
|
||||
step2_03_card_data_usage_update.sql - 卡累计流量 UPDATE
|
||||
step2_04_agent_wallet_init.sql - 代理钱包初始化(main + commission)
|
||||
step2_05_agent_wallet_transactions.sql - 代理钱包初始化流水
|
||||
|
||||
设计原则:
|
||||
- 所有 INSERT 通过 SELECT 子查询动态取 id,不硬编码新库 id
|
||||
- 使用 ON CONFLICT (...) WHERE ... DO NOTHING 保证幂等(PG 14+ 支持 partial index 推断)
|
||||
- 字符串 ' 转义为 ''
|
||||
- 卡的元数据(carrier/agent_id/msisdn/virtual_no/当前套餐)从奇成 fetch_card_meta 取,
|
||||
cards.csv 只承载 is_industry 覆盖,店铺归属完全由 mapping.agents 推导
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from . import iccid_utils
|
||||
from .csv_loader import CardRow, DeviceRow, ErrorRow
|
||||
from .legacy_query import (
|
||||
LegacyAgentBalance,
|
||||
LegacyCardMeta,
|
||||
LegacyCardUsage,
|
||||
LegacyCommissionAccount,
|
||||
)
|
||||
from .mapping_loader import Mapping, MissingTargetError
|
||||
|
||||
# 写死的默认值(对应原 migration.yaml,实际不需要业务侧关心)
|
||||
ORDER_NO_PREFIX_CARD = "MIG-CARD-"
|
||||
ALLOCATION_NO_PREFIX = "MIG-ALLOC-"
|
||||
METADATA_SOURCE = "qicheng"
|
||||
|
||||
|
||||
# ---------------- 通用工具 ----------------
|
||||
|
||||
|
||||
def _sql_str(value: Optional[str]) -> str:
|
||||
if value is None:
|
||||
return "NULL"
|
||||
s = str(value)
|
||||
return "'" + s.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _sql_jsonb(d: dict) -> str:
|
||||
import json
|
||||
|
||||
s = json.dumps(d, ensure_ascii=False, separators=(",", ":"))
|
||||
return "'" + s.replace("'", "''") + "'::jsonb"
|
||||
|
||||
|
||||
def _sql_decimal(value: Decimal) -> str:
|
||||
"""生成 SQL decimal 字面量,保留奇成小数 MB 精度。"""
|
||||
return format(value, "f")
|
||||
|
||||
|
||||
def _shop_id_sub(shop_code: Optional[str]) -> str:
|
||||
"""生成 shop_id 子查询表达式;空 shop_code 返回 'NULL'(平台库存)。"""
|
||||
if not shop_code:
|
||||
return "NULL"
|
||||
return f"(SELECT id FROM tb_shop WHERE shop_code = {_sql_str(shop_code)} AND deleted_at IS NULL LIMIT 1)"
|
||||
|
||||
|
||||
def _now_for_no() -> str:
|
||||
return datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
|
||||
|
||||
def _file_header(title: str) -> str:
|
||||
return (
|
||||
"-- ============================================================\n"
|
||||
f"-- {title}\n"
|
||||
f"-- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"
|
||||
"-- 重跑安全: ON CONFLICT DO NOTHING\n"
|
||||
"-- 建议执行: psql ... -1 -f <此文件>\n"
|
||||
"-- ============================================================\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_card_shop_code(card: CardRow, meta: LegacyCardMeta, mapping: Mapping) -> str:
|
||||
"""决定卡最终 shop_code:设备上的卡→空(跟设备走);否则按 agent 映射;查不到→平台库存。"""
|
||||
if card.bound_device_virtual_no:
|
||||
return ""
|
||||
if meta and meta.agent_id:
|
||||
shop = mapping.lookup_agent_shop_code(meta.agent_id)
|
||||
if shop:
|
||||
return shop
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_device_shop_code(
|
||||
device: DeviceRow,
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
mapping: Mapping,
|
||||
) -> str:
|
||||
"""决定设备最终 shop_code:按主卡(slot 1)的 agent 映射;无主卡或映射不到→平台库存。"""
|
||||
primary_iccid = device.sim_iccids.get(1) or next(iter(device.sim_iccids.values()), "")
|
||||
if not primary_iccid:
|
||||
return ""
|
||||
meta = card_metas.get(primary_iccid)
|
||||
if not meta or not meta.agent_id:
|
||||
return ""
|
||||
return mapping.lookup_agent_shop_code(meta.agent_id) or ""
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 脚本1:资产导入
|
||||
# ============================================================
|
||||
|
||||
|
||||
def write_step1(
|
||||
output_dir: Path,
|
||||
*,
|
||||
cards: list[CardRow],
|
||||
devices: list[DeviceRow],
|
||||
mapping: Mapping,
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
duplicate_iccid_19s: set[str],
|
||||
errors: list[ErrorRow],
|
||||
) -> None:
|
||||
"""生成 step1_* SQL + errors.csv + summary.txt。
|
||||
|
||||
会就地修改 errors 列表,把映射缺失等问题追加进去。
|
||||
|
||||
Args:
|
||||
card_metas: {iccid_19: LegacyCardMeta}
|
||||
duplicate_iccid_19s: 同一 iccid_19 在奇成命中多条 tbl_card 记录的集合,直接报错阻断
|
||||
"""
|
||||
user_id = mapping.migration_user_id
|
||||
|
||||
# 预检 carrier 映射 + 运营商一致性 + 重复命中,把异常卡剔除
|
||||
valid_cards = _filter_cards_by_carrier(cards, card_metas, duplicate_iccid_19s, mapping, errors)
|
||||
|
||||
cards_by_iccid_19 = {c.iccid_19: c for c in valid_cards}
|
||||
|
||||
_write_iot_cards(output_dir / "step1_01_iot_cards.sql", valid_cards, card_metas, mapping, user_id)
|
||||
_write_devices(output_dir / "step1_02_devices.sql", devices, card_metas, mapping, user_id)
|
||||
_write_asset_identifiers(output_dir / "step1_03_asset_identifiers.sql", valid_cards, devices, card_metas)
|
||||
_write_asset_wallets(output_dir / "step1_04_asset_wallets.sql", valid_cards, devices)
|
||||
_write_sim_bindings(output_dir / "step1_05_sim_bindings.sql", devices, cards_by_iccid_19)
|
||||
_write_shop_allocations(
|
||||
output_dir / "step1_06_shop_allocations.sql", valid_cards, devices, card_metas, mapping, user_id
|
||||
)
|
||||
|
||||
_write_errors(output_dir / "errors.csv", errors)
|
||||
_write_summary_step1(output_dir / "summary.txt", valid_cards, devices, errors)
|
||||
|
||||
|
||||
def _filter_cards_by_carrier(
|
||||
cards: list[CardRow],
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
duplicate_iccid_19s: set[str],
|
||||
mapping: Mapping,
|
||||
errors: list[ErrorRow],
|
||||
) -> list[CardRow]:
|
||||
"""筛掉:奇成查不到 / 命中多条 / carrier 映射缺失 / target_* 未填 / 运营商与 iccid 长度不一致 的卡。"""
|
||||
valid: list[CardRow] = []
|
||||
for c in cards:
|
||||
if c.iccid_19 in duplicate_iccid_19s:
|
||||
errors.append(ErrorRow(
|
||||
"cards.csv", c.line_no, "iccid_19", c.iccid_19,
|
||||
"legacy_duplicate", "奇成 tbl_card 中同一 iccid_19 命中多条记录,请业务方人工确认",
|
||||
))
|
||||
continue
|
||||
meta = card_metas.get(c.iccid_19)
|
||||
if meta is None:
|
||||
errors.append(ErrorRow(
|
||||
"cards.csv", c.line_no, "iccid_19", c.iccid_19,
|
||||
"not_in_legacy", "奇成 tbl_card 中找不到该 ICCID(iccid_19/iccid_20 都没命中)",
|
||||
))
|
||||
continue
|
||||
if not meta.category:
|
||||
errors.append(ErrorRow(
|
||||
"cards.csv", c.line_no, "iccid_19", c.iccid_19,
|
||||
"no_category", "奇成 tbl_card.category 为空,无法定位运营商映射",
|
||||
))
|
||||
continue
|
||||
carrier = mapping.lookup_carrier(meta.category)
|
||||
if carrier is None:
|
||||
errors.append(ErrorRow(
|
||||
"cards.csv", c.line_no, "category", str(meta.category),
|
||||
"carrier_not_in_mapping", f"mapping.yaml.carriers 未配置 category={meta.category} ({meta.category_name})",
|
||||
))
|
||||
continue
|
||||
try:
|
||||
carrier.require_target()
|
||||
except MissingTargetError as exc:
|
||||
errors.append(ErrorRow(
|
||||
"cards.csv", c.line_no, "category", str(meta.category),
|
||||
"carrier_target_missing", str(exc),
|
||||
))
|
||||
continue
|
||||
# 运营商类型与业务方 iccid_20 一致性校验
|
||||
err = iccid_utils.validate_carrier_iccid_match(c.iccid_20, carrier.target_carrier_type or "")
|
||||
if err is not None:
|
||||
errors.append(ErrorRow(
|
||||
"cards.csv", c.line_no, "iccid_20", c.iccid_20,
|
||||
err.code, err.message,
|
||||
))
|
||||
continue
|
||||
valid.append(c)
|
||||
return valid
|
||||
|
||||
|
||||
def _write_iot_cards(
|
||||
path: Path,
|
||||
cards: list[CardRow],
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
mapping: Mapping,
|
||||
user_id: int,
|
||||
) -> None:
|
||||
"""写 tb_iot_card。元数据从奇成 card_metas + mapping 推导,iccid_19/iccid_20 用业务方两列。"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step1_01 卡资产导入 (tb_iot_card)"))
|
||||
for c in cards:
|
||||
meta = card_metas[c.iccid_19]
|
||||
carrier = mapping.lookup_carrier(meta.category)
|
||||
assert carrier is not None # 已被 _filter_cards_by_carrier 保证
|
||||
|
||||
# 业务方直接给的两列;iccid 唯一键 = 有 iccid_20 用 iccid_20,否则 iccid_19
|
||||
iccid_20_expr = _sql_str(c.iccid_20) if c.iccid_20 else "NULL"
|
||||
|
||||
card_category = "industry" if c.is_industry else "normal"
|
||||
shop_code = _resolve_card_shop_code(c, meta, mapping)
|
||||
series = mapping.lookup_series(_meal_to_series_id(meta, mapping))
|
||||
series_id_expr = str(series.target_series_id) if series and series.target_series_id else "NULL"
|
||||
|
||||
f.write(
|
||||
"INSERT INTO tb_iot_card (\n"
|
||||
" iccid, iccid_19, iccid_20,\n"
|
||||
" carrier_id, carrier_type, carrier_name,\n"
|
||||
" card_category, batch_no,\n"
|
||||
" status, activation_status, real_name_status, network_status,\n"
|
||||
" asset_status, generation, is_standalone,\n"
|
||||
" realname_policy, stop_reason,\n"
|
||||
" msisdn, virtual_no, shop_id, series_id,\n"
|
||||
" device_virtual_no, gateway_extend,\n"
|
||||
" creator, updater, created_at, updated_at\n"
|
||||
") VALUES (\n"
|
||||
f" {_sql_str(c.iccid_full)}, {_sql_str(c.iccid_19)}, {iccid_20_expr},\n"
|
||||
f" {carrier.target_carrier_id}, {_sql_str(carrier.target_carrier_type)}, {_sql_str(carrier.target_carrier_name)},\n"
|
||||
f" {_sql_str(card_category)}, {_sql_str(mapping.migration_batch_no)},\n"
|
||||
" 1, 0, 0, 0,\n"
|
||||
" 1, 1, TRUE,\n"
|
||||
" 'after_order', '',\n"
|
||||
f" {_sql_str(meta.msisdn) if meta.msisdn else 'NULL'}, "
|
||||
f"{_sql_str(meta.virtual_no) if meta.virtual_no else 'NULL'}, "
|
||||
f"{_shop_id_sub(shop_code)}, {series_id_expr},\n"
|
||||
" '', '',\n"
|
||||
f" {user_id}, {user_id}, NOW(), NOW()\n"
|
||||
")\n"
|
||||
"ON CONFLICT (iccid) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _meal_to_series_id(meta: LegacyCardMeta, mapping: Mapping) -> str:
|
||||
"""由 meta 的 meal_id 经 mapping.packages 间接拿不到 series,所以这里直接靠
|
||||
mapping.series 用 legacy_series_id 反查就行。简化:返回空字符串(让上层走 series 映射)。
|
||||
|
||||
实际上 series_id 仅在套餐有 meal_series_id 时才有意义。考虑到 LegacyCardMeta 没存
|
||||
series_id,这里返回 '' — 让 series_id_expr 走 NULL 分支。后续若需要,scan_legacy
|
||||
可把 series_id 也下沉到 card_meta。
|
||||
"""
|
||||
_ = meta
|
||||
_ = mapping
|
||||
return ""
|
||||
|
||||
|
||||
def _write_devices(
|
||||
path: Path,
|
||||
devices: list[DeviceRow],
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
mapping: Mapping,
|
||||
user_id: int,
|
||||
) -> None:
|
||||
"""设备表 INSERT。
|
||||
|
||||
业务方不关心的字段都写默认值:
|
||||
device_name/device_model/manufacturer = NULL(可后台编辑)
|
||||
max_sim_slots = 4
|
||||
status / asset_status / online_status = 1 / 1 / 0
|
||||
enable_polling = TRUE,realname_policy = 'after_order'
|
||||
software_version = '',switch_mode = '0'(自动切卡)
|
||||
"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step1_02 设备资产导入 (tb_device)"))
|
||||
for d in devices:
|
||||
shop_code = _resolve_device_shop_code(d, card_metas, mapping)
|
||||
f.write(
|
||||
"INSERT INTO tb_device (\n"
|
||||
" virtual_no, max_sim_slots, imei,\n"
|
||||
" status, asset_status, generation, online_status,\n"
|
||||
" enable_polling, realname_policy,\n"
|
||||
" software_version, switch_mode,\n"
|
||||
" batch_no, shop_id,\n"
|
||||
" creator, updater, created_at, updated_at\n"
|
||||
") VALUES (\n"
|
||||
f" {_sql_str(d.virtual_no)}, {d.max_sim_slots}, "
|
||||
f"{_sql_str(d.imei) if d.imei else 'NULL'},\n"
|
||||
" 1, 1, 1, 0,\n"
|
||||
" TRUE, 'after_order',\n"
|
||||
" '', '0',\n"
|
||||
f" {_sql_str(mapping.migration_batch_no)}, {_shop_id_sub(shop_code)},\n"
|
||||
f" {user_id}, {user_id}, NOW(), NOW()\n"
|
||||
")\n"
|
||||
"ON CONFLICT (virtual_no) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_asset_identifiers(
|
||||
path: Path,
|
||||
cards: list[CardRow],
|
||||
devices: list[DeviceRow],
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
) -> None:
|
||||
"""卡的 iccid + 虚拟号(来自奇成)和设备的 virtual_no 都注册。"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step1_03 资产标识符注册 (tb_asset_identifier)"))
|
||||
for c in cards:
|
||||
meta = card_metas[c.iccid_19]
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n"
|
||||
f"SELECT {_sql_str(c.iccid_full)}, 'iot_card', id FROM tb_iot_card\n"
|
||||
f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL\n"
|
||||
"ON CONFLICT (identifier) DO NOTHING;\n\n"
|
||||
)
|
||||
if meta.virtual_no:
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n"
|
||||
f"SELECT {_sql_str(meta.virtual_no)}, 'iot_card', id FROM tb_iot_card\n"
|
||||
f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL\n"
|
||||
"ON CONFLICT (identifier) DO NOTHING;\n\n"
|
||||
)
|
||||
for d in devices:
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n"
|
||||
f"SELECT {_sql_str(d.virtual_no)}, 'device', id FROM tb_device\n"
|
||||
f"WHERE virtual_no = {_sql_str(d.virtual_no)} AND deleted_at IS NULL\n"
|
||||
"ON CONFLICT (identifier) DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_asset_wallets(path: Path, cards: list[CardRow], devices: list[DeviceRow]) -> None:
|
||||
"""卡和设备各一条钱包,初始余额 0。"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step1_04 资产钱包初始化 (tb_asset_wallet)"))
|
||||
for c in cards:
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_wallet (\n"
|
||||
" resource_type, resource_id, balance, frozen_balance,\n"
|
||||
" currency, status, version, shop_id_tag,\n"
|
||||
" created_at, updated_at\n"
|
||||
")\n"
|
||||
"SELECT 'iot_card', id, 0, 0,\n"
|
||||
" 'CNY', 1, 0, COALESCE(shop_id, 0),\n"
|
||||
" NOW(), NOW()\n"
|
||||
"FROM tb_iot_card\n"
|
||||
f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL\n"
|
||||
"ON CONFLICT (resource_type, resource_id) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
for d in devices:
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_wallet (\n"
|
||||
" resource_type, resource_id, balance, frozen_balance,\n"
|
||||
" currency, status, version, shop_id_tag,\n"
|
||||
" created_at, updated_at\n"
|
||||
")\n"
|
||||
"SELECT 'device', id, 0, 0,\n"
|
||||
" 'CNY', 1, 0, COALESCE(shop_id, 0),\n"
|
||||
" NOW(), NOW()\n"
|
||||
"FROM tb_device\n"
|
||||
f"WHERE virtual_no = {_sql_str(d.virtual_no)} AND deleted_at IS NULL\n"
|
||||
"ON CONFLICT (resource_type, resource_id) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_sim_bindings(
|
||||
path: Path,
|
||||
devices: list[DeviceRow],
|
||||
cards_by_iccid_19: dict[str, CardRow],
|
||||
) -> None:
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step1_05 设备-卡绑定 (tb_device_sim_binding)"))
|
||||
for d in devices:
|
||||
for slot, iccid_19 in sorted(d.sim_iccids.items()):
|
||||
card = cards_by_iccid_19.get(iccid_19)
|
||||
if card is None:
|
||||
continue # 该卡可能被 carrier 校验剔除,跳过此 slot 绑定
|
||||
is_current = "TRUE" if slot == d.current_slot else "FALSE"
|
||||
f.write(
|
||||
"INSERT INTO tb_device_sim_binding (\n"
|
||||
" device_id, iot_card_id, slot_position, bind_status, is_current,\n"
|
||||
" bind_time, creator, updater, created_at, updated_at\n"
|
||||
")\n"
|
||||
f"SELECT d.id, c.id, {slot}, 1, {is_current},\n"
|
||||
" NOW(), 0, 0, NOW(), NOW()\n"
|
||||
"FROM tb_device d\n"
|
||||
"CROSS JOIN tb_iot_card c\n"
|
||||
f"WHERE d.virtual_no = {_sql_str(d.virtual_no)} AND d.deleted_at IS NULL\n"
|
||||
f" AND c.iccid = {_sql_str(card.iccid_full)} AND c.deleted_at IS NULL\n"
|
||||
"ON CONFLICT (device_id, slot_position) WHERE bind_status = 1 AND deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_shop_allocations(
|
||||
path: Path,
|
||||
cards: list[CardRow],
|
||||
devices: list[DeviceRow],
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
mapping: Mapping,
|
||||
user_id: int,
|
||||
) -> None:
|
||||
"""只为带店铺归属的资产生成分配记录;设备上的卡跟随设备分配。"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step1_06 店铺分配记录 (tb_asset_allocation_record)"))
|
||||
no_ts = _now_for_no()
|
||||
|
||||
for c in cards:
|
||||
if c.bound_device_virtual_no:
|
||||
continue
|
||||
meta = card_metas[c.iccid_19]
|
||||
shop_code = _resolve_card_shop_code(c, meta, mapping)
|
||||
if not shop_code:
|
||||
continue
|
||||
allocation_no = f"{ALLOCATION_NO_PREFIX}CARD-{no_ts}-{c.line_no}"
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_allocation_record (\n"
|
||||
" allocation_no, allocation_type, asset_type, asset_id, asset_identifier,\n"
|
||||
" from_owner_type, to_owner_type, to_owner_id,\n"
|
||||
" operator_id, remark,\n"
|
||||
" creator, updater, created_at, updated_at\n"
|
||||
")\n"
|
||||
f"SELECT {_sql_str(allocation_no)}, 'allocate', 'iot_card', c.id, {_sql_str(c.iccid_full)},\n"
|
||||
" 'platform', 'shop', s.id,\n"
|
||||
f" {user_id}, '奇成数据迁移初始化分配',\n"
|
||||
f" {user_id}, {user_id}, NOW(), NOW()\n"
|
||||
"FROM tb_iot_card c\n"
|
||||
"CROSS JOIN tb_shop s\n"
|
||||
f"WHERE c.iccid = {_sql_str(c.iccid_full)} AND c.deleted_at IS NULL\n"
|
||||
f" AND s.shop_code = {_sql_str(shop_code)} AND s.deleted_at IS NULL;\n\n"
|
||||
)
|
||||
|
||||
for d in devices:
|
||||
shop_code = _resolve_device_shop_code(d, card_metas, mapping)
|
||||
if not shop_code:
|
||||
continue
|
||||
allocation_no = f"{ALLOCATION_NO_PREFIX}DEV-{no_ts}-{d.line_no}"
|
||||
f.write(
|
||||
"INSERT INTO tb_asset_allocation_record (\n"
|
||||
" allocation_no, allocation_type, asset_type, asset_id, asset_identifier,\n"
|
||||
" from_owner_type, to_owner_type, to_owner_id,\n"
|
||||
" operator_id, remark,\n"
|
||||
" creator, updater, created_at, updated_at\n"
|
||||
")\n"
|
||||
f"SELECT {_sql_str(allocation_no)}, 'allocate', 'device', dev.id, {_sql_str(d.virtual_no)},\n"
|
||||
" 'platform', 'shop', s.id,\n"
|
||||
f" {user_id}, '奇成数据迁移初始化分配',\n"
|
||||
f" {user_id}, {user_id}, NOW(), NOW()\n"
|
||||
"FROM tb_device dev\n"
|
||||
"CROSS JOIN tb_shop s\n"
|
||||
f"WHERE dev.virtual_no = {_sql_str(d.virtual_no)} AND dev.deleted_at IS NULL\n"
|
||||
f" AND s.shop_code = {_sql_str(shop_code)} AND s.deleted_at IS NULL;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_errors(path: Path, errors: list[ErrorRow]) -> None:
|
||||
with path.open("w", encoding="utf-8", newline="") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["source_file", "line_no", "field", "value", "error_code", "message"])
|
||||
for e in errors:
|
||||
writer.writerow(e.to_csv_row())
|
||||
|
||||
|
||||
def _write_summary_step1(path: Path, cards: list[CardRow], devices: list[DeviceRow], errors: list[ErrorRow]) -> None:
|
||||
bound = sum(1 for c in cards if c.bound_device_virtual_no)
|
||||
lines = [
|
||||
"奇成迁移 - 脚本1 资产导入摘要",
|
||||
"=" * 48,
|
||||
f"生成时间 : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
f"卡有效总数 : {len(cards)}",
|
||||
f" 绑设备 : {bound}",
|
||||
f" 独立卡 : {len(cards) - bound}",
|
||||
f"设备总数 : {len(devices)}",
|
||||
f"异常行 : {len(errors)} (详见 errors.csv)",
|
||||
"",
|
||||
"店铺归属:由 mapping.agents 自动推导(详见 step1_01/step1_02/step1_06 内的 shop_id 子查询)",
|
||||
]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 脚本2:关联数据导入
|
||||
# ============================================================
|
||||
|
||||
|
||||
def write_step2(
|
||||
output_dir: Path,
|
||||
*,
|
||||
cards: list[CardRow],
|
||||
mapping: Mapping,
|
||||
card_metas: dict[str, LegacyCardMeta],
|
||||
usage_snapshots: dict[str, LegacyCardUsage],
|
||||
commission_snapshots: dict[str, LegacyCommissionAccount],
|
||||
agent_balances: dict[str, LegacyAgentBalance],
|
||||
) -> list[str]:
|
||||
"""生成 step2_* SQL + warnings.csv + summary.txt。"""
|
||||
user_id = mapping.migration_user_id
|
||||
warnings: list[tuple[str, str, str]] = []
|
||||
# 卡 → package(从 card_meta.current_meal_id 经 mapping.packages 查 target)
|
||||
resolved_packages: dict[str, tuple[int, str]] = {} # iccid_19 → (target_package_id, expire_date)
|
||||
for c in cards:
|
||||
meta = card_metas.get(c.iccid_19)
|
||||
if meta is None or not meta.current_meal_id:
|
||||
continue
|
||||
pkg_map = mapping.lookup_package(meta.current_meal_id)
|
||||
if pkg_map is None:
|
||||
warnings.append(("package_not_in_mapping", c.iccid_19,
|
||||
f"奇成 meal_id={meta.current_meal_id} ({meta.current_meal_name}) 未在 mapping.yaml.packages"))
|
||||
continue
|
||||
try:
|
||||
pkg_map.require_target()
|
||||
except MissingTargetError as exc:
|
||||
warnings.append(("package_target_missing", c.iccid_19, str(exc)))
|
||||
continue
|
||||
resolved_packages[c.iccid_19] = (pkg_map.target_package_id, meta.current_expire_date or "")
|
||||
|
||||
# 用量预检:虚用量 > 0 但没找到当前生效套餐的卡,无法精确反算真用量
|
||||
# (按 flow_add_discount=0 处理,即真=虚,可能高估真用量;不阻断)
|
||||
for c in cards:
|
||||
snap = usage_snapshots.get(c.iccid_19)
|
||||
if snap is None or snap.virtual_used_mb <= 0:
|
||||
continue
|
||||
if not snap.has_active_package:
|
||||
warnings.append((
|
||||
"usage_without_active_package", c.iccid_19,
|
||||
f"奇成虚用量 {snap.virtual_used_mb}MB,但 tbl_card_life 中没有当前生效套餐"
|
||||
"(status=1),无法精确反算真用量,按 flow_add_discount=0 处理(真=虚)",
|
||||
))
|
||||
|
||||
_write_migration_orders(output_dir / "step2_01_migration_orders.sql", cards, resolved_packages, user_id)
|
||||
_write_package_usages(
|
||||
output_dir / "step2_02_package_usages.sql", cards, resolved_packages, usage_snapshots, user_id,
|
||||
)
|
||||
_write_card_data_usage(output_dir / "step2_03_card_data_usage_update.sql", cards, usage_snapshots)
|
||||
|
||||
# 代理钱包:只为 mapping.agents 中 target_shop_code 已填的项写入
|
||||
eligible_agents = [a for a in mapping.agents.values() if (a.target_shop_code or "").strip()]
|
||||
_write_agent_wallet_init(
|
||||
output_dir / "step2_04_agent_wallet_init.sql",
|
||||
eligible_agents, agent_balances, commission_snapshots, warnings,
|
||||
)
|
||||
_write_agent_wallet_transactions(
|
||||
output_dir / "step2_05_agent_wallet_transactions.sql",
|
||||
eligible_agents, agent_balances, commission_snapshots, user_id,
|
||||
)
|
||||
|
||||
_write_warnings(output_dir / "warnings.csv", warnings)
|
||||
_write_summary_step2(
|
||||
output_dir / "summary.txt",
|
||||
cards=cards,
|
||||
resolved=resolved_packages,
|
||||
agents=eligible_agents,
|
||||
warnings=warnings,
|
||||
)
|
||||
return [w[2] for w in warnings]
|
||||
|
||||
|
||||
def _make_card_order_no(iccid_full: str) -> str:
|
||||
"""生成稳定迁移订单号。
|
||||
|
||||
新库 order_no 限制为 varchar(30);MIG-CARD- + 20 位 ICCID 总长 29。
|
||||
固定使用 ICCID 还能保证 step2_01/step2_02 分开重跑时仍能匹配同一订单。
|
||||
"""
|
||||
return f"{ORDER_NO_PREFIX_CARD}{iccid_full}"
|
||||
|
||||
|
||||
def _write_migration_orders(
|
||||
path: Path,
|
||||
cards: list[CardRow],
|
||||
resolved_packages: dict[str, tuple[int, str]],
|
||||
user_id: int,
|
||||
) -> None:
|
||||
"""伪订单关键字段(详见方案文档 10.2):source=admin / buyer_type=personal /
|
||||
buyer_id=0 / payment_method=offline / payment_status=2 / commission_status=2 /
|
||||
commission_result=2 / amount=0。"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step2_01 迁移伪订单 (tb_order)"))
|
||||
for c in cards:
|
||||
if c.iccid_19 not in resolved_packages:
|
||||
continue
|
||||
order_no = _make_card_order_no(c.iccid_full)
|
||||
f.write(
|
||||
"INSERT INTO tb_order (\n"
|
||||
" order_no, order_type, buyer_type, buyer_id,\n"
|
||||
" iot_card_id, asset_identifier,\n"
|
||||
" total_amount, actual_paid_amount,\n"
|
||||
" payment_method, payment_status, paid_at,\n"
|
||||
" commission_status, commission_result, commission_config_version,\n"
|
||||
" source, generation,\n"
|
||||
" operator_account_type, operator_account_name,\n"
|
||||
" creator, updater, created_at, updated_at\n"
|
||||
")\n"
|
||||
f"SELECT {_sql_str(order_no)}, 'single_card', 'personal', 0,\n"
|
||||
f" c.id, {_sql_str(c.iccid_full)},\n"
|
||||
" 0, 0,\n"
|
||||
" 'offline', 2, NOW(),\n"
|
||||
" 2, 2, 0,\n"
|
||||
" 'admin', 1,\n"
|
||||
" 'platform', '奇成数据迁移',\n"
|
||||
f" {user_id}, {user_id}, NOW(), NOW()\n"
|
||||
"FROM tb_iot_card c\n"
|
||||
f"WHERE c.iccid = {_sql_str(c.iccid_full)} AND c.deleted_at IS NULL\n"
|
||||
"ON CONFLICT (order_no) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_package_usages(
|
||||
path: Path,
|
||||
cards: list[CardRow],
|
||||
resolved_packages: dict[str, tuple[int, str]],
|
||||
usage_snapshots: dict[str, LegacyCardUsage],
|
||||
user_id: int,
|
||||
) -> None:
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step2_02 套餐使用记录 (tb_package_usage)"))
|
||||
for c in cards:
|
||||
if c.iccid_19 not in resolved_packages:
|
||||
continue
|
||||
target_pkg_id, expire = resolved_packages[c.iccid_19]
|
||||
order_no = _make_card_order_no(c.iccid_full)
|
||||
expires_expr = f"{_sql_str(expire)}::timestamp" if expire else "NULL"
|
||||
snap = usage_snapshots.get(c.iccid_19)
|
||||
used_mb = snap.real_used_mb_floor if snap is not None else 0
|
||||
effective_limit_expr = (
|
||||
"CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 "
|
||||
"THEN p.virtual_data_mb ELSE p.real_data_mb END"
|
||||
)
|
||||
usage_expr = f"LEAST({used_mb}, {effective_limit_expr})"
|
||||
f.write(
|
||||
"INSERT INTO tb_package_usage (\n"
|
||||
" order_id, order_no, package_id, usage_type,\n"
|
||||
" iot_card_id,\n"
|
||||
" data_limit_mb, data_usage_mb,\n"
|
||||
" virtual_total_mb_snapshot, display_gain_ratio_snapshot, enable_virtual_data_snapshot,\n"
|
||||
" status, priority, generation,\n"
|
||||
" activated_at, expires_at,\n"
|
||||
" package_name,\n"
|
||||
" package_price_config_status, package_is_gift,\n"
|
||||
" data_reset_cycle,\n"
|
||||
" creator, updater, created_at, updated_at\n"
|
||||
")\n"
|
||||
"SELECT o.id, o.order_no, p.id, 'single_card',\n"
|
||||
" o.iot_card_id,\n"
|
||||
f" p.real_data_mb, {usage_expr},\n"
|
||||
f" {effective_limit_expr},\n"
|
||||
" CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 THEN p.virtual_ratio ELSE 1.0 END,\n"
|
||||
" CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 THEN TRUE ELSE FALSE END,\n"
|
||||
f" CASE WHEN {effective_limit_expr} > 0 AND {usage_expr} >= {effective_limit_expr} THEN 2 ELSE 1 END, 1, 1,\n"
|
||||
f" NULL, {expires_expr},\n"
|
||||
" p.package_name,\n"
|
||||
" p.price_config_status, p.is_gift,\n"
|
||||
" p.data_reset_cycle,\n"
|
||||
f" {user_id}, {user_id}, NOW(), NOW()\n"
|
||||
"FROM tb_order o\n"
|
||||
f"JOIN tb_package p ON p.id = {target_pkg_id} AND p.deleted_at IS NULL\n"
|
||||
f"WHERE o.order_no = {_sql_str(order_no)} AND o.deleted_at IS NULL\n"
|
||||
"ON CONFLICT (order_id, package_id) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_card_data_usage(path: Path, cards: list[CardRow], usage_snapshots: dict[str, LegacyCardUsage]) -> None:
|
||||
"""写卡累计流量基线 UPDATE。
|
||||
|
||||
三个字段都按"真用量"语义落库:
|
||||
- ``data_usage_mb``(BIGINT)= ``real_used_mb_floor``,sub-MB 上取整为 1
|
||||
- ``current_month_usage_mb``(numeric)= ``real_used_mb`` 原始 Decimal,保留小数
|
||||
- ``last_gateway_reading_mb``(double)= 同上,用作下次轮询增量计算的基线
|
||||
"""
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step2_03 卡累计流量更新 (tb_iot_card 流量基线)"))
|
||||
for c in cards:
|
||||
snap = usage_snapshots.get(c.iccid_19)
|
||||
if snap is None or snap.real_used_mb <= 0:
|
||||
continue
|
||||
real_decimal = _sql_decimal(snap.real_used_mb)
|
||||
real_int = snap.real_used_mb_floor
|
||||
f.write(
|
||||
"UPDATE tb_iot_card\n"
|
||||
"SET\n"
|
||||
f" data_usage_mb = {real_int},\n"
|
||||
f" current_month_usage_mb = {real_decimal},\n"
|
||||
f" last_gateway_reading_mb = {real_decimal},\n"
|
||||
" current_month_start_date = COALESCE(current_month_start_date, DATE_TRUNC('month', NOW())::date),\n"
|
||||
" last_data_check_at = NOW(),\n"
|
||||
" last_sync_time = NOW(),\n"
|
||||
" updated_at = NOW()\n"
|
||||
f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_agent_wallet_init(
|
||||
path: Path,
|
||||
eligible_agents,
|
||||
agent_balances: dict[str, LegacyAgentBalance],
|
||||
commission_snapshots: dict[str, LegacyCommissionAccount],
|
||||
warnings: list[tuple[str, str, str]],
|
||||
) -> None:
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step2_04 代理钱包初始化 (tb_agent_wallet, main + commission)"))
|
||||
for a in eligible_agents:
|
||||
balance = agent_balances.get(a.legacy_agent_id)
|
||||
commission = commission_snapshots.get(a.legacy_agent_id)
|
||||
if balance is None and commission is None:
|
||||
warnings.append((
|
||||
"agent_no_legacy", a.legacy_agent_id,
|
||||
f"奇成代理 {a.legacy_agent_id} ({a.legacy_agent_name}) 没有 tbl_agent / 佣金账户记录",
|
||||
))
|
||||
continue
|
||||
main_balance = balance.balance_fen if balance else 0
|
||||
comm_balance = commission.can_draw_amount_fen if commission else 0
|
||||
comm_frozen = commission.applying_draw_amount_fen if commission else 0
|
||||
shop_sub = _shop_id_sub(a.target_shop_code)
|
||||
|
||||
f.write(
|
||||
"INSERT INTO tb_agent_wallet (\n"
|
||||
" shop_id, wallet_type, balance, frozen_balance,\n"
|
||||
" currency, status, version, shop_id_tag,\n"
|
||||
" created_at, updated_at\n"
|
||||
") VALUES (\n"
|
||||
f" {shop_sub}, 'main', {main_balance}, 0,\n"
|
||||
f" 'CNY', 1, 0, {shop_sub},\n"
|
||||
" NOW(), NOW()\n"
|
||||
")\n"
|
||||
"ON CONFLICT (shop_id, wallet_type) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
f.write(
|
||||
"INSERT INTO tb_agent_wallet (\n"
|
||||
" shop_id, wallet_type, balance, frozen_balance,\n"
|
||||
" currency, status, version, shop_id_tag,\n"
|
||||
" created_at, updated_at\n"
|
||||
") VALUES (\n"
|
||||
f" {shop_sub}, 'commission', {comm_balance}, {comm_frozen},\n"
|
||||
f" 'CNY', 1, 0, {shop_sub},\n"
|
||||
" NOW(), NOW()\n"
|
||||
")\n"
|
||||
"ON CONFLICT (shop_id, wallet_type) WHERE deleted_at IS NULL DO NOTHING;\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_agent_wallet_transactions(
|
||||
path: Path,
|
||||
eligible_agents,
|
||||
agent_balances: dict[str, LegacyAgentBalance],
|
||||
commission_snapshots: dict[str, LegacyCommissionAccount],
|
||||
user_id: int,
|
||||
) -> None:
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(_file_header("step2_05 代理钱包初始化流水 (tb_agent_wallet_transaction)"))
|
||||
for a in eligible_agents:
|
||||
balance = agent_balances.get(a.legacy_agent_id)
|
||||
commission = commission_snapshots.get(a.legacy_agent_id)
|
||||
if balance is None and commission is None:
|
||||
continue
|
||||
main_amount = balance.balance_fen if balance else 0
|
||||
comm_amount = commission.can_draw_amount_fen if commission else 0
|
||||
comm_frozen = commission.applying_draw_amount_fen if commission else 0
|
||||
shop_sub = _shop_id_sub(a.target_shop_code)
|
||||
|
||||
for wallet_type, amount in (("main", main_amount), ("commission", comm_amount)):
|
||||
if amount <= 0:
|
||||
continue
|
||||
meta = {
|
||||
"source": METADATA_SOURCE,
|
||||
"legacy_agent_id": a.legacy_agent_id,
|
||||
"legacy_agent_name": a.legacy_agent_name,
|
||||
"legacy_field": "balance_money" if wallet_type == "main" else "can_draw_amount",
|
||||
}
|
||||
if wallet_type == "commission":
|
||||
meta["legacy_applying_draw_amount_fen"] = comm_frozen
|
||||
if commission:
|
||||
meta["legacy_total_commission_fen"] = commission.total_commission_amount_fen
|
||||
meta["legacy_total_draw_fen"] = commission.total_draw_amount_fen
|
||||
remark = (
|
||||
f"奇成数据迁移初始化({'主钱包' if wallet_type == 'main' else '分佣钱包'})"
|
||||
f"源 agent_id={a.legacy_agent_id}"
|
||||
)
|
||||
f.write(
|
||||
"INSERT INTO tb_agent_wallet_transaction (\n"
|
||||
" agent_wallet_id, shop_id, user_id,\n"
|
||||
" transaction_type, amount, balance_before, balance_after,\n"
|
||||
" status, reference_type, reference_id,\n"
|
||||
" remark, metadata,\n"
|
||||
" creator, shop_id_tag, created_at, updated_at\n"
|
||||
")\n"
|
||||
f"SELECT w.id, w.shop_id, {user_id},\n"
|
||||
f" 'recharge', {amount}, 0, {amount},\n"
|
||||
" 1, 'topup', NULL,\n"
|
||||
f" {_sql_str(remark)}, {_sql_jsonb(meta)},\n"
|
||||
f" {user_id}, w.shop_id, NOW(), NOW()\n"
|
||||
"FROM tb_agent_wallet w\n"
|
||||
f"WHERE w.shop_id = {shop_sub}\n"
|
||||
f" AND w.wallet_type = {_sql_str(wallet_type)}\n"
|
||||
" AND w.deleted_at IS NULL\n"
|
||||
" AND NOT EXISTS (\n"
|
||||
" SELECT 1 FROM tb_agent_wallet_transaction t\n"
|
||||
" WHERE t.agent_wallet_id = w.id\n"
|
||||
f" AND t.metadata->>'source' = {_sql_str(METADATA_SOURCE)}\n"
|
||||
f" AND t.metadata->>'legacy_agent_id' = {_sql_str(a.legacy_agent_id)}\n"
|
||||
f" AND t.metadata->>'legacy_field' = {_sql_str(meta['legacy_field'])}\n"
|
||||
" AND t.deleted_at IS NULL\n"
|
||||
" );\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_warnings(path: Path, warnings: list[tuple[str, str, str]]) -> None:
|
||||
with path.open("w", encoding="utf-8", newline="") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["kind", "key", "message"])
|
||||
for kind, key, message in warnings:
|
||||
writer.writerow([kind, key, message])
|
||||
|
||||
|
||||
def _write_summary_step2(path: Path, *, cards, resolved, agents, warnings) -> None:
|
||||
lines = [
|
||||
"奇成迁移 - 脚本2 关联数据导入摘要",
|
||||
"=" * 48,
|
||||
f"生成时间 : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
f"卡总数 : {len(cards)}",
|
||||
f" 生成伪订单+套餐 : {len(resolved)}",
|
||||
f"代理钱包初始化 : {len(agents)}",
|
||||
f"警告条目 : {len(warnings)} (详见 warnings.csv)",
|
||||
]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
Reference in New Issue
Block a user