From 58863e8ec5cb830a81aaebbaf74fcda7ace889e1 Mon Sep 17 00:00:00 2001 From: Break Date: Wed, 27 May 2026 11:10:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=BF=81=E7=A7=BB=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/admin-openapi.yaml | 20 + docs/脚本/奇成数据迁移方案.md | 611 +++++++++++++ scripts/migration/.python-version | 1 + scripts/migration/README.md | 227 +++++ scripts/migration/config/main_test_dsn.yaml | 11 + scripts/migration/config/mapping.yaml.example | 47 + scripts/migration/lib/__init__.py | 9 + scripts/migration/lib/csv_loader.py | 272 ++++++ scripts/migration/lib/iccid_utils.py | 158 ++++ scripts/migration/lib/legacy_query.py | 510 +++++++++++ scripts/migration/lib/mapping_loader.py | 245 +++++ scripts/migration/lib/sql_builder.py | 845 ++++++++++++++++++ scripts/migration/migrate_assets.py | 67 ++ scripts/migration/migrate_runtime.py | 68 ++ scripts/migration/output/.gitkeep | 0 scripts/migration/requirements.txt | 4 + scripts/migration/resources/README.md | 49 + scripts/migration/resources/cards.csv.example | 4 + .../migration/resources/devices.csv.example | 4 + scripts/migration/scan_legacy.py | 191 ++++ 20 files changed, 3343 insertions(+) create mode 100644 docs/脚本/奇成数据迁移方案.md create mode 100644 scripts/migration/.python-version create mode 100644 scripts/migration/README.md create mode 100644 scripts/migration/config/main_test_dsn.yaml create mode 100644 scripts/migration/config/mapping.yaml.example create mode 100644 scripts/migration/lib/__init__.py create mode 100644 scripts/migration/lib/csv_loader.py create mode 100644 scripts/migration/lib/iccid_utils.py create mode 100644 scripts/migration/lib/legacy_query.py create mode 100644 scripts/migration/lib/mapping_loader.py create mode 100644 scripts/migration/lib/sql_builder.py create mode 100644 scripts/migration/migrate_assets.py create mode 100644 scripts/migration/migrate_runtime.py create mode 100644 scripts/migration/output/.gitkeep create mode 100644 scripts/migration/requirements.txt create mode 100644 scripts/migration/resources/README.md create mode 100644 scripts/migration/resources/cards.csv.example create mode 100644 scripts/migration/resources/devices.csv.example create mode 100644 scripts/migration/scan_legacy.py diff --git a/docs/admin-openapi.yaml b/docs/admin-openapi.yaml index a699fb9..9609a66 100644 --- a/docs/admin-openapi.yaml +++ b/docs/admin-openapi.yaml @@ -1348,6 +1348,9 @@ components: type: string device_realtime: $ref: '#/components/schemas/DtoDeviceGatewayInfo' + gateway_card_imei: + description: 插拔卡业务 IMEI（由 Gateway 卡状态接口同步，非设备自身 IMEI，无业务含义，仅供查看） + type: string gateway_extend: description: Gateway 卡状态扩展字段，原样返回上游 extend，用于展示运营商侧实际停机原因 type: string @@ -1501,6 +1504,9 @@ components: enable_virtual_data: description: 当前主套餐是否启用虚流量（按套餐使用记录快照返回） type: boolean + gateway_card_imei: + description: 插拔卡业务 IMEI（由 Gateway 卡状态接口同步，非设备自身 IMEI，无业务含义，仅供查看） + type: string gateway_extend: description: Gateway 卡状态扩展字段，原样返回上游 extend，用于展示运营商侧实际停机原因 type: string @@ -3492,6 +3498,11 @@ components: description: 退款原因 maxLength: 1000 type: string + refund_voucher_key: + description: 退款凭证对象存储file_key（通过/storage/upload-url上传图片后获得） + maxLength: 500 + minLength: 1 + type: string requested_refund_amount: description: 申请退款金额（分） minimum: 1 @@ -3500,6 +3511,7 @@ components: - order_id - actual_received_amount - requested_refund_amount + - refund_voucher_key type: object DtoCreateRoleRequest: properties: @@ -7117,6 +7129,9 @@ components: refund_reason: description: 退款原因 type: string + refund_voucher_key: + description: 退款凭证对象存储file_key + type: string reject_reason: description: 拒绝原因 type: string @@ -7184,6 +7199,11 @@ components: maxLength: 1000 nullable: true type: string + refund_voucher_key: + description: 退款凭证对象存储file_key（重新提交时可替换；历史记录缺失时必填） + maxLength: 500 + nullable: true + type: string requested_refund_amount: description: 申请退款金额（分） minimum: 1 diff --git a/docs/脚本/奇成数据迁移方案.md b/docs/脚本/奇成数据迁移方案.md new file mode 100644 index 0000000..d36b26e --- /dev/null +++ b/docs/脚本/奇成数据迁移方案.md @@ -0,0 +1,611 @@ +# 奇成数据迁移方案 + +> 目标：把奇成（旧 MySQL `kyhl` 库）的存量卡和设备迁移到新系统（PostgreSQL），输出 SQL 脚本供线上手动执行。 +> 范围：当前态资产 + 当前生效套餐 + 累计已用流量 + 代理佣金总值。**不迁移历史明细**。 + +--- + +## 一、整体架构 + +``` + 业务方提供 + ┌──────────────────────┐ + │ resources/cards.csv │ + │ resources/devices.csv│ + └──────────┬───────────┘ + │ + 我们维护映射配置 + ┌──────────▼───────────┐ + │ config/carrier.yaml │ + │ config/series.yaml │ + │ config/package.yaml │ + │ config/shop.yaml │ + └──────────┬───────────┘ + │ + ┌──────────▼───────────┐ + │ 脚本1：资产导入 │ ← 纯离线，不查奇成 + │ migrate_assets.py │ + └──────────┬───────────┘ + │ + ▼ + ┌──────────────────────┐ + │ output/step1_*.sql │ ← 手动在线上执行 + └──────────┬───────────┘ + │ (执行完，新库已有这些 iccid) + │ + ┌──────────▼───────────┐ + │ 脚本2：关联数据导入 │ ← 直连奇成只读查询 + │ migrate_runtime.py │ ← 直连新库测试环境拿 id + └──────────┬───────────┘ + │ + ▼ + ┌──────────────────────┐ + │ output/step2_*.sql │ ← 手动在线上执行 + └──────────────────────┘ +``` + +**关键约束** +- 两个脚本都**不直接写入**线上库，只生成 `.sql` 文件 +- 脚本对奇成线上库**只 SELECT**，DB 连接层强制只读事务（`SET TRANSACTION READ ONLY`） +- 所有 INSERT 使用 `ON CONFLICT DO NOTHING`，支持重跑 + +--- + +## 二、业务方提供的资产清单 + +### 2.1 `resources/cards.csv` 卡资产清单 + +每张要迁移的卡一行（**包含独立卡和设备上的卡**）。 + +| 列名 | 必填 | 说明 | 示例 | +|------|------|------|------| +| `iccid` | 是 | ICCID。**19 位或 20 位**，业务方按奇成原值给（脚本根据 carrier_name 反查 carrier_type 判断长度规则） | `89852000263338772439` | +| `msisdn` | 否 | 手机号 / 接入号（奇成 `tbl_card.phone`） | `454006333877243` | +| `carrier_name` | 是 | 奇成的运营商名称，**必须在 `config/carrier.yaml` 中能匹配到** | `移动全国卡` | +| `package_name` | 否 | 当前生效套餐名（奇成 `tbl_card_life.meal_name`）。留空表示无套餐 | `Y星网专享年卡套餐每月300M（12个月）` | +| `package_expires_at` | 否 | 当前套餐到期时间（奇成 `tbl_card_life.expire_date`），格式 `YYYY-MM-DD HH:MM:SS`。无套餐留空 | `2027-04-28 16:00:00` | +| `is_industry` | 是 | 是否行业卡。`true`/`false` | `false` | +| `virtual_no` | 否 | 卡虚拟号（奇成 `tbl_virtual_number.vcode`），无则留空 | `8960066754` | +| `target_shop_code` | 否 | 目标店铺编码（`tb_shop.shop_code`）。**留空 = 进平台库存**；若该卡在设备上则**强制留空**（跟随设备） | `SHOP20260428103424CZZQ` | + +### 2.2 `resources/devices.csv` 设备资产清单 + +每台要迁移的设备一行。 + +| 列名 | 必填 | 说明 | 示例 | +|------|------|------|------| +| `imei` | 否 | 设备 IMEI（非蜂窝设备可留空） | `860000000000001` | +| `virtual_no` | 是 | 设备虚拟号/别名（业务方自定义，全局唯一） | `DEV-202604-0001` | +| `device_name` | 否 | 设备名称 | `张三家充电宝` | +| `device_model` | 否 | 设备型号 | `CB-PRO-V2` | +| `manufacturer` | 否 | 制造商 | `深圳XX` | +| `max_sim_slots` | 否 | 插槽数，默认 4 | `4` | +| `sim_iccid_1` | 否 | 插槽1的 iccid，必须在 cards.csv 中存在 | `89852000...` | +| `sim_iccid_2` | 否 | 插槽2的 iccid | `89860623...` | +| `sim_iccid_3` | 否 | 插槽3的 iccid | | +| `sim_iccid_4` | 否 | 插槽4的 iccid | | +| `current_slot` | 否 | 当前使用的插槽位置（1-4），用于 `tb_device_sim_binding.is_current` | `1` | +| `target_shop_code` | 否 | 目标店铺编码。**留空 = 进平台库存** | `SHOP20260428103424CZZQ` | + +**注意** +- 设备上的卡，其 `cards.csv` 中 `target_shop_code` 必须留空（一致性校验由脚本检查） +- `cards.csv` 必须先到位，`devices.csv` 的 `sim_iccid_*` 才能被校验 +- 业务方分别准备这两份是合理的，奇成 `tbl_card_relate`（主卡+副卡1+副卡2）的关系业务方应已掌握 + +--- + +## 三、ICCID 19/20 位的处理规则（关键） + +新系统设计：`tb_iot_card.iccid_19` 和 `iccid_20` 两个字段。 + +| 运营商类型 | 标准 ICCID 长度 | iccid_19 | iccid_20 | +|----------|---------------|----------|----------| +| **CTCC（电信）** | 19 位 | = iccid 全值 | NULL | +| **CMCC（移动）** | 20 位 | = iccid 前 19 位 | = iccid 全值 | +| **CUCC（联通）** | 20 位 | = iccid 前 19 位 | = iccid 全值 | +| **CBN（广电）** | 20 位 | = iccid 前 19 位 | = iccid 全值 | + +**脚本判定流程**： +``` +1. 读 carrier_name → 查 config/carrier.yaml 得到 carrier_type +2. 根据 carrier_type 决定标准长度（CTCC=19, 其他=20） +3. 校验 csv 中 iccid 实际长度： + - 长度匹配 → 正常拆分 + - 长度=20 但 carrier_type=CTCC → 警告（电信卡居然是 20 位？写入 errors.csv 让业务方确认） + - 长度=19 但 carrier_type≠CTCC → 警告（缺校验位？写入 errors.csv） + - 长度 ∉ {19, 20} → 拒绝（奇成里有 18/21/22 位的脏数据，必须剔除） +4. 写 tb_iot_card 时同时填 iccid / iccid_19 / iccid_20 +``` + +异常的 iccid 不进入 SQL 输出，统一写到 `output/errors.csv`，业务方人工修复后重跑。 + +--- + +## 四、我们维护的映射配置 + +迁移前**必须**在新系统建好店铺、运营商、套餐系列、套餐，然后填好以下映射。 + +### 4.1 `config/carrier_mapping.yaml` + +```yaml +# 奇成的运营商名称（业务方在 cards.csv 中填的）→ 新系统 carrier_id + type +carriers: + - legacy_name: "移动全国卡" + target_carrier_id: 1580 + target_carrier_type: CMCC + target_carrier_name: "移动15-BE20030985258" + - legacy_name: "电信行业卡" + target_carrier_id: 1583 + target_carrier_type: CTCC + target_carrier_name: "电信10-qiangukeji" + # ... 业务方提供的所有运营商都要列出 +``` + +### 4.2 `config/series_mapping.yaml` + +```yaml +# 奇成套餐系列名 → 新系统 series_id +# （根据业务方提供的 package_name，反查 tbl_set_meal → meal_series_id → tbl_set_meal_series.name） +series: + - legacy_series_name: "SXKJ专享" + target_series_id: 5 + - legacy_series_name: "移动全国卡月卡" + target_series_id: 6 +``` + +### 4.3 `config/package_mapping.yaml` + +```yaml +# 奇成套餐名 → 新系统 package_id（cards.csv 的 package_name 直接对照这里） +packages: + - legacy_package_name: "Y星网专享年卡套餐每月300M（12个月）" + target_package_id: 123 + - legacy_package_name: "移动全国卡100G（30天）" + target_package_id: 124 +``` + +> 注：奇成的 `tbl_set_meal` 套餐表非常大，**不需要全部映射**，只映射本次迁移涉及的套餐（脚本预扫描 cards.csv 收集 package_name 集合，缺失的列出来让我们补）。 + +### 4.4 `config/shop_mapping.yaml`（可选别名） + +```yaml +# 如果业务方愿意填 shop_code，本文件可以省略 +# 如果业务方只想填别名，这里维护 alias → shop_code 映射 +aliases: + shop_a: "SHOP20260428103424CZZQ" + shop_b: "SHOP20260429150917OQKA" +``` + +--- + +## 五、脚本1：资产导入（`migrate_assets.py`） + +### 5.1 输入 +- `resources/cards.csv` +- `resources/devices.csv` +- `config/*.yaml` + +### 5.2 处理流程 + +``` +1. 加载所有 yaml 映射 → 构建 (legacy_name → target_id) 字典 +2. 读 cards.csv： + a. 行级校验：iccid 格式、carrier_name 是否在映射中、is_industry 合法 + b. 19/20 位拆分（见第三节） + c. shop_code → shop_id（查新系统测试库 / 我们维护的映射） + d. 异常行写 errors.csv +3. 读 devices.csv： + a. 校验 sim_iccid_* 必须在 cards.csv 中存在 + b. 校验设备上的卡其 cards.csv.target_shop_code 必须为空 + c. virtual_no 重复性校验（同设备文件内 + 跨 cards.csv 的 virtual_no） +4. 生成 SQL（见 5.3） +``` + +### 5.3 输出文件 + +按表分块，便于逐表 review 和按需重跑： + +``` +output/ + step1_01_iot_cards.sql # INSERT INTO tb_iot_card ... + step1_02_devices.sql # INSERT INTO tb_device ... + step1_03_asset_identifiers.sql # INSERT INTO tb_asset_identifier ... + step1_04_asset_wallets.sql # INSERT INTO tb_asset_wallet ... + step1_05_sim_bindings.sql # INSERT INTO tb_device_sim_binding ... + step1_06_shop_allocations.sql # INSERT INTO tb_asset_allocation_record（如有 shop_id） + errors.csv # 异常行清单 + summary.txt # 统计：处理 X 张卡、Y 台设备、Z 条异常 +``` + +### 5.4 SQL 生成规则 + +#### 关于 ID 生成 + +新系统的 PK 是 `bigint` 自增。脚本生成 SQL 时使用 CTE 模式，让 PostgreSQL 自己分配 id： + +```sql +-- 示例：插入一张卡，并用 RETURNING 拿到 id，立刻写关联表 +WITH inserted_card AS ( + INSERT INTO tb_iot_card ( + iccid, iccid_19, iccid_20, carrier_id, carrier_type, carrier_name, + card_category, batch_no, status, activation_status, real_name_status, + network_status, stop_reason, realname_policy, is_standalone, + msisdn, virtual_no, shop_id, creator, updater, created_at, updated_at + ) VALUES ( + '89852000263338772439', '8985200026333877243', '89852000263338772439', + 1580, 'CMCC', '移动15-BE20030985258', + 'normal', 'MIGRATION-20260525', 1, 0, 0, + 0, '无套餐', 'after_order', true, + '454006333877243', '8960066754', NULL, 0, 0, NOW(), NOW() + ) + ON CONFLICT (iccid) WHERE deleted_at IS NULL DO NOTHING + RETURNING id +) +-- 同步写 asset_identifier +, ins_ai_iccid AS ( + INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id) + SELECT '89852000263338772439', 'iot_card', id FROM inserted_card + ON CONFLICT (identifier) DO NOTHING +) +, ins_ai_vno AS ( + INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id) + SELECT '8960066754', 'iot_card', id FROM inserted_card + WHERE '8960066754' <> '' + ON CONFLICT (identifier) DO NOTHING +) +-- 同步建钱包 +INSERT INTO tb_asset_wallet ( + resource_type, resource_id, balance, frozen_balance, + currency, status, version, shop_id_tag, created_at, updated_at +) +SELECT 'iot_card', id, 0, 0, 'CNY', 1, 0, COALESCE(0, 0), NOW(), NOW() +FROM inserted_card +ON CONFLICT (resource_type, resource_id) WHERE deleted_at IS NULL DO NOTHING; +``` + +每张卡一个 CTE 块，脚本生成几万个块。**优势**：原子性 + 幂等 + 不用预先分配 id 段。 + +#### 关于设备-卡绑定 + +绑定 SQL 需要先拿到 device.id 和 card.id，用相同的 CTE 模式： + +```sql +WITH d AS ( + SELECT id FROM tb_device WHERE virtual_no = 'DEV-202604-0001' LIMIT 1 +), c1 AS ( + SELECT id FROM tb_iot_card WHERE iccid = '89852000263338772439' AND deleted_at IS NULL LIMIT 1 +) +INSERT INTO tb_device_sim_binding (device_id, iot_card_id, slot_position, bind_status, is_current, bind_time, created_at, updated_at) +SELECT d.id, c1.id, 1, 1, true, NOW(), NOW(), NOW() +FROM d CROSS JOIN c1 +ON CONFLICT (device_id, slot_position) WHERE bind_status = 1 AND deleted_at IS NULL DO NOTHING; +-- 触发器自动更新 tb_iot_card.is_standalone = false +``` + +#### 关于分配记录（shop_id != NULL 的情况） + +```sql +-- 在 iot_card 写入 shop_id 的同时，补一条分配记录 +WITH c AS ( + SELECT id FROM tb_iot_card WHERE iccid = '...' AND deleted_at IS NULL +) +INSERT INTO tb_asset_allocation_record ( + allocation_no, allocation_type, asset_type, asset_id, asset_identifier, + from_owner_type, to_owner_type, to_owner_id, operator_id, created_at +) +SELECT 'MIG-' || TO_CHAR(NOW(), 'YYYYMMDDHH24MISS') || '-' || c.id, + 'allocate', 'iot_card', c.id, '...', + 'platform', 'shop', , 0, NOW() +FROM c; +``` + +--- + +## 六、脚本2：关联数据导入（`migrate_runtime.py`） + +### 6.1 前提 +- 脚本1已经执行，新库已有这批 iccid 的卡 +- 业务方已确认每张卡当前对应的套餐 ID（在脚本1阶段维护好 `package_mapping.yaml`） + +### 6.2 输入 +- `resources/cards.csv`（同脚本1） +- `config/*.yaml`（同脚本1，额外加 `config/migration.yaml` 配置 migration_user_id、伪订单前缀等） +- 奇成只读 DSN（连接配置） +- 新系统**测试环境**只读 DSN（仅用于拿 iccid → id 的映射，验证脚本逻辑；线上 id 在 step2 SQL 里用 SELECT 子查询动态取，**不硬编码**） + +### 6.3 处理流程 + +``` +1. 读 cards.csv 得到 iccid 列表 +2. 连奇成（只读事务）批量查： + - tbl_card_life WHERE iccid_mark IN (...) AND status=1 ORDER BY expire_date DESC + → 每张卡取最新生效的套餐：meal_id, meal_name, start_date, expire_date + - tbl_card WHERE iccid_mark IN (...) → total_bytes_cnt (GB) + - tbl_agent_commission_account → 按 agent_id 聚合 can_draw_amount, total_commision_amount + - tbl_agent，根据 cards.csv 中 agent 信息映射出 shop_id +3. 对每张卡： + a. 套餐映射：legacy_meal_name → package_id（从 yaml 或业务方 csv） + b. 生成伪订单 SQL（order_no = "MIG-CARD-" + iccid 后8位 + 时间戳） + c. 生成 tb_package_usage SQL（关联伪订单 + 计算 snapshot 字段） + d. 生成 UPDATE tb_iot_card SET data_usage_mb = ... （把 GB 转 MB） +4. 对每个 shop（按代理映射出来的）： + a. 生成 INSERT/UPDATE tb_agent_wallet（总值） + b. 生成一条 tb_agent_wallet_transaction（type=initial_migration, amount=总值, remark=源奇成代理ID） +5. 输出 SQL +``` + +### 6.4 输出文件 + +``` +output/ + step2_01_migration_orders.sql # 伪订单 + step2_02_package_usages.sql # 套餐使用记录 + step2_03_card_data_usage_update.sql # 卡累计流量 UPDATE + step2_04_agent_wallet_init.sql # 代理钱包总值 + step2_05_agent_wallet_transactions.sql # 迁移流水 + warnings.csv # 警告：找不到当前生效套餐、无法映射的代理等 + summary.txt +``` + +### 6.5 伪订单 + 套餐使用记录的 SQL 形态 + +```sql +-- 伪订单（一条 SQL 同时插订单和 package_usage） +WITH new_order AS ( + INSERT INTO tb_order ( + order_no, order_type, buyer_type, buyer_id, iot_card_id, + total_amount, payment_method, payment_status, paid_at, + source, generation, creator, updater, created_at, updated_at + ) + SELECT 'MIG-CARD-' || RIGHT('89852000263338772439', 8) || '-' || TO_CHAR(NOW(), 'YYYYMMDDHH24MISS'), + 'single_card', 'personal', 0, c.id, + 0, 'offline', 2, NOW(), + 'migration', 1, 0, 0, NOW(), NOW() + FROM tb_iot_card c WHERE c.iccid = '89852000263338772439' AND c.deleted_at IS NULL + ON CONFLICT (order_no) DO NOTHING + RETURNING id, order_no, iot_card_id +) +INSERT INTO tb_package_usage ( + order_id, order_no, package_id, usage_type, iot_card_id, + data_limit_mb, virtual_total_mb_snapshot, display_gain_ratio_snapshot, + enable_virtual_data_snapshot, status, priority, + activated_at, expires_at, + package_name, package_price_config_status, package_is_gift, + data_reset_cycle, next_reset_at, generation, + creator, updater, created_at, updated_at +) +SELECT no.id, no.order_no, p.id, 'single_card', no.iot_card_id, + p.real_data_mb, + CASE WHEN p.enable_virtual_data THEN p.virtual_data_mb ELSE p.real_data_mb END, + p.virtual_ratio, p.enable_virtual_data, + 1, 1, + NULL, -- activated_at 奇成多数为 NULL + '2027-04-28 16:00:00'::timestamp, -- 从奇成 tbl_card_life.expire_date 拿 + p.package_name, p.price_config_status, p.is_gift, + p.data_reset_cycle, NULL, + 1, 0, 0, NOW(), NOW() +FROM new_order no +JOIN tb_package p ON p.id = 123 AND p.deleted_at IS NULL -- package_id 来自映射 +ON CONFLICT DO NOTHING; +``` + +--- + +## 七、运行流程 + +### 7.1 准备阶段（在我们的开发/测试机上） + +```bash +# 1. 业务方提供 +cp 业务方发的*.csv scripts/migration/resources/ + +# 2. 我们/业务方在新系统线上建好店铺/运营商/套餐系列/套餐 +# （另行手动完成，本脚本不负责） + +# 3. 我们维护映射 yaml +vim scripts/migration/config/*.yaml + +# 4. 跑脚本1（离线，只读 csv + yaml） +cd scripts/migration +python migrate_assets.py +# 看 output/errors.csv，找业务方修复后重跑 + +# 5. 拿 step1 SQL 在测试环境跑一遍，确认无报错 +psql -h 测试库 -f output/step1_01_iot_cards.sql +psql -h 测试库 -f output/step1_02_devices.sql +... (按编号顺序) + +# 6. 跑脚本2（直连奇成 + 测试环境读 id） +python migrate_runtime.py +# 测试环境跑一遍 step2 SQL，验证最终态 + +# 7. 验证 OK，把所有 .sql 文件交给运维 +``` + +### 7.2 线上执行（用户手动） + +按编号顺序执行： + +```bash +# 第一波：资产 +psql -h 线上库 -1 -f step1_01_iot_cards.sql +psql -h 线上库 -1 -f step1_02_devices.sql +psql -h 线上库 -1 -f step1_03_asset_identifiers.sql +psql -h 线上库 -1 -f step1_04_asset_wallets.sql +psql -h 线上库 -1 -f step1_05_sim_bindings.sql +psql -h 线上库 -1 -f step1_06_shop_allocations.sql + +# 第二波：关联数据 +psql -h 线上库 -1 -f step2_01_migration_orders.sql +psql -h 线上库 -1 -f step2_02_package_usages.sql +psql -h 线上库 -1 -f step2_03_card_data_usage_update.sql +psql -h 线上库 -1 -f step2_04_agent_wallet_init.sql +psql -h 线上库 -1 -f step2_05_agent_wallet_transactions.sql +``` + +`-1` 单事务执行，失败自动回滚整文件。 + +--- + +## 八、目录结构（最终交付） + +``` +scripts/migration/ + README.md # 操作手册 + requirements.txt # python 依赖（pymysql, psycopg2, pyyaml, pandas） + migrate_assets.py # 脚本1 + migrate_runtime.py # 脚本2 + lib/ + legacy_query.py # 奇成只读查询封装 + csv_loader.py # csv 读取与校验 + mapping_loader.py # yaml 映射加载 + iccid_utils.py # 19/20 位拆分（移植 pkg/utils/iccid.go） + sql_builder.py # SQL 生成（CTE 模板） + resources/ + cards.csv # 业务方提供 + devices.csv # 业务方提供 + config/ + carrier_mapping.yaml + series_mapping.yaml + package_mapping.yaml + shop_mapping.yaml + migration.yaml # 全局配置（伪订单前缀、migration_user_id 等） + legacy_dsn.yaml # 奇成连接（.gitignore） + main_test_dsn.yaml # 新系统测试库连接（.gitignore） + output/ + step1_*.sql + step2_*.sql + errors.csv + warnings.csv + summary.txt +``` + +--- + +## 九、风险与未决事项 + +| # | 风险 | 应对 | +|---|------|------| +| 1 | 业务方填的 carrier_name / package_name 与奇成不一致 → 映射失败 | 脚本预扫描，把所有未匹配项写到 unmatched.csv 让业务方修正 | +| 2 | 奇成的 iccid 有 18/21/22 位脏数据 | 脚本第三节规则剔除，进 errors.csv | +| 3 | 同一 iccid 被业务方填多次 | 脚本去重 + 警告 | +| 4 | 设备上的卡在 cards.csv 里又写了 target_shop_code | 一致性校验，进 errors.csv | +| 5 | 奇成 tbl_card_life 一张卡有多条 status=1 记录 | 取 expire_date 最大的那条，warnings.csv 记录 | +| 6 | 套餐 ID 映射不全 | 脚本预扫描，缺的列出来，业务方/我们补全 yaml | +| 7 | tb_iot_card.is_standalone 由触发器维护 | 脚本写入时设 true，触发器会因为后续 binding 自动转 false | +| 8 | 代理→shop 映射不明确 | 需要业务方提供 agent_id → shop_code 映射表，作为 config/agent_shop_mapping.yaml | +| 9 | 线上库 sequence 与测试库不同 | 用 CTE + RETURNING 模式，不硬编码 id | +| 10 | 重复执行 | ON CONFLICT DO NOTHING 保证幂等 | + +--- + +## 十、待决事项的最终答案 + +### 10.1 代理→店铺映射（业务方提供） + +新增 `config/agent_shop_mapping.yaml`，由业务方维护： + +```yaml +# 奇成 agent_id → 新系统 shop_code 映射 +mappings: + - legacy_agent_id: "5846ACF63BD94ED2A26072BA5411B197" + legacy_agent_name: "SX20250721101058" + target_shop_code: "SHOP20260428103424CZZQ" + # ... +``` + +脚本2 通过这份映射把奇成佣金归属到新系统的 shop_id 上。映射不全的奇成代理，其佣金进 `warnings.csv` 让业务方补全。 + +### 10.2 伪订单字段填法（全部复用现有常量，不动状态机） + +`tb_order.buyer_id` 是 `NOT NULL bigint`，**平台自营/赠送场景代码里就是填 0**： +- `internal/service/order/service.go:251`（赠送场景） +- `internal/service/order/service.go:271`（平台自营） + +迁移伪订单字段最终值： + +| 字段 | 值 | 说明 | +|------|-----|------| +| `order_no` | `MIG-CARD--<时间戳>` / `MIG-DEV--<时间戳>` | 前缀就是迁移数据的唯一识别标记 | +| `source` | `'admin'`（复用现有） | 不新增 `migration` 取值 | +| `buyer_type` | `'personal'` | 与 buyer_id=0 配对 | +| `buyer_id` | `0` | 与赠送/平台自营一致 | +| `payment_method` | `'offline'`（复用现有 `model/order.go:99`） | 已有常量，只是字符串快照，不触发状态机 | +| `payment_status` | `2`（已支付） | 避免被支付回调扫到 | +| `paid_at` | `NOW()` | 已支付时间 | +| `total_amount` / `actual_paid_amount` | `0` | 避免汇总统计错乱 | +| `commission_status` | `2`（已完成） | 避免被佣金调度器扫描 | +| `commission_result` | `2`（无佣金） | 不进入佣金链路 | +| `operator_account_type` | `'platform'` | 操作者为平台 | +| `operator_account_name` | `'数据迁移'` | 备注 | + +**识别迁移订单**：`WHERE order_no LIKE 'MIG-%'`（不依赖 source 字段） + +### 10.3 shop_id_tag：平台库存填 0，有店铺填 shop_id + +`tb_asset_wallet.shop_id_tag` 是 `NOT NULL bigint`，代码里平台库存场景就是填 0： +- `internal/service/exchange/service.go:297-300` +- `internal/service/client_order/service.go:723`：`shopTag := 0; if card.ShopID != nil { shopTag = *card.ShopID }` + +迁移直接套这个逻辑。同理 `enterprise_id_tag` 可空，统一填 NULL（迁移阶段没有企业归属）。 + +### 10.4 代理佣金字段映射 + +新系统 `tb_agent_wallet` 设计是**双钱包**：每个店铺有两条记录，`wallet_type` 区分： +- `wallet_type='main'`：主钱包（充值消费的钱） +- `wallet_type='commission'`：分佣钱包（佣金） + +映射关系： + +| 新系统字段 | 奇成字段 | 说明 | +|-----------|---------|------| +| `main.balance` | `tbl_agent.balance_money` × 100 | 充值余额（奇成是元，新系统是分） | +| `commission.balance` | `tbl_agent_commission_account.can_draw_amount` × 100 | 当前可提现佣金 | +| `commission.frozen_balance` | `tbl_agent_commission_account.applying_draw_amount` × 100 | 申请中的佣金 | + +`total_commision_amount`（累计总佣金）和 `total_draw_amount`（已提现累计）是**历史明细汇总值**，新系统的 `tb_agent_wallet` 没有对应字段；如果业务方需要这些历史数据展示，可以写到迁移流水的 `metadata` JSONB 里作为参考，但不影响余额计算。 + +每个 shop 生成 **2 条** `tb_agent_wallet`（main + commission）+ **2 条** `tb_agent_wallet_transaction`，全部复用现有常量： + +| 字段 | 值 | 说明 | +|------|-----|------| +| `transaction_type` | `'recharge'`（复用） | 表示"初始化余额" | +| `transaction_subtype` | `NULL` | 字段允许 NULL，原本为 deduct 细分预留 | +| `reference_type` | `'topup'`（复用 `pkg/constants/wallet.go:115`） | 与 recharge 配对 | +| `reference_id` | `NULL` | 不关联具体业务，避免误关联 | +| `amount` | 奇成余额 × 100（元→分） | | +| `balance_before` | `0` | 迁移前为 0 | +| `balance_after` | 同 amount | | +| `status` | `1`（成功） | | +| `remark` | `"奇成数据迁移初始化，源 agent_id=, agent_name="` | 文本识别 | +| `metadata` | `{"source":"qicheng","legacy_agent_id":"...","legacy_field":"can_draw_amount"}` | JSON 结构化识别 | + +**识别迁移流水**：`WHERE remark LIKE '%奇成迁移%' OR metadata->>'source' = 'qicheng'` + +--- + +## 十一、不破坏状态机的总体策略 + +**核心原则**：本次迁移**不新增任何常量**，全部复用现有取值。识别迁移数据靠 `order_no` 前缀 + `remark` / `metadata` 字段，而不是污染枚举字段。 + +**避免触发后续状态机的关键设置已散落在 10.2/10.4**，汇总如下： +- 伪订单：`payment_status=2`(已支付) + `commission_status=2`(已完成) + `commission_result=2`(无佣金) → 不被支付回调、佣金调度器扫到 +- 钱包流水：`reference_id=NULL` → 不与真实业务关联，不被对账任务扫到 +- 金额：迁移订单 `total_amount=0`、`actual_paid_amount=0` → 即使被汇总统计扫到也不污染收入数据 + +**遗留风险**： +- 后台订单列表若按 `source='admin'` 过滤，会看到迁移订单；建议提前做一个常用过滤条件（`order_no NOT LIKE 'MIG-%'`），或者前端订单列表加个"排除迁移数据"开关 +- `payment_method='offline'` 在后台报表里会归类到"线下支付"统计中；如果运营对此敏感，需在报表 SQL 中加 `order_no NOT LIKE 'MIG-%'` 过滤 + +--- + +## 十二、实施顺序（审查通过后） + +1. 业务方启动准备：明确 `cards.csv` / `devices.csv` / `agent_shop_mapping.yaml` 的格式，开始整理 +2. 我们这边： + - 写脚本1（migrate_assets.py + lib + 模板） + - 在测试库验证脚本1 + - 写脚本2（migrate_runtime.py） + - 在测试库验证脚本2 + - 写 README.md 操作手册（含"如何在报表中排除迁移数据"提示） +3. 业务方提供数据 → 全流程联调一次 → 交付给运维上线 diff --git a/scripts/migration/.python-version b/scripts/migration/.python-version new file mode 100644 index 0000000..976544c --- /dev/null +++ b/scripts/migration/.python-version @@ -0,0 +1 @@ +3.13.7 diff --git a/scripts/migration/README.md b/scripts/migration/README.md new file mode 100644 index 0000000..0da132d --- /dev/null +++ b/scripts/migration/README.md @@ -0,0 +1,227 @@ +# 奇成数据迁移脚本 + +> 详细方案见 [`docs/脚本/奇成数据迁移方案.md`](../../docs/脚本/奇成数据迁移方案.md)。 + +## 1. 四步走 + +``` +业务方 我们 运维 + │ │ │ + │ cards.csv │ scan_legacy.py → mapping.yaml │ + │ devices.csv │ ↓ 人工填 target_* │ + ├─────────────▶│ migrate_assets.py → step1_*.sql │ + │ │ migrate_runtime.py → step2_*.sql │ + │ ├────────────────────────────────────▶│ psql -1 -f step*.sql + │ │ │ migration-finalize + │ │ │ 重建轮询队列/缓存 +``` + +## 2. 文件清单 + +``` +scripts/migration/ +├── README.md +├── requirements.txt +├── scan_legacy.py ← 第1步:扫描奇成,生成 mapping.yaml 模板 +├── migrate_assets.py ← 第2步:生成资产 SQL +├── migrate_runtime.py ← 第3步:生成关联数据 SQL +├── lib/ ← 公共库 +├── resources/ +│ ├── README.md +│ ├── cards.csv.example ← 业务方提供:iccid + 可选覆盖列 +│ └── devices.csv.example ← 业务方提供:设备 + 卡槽 +├── config/ +│ ├── mapping.yaml.example ← 唯一映射配置(全局 + carrier + package + series + agent) +│ └── legacy_dsn.yaml.example← 奇成 MySQL 只读 DSN +└── output/ ← 生成的 SQL 与日志 +``` + +## 3. 详细流程 + +### 3.1 业务方提供数据 + +把两个 CSV 放到 `resources/` 下(命名为 `cards.csv` / `devices.csv`): + +- **cards.csv**:`iccid_19` 必填(所有卡前 19 位),`iccid_20` 视运营商必填或留空,`is_industry` 可选 +- **devices.csv**:`virtual_no` 必填,`sim_iccid_*` 引用 `cards.csv` 中的 ICCID(可填 19 或 20 位) + +字段说明详见 [`resources/README.md`](resources/README.md)。 + +### 3.2 我们扫描 + 填映射 + +```bash +pip install -r requirements.txt +cp config/legacy_dsn.yaml.example config/legacy_dsn.yaml # 填真实 DSN +cp config/mapping.yaml.example config/mapping.yaml # 第一次跑要有这个文件 + +# 扫描奇成,把发现的 carrier/package/agent 写入 mapping.yaml +python3 scan_legacy.py + +# 看 config/mapping_todo.txt,把 target_* 全部填好 +vim config/mapping.yaml +``` + +`scan_legacy.py` 可以反复跑(增量合并),已填的 `target_*` 不会被覆盖。 + +### 3.3 生成 SQL + +```bash +python3 migrate_assets.py # → output/step1_*.sql + errors.csv +python3 migrate_runtime.py # → output/step2_*.sql + warnings.csv +``` + +如果 `errors.csv` 有内容,意味着 `mapping.yaml` 还没填完整、或者某些 ICCID 在奇成查不到,处理完再重跑。 + +### 3.4 测试环境 / 线上执行 + +```bash +# 按编号顺序执行 +psql -h $HOST -1 -f output/step1_01_iot_cards.sql +psql -h $HOST -1 -f output/step1_02_devices.sql +psql -h $HOST -1 -f output/step1_03_asset_identifiers.sql +psql -h $HOST -1 -f output/step1_04_asset_wallets.sql +psql -h $HOST -1 -f output/step1_05_sim_bindings.sql +psql -h $HOST -1 -f output/step1_06_shop_allocations.sql + +psql -h $HOST -1 -f output/step2_01_migration_orders.sql +psql -h $HOST -1 -f output/step2_02_package_usages.sql +psql -h $HOST -1 -f output/step2_03_card_data_usage_update.sql +psql -h $HOST -1 -f output/step2_04_agent_wallet_init.sql +psql -h $HOST -1 -f output/step2_05_agent_wallet_transactions.sql +``` + +`-1` = 单事务,失败回滚整文件;`ON CONFLICT DO NOTHING` 保证重跑安全。 + +SQL 导入完成后必须按批次执行迁移收尾。SQL 只负责落库,不会触发 Service 生命周期, +也不会自动写入 Redis 轮询队列或卡缓存: + +```bash +# 先看本批次会处理多少卡、会写哪些轮询任务,不会写 Redis +go run ./cmd/migration-finalize --batch-no MIGRATION-QICHENG-YYYYMMDD + +# dry-run 无误后再执行,会按 tb_polling_config 重建本批卡的轮询队列和卡缓存 +go run ./cmd/migration-finalize --batch-no MIGRATION-QICHENG-YYYYMMDD --apply +``` + +## 4. 配置说明 (`mapping.yaml`) + +唯一需要人工填写的文件: + +```yaml +migration_user_id: 0 # 后台超级管理员 ID,作为所有 creator/updater +migration_batch_no: "MIGRATION-QICHENG-YYYYMMDD" # 本次迁移批次号,线上每批必须唯一 + +carriers: # 由 scan_legacy 自动发现 legacy_*,你只填 target_* + - legacy_category: 9 + legacy_category_name: "中国联通" + target_carrier_id: 1585 ← 必填:新库 tb_carrier.id + target_carrier_type: CUCC ← 必填:CMCC/CUCC/CTCC/CBN + target_carrier_name: "联通22-..." + +packages: # 同理 + - legacy_meal_id: "D6554244711498752" + legacy_meal_name: "三网充电宝全国畅玩年卡套餐..." + target_package_id: 2 ← 必填:新库 tb_package.id + +series: # 可选(留 null 表示不写入 tb_iot_card.series_id) + - legacy_series_id: "..." + legacy_series_name: "国内卡 无预存30天" + target_series_id: null + +agents: # 决定所有卡/设备的店铺归属 + - legacy_agent_id: "D15EC779767B412A82B8448AC2324F04" + legacy_agent_name: "叶发伟" + target_shop_code: "SHOP20260428103424CZZQ" # 留 null = 该代理的卡进平台库存 +``` + +### 店铺归属是怎么决定的 + +CSV 里**没有** `target_shop_code` 列,所有归属都从 `mapping.agents` 推导: + +| 资产 | 归属规则 | +|------|---------| +| 独立卡 | 奇成 `tbl_card.agent_id` → `mapping.agents[].target_shop_code` | +| 设备 | 主卡(`sim_iccid_1`)的 agent_id → 同上 | +| 设备上的卡 | 跟随设备(`shop_id=NULL`,由触发器+绑定关系维护) | + +如果 agent 在 `mapping.agents` 里 `target_shop_code` 留空 → 该资产进平台库存。 + +## 5. 识别迁移数据(后台报表过滤用) + +| 数据 | 识别条件 | +|------|---------| +| 迁移伪订单 | `WHERE order_no LIKE 'MIG-%'` | +| 迁移分配记录 | `WHERE allocation_no LIKE 'MIG-ALLOC-%'` | +| 迁移钱包流水 | `WHERE metadata->>'source' = 'qicheng'` | + +迁移伪订单号固定为 `MIG-CARD-`: + +- 20 位卡总长 29,19 位卡总长 28,符合新库 `order_no varchar(30)` 限制。 +- 订单号不包含生成时间,`step2_01_migration_orders.sql` 和 `step2_02_package_usages.sql` 分开重跑也能稳定匹配。 + +## 6. 流量用量迁移规则(真用量 / 虚用量) + +奇成与新系统的"虚流量"模型口径不同,必须做一次反算才能落对地方。 + +### 6.1 两边的口径差异 + +| | 奇成 | 新系统 | +|---|---|---| +| 用量存储 | `tbl_card.total_bytes_cnt`(varchar 小数 MB)= **虚用量**(真 × `(1 + flow_add_discount/100)`) | `tb_iot_card.data_usage_mb`(BIGINT MB)= **真用量**(网关返回的物理读数) | +| 加成配置 | `tbl_set_meal.flow_add_discount`(0~100 百分比,作用于"已用量") | `tb_package.enable_virtual_data` + `virtual_data_mb` + `virtual_ratio`(`= real_data_mb / virtual_data_mb`) | +| 用户看到的"已用" | = 数据库 `total_bytes_cnt`(已加成) | 前端用 `data_usage_mb × display_gain_ratio_snapshot` 现算 | + +### 6.2 反算公式 + +迁移脚本对每张卡按下式得到真用量,再写入 `data_usage_mb` / `current_month_usage_mb` / `last_gateway_reading_mb`: + +``` +real_used_mb = total_bytes_cnt / (1 + flow_add_discount / 100) +``` + +其中 `flow_add_discount` 取自该卡在 `tbl_card_life`(status=1, expire_date 最大)对应的当前生效套餐。 + +### 6.3 精度与近似 + +- **跨多个套餐周期的累计**:`total_bytes_cnt` 是按各时段对应套餐虚比加权累加的,这里只用"当前生效套餐"的虚比反算,**有误差但可接受**(奇成没有按套餐分段的用量明细,无更好办法)。 +- **无当前生效套餐却有 `total_bytes_cnt`**:按 `flow_add_discount=0` 处理(真=虚),并写入 `warnings.csv`(`kind=usage_without_active_package`)供人工核查。 +- **整数取整规则**(`data_usage_mb` 是 BIGINT): + - `real_used_mb = 0` → 写 0 + - `0 < real_used_mb < 1` → **上取整为 1**,避免"明明用过的卡显示 0" + - `real_used_mb ≥ 1` → 向下取整 +- **小数字段保留精度**:`current_month_usage_mb`(numeric)与 `last_gateway_reading_mb`(double)直接写 Decimal 真值,作为下次轮询增量计算的基线,**避免轮询第一次回写时产生跳变**。 + +### 6.4 套餐使用记录的虚流量快照 + +`tb_package_usage` 的 `virtual_total_mb_snapshot` / `display_gain_ratio_snapshot` / `enable_virtual_data_snapshot` 三个字段**全部从新系统 `tb_package` 当场快照**,与奇成无关: + +| 快照字段 | 取值 | +|---|---| +| `virtual_total_mb_snapshot` | 启用虚:`virtual_data_mb`;否则:`real_data_mb` | +| `display_gain_ratio_snapshot` | 启用虚:`virtual_ratio`(即 `real_data_mb/virtual_data_mb`);否则:`1.0` | +| `enable_virtual_data_snapshot` | 启用虚:`TRUE`;否则:`FALSE` | +| `data_limit_mb` | 始终 `real_data_mb`(真总量) | +| `data_usage_mb` | `LEAST(real_used_mb_floor, virtual_total_mb_snapshot)`(防卫上限) | + +## 7. 重要约束 + +- ❗ 所有奇成查询走 `SET TRANSACTION READ ONLY`,严禁误写。 +- ❗ 所有 INSERT 用 `ON CONFLICT DO NOTHING`,严禁更新已有数据。 +- ❗ 伪订单的 `payment_status=2 commission_status=2 commission_result=2`,避免被支付回调、佣金调度器扫到。 +- ❗ 迁移卡默认 `status=1(在库)` / `activation_status=0(未激活)`,激活由后续业务流程驱动。 + +## 8. 常见问题 + +| 现象 | 排查 | +|------|------| +| `scan_legacy.py` 报 `pymysql 缺失` | `pip install -r requirements.txt` | +| `errors.csv` 出现 `carrier_not_in_mapping` | `mapping.yaml.carriers` 没覆盖到这个 category,重跑 `scan_legacy.py` 自动补 | +| `errors.csv` 出现 `carrier_target_missing` | `mapping.yaml.carriers` 里有这一项但 `target_*` 还是 null,手动填 | +| `errors.csv` 出现 `not_in_legacy` | 该 ICCID 在奇成 `tbl_card` 查不到,业务方确认 | +| `errors.csv` 出现 `legacy_duplicate` | 同一 `iccid_19` 在奇成命中多条 `tbl_card`,业务方人工确认是哪张物理卡 | +| `errors.csv` 出现 `carrier_length_conflict` | 业务方填的 `iccid_20` 与奇成查到的运营商不匹配(CTCC 不该有 20 位 / 非 CTCC 必须有 20 位) | +| `errors.csv` 出现 `iccid_20_prefix_mismatch` | `iccid_20` 前 19 位 ≠ `iccid_19`,业务方校对 | +| `warnings.csv` 出现 `agent_no_legacy` | `mapping.agents` 配了但奇成没有 tbl_agent 记录,业务方确认 | +| `warnings.csv` 出现 `usage_without_active_package` | 该卡奇成 `total_bytes_cnt > 0` 但 `tbl_card_life` 中没有 status=1 的当前生效套餐,无法精确反算真用量(按真=虚处理),详见 §6.3 | +| 线上报 `null value in column "creator"` | `migration_user_id` 是 0,某些约束可能拒绝;改成真实管理员 ID | +| 线上报 `null shop_id` | `mapping.agents.target_shop_code` 指向的店铺在新库不存在,先建店铺 | diff --git a/scripts/migration/config/main_test_dsn.yaml b/scripts/migration/config/main_test_dsn.yaml new file mode 100644 index 0000000..8c1557d --- /dev/null +++ b/scripts/migration/config/main_test_dsn.yaml @@ -0,0 +1,11 @@ +# 新系统 PostgreSQL 测试库连接配置(只读) +# 脚本仅用它做"测试环境 id 探测",生成的线上 SQL 不会硬编码这些 id +# 实际使用时复制为 main_test_dsn.yaml(已 .gitignore) + +host: "cxd.whcxd.cn" +port: 16159 +user: "erp_pgsql" +password: "erp_2025" +database: "junhong_cmp_test" +sslmode: "disable" +connect_timeout: 10 diff --git a/scripts/migration/config/mapping.yaml.example b/scripts/migration/config/mapping.yaml.example new file mode 100644 index 0000000..65961a0 --- /dev/null +++ b/scripts/migration/config/mapping.yaml.example @@ -0,0 +1,47 @@ +# 奇成迁移映射配置(单文件) +# 复制为 mapping.yaml 后使用,会被 scan_legacy.py 自动增量补全 +# +# 工作流: +# 1. 业务方提供 resources/cards.csv(只需 iccid 列) +# 2. python3 scan_legacy.py → 自动从奇成扫描元数据,把发现的 carrier/package/agent 补到本文件 +# 3. 人工填写所有 target_* 字段 +# 4. python3 migrate_assets.py / migrate_runtime.py 正常出 SQL + +# ============== 全局配置 ============== +# 迁移操作者(creator/updater/operator_id),通常是后台超级管理员账号 ID +# 上线前确认: SELECT id FROM tb_account WHERE login_account = 'admin' LIMIT 1; +migration_user_id: 0 + +# 本次迁移批次号。线上每次迁移必须使用独立批次,用于导入后收尾、验收和回滚定位。 +migration_batch_no: "MIGRATION-QICHENG-YYYYMMDD" + +# ============== 运营商映射 ============== +# scan_legacy 会按"奇成 tbl_card.category"分组写入,你只需填 target_* +carriers: + - legacy_category: 9 + legacy_category_name: "中国联通" + target_carrier_id: null # 必填:新库 tb_carrier.id + target_carrier_type: null # 必填:CMCC / CUCC / CTCC / CBN + target_carrier_name: null # 必填:新库 carrier_name 快照 + +# ============== 套餐映射 ============== +# scan_legacy 会按"奇成 tbl_card_life.meal_id"分组写入 +packages: + - legacy_meal_id: "D6554244711498752" + legacy_meal_name: "三网充电宝全国畅玩年卡套餐1000G/月（12个月）" + target_package_id: null # 必填:新库 tb_package.id + +# ============== 套餐系列映射(可选) ============== +# 仅当你需要给迁移卡设置 tb_iot_card.series_id 时才填 +series: + - legacy_series_id: "95A038B9D3014F9DA1539EC092D9DBAA" + legacy_series_name: "国内卡 无预存30天" + target_series_id: null # 可选:新库 tb_package_series.id,null=不写入 + +# ============== 代理-店铺映射 ============== +# scan_legacy 会按"奇成 tbl_card.agent_id"分组写入,你只需填 target_shop_code +# 留空(null)的代理:其卡进平台库存(除非 cards.csv 里显式指定 target_shop_code) +agents: + - legacy_agent_id: "D15EC779767B412A82B8448AC2324F04" + legacy_agent_name: "叶发伟" + target_shop_code: null # 新库 tb_shop.shop_code,null=不归任何店铺 diff --git a/scripts/migration/lib/__init__.py b/scripts/migration/lib/__init__.py new file mode 100644 index 0000000..4f4bd5d --- /dev/null +++ b/scripts/migration/lib/__init__.py @@ -0,0 +1,9 @@ +"""奇成数据迁移脚本公共库。 + +模块组成: + iccid_utils - ICCID 长度校验 / 19-20 位拆分 + csv_loader - cards.csv / devices.csv 读取与清洗 + mapping_loader - yaml 映射文件加载 + legacy_query - 奇成 MySQL 只读查询封装 + sql_builder - 输出 SQL 构造(CTE + ON CONFLICT) +""" diff --git a/scripts/migration/lib/csv_loader.py b/scripts/migration/lib/csv_loader.py new file mode 100644 index 0000000..2e70e55 --- /dev/null +++ b/scripts/migration/lib/csv_loader.py @@ -0,0 +1,272 @@ +"""cards.csv / devices.csv 加载与跨文件一致性校验。 + +新设计:CSV 只承载"业务方决策"—— + - cards.csv 只读 iccid + 可选覆盖列(target_shop_code、is_industry) + - devices.csv 读设备元数据 + sim_iccid_* + - 卡的运营商、套餐、msisdn、agent_id 等元数据全部从奇成查(见 legacy_query.fetch_card_meta) +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +CARD_HEADERS_REQUIRED = ("iccid_19",) +# 设备列至少包含 imei 或 virtual_no 之一(运行时校验,允许只填一列) +DEVICE_HEADERS_AT_LEAST_ONE = ("imei", "virtual_no") + + +@dataclass +class CardRow: + """cards.csv 单行的解析结果(只含业务方决策字段)。 + + 业务方必须同时提供 iccid_19 和 iccid_20(20 位运营商): + - iccid_19:前 19 位,所有卡必填,作为跨文件稳定 ID + - iccid_20:20 位完整 ICCID,CMCC/CUCC/CBN 必填、CTCC 必须留空 + - 运营商类型与 iccid_20 的一致性在查完奇成后由 sql_builder 校验 + 新系统 iccid 唯一键 = iccid_20 优先,否则 iccid_19。 + """ + + line_no: int # 原始 CSV 行号(从 1 开始,含表头) + iccid_19: str + iccid_20: str = "" # 留空表示 CTCC(查完奇成会再校验) + is_industry: bool = False # 默认 False(普通卡) + + # 跨文件校验后填充 + bound_device_virtual_no: str = "" + + @property + def iccid_full(self) -> str: + """完整 ICCID:有 iccid_20 用 iccid_20,否则用 iccid_19。""" + return self.iccid_20 or self.iccid_19 + + +@dataclass +class DeviceRow: + """devices.csv 单行的解析结果。 + + 简化为业务方核心关心的列:imei / virtual_no / sim_iccid_*。 + 店铺归属由"主卡 sim_iccid_1 的 agent_id → mapping.agents.target_shop_code"推导。 + 其他字段(device_name/device_model/manufacturer 等)直接用默认值,不暴露给业务方。 + """ + + line_no: int + imei: str + virtual_no: str # 留空时用 imei 兜底,见 _load_devices + sim_iccids: dict[int, str] # slot → iccid(只保留非空) + max_sim_slots: int = 4 # 默认 4 槽位 + current_slot: int = 1 # 默认 slot 1 为当前生效 + + +@dataclass +class ErrorRow: + """异常行汇总(写入 output/errors.csv)。""" + + source_file: str + line_no: int + field: str + value: str + error_code: str + message: str + + def to_csv_row(self) -> list[str]: + return [self.source_file, str(self.line_no), self.field, self.value, self.error_code, self.message] + + +@dataclass +class LoadResult: + cards: list[CardRow] = field(default_factory=list) + devices: list[DeviceRow] = field(default_factory=list) + errors: list[ErrorRow] = field(default_factory=list) + + +# ---------------- 字段清洗 ---------------- + + +def _normalize_cell(raw: str) -> str: + if raw is None: + return "" + s = raw.strip() + if s.startswith(""): + s = s[1:] + if s.startswith("'"): + s = s[1:] + s = s.replace(" ", "") + return s.strip() + + +def _parse_bool(raw: str) -> Optional[bool]: + v = _normalize_cell(raw).lower() + if v == "": + return False + if v in ("true", "1", "yes", "y", "t", "是", "行业卡"): + return True + if v in ("false", "0", "no", "n", "f", "否", "普通卡"): + return False + return None + + +def _parse_int(raw: str, default: int = 0) -> int: + v = _normalize_cell(raw) + if not v: + return default + try: + return int(v) + except ValueError: + return default + + +# ---------------- 主入口 ---------------- + + +def load_assets(resources_dir: Path) -> tuple[list[CardRow], list[DeviceRow], list[ErrorRow]]: + """读 cards.csv + devices.csv,做格式校验与跨文件一致性校验。 + + 返回:(合法卡列表, 合法设备列表, 异常行列表)。 + 元数据(carrier/agent_id/msisdn 等)由调用方再用 legacy_query 补充。 + """ + result = LoadResult() + + cards_path = resources_dir / "cards.csv" + if not cards_path.exists(): + raise FileNotFoundError(f"缺少 {cards_path},请按 resources/README.md 准备数据") + _load_cards(cards_path, result) + + devices_path = resources_dir / "devices.csv" + if devices_path.exists(): + _load_devices(devices_path, result) + + _cross_check(result) + + return result.cards, result.devices, result.errors + + +def _load_cards(path: Path, result: LoadResult) -> None: + import csv + + from . import iccid_utils + + with path.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + header = [_normalize_cell(h) for h in (reader.fieldnames or [])] + missing = [k for k in CARD_HEADERS_REQUIRED if k not in header] + if missing: + raise ValueError(f"{path} 缺少必备列: {missing}") + if "iccid_20" not in header: + raise ValueError(f"{path} 必须包含 iccid_20 列(20 位运营商卡填写,CTCC 留空)") + + seen_iccid_19: dict[str, int] = {} + for idx, raw in enumerate(reader, start=2): + iccid_19 = _normalize_cell(raw.get("iccid_19", "")) + iccid_20 = _normalize_cell(raw.get("iccid_20", "")) + + err = iccid_utils.validate_iccid_19(iccid_19) + if err is not None: + result.errors.append(ErrorRow("cards.csv", idx, "iccid_19", iccid_19, err.code, err.message)) + continue + + err = iccid_utils.validate_iccid_20(iccid_20, iccid_19) + if err is not None: + result.errors.append(ErrorRow("cards.csv", idx, "iccid_20", iccid_20, err.code, err.message)) + continue + + if iccid_19 in seen_iccid_19: + result.errors.append( + ErrorRow("cards.csv", idx, "iccid_19", iccid_19, "duplicate", + f"与第 {seen_iccid_19[iccid_19]} 行 iccid_19 重复") + ) + continue + seen_iccid_19[iccid_19] = idx + + is_industry = _parse_bool(raw.get("is_industry", "")) + if is_industry is None: + result.errors.append( + ErrorRow("cards.csv", idx, "is_industry", raw.get("is_industry", ""), "invalid_bool", "is_industry 必须是 true/false") + ) + continue + + result.cards.append( + CardRow(line_no=idx, iccid_19=iccid_19, iccid_20=iccid_20, is_industry=is_industry) + ) + + +def _load_devices(path: Path, result: LoadResult) -> None: + import csv + + with path.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + header = [_normalize_cell(h) for h in (reader.fieldnames or [])] + if not any(k in header for k in DEVICE_HEADERS_AT_LEAST_ONE): + raise ValueError(f"{path} 必须至少包含 imei 或 virtual_no 列") + + seen_virtual_no: dict[str, int] = {} + for idx, raw in enumerate(reader, start=2): + imei = _normalize_cell(raw.get("imei", "")) + virtual_no = _normalize_cell(raw.get("virtual_no", "")) + # virtual_no 留空时用 imei 兜底(业务方说"虚拟号一般会提供,没有就用 imei") + effective_vno = virtual_no or imei + if not effective_vno: + result.errors.append( + ErrorRow("devices.csv", idx, "virtual_no", "", "missing_identity", + "imei 和 virtual_no 不能同时为空") + ) + continue + if effective_vno in seen_virtual_no: + result.errors.append( + ErrorRow("devices.csv", idx, "virtual_no", effective_vno, "duplicate", + f"与第 {seen_virtual_no[effective_vno]} 行的设备号重复") + ) + continue + seen_virtual_no[effective_vno] = idx + + sim_iccids: dict[int, str] = {} + for slot in (1, 2, 3, 4): + v = _normalize_cell(raw.get(f"sim_iccid_{slot}", "")) + if v: + sim_iccids[slot] = v + + result.devices.append( + DeviceRow( + line_no=idx, + imei=imei, + virtual_no=effective_vno, + sim_iccids=sim_iccids, + ) + ) + + +def _cross_check(result: LoadResult) -> None: + """校验设备与卡之间的一致性,并把 bound_device_virtual_no 回填到 CardRow。 + + devices.csv 里 sim_iccid_X 业务方可能填 19 位也可能填 20 位, + 统一截取前 19 位回查 cards.iccid_19。 + """ + from . import iccid_utils + + cards_by_iccid_19: dict[str, CardRow] = {c.iccid_19: c for c in result.cards} + device_card_refs: dict[str, tuple[str, int]] = {} + + for device in result.devices: + for slot, raw_iccid in device.sim_iccids.items(): + key19 = iccid_utils.prefix_19(raw_iccid) + if key19 not in cards_by_iccid_19: + result.errors.append( + ErrorRow( + "devices.csv", device.line_no, f"sim_iccid_{slot}", raw_iccid, + "card_not_found", "引用的 ICCID(前 19 位)在 cards.csv 中不存在", + ) + ) + continue + if key19 in device_card_refs: + prev_vno, prev_line = device_card_refs[key19] + result.errors.append( + ErrorRow( + "devices.csv", device.line_no, f"sim_iccid_{slot}", raw_iccid, + "card_used_twice", f"该 ICCID 已被设备 {prev_vno}(第 {prev_line} 行)使用", + ) + ) + continue + device_card_refs[key19] = (device.virtual_no, device.line_no) + cards_by_iccid_19[key19].bound_device_virtual_no = device.virtual_no + # 标准化:slot 里的值统一改写为 iccid_19,sql_builder 直接用它查 cards / card_metas + device.sim_iccids[slot] = key19 diff --git a/scripts/migration/lib/iccid_utils.py b/scripts/migration/lib/iccid_utils.py new file mode 100644 index 0000000..89b346b --- /dev/null +++ b/scripts/migration/lib/iccid_utils.py @@ -0,0 +1,158 @@ +"""ICCID 19/20 位拆分与校验工具。 + +移植自 pkg/utils/iccid.go,并按运营商类型增加长度合规校验。 +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +# 运营商标准 ICCID 长度 +CARRIER_ICCID_LENGTH = { + "CTCC": 19, # 电信 + "CMCC": 20, # 移动 + "CUCC": 20, # 联通 + "CBN": 20, # 广电 +} + +VALID_CARRIER_TYPES = set(CARRIER_ICCID_LENGTH.keys()) + + +@dataclass(frozen=True) +class SplitResult: + """ICCID 拆分结果。""" + + iccid: str # 原始 ICCID(去空格) + iccid_19: str # 前 19 位(所有卡必填) + iccid_20: Optional[str] # 完整 20 位(仅 20 位运营商卡有值,19 位卡为 None → SQL NULL) + + +@dataclass(frozen=True) +class ValidationError: + """ICCID 校验失败信息。""" + + code: str # 错误码:length_invalid / length_mismatch + message: str # 中文描述 + + +def split_iccid(iccid: str) -> Optional[SplitResult]: + """按 ICCID 长度拆分双列存储值。 + + 与 Go 版本一致: + 19 位 → (iccid, iccid, None) + 20 位 → (iccid, iccid[:19], iccid) + 其他 → None(调用方按异常处理) + """ + iccid = iccid.strip() + if len(iccid) == 19: + return SplitResult(iccid=iccid, iccid_19=iccid, iccid_20=None) + if len(iccid) == 20: + return SplitResult(iccid=iccid, iccid_19=iccid[:19], iccid_20=iccid) + return None + + +def validate_and_split(iccid: str, carrier_type: str) -> tuple[Optional[SplitResult], Optional[ValidationError]]: + """按运营商类型校验 ICCID 长度并拆分。 + + 返回 (拆分结果, 错误信息) 二元组: + - 长度不是 19/20:返回 (None, length_invalid 错误) + - 长度与 carrier_type 不匹配:返回 (拆分结果, length_mismatch 警告) → 调用方决定要不要放行 + - 合法:返回 (拆分结果, None) + + 设计说明:不直接拒绝长度不匹配的情况(电信卡居然是 20 位 / 非电信卡是 19 位), + 保留拆分结果让脚本写入 errors.csv 让业务方确认。 + """ + iccid = iccid.strip() + if carrier_type not in VALID_CARRIER_TYPES: + return None, ValidationError( + code="carrier_type_invalid", + message=f"无效的运营商类型 {carrier_type!r},应为 {sorted(VALID_CARRIER_TYPES)} 之一", + ) + + result = split_iccid(iccid) + if result is None: + return None, ValidationError( + code="length_invalid", + message=f"ICCID 长度 {len(iccid)} 不合法,只支持 19/20 位", + ) + + expected_len = CARRIER_ICCID_LENGTH[carrier_type] + if len(iccid) != expected_len: + return result, ValidationError( + code="length_mismatch", + message=f"{carrier_type} 应为 {expected_len} 位但 ICCID 是 {len(iccid)} 位,请人工确认", + ) + + return result, None + + +def prefix_19(iccid: str) -> str: + """ICCID 前 19 位(稳定 ID,无校验位)。 + + 用于 devices.csv 引用 cards 时的兼容查找:业务方在 devices 里填 19 还是 20 位都接受, + 内部统一截取前 19 位回查 cards.iccid_19。 + """ + s = (iccid or "").strip() + if len(s) >= 19: + return s[:19] + return s + + +def validate_iccid_19(iccid_19: str) -> Optional[ValidationError]: + """iccid_19 必须 19 位纯数字。""" + if not iccid_19: + return ValidationError(code="iccid_19_required", message="iccid_19 不能为空") + if len(iccid_19) != 19: + return ValidationError( + code="iccid_19_length", + message=f"iccid_19 必须是 19 位,当前 {len(iccid_19)} 位", + ) + if not iccid_19.isdigit(): + return ValidationError(code="iccid_19_format", message="iccid_19 必须是纯数字") + return None + + +def validate_iccid_20(iccid_20: str, iccid_19: str) -> Optional[ValidationError]: + """iccid_20 可空(CTCC 卡);非空时必须 20 位纯数字且前 19 位 == iccid_19。""" + if not iccid_20: + return None + if len(iccid_20) != 20: + return ValidationError( + code="iccid_20_length", + message=f"iccid_20 必须是 20 位,当前 {len(iccid_20)} 位", + ) + if not iccid_20.isdigit(): + return ValidationError(code="iccid_20_format", message="iccid_20 必须是纯数字") + if iccid_20[:19] != iccid_19: + return ValidationError( + code="iccid_20_prefix_mismatch", + message=f"iccid_20 前 19 位 ({iccid_20[:19]!r}) 与 iccid_19 ({iccid_19!r}) 不一致", + ) + return None + + +def validate_carrier_iccid_match( + iccid_20: str, carrier_type: str +) -> Optional[ValidationError]: + """业务方填写的 iccid_20 与奇成查到的 carrier_type 是否匹配。 + + - CTCC(19 位运营商):iccid_20 必须为空 + - CMCC/CUCC/CBN(20 位运营商):iccid_20 必填 + """ + if carrier_type not in VALID_CARRIER_TYPES: + return ValidationError( + code="carrier_type_invalid", + message=f"未知运营商类型 {carrier_type!r}", + ) + expected_len = CARRIER_ICCID_LENGTH[carrier_type] + if expected_len == 19 and iccid_20: + return ValidationError( + code="carrier_length_conflict", + message=f"{carrier_type} 是 19 位运营商,iccid_20 应留空,实际填了 {iccid_20!r}", + ) + if expected_len == 20 and not iccid_20: + return ValidationError( + code="carrier_length_conflict", + message=f"{carrier_type} 是 {expected_len} 位运营商,iccid_20 必填", + ) + return None diff --git a/scripts/migration/lib/legacy_query.py b/scripts/migration/lib/legacy_query.py new file mode 100644 index 0000000..345f05f --- /dev/null +++ b/scripts/migration/lib/legacy_query.py @@ -0,0 +1,510 @@ +"""奇成 MySQL 只读查询封装。 + +接入 kyhl 库,仅 SELECT,强制 READ ONLY 事务防止误写。 +所有查询按 ICCID/agent_id 批量分块,避免单条 SQL 过大。 +""" +from __future__ import annotations + +import contextlib +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Iterable, Iterator, Optional + + +BATCH_SIZE = 500 + + +def _import_pymysql(): + """延迟导入 pymysql,这样脚本1(纯离线)不需要安装 pymysql 也能运行。""" + try: + import pymysql + from pymysql.cursors import DictCursor + return pymysql, DictCursor + except ImportError as exc: + raise ImportError("缺少 pymysql,请先 pip install -r requirements.txt") from exc + + +@dataclass(frozen=True) +class LegacyPackage: + """奇成当前生效套餐快照(取自 tbl_card_life)。""" + + iccid: str + meal_id: str + meal_name: str + start_date: Optional[str] + expire_date: Optional[str] + flow_size_mb: int # 套餐总量,单位 MB + status: int + + +@dataclass(frozen=True) +class LegacyCardUsage: + """奇成卡累计流量快照。 + + 奇成 ``tbl_card.total_bytes_cnt`` 存的是经过当前套餐 ``flow_add_discount`` + 加成后的"虚用量",我们这里同时拉取当前生效套餐(``tbl_card_life`` status=1 + 且 ``expire_date`` 最大)的 ``flow_add_discount``,反算回新系统期望的"真用量"。 + + 精度说明:跨多个套餐周期累计时,``total_bytes_cnt`` 是按各时段对应套餐 + 虚比加权累加的,这里只用"当前生效套餐"的虚比反算,**有误差但可接受**—— + 奇成没有按套餐分段的用量明细,无更好做法。 + """ + + iccid: str # 业务方视角的 iccid_19 + virtual_used_mb: Decimal # 奇成 total_bytes_cnt 原始值(虚 MB) + flow_add_discount_pct: Decimal # 当前生效套餐虚比百分比(0~100) + has_active_package: bool # 是否在 tbl_card_life 中找到当前生效套餐 + + @property + def real_used_mb(self) -> Decimal: + """反算真用量(Decimal MB,保留小数)。 + + 虚 = 真 × (1 + pct/100) → 真 = 虚 / (1 + pct/100) + """ + if self.virtual_used_mb <= 0: + return Decimal("0") + ratio = Decimal(1) + self.flow_add_discount_pct / Decimal(100) + return self.virtual_used_mb / ratio + + @property + def real_used_mb_floor(self) -> int: + """适配新系统 ``data_usage_mb``(BIGINT)的整数 MB。 + + 取整规则:0 → 0,(0, 1) → 1,≥1 → 向下取整。 + sub-MB 上取整为 1,避免"明明用过的卡显示 0"。 + """ + real = self.real_used_mb + if real <= 0: + return 0 + if real < 1: + return 1 + return int(real) + + +@dataclass(frozen=True) +class LegacyCommissionAccount: + """奇成代理佣金账户快照。""" + + agent_id: str + agent_name: str + can_draw_amount_fen: int # 当前可提现佣金(分) + applying_draw_amount_fen: int # 申请提现中佣金(分) + total_commission_amount_fen: int # 累计总佣金(分,做参考) + total_draw_amount_fen: int # 累计已提现(分,做参考) + + +@dataclass(frozen=True) +class LegacyAgentBalance: + """奇成代理主钱包余额快照(取自 tbl_agent.balance_money)。""" + + agent_id: str + balance_fen: int + + +@dataclass(frozen=True) +class LegacyCardMeta: + """奇成卡基础元数据(由 scan_legacy / migrate_assets 共享)。 + + 业务方提供的 iccid_19 + iccid_20 才是新系统的权威值,奇成的 iccid_mark 长度不可信。 + legacy_raw_iccid 仅作调试参考。 + """ + + legacy_raw_iccid: str # 奇成实际存的 iccid_mark(可能 19 或 20 位) + category: int # tbl_card.category(运营商分类编号) + category_name: str # tbl_vendor_category.category_name + agent_id: str + agent_name: str + msisdn: str # tbl_card.phone + virtual_no: str # tbl_virtual_number.vcode(如有) + current_meal_id: str # 当前生效套餐 ID(可空,取自 tbl_card_life status=1 expire_date 最大) + current_meal_name: str + current_expire_date: Optional[str] # ISO 字符串 + + +@dataclass(frozen=True) +class LegacyMealMeta: + """奇成套餐 + 系列元数据(由 scan_legacy 用)。""" + + meal_id: str + meal_name: str + series_id: str + series_name: str + + +# ---------------- 连接管理 ---------------- + + +@contextlib.contextmanager +def connect_readonly(dsn: dict) -> Iterator[Any]: + """打开只读连接。 + + 用法: + with connect_readonly(dsn) as conn: + rows = fetch_current_packages(conn, iccids) + """ + pymysql, DictCursor = _import_pymysql() + conn = pymysql.connect( + host=dsn["host"], + port=int(dsn.get("port", 3306)), + user=dsn["user"], + password=dsn["password"], + database=dsn["database"], + charset=dsn.get("charset", "utf8mb4"), + connect_timeout=int(dsn.get("connect_timeout", 10)), + cursorclass=DictCursor, + autocommit=False, + ) + try: + with conn.cursor() as cur: + cur.execute("SET SESSION TRANSACTION READ ONLY") + cur.execute("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ") + conn.begin() + yield conn + conn.rollback() # 只读事务,回滚即结束 + finally: + conn.close() + + +def _chunks(items: list[str], size: int = BATCH_SIZE) -> Iterator[list[str]]: + for start in range(0, len(items), size): + yield items[start : start + size] + + +def _to_fen(value) -> int: + """元 → 分。奇成的 decimal 是元单位。""" + if value is None: + return 0 + try: + d = Decimal(str(value)) + except Exception: + return 0 + return int((d * Decimal("100")).quantize(Decimal("1"))) + + +def _to_mb_decimal(value) -> Decimal: + """字符串/数字 → Decimal MB。奇成 total_bytes_cnt 是小数 MB 读数。""" + if value is None or value == "": + return Decimal("0") + try: + d = Decimal(str(value)) + if d < 0: + return Decimal("0") + return d + except Exception: + return Decimal("0") + + +# ---------------- 查询函数 ---------------- + + +def fetch_current_packages(conn, iccids: Iterable[str]) -> dict[str, LegacyPackage]: + """查每张卡的当前生效套餐:status=1 且 expire_date 最大。 + + 返回: iccid → LegacyPackage(找不到的 iccid 不在结果中) + """ + iccid_list = sorted({i for i in iccids if i}) + out: dict[str, LegacyPackage] = {} + if not iccid_list: + return out + + sql = """ + SELECT + iccid_mark, + meal_id, + meal_name, + start_date, + expire_date, + COALESCE(flow_size, 0) AS flow_size, + status + FROM tbl_card_life + WHERE iccid_mark IN %s + AND status = 1 + ORDER BY iccid_mark ASC, expire_date DESC + """ + seen: set[str] = set() + with conn.cursor() as cur: + for batch in _chunks(iccid_list): + cur.execute(sql, (tuple(batch),)) + for row in cur.fetchall(): + iccid = row["iccid_mark"] + if not iccid or iccid in seen: + continue + seen.add(iccid) + out[iccid] = LegacyPackage( + iccid=iccid, + meal_id=row.get("meal_id") or "", + meal_name=row.get("meal_name") or "", + start_date=row["start_date"].strftime("%Y-%m-%d %H:%M:%S") if row.get("start_date") else None, + expire_date=row["expire_date"].strftime("%Y-%m-%d %H:%M:%S") if row.get("expire_date") else None, + flow_size_mb=int(row.get("flow_size") or 0), + status=int(row.get("status") or 0), + ) + return out + + +def fetch_card_usage(conn, iccid_pairs: Iterable[tuple[str, str]]) -> dict[str, LegacyCardUsage]: + """查每张卡的累计已用流量,并同步取当前生效套餐的 flow_add_discount。 + + 奇成 ``tbl_card.total_bytes_cnt`` 是虚用量(已被套餐虚比加成)。 + 这里 JOIN 当前生效套餐(``tbl_card_life`` status=1 且 ``expire_date`` 最大) + + ``tbl_set_meal`` 取 ``flow_add_discount``,放进 ``LegacyCardUsage``; + 上层通过 ``real_used_mb_floor`` 反算真用量写入新系统。 + + Args: + iccid_pairs: (iccid_19, iccid_20) 列表;iccid_20 可为空字符串。 + + Returns: + {iccid_19: LegacyCardUsage}。 + """ + pairs = [(p[0], p[1]) for p in iccid_pairs if p[0]] + out: dict[str, LegacyCardUsage] = {} + if not pairs: + return out + + full_to_19: dict[str, str] = {} + for i19, i20 in pairs: + full_to_19[i19] = i19 + if i20: + full_to_19[i20] = i19 + full_set = sorted(full_to_19.keys()) + + # tbl_card 主表用 iccid_mark IN 全长精确匹配;tbl_card_life 从表用 + # LEFT(iccid_mark, 19) 兜底,兼容奇成两张表 iccid 长度不一致的脏数据。 + sql = """ + SELECT + c.iccid_mark, + c.total_bytes_cnt, + COALESCE(sm.flow_add_discount, 0) AS flow_add_discount, + CASE WHEN cl.iccid_mark IS NOT NULL THEN 1 ELSE 0 END AS has_active_package + FROM tbl_card c + LEFT JOIN ( + SELECT cl1.iccid_mark, cl1.meal_id + FROM tbl_card_life cl1 + INNER JOIN ( + SELECT iccid_mark, MAX(expire_date) AS max_expire + FROM tbl_card_life + WHERE status = 1 AND iccid_mark IN %s + GROUP BY iccid_mark + ) latest + ON latest.iccid_mark = cl1.iccid_mark + AND latest.max_expire = cl1.expire_date + WHERE cl1.status = 1 + ) cl ON LEFT(cl.iccid_mark, 19) = LEFT(c.iccid_mark, 19) + LEFT JOIN tbl_set_meal sm ON sm.id = cl.meal_id + WHERE c.iccid_mark IN %s + """ + with conn.cursor() as cur: + for batch in _chunks(full_set): + cur.execute(sql, (tuple(batch), tuple(batch))) + for row in cur.fetchall(): + legacy_iccid = row["iccid_mark"] + if not legacy_iccid: + continue + i19 = full_to_19.get(legacy_iccid) + if i19 is None: + continue + # 取第一个匹配,后续 fetch_card_meta 已经会把重复命中的卡标记为 errors + if i19 in out: + continue + out[i19] = LegacyCardUsage( + iccid=i19, + virtual_used_mb=_to_mb_decimal(row.get("total_bytes_cnt")), + flow_add_discount_pct=_to_mb_decimal(row.get("flow_add_discount")), + has_active_package=bool(row.get("has_active_package")), + ) + return out + + +def fetch_commission_accounts(conn, agent_ids: Iterable[str]) -> dict[str, LegacyCommissionAccount]: + """查代理佣金账户。""" + agent_list = sorted({a for a in agent_ids if a}) + out: dict[str, LegacyCommissionAccount] = {} + if not agent_list: + return out + + sql = """ + SELECT + agent_id, agent_name, + can_draw_amount, applying_draw_amount, + total_commision_amount, total_draw_amount + FROM tbl_agent_commission_account + WHERE agent_id IN %s + """ + with conn.cursor() as cur: + for batch in _chunks(agent_list): + cur.execute(sql, (tuple(batch),)) + for row in cur.fetchall(): + aid = row["agent_id"] + if not aid: + continue + out[aid] = LegacyCommissionAccount( + agent_id=aid, + agent_name=row.get("agent_name") or "", + can_draw_amount_fen=_to_fen(row.get("can_draw_amount")), + applying_draw_amount_fen=_to_fen(row.get("applying_draw_amount")), + total_commission_amount_fen=_to_fen(row.get("total_commision_amount")), + total_draw_amount_fen=_to_fen(row.get("total_draw_amount")), + ) + return out + + +def fetch_card_meta( + conn, + iccid_pairs: Iterable[tuple[str, str]], +) -> tuple[dict[str, LegacyCardMeta], set[str]]: + """查每张卡的元数据(卡本体 + 当前生效套餐 + 虚拟号)。 + + 匹配策略: + - tbl_card 主表用 `iccid_mark IN (iccid_19 ∪ iccid_20)` 全长精确; + 这样不会因前 19 位相同(校验位不同的两张 20 位卡)而误匹配多张 + - tbl_card_life / tbl_virtual_number 从表用 LEFT(iccid_mark, 19) 兜底, + 应对奇成两张表 iccid 长度不一致的脏数据 + + Args: + iccid_pairs: (iccid_19, iccid_20) 列表;iccid_20 可为空字符串(CTCC 卡)。 + + Returns: + (metas, duplicate_iccid_19): + - metas: {iccid_19: LegacyCardMeta} 单匹结果 + - duplicate_iccid_19: 一份业务方 iccid_19 命中奇成 tbl_card 多条记录的 set, + 这些卡不进 metas,sql_builder 会写 errors.csv 阻断 + """ + pairs = [(p[0], p[1]) for p in iccid_pairs if p[0]] + if not pairs: + return {}, set() + + # 全长 ICCID → 业务方 iccid_19。同时维护 iccid_20 优先级: + # 奇成 iccid_mark 命中时,优先取等于 iccid_20 的映射(20 位精确), + # 找不到再退到 iccid_19(奇成存了 19 位的兜底)。 + by_full: dict[str, str] = {} # 任意长度 ICCID → iccid_19 + for i19, i20 in pairs: + by_full[i19] = i19 + if i20: + by_full[i20] = i19 + full_set = sorted(by_full.keys()) + + sql = """ + SELECT + c.iccid_mark AS iccid, + COALESCE(c.category, 0) AS category, + COALESCE(vc.category_name, '') AS category_name, + COALESCE(c.agent_id, '') AS agent_id, + COALESCE(c.agent_name, '') AS agent_name, + COALESCE(c.phone, '') AS phone, + COALESCE(vn.vcode, '') AS vcode, + COALESCE(cl.meal_id, '') AS meal_id, + COALESCE(cl.meal_name, '') AS meal_name, + cl.expire_date AS expire_date + FROM tbl_card c + LEFT JOIN tbl_vendor_category vc ON vc.category = c.category + LEFT JOIN tbl_virtual_number vn ON LEFT(vn.iccid_mark, 19) = LEFT(c.iccid_mark, 19) + LEFT JOIN ( + SELECT cl1.iccid_mark, cl1.meal_id, cl1.meal_name, cl1.expire_date + FROM tbl_card_life cl1 + INNER JOIN ( + SELECT iccid_mark, MAX(expire_date) AS max_expire + FROM tbl_card_life + WHERE status = 1 AND iccid_mark IN %s + GROUP BY iccid_mark + ) latest + ON latest.iccid_mark = cl1.iccid_mark + AND latest.max_expire = cl1.expire_date + WHERE cl1.status = 1 + ) cl ON LEFT(cl.iccid_mark, 19) = LEFT(c.iccid_mark, 19) + WHERE c.iccid_mark IN %s + """ + + # 收集每个 iccid_19 命中的奇成行(可能多条 = 脏数据) + hits: dict[str, list[dict]] = {} + with conn.cursor() as cur: + for batch in _chunks(full_set): + cur.execute(sql, (tuple(batch), tuple(batch))) + for row in cur.fetchall(): + legacy_iccid = row["iccid"] + if not legacy_iccid: + continue + i19 = by_full.get(legacy_iccid) + if i19 is None: + # WHERE 已经限定 IN,理论不会出现;保险起见再按前 19 位反查 + i19 = by_full.get(legacy_iccid[:19]) + if i19 is None: + continue + hits.setdefault(i19, []).append(row) + + metas: dict[str, LegacyCardMeta] = {} + duplicates: set[str] = set() + for i19, rows in hits.items(): + if len(rows) > 1: + duplicates.add(i19) + continue + row = rows[0] + expire = row.get("expire_date") + expire_str = expire.strftime("%Y-%m-%d %H:%M:%S") if expire else None + metas[i19] = LegacyCardMeta( + legacy_raw_iccid=row["iccid"], + category=int(row.get("category") or 0), + category_name=row.get("category_name") or "", + agent_id=row.get("agent_id") or "", + agent_name=row.get("agent_name") or "", + msisdn=row.get("phone") or "", + virtual_no=row.get("vcode") or "", + current_meal_id=row.get("meal_id") or "", + current_meal_name=row.get("meal_name") or "", + current_expire_date=expire_str, + ) + return metas, duplicates + + +def fetch_meal_meta(conn, meal_ids: Iterable[str]) -> dict[str, LegacyMealMeta]: + """查套餐 + 系列元数据(供 scan_legacy 收集映射 legacy_* 字段)。""" + id_list = sorted({m for m in meal_ids if m}) + out: dict[str, LegacyMealMeta] = {} + if not id_list: + return out + sql = """ + SELECT + sm.id AS meal_id, + COALESCE(sm.name, '') AS meal_name, + COALESCE(sm.meal_series_id, '') AS series_id, + COALESCE(sms.name, '') AS series_name + FROM tbl_set_meal sm + LEFT JOIN tbl_set_meal_series sms ON sms.id = sm.meal_series_id + WHERE sm.id IN %s + """ + with conn.cursor() as cur: + for batch in _chunks(id_list): + cur.execute(sql, (tuple(batch),)) + for row in cur.fetchall(): + mid = row["meal_id"] + if not mid: + continue + out[mid] = LegacyMealMeta( + meal_id=mid, + meal_name=row.get("meal_name") or "", + series_id=row.get("series_id") or "", + series_name=row.get("series_name") or "", + ) + return out + + +def fetch_agent_balances(conn, agent_ids: Iterable[str]) -> dict[str, LegacyAgentBalance]: + """查代理主钱包余额(tbl_agent.balance_money,元单位)。""" + agent_list = sorted({a for a in agent_ids if a}) + out: dict[str, LegacyAgentBalance] = {} + if not agent_list: + return out + + sql = """ + SELECT id, balance_money + FROM tbl_agent + WHERE id IN %s + """ + with conn.cursor() as cur: + for batch in _chunks(agent_list): + cur.execute(sql, (tuple(batch),)) + for row in cur.fetchall(): + aid = row["id"] + if not aid: + continue + out[aid] = LegacyAgentBalance(agent_id=aid, balance_fen=_to_fen(row.get("balance_money"))) + return out diff --git a/scripts/migration/lib/mapping_loader.py b/scripts/migration/lib/mapping_loader.py new file mode 100644 index 0000000..34a89ef --- /dev/null +++ b/scripts/migration/lib/mapping_loader.py @@ -0,0 +1,245 @@ +"""mapping.yaml 单文件加载与增量合并。 + +设计要点: + - 一个文件搞定所有映射(migration_user_id / carriers / packages / series / agents) + - 由 scan_legacy.py 增量补全 legacy_* 字段;人工只需填 target_* + - lookup 时检查 target_* 是否已填,未填则抛 MissingTargetError 让调用方写入 errors.csv +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import yaml + + +class MissingTargetError(Exception): + """target_* 字段未填的具体错误,带 legacy 标识信息。""" + + def __init__(self, kind: str, legacy_key: str, legacy_name: str, missing_fields: list[str]): + self.kind = kind + self.legacy_key = legacy_key + self.legacy_name = legacy_name + self.missing_fields = missing_fields + super().__init__( + f"{kind} 映射未填 target_*: legacy_key={legacy_key!r} ({legacy_name!r}), 缺失={missing_fields}" + ) + + +@dataclass(frozen=True) +class CarrierMapping: + legacy_category: int + legacy_category_name: str + target_carrier_id: Optional[int] + target_carrier_type: Optional[str] + target_carrier_name: Optional[str] + + def require_target(self) -> "CarrierMapping": + missing = [ + n for n, v in ( + ("target_carrier_id", self.target_carrier_id), + ("target_carrier_type", self.target_carrier_type), + ("target_carrier_name", self.target_carrier_name), + ) if v in (None, "", 0) + ] + if missing: + raise MissingTargetError("carrier", str(self.legacy_category), self.legacy_category_name, missing) + return self + + +@dataclass(frozen=True) +class PackageMapping: + legacy_meal_id: str + legacy_meal_name: str + target_package_id: Optional[int] + + def require_target(self) -> "PackageMapping": + if self.target_package_id in (None, 0): + raise MissingTargetError("package", self.legacy_meal_id, self.legacy_meal_name, ["target_package_id"]) + return self + + +@dataclass(frozen=True) +class SeriesMapping: + legacy_series_id: str + legacy_series_name: str + target_series_id: Optional[int] # 允许 null,表示不写入新库 series_id + + +@dataclass(frozen=True) +class AgentMapping: + legacy_agent_id: str + legacy_agent_name: str + target_shop_code: Optional[str] # 允许 null,表示该代理的卡进平台库存 + + +@dataclass +class Mapping: + """整体映射(对应单个 mapping.yaml)。""" + + migration_user_id: int = 0 + migration_batch_no: str = "MIGRATION-QICHENG" + carriers: dict[int, CarrierMapping] = field(default_factory=dict) + packages: dict[str, PackageMapping] = field(default_factory=dict) + series: dict[str, SeriesMapping] = field(default_factory=dict) + agents: dict[str, AgentMapping] = field(default_factory=dict) + + def lookup_carrier(self, legacy_category: int) -> Optional[CarrierMapping]: + return self.carriers.get(int(legacy_category)) + + def lookup_package(self, legacy_meal_id: str) -> Optional[PackageMapping]: + return self.packages.get(str(legacy_meal_id).strip()) + + def lookup_series(self, legacy_series_id: str) -> Optional[SeriesMapping]: + if not legacy_series_id: + return None + return self.series.get(str(legacy_series_id).strip()) + + def lookup_agent_shop_code(self, legacy_agent_id: str) -> Optional[str]: + """按 agent_id 查 target_shop_code,空字符串视为 None(进平台库存)。""" + if not legacy_agent_id: + return None + agent = self.agents.get(str(legacy_agent_id).strip()) + if agent is None: + return None + code = (agent.target_shop_code or "").strip() + return code or None + + +# ---------------- 加载 ---------------- + + +MAPPING_FILE = "mapping.yaml" +DSN_FILE = "legacy_dsn.yaml" + + +def load_mapping(config_dir: Path) -> Mapping: + """加载 config/mapping.yaml。文件不存在时返回空 Mapping(供 scan_legacy 首次创建)。""" + path = config_dir / MAPPING_FILE + if not path.exists(): + return Mapping() + with path.open("r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + if not isinstance(raw, dict): + raise ValueError(f"{path} 顶层必须是 mapping 类型") + + m = Mapping( + migration_user_id=int(raw.get("migration_user_id", 0)), + migration_batch_no=_opt_str(raw.get("migration_batch_no")) or "MIGRATION-QICHENG", + ) + + for item in raw.get("carriers") or []: + c = CarrierMapping( + legacy_category=int(item["legacy_category"]), + legacy_category_name=str(item.get("legacy_category_name") or ""), + target_carrier_id=_opt_int(item.get("target_carrier_id")), + target_carrier_type=_opt_str(item.get("target_carrier_type")), + target_carrier_name=_opt_str(item.get("target_carrier_name")), + ) + m.carriers[c.legacy_category] = c + + for item in raw.get("packages") or []: + p = PackageMapping( + legacy_meal_id=str(item["legacy_meal_id"]).strip(), + legacy_meal_name=str(item.get("legacy_meal_name") or ""), + target_package_id=_opt_int(item.get("target_package_id")), + ) + m.packages[p.legacy_meal_id] = p + + for item in raw.get("series") or []: + s = SeriesMapping( + legacy_series_id=str(item["legacy_series_id"]).strip(), + legacy_series_name=str(item.get("legacy_series_name") or ""), + target_series_id=_opt_int(item.get("target_series_id")), + ) + m.series[s.legacy_series_id] = s + + for item in raw.get("agents") or []: + a = AgentMapping( + legacy_agent_id=str(item["legacy_agent_id"]).strip(), + legacy_agent_name=str(item.get("legacy_agent_name") or ""), + target_shop_code=_opt_str(item.get("target_shop_code")), + ) + m.agents[a.legacy_agent_id] = a + + return m + + +def merge_and_save(config_dir: Path, mapping: Mapping) -> Path: + """把 mapping 写回 config/mapping.yaml,保留人工已填的 target_* 字段。 + + 由 scan_legacy 调用:先 load,再用扫描结果 _补充_ legacy_* 项(不存在的追加),最后调用本函数写回。 + """ + config_dir.mkdir(parents=True, exist_ok=True) + path = config_dir / MAPPING_FILE + data = { + "migration_user_id": mapping.migration_user_id, + "migration_batch_no": mapping.migration_batch_no, + "carriers": [ + { + "legacy_category": c.legacy_category, + "legacy_category_name": c.legacy_category_name, + "target_carrier_id": c.target_carrier_id, + "target_carrier_type": c.target_carrier_type, + "target_carrier_name": c.target_carrier_name, + } + for c in sorted(mapping.carriers.values(), key=lambda x: x.legacy_category) + ], + "packages": [ + { + "legacy_meal_id": p.legacy_meal_id, + "legacy_meal_name": p.legacy_meal_name, + "target_package_id": p.target_package_id, + } + for p in sorted(mapping.packages.values(), key=lambda x: x.legacy_meal_id) + ], + "series": [ + { + "legacy_series_id": s.legacy_series_id, + "legacy_series_name": s.legacy_series_name, + "target_series_id": s.target_series_id, + } + for s in sorted(mapping.series.values(), key=lambda x: x.legacy_series_id) + ], + "agents": [ + { + "legacy_agent_id": a.legacy_agent_id, + "legacy_agent_name": a.legacy_agent_name, + "target_shop_code": a.target_shop_code, + } + for a in sorted(mapping.agents.values(), key=lambda x: x.legacy_agent_id) + ], + } + with path.open("w", encoding="utf-8") as f: + yaml.safe_dump(data, f, allow_unicode=True, sort_keys=False, default_flow_style=False) + return path + + +def load_legacy_dsn(config_dir: Path) -> dict: + """加载奇成 MySQL 只读连接配置。""" + path = config_dir / DSN_FILE + if not path.exists(): + raise FileNotFoundError(f"找不到 {path},请基于 legacy_dsn.yaml.example 创建并填写真实 DSN") + with path.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + for required in ("host", "port", "user", "password", "database"): + if required not in data: + raise ValueError(f"{path} 缺少必填项 {required}") + return data + + +# ---------------- 小工具 ---------------- + + +def _opt_int(v: Any) -> Optional[int]: + if v is None or v == "": + return None + return int(v) + + +def _opt_str(v: Any) -> Optional[str]: + if v is None: + return None + s = str(v).strip() + return s or None diff --git a/scripts/migration/lib/sql_builder.py b/scripts/migration/lib/sql_builder.py new file mode 100644 index 0000000..f6e8f2b --- /dev/null +++ b/scripts/migration/lib/sql_builder.py @@ -0,0 +1,845 @@ +"""SQL 输出构造。 + +输出文件: + step1_01_iot_cards.sql - 卡资产 + step1_02_devices.sql - 设备资产 + step1_03_asset_identifiers.sql- 全局标识符(iccid / virtual_no) + step1_04_asset_wallets.sql - 资产钱包 + step1_05_sim_bindings.sql - 设备-卡绑定 + step1_06_shop_allocations.sql - 店铺归属的分配记录 + + step2_01_migration_orders.sql - 迁移伪订单 + step2_02_package_usages.sql - 套餐使用记录 + step2_03_card_data_usage_update.sql - 卡累计流量 UPDATE + step2_04_agent_wallet_init.sql - 代理钱包初始化(main + commission) + step2_05_agent_wallet_transactions.sql - 代理钱包初始化流水 + +设计原则: + - 所有 INSERT 通过 SELECT 子查询动态取 id,不硬编码新库 id + - 使用 ON CONFLICT (...) WHERE ... DO NOTHING 保证幂等(PG 14+ 支持 partial index 推断) + - 字符串 ' 转义为 '' + - 卡的元数据(carrier/agent_id/msisdn/virtual_no/当前套餐)从奇成 fetch_card_meta 取, + cards.csv 只承载 is_industry 覆盖,店铺归属完全由 mapping.agents 推导 +""" +from __future__ import annotations + +import csv +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from typing import Optional + +from . import iccid_utils +from .csv_loader import CardRow, DeviceRow, ErrorRow +from .legacy_query import ( + LegacyAgentBalance, + LegacyCardMeta, + LegacyCardUsage, + LegacyCommissionAccount, +) +from .mapping_loader import Mapping, MissingTargetError + +# 写死的默认值(对应原 migration.yaml,实际不需要业务侧关心) +ORDER_NO_PREFIX_CARD = "MIG-CARD-" +ALLOCATION_NO_PREFIX = "MIG-ALLOC-" +METADATA_SOURCE = "qicheng" + + +# ---------------- 通用工具 ---------------- + + +def _sql_str(value: Optional[str]) -> str: + if value is None: + return "NULL" + s = str(value) + return "'" + s.replace("'", "''") + "'" + + +def _sql_jsonb(d: dict) -> str: + import json + + s = json.dumps(d, ensure_ascii=False, separators=(",", ":")) + return "'" + s.replace("'", "''") + "'::jsonb" + + +def _sql_decimal(value: Decimal) -> str: + """生成 SQL decimal 字面量,保留奇成小数 MB 精度。""" + return format(value, "f") + + +def _shop_id_sub(shop_code: Optional[str]) -> str: + """生成 shop_id 子查询表达式;空 shop_code 返回 'NULL'(平台库存)。""" + if not shop_code: + return "NULL" + return f"(SELECT id FROM tb_shop WHERE shop_code = {_sql_str(shop_code)} AND deleted_at IS NULL LIMIT 1)" + + +def _now_for_no() -> str: + return datetime.now().strftime("%Y%m%d%H%M%S") + + +def _file_header(title: str) -> str: + return ( + "-- ============================================================\n" + f"-- {title}\n" + f"-- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" + "-- 重跑安全: ON CONFLICT DO NOTHING\n" + "-- 建议执行: psql ... -1 -f <此文件>\n" + "-- ============================================================\n\n" + ) + + +def _resolve_card_shop_code(card: CardRow, meta: LegacyCardMeta, mapping: Mapping) -> str: + """决定卡最终 shop_code:设备上的卡→空(跟设备走);否则按 agent 映射;查不到→平台库存。""" + if card.bound_device_virtual_no: + return "" + if meta and meta.agent_id: + shop = mapping.lookup_agent_shop_code(meta.agent_id) + if shop: + return shop + return "" + + +def _resolve_device_shop_code( + device: DeviceRow, + card_metas: dict[str, LegacyCardMeta], + mapping: Mapping, +) -> str: + """决定设备最终 shop_code:按主卡(slot 1)的 agent 映射;无主卡或映射不到→平台库存。""" + primary_iccid = device.sim_iccids.get(1) or next(iter(device.sim_iccids.values()), "") + if not primary_iccid: + return "" + meta = card_metas.get(primary_iccid) + if not meta or not meta.agent_id: + return "" + return mapping.lookup_agent_shop_code(meta.agent_id) or "" + + +# ============================================================ +# 脚本1:资产导入 +# ============================================================ + + +def write_step1( + output_dir: Path, + *, + cards: list[CardRow], + devices: list[DeviceRow], + mapping: Mapping, + card_metas: dict[str, LegacyCardMeta], + duplicate_iccid_19s: set[str], + errors: list[ErrorRow], +) -> None: + """生成 step1_* SQL + errors.csv + summary.txt。 + + 会就地修改 errors 列表,把映射缺失等问题追加进去。 + + Args: + card_metas: {iccid_19: LegacyCardMeta} + duplicate_iccid_19s: 同一 iccid_19 在奇成命中多条 tbl_card 记录的集合,直接报错阻断 + """ + user_id = mapping.migration_user_id + + # 预检 carrier 映射 + 运营商一致性 + 重复命中,把异常卡剔除 + valid_cards = _filter_cards_by_carrier(cards, card_metas, duplicate_iccid_19s, mapping, errors) + + cards_by_iccid_19 = {c.iccid_19: c for c in valid_cards} + + _write_iot_cards(output_dir / "step1_01_iot_cards.sql", valid_cards, card_metas, mapping, user_id) + _write_devices(output_dir / "step1_02_devices.sql", devices, card_metas, mapping, user_id) + _write_asset_identifiers(output_dir / "step1_03_asset_identifiers.sql", valid_cards, devices, card_metas) + _write_asset_wallets(output_dir / "step1_04_asset_wallets.sql", valid_cards, devices) + _write_sim_bindings(output_dir / "step1_05_sim_bindings.sql", devices, cards_by_iccid_19) + _write_shop_allocations( + output_dir / "step1_06_shop_allocations.sql", valid_cards, devices, card_metas, mapping, user_id + ) + + _write_errors(output_dir / "errors.csv", errors) + _write_summary_step1(output_dir / "summary.txt", valid_cards, devices, errors) + + +def _filter_cards_by_carrier( + cards: list[CardRow], + card_metas: dict[str, LegacyCardMeta], + duplicate_iccid_19s: set[str], + mapping: Mapping, + errors: list[ErrorRow], +) -> list[CardRow]: + """筛掉:奇成查不到 / 命中多条 / carrier 映射缺失 / target_* 未填 / 运营商与 iccid 长度不一致 的卡。""" + valid: list[CardRow] = [] + for c in cards: + if c.iccid_19 in duplicate_iccid_19s: + errors.append(ErrorRow( + "cards.csv", c.line_no, "iccid_19", c.iccid_19, + "legacy_duplicate", "奇成 tbl_card 中同一 iccid_19 命中多条记录,请业务方人工确认", + )) + continue + meta = card_metas.get(c.iccid_19) + if meta is None: + errors.append(ErrorRow( + "cards.csv", c.line_no, "iccid_19", c.iccid_19, + "not_in_legacy", "奇成 tbl_card 中找不到该 ICCID(iccid_19/iccid_20 都没命中)", + )) + continue + if not meta.category: + errors.append(ErrorRow( + "cards.csv", c.line_no, "iccid_19", c.iccid_19, + "no_category", "奇成 tbl_card.category 为空,无法定位运营商映射", + )) + continue + carrier = mapping.lookup_carrier(meta.category) + if carrier is None: + errors.append(ErrorRow( + "cards.csv", c.line_no, "category", str(meta.category), + "carrier_not_in_mapping", f"mapping.yaml.carriers 未配置 category={meta.category} ({meta.category_name})", + )) + continue + try: + carrier.require_target() + except MissingTargetError as exc: + errors.append(ErrorRow( + "cards.csv", c.line_no, "category", str(meta.category), + "carrier_target_missing", str(exc), + )) + continue + # 运营商类型与业务方 iccid_20 一致性校验 + err = iccid_utils.validate_carrier_iccid_match(c.iccid_20, carrier.target_carrier_type or "") + if err is not None: + errors.append(ErrorRow( + "cards.csv", c.line_no, "iccid_20", c.iccid_20, + err.code, err.message, + )) + continue + valid.append(c) + return valid + + +def _write_iot_cards( + path: Path, + cards: list[CardRow], + card_metas: dict[str, LegacyCardMeta], + mapping: Mapping, + user_id: int, +) -> None: + """写 tb_iot_card。元数据从奇成 card_metas + mapping 推导,iccid_19/iccid_20 用业务方两列。""" + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step1_01 卡资产导入 (tb_iot_card)")) + for c in cards: + meta = card_metas[c.iccid_19] + carrier = mapping.lookup_carrier(meta.category) + assert carrier is not None # 已被 _filter_cards_by_carrier 保证 + + # 业务方直接给的两列;iccid 唯一键 = 有 iccid_20 用 iccid_20,否则 iccid_19 + iccid_20_expr = _sql_str(c.iccid_20) if c.iccid_20 else "NULL" + + card_category = "industry" if c.is_industry else "normal" + shop_code = _resolve_card_shop_code(c, meta, mapping) + series = mapping.lookup_series(_meal_to_series_id(meta, mapping)) + series_id_expr = str(series.target_series_id) if series and series.target_series_id else "NULL" + + f.write( + "INSERT INTO tb_iot_card (\n" + " iccid, iccid_19, iccid_20,\n" + " carrier_id, carrier_type, carrier_name,\n" + " card_category, batch_no,\n" + " status, activation_status, real_name_status, network_status,\n" + " asset_status, generation, is_standalone,\n" + " realname_policy, stop_reason,\n" + " msisdn, virtual_no, shop_id, series_id,\n" + " device_virtual_no, gateway_extend,\n" + " creator, updater, created_at, updated_at\n" + ") VALUES (\n" + f" {_sql_str(c.iccid_full)}, {_sql_str(c.iccid_19)}, {iccid_20_expr},\n" + f" {carrier.target_carrier_id}, {_sql_str(carrier.target_carrier_type)}, {_sql_str(carrier.target_carrier_name)},\n" + f" {_sql_str(card_category)}, {_sql_str(mapping.migration_batch_no)},\n" + " 1, 0, 0, 0,\n" + " 1, 1, TRUE,\n" + " 'after_order', '',\n" + f" {_sql_str(meta.msisdn) if meta.msisdn else 'NULL'}, " + f"{_sql_str(meta.virtual_no) if meta.virtual_no else 'NULL'}, " + f"{_shop_id_sub(shop_code)}, {series_id_expr},\n" + " '', '',\n" + f" {user_id}, {user_id}, NOW(), NOW()\n" + ")\n" + "ON CONFLICT (iccid) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _meal_to_series_id(meta: LegacyCardMeta, mapping: Mapping) -> str: + """由 meta 的 meal_id 经 mapping.packages 间接拿不到 series,所以这里直接靠 + mapping.series 用 legacy_series_id 反查就行。简化:返回空字符串(让上层走 series 映射)。 + + 实际上 series_id 仅在套餐有 meal_series_id 时才有意义。考虑到 LegacyCardMeta 没存 + series_id,这里返回 '' — 让 series_id_expr 走 NULL 分支。后续若需要,scan_legacy + 可把 series_id 也下沉到 card_meta。 + """ + _ = meta + _ = mapping + return "" + + +def _write_devices( + path: Path, + devices: list[DeviceRow], + card_metas: dict[str, LegacyCardMeta], + mapping: Mapping, + user_id: int, +) -> None: + """设备表 INSERT。 + + 业务方不关心的字段都写默认值: + device_name/device_model/manufacturer = NULL(可后台编辑) + max_sim_slots = 4 + status / asset_status / online_status = 1 / 1 / 0 + enable_polling = TRUE,realname_policy = 'after_order' + software_version = '',switch_mode = '0'(自动切卡) + """ + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step1_02 设备资产导入 (tb_device)")) + for d in devices: + shop_code = _resolve_device_shop_code(d, card_metas, mapping) + f.write( + "INSERT INTO tb_device (\n" + " virtual_no, max_sim_slots, imei,\n" + " status, asset_status, generation, online_status,\n" + " enable_polling, realname_policy,\n" + " software_version, switch_mode,\n" + " batch_no, shop_id,\n" + " creator, updater, created_at, updated_at\n" + ") VALUES (\n" + f" {_sql_str(d.virtual_no)}, {d.max_sim_slots}, " + f"{_sql_str(d.imei) if d.imei else 'NULL'},\n" + " 1, 1, 1, 0,\n" + " TRUE, 'after_order',\n" + " '', '0',\n" + f" {_sql_str(mapping.migration_batch_no)}, {_shop_id_sub(shop_code)},\n" + f" {user_id}, {user_id}, NOW(), NOW()\n" + ")\n" + "ON CONFLICT (virtual_no) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _write_asset_identifiers( + path: Path, + cards: list[CardRow], + devices: list[DeviceRow], + card_metas: dict[str, LegacyCardMeta], +) -> None: + """卡的 iccid + 虚拟号(来自奇成)和设备的 virtual_no 都注册。""" + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step1_03 资产标识符注册 (tb_asset_identifier)")) + for c in cards: + meta = card_metas[c.iccid_19] + f.write( + "INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n" + f"SELECT {_sql_str(c.iccid_full)}, 'iot_card', id FROM tb_iot_card\n" + f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL\n" + "ON CONFLICT (identifier) DO NOTHING;\n\n" + ) + if meta.virtual_no: + f.write( + "INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n" + f"SELECT {_sql_str(meta.virtual_no)}, 'iot_card', id FROM tb_iot_card\n" + f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL\n" + "ON CONFLICT (identifier) DO NOTHING;\n\n" + ) + for d in devices: + f.write( + "INSERT INTO tb_asset_identifier (identifier, asset_type, asset_id)\n" + f"SELECT {_sql_str(d.virtual_no)}, 'device', id FROM tb_device\n" + f"WHERE virtual_no = {_sql_str(d.virtual_no)} AND deleted_at IS NULL\n" + "ON CONFLICT (identifier) DO NOTHING;\n\n" + ) + + +def _write_asset_wallets(path: Path, cards: list[CardRow], devices: list[DeviceRow]) -> None: + """卡和设备各一条钱包,初始余额 0。""" + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step1_04 资产钱包初始化 (tb_asset_wallet)")) + for c in cards: + f.write( + "INSERT INTO tb_asset_wallet (\n" + " resource_type, resource_id, balance, frozen_balance,\n" + " currency, status, version, shop_id_tag,\n" + " created_at, updated_at\n" + ")\n" + "SELECT 'iot_card', id, 0, 0,\n" + " 'CNY', 1, 0, COALESCE(shop_id, 0),\n" + " NOW(), NOW()\n" + "FROM tb_iot_card\n" + f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL\n" + "ON CONFLICT (resource_type, resource_id) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + for d in devices: + f.write( + "INSERT INTO tb_asset_wallet (\n" + " resource_type, resource_id, balance, frozen_balance,\n" + " currency, status, version, shop_id_tag,\n" + " created_at, updated_at\n" + ")\n" + "SELECT 'device', id, 0, 0,\n" + " 'CNY', 1, 0, COALESCE(shop_id, 0),\n" + " NOW(), NOW()\n" + "FROM tb_device\n" + f"WHERE virtual_no = {_sql_str(d.virtual_no)} AND deleted_at IS NULL\n" + "ON CONFLICT (resource_type, resource_id) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _write_sim_bindings( + path: Path, + devices: list[DeviceRow], + cards_by_iccid_19: dict[str, CardRow], +) -> None: + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step1_05 设备-卡绑定 (tb_device_sim_binding)")) + for d in devices: + for slot, iccid_19 in sorted(d.sim_iccids.items()): + card = cards_by_iccid_19.get(iccid_19) + if card is None: + continue # 该卡可能被 carrier 校验剔除,跳过此 slot 绑定 + is_current = "TRUE" if slot == d.current_slot else "FALSE" + f.write( + "INSERT INTO tb_device_sim_binding (\n" + " device_id, iot_card_id, slot_position, bind_status, is_current,\n" + " bind_time, creator, updater, created_at, updated_at\n" + ")\n" + f"SELECT d.id, c.id, {slot}, 1, {is_current},\n" + " NOW(), 0, 0, NOW(), NOW()\n" + "FROM tb_device d\n" + "CROSS JOIN tb_iot_card c\n" + f"WHERE d.virtual_no = {_sql_str(d.virtual_no)} AND d.deleted_at IS NULL\n" + f" AND c.iccid = {_sql_str(card.iccid_full)} AND c.deleted_at IS NULL\n" + "ON CONFLICT (device_id, slot_position) WHERE bind_status = 1 AND deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _write_shop_allocations( + path: Path, + cards: list[CardRow], + devices: list[DeviceRow], + card_metas: dict[str, LegacyCardMeta], + mapping: Mapping, + user_id: int, +) -> None: + """只为带店铺归属的资产生成分配记录;设备上的卡跟随设备分配。""" + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step1_06 店铺分配记录 (tb_asset_allocation_record)")) + no_ts = _now_for_no() + + for c in cards: + if c.bound_device_virtual_no: + continue + meta = card_metas[c.iccid_19] + shop_code = _resolve_card_shop_code(c, meta, mapping) + if not shop_code: + continue + allocation_no = f"{ALLOCATION_NO_PREFIX}CARD-{no_ts}-{c.line_no}" + f.write( + "INSERT INTO tb_asset_allocation_record (\n" + " allocation_no, allocation_type, asset_type, asset_id, asset_identifier,\n" + " from_owner_type, to_owner_type, to_owner_id,\n" + " operator_id, remark,\n" + " creator, updater, created_at, updated_at\n" + ")\n" + f"SELECT {_sql_str(allocation_no)}, 'allocate', 'iot_card', c.id, {_sql_str(c.iccid_full)},\n" + " 'platform', 'shop', s.id,\n" + f" {user_id}, '奇成数据迁移初始化分配',\n" + f" {user_id}, {user_id}, NOW(), NOW()\n" + "FROM tb_iot_card c\n" + "CROSS JOIN tb_shop s\n" + f"WHERE c.iccid = {_sql_str(c.iccid_full)} AND c.deleted_at IS NULL\n" + f" AND s.shop_code = {_sql_str(shop_code)} AND s.deleted_at IS NULL;\n\n" + ) + + for d in devices: + shop_code = _resolve_device_shop_code(d, card_metas, mapping) + if not shop_code: + continue + allocation_no = f"{ALLOCATION_NO_PREFIX}DEV-{no_ts}-{d.line_no}" + f.write( + "INSERT INTO tb_asset_allocation_record (\n" + " allocation_no, allocation_type, asset_type, asset_id, asset_identifier,\n" + " from_owner_type, to_owner_type, to_owner_id,\n" + " operator_id, remark,\n" + " creator, updater, created_at, updated_at\n" + ")\n" + f"SELECT {_sql_str(allocation_no)}, 'allocate', 'device', dev.id, {_sql_str(d.virtual_no)},\n" + " 'platform', 'shop', s.id,\n" + f" {user_id}, '奇成数据迁移初始化分配',\n" + f" {user_id}, {user_id}, NOW(), NOW()\n" + "FROM tb_device dev\n" + "CROSS JOIN tb_shop s\n" + f"WHERE dev.virtual_no = {_sql_str(d.virtual_no)} AND dev.deleted_at IS NULL\n" + f" AND s.shop_code = {_sql_str(shop_code)} AND s.deleted_at IS NULL;\n\n" + ) + + +def _write_errors(path: Path, errors: list[ErrorRow]) -> None: + with path.open("w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["source_file", "line_no", "field", "value", "error_code", "message"]) + for e in errors: + writer.writerow(e.to_csv_row()) + + +def _write_summary_step1(path: Path, cards: list[CardRow], devices: list[DeviceRow], errors: list[ErrorRow]) -> None: + bound = sum(1 for c in cards if c.bound_device_virtual_no) + lines = [ + "奇成迁移 - 脚本1 资产导入摘要", + "=" * 48, + f"生成时间 : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + f"卡有效总数 : {len(cards)}", + f" 绑设备 : {bound}", + f" 独立卡 : {len(cards) - bound}", + f"设备总数 : {len(devices)}", + f"异常行 : {len(errors)} (详见 errors.csv)", + "", + "店铺归属:由 mapping.agents 自动推导(详见 step1_01/step1_02/step1_06 内的 shop_id 子查询)", + ] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +# ============================================================ +# 脚本2:关联数据导入 +# ============================================================ + + +def write_step2( + output_dir: Path, + *, + cards: list[CardRow], + mapping: Mapping, + card_metas: dict[str, LegacyCardMeta], + usage_snapshots: dict[str, LegacyCardUsage], + commission_snapshots: dict[str, LegacyCommissionAccount], + agent_balances: dict[str, LegacyAgentBalance], +) -> list[str]: + """生成 step2_* SQL + warnings.csv + summary.txt。""" + user_id = mapping.migration_user_id + warnings: list[tuple[str, str, str]] = [] + # 卡 → package(从 card_meta.current_meal_id 经 mapping.packages 查 target) + resolved_packages: dict[str, tuple[int, str]] = {} # iccid_19 → (target_package_id, expire_date) + for c in cards: + meta = card_metas.get(c.iccid_19) + if meta is None or not meta.current_meal_id: + continue + pkg_map = mapping.lookup_package(meta.current_meal_id) + if pkg_map is None: + warnings.append(("package_not_in_mapping", c.iccid_19, + f"奇成 meal_id={meta.current_meal_id} ({meta.current_meal_name}) 未在 mapping.yaml.packages")) + continue + try: + pkg_map.require_target() + except MissingTargetError as exc: + warnings.append(("package_target_missing", c.iccid_19, str(exc))) + continue + resolved_packages[c.iccid_19] = (pkg_map.target_package_id, meta.current_expire_date or "") + + # 用量预检:虚用量 > 0 但没找到当前生效套餐的卡,无法精确反算真用量 + # (按 flow_add_discount=0 处理,即真=虚,可能高估真用量;不阻断) + for c in cards: + snap = usage_snapshots.get(c.iccid_19) + if snap is None or snap.virtual_used_mb <= 0: + continue + if not snap.has_active_package: + warnings.append(( + "usage_without_active_package", c.iccid_19, + f"奇成虚用量 {snap.virtual_used_mb}MB,但 tbl_card_life 中没有当前生效套餐" + "(status=1),无法精确反算真用量,按 flow_add_discount=0 处理(真=虚)", + )) + + _write_migration_orders(output_dir / "step2_01_migration_orders.sql", cards, resolved_packages, user_id) + _write_package_usages( + output_dir / "step2_02_package_usages.sql", cards, resolved_packages, usage_snapshots, user_id, + ) + _write_card_data_usage(output_dir / "step2_03_card_data_usage_update.sql", cards, usage_snapshots) + + # 代理钱包:只为 mapping.agents 中 target_shop_code 已填的项写入 + eligible_agents = [a for a in mapping.agents.values() if (a.target_shop_code or "").strip()] + _write_agent_wallet_init( + output_dir / "step2_04_agent_wallet_init.sql", + eligible_agents, agent_balances, commission_snapshots, warnings, + ) + _write_agent_wallet_transactions( + output_dir / "step2_05_agent_wallet_transactions.sql", + eligible_agents, agent_balances, commission_snapshots, user_id, + ) + + _write_warnings(output_dir / "warnings.csv", warnings) + _write_summary_step2( + output_dir / "summary.txt", + cards=cards, + resolved=resolved_packages, + agents=eligible_agents, + warnings=warnings, + ) + return [w[2] for w in warnings] + + +def _make_card_order_no(iccid_full: str) -> str: + """生成稳定迁移订单号。 + + 新库 order_no 限制为 varchar(30);MIG-CARD- + 20 位 ICCID 总长 29。 + 固定使用 ICCID 还能保证 step2_01/step2_02 分开重跑时仍能匹配同一订单。 + """ + return f"{ORDER_NO_PREFIX_CARD}{iccid_full}" + + +def _write_migration_orders( + path: Path, + cards: list[CardRow], + resolved_packages: dict[str, tuple[int, str]], + user_id: int, +) -> None: + """伪订单关键字段(详见方案文档 10.2):source=admin / buyer_type=personal / + buyer_id=0 / payment_method=offline / payment_status=2 / commission_status=2 / + commission_result=2 / amount=0。""" + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step2_01 迁移伪订单 (tb_order)")) + for c in cards: + if c.iccid_19 not in resolved_packages: + continue + order_no = _make_card_order_no(c.iccid_full) + f.write( + "INSERT INTO tb_order (\n" + " order_no, order_type, buyer_type, buyer_id,\n" + " iot_card_id, asset_identifier,\n" + " total_amount, actual_paid_amount,\n" + " payment_method, payment_status, paid_at,\n" + " commission_status, commission_result, commission_config_version,\n" + " source, generation,\n" + " operator_account_type, operator_account_name,\n" + " creator, updater, created_at, updated_at\n" + ")\n" + f"SELECT {_sql_str(order_no)}, 'single_card', 'personal', 0,\n" + f" c.id, {_sql_str(c.iccid_full)},\n" + " 0, 0,\n" + " 'offline', 2, NOW(),\n" + " 2, 2, 0,\n" + " 'admin', 1,\n" + " 'platform', '奇成数据迁移',\n" + f" {user_id}, {user_id}, NOW(), NOW()\n" + "FROM tb_iot_card c\n" + f"WHERE c.iccid = {_sql_str(c.iccid_full)} AND c.deleted_at IS NULL\n" + "ON CONFLICT (order_no) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _write_package_usages( + path: Path, + cards: list[CardRow], + resolved_packages: dict[str, tuple[int, str]], + usage_snapshots: dict[str, LegacyCardUsage], + user_id: int, +) -> None: + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step2_02 套餐使用记录 (tb_package_usage)")) + for c in cards: + if c.iccid_19 not in resolved_packages: + continue + target_pkg_id, expire = resolved_packages[c.iccid_19] + order_no = _make_card_order_no(c.iccid_full) + expires_expr = f"{_sql_str(expire)}::timestamp" if expire else "NULL" + snap = usage_snapshots.get(c.iccid_19) + used_mb = snap.real_used_mb_floor if snap is not None else 0 + effective_limit_expr = ( + "CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 " + "THEN p.virtual_data_mb ELSE p.real_data_mb END" + ) + usage_expr = f"LEAST({used_mb}, {effective_limit_expr})" + f.write( + "INSERT INTO tb_package_usage (\n" + " order_id, order_no, package_id, usage_type,\n" + " iot_card_id,\n" + " data_limit_mb, data_usage_mb,\n" + " virtual_total_mb_snapshot, display_gain_ratio_snapshot, enable_virtual_data_snapshot,\n" + " status, priority, generation,\n" + " activated_at, expires_at,\n" + " package_name,\n" + " package_price_config_status, package_is_gift,\n" + " data_reset_cycle,\n" + " creator, updater, created_at, updated_at\n" + ")\n" + "SELECT o.id, o.order_no, p.id, 'single_card',\n" + " o.iot_card_id,\n" + f" p.real_data_mb, {usage_expr},\n" + f" {effective_limit_expr},\n" + " CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 THEN p.virtual_ratio ELSE 1.0 END,\n" + " CASE WHEN p.enable_virtual_data AND p.virtual_data_mb > 0 THEN TRUE ELSE FALSE END,\n" + f" CASE WHEN {effective_limit_expr} > 0 AND {usage_expr} >= {effective_limit_expr} THEN 2 ELSE 1 END, 1, 1,\n" + f" NULL, {expires_expr},\n" + " p.package_name,\n" + " p.price_config_status, p.is_gift,\n" + " p.data_reset_cycle,\n" + f" {user_id}, {user_id}, NOW(), NOW()\n" + "FROM tb_order o\n" + f"JOIN tb_package p ON p.id = {target_pkg_id} AND p.deleted_at IS NULL\n" + f"WHERE o.order_no = {_sql_str(order_no)} AND o.deleted_at IS NULL\n" + "ON CONFLICT (order_id, package_id) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _write_card_data_usage(path: Path, cards: list[CardRow], usage_snapshots: dict[str, LegacyCardUsage]) -> None: + """写卡累计流量基线 UPDATE。 + + 三个字段都按"真用量"语义落库: + - ``data_usage_mb``(BIGINT)= ``real_used_mb_floor``,sub-MB 上取整为 1 + - ``current_month_usage_mb``(numeric)= ``real_used_mb`` 原始 Decimal,保留小数 + - ``last_gateway_reading_mb``(double)= 同上,用作下次轮询增量计算的基线 + """ + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step2_03 卡累计流量更新 (tb_iot_card 流量基线)")) + for c in cards: + snap = usage_snapshots.get(c.iccid_19) + if snap is None or snap.real_used_mb <= 0: + continue + real_decimal = _sql_decimal(snap.real_used_mb) + real_int = snap.real_used_mb_floor + f.write( + "UPDATE tb_iot_card\n" + "SET\n" + f" data_usage_mb = {real_int},\n" + f" current_month_usage_mb = {real_decimal},\n" + f" last_gateway_reading_mb = {real_decimal},\n" + " current_month_start_date = COALESCE(current_month_start_date, DATE_TRUNC('month', NOW())::date),\n" + " last_data_check_at = NOW(),\n" + " last_sync_time = NOW(),\n" + " updated_at = NOW()\n" + f"WHERE iccid = {_sql_str(c.iccid_full)} AND deleted_at IS NULL;\n\n" + ) + + +def _write_agent_wallet_init( + path: Path, + eligible_agents, + agent_balances: dict[str, LegacyAgentBalance], + commission_snapshots: dict[str, LegacyCommissionAccount], + warnings: list[tuple[str, str, str]], +) -> None: + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step2_04 代理钱包初始化 (tb_agent_wallet, main + commission)")) + for a in eligible_agents: + balance = agent_balances.get(a.legacy_agent_id) + commission = commission_snapshots.get(a.legacy_agent_id) + if balance is None and commission is None: + warnings.append(( + "agent_no_legacy", a.legacy_agent_id, + f"奇成代理 {a.legacy_agent_id} ({a.legacy_agent_name}) 没有 tbl_agent / 佣金账户记录", + )) + continue + main_balance = balance.balance_fen if balance else 0 + comm_balance = commission.can_draw_amount_fen if commission else 0 + comm_frozen = commission.applying_draw_amount_fen if commission else 0 + shop_sub = _shop_id_sub(a.target_shop_code) + + f.write( + "INSERT INTO tb_agent_wallet (\n" + " shop_id, wallet_type, balance, frozen_balance,\n" + " currency, status, version, shop_id_tag,\n" + " created_at, updated_at\n" + ") VALUES (\n" + f" {shop_sub}, 'main', {main_balance}, 0,\n" + f" 'CNY', 1, 0, {shop_sub},\n" + " NOW(), NOW()\n" + ")\n" + "ON CONFLICT (shop_id, wallet_type) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + f.write( + "INSERT INTO tb_agent_wallet (\n" + " shop_id, wallet_type, balance, frozen_balance,\n" + " currency, status, version, shop_id_tag,\n" + " created_at, updated_at\n" + ") VALUES (\n" + f" {shop_sub}, 'commission', {comm_balance}, {comm_frozen},\n" + f" 'CNY', 1, 0, {shop_sub},\n" + " NOW(), NOW()\n" + ")\n" + "ON CONFLICT (shop_id, wallet_type) WHERE deleted_at IS NULL DO NOTHING;\n\n" + ) + + +def _write_agent_wallet_transactions( + path: Path, + eligible_agents, + agent_balances: dict[str, LegacyAgentBalance], + commission_snapshots: dict[str, LegacyCommissionAccount], + user_id: int, +) -> None: + with path.open("w", encoding="utf-8") as f: + f.write(_file_header("step2_05 代理钱包初始化流水 (tb_agent_wallet_transaction)")) + for a in eligible_agents: + balance = agent_balances.get(a.legacy_agent_id) + commission = commission_snapshots.get(a.legacy_agent_id) + if balance is None and commission is None: + continue + main_amount = balance.balance_fen if balance else 0 + comm_amount = commission.can_draw_amount_fen if commission else 0 + comm_frozen = commission.applying_draw_amount_fen if commission else 0 + shop_sub = _shop_id_sub(a.target_shop_code) + + for wallet_type, amount in (("main", main_amount), ("commission", comm_amount)): + if amount <= 0: + continue + meta = { + "source": METADATA_SOURCE, + "legacy_agent_id": a.legacy_agent_id, + "legacy_agent_name": a.legacy_agent_name, + "legacy_field": "balance_money" if wallet_type == "main" else "can_draw_amount", + } + if wallet_type == "commission": + meta["legacy_applying_draw_amount_fen"] = comm_frozen + if commission: + meta["legacy_total_commission_fen"] = commission.total_commission_amount_fen + meta["legacy_total_draw_fen"] = commission.total_draw_amount_fen + remark = ( + f"奇成数据迁移初始化({'主钱包' if wallet_type == 'main' else '分佣钱包'})" + f"源 agent_id={a.legacy_agent_id}" + ) + f.write( + "INSERT INTO tb_agent_wallet_transaction (\n" + " agent_wallet_id, shop_id, user_id,\n" + " transaction_type, amount, balance_before, balance_after,\n" + " status, reference_type, reference_id,\n" + " remark, metadata,\n" + " creator, shop_id_tag, created_at, updated_at\n" + ")\n" + f"SELECT w.id, w.shop_id, {user_id},\n" + f" 'recharge', {amount}, 0, {amount},\n" + " 1, 'topup', NULL,\n" + f" {_sql_str(remark)}, {_sql_jsonb(meta)},\n" + f" {user_id}, w.shop_id, NOW(), NOW()\n" + "FROM tb_agent_wallet w\n" + f"WHERE w.shop_id = {shop_sub}\n" + f" AND w.wallet_type = {_sql_str(wallet_type)}\n" + " AND w.deleted_at IS NULL\n" + " AND NOT EXISTS (\n" + " SELECT 1 FROM tb_agent_wallet_transaction t\n" + " WHERE t.agent_wallet_id = w.id\n" + f" AND t.metadata->>'source' = {_sql_str(METADATA_SOURCE)}\n" + f" AND t.metadata->>'legacy_agent_id' = {_sql_str(a.legacy_agent_id)}\n" + f" AND t.metadata->>'legacy_field' = {_sql_str(meta['legacy_field'])}\n" + " AND t.deleted_at IS NULL\n" + " );\n\n" + ) + + +def _write_warnings(path: Path, warnings: list[tuple[str, str, str]]) -> None: + with path.open("w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["kind", "key", "message"]) + for kind, key, message in warnings: + writer.writerow([kind, key, message]) + + +def _write_summary_step2(path: Path, *, cards, resolved, agents, warnings) -> None: + lines = [ + "奇成迁移 - 脚本2 关联数据导入摘要", + "=" * 48, + f"生成时间 : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + f"卡总数 : {len(cards)}", + f" 生成伪订单+套餐 : {len(resolved)}", + f"代理钱包初始化 : {len(agents)}", + f"警告条目 : {len(warnings)} (详见 warnings.csv)", + ] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scripts/migration/migrate_assets.py b/scripts/migration/migrate_assets.py new file mode 100644 index 0000000..7b23766 --- /dev/null +++ b/scripts/migration/migrate_assets.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""奇成数据迁移脚本1:资产导入。 + +工作流: + 1. 读 mapping.yaml(必须先跑过 scan_legacy.py) + 2. 读 cards.csv / devices.csv + 3. 连奇成查每张卡的元数据(category/agent_id/msisdn/virtual_no) + 4. 应用 mapping 把 legacy_* 翻译成 target_*,自动决定归属 + 5. 生成 output/step1_*.sql + errors.csv + summary.txt + +约束: + - 对奇成只读;所有 INSERT 用 ON CONFLICT 保证幂等 + - mapping 中任意 target_* 缺失会让对应卡进 errors.csv 跳过 + +用法: + python3 migrate_assets.py [--config-dir config] [--resources-dir resources] [--output-dir output] +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from lib import csv_loader, legacy_query, mapping_loader, sql_builder # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description="奇成数据迁移脚本1:资产导入") + parser.add_argument("--config-dir", default="config", help="yaml 配置目录") + parser.add_argument("--resources-dir", default="resources", help="CSV 资源目录") + parser.add_argument("--output-dir", default="output", help="SQL/CSV 输出目录") + args = parser.parse_args() + + base = Path(__file__).resolve().parent + config_dir = (base / args.config_dir).resolve() + resources_dir = (base / args.resources_dir).resolve() + output_dir = (base / args.output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + mapping = mapping_loader.load_mapping(config_dir) + cards, devices, errors = csv_loader.load_assets(resources_dir) + + # 连奇成查卡元数据(运营商分类、agent_id、msisdn、virtual_no、当前生效套餐) + iccid_pairs = [(c.iccid_19, c.iccid_20) for c in cards] + dsn = mapping_loader.load_legacy_dsn(config_dir) + with legacy_query.connect_readonly(dsn) as conn: + card_metas, dup_iccid_19 = legacy_query.fetch_card_meta(conn, iccid_pairs) + + sql_builder.write_step1( + output_dir, + cards=cards, + devices=devices, + mapping=mapping, + card_metas=card_metas, + duplicate_iccid_19s=dup_iccid_19, + errors=errors, + ) + + print(f"完成:卡 {len(cards)} 张,设备 {len(devices)} 台,异常 {len(errors)} 行") + print(f"SQL 输出: {output_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migration/migrate_runtime.py b/scripts/migration/migrate_runtime.py new file mode 100644 index 0000000..5830b6c --- /dev/null +++ b/scripts/migration/migrate_runtime.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""奇成数据迁移脚本2:关联数据导入。 + +前提:脚本1的 SQL 已在新系统执行(线上库已包含本次迁移所有 ICCID)。 + +工作流: + 1. 读 mapping.yaml + cards.csv + 2. 连奇成查:卡元数据(card_meta) + 累计流量(card_usage) + 代理佣金/主钱包余额 + 3. 卡 → 当前生效套餐 → mapping.packages → target_package_id + 4. 生成 step2_*.sql + +用法: + python3 migrate_runtime.py [--config-dir config] [--resources-dir resources] [--output-dir output] +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from lib import csv_loader, legacy_query, mapping_loader, sql_builder # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description="奇成数据迁移脚本2:关联数据导入") + parser.add_argument("--config-dir", default="config", help="yaml 配置目录") + parser.add_argument("--resources-dir", default="resources", help="CSV 资源目录") + parser.add_argument("--output-dir", default="output", help="SQL/CSV 输出目录") + args = parser.parse_args() + + base = Path(__file__).resolve().parent + config_dir = (base / args.config_dir).resolve() + resources_dir = (base / args.resources_dir).resolve() + output_dir = (base / args.output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + mapping = mapping_loader.load_mapping(config_dir) + cards, _devices, _errors = csv_loader.load_assets(resources_dir) + + iccid_pairs = [(c.iccid_19, c.iccid_20) for c in cards] + agent_ids = [a.legacy_agent_id for a in mapping.agents.values() if (a.target_shop_code or "").strip()] + + dsn = mapping_loader.load_legacy_dsn(config_dir) + with legacy_query.connect_readonly(dsn) as conn: + card_metas, _dup_iccid_19 = legacy_query.fetch_card_meta(conn, iccid_pairs) + usage_snapshots = legacy_query.fetch_card_usage(conn, iccid_pairs) + commission_snapshots = legacy_query.fetch_commission_accounts(conn, agent_ids) if agent_ids else {} + agent_balances = legacy_query.fetch_agent_balances(conn, agent_ids) if agent_ids else {} + + warnings = sql_builder.write_step2( + output_dir, + cards=cards, + mapping=mapping, + card_metas=card_metas, + usage_snapshots=usage_snapshots, + commission_snapshots=commission_snapshots, + agent_balances=agent_balances, + ) + + print(f"完成:卡 {len(cards)} 张,代理 {len(agent_ids)} 个,警告 {len(warnings)} 条") + print(f"SQL 输出: {output_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migration/output/.gitkeep b/scripts/migration/output/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/migration/requirements.txt b/scripts/migration/requirements.txt new file mode 100644 index 0000000..66fe448 --- /dev/null +++ b/scripts/migration/requirements.txt @@ -0,0 +1,4 @@ +# 奇成数据迁移脚本依赖 +# Python >= 3.10 推荐 +PyMySQL>=1.1.0 # 连接奇成 MySQL(只读) +PyYAML>=6.0 # 加载映射配置 diff --git a/scripts/migration/resources/README.md b/scripts/migration/resources/README.md new file mode 100644 index 0000000..0cad7e1 --- /dev/null +++ b/scripts/migration/resources/README.md @@ -0,0 +1,49 @@ +# 业务方资料投放目录 + +业务方按以下规则把数据放到本目录,**重命名去掉 `.example` 后缀**,即可被脚本读取。 + +> 设计原则:**业务方只填"是哪张卡"和"在哪台设备上",归属、运营商、套餐等元数据全部由 +> 脚本从奇成自动获取,通过 `config/mapping.yaml` 决定如何落到新系统。** + +## cards.csv 卡资产清单 + +每张要迁移的卡一行。**必须同时提供 iccid_19 和 iccid_20**(因为奇成历史数据 +长度不可信,业务方提供两列作为权威值)。 + +| 列名 | 必填 | 说明 | +|------|------|------| +| `iccid_19` | 是 | ICCID 前 19 位(所有卡都有,作为跨文件稳定 ID;必须 19 位纯数字) | +| `iccid_20` | 视运营商 | 完整 20 位 ICCID;CMCC/CUCC/CBN(20 位运营商) **必填**,CTCC(19 位)**必须留空**;非空时前 19 位必须等于 `iccid_19` | +| `is_industry` | 否 | 是否行业卡,留空 = `false`(普通卡) | + +运营商类型从奇成 `tbl_card.category` 自动推断,在查完奇成后会做一致性校验: +- CTCC 卡填了 `iccid_20` → `errors.csv: carrier_length_conflict` 阻断 +- 非 CTCC 卡 `iccid_20` 留空 → 同上阻断 +- `iccid_19` 在奇成 `tbl_card` 命中多条记录 → `errors.csv: legacy_duplicate` 阻断,请业务方人工确认 + +**店铺归属不在 CSV 里填**,由 `mapping.yaml.agents[].target_shop_code` 通过奇成 +`tbl_card.agent_id` 自动推导。如果该卡在奇成没有 agent_id 或代理映射未填 shop_code, +卡进平台库存。 + +## devices.csv 设备资产清单 + +每台要迁移的设备一行。 + +| 列名 | 必填 | 说明 | +|------|------|------| +| `imei` | 否 | 设备 IMEI(非蜂窝设备可留空) | +| `virtual_no` | 否 | 设备虚拟号/设备号。留空时用 `imei` 兜底;两者都为空则该行报错 | +| `sim_iccid_1` ~ `sim_iccid_4` | 否 | 各插槽 ICCID,引用 `cards.csv` 的卡。可填 19 或 20 位,内部统一按前 19 位回查 `cards.iccid_19`(至少要有 1 个) | + +**店铺归属规则**:用 `sim_iccid_1`(主卡)的 agent_id 走 `mapping.agents` +推导,与卡归属同口径。 + +**其他字段自动取默认值**:`device_name` / `device_model` / `manufacturer` 留空, +`max_sim_slots=4`,`current_slot=1`(以 `sim_iccid_1` 为当前生效卡)。 + +## 一致性约束 + +- `devices.csv` 任何 `sim_iccid_*` 引用的 ICCID **必须**在 `cards.csv` 中存在 +- 同一 ICCID 不允许在 `cards.csv` 中重复出现 +- `imei` 和 `virtual_no` 至少要有一个(用作设备唯一标识) +- 同一 ICCID 不能同时被多个设备/插槽引用 diff --git a/scripts/migration/resources/cards.csv.example b/scripts/migration/resources/cards.csv.example new file mode 100644 index 0000000..c4b2147 --- /dev/null +++ b/scripts/migration/resources/cards.csv.example @@ -0,0 +1,4 @@ +iccid_19,iccid_20,is_industry +8985200026333877243,89852000263338772439, +8986062300000000001,,true +8986006654000000000,89860066540000000002, diff --git a/scripts/migration/resources/devices.csv.example b/scripts/migration/resources/devices.csv.example new file mode 100644 index 0000000..d7bcd4d --- /dev/null +++ b/scripts/migration/resources/devices.csv.example @@ -0,0 +1,4 @@ +imei,virtual_no,sim_iccid_1,sim_iccid_2,sim_iccid_3,sim_iccid_4 +860000000000001,DEV-202604-0001,89852000263338772439,8986062300000000001,, +,DEV-202604-0002,89860066540000000002,,, +860000000000003,,89860000000000004,,, diff --git a/scripts/migration/scan_legacy.py b/scripts/migration/scan_legacy.py new file mode 100644 index 0000000..e0ee2e7 --- /dev/null +++ b/scripts/migration/scan_legacy.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""扫描奇成,自动生成/增量补全 config/mapping.yaml。 + +工作流: + 1. 读 resources/cards.csv 拿到 iccid 列表 + 2. 连奇成只读查询每张卡的 category / agent_id / 当前生效套餐 / 套餐系列 + 3. 加载已有的 config/mapping.yaml(若没有则创建空的) + 4. 把扫描到的 legacy_* 增量补到 mapping 里(已有项的 target_* 字段保留,新项的 target_* 为 null) + 5. 写回 config/mapping.yaml + 6. 输出"待人工填写"清单到 console + config/mapping_todo.txt + +用法: + python3 scan_legacy.py [--config-dir config] [--resources-dir resources] +""" +from __future__ import annotations + +import argparse +import csv +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from lib import legacy_query, mapping_loader # noqa: E402 +from lib.mapping_loader import ( # noqa: E402 + AgentMapping, + CarrierMapping, + Mapping, + PackageMapping, + SeriesMapping, +) + + +def _read_iccid_pairs(cards_csv: Path) -> list[tuple[str, str]]: + """从 cards.csv 读 iccid_19 + iccid_20 列(只关心这两列,其他列忽略)。 + + 返回 (iccid_19, iccid_20) 列表;iccid_20 可为空字符串(CTCC 卡留空)。 + """ + if not cards_csv.exists(): + raise FileNotFoundError(f"找不到 {cards_csv},请按 resources/README.md 准备") + pairs: dict[str, str] = {} + with cards_csv.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + headers = reader.fieldnames or [] + if "iccid_19" not in headers: + raise ValueError(f"{cards_csv} 必须包含 iccid_19 列") + if "iccid_20" not in headers: + raise ValueError(f"{cards_csv} 必须包含 iccid_20 列(CTCC 卡留空)") + for row in reader: + i19 = (row.get("iccid_19") or "").strip() + i20 = (row.get("iccid_20") or "").strip() + if i19: + pairs[i19] = i20 + return sorted(pairs.items()) + + +def main() -> int: + parser = argparse.ArgumentParser(description="奇成元数据预扫描,自动生成 mapping.yaml") + parser.add_argument("--config-dir", default="config", help="yaml 配置目录") + parser.add_argument("--resources-dir", default="resources", help="CSV 资源目录") + args = parser.parse_args() + + base = Path(__file__).resolve().parent + config_dir = (base / args.config_dir).resolve() + resources_dir = (base / args.resources_dir).resolve() + + iccid_pairs = _read_iccid_pairs(resources_dir / "cards.csv") + if not iccid_pairs: + print("cards.csv 中没有 iccid_19,无需扫描", file=sys.stderr) + return 1 + print(f"读到 {len(iccid_pairs)} 个 ICCID,连接奇成扫描元数据...") + + dsn = mapping_loader.load_legacy_dsn(config_dir) + with legacy_query.connect_readonly(dsn) as conn: + card_metas, dup_iccid_19 = legacy_query.fetch_card_meta(conn, iccid_pairs) + meal_ids = {m.current_meal_id for m in card_metas.values() if m.current_meal_id} + meal_metas = legacy_query.fetch_meal_meta(conn, meal_ids) + + if dup_iccid_19: + print(f"警告:{len(dup_iccid_19)} 张卡的 iccid_19 在奇成 tbl_card 命中多条记录:") + for x in sorted(dup_iccid_19): + print(f" - {x}") + print("这些卡 migrate_assets 阶段会写 errors.csv 阻断,请业务方人工确认。") + + existing = mapping_loader.load_mapping(config_dir) + merged = _merge(existing, card_metas, meal_metas) + out_path = mapping_loader.merge_and_save(config_dir, merged) + todo = _collect_todo(merged) + _write_todo(config_dir / "mapping_todo.txt", todo) + _print_summary(out_path, card_metas, meal_metas, todo) + return 0 + + +def _merge( + existing: Mapping, + card_metas: dict[str, "legacy_query.LegacyCardMeta"], + meal_metas: dict[str, "legacy_query.LegacyMealMeta"], +) -> Mapping: + """把扫描到的 legacy_* 增量合并到现有 mapping(保留已填的 target_*)。""" + merged = Mapping( + migration_user_id=existing.migration_user_id, + migration_batch_no=existing.migration_batch_no, + carriers=dict(existing.carriers), + packages=dict(existing.packages), + series=dict(existing.series), + agents=dict(existing.agents), + ) + + for meta in card_metas.values(): + # carrier + if meta.category and meta.category not in merged.carriers: + merged.carriers[meta.category] = CarrierMapping( + legacy_category=meta.category, + legacy_category_name=meta.category_name, + target_carrier_id=None, + target_carrier_type=None, + target_carrier_name=None, + ) + elif meta.category in merged.carriers and not merged.carriers[meta.category].legacy_category_name: + # 补全 legacy_category_name(如果之前是空的) + old = merged.carriers[meta.category] + merged.carriers[meta.category] = CarrierMapping( + legacy_category=old.legacy_category, + legacy_category_name=meta.category_name, + target_carrier_id=old.target_carrier_id, + target_carrier_type=old.target_carrier_type, + target_carrier_name=old.target_carrier_name, + ) + # agent + if meta.agent_id and meta.agent_id not in merged.agents: + merged.agents[meta.agent_id] = AgentMapping( + legacy_agent_id=meta.agent_id, + legacy_agent_name=meta.agent_name, + target_shop_code=None, + ) + + for mm in meal_metas.values(): + if mm.meal_id not in merged.packages: + merged.packages[mm.meal_id] = PackageMapping( + legacy_meal_id=mm.meal_id, + legacy_meal_name=mm.meal_name, + target_package_id=None, + ) + if mm.series_id and mm.series_id not in merged.series: + merged.series[mm.series_id] = SeriesMapping( + legacy_series_id=mm.series_id, + legacy_series_name=mm.series_name, + target_series_id=None, + ) + + return merged + + +def _collect_todo(mapping: Mapping) -> dict[str, list[str]]: + """收集所有待填写的项,按类别分组。""" + todo: dict[str, list[str]] = {"carriers": [], "packages": [], "agents": []} + for c in sorted(mapping.carriers.values(), key=lambda x: x.legacy_category): + if not c.target_carrier_id or not c.target_carrier_type or not c.target_carrier_name: + todo["carriers"].append(f"category={c.legacy_category} ({c.legacy_category_name})") + for p in sorted(mapping.packages.values(), key=lambda x: x.legacy_meal_id): + if not p.target_package_id: + todo["packages"].append(f"meal_id={p.legacy_meal_id} ({p.legacy_meal_name})") + for a in sorted(mapping.agents.values(), key=lambda x: x.legacy_agent_id): + if not (a.target_shop_code or "").strip(): + todo["agents"].append(f"agent_id={a.legacy_agent_id} ({a.legacy_agent_name})") + return todo + + +def _write_todo(path: Path, todo: dict[str, list[str]]) -> None: + lines = ["奇成迁移映射待填写清单", "=" * 40, ""] + for kind in ("carriers", "packages", "agents"): + lines.append(f"## {kind} (待填: {len(todo[kind])})") + for item in todo[kind]: + lines.append(f" - {item}") + lines.append("") + path.write_text("\n".join(lines), encoding="utf-8") + + +def _print_summary(out_path: Path, card_metas, meal_metas, todo: dict[str, list[str]]) -> None: + print(f"扫描完成,mapping 写入: {out_path}") + print(f" 奇成卡命中 : {len(card_metas)}") + print(f" 奇成套餐命中 : {len(meal_metas)}") + print(f" 待填 carriers : {len(todo['carriers'])}") + print(f" 待填 packages : {len(todo['packages'])}") + print(f" 待填 agents : {len(todo['agents'])}") + if any(todo.values()): + print(f"详见: {out_path.parent / 'mapping_todo.txt'}") + + +if __name__ == "__main__": + sys.exit(main())