From 88cc5e96ecf36ae38ed465633b9346e5131a6d33 Mon Sep 17 00:00:00 2001 From: break Date: Thu, 6 Aug 2026 09:35:00 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9A=82=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/tech-global-audit/审计覆盖基线.md | 22 +- README.md | 1 + cmd/worker/main.go | 75 +- docs/admin-openapi.yaml | 761 +++++++++ .../平台基础审计调查接口.md | 6 +- .../跨视角调查与前端导航契约.md | 113 ++ .../application/auditarchive/integration.go | 350 ++++ .../application/auditarchive/retention.go | 542 ++++++ internal/application/auditarchive/service.go | 412 +++++ internal/application/cardobservation/apply.go | 39 +- .../application/cardobservation/network.go | 38 + .../application/cardobservation/series.go | 20 +- .../application/cardobservation/traffic.go | 30 + internal/bootstrap/services.go | 2 +- internal/bootstrap/worker_services.go | 9 +- internal/handler/admin/audit.go | 117 +- internal/handler/callback/cmcc_realname.go | 2 +- internal/handler/callback/ctcc_realname.go | 2 +- internal/handler/callback/cucc_realname.go | 2 +- internal/handler/callback/payment.go | 2 +- internal/infrastructure/audit/registry.go | 21 + internal/infrastructure/audit/retention.go | 38 + .../cardobservation/series_runner.go | 221 ++- internal/model/dto/audit_dto.go | 61 +- internal/model/log_archive_run.go | 38 + .../polling/package_activation_handler.go | 21 +- internal/query/audit/events.go | 61 +- internal/query/audit/finance.go | 1519 +++++++++++++++++ internal/query/audit/resources.go | 20 +- internal/query/audit/risks.go | 299 ++++ internal/query/audit/subject_activities.go | 41 +- internal/query/audit/timeline.go | 317 ++++ internal/query/integration/logs.go | 76 +- internal/query/integration/overview.go | 32 +- internal/query/retention/retention.go | 115 ++ internal/routes/audit.go | 43 +- internal/service/carrier/service.go | 5 +- internal/service/enterprise_device/service.go | 8 +- internal/service/iot_card/unified_audit.go | 24 +- internal/service/order/service.go | 6 +- .../service/package/activation_service.go | 1 + internal/service/recharge_order/service.go | 7 +- internal/service/role/service.go | 22 +- internal/task/audit_daily_archive.go | 61 + internal/task/audit_monthly_retention.go | 64 + internal/task/auto_purchase.go | 17 +- internal/task/card_observation_series.go | 7 + internal/task/commission_calculation.go | 11 +- internal/task/integration_archive.go | 113 ++ internal/task/notification_cleanup.go | 3 +- internal/task/polling_carddata_handler.go | 1 + internal/task/polling_cardstatus_handler.go | 1 + internal/task/polling_package_handler.go | 1 + internal/task/polling_protect_handler.go | 1 + internal/task/polling_realname_handler.go | 1 + internal/task/polling_utils.go | 15 + m.csv | 1 + ...4_add_audit_investigation_indexes.down.sql | 19 + ...204_add_audit_investigation_indexes.up.sql | 32 + .../000205_create_log_archive_run.down.sql | 11 + .../000205_create_log_archive_run.up.sql | 53 + .../build-multi-view-audit-center/tasks.md | 24 +- pkg/constants/audit.go | 76 + pkg/constants/audit_archive.go | 37 + pkg/constants/constants.go | 2 + pkg/constants/wallet.go | 1 + pkg/errors/codes.go | 7 + pkg/errors/errors.go | 8 + pkg/errors/handler.go | 4 +- pkg/storage/s3.go | 26 + pkg/storage/storage.go | 9 + .../batch_purchase.cpython-313.pyc | Bin 0 -> 24764 bytes ...ets.example_待失效订单_20260803_111217.csv | 91 + ...ample_过期时间修改结果_20260803_112658.csv | 91 + scripts/legacy_device_export/devices.csv | 704 ++++---- 75 files changed, 6520 insertions(+), 513 deletions(-) create mode 100644 docs/feature-504-multi-view-audit-center/跨视角调查与前端导航契约.md create mode 100644 internal/application/auditarchive/integration.go create mode 100644 internal/application/auditarchive/retention.go create mode 100644 internal/application/auditarchive/service.go create mode 100644 internal/infrastructure/audit/retention.go create mode 100644 internal/model/log_archive_run.go create mode 100644 internal/query/audit/finance.go create mode 100644 internal/query/audit/risks.go create mode 100644 internal/query/audit/timeline.go create mode 100644 internal/query/retention/retention.go create mode 100644 internal/task/audit_daily_archive.go create mode 100644 internal/task/audit_monthly_retention.go create mode 100644 internal/task/integration_archive.go create mode 100644 m.csv create mode 100644 migrations/000204_add_audit_investigation_indexes.down.sql create mode 100644 migrations/000204_add_audit_investigation_indexes.up.sql create mode 100644 migrations/000205_create_log_archive_run.down.sql create mode 100644 migrations/000205_create_log_archive_run.up.sql create mode 100644 pkg/constants/audit_archive.go create mode 100644 scripts/batch_package_purchase/__pycache__/batch_purchase.cpython-313.pyc create mode 100644 scripts/batch_package_purchase/assets.example_待失效订单_20260803_111217.csv create mode 100644 scripts/batch_package_purchase/assets.example_过期时间修改结果_20260803_112658.csv diff --git a/.scratch/tech-global-audit/审计覆盖基线.md b/.scratch/tech-global-audit/审计覆盖基线.md index 0380aed..b29136e 100644 --- a/.scratch/tech-global-audit/审计覆盖基线.md +++ b/.scratch/tech-global-audit/审计覆盖基线.md @@ -80,9 +80,9 @@ | 代理订单主钱包退款回充 | `refund.approve` 与退款单、订单、原扣款钱包、原扣款流水和唯一退款流水同事务；重复退款流水不重复写成功事件 | 原成功扣款流水定位付款钱包并限定金额；退款审批、`tb_agent_wallet` 与唯一成功退款流水同事务形成权威事实 | N/A（本资金接缝不调用渠道或审批外部系统；原路渠道退款尚未实现） | 同事务写入 `wallet.agent_main.refunded`，消费者复核退款流水、原扣款事实、金额上限和资产快照 | | 代理商资金概况信用投影 | N/A（普通受权读取；不返回其他数据范围的资金事实，不执行资金或配置变更） | 只读投影 `tb_shop`、主/佣金钱包、提现汇总和主账号；派生金额不另建事实表 | N/A（无外部系统调用） | N/A（纯 Query 不产生可靠副作用） | | 受控系统配置更新 | N/A（用户已明确取消全局 Audit Event；仅超级管理员可更新代码注册 Key，未知 Key、非法类型和值域均拒绝） | `tb_system_config` 是配置值、类型、模块及更新人的 PostgreSQL 权威事实，更新后失效 Redis 缓存 | N/A（配置更新不调用外部系统；不得写 Integration Log 冒充配置审计） | N/A（配置更新不产生可靠异步副作用） | -| 电信实名结果回调 | N/A（Audit Event 已移出本 Change；运营商来源真实性验证也不在本票边界） | `tb_iot_card` 是实名状态、首次实名时间和逆转窗口的权威事实 | 每次入站先写 `tb_integration_log`，仅保存正文摘要；覆盖 `invalid_payload/ignored/not_found/conflict/success/failed` 终态 | 实名事实首次变化时由公共 `ApplyCardObservation` 同事务写入实名状态变化 Outbox；重复成功不重复写事件 | -| 移动实名成功回调 | N/A（Audit Event 已移出本 Change；不接入旧平台登录、MSISDN 补查或来源真实性验证） | `tb_iot_card` 是实名状态、首次实名时间和逆转窗口的权威事实 | 每次入站先写 `tb_integration_log`，仅保存正文摘要；覆盖 `invalid_payload/not_found/conflict/success/failed` 终态，并通过 pending 租约恢复中断处理 | 仅合法成功报文进入公共 `ApplyCardObservation`；实名事实首次变化时同事务写入 Outbox，重复成功不重复写事件 | -| 联通实名成功回调 | N/A（Audit Event 已移出本 Change；不复制旧 `inner_callback`、第三方推送或 Gateway 二次确认） | `tb_iot_card` 是实名状态、首次实名时间和逆转窗口的权威事实；上游 `dateChanged` 只用于幂等和留痕 | 每次入站先写 `tb_integration_log` 正文摘要；覆盖 `invalid_payload/not_found/conflict/success/failed`，关闭时记录 `ignored` | 仅合法成功报文进入公共 `ApplyCardObservation`；实名事实首次变化时同事务写入 Outbox，重复成功不重复写事件 | +| 电信实名结果回调 | 实名状态首次实际变化时写 `iot_card.realname_callback_sync`，使用 `external_system/callback`，关联 IoT 卡和入站 Integration Log；Audit Event 与卡状态、首次实名时间及实名变化 Outbox 同事务，审计失败回滚。已解析卡后的业务失败写独立短事务；重复成功不伪造事件 | `tb_iot_card` 是实名状态、首次实名时间和逆转窗口的权威事实 | 每次入站先写 `tb_integration_log`，仅保存正文摘要；覆盖 `invalid_payload/ignored/not_found/conflict/success/failed` 终态；未改变实名状态时只保留 Integration Log | 实名事实首次变化时由公共 `ApplyCardObservation` 同事务写入实名状态变化 Outbox；重复成功不重复写事件 | +| 移动实名成功回调 | 实名状态首次实际变化时写 `iot_card.realname_callback_sync`，使用 `external_system/callback`，关联 IoT 卡和入站 Integration Log；Audit Event 与卡状态、首次实名时间及实名变化 Outbox 同事务，审计失败回滚。已解析卡后的业务失败写独立短事务；重复成功不伪造事件 | `tb_iot_card` 是实名状态、首次实名时间和逆转窗口的权威事实 | 每次入站先写 `tb_integration_log`，仅保存正文摘要；覆盖 `invalid_payload/not_found/conflict/success/failed` 终态，并通过 pending 租约恢复中断处理；未改变实名状态时只保留 Integration Log | 仅合法成功报文进入公共 `ApplyCardObservation`；实名事实首次变化时同事务写入 Outbox，重复成功不重复写事件 | +| 联通实名成功回调 | 实名状态首次实际变化时写 `iot_card.realname_callback_sync`，使用 `external_system/callback`，关联 IoT 卡和入站 Integration Log；Audit Event 与卡状态、首次实名时间及实名变化 Outbox 同事务，审计失败回滚。已解析卡后的业务失败写独立短事务；重复成功不伪造事件 | `tb_iot_card` 是实名状态、首次实名时间和逆转窗口的权威事实；上游 `dateChanged` 只用于幂等和留痕 | 每次入站先写 `tb_integration_log` 正文摘要；覆盖 `invalid_payload/not_found/conflict/success/failed`，关闭时记录 `ignored`；未改变实名状态时只保留 Integration Log | 仅合法成功报文进入公共 `ApplyCardObservation`；实名事实首次变化时同事务写入 Outbox，重复成功不重复写事件 | | 联通解除实名回调 | N/A（只识别并留痕外部解除通知，不把单次回调作为本地实名逆转事实） | `tb_iot_card` 保持原实名状态、首次实名时间、检查时间及逆转计数，回调不写领域事实 | 每次入站先写 `tb_integration_log` 正文摘要；覆盖 `invalid_payload/not_found/conflict/ignored/failed` 终态并支持 pending 租约恢复 | N/A（不调用公共实名观测，不产生状态变化、停机或套餐事件） | ## 七月确认范围增量登记 @@ -182,6 +182,22 @@ - 轮询配置与人工动作已切换统一 Writer：轮询配置创建、更新、删除和启停使用 `polling_config.create/update/delete/update_status`，并发配置更新与计数重置使用 `polling_concurrency.update/reset`，告警规则创建、更新和删除使用 `polling_alert.create_rule/update_rule/delete_rule`；成功事件与对应 PostgreSQL 配置事实共用 GORM 事务，审计失败回滚配置写入。Redis 并发计数重置无法与 PostgreSQL 原子提交，审计失败时恢复重置前计数；人工单卡去重键在日志/审计事务失败或队列写入失败时移除，避免阻断原有重试。 - 单卡、批量、条件筛选人工触发和取消分别使用 `polling_manual_trigger.trigger_single/trigger_batch/trigger_by_condition/cancel_trigger`，记录真实后台账号、手动任务和实际卡资源；配置重名、每日触发上限、重复入队、越权取消和已结束任务取消等已定位资源的拒绝写独立短事务，二次失败保留原业务错误并记录 critical。`tb_polling_manual_trigger_log` 继续承担进度、结果与历史查询，不被 Audit Event 替代或停写；审计不复制 `CardIDs`、条件正文、通知渠道正文或其他安全凭据，只保存任务类型、触发方式、数量、状态及“条件/通知渠道是否配置”等安全事实。 - 实名、流量和卡状态轮询的每次真实 Gateway 尝试继续写 Integration Log；套餐/保护期轮询自身不伪造 Gateway 尝试，实际停复机复用共享 `StopResumeService` 的 Audit/Integration 边界。轮询配置、并发状态、告警历史、人工任务状态/历史和监控页面等普通运行查询均为 N/A；通知投递、任务 ledger、Audit Event 与 Integration Log 保持独立事实。`polling_cleanup` 数据清理配置未包含在 8.5 明确边界，本轮不借轮询审计扩展其 CRUD 或手动清理动作，继续保留在覆盖清单等待对应显式切片。 +- 外部 Callback 已按当前生产路由收口：微信/支付宝/富友支付回调继续复用 7.4 的支付、订单和充值事务审计，企微审批回调继续复用 8.1 的权威终态审计；电信、移动、联通实名成功回调在共享 `ApplyCardObservation` 事务中使用 `iot_card.realname_callback_sync`，真实 actor 为各运营商 `external_system/callback`，关联 IoT 卡和入站 Integration Log。重复支付、重复审批、重复实名及无状态变化只保留 Integration Log，不伪造成功 Audit Event；联通解除实名仍仅留痕且不改变本地实名事实。本切片不修改支付、运营商或企微协议。 +- 当前 31 个 Asynq Worker 已按 8.7 逐项复核：`iot_card:import`、`device:import`、`order_package:invalidate`、`asset_package:batch_order`、`commission:calculate`、`package:first_activation`、`package:queue_activation`、`order:expire`、`notification:cleanup`、`auto_purchase:after_recharge`、`wecom:approval:sync`、`wecom:approval:recovery`、`agent_recharge:recovery` 已复用前序纵向切片的统一 Audit Event；本切片新增 `card_observation:series` 及 `polling:realname/carddata/card_status` 的 `system_task/worker` 上下文，仅在实名、流量、网络或设备字段/当前卡槽实际变化时分别写 `iot_card.worker_realname_sync`、`iot_card.worker_traffic_sync`、`iot_card.worker_network_sync`、`device.worker_observation_sync`，并关联对应 Gateway Integration Log。`polling:package/protect` 不另造轮询动作，实际停复机继续复用 `iot_card.auto_stop/auto_start/auto_stop_reason_update`。上述链路传播 request/correlation/parent；重试失败写 failed 短事务，无实际变化仅保留 Integration Log 或任务运行事实，不伪造 success。 +- Worker N/A 与后续边界：`email:send` 仅模拟邮件投递；`export:dispatch/shard/finalize` 仅做导出技术装配；`commission_stats:update/sync/archive` 仅维护 Redis/PostgreSQL 统计投影；`polling:alert_check` 仅生成告警运行事实；`polling:data_cleanup` 仅执行既有轮询数据清理；`package:expiry_reminder` 仅生成可靠通知 Outbox；`daily_traffic:flush` 仅把 Redis 日流量 Domain Ledger 落盘，均不重复创建 Audit Event。`outbox:deliver` 自身是可靠投递技术入口，Relay/投递事实 N/A；其各业务消费者是否形成新业务事实按任务 8.9 逐项收口，不在 8.7 提前迁移。Scheduler 仅投递或创建任务的边界留给 8.8。 +- Scheduler 已按当前注册清单逐项复核：代理在线充值恢复、订单过期关闭和企微审批恢复在真实业务事实变化时使用 `scheduled_job/scheduler`；轮询 Scheduler 直接执行的套餐到期、后续权益接续、流量周期重置及套餐到期停机检查同样保留 `scheduled_job/scheduler` 操作者。套餐到期后的异步停机检查使用不受调度 tick 取消影响的原审计上下文，继续传播 actor、correlation 和 parent，不伪造人工操作者。 +- Scheduler N/A 与幂等边界：Asynq 周期注册、轮询心跳、队列深度检查、分片/手动队列出队、重复调度的 `Unique` 去重与失败回队只是技术调度或任务事实，不写 Audit Event，也不把后续 Worker 结果伪装成 Scheduler 成功。告警检查、轮询数据清理、通知保留清理、套餐临期提醒和日流量落盘继续按 8.7 的 Worker/Domain Ledger/Outbox 边界处理；重复调度依赖既有状态条件、领取租约、稳定事件 ID 和 Asynq `Unique`，无实际变化不伪造 success。Worker 消费逻辑未在 8.8 迁移。 +- 当前 14 个 Outbox 事件注册、12 个消费者实现已按 8.9 逐项复核：企微审批提交终态、审批标准决策分发、卡实名/流量/网络变化后续处理、代理在线充值入账和三类站内通知生成会形成新的内部业务事实，均通过 `outbox:deliver` 入口取得 `system_task/worker` actor，并沿用信封的 request/correlation/parent；实际变化继续复用 8.1、7.2、7.6 和 8.4 已接入的同事务 Audit Event。状态条件、处理租约、稳定事件 ID、业务唯一键和通知 `CreateIdempotent` 保证至少一次投递不会伪造重复 success；消费者返回成功只代表本次业务处理完成，Outbox 的 delivered 状态仍是独立投递事实。 +- Outbox 消费者 N/A 边界：卡观测序列请求仅幂等创建后续观测任务，代理主钱包预占/入账/退款消费者仅复核既有 Domain Ledger，扣款消费者仅复核流水并按阈值幂等追加新的通知 Outbox，均不把“校验通过、任务触发或二次投递成功”写成业务 Audit Event；后续观测或通知实际改变业务事实时由对应 Worker/Application 自身写 Audit Event。Relay 的领取、入队、续租、delivered/failed、退避和重投保持技术投递事实，不在 8.9 修改或审计化。 +- request/correlation 组合时间线只按非空稳定 ID 精确读取 `tb_audit_event`、`tb_integration_log` 和 `tb_outbox_event`，以 `record_source` 保留 Audit、外部交互及可靠投递边界；Outbox 只展示当前投递摘要，不把 delivered 解释为业务成功。Asynq 没有通用 PostgreSQL 历史表，Query 仅从已落库的导入、批量购包、套餐失效、设备批量和导出任务资源生成 `asynq_task` 摘要，不扫描 Redis、不展示技术重试。订单、支付、退款、充值、钱包/流水、套餐权益、审批和佣金等只生成 `domain_ledger_ref` 稳定引用，金额与状态仍以业务表为准并留给 9.2 专业资金视角。Access Log 只返回 request ID 供开发检索，不读取日志文件；历史缺少 request/correlation/parent、直接 Audit 关联或稳定资源时通过 fidelity 标记原样降级，不按相近时间、相似资源或相同 correlation 猜测技术尝试。节点统一按发生时间、`record_source`、稳定节点 ID 升序排列。 +- 资金调查时间线以平台只读 Query 组合 Audit Event、代理/资产钱包流水、代理钱包预占、订单、支付、退款、代理/个人资产充值、佣金、提现和审批当前业务事实；支持从店铺、钱包、订单、支付、退款、充值、审批、第三方交易号、操作者、时间或 correlation 中任一稳定条件进入，并在服务端解析已持久化关联，不要求前端补齐整条链路。节点按发生时间、`record_source`、稳定节点 ID 倒序分页，统一返回调查跳转引用；钱包流水的 `amount/balance_before/balance_after` 及各业务表金额字段明确标记为权威，Audit Event 金额只作操作摘要，冲突时不改写历史事件并以对应 Domain Ledger 为准。平台身份复用统一审计 Query 的 SuperAdmin/Platform 后端校验，不增加店铺数据范围，也不提供资金重算、状态修改、导出或恢复能力。 +- 风险调查视角只读聚合统一 Audit Event：高/严重风险、涉及订单/支付/退款/充值/钱包/佣金等资金资源、安全类别，以及 `failed/denied/partial/unknown` 结果进入固定风险集合；普通低风险成功事件明确排除。overview 在最长 31 天的显式时间范围内按风险、结果、action、来源和小时/日趋势聚合，events 复用统一事件批量投影、稳定倒序分页及 `investigation_refs`，可继续跳转事件、资源、actor 和 correlation。该视角不读取或修改业务状态，不产生 Audit Event、Domain Ledger、Integration Log 或 Outbox，也不提供处置工单、自动封禁、导出或恢复能力。 +- 跨视角查询性能仅补已确认路径的 PostgreSQL B-tree：保留事件时间、actor、scope、correlation 和资源时间线既有索引，补 action、result、risk、category、source 的稳定倒序分页索引，将 request/parent 索引补齐时间与 ID，并为资源 type+id/key 到事件的反向关联补索引。全局、actor、资源和风险事件先分页事件 ID，再批量投影事件与资源；Integration 列表同样先分页 ID，再批量读取列表字段，不加载正文 JSON。第一阶段不增加 JSONB 任意搜索、Redis 结果缓存、月分区或冷热联合查询。 +- 跨视角 HTTP 契约复用既有平台 `AuditHandler` 和生产/文档装配，仅新增 request、correlation、finance、risk 的只读 GET Handler/DTO/RouteSpec；身份继续只取认证上下文，未增加写路由、导出、处置、恢复或前端实现。前端导航文档逐行冻结 design 8.1-8.4 的源页面、前置接口、`response.data` 字段、入口可见条件、目标参数和降级规则，并以资产、订单、退款、钱包、通知、风险节点演示完整调用链。 +- Audit Event 与 Event Resource 每日冷归档使用独立 `audit:daily:archive` Asynq 任务和 `tb_log_archive_run` 轻量账本：按 `Asia/Shanghai` 前一完整自然日、事件 `created_at` 半开区间分页读取，每行保存一个完整事件及其全部 `resources[]`，生成 JSONL+gzip、manifest 和 SHA-256；对象 Key 按日期与 revision 稳定生成，上传后回读 metadata 核对大小、hash、事件数和资源数，成功重复任务直接复用，失败或对象不一致使用新 revision 且不覆盖旧对象。该基础设施任务不写业务 Audit Event，不读取 Integration/Access/Domain Ledger/Outbox/旧 operation log，不清理数据库，也不向业务 Writer 注入对象存储；对象存储失败只更新归档账本并由 Asynq 重试，业务审计写入继续正常执行。 +- Integration Log 冷归档复用同一对象存储、归档 Service 和 `tb_log_archive_run`：`integration:daily:archive` 按 `created_at` 保存前一自然日的结构化 JSONL+gzip 创建日快照；`integration:monthly:finalize` 在月初逐日按数据库当前内容重新生成并比较记录数与 SHA-256，首次终结、内容变化或对象 metadata 不一致时创建新的不可变 revision，旧对象不覆盖。月度复核仅把无 `pending` 记录且最终对象、manifest 均复核成功的日期标记 `is_final`；`pending`、对象损坏或复核失败会使任务失败并明确阻止后续清理。该切片不修改 Integration Writer、恢复语义或业务状态，也不删除 PostgreSQL 数据、不提供对象存储查询/恢复接口。 +- 月度留存清理使用 `audit:monthly:retention` Asynq 任务，在 `Asia/Shanghai` 每月 1 日 06:00 处理上一完整自然月：先补齐最后一日 Audit/Integration 归档并完成 Integration 最终 revision，再逐日核对 ledger、manifest、对象 metadata、压缩对象实际大小/SHA-256 及数据库数量。全月硬门禁通过后仅按 Event Resource → Audit Event → Integration Log 顺序对 `tb_audit_event_resource`、`tb_audit_event`、`tb_integration_log` 以 1000 行有界批次执行 GORM 物理删除，并复用 `tb_log_archive_run.cleanup_started_at/cleaned_at` 断点续跑；对象存储归档和 manifest 长期保留。清理结果以 `retention_worker/system_task` 写当月 `audit.retention_cleanup` Audit Event，资源为 `log_archive_month`；Domain Ledger、Integration Log 新写、Outbox 均为 N/A，因该事实是内部留存执行结果，不是业务状态、外部交互或可靠投递。Access Log、订单/支付/退款/钱包等 Domain Ledger、Outbox、Asynq 运行事实、手动轮询、旧 operation log 及其他业务表明确不删除。 +- 在线审计查询以 `tb_log_archive_run.cleaned_at/range_end` 作为真实清理边界：平台 Audit/Integration、request/correlation/finance/risk 及代理/企业活动响应统一返回 `retention{online_from,archived_before,timezone}`；缺省时间范围只查 PostgreSQL 在线窗口，显式早于或跨越边界返回 `CodeAuditDataArchived` 和当前边界。稳定事件或 Integration ID 只在在线库查找，不存在仍返回资源不存在；历史资源快照搜索和 Integration 尝试序列同样受边界限制。该 Query 切片不访问对象存储，不新增归档下载、恢复、冷热联合查询、导出或写路由，普通读取仍为 Audit Event/Domain Ledger/Integration Log/Outbox N/A。 ### 与审计接入分开保留的独立修复 diff --git a/README.md b/README.md index f347270..c1e5843 100644 --- a/README.md +++ b/README.md @@ -944,6 +944,7 @@ rdb.Set(ctx, key, status, time.Hour) ## 文档 - [平台基础审计调查接口](docs/feature-504-multi-view-audit-center/平台基础审计调查接口.md) +- [跨视角调查与前端导航契约](docs/feature-504-multi-view-audit-center/跨视角调查与前端导航契约.md) - [外部集成调查接口](docs/feature-504-multi-view-audit-center/外部集成调查接口.md) - [代理和企业主体资源活动接口](docs/feature-504-multi-view-audit-center/主体资源活动接口.md) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 2dc0f2b..c5a3234 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -16,6 +16,7 @@ import ( agentrechargeApp "github.com/break/junhong_cmp_fiber/internal/application/agentrecharge" approvalApp "github.com/break/junhong_cmp_fiber/internal/application/approval" + auditArchiveApp "github.com/break/junhong_cmp_fiber/internal/application/auditarchive" cardObservationApp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation" notificationApp "github.com/break/junhong_cmp_fiber/internal/application/notification" walletApp "github.com/break/junhong_cmp_fiber/internal/application/wallet" @@ -146,6 +147,7 @@ func runWorker(cfg *config.Config) { taskHandler.RegisterHandlers() registerWeComApprovalTasks(taskHandler.GetMux(), runtime, cfg, appLogger) registerAgentRechargeRecoveryTask(taskHandler.GetMux(), runtime, appLogger) + registerAuditArchiveTask(taskHandler.GetMux(), runtime, appLogger) outboxHandler := outbox.NewHandler(runtime.outboxConsumers) taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle) startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger) @@ -685,7 +687,7 @@ func startAsynqScheduler(cfg *config.Config, redisAddr string, appLogger *zap.Lo } }() - appLogger.Info("Asynq Scheduler 已启动（企微审批恢复: 每 2 分钟，套餐临期提醒: 上海时区每日 03:00）") + appLogger.Info("Asynq Scheduler 已启动（企微审批恢复: 每 2 分钟，套餐临期提醒: 上海时区每日 03:00，Audit 归档: 每日 04:00，Integration 归档: 每日 04:30/月初 05:00，月度留存清理: 月初 06:00）") return asynqScheduler } @@ -760,9 +762,80 @@ func registerAsynqScheduleTasks(asynqScheduler *asynq.Scheduler) error { ); err != nil { return fmt.Errorf("注册每日流量落盘定时任务失败: %w", err) } + if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 4 * * *", asynq.NewTask( + constants.TaskTypeAuditDailyArchive, + nil, + asynq.MaxRetry(10), + asynq.Timeout(2*time.Hour), + asynq.Unique(23*time.Hour), + asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAuditDailyArchive)), + )); err != nil { + return fmt.Errorf("注册统一审计每日冷归档定时任务失败: %w", err) + } + if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 30 4 * * *", asynq.NewTask( + constants.TaskTypeIntegrationDailyArchive, + nil, + asynq.MaxRetry(10), + asynq.Timeout(2*time.Hour), + asynq.Unique(23*time.Hour), + asynq.Queue(constants.QueueForTaskType(constants.TaskTypeIntegrationDailyArchive)), + )); err != nil { + return fmt.Errorf("注册 Integration Log 每日冷归档定时任务失败: %w", err) + } + if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 5 1 * *", asynq.NewTask( + constants.TaskTypeIntegrationMonthlyFinalize, + nil, + asynq.MaxRetry(10), + asynq.Timeout(6*time.Hour), + asynq.Unique(27*24*time.Hour), + asynq.Queue(constants.QueueForTaskType(constants.TaskTypeIntegrationMonthlyFinalize)), + )); err != nil { + return fmt.Errorf("注册 Integration Log 月度最终版本复核任务失败: %w", err) + } + if _, err := asynqScheduler.Register("CRON_TZ=Asia/Shanghai 0 6 1 * *", asynq.NewTask( + constants.TaskTypeAuditMonthlyRetention, + nil, + asynq.MaxRetry(10), + asynq.Timeout(12*time.Hour), + asynq.Unique(27*24*time.Hour), + asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAuditMonthlyRetention)), + )); err != nil { + return fmt.Errorf("注册月度日志留存清理任务失败: %w", err) + } return nil } +// registerAuditArchiveTask 注册 Audit 与 Integration 冷归档任务处理器。 +func registerAuditArchiveTask(mux *asynq.ServeMux, runtime *workerRuntime, appLogger *zap.Logger) { + if runtime.storageSvc == nil { + appLogger.Warn("对象存储未配置，审计归档任务将在执行时重试") + mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(nil, appLogger).Handle) + integrationHandler := task.NewIntegrationArchiveHandler(nil, appLogger) + mux.HandleFunc(constants.TaskTypeIntegrationDailyArchive, integrationHandler.HandleDaily) + mux.HandleFunc(constants.TaskTypeIntegrationMonthlyFinalize, integrationHandler.HandleMonthlyFinalize) + mux.HandleFunc(constants.TaskTypeAuditMonthlyRetention, task.NewAuditMonthlyRetentionHandler(nil, appLogger).Handle) + return + } + auditWriter, ok := runtime.workerResult.Services.PaymentAudit.(*auditInfra.Writer) + if !ok || auditWriter == nil { + appLogger.Fatal("初始化月度日志留存清理失败：统一审计 Writer 未配置") + } + service, err := auditArchiveApp.NewService(runtime.db, runtime.storageSvc.Provider(), constants.AuditArchiveInstanceID, auditWriter) + if err != nil { + appLogger.Fatal("初始化统一审计归档服务失败", zap.Error(err)) + } + mux.HandleFunc(constants.TaskTypeAuditDailyArchive, task.NewAuditDailyArchiveHandler(service, appLogger).Handle) + integrationHandler := task.NewIntegrationArchiveHandler(service, appLogger) + mux.HandleFunc(constants.TaskTypeIntegrationDailyArchive, integrationHandler.HandleDaily) + mux.HandleFunc(constants.TaskTypeIntegrationMonthlyFinalize, integrationHandler.HandleMonthlyFinalize) + mux.HandleFunc(constants.TaskTypeAuditMonthlyRetention, task.NewAuditMonthlyRetentionHandler(service, appLogger).Handle) + appLogger.Info("注册审计归档任务处理器", + zap.String("audit_task_type", constants.TaskTypeAuditDailyArchive), + zap.String("integration_daily_task_type", constants.TaskTypeIntegrationDailyArchive), + zap.String("integration_monthly_task_type", constants.TaskTypeIntegrationMonthlyFinalize), + zap.String("retention_task_type", constants.TaskTypeAuditMonthlyRetention)) +} + // createTaskHandler 创建并返回包含全部任务处理器的 Asynq Handler。 func createTaskHandler(runtime *workerRuntime, appLogger *zap.Logger) *queue.Handler { stopResumeSvc, _ := runtime.workerResult.Services.StopResumeService.(iot_card_svc.StopResumeServiceInterface) diff --git a/docs/admin-openapi.yaml b/docs/admin-openapi.yaml index 83f8d92..18431b7 100644 --- a/docs/admin-openapi.yaml +++ b/docs/admin-openapi.yaml @@ -130,6 +130,82 @@ components: user_agent: type: string type: object + AuditFinanceAmountAuthority: + properties: + authoritative: + type: boolean + conflict_rule: + type: string + field: + type: string + table: + type: string + type: object + AuditFinanceTimelineNode: + properties: + amount: + nullable: true + type: integer + amount_authority: + $ref: '#/components/schemas/AuditFinanceAmountAuthority' + balance_after: + nullable: true + type: integer + balance_before: + nullable: true + type: integer + code: + type: string + currency: + type: string + facts: + additionalProperties: {} + nullable: true + type: object + investigation_refs: + $ref: '#/components/schemas/AuditInvestigationRefs' + node_id: + type: string + occurred_at: + format: date-time + type: string + record_source: + type: string + result: + type: string + result_name: + type: string + shop_id: + minimum: 0 + nullable: true + type: integer + title: + type: string + wallet: + $ref: '#/components/schemas/AuditFinanceWalletRef' + type: object + AuditFinanceTimelinePage: + properties: + items: + items: + $ref: '#/components/schemas/AuditFinanceTimelineNode' + nullable: true + type: array + page: + type: integer + page_size: + type: integer + total: + type: integer + type: object + AuditFinanceWalletRef: + properties: + resource_type: + type: string + wallet_id: + minimum: 0 + type: integer + type: object AuditIntegrationRef: properties: integration_id: @@ -171,6 +247,76 @@ components: resource_type: type: string type: object + AuditLinkTimeline: + properties: + access_log_lookup_request_id: + nullable: true + type: string + correlation_id: + nullable: true + type: string + nodes: + items: + $ref: '#/components/schemas/AuditLinkTimelineNode' + nullable: true + type: array + request_id: + nullable: true + type: string + type: object + AuditLinkTimelineNode: + properties: + code: + type: string + correlation_id: + nullable: true + type: string + fidelity: + $ref: '#/components/schemas/AuditLinkageFidelity' + investigation_refs: + $ref: '#/components/schemas/AuditInvestigationRefs' + node_id: + type: string + occurred_at: + format: date-time + type: string + parent_event_id: + nullable: true + type: string + record_source: + type: string + reference_only: + type: boolean + request_id: + nullable: true + type: string + resources: + items: + $ref: '#/components/schemas/AuditInvestigationResourceRef' + nullable: true + type: array + result: + type: string + result_name: + type: string + summary: + type: string + title: + type: string + type: object + AuditLinkageFidelity: + properties: + correlation_available: + type: boolean + direct_audit_link_available: + type: boolean + parent_event_available: + type: boolean + request_available: + type: boolean + stable_resource_available: + type: boolean + type: object AuditResourceCandidate: properties: display_name: @@ -243,6 +389,88 @@ components: subject_visibility: type: string type: object + AuditRiskEventPage: + properties: + items: + items: + $ref: '#/components/schemas/AuditEventView' + nullable: true + type: array + page: + type: integer + page_size: + type: integer + total: + type: integer + type: object + AuditRiskNamedCount: + properties: + code: + type: string + count: + type: integer + name: + type: string + type: object + AuditRiskOverview: + properties: + actions: + items: + $ref: '#/components/schemas/AuditRiskNamedCount' + nullable: true + type: array + bucket: + type: string + results: + items: + $ref: '#/components/schemas/AuditRiskNamedCount' + nullable: true + type: array + risks: + items: + $ref: '#/components/schemas/AuditRiskNamedCount' + nullable: true + type: array + signals: + items: + $ref: '#/components/schemas/AuditRiskNamedCount' + nullable: true + type: array + sources: + items: + $ref: '#/components/schemas/AuditRiskNamedCount' + nullable: true + type: array + total: + type: integer + trend: + items: + $ref: '#/components/schemas/AuditRiskTrendPoint' + nullable: true + type: array + type: object + AuditRiskTrendPoint: + properties: + bucket_at: + format: date-time + type: string + denied: + type: integer + failed: + type: integer + finance: + type: integer + high_risk: + type: integer + partial: + type: integer + security: + type: integer + total: + type: integer + unknown: + type: integer + type: object AuditSubjectActivity: properties: action_code: @@ -1594,6 +1822,10 @@ components: reduction_pct: description: 展示增幅比例，公式为(real_total_mb / virtual_total_mb) - 1 type: number + renewal_price: + description: 当前主套餐续费价格（分，按当前销售渠道的生效零售价计算；无主套餐或当前渠道不可续费时为 null） + nullable: true + type: integer sn: description: 设备序列号 type: string @@ -15253,6 +15485,73 @@ paths: summary: 查询操作者行为时间线 tags: - 审计调查 + /api/admin/audit/correlations/{correlation_id}/timeline: + get: + description: correlation_id 来自审计、外部集成、可靠事件或业务详情的稳定引用。相同 correlation 只说明属于同一业务链路，不用于猜测技术重试。 + parameters: + - description: 跨请求、异步任务和外部交互的稳定业务链路ID + in: path + name: correlation_id + required: true + schema: + description: 跨请求、异步任务和外部交互的稳定业务链路ID + type: string + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/AuditLinkTimeline' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 查询业务关联时间线 + tags: + - 审计调查 /api/admin/audit/events: get: description: 筛选值来自调查人员输入或其他调查节点的稳定引用；身份范围只读取认证上下文。固定按发生时间和事件ID倒序，不提供导出、修改或删除。 @@ -15491,6 +15790,190 @@ paths: summary: 查询审计事件详情 tags: - 审计调查 + /api/admin/audit/finance/timeline: + get: + description: 可使用店铺、钱包、订单、支付、退款、充值、审批、交易号、操作者、时间或 correlation 中任一稳定条件进入；关联事实由服务端解析，金额以钱包流水及对应业务表为权威。 + parameters: + - description: 店铺ID + in: query + name: shop_id + schema: + description: 店铺ID + minimum: 0 + type: integer + - description: 代理或资产钱包ID + in: query + name: wallet_id + schema: + description: 代理或资产钱包ID + minimum: 0 + type: integer + - description: 订单ID + in: query + name: order_id + schema: + description: 订单ID + minimum: 0 + type: integer + - description: 订单编号 + in: query + name: order_no + schema: + description: 订单编号 + type: string + - description: 支付记录ID + in: query + name: payment_id + schema: + description: 支付记录ID + minimum: 0 + type: integer + - description: 支付单号 + in: query + name: payment_no + schema: + description: 支付单号 + type: string + - description: 退款单ID + in: query + name: refund_id + schema: + description: 退款单ID + minimum: 0 + type: integer + - description: 退款单号 + in: query + name: refund_no + schema: + description: 退款单号 + type: string + - description: 代理充值或个人资产充值ID + in: query + name: recharge_id + schema: + description: 代理充值或个人资产充值ID + minimum: 0 + type: integer + - description: 充值单号 + in: query + name: recharge_no + schema: + description: 充值单号 + type: string + - description: 审批实例ID + in: query + name: approval_instance_id + schema: + description: 审批实例ID + minimum: 0 + type: integer + - description: 第三方交易号 + in: query + name: third_party_trade_no + schema: + description: 第三方交易号 + type: string + - description: 操作者类型；与actor_id同时提供 + in: query + name: actor_kind + schema: + description: 操作者类型；与actor_id同时提供 + type: string + - description: 操作者稳定ID；与actor_kind同时提供 + in: query + name: actor_id + schema: + description: 操作者稳定ID；与actor_kind同时提供 + type: string + - description: 跨步骤业务链路ID + in: query + name: correlation_id + schema: + description: 跨步骤业务链路ID + type: string + - description: 开始时间（RFC3339，含时区） + in: query + name: created_from + schema: + description: 开始时间（RFC3339，含时区） + type: string + - description: 结束时间（RFC3339，含时区，不包含该时刻） + in: query + name: created_to + schema: + description: 结束时间（RFC3339，含时区，不包含该时刻） + type: string + - description: 页码，默认1 + in: query + name: page + schema: + description: 页码，默认1 + minimum: 1 + type: integer + - description: 每页数量，默认20，最大100 + in: query + name: page_size + schema: + description: 每页数量，默认20，最大100 + maximum: 100 + minimum: 1 + type: integer + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/AuditFinanceTimelinePage' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 查询资金调查时间线 + tags: + - 审计调查 /api/admin/audit/integrations: get: description: 组合筛选来自调查输入或关联视角稳定引用，固定按创建时间和记录ID倒序分页，不提供任意摘要搜索。 @@ -15951,6 +16434,73 @@ paths: summary: 查询外部集成交互总览 tags: - 审计调查 + /api/admin/audit/requests/{request_id}/timeline: + get: + description: request_id 来自审计或外部集成节点，也可由开发人员从 Access Log 粘贴。响应组合审计事件、外部交互、可靠事件和业务账本引用，但不扫描 Access Log 文件。 + parameters: + - description: HTTP请求关联ID，来自审计或外部集成节点，也可从Access Log粘贴 + in: path + name: request_id + required: true + schema: + description: HTTP请求关联ID，来自审计或外部集成节点，也可从Access Log粘贴 + type: string + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/AuditLinkTimeline' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 查询请求关联时间线 + tags: + - 审计调查 /api/admin/audit/resources/{resource_type}/{resource_id}/timeline: get: description: resource_type/resource_id 必须来自业务页面稳定字段、资源搜索结果或 investigation_refs。事件在资源作为 primary、affected 或 reference 时均会返回。 @@ -16153,6 +16703,217 @@ paths: summary: 精确搜索注册资源 tags: - 审计调查 + /api/admin/audit/risks/events: + get: + description: 筛选条件来自风险总览分桶或调查人员输入，明细返回统一 investigation_refs，可继续进入事件、资源、操作者和 correlation 视角；不提供处置或封禁能力。 + parameters: + - description: 开始时间（RFC3339，含时区，必填） + in: query + name: created_from + required: true + schema: + description: 开始时间（RFC3339，含时区，必填） + type: string + - description: 结束时间（RFC3339，含时区，不包含该时刻，最长31天，必填） + in: query + name: created_to + required: true + schema: + description: 结束时间（RFC3339，含时区，不包含该时刻，最长31天，必填） + type: string + - description: 风险等级 (low:低, normal:普通, high:高, critical:严重) + in: query + name: risk + schema: + description: 风险等级 (low:低, normal:普通, high:高, critical:严重) + type: string + - description: 结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知) + in: query + name: result + schema: + description: 结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知) + type: string + - description: 稳定动作编码 + in: query + name: action + schema: + description: 稳定动作编码 + type: string + - description: 来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调) + in: query + name: source + schema: + description: 来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调) + type: string + - description: 页码，默认1 + in: query + name: page + schema: + description: 页码，默认1 + minimum: 1 + type: integer + - description: 每页数量，默认20，最大100 + in: query + name: page_size + schema: + description: 每页数量，默认20，最大100 + maximum: 100 + minimum: 1 + type: integer + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/AuditRiskEventPage' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 查询风险事件明细 + tags: + - 审计调查 + /api/admin/audit/risks/overview: + get: + description: 必须提供最长31天的时间范围；只聚合高风险、资金、安全、失败、拒绝、部分成功和结果未知事件，普通低风险成功事件不计入。 + parameters: + - description: 开始时间（RFC3339，含时区，必填） + in: query + name: created_from + required: true + schema: + description: 开始时间（RFC3339，含时区，必填） + type: string + - description: 结束时间（RFC3339，含时区，不包含该时刻，最长31天，必填） + in: query + name: created_to + required: true + schema: + description: 结束时间（RFC3339，含时区，不包含该时刻，最长31天，必填） + type: string + - description: 风险等级 (low:低, normal:普通, high:高, critical:严重) + in: query + name: risk + schema: + description: 风险等级 (low:低, normal:普通, high:高, critical:严重) + type: string + - description: 结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知) + in: query + name: result + schema: + description: 结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知) + type: string + - description: 稳定动作编码 + in: query + name: action + schema: + description: 稳定动作编码 + type: string + - description: 来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调) + in: query + name: source + schema: + description: 来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调) + type: string + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/AuditRiskOverview' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 查询风险调查总览 + tags: + - 审计调查 /api/admin/authorizations: get: parameters: diff --git a/docs/feature-504-multi-view-audit-center/平台基础审计调查接口.md b/docs/feature-504-multi-view-audit-center/平台基础审计调查接口.md index 2bd7904..4e1d92b 100644 --- a/docs/feature-504-multi-view-audit-center/平台基础审计调查接口.md +++ b/docs/feature-504-multi-view-audit-center/平台基础审计调查接口.md @@ -11,6 +11,10 @@ | 操作者 | `GET /api/admin/audit/actors/{kind}/{id}/events` | `investigation_refs.actor_ref` 或平台账号选择器 | | 资源搜索 | `GET /api/admin/audit/resources/search` | 调查人员选择类型并输入精确业务标识 | | 资源时间线 | `GET /api/admin/audit/resources/{resource_type}/{resource_id}/timeline` | 业务响应稳定 ID、资源搜索结果或 `investigation_refs.resource_refs[]` | +| 请求链路 | `GET /api/admin/audit/requests/{request_id}/timeline` | 节点 `request_id` 或开发人员从 Access Log 粘贴 | +| 业务链路 | `GET /api/admin/audit/correlations/{correlation_id}/timeline` | 节点或业务详情中的稳定 `correlation_id` | +| 资金时间线 | `GET /api/admin/audit/finance/timeline` | 店铺、钱包、订单、支付、退款、充值、审批、交易号、操作者、时间或 correlation | +| 风险总览/明细 | `GET /api/admin/audit/risks/overview`、`GET /api/admin/audit/risks/events` | 调查时间范围及风险分桶稳定筛选 | 认证身份和数据范围只来自认证上下文，不接受 query/path/body 伪造。接口仅允许超级管理员和平台账号访问，全部为 GET，不提供导出、修改、删除、恢复或处置能力。 @@ -43,7 +47,7 @@ | 店铺主钱包流水 | path `shop_id`、资产类型与 ID | 资金视角使用 `shop_id`；资产 ID 存在时使用对应资源时间线 | | 资产钱包流水 | 上层 `wallet_id` | 资金视角使用 `wallet_id`；不解析业务编号前缀猜测资源 | -资金视角将在后续任务交付；当前文档只冻结其第一跳参数，缺失的支付、退款、钱包等关联由服务端 Query 解析。 +资金视角已交付；缺失的支付、退款、钱包等关联由服务端 Query 解析。完整逐页面映射和调用链见[跨视角调查与前端导航契约](跨视角调查与前端导航契约.md)。 ## 统一调查引用 diff --git a/docs/feature-504-multi-view-audit-center/跨视角调查与前端导航契约.md b/docs/feature-504-multi-view-audit-center/跨视角调查与前端导航契约.md new file mode 100644 index 0000000..7bb0240 --- /dev/null +++ b/docs/feature-504-multi-view-audit-center/跨视角调查与前端导航契约.md @@ -0,0 +1,113 @@ +# 跨视角调查与前端导航契约 + +本文对应 `build-multi-view-audit-center` 任务 9.5，冻结 request、correlation、资金和风险调查接口，以及现有业务页面进入审计中心的第一跳。所有字段路径均位于统一响应的 `response.data` 下；列表字段位于 `items[]`。本文只定义前端调用契约，不包含前端页面实现。 + +## 跨视角只读接口 + +| 视角 | 接口 | 入参来源 | 响应重点 | +|---|---|---|---| +| 请求链路 | `GET /api/admin/audit/requests/{request_id}/timeline` | 审计或 Integration 节点的 `investigation_refs.request_id`，或开发人员从 Access Log 粘贴 | `request_id`、`access_log_lookup_request_id`、`nodes[]` | +| 业务链路 | `GET /api/admin/audit/correlations/{correlation_id}/timeline` | 审计、Integration、Outbox、任务或业务详情中的稳定 correlation | `correlation_id`、`nodes[]` | +| 资金时间线 | `GET /api/admin/audit/finance/timeline` | 业务页面稳定 ID、调查节点引用或调查人员输入 | 分页 `items[]`、事实来源、金额权威、`investigation_refs` | +| 风险总览 | `GET /api/admin/audit/risks/overview` | 调查人员选择的 RFC3339 时间范围及可选筛选，最长 31 天 | 信号、风险、结果、动作、来源和趋势 | +| 风险明细 | `GET /api/admin/audit/risks/events` | 风险总览分桶携带相同筛选，或调查人员输入 | 分页风险事件及 `investigation_refs` | + +以上接口仅允许超级管理员和平台账号访问，全部为 GET。认证身份只来自认证上下文；不提供导出、修改、删除、风险处置、自动封禁、重试、补偿或恢复能力。 + +## 资产和组织页面逐行导航 + +| 源页面 | 前置接口 | `response.data` 稳定字段 | 入口名称与可见条件 | 目标接口与参数映射 | 降级行为 | +|---|---|---|---|---|---| +| 卡列表 | `GET /api/admin/iot-cards/standalone` | `items[].id/iccid/virtual_no/shop_id/device_virtual_no/authorized_enterprise_id` | 平台显示“审计记录”，要求 `id` 非零；代理显示“活动记录”，要求 `iccid` 非空；企业不从此列表进入 | 平台：`/audit/resources/iot_card/{id}/timeline`；代理：`/agent/resource-activities/iot_card/{iccid}` | 缺少对应 ID/ICCID 时隐藏；企业改从企业卡列表进入 | +| 设备列表 | `GET /api/admin/devices` | `items[].id/virtual_no/imei/sn/shop_id/bound_card_count/authorized_enterprise_id` | 平台要求 `id`；代理要求 `virtual_no`；企业不从此列表进入 | 平台：`/audit/resources/device/{id}/timeline`；代理：`/agent/resource-activities/device/{virtual_no}` | 缺少字段时隐藏；企业改从企业设备列表进入 | +| 设备卡槽 | `GET /api/admin/devices/{virtual_no}/cards` | `bindings[].id/iot_card_id/iccid/slot_position/is_current` | 平台可分别查看卡和绑定审计，要求相应 ID；代理可查看卡活动，要求 ICCID；企业仅在设备和卡均有效授权时显示卡活动 | 平台卡：`iot_card/{iot_card_id}`；平台绑定：`device_sim_binding/{bindings[].id}`；代理/企业卡：`iot_card/{iccid}` 的主体活动接口 | 不用绑定 ID 证明授权；设备时间线继续使用上层设备 ID/VirtualNo | +| 统一资产详情 | `GET /api/admin/assets/resolve/{identifier}` | `asset_type/asset_id/identifier/virtual_no/iccid/bound_device_id/cards[].card_id/exchange_trace[].asset_id/can_view` | 平台在 `asset_id` 存在时显示“审计记录”；代理在卡 ICCID 或设备 VirtualNo 存在时显示“活动记录”；企业不显示 | 平台：`card→iot_card/{asset_id}`、`device→device/{asset_id}`；代理：卡用 `iot_card/{iccid}`、设备用 `device/{virtual_no}` | 缺稳定字段时隐藏；换货轨迹仅 `can_view=true` 且资产 ID 存在时跳转；企业不得回退调用 resolve | +| 资产分配列表/详情 | `GET /api/admin/asset-allocation-records[/{id}]` | `items[].id/allocation_no/asset_type/asset_id/asset_identifier/from_owner_type/from_owner_id/to_owner_type/to_owner_id/related_device_id`；详情另有 `related_card_ids[]` | 平台要求记录或资产 ID；代理要求 `allocation_no` 且当前关联资产或店铺仍在范围内；企业无独立入口 | 平台：`asset_allocation_record/{id}` 及 `{asset_type}/{asset_id}`；代理：`/agent/resource-activities/asset_allocation_record/{allocation_no}` | 后端独立复核分配归属；不能因可打开旧详情就视为有权；企业从已授权资产活动查看结论 | +| 换货列表/详情 | `GET /api/admin/exchanges[/{id}]` | `items[].id/exchange_no/old_asset_type/old_asset_id/new_asset_type/new_asset_id/shop_id/submitter_id` | 平台要求换货或资产 ID；代理要求 `exchange_no` 且换货店铺仍在范围内；企业无独立入口 | 平台：`exchange_order/{id}`，旧新资产分别使用响应类型和 ID；代理：`/agent/resource-activities/exchange_order/{exchange_no}` | 旧新资产仅在各自仍可管理时开放；企业从有效授权资产活动查看结论 | +| 店铺列表/详情 | `GET /api/admin/shops[/{id}]` | `items[].id/shop_name/shop_code/parent_id/business_owner_account_id` 或详情同名字段 | 平台要求 `id`；代理要求 `shop_code` 且为自己或下级店铺；企业不显示 | 平台：`/audit/resources/shop/{id}/timeline`；代理：`/agent/resource-activities/shop/{shop_code}` | 缺字段或越权时隐藏/显示不可用，不搜索平台审计 | +| 企业列表 | `GET /api/admin/enterprises` | `items[].id/enterprise_name/enterprise_code/owner_shop_id` | 平台要求 `id`；代理要求 `enterprise_code` 且 owner shop 在范围内；企业自身不显示 | 平台：`enterprise/{id}`；代理：`/agent/resource-activities/enterprise/{enterprise_code}` | 当前没有企业详情接口，前端不得假设存在；企业 ID 不作为主体活动路径参数 | +| 企业卡列表 | `GET /api/admin/enterprises/{id}/cards` | `items[].id/iccid/virtual_no/device_id` | 平台要求 `id`；企业要求 `iccid` 且当前授权有效；代理不从此列表进入 | 平台：`iot_card/{id}`；企业：`/enterprise/resource-activities/iot_card/{iccid}` | 路由中的企业 ID 不作为授权证明；后端始终使用认证上下文复核 | +| 企业设备列表 | `GET /api/admin/enterprises/{id}/devices` | `items[].device_id/virtual_no` | 平台要求 `device_id`；企业要求 `virtual_no` 且当前授权有效；代理不从此列表进入 | 平台：`device/{device_id}`；企业：`/enterprise/resource-activities/device/{virtual_no}` | 字段为空或授权撤销时隐藏/显示活动不可用，不回退平台接口 | + +## 账号、交易和资金页面逐行导航 + +| 源页面 | 前置接口 | `response.data` 稳定字段 | 入口名称与可见条件 | 目标接口与参数映射 | 降级行为 | +|---|---|---|---|---|---| +| 账号列表/详情 | `GET /api/admin/accounts[/{id}]` | 列表 `items[].id`；详情 `id` | 平台在 ID 非零时显示“审计记录” | `/audit/resources/account/{id}/timeline` | 缺 ID 时隐藏，不按用户名搜索 | +| 店铺列表/详情 | `GET /api/admin/shops[/{id}]` | 列表 `items[].id`；详情 `id` | ID 非零时显示“审计记录”和“资金链路” | 审计：`shop/{id}`；资金：`/audit/finance/timeline?shop_id={id}` | 缺 ID 时两个入口均隐藏 | +| 企业列表 | `GET /api/admin/enterprises` | `items[].id` | ID 非零时显示“审计记录” | `/audit/resources/enterprise/{id}/timeline` | 不假设存在企业详情接口 | +| 订单列表/详情 | `GET /api/admin/orders[/{id}]` | `items[].id/order_no` 或详情 `id/order_no` | ID 非零时显示“审计记录”和“资金链路” | 审计：`order/{id}`；资金：`/audit/finance/timeline?order_id={id}` | 不要求前端补 payment/refund ID；缺 ID 时隐藏 | +| 退款列表/详情 | `GET /api/admin/refunds[/{id}]` | `items[].id/refund_no/order_id/approval_instance_id` 或详情同名字段 | ID 非零时显示审计和资金入口；审批 ID 非零时显示“审批审计” | 审计：`refund/{id}`；资金：`finance/timeline?refund_id={id}`；审批：`approval_instance/{approval_instance_id}` | 缺审批 ID 只隐藏审批入口，不解析退款编号猜测 | +| 代理充值列表/详情 | `GET /api/admin/agent-recharges[/{id}]` | `items[].id/recharge_no/shop_id/agent_wallet_id/approval_instance_id` 或详情同名字段 | ID 非零时显示审计和资金入口；审批 ID 非零时显示审批审计 | 审计：`agent_recharge/{id}`；资金：`finance/timeline?recharge_id={id}` | 缺 `payment_no` 由服务端关联，不要求前端补猜 | +| 代理在线充值结果 | `POST /api/admin/agent-recharges` | `recharge_id/recharge_no/payment_no` | `recharge_id` 非零时显示“资金链路” | `finance/timeline?recharge_id={recharge_id}`；可附加 `payment_no` 精确筛选 | `payment_no` 不直接构造 Integration 详情；缺 recharge ID 时隐藏 | +| 资产钱包 | `GET /api/admin/assets/{identifier}/wallet` | `wallet_id/resource_type/resource_id` | `wallet_id` 非零时显示“资金链路”；资源类型和 ID 齐全时显示“资产审计” | 资金：`finance/timeline?wallet_id={wallet_id}`；审计：`resources/{resource_type}/{resource_id}/timeline` | 两个入口独立判断；缺某组字段只隐藏对应入口 | +| 店铺资金概况 | `GET /api/admin/shops/fund-summary` | `items[].shop_id` | `shop_id` 非零时显示行内“资金链路” | `finance/timeline?shop_id={shop_id}` | 不要求该接口未返回的 agent wallet ID | +| 店铺主钱包流水 | `GET /api/admin/shops/{shop_id}/main-wallet/transactions` | path `shop_id`；`items[].id/asset_type/asset_id/asset_identifier` | 始终可按合法 path 显示资金入口；资产类型和 ID 齐全时显示资产审计 | 资金：`finance/timeline?shop_id={shop_id}`；审计：`resources/{asset_type}/{asset_id}/timeline` | 缺资产 ID 仍保留店铺资金入口，不按资产编号猜测 | +| 资产钱包流水 | `GET /api/admin/assets/{identifier}/wallet/transactions` | 上层钱包接口 `wallet_id`；`items[].id/reference_type/reference_no` | 上层 `wallet_id` 非零时显示“资金链路” | `finance/timeline?wallet_id={wallet_id}` | `reference_type/reference_no` 仅展示；需后端节点明确返回资源引用后才能继续跳转 | + +## 调查节点逐行跳转 + +| `investigation_refs` 字段 | 入口名称 | 可见条件 | 目标接口与参数 | 降级行为 | +|---|---|---|---|---| +| `event_id` | “事件详情” | 非空 | `GET /api/admin/audit/events/{event_id}` | 空值隐藏 | +| `actor_ref.kind/id` | “查看操作者行为” | kind 和 id 均非空 | `GET /api/admin/audit/actors/{kind}/{id}/events` | 任一缺失即隐藏，不用当前账号资料补齐 | +| `resource_refs[]` | “查看资源审计” | `resource_type/resource_id` 均非空 | `GET /api/admin/audit/resources/{resource_type}/{resource_id}/timeline` | 只有 Key 时先精确搜索；零或多命中不自动选择 | +| `request_id` | “查看请求链路” | 非空 | `GET /api/admin/audit/requests/{request_id}/timeline` | 空值隐藏，不扫描 Access Log 猜测 | +| `correlation_id` | “查看业务链路” | 非空 | `GET /api/admin/audit/correlations/{correlation_id}/timeline` | 空值隐藏，不按相近时间拼链路 | +| `integration_refs[].integration_id` | “查看外部交互” | 非空 | `GET /api/admin/audit/integrations/{integration_id}` | 空值隐藏，不使用数据库主键或相似资源猜测 | + +`actor_ref.kind` 第一阶段只使用 `account/openapi/system_task/scheduled_job/external_system`。代理和企业活动响应不得包含 `investigation_refs`。 + +## 六条完整调用链 + +### 资产详情 + +1. 调用 `GET /api/admin/assets/resolve/{identifier}`。 +2. 平台读取 `data.asset_type/asset_id`：卡将 `card` 转为 `iot_card`，调用 `GET /api/admin/audit/resources/iot_card/{asset_id}/timeline?page=1&page_size=20`；设备调用 `.../device/{asset_id}/timeline`。 +3. 代理读取 `data.iccid` 或 `data.virtual_no`，调用对应 `/agent/resource-activities/...`。 +4. 字段缺失时隐藏入口；企业不调用 resolve，也不回退平台接口。 + +### 订单 + +1. 调用 `GET /api/admin/orders/{id}`，读取 `data.id`。 +2. “审计记录”调用 `GET /api/admin/audit/resources/order/{id}/timeline`。 +3. “资金链路”调用 `GET /api/admin/audit/finance/timeline?order_id={id}&page=1&page_size=20`。 +4. Payment、Refund、钱包等关联由服务端解析，前端不补猜。 + +### 退款 + +1. 调用 `GET /api/admin/refunds/{id}`，读取 `data.id` 和可选 `data.approval_instance_id`。 +2. 审计调用 `resources/refund/{id}/timeline`，资金调用 `finance/timeline?refund_id={id}`。 +3. 审批实例 ID 非零时再调用 `resources/approval_instance/{approval_instance_id}/timeline`。 +4. 审批字段缺失只隐藏审批入口，不影响退款审计与资金链路。 + +### 钱包 + +1. 调用 `GET /api/admin/assets/{identifier}/wallet`，读取 `data.wallet_id/resource_type/resource_id`。 +2. “资金链路”调用 `finance/timeline?wallet_id={wallet_id}`。 +3. “资产审计”调用 `resources/{resource_type}/{resource_id}/timeline`。 +4. 两组稳定字段分别判断，不使用交易备注或 reference 编号前缀推断资源。 + +### 通知 + +1. 点击通知后先调用 `GET /api/admin/notifications/{id}/target`。 +2. 仅当 `data.available=true` 时展示跳转。 +3. `data.target_type=integration_log` 且 `data.target_key` 非空时，将 target key 原样作为 `integration_id` 调用 `GET /api/admin/audit/integrations/{target_key}`。 +4. 其他 target type 先进入对应业务详情，再按本文业务页面矩阵进入审计；不可用时只展示通知正文。 + +### 风险节点 + +1. 调用 `GET /api/admin/audit/risks/overview?created_from={from}&created_to={to}`。 +2. 点击风险、结果、动作或来源分桶时，将相同时间范围和对应稳定编码带入 `GET /api/admin/audit/risks/events`。 +3. 从明细 `items[].investigation_refs` 直接进入事件、操作者、资源、request、correlation 或 Integration 视角。 +4. 缺少的引用入口隐藏；风险中心不提供处置、封禁或恢复按钮。 + +## 统一降级与事实边界 + +- 缺少目标接口必需的稳定 ID 或 identifier 时隐藏入口，不按名称、中文描述、时间或编号前缀猜测。 +- 平台只有 Registry Key 时先调用精确资源搜索；零命中或多命中停留在搜索结果。 +- 已删除资源只要调查节点仍有稳定资源类型和 ID，就可查看事件快照时间线。 +- 代理或企业遇到越权、授权撤销或资源不存在时统一显示“活动不可用”，不回退平台调查、资源搜索或旧 operation log。 +- request/correlation 时间线的 `record_source` 保留 Audit Event、Integration Log、Outbox、Asynq 摘要和 Domain Ledger 引用的事实边界；Outbox 投递成功不等于业务成功。 +- 资金金额和余额以钱包流水及对应业务表为权威；Audit Event 仅用于解释谁做了什么，不用于资金重算。 +- 旧 operation log 仅由平台独立历史入口访问，不拼接到新 `/api/admin/audit/*`。 diff --git a/internal/application/auditarchive/integration.go b/internal/application/auditarchive/integration.go new file mode 100644 index 0000000..0b9f3e2 --- /dev/null +++ b/internal/application/auditarchive/integration.go @@ -0,0 +1,350 @@ +package auditarchive + +import ( + "compress/gzip" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/bytedance/sonic" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +type integrationArchiveFile struct { + path string + recordCount int64 + uncompressedBytes int64 + compressedBytes int64 + sha256 string +} + +type integrationArchiveManifest struct { + SchemaVersion string `json:"schema_version"` + Source string `json:"source"` + ArchiveDate string `json:"archive_date"` + Timezone string `json:"timezone"` + RangeStart time.Time `json:"range_start"` + RangeEnd time.Time `json:"range_end"` + InstanceID string `json:"instance_id"` + RecordCount int64 `json:"record_count"` + UncompressedBytes int64 `json:"uncompressed_bytes"` + CompressedBytes int64 `json:"compressed_bytes"` + ObjectKey string `json:"object_key"` + SHA256 string `json:"sha256"` + Revision int `json:"revision"` + GeneratedAt time.Time `json:"generated_at"` + Status string `json:"status"` + Final bool `json:"final"` +} + +// ArchivePreviousIntegrationDay 归档 Asia/Shanghai 前一完整自然日的 Integration Log 创建日快照。 +func (s *Service) ArchivePreviousIntegrationDay(ctx context.Context) error { + now := time.Now().In(s.location) + return s.ArchiveIntegrationDate(ctx, now.AddDate(0, 0, -1)) +} + +// ArchiveIntegrationDate 归档指定 Asia/Shanghai 自然日的 Integration Log 创建日快照。 +func (s *Service) ArchiveIntegrationDate(ctx context.Context, archiveDate time.Time) error { + return s.archiveIntegrationDate(ctx, archiveDate, false) +} + +// FinalizePreviousIntegrationMonth 复核并终结上一个完整自然月的 Integration Log 归档。 +func (s *Service) FinalizePreviousIntegrationMonth(ctx context.Context) error { + now := time.Now().In(s.location) + return s.FinalizeIntegrationMonth(ctx, now.AddDate(0, -1, 0)) +} + +// FinalizeIntegrationMonth 逐日复核指定完整自然月，并为变化内容创建最终 revision。 +func (s *Service) FinalizeIntegrationMonth(ctx context.Context, month time.Time) error { + monthStart := time.Date(month.In(s.location).Year(), month.In(s.location).Month(), 1, 0, 0, 0, 0, s.location) + currentMonth := time.Now().In(s.location) + currentMonthStart := time.Date(currentMonth.Year(), currentMonth.Month(), 1, 0, 0, 0, 0, s.location) + if !monthStart.Before(currentMonthStart) { + return fmt.Errorf("只能终结已经结束的 Integration Log 完整自然月") + } + for date := monthStart; date.Before(monthStart.AddDate(0, 1, 0)); date = date.AddDate(0, 0, 1) { + if err := s.archiveIntegrationDate(ctx, date, true); err != nil { + return fmt.Errorf("终结 %s Integration Log 归档失败: %w", date.Format(time.DateOnly), err) + } + } + return nil +} + +func (s *Service) archiveIntegrationDate(ctx context.Context, archiveDate time.Time, final bool) error { + if s.db == nil || s.store == nil { + return fmt.Errorf("Integration Log 归档数据库或对象存储未配置") + } + start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location) + end := start.AddDate(0, 0, 1) + today := time.Now().In(s.location) + if end.After(time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location)) { + return fmt.Errorf("Integration Log 只能归档已经结束的完整自然日") + } + + run, err := s.ensureIntegrationRun(ctx, start, end) + if err != nil { + return err + } + file, err := s.buildIntegrationArchiveFile(ctx, start, end) + if err != nil { + return err + } + defer os.Remove(file.path) + if final { + pending, pendingErr := s.integrationPendingCount(ctx, start, end) + if pendingErr != nil { + return pendingErr + } + if pending > 0 { + _ = s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID). + Updates(map[string]any{"is_final": false, "error_summary": "存在 pending Integration Log，无法形成最终归档", "updated_at": time.Now()}).Error + return fmt.Errorf("仍有 %d 条 pending Integration Log，无法形成最终归档", pending) + } + } + if run.Status == constants.ArchiveStatusSuccess && run.RecordCount == file.recordCount && run.SHA256 == file.sha256 { + valid, validateErr := s.validateIntegrationRun(ctx, run) + if validateErr == nil && valid && (!final || run.IsFinal) { + return nil + } + } + + acquired, err := s.acquireIntegrationRun(ctx, run) + if err != nil { + return err + } + if !acquired { + return fmt.Errorf("Integration Log 归档任务正在执行") + } + if err := s.uploadIntegrationArchive(ctx, run, file, final); err != nil { + s.markFailed(ctx, run.ID, err) + return err + } + return nil +} + +func (s *Service) ensureIntegrationRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) { + run := model.LogArchiveRun{ + Source: constants.IntegrationArchiveSource, ArchiveDate: start, InstanceID: s.instanceID, + SchemaVersion: constants.IntegrationArchiveSchemaVersion, Revision: 1, + Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end, + } + result := s.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}}, + DoNothing: true, + }).Create(&run) + if result.Error != nil { + return nil, fmt.Errorf("创建 Integration Log 归档账本失败: %w", result.Error) + } + if result.RowsAffected == 0 { + if err := s.db.WithContext(ctx).Where( + "source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?", + constants.IntegrationArchiveSource, start, s.instanceID, constants.IntegrationArchiveSchemaVersion, + ).First(&run).Error; err != nil { + return nil, fmt.Errorf("读取 Integration Log 归档账本失败: %w", err) + } + } + return &run, nil +} + +func (s *Service) acquireIntegrationRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) { + revision := run.Revision + if run.Status != constants.ArchiveStatusPending { + revision++ + } + now := time.Now() + result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, now.Add(-3*time.Hour)). + Updates(map[string]any{ + "status": constants.ArchiveStatusRunning, "revision": revision, "is_final": false, + "attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "", "completed_at": nil, "updated_at": now, + }) + if result.Error != nil { + return false, fmt.Errorf("锁定 Integration Log 归档任务失败: %w", result.Error) + } + if result.RowsAffected == 0 { + return false, nil + } + run.Revision = revision + return true, nil +} + +func (s *Service) buildIntegrationArchiveFile(ctx context.Context, start, end time.Time) (*integrationArchiveFile, error) { + temp, err := os.CreateTemp("", "integration-logs-*.jsonl.gz") + if err != nil { + return nil, fmt.Errorf("创建 Integration Log 归档临时文件失败: %w", err) + } + path := temp.Name() + failed := true + defer func() { + _ = temp.Close() + if failed { + _ = os.Remove(path) + } + }() + + hasher := sha256.New() + gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher)) + result := &integrationArchiveFile{path: path} + var lastID uint + for { + var logs []model.IntegrationLog + if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID). + Order("id ASC").Limit(archivePageSize).Find(&logs).Error; err != nil { + return nil, fmt.Errorf("读取 Integration Log 归档记录失败: %w", err) + } + if len(logs) == 0 { + break + } + for i := range logs { + line, marshalErr := sonic.Marshal(logs[i]) + if marshalErr != nil { + return nil, fmt.Errorf("序列化 Integration Log 归档记录失败: %w", marshalErr) + } + line = append(line, '\n') + if _, writeErr := gzipWriter.Write(line); writeErr != nil { + return nil, fmt.Errorf("写入 Integration Log 归档压缩流失败: %w", writeErr) + } + result.recordCount++ + result.uncompressedBytes += int64(len(line)) + } + lastID = logs[len(logs)-1].ID + } + if err := gzipWriter.Close(); err != nil { + return nil, fmt.Errorf("关闭 Integration Log 归档压缩流失败: %w", err) + } + if err := temp.Close(); err != nil { + return nil, fmt.Errorf("关闭 Integration Log 归档临时文件失败: %w", err) + } + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("读取 Integration Log 归档临时文件信息失败: %w", err) + } + result.compressedBytes = info.Size() + result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil)) + failed = false + return result, nil +} + +func (s *Service) uploadIntegrationArchive(ctx context.Context, run *model.LogArchiveRun, file *integrationArchiveFile, final bool) error { + count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd) + if err != nil { + return err + } + if count != file.recordCount { + return fmt.Errorf("Integration Log 归档生成期间记录数量发生变化") + } + objectKey, manifestKey := integrationObjectKeys(run.RangeStart, run.Revision) + metadata := integrationArchiveMetadata(file, run, final) + reader, err := os.Open(file.path) + if err != nil { + return fmt.Errorf("打开 Integration Log 归档临时文件失败: %w", err) + } + uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata) + closeErr := reader.Close() + if uploadErr != nil { + return fmt.Errorf("上传 Integration Log 归档对象失败: %w", uploadErr) + } + if closeErr != nil { + return fmt.Errorf("关闭 Integration Log 归档临时文件失败: %w", closeErr) + } + if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil { + return err + } + + generatedAt := time.Now().In(s.location) + manifest := integrationArchiveManifest{ + SchemaVersion: constants.IntegrationArchiveSchemaVersion, Source: constants.IntegrationArchiveSource, + ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone, + RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID, + RecordCount: file.recordCount, UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes, + ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision, GeneratedAt: generatedAt, + Status: constants.ArchiveStatusSuccess, Final: final, + } + manifestBytes, err := sonic.Marshal(manifest) + if err != nil { + return fmt.Errorf("序列化 Integration Log 归档清单失败: %w", err) + } + manifestMetadata := map[string]string{ + "source": constants.IntegrationArchiveSource, "data-sha256": file.sha256, + "revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(final), + } + if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil { + return fmt.Errorf("上传 Integration Log 归档清单失败: %w", err) + } + if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil { + return err + } + + completedAt := time.Now() + return s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(map[string]any{ + "status": constants.ArchiveStatusSuccess, "is_final": final, + "object_key": objectKey, "manifest_key": manifestKey, "record_count": file.recordCount, + "uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes, + "sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt, "updated_at": completedAt, + }).Error +} + +func (s *Service) validateIntegrationRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) { + metadata := map[string]string{ + "sha256": run.SHA256, "record-count": strconv.FormatInt(run.RecordCount, 10), + "revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(run.IsFinal), + } + if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil { + return false, nil + } + manifestMetadata := map[string]string{ + "source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256, + "revision": strconv.Itoa(run.Revision), "final": strconv.FormatBool(run.IsFinal), + } + if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil { + return false, nil + } + return true, nil +} + +func (s *Service) integrationRecordCount(ctx context.Context, start, end time.Time) (int64, error) { + var count int64 + if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}). + Where("created_at >= ? AND created_at < ?", start, end).Count(&count).Error; err != nil { + return 0, fmt.Errorf("统计 Integration Log 归档记录失败: %w", err) + } + return count, nil +} + +func (s *Service) integrationPendingCount(ctx context.Context, start, end time.Time) (int64, error) { + var count int64 + if err := s.db.WithContext(ctx).Model(&model.IntegrationLog{}). + Where("created_at >= ? AND created_at < ? AND result = ?", start, end, constants.IntegrationResultPending). + Count(&count).Error; err != nil { + return 0, fmt.Errorf("统计 pending Integration Log 失败: %w", err) + } + return count, nil +} + +func integrationObjectKeys(date time.Time, revision int) (string, string) { + prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day()) + name := fmt.Sprintf("integration-logs-%s-r%d", date.Format(time.DateOnly), revision) + return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json" +} + +func integrationArchiveMetadata(file *integrationArchiveFile, run *model.LogArchiveRun, final bool) map[string]string { + return map[string]string{ + "schema-version": constants.IntegrationArchiveSchemaVersion, + "source": constants.IntegrationArchiveSource, + "archive-date": run.RangeStart.Format(time.DateOnly), + "timezone": constants.AuditArchiveTimezone, + "record-count": strconv.FormatInt(file.recordCount, 10), + "sha256": file.sha256, + "revision": strconv.Itoa(run.Revision), + "final": strconv.FormatBool(final), + } +} diff --git a/internal/application/auditarchive/retention.go b/internal/application/auditarchive/retention.go new file mode 100644 index 0000000..84e746c --- /dev/null +++ b/internal/application/auditarchive/retention.go @@ -0,0 +1,542 @@ +package auditarchive + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/bytedance/sonic" + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +const maxManifestBytes = 1024 * 1024 + +// RetentionAudit 描述月度物理清理的统一审计事实。 +type RetentionAudit struct { + EventID string + Month string + Summary string + Result string + ErrorSummary string + RangeStart time.Time + RangeEnd time.Time + EventCount int64 + ResourceCount int64 + IntegrationCount int64 + ManifestKeys []string + DurationMS int64 +} + +// RetentionResult 是月度留存清理的结构化执行结果。 +type RetentionResult struct { + Month string + EventCount int64 + ResourceCount int64 + IntegrationCount int64 + ManifestKeys []string + Duration time.Duration +} + +type retentionRuns struct { + audit []*model.LogArchiveRun + integration []*model.LogArchiveRun +} + +// CleanupPreviousMonth 校验并物理清理上一个完整自然月的在线审计日志。 +func (s *Service) CleanupPreviousMonth(ctx context.Context) (RetentionResult, error) { + now := time.Now().In(s.location) + return s.CleanupMonth(ctx, now.AddDate(0, -1, 0)) +} + +// CleanupMonth 校验归档硬门禁后按固定顺序物理清理指定完整自然月。 +func (s *Service) CleanupMonth(ctx context.Context, month time.Time) (result RetentionResult, cleanupErr error) { + if s.db == nil || s.store == nil || s.audit == nil { + return result, fmt.Errorf("日志留存清理数据库、对象存储或审计 Writer 未配置") + } + start, end, err := s.retentionMonthRange(month) + if err != nil { + return result, err + } + startedAt := time.Now() + result.Month = start.Format("2006-01") + cleanupErr = s.executeRetention(ctx, start, end, &result) + result.Duration = time.Since(startedAt) + if auditErr := s.recordRetentionAudit(ctx, start, end, result, cleanupErr); auditErr != nil { + if cleanupErr != nil { + return result, fmt.Errorf("%w；记录留存清理失败审计失败: %v", cleanupErr, auditErr) + } + return result, fmt.Errorf("记录留存清理成功审计失败: %w", auditErr) + } + return result, cleanupErr +} + +func (s *Service) retentionMonthRange(month time.Time) (time.Time, time.Time, error) { + start := time.Date(month.In(s.location).Year(), month.In(s.location).Month(), 1, 0, 0, 0, 0, s.location) + end := start.AddDate(0, 1, 0) + now := time.Now().In(s.location) + currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, s.location) + if !end.Before(currentMonth) && !end.Equal(currentMonth) { + return time.Time{}, time.Time{}, fmt.Errorf("只能清理已经结束的完整自然月") + } + return start, end, nil +} + +func (s *Service) executeRetention(ctx context.Context, start, end time.Time, result *RetentionResult) error { + started, err := s.retentionCleanupStarted(ctx, start, end) + if err != nil { + return err + } + if !started { + lastDay := end.AddDate(0, 0, -1) + if err := s.ArchiveDate(ctx, lastDay); err != nil { + return fmt.Errorf("完成上月最后一天 Audit 归档失败: %w", err) + } + if err := s.ArchiveIntegrationDate(ctx, lastDay); err != nil { + return fmt.Errorf("完成上月最后一天 Integration Log 归档失败: %w", err) + } + if err := s.FinalizeIntegrationMonth(ctx, start); err != nil { + return err + } + } + runs, err := s.loadRetentionRuns(ctx, start, end) + if err != nil { + return err + } + if err := s.validateRetentionRuns(ctx, start, end, runs); err != nil { + return err + } + summarizeRetentionRuns(runs, result) + if err := s.cleanupAuditMonth(ctx, start, end, runs.audit); err != nil { + return err + } + return s.cleanupIntegrationMonth(ctx, start, end, runs.integration) +} + +func (s *Service) retentionCleanupStarted(ctx context.Context, start, end time.Time) (bool, error) { + var count int64 + err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NOT NULL", + start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID). + Count(&count).Error + if err != nil { + return false, fmt.Errorf("读取月度清理断点失败: %w", err) + } + return count > 0, nil +} + +func (s *Service) loadRetentionRuns(ctx context.Context, start, end time.Time) (retentionRuns, error) { + var rows []model.LogArchiveRun + err := s.db.WithContext(ctx).Where( + "source IN ? AND archive_date >= ? AND archive_date < ? AND instance_id = ?", + []string{constants.AuditArchiveSource, constants.IntegrationArchiveSource}, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID, + ).Order("archive_date ASC, source ASC").Find(&rows).Error + if err != nil { + return retentionRuns{}, fmt.Errorf("读取月度归档账本失败: %w", err) + } + days := int(end.Sub(start).Hours() / 24) + if len(rows) != days*2 { + return retentionRuns{}, fmt.Errorf("月度归档账本缺日：期望 %d 条，实际 %d 条", days*2, len(rows)) + } + runs := retentionRuns{audit: make([]*model.LogArchiveRun, 0, days), integration: make([]*model.LogArchiveRun, 0, days)} + for index := range rows { + run := &rows[index] + switch run.Source { + case constants.AuditArchiveSource: + runs.audit = append(runs.audit, run) + case constants.IntegrationArchiveSource: + runs.integration = append(runs.integration, run) + } + } + if len(runs.audit) != days || len(runs.integration) != days { + return retentionRuns{}, fmt.Errorf("月度 Audit 或 Integration 归档账本不完整") + } + return runs, nil +} + +func (s *Service) validateRetentionRuns(ctx context.Context, start, end time.Time, runs retentionRuns) error { + if err := validateCleanupLedgerState(runs.audit); err != nil { + return fmt.Errorf("Audit 清理断点非法: %w", err) + } + if err := validateCleanupLedgerState(runs.integration); err != nil { + return fmt.Errorf("Integration 清理断点非法: %w", err) + } + for index := range runs.audit { + date := start.AddDate(0, 0, index) + if err := s.validateAuditRetentionDay(ctx, date, runs.audit[index]); err != nil { + return fmt.Errorf("%s Audit 清理门禁失败: %w", date.Format(time.DateOnly), err) + } + if err := s.validateIntegrationRetentionDay(ctx, date, runs.integration[index]); err != nil { + return fmt.Errorf("%s Integration 清理门禁失败: %w", date.Format(time.DateOnly), err) + } + } + return nil +} + +func validateCleanupLedgerState(runs []*model.LogArchiveRun) error { + started, cleaned := 0, 0 + for _, run := range runs { + if run.CleanupStartedAt != nil { + started++ + } + if run.CleanedAt != nil { + cleaned++ + } + } + if started != 0 && started != len(runs) { + return fmt.Errorf("清理开始断点不是整月原子状态") + } + if cleaned != 0 && cleaned != len(runs) { + return fmt.Errorf("清理完成断点不是整月原子状态") + } + if cleaned > 0 && started == 0 { + return fmt.Errorf("清理完成但缺少开始断点") + } + return nil +} + +func (s *Service) validateAuditRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error { + if err := validateRunBase(run, date, constants.AuditArchiveSchemaVersion, false); err != nil { + return err + } + if err := s.validateAuditManifest(ctx, run); err != nil { + return err + } + events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd) + if err != nil { + return err + } + return validateRemainingCounts(run, events, resources) +} + +func (s *Service) validateIntegrationRetentionDay(ctx context.Context, date time.Time, run *model.LogArchiveRun) error { + if err := validateRunBase(run, date, constants.IntegrationArchiveSchemaVersion, true); err != nil { + return err + } + if err := s.validateIntegrationManifest(ctx, run); err != nil { + return err + } + count, err := s.integrationRecordCount(ctx, run.RangeStart, run.RangeEnd) + if err != nil { + return err + } + if run.CleanedAt != nil { + if count != 0 { + return fmt.Errorf("已标记清理完成但数据库仍有 %d 条记录", count) + } + return nil + } + if run.CleanupStartedAt != nil { + if count > run.RecordCount { + return fmt.Errorf("续跑窗口记录数超过最终归档数量") + } + return nil + } + file, err := s.buildIntegrationArchiveFile(ctx, run.RangeStart, run.RangeEnd) + if err != nil { + return err + } + defer os.Remove(file.path) + if file.recordCount != run.RecordCount || file.sha256 != run.SHA256 { + return fmt.Errorf("数据库当前 Integration 内容与最终 revision 不一致") + } + return nil +} + +func validateRunBase(run *model.LogArchiveRun, date time.Time, schema string, final bool) error { + if run.Status != constants.ArchiveStatusSuccess || run.SchemaVersion != schema { + return fmt.Errorf("归档状态或 schema version 不符合清理要求") + } + if run.ArchiveDate.Format(time.DateOnly) != date.Format(time.DateOnly) || + !run.RangeStart.Equal(date) || !run.RangeEnd.Equal(date.AddDate(0, 0, 1)) { + return fmt.Errorf("归档日期或半开时间范围不一致") + } + if final && !run.IsFinal { + return fmt.Errorf("Integration 最终 revision 尚未形成") + } + if run.ObjectKey == "" || run.ManifestKey == "" || run.SHA256 == "" { + return fmt.Errorf("归档对象、manifest 或 SHA-256 缺失") + } + return nil +} + +func validateRemainingCounts(run *model.LogArchiveRun, events, resources int64) error { + if run.CleanedAt != nil { + if events != 0 || resources != 0 { + return fmt.Errorf("已标记清理完成但数据库仍有事件或资源") + } + return nil + } + if run.CleanupStartedAt != nil { + if events > run.EventCount || resources > run.ResourceCount { + return fmt.Errorf("续跑窗口数量超过已归档数量") + } + return nil + } + if events != run.EventCount || resources != run.ResourceCount { + return fmt.Errorf("数据库事件或资源数量与 manifest 不一致") + } + return nil +} + +func (s *Service) validateAuditManifest(ctx context.Context, run *model.LogArchiveRun) error { + var manifest Manifest + if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil { + return err + } + if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || + manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone || + manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) || + manifest.EventCount != run.EventCount || manifest.ResourceCount != run.ResourceCount || + manifest.CompressedBytes != run.CompressedBytes || manifest.ObjectKey != run.ObjectKey || + manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision { + return fmt.Errorf("Audit manifest 与 ledger 不一致") + } + if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{ + "source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision), + }); err != nil { + return err + } + return s.verifyRetentionObject(ctx, run, false) +} + +func (s *Service) validateIntegrationManifest(ctx context.Context, run *model.LogArchiveRun) error { + var manifest integrationArchiveManifest + if err := s.readManifest(ctx, run.ManifestKey, &manifest); err != nil { + return err + } + if manifest.Source != run.Source || manifest.SchemaVersion != run.SchemaVersion || manifest.Status != constants.ArchiveStatusSuccess || !manifest.Final || + manifest.ArchiveDate != run.ArchiveDate.Format(time.DateOnly) || manifest.Timezone != constants.AuditArchiveTimezone || + manifest.InstanceID != run.InstanceID || !manifest.RangeStart.Equal(run.RangeStart) || !manifest.RangeEnd.Equal(run.RangeEnd) || + manifest.RecordCount != run.RecordCount || manifest.CompressedBytes != run.CompressedBytes || + manifest.ObjectKey != run.ObjectKey || manifest.SHA256 != run.SHA256 || manifest.Revision != run.Revision { + return fmt.Errorf("Integration manifest 与最终 ledger 不一致") + } + if err := s.verifyObject(ctx, run.ManifestKey, -1, map[string]string{ + "source": constants.IntegrationArchiveSource, "data-sha256": run.SHA256, + "revision": strconv.Itoa(run.Revision), "final": "true", + }); err != nil { + return err + } + return s.verifyRetentionObject(ctx, run, true) +} + +func (s *Service) readManifest(ctx context.Context, key string, target any) error { + object, err := s.store.Stat(ctx, key) + if err != nil { + return fmt.Errorf("读取 manifest metadata 失败: %w", err) + } + reader, err := s.store.Download(ctx, key) + if err != nil { + return fmt.Errorf("下载 manifest 失败: %w", err) + } + data, readErr := io.ReadAll(io.LimitReader(reader, maxManifestBytes+1)) + closeErr := reader.Close() + if readErr != nil { + return fmt.Errorf("读取 manifest 失败: %w", readErr) + } + if closeErr != nil { + return fmt.Errorf("关闭 manifest 对象失败: %w", closeErr) + } + if len(data) > maxManifestBytes || int64(len(data)) != object.Size { + return fmt.Errorf("manifest 大小非法或不完整") + } + if err := sonic.Unmarshal(data, target); err != nil { + return fmt.Errorf("解析 manifest 失败: %w", err) + } + return nil +} + +func (s *Service) verifyRetentionObject(ctx context.Context, run *model.LogArchiveRun, final bool) error { + metadata := map[string]string{ + "schema-version": run.SchemaVersion, "source": run.Source, + "archive-date": run.RangeStart.Format(time.DateOnly), "timezone": constants.AuditArchiveTimezone, + "sha256": run.SHA256, "revision": strconv.Itoa(run.Revision), + } + if run.Source == constants.AuditArchiveSource { + metadata["event-count"] = strconv.FormatInt(run.EventCount, 10) + metadata["resource-count"] = strconv.FormatInt(run.ResourceCount, 10) + } else { + metadata["record-count"] = strconv.FormatInt(run.RecordCount, 10) + metadata["final"] = strconv.FormatBool(final) + } + if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil { + return err + } + reader, err := s.store.Download(ctx, run.ObjectKey) + if err != nil { + return fmt.Errorf("下载归档对象复核 SHA-256 失败: %w", err) + } + hasher := sha256.New() + written, copyErr := io.Copy(hasher, reader) + closeErr := reader.Close() + if copyErr != nil { + return fmt.Errorf("读取归档对象复核 SHA-256 失败: %w", copyErr) + } + if closeErr != nil { + return fmt.Errorf("关闭归档对象失败: %w", closeErr) + } + if written != run.CompressedBytes || fmt.Sprintf("%x", hasher.Sum(nil)) != run.SHA256 { + return fmt.Errorf("归档对象大小或 SHA-256 复核失败") + } + return nil +} + +func summarizeRetentionRuns(runs retentionRuns, result *RetentionResult) { + result.ManifestKeys = make([]string, 0, len(runs.audit)+len(runs.integration)) + for _, run := range runs.audit { + result.EventCount += run.EventCount + result.ResourceCount += run.ResourceCount + result.ManifestKeys = append(result.ManifestKeys, run.ManifestKey) + } + for _, run := range runs.integration { + result.IntegrationCount += run.RecordCount + result.ManifestKeys = append(result.ManifestKeys, run.ManifestKey) + } +} + +func (s *Service) cleanupAuditMonth(ctx context.Context, start, end time.Time, runs []*model.LogArchiveRun) error { + if allRunsCleaned(runs) { + return nil + } + if err := s.markCleanupStarted(ctx, constants.AuditArchiveSource, start, end); err != nil { + return err + } + if err := s.deleteAuditResources(ctx, start, end); err != nil { + return err + } + if err := s.deleteAuditEvents(ctx, start, end); err != nil { + return err + } + return s.markCleaned(ctx, constants.AuditArchiveSource, start, end) +} + +func (s *Service) cleanupIntegrationMonth(ctx context.Context, start, end time.Time, runs []*model.LogArchiveRun) error { + if allRunsCleaned(runs) { + return nil + } + if err := s.markCleanupStarted(ctx, constants.IntegrationArchiveSource, start, end); err != nil { + return err + } + for { + subquery := s.db.Model(&model.IntegrationLog{}).Select("id"). + Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize) + deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.IntegrationLog{}) + if deleted.Error != nil { + return fmt.Errorf("分批物理删除 Integration Log 失败: %w", deleted.Error) + } + if deleted.RowsAffected == 0 { + break + } + } + return s.markCleaned(ctx, constants.IntegrationArchiveSource, start, end) +} + +func (s *Service) deleteAuditResources(ctx context.Context, start, end time.Time) error { + for { + subquery := s.db.Model(&model.AuditEventResource{}).Select("tb_audit_event_resource.id"). + Joins("JOIN tb_audit_event ON tb_audit_event.id = tb_audit_event_resource.audit_event_id"). + Where("tb_audit_event.created_at >= ? AND tb_audit_event.created_at < ?", start, end). + Order("tb_audit_event_resource.id ASC").Limit(constants.AuditRetentionDeleteBatchSize) + deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEventResource{}) + if deleted.Error != nil { + return fmt.Errorf("分批物理删除 Audit Event Resource 失败: %w", deleted.Error) + } + if deleted.RowsAffected == 0 { + return nil + } + } +} + +func (s *Service) deleteAuditEvents(ctx context.Context, start, end time.Time) error { + for { + subquery := s.db.Model(&model.AuditEvent{}).Select("id"). + Where("created_at >= ? AND created_at < ?", start, end).Order("id ASC").Limit(constants.AuditRetentionDeleteBatchSize) + deleted := s.db.WithContext(ctx).Where("id IN (?)", subquery).Delete(&model.AuditEvent{}) + if deleted.Error != nil { + return fmt.Errorf("分批物理删除 Audit Event 失败: %w", deleted.Error) + } + if deleted.RowsAffected == 0 { + return nil + } + } +} + +func (s *Service) markCleanupStarted(ctx context.Context, source string, start, end time.Time) error { + now := time.Now() + result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NULL", + source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID). + Updates(map[string]any{"cleanup_started_at": now, "updated_at": now}) + if result.Error != nil { + return fmt.Errorf("记录月度清理开始断点失败: %w", result.Error) + } + return s.validateCleanupMarkerCount(ctx, source, start, end, "cleanup_started_at IS NOT NULL", "开始") +} + +func (s *Service) markCleaned(ctx context.Context, source string, start, end time.Time) error { + now := time.Now() + result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND cleanup_started_at IS NOT NULL", + source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID). + Updates(map[string]any{"cleaned_at": now, "updated_at": now}) + if result.Error != nil { + return fmt.Errorf("记录月度清理完成断点失败: %w", result.Error) + } + return s.validateCleanupMarkerCount(ctx, source, start, end, "cleaned_at IS NOT NULL", "完成") +} + +func (s *Service) validateCleanupMarkerCount(ctx context.Context, source string, start, end time.Time, marker, label string) error { + var count int64 + err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("source = ? AND archive_date >= ? AND archive_date < ? AND instance_id = ? AND "+marker, + source, start.Format(time.DateOnly), end.Format(time.DateOnly), s.instanceID). + Count(&count).Error + if err != nil { + return fmt.Errorf("复核月度清理%s断点失败: %w", label, err) + } + expected := int64(end.Sub(start).Hours() / 24) + if count != expected { + return fmt.Errorf("月度清理%s断点不完整：期望 %d 条，实际 %d 条", label, expected, count) + } + return nil +} + +func allRunsCleaned(runs []*model.LogArchiveRun) bool { + return len(runs) > 0 && runs[0].CleanedAt != nil +} + +func (s *Service) recordRetentionAudit(ctx context.Context, start, end time.Time, result RetentionResult, cleanupErr error) error { + audit := RetentionAudit{ + Month: result.Month, RangeStart: start, RangeEnd: end, + EventCount: result.EventCount, ResourceCount: result.ResourceCount, + IntegrationCount: result.IntegrationCount, ManifestKeys: result.ManifestKeys, + DurationMS: result.Duration.Milliseconds(), Result: constants.AuditResultSuccess, + Summary: "完成已归档在线日志月度物理清理", + EventID: "evt_retention_" + strings.ReplaceAll(result.Month, "-", "_"), + } + if cleanupErr != nil { + audit.EventID = "" + audit.Result = constants.AuditResultFailed + audit.Summary = "已归档在线日志月度物理清理失败" + audit.ErrorSummary = truncateRetentionError(cleanupErr) + } + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + return s.audit.WriteRetentionCleanup(ctx, tx, audit) + }) +} + +func truncateRetentionError(err error) string { + value := []rune(err.Error()) + if len(value) > 500 { + value = value[:500] + } + return string(value) +} diff --git a/internal/application/auditarchive/service.go b/internal/application/auditarchive/service.go new file mode 100644 index 0000000..98a321f --- /dev/null +++ b/internal/application/auditarchive/service.go @@ -0,0 +1,412 @@ +// Package auditarchive 实现统一审计每日冷归档用例。 +package auditarchive + +import ( + "compress/gzip" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/bytedance/sonic" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/storage" +) + +const archivePageSize = 1000 + +// ObjectStore 是每日归档需要的最小对象存储能力。 +type ObjectStore interface { + UploadWithMetadata(context.Context, string, io.Reader, string, map[string]string) error + Stat(context.Context, string) (*storage.ObjectMetadata, error) + Download(context.Context, string) (io.ReadCloser, error) +} + +// RetentionAuditWriter 记录月度留存清理的系统审计事实。 +type RetentionAuditWriter interface { + WriteRetentionCleanup(context.Context, *gorm.DB, RetentionAudit) error +} + +// Service 编排审计归档生成、上传、复核和幂等账本更新。 +type Service struct { + db *gorm.DB + store ObjectStore + audit RetentionAuditWriter + instanceID string + location *time.Location +} + +// Manifest 是归档对象的完整性清单。 +type Manifest struct { + SchemaVersion string `json:"schema_version"` + Source string `json:"source"` + ArchiveDate string `json:"archive_date"` + Timezone string `json:"timezone"` + RangeStart time.Time `json:"range_start"` + RangeEnd time.Time `json:"range_end"` + InstanceID string `json:"instance_id"` + EventCount int64 `json:"event_count"` + ResourceCount int64 `json:"resource_count"` + UncompressedBytes int64 `json:"uncompressed_bytes"` + CompressedBytes int64 `json:"compressed_bytes"` + ObjectKey string `json:"object_key"` + SHA256 string `json:"sha256"` + Revision int `json:"revision"` + GeneratedAt time.Time `json:"generated_at"` + Status string `json:"status"` +} + +type archiveLine struct { + Event model.AuditEvent `json:"event"` + Resources []model.AuditEventResource `json:"resources"` +} + +type archiveFile struct { + path string + eventCount int64 + resourceCount int64 + uncompressedBytes int64 + compressedBytes int64 + sha256 string +} + +// NewService 创建统一审计每日冷归档服务。 +func NewService(db *gorm.DB, store ObjectStore, instanceID string, audit ...RetentionAuditWriter) (*Service, error) { + location, err := time.LoadLocation(constants.AuditArchiveTimezone) + if err != nil { + return nil, fmt.Errorf("加载审计归档时区失败: %w", err) + } + if strings.TrimSpace(instanceID) == "" { + instanceID = "audit-archive" + } + var auditWriter RetentionAuditWriter + if len(audit) > 0 { + auditWriter = audit[0] + } + return &Service{db: db, store: store, audit: auditWriter, instanceID: instanceID, location: location}, nil +} + +// ArchivePreviousDay 归档 Asia/Shanghai 前一完整自然日。 +func (s *Service) ArchivePreviousDay(ctx context.Context) error { + now := time.Now().In(s.location) + return s.ArchiveDate(ctx, now.AddDate(0, 0, -1)) +} + +// ArchiveDate 归档指定 Asia/Shanghai 自然日。 +func (s *Service) ArchiveDate(ctx context.Context, archiveDate time.Time) error { + if s.db == nil || s.store == nil { + return fmt.Errorf("审计归档数据库或对象存储未配置") + } + start := time.Date(archiveDate.In(s.location).Year(), archiveDate.In(s.location).Month(), archiveDate.In(s.location).Day(), 0, 0, 0, 0, s.location) + end := start.AddDate(0, 0, 1) + today := time.Now().In(s.location) + todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, s.location) + if end.After(todayStart) { + return fmt.Errorf("统一审计只能归档已经结束的完整自然日") + } + run, err := s.ensureRun(ctx, start, end) + if err != nil { + return err + } + if run.Status == constants.ArchiveStatusSuccess { + valid, validateErr := s.validateSuccessfulRun(ctx, run) + if validateErr == nil && valid { + return nil + } + } + + acquired, err := s.acquireRun(ctx, run) + if err != nil || !acquired { + return err + } + if err := s.execute(ctx, run); err != nil { + s.markFailed(ctx, run.ID, err) + return err + } + return nil +} + +func (s *Service) ensureRun(ctx context.Context, start, end time.Time) (*model.LogArchiveRun, error) { + run := model.LogArchiveRun{ + Source: constants.AuditArchiveSource, ArchiveDate: start, InstanceID: s.instanceID, + SchemaVersion: constants.AuditArchiveSchemaVersion, Revision: 1, + Status: constants.ArchiveStatusPending, RangeStart: start, RangeEnd: end, + } + result := s.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "source"}, {Name: "archive_date"}, {Name: "instance_id"}, {Name: "schema_version"}}, + DoNothing: true, + }).Create(&run) + if result.Error != nil { + return nil, fmt.Errorf("创建审计归档账本失败: %w", result.Error) + } + if result.RowsAffected == 0 { + if err := s.db.WithContext(ctx).Where( + "source = ? AND archive_date = ? AND instance_id = ? AND schema_version = ?", + constants.AuditArchiveSource, start, s.instanceID, constants.AuditArchiveSchemaVersion, + ).First(&run).Error; err != nil { + return nil, fmt.Errorf("读取审计归档账本失败: %w", err) + } + } + return &run, nil +} + +func (s *Service) acquireRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) { + revision := run.Revision + if run.Status == constants.ArchiveStatusFailed || run.Status == constants.ArchiveStatusSuccess || run.Status == constants.ArchiveStatusRunning { + revision++ + } + now := time.Now() + staleBefore := now.Add(-3 * time.Hour) + result := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("id = ? AND (status <> ? OR updated_at < ?)", run.ID, constants.ArchiveStatusRunning, staleBefore). + Updates(map[string]any{ + "status": constants.ArchiveStatusRunning, "revision": revision, + "attempt_count": gorm.Expr("attempt_count + 1"), "error_summary": "", + "completed_at": nil, "updated_at": now, + }) + if result.Error != nil { + return false, fmt.Errorf("锁定审计归档任务失败: %w", result.Error) + } + if result.RowsAffected == 0 { + return false, nil + } + run.Revision = revision + run.Status = constants.ArchiveStatusRunning + return true, nil +} + +func (s *Service) execute(ctx context.Context, run *model.LogArchiveRun) error { + file, err := s.buildArchiveFile(ctx, run.RangeStart, run.RangeEnd) + if err != nil { + return err + } + defer os.Remove(file.path) + + dbEvents, dbResources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd) + if err != nil { + return err + } + if dbEvents != file.eventCount || dbResources != file.resourceCount { + return fmt.Errorf("审计归档生成期间数据数量发生变化") + } + + objectKey, manifestKey := objectKeys(run.RangeStart, run.Revision) + metadata := archiveMetadata(file, run) + reader, err := os.Open(file.path) + if err != nil { + return fmt.Errorf("打开审计归档临时文件失败: %w", err) + } + uploadErr := s.store.UploadWithMetadata(ctx, objectKey, reader, "application/gzip", metadata) + closeErr := reader.Close() + if uploadErr != nil { + return fmt.Errorf("上传审计归档对象失败: %w", uploadErr) + } + if closeErr != nil { + return fmt.Errorf("关闭审计归档临时文件失败: %w", closeErr) + } + if err := s.verifyObject(ctx, objectKey, file.compressedBytes, metadata); err != nil { + return err + } + + generatedAt := time.Now().In(s.location) + manifest := Manifest{ + SchemaVersion: constants.AuditArchiveSchemaVersion, Source: constants.AuditArchiveSource, + ArchiveDate: run.RangeStart.In(s.location).Format(time.DateOnly), Timezone: constants.AuditArchiveTimezone, + RangeStart: run.RangeStart, RangeEnd: run.RangeEnd, InstanceID: s.instanceID, + EventCount: file.eventCount, ResourceCount: file.resourceCount, + UncompressedBytes: file.uncompressedBytes, CompressedBytes: file.compressedBytes, + ObjectKey: objectKey, SHA256: file.sha256, Revision: run.Revision, + GeneratedAt: generatedAt, Status: constants.ArchiveStatusSuccess, + } + manifestBytes, err := sonic.Marshal(manifest) + if err != nil { + return fmt.Errorf("序列化审计归档清单失败: %w", err) + } + manifestMetadata := map[string]string{"source": constants.AuditArchiveSource, "data-sha256": file.sha256, "revision": strconv.Itoa(run.Revision)} + if err := s.store.UploadWithMetadata(ctx, manifestKey, strings.NewReader(string(manifestBytes)), "application/json", manifestMetadata); err != nil { + return fmt.Errorf("上传审计归档清单失败: %w", err) + } + if err := s.verifyObject(ctx, manifestKey, int64(len(manifestBytes)), manifestMetadata); err != nil { + return err + } + + completedAt := time.Now() + updates := map[string]any{ + "status": constants.ArchiveStatusSuccess, "object_key": objectKey, "manifest_key": manifestKey, + "event_count": file.eventCount, "resource_count": file.resourceCount, + "uncompressed_bytes": file.uncompressedBytes, "compressed_bytes": file.compressedBytes, + "sha256": file.sha256, "generated_at": generatedAt, "completed_at": completedAt, + "updated_at": completedAt, + } + if err := s.db.WithContext(ctx).Model(&model.LogArchiveRun{}).Where("id = ?", run.ID).Updates(updates).Error; err != nil { + return fmt.Errorf("更新审计归档成功账本失败: %w", err) + } + return nil +} + +func (s *Service) buildArchiveFile(ctx context.Context, start, end time.Time) (*archiveFile, error) { + temp, err := os.CreateTemp("", "audit-events-*.jsonl.gz") + if err != nil { + return nil, fmt.Errorf("创建审计归档临时文件失败: %w", err) + } + path := temp.Name() + failed := true + defer func() { + _ = temp.Close() + if failed { + _ = os.Remove(path) + } + }() + + hasher := sha256.New() + gzipWriter := gzip.NewWriter(io.MultiWriter(temp, hasher)) + result := &archiveFile{path: path} + var lastID uint + for { + var events []model.AuditEvent + if err := s.db.WithContext(ctx).Where("created_at >= ? AND created_at < ? AND id > ?", start, end, lastID). + Order("id ASC").Limit(archivePageSize).Find(&events).Error; err != nil { + return nil, fmt.Errorf("读取审计归档事件失败: %w", err) + } + if len(events) == 0 { + break + } + ids := make([]uint, 0, len(events)) + for i := range events { + ids = append(ids, events[i].ID) + } + var resources []model.AuditEventResource + if err := s.db.WithContext(ctx).Where("audit_event_id IN ?", ids). + Order("audit_event_id ASC, sort_order ASC, id ASC").Find(&resources).Error; err != nil { + return nil, fmt.Errorf("读取审计归档资源失败: %w", err) + } + grouped := make(map[uint][]model.AuditEventResource, len(events)) + for i := range resources { + resource := resources[i] + grouped[resource.AuditEventID] = append(grouped[resource.AuditEventID], resource) + } + for i := range events { + eventResources := grouped[events[i].ID] + if eventResources == nil { + eventResources = []model.AuditEventResource{} + } + line, marshalErr := sonic.Marshal(archiveLine{Event: events[i], Resources: eventResources}) + if marshalErr != nil { + return nil, fmt.Errorf("序列化审计归档事件失败: %w", marshalErr) + } + line = append(line, '\n') + if _, writeErr := gzipWriter.Write(line); writeErr != nil { + return nil, fmt.Errorf("写入审计归档压缩流失败: %w", writeErr) + } + result.eventCount++ + result.resourceCount += int64(len(grouped[events[i].ID])) + result.uncompressedBytes += int64(len(line)) + } + lastID = events[len(events)-1].ID + } + if err := gzipWriter.Close(); err != nil { + return nil, fmt.Errorf("关闭审计归档压缩流失败: %w", err) + } + if err := temp.Close(); err != nil { + return nil, fmt.Errorf("关闭审计归档临时文件失败: %w", err) + } + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("读取审计归档临时文件信息失败: %w", err) + } + result.compressedBytes = info.Size() + result.sha256 = fmt.Sprintf("%x", hasher.Sum(nil)) + failed = false + return result, nil +} + +func (s *Service) databaseCounts(ctx context.Context, start, end time.Time) (int64, int64, error) { + var eventCount int64 + if err := s.db.WithContext(ctx).Model(&model.AuditEvent{}). + Where("created_at >= ? AND created_at < ?", start, end).Count(&eventCount).Error; err != nil { + return 0, 0, fmt.Errorf("统计审计归档事件失败: %w", err) + } + var resourceCount int64 + subquery := s.db.Model(&model.AuditEvent{}).Select("id").Where("created_at >= ? AND created_at < ?", start, end) + if err := s.db.WithContext(ctx).Model(&model.AuditEventResource{}). + Where("audit_event_id IN (?)", subquery).Count(&resourceCount).Error; err != nil { + return 0, 0, fmt.Errorf("统计审计归档资源失败: %w", err) + } + return eventCount, resourceCount, nil +} + +func (s *Service) validateSuccessfulRun(ctx context.Context, run *model.LogArchiveRun) (bool, error) { + events, resources, err := s.databaseCounts(ctx, run.RangeStart, run.RangeEnd) + if err != nil || events != run.EventCount || resources != run.ResourceCount { + return false, err + } + metadata := map[string]string{ + "sha256": run.SHA256, "event-count": strconv.FormatInt(run.EventCount, 10), + "resource-count": strconv.FormatInt(run.ResourceCount, 10), "revision": strconv.Itoa(run.Revision), + } + if err := s.verifyObject(ctx, run.ObjectKey, run.CompressedBytes, metadata); err != nil { + return false, nil + } + manifestMetadata := map[string]string{ + "source": constants.AuditArchiveSource, "data-sha256": run.SHA256, "revision": strconv.Itoa(run.Revision), + } + if err := s.verifyObject(ctx, run.ManifestKey, -1, manifestMetadata); err != nil { + return false, nil + } + return true, nil +} + +func (s *Service) verifyObject(ctx context.Context, key string, expectedSize int64, expectedMetadata map[string]string) error { + object, err := s.store.Stat(ctx, key) + if err != nil { + return fmt.Errorf("复核归档对象 metadata 失败: %w", err) + } + if expectedSize >= 0 && object.Size != expectedSize { + return fmt.Errorf("归档对象大小复核失败") + } + for name, value := range expectedMetadata { + if object.Metadata[strings.ToLower(name)] != value { + return fmt.Errorf("归档对象 metadata 字段 %s 复核失败", name) + } + } + return nil +} + +func (s *Service) markFailed(ctx context.Context, runID uint, archiveErr error) { + summary := []rune(archiveErr.Error()) + if len(summary) > 500 { + summary = summary[:500] + } + failedCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _ = s.db.WithContext(failedCtx).Model(&model.LogArchiveRun{}).Where("id = ?", runID).Updates(map[string]any{ + "status": constants.ArchiveStatusFailed, "error_summary": string(summary), "updated_at": time.Now(), + }).Error +} + +func objectKeys(date time.Time, revision int) (string, string) { + prefix := fmt.Sprintf("audit-archive/v1/%04d/%02d/%02d", date.Year(), date.Month(), date.Day()) + name := fmt.Sprintf("audit-events-%s-r%d", date.Format(time.DateOnly), revision) + return prefix + "/" + name + ".jsonl.gz", prefix + "/" + name + ".manifest.json" +} + +func archiveMetadata(file *archiveFile, run *model.LogArchiveRun) map[string]string { + return map[string]string{ + "schema-version": constants.AuditArchiveSchemaVersion, + "source": constants.AuditArchiveSource, + "archive-date": run.RangeStart.Format(time.DateOnly), + "timezone": constants.AuditArchiveTimezone, + "event-count": strconv.FormatInt(file.eventCount, 10), + "resource-count": strconv.FormatInt(file.resourceCount, 10), + "sha256": file.sha256, + "revision": strconv.Itoa(run.Revision), + } +} diff --git a/internal/application/cardobservation/apply.go b/internal/application/cardobservation/apply.go index 16ff296..9276187 100644 --- a/internal/application/cardobservation/apply.go +++ b/internal/application/cardobservation/apply.go @@ -84,7 +84,7 @@ func (s *Service) RecordCarrierCallbackFailure(ctx context.Context, card *model. } s.auditWriter.WriteCardStateFailure(ctx, StateAudit{ ActionCode: constants.AuditActionIotCardRealnameCallbackSynced, - Summary: "运营商回调同步 IoT 卡实名状态失败", Card: card, IntegrationID: integrationID, + Summary: "运营商回调同步 IoT 卡实名状态失败", Card: card, IntegrationID: integrationID, }, businessErr) } @@ -94,6 +94,7 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R return domain.RealnameDecision{}, errors.New(errors.CodeInternalError, "卡实名观测能力未完整配置") } var decision domain.RealnameDecision + var auditedCard *model.IotCard err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var card model.IotCard if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil { @@ -102,6 +103,7 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R } return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡失败") } + auditedCard = &card nextDecision, decisionErr := domain.ApplyRealname(domain.CardRealnameSnapshot{ CardID: card.ID, Status: card.RealNameStatus, FirstRealnameAt: card.FirstRealnameAt, ReversalCount: card.RealnameReversalCount, ReversalStartedAt: card.RealnameReversalStartedAt, @@ -185,11 +187,26 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置") } if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{ - ActionCode: constants.AuditActionIotCardRealnameCallbackSynced, - Summary: "运营商回调同步 IoT 卡实名状态", - Card: &card, + ActionCode: constants.AuditActionIotCardRealnameCallbackSynced, + Summary: "运营商回调同步 IoT 卡实名状态", + Card: &card, IntegrationID: observation.Metadata.ObservationID, - BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt}, + BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt}, + AfterData: map[string]any{ + "real_name_status": decision.AfterStatus, "first_realname_at": firstRealnameAfter(card.FirstRealnameAt, observation.Metadata.ObservedAt, decision.FirstVerified), + }, + }); err != nil { + return err + } + } else if workerObservationAudited(ctx) && decision.StatusChanged { + if s.auditWriter == nil { + return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置") + } + if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{ + ActionCode: constants.AuditActionIotCardWorkerRealnameSynced, + Summary: "Worker 同步 IoT 卡实名事实", Card: &card, + IntegrationID: observation.Metadata.ObservationID, + BeforeData: map[string]any{"real_name_status": card.RealNameStatus, "first_realname_at": card.FirstRealnameAt}, AfterData: map[string]any{ "real_name_status": decision.AfterStatus, "first_realname_at": firstRealnameAfter(card.FirstRealnameAt, observation.Metadata.ObservedAt, decision.FirstVerified), }, @@ -200,6 +217,13 @@ func (s *Service) ApplyCardObservation(ctx context.Context, observation domain.R return nil }) if err != nil { + if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil { + s.auditWriter.WriteCardStateFailure(ctx, StateAudit{ + ActionCode: constants.AuditActionIotCardWorkerRealnameSynced, + Summary: "Worker 同步 IoT 卡实名事实失败", Card: auditedCard, + IntegrationID: observation.Metadata.ObservationID, + }, err) + } return domain.RealnameDecision{}, err } if s.cache != nil { @@ -226,6 +250,11 @@ func manualRefreshAuditAction(ctx context.Context) (string, bool) { } } +func workerObservationAudited(ctx context.Context) bool { + linkage := auditcontext.From(ctx) + return linkage.ActorKind == constants.AuditActorSystemTask && linkage.Source == constants.AuditSourceWorker +} + func realnameChangedEventID(cardID uint, observationID string) string { prefix := "card-realname:" digest := sha256.Sum256([]byte(strconv.FormatUint(uint64(cardID), 10) + ":" + observationID + ":changed")) diff --git a/internal/application/cardobservation/network.go b/internal/application/cardobservation/network.go index e2e48d3..7d85c1b 100644 --- a/internal/application/cardobservation/network.go +++ b/internal/application/cardobservation/network.go @@ -33,6 +33,7 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai return domain.NetworkDecision{}, errors.New(errors.CodeInternalError, "卡网络观测能力未完整配置") } var decision domain.NetworkDecision + var auditedCard *model.IotCard err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var card model.IotCard if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil { @@ -41,6 +42,7 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai } return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡网络事实失败") } + auditedCard = &card nextDecision, decisionErr := domain.ApplyNetwork(domain.CardNetworkSnapshot{ CardID: card.ID, NetworkStatus: card.NetworkStatus, StopReason: card.StopReason, IsStandalone: card.IsStandalone, EnablePolling: card.EnablePolling, @@ -117,10 +119,46 @@ func (s *Service) ApplyNetworkObservation(ctx context.Context, observation domai }); err != nil { return err } + } else if workerObservationAudited(ctx) && stateChanged { + if s.auditWriter == nil { + return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置") + } + enablePolling := card.EnablePolling + if decision.StopPolling { + enablePolling = false + } + gatewayIMEI := card.GatewayCardIMEI + if decision.UpdateIMEI { + gatewayIMEI = decision.GatewayIMEI + } + if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{ + ActionCode: constants.AuditActionIotCardWorkerNetworkSynced, + Summary: "Worker 同步 IoT 卡网络事实", Card: &card, + IntegrationID: observation.Metadata.ObservationID, + BeforeData: map[string]any{ + "network_status": card.NetworkStatus, "stop_reason": card.StopReason, + "gateway_extend": card.GatewayExtend, "gateway_card_imei": card.GatewayCardIMEI, + "enable_polling": card.EnablePolling, + }, + AfterData: map[string]any{ + "network_status": decision.AfterStatus, "stop_reason": decision.StopReason, + "gateway_extend": decision.GatewayExtend, "gateway_card_imei": gatewayIMEI, + "enable_polling": enablePolling, + }, + }); err != nil { + return err + } } return nil }) if err != nil { + if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil { + s.auditWriter.WriteCardStateFailure(ctx, StateAudit{ + ActionCode: constants.AuditActionIotCardWorkerNetworkSynced, + Summary: "Worker 同步 IoT 卡网络事实失败", Card: auditedCard, + IntegrationID: observation.Metadata.ObservationID, + }, err) + } return domain.NetworkDecision{}, err } if s.cache != nil { diff --git a/internal/application/cardobservation/series.go b/internal/application/cardobservation/series.go index 92679f1..4fcb72b 100644 --- a/internal/application/cardobservation/series.go +++ b/internal/application/cardobservation/series.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" + "github.com/break/junhong_cmp_fiber/pkg/auditcontext" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" ) @@ -21,6 +22,7 @@ type SeriesRequest struct { Source string `json:"source"` RequestID string `json:"request_id,omitempty"` CorrelationID string `json:"correlation_id,omitempty"` + ParentEventID string `json:"parent_event_id,omitempty"` } // DeviceCardsSeriesRequest 描述需要在后台展开设备有效绑定卡的观测请求。 @@ -59,6 +61,7 @@ type SeriesTaskPayload struct { Source string `json:"source"` RequestID string `json:"request_id,omitempty"` CorrelationID string `json:"correlation_id,omitempty"` + ParentEventID string `json:"parent_event_id,omitempty"` } // RunResult 描述一次实际 Gateway 请求及公共观测应用结果。 @@ -135,7 +138,7 @@ func (s *SeriesTrigger) trigger(ctx context.Context, request SeriesRequest, cand if s == nil || s.coordinator == nil || s.scheduler == nil || s.logger == nil { return "", false, errors.New(errors.CodeInternalError, "卡观测序列触发能力未完整配置") } - request = normalizeSeriesTrace(request) + request = normalizeSeriesTrace(ctx, request) if err := validateSeriesRequest(request); err != nil { return "", false, err } @@ -171,6 +174,7 @@ func (s *SeriesTrigger) trigger(ctx context.Context, request SeriesRequest, cand Scene: originalRequest.Scene, ResourceType: originalRequest.ResourceType, ResourceID: originalRequest.ResourceID, SyncType: originalRequest.SyncType, ExpectedValue: originalRequest.ExpectedValue, Source: originalRequest.Source, RequestID: originalRequest.RequestID, CorrelationID: originalRequest.CorrelationID, + ParentEventID: originalRequest.ParentEventID, } if err := s.scheduler.Enqueue(ctx, payload); err != nil { s.coordinator.ReleaseSchedule(ctx, seriesID, attempt) @@ -303,12 +307,24 @@ func validateSeriesPayload(payload SeriesTaskPayload) error { return validateSeriesRequest(SeriesRequest{ Scene: payload.Scene, ResourceType: payload.ResourceType, ResourceID: payload.ResourceID, SyncType: payload.SyncType, Source: payload.Source, RequestID: payload.RequestID, CorrelationID: payload.CorrelationID, + ParentEventID: payload.ParentEventID, }) } -func normalizeSeriesTrace(request SeriesRequest) SeriesRequest { +func normalizeSeriesTrace(ctx context.Context, request SeriesRequest) SeriesRequest { + linkage := auditcontext.From(ctx) request.RequestID = strings.TrimSpace(request.RequestID) request.CorrelationID = strings.TrimSpace(request.CorrelationID) + request.ParentEventID = strings.TrimSpace(request.ParentEventID) + if request.RequestID == "" { + request.RequestID = strings.TrimSpace(linkage.RequestID) + } + if request.CorrelationID == "" { + request.CorrelationID = strings.TrimSpace(linkage.CorrelationID) + } + if request.ParentEventID == "" { + request.ParentEventID = strings.TrimSpace(linkage.ParentEventID) + } if request.RequestID == "" && request.CorrelationID == "" { traceID := uuid.NewString() request.RequestID = traceID diff --git a/internal/application/cardobservation/traffic.go b/internal/application/cardobservation/traffic.go index 759eb12..576d88a 100644 --- a/internal/application/cardobservation/traffic.go +++ b/internal/application/cardobservation/traffic.go @@ -32,6 +32,7 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai return domain.TrafficDecision{}, errors.New(errors.CodeInternalError, "卡流量观测能力未完整配置") } var decision domain.TrafficDecision + var auditedCard *model.IotCard err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var card model.IotCard if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", observation.CardID).First(&card).Error; err != nil { @@ -40,6 +41,7 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai } return errors.Wrap(errors.CodeDatabaseError, err, "锁定IoT卡流量事实失败") } + auditedCard = &card nextDecision, decisionErr := domain.ApplyTraffic(domain.CardTrafficSnapshot{ CardID: card.ID, DataUsageMB: card.DataUsageMB, CurrentMonthUsageMB: card.CurrentMonthUsageMB, CurrentMonthStartDate: card.CurrentMonthStartDate, LastMonthTotalMB: card.LastMonthTotalMB, @@ -99,10 +101,38 @@ func (s *Service) ApplyTrafficObservation(ctx context.Context, observation domai }); err != nil { return err } + } else if workerObservationAudited(ctx) && stateChanged { + if s.auditWriter == nil { + return errors.New(errors.CodeInternalError, "卡状态统一审计能力未配置") + } + if err := s.auditWriter.WriteCardStateAudit(ctx, tx, StateAudit{ + ActionCode: constants.AuditActionIotCardWorkerTrafficSynced, + Summary: "Worker 同步 IoT 卡流量事实", Card: &card, + IntegrationID: observation.Metadata.ObservationID, + BeforeData: map[string]any{ + "data_usage_mb": card.DataUsageMB, "current_month_usage_mb": card.CurrentMonthUsageMB, + "current_month_start_date": card.CurrentMonthStartDate, "last_month_total_mb": card.LastMonthTotalMB, + "last_gateway_reading_mb": card.LastGatewayReadingMB, + }, + AfterData: map[string]any{ + "data_usage_mb": decision.DataUsageMB, "current_month_usage_mb": decision.CurrentMonthUsageMB, + "current_month_start_date": decision.CurrentMonthStartDate, "last_month_total_mb": decision.LastMonthTotalMB, + "last_gateway_reading_mb": decision.LastGatewayReadingMB, "increment_mb": decision.IncrementMB, + }, + }); err != nil { + return err + } } return nil }) if err != nil { + if workerObservationAudited(ctx) && auditedCard != nil && s.auditWriter != nil { + s.auditWriter.WriteCardStateFailure(ctx, StateAudit{ + ActionCode: constants.AuditActionIotCardWorkerTrafficSynced, + Summary: "Worker 同步 IoT 卡流量事实失败", Card: auditedCard, + IntegrationID: observation.Metadata.ObservationID, + }, err) + } return domain.TrafficDecision{}, err } if s.cache != nil { diff --git a/internal/bootstrap/services.go b/internal/bootstrap/services.go index 807e5e4..65ef29d 100644 --- a/internal/bootstrap/services.go +++ b/internal/bootstrap/services.go @@ -191,7 +191,7 @@ func initServices(s *stores, deps *Dependencies) *services { ) cardObservationSeries := cardObservationApp.NewSeriesAttemptService( seriesCoordinator, - cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration), + cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, seriesIntegration, auditWriter), cardObservationInfra.NewSeriesAttemptLogger(seriesIntegration), ) observationSeries := cardObservationInfra.NewBestEffortSeriesDispatcher(seriesTrigger, deps.Logger, s.DeviceSimBinding, s.Carrier) diff --git a/internal/bootstrap/worker_services.go b/internal/bootstrap/worker_services.go index 77db317..dd716f0 100644 --- a/internal/bootstrap/worker_services.go +++ b/internal/bootstrap/worker_services.go @@ -102,11 +102,18 @@ func initWorkerServices(stores *queue.WorkerStores, deps *WorkerDependencies) *q cardObservationInfra.NewEventWriter(cardObservationOutbox), cardObservationInfra.NewCacheInvalidator(deps.Redis, deps.Logger), ) + iotCardAuditService := iotCardSvc.New( + deps.DB, stores.IotCard, stores.Shop, stores.AssetAllocationRecord, + stores.ShopPackageAllocation, stores.ShopSeriesAllocation, stores.PackageSeries, + deps.GatewayClient, deps.Logger, assetAudit, + ) + iotCardAuditService.SetAccessAudit(auditWriter) + cardObservationService.SetStateAuditWriter(iotCardAuditService) cardObservationIntegration := integrationlog.NewRepository(deps.DB) cardObservationSeriesCoordinator := cardObservationInfra.NewSeriesCoordinator(deps.Redis) cardObservationSeriesService := cardObservationApp.NewSeriesAttemptService( cardObservationSeriesCoordinator, - cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, cardObservationIntegration), + cardObservationInfra.NewSeriesRunner(deps.DB, deps.GatewayClient, cardObservationService, cardObservationIntegration, auditWriter), cardObservationInfra.NewSeriesAttemptLogger(cardObservationIntegration), ) diff --git a/internal/handler/admin/audit.go b/internal/handler/admin/audit.go index 71885e5..53d6f4d 100644 --- a/internal/handler/admin/audit.go +++ b/internal/handler/admin/audit.go @@ -121,15 +121,115 @@ func (h *AuditHandler) ResourceTimeline(c *fiber.Ctx) error { return response.Success(c, result) } +// RequestTimeline 查询指定 HTTP 请求关联的跨事实时间线。 +// GET /api/admin/audit/requests/:request_id/timeline +func (h *AuditHandler) RequestTimeline(c *fiber.Ctx) error { + requestID := c.Params("request_id") + if requestID == "" { + return errors.New(errors.CodeInvalidParam) + } + result, err := h.auditQuery.RequestTimeline(c.UserContext(), requestID) + if err != nil { + return err + } + return response.Success(c, result) +} + +// CorrelationTimeline 查询跨请求业务关联时间线。 +// GET /api/admin/audit/correlations/:correlation_id/timeline +func (h *AuditHandler) CorrelationTimeline(c *fiber.Ctx) error { + correlationID := c.Params("correlation_id") + if correlationID == "" { + return errors.New(errors.CodeInvalidParam) + } + result, err := h.auditQuery.CorrelationTimeline(c.UserContext(), correlationID) + if err != nil { + return err + } + return response.Success(c, result) +} + +// FinanceTimeline 查询资金审计与业务账本的组合时间线。 +// GET /api/admin/audit/finance/timeline +func (h *AuditHandler) FinanceTimeline(c *fiber.Ctx) error { + var request dto.AuditFinanceTimelineRequest + if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) { + return errors.New(errors.CodeInvalidParam) + } + from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo) + if err != nil { + return errors.New(errors.CodeInvalidParam) + } + result, err := h.auditQuery.FinanceTimeline(c.UserContext(), auditquery.FinanceFilter{ + ShopID: request.ShopID, WalletID: request.WalletID, OrderID: request.OrderID, OrderNo: request.OrderNo, + PaymentID: request.PaymentID, PaymentNo: request.PaymentNo, RefundID: request.RefundID, RefundNo: request.RefundNo, + RechargeID: request.RechargeID, RechargeNo: request.RechargeNo, ApprovalInstanceID: request.ApprovalInstanceID, + ThirdPartyTradeNo: request.ThirdPartyTradeNo, ActorKind: request.ActorKind, ActorID: request.ActorID, + CorrelationID: request.CorrelationID, CreatedFrom: from, CreatedTo: to, Page: request.Page, PageSize: request.PageSize, + }) + if err != nil { + return err + } + return response.Success(c, result) +} + +// RiskOverview 查询固定风险信号总览。 +// GET /api/admin/audit/risks/overview +func (h *AuditHandler) RiskOverview(c *fiber.Ctx) error { + var request dto.AuditRiskOverviewRequest + if err := c.QueryParser(&request); err != nil { + return errors.New(errors.CodeInvalidParam) + } + filter, err := riskFilter(request.AuditRiskFilterRequest, 0, 0) + if err != nil { + return err + } + result, err := h.auditQuery.RiskOverview(c.UserContext(), filter) + if err != nil { + return err + } + return response.Success(c, result) +} + +// RiskEvents 查询固定风险集合的事件明细。 +// GET /api/admin/audit/risks/events +func (h *AuditHandler) RiskEvents(c *fiber.Ctx) error { + var request dto.AuditRiskEventsRequest + if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) { + return errors.New(errors.CodeInvalidParam) + } + filter, err := riskFilter(request.AuditRiskFilterRequest, request.Page, request.PageSize) + if err != nil { + return err + } + result, err := h.auditQuery.RiskEvents(c.UserContext(), filter) + if err != nil { + return err + } + return response.Success(c, result) +} + +func riskFilter(request dto.AuditRiskFilterRequest, page, pageSize int) (auditquery.RiskFilter, error) { + from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo) + if err != nil { + return auditquery.RiskFilter{}, errors.New(errors.CodeInvalidParam) + } + return auditquery.RiskFilter{ + CreatedFrom: from, CreatedTo: to, Risk: request.Risk, Result: request.Result, + Action: request.Action, Source: request.Source, Page: page, PageSize: pageSize, + }, nil +} + // AgentResourceActivities 查询代理范围内的安全资源活动。 // GET /api/admin/agent/resource-activities/:resource_type/:identifier func (h *AuditHandler) AgentResourceActivities(c *fiber.Ctx) error { - request, err := subjectActivityRequest(c) + request, from, to, err := subjectActivityRequest(c) if err != nil { return err } result, err := h.auditQuery.AgentResourceActivities(c.UserContext(), auditquery.SubjectActivityFilter{ ResourceType: request.ResourceType, Identifier: request.Identifier, + CreatedFrom: from, CreatedTo: to, Page: request.Page, PageSize: request.PageSize, }) if err != nil { @@ -141,12 +241,13 @@ func (h *AuditHandler) AgentResourceActivities(c *fiber.Ctx) error { // EnterpriseResourceActivities 查询企业当前有效授权资产的安全资源活动。 // GET /api/admin/enterprise/resource-activities/:resource_type/:identifier func (h *AuditHandler) EnterpriseResourceActivities(c *fiber.Ctx) error { - request, err := subjectActivityRequest(c) + request, from, to, err := subjectActivityRequest(c) if err != nil { return err } result, err := h.auditQuery.EnterpriseResourceActivities(c.UserContext(), auditquery.SubjectActivityFilter{ ResourceType: request.ResourceType, Identifier: request.Identifier, + CreatedFrom: from, CreatedTo: to, Page: request.Page, PageSize: request.PageSize, }) if err != nil { @@ -155,17 +256,21 @@ func (h *AuditHandler) EnterpriseResourceActivities(c *fiber.Ctx) error { return response.Success(c, result) } -func subjectActivityRequest(c *fiber.Ctx) (dto.SubjectResourceActivityRequest, error) { +func subjectActivityRequest(c *fiber.Ctx) (dto.SubjectResourceActivityRequest, *time.Time, *time.Time, error) { var request dto.SubjectResourceActivityRequest if err := c.QueryParser(&request); err != nil || invalidAuditPage(request.Page, request.PageSize) { - return request, errors.New(errors.CodeInvalidParam) + return request, nil, nil, errors.New(errors.CodeInvalidParam) } request.ResourceType = c.Params("resource_type") request.Identifier = c.Params("identifier") if request.ResourceType == "" || request.Identifier == "" { - return request, errors.New(errors.CodeInvalidParam) + return request, nil, nil, errors.New(errors.CodeInvalidParam) } - return request, nil + from, to, err := auditTimeRange(request.CreatedFrom, request.CreatedTo) + if err != nil { + return request, nil, nil, errors.New(errors.CodeInvalidParam) + } + return request, from, to, nil } // IntegrationOverview 查询外部集成交互总览。 diff --git a/internal/handler/callback/cmcc_realname.go b/internal/handler/callback/cmcc_realname.go index 59dcc31..7af5050 100644 --- a/internal/handler/callback/cmcc_realname.go +++ b/internal/handler/callback/cmcc_realname.go @@ -109,7 +109,7 @@ func (h *CMCCRealnameHandler) process(ctx context.Context, body []byte, contentT }, }) if err != nil { - h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err) + h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err) return h.fail(ctx, log.IntegrationID, err) } if h.series != nil { diff --git a/internal/handler/callback/ctcc_realname.go b/internal/handler/callback/ctcc_realname.go index 7e6eb36..7295a5a 100644 --- a/internal/handler/callback/ctcc_realname.go +++ b/internal/handler/callback/ctcc_realname.go @@ -135,7 +135,7 @@ func (h *CTCCRealnameHandler) process(ctx context.Context, body []byte, contentT }, }) if err != nil { - h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err) + h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err) return h.failPending(ctx, log.IntegrationID, err) } if h.series != nil { diff --git a/internal/handler/callback/cucc_realname.go b/internal/handler/callback/cucc_realname.go index 01b4922..d6987e6 100644 --- a/internal/handler/callback/cucc_realname.go +++ b/internal/handler/callback/cucc_realname.go @@ -108,7 +108,7 @@ func (h *CUCCRealnameHandler) process(ctx context.Context, body []byte, contentT }, }) if err != nil { - h.observation.RecordCarrierCallbackFailure(ctx, &card, log.IntegrationID, err) + h.observation.RecordCarrierCallbackFailure(ctx, card, log.IntegrationID, err) return h.fail(ctx, log.IntegrationID, err) } if h.series != nil { diff --git a/internal/handler/callback/payment.go b/internal/handler/callback/payment.go index 6d72498..a115476 100644 --- a/internal/handler/callback/payment.go +++ b/internal/handler/callback/payment.go @@ -303,7 +303,7 @@ func (h *PaymentHandler) confirmAgentRechargePayment(ctx context.Context, callba } if log.Result == constants.IntegrationResultPending { resolvedResourceID := strconv.FormatUint(uint64(result.PaymentID), 10) - _, err = h.integration.Complete(ctx, log.IntegrationID, integrationlog.Completion{ + _, err := h.integration.Complete(ctx, log.IntegrationID, integrationlog.Completion{ Result: constants.IntegrationResultSuccess, ProviderCode: "SUCCESS", ResponseSummary: map[string]any{"confirmed": true, "already_confirmed": result.AlreadyConfirmed}, StateChanged: !result.AlreadyConfirmed, ResourceID: &resolvedResourceID, diff --git a/internal/infrastructure/audit/registry.go b/internal/infrastructure/audit/registry.go index 9705b34..cfafa8d 100644 --- a/internal/infrastructure/audit/registry.go +++ b/internal/infrastructure/audit/registry.go @@ -105,6 +105,9 @@ func NewRegistry() *Registry { iotCardRealnameCallbackSynced := iotCardAction(constants.AuditActionIotCardRealnameCallbackSynced, "运营商回调同步 IoT 卡实名状态", constants.AuditActorExternalSystem, constants.AuditSourceCallback) iotCardManualRefreshed := iotCardAction(constants.AuditActionIotCardManualRefreshed, "人工刷新 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI) iotCardPersonalRefreshed := iotCardAction(constants.AuditActionIotCardPersonalRefreshed, "个人客户刷新 IoT 卡", constants.AuditActorPersonalCustomer, constants.AuditSourcePersonalAPI) + iotCardWorkerRealnameSynced := iotCardAction(constants.AuditActionIotCardWorkerRealnameSynced, "Worker 同步 IoT 卡实名事实", constants.AuditActorSystemTask, constants.AuditSourceWorker) + iotCardWorkerTrafficSynced := iotCardAction(constants.AuditActionIotCardWorkerTrafficSynced, "Worker 同步 IoT 卡流量事实", constants.AuditActorSystemTask, constants.AuditSourceWorker) + iotCardWorkerNetworkSynced := iotCardAction(constants.AuditActionIotCardWorkerNetworkSynced, "Worker 同步 IoT 卡网络事实", constants.AuditActorSystemTask, constants.AuditSourceWorker) iotCardManualStopped := iotCardAction(constants.AuditActionIotCardManualStopped, "人工停用 IoT 卡网络", constants.AuditActorAccount, constants.AuditSourceAdminAPI) iotCardManualStarted := iotCardAction(constants.AuditActionIotCardManualStarted, "人工恢复 IoT 卡网络", constants.AuditActorAccount, constants.AuditSourceAdminAPI) iotCardAutoStopped := iotCardAction(constants.AuditActionIotCardAutoStopped, "自动停用 IoT 卡网络", constants.AuditActorSystemTask, constants.AuditSourceWorker) @@ -131,6 +134,7 @@ func NewRegistry() *Registry { deviceCardBound := deviceAction(constants.AuditActionDeviceCardBound, "设备绑定 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI) deviceCardUnbound := deviceAction(constants.AuditActionDeviceCardUnbound, "设备解绑 IoT 卡", constants.AuditActorAccount, constants.AuditSourceAdminAPI) deviceCurrentCardSwitched := deviceExternalAction(constants.AuditActionDeviceCurrentCardSwitched, "切换设备当前卡", true) + deviceWorkerObservationSynced := deviceAction(constants.AuditActionDeviceWorkerObservationSynced, "Worker 同步设备观测事实", constants.AuditActorSystemTask, constants.AuditSourceWorker) cardExchangeCreated := cardExchangeAction(constants.AuditActionCardExchangeCreated, "创建卡换货单", constants.AuditRiskNormal, false) cardExchangeShippingInfoSubmitted := cardExchangeAction(constants.AuditActionCardExchangeShippingInfoSubmitted, "提交卡换货收货信息", constants.AuditRiskHigh, true) cardExchangeShipped := cardExchangeAction(constants.AuditActionCardExchangeShipped, "卡换货发货", constants.AuditRiskNormal, false) @@ -205,6 +209,14 @@ func NewRegistry() *Registry { notificationReadAll.AllowedOrigins = []ActionOrigin{{Actor: constants.AuditActorPersonalCustomer, Source: constants.AuditSourcePersonalAPI}} notificationCleanup := notificationAction(constants.AuditActionNotificationCleanup, "清理过期通知", constants.AuditResourceNotificationCleanupBatch, constants.AuditActorSystemTask, constants.AuditSourceWorker) notificationCleanupItem := notificationAction(constants.AuditActionNotificationCleanupItem, "清理单条过期通知", constants.AuditResourceNotification, constants.AuditActorSystemTask, constants.AuditSourceWorker) + retentionCleanup := ActionDefinition{ + Code: constants.AuditActionLogRetentionCleanup, Name: "清理已归档在线日志", + Category: constants.AuditCategoryReliability, Risk: constants.AuditRiskHigh, + PrimaryResource: constants.AuditResourceLogArchiveMonth, AllowedActor: constants.AuditActorSystemTask, + Source: constants.AuditSourceWorker, RequireTransaction: true, + DefaultVisibility: constants.AuditSubjectInternalOnly, + AllowedVisibility: []string{constants.AuditSubjectInternalOnly}, + } pollingConfigCreated := pollingAction(constants.AuditActionPollingConfigCreated, "创建轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh) pollingConfigUpdated := pollingAction(constants.AuditActionPollingConfigUpdated, "更新轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh) pollingConfigDeleted := pollingAction(constants.AuditActionPollingConfigDeleted, "删除轮询配置", constants.AuditResourcePollingConfig, constants.AuditRiskHigh) @@ -407,6 +419,9 @@ func NewRegistry() *Registry { constants.AuditActionIotCardRealnameCallbackSynced: iotCardRealnameCallbackSynced, constants.AuditActionIotCardManualRefreshed: iotCardManualRefreshed, constants.AuditActionIotCardPersonalRefreshed: iotCardPersonalRefreshed, + constants.AuditActionIotCardWorkerRealnameSynced: iotCardWorkerRealnameSynced, + constants.AuditActionIotCardWorkerTrafficSynced: iotCardWorkerTrafficSynced, + constants.AuditActionIotCardWorkerNetworkSynced: iotCardWorkerNetworkSynced, constants.AuditActionIotCardManualStopped: iotCardManualStopped, constants.AuditActionIotCardManualStarted: iotCardManualStarted, constants.AuditActionIotCardAutoStopped: iotCardAutoStopped, @@ -432,6 +447,7 @@ func NewRegistry() *Registry { constants.AuditActionDeviceCardBound: deviceCardBound, constants.AuditActionDeviceCardUnbound: deviceCardUnbound, constants.AuditActionDeviceCurrentCardSwitched: deviceCurrentCardSwitched, + constants.AuditActionDeviceWorkerObservationSynced: deviceWorkerObservationSynced, constants.AuditActionCardExchangeCreated: cardExchangeCreated, constants.AuditActionCardExchangeShippingInfoSubmitted: cardExchangeShippingInfoSubmitted, constants.AuditActionCardExchangeShipped: cardExchangeShipped, @@ -478,6 +494,7 @@ func NewRegistry() *Registry { constants.AuditActionNotificationReadAll: notificationReadAll, constants.AuditActionNotificationCleanup: notificationCleanup, constants.AuditActionNotificationCleanupItem: notificationCleanupItem, + constants.AuditActionLogRetentionCleanup: retentionCleanup, constants.AuditActionPollingConfigCreated: pollingConfigCreated, constants.AuditActionPollingConfigUpdated: pollingConfigUpdated, constants.AuditActionPollingConfigDeleted: pollingConfigDeleted, @@ -608,6 +625,10 @@ func NewRegistry() *Registry { "resource_type", "resource_id", "resource_key", "correlation_id", }, }, + constants.AuditResourceLogArchiveMonth: { + Type: constants.AuditResourceLogArchiveMonth, Name: "日志归档自然月", + IdentityFields: []string{"month", "timezone", "range_start", "range_end"}, + }, constants.AuditResourceDeviceBatchTask: { Type: constants.AuditResourceDeviceBatchTask, Name: "设备批量分配任务", IdentityFields: []string{"task_no", "operation_type"}, diff --git a/internal/infrastructure/audit/retention.go b/internal/infrastructure/audit/retention.go new file mode 100644 index 0000000..8ed453d --- /dev/null +++ b/internal/infrastructure/audit/retention.go @@ -0,0 +1,38 @@ +package audit + +import ( + "context" + + "gorm.io/gorm" + + auditarchive "github.com/break/junhong_cmp_fiber/internal/application/auditarchive" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// WriteRetentionCleanup 将月度物理清理结果写入当前在线月份的统一审计事件。 +func (w *Writer) WriteRetentionCleanup(ctx context.Context, tx *gorm.DB, input auditarchive.RetentionAudit) error { + return w.Append(ctx, tx, AppendInput{ + EventID: input.EventID, ActionCode: constants.AuditActionLogRetentionCleanup, + Summary: input.Summary, + Actor: ActorInput{ + Kind: constants.AuditActorSystemTask, ID: constants.AuditActorIDRetentionWorker, Name: "日志留存清理任务", + }, + Source: constants.AuditSourceWorker, ScopeType: constants.AuditScopePlatform, + Result: input.Result, ErrorSummary: input.ErrorSummary, + CorrelationID: "retention:" + input.Month, + Metadata: map[string]any{ + "event_count": input.EventCount, "resource_count": input.ResourceCount, + "integration_count": input.IntegrationCount, "manifest_keys": input.ManifestKeys, + "duration_ms": input.DurationMS, + }, + Resources: []ResourceInput{{ + Type: constants.AuditResourceLogArchiveMonth, Key: input.Month, DisplayName: input.Month, + Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleRetentionMonth, + IdentitySnapshot: map[string]any{ + "month": input.Month, "timezone": constants.AuditArchiveTimezone, + "range_start": input.RangeStart, "range_end": input.RangeEnd, + }, + SubjectVisibility: constants.AuditSubjectInternalOnly, + }}, + }) +} diff --git a/internal/infrastructure/cardobservation/series_runner.go b/internal/infrastructure/cardobservation/series_runner.go index 023b984..e122d88 100644 --- a/internal/infrastructure/cardobservation/series_runner.go +++ b/internal/infrastructure/cardobservation/series_runner.go @@ -12,9 +12,11 @@ import ( cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation" carddomain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation" "github.com/break/junhong_cmp_fiber/internal/gateway" + "github.com/break/junhong_cmp_fiber/internal/infrastructure/audit" "github.com/break/junhong_cmp_fiber/internal/infrastructure/integrationlog" "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/internal/store/postgres" + "github.com/break/junhong_cmp_fiber/pkg/auditcontext" "github.com/break/junhong_cmp_fiber/pkg/constants" apperrors "github.com/break/junhong_cmp_fiber/pkg/errors" ) @@ -77,11 +79,15 @@ type SeriesRunner struct { observation *cardapp.Service integration *integrationlog.Repository carrier *postgres.CarrierStore + auditWriter *audit.Writer } // NewSeriesRunner 创建卡观测序列 Gateway 执行器。 -func NewSeriesRunner(db *gorm.DB, gatewayClient *gateway.Client, observation *cardapp.Service, integration *integrationlog.Repository) *SeriesRunner { - return &SeriesRunner{db: db, gateway: gatewayClient, observation: observation, integration: integration, carrier: postgres.NewCarrierStore(db)} +func NewSeriesRunner(db *gorm.DB, gatewayClient *gateway.Client, observation *cardapp.Service, integration *integrationlog.Repository, auditWriter *audit.Writer) *SeriesRunner { + return &SeriesRunner{ + db: db, gateway: gatewayClient, observation: observation, integration: integration, + carrier: postgres.NewCarrierStore(db), auditWriter: auditWriter, + } } // Provider 返回用于请求互斥的运营商接入标识。 @@ -188,7 +194,7 @@ func (r *SeriesRunner) runDeviceInfo(ctx context.Context, payload cardapp.Series response, runErr := r.gateway.SyncDeviceInfo(ctx, &gateway.SyncDeviceInfoReq{CardNo: deviceGatewayIdentifier(device)}) result := cardapp.RunResult{} if runErr == nil { - result.StateChanged, runErr = r.applyDeviceInfo(ctx, device, response) + result.StateChanged, runErr = r.applyDeviceInfo(ctx, device, response, attempt.IntegrationID) } completionResult := constants.IntegrationResultSuccess if runErr != nil { @@ -222,7 +228,7 @@ func (r *SeriesRunner) startDeviceAttempt(ctx context.Context, payload cardapp.S }) } -func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device, response *gateway.SyncDeviceInfoResp) (bool, error) { +func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device, response *gateway.SyncDeviceInfoResp, integrationID string) (bool, error) { if response == nil { return false, apperrors.New(apperrors.CodeGatewayError, "Gateway 设备信息响应为空") } @@ -236,21 +242,19 @@ func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device if lastOnlineTime := parseGatewayTime(response.LastOnlineTime); lastOnlineTime != nil { updates["last_online_time"] = lastOnlineTime } - var currentBinding struct { - SlotPosition int - } - currentSlotErr := r.db.WithContext(ctx).Model(&model.DeviceSimBinding{}). - Select("slot_position"). - Where("device_id = ? AND bind_status = ? AND is_current = ?", device.ID, constants.BindStatusBound, true). - Take(¤tBinding).Error - if currentSlotErr != nil && currentSlotErr != gorm.ErrRecordNotFound { - return false, apperrors.Wrap(apperrors.CodeDatabaseError, currentSlotErr, "读取设备当前槽位失败") - } - changed := device.OnlineStatus != int(response.OnlineStatus) || + deviceChanged := device.OnlineStatus != int(response.OnlineStatus) || device.SoftwareVersion != string(response.SoftwareVersion) || - device.SwitchMode != string(response.SwitchMode) || - currentBinding.SlotPosition != int(response.CurrentSlotNo) + device.SwitchMode != string(response.SwitchMode) + changed := deviceChanged err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + state, err := loadDeviceObservationState(ctx, tx, device.ID, int(response.CurrentSlotNo)) + if err != nil { + return err + } + beforeSlot := bindingSlot(state.current) + afterSlot := bindingSlot(state.target) + changed = deviceChanged || beforeSlot != int(response.CurrentSlotNo) + auditChanged := deviceChanged || beforeSlot != afterSlot if err := tx.Model(&model.Device{}).Where("id = ?", device.ID).Updates(updates).Error; err != nil { return err } @@ -259,19 +263,192 @@ func (r *SeriesRunner) applyDeviceInfo(ctx context.Context, device *model.Device Update("is_current", false).Error; err != nil { return err } - if int(response.CurrentSlotNo) <= 0 { + if int(response.CurrentSlotNo) > 0 { + if err := tx.Model(&model.DeviceSimBinding{}). + Where("device_id = ? AND slot_position = ? AND bind_status = ?", device.ID, int(response.CurrentSlotNo), constants.BindStatusBound). + Update("is_current", true).Error; err != nil { + return err + } + } + if !auditChanged { return nil } - return tx.Model(&model.DeviceSimBinding{}). - Where("device_id = ? AND slot_position = ? AND bind_status = ?", device.ID, int(response.CurrentSlotNo), constants.BindStatusBound). - Update("is_current", true).Error + return r.appendDeviceObservationAudit(ctx, tx, device, state, beforeSlot, afterSlot, integrationID, map[string]any{ + "online_status": device.OnlineStatus, "software_version": device.SoftwareVersion, + "switch_mode": device.SwitchMode, "current_slot": beforeSlot, + }, map[string]any{ + "online_status": int(response.OnlineStatus), "software_version": string(response.SoftwareVersion), + "switch_mode": string(response.SwitchMode), "current_slot": afterSlot, + }) }) if err != nil { - return false, apperrors.Wrap(apperrors.CodeDatabaseError, err, "回写设备 Gateway 信息失败") + wrapped := apperrors.Wrap(apperrors.CodeDatabaseError, err, "回写设备 Gateway 信息失败") + r.recordDeviceObservationFailure(ctx, device, integrationID, wrapped) + return false, wrapped } return changed, nil } +type deviceObservationState struct { + bindings []model.DeviceSimBinding + cards map[uint]*model.IotCard + current *model.DeviceSimBinding + target *model.DeviceSimBinding +} + +func loadDeviceObservationState(ctx context.Context, tx *gorm.DB, deviceID uint, targetSlot int) (*deviceObservationState, error) { + state := &deviceObservationState{cards: make(map[uint]*model.IotCard)} + if err := tx.WithContext(ctx).Where("device_id = ? AND bind_status = ?", deviceID, constants.BindStatusBound). + Order("slot_position ASC").Find(&state.bindings).Error; err != nil { + return nil, apperrors.Wrap(apperrors.CodeDatabaseError, err, "读取设备有效卡槽失败") + } + cardIDs := make([]uint, 0, len(state.bindings)) + for index := range state.bindings { + binding := &state.bindings[index] + cardIDs = append(cardIDs, binding.IotCardID) + if binding.IsCurrent && state.current == nil { + state.current = binding + } + if targetSlot > 0 && binding.SlotPosition == targetSlot { + state.target = binding + } + } + if len(cardIDs) == 0 { + return state, nil + } + var cards []*model.IotCard + if err := tx.WithContext(ctx).Where("id IN ?", cardIDs).Find(&cards).Error; err != nil { + return nil, apperrors.Wrap(apperrors.CodeDatabaseError, err, "读取设备卡槽关联 IoT 卡失败") + } + for _, card := range cards { + state.cards[card.ID] = card + } + return state, nil +} + +func bindingSlot(binding *model.DeviceSimBinding) int { + if binding == nil { + return 0 + } + return binding.SlotPosition +} + +func (r *SeriesRunner) appendDeviceObservationAudit( + ctx context.Context, + tx *gorm.DB, + device *model.Device, + state *deviceObservationState, + beforeSlot, afterSlot int, + integrationID string, + beforeData, afterData map[string]any, +) error { + if r.auditWriter == nil { + return apperrors.New(apperrors.CodeInternalError, "设备观测统一审计能力未配置") + } + deviceID := strconv.FormatUint(uint64(device.ID), 10) + resources := []audit.ResourceInput{{ + Type: constants.AuditResourceDevice, ID: &deviceID, + Key: audit.DeviceResourceKey(device), DisplayName: device.VirtualNo, + Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleDeviceTarget, + IdentitySnapshot: audit.DeviceIdentitySnapshot(device), BeforeData: beforeData, AfterData: afterData, + SubjectVisibility: constants.AuditSubjectResult, SubjectSummary: "Worker 同步设备观测事实", + }} + if beforeSlot != afterSlot { + resources = appendDeviceObservationBindingResources(resources, device, state.current, state.cards[stateCardID(state.current)], false, + constants.AuditResourceRoleDeviceOldCurrentCard, constants.AuditResourceRoleDeviceOldCurrentBinding) + resources = appendDeviceObservationBindingResources(resources, device, state.target, state.cards[stateCardID(state.target)], true, + constants.AuditResourceRoleDeviceNewCurrentCard, constants.AuditResourceRoleDeviceNewCurrentBinding) + } + resources = append(resources, deviceObservationIntegrationResource(ctx, device, integrationID)) + return r.auditWriter.Append(ctx, tx, audit.AppendInput{ + ActionCode: constants.AuditActionDeviceWorkerObservationSynced, + Summary: "Worker 同步设备观测事实", ScopeType: constants.AuditScopePlatform, + Result: constants.AuditResultSuccess, Resources: resources, + }) +} + +func appendDeviceObservationBindingResources( + resources []audit.ResourceInput, + device *model.Device, + binding *model.DeviceSimBinding, + card *model.IotCard, + afterCurrent bool, + cardRole, bindingRole string, +) []audit.ResourceInput { + if binding == nil { + return resources + } + if card != nil { + cardID := strconv.FormatUint(uint64(card.ID), 10) + resources = append(resources, audit.ResourceInput{ + Type: constants.AuditResourceIotCard, ID: &cardID, + Key: audit.IotCardResourceKey(card), DisplayName: card.ICCID, + Relation: constants.AuditResourceRelationAffected, Role: cardRole, + IdentitySnapshot: audit.IotCardIdentitySnapshot(card), + BeforeData: map[string]any{"is_current": binding.IsCurrent}, AfterData: map[string]any{"is_current": afterCurrent}, + SubjectVisibility: constants.AuditSubjectResult, SubjectSummary: "设备当前卡槽已同步", + }) + } + bindingID := strconv.FormatUint(uint64(binding.ID), 10) + identity := map[string]any{ + "id": binding.ID, "device_id": binding.DeviceID, "device_virtual_no": device.VirtualNo, + "slot_position": binding.SlotPosition, "iot_card_id": binding.IotCardID, "is_current": afterCurrent, + } + if card != nil { + identity["iccid"] = card.ICCID + identity["virtual_no"] = card.VirtualNo + } + return append(resources, audit.ResourceInput{ + Type: constants.AuditResourceDeviceSIMBinding, ID: &bindingID, + Key: bindingID, DisplayName: device.VirtualNo, + Relation: constants.AuditResourceRelationAffected, Role: bindingRole, + IdentitySnapshot: identity, + BeforeData: map[string]any{"is_current": binding.IsCurrent}, AfterData: map[string]any{"is_current": afterCurrent}, + SubjectVisibility: constants.AuditSubjectInternalOnly, + }) +} + +func stateCardID(binding *model.DeviceSimBinding) uint { + if binding == nil { + return 0 + } + return binding.IotCardID +} + +func deviceObservationIntegrationResource(ctx context.Context, device *model.Device, integrationID string) audit.ResourceInput { + deviceID := strconv.FormatUint(uint64(device.ID), 10) + return audit.ResourceInput{ + Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: integrationID, + Relation: constants.AuditResourceRelationReference, Role: constants.AuditResourceRoleWorkerIntegration, + IdentitySnapshot: map[string]any{ + "integration_id": integrationID, "provider": constants.IntegrationProviderGateway, + "direction": constants.IntegrationDirectionOutbound, "operation": constants.IntegrationOperationGatewayDeviceInfo, + "resource_type": constants.CardObservationResourceTypeDevice, "resource_id": deviceID, + "resource_key": "device:" + deviceID, "correlation_id": auditcontext.From(ctx).CorrelationID, + }, + SubjectVisibility: constants.AuditSubjectInternalOnly, + } +} + +func (r *SeriesRunner) recordDeviceObservationFailure(ctx context.Context, device *model.Device, integrationID string, businessErr error) { + deviceID := strconv.FormatUint(uint64(device.ID), 10) + r.auditWriter.RecordFailure(ctx, r.db, audit.AppendInput{ + ActionCode: constants.AuditActionDeviceWorkerObservationSynced, + Summary: "Worker 同步设备观测事实失败", ScopeType: constants.AuditScopePlatform, + Resources: []audit.ResourceInput{{ + Type: constants.AuditResourceDevice, ID: &deviceID, + Key: audit.DeviceResourceKey(device), DisplayName: device.VirtualNo, + Relation: constants.AuditResourceRelationPrimary, Role: constants.AuditResourceRoleDeviceTarget, + IdentitySnapshot: audit.DeviceIdentitySnapshot(device), + BeforeData: map[string]any{ + "online_status": device.OnlineStatus, "software_version": device.SoftwareVersion, + "switch_mode": device.SwitchMode, + }, + SubjectVisibility: constants.AuditSubjectResult, + }, deviceObservationIntegrationResource(ctx, device, integrationID)}, + }, businessErr) +} + func (r *SeriesRunner) loadDevice(ctx context.Context, payload cardapp.SeriesTaskPayload) (*model.Device, error) { if payload.ResourceType != constants.CardObservationResourceTypeDevice { return nil, apperrors.New(apperrors.CodeInvalidParam, "设备信息观测资源类型无效") diff --git a/internal/model/dto/audit_dto.go b/internal/model/dto/audit_dto.go index ea8754a..28c6a6c 100644 --- a/internal/model/dto/audit_dto.go +++ b/internal/model/dto/audit_dto.go @@ -62,18 +62,75 @@ type AuditResourceTimelineRequest struct { PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量，默认20，最大100"` } +// AuditRequestTimelineParams 是请求链路时间线的路径参数。 +type AuditRequestTimelineParams struct { + RequestID string `json:"request_id" path:"request_id" required:"true" description:"HTTP请求关联ID，来自审计或外部集成节点，也可从Access Log粘贴"` +} + +// AuditCorrelationTimelineParams 是业务关联时间线的路径参数。 +type AuditCorrelationTimelineParams struct { + CorrelationID string `json:"correlation_id" path:"correlation_id" required:"true" description:"跨请求、异步任务和外部交互的稳定业务链路ID"` +} + +// AuditFinanceTimelineRequest 是资金调查时间线的稳定业务筛选参数。 +type AuditFinanceTimelineRequest struct { + ShopID uint `json:"shop_id" query:"shop_id" description:"店铺ID"` + WalletID uint `json:"wallet_id" query:"wallet_id" description:"代理或资产钱包ID"` + OrderID uint `json:"order_id" query:"order_id" description:"订单ID"` + OrderNo string `json:"order_no" query:"order_no" description:"订单编号"` + PaymentID uint `json:"payment_id" query:"payment_id" description:"支付记录ID"` + PaymentNo string `json:"payment_no" query:"payment_no" description:"支付单号"` + RefundID uint `json:"refund_id" query:"refund_id" description:"退款单ID"` + RefundNo string `json:"refund_no" query:"refund_no" description:"退款单号"` + RechargeID uint `json:"recharge_id" query:"recharge_id" description:"代理充值或个人资产充值ID"` + RechargeNo string `json:"recharge_no" query:"recharge_no" description:"充值单号"` + ApprovalInstanceID uint `json:"approval_instance_id" query:"approval_instance_id" description:"审批实例ID"` + ThirdPartyTradeNo string `json:"third_party_trade_no" query:"third_party_trade_no" description:"第三方交易号"` + ActorKind string `json:"actor_kind" query:"actor_kind" description:"操作者类型；与actor_id同时提供"` + ActorID string `json:"actor_id" query:"actor_id" description:"操作者稳定ID；与actor_kind同时提供"` + CorrelationID string `json:"correlation_id" query:"correlation_id" description:"跨步骤业务链路ID"` + CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间（RFC3339，含时区）"` + CreatedTo string `json:"created_to" query:"created_to" description:"结束时间（RFC3339，含时区，不包含该时刻）"` + Page int `json:"page" query:"page" minimum:"1" description:"页码，默认1"` + PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量，默认20，最大100"` +} + +// AuditRiskFilterRequest 是风险总览和明细共用的受控筛选参数。 +type AuditRiskFilterRequest struct { + CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间（RFC3339，含时区）；默认从在线窗口开始"` + CreatedTo string `json:"created_to" query:"created_to" description:"结束时间（RFC3339，含时区，不包含该时刻，最长31天）；默认当前时间"` + Risk string `json:"risk" query:"risk" description:"风险等级 (low:低, normal:普通, high:高, critical:严重)"` + Result string `json:"result" query:"result" description:"结果 (success:成功, failed:失败, denied:拒绝, partial:部分成功, unknown:未知)"` + Action string `json:"action" query:"action" description:"稳定动作编码"` + Source string `json:"source" query:"source" description:"来源 (admin_api:后台管理API, personal_api:个人客户API, openapi:代理OpenAPI, worker:异步Worker, scheduler:计划任务, callback:外部系统回调)"` +} + +// AuditRiskOverviewRequest 是风险总览请求参数。 +type AuditRiskOverviewRequest struct { + AuditRiskFilterRequest +} + +// AuditRiskEventsRequest 是风险事件明细请求参数。 +type AuditRiskEventsRequest struct { + AuditRiskFilterRequest + Page int `json:"page" query:"page" minimum:"1" description:"页码，默认1"` + PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量，默认20，最大100"` +} + // SubjectResourceActivityRequest 是代理和企业安全资源活动的路径及分页参数。 type SubjectResourceActivityRequest struct { ResourceType string `json:"resource_type" path:"resource_type" required:"true" description:"资源类型 (iot_card:IoT卡, device:设备, asset_allocation_record:资产分配记录, exchange_order:换货单, shop:店铺, enterprise:企业)"` Identifier string `json:"identifier" path:"identifier" required:"true" description:"业务稳定标识；卡使用ICCID，设备使用VirtualNo，其他资源使用对应业务编号"` + CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间（RFC3339，含时区）；默认从在线窗口开始"` + CreatedTo string `json:"created_to" query:"created_to" description:"结束时间（RFC3339，含时区，不包含该时刻）；默认当前时间"` Page int `json:"page" query:"page" minimum:"1" description:"页码，默认1"` PageSize int `json:"page_size" query:"page_size" minimum:"1" maximum:"100" description:"每页数量，默认20，最大100"` } // IntegrationFilterRequest 是外部集成调查的公共受控筛选参数。 type IntegrationFilterRequest struct { - CreatedFrom string `json:"created_from" query:"created_from" required:"true" description:"开始时间（RFC3339，含时区，必填）"` - CreatedTo string `json:"created_to" query:"created_to" required:"true" description:"结束时间（RFC3339，含时区，不包含该时刻，必填）"` + CreatedFrom string `json:"created_from" query:"created_from" description:"开始时间（RFC3339，含时区）；默认从在线窗口开始"` + CreatedTo string `json:"created_to" query:"created_to" description:"结束时间（RFC3339，含时区，不包含该时刻）；默认当前时间"` IntegrationID string `json:"integration_id" query:"integration_id" description:"稳定外部集成记录ID"` Provider string `json:"provider" query:"provider" description:"外部服务提供方稳定编码"` Direction string `json:"direction" query:"direction" description:"交互方向 (inbound:入站, outbound:出站)"` diff --git a/internal/model/log_archive_run.go b/internal/model/log_archive_run.go new file mode 100644 index 0000000..6a96996 --- /dev/null +++ b/internal/model/log_archive_run.go @@ -0,0 +1,38 @@ +package model + +import "time" + +// LogArchiveRun 是日志冷归档运行账本，不保存日志正文。 +type LogArchiveRun struct { + ID uint `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + Source string `gorm:"column:source;type:varchar(32);not null" json:"source"` + ArchiveDate time.Time `gorm:"column:archive_date;type:date;not null" json:"archive_date"` + InstanceID string `gorm:"column:instance_id;type:varchar(100);not null" json:"instance_id"` + SchemaVersion string `gorm:"column:schema_version;type:varchar(32);not null" json:"schema_version"` + Revision int `gorm:"column:revision;not null;default:1" json:"revision"` + Status string `gorm:"column:status;type:varchar(16);not null" json:"status"` + IsFinal bool `gorm:"column:is_final;not null;default:false" json:"is_final"` + RangeStart time.Time `gorm:"column:range_start;type:timestamptz;not null" json:"range_start"` + RangeEnd time.Time `gorm:"column:range_end;type:timestamptz;not null" json:"range_end"` + ObjectKey string `gorm:"column:object_key;type:varchar(500);not null;default:''" json:"object_key"` + ManifestKey string `gorm:"column:manifest_key;type:varchar(500);not null;default:''" json:"manifest_key"` + EventCount int64 `gorm:"column:event_count;not null;default:0" json:"event_count"` + ResourceCount int64 `gorm:"column:resource_count;not null;default:0" json:"resource_count"` + RecordCount int64 `gorm:"column:record_count;not null;default:0" json:"record_count"` + UncompressedBytes int64 `gorm:"column:uncompressed_bytes;not null;default:0" json:"uncompressed_bytes"` + CompressedBytes int64 `gorm:"column:compressed_bytes;not null;default:0" json:"compressed_bytes"` + SHA256 string `gorm:"column:sha256;type:varchar(64);not null;default:''" json:"sha256"` + AttemptCount int `gorm:"column:attempt_count;not null;default:0" json:"attempt_count"` + ErrorSummary string `gorm:"column:error_summary;type:varchar(500);not null;default:''" json:"error_summary"` + GeneratedAt *time.Time `gorm:"column:generated_at" json:"generated_at,omitempty"` + CompletedAt *time.Time `gorm:"column:completed_at" json:"completed_at,omitempty"` + CleanupStartedAt *time.Time `gorm:"column:cleanup_started_at" json:"cleanup_started_at,omitempty"` + CleanedAt *time.Time `gorm:"column:cleaned_at" json:"cleaned_at,omitempty"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"` +} + +// TableName 返回日志冷归档运行账本表名。 +func (LogArchiveRun) TableName() string { + return "tb_log_archive_run" +} diff --git a/internal/polling/package_activation_handler.go b/internal/polling/package_activation_handler.go index 0d99d80..cc93f68 100644 --- a/internal/polling/package_activation_handler.go +++ b/internal/polling/package_activation_handler.go @@ -41,6 +41,13 @@ type PackageActivationHandler struct { logger *zap.Logger } +func workerCorrelation(correlationID string, task *asynq.Task) string { + if correlationID != "" { + return correlationID + } + return task.ResultWriter().TaskID() +} + // PackageActivationPayload 套餐激活任务载荷 type PackageActivationPayload struct { PackageUsageID uint `json:"package_usage_id"` @@ -48,6 +55,9 @@ type PackageActivationPayload struct { CarrierID uint `json:"carrier_id"` ActivationType string `json:"activation_type"` // "queue" 或 "realname" Timestamp int64 `json:"timestamp"` + RequestID string `json:"request_id,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + ParentEventID string `json:"parent_event_id,omitempty"` } // NewPackageActivationHandler 创建套餐激活检查处理器 @@ -397,10 +407,11 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c if h.stopResumeCallback == nil { return } + detachedCtx := context.WithoutCancel(ctx) if carrierType == "iot_card" { go func() { - if err := h.stopResumeCallback.CheckAndStopCard(context.Background(), carrierID); err != nil { + if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, carrierID); err != nil { h.logger.Error("套餐过期后停机失败", zap.Uint("card_id", carrierID), zap.Error(err)) @@ -420,7 +431,7 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c for _, b := range bindings { cardID := b.IotCardID go func(cID uint) { - if err := h.stopResumeCallback.CheckAndStopCard(context.Background(), cID); err != nil { + if err := h.stopResumeCallback.CheckAndStopCard(detachedCtx, cID); err != nil { h.logger.Error("套餐过期后停机失败", zap.Uint("card_id", cID), zap.Error(err)) @@ -432,12 +443,16 @@ func (h *PackageActivationHandler) triggerStopAfterExpiry(ctx context.Context, c // enqueueActivationTask 提交套餐激活任务到 Asynq func (h *PackageActivationHandler) enqueueActivationTask(ctx context.Context, packageUsageID uint, carrierType string, carrierID uint, activationType string) error { + linkage := auditcontext.From(ctx) payload := PackageActivationPayload{ PackageUsageID: packageUsageID, CarrierType: carrierType, CarrierID: carrierID, ActivationType: activationType, Timestamp: time.Now().Unix(), + RequestID: linkage.RequestID, + CorrelationID: linkage.CorrelationID, + ParentEventID: linkage.ParentEventID, } payloadBytes, err := sonic.Marshal(payload) @@ -477,6 +492,7 @@ func (h *PackageActivationHandler) HandlePackageQueueActivation(ctx context.Cont ctx = auditcontext.With(ctx, auditcontext.Context{ ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypePackageQueueActivation, ActorName: "套餐排队激活任务", Source: constants.AuditSourceWorker, + RequestID: payload.RequestID, CorrelationID: workerCorrelation(payload.CorrelationID, t), ParentEventID: payload.ParentEventID, }) h.logger.Info("开始执行套餐激活", @@ -534,6 +550,7 @@ func (h *PackageActivationHandler) HandlePackageFirstActivation(ctx context.Cont ctx = auditcontext.With(ctx, auditcontext.Context{ ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypePackageFirstActivation, ActorName: "套餐首次实名激活任务", Source: constants.AuditSourceWorker, + RequestID: payload.RequestID, CorrelationID: workerCorrelation(payload.CorrelationID, t), ParentEventID: payload.ParentEventID, }) if payload.CarrierType == "" || payload.CarrierID == 0 { diff --git a/internal/query/audit/events.go b/internal/query/audit/events.go index dd7a766..c6fa53e 100644 --- a/internal/query/audit/events.go +++ b/internal/query/audit/events.go @@ -10,6 +10,7 @@ import ( "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" "github.com/break/junhong_cmp_fiber/pkg/middleware" @@ -39,10 +40,17 @@ type EventFilter struct { // EventPage 是平台全局事件稳定分页结果。 type EventPage struct { - Total int64 `json:"total"` - Page int `json:"page"` - PageSize int `json:"page_size"` - Items []EventView `json:"items"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Items []EventView `json:"items"` + Retention retentionquery.Info `json:"retention"` +} + +// EventDetail 是单个审计事件及在线留存边界。 +type EventDetail struct { + EventView + Retention retentionquery.Info `json:"retention"` } // EventView 是不暴露 GORM Model 的审计事件投影。 @@ -147,6 +155,14 @@ func (q *Query) List(ctx context.Context, filter EventFilter) (*EventPage, error if err := q.authorize(ctx); err != nil { return nil, err } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo) + if err != nil { + return nil, err + } if !validEventFilter(filter) { return nil, errors.New(errors.CodeInvalidParam) } @@ -156,16 +172,33 @@ func (q *Query) List(ctx context.Context, filter EventFilter) (*EventPage, error if err := query.Count(&total).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计审计事件失败") } - rows := make([]model.AuditEvent, 0, filter.PageSize) - if err := query.Order("occurred_at DESC, id DESC"). - Offset((filter.Page - 1) * filter.PageSize).Limit(filter.PageSize).Find(&rows).Error; err != nil { - return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询审计事件失败") + rows, err := q.loadEventPage(ctx, query, filter.Page, filter.PageSize) + if err != nil { + return nil, err } items, err := q.project(ctx, rows) if err != nil { return nil, err } - return &EventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil + return &EventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil +} + +// loadEventPage 先分页主键，再批量读取事件宽行，避免排序阶段加载 JSON 字段。 +func (q *Query) loadEventPage(ctx context.Context, query *gorm.DB, page, pageSize int) ([]model.AuditEvent, error) { + ids := make([]uint, 0, pageSize) + if err := query.Select("id").Order("occurred_at DESC, id DESC"). + Offset((page-1)*pageSize).Limit(pageSize).Pluck("id", &ids).Error; err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询审计事件分页ID失败") + } + rows := make([]model.AuditEvent, 0, len(ids)) + if len(ids) == 0 { + return rows, nil + } + if err := q.db.WithContext(ctx).Where("id IN ?", ids). + Order("occurred_at DESC, id DESC").Find(&rows).Error; err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "批量投影审计事件失败") + } + return rows, nil } func validEventFilter(filter EventFilter) bool { @@ -192,15 +225,19 @@ func validOptionalValue(value string, allowed ...string) bool { } // Get 查询平台范围的单个稳定审计事件详情。 -func (q *Query) Get(ctx context.Context, eventID string) (*EventView, error) { +func (q *Query) Get(ctx context.Context, eventID string) (*EventDetail, error) { if err := q.authorize(ctx); err != nil { return nil, err } if eventID == "" { return nil, errors.New(errors.CodeInvalidParam) } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } var row model.AuditEvent - if err := q.db.WithContext(ctx).Where("event_id = ?", eventID).First(&row).Error; err != nil { + if err := q.db.WithContext(ctx).Where("event_id = ? AND occurred_at >= ?", eventID, retention.OnlineFrom.UTC()).First(&row).Error; err != nil { if err == gorm.ErrRecordNotFound { return nil, errors.New(errors.CodeNotFound, "审计事件不存在") } @@ -210,7 +247,7 @@ func (q *Query) Get(ctx context.Context, eventID string) (*EventView, error) { if err != nil { return nil, err } - return &items[0], nil + return &EventDetail{EventView: items[0], Retention: retention}, nil } func (q *Query) authorize(ctx context.Context) error { diff --git a/internal/query/audit/finance.go b/internal/query/audit/finance.go new file mode 100644 index 0000000..a0041b7 --- /dev/null +++ b/internal/query/audit/finance.go @@ -0,0 +1,1519 @@ +package audit + +import ( + "context" + "sort" + "strconv" + "strings" + "time" + + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// FinanceFilter 定义资金调查时间线的稳定业务筛选。 +type FinanceFilter struct { + ShopID uint + WalletID uint + OrderID uint + OrderNo string + PaymentID uint + PaymentNo string + RefundID uint + RefundNo string + RechargeID uint + RechargeNo string + ApprovalInstanceID uint + ThirdPartyTradeNo string + ActorKind string + ActorID string + CorrelationID string + CreatedFrom *time.Time + CreatedTo *time.Time + Page int + PageSize int +} + +// FinanceTimelinePage 是资金多源投影的稳定分页结果。 +type FinanceTimelinePage struct { + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Items []FinanceTimelineNode `json:"items"` + Retention retentionquery.Info `json:"retention"` +} + +// FinanceTimelineNode 是明确事实来源和金额权威的资金时间线节点。 +type FinanceTimelineNode struct { + RecordSource string `json:"record_source"` + NodeID string `json:"node_id"` + OccurredAt time.Time `json:"occurred_at"` + Code string `json:"code"` + Title string `json:"title"` + Result string `json:"result"` + ResultName string `json:"result_name"` + Amount *int64 `json:"amount"` + BalanceBefore *int64 `json:"balance_before"` + BalanceAfter *int64 `json:"balance_after"` + Currency string `json:"currency"` + ShopID *uint `json:"shop_id"` + Wallet *FinanceWalletRef `json:"wallet"` + AmountAuthority FinanceAmountAuthority `json:"amount_authority"` + Facts map[string]any `json:"facts"` + InvestigationRefs InvestigationRefs `json:"investigation_refs"` +} + +// FinanceWalletRef 是资金节点关联的钱包稳定引用。 +type FinanceWalletRef struct { + ResourceType string `json:"resource_type"` + WalletID uint `json:"wallet_id"` +} + +// FinanceAmountAuthority 说明当前金额是否为业务权威及其字段来源。 +type FinanceAmountAuthority struct { + Authoritative bool `json:"authoritative"` + Table string `json:"table"` + Field string `json:"field"` + ConflictRule string `json:"conflict_rule"` +} + +type financeRefs struct { + seeded bool + shops map[uint]struct{} + agentWallets map[uint]struct{} + assetWallets map[uint]struct{} + agentTxs map[uint]struct{} + assetTxs map[uint]struct{} + reservations map[uint]struct{} + orders map[uint]struct{} + orderNos map[string]struct{} + payments map[uint]struct{} + paymentNos map[string]struct{} + refunds map[uint]struct{} + refundNos map[string]struct{} + agentRecharges map[uint]struct{} + rechargeOrders map[uint]struct{} + rechargeNos map[string]struct{} + approvals map[uint]struct{} + commissions map[uint]struct{} + withdrawals map[uint]struct{} + tradeNos map[string]struct{} +} + +// FinanceTimeline 查询资金审计与业务账本的只读组合时间线。 +func (q *Query) FinanceTimeline(ctx context.Context, filter FinanceFilter) (*FinanceTimelinePage, error) { + if err := q.authorize(ctx); err != nil { + return nil, err + } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo) + if err != nil { + return nil, err + } + if !validFinanceFilter(filter) { + return nil, errors.New(errors.CodeInvalidParam) + } + filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize) + refs := newFinanceRefs(filter) + if err := q.seedFinanceActorAndCorrelation(ctx, filter, refs); err != nil { + return nil, err + } + if err := q.expandFinanceRefs(ctx, refs); err != nil { + return nil, err + } + auditRows, auditTotal, err := q.loadFinanceAuditRows(ctx, filter, refs) + if err != nil { + return nil, err + } + if err := q.addAuditFinanceRefs(ctx, auditRows, refs); err != nil { + return nil, err + } + if err := q.expandFinanceRefs(ctx, refs); err != nil { + return nil, err + } + + limit := filter.Page * filter.PageSize + nodes, total, err := q.loadFinanceNodes(ctx, filter, refs, auditRows, auditTotal, limit) + if err != nil { + return nil, err + } + sort.Slice(nodes, func(i, j int) bool { + if nodes[i].OccurredAt.Equal(nodes[j].OccurredAt) { + if nodes[i].RecordSource == nodes[j].RecordSource { + return financeNodeIDAfter(nodes[i].NodeID, nodes[j].NodeID) + } + return nodes[i].RecordSource > nodes[j].RecordSource + } + return nodes[i].OccurredAt.After(nodes[j].OccurredAt) + }) + start := (filter.Page - 1) * filter.PageSize + if start > len(nodes) { + start = len(nodes) + } + end := start + filter.PageSize + if end > len(nodes) { + end = len(nodes) + } + items := nodes[start:end] + if items == nil { + items = []FinanceTimelineNode{} + } + return &FinanceTimelinePage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil +} + +// validFinanceFilter 拒绝无条件全表扫描及不完整的操作者筛选。 +func validFinanceFilter(filter FinanceFilter) bool { + if filter.Page < 0 || filter.PageSize < 0 || filter.PageSize > constants.MaxPageSize { + return false + } + if filter.CreatedFrom != nil && filter.CreatedTo != nil && !filter.CreatedFrom.Before(*filter.CreatedTo) { + return false + } + if filter.ActorKind != "" && !validActorKind(filter.ActorKind) { + return false + } + if (filter.ActorKind == "") != (filter.ActorID == "") { + return false + } + hasStableCondition := filter.ShopID != 0 || filter.WalletID != 0 || filter.OrderID != 0 || filter.OrderNo != "" || + filter.PaymentID != 0 || filter.PaymentNo != "" || filter.RefundID != 0 || filter.RefundNo != "" || + filter.RechargeID != 0 || filter.RechargeNo != "" || filter.ApprovalInstanceID != 0 || + filter.ThirdPartyTradeNo != "" || filter.ActorID != "" || filter.CorrelationID != "" + return hasStableCondition || (filter.CreatedFrom != nil && filter.CreatedTo != nil) +} + +// newFinanceRefs 将调用方提供的稳定条件初始化为关联解析种子。 +func newFinanceRefs(filter FinanceFilter) *financeRefs { + refs := &financeRefs{ + shops: make(map[uint]struct{}), agentWallets: make(map[uint]struct{}), assetWallets: make(map[uint]struct{}), + agentTxs: make(map[uint]struct{}), assetTxs: make(map[uint]struct{}), reservations: make(map[uint]struct{}), + orders: make(map[uint]struct{}), orderNos: make(map[string]struct{}), payments: make(map[uint]struct{}), + paymentNos: make(map[string]struct{}), refunds: make(map[uint]struct{}), refundNos: make(map[string]struct{}), + agentRecharges: make(map[uint]struct{}), rechargeOrders: make(map[uint]struct{}), rechargeNos: make(map[string]struct{}), + approvals: make(map[uint]struct{}), commissions: make(map[uint]struct{}), withdrawals: make(map[uint]struct{}), + tradeNos: make(map[string]struct{}), + } + addUint(refs.shops, filter.ShopID) + addUint(refs.agentWallets, filter.WalletID) + addUint(refs.assetWallets, filter.WalletID) + addUint(refs.orders, filter.OrderID) + addString(refs.orderNos, filter.OrderNo) + addUint(refs.payments, filter.PaymentID) + addString(refs.paymentNos, filter.PaymentNo) + addUint(refs.refunds, filter.RefundID) + addString(refs.refundNos, filter.RefundNo) + addUint(refs.agentRecharges, filter.RechargeID) + addUint(refs.rechargeOrders, filter.RechargeID) + addString(refs.rechargeNos, filter.RechargeNo) + addUint(refs.approvals, filter.ApprovalInstanceID) + addString(refs.tradeNos, filter.ThirdPartyTradeNo) + refs.seeded = filter.ShopID != 0 || filter.WalletID != 0 || filter.OrderID != 0 || filter.OrderNo != "" || + filter.PaymentID != 0 || filter.PaymentNo != "" || filter.RefundID != 0 || filter.RefundNo != "" || + filter.RechargeID != 0 || filter.RechargeNo != "" || filter.ApprovalInstanceID != 0 || filter.ThirdPartyTradeNo != "" + return refs +} + +// seedFinanceActorAndCorrelation 使用明确持久化字段解析操作者和业务链路，不按时间邻近猜测。 +func (q *Query) seedFinanceActorAndCorrelation(ctx context.Context, filter FinanceFilter, refs *financeRefs) error { + if filter.CorrelationID != "" { + rows := []model.ApprovalInstance{} + if err := q.db.WithContext(ctx).Where("correlation_id = ?", filter.CorrelationID).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联审批链路失败") + } + for _, row := range rows { + addUint(refs.approvals, row.ID) + } + } + if filter.WalletID != 0 { + if err := q.seedFinanceWalletReferences(ctx, filter, refs); err != nil { + return err + } + } + if filter.ActorKind != constants.AuditActorAccount { + return nil + } + actorID64, err := strconv.ParseUint(filter.ActorID, 10, 64) + if err != nil { + return nil + } + actorID := uint(actorID64) + orderRows := []model.Order{} + orderQuery := applyFinanceTime(q.db.WithContext(ctx).Where("operator_account_id = ?", actorID), filter, "updated_at") + if err := orderQuery.Limit(filter.Page * filter.PageSize).Find(&orderRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析操作者关联订单失败") + } + for _, row := range orderRows { + addUint(refs.orders, row.ID) + } + agentRows := []model.AgentRechargeRecord{} + agentQuery := applyFinanceTime(q.db.WithContext(ctx).Where("user_id = ?", actorID), filter, "updated_at") + if err := agentQuery.Limit(filter.Page * filter.PageSize).Find(&agentRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析操作者关联代理充值失败") + } + for _, row := range agentRows { + addUint(refs.agentRecharges, row.ID) + } + rechargeRows := []model.RechargeOrder{} + rechargeQuery := applyFinanceTime(q.db.WithContext(ctx).Where("user_id = ?", actorID), filter, "updated_at") + if err := rechargeQuery.Limit(filter.Page * filter.PageSize).Find(&rechargeRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析操作者关联资产充值失败") + } + for _, row := range rechargeRows { + addUint(refs.rechargeOrders, row.ID) + } + approvalRows := []model.ApprovalInstance{} + approvalQuery := applyFinanceTime(q.db.WithContext(ctx).Where("submitter_account_id = ?", actorID), filter, "status_changed_at") + if err := approvalQuery.Limit(filter.Page * filter.PageSize).Find(&approvalRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析操作者关联审批失败") + } + for _, row := range approvalRows { + addUint(refs.approvals, row.ID) + } + return nil +} + +// seedFinanceWalletReferences 从当前分页窗口的唯一流水引用解析业务单据。 +func (q *Query) seedFinanceWalletReferences(ctx context.Context, filter FinanceFilter, refs *financeRefs) error { + limit := filter.Page * filter.PageSize + agentRows := []model.AgentWalletTransaction{} + agentQuery := applyFinanceTime(q.db.WithContext(ctx).Where("agent_wallet_id = ?", filter.WalletID), filter, "created_at") + if err := agentQuery.Order("created_at DESC, id DESC").Limit(limit).Find(&agentRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析代理钱包关联业务失败") + } + for _, row := range agentRows { + addUint(refs.agentTxs, row.ID) + if row.ReferenceType == nil || row.ReferenceID == nil { + continue + } + switch *row.ReferenceType { + case constants.ReferenceTypeOrder: + addUint(refs.orders, *row.ReferenceID) + case constants.ReferenceTypeRefund: + addUint(refs.refunds, *row.ReferenceID) + case constants.ReferenceTypeTopup: + addUint(refs.agentRecharges, *row.ReferenceID) + case constants.ReferenceTypeCommission: + addUint(refs.commissions, *row.ReferenceID) + case constants.ReferenceTypeWithdrawal: + addUint(refs.withdrawals, *row.ReferenceID) + } + } + assetRows := []model.AssetWalletTransaction{} + assetQuery := applyFinanceTime(q.db.WithContext(ctx).Where("asset_wallet_id = ?", filter.WalletID), filter, "created_at") + if err := assetQuery.Order("created_at DESC, id DESC").Limit(limit).Find(&assetRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资产钱包关联业务失败") + } + for _, row := range assetRows { + addUint(refs.assetTxs, row.ID) + if row.ReferenceType == nil || row.ReferenceNo == nil { + continue + } + switch *row.ReferenceType { + case constants.ReferenceTypeOrder: + addString(refs.orderNos, *row.ReferenceNo) + case constants.ReferenceTypeRefund: + addString(refs.refundNos, *row.ReferenceNo) + case constants.ReferenceTypeTopup: + addString(refs.rechargeNos, *row.ReferenceNo) + case constants.ReferenceTypeRecharge: + addString(refs.paymentNos, *row.ReferenceNo) + } + } + reservationRows := []model.AgentWalletReservation{} + reservationQuery := applyFinanceTime(q.db.WithContext(ctx).Where("agent_wallet_id = ?", filter.WalletID), filter, "updated_at") + if err := reservationQuery.Order("updated_at DESC, id DESC").Limit(limit).Find(&reservationRows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析代理钱包预占关联业务失败") + } + for _, row := range reservationRows { + addUint(refs.reservations, row.ID) + if row.ReferenceType == constants.ReferenceTypeOrder { + addUint(refs.orders, row.ReferenceID) + } + } + return nil +} + +// expandFinanceRefs 以固定轮次展开支付、订单、退款、充值和审批的确定性关系。 +func (q *Query) expandFinanceRefs(ctx context.Context, refs *financeRefs) error { + for range 2 { + if err := q.expandFinanceRecords(ctx, refs); err != nil { + return err + } + } + return nil +} + +// expandFinanceRecords 批量读取各业务表，避免按时间线节点逐条回查。 +func (q *Query) expandFinanceRecords(ctx context.Context, refs *financeRefs) error { + if err := q.expandOrders(ctx, refs); err != nil { + return err + } + if err := q.expandPayments(ctx, refs); err != nil { + return err + } + if err := q.expandRefunds(ctx, refs); err != nil { + return err + } + if err := q.expandRecharges(ctx, refs); err != nil { + return err + } + return q.expandApprovals(ctx, refs) +} + +// expandOrders 解析订单编号、店铺和后续支付所需的内部 ID。 +func (q *Query) expandOrders(ctx context.Context, refs *financeRefs) error { + conditions, args := make([]string, 0, 2), make([]any, 0, 2) + if len(refs.orders) > 0 { + conditions, args = append(conditions, "id IN ?"), append(args, uintKeys(refs.orders)) + } + if len(refs.orderNos) > 0 { + conditions, args = append(conditions, "order_no IN ?"), append(args, stringKeys(refs.orderNos)) + } + if len(conditions) == 0 { + return nil + } + rows := []model.Order{} + if err := q.db.WithContext(ctx).Where(strings.Join(conditions, " OR "), args...).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联订单失败") + } + for _, row := range rows { + addUint(refs.orders, row.ID) + addString(refs.orderNos, row.OrderNo) + } + return nil +} + +// expandPayments 解析支付单、渠道交易号及其明确业务单类型。 +func (q *Query) expandPayments(ctx context.Context, refs *financeRefs) error { + conditions, args := make([]string, 0, 5), make([]any, 0, 5) + if len(refs.payments) > 0 { + conditions, args = append(conditions, "id IN ?"), append(args, uintKeys(refs.payments)) + } + if len(refs.paymentNos) > 0 { + conditions, args = append(conditions, "payment_no IN ?"), append(args, stringKeys(refs.paymentNos)) + } + if len(refs.tradeNos) > 0 { + conditions, args = append(conditions, "third_party_trade_no IN ?"), append(args, stringKeys(refs.tradeNos)) + } + if len(refs.orders) > 0 { + conditions, args = append(conditions, "order_type = ? AND order_id IN ?"), append(args, model.PaymentOrderTypePackage, uintKeys(refs.orders)) + } + if len(refs.agentRecharges) > 0 { + conditions, args = append(conditions, "order_type = ? AND order_id IN ?"), append(args, model.PaymentOrderTypeAgentRecharge, uintKeys(refs.agentRecharges)) + } + if len(refs.rechargeOrders) > 0 { + conditions, args = append(conditions, "order_type = ? AND order_id IN ?"), append(args, model.PaymentOrderTypeRecharge, uintKeys(refs.rechargeOrders)) + } + if len(conditions) == 0 { + return nil + } + rows := []model.Payment{} + if err := q.db.WithContext(ctx).Where("("+strings.Join(conditions, ") OR (")+")", args...).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联支付失败") + } + for _, row := range rows { + addUint(refs.payments, row.ID) + addString(refs.paymentNos, row.PaymentNo) + addString(refs.tradeNos, row.ThirdPartyTradeNo) + switch row.OrderType { + case model.PaymentOrderTypePackage: + addUint(refs.orders, row.OrderID) + case model.PaymentOrderTypeAgentRecharge: + addUint(refs.agentRecharges, row.OrderID) + case model.PaymentOrderTypeRecharge: + addUint(refs.rechargeOrders, row.OrderID) + } + } + return nil +} + +// expandRefunds 解析退款与订单、审批的稳定关系。 +func (q *Query) expandRefunds(ctx context.Context, refs *financeRefs) error { + conditions, args := make([]string, 0, 4), make([]any, 0, 4) + if len(refs.refunds) > 0 { + conditions, args = append(conditions, "id IN ?"), append(args, uintKeys(refs.refunds)) + } + if len(refs.refundNos) > 0 { + conditions, args = append(conditions, "refund_no IN ?"), append(args, stringKeys(refs.refundNos)) + } + if len(refs.orders) > 0 { + conditions, args = append(conditions, "order_id IN ?"), append(args, uintKeys(refs.orders)) + } + if len(refs.approvals) > 0 { + conditions, args = append(conditions, "approval_instance_id IN ?"), append(args, uintKeys(refs.approvals)) + } + if len(conditions) == 0 { + return nil + } + rows := []model.RefundRequest{} + if err := q.db.WithContext(ctx).Where(strings.Join(conditions, " OR "), args...).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联退款失败") + } + for _, row := range rows { + addUint(refs.refunds, row.ID) + addString(refs.refundNos, row.RefundNo) + addUint(refs.orders, row.OrderID) + if row.ApprovalInstanceID != nil { + addUint(refs.approvals, *row.ApprovalInstanceID) + } + } + return nil +} + +// expandRecharges 同时解析代理充值和个人资产充值两类既有业务单。 +func (q *Query) expandRecharges(ctx context.Context, refs *financeRefs) error { + agentConditions, agentArgs := make([]string, 0, 4), make([]any, 0, 4) + if len(refs.agentRecharges) > 0 { + agentConditions, agentArgs = append(agentConditions, "id IN ?"), append(agentArgs, uintKeys(refs.agentRecharges)) + } + if len(refs.rechargeNos) > 0 { + agentConditions, agentArgs = append(agentConditions, "recharge_no IN ?"), append(agentArgs, stringKeys(refs.rechargeNos)) + } + if len(refs.tradeNos) > 0 { + agentConditions, agentArgs = append(agentConditions, "payment_transaction_id IN ?"), append(agentArgs, stringKeys(refs.tradeNos)) + } + if len(refs.approvals) > 0 { + agentConditions, agentArgs = append(agentConditions, "approval_instance_id IN ?"), append(agentArgs, uintKeys(refs.approvals)) + } + if len(agentConditions) > 0 { + rows := []model.AgentRechargeRecord{} + if err := q.db.WithContext(ctx).Where(strings.Join(agentConditions, " OR "), agentArgs...).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联代理充值失败") + } + for _, row := range rows { + addUint(refs.agentRecharges, row.ID) + addString(refs.rechargeNos, row.RechargeNo) + if row.ApprovalInstanceID != nil { + addUint(refs.approvals, *row.ApprovalInstanceID) + } + if row.PaymentTransactionID != nil { + addString(refs.tradeNos, *row.PaymentTransactionID) + } + } + } + personalConditions, personalArgs := make([]string, 0, 2), make([]any, 0, 2) + if len(refs.rechargeOrders) > 0 { + personalConditions, personalArgs = append(personalConditions, "id IN ?"), append(personalArgs, uintKeys(refs.rechargeOrders)) + } + if len(refs.rechargeNos) > 0 { + personalConditions, personalArgs = append(personalConditions, "recharge_order_no IN ?"), append(personalArgs, stringKeys(refs.rechargeNos)) + } + if len(personalConditions) == 0 { + return nil + } + rows := []model.RechargeOrder{} + if err := q.db.WithContext(ctx).Where(strings.Join(personalConditions, " OR "), personalArgs...).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联资产充值失败") + } + for _, row := range rows { + addUint(refs.rechargeOrders, row.ID) + addString(refs.rechargeNos, row.RechargeOrderNo) + } + return nil +} + +// expandApprovals 只按审批业务类型关联退款或线下代理充值。 +func (q *Query) expandApprovals(ctx context.Context, refs *financeRefs) error { + conditions, args := make([]string, 0, 3), make([]any, 0, 3) + if len(refs.approvals) > 0 { + conditions, args = append(conditions, "id IN ?"), append(args, uintKeys(refs.approvals)) + } + if len(refs.refunds) > 0 { + conditions, args = append(conditions, "business_type = ? AND business_id IN ?"), append(args, constants.ApprovalBusinessTypeRefund, uintKeys(refs.refunds)) + } + if len(refs.agentRecharges) > 0 { + conditions, args = append(conditions, "business_type = ? AND business_id IN ?"), append(args, constants.ApprovalBusinessTypeOfflineRecharge, uintKeys(refs.agentRecharges)) + } + if len(conditions) == 0 { + return nil + } + rows := []model.ApprovalInstance{} + if err := q.db.WithContext(ctx).Where("("+strings.Join(conditions, ") OR (")+")", args...).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金关联审批失败") + } + for _, row := range rows { + addUint(refs.approvals, row.ID) + switch row.BusinessType { + case constants.ApprovalBusinessTypeRefund: + addUint(refs.refunds, row.BusinessID) + case constants.ApprovalBusinessTypeOfflineRecharge: + addUint(refs.agentRecharges, row.BusinessID) + } + } + return nil +} + +// loadFinanceAuditRows 只读取具有资金资源的审计事件，并保留操作者权威。 +func (q *Query) loadFinanceAuditRows(ctx context.Context, filter FinanceFilter, refs *financeRefs) ([]model.AuditEvent, int64, error) { + resourceTypes := []string{ + constants.AuditResourceOrder, constants.AuditResourcePayment, constants.AuditResourceRefund, + constants.AuditResourceAgentRecharge, constants.AuditResourceRechargeOrder, + constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction, + constants.AuditResourceAgentWalletReservation, constants.AuditResourceAssetWallet, + constants.AuditResourceAssetWalletTransaction, constants.AuditResourceCommissionRecord, + constants.AuditResourceCommissionWithdrawal, constants.AuditResourceApprovalInstance, + } + query := q.db.WithContext(ctx).Model(&model.AuditEvent{}). + Where("EXISTS (?)", q.db.Table("tb_audit_event_resource AS finance_resource").Select("1"). + Where("finance_resource.audit_event_id = tb_audit_event.id"). + Where("finance_resource.resource_type IN ?", resourceTypes)) + query = applyFinanceTime(query, filter, "occurred_at") + if filter.ActorKind != "" { + query = query.Where("actor_kind = ? AND actor_id = ?", filter.ActorKind, filter.ActorID) + } + if filter.CorrelationID != "" { + query = query.Where("correlation_id = ?", filter.CorrelationID) + } + if refs.seeded { + predicate, args := financeResourcePredicate("matched_resource", refs) + if predicate == "" { + return []model.AuditEvent{}, 0, nil + } + query = query.Where("EXISTS (?)", q.db.Table("tb_audit_event_resource AS matched_resource").Select("1"). + Where("matched_resource.audit_event_id = tb_audit_event.id").Where(predicate, args...)) + } + var total int64 + if err := query.Count(&total).Error; err != nil { + return nil, 0, errors.Wrap(errors.CodeDatabaseError, err, "统计资金关联审计事件失败") + } + rows := []model.AuditEvent{} + if err := query.Order("occurred_at DESC, event_id DESC").Limit(filter.Page * filter.PageSize).Find(&rows).Error; err != nil { + return nil, 0, errors.Wrap(errors.CodeDatabaseError, err, "查询资金关联审计事件失败") + } + return rows, total, nil +} + +// addAuditFinanceRefs 从事件资源读取稳定业务 ID，不解析摘要或中文描述。 +func (q *Query) addAuditFinanceRefs(ctx context.Context, events []model.AuditEvent, refs *financeRefs) error { + if len(events) == 0 { + return nil + } + ids := make([]uint, 0, len(events)) + for _, event := range events { + ids = append(ids, event.ID) + } + rows := []model.AuditEventResource{} + if err := q.db.WithContext(ctx).Where("audit_event_id IN ?", ids).Find(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "解析资金审计资源失败") + } + for _, row := range rows { + id, ok := parseResourceUint(row.ResourceID) + switch row.ResourceType { + case constants.AuditResourceShop: + // 店铺只接受显式筛选，避免单笔业务链路扩散成整个店铺资金历史。 + case constants.AuditResourceOrder: + if ok { + addUint(refs.orders, id) + } + addString(refs.orderNos, row.ResourceKey) + case constants.AuditResourcePayment: + if ok { + addUint(refs.payments, id) + } + addString(refs.paymentNos, row.ResourceKey) + case constants.AuditResourceRefund: + if ok { + addUint(refs.refunds, id) + } + addString(refs.refundNos, row.ResourceKey) + case constants.AuditResourceAgentRecharge: + if ok { + addUint(refs.agentRecharges, id) + } + addString(refs.rechargeNos, row.ResourceKey) + case constants.AuditResourceRechargeOrder: + if ok { + addUint(refs.rechargeOrders, id) + } + addString(refs.rechargeNos, row.ResourceKey) + case constants.AuditResourceAgentWallet: + // 钱包只接受显式筛选，关联业务使用唯一流水资源继续解析。 + case constants.AuditResourceAssetWallet: + // 钱包只接受显式筛选，关联业务使用唯一流水资源继续解析。 + case constants.AuditResourceAgentWalletTransaction: + if ok { + addUint(refs.agentTxs, id) + } + case constants.AuditResourceAssetWalletTransaction: + if ok { + addUint(refs.assetTxs, id) + } + case constants.AuditResourceAgentWalletReservation: + if ok { + addUint(refs.reservations, id) + } + case constants.AuditResourceApprovalInstance: + if ok { + addUint(refs.approvals, id) + } + case constants.AuditResourceCommissionRecord: + if ok { + addUint(refs.commissions, id) + } + case constants.AuditResourceCommissionWithdrawal: + if ok { + addUint(refs.withdrawals, id) + } + } + } + return nil +} + +// financeResourcePredicate 生成仅包含已注册资金资源的参数化匹配条件。 +func financeResourcePredicate(alias string, refs *financeRefs) (string, []any) { + type resourceMatch struct { + resourceType string + ids map[uint]struct{} + keys map[string]struct{} + } + matches := []resourceMatch{ + {constants.AuditResourceShop, refs.shops, nil}, + {constants.AuditResourceOrder, refs.orders, refs.orderNos}, + {constants.AuditResourcePayment, refs.payments, refs.paymentNos}, + {constants.AuditResourceRefund, refs.refunds, refs.refundNos}, + {constants.AuditResourceAgentRecharge, refs.agentRecharges, refs.rechargeNos}, + {constants.AuditResourceRechargeOrder, refs.rechargeOrders, refs.rechargeNos}, + {constants.AuditResourceAgentWallet, refs.agentWallets, nil}, + {constants.AuditResourceAssetWallet, refs.assetWallets, nil}, + {constants.AuditResourceAgentWalletTransaction, refs.agentTxs, nil}, + {constants.AuditResourceAssetWalletTransaction, refs.assetTxs, nil}, + {constants.AuditResourceAgentWalletReservation, refs.reservations, nil}, + {constants.AuditResourceApprovalInstance, refs.approvals, nil}, + {constants.AuditResourceCommissionRecord, refs.commissions, nil}, + {constants.AuditResourceCommissionWithdrawal, refs.withdrawals, nil}, + } + conditions, args := make([]string, 0, len(matches)*2), make([]any, 0, len(matches)*2) + for _, match := range matches { + if len(match.ids) > 0 { + conditions = append(conditions, "("+alias+".resource_type = ? AND "+alias+".resource_id IN ?)") + args = append(args, match.resourceType, stringUintKeys(match.ids)) + } + if len(match.keys) > 0 { + conditions = append(conditions, "("+alias+".resource_type = ? AND "+alias+".resource_key IN ?)") + args = append(args, match.resourceType, stringKeys(match.keys)) + } + } + return strings.Join(conditions, " OR "), args +} + +// loadFinanceNodes 以固定查询数读取各事实源后统一分页，不产生节点级 N+1。 +func (q *Query) loadFinanceNodes(ctx context.Context, filter FinanceFilter, refs *financeRefs, auditRows []model.AuditEvent, auditTotal int64, limit int) ([]FinanceTimelineNode, int64, error) { + nodes, err := q.financeAuditNodes(ctx, auditRows) + if err != nil { + return nil, 0, err + } + total := auditTotal + loaders := []func() ([]FinanceTimelineNode, int64, error){ + func() ([]FinanceTimelineNode, int64, error) { + return q.loadAgentWalletFinance(ctx, filter, refs, limit) + }, + func() ([]FinanceTimelineNode, int64, error) { + return q.loadAssetWalletFinance(ctx, filter, refs, limit) + }, + func() ([]FinanceTimelineNode, int64, error) { + return q.loadReservationFinance(ctx, filter, refs, limit) + }, + func() ([]FinanceTimelineNode, int64, error) { return q.loadOrderFinance(ctx, filter, refs, limit) }, + func() ([]FinanceTimelineNode, int64, error) { return q.loadPaymentFinance(ctx, filter, refs, limit) }, + func() ([]FinanceTimelineNode, int64, error) { return q.loadRefundFinance(ctx, filter, refs, limit) }, + func() ([]FinanceTimelineNode, int64, error) { + return q.loadAgentRechargeFinance(ctx, filter, refs, limit) + }, + func() ([]FinanceTimelineNode, int64, error) { + return q.loadRechargeOrderFinance(ctx, filter, refs, limit) + }, + func() ([]FinanceTimelineNode, int64, error) { return q.loadCommissionFinance(ctx, filter, refs, limit) }, + func() ([]FinanceTimelineNode, int64, error) { return q.loadWithdrawalFinance(ctx, filter, refs, limit) }, + func() ([]FinanceTimelineNode, int64, error) { return q.loadApprovalFinance(ctx, filter, refs, limit) }, + } + for _, load := range loaders { + items, count, loadErr := load() + if loadErr != nil { + return nil, 0, loadErr + } + nodes = append(nodes, items...) + total += count + } + return nodes, total, nil +} + +// financeAuditNodes 投影操作者事实，明确其金额不是资金权威。 +func (q *Query) financeAuditNodes(ctx context.Context, rows []model.AuditEvent) ([]FinanceTimelineNode, error) { + events, err := q.project(ctx, rows) + if err != nil { + return nil, err + } + integrationRows := []model.IntegrationLog{} + ids := make([]uint, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + if len(ids) > 0 { + if err := q.db.WithContext(ctx).Where("audit_event_id IN ?", ids).Find(&integrationRows).Error; err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询资金审计外部交互引用失败") + } + } + integrationByAudit := integrationRefsByAuditID(integrationRows) + nodes := make([]FinanceTimelineNode, 0, len(events)) + for index, event := range events { + refs := event.InvestigationRefs + refs.IntegrationRefs = uniqueIntegrationRefs(append(refs.IntegrationRefs, integrationByAudit[rows[index].ID]...)) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceAuditEvent, NodeID: event.EventID, OccurredAt: event.OccurredAt, + Code: event.ActionCode, Title: event.ActionName, Result: event.Result, ResultName: auditResultName(event.Result), + AmountAuthority: nonAuthoritativeAuditAmount(), + Facts: map[string]any{"summary": event.Summary, "metadata": event.Metadata, "risk_level": event.RiskLevel}, + InvestigationRefs: refs, + }) + } + return nodes, nil +} + +// loadAgentWalletFinance 读取代理钱包金额与余额权威流水。 +func (q *Query) loadAgentWalletFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.AgentWalletTransaction{}) + query = applyFinanceTime(query, filter, "created_at") + conditions, args := make([]string, 0, 8), make([]any, 0, 8) + appendUintCondition(&conditions, &args, "id", refs.agentTxs) + appendUintCondition(&conditions, &args, "agent_wallet_id", refs.agentWallets) + appendUintCondition(&conditions, &args, "shop_id", refs.shops) + appendReferenceIDCondition(&conditions, &args, constants.ReferenceTypeOrder, refs.orders) + appendReferenceIDCondition(&conditions, &args, constants.ReferenceTypeRefund, refs.refunds) + appendReferenceIDCondition(&conditions, &args, constants.ReferenceTypeTopup, refs.agentRecharges) + appendReferenceIDCondition(&conditions, &args, constants.ReferenceTypeCommission, refs.commissions) + appendReferenceIDCondition(&conditions, &args, constants.ReferenceTypeWithdrawal, refs.withdrawals) + appendAccountActorCondition(&conditions, &args, filter, "user_id", "creator") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.AgentWalletTransaction](query, "created_at DESC, id DESC", limit, "代理钱包流水") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount, before, after := row.Amount, row.BalanceBefore, row.BalanceAfter + resourceID := strconv.FormatUint(uint64(row.ID), 10) + refsView := ledgerRefs(constants.AuditResourceAgentWalletTransaction, resourceID, resourceID, row.UserID) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceAgentWallet, row.AgentWalletID, "")) + refsView.ResourceRefs = append(refsView.ResourceRefs, agentTransactionBusinessRef(row)...) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceAgentWalletTransaction, NodeID: resourceID, OccurredAt: row.CreatedAt, + Code: row.TransactionType, Title: "代理钱包" + constants.GetAgentTransactionTypeName(row.TransactionType), + Result: strconv.Itoa(row.Status), ResultName: constants.GetTransactionStatusName(row.Status), + Amount: &amount, BalanceBefore: &before, BalanceAfter: &after, Currency: "CNY", ShopID: &row.ShopID, + Wallet: &FinanceWalletRef{ResourceType: constants.AuditResourceAgentWallet, WalletID: row.AgentWalletID}, + AmountAuthority: authoritativeAmount("tb_agent_wallet_transaction", "amount,balance_before,balance_after"), + Facts: map[string]any{"reference_type": row.ReferenceType, "reference_id": row.ReferenceID, "transaction_subtype": row.TransactionSubtype, "asset_type": row.AssetType, "asset_id": row.AssetID, "asset_identifier": row.AssetIdentifier}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadAssetWalletFinance 读取卡或设备钱包金额与余额权威流水。 +func (q *Query) loadAssetWalletFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.AssetWalletTransaction{}) + query = applyFinanceTime(query, filter, "created_at") + conditions, args := make([]string, 0, 6), make([]any, 0, 6) + appendUintCondition(&conditions, &args, "id", refs.assetTxs) + appendUintCondition(&conditions, &args, "asset_wallet_id", refs.assetWallets) + appendUintCondition(&conditions, &args, "shop_id_tag", refs.shops) + appendReferenceNoCondition(&conditions, &args, constants.ReferenceTypeOrder, refs.orderNos) + appendReferenceNoCondition(&conditions, &args, constants.ReferenceTypeRefund, refs.refundNos) + appendReferenceNoCondition(&conditions, &args, constants.ReferenceTypeTopup, refs.rechargeNos) + appendReferenceNoCondition(&conditions, &args, constants.ReferenceTypeRecharge, refs.paymentNos) + appendAccountActorCondition(&conditions, &args, filter, "user_id", "creator") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.AssetWalletTransaction](query, "created_at DESC, id DESC", limit, "资产钱包流水") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount, before, after := row.Amount, row.BalanceBefore, row.BalanceAfter + resourceID := strconv.FormatUint(uint64(row.ID), 10) + refsView := ledgerRefs(constants.AuditResourceAssetWalletTransaction, resourceID, resourceID, row.UserID) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceAssetWallet, row.AssetWalletID, "")) + refsView.ResourceRefs = append(refsView.ResourceRefs, assetTransactionBusinessRef(row)...) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceAssetWalletTransaction, NodeID: resourceID, OccurredAt: row.CreatedAt, + Code: row.TransactionType, Title: "资产钱包" + assetTransactionTypeName(row.TransactionType), + Result: strconv.Itoa(row.Status), ResultName: constants.GetTransactionStatusName(row.Status), + Amount: &amount, BalanceBefore: &before, BalanceAfter: &after, Currency: "CNY", ShopID: &row.ShopIDTag, + Wallet: &FinanceWalletRef{ResourceType: constants.AuditResourceAssetWallet, WalletID: row.AssetWalletID}, + AmountAuthority: authoritativeAmount("tb_asset_wallet_transaction", "amount,balance_before,balance_after"), + Facts: map[string]any{"reference_type": row.ReferenceType, "reference_no": row.ReferenceNo, "resource_type": row.ResourceType, "resource_id": row.ResourceID}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadReservationFinance 读取代理主钱包预占及唯一终态。 +func (q *Query) loadReservationFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.AgentWalletReservation{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 4), make([]any, 0, 4) + appendUintCondition(&conditions, &args, "id", refs.reservations) + appendUintCondition(&conditions, &args, "agent_wallet_id", refs.agentWallets) + appendUintCondition(&conditions, &args, "shop_id", refs.shops) + appendReservationReferenceCondition(&conditions, &args, constants.ReferenceTypeOrder, refs.orders) + appendAccountActorCondition(&conditions, &args, filter, "creator") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.AgentWalletReservation](query, "updated_at DESC, id DESC", limit, "代理钱包预占") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.Amount + resourceID := strconv.FormatUint(uint64(row.ID), 10) + refsView := ledgerRefs(constants.AuditResourceAgentWalletReservation, resourceID, row.ReferenceType+":"+strconv.FormatUint(uint64(row.ReferenceID), 10), row.Creator) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceAgentWallet, row.AgentWalletID, "")) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceAgentWalletReservation, NodeID: resourceID, OccurredAt: row.UpdatedAt, + Code: row.ReferenceType, Title: "代理钱包资金预占", Result: strconv.Itoa(row.Status), ResultName: reservationStatusName(row.Status), + Amount: &amount, Currency: "CNY", ShopID: &row.ShopID, + Wallet: &FinanceWalletRef{ResourceType: constants.AuditResourceAgentWallet, WalletID: row.AgentWalletID}, + AmountAuthority: authoritativeAmount("tb_agent_wallet_reservation", "amount"), + Facts: map[string]any{"reference_type": row.ReferenceType, "reference_id": row.ReferenceID, "completed_at": row.CompletedAt}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadOrderFinance 读取订单金额和当前支付事实。 +func (q *Query) loadOrderFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.Order{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 5), make([]any, 0, 5) + appendUintCondition(&conditions, &args, "id", refs.orders) + appendStringCondition(&conditions, &args, "order_no", refs.orderNos) + if len(refs.shops) > 0 { + conditions = append(conditions, "(buyer_type = ? AND buyer_id IN ?) OR seller_shop_id IN ?") + args = append(args, model.BuyerTypeAgent, uintKeys(refs.shops), uintKeys(refs.shops)) + } + appendAccountActorCondition(&conditions, &args, filter, "operator_account_id") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.Order](query, "updated_at DESC, id DESC", limit, "订单资金事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.TotalAmount + shopID := orderShopID(row) + refsView := ledgerRefs(constants.AuditResourceOrder, strconv.FormatUint(uint64(row.ID), 10), row.OrderNo, pointerUintValue(row.OperatorAccountID)) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceOrder, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: row.PaymentMethod, Title: "订单 " + row.OrderNo, Result: strconv.Itoa(row.PaymentStatus), ResultName: constants.GetOrderPaymentStatusName(row.PaymentStatus), + Amount: &amount, Currency: "CNY", ShopID: shopID, + AmountAuthority: authoritativeAmount("tb_order", "total_amount"), + Facts: map[string]any{"order_no": row.OrderNo, "actual_paid_amount": row.ActualPaidAmount, "payment_method": row.PaymentMethod, "buyer_type": row.BuyerType, "buyer_id": row.BuyerID, "commission_status": row.CommissionStatus, "commission_result": row.CommissionResult}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadPaymentFinance 读取支付金额、渠道交易号和当前支付状态。 +func (q *Query) loadPaymentFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.Payment{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 6), make([]any, 0, 6) + appendUintCondition(&conditions, &args, "id", refs.payments) + appendStringCondition(&conditions, &args, "payment_no", refs.paymentNos) + appendStringCondition(&conditions, &args, "third_party_trade_no", refs.tradeNos) + appendTypedOrderCondition(&conditions, &args, model.PaymentOrderTypePackage, refs.orders) + appendTypedOrderCondition(&conditions, &args, model.PaymentOrderTypeAgentRecharge, refs.agentRecharges) + appendTypedOrderCondition(&conditions, &args, model.PaymentOrderTypeRecharge, refs.rechargeOrders) + if filter.ShopID != 0 { + conditions = append(conditions, `(order_type = ? AND EXISTS (SELECT 1 FROM tb_order o WHERE o.id = tb_payment.order_id AND o.deleted_at IS NULL AND ((o.buyer_type = ? AND o.buyer_id = ?) OR o.seller_shop_id = ?))) OR (order_type = ? AND EXISTS (SELECT 1 FROM tb_agent_recharge_record ar WHERE ar.id = tb_payment.order_id AND ar.deleted_at IS NULL AND ar.shop_id = ?)) OR (order_type = ? AND EXISTS (SELECT 1 FROM tb_recharge_order ro WHERE ro.id = tb_payment.order_id AND ro.deleted_at IS NULL AND ro.shop_id_tag = ?))`) + args = append(args, model.PaymentOrderTypePackage, model.BuyerTypeAgent, filter.ShopID, filter.ShopID, model.PaymentOrderTypeAgentRecharge, filter.ShopID, model.PaymentOrderTypeRecharge, filter.ShopID) + } + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.Payment](query, "updated_at DESC, id DESC", limit, "支付资金事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.Amount + refsView := ledgerRefs(constants.AuditResourcePayment, strconv.FormatUint(uint64(row.ID), 10), row.PaymentNo, 0) + refsView.ResourceRefs = append(refsView.ResourceRefs, paymentBusinessRefs(row)...) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourcePayment, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: row.PaymentMethod, Title: "支付单 " + row.PaymentNo, Result: strconv.Itoa(row.Status), ResultName: constants.GetPaymentRecordStatusName(row.Status), + Amount: &amount, Currency: "CNY", AmountAuthority: authoritativeAmount("tb_payment", "amount"), + Facts: map[string]any{"payment_no": row.PaymentNo, "order_type": row.OrderType, "order_id": row.OrderID, "third_party_trade_no": row.ThirdPartyTradeNo, "paid_at": row.PaidAt}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadRefundFinance 优先展示已批准金额，否则展示申请金额并声明实际字段。 +func (q *Query) loadRefundFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.RefundRequest{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 6), make([]any, 0, 6) + appendUintCondition(&conditions, &args, "id", refs.refunds) + appendStringCondition(&conditions, &args, "refund_no", refs.refundNos) + appendUintCondition(&conditions, &args, "order_id", refs.orders) + appendUintCondition(&conditions, &args, "approval_instance_id", refs.approvals) + appendUintCondition(&conditions, &args, "shop_id", refs.shops) + appendAccountActorCondition(&conditions, &args, filter, "processor_id") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.RefundRequest](query, "updated_at DESC, id DESC", limit, "退款资金事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount, field := row.RequestedRefundAmount, "requested_refund_amount" + if row.ApprovedRefundAmount != nil { + amount, field = *row.ApprovedRefundAmount, "approved_refund_amount" + } + refsView := ledgerRefs(constants.AuditResourceRefund, strconv.FormatUint(uint64(row.ID), 10), row.RefundNo, pointerUintValue(row.ProcessorID)) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceOrder, row.OrderID, row.OrderNo)) + if row.ApprovalInstanceID != nil { + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceApprovalInstance, *row.ApprovalInstanceID, "")) + } + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceRefund, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: "refund", Title: "退款单 " + row.RefundNo, Result: strconv.Itoa(row.Status), ResultName: constants.GetRefundStatusName(row.Status), + Amount: &amount, Currency: "CNY", ShopID: row.ShopID, AmountAuthority: authoritativeAmount("tb_refund_request", field), + Facts: map[string]any{"refund_no": row.RefundNo, "order_id": row.OrderID, "order_no": row.OrderNo, "actual_received_amount": row.ActualReceivedAmount, "requested_refund_amount": row.RequestedRefundAmount, "approved_refund_amount": row.ApprovedRefundAmount, "approval_instance_id": row.ApprovalInstanceID}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadAgentRechargeFinance 读取代理充值金额、支付和审批事实。 +func (q *Query) loadAgentRechargeFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.AgentRechargeRecord{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 7), make([]any, 0, 7) + appendUintCondition(&conditions, &args, "id", refs.agentRecharges) + appendStringCondition(&conditions, &args, "recharge_no", refs.rechargeNos) + appendStringCondition(&conditions, &args, "payment_transaction_id", refs.tradeNos) + appendUintCondition(&conditions, &args, "agent_wallet_id", refs.agentWallets) + appendUintCondition(&conditions, &args, "approval_instance_id", refs.approvals) + appendUintCondition(&conditions, &args, "shop_id", refs.shops) + appendAccountActorCondition(&conditions, &args, filter, "user_id") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.AgentRechargeRecord](query, "updated_at DESC, id DESC", limit, "代理充值事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.Amount + refsView := ledgerRefs(constants.AuditResourceAgentRecharge, strconv.FormatUint(uint64(row.ID), 10), row.RechargeNo, row.UserID) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceAgentWallet, row.AgentWalletID, "")) + if row.ApprovalInstanceID != nil { + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceApprovalInstance, *row.ApprovalInstanceID, "")) + } + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceAgentRecharge, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: row.PaymentMethod, Title: "代理充值单 " + row.RechargeNo, Result: strconv.Itoa(row.Status), ResultName: constants.GetRechargeStatusName(row.Status), + Amount: &amount, Currency: "CNY", ShopID: &row.ShopID, + Wallet: &FinanceWalletRef{ResourceType: constants.AuditResourceAgentWallet, WalletID: row.AgentWalletID}, + AmountAuthority: authoritativeAmount("tb_agent_recharge_record", "amount"), + Facts: map[string]any{"recharge_no": row.RechargeNo, "payment_method": row.PaymentMethod, "payment_transaction_id": row.PaymentTransactionID, "approval_instance_id": row.ApprovalInstanceID, "paid_at": row.PaidAt, "completed_at": row.CompletedAt}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadRechargeOrderFinance 读取个人资产充值金额及自动购包状态。 +func (q *Query) loadRechargeOrderFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.RechargeOrder{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 5), make([]any, 0, 5) + appendUintCondition(&conditions, &args, "id", refs.rechargeOrders) + appendStringCondition(&conditions, &args, "recharge_order_no", refs.rechargeNos) + appendUintCondition(&conditions, &args, "asset_wallet_id", refs.assetWallets) + appendUintCondition(&conditions, &args, "shop_id_tag", refs.shops) + appendAccountActorCondition(&conditions, &args, filter, "user_id") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.RechargeOrder](query, "updated_at DESC, id DESC", limit, "资产充值事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.Amount + refsView := ledgerRefs(constants.AuditResourceRechargeOrder, strconv.FormatUint(uint64(row.ID), 10), row.RechargeOrderNo, row.UserID) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceAssetWallet, row.AssetWalletID, "")) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceRechargeOrder, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: row.OperatorType, Title: "资产充值单 " + row.RechargeOrderNo, Result: strconv.Itoa(row.Status), ResultName: rechargeOrderStatusName(row.Status), + Amount: &amount, Currency: "CNY", ShopID: &row.ShopIDTag, + Wallet: &FinanceWalletRef{ResourceType: constants.AuditResourceAssetWallet, WalletID: row.AssetWalletID}, + AmountAuthority: authoritativeAmount("tb_recharge_order", "amount"), + Facts: map[string]any{"recharge_order_no": row.RechargeOrderNo, "resource_type": row.ResourceType, "resource_id": row.ResourceID, "paid_at": row.PaidAt, "auto_purchase_status": row.AutoPurchaseStatus}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadCommissionFinance 读取佣金金额、状态和入账后余额。 +func (q *Query) loadCommissionFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.CommissionRecord{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 4), make([]any, 0, 4) + appendUintCondition(&conditions, &args, "id", refs.commissions) + appendUintCondition(&conditions, &args, "order_id", refs.orders) + appendUintCondition(&conditions, &args, "shop_id", refs.shops) + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.CommissionRecord](query, "updated_at DESC, id DESC", limit, "佣金事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.Amount + refsView := ledgerRefs(constants.AuditResourceCommissionRecord, strconv.FormatUint(uint64(row.ID), 10), strconv.FormatUint(uint64(row.ID), 10), 0) + refsView.ResourceRefs = append(refsView.ResourceRefs, financeResourceRef(constants.AuditResourceOrder, row.OrderID, "")) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceCommissionRecord, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: row.CommissionSource, Title: "佣金记录", Result: strconv.Itoa(row.Status), ResultName: constants.GetCommissionRecordStatusName(row.Status), + Amount: &amount, BalanceAfter: int64Pointer(row.BalanceAfter), Currency: "CNY", ShopID: &row.ShopID, + AmountAuthority: authoritativeAmount("tb_commission_record", "amount,balance_after"), + Facts: map[string]any{"order_id": row.OrderID, "commission_source": row.CommissionSource, "released_at": row.ReleasedAt}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadWithdrawalFinance 读取提现申请金额、手续费和实际到账金额。 +func (q *Query) loadWithdrawalFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.CommissionWithdrawalRequest{}) + query = applyFinanceTime(query, filter, "updated_at") + conditions, args := make([]string, 0, 4), make([]any, 0, 4) + appendUintCondition(&conditions, &args, "id", refs.withdrawals) + appendUintCondition(&conditions, &args, "shop_id", refs.shops) + appendAccountActorCondition(&conditions, &args, filter, "applicant_id", "processor_id") + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.CommissionWithdrawalRequest](query, "updated_at DESC, id DESC", limit, "佣金提现事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + amount := row.Amount + refsView := ledgerRefs(constants.AuditResourceCommissionWithdrawal, strconv.FormatUint(uint64(row.ID), 10), row.WithdrawalNo, row.ApplicantID) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceCommissionWithdrawal, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.UpdatedAt, + Code: row.WithdrawalMethod, Title: "佣金提现单 " + row.WithdrawalNo, Result: strconv.Itoa(row.Status), ResultName: constants.GetWithdrawalStatusName(row.Status), + Amount: &amount, Currency: "CNY", ShopID: &row.ShopID, + AmountAuthority: authoritativeAmount("tb_commission_withdrawal_request", "amount"), + Facts: map[string]any{"fee": row.Fee, "actual_amount": row.ActualAmount, "payment_type": row.PaymentType, "processed_at": row.ProcessedAt, "paid_at": row.PaidAt}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +// loadApprovalFinance 读取审批状态，明确审批表不提供金额权威。 +func (q *Query) loadApprovalFinance(ctx context.Context, filter FinanceFilter, refs *financeRefs, limit int) ([]FinanceTimelineNode, int64, error) { + query := q.db.WithContext(ctx).Model(&model.ApprovalInstance{}) + query = applyFinanceTime(query, filter, "status_changed_at") + conditions, args := make([]string, 0, 5), make([]any, 0, 5) + appendUintCondition(&conditions, &args, "id", refs.approvals) + appendApprovalBusinessCondition(&conditions, &args, constants.ApprovalBusinessTypeRefund, refs.refunds) + appendApprovalBusinessCondition(&conditions, &args, constants.ApprovalBusinessTypeOfflineRecharge, refs.agentRecharges) + if filter.ShopID != 0 { + conditions = append(conditions, `(business_type = ? AND EXISTS (SELECT 1 FROM tb_refund_request r WHERE r.id = tb_approval_instance.business_id AND r.deleted_at IS NULL AND r.shop_id = ?)) OR (business_type = ? AND EXISTS (SELECT 1 FROM tb_agent_recharge_record ar WHERE ar.id = tb_approval_instance.business_id AND ar.deleted_at IS NULL AND ar.shop_id = ?))`) + args = append(args, constants.ApprovalBusinessTypeRefund, filter.ShopID, constants.ApprovalBusinessTypeOfflineRecharge, filter.ShopID) + } + appendAccountActorCondition(&conditions, &args, filter, "submitter_account_id") + if filter.CorrelationID != "" { + conditions, args = append(conditions, "correlation_id = ?"), append(args, filter.CorrelationID) + } + query = applyFinanceRelationship(query, filter, conditions, args) + rows, total, err := loadFinanceRows[model.ApprovalInstance](query, "status_changed_at DESC, id DESC", limit, "审批事实") + if err != nil { + return nil, 0, err + } + nodes := make([]FinanceTimelineNode, 0, len(rows)) + for _, row := range rows { + refsView := ledgerRefs(constants.AuditResourceApprovalInstance, strconv.FormatUint(uint64(row.ID), 10), strconv.FormatUint(uint64(row.ID), 10), row.SubmitterAccountID) + refsView.CorrelationID = stringPointer(row.CorrelationID) + refsView.ResourceRefs = append(refsView.ResourceRefs, approvalBusinessRefs(row)...) + nodes = append(nodes, FinanceTimelineNode{ + RecordSource: constants.AuditRecordSourceApprovalInstance, NodeID: strconv.FormatUint(uint64(row.ID), 10), OccurredAt: row.StatusChangedAt, + Code: row.BusinessType, Title: "审批实例", Result: strconv.Itoa(row.Status), ResultName: constants.GetApprovalStatusName(row.Status), + AmountAuthority: FinanceAmountAuthority{Authoritative: false, Table: "tb_approval_instance", ConflictRule: "审批表只对审批状态负责，不提供金额权威"}, + Facts: map[string]any{"business_type": row.BusinessType, "business_id": row.BusinessID, "provider": row.Provider, "external_ref": row.ExternalRef, "correlation_id": row.CorrelationID}, + InvestigationRefs: refsView, + }) + } + return nodes, total, nil +} + +func loadFinanceRows[T any](query *gorm.DB, order string, limit int, sourceName string) ([]T, int64, error) { + var total int64 + if err := query.Count(&total).Error; err != nil { + return nil, 0, errors.Wrap(errors.CodeDatabaseError, err, "统计"+sourceName+"失败") + } + rows := make([]T, 0, limit) + if err := query.Order(order).Limit(limit).Find(&rows).Error; err != nil { + return nil, 0, errors.Wrap(errors.CodeDatabaseError, err, "查询"+sourceName+"失败") + } + return rows, total, nil +} + +func applyFinanceTime(query *gorm.DB, filter FinanceFilter, column string) *gorm.DB { + if filter.CreatedFrom != nil { + query = query.Where(column+" >= ?", filter.CreatedFrom.UTC()) + } + if filter.CreatedTo != nil { + query = query.Where(column+" < ?", filter.CreatedTo.UTC()) + } + return query +} + +func applyFinanceRelationship(query *gorm.DB, filter FinanceFilter, conditions []string, args []any) *gorm.DB { + if !financeRelationshipRequired(filter) { + return query + } + if len(conditions) == 0 { + return query.Where("1 = 0") + } + return query.Where("("+strings.Join(conditions, ") OR (")+")", args...) +} + +func financeRelationshipRequired(filter FinanceFilter) bool { + return filter.ShopID != 0 || filter.WalletID != 0 || filter.OrderID != 0 || filter.OrderNo != "" || + filter.PaymentID != 0 || filter.PaymentNo != "" || filter.RefundID != 0 || filter.RefundNo != "" || + filter.RechargeID != 0 || filter.RechargeNo != "" || filter.ApprovalInstanceID != 0 || + filter.ThirdPartyTradeNo != "" || filter.ActorKind != "" || filter.CorrelationID != "" +} + +func appendUintCondition(conditions *[]string, args *[]any, column string, values map[uint]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, column+" IN ?") + *args = append(*args, uintKeys(values)) +} + +func appendStringCondition(conditions *[]string, args *[]any, column string, values map[string]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, column+" IN ?") + *args = append(*args, stringKeys(values)) +} + +func appendReferenceIDCondition(conditions *[]string, args *[]any, referenceType string, values map[uint]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, "reference_type = ? AND reference_id IN ?") + *args = append(*args, referenceType, uintKeys(values)) +} + +func appendReferenceNoCondition(conditions *[]string, args *[]any, referenceType string, values map[string]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, "reference_type = ? AND reference_no IN ?") + *args = append(*args, referenceType, stringKeys(values)) +} + +func appendReservationReferenceCondition(conditions *[]string, args *[]any, referenceType string, values map[uint]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, "reference_type = ? AND reference_id IN ?") + *args = append(*args, referenceType, uintKeys(values)) +} + +func appendTypedOrderCondition(conditions *[]string, args *[]any, orderType string, values map[uint]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, "order_type = ? AND order_id IN ?") + *args = append(*args, orderType, uintKeys(values)) +} + +func appendApprovalBusinessCondition(conditions *[]string, args *[]any, businessType string, values map[uint]struct{}) { + if len(values) == 0 { + return + } + *conditions = append(*conditions, "business_type = ? AND business_id IN ?") + *args = append(*args, businessType, uintKeys(values)) +} + +func appendAccountActorCondition(conditions *[]string, args *[]any, filter FinanceFilter, columns ...string) { + if filter.ActorKind != constants.AuditActorAccount || filter.ActorID == "" { + return + } + actorID, err := strconv.ParseUint(filter.ActorID, 10, 64) + if err != nil { + return + } + for _, column := range columns { + *conditions = append(*conditions, column+" = ?") + *args = append(*args, uint(actorID)) + } +} + +func ledgerRefs(resourceType, resourceID, resourceKey string, actorID uint) InvestigationRefs { + refs := InvestigationRefs{ + ResourceRefs: []InvestigationResourceRef{{ResourceType: resourceType, ResourceID: stringPointer(resourceID), ResourceKey: resourceKey, DisplayName: resourceKey}}, + IntegrationRefs: []IntegrationRef{}, + } + if actorID != 0 { + refs.ActorRef = &ActorRef{Kind: constants.AuditActorAccount, ID: strconv.FormatUint(uint64(actorID), 10)} + } + return refs +} + +func financeResourceRef(resourceType string, resourceID uint, resourceKey string) InvestigationResourceRef { + id := strconv.FormatUint(uint64(resourceID), 10) + if resourceKey == "" { + resourceKey = id + } + return InvestigationResourceRef{ResourceType: resourceType, ResourceID: &id, ResourceKey: resourceKey, DisplayName: resourceKey} +} + +// agentTransactionBusinessRef 将显式 reference_type 转为业务资源引用。 +func agentTransactionBusinessRef(row model.AgentWalletTransaction) []InvestigationResourceRef { + if row.ReferenceType == nil || row.ReferenceID == nil { + return nil + } + resourceType := "" + switch *row.ReferenceType { + case constants.ReferenceTypeOrder: + resourceType = constants.AuditResourceOrder + case constants.ReferenceTypeRefund: + resourceType = constants.AuditResourceRefund + case constants.ReferenceTypeTopup: + resourceType = constants.AuditResourceAgentRecharge + case constants.ReferenceTypeCommission: + resourceType = constants.AuditResourceCommissionRecord + case constants.ReferenceTypeWithdrawal: + resourceType = constants.AuditResourceCommissionWithdrawal + } + if resourceType == "" { + return nil + } + return []InvestigationResourceRef{financeResourceRef(resourceType, *row.ReferenceID, "")} +} + +// assetTransactionBusinessRef 保留只有业务编号而没有内部 ID 的历史引用。 +func assetTransactionBusinessRef(row model.AssetWalletTransaction) []InvestigationResourceRef { + if row.ReferenceType == nil || row.ReferenceNo == nil || *row.ReferenceNo == "" { + return nil + } + resourceType := "" + switch *row.ReferenceType { + case constants.ReferenceTypeOrder: + resourceType = constants.AuditResourceOrder + case constants.ReferenceTypeRefund: + resourceType = constants.AuditResourceRefund + case constants.ReferenceTypeRecharge: + resourceType = constants.AuditResourcePayment + } + if resourceType == "" { + return nil + } + return []InvestigationResourceRef{{ResourceType: resourceType, ResourceKey: *row.ReferenceNo, DisplayName: *row.ReferenceNo}} +} + +// paymentBusinessRefs 按支付单声明的 order_type 定位业务单类型。 +func paymentBusinessRefs(row model.Payment) []InvestigationResourceRef { + resourceType := "" + switch row.OrderType { + case model.PaymentOrderTypePackage: + resourceType = constants.AuditResourceOrder + case model.PaymentOrderTypeAgentRecharge: + resourceType = constants.AuditResourceAgentRecharge + case model.PaymentOrderTypeRecharge: + resourceType = constants.AuditResourceRechargeOrder + } + if resourceType == "" { + return nil + } + return []InvestigationResourceRef{financeResourceRef(resourceType, row.OrderID, "")} +} + +// approvalBusinessRefs 按审批实例声明的业务类型生成稳定跳转。 +func approvalBusinessRefs(row model.ApprovalInstance) []InvestigationResourceRef { + resourceType := "" + switch row.BusinessType { + case constants.ApprovalBusinessTypeRefund: + resourceType = constants.AuditResourceRefund + case constants.ApprovalBusinessTypeOfflineRecharge: + resourceType = constants.AuditResourceAgentRecharge + } + if resourceType == "" { + return nil + } + return []InvestigationResourceRef{financeResourceRef(resourceType, row.BusinessID, "")} +} + +func authoritativeAmount(table, field string) FinanceAmountAuthority { + return FinanceAmountAuthority{ + Authoritative: true, Table: table, Field: field, + ConflictRule: "金额冲突时以该业务表字段为准，不修改历史 Audit Event", + } +} + +func nonAuthoritativeAuditAmount() FinanceAmountAuthority { + return FinanceAmountAuthority{ + Authoritative: false, Table: "tb_audit_event", + ConflictRule: "Audit Event 只解释操作者与动作，金额以钱包流水及对应业务表为准", + } +} + +func auditResultName(result string) string { + switch result { + case constants.AuditResultSuccess: + return "成功" + case constants.AuditResultFailed: + return "失败" + case constants.AuditResultDenied: + return "拒绝" + case constants.AuditResultPartial: + return "部分成功" + case constants.AuditResultUnknown: + return "结果未知" + default: + return "未知" + } +} + +func reservationStatusName(status int) string { + switch status { + case constants.AgentWalletReservationStatusFrozen: + return "已冻结" + case constants.AgentWalletReservationStatusReleased: + return "已释放" + case constants.AgentWalletReservationStatusCompleted: + return "已完成扣除" + default: + return "未知" + } +} + +func assetTransactionTypeName(transactionType string) string { + switch transactionType { + case constants.AssetTransactionTypeRecharge: + return "充值" + case constants.AssetTransactionTypeDeduct: + return "扣款" + case constants.AssetTransactionTypeRefund: + return "退款" + case constants.AssetTransactionTypeExchange: + return "换货迁移" + default: + return "未知变动" + } +} + +func rechargeOrderStatusName(status int) string { + switch status { + case model.RechargeOrderStatusPending: + return "待支付" + case model.RechargeOrderStatusPaid: + return "已支付" + case model.RechargeOrderStatusClosed: + return "已关闭" + case model.RechargeOrderStatusRefunded: + return "已退款" + default: + return "未知" + } +} + +func orderShopID(row model.Order) *uint { + if row.BuyerType == model.BuyerTypeAgent { + id := row.BuyerID + return &id + } + return row.SellerShopID +} + +func pointerUintValue(value *uint) uint { + if value == nil { + return 0 + } + return *value +} + +func int64Pointer(value int64) *int64 { + return &value +} + +func parseResourceUint(value *string) (uint, bool) { + if value == nil || *value == "" { + return 0, false + } + parsed, err := strconv.ParseUint(*value, 10, 64) + if err != nil { + return 0, false + } + return uint(parsed), true +} + +func addUint(values map[uint]struct{}, value uint) { + if value != 0 { + values[value] = struct{}{} + } +} + +func addString(values map[string]struct{}, value string) { + value = strings.TrimSpace(value) + if value != "" { + values[value] = struct{}{} + } +} + +func uintKeys(values map[uint]struct{}) []uint { + keys := make([]uint, 0, len(values)) + for value := range values { + keys = append(keys, value) + } + return keys +} + +func stringUintKeys(values map[uint]struct{}) []string { + keys := make([]string, 0, len(values)) + for value := range values { + keys = append(keys, strconv.FormatUint(uint64(value), 10)) + } + return keys +} + +func stringKeys(values map[string]struct{}) []string { + keys := make([]string, 0, len(values)) + for value := range values { + keys = append(keys, value) + } + return keys +} + +func financeNodeIDAfter(left, right string) bool { + leftID, leftErr := strconv.ParseUint(left, 10, 64) + rightID, rightErr := strconv.ParseUint(right, 10, 64) + if leftErr == nil && rightErr == nil { + return leftID > rightID + } + return left > right +} diff --git a/internal/query/audit/resources.go b/internal/query/audit/resources.go index f2a5bf3..eb60573 100644 --- a/internal/query/audit/resources.go +++ b/internal/query/audit/resources.go @@ -9,6 +9,7 @@ import ( "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" ) @@ -17,16 +18,18 @@ import ( type ResourceSearchFilter struct { ResourceType string Keyword string + OnlineFrom time.Time Page int PageSize int } // ResourceSearchPage 是资源候选稳定分页结果。 type ResourceSearchPage struct { - Total int64 `json:"total"` - Page int `json:"page"` - PageSize int `json:"page_size"` - Items []ResourceCandidate `json:"items"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Items []ResourceCandidate `json:"items"` + Retention retentionquery.Info `json:"retention"` } // ResourceCandidate 是当前业务表或历史事件快照解析出的稳定资源候选。 @@ -59,6 +62,11 @@ func (q *Query) SearchResources(ctx context.Context, filter ResourceSearchFilter if filter.Keyword == "" || !searchableResourceType(filter.ResourceType) { return nil, errors.New(errors.CodeInvalidParam) } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.OnlineFrom = retention.OnlineFrom filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize) items, total, err := q.searchCurrent(ctx, filter) if err != nil { @@ -70,7 +78,7 @@ func (q *Query) SearchResources(ctx context.Context, filter ResourceSearchFilter return nil, err } } - return &ResourceSearchPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil + return &ResourceSearchPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil } // ResourceTimeline 查询注册资源作为任意关系参与的统一事件时间线。 @@ -196,7 +204,7 @@ func (q *Query) searchHistorical(ctx context.Context, filter ResourceSearchFilte func (q *Query) historicalIdentifierQuery(ctx context.Context, filter ResourceSearchFilter) *gorm.DB { query := q.db.WithContext(ctx).Model(&model.AuditEventResource{}). - Where("resource_type = ? AND resource_id IS NOT NULL", filter.ResourceType) + Where("resource_type = ? AND resource_id IS NOT NULL AND created_at >= ?", filter.ResourceType, filter.OnlineFrom.UTC()) switch filter.ResourceType { case constants.AuditResourceIotCard: return query.Where("resource_key = ? OR identity_snapshot ->> 'iccid' = ? OR identity_snapshot ->> 'iccid_19' = ? OR identity_snapshot ->> 'iccid_20' = ? OR identity_snapshot ->> 'virtual_no' = ?", filter.Keyword, filter.Keyword, filter.Keyword, filter.Keyword, filter.Keyword) diff --git a/internal/query/audit/risks.go b/internal/query/audit/risks.go new file mode 100644 index 0000000..dea5935 --- /dev/null +++ b/internal/query/audit/risks.go @@ -0,0 +1,299 @@ +package audit + +import ( + "context" + "time" + + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// RiskFilter 定义固定风险调查视角的时间范围与筛选条件。 +type RiskFilter struct { + CreatedFrom *time.Time + CreatedTo *time.Time + Risk string + Result string + Action string + Source string + Page int + PageSize int +} + +// RiskOverview 是风险信号、固定维度与时间趋势的只读聚合。 +type RiskOverview struct { + Total int64 `json:"total"` + Bucket string `json:"bucket"` + Signals []RiskNamedCount `json:"signals"` + Risks []RiskNamedCount `json:"risks"` + Results []RiskNamedCount `json:"results"` + Actions []RiskNamedCount `json:"actions"` + Sources []RiskNamedCount `json:"sources"` + Trend []RiskTrendPoint `json:"trend"` + Retention retentionquery.Info `json:"retention"` +} + +// RiskNamedCount 是风险聚合维度的稳定编码、中文名称和数量。 +type RiskNamedCount struct { + Code string `json:"code"` + Name string `json:"name"` + Count int64 `json:"count"` +} + +// RiskTrendPoint 是固定时间桶内的风险信号趋势。 +type RiskTrendPoint struct { + BucketAt time.Time `json:"bucket_at"` + Total int64 `json:"total"` + HighRisk int64 `json:"high_risk"` + Finance int64 `json:"finance"` + Security int64 `json:"security"` + Failed int64 `json:"failed"` + Denied int64 `json:"denied"` + Partial int64 `json:"partial"` + Unknown int64 `json:"unknown"` +} + +// RiskEventPage 是风险事件的稳定分页结果。 +type RiskEventPage struct { + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Items []EventView `json:"items"` + Retention retentionquery.Info `json:"retention"` +} + +// RiskOverview 查询指定时间范围内的固定风险调查总览。 +func (q *Query) RiskOverview(ctx context.Context, filter RiskFilter) (*RiskOverview, error) { + if err := q.authorize(ctx); err != nil { + return nil, err + } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.AuditRiskQueryMaxRange) + if err != nil { + return nil, err + } + if !validRiskFilter(filter) { + return nil, errors.New(errors.CodeInvalidParam) + } + + base := q.applyRiskFilters(q.db.WithContext(ctx).Model(&model.AuditEvent{}), filter) + result := &RiskOverview{ + Bucket: riskTrendBucket(*filter.CreatedFrom, *filter.CreatedTo), + Signals: []RiskNamedCount{}, Risks: []RiskNamedCount{}, Results: []RiskNamedCount{}, + Actions: []RiskNamedCount{}, Sources: []RiskNamedCount{}, Trend: []RiskTrendPoint{}, + Retention: retention, + } + if err := loadRiskSignals(base, result); err != nil { + return nil, err + } + if err := loadRiskDimension(base, "risk_level", result); err != nil { + return nil, err + } + if err := loadRiskDimension(base, "result", result); err != nil { + return nil, err + } + if err := loadRiskDimension(base, "action_code", result); err != nil { + return nil, err + } + if err := loadRiskDimension(base, "source", result); err != nil { + return nil, err + } + if err := loadRiskTrend(base, result); err != nil { + return nil, err + } + return result, nil +} + +// RiskEvents 查询指定时间范围内的风险事件明细。 +func (q *Query) RiskEvents(ctx context.Context, filter RiskFilter) (*RiskEventPage, error) { + if err := q.authorize(ctx); err != nil { + return nil, err + } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.AuditRiskQueryMaxRange) + if err != nil { + return nil, err + } + if !validRiskFilter(filter) { + return nil, errors.New(errors.CodeInvalidParam) + } + filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize) + query := q.applyRiskFilters(q.db.WithContext(ctx).Model(&model.AuditEvent{}), filter) + var total int64 + if err := query.Count(&total).Error; err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计风险事件失败") + } + rows, err := q.loadEventPage(ctx, query, filter.Page, filter.PageSize) + if err != nil { + return nil, err + } + items, err := q.project(ctx, rows) + if err != nil { + return nil, err + } + return &RiskEventPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil +} + +func validRiskFilter(filter RiskFilter) bool { + if filter.CreatedFrom == nil || filter.CreatedTo == nil || !filter.CreatedFrom.Before(*filter.CreatedTo) || + filter.CreatedTo.Sub(*filter.CreatedFrom) > constants.AuditRiskQueryMaxRange { + return false + } + return validEventFilter(EventFilter{ + Risk: filter.Risk, Result: filter.Result, Action: filter.Action, Source: filter.Source, + Page: filter.Page, PageSize: filter.PageSize, + }) +} + +func (q *Query) applyRiskFilters(query *gorm.DB, filter RiskFilter) *gorm.DB { + query = q.applyFilters(query, EventFilter{ + CreatedFrom: filter.CreatedFrom, CreatedTo: filter.CreatedTo, + Risk: filter.Risk, Result: filter.Result, Action: filter.Action, Source: filter.Source, + }) + return query.Where(riskScopeSQL(), riskLevels(), constants.AuditCategorySecurity, abnormalResults(), financeResourceTypes()) +} + +func loadRiskSignals(query *gorm.DB, result *RiskOverview) error { + var row struct { + Total, HighRisk, Finance, Security, Failed, Denied, Partial, Unknown int64 + } + err := query.Select(`COUNT(*) AS total, + COUNT(*) FILTER (WHERE risk_level IN ?) AS high_risk, + COUNT(*) FILTER (WHERE EXISTS (SELECT 1 FROM tb_audit_event_resource aer WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?)) AS finance, + COUNT(*) FILTER (WHERE category = ?) AS security, + COUNT(*) FILTER (WHERE result = ?) AS failed, + COUNT(*) FILTER (WHERE result = ?) AS denied, + COUNT(*) FILTER (WHERE result = ?) AS partial, + COUNT(*) FILTER (WHERE result = ?) AS unknown`, riskLevels(), financeResourceTypes(), constants.AuditCategorySecurity, + constants.AuditResultFailed, constants.AuditResultDenied, constants.AuditResultPartial, constants.AuditResultUnknown).Scan(&row).Error + if err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险信号失败") + } + result.Total = row.Total + result.Signals = []RiskNamedCount{ + {Code: constants.AuditRiskSignalHighRisk, Name: "高风险", Count: row.HighRisk}, + {Code: constants.AuditRiskSignalFinance, Name: "资金", Count: row.Finance}, + {Code: constants.AuditRiskSignalSecurity, Name: "安全", Count: row.Security}, + {Code: constants.AuditRiskSignalFailed, Name: "失败", Count: row.Failed}, + {Code: constants.AuditRiskSignalDenied, Name: "拒绝", Count: row.Denied}, + {Code: constants.AuditRiskSignalPartial, Name: "部分成功", Count: row.Partial}, + {Code: constants.AuditRiskSignalUnknown, Name: "结果未知", Count: row.Unknown}, + } + return nil +} + +func loadRiskDimension(query *gorm.DB, column string, result *RiskOverview) error { + var rows []struct { + Code string + Name string + Count int64 + } + selectClause := column + " AS code, '' AS name, COUNT(*) AS count" + groupClause := column + if column == "action_code" { + selectClause = "action_code AS code, action_name AS name, COUNT(*) AS count" + groupClause = "action_code, action_name" + } + if err := query.Select(selectClause).Group(groupClause).Order(column + " ASC").Scan(&rows).Error; err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险维度失败") + } + items := make([]RiskNamedCount, 0, len(rows)) + for _, row := range rows { + name := row.Name + if name == "" { + name = riskDimensionName(column, row.Code) + } + items = append(items, RiskNamedCount{Code: row.Code, Name: name, Count: row.Count}) + } + switch column { + case "risk_level": + result.Risks = items + case "result": + result.Results = items + case "action_code": + result.Actions = items + case "source": + result.Sources = items + } + return nil +} + +func loadRiskTrend(query *gorm.DB, result *RiskOverview) error { + err := query.Select(`date_trunc(?, occurred_at) AS bucket_at, COUNT(*) AS total, + COUNT(*) FILTER (WHERE risk_level IN ?) AS high_risk, + COUNT(*) FILTER (WHERE EXISTS (SELECT 1 FROM tb_audit_event_resource aer WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ?)) AS finance, + COUNT(*) FILTER (WHERE category = ?) AS security, + COUNT(*) FILTER (WHERE result = ?) AS failed, + COUNT(*) FILTER (WHERE result = ?) AS denied, + COUNT(*) FILTER (WHERE result = ?) AS partial, + COUNT(*) FILTER (WHERE result = ?) AS unknown`, result.Bucket, riskLevels(), financeResourceTypes(), + constants.AuditCategorySecurity, constants.AuditResultFailed, constants.AuditResultDenied, + constants.AuditResultPartial, constants.AuditResultUnknown). + Group("bucket_at").Order("bucket_at ASC").Scan(&result.Trend).Error + if err != nil { + return errors.Wrap(errors.CodeDatabaseError, err, "聚合风险趋势失败") + } + return nil +} + +func riskScopeSQL() string { + return `(risk_level IN ? OR category = ? OR result IN ? OR EXISTS ( + SELECT 1 FROM tb_audit_event_resource aer + WHERE aer.audit_event_id = tb_audit_event.id AND aer.resource_type IN ? + ))` +} + +func riskLevels() []string { + return []string{constants.AuditRiskHigh, constants.AuditRiskCritical} +} + +func abnormalResults() []string { + return []string{constants.AuditResultFailed, constants.AuditResultDenied, constants.AuditResultPartial, constants.AuditResultUnknown} +} + +func financeResourceTypes() []string { + return []string{ + constants.AuditResourceOrder, constants.AuditResourceRefund, constants.AuditResourceAgentRecharge, + constants.AuditResourceRechargeOrder, constants.AuditResourceAssetWallet, constants.AuditResourceAssetWalletTransaction, + constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction, + constants.AuditResourceAgentWalletReservation, constants.AuditResourcePayment, + constants.AuditResourceCommissionRecord, constants.AuditResourceCommissionWithdrawal, + } +} + +func riskTrendBucket(from, to time.Time) string { + if to.Sub(from) <= constants.AuditRiskHourlyTrendMaxRange { + return "hour" + } + return "day" +} + +func riskDimensionName(column, code string) string { + names := map[string]map[string]string{ + "risk_level": { + constants.AuditRiskLow: "低", constants.AuditRiskNormal: "普通", + constants.AuditRiskHigh: "高", constants.AuditRiskCritical: "严重", + }, + "result": { + constants.AuditResultSuccess: "成功", constants.AuditResultFailed: "失败", + constants.AuditResultDenied: "拒绝", constants.AuditResultPartial: "部分成功", + constants.AuditResultUnknown: "结果未知", + }, + "source": { + constants.AuditSourceAdminAPI: "后台管理 API", constants.AuditSourcePersonalAPI: "个人客户 API", + constants.AuditSourceOpenAPI: "代理 OpenAPI", constants.AuditSourceWorker: "异步 Worker", + constants.AuditSourceScheduler: "计划任务", constants.AuditSourceCallback: "外部系统回调", + }, + } + return names[column][code] +} diff --git a/internal/query/audit/subject_activities.go b/internal/query/audit/subject_activities.go index aaa58b6..c685cce 100644 --- a/internal/query/audit/subject_activities.go +++ b/internal/query/audit/subject_activities.go @@ -9,6 +9,7 @@ import ( "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" "github.com/break/junhong_cmp_fiber/pkg/middleware" @@ -18,17 +19,20 @@ import ( type SubjectActivityFilter struct { ResourceType string Identifier string + CreatedFrom *time.Time + CreatedTo *time.Time Page int PageSize int } // SubjectActivityPage 是不包含平台调查字段的代理资源活动分页结果。 type SubjectActivityPage struct { - Resource SubjectResourceSummary `json:"resource"` - Total int64 `json:"total"` - Page int `json:"page"` - PageSize int `json:"page_size"` - Items []SubjectActivity `json:"items"` + Resource SubjectResourceSummary `json:"resource"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Items []SubjectActivity `json:"items"` + Retention retentionquery.Info `json:"retention"` } // SubjectActivity 是写入时已生成的主体安全活动投影。 @@ -78,11 +82,19 @@ func (q *Query) AgentResourceActivities(ctx context.Context, filter SubjectActiv return nil, errors.New(errors.CodeInvalidParam) } filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize) + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo) + if err != nil { + return nil, err + } target, err := q.resolveAgentTarget(ctx, filter.ResourceType, filter.Identifier, shopIDs) if err != nil { return nil, err } - return q.subjectActivitiesForTarget(ctx, filter, target, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) { + return q.subjectActivitiesForTarget(ctx, filter, target, retention, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) { return q.agentAllowedResourceIDs(ctx, resources, shopIDs) }) } @@ -100,21 +112,30 @@ func (q *Query) EnterpriseResourceActivities(ctx context.Context, filter Subject return nil, errors.New(errors.CodeInvalidParam) } filter.Page, filter.PageSize = normalizePage(filter.Page, filter.PageSize) + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit) + if err != nil { + return nil, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo) + if err != nil { + return nil, err + } target, err := q.resolveEnterpriseTarget(ctx, filter.ResourceType, filter.Identifier, enterpriseID) if err != nil { return nil, err } - return q.subjectActivitiesForTarget(ctx, filter, target, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) { + return q.subjectActivitiesForTarget(ctx, filter, target, retention, func(ctx context.Context, resources []model.AuditEventResource) (map[string]bool, error) { return q.enterpriseAllowedResourceIDs(ctx, resources, enterpriseID) }) } -func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectActivityFilter, target subjectTarget, authorize subjectResourceAuthorizer) (*SubjectActivityPage, error) { +func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectActivityFilter, target subjectTarget, retention retentionquery.Info, authorize subjectResourceAuthorizer) (*SubjectActivityPage, error) { resourceMatch := q.db.Table("tb_audit_event_resource AS target").Select("1"). Where("target.audit_event_id = tb_audit_event.id AND target.resource_type = ? AND target.resource_id = ?", filter.ResourceType, target.id). Where("target.subject_visibility IN ?", []string{constants.AuditSubjectResult, constants.AuditSubjectDetail}) - base := q.db.WithContext(ctx).Model(&model.AuditEvent{}).Where("EXISTS (?)", resourceMatch) + base := q.db.WithContext(ctx).Model(&model.AuditEvent{}). + Where("occurred_at >= ? AND occurred_at < ?", filter.CreatedFrom.UTC(), filter.CreatedTo.UTC()).Where("EXISTS (?)", resourceMatch) var total int64 if err := base.Count(&total).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计代理资源活动失败") @@ -131,7 +152,7 @@ func (q *Query) subjectActivitiesForTarget(ctx context.Context, filter SubjectAc if err != nil { return nil, err } - return &SubjectActivityPage{Resource: target.summary, Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil + return &SubjectActivityPage{Resource: target.summary, Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil } func (q *Query) resolveEnterpriseTarget(ctx context.Context, resourceType, identifier string, enterpriseID uint) (subjectTarget, error) { diff --git a/internal/query/audit/timeline.go b/internal/query/audit/timeline.go new file mode 100644 index 0000000..4ce2fd9 --- /dev/null +++ b/internal/query/audit/timeline.go @@ -0,0 +1,317 @@ +package audit + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// LinkTimeline 是 request 或 correlation 的跨事实只读时间线。 +type LinkTimeline struct { + RequestID *string `json:"request_id"` + CorrelationID *string `json:"correlation_id"` + AccessLogLookupRequestID *string `json:"access_log_lookup_request_id"` + Nodes []LinkTimelineNode `json:"nodes"` + Retention retentionquery.Info `json:"retention"` +} + +// LinkTimelineNode 是保留各事实源权威边界的时间线节点。 +type LinkTimelineNode struct { + RecordSource string `json:"record_source"` + NodeID string `json:"node_id"` + OccurredAt time.Time `json:"occurred_at"` + Code string `json:"code"` + Title string `json:"title"` + Result string `json:"result"` + ResultName string `json:"result_name"` + Summary string `json:"summary"` + ReferenceOnly bool `json:"reference_only"` + RequestID *string `json:"request_id"` + CorrelationID *string `json:"correlation_id"` + ParentEventID *string `json:"parent_event_id"` + Resources []InvestigationResourceRef `json:"resources"` + InvestigationRefs InvestigationRefs `json:"investigation_refs"` + Fidelity LinkageFidelity `json:"fidelity"` +} + +// LinkageFidelity 明确节点已有的稳定关联能力，不补猜历史缺失字段。 +type LinkageFidelity struct { + RequestAvailable bool `json:"request_available"` + CorrelationAvailable bool `json:"correlation_available"` + ParentEventAvailable bool `json:"parent_event_available"` + DirectAuditLinkAvailable bool `json:"direct_audit_link_available"` + StableResourceAvailable bool `json:"stable_resource_available"` +} + +// RequestTimeline 按精确 request ID 组合已持久化事实，不扫描 Access Log。 +func (q *Query) RequestTimeline(ctx context.Context, requestID string) (*LinkTimeline, error) { + return q.linkTimeline(ctx, "request_id", requestID) +} + +// CorrelationTimeline 按精确 correlation ID 组合跨请求业务链路。 +func (q *Query) CorrelationTimeline(ctx context.Context, correlationID string) (*LinkTimeline, error) { + return q.linkTimeline(ctx, "correlation_id", correlationID) +} + +func (q *Query) linkTimeline(ctx context.Context, column, value string) (*LinkTimeline, error) { + if err := q.authorize(ctx); err != nil { + return nil, err + } + if value == "" || (column != "request_id" && column != "correlation_id") { + return nil, errors.New(errors.CodeInvalidParam) + } + + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceAudit, retentionquery.SourceIntegration) + if err != nil { + return nil, err + } + auditRows, integrationRows, outboxRows, err := q.loadLinkRows(ctx, column, value, retention.OnlineFrom) + if err != nil { + return nil, err + } + events, err := q.project(ctx, auditRows) + if err != nil { + return nil, err + } + + nodes := make([]LinkTimelineNode, 0, len(events)+len(integrationRows)+len(outboxRows)) + integrationByAudit := integrationRefsByAuditID(integrationRows) + for index, event := range events { + refs := event.InvestigationRefs + refs.IntegrationRefs = append(refs.IntegrationRefs, integrationByAudit[auditRows[index].ID]...) + refs.IntegrationRefs = append(refs.IntegrationRefs, integrationResourceRefs(event.Resources)...) + refs.IntegrationRefs = uniqueIntegrationRefs(refs.IntegrationRefs) + nodes = append(nodes, auditTimelineNode(event, refs)) + nodes = append(nodes, resourceReferenceNodes(event, refs)...) + } + for _, row := range integrationRows { + nodes = append(nodes, integrationTimelineNode(row)) + } + for _, row := range outboxRows { + nodes = append(nodes, outboxTimelineNode(row)) + } + sort.Slice(nodes, func(i, j int) bool { + if nodes[i].OccurredAt.Equal(nodes[j].OccurredAt) { + if nodes[i].RecordSource == nodes[j].RecordSource { + return nodes[i].NodeID < nodes[j].NodeID + } + return nodes[i].RecordSource < nodes[j].RecordSource + } + return nodes[i].OccurredAt.Before(nodes[j].OccurredAt) + }) + + timeline := &LinkTimeline{Nodes: nodes, Retention: retention} + if timeline.Nodes == nil { + timeline.Nodes = []LinkTimelineNode{} + } + if column == "request_id" { + timeline.RequestID = stringPointer(value) + timeline.AccessLogLookupRequestID = stringPointer(value) + } else { + timeline.CorrelationID = stringPointer(value) + } + return timeline, nil +} + +func (q *Query) loadLinkRows(ctx context.Context, column, value string, onlineFrom time.Time) ([]model.AuditEvent, []model.IntegrationLog, []model.OutboxEvent, error) { + auditRows := []model.AuditEvent{} + if err := q.db.WithContext(ctx).Where(column+" = ? AND occurred_at >= ?", value, onlineFrom.UTC()).Find(&auditRows).Error; err != nil { + return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路审计事件失败") + } + integrationRows := []model.IntegrationLog{} + if err := q.db.WithContext(ctx).Where(column+" = ? AND created_at >= ?", value, onlineFrom.UTC()).Find(&integrationRows).Error; err != nil { + return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路外部交互失败") + } + outboxRows := []model.OutboxEvent{} + if err := q.db.WithContext(ctx).Where(column+" = ? AND created_at >= ?", value, onlineFrom.UTC()).Find(&outboxRows).Error; err != nil { + return nil, nil, nil, errors.Wrap(errors.CodeDatabaseError, err, "查询链路可靠事件失败") + } + return auditRows, integrationRows, outboxRows, nil +} + +func auditTimelineNode(event EventView, refs InvestigationRefs) LinkTimelineNode { + return LinkTimelineNode{ + RecordSource: constants.AuditRecordSourceAuditEvent, NodeID: event.EventID, + OccurredAt: event.OccurredAt, Code: event.ActionCode, Title: event.ActionName, + Result: event.Result, Summary: event.Summary, + RequestID: stringPointer(event.RequestID), CorrelationID: stringPointer(event.CorrelationID), + ParentEventID: stringPointer(event.ParentEventID), Resources: refs.ResourceRefs, InvestigationRefs: refs, + Fidelity: linkageFidelity(event.RequestID, event.CorrelationID, event.ParentEventID, true, len(refs.ResourceRefs) > 0), + } +} + +func integrationTimelineNode(row model.IntegrationLog) LinkTimelineNode { + resource := integrationResourceRef(row) + resources := make([]InvestigationResourceRef, 0, 1) + if resource != nil { + resources = append(resources, *resource) + } + refs := InvestigationRefs{ + ResourceRefs: resources, RequestID: row.RequestID, CorrelationID: row.CorrelationID, + IntegrationRefs: []IntegrationRef{{IntegrationID: row.IntegrationID}}, + } + return LinkTimelineNode{ + RecordSource: constants.AuditRecordSourceIntegrationLog, NodeID: row.IntegrationID, + OccurredAt: row.CreatedAt, Code: row.Operation, + Title: constants.IntegrationProviderName(row.Provider) + " · " + constants.IntegrationOperationName(row.Operation), + Result: row.Result, ResultName: constants.IntegrationResultName(row.Result), Summary: "外部交互事实", + RequestID: row.RequestID, CorrelationID: row.CorrelationID, Resources: resources, InvestigationRefs: refs, + Fidelity: linkageFidelity(pointerValue(row.RequestID), pointerValue(row.CorrelationID), "", row.AuditEventID != nil, resource != nil), + } +} + +func outboxTimelineNode(row model.OutboxEvent) LinkTimelineNode { + resourceType := row.ResourceType + if resourceType == "" { + resourceType = row.AggregateType + } + resourceID := row.ResourceID + if resourceID == "" { + resourceID = row.AggregateID + } + resourceKey := row.BusinessKey + if resourceKey == "" { + resourceKey = row.AggregateID + } + resource := InvestigationResourceRef{ResourceType: resourceType, ResourceID: stringPointer(resourceID), ResourceKey: resourceKey, DisplayName: resourceKey} + refs := InvestigationRefs{ + ResourceRefs: []InvestigationResourceRef{resource}, RequestID: stringPointer(row.RequestID), + CorrelationID: stringPointer(row.CorrelationID), IntegrationRefs: []IntegrationRef{}, + } + return LinkTimelineNode{ + RecordSource: constants.AuditRecordSourceOutboxEvent, NodeID: row.EventID, + OccurredAt: row.CreatedAt, Code: row.EventType, Title: "可靠事件：" + row.EventType, + Result: strconv.Itoa(row.Status), ResultName: constants.GetOutboxStatusName(row.Status), + Summary: fmt.Sprintf("%s/%s，重试 %d 次", row.AggregateType, row.AggregateID, row.RetryCount), + RequestID: stringPointer(row.RequestID), CorrelationID: stringPointer(row.CorrelationID), + ParentEventID: stringPointer(row.ParentEventID), Resources: refs.ResourceRefs, InvestigationRefs: refs, + Fidelity: linkageFidelity(row.RequestID, row.CorrelationID, row.ParentEventID, row.ParentEventID != "", resourceType != "" && resourceID != ""), + } +} + +func resourceReferenceNodes(event EventView, refs InvestigationRefs) []LinkTimelineNode { + nodes := make([]LinkTimelineNode, 0, len(event.Resources)) + for _, resource := range event.Resources { + recordSource := "" + titlePrefix := "" + summary := "" + switch { + case isAsynqTaskResource(resource.ResourceType): + recordSource = constants.AuditRecordSourceAsynqTask + titlePrefix = "异步任务：" + summary = "持久化任务资源摘要；不读取或推断 Redis 队列历史" + case isDomainLedgerResource(resource.ResourceType): + recordSource = constants.AuditRecordSourceDomainLedgerRef + titlePrefix = "业务账本引用：" + summary = "状态、金额及业务结论以对应业务表为准" + default: + continue + } + resourceRef := InvestigationResourceRef{ResourceType: resource.ResourceType, ResourceID: resource.ResourceID, ResourceKey: resource.ResourceKey, DisplayName: resource.DisplayName} + nodeRefs := refs + nodeRefs.ResourceRefs = []InvestigationResourceRef{resourceRef} + nodes = append(nodes, LinkTimelineNode{ + RecordSource: recordSource, + NodeID: fmt.Sprintf("%s:%s:%s:%s:%s", event.EventID, resource.ResourceType, pointerValue(resource.ResourceID), resource.ResourceKey, resource.Role), + OccurredAt: event.OccurredAt, Code: resource.ResourceType, Title: titlePrefix + resource.DisplayName, + Result: event.Result, Summary: summary, ReferenceOnly: true, + RequestID: stringPointer(event.RequestID), CorrelationID: stringPointer(event.CorrelationID), + ParentEventID: stringPointer(event.ParentEventID), Resources: nodeRefs.ResourceRefs, InvestigationRefs: nodeRefs, + Fidelity: linkageFidelity(event.RequestID, event.CorrelationID, event.ParentEventID, true, resource.ResourceID != nil || resource.ResourceKey != ""), + }) + } + return nodes +} + +func integrationRefsByAuditID(rows []model.IntegrationLog) map[uint][]IntegrationRef { + refs := make(map[uint][]IntegrationRef) + for _, row := range rows { + if row.AuditEventID != nil { + refs[*row.AuditEventID] = append(refs[*row.AuditEventID], IntegrationRef{IntegrationID: row.IntegrationID}) + } + } + return refs +} + +func integrationResourceRefs(resources []ResourceView) []IntegrationRef { + refs := make([]IntegrationRef, 0) + for _, resource := range resources { + if resource.ResourceType == constants.AuditResourceIntegrationLog && resource.ResourceKey != "" { + refs = append(refs, IntegrationRef{IntegrationID: resource.ResourceKey}) + } + } + return refs +} + +func uniqueIntegrationRefs(refs []IntegrationRef) []IntegrationRef { + unique := make([]IntegrationRef, 0, len(refs)) + seen := make(map[string]bool, len(refs)) + for _, ref := range refs { + if ref.IntegrationID == "" || seen[ref.IntegrationID] { + continue + } + seen[ref.IntegrationID] = true + unique = append(unique, ref) + } + return unique +} + +func integrationResourceRef(row model.IntegrationLog) *InvestigationResourceRef { + if row.ResourceType == nil || *row.ResourceType == "" { + return nil + } + ref := InvestigationResourceRef{ResourceType: *row.ResourceType, ResourceID: row.ResourceID} + if row.ResourceKey != nil { + ref.ResourceKey = *row.ResourceKey + ref.DisplayName = *row.ResourceKey + } + return &ref +} + +func linkageFidelity(requestID, correlationID, parentEventID string, directAuditLink, stableResource bool) LinkageFidelity { + return LinkageFidelity{ + RequestAvailable: requestID != "", CorrelationAvailable: correlationID != "", + ParentEventAvailable: parentEventID != "", DirectAuditLinkAvailable: directAuditLink, + StableResourceAvailable: stableResource, + } +} + +func pointerValue(value *string) string { + if value == nil { + return "" + } + return *value +} + +func isAsynqTaskResource(resourceType string) bool { + switch resourceType { + case constants.AuditResourceDeviceBatchTask, constants.AuditResourceIotCardImportTask, + constants.AuditResourceDeviceImportTask, constants.AuditResourceAssetPackageBatchOrderTask, + constants.AuditResourceOrderPackageInvalidateTask, constants.AuditResourceExportTask: + return true + default: + return false + } +} + +func isDomainLedgerResource(resourceType string) bool { + switch resourceType { + case constants.AuditResourceOrder, constants.AuditResourcePayment, constants.AuditResourceRefund, + constants.AuditResourceAgentRecharge, constants.AuditResourceRechargeOrder, + constants.AuditResourceAssetWallet, constants.AuditResourceAssetWalletTransaction, + constants.AuditResourceAgentWallet, constants.AuditResourceAgentWalletTransaction, + constants.AuditResourceAgentWalletReservation, constants.AuditResourcePackageUsage, + constants.AuditResourceApprovalInstance, constants.AuditResourceCommissionRecord, + constants.AuditResourceCommissionWithdrawal: + return true + default: + return false + } +} diff --git a/internal/query/integration/logs.go b/internal/query/integration/logs.go index b763662..61abd13 100644 --- a/internal/query/integration/logs.go +++ b/internal/query/integration/logs.go @@ -12,6 +12,7 @@ import ( "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" "github.com/break/junhong_cmp_fiber/pkg/middleware" @@ -35,10 +36,11 @@ type ListFilter struct { // ListPage 是按创建时间和主键稳定倒序的分页结果。 type ListPage struct { - Total int64 `json:"total"` - Page int `json:"page"` - PageSize int `json:"page_size"` - Items []ListItem `json:"items"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Items []ListItem `json:"items"` + Retention retentionquery.Info `json:"retention"` } // ListItem 是 Integration Log 列表投影。 @@ -81,6 +83,12 @@ type Detail struct { Fidelity FidelityView `json:"fidelity"` } +// DetailResponse 是外部交互详情及在线留存边界。 +type DetailResponse struct { + Detail + Retention retentionquery.Info `json:"retention"` +} + // AttemptView 是显式 trigger_series 下的单次技术尝试。 type AttemptView struct { IntegrationID string `json:"integration_id"` @@ -169,6 +177,10 @@ func (q *Query) List(ctx context.Context, filter ListFilter) (*ListPage, error) if err := q.authorize(ctx); err != nil { return nil, err } + filter, retention, err := q.normalizeOnlineFilter(ctx, filter) + if err != nil { + return nil, err + } if !validFilter(filter) { return nil, errors.New(errors.CodeInvalidParam) } @@ -178,27 +190,52 @@ func (q *Query) List(ctx context.Context, filter ListFilter) (*ListPage, error) if err := query.Count(&total).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "统计外部交互日志失败") } - rows := make([]model.IntegrationLog, 0, filter.PageSize) - if err := query.Order("created_at DESC, id DESC").Offset((filter.Page - 1) * filter.PageSize).Limit(filter.PageSize).Find(&rows).Error; err != nil { - return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互日志失败") + rows, err := q.loadListPage(ctx, query, filter.Page, filter.PageSize) + if err != nil { + return nil, err } items := make([]ListItem, len(rows)) for i, row := range rows { items[i] = projectListItem(row) } - return &ListPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items}, nil + return &ListPage{Total: total, Page: filter.Page, PageSize: filter.PageSize, Items: items, Retention: retention}, nil +} + +// loadListPage 先分页主键，再批量读取列表字段，避免加载正文摘要 JSON。 +func (q *Query) loadListPage(ctx context.Context, query *gorm.DB, page, pageSize int) ([]model.IntegrationLog, error) { + ids := make([]uint, 0, pageSize) + if err := query.Select("id").Order("created_at DESC, id DESC"). + Offset((page-1)*pageSize).Limit(pageSize).Pluck("id", &ids).Error; err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互日志分页ID失败") + } + rows := make([]model.IntegrationLog, 0, len(ids)) + if len(ids) == 0 { + return rows, nil + } + if err := q.db.WithContext(ctx).Select( + "id", "integration_id", "provider", "direction", "operation", + "resource_type", "resource_id", "resource_key", "result", "duration_ms", + "state_changed", "request_id", "correlation_id", "created_at", + ).Where("id IN ?", ids).Order("created_at DESC, id DESC").Find(&rows).Error; err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "批量投影外部交互日志失败") + } + return rows, nil } // Get 使用稳定 integration_id 查询结构化详情。 -func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error) { +func (q *Query) Get(ctx context.Context, integrationID string) (*DetailResponse, error) { if err := q.authorize(ctx); err != nil { return nil, err } if integrationID == "" { return nil, errors.New(errors.CodeInvalidParam) } + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceIntegration) + if err != nil { + return nil, err + } var row model.IntegrationLog - if err := q.db.WithContext(ctx).Where("integration_id = ?", integrationID).First(&row).Error; err != nil { + if err := q.db.WithContext(ctx).Where("integration_id = ? AND created_at >= ?", integrationID, retention.OnlineFrom.UTC()).First(&row).Error; err != nil { if err == gorm.ErrRecordNotFound { return nil, errors.New(errors.CodeNotFound, "外部交互日志不存在") } @@ -216,12 +253,12 @@ func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error) if err != nil { return nil, err } - attempts, err := q.loadAttempts(ctx, row) + attempts, err := q.loadAttempts(ctx, row, retention.OnlineFrom) if err != nil { return nil, err } providerMessage, providerMessageFidelity := safeProviderMessage(row.ProviderMessage) - return &Detail{ + return &DetailResponse{Detail: Detail{ Identity: IdentityView{IntegrationID: row.IntegrationID, Provider: row.Provider, ProviderName: constants.IntegrationProviderName(row.Provider), Direction: row.Direction, DirectionName: constants.IntegrationDirectionName(row.Direction), Operation: row.Operation, OperationName: constants.IntegrationOperationName(row.Operation), ExternalID: row.ExternalID}, Resource: resourceView(row), Trigger: TriggerView{Source: row.TriggerSource, Scene: row.TriggerScene, Series: row.TriggerSeries, Attempt: row.Attempt}, Result: ResultView{Code: row.Result, Name: constants.IntegrationResultName(row.Result), Category: constants.IntegrationResultCategory(row.Result), HTTPStatus: row.HTTPStatus, ProviderCode: row.ProviderCode, ProviderMessage: providerMessage, DurationMS: row.DurationMS, StateChanged: row.StateChanged, RecoveryStrategy: row.RecoveryStrategy}, @@ -235,13 +272,13 @@ func (q *Query) Get(ctx context.Context, integrationID string) (*Detail, error) ResourceIDAvailable: row.ResourceID != nil && *row.ResourceID != "", ProviderMessageFidelity: providerMessageFidelity, }, - }, nil + }, Retention: retention}, nil } -func (q *Query) loadAttempts(ctx context.Context, current model.IntegrationLog) ([]AttemptView, error) { +func (q *Query) loadAttempts(ctx context.Context, current model.IntegrationLog, onlineFrom time.Time) ([]AttemptView, error) { rows := []model.IntegrationLog{current} if current.TriggerSeries != nil && *current.TriggerSeries != "" { - if err := q.db.WithContext(ctx).Where("trigger_series = ?", *current.TriggerSeries). + if err := q.db.WithContext(ctx).Where("trigger_series = ? AND created_at >= ?", *current.TriggerSeries, onlineFrom.UTC()). Order("attempt ASC, created_at ASC, id ASC").Find(&rows).Error; err != nil { return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询外部交互尝试序列失败") } @@ -270,6 +307,15 @@ func (q *Query) authorize(ctx context.Context) error { return nil } +func (q *Query) normalizeOnlineFilter(ctx context.Context, filter ListFilter) (ListFilter, retentionquery.Info, error) { + retention, err := retentionquery.Load(ctx, q.db, retentionquery.SourceIntegration) + if err != nil { + return filter, retention, err + } + filter.CreatedFrom, filter.CreatedTo, err = retentionquery.NormalizeRange(retention, filter.CreatedFrom, filter.CreatedTo, constants.IntegrationQueryMaxRange) + return filter, retention, err +} + func validFilter(filter ListFilter) bool { if filter.CreatedFrom == nil || filter.CreatedTo == nil || !filter.CreatedFrom.Before(*filter.CreatedTo) || filter.CreatedTo.Sub(*filter.CreatedFrom) > constants.IntegrationQueryMaxRange { return false diff --git a/internal/query/integration/overview.go b/internal/query/integration/overview.go index a7ebd3e..4e928be 100644 --- a/internal/query/integration/overview.go +++ b/internal/query/integration/overview.go @@ -7,6 +7,7 @@ import ( "gorm.io/gorm" "github.com/break/junhong_cmp_fiber/internal/model" + retentionquery "github.com/break/junhong_cmp_fiber/internal/query/retention" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" ) @@ -19,17 +20,18 @@ type OverviewFilter struct { // Overview 是外部交互固定维度聚合结果。 type Overview struct { - Total int64 `json:"total"` - AnomalyCount int64 `json:"anomaly_count"` - UnknownCount int64 `json:"unknown_count"` - StalePendingCount int64 `json:"stale_pending_count"` - StateChangedCount int64 `json:"state_changed_count"` - AverageDurationMS float64 `json:"average_duration_ms"` - P95DurationMS float64 `json:"p95_duration_ms"` - Results []ResultCount `json:"results"` - Providers []NamedCount `json:"providers"` - Directions []NamedCount `json:"directions"` - Trend []TrendPoint `json:"trend"` + Total int64 `json:"total"` + AnomalyCount int64 `json:"anomaly_count"` + UnknownCount int64 `json:"unknown_count"` + StalePendingCount int64 `json:"stale_pending_count"` + StateChangedCount int64 `json:"state_changed_count"` + AverageDurationMS float64 `json:"average_duration_ms"` + P95DurationMS float64 `json:"p95_duration_ms"` + Results []ResultCount `json:"results"` + Providers []NamedCount `json:"providers"` + Directions []NamedCount `json:"directions"` + Trend []TrendPoint `json:"trend"` + Retention retentionquery.Info `json:"retention"` } // ResultCount 是原始结果及其派生类别计数。 @@ -66,11 +68,17 @@ func (q *Query) Overview(ctx context.Context, filter OverviewFilter) (*Overview, if filter.Bucket == "" { filter.Bucket = "hour" } + var retention retentionquery.Info + var err error + filter.ListFilter, retention, err = q.normalizeOnlineFilter(ctx, filter.ListFilter) + if err != nil { + return nil, err + } if !validFilter(filter.ListFilter) || (filter.Bucket != "hour" && filter.Bucket != "day") { return nil, errors.New(errors.CodeInvalidParam) } base := applyFilters(q.db.WithContext(ctx).Model(&model.IntegrationLog{}), filter.ListFilter) - result := &Overview{Results: []ResultCount{}, Providers: []NamedCount{}, Directions: []NamedCount{}, Trend: []TrendPoint{}} + result := &Overview{Results: []ResultCount{}, Providers: []NamedCount{}, Directions: []NamedCount{}, Trend: []TrendPoint{}, Retention: retention} if err := loadOverviewMetrics(base, result); err != nil { return nil, err } diff --git a/internal/query/retention/retention.go b/internal/query/retention/retention.go new file mode 100644 index 0000000..faef33f --- /dev/null +++ b/internal/query/retention/retention.go @@ -0,0 +1,115 @@ +// Package retention 提供在线审计查询的统一留存边界。 +package retention + +import ( + "context" + "time" + + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// Source 表示受在线留存边界约束的数据源。 +type Source string + +const ( + // SourceAudit 表示统一审计事件。 + SourceAudit Source = constants.AuditArchiveSource + // SourceIntegration 表示外部交互日志。 + SourceIntegration Source = constants.IntegrationArchiveSource +) + +// Info 是查询响应公开的在线留存边界。 +type Info struct { + OnlineFrom time.Time `json:"online_from" description:"当前可在线查询的最早时间"` + ArchivedBefore *time.Time `json:"archived_before" description:"早于该时间的数据已归档；尚未清理时为空"` + Timezone string `json:"timezone" description:"留存自然日时区"` +} + +// Load 从归档账本读取已完成物理清理的数据边界。 +func Load(ctx context.Context, db *gorm.DB, sources ...Source) (Info, error) { + location, err := time.LoadLocation(constants.AuditArchiveTimezone) + if err != nil { + return Info{}, errors.Wrap(errors.CodeInternalError, err, "加载审计留存时区失败") + } + now := time.Now().In(location) + info := Info{OnlineFrom: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location), Timezone: constants.AuditArchiveTimezone} + for _, source := range sources { + boundary, cleaned, err := sourceBoundary(ctx, db, source, location) + if err != nil { + return Info{}, err + } + if boundary.Before(info.OnlineFrom) && info.ArchivedBefore == nil { + info.OnlineFrom = boundary + } + if cleaned && (info.ArchivedBefore == nil || boundary.After(*info.ArchivedBefore)) { + value := boundary + info.ArchivedBefore = &value + info.OnlineFrom = boundary + } + } + return info, nil +} + +func sourceBoundary(ctx context.Context, db *gorm.DB, source Source, location *time.Location) (time.Time, bool, error) { + var cleanedEnd *time.Time + if err := db.WithContext(ctx).Model(&model.LogArchiveRun{}). + Where("source = ? AND cleaned_at IS NOT NULL", source). + Select("MAX(range_end)").Scan(&cleanedEnd).Error; err != nil { + return time.Time{}, false, errors.Wrap(errors.CodeDatabaseError, err, "查询审计留存清理边界失败") + } + if cleanedEnd != nil { + return cleanedEnd.In(location), true, nil + } + + var earliest *time.Time + table, column := "tb_audit_event", "occurred_at" + if source == SourceIntegration { + table, column = "tb_integration_log", "created_at" + } + if err := db.WithContext(ctx).Table(table).Select("MIN(" + column + ")").Scan(&earliest).Error; err != nil { + return time.Time{}, false, errors.Wrap(errors.CodeDatabaseError, err, "查询审计在线数据边界失败") + } + if earliest != nil { + return earliest.In(location), false, nil + } + now := time.Now().In(location) + return time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, location), false, nil +} + +// NormalizeRange 将缺省范围收敛到在线窗口，并拒绝归档或跨边界查询。 +func NormalizeRange(info Info, from, to *time.Time, maxRange ...time.Duration) (*time.Time, *time.Time, error) { + explicitFrom := from != nil + if from != nil && info.ArchivedBefore != nil && from.Before(info.OnlineFrom) { + return nil, nil, archivedError(info) + } + if to != nil && info.ArchivedBefore != nil && !to.After(info.OnlineFrom) { + return nil, nil, archivedError(info) + } + if from == nil { + value := info.OnlineFrom + from = &value + } + if to == nil { + value := time.Now() + to = &value + } + if len(maxRange) > 0 && maxRange[0] > 0 && to.Sub(*from) > maxRange[0] { + if explicitFrom { + return nil, nil, errors.New(errors.CodeInvalidParam) + } + value := to.Add(-maxRange[0]) + from = &value + } + if !from.Before(*to) { + return nil, nil, errors.New(errors.CodeInvalidParam) + } + return from, to, nil +} + +func archivedError(info Info) error { + return errors.NewWithData(errors.CodeAuditDataArchived, map[string]any{"retention": info}) +} diff --git a/internal/routes/audit.go b/internal/routes/audit.go index e547f40..25d53d5 100644 --- a/internal/routes/audit.go +++ b/internal/routes/audit.go @@ -15,13 +15,13 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc * agent := router.Group("/agent/resource-activities") Register(agent, doc, basePath+"/agent/resource-activities", "GET", "/:resource_type/:identifier", handler.AgentResourceActivities, RouteSpec{ Summary: "查询代理资源活动", - Description: "resource_type/identifier 来自代理当前业务页面稳定字段；店铺范围只读取认证上下文。仅返回写入时生成的安全业务结论和白名单详情，越权与不存在同错。", + Description: "resource_type/identifier 来自代理当前业务页面稳定字段；店铺范围只读取认证上下文。仅查询 retention 标明的在线窗口，归档范围不从对象存储读取。", Tags: []string{"资源活动"}, Input: new(dto.SubjectResourceActivityRequest), Output: new(auditquery.SubjectActivityPage), Auth: true, }) enterprise := router.Group("/enterprise/resource-activities") Register(enterprise, doc, basePath+"/enterprise/resource-activities", "GET", "/:resource_type/:identifier", handler.EnterpriseResourceActivities, RouteSpec{ Summary: "查询企业资源活动", - Description: "仅支持企业当前有效授权的卡和设备；企业身份只读取认证上下文，授权撤销后立即不可读取。响应不包含平台操作者、风险、内部前后值或外部交互内容。", + Description: "仅支持企业当前有效授权的卡和设备；企业身份只读取认证上下文。仅查询 retention 标明的在线窗口，响应不包含平台内部调查字段。", Tags: []string{"资源活动"}, Input: new(dto.SubjectResourceActivityRequest), Output: new(auditquery.SubjectActivityPage), Auth: true, }) @@ -30,13 +30,13 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc * Register(audit, doc, groupPath, "GET", "/events", handler.ListEvents, RouteSpec{ Summary: "查询全局审计事件", - Description: "筛选值来自调查人员输入或其他调查节点的稳定引用；身份范围只读取认证上下文。固定按发生时间和事件ID倒序，不提供导出、修改或删除。", + Description: "筛选值来自调查人员输入或其他调查节点的稳定引用；缺省只查 retention 标明的在线窗口，归档范围返回稳定错误。固定倒序分页，不提供导出、修改或删除。", Tags: []string{"审计调查"}, Input: new(dto.AuditEventListRequest), Output: new(auditquery.EventPage), Auth: true, }) Register(audit, doc, groupPath, "GET", "/events/:event_id", handler.GetEvent, RouteSpec{ Summary: "查询审计事件详情", - Description: "event_id 来自事件、资源、操作者或链路节点的 investigation_refs；返回全部资源快照和各资源 before/after。", - Tags: []string{"审计调查"}, Input: new(dto.AuditEventIDParams), Output: new(auditquery.EventView), Auth: true, + Description: "event_id 来自 investigation_refs；只查询在线 PostgreSQL，未命中仍返回资源不存在，不扫描对象存储。返回全部资源快照和各资源 before/after。", + Tags: []string{"审计调查"}, Input: new(dto.AuditEventIDParams), Output: new(auditquery.EventDetail), Auth: true, }) Register(audit, doc, groupPath, "GET", "/actors/:kind/:id/events", handler.ListActorEvents, RouteSpec{ Summary: "查询操作者行为时间线", @@ -54,20 +54,45 @@ func registerAuditRoutes(router fiber.Router, handler *admin.AuditHandler, doc * Description: "resource_type/resource_id 必须来自业务页面稳定字段、资源搜索结果或 investigation_refs。事件在资源作为 primary、affected 或 reference 时均会返回。", Tags: []string{"审计调查"}, Input: new(dto.AuditResourceTimelineRequest), Output: new(auditquery.EventPage), Auth: true, }) + Register(audit, doc, groupPath, "GET", "/requests/:request_id/timeline", handler.RequestTimeline, RouteSpec{ + Summary: "查询请求关联时间线", + Description: "request_id 来自审计或外部集成节点，也可从 Access Log 粘贴。只组合 retention 在线窗口内的持久化事实，不扫描 Access Log 或对象存储。", + Tags: []string{"审计调查"}, Input: new(dto.AuditRequestTimelineParams), Output: new(auditquery.LinkTimeline), Auth: true, + }) + Register(audit, doc, groupPath, "GET", "/correlations/:correlation_id/timeline", handler.CorrelationTimeline, RouteSpec{ + Summary: "查询业务关联时间线", + Description: "correlation_id 来自稳定调查引用。只组合 retention 在线窗口内的持久化事实；相同 correlation 不用于猜测技术重试。", + Tags: []string{"审计调查"}, Input: new(dto.AuditCorrelationTimelineParams), Output: new(auditquery.LinkTimeline), Auth: true, + }) + Register(audit, doc, groupPath, "GET", "/finance/timeline", handler.FinanceTimeline, RouteSpec{ + Summary: "查询资金调查时间线", + Description: "可使用任一稳定资金条件进入；缺省只查 retention 在线窗口，归档范围不返回部分结果。关联事实由服务端解析，金额以业务账本为权威。", + Tags: []string{"审计调查"}, Input: new(dto.AuditFinanceTimelineRequest), Output: new(auditquery.FinanceTimelinePage), Auth: true, + }) + Register(audit, doc, groupPath, "GET", "/risks/overview", handler.RiskOverview, RouteSpec{ + Summary: "查询风险调查总览", + Description: "时间范围最长31天，缺省时使用当前在线窗口；只聚合高风险、资金、安全、失败、拒绝、部分成功和结果未知事件。", + Tags: []string{"审计调查"}, Input: new(dto.AuditRiskOverviewRequest), Output: new(auditquery.RiskOverview), Auth: true, + }) + Register(audit, doc, groupPath, "GET", "/risks/events", handler.RiskEvents, RouteSpec{ + Summary: "查询风险事件明细", + Description: "筛选条件来自风险总览分桶或调查人员输入，缺省只查 retention 在线窗口；明细返回 investigation_refs，不提供处置或封禁能力。", + Tags: []string{"审计调查"}, Input: new(dto.AuditRiskEventsRequest), Output: new(auditquery.RiskEventPage), Auth: true, + }) // Integration 总览静态路径必须先于动态详情路径，避免 overview 被当作 integration_id。 Register(audit, doc, groupPath, "GET", "/integrations/overview", handler.IntegrationOverview, RouteSpec{ Summary: "查询外部集成交互总览", - Description: "筛选和时间范围来自调查输入或关联视角跳转，身份只来自认证上下文。总览区分成功、处理中、结果不确定、失败和未发送终态。", + Description: "筛选和时间范围来自调查输入或关联视角跳转；缺省只查 retention 在线窗口，归档范围返回稳定错误。总览区分五类结果。", Tags: []string{"审计调查"}, Input: new(dto.IntegrationOverviewRequest), Output: new(integrationquery.Overview), Auth: true, }) Register(audit, doc, groupPath, "GET", "/integrations", handler.ListIntegrations, RouteSpec{ Summary: "查询外部集成交互列表", - Description: "组合筛选来自调查输入或关联视角稳定引用，固定按创建时间和记录ID倒序分页，不提供任意摘要搜索。", + Description: "组合筛选来自调查输入或稳定引用；缺省只查 retention 在线窗口，归档范围不返回空页或部分结果。固定倒序分页，不提供任意摘要搜索。", Tags: []string{"审计调查"}, Input: new(dto.IntegrationListRequest), Output: new(integrationquery.ListPage), Auth: true, }) Register(audit, doc, groupPath, "GET", "/integrations/:integration_id", handler.GetIntegration, RouteSpec{ Summary: "查询外部集成交互详情", - Description: "integration_id 来自列表、通知目标 target_key 或调查节点稳定引用；只展示结构化详情和显式尝试序列，不提供重试、补偿、确认、绑定、恢复、修改、删除或导出。", - Tags: []string{"审计调查"}, Input: new(dto.IntegrationIDParams), Output: new(integrationquery.Detail), Auth: true, + Description: "integration_id 来自稳定引用；只查询在线 PostgreSQL，未命中仍返回资源不存在。展示结构化详情和在线尝试序列，不提供归档读取、恢复、修改、删除或导出。", + Tags: []string{"审计调查"}, Input: new(dto.IntegrationIDParams), Output: new(integrationquery.DetailResponse), Auth: true, }) } diff --git a/internal/service/carrier/service.go b/internal/service/carrier/service.go index dbe7267..b88cfa5 100644 --- a/internal/service/carrier/service.go +++ b/internal/service/carrier/service.go @@ -83,10 +83,6 @@ func (s *Service) Get(ctx context.Context, id uint) (*dto.CarrierResponse, error } return nil, errors.Wrap(errors.CodeInternalError, err, "获取运营商失败") } - if s.audit == nil { - return nil, errors.New(errors.CodeInvalidStatus, "运营商配置审计接缝未配置") - } - before := *carrier return s.toResponse(carrier), nil } @@ -103,6 +99,7 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateCarrierReq } return nil, errors.Wrap(errors.CodeInternalError, err, "获取运营商失败") } + before := *carrier if req.CarrierName != nil { carrier.CarrierName = *req.CarrierName diff --git a/internal/service/enterprise_device/service.go b/internal/service/enterprise_device/service.go index f9484bd..d73eacc 100644 --- a/internal/service/enterprise_device/service.go +++ b/internal/service/enterprise_device/service.go @@ -65,12 +65,12 @@ func (s *Service) AllocateDevices(ctx context.Context, enterpriseID uint, req *d return nil, errors.New(errors.CodeInvalidStatus, "企业设备授权审计接缝未配置") } if err := validateEnterpriseDeviceActor(ctx); err != nil { - s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, err) + s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, err) return nil, err } if err := middleware.CanManageEnterprise(ctx, enterpriseID, s.enterpriseStore); err != nil { permissionErr := errors.New(errors.CodeForbidden, "无权限操作该资源或资源不存在") - s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, permissionErr) + s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesAllocated, "向企业授权设备被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, permissionErr) return nil, permissionErr } enterprise, err := s.enterpriseStore.GetByID(ctx, enterpriseID) @@ -417,12 +417,12 @@ func (s *Service) RecallDevices(ctx context.Context, enterpriseID uint, req *dto return nil, errors.New(errors.CodeInvalidStatus, "企业设备授权审计接缝未配置") } if err := validateEnterpriseDeviceActor(ctx); err != nil { - s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, err) + s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, err) return nil, err } if err := middleware.CanManageEnterprise(ctx, enterpriseID, s.enterpriseStore); err != nil { permissionErr := errors.New(errors.CodeForbidden, "无权限操作该资源或资源不存在") - s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{ID: enterpriseID}, nil, nil, permissionErr) + s.recordDeviceFailure(ctx, constants.AuditActionEnterpriseDevicesRecalled, "回收企业设备授权被拒绝", &model.Enterprise{Model: gorm.Model{ID: enterpriseID}}, nil, nil, permissionErr) return nil, permissionErr } enterprise, err := s.enterpriseStore.GetByID(ctx, enterpriseID) diff --git a/internal/service/iot_card/unified_audit.go b/internal/service/iot_card/unified_audit.go index 3d3b5d9..1d07927 100644 --- a/internal/service/iot_card/unified_audit.go +++ b/internal/service/iot_card/unified_audit.go @@ -23,38 +23,46 @@ func (s *Service) SetAccessAudit(writer *audit.Writer) { s.auditWriter = writer } -// WriteCardStateAudit 将卡观测事务中的人工状态操作写入统一 Audit Event。 +// WriteCardStateAudit 将卡观测事务中的人工、回调或 Worker 状态变化写入统一 Audit Event。 func (s *Service) WriteCardStateAudit(ctx context.Context, tx *gorm.DB, input cardapp.StateAudit) error { extraResources := make([]audit.ResourceInput, 0, 1) if input.IntegrationID != "" { - extraResources = append(extraResources, callbackIntegrationAuditResource(ctx, input.IntegrationID)) + extraResources = append(extraResources, cardStateIntegrationAuditResource(ctx, input.IntegrationID)) } return s.appendCardLifecycleAudit(ctx, tx, input.ActionCode, input.Summary, constants.AuditResultSuccess, input.Card, input.BeforeData, input.AfterData, nil, extraResources...) } -// WriteCardStateFailure 使用独立短事务记录已解析卡资源后的回调失败。 +// WriteCardStateFailure 使用独立短事务记录已解析卡资源后的回调或 Worker 失败。 func (s *Service) WriteCardStateFailure(ctx context.Context, input cardapp.StateAudit, businessErr error) { extraResources := make([]audit.ResourceInput, 0, 1) if input.IntegrationID != "" { - extraResources = append(extraResources, callbackIntegrationAuditResource(ctx, input.IntegrationID)) + extraResources = append(extraResources, cardStateIntegrationAuditResource(ctx, input.IntegrationID)) } s.recordCardLifecycleFailure(ctx, input.ActionCode, input.Summary, constants.AuditResultFailed, input.Card, input.Card.ID, businessErr, extraResources...) } -func callbackIntegrationAuditResource(ctx context.Context, integrationID string) audit.ResourceInput { +func cardStateIntegrationAuditResource(ctx context.Context, integrationID string) audit.ResourceInput { linkage := auditcontext.From(ctx) correlationID := linkage.CorrelationID if correlationID == "" { correlationID = linkage.RequestID } + direction := constants.IntegrationDirectionInbound + role := constants.AuditResourceRoleCallbackIntegration + provider := linkage.ActorID + if linkage.Source == constants.AuditSourceWorker { + direction = constants.IntegrationDirectionOutbound + role = constants.AuditResourceRoleWorkerIntegration + provider = constants.IntegrationProviderGateway + } return audit.ResourceInput{ Type: constants.AuditResourceIntegrationLog, Key: integrationID, DisplayName: integrationID, - Relation: constants.AuditResourceRelationReference, Role: constants.AuditResourceRoleCallbackIntegration, + Relation: constants.AuditResourceRelationReference, Role: role, IdentitySnapshot: map[string]any{ - "integration_id": integrationID, "provider": linkage.ActorID, - "direction": constants.IntegrationDirectionInbound, "correlation_id": correlationID, + "integration_id": integrationID, "provider": provider, + "direction": direction, "correlation_id": correlationID, }, SubjectVisibility: constants.AuditSubjectInternalOnly, } diff --git a/internal/service/order/service.go b/internal/service/order/service.go index 5a086e6..b0773c7 100644 --- a/internal/service/order/service.go +++ b/internal/service/order/service.go @@ -2709,8 +2709,12 @@ func (s *Service) enqueueCommissionCalculation(ctx context.Context, orderID uint return } + linkage := auditcontext.From(ctx) // 直接传 map，由 EnqueueTask 内部统一序列化一次（传 []byte 会导致 sonic.Marshal 二次 base64 编码） - if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeCommission, map[string]any{"order_id": orderID}); err != nil { + if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeCommission, map[string]any{ + "order_id": orderID, "request_id": linkage.RequestID, "correlation_id": linkage.CorrelationID, + "parent_event_id": linkage.ParentEventID, + }); err != nil { s.logger.Error("佣金计算任务入队失败", zap.Uint("order_id", orderID), zap.Error(err), diff --git a/internal/service/package/activation_service.go b/internal/service/package/activation_service.go index dadfce7..fd7617d 100644 --- a/internal/service/package/activation_service.go +++ b/internal/service/package/activation_service.go @@ -14,6 +14,7 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/zap" "gorm.io/gorm" + "gorm.io/gorm/clause" ) // ResumeCallback 复机回调接口 diff --git a/internal/service/recharge_order/service.go b/internal/service/recharge_order/service.go index 111b50c..2230884 100644 --- a/internal/service/recharge_order/service.go +++ b/internal/service/recharge_order/service.go @@ -9,6 +9,7 @@ import ( "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/internal/store/postgres" "github.com/break/junhong_cmp_fiber/internal/task" + "github.com/break/junhong_cmp_fiber/pkg/auditcontext" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" "github.com/break/junhong_cmp_fiber/pkg/queue" @@ -227,7 +228,11 @@ func (s *Service) HandlePaymentCallback(ctx context.Context, paymentNo string, p linkedIDs := rechargeOrder.LinkedPackageIDs if len(linkedIDs) > 0 { - taskPayload := task.AutoPurchasePayload{RechargeOrderID: rechargeOrder.ID} + linkage := auditcontext.From(ctx) + taskPayload := task.AutoPurchasePayload{ + RechargeOrderID: rechargeOrder.ID, RequestID: linkage.RequestID, + CorrelationID: linkage.CorrelationID, ParentEventID: linkage.ParentEventID, + } if err := s.queueClient.EnqueueTask(ctx, constants.TaskTypeAutoPurchaseAfterRecharge, taskPayload, asynq.MaxRetry(3), asynq.Queue(constants.QueueForTaskType(constants.TaskTypeAutoPurchaseAfterRecharge)), diff --git a/internal/service/role/service.go b/internal/service/role/service.go index eab5e39..02f4dec 100644 --- a/internal/service/role/service.go +++ b/internal/service/role/service.go @@ -132,12 +132,12 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateRoleReques if req.RoleName != nil && *req.RoleName != role.RoleName { exists, err := s.roleStore.ExistsByName(ctx, *req.RoleName, id) if err != nil { - s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, beforeData, nil, err) + s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, nil, beforeData, err) return nil, errors.Wrap(errors.CodeInternalError, err, "检查角色名失败") } if exists { appErr := errors.New(errors.CodeRoleNameExists) - s.recordFailure(ctx, constants.AuditActionRoleUpdated, "拒绝更新重复角色名", constants.AuditResultDenied, role, beforeData, nil, appErr) + s.recordFailure(ctx, constants.AuditActionRoleUpdated, "拒绝更新重复角色名", constants.AuditResultDenied, role, nil, beforeData, appErr) return nil, appErr } role.RoleName = *req.RoleName @@ -162,7 +162,7 @@ func (s *Service) Update(ctx context.Context, id uint, req *dto.UpdateRoleReques OperatorID: currentUserID, Role: role, BeforeData: beforeData, AfterData: roleAuditData(role), }) }); err != nil { - s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, beforeData, nil, err) + s.recordFailure(ctx, constants.AuditActionRoleUpdated, "更新角色失败", constants.AuditResultFailed, role, nil, beforeData, err) return nil, errors.Wrap(errors.CodeInternalError, err, "更新角色失败") } @@ -181,19 +181,19 @@ func (s *Service) Delete(ctx context.Context, id uint) error { accountCount, err := s.accountRoleStore.CountByRoleID(ctx, id) if err != nil { - s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err) + s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err) return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败") } shopCount, err := s.shopRoleStore.CountByRoleID(ctx, id) if err != nil { - s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err) + s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err) return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败") } if accountCount > 0 || shopCount > 0 { appErr := errors.New(errors.CodeRoleInUse, fmt.Sprintf("该角色已分配给 %d 个账号、%d 个店铺，请先移除相关分配后再删除", accountCount, shopCount)) - s.recordFailure(ctx, constants.AuditActionRoleDeleted, "拒绝删除使用中的角色", constants.AuditResultDenied, role, roleAuditData(role), nil, appErr) + s.recordFailure(ctx, constants.AuditActionRoleDeleted, "拒绝删除使用中的角色", constants.AuditResultDenied, role, nil, roleAuditData(role), appErr) return appErr } @@ -208,7 +208,7 @@ func (s *Service) Delete(ctx context.Context, id uint) error { OperatorID: operatorID, Role: role, BeforeData: beforeData, AfterData: map[string]any{"deleted": true}, }) }); err != nil { - s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, beforeData, nil, err) + s.recordFailure(ctx, constants.AuditActionRoleDeleted, "删除角色失败", constants.AuditResultFailed, role, nil, beforeData, err) return errors.Wrap(errors.CodeInternalError, err, "删除角色失败") } @@ -510,19 +510,19 @@ func (s *Service) UpdateStatus(ctx context.Context, id uint, status int) error { if status == constants.StatusDisabled { accountCount, err := s.accountRoleStore.CountByRoleID(ctx, id) if err != nil { - s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err) + s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err) return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败") } shopCount, err := s.shopRoleStore.CountByRoleID(ctx, id) if err != nil { - s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, roleAuditData(role), nil, err) + s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, roleAuditData(role), err) return errors.Wrap(errors.CodeInternalError, err, "检查角色分配情况失败") } if accountCount > 0 || shopCount > 0 { appErr := errors.New(errors.CodeRoleInUse, fmt.Sprintf("该角色已分配给 %d 个账号、%d 个店铺，请先移除相关分配后再禁用", accountCount, shopCount)) - s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "拒绝禁用使用中的角色", constants.AuditResultDenied, role, roleAuditData(role), nil, appErr) + s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "拒绝禁用使用中的角色", constants.AuditResultDenied, role, nil, roleAuditData(role), appErr) return appErr } } @@ -540,7 +540,7 @@ func (s *Service) UpdateStatus(ctx context.Context, id uint, status int) error { OperatorID: currentUserID, Role: role, BeforeData: beforeData, AfterData: roleAuditData(role), }) }); err != nil { - s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, beforeData, nil, err) + s.recordFailure(ctx, constants.AuditActionRoleStatusUpdated, "更新角色状态失败", constants.AuditResultFailed, role, nil, beforeData, err) return errors.Wrap(errors.CodeInternalError, err, "更新角色状态失败") } diff --git a/internal/task/audit_daily_archive.go b/internal/task/audit_daily_archive.go new file mode 100644 index 0000000..ea9d626 --- /dev/null +++ b/internal/task/audit_daily_archive.go @@ -0,0 +1,61 @@ +package task + +import ( + "context" + "fmt" + "time" + + "github.com/bytedance/sonic" + "github.com/hibiken/asynq" + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/application/auditarchive" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// AuditDailyArchivePayload 是人工补档时可选的任务载荷。 +type AuditDailyArchivePayload struct { + ArchiveDate string `json:"archive_date"` +} + +// AuditDailyArchiveHandler 处理统一审计每日冷归档任务。 +type AuditDailyArchiveHandler struct { + service *auditarchive.Service + logger *zap.Logger +} + +// NewAuditDailyArchiveHandler 创建统一审计每日冷归档任务处理器。 +func NewAuditDailyArchiveHandler(service *auditarchive.Service, logger *zap.Logger) *AuditDailyArchiveHandler { + return &AuditDailyArchiveHandler{service: service, logger: logger} +} + +// Handle 执行前一完整自然日归档，或按任务载荷补档指定自然日。 +func (h *AuditDailyArchiveHandler) Handle(ctx context.Context, task *asynq.Task) error { + if h.service == nil { + return fmt.Errorf("统一审计归档服务未配置") + } + var err error + if len(task.Payload()) == 0 { + err = h.service.ArchivePreviousDay(ctx) + } else { + var payload AuditDailyArchivePayload + if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil { + return fmt.Errorf("解析统一审计归档任务载荷失败: %w", unmarshalErr) + } + location, locationErr := time.LoadLocation(constants.AuditArchiveTimezone) + if locationErr != nil { + return fmt.Errorf("加载统一审计归档时区失败: %w", locationErr) + } + archiveDate, parseErr := time.ParseInLocation(time.DateOnly, payload.ArchiveDate, location) + if parseErr != nil { + return fmt.Errorf("解析统一审计归档日期失败: %w", parseErr) + } + err = h.service.ArchiveDate(ctx, archiveDate) + } + if err != nil { + h.logger.Error("统一审计每日冷归档失败", zap.Error(err)) + return err + } + h.logger.Info("统一审计每日冷归档完成") + return nil +} diff --git a/internal/task/audit_monthly_retention.go b/internal/task/audit_monthly_retention.go new file mode 100644 index 0000000..3aaf1e9 --- /dev/null +++ b/internal/task/audit_monthly_retention.go @@ -0,0 +1,64 @@ +package task + +import ( + "context" + "fmt" + "time" + + "github.com/bytedance/sonic" + "github.com/hibiken/asynq" + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/application/auditarchive" +) + +// AuditMonthlyRetentionPayload 是人工补跑月度清理时可选的任务载荷。 +type AuditMonthlyRetentionPayload struct { + ArchiveMonth string `json:"archive_month"` +} + +// AuditMonthlyRetentionHandler 处理归档完整性门禁与上月在线日志物理清理。 +type AuditMonthlyRetentionHandler struct { + service *auditarchive.Service + logger *zap.Logger +} + +// NewAuditMonthlyRetentionHandler 创建月度日志留存清理处理器。 +func NewAuditMonthlyRetentionHandler(service *auditarchive.Service, logger *zap.Logger) *AuditMonthlyRetentionHandler { + return &AuditMonthlyRetentionHandler{service: service, logger: logger} +} + +// Handle 校验整月归档后按固定顺序分批物理删除 PostgreSQL 在线日志。 +func (h *AuditMonthlyRetentionHandler) Handle(ctx context.Context, task *asynq.Task) error { + if h.service == nil { + return fmt.Errorf("月度日志留存清理服务未配置") + } + startedAt := time.Now() + var result auditarchive.RetentionResult + var err error + if len(task.Payload()) == 0 { + result, err = h.service.CleanupPreviousMonth(ctx) + } else { + var payload AuditMonthlyRetentionPayload + if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil { + return fmt.Errorf("解析月度日志留存清理任务载荷失败: %w", unmarshalErr) + } + month, parseErr := parseArchiveMonth(payload.ArchiveMonth) + if parseErr != nil { + return parseErr + } + result, err = h.service.CleanupMonth(ctx, month) + } + fields := []zap.Field{ + zap.String("archive_month", result.Month), zap.Int64("audit_event_count", result.EventCount), + zap.Int64("event_resource_count", result.ResourceCount), zap.Int64("integration_log_count", result.IntegrationCount), + zap.Duration("duration", time.Since(startedAt)), zap.Int("manifest_count", len(result.ManifestKeys)), + } + if err != nil { + fields = append(fields, zap.String("severity", "critical"), zap.Error(err)) + h.logger.Error("月度日志留存清理失败，PostgreSQL 整月清理已阻断或等待断点续跑", fields...) + return err + } + h.logger.Info("月度日志留存清理完成", fields...) + return nil +} diff --git a/internal/task/auto_purchase.go b/internal/task/auto_purchase.go index 33c402f..a9725bd 100644 --- a/internal/task/auto_purchase.go +++ b/internal/task/auto_purchase.go @@ -27,7 +27,10 @@ import ( // AutoPurchasePayload 充值后自动购包任务载荷 type AutoPurchasePayload struct { - RechargeOrderID uint `json:"recharge_order_id"` + RechargeOrderID uint `json:"recharge_order_id"` + RequestID string `json:"request_id,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + ParentEventID string `json:"parent_event_id,omitempty"` } // AutoPurchaseHandler 充值后自动购包任务处理器 @@ -123,10 +126,14 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task) h.logger.Error("查询充值订单失败", zap.Uint("recharge_order_id", payload.RechargeOrderID), zap.Error(err)) return err } + correlationID := payload.CorrelationID + if correlationID == "" { + correlationID = rechargeOrder.RechargeOrderNo + } ctx = auditcontext.With(ctx, auditcontext.Context{ ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeAutoPurchaseAfterRecharge, ActorName: "充值后自动购包任务", Source: constants.AuditSourceWorker, - CorrelationID: rechargeOrder.RechargeOrderNo, + RequestID: payload.RequestID, CorrelationID: correlationID, ParentEventID: payload.ParentEventID, }) if rechargeOrder.AutoPurchaseStatus == constants.AutoPurchaseStatusSuccess { @@ -301,7 +308,11 @@ func (h *AutoPurchaseHandler) ProcessTask(ctx context.Context, task *asynq.Task) // 事务提交成功后触发佣金计算（不在事务内，防止任务提交后事务回滚的数据一致性问题） if h.asynqClient != nil && createdOrderID > 0 { - payloadBytes, marshalErr := sonic.Marshal(map[string]any{"order_id": createdOrderID}) + linkage := auditcontext.From(ctx) + payloadBytes, marshalErr := sonic.Marshal(CommissionCalculationPayload{ + OrderID: createdOrderID, RequestID: linkage.RequestID, + CorrelationID: linkage.CorrelationID, ParentEventID: linkage.ParentEventID, + }) if marshalErr != nil { h.logger.Warn("佣金任务载荷序列化失败", zap.Uint("order_id", createdOrderID), diff --git a/internal/task/card_observation_series.go b/internal/task/card_observation_series.go index faa0a4b..a978152 100644 --- a/internal/task/card_observation_series.go +++ b/internal/task/card_observation_series.go @@ -8,6 +8,8 @@ import ( "go.uber.org/zap" cardapp "github.com/break/junhong_cmp_fiber/internal/application/cardobservation" + "github.com/break/junhong_cmp_fiber/pkg/auditcontext" + "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/break/junhong_cmp_fiber/pkg/errors" ) @@ -32,6 +34,11 @@ func (h *CardObservationSeriesHandler) Handle(ctx context.Context, task *asynq.T h.logger.Error("解析卡观测序列任务载荷失败", zap.Error(err)) return errors.Wrap(errors.CodeInvalidParam, err, "卡观测序列任务载荷无法解析") } + ctx = auditcontext.With(ctx, auditcontext.Context{ + ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeCardObservationSeries, + ActorName: "卡观测序列任务", Source: constants.AuditSourceWorker, + RequestID: payload.RequestID, CorrelationID: payload.CorrelationID, ParentEventID: payload.ParentEventID, + }) if err := h.service.Execute(ctx, payload); err != nil { h.logger.Warn("卡观测序列当前尝试失败", zap.String("series_id", payload.SeriesID), zap.Int("attempt", payload.Attempt), zap.Error(err)) diff --git a/internal/task/commission_calculation.go b/internal/task/commission_calculation.go index 906b7ad..4c429b7 100644 --- a/internal/task/commission_calculation.go +++ b/internal/task/commission_calculation.go @@ -18,7 +18,10 @@ const ( ) type CommissionCalculationPayload struct { - OrderID uint `json:"order_id"` + OrderID uint `json:"order_id"` + RequestID string `json:"request_id,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + ParentEventID string `json:"parent_event_id,omitempty"` } type CommissionCalculationHandler struct { @@ -48,9 +51,15 @@ func (h *CommissionCalculationHandler) HandleCommissionCalculation(ctx context.C ) return asynq.SkipRetry } + correlationID := payload.CorrelationID + if correlationID == "" { + correlationID = task.ResultWriter().TaskID() + } ctx = auditcontext.With(ctx, auditcontext.Context{ ActorKind: constants.AuditActorSystemTask, ActorID: constants.AuditActorIDCommissionCalculationWorker, ActorName: "订单佣金计算任务", Source: constants.AuditSourceWorker, + RequestID: payload.RequestID, CorrelationID: correlationID, + ParentEventID: payload.ParentEventID, }) if err := h.service.CalculateCommission(ctx, payload.OrderID); err != nil { diff --git a/internal/task/integration_archive.go b/internal/task/integration_archive.go new file mode 100644 index 0000000..6ccd9ad --- /dev/null +++ b/internal/task/integration_archive.go @@ -0,0 +1,113 @@ +package task + +import ( + "context" + "fmt" + "time" + + "github.com/bytedance/sonic" + "github.com/hibiken/asynq" + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/application/auditarchive" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// IntegrationDailyArchivePayload 是人工补档时可选的任务载荷。 +type IntegrationDailyArchivePayload struct { + ArchiveDate string `json:"archive_date"` +} + +// IntegrationMonthlyFinalizePayload 是人工月度复核时可选的任务载荷。 +type IntegrationMonthlyFinalizePayload struct { + ArchiveMonth string `json:"archive_month"` +} + +// IntegrationArchiveHandler 处理 Integration Log 每日归档与月度最终复核。 +type IntegrationArchiveHandler struct { + service *auditarchive.Service + logger *zap.Logger +} + +// NewIntegrationArchiveHandler 创建 Integration Log 归档任务处理器。 +func NewIntegrationArchiveHandler(service *auditarchive.Service, logger *zap.Logger) *IntegrationArchiveHandler { + return &IntegrationArchiveHandler{service: service, logger: logger} +} + +// HandleDaily 执行前一完整自然日归档，或按任务载荷补档指定自然日。 +func (h *IntegrationArchiveHandler) HandleDaily(ctx context.Context, task *asynq.Task) error { + if h.service == nil { + return fmt.Errorf("Integration Log 归档服务未配置") + } + var err error + if len(task.Payload()) == 0 { + err = h.service.ArchivePreviousIntegrationDay(ctx) + } else { + var payload IntegrationDailyArchivePayload + if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil { + return fmt.Errorf("解析 Integration Log 每日归档任务载荷失败: %w", unmarshalErr) + } + date, parseErr := parseArchiveDate(payload.ArchiveDate) + if parseErr != nil { + return parseErr + } + err = h.service.ArchiveIntegrationDate(ctx, date) + } + if err != nil { + h.logger.Error("Integration Log 每日冷归档失败", zap.Error(err)) + return err + } + h.logger.Info("Integration Log 每日冷归档完成") + return nil +} + +// HandleMonthlyFinalize 执行上一个完整自然月复核，或按任务载荷复核指定月份。 +func (h *IntegrationArchiveHandler) HandleMonthlyFinalize(ctx context.Context, task *asynq.Task) error { + if h.service == nil { + return fmt.Errorf("Integration Log 归档服务未配置") + } + var err error + if len(task.Payload()) == 0 { + err = h.service.FinalizePreviousIntegrationMonth(ctx) + } else { + var payload IntegrationMonthlyFinalizePayload + if unmarshalErr := sonic.Unmarshal(task.Payload(), &payload); unmarshalErr != nil { + return fmt.Errorf("解析 Integration Log 月度复核任务载荷失败: %w", unmarshalErr) + } + month, parseErr := parseArchiveMonth(payload.ArchiveMonth) + if parseErr != nil { + return parseErr + } + err = h.service.FinalizeIntegrationMonth(ctx, month) + } + if err != nil { + h.logger.Error("Integration Log 月度最终版本复核失败，后续清理必须阻止", zap.Error(err)) + return err + } + h.logger.Info("Integration Log 月度最终版本复核完成") + return nil +} + +func parseArchiveDate(value string) (time.Time, error) { + location, err := time.LoadLocation(constants.AuditArchiveTimezone) + if err != nil { + return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err) + } + date, err := time.ParseInLocation(time.DateOnly, value, location) + if err != nil { + return time.Time{}, fmt.Errorf("解析 Integration Log 归档日期失败: %w", err) + } + return date, nil +} + +func parseArchiveMonth(value string) (time.Time, error) { + location, err := time.LoadLocation(constants.AuditArchiveTimezone) + if err != nil { + return time.Time{}, fmt.Errorf("加载 Integration Log 归档时区失败: %w", err) + } + month, err := time.ParseInLocation("2006-01", value, location) + if err != nil { + return time.Time{}, fmt.Errorf("解析 Integration Log 归档月份失败: %w", err) + } + return month, nil +} diff --git a/internal/task/notification_cleanup.go b/internal/task/notification_cleanup.go index 1c7116f..991780b 100644 --- a/internal/task/notification_cleanup.go +++ b/internal/task/notification_cleanup.go @@ -23,10 +23,11 @@ func NewNotificationCleanupHandler(service *notificationinfra.CleanupService, lo } // Handle 执行有界、可重入的通知分批清理。 -func (h *NotificationCleanupHandler) Handle(ctx context.Context, _ *asynq.Task) error { +func (h *NotificationCleanupHandler) Handle(ctx context.Context, task *asynq.Task) error { ctx = auditcontext.With(ctx, auditcontext.Context{ ActorKind: constants.AuditActorSystemTask, ActorID: constants.TaskTypeNotificationCleanup, ActorName: "站内通知清理任务", Source: constants.AuditSourceWorker, + CorrelationID: task.ResultWriter().TaskID(), }) h.logger.Info("开始执行站内通知保留清理") if err := h.service.Run(ctx); err != nil { diff --git a/internal/task/polling_carddata_handler.go b/internal/task/polling_carddata_handler.go index 254b0e7..331437a 100644 --- a/internal/task/polling_carddata_handler.go +++ b/internal/task/polling_carddata_handler.go @@ -77,6 +77,7 @@ func (h *PollingCarddataHandler) Handle(ctx context.Context, task *asynq.Task) e if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil { return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 流量 Integration Log 失败", logErr) } + ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCarddata, "卡流量轮询任务", attempt.IntegrationID) if h.observation == nil || h.carrier == nil { return h.failAndRequeue(ctx, cardID, startedAt, "卡流量观测能力未配置", nil) } diff --git a/internal/task/polling_cardstatus_handler.go b/internal/task/polling_cardstatus_handler.go index 576f86d..4dcc863 100644 --- a/internal/task/polling_cardstatus_handler.go +++ b/internal/task/polling_cardstatus_handler.go @@ -76,6 +76,7 @@ func (h *PollingCardStatusHandler) Handle(ctx context.Context, task *asynq.Task) if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil { return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 网络 Integration Log 失败", logErr) } + ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingCardStatus, "卡网络状态轮询任务", attempt.IntegrationID) if h.observation == nil { return h.failAndRequeue(ctx, cardID, startedAt, "卡网络观测能力未配置", nil) } diff --git a/internal/task/polling_package_handler.go b/internal/task/polling_package_handler.go index b3f525f..297b2b5 100644 --- a/internal/task/polling_package_handler.go +++ b/internal/task/polling_package_handler.go @@ -44,6 +44,7 @@ func (h *PollingPackageHandler) Handle(ctx context.Context, t *asynq.Task) error if !ok { return nil } + ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingPackage, "套餐状态轮询任务", t.ResultWriter().TaskID()) if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingPackage) { h.base.logger.Debug("并发已满，重新入队", zap.Uint("card_id", cardID)) diff --git a/internal/task/polling_protect_handler.go b/internal/task/polling_protect_handler.go index 1a2131d..b0e2602 100644 --- a/internal/task/polling_protect_handler.go +++ b/internal/task/polling_protect_handler.go @@ -52,6 +52,7 @@ func (h *PollingProtectHandler) Handle(ctx context.Context, t *asynq.Task) error if !ok { return nil } + ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingProtect, "保护期一致性轮询任务", t.ResultWriter().TaskID()) if !h.base.acquireConcurrency(ctx, constants.TaskTypePollingProtect) { h.base.logger.Debug("并发已满，重新入队", zap.Uint("card_id", cardID)) diff --git a/internal/task/polling_realname_handler.go b/internal/task/polling_realname_handler.go index 1d46a91..4761f57 100644 --- a/internal/task/polling_realname_handler.go +++ b/internal/task/polling_realname_handler.go @@ -68,6 +68,7 @@ func (h *PollingRealnameHandler) Handle(ctx context.Context, task *asynq.Task) e if logErr := completeGatewayAttempt(ctx, h.integration, attempt, true, attemptStartedAt); logErr != nil { return h.failAndRequeue(ctx, cardID, startedAt, "完成 Gateway 实名 Integration Log 失败", logErr) } + ctx = withPollingWorkerAuditContext(ctx, constants.TaskTypePollingRealname, "实名状态轮询任务", attempt.IntegrationID) if h.observation == nil { return h.failAndRequeue(ctx, cardID, startedAt, "卡实名观测能力未配置", nil) } diff --git a/internal/task/polling_utils.go b/internal/task/polling_utils.go index 3420775..18ae3c7 100644 --- a/internal/task/polling_utils.go +++ b/internal/task/polling_utils.go @@ -1,14 +1,29 @@ package task import ( + "context" "strconv" "time" "github.com/bytedance/sonic" "go.uber.org/zap" "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/pkg/auditcontext" + "github.com/break/junhong_cmp_fiber/pkg/constants" ) +// withPollingWorkerAuditContext 为实际改变业务事实的轮询任务补充真实系统操作者与链路。 +func withPollingWorkerAuditContext(ctx context.Context, taskType, taskName, correlationID string) context.Context { + if correlationID == "" { + correlationID = taskType + } + return auditcontext.With(ctx, auditcontext.Context{ + ActorKind: constants.AuditActorSystemTask, ActorID: taskType, + ActorName: taskName, Source: constants.AuditSourceWorker, CorrelationID: correlationID, + }) +} + // shortTaskType 从完整任务类型中提取简短名称（如 polling:carddata → carddata） func shortTaskType(fullTaskType string) string { for i := len(fullTaskType) - 1; i >= 0; i-- { diff --git a/m.csv b/m.csv new file mode 100644 index 0000000..2c9ae84 --- /dev/null +++ b/m.csv @@ -0,0 +1 @@ +89861590172420377385 diff --git a/migrations/000204_add_audit_investigation_indexes.down.sql b/migrations/000204_add_audit_investigation_indexes.down.sql new file mode 100644 index 0000000..23bef6a --- /dev/null +++ b/migrations/000204_add_audit_investigation_indexes.down.sql @@ -0,0 +1,19 @@ +-- 回滚统一审计跨视角查询索引，并恢复 request 与 parent 的原始精确查询索引。 +DROP INDEX idx_audit_resource_key_event; +DROP INDEX idx_audit_resource_id_event; + +DROP INDEX idx_audit_event_parent; +CREATE INDEX idx_audit_event_parent + ON tb_audit_event (parent_event_id) + WHERE parent_event_id <> ''; + +DROP INDEX idx_audit_event_request; +CREATE INDEX idx_audit_event_request + ON tb_audit_event (request_id) + WHERE request_id <> ''; + +DROP INDEX idx_audit_event_source_occurred; +DROP INDEX idx_audit_event_category_occurred; +DROP INDEX idx_audit_event_risk_occurred; +DROP INDEX idx_audit_event_result_occurred; +DROP INDEX idx_audit_event_action_occurred; diff --git a/migrations/000204_add_audit_investigation_indexes.up.sql b/migrations/000204_add_audit_investigation_indexes.up.sql new file mode 100644 index 0000000..5c9d758 --- /dev/null +++ b/migrations/000204_add_audit_investigation_indexes.up.sql @@ -0,0 +1,32 @@ +-- 为统一审计跨视角查询补充受控 B-tree 索引，不增加 JSONB 搜索、缓存或分区。 +CREATE INDEX idx_audit_event_action_occurred + ON tb_audit_event (action_code, occurred_at DESC, id DESC); + +CREATE INDEX idx_audit_event_result_occurred + ON tb_audit_event (result, occurred_at DESC, id DESC); + +CREATE INDEX idx_audit_event_risk_occurred + ON tb_audit_event (risk_level, occurred_at DESC, id DESC); + +CREATE INDEX idx_audit_event_category_occurred + ON tb_audit_event (category, occurred_at DESC, id DESC); + +CREATE INDEX idx_audit_event_source_occurred + ON tb_audit_event (source, occurred_at DESC, id DESC); + +DROP INDEX idx_audit_event_request; +CREATE INDEX idx_audit_event_request + ON tb_audit_event (request_id, occurred_at DESC, id DESC) + WHERE request_id <> ''; + +DROP INDEX idx_audit_event_parent; +CREATE INDEX idx_audit_event_parent + ON tb_audit_event (parent_event_id, occurred_at DESC, id DESC) + WHERE parent_event_id <> ''; + +CREATE INDEX idx_audit_resource_id_event + ON tb_audit_event_resource (resource_type, resource_id, audit_event_id) + WHERE resource_id IS NOT NULL; + +CREATE INDEX idx_audit_resource_key_event + ON tb_audit_event_resource (resource_type, resource_key, audit_event_id); diff --git a/migrations/000205_create_log_archive_run.down.sql b/migrations/000205_create_log_archive_run.down.sql new file mode 100644 index 0000000..a36b9e7 --- /dev/null +++ b/migrations/000205_create_log_archive_run.down.sql @@ -0,0 +1,11 @@ +LOCK TABLE tb_log_archive_run IN ACCESS EXCLUSIVE MODE; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM tb_log_archive_run LIMIT 1) THEN + RAISE EXCEPTION 'tb_log_archive_run 已存在归档运行事实，禁止回滚迁移'; + END IF; +END; +$$; + +DROP TABLE tb_log_archive_run; diff --git a/migrations/000205_create_log_archive_run.up.sql b/migrations/000205_create_log_archive_run.up.sql new file mode 100644 index 0000000..9cfee1e --- /dev/null +++ b/migrations/000205_create_log_archive_run.up.sql @@ -0,0 +1,53 @@ +CREATE TABLE tb_log_archive_run ( + id bigserial PRIMARY KEY, + source varchar(32) NOT NULL, + archive_date date NOT NULL, + instance_id varchar(100) NOT NULL, + schema_version varchar(32) NOT NULL, + revision integer NOT NULL DEFAULT 1, + status varchar(16) NOT NULL, + is_final boolean NOT NULL DEFAULT false, + range_start timestamptz NOT NULL, + range_end timestamptz NOT NULL, + object_key varchar(500) NOT NULL DEFAULT '', + manifest_key varchar(500) NOT NULL DEFAULT '', + event_count bigint NOT NULL DEFAULT 0, + resource_count bigint NOT NULL DEFAULT 0, + record_count bigint NOT NULL DEFAULT 0, + uncompressed_bytes bigint NOT NULL DEFAULT 0, + compressed_bytes bigint NOT NULL DEFAULT 0, + sha256 varchar(64) NOT NULL DEFAULT '', + attempt_count integer NOT NULL DEFAULT 0, + error_summary varchar(500) NOT NULL DEFAULT '', + generated_at timestamptz, + completed_at timestamptz, + cleanup_started_at timestamptz, + cleaned_at timestamptz, + created_at timestamptz NOT NULL DEFAULT NOW(), + updated_at timestamptz NOT NULL DEFAULT NOW(), + CONSTRAINT uq_log_archive_run UNIQUE (source, archive_date, instance_id, schema_version), + CONSTRAINT ck_log_archive_run_status CHECK (status IN ('pending', 'running', 'success', 'failed')), + CONSTRAINT ck_log_archive_run_revision CHECK (revision > 0), + CONSTRAINT ck_log_archive_run_range CHECK (range_end > range_start), + CONSTRAINT ck_log_archive_run_counts CHECK ( + event_count >= 0 AND resource_count >= 0 AND record_count >= 0 + AND uncompressed_bytes >= 0 AND compressed_bytes >= 0 AND attempt_count >= 0 + ) +); + +CREATE INDEX idx_log_archive_run_date_status + ON tb_log_archive_run (archive_date, status, source); + +COMMENT ON TABLE tb_log_archive_run IS '日志冷归档运行账本，不保存日志正文'; +COMMENT ON COLUMN tb_log_archive_run.source IS '归档数据源稳定编码'; +COMMENT ON COLUMN tb_log_archive_run.archive_date IS 'Asia/Shanghai 归档自然日'; +COMMENT ON COLUMN tb_log_archive_run.instance_id IS '归档任务实例标识'; +COMMENT ON COLUMN tb_log_archive_run.schema_version IS '归档 JSONL 结构版本'; +COMMENT ON COLUMN tb_log_archive_run.revision IS '同一归档日不可变对象版本'; +COMMENT ON COLUMN tb_log_archive_run.status IS '归档状态：pending、running、success 或 failed'; +COMMENT ON COLUMN tb_log_archive_run.is_final IS '是否已按数据库当前内容形成月度最终版本'; +COMMENT ON COLUMN tb_log_archive_run.object_key IS 'JSONL gzip 对象 Key'; +COMMENT ON COLUMN tb_log_archive_run.manifest_key IS '归档 manifest 对象 Key'; +COMMENT ON COLUMN tb_log_archive_run.sha256 IS 'gzip 对象内容 SHA-256'; +COMMENT ON COLUMN tb_log_archive_run.cleanup_started_at IS '月度物理清理已通过门禁并开始执行的时间'; +COMMENT ON COLUMN tb_log_archive_run.cleaned_at IS '后续月度在线数据清理完成时间'; diff --git a/openspec/changes/build-multi-view-audit-center/tasks.md b/openspec/changes/build-multi-view-audit-center/tasks.md index 0c96e24..dc3fac2 100644 --- a/openspec/changes/build-multi-view-audit-center/tasks.md +++ b/openspec/changes/build-multi-view-audit-center/tasks.md @@ -82,24 +82,24 @@ - [x] 8.3 迁移卡/设备导入、资产套餐批购、订单套餐失效和导出任务创建/取消纵向用例，记录任务、文件名/目标/操作者和批量根子结果；对象存储签名 URL 不入审计，审计中心自身仍不提供导出。【主：Application + Asynq｜辅：Object Storage｜边界：现有导入批量和业务导出任务动作｜不迁移：导出 DataSource 内容｜验证：结构化 payload、幂等、partial、凭据删除和任务/资源时间线核对通过，LSP 无诊断】 - [x] 8.4 迁移通知生成、投递、单条/全部已读和清理纵向用例，低风险写仍登记 action，系统清理使用系统 actor，Audit Event 不替代通知或 Outbox 投递事实。【主：简单写/Application + Worker｜边界：通知状态变化｜不迁移：通知查询｜验证：人工/系统 actor、低风险默认展示、Outbox 幂等和资源关联核对通过，LSP 无诊断】 - [x] 8.5 迁移轮询配置、并发配置、告警规则、人工触发/取消和状态变化纵向用例；手动轮询表继续承担进度/结果，实际 Gateway 尝试写 Integration Log，人工触发动作另写 Audit Event。【主：Application + Adapter｜辅：Scheduler/Query｜边界：轮询配置与人工动作｜不迁移：删除手动任务表和普通运行查询｜验证：配置/触发/拒绝、任务 ledger 保留、Integration/Audit 分界和 actor 核对通过，LSP 无诊断】 -- [ ] 8.6 迁移支付、运营商实名和企微等外部 Callback 中实际改变内部事实的入口，使用 external actor；无状态变化只保留 Integration Log。【主：Application/Infrastructure Adapter｜边界：当前外部回调｜不迁移：渠道协议｜验证：逐回调 action、幂等、资源解析、Integration/Audit 分界和链路核对通过，LSP 无诊断】 -- [ ] 8.7 迁移当前 Worker 中实际改变内部业务事实的入口，使用 system_task actor 并传播 correlation/parent；纯投递或技术装配按清单 N/A。【主：Application/Asynq｜边界：当前 Worker 写入口｜不迁移：Scheduler 与 Callback｜验证：逐 Worker 覆盖、幂等、失败重试和未登记动作门禁 PASS，LSP 无诊断】 -- [ ] 8.8 迁移当前 Scheduler 中实际创建任务、改变配置/状态或产生业务事实的入口，使用 scheduled_job actor；实施时以当前清单为准，不硬编码历史数量。【主：Application/Scheduler｜边界：当前计划任务｜不迁移：Worker 消费逻辑｜验证：逐 Scheduler 覆盖、重复调度幂等、父子链路和 N/A 理由 PASS，LSP 无诊断】 -- [ ] 8.9 迁移当前 Outbox 消费者中实际形成新业务事实的入口，保留 Outbox 投递事实并为内部变化写 system_task Audit Event，不把投递成功伪装成业务成功。【主：Application/Outbox Consumer｜边界：当前可靠事件消费者｜不迁移：Relay 实现｜验证：逐消费者覆盖、至少一次幂等、业务/投递结果分离和 correlation 核对通过，LSP 无诊断】 +- [x] 8.6 迁移支付、运营商实名和企微等外部 Callback 中实际改变内部事实的入口，使用 external actor；无状态变化只保留 Integration Log。【主：Application/Infrastructure Adapter｜边界：当前外部回调｜不迁移：渠道协议｜验证：逐回调 action、幂等、资源解析、Integration/Audit 分界和链路核对通过，LSP 无诊断】 +- [x] 8.7 迁移当前 Worker 中实际改变内部业务事实的入口，使用 system_task actor 并传播 correlation/parent；纯投递或技术装配按清单 N/A。【主：Application/Asynq｜边界：当前 Worker 写入口｜不迁移：Scheduler 与 Callback｜验证：逐 Worker 覆盖、幂等、失败重试和未登记动作门禁 PASS，LSP 无诊断】 +- [x] 8.8 迁移当前 Scheduler 中实际创建任务、改变配置/状态或产生业务事实的入口，使用 scheduled_job actor；实施时以当前清单为准，不硬编码历史数量。【主：Application/Scheduler｜边界：当前计划任务｜不迁移：Worker 消费逻辑｜验证：逐 Scheduler 覆盖、重复调度幂等、父子链路和 N/A 理由 PASS，LSP 无诊断】 +- [x] 8.9 迁移当前 Outbox 消费者中实际形成新业务事实的入口，保留 Outbox 投递事实并为内部变化写 system_task Audit Event，不把投递成功伪装成业务成功。【主：Application/Outbox Consumer｜边界：当前可靠事件消费者｜不迁移：Relay 实现｜验证：逐消费者覆盖、至少一次幂等、业务/投递结果分离和 correlation 核对通过，LSP 无诊断】 ## 9. 跨视角调查与性能收口 -- [ ] 9.1 交付 request 和 correlation 组合时间线，按 `record_source` 组合 Audit Event、Integration Log、Outbox/Asynq 摘要及 Domain Ledger 引用，不扫描 Access Log、不猜测历史链路或技术重试。【主：Query｜边界：跨事实只读投影｜不迁移：Access Log 存储与关系图｜验证：支付、退款、审批、异步链路、历史缺字段和稳定排序核对通过，LSP 无诊断】 -- [ ] 9.2 交付资金调查时间线，按店铺、钱包、订单、支付、退款、充值、审批、交易号、操作者、时间和 correlation 查询，并明确金额权威来自钱包流水及业务表。【主：Query｜边界：资金多源投影｜不迁移：资金重算或状态修改｜验证：余额变化、退款、充值、佣金、来源冲突和权限身份核对通过，LSP 无诊断】 -- [ ] 9.3 交付风险 overview/events，聚合高风险、资金、安全、失败、拒绝、partial 和 unknown，支持跳转事件、资源、actor 和 correlation；不建设处置工单或自动封禁。【主：Query｜边界：固定风险调查视角｜不迁移：风控决策系统｜验证：计数/趋势、低风险排除、跳转和时间范围性能核对通过，LSP 无诊断】 -- [ ] 9.4 为事件、资源、actor、action/result/risk、scope、request、correlation、parent 及跨视角常用过滤补最小索引，使用先分页 ID 后批量投影避免 N+1，不为第一阶段增加 JSONB 任意模糊搜索或 Redis 结果缓存。【主：Query/Infrastructure + Migration｜边界：已确认查询路径｜不迁移：数据库月分区与冷热联合查询｜验证：迁移 up/down、EXPLAIN/基准、数据库 <50ms、API P95/P99 和并发分页核对通过，LSP 无诊断】 -- [ ] 9.5 完成跨视角 Handler/DTO/RouteSpec、生产装配、共享文档 Handler、`cmd/api/docs.go`、`cmd/gendocs/main.go` 和中文功能/API 文档，前端契约必须逐行呈现 design 8.1-8.4 的源页面、前置接口、`response.data` 字段、入口名称/可见条件、目标接口、参数映射和降级行为，不得只罗列审计 API。【主：API/Documentation｜边界：9.1-9.3 查询接口｜不迁移：前端页面实现｜验证：以资产详情、订单、退款、钱包、通知、风险节点逐条演示完整调用链，两条 OpenAPI、README 索引和构建检查 PASS，LSP 无诊断】 +- [x] 9.1 交付 request 和 correlation 组合时间线，按 `record_source` 组合 Audit Event、Integration Log、Outbox/Asynq 摘要及 Domain Ledger 引用，不扫描 Access Log、不猜测历史链路或技术重试。【主：Query｜边界：跨事实只读投影｜不迁移：Access Log 存储与关系图｜验证：支付、退款、审批、异步链路、历史缺字段和稳定排序核对通过，LSP 无诊断】 +- [x] 9.2 交付资金调查时间线，按店铺、钱包、订单、支付、退款、充值、审批、交易号、操作者、时间和 correlation 查询，并明确金额权威来自钱包流水及业务表。【主：Query｜边界：资金多源投影｜不迁移：资金重算或状态修改｜验证：余额变化、退款、充值、佣金、来源冲突和权限身份核对通过，LSP 无诊断】 +- [x] 9.3 交付风险 overview/events，聚合高风险、资金、安全、失败、拒绝、partial 和 unknown，支持跳转事件、资源、actor 和 correlation；不建设处置工单或自动封禁。【主：Query｜边界：固定风险调查视角｜不迁移：风控决策系统｜验证：计数/趋势、低风险排除、跳转和时间范围性能核对通过，LSP 无诊断】 +- [x] 9.4 为事件、资源、actor、action/result/risk、scope、request、correlation、parent 及跨视角常用过滤补最小索引，使用先分页 ID 后批量投影避免 N+1，不为第一阶段增加 JSONB 任意模糊搜索或 Redis 结果缓存。【主：Query/Infrastructure + Migration｜边界：已确认查询路径｜不迁移：数据库月分区与冷热联合查询｜验证：迁移 up/down、EXPLAIN/基准、数据库 <50ms、API P95/P99 和并发分页核对通过，LSP 无诊断】 +- [x] 9.5 完成跨视角 Handler/DTO/RouteSpec、生产装配、共享文档 Handler、`cmd/api/docs.go`、`cmd/gendocs/main.go` 和中文功能/API 文档，前端契约必须逐行呈现 design 8.1-8.4 的源页面、前置接口、`response.data` 字段、入口名称/可见条件、目标接口、参数映射和降级行为，不得只罗列审计 API。【主：API/Documentation｜边界：9.1-9.3 查询接口｜不迁移：前端页面实现｜验证：以资产详情、订单、退款、钱包、通知、风险节点逐条演示完整调用链，两条 OpenAPI、README 索引和构建检查 PASS，LSP 无诊断】 ## 10. 每日冷归档与月初受控清理 -- [ ] 10.1 交付 Audit Event + Event Resource 每日归档纵向切片：新增无外键 `tb_log_archive_run`、按 `Asia/Shanghai` 前一完整自然日读取、事件携带完整 resources 的 JSONL+gzip、manifest/SHA-256、对象 metadata 复核、稳定对象 Key 与重复任务幂等；复用现有对象存储和 Asynq，不阻塞业务 Writer。【主：Infrastructure + Application/Asynq｜边界：Audit 每日冷归档完整闭环｜不迁移：Integration、Access、清理与归档查询｜验证：迁移 up/down、空日/大日、事件资源计数、对象损坏、重复投递、存储故障和业务写入隔离核对通过，LSP 无诊断】 -- [ ] 10.2 交付 Integration Log 每日归档及月度最终 revision 纵向切片：每日保存创建日快照，月初按数据库当前内容复核可变记录，内容变化时创建不可变新 revision，pending 或不一致阻止清理，不覆盖旧对象。【主：Infrastructure + Application/Asynq｜边界：Integration 冷归档最终性｜不迁移：Integration Writer 语义、恢复和对象存储查询｜验证：pending→success/unknown、跨月更新、hash 变化、重复复核、无法终结和对象版本核对通过，LSP 无诊断】 -- [ ] 10.3 交付月初整月数据库物理删除纵向切片：先完成上月最后一天归档，再校验全部日期的 Audit/Integration manifest、Audit 事件与资源数、Integration 最终 revision、对象大小和 SHA-256；通过后使用 GORM、索引和有界批次按先 Event Resource 后 Event、再 Integration 的受控流程物理 `DELETE` PostgreSQL 上月数据，任务可从 ledger 断点继续，并写当前月 `retention_worker` Audit Event；审计 Model 不含 `gorm.DeletedAt`，不写归档状态、不迁移数据库历史表、不删除对象存储备份。【主：Application + Infrastructure/Asynq｜边界：归档门禁与 PostgreSQL 上月数据物理删除｜不迁移：Access Log、Domain Ledger、Outbox、旧 operation log、对象存储生命周期、分区改造和人工清表｜验证：缺日/hash 不一致/部分对象/pending 全部阻断，物理行数归零、批次中断续跑、窗口隔离、对象长期保留、清理事件和数据库膨胀指标核对通过，LSP 无诊断】 +- [x] 10.1 交付 Audit Event + Event Resource 每日归档纵向切片：新增无外键 `tb_log_archive_run`、按 `Asia/Shanghai` 前一完整自然日读取、事件携带完整 resources 的 JSONL+gzip、manifest/SHA-256、对象 metadata 复核、稳定对象 Key 与重复任务幂等；复用现有对象存储和 Asynq，不阻塞业务 Writer。【主：Infrastructure + Application/Asynq｜边界：Audit 每日冷归档完整闭环｜不迁移：Integration、Access、清理与归档查询｜验证：迁移 up/down、空日/大日、事件资源计数、对象损坏、重复投递、存储故障和业务写入隔离核对通过，LSP 无诊断】 +- [x] 10.2 交付 Integration Log 每日归档及月度最终 revision 纵向切片：每日保存创建日快照，月初按数据库当前内容复核可变记录，内容变化时创建不可变新 revision，pending 或不一致阻止清理，不覆盖旧对象。【主：Infrastructure + Application/Asynq｜边界：Integration 冷归档最终性｜不迁移：Integration Writer 语义、恢复和对象存储查询｜验证：pending→success/unknown、跨月更新、hash 变化、重复复核、无法终结和对象版本核对通过，LSP 无诊断】 +- [x] 10.3 交付月初整月数据库物理删除纵向切片：先完成上月最后一天归档，再校验全部日期的 Audit/Integration manifest、Audit 事件与资源数、Integration 最终 revision、对象大小和 SHA-256；通过后使用 GORM、索引和有界批次按先 Event Resource 后 Event、再 Integration 的受控流程物理 `DELETE` PostgreSQL 上月数据，任务可从 ledger 断点继续，并写当前月 `retention_worker` Audit Event；审计 Model 不含 `gorm.DeletedAt`，不写归档状态、不迁移数据库历史表、不删除对象存储备份。【主：Application + Infrastructure/Asynq｜边界：归档门禁与 PostgreSQL 上月数据物理删除｜不迁移：Access Log、Domain Ledger、Outbox、旧 operation log、对象存储生命周期、分区改造和人工清表｜验证：缺日/hash 不一致/部分对象/pending 全部阻断，物理行数归零、批次中断续跑、窗口隔离、对象长期保留、清理事件和数据库膨胀指标核对通过，LSP 无诊断】 - [ ] 10.4 为平台 Audit/Integration 及代理/企业活动 DTO 增加 `retention{online_from,archived_before,timezone}`，显式时间范围早于或跨越在线边界时返回稳定已归档错误；默认只查在线窗口，ID-only 不存在仍按资源不存在处理，不访问对象存储，也不新增下载或恢复路由。【主：Query/API｜边界：在线查询留存语义｜不迁移：冷热联合查询和前端页面实现｜验证：在线、已归档、跨边界、默认范围、空在线月、路由/OpenAPI 无归档读写能力核对通过，LSP 无诊断】 - [ ] 10.5 先以“只归档、不清理”开关运行并验收一个完整自然月，记录每日成功率、积压、对象大小、压缩率、hash/计数差异、Integration revision、清理预估耗时和告警；只有月度演练全部通过后才启用清理开关。【主：Release/Observability｜边界：留存能力灰度启用｜不迁移：Access Log、自动恢复和分区改造｜验证：完整月 manifest、故障重试、月度 dry-run、监控阈值、操作手册和启停回滚证据齐全】 diff --git a/pkg/constants/audit.go b/pkg/constants/audit.go index f020678..5dc8b76 100644 --- a/pkg/constants/audit.go +++ b/pkg/constants/audit.go @@ -1,5 +1,63 @@ package constants +import "time" + +const ( + // AuditRiskQueryMaxRange 是风险调查允许的最大连续时间范围。 + AuditRiskQueryMaxRange = 31 * 24 * time.Hour + // AuditRiskHourlyTrendMaxRange 是风险趋势使用小时粒度的最大时间范围。 + AuditRiskHourlyTrendMaxRange = 72 * time.Hour + // AuditRiskSignalHighRisk 表示高风险或严重风险信号。 + AuditRiskSignalHighRisk = "high_risk" + // AuditRiskSignalFinance 表示涉及资金资源的风险信号。 + AuditRiskSignalFinance = "finance" + // AuditRiskSignalSecurity 表示安全类别风险信号。 + AuditRiskSignalSecurity = "security" + // AuditRiskSignalFailed 表示执行失败信号。 + AuditRiskSignalFailed = "failed" + // AuditRiskSignalDenied 表示业务或权限拒绝信号。 + AuditRiskSignalDenied = "denied" + // AuditRiskSignalPartial 表示部分成功信号。 + AuditRiskSignalPartial = "partial" + // AuditRiskSignalUnknown 表示结果未知信号。 + AuditRiskSignalUnknown = "unknown" +) + +const ( + // AuditRecordSourceAuditEvent 表示统一内部审计事件节点。 + AuditRecordSourceAuditEvent = "audit_event" + // AuditRecordSourceIntegrationLog 表示外部交互日志节点。 + AuditRecordSourceIntegrationLog = "integration_log" + // AuditRecordSourceOutboxEvent 表示可靠事件投递摘要节点。 + AuditRecordSourceOutboxEvent = "outbox_event" + // AuditRecordSourceAsynqTask 表示具有持久化任务资源的异步任务摘要节点。 + AuditRecordSourceAsynqTask = "asynq_task" + // AuditRecordSourceDomainLedgerRef 表示业务权威表的稳定引用节点。 + AuditRecordSourceDomainLedgerRef = "domain_ledger_ref" + // AuditRecordSourceAgentWalletTransaction 表示代理钱包权威流水节点。 + AuditRecordSourceAgentWalletTransaction = "agent_wallet_transaction" + // AuditRecordSourceAssetWalletTransaction 表示资产钱包权威流水节点。 + AuditRecordSourceAssetWalletTransaction = "asset_wallet_transaction" + // AuditRecordSourceAgentWalletReservation 表示代理钱包预占事实节点。 + AuditRecordSourceAgentWalletReservation = "agent_wallet_reservation" + // AuditRecordSourceOrder 表示订单业务事实节点。 + AuditRecordSourceOrder = "order" + // AuditRecordSourcePayment 表示支付业务事实节点。 + AuditRecordSourcePayment = "payment" + // AuditRecordSourceRefund 表示退款业务事实节点。 + AuditRecordSourceRefund = "refund" + // AuditRecordSourceAgentRecharge 表示代理充值业务事实节点。 + AuditRecordSourceAgentRecharge = "agent_recharge" + // AuditRecordSourceRechargeOrder 表示个人资产充值业务事实节点。 + AuditRecordSourceRechargeOrder = "recharge_order" + // AuditRecordSourceCommissionRecord 表示佣金业务事实节点。 + AuditRecordSourceCommissionRecord = "commission_record" + // AuditRecordSourceCommissionWithdrawal 表示佣金提现业务事实节点。 + AuditRecordSourceCommissionWithdrawal = "commission_withdrawal" + // AuditRecordSourceApprovalInstance 表示审批业务事实节点。 + AuditRecordSourceApprovalInstance = "approval_instance" +) + const ( // AuditActionAccountCreated 表示创建账号。 AuditActionAccountCreated = "account.created" @@ -105,6 +163,12 @@ const ( AuditActionIotCardManualRefreshed = "iot_card.manual_refresh" // AuditActionIotCardPersonalRefreshed 表示个人客户刷新名下 IoT 卡并同步实际变化的内部事实。 AuditActionIotCardPersonalRefreshed = "iot_card.personal_refresh" + // AuditActionIotCardWorkerRealnameSynced 表示 Worker 观测并同步 IoT 卡实名事实。 + AuditActionIotCardWorkerRealnameSynced = "iot_card.worker_realname_sync" + // AuditActionIotCardWorkerTrafficSynced 表示 Worker 观测并同步 IoT 卡流量事实。 + AuditActionIotCardWorkerTrafficSynced = "iot_card.worker_traffic_sync" + // AuditActionIotCardWorkerNetworkSynced 表示 Worker 观测并同步 IoT 卡网络事实。 + AuditActionIotCardWorkerNetworkSynced = "iot_card.worker_network_sync" // AuditActionIotCardManualStopped 表示人工停用 IoT 卡网络。 AuditActionIotCardManualStopped = "iot_card.manual_stop" // AuditActionIotCardManualStarted 表示人工恢复 IoT 卡网络。 @@ -155,6 +219,8 @@ const ( AuditActionDeviceCardUnbound = "device.unbind_card" // AuditActionDeviceCurrentCardSwitched 表示切换设备当前使用的 IoT 卡。 AuditActionDeviceCurrentCardSwitched = "device.switch_current_card" + // AuditActionDeviceWorkerObservationSynced 表示 Worker 根据 Gateway 观测同步设备及当前卡槽事实。 + AuditActionDeviceWorkerObservationSynced = "device.worker_observation_sync" // AuditActionCardExchangeCreated 表示创建卡换货单。 AuditActionCardExchangeCreated = "exchange.card.create" // AuditActionCardExchangeShippingInfoSubmitted 表示个人客户提交卡换货收货信息。 @@ -369,6 +435,8 @@ const ( AuditActionNotificationCleanup = "notification.cleanup" // AuditActionNotificationCleanupItem 表示系统清理单条过期通知。 AuditActionNotificationCleanupItem = "notification.cleanup_item" + // AuditActionLogRetentionCleanup 表示留存任务物理清理已归档在线日志。 + AuditActionLogRetentionCleanup = "audit.retention_cleanup" // AuditActionPollingConfigCreated 表示创建轮询配置。 AuditActionPollingConfigCreated = "polling_config.create" // AuditActionPollingConfigUpdated 表示更新轮询配置。 @@ -466,6 +534,8 @@ const ( AuditResourceOutboxEvent = "outbox_event" // AuditResourceIntegrationLog 表示外部集成日志资源。 AuditResourceIntegrationLog = "integration_log" + // AuditResourceLogArchiveMonth 表示已归档日志自然月。 + AuditResourceLogArchiveMonth = "log_archive_month" // AuditResourceDeviceBatchTask 表示设备批量分配任务资源。 AuditResourceDeviceBatchTask = "device_batch_task" // AuditResourceIotCardImportTask 表示 IoT 卡导入任务资源。 @@ -598,10 +668,14 @@ const ( AuditResourceRoleApprovalIntegration = "approval_integration" // AuditResourceRoleCallbackIntegration 表示外部回调对应的集成交互事实。 AuditResourceRoleCallbackIntegration = "callback_integration" + // AuditResourceRoleWorkerIntegration 表示 Worker 外部观测对应的集成交互事实。 + AuditResourceRoleWorkerIntegration = "worker_integration" // AuditResourceRoleApprovalOutbox 表示审批链路对应的可靠 Outbox 事实。 AuditResourceRoleApprovalOutbox = "approval_outbox" // AuditResourceRoleBatchTask 表示批量根事件的任务资源。 AuditResourceRoleBatchTask = "batch_task" + // AuditResourceRoleRetentionMonth 表示留存清理目标自然月。 + AuditResourceRoleRetentionMonth = "retention_month" // AuditResourceRoleBatchItem 表示批量子事件的单项资源。 AuditResourceRoleBatchItem = "batch_item" // AuditResourceRoleNotificationTarget 表示本次写操作的通知资源。 @@ -897,6 +971,8 @@ const ( AuditActorIDRefundCommissionPostProcessing = "refund_commission_post_processing" // AuditActorIDCommissionCalculationWorker 表示订单佣金计算任务。 AuditActorIDCommissionCalculationWorker = "commission_calculation_worker" + // AuditActorIDRetentionWorker 表示日志留存清理任务。 + AuditActorIDRetentionWorker = "retention_worker" // AuditSourceAdminAPI 表示后台管理 API 入口。 AuditSourceAdminAPI = "admin_api" // AuditSourcePersonalAPI 表示个人客户 API 入口。 diff --git a/pkg/constants/audit_archive.go b/pkg/constants/audit_archive.go new file mode 100644 index 0000000..8750ba1 --- /dev/null +++ b/pkg/constants/audit_archive.go @@ -0,0 +1,37 @@ +package constants + +const ( + // TaskTypeAuditDailyArchive 表示统一审计每日冷归档任务。 + TaskTypeAuditDailyArchive = "audit:daily:archive" + // TaskTypeIntegrationDailyArchive 表示 Integration Log 每日冷归档任务。 + TaskTypeIntegrationDailyArchive = "integration:daily:archive" + // TaskTypeIntegrationMonthlyFinalize 表示 Integration Log 月度最终版本复核任务。 + TaskTypeIntegrationMonthlyFinalize = "integration:monthly:finalize" + // TaskTypeAuditMonthlyRetention 表示审计日志月度物理清理任务。 + TaskTypeAuditMonthlyRetention = "audit:monthly:retention" + + // AuditArchiveSource 表示 Audit Event 与 Event Resource 归档数据源。 + AuditArchiveSource = "audit" + // AuditArchiveSchemaVersion 表示审计归档 JSONL 结构版本。 + AuditArchiveSchemaVersion = "v1" + // AuditArchiveTimezone 表示审计归档自然日时区。 + AuditArchiveTimezone = "Asia/Shanghai" + // AuditArchiveInstanceID 表示当前单库审计归档实例。 + AuditArchiveInstanceID = "primary" + // IntegrationArchiveSource 表示 Integration Log 归档数据源。 + IntegrationArchiveSource = "integration" + // IntegrationArchiveSchemaVersion 表示 Integration Log 归档 JSONL 结构版本。 + IntegrationArchiveSchemaVersion = "v1" + + // ArchiveStatusPending 表示归档任务等待执行。 + ArchiveStatusPending = "pending" + // ArchiveStatusRunning 表示归档任务正在执行。 + ArchiveStatusRunning = "running" + // ArchiveStatusSuccess 表示归档对象与 metadata 已复核成功。 + ArchiveStatusSuccess = "success" + // ArchiveStatusFailed 表示归档任务执行失败并等待重试。 + ArchiveStatusFailed = "failed" + + // AuditRetentionDeleteBatchSize 表示月度物理清理单批删除上限。 + AuditRetentionDeleteBatchSize = 1000 +) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 818d613..48bd712 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -298,6 +298,8 @@ func QueueForTaskType(taskType string) string { return QueueDataCleanup case TaskTypeDailyTrafficFlush: return QueueDailyTrafficFlush + case TaskTypeAuditDailyArchive, TaskTypeIntegrationDailyArchive, TaskTypeIntegrationMonthlyFinalize, TaskTypeAuditMonthlyRetention: + return QueueDataCleanup case TaskTypeOutboxDeliver: return QueueOutboxDeliver case TaskTypeCardObservationSeries: diff --git a/pkg/constants/wallet.go b/pkg/constants/wallet.go index f28a505..92ff090 100644 --- a/pkg/constants/wallet.go +++ b/pkg/constants/wallet.go @@ -152,6 +152,7 @@ const ( ReferenceTypeCommission = "commission" // 分佣 ReferenceTypeWithdrawal = "withdrawal" // 提现 ReferenceTypeTopup = "topup" // 充值 + ReferenceTypeRecharge = "recharge" // 个人资产充值支付 ReferenceTypeRefund = "refund" // 退款 ReferenceTypeExchange = "exchange" // 换货 ReferenceTypeManualAdjustment = "manual_adjustment" // 人工余额调整 diff --git a/pkg/errors/codes.go b/pkg/errors/codes.go index 9b4df36..2dcdeb7 100644 --- a/pkg/errors/codes.go +++ b/pkg/errors/codes.go @@ -173,6 +173,9 @@ const ( CodeWeComApplicationNotFound = 1210 // 企业微信应用配置不存在 CodeWeComCredentialInvalid = 1211 // 企业微信加密凭据不可用 + // 审计留存相关错误 (1220-1229) + CodeAuditDataArchived = 1220 // 查询范围已归档，当前不支持在线查询 + // 服务端错误 (2000-2999) -> 5xx HTTP 状态码 CodeInternalError = 2001 // 内部服务器错误 CodeDatabaseError = 2002 // 数据库错误 @@ -316,6 +319,7 @@ var allErrorCodes = []int{ CodePaymentMethodUnavailable, CodeWeComApplicationNotFound, CodeWeComCredentialInvalid, + CodeAuditDataArchived, CodeInternalError, CodeDatabaseError, CodeRedisError, @@ -453,6 +457,7 @@ var errorMessages = map[int]string{ CodePaymentMethodUnavailable: "当前资产不支持所选支付方式", CodeWeComApplicationNotFound: "企业微信应用配置不存在或已禁用", CodeWeComCredentialInvalid: "企业微信凭据配置无效", + CodeAuditDataArchived: "数据已归档，第一阶段不支持在线查询", CodeInvalidCredentials: "用户名或密码错误", CodeAccountLocked: "账号已锁定", CodePasswordExpired: "密码已过期", @@ -499,6 +504,8 @@ func GetHTTPStatus(code int) int { return 403 // Forbidden case CodeNotFound: return 404 // Not Found + case CodeAuditDataArchived: + return 410 // Gone case CodeConflict, CodeUsernameExists, CodePhoneExists, diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 32b36bf..0dcc3ce 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -28,6 +28,7 @@ type AppError struct { Code int // 应用错误码 Message string // 错误消息 Err error // 底层错误（可选） + Data any // 可安全返回的结构化错误上下文（可选） } func (e *AppError) Error() string { @@ -59,6 +60,13 @@ func New(code int, customMsg ...string) *AppError { } } +// NewWithData 创建携带安全结构化上下文的 AppError。 +func NewWithData(code int, data any, customMsg ...string) *AppError { + err := New(code, customMsg...) + err.Data = data + return err +} + // Wrap 用错误码和消息包装现有错误 // 优先使用 errorMessages 映射表中的消息，允许通过可选参数覆盖 // 用法： diff --git a/pkg/errors/handler.go b/pkg/errors/handler.go index ce7b172..4108248 100644 --- a/pkg/errors/handler.go +++ b/pkg/errors/handler.go @@ -51,6 +51,7 @@ func handleError(c *fiber.Ctx, err error, logger *zap.Logger) error { var code int var message string var httpStatus int + var data any var appErr *AppError var fiberErr *fiber.Error @@ -60,6 +61,7 @@ func handleError(c *fiber.Ctx, err error, logger *zap.Logger) error { code = appErr.Code message = appErr.Message httpStatus = GetHTTPStatus(appErr.Code) + data = appErr.Data // 记录错误日志（包含完整上下文） logFields := append(errCtx.ToLogFields(), @@ -116,7 +118,7 @@ func handleError(c *fiber.Ctx, err error, logger *zap.Logger) error { // 6. 返回统一 JSON 响应 errResp := c.Status(httpStatus).JSON(fiber.Map{ "code": code, - "data": nil, + "data": data, "msg": message, "timestamp": time.Now().Format(time.RFC3339), }) diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index 6763eea..eb1b2c2 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -59,11 +59,17 @@ func NewS3Provider(cfg *config.StorageConfig) (*S3Provider, error) { } func (p *S3Provider) Upload(ctx context.Context, key string, reader io.Reader, contentType string) error { + return p.UploadWithMetadata(ctx, key, reader, contentType, nil) +} + +// UploadWithMetadata 上传对象并保存用于完整性复核的 metadata。 +func (p *S3Provider) UploadWithMetadata(ctx context.Context, key string, reader io.Reader, contentType string, metadata map[string]string) error { input := &s3manager.UploadInput{ Bucket: aws.String(p.bucket), Key: aws.String(key), Body: reader, ContentType: aws.String(contentType), + Metadata: aws.StringMap(metadata), } _, err := p.uploader.UploadWithContext(ctx, input) @@ -73,6 +79,26 @@ func (p *S3Provider) Upload(ctx context.Context, key string, reader io.Reader, c return nil } +// Stat 读取对象大小、内容类型和 metadata。 +func (p *S3Provider) Stat(ctx context.Context, key string) (*ObjectMetadata, error) { + result, err := p.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(p.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("读取对象 metadata 失败: %w", err) + } + metadata := make(map[string]string, len(result.Metadata)) + for name, value := range result.Metadata { + metadata[strings.ToLower(name)] = aws.StringValue(value) + } + return &ObjectMetadata{ + Size: aws.Int64Value(result.ContentLength), + ContentType: aws.StringValue(result.ContentType), + Metadata: metadata, + }, nil +} + func (p *S3Provider) Download(ctx context.Context, key string) (io.ReadCloser, error) { input := &s3.GetObjectInput{ Bucket: aws.String(p.bucket), diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 1128173..6a9781b 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -8,6 +8,8 @@ import ( type Provider interface { Upload(ctx context.Context, key string, reader io.Reader, contentType string) error + UploadWithMetadata(ctx context.Context, key string, reader io.Reader, contentType string, metadata map[string]string) error + Stat(ctx context.Context, key string) (*ObjectMetadata, error) Download(ctx context.Context, key string) (io.ReadCloser, error) DownloadToTemp(ctx context.Context, key string) (localPath string, cleanup func(), err error) Delete(ctx context.Context, key string) error @@ -15,3 +17,10 @@ type Provider interface { GetUploadURL(ctx context.Context, key string, contentType string, expires time.Duration) (string, error) GetDownloadURL(ctx context.Context, key string, expires time.Duration) (string, error) } + +// ObjectMetadata 是对象存储返回的受控对象属性。 +type ObjectMetadata struct { + Size int64 + ContentType string + Metadata map[string]string +} diff --git a/scripts/batch_package_purchase/__pycache__/batch_purchase.cpython-313.pyc b/scripts/batch_package_purchase/__pycache__/batch_purchase.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cf42f1ec3dec36fab6caa5c41176277dc19bd8a GIT binary patch literal 24764 zcmb_^3v^T0neNfcdfJjDzh4M!0|sm$ya|Q?HV_Dp`k;iyNmXGB{{DZTtLf<`0ndoO?Y=b|1>ukMLwR)Eqq@3IEeMANQIG^tC91cm zHmfDIll+=38cD<6T1m^^I!VXgX;K<{>m@yV8zckX+AYS-CdtI|>b96Snzr){xp zwn!G{*KbMRoFQc}zhR4YvrV!wzi~_EX1iqH?2sJnw`oh(=4>f@bB>gQ-)1RSv`Bdi z1u^}8m6VTn2Hpktm}Ur~RkW>8Uuio?OJ`-BYqe!>f>{vCw>k7AQF+r&u8t76|ugrRm}vv55I*h;zka<|`HFi6zWe zCRU2mn6F%{5}nL9Q=BhOXTDkD0&xcO%@!AmrOa0$-XfMU-<%TR`hVggv7E)uy*@4x zA}$tZvQ+cLCE_gRtGuU5oV{|FAb9cw;U0b6_hJK>^qL{uQ#m6(vP)Dv$|*Eynlu$l z1L+7DJ^0D^GhJgJeK7LLK=hp><0revo_uZe*!zDv|JqppxoFp`(H9O!JC8Wmi+BF% z{PW`--D5v_a^&o(v4JNpztR`&elglLFm~oGBpQA3ooLtFv4Xmv9~tc6#r?74ag%BM zT+i5D(Czdtf~aP-wrMm~FE>?f~MA>%Ya z6{?8YYWA(B zl{@|~aL50Zoc|4HbsEAVy5@$4=EjJrxzX!uZ{E}Fl_I7Gnx*!Dr^W4S{o05$V@F>b zeeQU)>*KF2^ova&hGne(>94H_Ubs}ksj66D!7M*djv@Z z3ZxcQlBQe`)u1<;dvv097Ggymd}++5m(rRH6~>5Zt>5o$zungsXdf>}lE8d8BcHz! z{h)_dmUCqAgUhcyncTK$*R#MCM)+s#ic9IXC+-N_98YhTW-2LCPttNk|Lv26=`yhCGGe++73PSE7H^%` zA82VGuS8LSIdJ%NVPB8Ff5J%#oMZhTkAC<#CNBE&3(>Q$jC_7#^s#=-lvIG^ku(~> zfd8B9#^#3hdd${5C$`4D&L29rwfejf?e5mbhomA(X^=eo-Rvig?>4CfE*omzO=_A{ zp*YkYe>NN%lEs%S)+e@vP18c!Y2l2_r?>JPlsNUk+=-a~&cI-ZOl*kE@LA%%bmq&z zuvsf?5$lfF9BvP^H?;2ajxR-R0dn}*fl7`HzQ-QfpVPh!6YTn!4-p1B0hTg^_ zF!oBcwxJaila0E5AM#=$MhuLKVo$L#Pd)q#&W!aP9}WL)my87=Z9zCa^NFqD z%)*eiFl^!hv``cCK^Z5_M~oAxuF^`MJRh8vNV+C%g)U-`k;@I9<`!=wBckK$kR`AP z&e(~*%O}tBMHn3b(|T|0#gC((9f@}JkDYpZ^v!OJF<+>BebUNg3#=kdnuh$6lbq?~ z%pj+f9NKYH*WRrZMh;c&CpqkCVOU$u*BXUS>RU{nQC;1Om=t|8iyFqiv5rJ7^QDRD zqK^6WVuqN;de=F( z%&Sk1^*G@=)O{DkJa}WcfXWkm^{P^;ZXcISRkY9Pok3ZGbRqz}ZT&lX*3@Qoe zEvgg=!zHxD<99h*Vk065O+tlwTSPU#LK8^?9R^j57kUB*j(d;RYXjIDGj?;)wS5K419+$43NCSJ6 zcEsyIZ(O>R<(VdxmzGpsT0JqP)a9XNuCzKL4O1TFOrq4>h;@=Q6K170jLJ>(mkAY? zz(N$xh0~bW{L#A*Y%U)?7ya~b^pl^Cef~=H^*5bs8A*5UY`x#>!!C?yN#u=a_Ir0@ zUl;o-HNxqF(JgjavFNOA6tQ5}OA)=i>8yz_0q{LOpX)COebhpF>Tlanb zR$s()$M%i4Z*%WhyK_?{UGn)`$GDd?7XggAM$#Lb{cSCthujZ% zS_0mPEuJyPsU(s;BO1uuR6w&20*0>vk_2%{&eL>^s8r}BY#Bk?Le3qOCk;hGn30xI zNF6!b;rM9`Kv(GDv_;3n?MT)X!ootgQjV|TbO@I+vxhQEgPEmai+$LV6V9K0VM)+( zd)S^cVHE6T6N1ie>P)+45p2ahcOI`lS|74jbZUpK`8_j_S01hG%Nnv)1+7)X_Oh^j zX72;19y#$yZ|lIW^ZUN639bH)TwOOIs6O0qwM#P}hy~G27xw!KE%t?u%a(&@lZB&b=j+jP zXUAXqF#7ycu|p|4V-g1^;vPxc?`diAwx8Cp6F|hQ$kfe^{zyif=b?RIfbM->9041p zZOBlemUdD;)AqWW+B&u_E3|x(&_dv)n377NjwY6cr2$fYyVuPGrAXnF)sY;z8~OZ1 zHn2#n7~JVW>vXO6`MSup4Pbohyx>=UiHaae1&=MWag#R<1@YJId5k+t`DxOPhJh z<5Cl;cA+$^E;EE2eTh&{f)aanwbc=vFy#>~E7WSiXK)$n;-W|*iNa3n;^Igm4uVv= zn6X0bGuAtlP?SQvY|4|w)Z^lV%P3k`X#8psu&5{~0)ta16Q&FE1Z5N&ArkFTJB4Y2 z6aSioYRxoZk7~M*5H<9I2V%3&-P8?nwujgzqidJdD>eg{{dZ~eNW}`12#`vTT62TL&WCOCOI%826I$0P@^(N zIxFL##=@!vC^(J8nYQ)=CIU+;*EWGHAF!-%^|ixOx$~hm?*aWfuSfDq&Pe*&0LZn} ze1KI_p^a#8C}6xgqTlQF5Y87#<0QR8#f}9Lz2t?s&_F<3nwOw9KPVX!Curp|X(2$% zFm^;#grN$1M9Vh1t}(E$4W$C=qEj)gZGPyWmCdWrg77mLi=`l3fmIO6#0Wv6p<>+p zXfBal&jk#U>LZq&{5ZogK>c=+d6=M6Vdkvo?>7@2)3+FZP;uNnoEWqxkpyMw6dpZ zp!ULjLC1H79J_*!UGjIoA98p)YcA!?4i~JxqSH7`o%#ubU~wQzc;@oXjosUVmeQ}x zws6IQf$RajJd47%2Q6hI_WZCd>qz>c^j>YyRu<084rdjH9r;&`I#!6qX56S6&dlq{ z?Vopg-fuFO4K5ECmL9J@TK)Q(p~8j1!i58~E>y^c3+2K)x(&m*(8)taAY-&&KS`Grjd|1Wa#$gQ&0RBL`wt%Dyz%6b&X7Tz{s6zm6Wf4&rTd%(Qf zNtA65NWO@cP&B04LJ3$Zl61N6Ma5TP;H!$o6V>JMP2?leB(apVtWgqMi7X{siOiRX zHzdW=ay^id0H+AoOR6Z5$t1Bg&WcJD&sO_9s-7x3U`pUqN$s)R7^4A+79=-0d&%*U zL#&x?fRqT%YKkR?vik3a^R)0g?QPezYTdMJ28XWr+8ndabge+GTXoH#)~!`d7%Frd zRhMjz2_1PR(gdr0LeIPg!9Huk$h;=ORxn{^UW;JL`b#=_nGnD#Yd}5mCoTyzGD+au z_&;&~6)GP=5vlmUrk=zn_N3z7P_6-(Q1~Fp2P-tr8CXRqOm>>RWLrS zytfes3kgdO4e2+cs_Aw$Z(|W-B0pb!)|1 zHxyzKAwb)ZA4r_*klKk*($anQT=dW``VQddCAwnpzB503p2?Otz7+W}o=WrP2Loq=0WFHW1#YH}!^X`o`b`^Wv! z&z?jp#@~IF4OTzYlfapyCqF`YV8lNNG)vw_=`e~oV5y9YJyj{?g| z^AJajrp$v&tgYL&c57{b_y?f#(XQ^1&tB)a6KP(dLmfH$;@C6qMW5~CFwo5_RLY|| zN_c2s}Q~%hRGb!g)$xM^;*yquwUKsoQb?A%v z!j8W9bhPWo2Mm>!0Fa?%3=k&46)J3Y$71vPI#)!GefDznnX}Q(0rEWc4xjsI_w$!u zKLarCfD!Y-g*Rf9cvjyQn1vS>4Rl{znjHJ-Q#4AaUczqr)A>$bu5;0XKb=2_89sv4 z(ktY!m4Y#WCdMlP*J^t0C_TSk6R1M;Xnz+<sIrQ-Cm)9VGQiKnfch-H2sj5p^qMXdnSz-veBz zcmeH+;6zOlGE$M!JdOR5{180_jPu^ur6d<7EW)T&L*`x0(lb1Bh zlEX2~`CShz0{Zh3#nV5wZw&cXxn@UX!VwibrdcTB3oVjnUW1R`W(A;%PB|3&(h@j6!6l+`h1wx(`U6yE z5YO5kbQv7Nokl-u_ft3|KBLPtDR&J< zHx7`v;u;Onuo9;?PbtdQ)WsnZrN4l8OfEfiJ?2z-5pG!&@4Hjj`T$BXiRmO+`pgNK zs1bs?n1Qi2r_!z{GgzOa)JnQf5B2y>LD=C!MTFPfh8c~h1e#GpRAn%@RwA3Qzq)t(Z-GzSUF49z{W1KBXdnV6inX+e8 zN;{d3DY3iql^U_HRApF=PW<_ddxRSUTiYJyz*a>;fR2PIWIWjZL_2>332M9}Cf$Im zf*(YW^~HG6!52oKdv~<+z42#Wh@O0bOEe@j=AyvSgP(z=fO{qhPS7z0XmZGyhcjCw z8SUxktQD~vA}Z+{BSVua{mo6%%Se10A{E14nte?Xz0bR!40FbbK9MpD+8x+25ljn9 zEoXoZsZNr~K7VTb1Ps&2aEy%1I*(A>&%8N)^r=`+(XappT!2;4d*g4sdHM9&XvcX8 z%T(AIV7LiN85w+$mCDlLH}5Cfi9DoX8W4Tzrz4*pbp~j2vGIB3=vaS$^y432KK9tJ8*65E1fAMPZ(i`&<)zrDse+WBhS zvaBKlNZW(W{&s)FbSEkJV#pX&fqP)x5z&%4TtovAoiGjp{31qZGxvD`MS7)s(cp-t z1u4>iMS6XW5$NjnlPt2Sb}i|2NSGCw(G<|V)@ENtdc=UTU~bjE7kSt?M)H{bmz&v; z?S9Ceu$q%PsC);H_5oSh*Vg`!+wVnvT56s23F7(6pyNYC$5oVR?B(bBBYm3$RjhZv zpDii{nxi*YE}D~ORQeL7{2G7$KVpGFYhreEtvR@cbL-)ZJh||ef#m~p2nYFctX}`VOtKz z`Fii_y};EsQ^Y`rp9C|(AP(6svBZV?O_ zP|zN!I8+fVtQkxj^j}baVh$D7gzPn)`eAcM=L65(+G7uy3&R#iPw`2A-_o}q9;ge> zS|-n2E-&9A=iU*tV1(Gv*IsD4@Lf57dnjXvY}oOYIg>*3YeE^dvY|F?&g|IwAAi18 z$l0j+iyn3V)>!?GzZ{eOg>%lvT+Po+c^mD9pUuo7cjda-2>O*>M{cfZW4V43#i4Bs zs-wF4GkE@H0S!y4Nd;{UkrwvUQ8`l{h(m(HTzsk|vlLYV>roh~!kmcN6B`oo2faz6 z{5Tqws05k`w4AgpO$oh&I#7ue(SbKWLPno`mK$b)u7FQ)pkie3taG!teVa4d|H;^g zC%|(E3Z>mIJqCw%y$4EOE2XLN5-QJe^?FKi4;&D1I$24-gyW|M#}frLlZ``JbAwrPJD14XB?DP- z{{5Oxu-Jyo(}Lz{K#Fx)ze9cv-rx91eCx?ss~5h|XRR&Kd{L}|n|fYv!CcY2DkDrf zswNVl@t*pKae)W(MZAbTE>(SecCVW+@Vg{EU)(^eEJ=Vz{`l-&mp?Y&lWH?4HD8w+ zq!EG}@z^PkB0Bn@rLH6LIOS2|l~M3aZg;%jWr9mxhVg-Zdk%`=4F57iT4FY+%#e6w z@U`f(C&n3X1F%4gh-qeE7($?v#1;z}c?<++@HAsE`euJ)b5k>H&?H)?EK|h5j7r+P zjg#!qq&q1AIT}orpEN5S!l>DbRTMT`yOtkZ-n{|Dq@!k7pV?h@Ogp464(f}Ea4vrC zA=!|}gEd3?qM*JAnL5prLr5cYXywbRIt^jx>>=lZpmV{H^Om6VmgoK5%gH3;wm}0N z)Wwf=@x6$K&_Rh&jhZQ|%Yz+qgSBUQ(CX`E7m{Mq ziu#-KV&+vhD?8=k5Jy9$%_-yciFltT>8OAc3JwHX(V&F}Su_GmH3c?dOs`*Wqd)D0 z%`;(AZ@d|!Tg+N2+IbWIWR#;dBnGF`|Lll|&d7lY=2=rbkpx=>r5b zU&BBKNCf354q3G_%P_v0xs5f<8rV9HFI?$SWI!lew2b^w;lLQZ54JN=FfN5i-$ev- z?6hp3!Y?day_s0!>_fAAmh_gLntft+--?0Vp&2Ve_LZG_t|C0SsCWO{tAUzmX~u;v7Uc^an>q&-`TUQ`~(yJB|xD-JPRH-idyG1RZ6^IV1ru29S+q$`+?IgmR#9 zXZ)1LG6vrf1C_{`PW{S;2mQ*-IaJ(vi>$3WxxP>L_GWmlpI2fwY+kjzO`HoV37;%n3{oIC2$|m zR^i}NVdPR1@q&m6XG)K+k(2>sSKx1LdB7`S>)=S`!+iqcaC->yju`hhx9?>)X<&ri z?%fBYQ`{~9qR!7zCJwI?sv9v<1{~61LP3iY6owyj9I^7lNL%7JXk3D z%prYlP@g-b&kySJhpmq8%{WR7S?2_;b3)d6L)OJX>*A1gsccv}Y|A@hdCBr}#*l4R z&^D{j8nP|#xIJvn=Au{c%2R7jtdVCd8d!ay;zHq*PyM zs6)Wir;d6v@kCkd^Dt(FfgJi8Z;Fu(1Cf77c9)m=H@uSD9N?2=&T9;-OEpFYSc)XLg zcQ#L(?Cx5u@MSFjw7@#Vjhyd^KGn%pKbKz|0FuC!QH1Vvzf1g-UF_p(qcLVXovfY0 z&Imk=Vz?e8cFmC8_QQ=rrd>lpfFokGo@=`j>aqC=tyNslHQw>s*yqnUQ|RQJmp?do z`NjU|TTev0dT?)I{HYf%e|jbW>=Pow$6Tqz8#-CP^%_$n@s98=@OaYAFj$Lg;i60( z#_%+PTmZ^l*L5dD;+%BhrR&v6dcmryD%ck>3l4T_+JZ(&KOu*}KwO)9$WPc>V>6JH zWfov#L&!W=*3RWLB6-JSrE?nb zkBmkn*lS{MCJPlH=&DqEJ+VhJgv9v__+ZlBNGe;KbV`e}@nnd2Qk^W zKs~8^y|QDmBXJ=_G!oRk3Hy%Z-c%MOE;UTGVHHvZ)k{)27NiE+IrYJ=Pfu7e)Dcgb z@&Ikn;&PL91+_fR{D70zaGW$%|3xDM*&z%kwZN7-WejZ3yutX|6a8;;RAlk{V@wOo z?;>-#8YF^{?XE-P9gmHiJ2&#_;pB^sL}Ez5<~gEgKu)>=gF-^TNe*_$mW%SyV*}AQ zI*>zJi&D5{BEOA$cmPBM5e%n|TMAGB(azCV`bW<7;G!}vNh6G#z9$(r;u0K;CJ?i2 zoEV(6o16x!f;P6($$)NRF$A9@*#<~+4^p&|MJteSI{g}b<3s`sCJFNLi8JixHd&2~ z9_b~pxcfz-$#1*ORsLP!;0gf@nkL{whdL(VyJm^m69bQrKB zJq}-mPWplZ){{e$LPW=aTV@|Qh3rVbqD17>60!LTo(_RFylKTn?aE=Tv2%ORrsLa> zZtq(kDp~+b!jI+*6v?L57qx3x;`J#49*c`jHmMm(J`432!>gr;+hw*KJ)JOYW z_EaY3E{*a%1Z;3xH|%j~Z+d%8mjn~0Oo{v{$b-V{VWOTgS6mdqa!8%x=2AFrkEwmh zx8L;jap_yBJ6;Etq)We@bjyZ>@Pb-NQ8ML$zL*v;9+;))avyRVQnn^e*cDO(qij-% zu?MhMf#OfR5gIE$5r*=O$!!(2=$$Tg`;yue*Y3KKZ@r~%X`DqUEp?faN=lt4&Onqr zqM^#@!j*#50B#Y%rh7+`SVtAezcoJ4X0dUzsIscxvSaids9T^U5lFd>He z;J6?p0ieJFIWoapzz*~Thw!G|NI2sxa+1*b|*lnj_c)46mxY%RG|=p-Zca7pPZ_TRM`sT#4^JyU{>afshV#mM8$)?>am#qPsG@I9f7R)#!K~1n)uEy_;i9r(XJy}g z{l3$_!S98t?g%-zUU8T*GbVC`Y^dx?j;9|@kJ-s*7hjp4Nq53Bji!IURwh_Vt_ltv znwZ_)@bYZ5?fQm<3uc6;O^4x2%)e&eTu z%j31iT8r>Yi)lly=9f83H`q0QZ&AVj_jVoJ1`;KdhiEI~JL|xWfZ%9p@~1v2g<{-l zgjus1h@HllhCh80=ZTBGxIUy!3D+ju`WU|rzi}DoL(u(>iY$BBg&vY`raZKZ=y86F zy$%HI_yHN>GZ{<6Drlo>(^hPGQ1G_%2~M)hzU${pVqDsnj*CPj!cBQx>2*0%g78zJ ztY}SoUEcNaajvisJOcENS(T*KlL*Bg(LxqAV!F!!H^XJAi_={t1+z!7qI8<@DG$VQ zL!t%Sn_c=?4f-3@V7p-rnf23^!co@@#h-X9^`M>hNomUxX%t^lS_ev-se~uqXt||c z862Sw`0&(6%%U}SbDJzS>M7a^OCM0iMo0&!Jw^NK=P9)j#q@y7CyiIm-_kCpuzj9O z2Tb}E)PsGsQMGkGETINh+T*`LXX}Z2v!{-T^pq#rw#`W;?1d4ojoF|{nEQAoI}~C$afVEEK*JQSW-(PU8eU-?@{=V;6zmQ{BTAn zz`ZbEAx=)`Fd`lh2e@>AYx~vAlIUKiVj=Tdz`#?G`k-3E=nK+!;6OX@3%r^SfuAv$ z3x_HC0LRBaMoyT@kcjcvvC$9SA`Q{G50rkt&JsL%x%+9}Y5e4$kU?*^1&DU=aiZ8a zsDZqtPKE~4A&$yr#yidcW`&*Wkd+-qlTl^l9LHP<75U6U7Xuj4?YqCRS>i{uj}ave z!ikt_aL4{GsTr0y5%Yd}^E-JF+)4`g_ey6eC2{YFfyLr0HA&LS7{xM)gb4&#E7wUd zQOb)?2Ept{irCh=c5ZUl*41sVbFbfCQ!7G~Md-B7-{SSQRb=o}VnQluqRi$byg6b} z?5!i|vF{Av>jMzHB9=YuQ!=q%_+kQtGB!oHQ_j1mYxXsIACySaiRe)#eaXj9h=wA% z@HZD=^f7yEiL_}n$^QR@1IfiM=xjsUoS-&mNLv)t7WM8NsDWutP`e_WmEEy5?8xf4 zW7uNtPJ3=kN9`qJ){rqjXw2^^?K2P7$>hK7LJb+}l)O|Dw%GoxHsxhp(WTR81ty9F zqjkto5Hu8AG!%smmM+sl)AN=QZpZUmX2(-IoKx0&=Rj>JXG7;kvWnS$aC^_j<6Dky zxoDXkw&rv_+VyDIR@|Ezw9V*FAGT%psCvqKa(h+1<-NIm=7HLw#WlgjHS(g`3-`+6 z-9utSP;8Lv8s!}CZ*6-n*@}8qLEH3h3vz~Ya(i^$`+GdSx}N>L_Yag0-Lfut%Q|`C zdVIv7MpZg+ld z%q|Lwj~5&*__I-$X~$*7^h|;gpTFahHHRaHy$$%PLf>8e_nf}xV)fQgc-+g(PY;s-H-idHP32~$#Oq~zeruXg(&0N(9bF1|iro;KmcU4r_Z;iKo zQ;eem&p=ffKQ8Zb);X1YiMPVuecD~?82bhHN}!}-T&%+= zYU@f9`4peh8>M|7jC4}|*mnbz-eFYBm7Ke5skBb<_UP(o#ZxArl(hN^B~&!k&sF>Y zq++uH5yYH&Ob|24kUgoSbl~uG)SdE(8HrL8A{0c%cx@1h678{06>buOk4wY8g^+Tt zTy#Al)%vu>AdvX+sXi{eEmi{#!ARl|l&C+GM6X5a`dgH4)X{N&$|KrIoCU1m@afm; zo3i*D-wIf&Bt*#&^d|Kz`)@0=DJTAofLw->$1H)n2mppuk^BKqsM8d68+}BS3oj5e z0<~xdS7^m_Xo>rPWD-0@rM511CD6&$XvttX?56c}6A%{x4;Yjyf$Zj#a=*~Y>lr!s z4!bglgscW7tuuPQ1DE%hEXWjU%7+S?8)G@6Cr{y`n{xq^s$$o^rbZD3Py`BCO}_wx zis(QW&qgzqEK>1k*q3uPATz2^^mouMba3nV08MLhHWK(g=z*?}q)dEQ@%4`YIssni zLQTx3o+ME^)R6!@@H8KLMPz&}c9}6iB#%tHNd#0b9agj6(ex$h5~{f-LHa+)z@%R3 zKj8z+^aTY>maZfd_NHWp!?Xt67zxdh{x8yQV*)hmk_q<)@d-IH2!HhhzE~x#1RopR5u%>{LVX)DDT{S|*%I5%T?> zoG-~CQH*N`{)nHEG(LpVM-;c3oPQ-}jGS}itRrU+Ik%ESlBkr1G9sBp*+IAuLy9ngfFi{)ucKDuw<9yw#(MZ&sxI7AtogFI9R6QxRdDG;hqPkcZrGN+h}HCs z9ZGq2kFshT9a}-lys-JZfGEBhSRz{sj=%M55`a zG22)L*c6=fu$@Xh;vlqv@3caSXSe5mN;+ z)hh8%(U3?P$%uu=z9qxIaYN!aCwMOM>1j+qBOegHVLZIKML003#Q)-=HClO?)if-{SQL%AB4g`2zh@Ha{ft} z_fNuH+#AlA-eC%-PwOy69ObfrL$uze?yy|R64d7IvY;yWFKVqSYeIl?RU@cjcLLv) zj6|$XWyh!dRQ6abd{=CVSOdkviN(TqEk~tV-IIS!!0RuDyH%=Pmo z+8wHSNcBCHLACT+jzwj=R-#wo0Gg#%t+-}VtL9%b>41=5vt_B~U8}OI(kJFDR+++< zf(Z@YFdCiE;f;=2awhcTGYFYk6GrlxgtY8yX7c$|D{<@&TZ$$$x7

QTuM)JD&I2J$epsu2X4B)F4(@#J Q&fjXYCNzS!fEm&JKW+qKBme*a literal 0 HcmV?d00001 diff --git a/scripts/batch_package_purchase/assets.example_待失效订单_20260803_111217.csv b/scripts/batch_package_purchase/assets.example_待失效订单_20260803_111217.csv new file mode 100644 index 0000000..e18061a --- /dev/null +++ b/scripts/batch_package_purchase/assets.example_待失效订单_20260803_111217.csv @@ -0,0 +1,91 @@ +order_no +ORD20260803110607615595 +ORD20260803110608277358 +ORD20260803110609298852 +ORD20260803110609993771 +ORD20260803110610862222 +ORD20260803110611648789 +ORD20260803110612391555 +ORD20260803110613360237 +ORD20260803110614033440 +ORD20260803110615619499 +ORD20260803110616321048 +ORD20260803110616305693 +ORD20260803110617261387 +ORD20260803110618333268 +ORD20260803110619282396 +ORD20260803110620338587 +ORD20260803110620648002 +ORD20260803110621100796 +ORD20260803110621377665 +ORD20260803110622985501 +ORD20260803110623744573 +ORD20260803110624555820 +ORD20260803110625257789 +ORD20260803110626061375 +ORD20260803110626383282 +ORD20260803110628343084 +ORD20260803110630203516 +ORD20260803110630817942 +ORD20260803110631500070 +ORD20260803110632923546 +ORD20260803110633703072 +ORD20260803110634423001 +ORD20260803110635743929 +ORD20260803110635838650 +ORD20260803110636164609 +ORD20260803110637042987 +ORD20260803110638462120 +ORD20260803110638820678 +ORD20260803110639694789 +ORD20260803110639666282 +ORD20260803110640131286 +ORD20260803110640764346 +ORD20260803110641546312 +ORD20260803110641801253 +ORD20260803110642118405 +ORD20260803110643965799 +ORD20260803110644712849 +ORD20260803110645045631 +ORD20260803110645785900 +ORD20260803110646330720 +ORD20260803110857032891 +ORD20260803110717724848 +ORD20260803110717967630 +ORD20260803110718644411 +ORD20260803110719642078 +ORD20260803110720131946 +ORD20260803110720239975 +ORD20260803110721523329 +ORD20260803110721864337 +ORD20260803110722867379 +ORD20260803110722886903 +ORD20260803110723862079 +ORD20260803110724441746 +ORD20260803110724799240 +ORD20260803110725852929 +ORD20260803110725604711 +ORD20260803110726473340 +ORD20260803110726008729 +ORD20260803110727813949 +ORD20260803110727095324 +ORD20260803110728413520 +ORD20260803110728903686 +ORD20260803110729965364 +ORD20260803110729067178 +ORD20260803110730751445 +ORD20260803110731653883 +ORD20260803110731038843 +ORD20260803110732876903 +ORD20260803110732344625 +ORD20260803110733592658 +ORD20260803110734238582 +ORD20260803110734200187 +ORD20260803110736908880 +ORD20260803110737672069 +ORD20260803110738546467 +ORD20260803110738284338 +ORD20260803110739256634 +ORD20260803110740462025 +ORD20260803110740132007 +ORD20260803110741839255 diff --git a/scripts/batch_package_purchase/assets.example_过期时间修改结果_20260803_112658.csv b/scripts/batch_package_purchase/assets.example_过期时间修改结果_20260803_112658.csv new file mode 100644 index 0000000..88c7e2a --- /dev/null +++ b/scripts/batch_package_purchase/assets.example_过期时间修改结果_20260803_112658.csv @@ -0,0 +1,91 @@ +line_no,identifier,package_usage_id,package_name,old_expires_at,new_expires_at,status,http_status,code,msg +2,8986032445201075309,34878,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +3,8986032445201075310,34879,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +4,8986032445201075311,34880,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +5,8986032445201075312,34881,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +6,8986032445201075313,34882,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +7,8986032445201075314,34883,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +8,8986032445201075315,34884,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +9,8986032445201075316,34885,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +10,8986032445201075317,34886,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +11,8986032445201075318,34887,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +12,8986032445201075319,34888,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +13,8986032445201075320,34889,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +14,8986032445201075321,34890,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +15,8986032445201075322,34891,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +16,8986032445201075323,34892,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +17,8986032445201075324,34893,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +18,8986032445201075325,34894,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +19,8986032445201075326,34895,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +20,8986032445201075327,34896,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +21,8986032445201075328,34897,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +22,8986032445201075329,34898,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +23,8986032445201075330,34899,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +24,8986032445201075331,34900,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +25,8986032445201075332,34901,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +26,8986032445201075333,34902,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +27,8986032445201075334,34903,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +28,8986032445201075335,34904,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +29,8986032445201075336,34905,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +30,8986032445201075337,34906,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +31,8986032445201075338,34907,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +32,8986032445201075339,34908,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +33,8986032445201075340,34909,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +34,8986032445201075341,34910,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +35,8986032445201075342,34911,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +36,8986032445201075343,34912,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +37,8986032445201075344,34913,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +38,8986032445201075345,34914,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +39,8986032445201075346,34915,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +40,8986032445201075347,34916,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +41,8986032445201075348,34917,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +42,8986032445201075349,34918,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +43,8986032445201075350,34919,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +44,8986032445201075351,34920,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +45,8986032445201075352,34921,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +46,8986032445201075353,34922,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +47,8986032445201075354,34923,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +48,8986032445201075355,34924,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +49,8986032445201075356,34925,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +50,8986032445201075357,34926,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +51,8986032445201075358,34927,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +52,8986032445201075359,34967,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +53,8986032445201075360,34928,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +54,8986032445201075361,34929,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +55,8986032445201075362,34930,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +56,8986032445201075363,34931,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,失败,,,请求失败： +57,8986032445201075364,34932,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +58,8986032445201075365,34933,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +59,8986032445201075366,34934,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +60,8986032445201075367,34935,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +61,8986032445201075368,34936,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +62,8986032445201075369,34937,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +63,8986032445201075370,34938,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +64,8986032445201075371,34939,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +65,8986032445201075372,34940,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +66,8986032445201075373,34941,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +67,8986032445201075374,34942,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +68,8986032445201075375,34943,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +69,8986032445201075376,34944,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +70,8986032445201075377,34945,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +71,8986032445201075378,34946,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +72,8986032445201075379,34947,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +73,8986032445201075380,34948,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +74,8986032445201075381,34949,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +75,8986032445201075382,34950,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +76,8986032445201075383,34951,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +77,8986032445201075384,34952,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +78,8986032445201075385,34953,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +79,8986032445201075386,34954,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +80,8986032445201075387,34955,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +81,8986032445201075388,34956,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +82,8986032445201075389,34957,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +83,8986032445201075390,34958,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +84,8986032445201075391,34959,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +85,8986032445201075392,34960,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +86,8986032445201075393,34961,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +87,8986032445201075394,34962,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +88,8986032445201075395,34963,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +89,8986032445201075396,34964,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +90,8986032445201075397,34965,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success +91,8986032445201075398,34966,星网专享年卡套餐每月1G（12个月）,2027-08-03T23:59:59+08:00,2027-04-08 23:59:59,成功,200,0,success diff --git a/scripts/legacy_device_export/devices.csv b/scripts/legacy_device_export/devices.csv index ee59bc9..18c1c86 100644 --- a/scripts/legacy_device_export/devices.csv +++ b/scripts/legacy_device_export/devices.csv @@ -1,352 +1,352 @@ -imei -862639075986135 -862639075972853 -862639076004532 -862639076024787 -862639076035379 -862639076007840 -862639075987521 -862639075981524 -862639076008442 -862639076022815 -862639076016668 -862639075990475 -862639076036872 -862639076037805 -862639076030503 -862639076004870 -862639076020678 -862639076023888 -862639075998429 -862639075989949 -862639076016726 -862639076022922 -862639076004284 -862639075982696 -862639076026915 -862639076031998 -862639075979866 -862639076010695 -862639075977365 -862639076028531 -862639075979502 -862639076015843 -862639075974404 -862639076016171 -862639076037201 -862639075984924 -862639075981144 -862639076009861 -862639075991663 -862639076017195 -862639075973158 -862639076002122 -862639076005844 -862639076015421 -862639076015462 -862639075980880 -862639076008434 -862639076016932 -862639075971343 -862639076032764 -862639075972119 -862639076017609 -862639076025933 -862639075972887 -862639075978769 -862639076030594 -862639076000373 -862639076018862 -862639075979213 -862639076039637 -862639076026485 -862639076000480 -862639076006040 -862639076034802 -862639076001504 -862639076021890 -862639075987182 -862639076032012 -862639076024902 -862639075994386 -862639075972770 -862639076026840 -862639075985533 -862639075991515 -862639075974115 -862639075979155 -862639075991853 -862639076030073 -862639076015876 -862639076037821 -862639076002403 -862639075974560 -862639075980245 -862639075973539 -862639076030537 -862639075993883 -862639076020249 -862639075987778 -862639075994683 -862639076032251 -862639075984882 -862639075993693 -862639076017427 -862639076033853 -862639075994998 -862639075987331 -862639075970535 -862639076039165 -862639076001363 -862639076022450 -862639076019886 -862639076039686 -862639076015082 -862639075980781 -862639075996506 -862639075993313 -862639076030479 -862639076029760 -862639076022591 -862639076011024 -862639075971079 -862639076023672 -862639076022716 -862639075989683 -862639075980211 -862639076002395 -862639076002221 -862639076009085 -862639076010448 -862639076003500 -862639076009036 -862639076015579 -862639075984262 -862639075977233 -862639075999187 -862639075978546 -862639075976300 -862639075999732 -862639076028481 -862639076029810 -862639076028697 -862639075980492 -862639076002064 -862639076000209 -862639075987422 -862639076013905 -862639076030099 -862639075995953 -862639076036781 -862639076037177 -862639076030784 -862639076017500 -862639076014754 -862639076004219 -862639075979247 -862639075993354 -862639075998460 -862639076024969 -862639075994923 -862639076021833 -862639076007337 -862639075971327 -862639076003203 -862639076008897 -862639075975641 -862639075993701 -862639075975625 -862639076016353 -862639076018425 -862639076012089 -862639075983595 -862639075998254 -862639075999369 -862639075970543 -862639076029968 -862639076015769 -862639075987448 -862639075987505 -862639075993164 -862639075993289 -862639076035692 -862639075982175 -862639075992166 -862639075996118 -862639076023557 -862639076033168 -862639075976730 -862639075996324 -862639075974677 -862639076019241 -862639076032731 -862639076023607 -862639075997702 -862639076007014 -862639076001231 -862639075972507 -862639075993230 -862639075976490 -862639075995862 -862639076002353 -862639076026741 -862639075975500 -862639076029281 -862639076030180 -862639075979510 -862639075975245 -862639075996928 -862639075979270 -862639076022658 -862639076023474 -862639076001629 -862639076000282 -862639075996159 -862639075980310 -862639075980070 -862639076003633 -862639075980047 -862639075975765 -862639076028655 -862639076017971 -862639076032327 -862639076002841 -862639076021478 -862639075973430 -862639075996589 -862639075976888 -862639076003815 -862639076016429 -862639076010315 -862639076010901 -862639076004391 -862639075988305 -862639076031949 -862639075970949 -862639076003898 -862639075978587 -862639075984007 -862639075970220 -862639075970816 -862639076014580 -862639076026725 -862639076015074 -862639076018961 -862639075991499 -862639075974834 -862639076022302 -862639075996423 -862639075983371 -862639075995441 -862639075977829 -862639075972267 -862639076020447 -862639075987398 -862639076015231 -862639075978314 -862639076027897 -862639076023045 -862639076033846 -862639076003104 -862639075986663 -862639076035890 -862639076010067 -862639076031071 -862639075990061 -862639075983462 -862639075982480 -862639075987497 -862639076022955 -862639076017997 -862639075975922 -862639076022484 -862639076019340 -862639075997066 -862639076030214 -862639075980344 -862639076037359 -862639075985558 -862639076028986 -862639076038134 -862639076023896 -862639076002635 -862639075976458 -862639076006529 -862639076022252 -862639075985954 -862639076019894 -862639075980864 -862639076017179 -862639076026279 -862639076018607 -862639075988354 -862639076030966 -862639076001736 -862639075972259 -862639076001611 -862639075973927 -862639075974909 -862639076003872 -862639076028465 -862639076012642 -862639076023078 -862639076019290 -862639076029596 -862639075973950 -862639075970808 -862639076028275 -862639076011669 -862639076027061 -862639075993321 -862639075975732 -862639076030982 -862639076031881 -862639076026675 -862639076002551 -862639076017716 -862639076039264 -862639075970832 -862639076011321 -862639075983132 -862639075998270 -862639075999302 -862639076006727 -862639075973307 -862639076014721 -862639076020991 -862639075985897 -862639075978462 -862639075993222 -862639076007832 -862639076011339 -862639075994378 -862639075977597 -862639075988271 -862639076026709 -862639075998072 -862639076035940 -862639076033895 -862639076012659 -862639075980534 -862639075986614 -862639076036435 -862639075970725 -862639075998296 -862639076018953 -862639075978843 -862639075999609 -862639076032640 -862639076003674 -862639076038704 -862639075990657 -862639076003930 -862639075992315 -862639075981029 -862639076020603 -862639075999385 -862639076024183 -862639076035932 -862639076017823 -862639075971855 -862639076021668 -862639076026600 +imei +862639075986135 +862639075972853 +862639076004532 +862639076024787 +862639076035379 +862639076007840 +862639075987521 +862639075981524 +862639076008442 +862639076022815 +862639076016668 +862639075990475 +862639076036872 +862639076037805 +862639076030503 +862639076004870 +862639076020678 +862639076023888 +862639075998429 +862639075989949 +862639076016726 +862639076022922 +862639076004284 +862639075982696 +862639076026915 +862639076031998 +862639075979866 +862639076010695 +862639075977365 +862639076028531 +862639075979502 +862639076015843 +862639075974404 +862639076016171 +862639076037201 +862639075984924 +862639075981144 +862639076009861 +862639075991663 +862639076017195 +862639075973158 +862639076002122 +862639076005844 +862639076015421 +862639076015462 +862639075980880 +862639076008434 +862639076016932 +862639075971343 +862639076032764 +862639075972119 +862639076017609 +862639076025933 +862639075972887 +862639075978769 +862639076030594 +862639076000373 +862639076018862 +862639075979213 +862639076039637 +862639076026485 +862639076000480 +862639076006040 +862639076034802 +862639076001504 +862639076021890 +862639075987182 +862639076032012 +862639076024902 +862639075994386 +862639075972770 +862639076026840 +862639075985533 +862639075991515 +862639075974115 +862639075979155 +862639075991853 +862639076030073 +862639076015876 +862639076037821 +862639076002403 +862639075974560 +862639075980245 +862639075973539 +862639076030537 +862639075993883 +862639076020249 +862639075987778 +862639075994683 +862639076032251 +862639075984882 +862639075993693 +862639076017427 +862639076033853 +862639075994998 +862639075987331 +862639075970535 +862639076039165 +862639076001363 +862639076022450 +862639076019886 +862639076039686 +862639076015082 +862639075980781 +862639075996506 +862639075993313 +862639076030479 +862639076029760 +862639076022591 +862639076011024 +862639075971079 +862639076023672 +862639076022716 +862639075989683 +862639075980211 +862639076002395 +862639076002221 +862639076009085 +862639076010448 +862639076003500 +862639076009036 +862639076015579 +862639075984262 +862639075977233 +862639075999187 +862639075978546 +862639075976300 +862639075999732 +862639076028481 +862639076029810 +862639076028697 +862639075980492 +862639076002064 +862639076000209 +862639075987422 +862639076013905 +862639076030099 +862639075995953 +862639076036781 +862639076037177 +862639076030784 +862639076017500 +862639076014754 +862639076004219 +862639075979247 +862639075993354 +862639075998460 +862639076024969 +862639075994923 +862639076021833 +862639076007337 +862639075971327 +862639076003203 +862639076008897 +862639075975641 +862639075993701 +862639075975625 +862639076016353 +862639076018425 +862639076012089 +862639075983595 +862639075998254 +862639075999369 +862639075970543 +862639076029968 +862639076015769 +862639075987448 +862639075987505 +862639075993164 +862639075993289 +862639076035692 +862639075982175 +862639075992166 +862639075996118 +862639076023557 +862639076033168 +862639075976730 +862639075996324 +862639075974677 +862639076019241 +862639076032731 +862639076023607 +862639075997702 +862639076007014 +862639076001231 +862639075972507 +862639075993230 +862639075976490 +862639075995862 +862639076002353 +862639076026741 +862639075975500 +862639076029281 +862639076030180 +862639075979510 +862639075975245 +862639075996928 +862639075979270 +862639076022658 +862639076023474 +862639076001629 +862639076000282 +862639075996159 +862639075980310 +862639075980070 +862639076003633 +862639075980047 +862639075975765 +862639076028655 +862639076017971 +862639076032327 +862639076002841 +862639076021478 +862639075973430 +862639075996589 +862639075976888 +862639076003815 +862639076016429 +862639076010315 +862639076010901 +862639076004391 +862639075988305 +862639076031949 +862639075970949 +862639076003898 +862639075978587 +862639075984007 +862639075970220 +862639075970816 +862639076014580 +862639076026725 +862639076015074 +862639076018961 +862639075991499 +862639075974834 +862639076022302 +862639075996423 +862639075983371 +862639075995441 +862639075977829 +862639075972267 +862639076020447 +862639075987398 +862639076015231 +862639075978314 +862639076027897 +862639076023045 +862639076033846 +862639076003104 +862639075986663 +862639076035890 +862639076010067 +862639076031071 +862639075990061 +862639075983462 +862639075982480 +862639075987497 +862639076022955 +862639076017997 +862639075975922 +862639076022484 +862639076019340 +862639075997066 +862639076030214 +862639075980344 +862639076037359 +862639075985558 +862639076028986 +862639076038134 +862639076023896 +862639076002635 +862639075976458 +862639076006529 +862639076022252 +862639075985954 +862639076019894 +862639075980864 +862639076017179 +862639076026279 +862639076018607 +862639075988354 +862639076030966 +862639076001736 +862639075972259 +862639076001611 +862639075973927 +862639075974909 +862639076003872 +862639076028465 +862639076012642 +862639076023078 +862639076019290 +862639076029596 +862639075973950 +862639075970808 +862639076028275 +862639076011669 +862639076027061 +862639075993321 +862639075975732 +862639076030982 +862639076031881 +862639076026675 +862639076002551 +862639076017716 +862639076039264 +862639075970832 +862639076011321 +862639075983132 +862639075998270 +862639075999302 +862639076006727 +862639075973307 +862639076014721 +862639076020991 +862639075985897 +862639075978462 +862639075993222 +862639076007832 +862639076011339 +862639075994378 +862639075977597 +862639075988271 +862639076026709 +862639075998072 +862639076035940 +862639076033895 +862639076012659 +862639075980534 +862639075986614 +862639076036435 +862639075970725 +862639075998296 +862639076018953 +862639075978843 +862639075999609 +862639076032640 +862639076003674 +862639076038704 +862639075990657 +862639076003930 +862639075992315 +862639075981029 +862639076020603 +862639075999385 +862639076024183 +862639076035932 +862639076017823 +862639075971855 +862639076021668 +862639076026600