From 17782d5f8ebe717638def759fa1d51dfa2f78da5 Mon Sep 17 00:00:00 2001 From: break Date: Thu, 23 Jul 2026 17:52:48 +0900 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E4=B8=83=E6=9C=88=E8=BF=AD?= =?UTF-8?q?=E4=BB=A3=E5=85=AC=E5=85=B1=E6=8A=80=E6=9C=AF=E5=9F=BA=E7=A1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/tech-public-foundation/PRD.md | 2 +- ...01-public-migration-ownership-and-gates.md | 15 +- .../02-transactional-public-outbox-write.md | 15 +- .../03-outbox-at-least-once-delivery.md | 15 +- .../04-outbox-observability-and-recovery.md | 15 +- .../issues/05-command-idempotency-contract.md | 15 +- .../issues/06-unified-async-task-contract.md | 15 +- .../07-controlled-system-config-read.md | 15 +- .../08-controlled-system-config-update.md | 15 +- .../09-access-log-recursive-redaction.md | 15 +- .../10-sensitive-route-safe-summaries.md | 15 +- ...ublic-migration-and-rollback-validation.md | 15 +- ...n-release-gate-and-integration-contract.md | 16 +- README.md | 4 + cmd/foundation-check/main.go | 57 +++ cmd/worker/main.go | 50 ++- docs/admin-openapi.yaml | 275 +++++++++++++++ .../创建命令幂等契约.md | 19 + docs/tech-public-foundation/功能总结.md | 69 ++++ .../数据库对象所有权与发布门禁.md | 22 ++ .../统一异步任务与前端轮询契约.md | 13 + .../迁移发布与数据安全回滚.md | 22 ++ internal/application/outbox/recovery.go | 154 +++++++++ .../outbox/recovery_integration_test.go | 129 +++++++ internal/application/systemconfig/update.go | 178 ++++++++++ .../systemconfig/update_integration_test.go | 141 ++++++++ internal/bootstrap/dependencies.go | 24 +- internal/bootstrap/handlers.go | 18 + internal/bootstrap/types.go | 1 + internal/handler/admin/system_config.go | 54 +++ internal/infrastructure/asynctask/store.go | 117 +++++++ .../asynctask/store_integration_test.go | 130 +++++++ .../infrastructure/messaging/outbox/relay.go | 326 ++++++++++++++++++ .../outbox/relay_integration_test.go | 253 ++++++++++++++ .../messaging/outbox/repository.go | 78 +++++ .../outbox/repository_integration_test.go | 105 ++++++ .../infrastructure/releasegate/checker.go | 324 +++++++++++++++++ .../releasegate/checker_test.go | 20 ++ .../releasegate/migration_integration_test.go | 222 ++++++++++++ .../infrastructure/systemconfig/reader.go | 149 ++++++++ .../systemconfig/reader_integration_test.go | 115 ++++++ .../infrastructure/systemconfig/registry.go | 183 ++++++++++ .../systemconfig/registry_test.go | 61 ++++ internal/model/dto/export_task_dto.go | 7 + internal/model/dto/system_config_dto.go | 46 +++ internal/model/outbox_event.go | 39 +++ internal/model/system_config.go | 24 ++ internal/query/outbox/metrics.go | 143 ++++++++ .../query/outbox/metrics_integration_test.go | 73 ++++ internal/query/systemconfig/list.go | 73 ++++ internal/routes/admin.go | 3 + internal/routes/system_config.go | 31 ++ .../routes/system_config_integration_test.go | 236 +++++++++++++ internal/service/export_task/service.go | 14 + internal/testutil/outbox.go | 27 ++ internal/testutil/system_config.go | 22 ++ .../000165_create_public_outbox.down.sql | 11 + migrations/000165_create_public_outbox.up.sql | 44 +++ .../000166_create_system_config.down.sql | 11 + migrations/000166_create_system_config.up.sql | 22 ++ pkg/asynctask/contract.go | 103 ++++++ pkg/asynctask/contract_test.go | 105 ++++++ pkg/asynctask/state.go | 53 +++ pkg/constants/constants.go | 36 +- pkg/constants/outbox.go | 45 +++ pkg/constants/system_config.go | 24 ++ pkg/idempotency/fingerprint.go | 124 +++++++ pkg/idempotency/fingerprint_test.go | 58 ++++ pkg/idempotency/postgres_integration_test.go | 86 +++++ pkg/logger/access_policy.go | 130 +++++++ pkg/logger/middleware.go | 112 ++++-- pkg/logger/middleware_test.go | 154 +++++++++ pkg/openapi/handlers.go | 1 + pkg/queue/client.go | 7 +- pkg/queue/client_test.go | 17 + scripts/test-tech-public-foundation.sh | 27 ++ 76 files changed, 5246 insertions(+), 158 deletions(-) create mode 100644 cmd/foundation-check/main.go create mode 100644 docs/tech-public-foundation/创建命令幂等契约.md create mode 100644 docs/tech-public-foundation/功能总结.md create mode 100644 docs/tech-public-foundation/数据库对象所有权与发布门禁.md create mode 100644 docs/tech-public-foundation/统一异步任务与前端轮询契约.md create mode 100644 docs/tech-public-foundation/迁移发布与数据安全回滚.md create mode 100644 internal/application/outbox/recovery.go create mode 100644 internal/application/outbox/recovery_integration_test.go create mode 100644 internal/application/systemconfig/update.go create mode 100644 internal/application/systemconfig/update_integration_test.go create mode 100644 internal/handler/admin/system_config.go create mode 100644 internal/infrastructure/asynctask/store.go create mode 100644 internal/infrastructure/asynctask/store_integration_test.go create mode 100644 internal/infrastructure/messaging/outbox/relay.go create mode 100644 internal/infrastructure/messaging/outbox/relay_integration_test.go create mode 100644 internal/infrastructure/messaging/outbox/repository.go create mode 100644 internal/infrastructure/messaging/outbox/repository_integration_test.go create mode 100644 internal/infrastructure/releasegate/checker.go create mode 100644 internal/infrastructure/releasegate/checker_test.go create mode 100644 internal/infrastructure/releasegate/migration_integration_test.go create mode 100644 internal/infrastructure/systemconfig/reader.go create mode 100644 internal/infrastructure/systemconfig/reader_integration_test.go create mode 100644 internal/infrastructure/systemconfig/registry.go create mode 100644 internal/infrastructure/systemconfig/registry_test.go create mode 100644 internal/model/dto/system_config_dto.go create mode 100644 internal/model/outbox_event.go create mode 100644 internal/model/system_config.go create mode 100644 internal/query/outbox/metrics.go create mode 100644 internal/query/outbox/metrics_integration_test.go create mode 100644 internal/query/systemconfig/list.go create mode 100644 internal/routes/system_config.go create mode 100644 internal/routes/system_config_integration_test.go create mode 100644 internal/testutil/outbox.go create mode 100644 internal/testutil/system_config.go create mode 100644 migrations/000165_create_public_outbox.down.sql create mode 100644 migrations/000165_create_public_outbox.up.sql create mode 100644 migrations/000166_create_system_config.down.sql create mode 100644 migrations/000166_create_system_config.up.sql create mode 100644 pkg/asynctask/contract.go create mode 100644 pkg/asynctask/contract_test.go create mode 100644 pkg/asynctask/state.go create mode 100644 pkg/constants/outbox.go create mode 100644 pkg/constants/system_config.go create mode 100644 pkg/idempotency/fingerprint.go create mode 100644 pkg/idempotency/fingerprint_test.go create mode 100644 pkg/idempotency/postgres_integration_test.go create mode 100644 pkg/logger/access_policy.go create mode 100644 pkg/logger/middleware_test.go create mode 100644 pkg/queue/client_test.go create mode 100755 scripts/test-tech-public-foundation.sh diff --git a/.scratch/tech-public-foundation/PRD.md b/.scratch/tech-public-foundation/PRD.md index c29c94f..9d68322 100644 --- a/.scratch/tech-public-foundation/PRD.md +++ b/.scratch/tech-public-foundation/PRD.md @@ -1,6 +1,6 @@ # PRD：TECH 七月迭代公共开发基础 -Status: ready-for-agent +Status: completed ## Problem Statement diff --git a/.scratch/tech-public-foundation/issues/01-public-migration-ownership-and-gates.md b/.scratch/tech-public-foundation/issues/01-public-migration-ownership-and-gates.md index 47cd307..e586489 100644 --- a/.scratch/tech-public-foundation/issues/01-public-migration-ownership-and-gates.md +++ b/.scratch/tech-public-foundation/issues/01-public-migration-ownership-and-gates.md @@ -4,16 +4,15 @@ **Blocked by:** None — can start immediately -**Status:** ready-for-agent +**Status:** completed **架构通道:** Infrastructure。 **完整业务边界:** 本票收口公共数据库对象的所有权登记、迁移前置检查、后置检查、失败退出、可重入和安全输出契约。明确不创建下游业务表，不迁移历史业务数据，不接管审计、通知或业务 PRD 拥有的迁移。 -- [ ] 公共 Outbox、系统配置及其公共索引、约束和初始化数据均有唯一迁移所有者，下游只能声明依赖，不能复制公共 DDL。 -- [ ] 检查能够发现目标对象定义不一致、唯一键冲突、必填字段空值、非法枚举、未完成任务、未投递事件、长租约和依赖版本问题，并以非零状态阻断发布。 -- [ ] 前置和后置检查可以重复运行；重复执行不产生新业务事实，`IF NOT EXISTS` 不会掩盖已有对象定义不一致。 -- [ ] 后置检查验证约束、关键索引、异常计数和读写冒烟，输出仅包含计数、错误码与安全标识，不泄露敏感值。 -- [ ] 发布说明明确哪些结构可安全回滚、哪些已有事实只能停止生产者后向前修复，以及迁移异常时的停止条件。 -- [ ] 自动化验证覆盖空数据库、兼容存量数据库、异常数据和重复执行场景，不执行全库清理。 - +- [x] 公共 Outbox、系统配置及其公共索引、约束和初始化数据均有唯一迁移所有者，下游只能声明依赖，不能复制公共 DDL。 +- [x] 检查能够发现目标对象定义不一致、唯一键冲突、必填字段空值、非法枚举、未完成任务、未投递事件、长租约和依赖版本问题，并以非零状态阻断发布。 +- [x] 前置和后置检查可以重复运行；重复执行不产生新业务事实，`IF NOT EXISTS` 不会掩盖已有对象定义不一致。 +- [x] 后置检查验证约束、关键索引、异常计数和读写冒烟，输出仅包含计数、错误码与安全标识，不泄露敏感值。 +- [x] 发布说明明确哪些结构可安全回滚、哪些已有事实只能停止生产者后向前修复，以及迁移异常时的停止条件。 +- [x] 自动化验证覆盖空数据库、兼容存量数据库、异常数据和重复执行场景，不执行全库清理。 diff --git a/.scratch/tech-public-foundation/issues/02-transactional-public-outbox-write.md b/.scratch/tech-public-foundation/issues/02-transactional-public-outbox-write.md index 34ab73a..036670b 100644 --- a/.scratch/tech-public-foundation/issues/02-transactional-public-outbox-write.md +++ b/.scratch/tech-public-foundation/issues/02-transactional-public-outbox-write.md @@ -4,16 +4,15 @@ **Blocked by:** 01 — 建立公共迁移所有权与检查门禁 -**Status:** ready-for-agent +**Status:** completed **架构通道:** Application + Port/Adapter。 **完整业务边界:** 本票收口公共 Outbox 模型、事件信封、事务内追加 Port 和一个可观察的示例写入链路。明确不定义下游业务事件含义，不实现业务消费者，不引入 UnitOfWork、事务工厂或全仓事务重构。 -- [ ] 公共 Outbox 具有稳定唯一的事件 ID、事件类型、载荷版本、聚合与资源定位、请求与关联标识、结构化载荷、投递生命周期、重试、租约和安全错误摘要字段。 -- [ ] Outbox 内部状态固定为 `1=待投递、2=投递中、3=已投递、4=投递失败`，常量、模型注释和公开说明保持一致。 -- [ ] 业务事实和 Outbox 使用同一 GORM 事务句柄；任一写入失败时二者均不可见，未提交事件不会被投递侧读取。 -- [ ] 事件 ID、业务键及必要快照在事务内生成并持久化，重试过程中不会重新生成事件身份。 -- [ ] 事务内不执行 Redis、Asynq、HTTP、对象存储或其他外部调用，Domain 不依赖 GORM。 -- [ ] PostgreSQL 集成测试覆盖事务成功、业务写入失败、Outbox 写入失败、事件 ID 唯一约束和回滚行为。 - +- [x] 公共 Outbox 具有稳定唯一的事件 ID、事件类型、载荷版本、聚合与资源定位、请求与关联标识、结构化载荷、投递生命周期、重试、租约和安全错误摘要字段。 +- [x] Outbox 内部状态固定为 `1=待投递、2=投递中、3=已投递、4=投递失败`，常量、模型注释和公开说明保持一致。 +- [x] 业务事实和 Outbox 使用同一 GORM 事务句柄；任一写入失败时二者均不可见，未提交事件不会被投递侧读取。 +- [x] 事件 ID、业务键及必要快照在事务内生成并持久化，重试过程中不会重新生成事件身份。 +- [x] 事务内不执行 Redis、Asynq、HTTP、对象存储或其他外部调用，Domain 不依赖 GORM。 +- [x] PostgreSQL 集成测试覆盖事务成功、业务写入失败、Outbox 写入失败、事件 ID 唯一约束和回滚行为。 diff --git a/.scratch/tech-public-foundation/issues/03-outbox-at-least-once-delivery.md b/.scratch/tech-public-foundation/issues/03-outbox-at-least-once-delivery.md index f8a4a23..2665974 100644 --- a/.scratch/tech-public-foundation/issues/03-outbox-at-least-once-delivery.md +++ b/.scratch/tech-public-foundation/issues/03-outbox-at-least-once-delivery.md @@ -4,16 +4,15 @@ **Blocked by:** 02 — 在业务事务中可靠写入公共 Outbox -**Status:** ready-for-agent +**Status:** completed **架构通道:** Infrastructure。 **完整业务边界:** 本票收口 Relay 的领取、租约、续租、投递、完成、失败、退避与恢复闭环，并通过公开 Asynq Handler 验证结构化信封。明确不实现业务消费者副作用，不承诺精确一次，不迁移未触碰的旧队列生产者。 -- [ ] Relay 以小批量条件领取或跳锁方式取得处理权，领取、续租、完成和失败均校验当前状态与租约所有者。 -- [ ] Relay 调用统一队列客户端时传 struct 或 map；传入 `[]byte` 被明确拒绝并有回归测试防止二次序列化为 Base64。 -- [ ] 入队成功后事件标记为已投递；模拟入队成功但标记前崩溃时，恢复投递仍使用原事件 ID、载荷和关联标识。 -- [ ] 瞬时失败按有上限的指数退避安排下次领取，达到最大重试或永久失败时保留记录并产生中文安全告警。 -- [ ] 多 Relay 并发时同一时刻只有租约所有者能够完成事件；Worker 崩溃后其他实例可在租约过期后恢复领取。 -- [ ] 真实 PostgreSQL、Redis 和 Asynq 链路测试覆盖事务写入、Relay、公开 Handler 与可观察消费结果，不依赖 Relay 私有函数断言。 - +- [x] Relay 以小批量条件领取或跳锁方式取得处理权，领取、续租、完成和失败均校验当前状态与租约所有者。 +- [x] Relay 调用统一队列客户端时传 struct 或 map；传入 `[]byte` 被明确拒绝并有回归测试防止二次序列化为 Base64。 +- [x] 入队成功后事件标记为已投递；模拟入队成功但标记前崩溃时，恢复投递仍使用原事件 ID、载荷和关联标识。 +- [x] 瞬时失败按有上限的指数退避安排下次领取，达到最大重试或永久失败时保留记录并产生中文安全告警。 +- [x] 多 Relay 并发时同一时刻只有租约所有者能够完成事件；Worker 崩溃后其他实例可在租约过期后恢复领取。 +- [x] 真实 PostgreSQL、Redis 和 Asynq 链路测试覆盖事务写入、Relay、公开 Handler 与可观察消费结果，不依赖 Relay 私有函数断言。 diff --git a/.scratch/tech-public-foundation/issues/04-outbox-observability-and-recovery.md b/.scratch/tech-public-foundation/issues/04-outbox-observability-and-recovery.md index e37fd1e..8f33fc5 100644 --- a/.scratch/tech-public-foundation/issues/04-outbox-observability-and-recovery.md +++ b/.scratch/tech-public-foundation/issues/04-outbox-observability-and-recovery.md @@ -4,16 +4,15 @@ **Blocked by:** 03 — 完成 Outbox 到 Asynq 的至少一次投递闭环 -**Status:** ready-for-agent +**Status:** completed **架构通道:** 主通道为 Query，辅助通道为 Infrastructure。 **完整业务边界:** 本票收口 Outbox 运行状态查询、指标、告警和受控恢复用例。明确不实现 Audit Event 模型或查询，不删除 Outbox，不修改已投递事件内容，不跳过消费者幂等检查。 -- [ ] 查询和指标覆盖待投递量、最老待投递年龄、处理中、过期租约、成功率、重试分布、最终失败和按事件类型的积压。 -- [ ] 日志、指标和告警使用事件 ID、关联 ID 与安全资源标识串联，不把完整载荷或敏感值作为日志字段或指标标签。 -- [ ] 受控重放只接受明确选择的失败或滞留事件，保留原事件 ID、载荷和关联标识，并记录操作者、原因和恢复批次。 -- [ ] 租约释放只作用于符合状态与过期条件的事件，不能越过当前租约所有者直接修改正在有效处理的事件。 -- [ ] 人工恢复通过统一审计 Port 记录；审计不可用时遵循明确的失败策略，但本票不创建独立审计表。 -- [ ] 测试覆盖积压统计、阈值告警、最终失败、选择性重放、租约释放和越权/非法状态拒绝。 - +- [x] 查询和指标覆盖待投递量、最老待投递年龄、处理中、过期租约、成功率、重试分布、最终失败和按事件类型的积压。 +- [x] 日志、指标和告警使用事件 ID、关联 ID 与安全资源标识串联，不把完整载荷或敏感值作为日志字段或指标标签。 +- [x] 受控重放只接受明确选择的失败或滞留事件，保留原事件 ID、载荷和关联标识，并记录操作者、原因和恢复批次。 +- [x] 租约释放只作用于符合状态与过期条件的事件，不能越过当前租约所有者直接修改正在有效处理的事件。 +- [x] 人工恢复通过统一审计 Port 记录；审计不可用时遵循明确的失败策略，但本票不创建独立审计表。 +- [x] 测试覆盖积压统计、阈值告警、最终失败、选择性重放、租约释放和越权/非法状态拒绝。 diff --git a/.scratch/tech-public-foundation/issues/05-command-idempotency-contract.md b/.scratch/tech-public-foundation/issues/05-command-idempotency-contract.md index 60e3419..7425071 100644 --- a/.scratch/tech-public-foundation/issues/05-command-idempotency-contract.md +++ b/.scratch/tech-public-foundation/issues/05-command-idempotency-contract.md @@ -4,16 +4,15 @@ **Blocked by:** None — can start immediately -**Status:** ready-for-agent +**Status:** completed **架构通道:** Application + Port/Adapter。 **完整业务边界:** 本票收口请求指纹、幂等作用域、冲突分类和并发职责的可复用构件与公开示例。明确不建立万能幂等表，不迁移未触碰的旧订单、钱包或状态机，不用 Redis 替代数据库事实。 -- [ ] 请求指纹只包含影响业务结果的规范化字段，排除时间戳、签名、Token 等易变传输字段，并携带算法版本。 -- [ ] 同一作用域内相同请求 ID 和相同指纹返回原结果；不同指纹返回统一幂等冲突且不覆盖既有事实。 -- [ ] 作用域至少区分调用主体与操作类型；不同主体使用相同请求 ID 不会相互污染。 -- [ ] 并发首次提交由 PostgreSQL 唯一约束裁决，应用层预查仅用于友好返回；Redis 不可用、过期或主从切换不影响最终正确性。 -- [ ] 文档和测试明确区分请求 ID、事件 ID、业务唯一键、状态条件更新、钱包版本、Worker 租约和 Redis 防并发的职责。 -- [ ] 集成测试覆盖相同请求重放、指纹冲突、不同主体、并发首写和 Redis 故障，不修改未触碰业务模块。 - +- [x] 请求指纹只包含影响业务结果的规范化字段，排除时间戳、签名、Token 等易变传输字段，并携带算法版本。 +- [x] 同一作用域内相同请求 ID 和相同指纹返回原结果；不同指纹返回统一幂等冲突且不覆盖既有事实。 +- [x] 作用域至少区分调用主体与操作类型；不同主体使用相同请求 ID 不会相互污染。 +- [x] 并发首次提交由 PostgreSQL 唯一约束裁决，应用层预查仅用于友好返回；Redis 不可用、过期或主从切换不影响最终正确性。 +- [x] 文档和测试明确区分请求 ID、事件 ID、业务唯一键、状态条件更新、钱包版本、Worker 租约和 Redis 防并发的职责。 +- [x] 集成测试覆盖相同请求重放、指纹冲突、不同主体、并发首写和 Redis 故障，不修改未触碰业务模块。 diff --git a/.scratch/tech-public-foundation/issues/06-unified-async-task-contract.md b/.scratch/tech-public-foundation/issues/06-unified-async-task-contract.md index 622fafc..b8fc6d4 100644 --- a/.scratch/tech-public-foundation/issues/06-unified-async-task-contract.md +++ b/.scratch/tech-public-foundation/issues/06-unified-async-task-contract.md @@ -4,16 +4,15 @@ **Blocked by:** None — can start immediately -**Status:** ready-for-agent +**Status:** completed **架构通道:** 主通道为 Application，辅助通道为 Query。 **完整业务边界:** 本票收口公共任务状态、公开投影、领取与终态更新、结构化队列载荷及前端轮询契约，并以现有导出任务作为兼容基准。明确不创建万能任务表，不迁移批量订购、设备分配、导入任务或其他未触碰业务任务。 -- [ ] 公共状态固定为 `1=待处理、2=处理中、3=已完成、4=已失败、5=已取消`，响应同时返回对应中文状态名。 -- [ ] 公开查询契约至少包含任务 ID、状态与名称、总数、成功数、失败数、进度、安全失败摘要、开始、完成和更新时间。 -- [ ] 业务项处理完成后满足 `total_count = success_count + failed_count`；部分成功和全部业务项失败均为已完成，不增加“部分成功”状态。 -- [ ] 待处理领取、终态进入和取消均使用预期状态条件更新；重复 Handler 遇到终态不会重新制造业务副作用。 -- [ ] 处理中任务具有租约或等价恢复记录，进程重启或重复投递后能从 PostgreSQL 事实恢复；队列载荷只包含最小结构化标识且禁止 `[]byte`。 -- [ ] 契约测试覆盖全成功、部分成功、全部业务项失败、整体失败、取消、重复消费和过期任务恢复，并记录跨仓轮询、隐藏暂停与刷新恢复语义。 - +- [x] 公共状态固定为 `1=待处理、2=处理中、3=已完成、4=已失败、5=已取消`，响应同时返回对应中文状态名。 +- [x] 公开查询契约至少包含任务 ID、状态与名称、总数、成功数、失败数、进度、安全失败摘要、开始、完成和更新时间。 +- [x] 业务项处理完成后满足 `total_count = success_count + failed_count`；部分成功和全部业务项失败均为已完成，不增加“部分成功”状态。 +- [x] 待处理领取、终态进入和取消均使用预期状态条件更新；重复 Handler 遇到终态不会重新制造业务副作用。 +- [x] 处理中任务具有租约或等价恢复记录，进程重启或重复投递后能从 PostgreSQL 事实恢复；队列载荷只包含最小结构化标识且禁止 `[]byte`。 +- [x] 契约测试覆盖全成功、部分成功、全部业务项失败、整体失败、取消、重复消费和过期任务恢复，并记录跨仓轮询、隐藏暂停与刷新恢复语义。 diff --git a/.scratch/tech-public-foundation/issues/07-controlled-system-config-read.md b/.scratch/tech-public-foundation/issues/07-controlled-system-config-read.md index b93cf87..47e149b 100644 --- a/.scratch/tech-public-foundation/issues/07-controlled-system-config-read.md +++ b/.scratch/tech-public-foundation/issues/07-controlled-system-config-read.md @@ -4,16 +4,15 @@ **Blocked by:** 01 — 建立公共迁移所有权与检查门禁 -**Status:** ready-for-agent +**Status:** completed **架构通道:** 主通道为 Query，辅助通道为 Infrastructure。 **完整业务边界:** 本票收口系统配置表、代码注册表、启动校验、缓存读取和超级管理员列表 API。明确不注册支付等具体业务 Key，不允许创建任意 Key，不提供原始 JSON 自由编辑器；新增 Handler 必须同步文档生成器。 -- [ ] 配置存储支持唯一 Key、字符串化值、`string/int/bool/json` 类型、模块、中文说明、只读与敏感属性、创建更新人与时间，且无外键或 GORM 关联标签。 -- [ ] 注册表定义稳定 Key、模块、类型、值域或枚举、默认值、只读、敏感和控件提示；重复 Key、类型冲突或非法默认值在启动或验证阶段失败。 -- [ ] 只有已认证超级管理员可按模块分页查询配置；权限不足返回统一 403，不伪装为空数据。 -- [ ] 未注册数据库 Key 默认不可写，最多按只读、可诊断方式展示；敏感配置值按注册策略脱敏。 -- [ ] PostgreSQL 是唯一事实来源；Redis 未命中、超时或不可用时回退数据库并尝试回填，缓存 Key 和默认 TTL 遵循公共常量。 -- [ ] 真实 Fiber、认证、GORM 和 Redis 测试覆盖模块过滤、四种类型、缓存命中与回退、未注册 Key、敏感值和权限边界；OpenAPI 文档生成器同步更新。 - +- [x] 配置存储支持唯一 Key、字符串化值、`string/int/bool/json` 类型、模块、中文说明、只读与敏感属性、创建更新人与时间，且无外键或 GORM 关联标签。 +- [x] 注册表定义稳定 Key、模块、类型、值域或枚举、默认值、只读、敏感和控件提示；重复 Key、类型冲突或非法默认值在启动或验证阶段失败。 +- [x] 只有已认证超级管理员可按模块分页查询配置；权限不足返回统一 403，不伪装为空数据。 +- [x] 未注册数据库 Key 默认不可写，最多按只读、可诊断方式展示；敏感配置值按注册策略脱敏。 +- [x] PostgreSQL 是唯一事实来源；Redis 未命中、超时或不可用时回退数据库并尝试回填，缓存 Key 和默认 TTL 遵循公共常量。 +- [x] 真实 Fiber、认证、GORM 和 Redis 测试覆盖模块过滤、四种类型、缓存命中与回退、未注册 Key、敏感值和权限边界；OpenAPI 文档生成器同步更新。 diff --git a/.scratch/tech-public-foundation/issues/08-controlled-system-config-update.md b/.scratch/tech-public-foundation/issues/08-controlled-system-config-update.md index 1eea126..e7869db 100644 --- a/.scratch/tech-public-foundation/issues/08-controlled-system-config-update.md +++ b/.scratch/tech-public-foundation/issues/08-controlled-system-config-update.md @@ -4,16 +4,15 @@ **Blocked by:** 07 — 交付受控系统配置注册与查询闭环 -**Status:** ready-for-agent +**Status:** completed **架构通道:** 主通道为简单写 Application 事务脚本，辅助通道为 Port/Adapter。 **完整业务边界:** 本票收口单 Key 更新、权限、类型和值域校验、事务写入、缓存失效和审计 Port。明确不实现全局 Audit Event 模型，不提供无约束批量覆盖，不接管各业务模块的具体值域或生效规则；新增 Handler 必须同步文档生成器。 -- [ ] 只有超级管理员可以更新配置；未注册、只读、类型错误、非法 JSON、越界或枚举外值均返回统一中文错误且不修改事实。 -- [ ] 配置更新使用现有 GORM 显式事务完成事实写入和审计接缝调用；任一步事务内写入失败时整体回滚。 -- [ ] 审计信息包含操作者、操作类型、中文描述、变更前后事实和请求关联标识，但本票不创建独立配置审计表。 -- [ ] 提交成功后失效对应 Redis 缓存；失效失败不回滚 PostgreSQL，产生包含组件、错误码、时间窗口和安全标识的中文告警。 -- [ ] 数据库值不可解析或越界时不能静默使用错误值，按注册策略返回最后验证值或安全默认值并告警。 -- [ ] 真实 Fiber、认证、GORM 和 Redis 测试覆盖更新成功、并发更新、权限、未注册、只读、四种类型、事务回滚、缓存失效失败和审计事实；OpenAPI 文档生成器同步更新。 - +- [x] 只有超级管理员可以更新配置；未注册、只读、类型错误、非法 JSON、越界或枚举外值均返回统一中文错误且不修改事实。 +- [x] 配置更新使用现有 GORM 显式事务完成事实写入和审计接缝调用；任一步事务内写入失败时整体回滚。 +- [x] 审计信息包含操作者、操作类型、中文描述、变更前后事实和请求关联标识，但本票不创建独立配置审计表。 +- [x] 提交成功后失效对应 Redis 缓存；失效失败不回滚 PostgreSQL，产生包含组件、错误码、时间窗口和安全标识的中文告警。 +- [x] 数据库值不可解析或越界时不能静默使用错误值，按注册策略返回最后验证值或安全默认值并告警。 +- [x] 真实 Fiber、认证、GORM 和 Redis 测试覆盖更新成功、并发更新、权限、未注册、只读、四种类型、事务回滚、缓存失效失败和审计事实；OpenAPI 文档生成器同步更新。 diff --git a/.scratch/tech-public-foundation/issues/09-access-log-recursive-redaction.md b/.scratch/tech-public-foundation/issues/09-access-log-recursive-redaction.md index 005da08..7fb8bb7 100644 --- a/.scratch/tech-public-foundation/issues/09-access-log-recursive-redaction.md +++ b/.scratch/tech-public-foundation/issues/09-access-log-recursive-redaction.md @@ -4,16 +4,15 @@ **Blocked by:** None — can start immediately -**Status:** ready-for-agent +**Status:** completed **架构通道:** Infrastructure。 **完整业务边界:** 本票收口公共敏感字段注册、query/请求/响应 JSON 脱敏、截断与访问日志元数据。明确不修改 Audit Event 或 Integration Log 模型、Writer 与查询，不改变业务响应内容。 -- [ ] 公共敏感字段至少覆盖密码、口令、Token、Authorization、Cookie、密钥、Secret、签名、Nonce、验证码、支付凭证和私密 URL，匹配大小写不敏感。 -- [ ] query、请求 JSON 和响应 JSON 复用同一递归规则，嵌套对象、数组、非字符串敏感字段均被不可逆替换。 -- [ ] 请求体和响应体分别先脱敏后按 50KB 截断，并输出可机器识别的截断状态，不因序列化失败回退记录未脱敏 JSON。 -- [ ] 脱敏后仍保留方法、路径、安全 query、状态、耗时、请求 ID、IP、User-Agent、用户标识及请求/响应摘要。 -- [ ] 日志、注释和告警均使用中文，用户可见响应继续使用统一错误与响应格式。 -- [ ] 真实 Fiber 测试捕获最终 JSON 日志，覆盖嵌套结构、数组、大小写变体、query、请求与响应、超长 body 和无法序列化场景。 - +- [x] 公共敏感字段至少覆盖密码、口令、Token、Authorization、Cookie、密钥、Secret、签名、Nonce、验证码、支付凭证和私密 URL，匹配大小写不敏感。 +- [x] query、请求 JSON 和响应 JSON 复用同一递归规则，嵌套对象、数组、非字符串敏感字段均被不可逆替换。 +- [x] 请求体和响应体分别先脱敏后按 50KB 截断，并输出可机器识别的截断状态，不因序列化失败回退记录未脱敏 JSON。 +- [x] 脱敏后仍保留方法、路径、安全 query、状态、耗时、请求 ID、IP、User-Agent、用户标识及请求/响应摘要。 +- [x] 日志、注释和告警均使用中文，用户可见响应继续使用统一错误与响应格式。 +- [x] 真实 Fiber 测试捕获最终 JSON 日志，覆盖嵌套结构、数组、大小写变体、query、请求与响应、超长 body 和无法序列化场景。 diff --git a/.scratch/tech-public-foundation/issues/10-sensitive-route-safe-summaries.md b/.scratch/tech-public-foundation/issues/10-sensitive-route-safe-summaries.md index 5fe6047..c6bfdb4 100644 --- a/.scratch/tech-public-foundation/issues/10-sensitive-route-safe-summaries.md +++ b/.scratch/tech-public-foundation/issues/10-sensitive-route-safe-summaries.md @@ -4,16 +4,15 @@ **Blocked by:** 09 — 统一 Access Log 请求与响应递归脱敏 -**Status:** ready-for-agent +**Status:** completed **架构通道:** Infrastructure。 **完整业务边界:** 本票收口路由级敏感策略、非 JSON 安全降级和固定回归矩阵。明确不实现登录、支付、企微、文件或导出业务逻辑，不记录完整回调正文、文件内容或临时访问能力。 -- [ ] 登录和 Token 接口不记录密码、验证码、访问令牌、刷新令牌或会话标识，只保留成功状态和必要主体标识。 -- [ ] 支付接口不记录支付凭证、银行卡敏感信息、二维码原文、跳转链接、渠道密钥或完整签名，只保留安全订单号、渠道类型、结果码和金额摘要。 -- [ ] 企微回调不记录加密包、解密正文、签名、Nonce、通讯录敏感字段或完整响应，只保留事件类型、安全标识、大小、哈希和处理结果。 -- [ ] 文件与导出接口不记录 multipart、二进制、Base64、文件字节、临时凭证或签名下载地址，只保留脱敏文件名、类型、大小、数量、任务标识和结果。 -- [ ] 敏感路由解析失败时只记录字段存在性、长度、内容类型、安全哈希和截断标志；普通非敏感文本也必须经过明确路由策略才可记录。 -- [ ] 真实 Fiber 回归矩阵覆盖 JSON、XML、表单、multipart、二进制、超长和不可解析载荷，并断言日志中不存在测试凭证、签名、回调原文或文件字节。 - +- [x] 登录和 Token 接口不记录密码、验证码、访问令牌、刷新令牌或会话标识，只保留成功状态和必要主体标识。 +- [x] 支付接口不记录支付凭证、银行卡敏感信息、二维码原文、跳转链接、渠道密钥或完整签名，只保留安全订单号、渠道类型、结果码和金额摘要。 +- [x] 企微回调不记录加密包、解密正文、签名、Nonce、通讯录敏感字段或完整响应，只保留事件类型、安全标识、大小、哈希和处理结果。 +- [x] 文件与导出接口不记录 multipart、二进制、Base64、文件字节、临时凭证或签名下载地址，只保留脱敏文件名、类型、大小、数量、任务标识和结果。 +- [x] 敏感路由解析失败时只记录字段存在性、长度、内容类型、安全哈希和截断标志；普通非敏感文本也必须经过明确路由策略才可记录。 +- [x] 真实 Fiber 回归矩阵覆盖 JSON、XML、表单、multipart、二进制、超长和不可解析载荷，并断言日志中不存在测试凭证、签名、回调原文或文件字节。 diff --git a/.scratch/tech-public-foundation/issues/11-public-migration-and-rollback-validation.md b/.scratch/tech-public-foundation/issues/11-public-migration-and-rollback-validation.md index dfb522b..61ef0be 100644 --- a/.scratch/tech-public-foundation/issues/11-public-migration-and-rollback-validation.md +++ b/.scratch/tech-public-foundation/issues/11-public-migration-and-rollback-validation.md @@ -4,16 +4,15 @@ **Blocked by:** 02 — 在业务事务中可靠写入公共 Outbox；07 — 交付受控系统配置注册与查询闭环 -**Status:** ready-for-agent +**Status:** completed **架构通道:** Infrastructure。 **完整业务边界:** 本票收口公共 Outbox 和系统配置对象的迁移链路、兼容数据验证、可重入回填、后置校验和回滚边界。明确不迁移下游业务表，不删除已产生的业务事实、Outbox、审计、通知或任务结果。 -- [ ] 空数据库和兼容存量数据库均可执行正向迁移，公共表、索引、约束和初始化数据只创建一次且定义符合契约。 -- [ ] 构造重复事件 ID、非法配置、唯一键冲突、处理中任务、未投递事件和长租约时，前置检查以非零状态失败且不执行破坏性写入。 -- [ ] 需要回填时按稳定主键分批、记录进度并可中断重跑；重复执行不生成重复事实，最终行数守恒。 -- [ ] 后置校验覆盖约束生效、异常计数归零、关键索引可用和读写冒烟，输出不包含敏感数据。 -- [ ] 未产生业务数据的新增结构可在验证后回滚；已有 Outbox 或配置事实后，回滚流程停止生产者和 Relay、保留事实并向前修复，禁止删表清理。 -- [ ] 迁移说明记录发布顺序、停止条件、恢复步骤和测试数据隔离策略，并通过相关迁移与集成测试。 - +- [x] 空数据库和兼容存量数据库均可执行正向迁移，公共表、索引、约束和初始化数据只创建一次且定义符合契约。 +- [x] 构造重复事件 ID、非法配置、唯一键冲突、处理中任务、未投递事件和长租约时，前置检查以非零状态失败且不执行破坏性写入。 +- [x] 需要回填时按稳定主键分批、记录进度并可中断重跑；重复执行不生成重复事实，最终行数守恒。 +- [x] 后置校验覆盖约束生效、异常计数归零、关键索引可用和读写冒烟，输出不包含敏感数据。 +- [x] 未产生业务数据的新增结构可在验证后回滚；已有 Outbox 或配置事实后，回滚流程停止生产者和 Relay、保留事实并向前修复，禁止删表清理。 +- [x] 迁移说明记录发布顺序、停止条件、恢复步骤和测试数据隔离策略，并通过相关迁移与集成测试。 diff --git a/.scratch/tech-public-foundation/issues/12-foundation-release-gate-and-integration-contract.md b/.scratch/tech-public-foundation/issues/12-foundation-release-gate-and-integration-contract.md index 1b53d2a..8a09766 100644 --- a/.scratch/tech-public-foundation/issues/12-foundation-release-gate-and-integration-contract.md +++ b/.scratch/tech-public-foundation/issues/12-foundation-release-gate-and-integration-contract.md @@ -4,16 +4,16 @@ **Blocked by:** 03 — 完成 Outbox 到 Asynq 的至少一次投递闭环；04 — 提供 Outbox 监控和受控恢复能力；05 — 提供创建命令幂等与并发职责契约；06 — 冻结统一异步任务五态和查询契约；08 — 交付系统配置更新、权限和审计闭环；10 — 为敏感接口提供安全摘要策略；11 — 验证公共对象迁移与数据安全回滚边界 -**Status:** ready-for-agent +**Status:** completed **架构通道:** 主通道为 Infrastructure，辅助通道为 Application 与 Query 契约。 **完整业务边界:** 本票收口公共能力的端到端发布门禁、运行手册、跨仓前端契约和下游接入说明。明确不实现前端代码、Audit Event、Integration Log、站内通知、业务事件消费者或任何下游领域规则，不借验收迁移未触碰旧模块。 -- [ ] 整体验收通过公开接缝验证业务事务写入、Outbox Relay、Asynq Handler、重复投递、租约恢复和可观察消费结果。 -- [ ] PostgreSQL、Redis 和 Asynq 测试使用隔离数据与唯一前缀，只清理本次创建的数据，不执行全库或全缓存清空。 -- [ ] 发布顺序明确为迁移与检查、兼容 API、Relay/Worker、依赖消费者、前端；生产者不得在消费者和监控就绪前制造不可见积压。 -- [ ] 停止条件覆盖迁移异常、Outbox 持续积压或租约大量过期、配置读写不一致、脱敏回归失败和关键任务无法恢复。 -- [ ] 下游接入说明明确公共基础提供与不提供的能力，以及审计、通知和各业务 PRD 自行拥有的模型、状态机、业务唯一键、失败明细和消费者幂等。 -- [ ] 前端跨仓契约记录加载、真实空态、筛选空态、403、失败重试、任务 ID 恢复、2/3/5 秒退避、最长 10 秒、页面隐藏暂停和恢复立即刷新。 -- [ ] 中文功能总结覆盖关键流程、异常闭环、发布回滚、监控恢复和待决策项，README 增加索引；所有公共外部行为测试通过后方可标记基础就绪。 +- [x] 整体验收通过公开接缝验证业务事务写入、Outbox Relay、Asynq Handler、重复投递、租约恢复和可观察消费结果。 +- [x] PostgreSQL、Redis 和 Asynq 测试使用隔离数据与唯一前缀，只清理本次创建的数据，不执行全库或全缓存清空。 +- [x] 发布顺序明确为迁移与检查、兼容 API、Relay/Worker、依赖消费者、前端；生产者不得在消费者和监控就绪前制造不可见积压。 +- [x] 停止条件覆盖迁移异常、Outbox 持续积压或租约大量过期、配置读写不一致、脱敏回归失败和关键任务无法恢复。 +- [x] 下游接入说明明确公共基础提供与不提供的能力，以及审计、通知和各业务 PRD 自行拥有的模型、状态机、业务唯一键、失败明细和消费者幂等。 +- [x] 前端跨仓契约记录加载、真实空态、筛选空态、403、失败重试、任务 ID 恢复、2/3/5 秒退避、最长 10 秒、页面隐藏暂停和恢复立即刷新。 +- [x] 中文功能总结覆盖关键流程、异常闭环、发布回滚、监控恢复和待决策项，README 增加索引；所有公共外部行为测试通过后方可标记基础就绪。 diff --git a/README.md b/README.md index d31074a..7a9ae49 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,10 @@ default: ## 核心功能 +### 公共技术基础 + +公共 Outbox、命令幂等、统一异步任务、受控系统配置、迁移门禁和 Access Log 安全策略的能力边界、发布顺序与下游接入方式见[公共技术基础功能总结](docs/tech-public-foundation/功能总结.md)。 + ### 账号管理重构（2025-02） 统一了账号管理和认证接口架构，消除了路由冗余，修复了越权漏洞，添加了完整的操作审计。 diff --git a/cmd/foundation-check/main.go b/cmd/foundation-check/main.go new file mode 100644 index 0000000..e101cce --- /dev/null +++ b/cmd/foundation-check/main.go @@ -0,0 +1,57 @@ +// Command foundation-check 执行公共基础迁移前后发布门禁。 +package main + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + "github.com/bytedance/sonic" + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/releasegate" + "github.com/break/junhong_cmp_fiber/pkg/config" + "github.com/break/junhong_cmp_fiber/pkg/database" +) + +func main() { + phase := flag.String("phase", releasegate.PhasePre, "检查阶段：pre 或 post") + timeout := flag.Duration("timeout", time.Minute, "检查超时时间") + flag.Parse() + + if err := run(*phase, *timeout); err != nil { + fmt.Fprintf(os.Stderr, "公共基础发布门禁失败：%v\n", err) + os.Exit(1) + } +} + +func run(phase string, timeout time.Duration) error { + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("加载配置失败：%w", err) + } + db, err := database.InitPostgreSQL(&cfg.Database, zap.NewNop()) + if err != nil { + return fmt.Errorf("连接 PostgreSQL 失败：%w", err) + } + if sqlDB, dbErr := db.DB(); dbErr == nil { + defer func() { _ = sqlDB.Close() }() + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + report, err := releasegate.NewChecker(db).Run(ctx, phase) + if err != nil { + return err + } + encoded, err := sonic.Marshal(report) + if err != nil { + return fmt.Errorf("序列化门禁报告失败：%w", err) + } + fmt.Println(string(encoded)) + if !report.Passed() { + return fmt.Errorf("发现阻断项，请按稳定错误码处理后重试") + } + return nil +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 6134e5a..aadd009 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -16,6 +16,7 @@ import ( "github.com/break/junhong_cmp_fiber/internal/bootstrap" "github.com/break/junhong_cmp_fiber/internal/gateway" + "github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox" "github.com/break/junhong_cmp_fiber/internal/model" "github.com/break/junhong_cmp_fiber/internal/polling" iot_card_svc "github.com/break/junhong_cmp_fiber/internal/service/iot_card" @@ -36,6 +37,7 @@ const ( workerModulePollingInitializer = "polling_initializer" workerModulePollingScheduler = "polling_scheduler" workerModuleAsynqScheduler = "asynq_scheduler" + workerModuleOutboxRelay = "outbox_relay" importRescueLimit = 500 // 启动补偿单次扫描的最大导入任务数 ) @@ -55,6 +57,8 @@ type workerRuntime struct { asynqClient *asynq.Client workerResult *bootstrap.WorkerBootstrapResult workerServer *queue.Server + outboxQueueClient *queue.Client + outboxConsumers *outbox.ConsumerRegistry pollingConfigMgr *polling.PollingConfigManager pollingQueueMgr *polling.PollingQueueManager pollingIotCardStore *postgres.IotCardStore @@ -124,6 +128,9 @@ func runWorker(cfg *config.Config) { taskHandler := createTaskHandler(runtime, appLogger) taskHandler.RegisterHandlers() + outboxHandler := outbox.NewHandler(runtime.outboxConsumers) + taskHandler.GetMux().HandleFunc(constants.TaskTypeOutboxDeliver, outboxHandler.Handle) + startOutboxRelay(ctx, runtime, cfg.Worker.InstanceName, appLogger) rescuePendingImportTasks(ctx, runtime, appLogger) appLogger.Info("Worker 服务器配置完成", @@ -149,7 +156,7 @@ func runWorker(cfg *config.Config) { // buildWorkerModuleStatus 根据角色生成启用/禁用模块列表，供启动日志和人工验收使用。 func buildWorkerModuleStatus(role string) workerModuleStatus { status := workerModuleStatus{ - enabled: []string{workerModuleQueueServer}, + enabled: []string{workerModuleQueueServer, workerModuleOutboxRelay}, } if runsSingletonModules(role) { @@ -233,6 +240,7 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L } workerServer := queue.NewServer(redisClient, &cfg.Queue, appLogger) + outboxQueueClient := queue.NewClient(redisClient, appLogger) pollingConfigStore := postgres.NewPollingConfigStore(db) pollingConfigMgr := polling.NewPollingConfigManager(pollingConfigStore, redisClient, appLogger) @@ -275,6 +283,8 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L asynqClient: asynqClient, workerResult: workerResult, workerServer: workerServer, + outboxQueueClient: outboxQueueClient, + outboxConsumers: outbox.NewConsumerRegistry(), pollingConfigMgr: pollingConfigMgr, pollingQueueMgr: pollingQueueMgr, pollingIotCardStore: pollingIotCardStore, @@ -285,6 +295,11 @@ func initWorkerRuntime(ctx context.Context, cfg *config.Config, appLogger *zap.L // close 在 Worker 退出时关闭共享客户端与数据库连接。 func (r *workerRuntime) close(appLogger *zap.Logger) { + if r.outboxQueueClient != nil { + if err := r.outboxQueueClient.Close(); err != nil { + appLogger.Error("关闭 Outbox 队列客户端失败", zap.Error(err)) + } + } if r.asynqClient != nil { if err := r.asynqClient.Close(); err != nil { appLogger.Error("关闭 Asynq 客户端失败", zap.Error(err)) @@ -305,6 +320,39 @@ func (r *workerRuntime) close(appLogger *zap.Logger) { } } +// startOutboxRelay 启动公共 Outbox 的可取消轮询；多个 Worker 依靠 PostgreSQL 租约并发协调。 +func startOutboxRelay(ctx context.Context, runtime *workerRuntime, instanceName string, appLogger *zap.Logger) { + owner := instanceName + if owner == "" { + owner = fmt.Sprintf("worker-%d", os.Getpid()) + } + relay, err := outbox.NewRelay( + runtime.db, + outbox.NewQueuePublisher(runtime.outboxQueueClient), + appLogger, + outbox.RelayOptions{Owner: owner}, + ) + if err != nil { + appLogger.Fatal("初始化 Outbox Relay 失败", zap.Error(err)) + } + go func() { + ticker := time.NewTicker(constants.OutboxRelayPollInterval) + defer ticker.Stop() + for { + if _, err := relay.ProcessBatch(ctx); err != nil && ctx.Err() == nil { + appLogger.Error("Outbox Relay 批次处理失败", + zap.String("component", "outbox_relay"), zap.String("error_code", "OUTBOX_RELAY_BATCH_FAILED"), zap.Error(err)) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + appLogger.Info("Outbox Relay 已启动", zap.String("lease_owner", owner)) +} + // startPollingInitializer 为 leader/all 启动渐进式初始化器，并保留配置变更重启逻辑。 func startPollingInitializer(ctx context.Context, runtime *workerRuntime, appLogger *zap.Logger) *polling.PollingInitializer { pollingInitializer := polling.NewPollingInitializer( diff --git a/docs/admin-openapi.yaml b/docs/admin-openapi.yaml index 693789b..addb4f1 100644 --- a/docs/admin-openapi.yaml +++ b/docs/admin-openapi.yaml @@ -5406,9 +5406,18 @@ components: download_url: description: 下载链接（仅已完成任务返回） type: string + error_code: + description: 安全错误码 + type: string error_message: description: 错误信息 type: string + error_summary: + description: 安全失败摘要 + type: string + failed_count: + description: 统一任务失败数 + type: integer failed_shards: description: 失败分片数 type: integer @@ -5442,18 +5451,32 @@ components: status_name: description: 任务状态名称（中文） type: string + success_count: + description: 统一任务成功数 + type: integer success_shards: description: 成功分片数 type: integer + task_id: + description: 任务ID + minimum: 0 + type: integer task_no: description: 任务编号 type: string + total_count: + description: 统一任务总数 + type: integer total_rows: description: 总行数 type: integer total_shards: description: 总分片数 type: integer + updated_at: + description: 更新时间 + format: date-time + type: string type: object DtoExportTaskItem: properties: @@ -5486,9 +5509,18 @@ components: creator_user_type: description: 创建人用户类型 (2:平台, 3:代理, 4:企业) type: integer + error_code: + description: 安全错误码 + type: string error_message: description: 错误信息 type: string + error_summary: + description: 安全失败摘要 + type: string + failed_count: + description: 统一任务失败数 + type: integer failed_shards: description: 失败分片数 type: integer @@ -5522,18 +5554,32 @@ components: status_name: description: 任务状态名称（中文） type: string + success_count: + description: 统一任务成功数 + type: integer success_shards: description: 成功分片数 type: integer + task_id: + description: 任务ID + minimum: 0 + type: integer task_no: description: 任务编号 type: string + total_count: + description: 统一任务总数 + type: integer total_rows: description: 总行数 type: integer total_shards: description: 总分片数 type: integer + updated_at: + description: 更新时间 + format: date-time + type: string type: object DtoFailedDeviceItem: properties: @@ -8680,6 +8726,72 @@ components: required: - target_iccid type: object + DtoSystemConfigItem: + properties: + config_key: + description: 稳定配置 Key + type: string + control: + description: 前端控件提示 + type: string + description: + description: 中文说明 + type: string + enum_values: + description: 允许的枚举值 + items: + type: string + type: array + max: + description: 整数最大值 + nullable: true + type: integer + min: + description: 整数最小值 + nullable: true + type: integer + module: + description: 所属模块 + type: string + readonly: + description: 是否只读 + type: boolean + registered: + description: 是否已在代码注册 + type: boolean + sensitive: + description: 是否敏感 + type: boolean + updated_at: + description: 最近更新时间 + format: date-time + nullable: true + type: string + value: + description: 配置值；敏感值按注册策略脱敏 + type: string + value_type: + description: 值类型 (string:字符串, int:整数, bool:布尔, json:JSON) + type: string + type: object + DtoSystemConfigListResponse: + properties: + list: + description: 配置列表 + items: + $ref: '#/components/schemas/DtoSystemConfigItem' + nullable: true + type: array + page: + description: 页码 + type: integer + page_size: + description: 每页数量 + type: integer + total: + description: 总数量 + type: integer + type: object DtoTriggerBatchReq: properties: card_ids: @@ -9389,6 +9501,18 @@ components: required: - status type: object + DtoUpdateSystemConfigParams: + properties: + key: + description: 稳定配置 Key + type: string + value: + description: 字符串化配置值 + type: string + required: + - key + - value + type: object DtoUpdateWechatConfigParams: properties: ali_app_id: @@ -24676,6 +24800,157 @@ paths: summary: 查询操作密码是否已设置 tags: - 超级管理员 + /api/admin/system-configs: + get: + parameters: + - description: 模块筛选 + in: query + name: module + schema: + description: 模块筛选 + type: string + - description: 页码 + in: query + name: page + schema: + description: 页码 + minimum: 1 + type: integer + - description: 每页数量 + in: query + name: page_size + schema: + description: 每页数量 + maximum: 100 + minimum: 1 + type: integer + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/DtoSystemConfigListResponse' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 查询受控系统配置 + tags: + - 系统配置 + /api/admin/system-configs/{key}: + put: + parameters: + - description: 稳定配置 Key + in: path + name: key + required: true + schema: + description: 稳定配置 Key + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DtoUpdateSystemConfigParams' + responses: + "200": + content: + application/json: + schema: + properties: + code: + description: 响应码 + example: 0 + type: integer + data: + $ref: '#/components/schemas/DtoSystemConfigItem' + msg: + description: 响应消息 + example: success + type: string + timestamp: + description: 时间戳 + format: date-time + type: string + required: + - code + - msg + - data + - timestamp + type: object + description: 成功 + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 请求参数错误 + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 未认证或认证已过期 + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 无权访问 + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 服务器内部错误 + security: + - BearerAuth: [] + summary: 更新受控系统配置 + tags: + - 系统配置 /api/admin/wechat-configs: get: parameters: diff --git a/docs/tech-public-foundation/创建命令幂等契约.md b/docs/tech-public-foundation/创建命令幂等契约.md new file mode 100644 index 0000000..198a80e --- /dev/null +++ b/docs/tech-public-foundation/创建命令幂等契约.md @@ -0,0 +1,19 @@ +# 创建命令幂等与并发职责契约 + +创建类命令使用“调用主体 + 操作类型 + `request_id`”作为入口身份，并把影响业务结果的专用命令 DTO 计算为带版本的 SHA-256 指纹。时间戳、签名、Token、Authorization、Cookie 和 Nonce 等易变传输字段不进入指纹；业务模块应尽量直接传专用 DTO，避免把任意 HTTP 请求整体作为业务身份。 + +同一作用域中，相同 `request_id` 与相同指纹返回已保存的业务结果；指纹不同返回统一冲突且不得覆盖原事实。不同主体或不同操作类型即使复用同一 `request_id`，也属于不同命令。 + +应用层预查只改善返回体验。并发首写必须由业务表或业务幂等事实上的 PostgreSQL 唯一约束裁决，并与业务事实处于同一事务。公共基础不创建万能幂等表，业务模块自行保存结果定位和稳定业务唯一键。 + +各类身份与并发职责不得混用： + +| 构件 | 唯一职责 | +|---|---| +| `request_id` + 指纹 | 识别入口命令重放与内容冲突 | +| `event_id` | 识别至少一次投递中的同一事件 | +| 业务唯一键/唯一流水 | 防止同一业务副作用重复落库 | +| 状态条件更新 | 裁决状态机是否可从预期状态推进 | +| 钱包 `version` | 裁决余额、冻结额等并发数值更新 | +| Worker 租约 | 暂时决定当前处理者，不证明副作用尚未发生 | +| Redis 锁或 `SETNX` | 降低热点并发；故障、过期或切主不能破坏最终正确性 | diff --git a/docs/tech-public-foundation/功能总结.md b/docs/tech-public-foundation/功能总结.md new file mode 100644 index 0000000..0c3a268 --- /dev/null +++ b/docs/tech-public-foundation/功能总结.md @@ -0,0 +1,69 @@ +# 公共技术基础功能总结 + +## 交付边界 + +本能力提供公共 Outbox 与 Relay、创建命令幂等原语、统一异步任务五态、受控系统配置、数据库发布门禁，以及 Access Log 递归脱敏和敏感路由安全摘要。公共基础只定义稳定接缝和基础设施语义，不拥有 Audit Event、Integration Log、站内通知、业务模型、业务状态机、业务唯一键、任务失败明细或业务事件消费者。 + +各业务 PRD 必须自行实现消费者副作用幂等，并决定使用事件 ID、业务唯一键、状态条件更新或版本号裁决重复消费。审计和通知由对应公共能力实现后注入现有 Port；系统配置更新在审计 Port 不可用时失败关闭，不另建临时审计表。 + +## 关键流程 + +### 事务事件与至少一次投递 + +业务 Application 在同一 GORM 事务中写入业务事实和公共 Outbox，事务内不调用 Redis、Asynq、HTTP 或对象存储。Relay 使用行锁跳过竞争记录并写入租约，再通过统一队列客户端把结构化信封交给 Asynq。公开 Handler 按稳定事件类型分发给业务消费者；入队成功但状态未落库时允许使用原事件 ID、载荷和关联标识重复投递。 + +瞬时错误采用有上限的指数退避；达到最大次数或收到明确永久错误后保留最终失败事实。租约过期后其他实例可以恢复领取，有效租约不能被人工释放或其他实例完成。 + +### 幂等与异步任务 + +创建命令指纹只包含影响业务结果的规范化字段，并携带算法版本。PostgreSQL 唯一约束是并发首次提交的最终裁决；Redis 只用于减少并发和改善体验。请求 ID、事件 ID、业务唯一键、状态条件、数值版本和 Worker 租约不得互相替代。 + +异步任务固定为 `1=待处理、2=处理中、3=已完成、4=已失败、5=已取消`。任务表由各业务拥有，公共 Adapter 通过可配置列映射执行条件领取、租约恢复、完成和取消，不建设万能任务表。部分成功或业务项全部失败仍为已完成，并满足总数等于成功数加失败数。 + +### 受控系统配置 + +业务模块在代码注册表中声明稳定 Key、类型、默认值、值域、只读和敏感策略。PostgreSQL 是唯一事实来源，Redis 仅作短缓存；缓存异常会回退数据库并产生不含配置值的安全告警。只有超级管理员可以分页查询或更新，未注册 Key 强制只读，敏感值只返回“已配置”。更新事实和审计接缝处于同一事务，提交后只失效对应缓存 Key。 + +### 日志安全 + +Access Log 对 query、请求 JSON 和响应 JSON 复用大小写不敏感的递归脱敏规则，并在脱敏后执行 50KB 截断。登录、支付、企微回调、文件和导出路由只记录白名单安全摘要；JSON、XML、表单、multipart、二进制和无法解析载荷均不得回退记录原文。 + +## 异常闭环与运行监控 + +发布和运行监控至少包含 Outbox 待投递量、最老待投递年龄、处理中数量、过期租约、成功率、重试分布、最终失败和按事件类型积压。日志和指标只使用事件 ID、关联 ID、安全资源标识、错误码和计数，禁止把完整载荷或敏感值作为标签。 + +人工恢复只接受显式选中的失败或过期租约事件。重放保留原事件身份与内容，记录操作者、中文原因和恢复批次；审计写入失败时整个恢复事务回滚。持续积压、租约大量过期、配置读写不一致、脱敏回归失败或关键任务无法恢复时停止放量并保持事实不变。 + +## 发布与回滚 + +发布顺序固定如下： + +1. 执行只读前置检查，部署 165/166 迁移，再执行后置检查。 +2. 部署兼容 API。 +3. 部署 Relay/Worker 并确认指标、日志和告警可用。 +4. 部署依赖消费者并验证消费者幂等和可观察结果。 +5. 最后允许前端和业务生产者放量；消费者和监控就绪前禁止制造新积压。 + +新增表为空时可以执行 down 迁移。已有 Outbox 或配置事实后必须停止生产者和 Relay、保留事实、回滚无数据风险的应用版本并向前修复，禁止删表或清空数据降级。详细命令、停止条件和测试隔离方式见[迁移发布与数据安全回滚](迁移发布与数据安全回滚.md)。 + +## 前端跨仓契约 + +首次加载显示占位，成功且真实无数据时才显示空态，筛选无结果时提供清除入口。403 显示无权限且不重试；瞬时错误保留已有数据和输入并提供显式重试。创建成功后保存任务 ID，刷新或重新进入页面后恢复查询，不得重新创建任务。 + +待处理或处理中按 2 秒、3 秒、5 秒退避轮询，最长间隔 10 秒；终态停止。页面隐藏时暂停，恢复可见后立即刷新一次。字段和状态语义见[统一异步任务与前端轮询契约](统一异步任务与前端轮询契约.md)。 + +## 下游接入清单 + +- 生产者：在 Application 的既有 GORM 事务中调用 Outbox Repository，保存稳定事件类型、版本、事件 ID、业务键和关联标识。 +- 消费者：在 Worker 组合根注册稳定事件类型，实现重复投递无副作用的 `EventConsumer`，并提供业务结果的可观察断言。 +- 配置所有者：注册本模块拥有的配置定义；不得复制公共表 DDL，也不得由公共基础猜测业务 Key。 +- 异步任务所有者：保留业务任务表和失败明细，把公共五态投影到 API，并采用租约或等价 PostgreSQL 恢复事实。 +- 发布负责人：运行前后置门禁，确认 Relay、消费者、监控和前端依次就绪后再放量。 + +## 待决策项 + +- Audit Event 公共实现完成后，需要在系统配置更新和 Outbox 人工恢复的组合根注入正式审计 Adapter。 +- 各下游 PRD 需要分别确认事件类型、载荷版本、消费者幂等键、业务失败明细和通知策略；公共基础不预先注册这些内容。 +- 生产阈值需结合容量基线确定待投递年龄、积压量、过期租约比例和成功率告警值；当前公共 Query 提供指标与阈值计算接缝，不固化业务容量数字。 + +只有真实 PostgreSQL、Redis、Asynq、Fiber 接缝测试、全量 Go 测试和累计差异评审全部通过后，才可把本基础标记为可供下游接入。 diff --git a/docs/tech-public-foundation/数据库对象所有权与发布门禁.md b/docs/tech-public-foundation/数据库对象所有权与发布门禁.md new file mode 100644 index 0000000..f449154 --- /dev/null +++ b/docs/tech-public-foundation/数据库对象所有权与发布门禁.md @@ -0,0 +1,22 @@ +# 公共数据库对象所有权与发布门禁 + +## 对象所有权 + +| 数据库对象 | 唯一迁移所有者 | 下游接入方式 | +|---|---|---| +| `tb_outbox_event` 及公共索引、约束 | `tech-public-foundation` | 依赖公共 Outbox Port，禁止复制 DDL | +| `tb_system_config` 及公共索引、约束 | `tech-public-foundation` | 注册业务 Key，禁止复制 DDL 或创建任意 Key | +| Audit Event、Integration Log | `tech-global-audit` | 公共基础只依赖审计 Port | +| 通知、业务任务、业务失败明细 | 对应业务 PRD | 复用公共契约，保留自己的业务表 | + +## 门禁执行 + +迁移和放量前运行前置检查，迁移后运行后置检查。检查只读取 PostgreSQL，重复执行不会写入业务事实；输出仅包含稳定错误码、对象名和计数，不输出事件载荷或配置值。 + +前置检查阻断以下情况：依赖迁移版本不足或处于脏状态、已有对象定义冲突、重复事件 ID 或配置 Key、必填字段空值、非法状态/类型、未投递事件和过期租约。后置检查还要求公共表及约定字段已经存在，并执行约束、索引和读写冒烟验证。 + +## 发布与回滚边界 + +发布顺序固定为：迁移与检查、兼容 API、Relay/Worker、依赖消费者、前端。迁移异常、Outbox 持续积压或租约大量过期、配置读写不一致、访问日志脱敏回归失败、关键任务无法恢复时停止放量。 + +尚未产生事实的新结构可在验证后回滚。公共表已经保存 Outbox 或配置事实后，必须先停止生产者和 Relay，保留事实并向前修复；禁止通过降级删表清理。 diff --git a/docs/tech-public-foundation/统一异步任务与前端轮询契约.md b/docs/tech-public-foundation/统一异步任务与前端轮询契约.md new file mode 100644 index 0000000..be38c77 --- /dev/null +++ b/docs/tech-public-foundation/统一异步任务与前端轮询契约.md @@ -0,0 +1,13 @@ +# 统一异步任务与前端轮询契约 + +业务任务固定使用五态：`1=待处理、2=处理中、3=已完成、4=已失败、5=已取消`，所有公开投影同时返回中文状态名。各业务继续拥有自己的任务表、任务项和失败明细；公共基础不建设万能任务表。 + +公开投影至少包含 `task_id`、状态与状态名、总数、成功数、失败数、进度、安全错误码与中文摘要、开始时间、完成时间和更新时间。业务项到达处理终点时使用“已完成”：部分成功和全部业务项失败均通过计数表达，且必须满足 `total_count = success_count + failed_count`；只有整体无法建立或执行到业务终点时才使用“已失败”。 + +领取必须使用 PostgreSQL 预期状态条件更新。待处理任务可以进入处理中；处理中任务保存租约所有者与到期时间，仅过期租约可被其他 Worker 恢复。续租和进入终态都校验当前状态、租约所有者与有效期。队列重复投递遇到终态时直接返回，不制造第二次业务副作用。取消仅适用于业务明确支持的任务，并从待处理或处理中条件更新到已取消。 + +调用统一 `EnqueueTask` 时载荷只包含 `task_id`、必要分片标识和关联标识，并以 struct 或 map 传入；禁止预序列化为 `[]byte`。 + +前端首次加载显示占位；成功且确实无数据时才显示空态，筛选无结果提供清除筛选入口。403 显示无权限且不提供重试。瞬时失败保留已有数据和输入并提供显式重试。创建成功后持久保存 `task_id`，刷新或重新进入页面后恢复查询。 + +待处理或处理中按 2 秒、3 秒、5 秒逐步退避，最长不超过 10 秒；终态停止轮询。页面隐藏时暂停，恢复可见后立即刷新一次，再按最新状态继续。网络恢复和页面刷新不得重新创建任务。 diff --git a/docs/tech-public-foundation/迁移发布与数据安全回滚.md b/docs/tech-public-foundation/迁移发布与数据安全回滚.md new file mode 100644 index 0000000..82e6d44 --- /dev/null +++ b/docs/tech-public-foundation/迁移发布与数据安全回滚.md @@ -0,0 +1,22 @@ +# 公共基础迁移发布与数据安全回滚 + +## 执行顺序 + +1. 使用 `./scripts/test-tech-public-foundation.sh` 完成真实依赖公共接缝验收，再使用 `go run ./cmd/foundation-check --phase pre` 执行只读前置检查。 +2. 依次执行 `000165_create_public_outbox`、`000166_create_system_config` 正向迁移。 +3. 使用 `go run ./cmd/foundation-check --phase post` 验证字段、约束、索引、异常计数和事务内读写冒烟。 +4. 部署兼容 API 和 Outbox Relay/Worker，确认监控就绪后再部署依赖消费者，最后允许前端或生产者放量。 + +门禁报告只输出稳定错误码、数据库对象安全标识和计数。依赖迁移版本不足或脏状态、对象定义冲突、重复身份、空值、非法枚举、未完成任务、未投递事件和过期租约均以非零状态阻断。 + +## 数据迁移与隔离 + +165/166 只创建新的公共表和索引，不回填历史业务数据，因此没有批次回填步骤。后续确需回填时必须按稳定主键分批记录进度，允许中断重跑并验证行数守恒；不得借本次迁移扫描或改写下游业务表。 + +自动化验收在独立 PostgreSQL schema 和测试事务内执行，只删除本次创建的 schema 或依赖事务回滚，不执行全库清理。Redis/Asynq 测试使用唯一事件 ID 和配置 Key，并只删除本次任务与缓存键。 + +## 回滚边界 + +新增表为空时，down 迁移允许删除结构。任一公共表已经保存事件或配置事实时，down 迁移主动失败；发布负责人必须停止新生产者、停止 Relay 领取、保留已有事实，回滚无数据风险的应用版本，并修复后向前恢复。禁止通过删表、清空 Outbox 或删除配置事实完成降级。 + +停止条件包括：迁移门禁异常、Outbox 持续积压或过期租约大量增加、配置读写不一致、Access Log 脱敏回归失败和关键任务无法恢复。 diff --git a/internal/application/outbox/recovery.go b/internal/application/outbox/recovery.go new file mode 100644 index 0000000..44e0e3e --- /dev/null +++ b/internal/application/outbox/recovery.go @@ -0,0 +1,154 @@ +// Package outbox 提供公共 Outbox 的受控人工恢复用例。 +package outbox + +import ( + "context" + stderrors "errors" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" + pkgerrors "github.com/break/junhong_cmp_fiber/pkg/errors" +) + +// Operator 表示人工恢复操作者的授权快照。 +type Operator struct { + ID uint + SuperAdmin bool + RequestID string + CorrelationID string +} + +// RecoveryAudit 是交给统一 Audit Port 的安全恢复事实。 +type RecoveryAudit struct { + OperatorID uint + OperationType string + Description string + EventIDs []string + Reason string + BatchID string + RequestID string + CorrelationID string +} + +// AuditWriter 是 tech-global-audit 提供实现的统一审计接缝。 +type AuditWriter interface { + WriteRecovery(ctx context.Context, tx *gorm.DB, audit RecoveryAudit) error +} + +// RecoveryService 执行选择性重放和过期租约释放。 +type RecoveryService struct { + db *gorm.DB + audit AuditWriter + now func() time.Time +} + +// NewRecoveryService 创建受控恢复用例；审计接缝不可缺失。 +func NewRecoveryService(db *gorm.DB, audit AuditWriter, now func() time.Time) (*RecoveryService, error) { + if db == nil || audit == nil { + return nil, stderrors.New("Outbox 恢复必须配置数据库和统一审计接缝") + } + if now == nil { + now = time.Now + } + return &RecoveryService{db: db, audit: audit, now: now}, nil +} + +// Replay 只重放明确选择的最终失败或租约过期事件，并保留原始内容和身份。 +func (s *RecoveryService) Replay(ctx context.Context, operator Operator, ids []uint, reason string) (string, error) { + if err := validateCommand(operator, ids, reason); err != nil { + return "", err + } + batchID := uuid.NewString() + now := s.now().UTC() + err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + events, err := loadSelectedForUpdate(tx, ids) + if err != nil { + return err + } + if len(events) != len(ids) { + return pkgerrors.New(pkgerrors.CodeInvalidStatus, "选择的事件不存在或状态不允许重放") + } + eventIDs := make([]string, 0, len(events)) + for _, event := range events { + allowed := event.Status == constants.OutboxStatusFailed || + (event.Status == constants.OutboxStatusDelivering && event.LeaseExpiresAt != nil && !event.LeaseExpiresAt.After(now)) + if !allowed { + return pkgerrors.New(pkgerrors.CodeInvalidStatus, "选择的事件不存在或状态不允许重放") + } + eventIDs = append(eventIDs, event.EventID) + } + result := tx.Model(&model.OutboxEvent{}).Where("id IN ?", ids).Updates(map[string]any{ + "status": constants.OutboxStatusPending, "next_attempt_at": now, + "lease_owner": nil, "lease_expires_at": nil, "updated_at": now, + }) + if result.Error != nil { + return result.Error + } + return s.audit.WriteRecovery(ctx, tx, RecoveryAudit{ + OperatorID: operator.ID, OperationType: "outbox_replay", Description: "人工重放 Outbox 事件", + EventIDs: eventIDs, Reason: reason, BatchID: batchID, + RequestID: operator.RequestID, CorrelationID: operator.CorrelationID, + }) + }) + return batchID, err +} + +// ReleaseExpiredLeases 只释放明确选择且已经过期的投递租约。 +func (s *RecoveryService) ReleaseExpiredLeases(ctx context.Context, operator Operator, ids []uint, reason string) (string, error) { + if err := validateCommand(operator, ids, reason); err != nil { + return "", err + } + batchID := uuid.NewString() + now := s.now().UTC() + err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + events, err := loadSelectedForUpdate(tx, ids) + if err != nil { + return err + } + if len(events) != len(ids) { + return pkgerrors.New(pkgerrors.CodeInvalidStatus, "选择的租约不存在或仍然有效") + } + eventIDs := make([]string, 0, len(events)) + for _, event := range events { + if event.Status != constants.OutboxStatusDelivering || event.LeaseExpiresAt == nil || event.LeaseExpiresAt.After(now) { + return pkgerrors.New(pkgerrors.CodeInvalidStatus, "选择的租约不存在或仍然有效") + } + eventIDs = append(eventIDs, event.EventID) + } + result := tx.Model(&model.OutboxEvent{}).Where("id IN ? AND status = ? AND lease_expires_at <= ?", ids, constants.OutboxStatusDelivering, now). + Updates(map[string]any{ + "status": constants.OutboxStatusPending, "next_attempt_at": now, + "lease_owner": nil, "lease_expires_at": nil, "updated_at": now, + }) + if result.Error != nil { + return result.Error + } + return s.audit.WriteRecovery(ctx, tx, RecoveryAudit{ + OperatorID: operator.ID, OperationType: "outbox_release_expired_lease", Description: "人工释放 Outbox 过期租约", + EventIDs: eventIDs, Reason: reason, BatchID: batchID, + RequestID: operator.RequestID, CorrelationID: operator.CorrelationID, + }) + }) + return batchID, err +} + +func validateCommand(operator Operator, ids []uint, reason string) error { + if !operator.SuperAdmin { + return pkgerrors.New(pkgerrors.CodeForbidden) + } + if operator.ID == 0 || len(ids) == 0 || reason == "" { + return pkgerrors.New(pkgerrors.CodeInvalidParam) + } + return nil +} + +func loadSelectedForUpdate(tx *gorm.DB, ids []uint) ([]model.OutboxEvent, error) { + var events []model.OutboxEvent + err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id IN ?", ids).Order("id ASC").Find(&events).Error + return events, err +} diff --git a/internal/application/outbox/recovery_integration_test.go b/internal/application/outbox/recovery_integration_test.go new file mode 100644 index 0000000..e8a213a --- /dev/null +++ b/internal/application/outbox/recovery_integration_test.go @@ -0,0 +1,129 @@ +package outbox_test + +import ( + "context" + stderrors "errors" + "testing" + "time" + + "gorm.io/gorm" + + outboxapp "github.com/break/junhong_cmp_fiber/internal/application/outbox" + infraoutbox "github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/testutil" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +type databaseAuditWriter struct { + fail bool +} + +func (w *databaseAuditWriter) WriteRecovery(_ context.Context, tx *gorm.DB, audit outboxapp.RecoveryAudit) error { + if w.fail { + return stderrors.New("测试审计不可用") + } + return tx.Exec(`INSERT INTO test_outbox_recovery_audit (batch_id, operation_type, event_count) + VALUES (?, ?, ?)`, audit.BatchID, audit.OperationType, len(audit.EventIDs)).Error +} + +func TestSelectiveReplayPreservesIdentityAndWritesAudit(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + testutil.CreateTemporaryOutboxTable(t, db) + createTemporaryRecoveryAuditTable(t, db) + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + repository := infraoutbox.NewRepository() + event, err := repository.Append(context.Background(), db, infraoutbox.Envelope{ + EventID: "recovery-stable", EventType: "example.recovery", AggregateType: "example", AggregateID: "1", + ResourceType: "example", ResourceID: "1", CorrelationID: "correlation-stable", + Payload: struct { + Secret string `json:"secret"` + }{Secret: "preserved-in-database-only"}, + }) + if err != nil { + t.Fatalf("准备恢复事件失败：%v", err) + } + if err := db.Model(&model.OutboxEvent{}).Where("id = ?", event.ID).Updates(map[string]any{ + "status": constants.OutboxStatusFailed, "retry_count": 10, "last_error_code": "FINAL", + }).Error; err != nil { + t.Fatalf("设置最终失败状态失败：%v", err) + } + var storedBefore model.OutboxEvent + if err := db.First(&storedBefore, event.ID).Error; err != nil { + t.Fatalf("读取重放前事件失败：%v", err) + } + + service, err := outboxapp.NewRecoveryService(db, &databaseAuditWriter{}, func() time.Time { return now }) + if err != nil { + t.Fatalf("创建恢复服务失败：%v", err) + } + batchID, err := service.Replay(context.Background(), outboxapp.Operator{ + ID: 7, SuperAdmin: true, RequestID: "request-recovery", CorrelationID: "correlation-recovery", + }, []uint{event.ID}, "确认消费者已具备幂等保护") + if err != nil || batchID == "" { + t.Fatalf("选择性重放失败：%v", err) + } + var recovered model.OutboxEvent + if err := db.First(&recovered, event.ID).Error; err != nil { + t.Fatalf("读取重放事件失败：%v", err) + } + if recovered.Status != constants.OutboxStatusPending || recovered.EventID != event.EventID || + recovered.CorrelationID != event.CorrelationID || string(recovered.Payload) != string(storedBefore.Payload) { + t.Fatalf("重放改变了事件身份或内容：%+v", recovered) + } + var auditCount int64 + if err := db.Table("test_outbox_recovery_audit").Where("batch_id = ?", batchID).Count(&auditCount).Error; err != nil || auditCount != 1 { + t.Fatalf("恢复审计事实缺失：%v，数量：%d", err, auditCount) + } +} + +func TestRecoveryRejectsActiveLeaseUnauthorizedOperatorAndAuditFailure(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + testutil.CreateTemporaryOutboxTable(t, db) + createTemporaryRecoveryAuditTable(t, db) + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + repository := infraoutbox.NewRepository() + event, err := repository.Append(context.Background(), db, infraoutbox.Envelope{ + EventID: "active-lease", EventType: "example.recovery", AggregateType: "example", AggregateID: "2", + ResourceType: "example", ResourceID: "2", Payload: struct { + Visible bool `json:"visible"` + }{Visible: true}, + }) + if err != nil { + t.Fatalf("准备有效租约事件失败：%v", err) + } + if err := db.Model(&model.OutboxEvent{}).Where("id = ?", event.ID).Updates(map[string]any{ + "status": constants.OutboxStatusDelivering, "lease_owner": "active-worker", "lease_expires_at": now.Add(time.Minute), + }).Error; err != nil { + t.Fatalf("设置有效租约失败：%v", err) + } + service, _ := outboxapp.NewRecoveryService(db, &databaseAuditWriter{}, func() time.Time { return now }) + if _, err := service.ReleaseExpiredLeases(context.Background(), outboxapp.Operator{ID: 7, SuperAdmin: true}, []uint{event.ID}, "测试释放"); err == nil { + t.Fatal("不得释放仍然有效的租约") + } + if _, err := service.Replay(context.Background(), outboxapp.Operator{ID: 8, SuperAdmin: false}, []uint{event.ID}, "越权测试"); err == nil { + t.Fatal("非超级管理员不得执行人工恢复") + } + + if err := db.Model(&model.OutboxEvent{}).Where("id = ?", event.ID).Update("status", constants.OutboxStatusFailed).Error; err != nil { + t.Fatalf("设置失败事件失败：%v", err) + } + failClosed, _ := outboxapp.NewRecoveryService(db, &databaseAuditWriter{fail: true}, func() time.Time { return now }) + if _, err := failClosed.Replay(context.Background(), outboxapp.Operator{ID: 7, SuperAdmin: true}, []uint{event.ID}, "审计失败测试"); err == nil { + t.Fatal("统一审计不可用时恢复事务必须失败") + } + var unchanged model.OutboxEvent + if err := db.First(&unchanged, event.ID).Error; err != nil || unchanged.Status != constants.OutboxStatusFailed { + t.Fatalf("审计失败后事件状态未回滚：%v，状态：%d", err, unchanged.Status) + } +} + +func createTemporaryRecoveryAuditTable(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TEMP TABLE test_outbox_recovery_audit ( + id bigserial PRIMARY KEY, batch_id varchar(64) NOT NULL, + operation_type varchar(100) NOT NULL, event_count integer NOT NULL + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建恢复审计测试表失败：%v", err) + } +} diff --git a/internal/application/systemconfig/update.go b/internal/application/systemconfig/update.go new file mode 100644 index 0000000..b84a8fb --- /dev/null +++ b/internal/application/systemconfig/update.go @@ -0,0 +1,178 @@ +// Package systemconfig 提供受控系统配置的简单写事务脚本。 +package systemconfig + +import ( + "context" + "crypto/sha256" + "encoding/hex" + stderrors "errors" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + configinfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/model/dto" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" + "github.com/break/junhong_cmp_fiber/pkg/middleware" +) + +// ChangeAudit 是系统配置更新交给统一审计 Port 的事实。 +type ChangeAudit struct { + OperatorID uint + OperationType string + Description string + ConfigKey string + BeforeData map[string]any + AfterData map[string]any + RequestID string + CorrelationID string +} + +// AuditWriter 由 tech-global-audit 提供事务内实现。 +type AuditWriter interface { + WriteConfigChange(ctx context.Context, tx *gorm.DB, audit ChangeAudit) error +} + +// UnavailableAuditWriter 是统一审计尚未注入时的失败关闭策略。 +type UnavailableAuditWriter struct{} + +// WriteConfigChange 拒绝在缺少统一审计时修改敏感配置。 +func (UnavailableAuditWriter) WriteConfigChange(_ context.Context, _ *gorm.DB, _ ChangeAudit) error { + return stderrors.New("统一审计接缝尚未配置") +} + +// UpdateService 执行单 Key 校验、事务更新、审计和提交后缓存失效。 +type UpdateService struct { + db *gorm.DB + registry *configinfra.Registry + cache configinfra.Cache + audit AuditWriter + alerts configinfra.AlertSink + now func() time.Time +} + +// NewUpdateService 创建系统配置更新事务脚本。 +func NewUpdateService( + db *gorm.DB, + registry *configinfra.Registry, + cache configinfra.Cache, + audit AuditWriter, + alerts configinfra.AlertSink, + now func() time.Time, +) *UpdateService { + if audit == nil { + audit = UnavailableAuditWriter{} + } + if now == nil { + now = time.Now + } + return &UpdateService{db: db, registry: registry, cache: cache, audit: audit, alerts: alerts, now: now} +} + +// Execute 更新一个已注册且非只读的配置 Key。 +func (s *UpdateService) Execute(ctx context.Context, key string, request dto.UpdateSystemConfigRequest) (*dto.SystemConfigItem, error) { + if middleware.GetUserTypeFromContext(ctx) != constants.UserTypeSuperAdmin { + return nil, errors.New(errors.CodeForbidden) + } + operatorID := middleware.GetUserIDFromContext(ctx) + if operatorID == 0 || key == "" { + return nil, errors.New(errors.CodeInvalidParam) + } + definition, registered := s.registry.Get(key) + if !registered { + return nil, errors.New(errors.CodeInvalidParam, "系统配置 Key 未注册") + } + if definition.Readonly { + return nil, errors.New(errors.CodeInvalidStatus, "系统配置为只读，不能更新") + } + if err := configinfra.ValidateValue(definition, request.Value); err != nil { + return nil, errors.New(errors.CodeInvalidParam, "系统配置值不符合注册规则") + } + now := s.now().UTC() + var saved model.SystemConfig + err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // 同一 Key 的首次创建和后续更新都由 PostgreSQL 事务级咨询锁串行裁决。 + if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtext(?))", key).Error; err != nil { + return err + } + var existing model.SystemConfig + findErr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("config_key = ?", key).First(&existing).Error + beforeValue := definition.DefaultValue + if findErr == nil { + beforeValue = existing.ConfigValue + } else if findErr != gorm.ErrRecordNotFound { + return findErr + } + if findErr == gorm.ErrRecordNotFound { + existing = model.SystemConfig{ + ConfigKey: key, ConfigValue: request.Value, ValueType: definition.ValueType, + Module: definition.Module, Description: definition.Description, + IsReadonly: definition.Readonly, IsSensitive: definition.Sensitive, + Creator: operatorID, Updater: operatorID, CreatedAt: now, UpdatedAt: now, + } + if err := tx.Create(&existing).Error; err != nil { + return err + } + } else { + if err := tx.Model(&model.SystemConfig{}).Where("id = ?", existing.ID).Updates(map[string]any{ + "config_value": request.Value, "value_type": definition.ValueType, + "module": definition.Module, "description": definition.Description, + "is_readonly": definition.Readonly, "is_sensitive": definition.Sensitive, + "updater": operatorID, "updated_at": now, + }).Error; err != nil { + return err + } + existing.ConfigValue = request.Value + existing.Updater = operatorID + existing.UpdatedAt = now + } + requestID := "" + if value := middleware.GetRequestIDFromContext(ctx); value != nil { + requestID = *value + } + if err := s.audit.WriteConfigChange(ctx, tx, ChangeAudit{ + OperatorID: operatorID, OperationType: "system_config_update", Description: "更新受控系统配置", + ConfigKey: key, + BeforeData: map[string]any{"config_key": key, "value": auditValue(definition, beforeValue)}, + AfterData: map[string]any{"config_key": key, "value": auditValue(definition, request.Value)}, + RequestID: requestID, CorrelationID: requestID, + }); err != nil { + return err + } + saved = existing + return nil + }) + if err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "更新系统配置失败") + } + s.registry.Remember(key, request.Value) + if s.cache != nil { + if err := s.cache.Delete(ctx, constants.RedisSystemConfigKey(key)); err != nil { + if s.alerts != nil { + s.alerts.Warn(ctx, "SYSTEM_CONFIG_CACHE_INVALIDATE_FAILED", "system_config", key, "系统配置缓存失效失败，数据库事实已提交") + } + } + } + value := saved.ConfigValue + if definition.Sensitive && value != "" { + value = "[已配置]" + } + updatedAt := saved.UpdatedAt + return &dto.SystemConfigItem{ + ConfigKey: key, Value: value, ValueType: definition.ValueType, Module: definition.Module, + Description: definition.Description, Readonly: definition.Readonly, Sensitive: definition.Sensitive, + Registered: true, Control: definition.Control, EnumValues: definition.EnumValues, + Min: definition.Min, Max: definition.Max, UpdatedAt: &updatedAt, + }, nil +} + +func auditValue(definition configinfra.Definition, value string) string { + if !definition.Sensitive { + return value + } + sum := sha256.Sum256([]byte(value)) + return "[敏感值 sha256:" + hex.EncodeToString(sum[:8]) + "]" +} diff --git a/internal/application/systemconfig/update_integration_test.go b/internal/application/systemconfig/update_integration_test.go new file mode 100644 index 0000000..43a9b2c --- /dev/null +++ b/internal/application/systemconfig/update_integration_test.go @@ -0,0 +1,141 @@ +package systemconfig_test + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/bytedance/sonic" + "github.com/google/uuid" + "gorm.io/driver/postgres" + "gorm.io/gorm" + + systemconfigapp "github.com/break/junhong_cmp_fiber/internal/application/systemconfig" + systemconfiginfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/model/dto" + "github.com/break/junhong_cmp_fiber/pkg/config" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/middleware" +) + +type transactionAuditWriter struct{} + +func (transactionAuditWriter) WriteConfigChange(_ context.Context, tx *gorm.DB, audit systemconfigapp.ChangeAudit) error { + before, _ := sonic.Marshal(audit.BeforeData) + after, _ := sonic.Marshal(audit.AfterData) + return tx.Exec(`INSERT INTO test_system_config_audit + (config_key, operator_id, before_data, after_data) VALUES (?, ?, ?::jsonb, ?::jsonb)`, + audit.ConfigKey, audit.OperatorID, string(before), string(after)).Error +} + +func TestConcurrentFirstSystemConfigUpdatesAreSerializedByPostgres(t *testing.T) { + if os.Getenv("JUNHONG_DATABASE_HOST") == "" { + t.Skip("未加载 .env.local，跳过依赖真实 PostgreSQL 的并发集成测试") + } + cfg, err := config.Load() + if err != nil { + t.Fatalf("加载测试配置失败：%v", err) + } + schema := "test_system_config_" + strings.ReplaceAll(uuid.NewString(), "-", "") + adminDB := openSchemaDatabase(t, &cfg.Database, "public") + if err := adminDB.Exec("CREATE SCHEMA " + schema).Error; err != nil { + t.Fatalf("创建隔离 schema 失败：%v", err) + } + t.Cleanup(func() { + _ = adminDB.Exec("DROP SCHEMA " + schema + " CASCADE").Error + closeDatabase(adminDB) + }) + + firstDB := openSchemaDatabase(t, &cfg.Database, schema) + secondDB := openSchemaDatabase(t, &cfg.Database, schema) + t.Cleanup(func() { + closeDatabase(firstDB) + closeDatabase(secondDB) + }) + createConcurrentConfigTables(t, firstDB) + registry := systemconfiginfra.NewRegistry() + definition := systemconfiginfra.Definition{ + Key: "foundation.concurrent.value", Module: "foundation", ValueType: constants.SystemConfigTypeString, + DefaultValue: "default", Description: "并发配置测试值", + } + if err := registry.Register(definition); err != nil { + t.Fatalf("注册并发测试配置失败：%v", err) + } + ctx := middleware.SetUserContext(context.Background(), &middleware.UserContextInfo{UserID: 7, UserType: constants.UserTypeSuperAdmin}) + services := []*systemconfigapp.UpdateService{ + systemconfigapp.NewUpdateService(firstDB, registry, nil, transactionAuditWriter{}, nil, nil), + systemconfigapp.NewUpdateService(secondDB, registry, nil, transactionAuditWriter{}, nil, nil), + } + values := []string{"first", "second"} + start := make(chan struct{}) + errorsFound := make(chan error, len(services)) + var waitGroup sync.WaitGroup + for index := range services { + waitGroup.Add(1) + go func(index int) { + defer waitGroup.Done() + <-start + _, executeErr := services[index].Execute(ctx, definition.Key, dto.UpdateSystemConfigRequest{Value: values[index]}) + errorsFound <- executeErr + }(index) + } + close(start) + waitGroup.Wait() + close(errorsFound) + for executeErr := range errorsFound { + if executeErr != nil { + t.Fatalf("并发更新不应产生唯一键冲突：%v", executeErr) + } + } + var stored model.SystemConfig + if err := firstDB.Where("config_key = ?", definition.Key).First(&stored).Error; err != nil { + t.Fatalf("读取并发更新结果失败：%v", err) + } + if stored.ConfigValue != values[0] && stored.ConfigValue != values[1] { + t.Fatalf("并发更新保存了非法结果：%q", stored.ConfigValue) + } + var auditCount int64 + if err := firstDB.Table("test_system_config_audit").Where("config_key = ?", definition.Key).Count(&auditCount).Error; err != nil || auditCount != 2 { + t.Fatalf("并发更新审计事实不完整：错误=%v，数量=%d", err, auditCount) + } +} + +func openSchemaDatabase(t *testing.T, cfg *config.DatabaseConfig, schema string) *gorm.DB { + t.Helper() + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s search_path=%s TimeZone=Asia/Shanghai", + cfg.Host, cfg.Port, cfg.User, cfg.Password, cfg.DBName, cfg.SSLMode, schema) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{SkipDefaultTransaction: true}) + if err != nil { + t.Fatalf("连接隔离 PostgreSQL 失败：%v", err) + } + return db +} + +func closeDatabase(db *gorm.DB) { + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } +} + +func createConcurrentConfigTables(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TABLE tb_system_config ( + id bigserial PRIMARY KEY, config_key varchar(150) NOT NULL UNIQUE, config_value text NOT NULL, + value_type varchar(20) NOT NULL, module varchar(100) NOT NULL, description varchar(500) NOT NULL, + is_readonly boolean NOT NULL DEFAULT false, is_sensitive boolean NOT NULL DEFAULT false, + creator bigint NOT NULL DEFAULT 0, updater bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT NOW(), updated_at timestamptz NOT NULL DEFAULT NOW() + )`).Error; err != nil { + t.Fatalf("创建隔离配置表失败：%v", err) + } + if err := db.Exec(`CREATE TABLE test_system_config_audit ( + id bigserial PRIMARY KEY, config_key varchar(150) NOT NULL, operator_id bigint NOT NULL, + before_data jsonb NOT NULL, after_data jsonb NOT NULL + )`).Error; err != nil { + t.Fatalf("创建隔离审计表失败：%v", err) + } +} diff --git a/internal/bootstrap/dependencies.go b/internal/bootstrap/dependencies.go index 4644e03..9052073 100644 --- a/internal/bootstrap/dependencies.go +++ b/internal/bootstrap/dependencies.go @@ -1,7 +1,9 @@ package bootstrap import ( + systemConfigApp "github.com/break/junhong_cmp_fiber/internal/application/systemconfig" "github.com/break/junhong_cmp_fiber/internal/gateway" + systemConfigInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" "github.com/break/junhong_cmp_fiber/internal/service/verification" "github.com/break/junhong_cmp_fiber/pkg/auth" "github.com/break/junhong_cmp_fiber/pkg/queue" @@ -15,14 +17,16 @@ import ( // Dependencies 封装所有基础依赖 // 这些是应用启动时初始化的核心组件 type Dependencies struct { - DB *gorm.DB // PostgreSQL 数据库连接 - Redis *redis.Client // Redis 客户端 - Logger *zap.Logger // 应用日志器 - JWTManager *auth.JWTManager // JWT 管理器（个人客户认证） - TokenManager *auth.TokenManager // Token 管理器（后台和H5认证） - VerificationService *verification.Service // 验证码服务 - QueueClient *queue.Client // Asynq 任务队列客户端 - StorageService *storage.Service // 对象存储服务（可选，配置缺失时为 nil） - GatewayClient *gateway.Client // Gateway API 客户端（可选，配置缺失时为 nil） - WechatPayment wechat.PaymentServiceInterface // 微信支付服务（可选） + DB *gorm.DB // PostgreSQL 数据库连接 + Redis *redis.Client // Redis 客户端 + Logger *zap.Logger // 应用日志器 + JWTManager *auth.JWTManager // JWT 管理器（个人客户认证） + TokenManager *auth.TokenManager // Token 管理器（后台和H5认证） + VerificationService *verification.Service // 验证码服务 + QueueClient *queue.Client // Asynq 任务队列客户端 + StorageService *storage.Service // 对象存储服务（可选，配置缺失时为 nil） + GatewayClient *gateway.Client // Gateway API 客户端（可选，配置缺失时为 nil） + WechatPayment wechat.PaymentServiceInterface // 微信支付服务（可选） + SystemConfigRegistry *systemConfigInfra.Registry // 业务模块共享的受控配置注册表（可选） + SystemConfigAudit systemConfigApp.AuditWriter // 统一配置变更审计 Port（可选，缺失时更新失败关闭） } diff --git a/internal/bootstrap/handlers.go b/internal/bootstrap/handlers.go index a513ff9..099162f 100644 --- a/internal/bootstrap/handlers.go +++ b/internal/bootstrap/handlers.go @@ -1,15 +1,18 @@ package bootstrap import ( + systemConfigApp "github.com/break/junhong_cmp_fiber/internal/application/systemconfig" "github.com/break/junhong_cmp_fiber/internal/handler/admin" "github.com/break/junhong_cmp_fiber/internal/handler/app" authHandler "github.com/break/junhong_cmp_fiber/internal/handler/auth" "github.com/break/junhong_cmp_fiber/internal/handler/callback" openapiHandler "github.com/break/junhong_cmp_fiber/internal/handler/openapi" + systemConfigInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" pollingPkg "github.com/break/junhong_cmp_fiber/internal/polling" assetQuery "github.com/break/junhong_cmp_fiber/internal/query/asset" exchangeQuery "github.com/break/junhong_cmp_fiber/internal/query/exchange" packageExpiryQuery "github.com/break/junhong_cmp_fiber/internal/query/packageexpiry" + systemConfigQuery "github.com/break/junhong_cmp_fiber/internal/query/systemconfig" clientOrderSvc "github.com/break/junhong_cmp_fiber/internal/service/client_order" pollingSvcPkg "github.com/break/junhong_cmp_fiber/internal/service/polling" rechargeOrderSvc "github.com/break/junhong_cmp_fiber/internal/service/recharge_order" @@ -76,6 +79,20 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers { deps.Redis, deps.Logger, ) + systemConfigRegistry := deps.SystemConfigRegistry + if systemConfigRegistry == nil { + systemConfigRegistry = systemConfigInfra.NewRegistry() + } + systemConfigAlerts := systemConfigInfra.NewLogAlertSink(deps.Logger) + var systemConfigCache systemConfigInfra.Cache + if deps.Redis != nil { + systemConfigCache = systemConfigInfra.NewRedisCache(deps.Redis) + } + systemConfigReader := systemConfigInfra.NewReader(deps.DB, systemConfigRegistry, systemConfigCache, systemConfigAlerts) + systemConfigList := systemConfigQuery.NewListQuery(systemConfigReader) + systemConfigUpdate := systemConfigApp.NewUpdateService( + deps.DB, systemConfigRegistry, systemConfigCache, deps.SystemConfigAudit, systemConfigAlerts, nil, + ) return &Handlers{ Auth: authHandler.NewHandler(svc.Auth, validate), @@ -150,6 +167,7 @@ func initHandlers(svc *services, deps *Dependencies) *Handlers { OrderPackageInvalidate: admin.NewOrderPackageInvalidateHandler(svc.OrderPackageInvalidate), ClientWechat: app.NewClientWechatHandler(svc.WechatConfig, deps.Redis, deps.Logger), SuperAdmin: admin.NewSuperAdminHandler(svc.OperationPassword), + SystemConfig: admin.NewSystemConfigHandler(systemConfigList, systemConfigUpdate), AgentOpenAPI: openapiHandler.NewHandler(svc.AgentOpenAPI, validate), } } diff --git a/internal/bootstrap/types.go b/internal/bootstrap/types.go index 06cad8d..f66035b 100644 --- a/internal/bootstrap/types.go +++ b/internal/bootstrap/types.go @@ -66,6 +66,7 @@ type Handlers struct { OrderPackageInvalidate *admin.OrderPackageInvalidateHandler ClientWechat *app.ClientWechatHandler SuperAdmin *admin.SuperAdminHandler + SystemConfig *admin.SystemConfigHandler AgentOpenAPI *openapiHandler.Handler } diff --git a/internal/handler/admin/system_config.go b/internal/handler/admin/system_config.go new file mode 100644 index 0000000..9592d5e --- /dev/null +++ b/internal/handler/admin/system_config.go @@ -0,0 +1,54 @@ +package admin + +import ( + "github.com/gofiber/fiber/v2" + + configapp "github.com/break/junhong_cmp_fiber/internal/application/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/model/dto" + configquery "github.com/break/junhong_cmp_fiber/internal/query/systemconfig" + "github.com/break/junhong_cmp_fiber/pkg/errors" + "github.com/break/junhong_cmp_fiber/pkg/response" +) + +// SystemConfigHandler 提供受控系统配置查询和单 Key 更新接口。 +type SystemConfigHandler struct { + listQuery *configquery.ListQuery + updateService *configapp.UpdateService +} + +// NewSystemConfigHandler 创建系统配置 Handler。 +func NewSystemConfigHandler(listQuery *configquery.ListQuery, updateService *configapp.UpdateService) *SystemConfigHandler { + return &SystemConfigHandler{listQuery: listQuery, updateService: updateService} +} + +// List 查询受控系统配置列表。 +// GET /api/admin/system-configs +func (h *SystemConfigHandler) List(c *fiber.Ctx) error { + var request dto.SystemConfigListRequest + if err := c.QueryParser(&request); err != nil { + return errors.New(errors.CodeInvalidParam) + } + result, err := h.listQuery.Execute(c.UserContext(), request) + if err != nil { + return err + } + return response.Success(c, result) +} + +// Update 更新一个已注册且允许修改的系统配置。 +// PUT /api/admin/system-configs/:key +func (h *SystemConfigHandler) Update(c *fiber.Ctx) error { + key := c.Params("key") + if key == "" { + return errors.New(errors.CodeInvalidParam) + } + var request dto.UpdateSystemConfigRequest + if err := c.BodyParser(&request); err != nil { + return errors.New(errors.CodeInvalidParam) + } + result, err := h.updateService.Execute(c.UserContext(), key, request) + if err != nil { + return err + } + return response.Success(c, result) +} diff --git a/internal/infrastructure/asynctask/store.go b/internal/infrastructure/asynctask/store.go new file mode 100644 index 0000000..4bf0116 --- /dev/null +++ b/internal/infrastructure/asynctask/store.go @@ -0,0 +1,117 @@ +// Package asynctask 提供业务自有任务表复用的 PostgreSQL 条件更新 Adapter。 +package asynctask + +import ( + "context" + stderrors "errors" + "regexp" + "time" + + "gorm.io/gorm" + + contract "github.com/break/junhong_cmp_fiber/pkg/asynctask" +) + +var identifierPattern = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + +// Definition 描述业务自有任务表如何映射统一字段。 +type Definition struct { + Table string + IDColumn string + StatusColumn string + LeaseOwnerColumn string + LeaseExpiresColumn string + TotalColumn string + SuccessColumn string + FailedColumn string + ProgressColumn string + ErrorCodeColumn string + ErrorSummaryColumn string + StartedAtColumn string + CompletedAtColumn string + UpdatedAtColumn string +} + +// Store 对一个已注册的业务任务表执行统一条件更新。 +type Store struct { + db *gorm.DB + definition Definition +} + +// NewStore 创建业务任务表 Adapter，不创建或迁移任何万能任务表。 +func NewStore(db *gorm.DB, definition Definition) (*Store, error) { + columns := []string{ + definition.Table, definition.IDColumn, definition.StatusColumn, definition.LeaseOwnerColumn, + definition.LeaseExpiresColumn, definition.TotalColumn, definition.SuccessColumn, + definition.FailedColumn, definition.ProgressColumn, definition.ErrorCodeColumn, + definition.ErrorSummaryColumn, definition.StartedAtColumn, definition.CompletedAtColumn, definition.UpdatedAtColumn, + } + for _, identifier := range columns { + if !identifierPattern.MatchString(identifier) { + return nil, stderrors.New("异步任务表或字段标识不合法") + } + } + return &Store{db: db, definition: definition}, nil +} + +// Claim 使用预期状态和过期租约条件领取任务。 +func (s *Store) Claim(ctx context.Context, id any, owner string, now time.Time, duration time.Duration) (bool, error) { + if owner == "" || duration <= 0 { + return false, stderrors.New("任务租约所有者和时长不能为空") + } + d := s.definition + result := s.db.WithContext(ctx).Table(d.Table). + Where(d.IDColumn+" = ? AND ("+d.StatusColumn+" = ? OR ("+d.StatusColumn+" = ? AND "+d.LeaseExpiresColumn+" <= ?))", + id, contract.StatusPending, contract.StatusProcessing, now). + Updates(map[string]any{ + d.StatusColumn: contract.StatusProcessing, d.LeaseOwnerColumn: owner, + d.LeaseExpiresColumn: now.Add(duration), d.StartedAtColumn: gorm.Expr("COALESCE("+d.StartedAtColumn+", ?)", now), + d.UpdatedAtColumn: now, + }) + return result.RowsAffected == 1, result.Error +} + +// Renew 只允许当前所有者在租约有效时续期处理中任务。 +func (s *Store) Renew(ctx context.Context, id any, owner string, now time.Time, duration time.Duration) (bool, error) { + if owner == "" || duration <= 0 { + return false, stderrors.New("任务租约所有者和时长不能为空") + } + d := s.definition + updated := s.db.WithContext(ctx).Table(d.Table). + Where(d.IDColumn+" = ? AND "+d.StatusColumn+" = ? AND "+d.LeaseOwnerColumn+" = ? AND "+d.LeaseExpiresColumn+" > ?", + id, contract.StatusProcessing, owner, now). + Updates(map[string]any{d.LeaseExpiresColumn: now.Add(duration), d.UpdatedAtColumn: now}) + return updated.RowsAffected == 1, updated.Error +} + +// Finish 只允许有效租约所有者把处理中任务推进到统一终态。 +func (s *Store) Finish(ctx context.Context, id any, owner string, result contract.TerminalResult, now time.Time) (bool, error) { + projection, err := contract.NewTerminalProjection(result) + if err != nil { + return false, err + } + d := s.definition + updated := s.db.WithContext(ctx).Table(d.Table). + Where(d.IDColumn+" = ? AND "+d.StatusColumn+" = ? AND "+d.LeaseOwnerColumn+" = ? AND "+d.LeaseExpiresColumn+" > ?", + id, contract.StatusProcessing, owner, now). + Updates(map[string]any{ + d.StatusColumn: projection.Status, d.TotalColumn: projection.TotalCount, + d.SuccessColumn: projection.SuccessCount, d.FailedColumn: projection.FailedCount, + d.ProgressColumn: projection.Progress, d.ErrorCodeColumn: projection.ErrorCode, + d.ErrorSummaryColumn: projection.ErrorSummary, d.CompletedAtColumn: now, + d.LeaseOwnerColumn: nil, d.LeaseExpiresColumn: nil, d.UpdatedAtColumn: now, + }) + return updated.RowsAffected == 1, updated.Error +} + +// Cancel 只允许业务明确支持时从待处理或处理中进入已取消。 +func (s *Store) Cancel(ctx context.Context, id any, now time.Time) (bool, error) { + d := s.definition + updated := s.db.WithContext(ctx).Table(d.Table). + Where(d.IDColumn+" = ? AND "+d.StatusColumn+" IN ?", id, []int{contract.StatusPending, contract.StatusProcessing}). + Updates(map[string]any{ + d.StatusColumn: contract.StatusCancelled, d.ProgressColumn: 100, + d.CompletedAtColumn: now, d.LeaseOwnerColumn: nil, d.LeaseExpiresColumn: nil, d.UpdatedAtColumn: now, + }) + return updated.RowsAffected == 1, updated.Error +} diff --git a/internal/infrastructure/asynctask/store_integration_test.go b/internal/infrastructure/asynctask/store_integration_test.go new file mode 100644 index 0000000..ba02ca7 --- /dev/null +++ b/internal/infrastructure/asynctask/store_integration_test.go @@ -0,0 +1,130 @@ +package asynctask_test + +import ( + "context" + "testing" + "time" + + "gorm.io/gorm" + + storepkg "github.com/break/junhong_cmp_fiber/internal/infrastructure/asynctask" + "github.com/break/junhong_cmp_fiber/internal/testutil" + contract "github.com/break/junhong_cmp_fiber/pkg/asynctask" +) + +func TestPostgresTaskTransitionsAreConditionalAndRecoverExpiredLease(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTaskContractTable(t, db) + store, err := storepkg.NewStore(db, taskDefinition()) + if err != nil { + t.Fatalf("创建任务契约 Store 失败：%v", err) + } + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + if err := db.Exec("INSERT INTO test_async_contract_task (id, status, updated_at) VALUES (1, 1, ?)", now).Error; err != nil { + t.Fatalf("准备待处理任务失败：%v", err) + } + claimed, err := store.Claim(context.Background(), 1, "worker-a", now, time.Minute) + if err != nil || !claimed { + t.Fatalf("领取待处理任务失败：%v，领取：%v", err, claimed) + } + claimed, err = store.Claim(context.Background(), 1, "worker-b", now.Add(30*time.Second), time.Minute) + if err != nil || claimed { + t.Fatalf("不得抢占有效租约：%v，领取：%v", err, claimed) + } + claimed, err = store.Claim(context.Background(), 1, "worker-b", now.Add(2*time.Minute), time.Minute) + if err != nil || !claimed { + t.Fatalf("过期任务应由新 Worker 恢复：%v，领取：%v", err, claimed) + } + renewed, err := store.Renew(context.Background(), 1, "worker-a", now.Add(150*time.Second), 2*time.Minute) + if err != nil || renewed { + t.Fatalf("旧租约所有者不得续租：%v，续租：%v", err, renewed) + } + renewed, err = store.Renew(context.Background(), 1, "worker-b", now.Add(150*time.Second), 2*time.Minute) + if err != nil || !renewed { + t.Fatalf("有效租约所有者续租失败：%v，续租：%v", err, renewed) + } + claimed, err = store.Claim(context.Background(), 1, "worker-c", now.Add(3*time.Minute), time.Minute) + if err != nil || claimed { + t.Fatalf("续租后不得被其他 Worker 领取：%v，领取：%v", err, claimed) + } + finished, err := store.Finish(context.Background(), 1, "worker-a", contract.TerminalResult{ + TaskID: "1", Status: contract.StatusCompleted, TotalCount: 10, SuccessCount: 7, FailedCount: 3, UpdatedAt: now, + }, now.Add(3*time.Minute)) + if err != nil || finished { + t.Fatalf("旧租约所有者不得完成任务：%v，完成：%v", err, finished) + } + finished, err = store.Finish(context.Background(), 1, "worker-b", contract.TerminalResult{ + TaskID: "1", Status: contract.StatusCompleted, TotalCount: 10, SuccessCount: 7, FailedCount: 3, UpdatedAt: now, + }, now.Add(3*time.Minute)) + if err != nil || !finished { + t.Fatalf("当前租约所有者完成任务失败：%v，完成：%v", err, finished) + } + claimed, err = store.Claim(context.Background(), 1, "worker-c", now.Add(4*time.Minute), time.Minute) + if err != nil || claimed { + t.Fatalf("终态重复消费必须无副作用：%v，领取：%v", err, claimed) + } + cancelled, err := store.Cancel(context.Background(), 1, now.Add(4*time.Minute)) + if err != nil || cancelled { + t.Fatalf("终态任务不得再次取消：%v，取消：%v", err, cancelled) + } + + var row struct { + Status int + Total int + Success int + Failed int + } + if err := db.Table("test_async_contract_task").Select("status, total_count AS total, success_count AS success, failed_count AS failed").Where("id = 1").Scan(&row).Error; err != nil { + t.Fatalf("读取任务终态失败：%v", err) + } + if row.Status != contract.StatusCompleted || row.Total != 10 || row.Success != 7 || row.Failed != 3 { + t.Fatalf("任务终态计数错误：%+v", row) + } +} + +func TestPostgresTaskContractSupportsWholeFailureAndCancellation(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTaskContractTable(t, db) + store, _ := storepkg.NewStore(db, taskDefinition()) + now := time.Now().UTC() + if err := db.Exec("INSERT INTO test_async_contract_task (id, status, updated_at) VALUES (2, 1, ?), (3, 1, ?)", now, now).Error; err != nil { + t.Fatalf("准备任务失败：%v", err) + } + claimed, _ := store.Claim(context.Background(), 2, "worker", now, time.Minute) + if !claimed { + t.Fatal("整体失败任务领取失败") + } + finished, err := store.Finish(context.Background(), 2, "worker", contract.TerminalResult{ + TaskID: "2", Status: contract.StatusFailed, ErrorCode: "FILE_PARSE_FAILED", ErrorSummary: "文件无法解析", UpdatedAt: now, + }, now) + if err != nil || !finished { + t.Fatalf("整体失败终态更新失败：%v", err) + } + cancelled, err := store.Cancel(context.Background(), 3, now) + if err != nil || !cancelled { + t.Fatalf("待处理任务取消失败：%v", err) + } +} + +func taskDefinition() storepkg.Definition { + return storepkg.Definition{ + Table: "test_async_contract_task", IDColumn: "id", StatusColumn: "status", + LeaseOwnerColumn: "lease_owner", LeaseExpiresColumn: "lease_expires_at", + TotalColumn: "total_count", SuccessColumn: "success_count", FailedColumn: "failed_count", + ProgressColumn: "progress", ErrorCodeColumn: "error_code", ErrorSummaryColumn: "error_summary", + StartedAtColumn: "started_at", CompletedAtColumn: "completed_at", UpdatedAtColumn: "updated_at", + } +} + +func createTaskContractTable(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TEMP TABLE test_async_contract_task ( + id bigint PRIMARY KEY, status integer NOT NULL, total_count integer NOT NULL DEFAULT 0, + success_count integer NOT NULL DEFAULT 0, failed_count integer NOT NULL DEFAULT 0, + progress integer NOT NULL DEFAULT 0, error_code varchar(100) NOT NULL DEFAULT '', + error_summary varchar(500) NOT NULL DEFAULT '', lease_owner varchar(100), lease_expires_at timestamptz, + started_at timestamptz, completed_at timestamptz, updated_at timestamptz NOT NULL + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建任务契约测试表失败：%v", err) + } +} diff --git a/internal/infrastructure/messaging/outbox/relay.go b/internal/infrastructure/messaging/outbox/relay.go new file mode 100644 index 0000000..a83a6da --- /dev/null +++ b/internal/infrastructure/messaging/outbox/relay.go @@ -0,0 +1,326 @@ +package outbox + +import ( + "context" + stderrors "errors" + "math" + "sync" + "time" + + "github.com/bytedance/sonic" + "github.com/hibiken/asynq" + "go.uber.org/zap" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/queue" +) + +// DeliveryEnvelope 是 Relay 原样传播到 Asynq 的公共结构化信封。 +type DeliveryEnvelope struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + PayloadVersion int `json:"payload_version"` + AggregateType string `json:"aggregate_type"` + AggregateID string `json:"aggregate_id"` + ResourceType string `json:"resource_type"` + ResourceID string `json:"resource_id"` + BusinessKey string `json:"business_key,omitempty"` + RequestID string `json:"request_id,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + Payload sonic.NoCopyRawMessage `json:"payload"` +} + +// Publisher 是 Relay 的队列边界。 +type Publisher interface { + Publish(ctx context.Context, envelope DeliveryEnvelope) error +} + +// PermanentError 表示重试无法修复的投递错误。 +type PermanentError struct { + err error +} + +// Error 返回安全的错误文本，仅供内部日志和错误链判断使用。 +func (e *PermanentError) Error() string { + return e.err.Error() +} + +// Unwrap 返回原始错误。 +func (e *PermanentError) Unwrap() error { + return e.err +} + +// Permanent 将不可恢复错误标记为永久失败，Relay 会直接保留最终失败事实。 +func Permanent(err error) error { + if err == nil { + return nil + } + return &PermanentError{err: err} +} + +// QueuePublisher 使用项目统一队列客户端发布结构化信封。 +type QueuePublisher struct { + client *queue.Client +} + +// NewQueuePublisher 创建项目统一队列客户端 Adapter。 +func NewQueuePublisher(client *queue.Client) *QueuePublisher { + return &QueuePublisher{client: client} +} + +// Publish 将公共信封作为 struct 入队，禁止调用方预序列化。 +func (p *QueuePublisher) Publish(ctx context.Context, envelope DeliveryEnvelope) error { + return p.client.EnqueueTask(ctx, constants.TaskTypeOutboxDeliver, envelope) +} + +// RelayOptions 控制单个 Relay 实例的领取与重试行为。 +type RelayOptions struct { + Owner string + BatchSize int + LeaseDuration time.Duration + Now func() time.Time +} + +// Relay 完成公共 Outbox 的租约领取和至少一次队列投递。 +type Relay struct { + db *gorm.DB + publisher Publisher + logger *zap.Logger + options RelayOptions +} + +// NewRelay 创建公共 Outbox Relay。 +func NewRelay(db *gorm.DB, publisher Publisher, logger *zap.Logger, options RelayOptions) (*Relay, error) { + if db == nil || publisher == nil || options.Owner == "" { + return nil, stderrors.New("Outbox Relay 依赖和租约所有者不能为空") + } + if options.BatchSize <= 0 { + options.BatchSize = constants.OutboxDefaultBatchSize + } + if options.LeaseDuration <= 0 { + options.LeaseDuration = constants.OutboxDefaultLeaseDuration + } + if options.Now == nil { + options.Now = time.Now + } + if logger == nil { + logger = zap.NewNop() + } + return &Relay{db: db, publisher: publisher, logger: logger, options: options}, nil +} + +// ProcessBatch 领取并投递一批到期事件。 +func (r *Relay) ProcessBatch(ctx context.Context) (int, error) { + events, err := r.ClaimBatch(ctx) + if err != nil { + return 0, err + } + processed := 0 + for _, event := range events { + if err := r.publisher.Publish(ctx, deliveryEnvelope(event)); err != nil { + var permanentError *PermanentError + permanent := stderrors.As(err, &permanentError) + code := "OUTBOX_ENQUEUE_FAILED" + summary := "队列暂时不可用" + if permanent { + code = "OUTBOX_PERMANENT_FAILURE" + summary = "事件无法投递，已停止自动重试" + } + if failErr := r.markFailed(ctx, event, code, summary, permanent); failErr != nil { + return processed, failErr + } + r.logger.Warn("Outbox 事件投递失败", + zap.String("event_id", event.EventID), zap.String("correlation_id", event.CorrelationID), + zap.String("error_code", code), zap.Bool("permanent", permanent)) + continue + } + if err := r.MarkDelivered(ctx, event.ID); err != nil { + // 入队成功但标记失败时保留租约，过期后会使用同一 event_id 再次投递。 + return processed, err + } + processed++ + } + return processed, nil +} + +// ClaimBatch 通过行锁跳过竞争行，并领取待投递或租约过期事件。 +func (r *Relay) ClaimBatch(ctx context.Context) ([]model.OutboxEvent, error) { + now := r.options.Now().UTC() + expiresAt := now.Add(r.options.LeaseDuration) + claimed := make([]model.OutboxEvent, 0, r.options.BatchSize) + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var candidates []model.OutboxEvent + if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}). + Where("(status = ? AND next_attempt_at <= ?) OR (status = ? AND lease_expires_at <= ?)", + constants.OutboxStatusPending, now, constants.OutboxStatusDelivering, now). + Order("created_at ASC, id ASC").Limit(r.options.BatchSize).Find(&candidates).Error; err != nil { + return err + } + for index := range candidates { + result := tx.Model(&model.OutboxEvent{}). + Where("id = ? AND ((status = ? AND next_attempt_at <= ?) OR (status = ? AND lease_expires_at <= ?))", + candidates[index].ID, constants.OutboxStatusPending, now, constants.OutboxStatusDelivering, now). + Updates(map[string]any{ + "status": constants.OutboxStatusDelivering, "lease_owner": r.options.Owner, + "lease_expires_at": expiresAt, "updated_at": now, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 1 { + candidates[index].Status = constants.OutboxStatusDelivering + candidates[index].LeaseOwner = &r.options.Owner + candidates[index].LeaseExpiresAt = &expiresAt + claimed = append(claimed, candidates[index]) + } + } + return nil + }) + return claimed, err +} + +// RenewLease 仅允许当前租约所有者续租投递中的事件。 +func (r *Relay) RenewLease(ctx context.Context, eventID uint) (bool, error) { + now := r.options.Now().UTC() + result := r.db.WithContext(ctx).Model(&model.OutboxEvent{}). + Where("id = ? AND status = ? AND lease_owner = ? AND lease_expires_at > ?", + eventID, constants.OutboxStatusDelivering, r.options.Owner, now). + Updates(map[string]any{"lease_expires_at": now.Add(r.options.LeaseDuration), "updated_at": now}) + return result.RowsAffected == 1, result.Error +} + +// MarkDelivered 仅允许当前租约所有者把事件标记为已投递。 +func (r *Relay) MarkDelivered(ctx context.Context, eventID uint) error { + now := r.options.Now().UTC() + result := r.db.WithContext(ctx).Model(&model.OutboxEvent{}). + Where("id = ? AND status = ? AND lease_owner = ?", eventID, constants.OutboxStatusDelivering, r.options.Owner). + Updates(map[string]any{ + "status": constants.OutboxStatusDelivered, "delivered_at": now, + "lease_owner": nil, "lease_expires_at": nil, "last_error_code": "", + "last_error_summary": "", "updated_at": now, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return stderrors.New("Outbox 投递完成条件不满足") + } + return nil +} + +// MarkFailed 记录安全错误并退避；达到上限后保留最终失败事实。 +func (r *Relay) MarkFailed(ctx context.Context, event model.OutboxEvent, code, summary string) error { + return r.markFailed(ctx, event, code, summary, false) +} + +func (r *Relay) markFailed(ctx context.Context, event model.OutboxEvent, code, summary string, permanent bool) error { + now := r.options.Now().UTC() + retryCount := event.RetryCount + 1 + status := constants.OutboxStatusPending + nextAttemptAt := now.Add(backoff(retryCount)) + if permanent || retryCount >= event.MaxRetries { + status = constants.OutboxStatusFailed + nextAttemptAt = now + r.logger.Error("Outbox 事件停止自动重试", + zap.String("event_id", event.EventID), zap.String("correlation_id", event.CorrelationID), + zap.String("error_code", code), zap.Bool("permanent", permanent)) + } + result := r.db.WithContext(ctx).Model(&model.OutboxEvent{}). + Where("id = ? AND status = ? AND lease_owner = ?", event.ID, constants.OutboxStatusDelivering, r.options.Owner). + Updates(map[string]any{ + "status": status, "retry_count": retryCount, "next_attempt_at": nextAttemptAt, + "lease_owner": nil, "lease_expires_at": nil, "last_error_code": code, + "last_error_summary": summary, "updated_at": now, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return stderrors.New("Outbox 失败更新条件不满足") + } + return nil +} + +func deliveryEnvelope(event model.OutboxEvent) DeliveryEnvelope { + return DeliveryEnvelope{ + EventID: event.EventID, EventType: event.EventType, PayloadVersion: event.PayloadVersion, + AggregateType: event.AggregateType, AggregateID: event.AggregateID, + ResourceType: event.ResourceType, ResourceID: event.ResourceID, BusinessKey: event.BusinessKey, + RequestID: event.RequestID, CorrelationID: event.CorrelationID, + Payload: sonic.NoCopyRawMessage(event.Payload), + } +} + +func backoff(retryCount int) time.Duration { + delay := float64(constants.OutboxBaseRetryDelay) * math.Pow(2, float64(retryCount-1)) + if delay > float64(constants.OutboxMaxRetryDelay) { + return constants.OutboxMaxRetryDelay + } + return time.Duration(delay) +} + +// EventConsumer 是业务消费者公开实现的事件处理边界。 +type EventConsumer interface { + Consume(ctx context.Context, envelope DeliveryEnvelope) error +} + +// ConsumerRegistry 按稳定事件类型分发到业务消费者。 +type ConsumerRegistry struct { + mu sync.RWMutex + consumers map[string]EventConsumer +} + +// NewConsumerRegistry 创建空消费者注册表。 +func NewConsumerRegistry() *ConsumerRegistry { + return &ConsumerRegistry{consumers: map[string]EventConsumer{}} +} + +// Register 注册一个由业务 PRD 拥有的事件消费者。 +func (r *ConsumerRegistry) Register(eventType string, consumer EventConsumer) error { + if eventType == "" || consumer == nil { + return stderrors.New("Outbox 消费者注册信息不完整") + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.consumers[eventType]; exists { + return stderrors.New("Outbox 事件类型重复注册") + } + r.consumers[eventType] = consumer + return nil +} + +// Consume 将公共信封交给对应业务消费者；公共层不实现业务副作用。 +func (r *ConsumerRegistry) Consume(ctx context.Context, envelope DeliveryEnvelope) error { + r.mu.RLock() + consumer := r.consumers[envelope.EventType] + r.mu.RUnlock() + if consumer == nil { + return stderrors.New("Outbox 事件消费者尚未注册") + } + return consumer.Consume(ctx, envelope) +} + +// Handler 是公共 Outbox Asynq 任务 Handler。 +type Handler struct { + consumer EventConsumer +} + +// NewHandler 创建公共 Outbox Asynq Handler。 +func NewHandler(consumer EventConsumer) *Handler { + return &Handler{consumer: consumer} +} + +// Handle 解析结构化信封并调用公开消费者边界。 +func (h *Handler) Handle(ctx context.Context, task *asynq.Task) error { + var envelope DeliveryEnvelope + if err := sonic.Unmarshal(task.Payload(), &envelope); err != nil { + return err + } + if envelope.EventID == "" || envelope.EventType == "" || envelope.PayloadVersion <= 0 { + return stderrors.New("Outbox 事件信封不完整") + } + return h.consumer.Consume(ctx, envelope) +} diff --git a/internal/infrastructure/messaging/outbox/relay_integration_test.go b/internal/infrastructure/messaging/outbox/relay_integration_test.go new file mode 100644 index 0000000..d91821b --- /dev/null +++ b/internal/infrastructure/messaging/outbox/relay_integration_test.go @@ -0,0 +1,253 @@ +package outbox_test + +import ( + "context" + stderrors "errors" + "sync" + "testing" + "time" + + "github.com/bytedance/sonic" + "github.com/hibiken/asynq" + "go.uber.org/zap" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/testutil" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/queue" +) + +type recordingPublisher struct { + mu sync.Mutex + envelopes []outbox.DeliveryEnvelope + err error +} + +func (p *recordingPublisher) Publish(_ context.Context, envelope outbox.DeliveryEnvelope) error { + p.mu.Lock() + defer p.mu.Unlock() + p.envelopes = append(p.envelopes, envelope) + return p.err +} + +func TestExpiredLeaseRedeliversOriginalEnvelope(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTemporaryOutboxTables(t, db) + repository := outbox.NewRepository() + event, err := repository.Append(context.Background(), db, newEnvelope("event-stable", "business-stable")) + if err != nil { + t.Fatalf("准备 Outbox 事件失败：%v", err) + } + + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + firstPublisher := &recordingPublisher{} + first, err := outbox.NewRelay(db, firstPublisher, zap.NewNop(), outbox.RelayOptions{ + Owner: "relay-a", BatchSize: 1, LeaseDuration: time.Second, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("创建首个 Relay 失败：%v", err) + } + claimed, err := first.ClaimBatch(context.Background()) + if err != nil || len(claimed) != 1 { + t.Fatalf("首个 Relay 领取失败：%v，数量：%d", err, len(claimed)) + } + // 模拟入队成功但数据库标记前崩溃：公开队列信封已经可观察，但租约没有完成。 + if err := firstPublisher.Publish(context.Background(), deliveryFromModel(claimed[0])); err != nil { + t.Fatalf("模拟首次入队失败：%v", err) + } + + now = now.Add(2 * time.Second) + secondPublisher := &recordingPublisher{} + second, err := outbox.NewRelay(db, secondPublisher, zap.NewNop(), outbox.RelayOptions{ + Owner: "relay-b", BatchSize: 1, LeaseDuration: time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("创建恢复 Relay 失败：%v", err) + } + processed, err := second.ProcessBatch(context.Background()) + if err != nil || processed != 1 { + t.Fatalf("恢复投递失败：%v，数量：%d", err, processed) + } + if len(firstPublisher.envelopes) != 1 || len(secondPublisher.envelopes) != 1 { + t.Fatalf("至少一次投递次数不正确：%d/%d", len(firstPublisher.envelopes), len(secondPublisher.envelopes)) + } + firstEnvelope := firstPublisher.envelopes[0] + secondEnvelope := secondPublisher.envelopes[0] + if firstEnvelope.EventID != event.EventID || secondEnvelope.EventID != event.EventID || + firstEnvelope.CorrelationID != secondEnvelope.CorrelationID || string(firstEnvelope.Payload) != string(secondEnvelope.Payload) { + t.Fatalf("恢复投递改变了事件身份或载荷：%+v / %+v", firstEnvelope, secondEnvelope) + } +} + +func TestRelayFailureBackoffAndFinalFailure(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTemporaryOutboxTables(t, db) + repository := outbox.NewRepository() + event, err := repository.Append(context.Background(), db, newEnvelope("event-retry", "business-retry")) + if err != nil { + t.Fatalf("准备 Outbox 事件失败：%v", err) + } + if err := db.Model(&model.OutboxEvent{}).Where("id = ?", event.ID).Update("max_retries", 2).Error; err != nil { + t.Fatalf("设置最大重试次数失败：%v", err) + } + + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + publisher := &recordingPublisher{err: stderrors.New("测试队列失败")} + relay, err := outbox.NewRelay(db, publisher, zap.NewNop(), outbox.RelayOptions{ + Owner: "relay-retry", BatchSize: 1, LeaseDuration: time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("创建 Relay 失败：%v", err) + } + if _, err := relay.ProcessBatch(context.Background()); err != nil { + t.Fatalf("记录首次失败失败：%v", err) + } + var afterFirst model.OutboxEvent + if err := db.First(&afterFirst, event.ID).Error; err != nil { + t.Fatalf("读取首次失败事实失败：%v", err) + } + if afterFirst.Status != constants.OutboxStatusPending || afterFirst.RetryCount != 1 || !afterFirst.NextAttemptAt.After(now) { + t.Fatalf("首次失败未按退避重试：%+v", afterFirst) + } + + now = afterFirst.NextAttemptAt + if _, err := relay.ProcessBatch(context.Background()); err != nil { + t.Fatalf("记录最终失败失败：%v", err) + } + var final model.OutboxEvent + if err := db.First(&final, event.ID).Error; err != nil { + t.Fatalf("读取最终失败事实失败：%v", err) + } + if final.Status != constants.OutboxStatusFailed || final.RetryCount != 2 || final.LastErrorSummary != "队列暂时不可用" { + t.Fatalf("最终失败事实不正确：%+v", final) + } +} + +func TestRelayPermanentFailureStopsRetryImmediately(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTemporaryOutboxTables(t, db) + repository := outbox.NewRepository() + event, err := repository.Append(context.Background(), db, newEnvelope("event-permanent", "business-permanent")) + if err != nil { + t.Fatalf("准备永久失败事件失败：%v", err) + } + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + publisher := &recordingPublisher{err: outbox.Permanent(stderrors.New("载荷版本不受支持"))} + relay, err := outbox.NewRelay(db, publisher, zap.NewNop(), outbox.RelayOptions{ + Owner: "relay-permanent", BatchSize: 1, LeaseDuration: time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("创建 Relay 失败：%v", err) + } + if _, err := relay.ProcessBatch(context.Background()); err != nil { + t.Fatalf("记录永久失败事实失败：%v", err) + } + var failed model.OutboxEvent + if err := db.First(&failed, event.ID).Error; err != nil { + t.Fatalf("读取永久失败事实失败：%v", err) + } + if failed.Status != constants.OutboxStatusFailed || failed.RetryCount != 1 || failed.LastErrorCode != "OUTBOX_PERMANENT_FAILURE" { + t.Fatalf("永久失败未立即停止重试：%+v", failed) + } +} + +type recordingConsumer struct { + envelope outbox.DeliveryEnvelope +} + +func (c *recordingConsumer) Consume(_ context.Context, envelope outbox.DeliveryEnvelope) error { + c.envelope = envelope + return nil +} + +func TestPublicAsynqHandlerObservesStructuredEnvelope(t *testing.T) { + t.Parallel() + + envelope := outbox.DeliveryEnvelope{ + EventID: "event-handler", EventType: "foundation.example.created", PayloadVersion: 1, + CorrelationID: "correlation-handler", Payload: sonic.NoCopyRawMessage(`{"visible":true}`), + } + payload, err := sonic.Marshal(envelope) + if err != nil { + t.Fatalf("序列化公开信封失败：%v", err) + } + consumer := &recordingConsumer{} + handler := outbox.NewHandler(consumer) + if err := handler.Handle(context.Background(), asynq.NewTask(constants.TaskTypeOutboxDeliver, payload)); err != nil { + t.Fatalf("公开 Handler 处理失败：%v", err) + } + if consumer.envelope.EventID != envelope.EventID || consumer.envelope.CorrelationID != envelope.CorrelationID { + t.Fatalf("公开 Handler 未原样传播身份：%+v", consumer.envelope) + } +} + +func TestPostgresRelayRedisAsynqAndPublicHandlerChain(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + redisClient := testutil.NewRedisClient(t) + createTemporaryOutboxTables(t, db) + + repository := outbox.NewRepository() + event, err := repository.Append(context.Background(), db, newEnvelope("event-real-chain", "business-real-chain")) + if err != nil { + t.Fatalf("准备真实链路事件失败：%v", err) + } + queueClient := queue.NewClient(redisClient, zap.NewNop()) + t.Cleanup(func() { _ = queueClient.Close() }) + publisher := outbox.NewQueuePublisher(queueClient) + relay, err := outbox.NewRelay(db, publisher, zap.NewNop(), outbox.RelayOptions{Owner: "relay-real-chain", BatchSize: 1}) + if err != nil { + t.Fatalf("创建真实链路 Relay 失败：%v", err) + } + processed, err := relay.ProcessBatch(context.Background()) + if err != nil || processed != 1 { + t.Fatalf("真实链路投递失败：%v，数量：%d", err, processed) + } + + options := redisClient.Options() + inspector := asynq.NewInspector(asynq.RedisClientOpt{Addr: options.Addr, Password: options.Password, DB: options.DB}) + t.Cleanup(func() { _ = inspector.Close() }) + tasks, err := inspector.ListPendingTasks(constants.QueueOutboxDeliver, asynq.PageSize(1000)) + if err != nil { + t.Fatalf("检查 Asynq 待处理任务失败：%v", err) + } + var matched *asynq.TaskInfo + for _, info := range tasks { + var queued outbox.DeliveryEnvelope + if sonic.Unmarshal(info.Payload, &queued) == nil && queued.EventID == event.EventID { + matched = info + break + } + } + if matched == nil { + t.Fatal("真实 Asynq 队列中未找到本次公共事件") + } + t.Cleanup(func() { _ = inspector.DeleteTask(matched.Queue, matched.ID) }) + + consumer := &recordingConsumer{} + handler := outbox.NewHandler(consumer) + if err := handler.Handle(context.Background(), asynq.NewTask(matched.Type, matched.Payload)); err != nil { + t.Fatalf("公开 Handler 处理真实队列载荷失败：%v", err) + } + if consumer.envelope.EventID != event.EventID || consumer.envelope.CorrelationID != event.CorrelationID { + t.Fatalf("真实链路未原样传播事件身份：%+v", consumer.envelope) + } + + var delivered model.OutboxEvent + if err := db.First(&delivered, event.ID).Error; err != nil { + t.Fatalf("读取已投递事件失败：%v", err) + } + if delivered.Status != constants.OutboxStatusDelivered || delivered.DeliveredAt == nil { + t.Fatalf("真实链路未完成 Outbox 状态：%+v", delivered) + } +} + +func deliveryFromModel(event model.OutboxEvent) outbox.DeliveryEnvelope { + return outbox.DeliveryEnvelope{ + EventID: event.EventID, EventType: event.EventType, PayloadVersion: event.PayloadVersion, + AggregateType: event.AggregateType, AggregateID: event.AggregateID, + ResourceType: event.ResourceType, ResourceID: event.ResourceID, BusinessKey: event.BusinessKey, + RequestID: event.RequestID, CorrelationID: event.CorrelationID, + Payload: sonic.NoCopyRawMessage(event.Payload), + } +} diff --git a/internal/infrastructure/messaging/outbox/repository.go b/internal/infrastructure/messaging/outbox/repository.go new file mode 100644 index 0000000..7e902c8 --- /dev/null +++ b/internal/infrastructure/messaging/outbox/repository.go @@ -0,0 +1,78 @@ +// Package outbox 实现公共 Outbox 的持久化与投递基础设施。 +package outbox + +import ( + "context" + stderrors "errors" + "strings" + "time" + + "github.com/bytedance/sonic" + "github.com/google/uuid" + "gorm.io/datatypes" + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/asynctask" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// Envelope 是业务 UseCase 在事务内追加的公共事件信封。 +type Envelope struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + PayloadVersion int `json:"payload_version"` + AggregateType string `json:"aggregate_type"` + AggregateID string `json:"aggregate_id"` + ResourceType string `json:"resource_type"` + ResourceID string `json:"resource_id"` + BusinessKey string `json:"business_key,omitempty"` + RequestID string `json:"request_id,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + Payload any `json:"payload"` +} + +// Repository 通过调用方显式传入的 GORM 事务句柄追加事件。 +type Repository struct{} + +// NewRepository 创建公共 Outbox Repository。 +func NewRepository() *Repository { + return &Repository{} +} + +// Append 在业务事务中持久化稳定事件身份和必要快照。 +func (r *Repository) Append(ctx context.Context, tx *gorm.DB, envelope Envelope) (*model.OutboxEvent, error) { + if tx == nil { + return nil, stderrors.New("Outbox 追加必须传入 GORM 事务句柄") + } + if strings.TrimSpace(envelope.EventType) == "" || strings.TrimSpace(envelope.AggregateType) == "" || + strings.TrimSpace(envelope.AggregateID) == "" || strings.TrimSpace(envelope.ResourceType) == "" || + strings.TrimSpace(envelope.ResourceID) == "" { + return nil, stderrors.New("Outbox 事件类型和资源定位不能为空") + } + if err := asynctask.ValidatePayload(envelope.Payload); err != nil { + return nil, err + } + payload, err := sonic.Marshal(envelope.Payload) + if err != nil { + return nil, err + } + if envelope.EventID == "" { + envelope.EventID = uuid.NewString() + } + if envelope.PayloadVersion <= 0 { + envelope.PayloadVersion = 1 + } + now := time.Now().UTC() + event := &model.OutboxEvent{ + EventID: envelope.EventID, EventType: envelope.EventType, PayloadVersion: envelope.PayloadVersion, + AggregateType: envelope.AggregateType, AggregateID: envelope.AggregateID, + ResourceType: envelope.ResourceType, ResourceID: envelope.ResourceID, BusinessKey: envelope.BusinessKey, + RequestID: envelope.RequestID, CorrelationID: envelope.CorrelationID, Payload: datatypes.JSON(payload), + Status: constants.OutboxStatusPending, MaxRetries: constants.OutboxDefaultMaxRetries, NextAttemptAt: now, + } + if err := tx.WithContext(ctx).Create(event).Error; err != nil { + return nil, err + } + return event, nil +} diff --git a/internal/infrastructure/messaging/outbox/repository_integration_test.go b/internal/infrastructure/messaging/outbox/repository_integration_test.go new file mode 100644 index 0000000..5be7f26 --- /dev/null +++ b/internal/infrastructure/messaging/outbox/repository_integration_test.go @@ -0,0 +1,105 @@ +package outbox_test + +import ( + "context" + stderrors "errors" + "testing" + + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox" + "github.com/break/junhong_cmp_fiber/internal/testutil" +) + +func TestBusinessFactAndOutboxCommitAndRollbackTogether(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTemporaryOutboxTables(t, db) + repository := outbox.NewRepository() + + err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec("INSERT INTO test_foundation_business_fact (business_key) VALUES (?)", "fact-success").Error; err != nil { + return err + } + _, err := repository.Append(context.Background(), tx, newEnvelope("event-success", "fact-success")) + return err + }) + if err != nil { + t.Fatalf("提交业务事实和 Outbox 失败：%v", err) + } + assertTableCount(t, db, "test_foundation_business_fact", 1) + assertTableCount(t, db, "tb_outbox_event", 1) + + err = db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec("INSERT INTO test_foundation_business_fact (business_key) VALUES (?)", "fact-rollback").Error; err != nil { + return err + } + if _, err := repository.Append(context.Background(), tx, newEnvelope("event-rollback", "fact-rollback")); err != nil { + return err + } + return stderrors.New("注入业务失败") + }) + if err == nil { + t.Fatal("注入业务失败时事务应回滚") + } + assertTableCount(t, db, "test_foundation_business_fact", 1) + assertTableCount(t, db, "tb_outbox_event", 1) +} + +func TestOutboxUniqueFailureRollsBackBusinessFact(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + createTemporaryOutboxTables(t, db) + repository := outbox.NewRepository() + + if err := db.Transaction(func(tx *gorm.DB) error { + _, err := repository.Append(context.Background(), tx, newEnvelope("event-duplicate", "first")) + return err + }); err != nil { + t.Fatalf("准备重复事件失败：%v", err) + } + + err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec("INSERT INTO test_foundation_business_fact (business_key) VALUES (?)", "must-rollback").Error; err != nil { + return err + } + _, appendErr := repository.Append(context.Background(), tx, newEnvelope("event-duplicate", "second")) + return appendErr + }) + if err == nil { + t.Fatal("重复事件 ID 必须导致 Outbox 写入失败") + } + assertTableCount(t, db, "test_foundation_business_fact", 0) + assertTableCount(t, db, "tb_outbox_event", 1) +} + +func newEnvelope(eventID, businessKey string) outbox.Envelope { + return outbox.Envelope{ + EventID: eventID, EventType: "foundation.example.created", PayloadVersion: 1, + AggregateType: "example", AggregateID: businessKey, + ResourceType: "example", ResourceID: businessKey, BusinessKey: businessKey, + RequestID: "request-1", CorrelationID: "correlation-1", + Payload: struct { + BusinessKey string `json:"business_key"` + }{BusinessKey: businessKey}, + } +} + +func createTemporaryOutboxTables(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TEMP TABLE test_foundation_business_fact ( + id bigserial PRIMARY KEY, business_key varchar(100) NOT NULL UNIQUE + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建业务事实测试表失败：%v", err) + } + testutil.CreateTemporaryOutboxTable(t, db) +} + +func assertTableCount(t *testing.T, db *gorm.DB, table string, expected int64) { + t.Helper() + var count int64 + if err := db.Table(table).Count(&count).Error; err != nil { + t.Fatalf("统计表 %s 失败：%v", table, err) + } + if count != expected { + t.Fatalf("表 %s 行数错误：得到 %d，期望 %d", table, count, expected) + } +} diff --git a/internal/infrastructure/releasegate/checker.go b/internal/infrastructure/releasegate/checker.go new file mode 100644 index 0000000..f1ee1a2 --- /dev/null +++ b/internal/infrastructure/releasegate/checker.go @@ -0,0 +1,324 @@ +// Package releasegate 提供公共基础数据库对象的发布检查门禁。 +package releasegate + +import ( + "context" + stderrors "errors" + "sort" + "strings" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +const ( + // PhasePre 表示迁移或发布前检查。 + PhasePre = "pre" + // PhasePost 表示迁移后的定义与冒烟检查。 + PhasePost = "post" +) + +// Severity 表示门禁检查结果级别。 +type Severity string + +const ( + // SeverityInfo 表示安全的计数或对象标识。 + SeverityInfo Severity = "info" + // SeverityBlock 表示必须阻断发布的问题。 + SeverityBlock Severity = "block" +) + +// Finding 是不包含业务敏感值的门禁检查结果。 +type Finding struct { + Code string `json:"code"` + Severity Severity `json:"severity"` + Object string `json:"object"` + Count int64 `json:"count"` + Summary string `json:"summary"` +} + +// Report 是可重复执行的公共基础门禁报告。 +type Report struct { + Phase string `json:"phase"` + Findings []Finding `json:"findings"` +} + +// Passed 判断报告是否允许继续发布。 +func (r Report) Passed() bool { + for _, finding := range r.Findings { + if finding.Severity == SeverityBlock { + return false + } + } + return true +} + +// Checker 使用 PostgreSQL 事实检查公共对象定义与异常数据。 +type Checker struct { + db *gorm.DB + schema string +} + +// NewChecker 创建公共发布门禁检查器。 +func NewChecker(db *gorm.DB) *Checker { + return &Checker{db: db, schema: "public"} +} + +// NewCheckerWithSchema 创建隔离 schema 的迁移验收检查器。 +func NewCheckerWithSchema(db *gorm.DB, schema string) *Checker { + if strings.TrimSpace(schema) == "" { + schema = "public" + } + return &Checker{db: db, schema: schema} +} + +type tableContract struct { + name string + columns map[string]string +} + +var publicContracts = []tableContract{ + {name: "tb_outbox_event", columns: map[string]string{ + "event_id": "character varying", "event_type": "character varying", "payload": "jsonb", + "status": "integer", "retry_count": "integer", "next_attempt_at": "timestamp with time zone", + "lease_owner": "character varying", "lease_expires_at": "timestamp with time zone", + }}, + {name: "tb_system_config", columns: map[string]string{ + "config_key": "character varying", "config_value": "text", "value_type": "character varying", + "module": "character varying", "is_readonly": "boolean", "is_sensitive": "boolean", + }}, +} + +// Run 执行指定阶段检查；检查只读且可重复运行。 +func (c *Checker) Run(ctx context.Context, phase string) (Report, error) { + report := Report{Phase: phase, Findings: make([]Finding, 0)} + if phase != PhasePre && phase != PhasePost { + return report, stderrors.New("公共发布门禁阶段不受支持") + } + version, dirty, err := c.migrationVersion(ctx) + if err != nil { + return report, err + } + minimumVersion := uint(164) + if phase == PhasePost { + minimumVersion = 166 + } + if dirty || version < minimumVersion { + report.Findings = append(report.Findings, Finding{ + Code: "FOUNDATION_DEPENDENCY_VERSION", Severity: SeverityBlock, + Object: "schema_migrations", Count: int64(version), Summary: "迁移依赖版本未满足或数据库处于脏状态", + }) + } + + for _, contract := range publicContracts { + exists, err := c.tableExists(ctx, contract.name) + if err != nil { + return report, err + } + if !exists { + severity := SeverityInfo + if phase == PhasePost { + severity = SeverityBlock + } + report.Findings = append(report.Findings, Finding{ + Code: "FOUNDATION_OBJECT_MISSING", Severity: severity, Object: contract.name, + Summary: "公共对象尚不存在", + }) + continue + } + definitionFindings, err := c.checkColumns(ctx, contract) + if err != nil { + return report, err + } + report.Findings = append(report.Findings, definitionFindings...) + } + if phase == PhasePost { + objectFindings, err := c.checkIndexesAndConstraints(ctx) + if err != nil { + return report, err + } + report.Findings = append(report.Findings, objectFindings...) + if err := c.smokeWrite(ctx); err != nil { + report.Findings = append(report.Findings, Finding{ + Code: "FOUNDATION_READ_WRITE_SMOKE_FAILED", Severity: SeverityBlock, + Object: "tech-public-foundation", Count: 1, Summary: "公共对象读写冒烟失败", + }) + } + } + + anomalyFindings, err := c.checkAnomalies(ctx) + if err != nil { + return report, err + } + report.Findings = append(report.Findings, anomalyFindings...) + sort.Slice(report.Findings, func(i, j int) bool { + if report.Findings[i].Object == report.Findings[j].Object { + return report.Findings[i].Code < report.Findings[j].Code + } + return report.Findings[i].Object < report.Findings[j].Object + }) + return report, nil +} + +func (c *Checker) migrationVersion(ctx context.Context) (uint, bool, error) { + var result struct { + Version uint `gorm:"column:version"` + Dirty bool `gorm:"column:dirty"` + } + err := c.db.WithContext(ctx).Raw("SELECT version, dirty FROM schema_migrations LIMIT 1").Scan(&result).Error + return result.Version, result.Dirty, err +} + +func (c *Checker) tableExists(ctx context.Context, table string) (bool, error) { + var exists bool + err := c.db.WithContext(ctx).Raw("SELECT to_regclass(?) IS NOT NULL", c.schema+"."+table).Scan(&exists).Error + return exists, err +} + +func (c *Checker) checkColumns(ctx context.Context, contract tableContract) ([]Finding, error) { + var rows []struct { + Name string `gorm:"column:column_name"` + Type string `gorm:"column:data_type"` + } + err := c.db.WithContext(ctx).Raw(` + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = ? AND table_name = ?`, c.schema, contract.name).Scan(&rows).Error + if err != nil { + return nil, err + } + actual := make(map[string]string, len(rows)) + for _, row := range rows { + actual[row.Name] = row.Type + } + findings := make([]Finding, 0) + for column, expectedType := range contract.columns { + actualType, exists := actual[column] + if !exists || !strings.EqualFold(actualType, expectedType) { + findings = append(findings, Finding{ + Code: "FOUNDATION_DEFINITION_MISMATCH", Severity: SeverityBlock, + Object: contract.name + "." + column, Count: 1, Summary: "公共对象字段定义不符合契约", + }) + } + } + return findings, nil +} + +func (c *Checker) checkAnomalies(ctx context.Context) ([]Finding, error) { + checks := []struct { + table string + code string + summary string + query string + }{ + {"tb_outbox_event", "OUTBOX_DUPLICATE_EVENT_ID", "存在重复事件 ID", `SELECT COUNT(*) FROM (SELECT event_id FROM tb_outbox_event GROUP BY event_id HAVING COUNT(*) > 1) AS conflicts`}, + {"tb_outbox_event", "OUTBOX_INVALID_REQUIRED_FIELD", "存在必填字段空值", `SELECT COUNT(*) FROM tb_outbox_event WHERE event_id IS NULL OR event_type IS NULL OR payload IS NULL`}, + {"tb_outbox_event", "OUTBOX_INVALID_STATUS", "存在非法投递状态", `SELECT COUNT(*) FROM tb_outbox_event WHERE status NOT IN (1,2,3,4)`}, + {"tb_outbox_event", "OUTBOX_UNDELIVERED", "存在未投递事件", `SELECT COUNT(*) FROM tb_outbox_event WHERE status IN (1,2,4)`}, + {"tb_outbox_event", "OUTBOX_EXPIRED_LEASE", "存在过期租约", `SELECT COUNT(*) FROM tb_outbox_event WHERE status = 2 AND lease_expires_at < NOW()`}, + {"tb_system_config", "SYSTEM_CONFIG_DUPLICATE_KEY", "存在重复配置 Key", `SELECT COUNT(*) FROM (SELECT config_key FROM tb_system_config GROUP BY config_key HAVING COUNT(*) > 1) AS conflicts`}, + {"tb_system_config", "SYSTEM_CONFIG_INVALID_REQUIRED_FIELD", "存在配置必填字段空值", `SELECT COUNT(*) FROM tb_system_config WHERE config_key IS NULL OR config_value IS NULL OR value_type IS NULL OR module IS NULL`}, + {"tb_system_config", "SYSTEM_CONFIG_INVALID_TYPE", "存在非法配置类型", `SELECT COUNT(*) FROM tb_system_config WHERE value_type NOT IN ('string','int','bool','json')`}, + } + findings := make([]Finding, 0, len(checks)) + for _, check := range checks { + exists, err := c.tableExists(ctx, check.table) + if err != nil { + return nil, err + } + if !exists { + continue + } + var count int64 + if err := c.db.WithContext(ctx).Raw(check.query).Scan(&count).Error; err != nil { + return nil, err + } + if count > 0 { + findings = append(findings, Finding{ + Code: check.code, Severity: SeverityBlock, Object: check.table, + Count: count, Summary: check.summary, + }) + } + } + for _, table := range []string{"tb_export_task", "tb_iot_card_import_task", "tb_device_import_task", "tb_order_package_invalidate_task"} { + exists, err := c.tableExists(ctx, table) + if err != nil { + return nil, err + } + if !exists { + continue + } + var count int64 + if err := c.db.WithContext(ctx).Table(table).Where("status IN ?", []int{1, 2}).Count(&count).Error; err != nil { + return nil, err + } + if count > 0 { + findings = append(findings, Finding{ + Code: "ASYNC_TASK_UNFINISHED", Severity: SeverityBlock, Object: table, + Count: count, Summary: "存在待处理或处理中任务", + }) + } + } + return findings, nil +} + +func (c *Checker) checkIndexesAndConstraints(ctx context.Context) ([]Finding, error) { + expectedConstraints := map[string][]string{ + "tb_outbox_event": {"uq_outbox_event_id", "ck_outbox_event_status", "ck_outbox_event_payload_version", "ck_outbox_event_retry_count"}, + "tb_system_config": {"uq_system_config_key", "ck_system_config_value_type"}, + } + expectedIndexes := map[string][]string{ + "tb_outbox_event": {"idx_outbox_event_claim", "idx_outbox_event_type_status", "idx_outbox_event_aggregate", "idx_outbox_event_resource", "idx_outbox_event_request_id", "idx_outbox_event_correlation_id"}, + "tb_system_config": {"idx_system_config_module_key"}, + } + findings := make([]Finding, 0) + for table, names := range expectedConstraints { + for _, name := range names { + var count int64 + if err := c.db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM information_schema.table_constraints + WHERE table_schema = ? AND table_name = ? AND constraint_name = ?`, c.schema, table, name).Scan(&count).Error; err != nil { + return nil, err + } + if count != 1 { + findings = append(findings, Finding{Code: "FOUNDATION_CONSTRAINT_MISSING", Severity: SeverityBlock, Object: table + "." + name, Count: count, Summary: "公共约束缺失或重复"}) + } + } + } + for table, names := range expectedIndexes { + for _, name := range names { + var count int64 + if err := c.db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM pg_indexes + WHERE schemaname = ? AND tablename = ? AND indexname = ?`, c.schema, table, name).Scan(&count).Error; err != nil { + return nil, err + } + if count != 1 { + findings = append(findings, Finding{Code: "FOUNDATION_INDEX_MISSING", Severity: SeverityBlock, Object: table + "." + name, Count: count, Summary: "公共索引缺失或重复"}) + } + } + } + return findings, nil +} + +var errSmokeRollback = stderrors.New("公共对象冒烟回滚") + +func (c *Checker) smokeWrite(ctx context.Context) error { + err := c.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + eventID := "smoke-" + uuid.NewString() + if err := tx.Exec(`INSERT INTO tb_outbox_event + (event_id, event_type, payload_version, aggregate_type, aggregate_id, resource_type, resource_id, payload) + VALUES (?, 'foundation.smoke', 1, 'foundation', 'smoke', 'foundation', 'smoke', '{}'::jsonb)`, eventID).Error; err != nil { + return err + } + configKey := "foundation.smoke." + uuid.NewString() + if err := tx.Exec(`INSERT INTO tb_system_config + (config_key, config_value, value_type, module, description) + VALUES (?, 'true', 'bool', 'foundation', '公共对象读写冒烟')`, configKey).Error; err != nil { + return err + } + return errSmokeRollback + }) + if stderrors.Is(err, errSmokeRollback) { + return nil + } + return err +} diff --git a/internal/infrastructure/releasegate/checker_test.go b/internal/infrastructure/releasegate/checker_test.go new file mode 100644 index 0000000..907a4ec --- /dev/null +++ b/internal/infrastructure/releasegate/checker_test.go @@ -0,0 +1,20 @@ +package releasegate + +import "testing" + +func TestReportBlocksOnlyOnBlockingFindings(t *testing.T) { + t.Parallel() + + report := Report{Phase: PhasePre, Findings: []Finding{{ + Code: "FOUNDATION_OBJECT_MISSING", Severity: SeverityInfo, Object: "tb_outbox_event", + }}} + if !report.Passed() { + t.Fatal("迁移前公共对象尚未创建不应单独阻断发布") + } + report.Findings = append(report.Findings, Finding{ + Code: "OUTBOX_UNDELIVERED", Severity: SeverityBlock, Object: "tb_outbox_event", Count: 1, + }) + if report.Passed() { + t.Fatal("存在未投递事件时必须阻断发布") + } +} diff --git a/internal/infrastructure/releasegate/migration_integration_test.go b/internal/infrastructure/releasegate/migration_integration_test.go new file mode 100644 index 0000000..97e2edf --- /dev/null +++ b/internal/infrastructure/releasegate/migration_integration_test.go @@ -0,0 +1,222 @@ +package releasegate_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/releasegate" + "github.com/break/junhong_cmp_fiber/internal/testutil" +) + +func TestPublicFoundationMigrationsOnEmptyAndCompatibleDatabase(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + schema := prepareMigrationSchema(t, db, 164) + checker := releasegate.NewCheckerWithSchema(db, schema) + + pre, err := checker.Run(context.Background(), releasegate.PhasePre) + if err != nil { + t.Fatalf("执行迁移前检查失败：%v", err) + } + if !pre.Passed() { + t.Fatalf("空数据库迁移前检查不应阻断：%+v", pre.Findings) + } + if err := db.Exec("CREATE TABLE compatible_existing_data (id bigint PRIMARY KEY, value text NOT NULL)").Error; err != nil { + t.Fatalf("准备兼容存量表失败：%v", err) + } + if err := db.Exec("INSERT INTO compatible_existing_data (id, value) VALUES (1, 'keep')").Error; err != nil { + t.Fatalf("准备兼容存量数据失败：%v", err) + } + + executeMigration(t, db, "000165_create_public_outbox.up.sql") + executeMigration(t, db, "000166_create_system_config.up.sql") + if err := db.Exec("UPDATE schema_migrations SET version = 166, dirty = false").Error; err != nil { + t.Fatalf("更新隔离迁移版本失败：%v", err) + } + + post, err := checker.Run(context.Background(), releasegate.PhasePost) + if err != nil { + t.Fatalf("执行迁移后检查失败：%v", err) + } + if !post.Passed() { + t.Fatalf("迁移后检查应通过：%+v", post.Findings) + } + postAgain, err := checker.Run(context.Background(), releasegate.PhasePost) + if err != nil || !postAgain.Passed() { + t.Fatalf("迁移后检查必须可重复执行：%v，结果：%+v", err, postAgain.Findings) + } + var compatibleCount int64 + if err := db.Table("compatible_existing_data").Count(&compatibleCount).Error; err != nil || compatibleCount != 1 { + t.Fatalf("迁移破坏了兼容存量数据：%v，数量：%d", err, compatibleCount) + } + + if err := db.Transaction(func(tx *gorm.DB) error { + return executeMigrationWithError(tx, "000165_create_public_outbox.up.sql") + }); err == nil { + t.Fatal("公共对象已创建时不得通过重复 DDL 静默掩盖定义") + } +} + +func TestMigrationGateBlocksAnomaliesWithoutDestructiveWrites(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + schema := prepareMigrationSchema(t, db, 166) + executeMigration(t, db, "000165_create_public_outbox.up.sql") + executeMigration(t, db, "000166_create_system_config.up.sql") + if err := db.Exec(`INSERT INTO tb_outbox_event + (event_id, event_type, payload_version, aggregate_type, aggregate_id, resource_type, resource_id, payload) + VALUES ('blocked-event', 'foundation.blocked', 1, 'example', '1', 'example', '1', '{}'::jsonb)`).Error; err != nil { + t.Fatalf("准备未投递异常失败：%v", err) + } + if err := db.Exec("ALTER TABLE tb_outbox_event DROP CONSTRAINT ck_outbox_event_status").Error; err != nil { + t.Fatalf("准备非法状态定义失败：%v", err) + } + if err := db.Exec(`INSERT INTO tb_outbox_event + (event_id, event_type, payload_version, aggregate_type, aggregate_id, resource_type, resource_id, payload, status) + VALUES ('invalid-status', 'foundation.invalid', 1, 'example', '2', 'example', '2', '{}'::jsonb, 9)`).Error; err != nil { + t.Fatalf("准备非法状态数据失败：%v", err) + } + + report, err := releasegate.NewCheckerWithSchema(db, schema).Run(context.Background(), releasegate.PhasePost) + if err != nil { + t.Fatalf("执行异常门禁失败：%v", err) + } + if report.Passed() || !hasFinding(report, "OUTBOX_UNDELIVERED") || !hasFinding(report, "OUTBOX_INVALID_STATUS") || !hasFinding(report, "FOUNDATION_CONSTRAINT_MISSING") { + t.Fatalf("异常数据和定义未被完整阻断：%+v", report.Findings) + } + var count int64 + if err := db.Table("tb_outbox_event").Count(&count).Error; err != nil || count != 2 { + t.Fatalf("只读门禁修改了异常数据：%v，数量：%d", err, count) + } +} + +func TestDownMigrationAllowsEmptyStructuresAndRejectsExistingFacts(t *testing.T) { + t.Run("空结构可回滚", func(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + schema := prepareMigrationSchema(t, db, 166) + executeMigration(t, db, "000165_create_public_outbox.up.sql") + executeMigration(t, db, "000166_create_system_config.up.sql") + executeMigration(t, db, "000166_create_system_config.down.sql") + executeMigration(t, db, "000165_create_public_outbox.down.sql") + for _, table := range []string{"tb_outbox_event", "tb_system_config"} { + var exists bool + if err := db.Raw("SELECT to_regclass(?) IS NOT NULL", schema+"."+table).Scan(&exists).Error; err != nil || exists { + t.Fatalf("空结构回滚失败：%s，错误：%v", table, err) + } + } + }) + + t.Run("已有事实拒绝删表", func(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + _ = prepareMigrationSchema(t, db, 166) + executeMigration(t, db, "000165_create_public_outbox.up.sql") + executeMigration(t, db, "000166_create_system_config.up.sql") + if err := db.Exec(`INSERT INTO tb_system_config + (config_key, config_value, value_type, module, description) + VALUES ('foundation.test.fact', 'true', 'bool', 'foundation', '回滚保护测试')`).Error; err != nil { + t.Fatalf("准备配置事实失败：%v", err) + } + if err := db.Transaction(func(tx *gorm.DB) error { + return executeMigrationWithError(tx, "000166_create_system_config.down.sql") + }); err == nil || !strings.Contains(err.Error(), "禁止删表回滚") { + t.Fatalf("已有事实时 down 迁移必须明确拒绝：%v", err) + } + var count int64 + if err := db.Table("tb_system_config").Count(&count).Error; err != nil || count != 1 { + t.Fatalf("拒绝回滚后配置事实丢失：%v，数量：%d", err, count) + } + }) +} + +func prepareMigrationSchema(t *testing.T, db *gorm.DB, version uint) string { + t.Helper() + schema := "foundation_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + if err := db.Exec(fmt.Sprintf(`CREATE SCHEMA %s`, schema)).Error; err != nil { + t.Fatalf("创建隔离迁移 schema 失败：%v", err) + } + if err := db.Exec(fmt.Sprintf(`SET LOCAL search_path TO %s`, schema)).Error; err != nil { + t.Fatalf("切换隔离迁移 schema 失败：%v", err) + } + if err := db.Exec("CREATE TABLE schema_migrations (version bigint NOT NULL, dirty boolean NOT NULL)").Error; err != nil { + t.Fatalf("创建隔离迁移版本表失败：%v", err) + } + if err := db.Exec("INSERT INTO schema_migrations (version, dirty) VALUES (?, false)", version).Error; err != nil { + t.Fatalf("写入隔离迁移版本失败：%v", err) + } + return schema +} + +func executeMigration(t *testing.T, db *gorm.DB, name string) { + t.Helper() + if err := executeMigrationWithError(db, name); err != nil { + t.Fatalf("执行迁移 %s 失败：%v", name, err) + } +} + +func executeMigrationWithError(db *gorm.DB, name string) error { + path := filepath.Join("..", "..", "..", "migrations", name) + content, err := os.ReadFile(path) + if err != nil { + return err + } + for _, statement := range splitSQLStatements(string(content)) { + if err := db.Exec(statement).Error; err != nil { + return err + } + } + return nil +} + +func splitSQLStatements(content string) []string { + statements := make([]string, 0) + start := 0 + inSingleQuote := false + inDoubleQuote := false + inDollarBlock := false + for index := 0; index < len(content); index++ { + if index+1 < len(content) && content[index:index+2] == "$$" && !inSingleQuote && !inDoubleQuote { + inDollarBlock = !inDollarBlock + index++ + continue + } + if inDollarBlock { + continue + } + switch content[index] { + case '\'': + if !inDoubleQuote { + inSingleQuote = !inSingleQuote + } + case '"': + if !inSingleQuote { + inDoubleQuote = !inDoubleQuote + } + case ';': + if !inSingleQuote && !inDoubleQuote { + statement := strings.TrimSpace(content[start : index+1]) + if statement != "" { + statements = append(statements, statement) + } + start = index + 1 + } + } + } + if tail := strings.TrimSpace(content[start:]); tail != "" { + statements = append(statements, tail) + } + return statements +} + +func hasFinding(report releasegate.Report, code string) bool { + for _, finding := range report.Findings { + if finding.Code == code { + return true + } + } + return false +} diff --git a/internal/infrastructure/systemconfig/reader.go b/internal/infrastructure/systemconfig/reader.go new file mode 100644 index 0000000..214e0d3 --- /dev/null +++ b/internal/infrastructure/systemconfig/reader.go @@ -0,0 +1,149 @@ +package systemconfig + +import ( + "context" + stderrors "errors" + "sort" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// AlertSink 接收配置读取和缓存故障的中文安全告警。 +type AlertSink interface { + Warn(ctx context.Context, code, component, safeID, summary string) +} + +// LogAlertSink 使用应用日志承接安全告警。 +type LogAlertSink struct { + logger *zap.Logger +} + +// NewLogAlertSink 创建系统配置日志告警 Adapter。 +func NewLogAlertSink(logger *zap.Logger) *LogAlertSink { + if logger == nil { + logger = zap.NewNop() + } + return &LogAlertSink{logger: logger} +} + +// Warn 记录不包含配置值的中文安全告警。 +func (s *LogAlertSink) Warn(_ context.Context, code, component, safeID, summary string) { + s.logger.Warn(summary, zap.String("error_code", code), zap.String("component", component), zap.String("safe_id", safeID)) +} + +// Reader 提供以 PostgreSQL 为唯一事实来源的缓存读取能力。 +type Reader struct { + db *gorm.DB + registry *Registry + cache Cache + alerts AlertSink +} + +func (r *Reader) warn(ctx context.Context, code, key, summary string) { + if r.alerts != nil { + r.alerts.Warn(ctx, code, "system_config", key, summary) + } +} + +// NewReader 创建系统配置读取器。 +func NewReader(db *gorm.DB, registry *Registry, cache Cache, alerts AlertSink) *Reader { + return &Reader{db: db, registry: registry, cache: cache, alerts: alerts} +} + +// Get 读取单个已注册配置；Redis 故障时回退 PostgreSQL。 +func (r *Reader) Get(ctx context.Context, key string) (string, error) { + definition, registered := r.registry.Get(key) + if !registered { + return "", stderrors.New("系统配置 Key 未注册") + } + cacheKey := constants.RedisSystemConfigKey(key) + if r.cache != nil { + if value, err := r.cache.Get(ctx, cacheKey); err == nil { + if ValidateValue(definition, value) == nil { + r.registry.Remember(key, value) + return value, nil + } + r.warn(ctx, "SYSTEM_CONFIG_CACHE_INVALID", key, "系统配置缓存值非法，已回退 PostgreSQL") + } else if err != redis.Nil { + r.warn(ctx, "SYSTEM_CONFIG_CACHE_READ_FAILED", key, "系统配置缓存读取失败，已回退 PostgreSQL") + } + } + var record model.SystemConfig + err := r.db.WithContext(ctx).Where("config_key = ?", key).First(&record).Error + if err == gorm.ErrRecordNotFound { + return definition.DefaultValue, nil + } + if err != nil { + return "", err + } + value := record.ConfigValue + if ValidateValue(definition, value) != nil { + value = r.registry.LastValidatedOrDefault(definition) + r.warn(ctx, "SYSTEM_CONFIG_DATABASE_VALUE_INVALID", key, "数据库配置值非法，已使用最近验证值或安全默认值") + } else { + r.registry.Remember(key, value) + } + if r.cache != nil { + if err := r.cache.Set(ctx, cacheKey, value, constants.SystemConfigCacheTTL); err != nil { + r.warn(ctx, "SYSTEM_CONFIG_CACHE_WRITE_FAILED", key, "系统配置缓存回填失败") + } + } + return value, nil +} + +// ListItem 是查询层组装 DTO 所需的稳定投影。 +type ListItem struct { + Definition Definition + Record *model.SystemConfig + Value string + Registered bool +} + +// List 合并代码注册表和数据库遗留记录；未注册记录强制只读。 +func (r *Reader) List(ctx context.Context, module string) ([]ListItem, error) { + var records []model.SystemConfig + query := r.db.WithContext(ctx).Order("config_key ASC") + if module != "" { + query = query.Where("module = ?", module) + } + if err := query.Find(&records).Error; err != nil { + return nil, err + } + byKey := make(map[string]*model.SystemConfig, len(records)) + for index := range records { + byKey[records[index].ConfigKey] = &records[index] + } + items := make([]ListItem, 0, len(records)+len(r.registry.List())) + for _, definition := range r.registry.List() { + if module != "" && definition.Module != module { + continue + } + record := byKey[definition.Key] + value := definition.DefaultValue + if record != nil { + value = record.ConfigValue + if ValidateValue(definition, value) != nil { + value = r.registry.LastValidatedOrDefault(definition) + r.warn(ctx, "SYSTEM_CONFIG_DATABASE_VALUE_INVALID", definition.Key, "数据库配置值非法，列表已使用安全值") + } else { + r.registry.Remember(definition.Key, value) + } + delete(byKey, definition.Key) + } + items = append(items, ListItem{Definition: definition, Record: record, Value: value, Registered: true}) + } + for _, record := range byKey { + definition := Definition{ + Key: record.ConfigKey, Module: record.Module, ValueType: record.ValueType, + Description: record.Description, Readonly: true, Sensitive: record.IsSensitive, Control: "readonly", + } + items = append(items, ListItem{Definition: definition, Record: record, Value: record.ConfigValue, Registered: false}) + } + sort.Slice(items, func(i, j int) bool { return items[i].Definition.Key < items[j].Definition.Key }) + return items, nil +} diff --git a/internal/infrastructure/systemconfig/reader_integration_test.go b/internal/infrastructure/systemconfig/reader_integration_test.go new file mode 100644 index 0000000..30b3979 --- /dev/null +++ b/internal/infrastructure/systemconfig/reader_integration_test.go @@ -0,0 +1,115 @@ +package systemconfig_test + +import ( + "context" + stderrors "errors" + "sync" + "testing" + "time" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/testutil" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +type unavailableCache struct{} + +func (unavailableCache) Get(context.Context, string) (string, error) { + return "", stderrors.New("测试缓存不可用") +} + +func (unavailableCache) Set(context.Context, string, string, time.Duration) error { + return stderrors.New("测试缓存不可用") +} + +func (unavailableCache) Delete(context.Context, string) error { + return stderrors.New("测试缓存不可用") +} + +type alertRecorder struct { + mu sync.Mutex + codes []string +} + +func (r *alertRecorder) Warn(_ context.Context, code, _, _, _ string) { + r.mu.Lock() + defer r.mu.Unlock() + r.codes = append(r.codes, code) +} + +func TestReaderUsesRedisHitAndFallsBackToPostgresOnMiss(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + redisClient := testutil.NewRedisClient(t) + testutil.CreateTemporarySystemConfigTable(t, db) + registry := systemconfig.NewRegistry() + definition := systemconfig.Definition{ + Key: "foundation.reader.limit", Module: "foundation", ValueType: constants.SystemConfigTypeInt, + DefaultValue: "10", Description: "读取器测试上限", + } + if err := registry.Register(definition); err != nil { + t.Fatalf("注册系统配置失败：%v", err) + } + if err := db.Create(&model.SystemConfig{ + ConfigKey: definition.Key, ConfigValue: "20", ValueType: definition.ValueType, + Module: definition.Module, Description: definition.Description, + }).Error; err != nil { + t.Fatalf("准备数据库配置失败：%v", err) + } + cacheKey := constants.RedisSystemConfigKey(definition.Key) + t.Cleanup(func() { _ = redisClient.Del(context.Background(), cacheKey).Err() }) + if err := redisClient.Set(context.Background(), cacheKey, "30", constants.SystemConfigCacheTTL).Err(); err != nil { + t.Fatalf("准备 Redis 缓存失败：%v", err) + } + reader := systemconfig.NewReader(db, registry, systemconfig.NewRedisCache(redisClient), nil) + + value, err := reader.Get(context.Background(), definition.Key) + if err != nil || value != "30" { + t.Fatalf("Redis 命中结果错误：值=%q，错误=%v", value, err) + } + if err := redisClient.Del(context.Background(), cacheKey).Err(); err != nil { + t.Fatalf("清理 Redis 缓存失败：%v", err) + } + value, err = reader.Get(context.Background(), definition.Key) + if err != nil || value != "20" { + t.Fatalf("Redis 未命中时未回退 PostgreSQL：值=%q，错误=%v", value, err) + } + cached, err := redisClient.Get(context.Background(), cacheKey).Result() + if err != nil || cached != "20" { + t.Fatalf("PostgreSQL 结果未回填 Redis：值=%q，错误=%v", cached, err) + } + ttl, err := redisClient.TTL(context.Background(), cacheKey).Result() + if err != nil || ttl <= 0 || ttl > constants.SystemConfigCacheTTL { + t.Fatalf("Redis 回填 TTL 不符合公共常量：TTL=%s，错误=%v", ttl, err) + } +} + +func TestReaderFallsBackToPostgresWhenCacheIsUnavailable(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + testutil.CreateTemporarySystemConfigTable(t, db) + registry := systemconfig.NewRegistry() + definition := systemconfig.Definition{ + Key: "foundation.reader.fallback", Module: "foundation", ValueType: constants.SystemConfigTypeString, + DefaultValue: "default", Description: "缓存故障回退测试值", + } + if err := registry.Register(definition); err != nil { + t.Fatalf("注册系统配置失败：%v", err) + } + if err := db.Create(&model.SystemConfig{ + ConfigKey: definition.Key, ConfigValue: "database", ValueType: definition.ValueType, + Module: definition.Module, Description: definition.Description, + }).Error; err != nil { + t.Fatalf("准备数据库配置失败：%v", err) + } + alerts := &alertRecorder{} + reader := systemconfig.NewReader(db, registry, unavailableCache{}, alerts) + value, err := reader.Get(context.Background(), definition.Key) + if err != nil || value != "database" { + t.Fatalf("缓存不可用时未回退 PostgreSQL：值=%q，错误=%v", value, err) + } + alerts.mu.Lock() + defer alerts.mu.Unlock() + if len(alerts.codes) < 2 || alerts.codes[0] != "SYSTEM_CONFIG_CACHE_READ_FAILED" { + t.Fatalf("缓存读写故障未产生安全告警：%v", alerts.codes) + } +} diff --git a/internal/infrastructure/systemconfig/registry.go b/internal/infrastructure/systemconfig/registry.go new file mode 100644 index 0000000..4232d30 --- /dev/null +++ b/internal/infrastructure/systemconfig/registry.go @@ -0,0 +1,183 @@ +// Package systemconfig 实现受控系统配置注册、缓存和持久化 Adapter。 +package systemconfig + +import ( + "context" + stderrors "errors" + "strconv" + "strings" + "sync" + "time" + + "github.com/bytedance/sonic" + "github.com/redis/go-redis/v9" + + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// Definition 是业务模块拥有的配置 Key 注册定义。 +type Definition struct { + Key string + Module string + ValueType string + DefaultValue string + Description string + Readonly bool + Sensitive bool + Control string + EnumValues []string + Min *int64 + Max *int64 +} + +// Registry 保存可写配置的代码权威定义。 +type Registry struct { + mu sync.RWMutex + definitions map[string]Definition + lastValidated map[string]string +} + +// NewRegistry 创建空注册表；具体业务 Key 由所属模块注册。 +func NewRegistry() *Registry { + return &Registry{definitions: map[string]Definition{}, lastValidated: map[string]string{}} +} + +// Register 注册一个业务模块拥有的配置 Key。 +func (r *Registry) Register(definition Definition) error { + if !validKey(definition.Key) || strings.TrimSpace(definition.Module) == "" || strings.TrimSpace(definition.Description) == "" { + return stderrors.New("系统配置注册信息不完整") + } + if err := ValidateValue(definition, definition.DefaultValue); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + if existing, exists := r.definitions[definition.Key]; exists { + if existing.ValueType != definition.ValueType { + return stderrors.New("系统配置 Key 存在类型冲突") + } + return stderrors.New("系统配置 Key 重复注册") + } + r.definitions[definition.Key] = definition + r.lastValidated[definition.Key] = definition.DefaultValue + return nil +} + +// Get 查询已注册定义。 +func (r *Registry) Get(key string) (Definition, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + definition, exists := r.definitions[key] + return definition, exists +} + +// List 返回注册定义快照。 +func (r *Registry) List() []Definition { + r.mu.RLock() + defer r.mu.RUnlock() + items := make([]Definition, 0, len(r.definitions)) + for _, definition := range r.definitions { + items = append(items, definition) + } + return items +} + +// Remember 保存最近一次通过注册校验的值。 +func (r *Registry) Remember(key, value string) { + r.mu.Lock() + defer r.mu.Unlock() + r.lastValidated[key] = value +} + +// LastValidatedOrDefault 返回最近验证值或代码安全默认值。 +func (r *Registry) LastValidatedOrDefault(definition Definition) string { + r.mu.RLock() + defer r.mu.RUnlock() + if value, exists := r.lastValidated[definition.Key]; exists { + return value + } + return definition.DefaultValue +} + +// ValidateValue 按注册类型、枚举和值域校验字符串化配置值。 +func ValidateValue(definition Definition, value string) error { + switch definition.ValueType { + case constants.SystemConfigTypeString: + case constants.SystemConfigTypeInt: + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return stderrors.New("系统配置值必须是整数") + } + if definition.Min != nil && parsed < *definition.Min { + return stderrors.New("系统配置值小于允许范围") + } + if definition.Max != nil && parsed > *definition.Max { + return stderrors.New("系统配置值大于允许范围") + } + case constants.SystemConfigTypeBool: + if value != "true" && value != "false" { + return stderrors.New("系统配置值必须是 true 或 false") + } + case constants.SystemConfigTypeJSON: + var parsed any + if sonic.Unmarshal([]byte(value), &parsed) != nil { + return stderrors.New("系统配置值必须是合法 JSON") + } + default: + return stderrors.New("系统配置类型不受支持") + } + if len(definition.EnumValues) > 0 { + for _, allowed := range definition.EnumValues { + if value == allowed { + return nil + } + } + return stderrors.New("系统配置值不在允许枚举中") + } + return nil +} + +func validKey(key string) bool { + parts := strings.Split(key, ".") + if len(parts) < 3 { + return false + } + for _, part := range parts { + if strings.TrimSpace(part) == "" { + return false + } + } + return true +} + +// Cache 是系统配置使用的最小缓存边界。 +type Cache interface { + Get(ctx context.Context, key string) (string, error) + Set(ctx context.Context, key, value string, ttl time.Duration) error + Delete(ctx context.Context, key string) error +} + +// RedisCache 使用 Redis 实现系统配置短期缓存。 +type RedisCache struct { + client *redis.Client +} + +// NewRedisCache 创建系统配置 Redis Adapter。 +func NewRedisCache(client *redis.Client) *RedisCache { + return &RedisCache{client: client} +} + +// Get 读取缓存值。 +func (c *RedisCache) Get(ctx context.Context, key string) (string, error) { + return c.client.Get(ctx, key).Result() +} + +// Set 回填缓存值。 +func (c *RedisCache) Set(ctx context.Context, key, value string, ttl time.Duration) error { + return c.client.Set(ctx, key, value, ttl).Err() +} + +// Delete 失效单 Key 缓存。 +func (c *RedisCache) Delete(ctx context.Context, key string) error { + return c.client.Del(ctx, key).Err() +} diff --git a/internal/infrastructure/systemconfig/registry_test.go b/internal/infrastructure/systemconfig/registry_test.go new file mode 100644 index 0000000..dad52de --- /dev/null +++ b/internal/infrastructure/systemconfig/registry_test.go @@ -0,0 +1,61 @@ +package systemconfig_test + +import ( + "testing" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +func TestRegistryRejectsDuplicateTypeConflictAndInvalidDefault(t *testing.T) { + t.Parallel() + + registry := systemconfig.NewRegistry() + definition := systemconfig.Definition{ + Key: "foundation.example.limit", Module: "foundation", ValueType: constants.SystemConfigTypeInt, + DefaultValue: "10", Description: "示例限制", + } + if err := registry.Register(definition); err != nil { + t.Fatalf("注册合法配置失败：%v", err) + } + if err := registry.Register(definition); err == nil { + t.Fatal("重复 Key 必须被拒绝") + } + conflict := definition + conflict.ValueType = constants.SystemConfigTypeString + if err := registry.Register(conflict); err == nil { + t.Fatal("同 Key 类型冲突必须被拒绝") + } + invalid := definition + invalid.Key = "foundation.example.invalid" + invalid.DefaultValue = "not-int" + if err := registry.Register(invalid); err == nil { + t.Fatal("非法默认值必须在注册阶段失败") + } +} + +func TestValidateValueSupportsFourControlledTypesAndBounds(t *testing.T) { + t.Parallel() + + minimum, maximum := int64(1), int64(10) + cases := []systemconfig.Definition{ + {Key: "a.b.string", Module: "a", ValueType: constants.SystemConfigTypeString, DefaultValue: "x", Description: "字符串"}, + {Key: "a.b.int", Module: "a", ValueType: constants.SystemConfigTypeInt, DefaultValue: "5", Description: "整数", Min: &minimum, Max: &maximum}, + {Key: "a.b.bool", Module: "a", ValueType: constants.SystemConfigTypeBool, DefaultValue: "true", Description: "布尔"}, + {Key: "a.b.json", Module: "a", ValueType: constants.SystemConfigTypeJSON, DefaultValue: `{"enabled":true}`, Description: "JSON"}, + } + for _, definition := range cases { + if err := systemconfig.ValidateValue(definition, definition.DefaultValue); err != nil { + t.Fatalf("合法 %s 值未通过：%v", definition.ValueType, err) + } + } + if err := systemconfig.ValidateValue(cases[1], "11"); err == nil { + t.Fatal("越界整数必须被拒绝") + } + if err := systemconfig.ValidateValue(cases[2], "yes"); err == nil { + t.Fatal("非法布尔值必须被拒绝") + } + if err := systemconfig.ValidateValue(cases[3], "{"); err == nil { + t.Fatal("非法 JSON 必须被拒绝") + } +} diff --git a/internal/model/dto/export_task_dto.go b/internal/model/dto/export_task_dto.go index 836370a..97e45ae 100644 --- a/internal/model/dto/export_task_dto.go +++ b/internal/model/dto/export_task_dto.go @@ -31,6 +31,7 @@ type ListExportTaskRequest struct { // ExportTaskItem 导出任务列表项。 type ExportTaskItem struct { ID uint `json:"id" description:"任务ID"` + TaskID uint `json:"task_id" description:"任务ID"` TaskNo string `json:"task_no" description:"任务编号"` Scene string `json:"scene" description:"导出场景 (device:设备, iot_card:IoT卡, order:订单)"` Format string `json:"format" description:"导出格式 (xlsx:Excel, csv:CSV)"` @@ -42,10 +43,16 @@ type ExportTaskItem struct { TotalShards int `json:"total_shards" description:"总分片数"` SuccessShards int `json:"success_shards" description:"成功分片数"` FailedShards int `json:"failed_shards" description:"失败分片数"` + TotalCount int `json:"total_count" description:"统一任务总数"` + SuccessCount int `json:"success_count" description:"统一任务成功数"` + FailedCount int `json:"failed_count" description:"统一任务失败数"` CancelRequested bool `json:"cancel_requested" description:"是否已请求取消"` FileKey string `json:"file_key,omitempty" description:"导出文件Key"` ErrorMessage string `json:"error_message,omitempty" description:"错误信息"` + ErrorCode string `json:"error_code,omitempty" description:"安全错误码"` + ErrorSummary string `json:"error_summary,omitempty" description:"安全失败摘要"` CreatedAt time.Time `json:"created_at" description:"创建时间"` + UpdatedAt time.Time `json:"updated_at" description:"更新时间"` StartedAt *time.Time `json:"started_at,omitempty" description:"开始处理时间"` CompletedAt *time.Time `json:"completed_at,omitempty" description:"完成时间"` CreatorUserID uint `json:"creator_user_id" description:"创建人用户ID"` diff --git a/internal/model/dto/system_config_dto.go b/internal/model/dto/system_config_dto.go new file mode 100644 index 0000000..537c693 --- /dev/null +++ b/internal/model/dto/system_config_dto.go @@ -0,0 +1,46 @@ +package dto + +import "time" + +// SystemConfigListRequest 是系统配置分页查询参数。 +type SystemConfigListRequest struct { + Module string `json:"module" query:"module" validate:"omitempty,max=100" description:"模块筛选"` + Page int `json:"page" query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码"` + PageSize int `json:"page_size" query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页数量"` +} + +// SystemConfigItem 是超级管理员可见的受控配置投影。 +type SystemConfigItem struct { + ConfigKey string `json:"config_key" description:"稳定配置 Key"` + Value string `json:"value" description:"配置值；敏感值按注册策略脱敏"` + ValueType string `json:"value_type" description:"值类型 (string:字符串, int:整数, bool:布尔, json:JSON)"` + Module string `json:"module" description:"所属模块"` + Description string `json:"description" description:"中文说明"` + Readonly bool `json:"readonly" description:"是否只读"` + Sensitive bool `json:"sensitive" description:"是否敏感"` + Registered bool `json:"registered" description:"是否已在代码注册"` + Control string `json:"control" description:"前端控件提示"` + EnumValues []string `json:"enum_values,omitempty" description:"允许的枚举值"` + Min *int64 `json:"min,omitempty" description:"整数最小值"` + Max *int64 `json:"max,omitempty" description:"整数最大值"` + UpdatedAt *time.Time `json:"updated_at,omitempty" description:"最近更新时间"` +} + +// SystemConfigListResponse 是系统配置分页结果。 +type SystemConfigListResponse struct { + List []SystemConfigItem `json:"list" description:"配置列表"` + Total int64 `json:"total" description:"总数量"` + Page int `json:"page" description:"页码"` + PageSize int `json:"page_size" description:"每页数量"` +} + +// UpdateSystemConfigRequest 是单 Key 更新请求。 +type UpdateSystemConfigRequest struct { + Value string `json:"value" validate:"required" required:"true" description:"字符串化配置值"` +} + +// UpdateSystemConfigParams 组合路径和请求体文档参数。 +type UpdateSystemConfigParams struct { + Key string `json:"key" path:"key" validate:"required" required:"true" description:"稳定配置 Key"` + UpdateSystemConfigRequest +} diff --git a/internal/model/outbox_event.go b/internal/model/outbox_event.go new file mode 100644 index 0000000..57554a7 --- /dev/null +++ b/internal/model/outbox_event.go @@ -0,0 +1,39 @@ +package model + +import ( + "time" + + "gorm.io/datatypes" +) + +// OutboxEvent 是跨进程可靠事件的权威公共持久化模型。 +type OutboxEvent struct { + ID uint `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + EventID string `gorm:"column:event_id;type:varchar(64);not null;uniqueIndex:uq_outbox_event_id" json:"event_id"` + EventType string `gorm:"column:event_type;type:varchar(150);not null;index:idx_outbox_event_type_status,priority:1" json:"event_type"` + PayloadVersion int `gorm:"column:payload_version;type:int;not null;default:1" json:"payload_version"` + AggregateType string `gorm:"column:aggregate_type;type:varchar(100);not null" json:"aggregate_type"` + AggregateID string `gorm:"column:aggregate_id;type:varchar(100);not null" json:"aggregate_id"` + ResourceType string `gorm:"column:resource_type;type:varchar(100);not null" json:"resource_type"` + ResourceID string `gorm:"column:resource_id;type:varchar(100);not null" json:"resource_id"` + BusinessKey string `gorm:"column:business_key;type:varchar(150);not null;default:''" json:"business_key,omitempty"` + RequestID string `gorm:"column:request_id;type:varchar(100);not null;default:'';index" json:"request_id,omitempty"` + CorrelationID string `gorm:"column:correlation_id;type:varchar(100);not null;default:'';index" json:"correlation_id,omitempty"` + Payload datatypes.JSON `gorm:"column:payload;type:jsonb;not null" json:"payload"` + Status int `gorm:"column:status;type:int;not null;default:1;index:idx_outbox_event_type_status,priority:2" json:"status"` + RetryCount int `gorm:"column:retry_count;type:int;not null;default:0" json:"retry_count"` + MaxRetries int `gorm:"column:max_retries;type:int;not null;default:10" json:"max_retries"` + NextAttemptAt time.Time `gorm:"column:next_attempt_at;type:timestamptz;not null;index:idx_outbox_event_claim,priority:2" json:"next_attempt_at"` + LeaseOwner *string `gorm:"column:lease_owner;type:varchar(100)" json:"lease_owner,omitempty"` + LeaseExpiresAt *time.Time `gorm:"column:lease_expires_at;type:timestamptz;index:idx_outbox_event_claim,priority:3" json:"lease_expires_at,omitempty"` + LastErrorCode string `gorm:"column:last_error_code;type:varchar(100);not null;default:''" json:"last_error_code,omitempty"` + LastErrorSummary string `gorm:"column:last_error_summary;type:varchar(500);not null;default:''" json:"last_error_summary,omitempty"` + DeliveredAt *time.Time `gorm:"column:delivered_at;type:timestamptz" json:"delivered_at,omitempty"` + CreatedAt time.Time `gorm:"column:created_at;type:timestamptz;not null;autoCreateTime;index:idx_outbox_event_claim,priority:4" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at;type:timestamptz;not null;autoUpdateTime" json:"updated_at"` +} + +// TableName 返回公共 Outbox 表名。 +func (OutboxEvent) TableName() string { + return "tb_outbox_event" +} diff --git a/internal/model/system_config.go b/internal/model/system_config.go new file mode 100644 index 0000000..a175f3c --- /dev/null +++ b/internal/model/system_config.go @@ -0,0 +1,24 @@ +package model + +import "time" + +// SystemConfig 是受控系统配置的 PostgreSQL 持久化模型。 +type SystemConfig struct { + ID uint `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + ConfigKey string `gorm:"column:config_key;type:varchar(150);not null;uniqueIndex:uq_system_config_key" json:"config_key"` + ConfigValue string `gorm:"column:config_value;type:text;not null" json:"config_value"` + ValueType string `gorm:"column:value_type;type:varchar(20);not null" json:"value_type"` + Module string `gorm:"column:module;type:varchar(100);not null;index:idx_system_config_module_key,priority:1" json:"module"` + Description string `gorm:"column:description;type:varchar(500);not null" json:"description"` + IsReadonly bool `gorm:"column:is_readonly;type:boolean;not null;default:false" json:"is_readonly"` + IsSensitive bool `gorm:"column:is_sensitive;type:boolean;not null;default:false" json:"is_sensitive"` + Creator uint `gorm:"column:creator;not null;default:0" json:"creator"` + Updater uint `gorm:"column:updater;not null;default:0" json:"updater"` + CreatedAt time.Time `gorm:"column:created_at;type:timestamptz;not null;autoCreateTime" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at;type:timestamptz;not null;autoUpdateTime" json:"updated_at"` +} + +// TableName 返回受控系统配置表名。 +func (SystemConfig) TableName() string { + return "tb_system_config" +} diff --git a/internal/query/outbox/metrics.go b/internal/query/outbox/metrics.go new file mode 100644 index 0000000..e387723 --- /dev/null +++ b/internal/query/outbox/metrics.go @@ -0,0 +1,143 @@ +// Package outbox 提供公共 Outbox 的只读运行状态投影。 +package outbox + +import ( + "context" + "time" + + "gorm.io/gorm" + + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +// EventTypeBacklog 表示某事件类型的未投递积压。 +type EventTypeBacklog struct { + EventType string `json:"event_type"` + Count int64 `json:"count"` +} + +// RetryBucket 表示某重试次数上的事件数量。 +type RetryBucket struct { + RetryCount int `json:"retry_count"` + Count int64 `json:"count"` +} + +// Metrics 是 Outbox 运维公开指标投影。 +type Metrics struct { + PendingCount int64 `json:"pending_count"` + OldestPendingAgeSecs int64 `json:"oldest_pending_age_seconds"` + DeliveringCount int64 `json:"delivering_count"` + ExpiredLeaseCount int64 `json:"expired_lease_count"` + DeliveredInWindow int64 `json:"delivered_in_window"` + FinalFailedCount int64 `json:"final_failed_count"` + DeliverySuccessRate float64 `json:"delivery_success_rate"` + RetryDistribution []RetryBucket `json:"retry_distribution"` + BacklogByEventType []EventTypeBacklog `json:"backlog_by_event_type"` + ObservedAt time.Time `json:"observed_at"` +} + +// Query 查询 PostgreSQL 中的 Outbox 运行事实。 +type Query struct { + db *gorm.DB + now func() time.Time +} + +// NewQuery 创建 Outbox 指标查询。 +func NewQuery(db *gorm.DB, now func() time.Time) *Query { + if now == nil { + now = time.Now + } + return &Query{db: db, now: now} +} + +// GetMetrics 查询指定统计窗口的积压、成功率、重试和最终失败。 +func (q *Query) GetMetrics(ctx context.Context, window time.Duration) (Metrics, error) { + now := q.now().UTC() + if window <= 0 { + window = time.Hour + } + metrics := Metrics{ObservedAt: now, RetryDistribution: []RetryBucket{}, BacklogByEventType: []EventTypeBacklog{}} + base := func() *gorm.DB { return q.db.WithContext(ctx).Model(&model.OutboxEvent{}) } + if err := base().Where("status = ?", constants.OutboxStatusPending).Count(&metrics.PendingCount).Error; err != nil { + return metrics, err + } + if err := base().Where("status = ?", constants.OutboxStatusDelivering).Count(&metrics.DeliveringCount).Error; err != nil { + return metrics, err + } + if err := base().Where("status = ? AND lease_expires_at <= ?", constants.OutboxStatusDelivering, now). + Count(&metrics.ExpiredLeaseCount).Error; err != nil { + return metrics, err + } + if err := base().Where("status = ?", constants.OutboxStatusFailed).Count(&metrics.FinalFailedCount).Error; err != nil { + return metrics, err + } + var oldestEvent model.OutboxEvent + err := base().Select("created_at").Where("status = ?", constants.OutboxStatusPending). + Order("created_at ASC").Take(&oldestEvent).Error + if err != nil && err != gorm.ErrRecordNotFound { + return metrics, err + } + if err == nil && oldestEvent.CreatedAt.Before(now) { + metrics.OldestPendingAgeSecs = int64(now.Sub(oldestEvent.CreatedAt).Seconds()) + } + windowStart := now.Add(-window) + if err := base().Where("status = ? AND delivered_at >= ?", constants.OutboxStatusDelivered, windowStart). + Count(&metrics.DeliveredInWindow).Error; err != nil { + return metrics, err + } + var failedInWindow int64 + if err := base().Where("status = ? AND updated_at >= ?", constants.OutboxStatusFailed, windowStart). + Count(&failedInWindow).Error; err != nil { + return metrics, err + } + totalTerminal := metrics.DeliveredInWindow + failedInWindow + if totalTerminal > 0 { + metrics.DeliverySuccessRate = float64(metrics.DeliveredInWindow) / float64(totalTerminal) + } + if err := base().Select("retry_count, COUNT(*) AS count").Where("retry_count > 0"). + Group("retry_count").Order("retry_count ASC").Scan(&metrics.RetryDistribution).Error; err != nil { + return metrics, err + } + if err := base().Select("event_type, COUNT(*) AS count").Where("status IN ?", []int{ + constants.OutboxStatusPending, constants.OutboxStatusDelivering, constants.OutboxStatusFailed, + }).Group("event_type").Order("event_type ASC").Scan(&metrics.BacklogByEventType).Error; err != nil { + return metrics, err + } + return metrics, nil +} + +// Thresholds 是不包含高基数字段的 Outbox 告警阈值。 +type Thresholds struct { + PendingCount int64 + OldestPendingAge time.Duration + ExpiredLeaseCount int64 + FinalFailedCount int64 +} + +// Alert 是可交给现有告警通道的中文安全摘要。 +type Alert struct { + Code string `json:"code"` + Component string `json:"component"` + Count int64 `json:"count"` + Summary string `json:"summary"` + Window string `json:"window"` +} + +// EvaluateAlerts 根据聚合指标生成低基数告警，不包含 payload 或敏感配置。 +func EvaluateAlerts(metrics Metrics, thresholds Thresholds) []Alert { + alerts := make([]Alert, 0, 4) + if thresholds.PendingCount > 0 && metrics.PendingCount >= thresholds.PendingCount { + alerts = append(alerts, Alert{Code: "OUTBOX_PENDING_HIGH", Component: "outbox", Count: metrics.PendingCount, Summary: "Outbox 待投递事件持续积压", Window: "当前快照"}) + } + if thresholds.OldestPendingAge > 0 && metrics.OldestPendingAgeSecs >= int64(thresholds.OldestPendingAge.Seconds()) { + alerts = append(alerts, Alert{Code: "OUTBOX_OLDEST_PENDING_HIGH", Component: "outbox", Count: metrics.OldestPendingAgeSecs, Summary: "Outbox 最老待投递事件超过阈值", Window: "秒"}) + } + if thresholds.ExpiredLeaseCount > 0 && metrics.ExpiredLeaseCount >= thresholds.ExpiredLeaseCount { + alerts = append(alerts, Alert{Code: "OUTBOX_EXPIRED_LEASE_HIGH", Component: "outbox", Count: metrics.ExpiredLeaseCount, Summary: "Outbox 过期租约数量超过阈值", Window: "当前快照"}) + } + if thresholds.FinalFailedCount > 0 && metrics.FinalFailedCount >= thresholds.FinalFailedCount { + alerts = append(alerts, Alert{Code: "OUTBOX_FINAL_FAILED_HIGH", Component: "outbox", Count: metrics.FinalFailedCount, Summary: "Outbox 最终失败事件需要人工处理", Window: "当前快照"}) + } + return alerts +} diff --git a/internal/query/outbox/metrics_integration_test.go b/internal/query/outbox/metrics_integration_test.go new file mode 100644 index 0000000..3764cf4 --- /dev/null +++ b/internal/query/outbox/metrics_integration_test.go @@ -0,0 +1,73 @@ +package outbox_test + +import ( + "context" + "testing" + "time" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/messaging/outbox" + "github.com/break/junhong_cmp_fiber/internal/model" + outboxquery "github.com/break/junhong_cmp_fiber/internal/query/outbox" + "github.com/break/junhong_cmp_fiber/internal/testutil" + "github.com/break/junhong_cmp_fiber/pkg/constants" +) + +func TestMetricsCoverBacklogRetriesFailuresAndThresholdAlerts(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + testutil.CreateTemporaryOutboxTable(t, db) + repository := outbox.NewRepository() + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + + states := []struct { + id string + eventType string + status int + retries int + }{ + {id: "metrics-pending", eventType: "example.a", status: constants.OutboxStatusPending}, + {id: "metrics-delivering", eventType: "example.a", status: constants.OutboxStatusDelivering, retries: 1}, + {id: "metrics-delivered", eventType: "example.b", status: constants.OutboxStatusDelivered}, + {id: "metrics-failed", eventType: "example.b", status: constants.OutboxStatusFailed, retries: 3}, + } + for _, state := range states { + event, err := repository.Append(context.Background(), db, outbox.Envelope{ + EventID: state.id, EventType: state.eventType, AggregateType: "example", AggregateID: state.id, + ResourceType: "example", ResourceID: state.id, Payload: struct { + Visible bool `json:"visible"` + }{Visible: true}, + }) + if err != nil { + t.Fatalf("准备指标事件失败：%v", err) + } + updates := map[string]any{"status": state.status, "retry_count": state.retries, "created_at": now.Add(-2 * time.Minute), "updated_at": now} + if state.status == constants.OutboxStatusDelivering { + updates["lease_owner"] = "dead-worker" + updates["lease_expires_at"] = now.Add(-time.Minute) + } + if state.status == constants.OutboxStatusDelivered { + updates["delivered_at"] = now.Add(-time.Minute) + } + if err := db.Model(&model.OutboxEvent{}).Where("id = ?", event.ID).Updates(updates).Error; err != nil { + t.Fatalf("设置指标事件状态失败：%v", err) + } + } + + query := outboxquery.NewQuery(db, func() time.Time { return now }) + metrics, err := query.GetMetrics(context.Background(), time.Hour) + if err != nil { + t.Fatalf("查询 Outbox 指标失败：%v", err) + } + if metrics.PendingCount != 1 || metrics.DeliveringCount != 1 || metrics.ExpiredLeaseCount != 1 || + metrics.DeliveredInWindow != 1 || metrics.FinalFailedCount != 1 || metrics.OldestPendingAgeSecs != 120 { + t.Fatalf("Outbox 指标不完整：%+v", metrics) + } + if len(metrics.RetryDistribution) != 2 || len(metrics.BacklogByEventType) != 2 { + t.Fatalf("重试分布或事件类型积压不完整：%+v", metrics) + } + alerts := outboxquery.EvaluateAlerts(metrics, outboxquery.Thresholds{ + PendingCount: 1, OldestPendingAge: time.Minute, ExpiredLeaseCount: 1, FinalFailedCount: 1, + }) + if len(alerts) != 4 { + t.Fatalf("阈值告警数量错误：%+v", alerts) + } +} diff --git a/internal/query/systemconfig/list.go b/internal/query/systemconfig/list.go new file mode 100644 index 0000000..b9df7b4 --- /dev/null +++ b/internal/query/systemconfig/list.go @@ -0,0 +1,73 @@ +// Package systemconfig 提供受控系统配置的超级管理员读取投影。 +package systemconfig + +import ( + "context" + + "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/model/dto" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/errors" + "github.com/break/junhong_cmp_fiber/pkg/middleware" +) + +// ListQuery 按模块分页查询代码注册配置和未注册遗留记录。 +type ListQuery struct { + reader *systemconfig.Reader +} + +// NewListQuery 创建系统配置列表查询。 +func NewListQuery(reader *systemconfig.Reader) *ListQuery { + return &ListQuery{reader: reader} +} + +// Execute 执行超级管理员系统配置查询。 +func (q *ListQuery) Execute(ctx context.Context, request dto.SystemConfigListRequest) (*dto.SystemConfigListResponse, error) { + if middleware.GetUserTypeFromContext(ctx) != constants.UserTypeSuperAdmin { + return nil, errors.New(errors.CodeForbidden) + } + page := request.Page + if page <= 0 { + page = 1 + } + pageSize := request.PageSize + if pageSize <= 0 { + pageSize = constants.DefaultPageSize + } + if pageSize > constants.MaxPageSize { + pageSize = constants.MaxPageSize + } + items, err := q.reader.List(ctx, request.Module) + if err != nil { + return nil, errors.Wrap(errors.CodeDatabaseError, err, "查询系统配置失败") + } + total := len(items) + start := (page - 1) * pageSize + if start > total { + start = total + } + end := start + pageSize + if end > total { + end = total + } + result := make([]dto.SystemConfigItem, 0, end-start) + for _, item := range items[start:end] { + value := item.Value + if item.Definition.Sensitive && value != "" { + value = "[已配置]" + } + projection := dto.SystemConfigItem{ + ConfigKey: item.Definition.Key, Value: value, ValueType: item.Definition.ValueType, + Module: item.Definition.Module, Description: item.Definition.Description, + Readonly: item.Definition.Readonly || !item.Registered, Sensitive: item.Definition.Sensitive, + Registered: item.Registered, Control: item.Definition.Control, + EnumValues: item.Definition.EnumValues, Min: item.Definition.Min, Max: item.Definition.Max, + } + if item.Record != nil { + updatedAt := item.Record.UpdatedAt + projection.UpdatedAt = &updatedAt + } + result = append(result, projection) + } + return &dto.SystemConfigListResponse{List: result, Total: int64(total), Page: page, PageSize: pageSize}, nil +} diff --git a/internal/routes/admin.go b/internal/routes/admin.go index ac26298..f2f7cc7 100644 --- a/internal/routes/admin.go +++ b/internal/routes/admin.go @@ -131,4 +131,7 @@ func RegisterAdminRoutes(router fiber.Router, handlers *bootstrap.Handlers, midd if handlers.SuperAdmin != nil { registerSuperAdminRoutes(authGroup, handlers.SuperAdmin, doc, basePath) } + if handlers.SystemConfig != nil { + registerSystemConfigRoutes(authGroup, handlers.SystemConfig, doc, basePath) + } } diff --git a/internal/routes/system_config.go b/internal/routes/system_config.go new file mode 100644 index 0000000..8411137 --- /dev/null +++ b/internal/routes/system_config.go @@ -0,0 +1,31 @@ +package routes + +import ( + "github.com/gofiber/fiber/v2" + + "github.com/break/junhong_cmp_fiber/internal/handler/admin" + "github.com/break/junhong_cmp_fiber/internal/model/dto" + "github.com/break/junhong_cmp_fiber/pkg/openapi" +) + +// registerSystemConfigRoutes 注册受控系统配置路由。 +func registerSystemConfigRoutes(router fiber.Router, handler *admin.SystemConfigHandler, doc *openapi.Generator, basePath string) { + configs := router.Group("/system-configs") + groupPath := basePath + "/system-configs" + + Register(configs, doc, groupPath, "GET", "", handler.List, RouteSpec{ + Summary: "查询受控系统配置", + Tags: []string{"系统配置"}, + Input: new(dto.SystemConfigListRequest), + Output: new(dto.SystemConfigListResponse), + Auth: true, + }) + + Register(configs, doc, groupPath, "PUT", "/:key", handler.Update, RouteSpec{ + Summary: "更新受控系统配置", + Tags: []string{"系统配置"}, + Input: new(dto.UpdateSystemConfigParams), + Output: new(dto.SystemConfigItem), + Auth: true, + }) +} diff --git a/internal/routes/system_config_integration_test.go b/internal/routes/system_config_integration_test.go new file mode 100644 index 0000000..a6e6b00 --- /dev/null +++ b/internal/routes/system_config_integration_test.go @@ -0,0 +1,236 @@ +package routes + +import ( + "context" + stderrors "errors" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/bytedance/sonic" + "github.com/gofiber/fiber/v2" + "go.uber.org/zap" + "gorm.io/gorm" + + systemConfigApp "github.com/break/junhong_cmp_fiber/internal/application/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/handler/admin" + systemConfigInfra "github.com/break/junhong_cmp_fiber/internal/infrastructure/systemconfig" + internalMiddleware "github.com/break/junhong_cmp_fiber/internal/middleware" + "github.com/break/junhong_cmp_fiber/internal/model" + "github.com/break/junhong_cmp_fiber/internal/model/dto" + systemConfigQuery "github.com/break/junhong_cmp_fiber/internal/query/systemconfig" + "github.com/break/junhong_cmp_fiber/internal/testutil" + "github.com/break/junhong_cmp_fiber/pkg/constants" + "github.com/break/junhong_cmp_fiber/pkg/middleware" +) + +type configAuditWriter struct { + fail bool +} + +func (w *configAuditWriter) WriteConfigChange(_ context.Context, tx *gorm.DB, audit systemConfigApp.ChangeAudit) error { + if w.fail { + return stderrors.New("测试审计写入失败") + } + before, _ := sonic.Marshal(audit.BeforeData) + after, _ := sonic.Marshal(audit.AfterData) + return tx.Exec(`INSERT INTO test_system_config_audit + (config_key, operator_id, request_id, before_data, after_data) + VALUES (?, ?, ?, ?::jsonb, ?::jsonb)`, audit.ConfigKey, audit.OperatorID, audit.RequestID, string(before), string(after)).Error +} + +type recordingAlerts struct { + mu sync.Mutex + codes []string +} + +func (a *recordingAlerts) Warn(_ context.Context, code, _, _, _ string) { + a.mu.Lock() + defer a.mu.Unlock() + a.codes = append(a.codes, code) +} + +type failingCache struct{} + +func (failingCache) Get(context.Context, string) (string, error) { + return "", stderrors.New("缓存不可用") +} +func (failingCache) Set(context.Context, string, string, time.Duration) error { + return stderrors.New("缓存不可用") +} +func (failingCache) Delete(context.Context, string) error { return stderrors.New("缓存不可用") } + +func TestSystemConfigFiberListAndUpdateUseRealPostgresAndRedis(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + redisClient := testutil.NewRedisClient(t) + testutil.CreateTemporarySystemConfigTable(t, db) + createTemporarySystemConfigAuditTable(t, db) + registry := newSystemConfigTestRegistry(t) + cache := systemConfigInfra.NewRedisCache(redisClient) + alerts := &recordingAlerts{} + reader := systemConfigInfra.NewReader(db, registry, cache, alerts) + handler := admin.NewSystemConfigHandler( + systemConfigQuery.NewListQuery(reader), + systemConfigApp.NewUpdateService(db, registry, cache, &configAuditWriter{}, alerts, nil), + ) + + if err := db.Create(&model.SystemConfig{ + ConfigKey: "legacy.unknown.value", ConfigValue: "legacy", ValueType: constants.SystemConfigTypeString, + Module: "legacy", Description: "未注册遗留配置", IsSensitive: false, + }).Error; err != nil { + t.Fatalf("准备未注册配置失败：%v", err) + } + if err := redisClient.Set(context.Background(), constants.RedisSystemConfigKey("foundation.demo.int"), "7", constants.SystemConfigCacheTTL).Err(); err != nil { + t.Fatalf("准备 Redis 配置缓存失败：%v", err) + } + t.Cleanup(func() { + for _, definition := range registry.List() { + _ = redisClient.Del(context.Background(), constants.RedisSystemConfigKey(definition.Key)).Err() + } + }) + + app := newSystemConfigTestApp(handler, constants.UserTypeSuperAdmin) + listResponse, err := app.Test(httptest.NewRequest("GET", "/api/admin/system-configs?module=foundation&page=1&page_size=20", nil)) + if err != nil { + t.Fatalf("查询系统配置接口失败：%v", err) + } + if listResponse.StatusCode != fiber.StatusOK { + t.Fatalf("查询系统配置状态码错误：%d", listResponse.StatusCode) + } + var listBody struct { + Code int `json:"code"` + Data dto.SystemConfigListResponse `json:"data"` + } + if err := sonic.ConfigDefault.NewDecoder(listResponse.Body).Decode(&listBody); err != nil { + t.Fatalf("解析系统配置列表响应失败：%v", err) + } + _ = listResponse.Body.Close() + if listBody.Code != 0 || listBody.Data.Total != 4 || len(listBody.Data.List) != 4 { + t.Fatalf("四种注册类型未完整返回：%+v", listBody) + } + for _, item := range listBody.Data.List { + if item.Sensitive && item.Value != "[已配置]" { + t.Fatalf("敏感配置未脱敏：%+v", item) + } + } + + updateRequest := httptest.NewRequest("PUT", "/api/admin/system-configs/foundation.demo.int", strings.NewReader(`{"value":"8"}`)) + updateRequest.Header.Set("Content-Type", "application/json") + updateResponse, err := app.Test(updateRequest) + if err != nil { + t.Fatalf("更新系统配置接口失败：%v", err) + } + if updateResponse.StatusCode != fiber.StatusOK { + t.Fatalf("更新系统配置状态码错误：%d", updateResponse.StatusCode) + } + _ = updateResponse.Body.Close() + var stored model.SystemConfig + if err := db.Where("config_key = ?", "foundation.demo.int").First(&stored).Error; err != nil || stored.ConfigValue != "8" { + t.Fatalf("系统配置数据库事实错误：%v，记录：%+v", err, stored) + } + var auditCount int64 + if err := db.Table("test_system_config_audit").Where("config_key = ? AND operator_id = ?", stored.ConfigKey, 7).Count(&auditCount).Error; err != nil || auditCount != 1 { + t.Fatalf("系统配置审计事实错误：%v，数量：%d", err, auditCount) + } + if exists, err := redisClient.Exists(context.Background(), constants.RedisSystemConfigKey(stored.ConfigKey)).Result(); err != nil || exists != 0 { + t.Fatalf("更新后缓存未失效：%v，存在：%d", err, exists) + } +} + +func TestSystemConfigPermissionsValidationRollbackAndCacheFailure(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + testutil.CreateTemporarySystemConfigTable(t, db) + createTemporarySystemConfigAuditTable(t, db) + registry := newSystemConfigTestRegistry(t) + alerts := &recordingAlerts{} + reader := systemConfigInfra.NewReader(db, registry, failingCache{}, alerts) + service := systemConfigApp.NewUpdateService(db, registry, failingCache{}, &configAuditWriter{}, alerts, nil) + handler := admin.NewSystemConfigHandler(systemConfigQuery.NewListQuery(reader), service) + + forbiddenApp := newSystemConfigTestApp(handler, constants.UserTypePlatform) + response, err := forbiddenApp.Test(httptest.NewRequest("GET", "/api/admin/system-configs", nil)) + if err != nil { + t.Fatalf("执行无权限查询失败：%v", err) + } + if response.StatusCode != fiber.StatusForbidden { + t.Fatalf("权限不足必须返回 403，得到：%d", response.StatusCode) + } + _ = response.Body.Close() + + ctx := middleware.SetUserContext(context.Background(), &middleware.UserContextInfo{UserID: 7, UserType: constants.UserTypeSuperAdmin}) + invalidCases := []struct { + key string + value string + }{ + {key: "foundation.demo.int", value: "not-int"}, + {key: "foundation.demo.bool", value: "yes"}, + {key: "foundation.demo.json", value: "{"}, + {key: "foundation.unknown.value", value: "x"}, + {key: "foundation.demo.readonly", value: "x"}, + } + for _, item := range invalidCases { + if _, err := service.Execute(ctx, item.key, dto.UpdateSystemConfigRequest{Value: item.value}); err == nil { + t.Fatalf("非法配置更新应失败：%s=%s", item.key, item.value) + } + } + + failClosed := systemConfigApp.NewUpdateService(db, registry, failingCache{}, &configAuditWriter{fail: true}, alerts, nil) + if _, err := failClosed.Execute(ctx, "foundation.demo.string", dto.UpdateSystemConfigRequest{Value: "must-rollback"}); err == nil { + t.Fatal("审计写入失败时配置事务必须回滚") + } + var count int64 + if err := db.Model(&model.SystemConfig{}).Where("config_key = ?", "foundation.demo.string").Count(&count).Error; err != nil || count != 0 { + t.Fatalf("审计失败后配置事实未回滚：%v，数量：%d", err, count) + } + + if _, err := service.Execute(ctx, "foundation.demo.string", dto.UpdateSystemConfigRequest{Value: "committed"}); err != nil { + t.Fatalf("缓存失效失败不应回滚数据库事实：%v", err) + } + if len(alerts.codes) == 0 { + t.Fatal("缓存故障必须产生可观察告警") + } +} + +func newSystemConfigTestRegistry(t *testing.T) *systemConfigInfra.Registry { + t.Helper() + registry := systemConfigInfra.NewRegistry() + minimum, maximum := int64(1), int64(10) + definitions := []systemConfigInfra.Definition{ + {Key: "foundation.demo.string", Module: "foundation", ValueType: constants.SystemConfigTypeString, DefaultValue: "default", Description: "字符串示例", Control: "input"}, + {Key: "foundation.demo.int", Module: "foundation", ValueType: constants.SystemConfigTypeInt, DefaultValue: "5", Description: "整数示例", Control: "number", Min: &minimum, Max: &maximum}, + {Key: "foundation.demo.bool", Module: "foundation", ValueType: constants.SystemConfigTypeBool, DefaultValue: "true", Description: "布尔示例", Control: "switch"}, + {Key: "foundation.demo.json", Module: "foundation", ValueType: constants.SystemConfigTypeJSON, DefaultValue: `{"enabled":true}`, Description: "JSON 示例", Control: "structured", Sensitive: true}, + {Key: "foundation.demo.readonly", Module: "foundation-readonly", ValueType: constants.SystemConfigTypeString, DefaultValue: "fixed", Description: "只读示例", Control: "readonly", Readonly: true}, + } + for _, definition := range definitions { + if err := registry.Register(definition); err != nil { + t.Fatalf("注册测试配置失败：%v", err) + } + } + return registry +} + +func newSystemConfigTestApp(handler *admin.SystemConfigHandler, userType int) *fiber.App { + app := fiber.New(fiber.Config{JSONEncoder: sonic.Marshal, JSONDecoder: sonic.Unmarshal, ErrorHandler: internalMiddleware.ErrorHandler(zap.NewNop())}) + api := app.Group("/api/admin", func(c *fiber.Ctx) error { + ctx := middleware.SetUserContext(c.UserContext(), &middleware.UserContextInfo{UserID: 7, UserType: userType}) + ctx = context.WithValue(ctx, constants.ContextKeyRequestID, "request-system-config") + c.SetUserContext(ctx) + return c.Next() + }) + registerSystemConfigRoutes(api, handler, nil, "/api/admin") + return app +} + +func createTemporarySystemConfigAuditTable(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TEMP TABLE test_system_config_audit ( + id bigserial PRIMARY KEY, config_key varchar(150) NOT NULL, + operator_id bigint NOT NULL, request_id varchar(100) NOT NULL, + before_data jsonb NOT NULL, after_data jsonb NOT NULL + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建系统配置审计测试表失败：%v", err) + } +} diff --git a/internal/service/export_task/service.go b/internal/service/export_task/service.go index 37f6d26..de16be1 100644 --- a/internal/service/export_task/service.go +++ b/internal/service/export_task/service.go @@ -279,6 +279,7 @@ func (s *Service) CancelTask(ctx context.Context, id uint) (*dto.CancelExportTas func toTaskItemDTO(task *model.ExportTask) *dto.ExportTaskItem { return &dto.ExportTaskItem{ ID: task.ID, + TaskID: task.ID, TaskNo: task.TaskNo, Scene: task.Scene, Format: task.Format, @@ -290,10 +291,16 @@ func toTaskItemDTO(task *model.ExportTask) *dto.ExportTaskItem { TotalShards: task.TotalShards, SuccessShards: task.SuccessShards, FailedShards: task.FailedShards, + TotalCount: task.TotalShards, + SuccessCount: task.SuccessShards, + FailedCount: task.FailedShards, CancelRequested: task.CancelRequested, FileKey: task.FileKey, ErrorMessage: task.ErrorMessage, + ErrorCode: exportTaskErrorCode(task), + ErrorSummary: task.ErrorMessage, CreatedAt: task.CreatedAt, + UpdatedAt: task.UpdatedAt, StartedAt: task.StartedAt, CompletedAt: task.CompletedAt, CreatorUserID: task.CreatorUserID, @@ -302,3 +309,10 @@ func toTaskItemDTO(task *model.ExportTask) *dto.ExportTaskItem { CreatorEnterpriseID: task.CreatorEnterpriseID, } } + +func exportTaskErrorCode(task *model.ExportTask) string { + if task.ErrorMessage == "" { + return "" + } + return "EXPORT_TASK_FAILED" +} diff --git a/internal/testutil/outbox.go b/internal/testutil/outbox.go new file mode 100644 index 0000000..1f6389f --- /dev/null +++ b/internal/testutil/outbox.go @@ -0,0 +1,27 @@ +package testutil + +import ( + "testing" + + "gorm.io/gorm" +) + +// CreateTemporaryOutboxTable 创建事务内自动回滚的公共 Outbox 测试表。 +func CreateTemporaryOutboxTable(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TEMP TABLE tb_outbox_event ( + id bigserial PRIMARY KEY, event_id varchar(64) NOT NULL UNIQUE, + event_type varchar(150) NOT NULL, payload_version integer NOT NULL, + aggregate_type varchar(100) NOT NULL, aggregate_id varchar(100) NOT NULL, + resource_type varchar(100) NOT NULL, resource_id varchar(100) NOT NULL, + business_key varchar(150) NOT NULL DEFAULT '', request_id varchar(100) NOT NULL DEFAULT '', + correlation_id varchar(100) NOT NULL DEFAULT '', payload jsonb NOT NULL, + status integer NOT NULL, retry_count integer NOT NULL DEFAULT 0, + max_retries integer NOT NULL, next_attempt_at timestamptz NOT NULL, + lease_owner varchar(100), lease_expires_at timestamptz, + last_error_code varchar(100) NOT NULL DEFAULT '', last_error_summary varchar(500) NOT NULL DEFAULT '', + delivered_at timestamptz, created_at timestamptz NOT NULL DEFAULT NOW(), updated_at timestamptz NOT NULL DEFAULT NOW() + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建 Outbox 测试表失败：%v", err) + } +} diff --git a/internal/testutil/system_config.go b/internal/testutil/system_config.go new file mode 100644 index 0000000..5ecc889 --- /dev/null +++ b/internal/testutil/system_config.go @@ -0,0 +1,22 @@ +package testutil + +import ( + "testing" + + "gorm.io/gorm" +) + +// CreateTemporarySystemConfigTable 创建事务内自动回滚的系统配置测试表。 +func CreateTemporarySystemConfigTable(t *testing.T, db *gorm.DB) { + t.Helper() + if err := db.Exec(`CREATE TEMP TABLE tb_system_config ( + id bigserial PRIMARY KEY, config_key varchar(150) NOT NULL UNIQUE, + config_value text NOT NULL, value_type varchar(20) NOT NULL, + module varchar(100) NOT NULL, description varchar(500) NOT NULL, + is_readonly boolean NOT NULL DEFAULT false, is_sensitive boolean NOT NULL DEFAULT false, + creator bigint NOT NULL DEFAULT 0, updater bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT NOW(), updated_at timestamptz NOT NULL DEFAULT NOW() + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建系统配置测试表失败：%v", err) + } +} diff --git a/migrations/000165_create_public_outbox.down.sql b/migrations/000165_create_public_outbox.down.sql new file mode 100644 index 0000000..6d25027 --- /dev/null +++ b/migrations/000165_create_public_outbox.down.sql @@ -0,0 +1,11 @@ +-- 已产生可靠事件后禁止通过降级删除事实，只允许空表回滚。 +DO $$ +BEGIN + IF to_regclass('tb_outbox_event') IS NOT NULL + AND EXISTS (SELECT 1 FROM tb_outbox_event LIMIT 1) THEN + RAISE EXCEPTION 'tb_outbox_event 已存在事件事实，禁止删表回滚，请停止生产者和 Relay 后向前修复'; + END IF; +END +$$; + +DROP TABLE IF EXISTS tb_outbox_event; diff --git a/migrations/000165_create_public_outbox.up.sql b/migrations/000165_create_public_outbox.up.sql new file mode 100644 index 0000000..b2ccac6 --- /dev/null +++ b/migrations/000165_create_public_outbox.up.sql @@ -0,0 +1,44 @@ +-- 创建公共 Outbox；该表及全部公共索引、约束只由 tech-public-foundation 维护。 +CREATE TABLE tb_outbox_event ( + id bigserial PRIMARY KEY, + event_id varchar(64) NOT NULL, + event_type varchar(150) NOT NULL, + payload_version integer NOT NULL DEFAULT 1, + aggregate_type varchar(100) NOT NULL, + aggregate_id varchar(100) NOT NULL, + resource_type varchar(100) NOT NULL, + resource_id varchar(100) NOT NULL, + business_key varchar(150) NOT NULL DEFAULT '', + request_id varchar(100) NOT NULL DEFAULT '', + correlation_id varchar(100) NOT NULL DEFAULT '', + payload jsonb NOT NULL, + status integer NOT NULL DEFAULT 1, + retry_count integer NOT NULL DEFAULT 0, + max_retries integer NOT NULL DEFAULT 10, + next_attempt_at timestamptz NOT NULL DEFAULT NOW(), + lease_owner varchar(100), + lease_expires_at timestamptz, + last_error_code varchar(100) NOT NULL DEFAULT '', + last_error_summary varchar(500) NOT NULL DEFAULT '', + delivered_at timestamptz, + created_at timestamptz NOT NULL DEFAULT NOW(), + updated_at timestamptz NOT NULL DEFAULT NOW(), + CONSTRAINT uq_outbox_event_id UNIQUE (event_id), + CONSTRAINT ck_outbox_event_status CHECK (status IN (1, 2, 3, 4)), + CONSTRAINT ck_outbox_event_payload_version CHECK (payload_version > 0), + CONSTRAINT ck_outbox_event_retry_count CHECK (retry_count >= 0 AND max_retries > 0) +); + +CREATE INDEX idx_outbox_event_claim + ON tb_outbox_event (status, next_attempt_at, lease_expires_at, created_at); +CREATE INDEX idx_outbox_event_type_status + ON tb_outbox_event (event_type, status); +CREATE INDEX idx_outbox_event_aggregate + ON tb_outbox_event (aggregate_type, aggregate_id, created_at); +CREATE INDEX idx_outbox_event_resource + ON tb_outbox_event (resource_type, resource_id, created_at); +CREATE INDEX idx_outbox_event_request_id ON tb_outbox_event (request_id) WHERE request_id <> ''; +CREATE INDEX idx_outbox_event_correlation_id ON tb_outbox_event (correlation_id) WHERE correlation_id <> ''; + +COMMENT ON TABLE tb_outbox_event IS '公共可靠事件 Outbox'; +COMMENT ON COLUMN tb_outbox_event.status IS '投递状态 1=待投递, 2=投递中, 3=已投递, 4=投递失败'; diff --git a/migrations/000166_create_system_config.down.sql b/migrations/000166_create_system_config.down.sql new file mode 100644 index 0000000..add6eb9 --- /dev/null +++ b/migrations/000166_create_system_config.down.sql @@ -0,0 +1,11 @@ +-- 已产生配置事实后禁止通过降级删除，只允许空表回滚。 +DO $$ +BEGIN + IF to_regclass('tb_system_config') IS NOT NULL + AND EXISTS (SELECT 1 FROM tb_system_config LIMIT 1) THEN + RAISE EXCEPTION 'tb_system_config 已存在配置事实，禁止删表回滚，请停止写入后向前修复'; + END IF; +END +$$; + +DROP TABLE IF EXISTS tb_system_config; diff --git a/migrations/000166_create_system_config.up.sql b/migrations/000166_create_system_config.up.sql new file mode 100644 index 0000000..ea43c0c --- /dev/null +++ b/migrations/000166_create_system_config.up.sql @@ -0,0 +1,22 @@ +-- 创建受控系统配置表；具体业务 Key 由所属业务模块注册，不在公共迁移中预置。 +CREATE TABLE tb_system_config ( + id bigserial PRIMARY KEY, + config_key varchar(150) NOT NULL, + config_value text NOT NULL, + value_type varchar(20) NOT NULL, + module varchar(100) NOT NULL, + description varchar(500) NOT NULL, + is_readonly boolean NOT NULL DEFAULT false, + is_sensitive boolean NOT NULL DEFAULT false, + creator bigint NOT NULL DEFAULT 0, + updater bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT NOW(), + updated_at timestamptz NOT NULL DEFAULT NOW(), + CONSTRAINT uq_system_config_key UNIQUE (config_key), + CONSTRAINT ck_system_config_value_type CHECK (value_type IN ('string', 'int', 'bool', 'json')) +); + +CREATE INDEX idx_system_config_module_key ON tb_system_config (module, config_key); + +COMMENT ON TABLE tb_system_config IS '受控系统配置表'; +COMMENT ON COLUMN tb_system_config.value_type IS '值类型 string/int/bool/json'; diff --git a/pkg/asynctask/contract.go b/pkg/asynctask/contract.go new file mode 100644 index 0000000..d740b3d --- /dev/null +++ b/pkg/asynctask/contract.go @@ -0,0 +1,103 @@ +// Package asynctask 定义业务异步任务的统一公开契约。 +package asynctask + +import ( + stderrors "errors" + "reflect" + "time" +) + +const ( + // StatusPending 表示任务待处理。 + StatusPending = 1 + // StatusProcessing 表示任务处理中。 + StatusProcessing = 2 + // StatusCompleted 表示任务已到达业务处理终点。 + StatusCompleted = 3 + // StatusFailed 表示任务整体无法执行到业务终点。 + StatusFailed = 4 + // StatusCancelled 表示业务明确支持且已经取消任务。 + StatusCancelled = 5 +) + +var statusNames = map[int]string{ + StatusPending: "待处理", + StatusProcessing: "处理中", + StatusCompleted: "已完成", + StatusFailed: "已失败", + StatusCancelled: "已取消", +} + +// Projection 是跨业务统一的任务公开投影。 +type Projection struct { + TaskID string `json:"task_id"` + Status int `json:"status"` + StatusName string `json:"status_name"` + TotalCount int `json:"total_count"` + SuccessCount int `json:"success_count"` + FailedCount int `json:"failed_count"` + Progress int `json:"progress"` + ErrorCode string `json:"error_code,omitempty"` + ErrorSummary string `json:"error_summary,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// TerminalResult 是构造终态投影所需的业务结果。 +type TerminalResult struct { + TaskID string + Status int + TotalCount int + SuccessCount int + FailedCount int + ErrorCode string + ErrorSummary string + StartedAt *time.Time + CompletedAt *time.Time + UpdatedAt time.Time +} + +// StatusName 返回固定任务状态的中文名称。 +func StatusName(status int) string { + return statusNames[status] +} + +// IsTerminal 判断状态是否为统一终态。 +func IsTerminal(status int) bool { + return status == StatusCompleted || status == StatusFailed || status == StatusCancelled +} + +// NewTerminalProjection 校验终态守恒并构造公开投影。 +func NewTerminalProjection(result TerminalResult) (Projection, error) { + if !IsTerminal(result.Status) { + return Projection{}, stderrors.New("任务状态不是终态") + } + if result.TotalCount < 0 || result.SuccessCount < 0 || result.FailedCount < 0 { + return Projection{}, stderrors.New("任务结果计数不能为负数") + } + if result.Status == StatusCompleted && result.TotalCount != result.SuccessCount+result.FailedCount { + return Projection{}, stderrors.New("已完成任务的结果计数不守恒") + } + return Projection{ + TaskID: result.TaskID, Status: result.Status, StatusName: StatusName(result.Status), + TotalCount: result.TotalCount, SuccessCount: result.SuccessCount, FailedCount: result.FailedCount, + Progress: 100, ErrorCode: result.ErrorCode, ErrorSummary: result.ErrorSummary, + StartedAt: result.StartedAt, CompletedAt: result.CompletedAt, UpdatedAt: result.UpdatedAt, + }, nil +} + +// ValidatePayload 确保统一队列客户端接收 struct 或 map，而非预序列化字节。 +func ValidatePayload(payload any) error { + if payload == nil { + return stderrors.New("任务载荷不能为空") + } + kind := reflect.TypeOf(payload).Kind() + if kind != reflect.Struct && kind != reflect.Map && kind != reflect.Ptr { + return stderrors.New("任务载荷必须是结构体或 map") + } + if kind == reflect.Ptr && reflect.TypeOf(payload).Elem().Kind() != reflect.Struct { + return stderrors.New("任务载荷指针必须指向结构体") + } + return nil +} diff --git a/pkg/asynctask/contract_test.go b/pkg/asynctask/contract_test.go new file mode 100644 index 0000000..2e8f9cc --- /dev/null +++ b/pkg/asynctask/contract_test.go @@ -0,0 +1,105 @@ +package asynctask_test + +import ( + "testing" + "time" + + "github.com/break/junhong_cmp_fiber/pkg/asynctask" +) + +func TestCompletedTaskUsesCountsForPartialAndAllItemFailures(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + cases := []struct { + name string + success int + failed int + }{ + {name: "部分成功", success: 7, failed: 3}, + {name: "业务项全部失败", success: 0, failed: 10}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + projection, err := asynctask.NewTerminalProjection(asynctask.TerminalResult{ + TaskID: "task-1", Status: asynctask.StatusCompleted, + TotalCount: 10, SuccessCount: tc.success, FailedCount: tc.failed, + StartedAt: &now, CompletedAt: &now, UpdatedAt: now, + }) + if err != nil { + t.Fatalf("构造任务投影失败：%v", err) + } + if projection.Status != asynctask.StatusCompleted || projection.StatusName != "已完成" { + t.Fatalf("业务项处理到终点必须为已完成：%+v", projection) + } + if projection.Progress != 100 { + t.Fatalf("终态进度必须为 100，得到：%d", projection.Progress) + } + }) + } +} + +func TestTerminalProjectionRejectsBrokenCountInvariant(t *testing.T) { + t.Parallel() + + _, err := asynctask.NewTerminalProjection(asynctask.TerminalResult{ + TaskID: "task-2", Status: asynctask.StatusCompleted, + TotalCount: 10, SuccessCount: 5, FailedCount: 4, + UpdatedAt: time.Now(), + }) + if err == nil { + t.Fatal("终态计数不守恒时应拒绝构造公开投影") + } +} + +func TestStructuredPayloadRejectsBytes(t *testing.T) { + t.Parallel() + + if err := asynctask.ValidatePayload([]byte(`{"task_id":1}`)); err == nil { + t.Fatal("预序列化 []byte 载荷必须被拒绝") + } + if err := asynctask.ValidatePayload(struct { + TaskID uint `json:"task_id"` + }{TaskID: 1}); err != nil { + t.Fatalf("结构化载荷应通过校验：%v", err) + } +} + +func TestClaimRecoversOnlyExpiredProcessingTask(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 7, 23, 10, 0, 0, 0, time.UTC) + activeExpiry := now.Add(time.Minute) + active, err := asynctask.Claim(asynctask.LeaseState{ + Status: asynctask.StatusProcessing, LeaseOwner: "worker-a", LeaseExpiresAt: &activeExpiry, + }, "worker-b", now, time.Minute) + if err != nil { + t.Fatalf("检查有效租约失败：%v", err) + } + if active.Claimed { + t.Fatal("其他 Worker 不得抢占有效租约") + } + + expiredAt := now.Add(-time.Second) + recovered, err := asynctask.Claim(asynctask.LeaseState{ + Status: asynctask.StatusProcessing, LeaseOwner: "worker-a", LeaseExpiresAt: &expiredAt, + }, "worker-b", now, time.Minute) + if err != nil { + t.Fatalf("恢复过期任务失败：%v", err) + } + if !recovered.Claimed || recovered.State.LeaseOwner != "worker-b" { + t.Fatalf("过期任务应由新 Worker 恢复：%+v", recovered) + } +} + +func TestTerminalTaskCannotBeClaimedOrCancelledAgain(t *testing.T) { + t.Parallel() + + result, err := asynctask.Claim(asynctask.LeaseState{Status: asynctask.StatusCompleted}, "worker-a", time.Now(), time.Minute) + if err != nil { + t.Fatalf("检查终态任务失败：%v", err) + } + if result.Claimed || asynctask.CanCancel(asynctask.StatusCompleted) { + t.Fatal("终态任务不得被重复消费或取消") + } +} diff --git a/pkg/asynctask/state.go b/pkg/asynctask/state.go new file mode 100644 index 0000000..365bbba --- /dev/null +++ b/pkg/asynctask/state.go @@ -0,0 +1,53 @@ +package asynctask + +import ( + stderrors "errors" + "time" +) + +// LeaseState 是业务任务持久化层应保存的最小状态与租约事实。 +type LeaseState struct { + Status int + LeaseOwner string + LeaseExpiresAt *time.Time +} + +// ClaimResult 表示条件领取后的新事实。 +type ClaimResult struct { + State LeaseState + Claimed bool +} + +// Claim 按统一语义判断待处理或租约过期任务能否被领取。 +// +// 业务持久化 Adapter 必须把同样的条件放进 PostgreSQL UPDATE 的 WHERE 中； +// 本函数用于跨业务共享状态判定，不能替代数据库条件更新。 +func Claim(current LeaseState, owner string, now time.Time, leaseDuration time.Duration) (ClaimResult, error) { + if owner == "" || leaseDuration <= 0 { + return ClaimResult{}, stderrors.New("任务租约所有者和时长不能为空") + } + claimable := current.Status == StatusPending + if current.Status == StatusProcessing && current.LeaseExpiresAt != nil && !current.LeaseExpiresAt.After(now) { + claimable = true + } + if !claimable { + return ClaimResult{State: current}, nil + } + expiresAt := now.Add(leaseDuration) + return ClaimResult{ + State: LeaseState{Status: StatusProcessing, LeaseOwner: owner, LeaseExpiresAt: &expiresAt}, + Claimed: true, + }, nil +} + +// CanFinish 判断当前租约所有者能否从处理中进入终态。 +func CanFinish(current LeaseState, owner string, now time.Time) bool { + return current.Status == StatusProcessing && + current.LeaseOwner == owner && + current.LeaseExpiresAt != nil && current.LeaseExpiresAt.After(now) +} + +// CanCancel 判断业务支持取消时是否可用预期状态条件更新。 +func CanCancel(status int) bool { + return status == StatusPending || status == StatusProcessing +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 160aa89..f42f154 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,6 +1,10 @@ package constants -import "time" +import ( + "time" + + "github.com/break/junhong_cmp_fiber/pkg/asynctask" +) // Fiber Locals 的上下文键 const ( @@ -79,6 +83,7 @@ const ( TaskTypeAlertCheck = "alert:check" // 告警检查 TaskTypeDataCleanup = "data:cleanup" // 数据清理 TaskTypeDailyTrafficFlush = "traffic:daily:flush" // 每日流量落盘 + TaskTypeOutboxDeliver = "outbox:deliver" // 公共 Outbox 事件投递 ) // 用户状态常量 @@ -219,6 +224,7 @@ const ( QueueAlertCheck = TaskTypeAlertCheck // 告警检查任务队列 QueueDataCleanup = TaskTypeDataCleanup // 数据清理任务队列 QueueDailyTrafficFlush = TaskTypeDailyTrafficFlush // 每日流量落盘任务队列 + QueueOutboxDeliver = TaskTypeOutboxDeliver // 公共 Outbox 投递队列 DefaultRetryMax = 5 // 默认任务最大重试次数 DefaultTimeout = 10 * time.Minute // 默认任务超时时间 @@ -276,6 +282,8 @@ func QueueForTaskType(taskType string) string { return QueueDataCleanup case TaskTypeDailyTrafficFlush: return QueueDailyTrafficFlush + case TaskTypeOutboxDeliver: + return QueueOutboxDeliver default: return QueueDefault } @@ -309,6 +317,7 @@ func DefaultTaskQueueWeights() map[string]int { QueuePackageDataReset: 1, QueueDataCleanup: 1, QueueDailyTrafficFlush: 1, + QueueOutboxDeliver: 4, QueueDefault: 1, QueueLow: 1, } @@ -329,11 +338,11 @@ const ( // 导出主任务状态常量 const ( - ExportTaskStatusPending = 1 // 待处理 - ExportTaskStatusProcessing = 2 // 处理中 - ExportTaskStatusCompleted = 3 // 已完成 - ExportTaskStatusFailed = 4 // 已失败 - ExportTaskStatusCancelled = 5 // 已取消 + ExportTaskStatusPending = asynctask.StatusPending // 待处理 + ExportTaskStatusProcessing = asynctask.StatusProcessing // 处理中 + ExportTaskStatusCompleted = asynctask.StatusCompleted // 已完成 + ExportTaskStatusFailed = asynctask.StatusFailed // 已失败 + ExportTaskStatusCancelled = asynctask.StatusCancelled // 已取消 ) // 导出分片任务状态常量 @@ -450,20 +459,11 @@ func GetShelfStatusName(status int) string { // GetExportTaskStatusName 获取导出主任务状态名称 func GetExportTaskStatusName(status int) string { - switch status { - case ExportTaskStatusPending: - return "待处理" - case ExportTaskStatusProcessing: - return "处理中" - case ExportTaskStatusCompleted: - return "已完成" - case ExportTaskStatusFailed: - return "已失败" - case ExportTaskStatusCancelled: - return "已取消" - default: + name := asynctask.StatusName(status) + if name == "" { return "未知" } + return name } // GetExportShardStatusName 获取导出分片任务状态名称 diff --git a/pkg/constants/outbox.go b/pkg/constants/outbox.go new file mode 100644 index 0000000..9ffd9a0 --- /dev/null +++ b/pkg/constants/outbox.go @@ -0,0 +1,45 @@ +package constants + +import "time" + +const ( + // OutboxStatusPending 表示事件待投递。 + OutboxStatusPending = 1 + // OutboxStatusDelivering 表示事件已被 Relay 租约领取。 + OutboxStatusDelivering = 2 + // OutboxStatusDelivered 表示事件已成功提交到队列。 + OutboxStatusDelivered = 3 + // OutboxStatusFailed 表示事件达到最终失败或等待人工重放。 + OutboxStatusFailed = 4 +) + +const ( + // OutboxDefaultMaxRetries 是公共 Outbox 默认最大重试次数。 + OutboxDefaultMaxRetries = 10 + // OutboxDefaultLeaseDuration 是 Relay 默认租约时长。 + OutboxDefaultLeaseDuration = time.Minute + // OutboxDefaultBatchSize 是 Relay 默认单批领取数。 + OutboxDefaultBatchSize = 100 + // OutboxBaseRetryDelay 是瞬时失败指数退避的基础时长。 + OutboxBaseRetryDelay = 5 * time.Second + // OutboxMaxRetryDelay 是瞬时失败指数退避的上限。 + OutboxMaxRetryDelay = 30 * time.Minute + // OutboxRelayPollInterval 是 Worker 扫描到期事件的默认间隔。 + OutboxRelayPollInterval = time.Second +) + +// GetOutboxStatusName 返回 Outbox 内部状态中文名称。 +func GetOutboxStatusName(status int) string { + switch status { + case OutboxStatusPending: + return "待投递" + case OutboxStatusDelivering: + return "投递中" + case OutboxStatusDelivered: + return "已投递" + case OutboxStatusFailed: + return "投递失败" + default: + return "未知" + } +} diff --git a/pkg/constants/system_config.go b/pkg/constants/system_config.go new file mode 100644 index 0000000..eeb0a14 --- /dev/null +++ b/pkg/constants/system_config.go @@ -0,0 +1,24 @@ +package constants + +import ( + "fmt" + "time" +) + +const ( + // SystemConfigTypeString 表示字符串配置。 + SystemConfigTypeString = "string" + // SystemConfigTypeInt 表示整数配置。 + SystemConfigTypeInt = "int" + // SystemConfigTypeBool 表示布尔配置。 + SystemConfigTypeBool = "bool" + // SystemConfigTypeJSON 表示 JSON 配置。 + SystemConfigTypeJSON = "json" + // SystemConfigCacheTTL 是系统配置默认缓存时长。 + SystemConfigCacheTTL = 5 * time.Minute +) + +// RedisSystemConfigKey 生成单个系统配置的 Redis 缓存键。 +func RedisSystemConfigKey(configKey string) string { + return fmt.Sprintf("system_config:value:%s", configKey) +} diff --git a/pkg/idempotency/fingerprint.go b/pkg/idempotency/fingerprint.go new file mode 100644 index 0000000..2b31c96 --- /dev/null +++ b/pkg/idempotency/fingerprint.go @@ -0,0 +1,124 @@ +// Package idempotency 提供创建类命令的幂等身份与请求指纹公共契约。 +package idempotency + +import ( + "crypto/sha256" + "encoding/hex" + "reflect" + "strings" + + "github.com/bytedance/sonic" +) + +const ( + // FingerprintAlgorithmV1 是请求指纹算法的首个稳定版本。 + FingerprintAlgorithmV1 = "sha256-canonical-json-v1" +) + +// Scope 表示请求 ID 的幂等作用域。 +type Scope struct { + Subject string `json:"subject"` + Operation string `json:"operation"` +} + +// FingerprintValue 表示带算法版本的请求指纹。 +type FingerprintValue struct { + Algorithm string `json:"algorithm"` + Value string `json:"value"` +} + +// Record 表示业务模块自行持久化的幂等事实最小投影。 +type Record struct { + Scope Scope `json:"scope"` + RequestID string `json:"request_id"` + Fingerprint FingerprintValue `json:"fingerprint"` +} + +// ReplayKind 表示已有事实与本次命令的关系。 +type ReplayKind string + +const ( + // ReplaySame 表示相同命令重放，应返回原业务结果。 + ReplaySame ReplayKind = "same" + // ReplayConflict 表示同作用域、同请求 ID 携带了不同业务内容。 + ReplayConflict ReplayKind = "conflict" + // ReplayUnrelated 表示不是同一幂等作用域内的命令。 + ReplayUnrelated ReplayKind = "unrelated" +) + +var volatileFieldNames = map[string]struct{}{ + "authorization": {}, + "cookie": {}, + "nonce": {}, + "sign": {}, + "signature": {}, + "timestamp": {}, + "token": {}, +} + +// Fingerprint 对影响业务结果的规范化字段计算稳定指纹。 +// +// 调用方仍应优先传入专用命令 DTO；本函数会递归排除约定的易变传输字段， +// 并去除字符串首尾空白，避免签名、令牌或对象字段顺序污染业务身份。 +func Fingerprint(command any) (FingerprintValue, error) { + raw, err := sonic.ConfigStd.Marshal(command) + if err != nil { + return FingerprintValue{}, err + } + var value any + if err := sonic.Unmarshal(raw, &value); err != nil { + return FingerprintValue{}, err + } + normalized := normalize(value) + canonical, err := sonic.ConfigStd.Marshal(normalized) + if err != nil { + return FingerprintValue{}, err + } + sum := sha256.Sum256(canonical) + return FingerprintValue{ + Algorithm: FingerprintAlgorithmV1, + Value: hex.EncodeToString(sum[:]), + }, nil +} + +// ValidateScope 校验幂等作用域和稳定请求 ID 是否完整。 +func ValidateScope(scope Scope, requestID string) bool { + return strings.TrimSpace(scope.Subject) != "" && + strings.TrimSpace(scope.Operation) != "" && + strings.TrimSpace(requestID) != "" +} + +// Classify 比较已有幂等事实与本次命令。 +func Classify(existing Record, scope Scope, requestID string, fingerprint FingerprintValue) ReplayKind { + if existing.Scope != scope || existing.RequestID != requestID { + return ReplayUnrelated + } + if reflect.DeepEqual(existing.Fingerprint, fingerprint) { + return ReplaySame + } + return ReplayConflict +} + +func normalize(value any) any { + switch typed := value.(type) { + case map[string]any: + result := make(map[string]any, len(typed)) + for key, item := range typed { + if _, volatile := volatileFieldNames[strings.ToLower(strings.TrimSpace(key))]; volatile { + continue + } + result[key] = normalize(item) + } + return result + case []any: + result := make([]any, len(typed)) + for index, item := range typed { + result[index] = normalize(item) + } + return result + case string: + return strings.TrimSpace(typed) + default: + return value + } +} diff --git a/pkg/idempotency/fingerprint_test.go b/pkg/idempotency/fingerprint_test.go new file mode 100644 index 0000000..01398d1 --- /dev/null +++ b/pkg/idempotency/fingerprint_test.go @@ -0,0 +1,58 @@ +package idempotency_test + +import ( + "testing" + + "github.com/break/junhong_cmp_fiber/pkg/idempotency" +) + +func TestFingerprintIgnoresTransportFieldsAndNormalizesObjectOrder(t *testing.T) { + t.Parallel() + + left, err := idempotency.Fingerprint(map[string]any{ + "amount": 100, + "remark": " 月度充值 ", + "token": "token-a", + "nested": map[string]any{"enabled": true, "count": 2}, + }) + if err != nil { + t.Fatalf("计算第一个请求指纹失败：%v", err) + } + right, err := idempotency.Fingerprint(map[string]any{ + "nested": map[string]any{"count": 2, "enabled": true}, + "timestamp": 1720000000, + "remark": "月度充值", + "amount": 100, + }) + if err != nil { + t.Fatalf("计算第二个请求指纹失败：%v", err) + } + + if left.Algorithm != idempotency.FingerprintAlgorithmV1 { + t.Fatalf("算法版本不正确：%s", left.Algorithm) + } + if left.Value != right.Value { + t.Fatalf("仅传输字段或对象顺序变化不应改变指纹：%s != %s", left.Value, right.Value) + } +} + +func TestClassifyReplayDistinguishesScopeAndConflict(t *testing.T) { + t.Parallel() + + original := idempotency.Record{ + Scope: idempotency.Scope{Subject: "account:7", Operation: "agent-recharge.create"}, + RequestID: "request-1", + Fingerprint: idempotency.FingerprintValue{Algorithm: idempotency.FingerprintAlgorithmV1, Value: "abc"}, + } + + if got := idempotency.Classify(original, original.Scope, "request-1", original.Fingerprint); got != idempotency.ReplaySame { + t.Fatalf("相同命令应识别为重放，得到：%s", got) + } + if got := idempotency.Classify(original, original.Scope, "request-1", idempotency.FingerprintValue{Algorithm: idempotency.FingerprintAlgorithmV1, Value: "def"}); got != idempotency.ReplayConflict { + t.Fatalf("同作用域同请求 ID 的不同内容应冲突，得到：%s", got) + } + otherScope := idempotency.Scope{Subject: "account:8", Operation: original.Scope.Operation} + if got := idempotency.Classify(original, otherScope, "request-1", original.Fingerprint); got != idempotency.ReplayUnrelated { + t.Fatalf("不同主体不应相互污染，得到：%s", got) + } +} diff --git a/pkg/idempotency/postgres_integration_test.go b/pkg/idempotency/postgres_integration_test.go new file mode 100644 index 0000000..4da6114 --- /dev/null +++ b/pkg/idempotency/postgres_integration_test.go @@ -0,0 +1,86 @@ +package idempotency_test + +import ( + "context" + "sync" + "testing" + + "github.com/break/junhong_cmp_fiber/internal/testutil" + "github.com/break/junhong_cmp_fiber/pkg/idempotency" +) + +func TestPostgresUniqueConstraintArbitratesConcurrentFirstSubmission(t *testing.T) { + db := testutil.NewPostgresTransaction(t) + if err := db.Exec(` + CREATE TEMP TABLE test_command_idempotency ( + subject varchar(100) NOT NULL, + operation varchar(100) NOT NULL, + request_id varchar(100) NOT NULL, + fingerprint varchar(128) NOT NULL, + result_id bigint NOT NULL, + UNIQUE (subject, operation, request_id) + ) ON COMMIT DROP`).Error; err != nil { + t.Fatalf("创建幂等契约测试表失败：%v", err) + } + + fingerprint, err := idempotency.Fingerprint(map[string]any{"amount": 100}) + if err != nil { + t.Fatalf("计算指纹失败：%v", err) + } + + start := make(chan struct{}) + results := make(chan int64, 2) + errs := make(chan error, 2) + var wait sync.WaitGroup + for _, resultID := range []int64{101, 102} { + wait.Add(1) + go func(id int64) { + defer wait.Done() + <-start + result := db.WithContext(context.Background()).Exec(` + INSERT INTO test_command_idempotency + (subject, operation, request_id, fingerprint, result_id) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (subject, operation, request_id) DO NOTHING`, + "account:7", "agent-recharge.create", "request-1", fingerprint.Value, id, + ) + results <- result.RowsAffected + errs <- result.Error + }(resultID) + } + close(start) + wait.Wait() + close(results) + close(errs) + + var inserted int64 + for resultErr := range errs { + if resultErr != nil { + t.Fatalf("并发写入失败：%v", resultErr) + } + } + for affected := range results { + inserted += affected + } + if inserted != 1 { + t.Fatalf("并发首写必须只有一个数据库事实，实际插入：%d", inserted) + } + + var count int64 + if err := db.Table("test_command_idempotency").Count(&count).Error; err != nil { + t.Fatalf("统计幂等事实失败：%v", err) + } + if count != 1 { + t.Fatalf("幂等事实数量错误：%d", count) + } +} + +func TestIdempotencyScopeDoesNotDependOnRedis(t *testing.T) { + t.Parallel() + + scope := idempotency.Scope{Subject: "account:7", Operation: "example.create"} + if !idempotency.ValidateScope(scope, "request-1") { + t.Fatal("完整作用域和请求 ID 应通过校验") + } + // 公共幂等契约没有 Redis 参数；Redis 仅可由业务作为削峰优化，不能参与最终裁决。 +} diff --git a/pkg/logger/access_policy.go b/pkg/logger/access_policy.go new file mode 100644 index 0000000..2d4747f --- /dev/null +++ b/pkg/logger/access_policy.go @@ -0,0 +1,130 @@ +package logger + +import ( + "net/url" + "path/filepath" + "strings" + + "github.com/bytedance/sonic" +) + +// AccessPolicy 是敏感路由的安全摘要策略。 +type AccessPolicy struct { + Name string + Sensitive bool + SafeFields map[string]struct{} + FileFields map[string]struct{} +} + +var ( + loginPolicy = AccessPolicy{ + Name: "login_token", Sensitive: true, + SafeFields: fieldSet("username", "user_id", "account_id", "success", "result_code"), + } + paymentPolicy = AccessPolicy{ + Name: "payment", Sensitive: true, + SafeFields: fieldSet("order_no", "payment_no", "channel", "payment_method", "result_code", "amount", "status"), + } + wecomPolicy = AccessPolicy{ + Name: "wecom_callback", Sensitive: true, + SafeFields: fieldSet("event_type", "resource_type", "resource_id", "result_code", "status"), + } + filePolicy = AccessPolicy{ + Name: "file_export", Sensitive: true, + SafeFields: fieldSet("content_type", "file_size", "size", "count", "task_id", "status", "result_code"), + FileFields: fieldSet("file_name", "filename", "name"), + } + defaultPolicy = AccessPolicy{Name: "default_json"} +) + +func fieldSet(fields ...string) map[string]struct{} { + result := make(map[string]struct{}, len(fields)) + for _, field := range fields { + result[field] = struct{}{} + } + return result +} + +func policyForPath(path string) AccessPolicy { + normalized := strings.ToLower(path) + switch { + case strings.Contains(normalized, "/login"), strings.Contains(normalized, "token"): + return loginPolicy + case strings.Contains(normalized, "/callback") && (strings.Contains(normalized, "wecom") || strings.Contains(normalized, "wework")): + return wecomPolicy + case strings.Contains(normalized, "payment"), strings.Contains(normalized, "wechat-pay"), + strings.Contains(normalized, "alipay"), strings.Contains(normalized, "fuiou-pay"): + return paymentPolicy + case strings.Contains(normalized, "/storage"), strings.Contains(normalized, "/upload"), + strings.Contains(normalized, "/download"), strings.Contains(normalized, "/export"): + return filePolicy + default: + return defaultPolicy + } +} + +func sanitizeWithPolicy(raw []byte, contentType string, policy AccessPolicy) SanitizedContent { + if len(raw) == 0 { + return SanitizedContent{} + } + if !policy.Sensitive { + return sanitizeBody(raw, BodyPolicyJSON) + } + summary := make(map[string]any) + normalizedContentType := strings.ToLower(contentType) + switch { + case strings.Contains(normalizedContentType, "json"): + var payload any + if sonic.Unmarshal(raw, &payload) == nil { + collectSafeFields(payload, policy, summary) + } + case strings.Contains(normalizedContentType, "x-www-form-urlencoded"): + if values, err := url.ParseQuery(string(raw)); err == nil { + for key, items := range values { + collectSafeScalar(key, strings.Join(items, ","), policy, summary) + } + } + } + content := "[仅记录安全摘要]" + if len(summary) > 0 { + if encoded, err := sonic.Marshal(summary); err == nil { + content, _ = truncateBody(encoded, MaxBodyLogSize) + } + } + return SanitizedContent{Content: content, Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize} +} + +func collectSafeFields(value any, policy AccessPolicy, output map[string]any) { + switch typed := value.(type) { + case map[string]any: + for key, item := range typed { + switch scalar := item.(type) { + case string: + collectSafeScalar(key, scalar, policy, output) + case float64, bool: + if _, allowed := policy.SafeFields[strings.ToLower(key)]; allowed { + output[key] = scalar + } + default: + collectSafeFields(item, policy, output) + } + } + case []any: + for _, item := range typed { + collectSafeFields(item, policy, output) + } + } +} + +func collectSafeScalar(key, value string, policy AccessPolicy, output map[string]any) { + normalized := strings.ToLower(key) + if _, isFileName := policy.FileFields[normalized]; isFileName { + base := filepath.Base(value) + hash := digest([]byte(base)) + output[key] = "sha256:" + hash[:16] + return + } + if _, allowed := policy.SafeFields[normalized]; allowed { + output[key] = value + } +} diff --git a/pkg/logger/middleware.go b/pkg/logger/middleware.go index ac96788..c7b19cf 100644 --- a/pkg/logger/middleware.go +++ b/pkg/logger/middleware.go @@ -2,6 +2,8 @@ package logger import ( "context" + "crypto/sha256" + "encoding/hex" "net/url" "strings" "time" @@ -15,20 +17,38 @@ import ( const ( // MaxBodyLogSize 限制记录的请求/响应 body 大小为 50KB MaxBodyLogSize = 50 * 1024 + redactedValue = "[已脱敏]" ) +// BodyPolicy 表示访问日志正文记录策略。 +type BodyPolicy int + +const ( + // BodyPolicyJSON 表示只允许记录脱敏后的 JSON 正文。 + BodyPolicyJSON BodyPolicy = iota + 1 + // BodyPolicySummary 表示只记录不可逆安全摘要。 + BodyPolicySummary +) + +// SanitizedContent 表示访问日志可安全记录的正文或查询摘要。 +type SanitizedContent struct { + Content string + Size int + SHA256 string + Truncated bool +} + // truncateBody 截断 body 到指定大小 -func truncateBody(body []byte, maxSize int) string { +func truncateBody(body []byte, maxSize int) (string, bool) { if len(body) == 0 { - return "" + return "", false } if len(body) <= maxSize { - return string(body) + return string(body), false } - // 超过限制，截断并添加提示 - return string(body[:maxSize]) + "... (truncated)" + return string(body[:maxSize]), true } // maskSensitiveValue 按字段名判断并脱敏访问日志中的敏感值 @@ -36,25 +56,20 @@ func maskSensitiveValue(key, value string) string { if value == "" { return value } - normalized := strings.ToLower(key) - if strings.Contains(normalized, "password") || - strings.Contains(normalized, "sign") || - strings.Contains(normalized, "nonce") || - strings.Contains(normalized, "token") || - strings.Contains(normalized, "secret") { - return "***" + if shouldMaskField(key) { + return redactedValue } return value } // sanitizeQuery 脱敏 query 中的密码、签名、nonce、token 等字段 -func sanitizeQuery(rawQuery string) string { +func sanitizeQuery(rawQuery string) SanitizedContent { if rawQuery == "" { - return "" + return SanitizedContent{} } values, err := url.ParseQuery(rawQuery) if err != nil { - return rawQuery + return summarize([]byte(rawQuery)) } for key, items := range values { for index, item := range items { @@ -62,24 +77,38 @@ func sanitizeQuery(rawQuery string) string { } values[key] = items } - return values.Encode() + content, truncated := truncateBody([]byte(values.Encode()), MaxBodyLogSize) + return SanitizedContent{Content: content, Size: len(rawQuery), SHA256: digest([]byte(rawQuery)), Truncated: truncated} } -// sanitizeBody 脱敏 JSON 请求体后再写入访问日志 -func sanitizeBody(rawBody []byte) string { +// sanitizeBody 按策略脱敏正文后再写入访问日志。 +func sanitizeBody(rawBody []byte, policy BodyPolicy) SanitizedContent { if len(rawBody) == 0 { - return "" + return SanitizedContent{} + } + if policy == BodyPolicySummary { + return summarize(rawBody) } var payload any if err := sonic.Unmarshal(rawBody, &payload); err != nil { - return truncateBody(rawBody, MaxBodyLogSize) + return summarize(rawBody) } sanitizeJSONValue(payload) data, err := sonic.Marshal(payload) if err != nil { - return truncateBody(rawBody, MaxBodyLogSize) + return summarize(rawBody) } - return truncateBody(data, MaxBodyLogSize) + content, truncated := truncateBody(data, MaxBodyLogSize) + return SanitizedContent{Content: content, Size: len(rawBody), SHA256: digest(rawBody), Truncated: truncated} +} + +func summarize(raw []byte) SanitizedContent { + return SanitizedContent{Content: "[仅记录安全摘要]", Size: len(raw), SHA256: digest(raw), Truncated: len(raw) > MaxBodyLogSize} +} + +func digest(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) } // sanitizeJSONValue 递归脱敏 JSON 对象中的敏感字段 @@ -92,7 +121,7 @@ func sanitizeJSONValue(value any) { continue } if shouldMaskField(key) { - typed[key] = "***" + typed[key] = redactedValue continue } sanitizeJSONValue(item) @@ -108,6 +137,14 @@ func sanitizeJSONValue(value any) { func shouldMaskField(key string) bool { normalized := strings.ToLower(key) return strings.Contains(normalized, "password") || + strings.Contains(normalized, "passwd") || + strings.Contains(normalized, "credential") || + strings.Contains(normalized, "authorization") || + strings.Contains(normalized, "cookie") || + strings.Contains(normalized, "key") || + strings.Contains(normalized, "url") || + strings.Contains(normalized, "qr_content") || + strings.Contains(normalized, "verification_code") || strings.Contains(normalized, "sign") || strings.Contains(normalized, "nonce") || strings.Contains(normalized, "token") || @@ -117,6 +154,12 @@ func shouldMaskField(key string) bool { // Middleware 创建 Fiber 日志中间件 // 记录所有 HTTP 请求到访问日志（包括请求和响应 body） func Middleware() fiber.Handler { + return MiddlewareWithLogger(GetAccessLogger()) +} + +// MiddlewareWithLogger 创建可注入访问日志器的 Fiber 中间件。 +// 生产环境使用 Middleware；该入口让集成测试捕获最终 JSON 日志而无需修改全局状态。 +func MiddlewareWithLogger(accessLogger *zap.Logger) fiber.Handler { return func(c *fiber.Ctx) error { // 记录请求开始时间 startTime := time.Now() @@ -136,10 +179,14 @@ func Middleware() fiber.Handler { c.SetUserContext(ctx) // 获取请求 body（在 c.Next() 之前读取） - requestBody := sanitizeBody(c.Body()) + policy := policyForPath(c.Path()) + requestBody := sanitizeWithPolicy(c.Body(), c.Get("Content-Type"), policy) // 获取 query 参数 queryParams := sanitizeQuery(string(c.Request().URI().QueryString())) + if policy.Sensitive && queryParams.Size > 0 { + queryParams = summarize([]byte(c.Request().URI().QueryString())) + } // 处理请求 err := c.Next() @@ -162,22 +209,29 @@ func Middleware() fiber.Handler { } // 获取响应 body - responseBody := truncateBody(c.Response().Body(), MaxBodyLogSize) + responseBody := sanitizeWithPolicy(c.Response().Body(), string(c.Response().Header.ContentType()), policy) // 记录访问日志 - accessLogger := GetAccessLogger() accessLogger.Info("", zap.String("method", c.Method()), zap.String("path", c.Path()), - zap.String("query", queryParams), + zap.String("body_policy", policy.Name), + zap.String("query", queryParams.Content), + zap.Bool("query_truncated", queryParams.Truncated), zap.Int("status", c.Response().StatusCode()), zap.Float64("duration_ms", float64(duration.Microseconds())/1000.0), zap.String("request_id", requestID), zap.String("ip", c.IP()), zap.String("user_agent", c.Get("User-Agent")), zap.Uint("user_id", userID), - zap.String("request_body", requestBody), - zap.String("response_body", responseBody), + zap.String("request_body", requestBody.Content), + zap.Int("request_body_size", requestBody.Size), + zap.String("request_body_sha256", requestBody.SHA256), + zap.Bool("request_body_truncated", requestBody.Truncated), + zap.String("response_body", responseBody.Content), + zap.Int("response_body_size", responseBody.Size), + zap.String("response_body_sha256", responseBody.SHA256), + zap.Bool("response_body_truncated", responseBody.Truncated), ) return err diff --git a/pkg/logger/middleware_test.go b/pkg/logger/middleware_test.go new file mode 100644 index 0000000..bad12ff --- /dev/null +++ b/pkg/logger/middleware_test.go @@ -0,0 +1,154 @@ +package logger + +import ( + "bytes" + "io" + "net/http/httptest" + "strings" + "testing" + + "github.com/gofiber/fiber/v2" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func TestSanitizeJSONRecursivelyMasksRequestAndResponseSecrets(t *testing.T) { + t.Parallel() + + raw := []byte(`{"Password":"p@ss","items":[{"access_token":"token-value","count":3}],"profile":{"private_url":"https://secret.example/a"}}`) + result := sanitizeBody(raw, BodyPolicyJSON) + if strings.Contains(result.Content, "p@ss") || strings.Contains(result.Content, "token-value") || strings.Contains(result.Content, "secret.example") { + t.Fatalf("访问日志仍包含敏感值：%s", result.Content) + } + if !strings.Contains(result.Content, redactedValue) { + t.Fatalf("访问日志未输出统一脱敏占位：%s", result.Content) + } + if result.Truncated { + t.Fatal("短载荷不应标记为截断") + } +} + +func TestFiberAccessLogSanitizesFinalRequestAndResponseJSON(t *testing.T) { + var output bytes.Buffer + encoderConfig := zap.NewProductionEncoderConfig() + log := zap.New(zapcore.NewCore( + zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(&output), zapcore.InfoLevel, + )) + app := fiber.New() + app.Use(func(c *fiber.Ctx) error { + c.Locals("requestid", "request-test-1") + return c.Next() + }) + app.Use(MiddlewareWithLogger(log)) + app.Post("/tokens", func(c *fiber.Ctx) error { + return c.JSON(fiber.Map{"access_token": "response-token", "data": fiber.Map{"ok": true}}) + }) + + request := httptest.NewRequest("POST", "/tokens?Authorization=query-secret&page=1", strings.NewReader(`{"password":"request-secret"}`)) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil { + t.Fatalf("执行 Fiber 请求失败：%v", err) + } + _, _ = io.Copy(io.Discard, response.Body) + _ = response.Body.Close() + + logged := output.String() + for _, secret := range []string{"request-secret", "response-token", "query-secret"} { + if strings.Contains(logged, secret) { + t.Fatalf("最终 JSON 访问日志泄露测试凭证 %q：%s", secret, logged) + } + } + for _, field := range []string{"request_id", "status", "duration_ms", "request_body_truncated", "response_body_truncated"} { + if !strings.Contains(logged, field) { + t.Fatalf("最终 JSON 访问日志缺少字段 %q：%s", field, logged) + } + } +} + +func TestSensitiveRouteMatrixNeverLogsRawPayload(t *testing.T) { + cases := []struct { + name string + path string + contentType string + body string + secret string + }{ + {name: "登录JSON", path: "/api/auth/login", contentType: "application/json", body: `{"username":"safe-user","password":"json-secret"}`, secret: "json-secret"}, + {name: "企微XML", path: "/api/callback/wecom", contentType: "application/xml", body: `xml-secret`, secret: "xml-secret"}, + {name: "支付表单", path: "/api/callback/alipay", contentType: "application/x-www-form-urlencoded", body: "order_no=SAFE-1&sign=form-secret", secret: "form-secret"}, + {name: "文件multipart", path: "/api/admin/storage/upload", contentType: "multipart/form-data; boundary=x", body: "--x\r\nContent-Disposition: form-data; name=\"file\"; filename=\"secret.bin\"\r\nContent-Type: application/octet-stream\r\n\r\nfile-bytes-secret\r\n--x--\r\n", secret: "file-bytes-secret"}, + {name: "导出二进制", path: "/api/admin/export-tasks/1/download", contentType: "application/octet-stream", body: "binary-secret", secret: "binary-secret"}, + {name: "支付不可解析", path: "/api/callback/wechat-pay", contentType: "application/json", body: `{"sign":"broken-secret"`, secret: "broken-secret"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var output bytes.Buffer + log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel)) + app := fiber.New() + app.Use(MiddlewareWithLogger(log)) + app.Post(tc.path, func(c *fiber.Ctx) error { + return c.Status(fiber.StatusOK).Type("json").SendString(`{"access_token":"response-secret","status":"ok"}`) + }) + request := httptest.NewRequest("POST", tc.path+"?signature=query-secret", strings.NewReader(tc.body)) + request.Header.Set("Content-Type", tc.contentType) + response, err := app.Test(request) + if err != nil { + t.Fatalf("执行敏感路由请求失败：%v", err) + } + _, _ = io.Copy(io.Discard, response.Body) + _ = response.Body.Close() + logged := output.String() + for _, secret := range []string{tc.secret, "query-secret", "response-secret"} { + if strings.Contains(logged, secret) { + t.Fatalf("敏感路由日志泄露 %q：%s", secret, logged) + } + } + if !strings.Contains(logged, "body_policy") || !strings.Contains(logged, "sha256") { + t.Fatalf("敏感路由日志缺少策略或摘要：%s", logged) + } + }) + } +} + +func TestSensitiveRouteLongBodyRecordsTruncationWithoutRawSecret(t *testing.T) { + var output bytes.Buffer + log := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(&output), zapcore.InfoLevel)) + app := fiber.New() + app.Use(MiddlewareWithLogger(log)) + app.Post("/api/admin/storage/upload", func(c *fiber.Ctx) error { return c.SendStatus(fiber.StatusOK) }) + secret := strings.Repeat("long-file-secret", MaxBodyLogSize) + request := httptest.NewRequest("POST", "/api/admin/storage/upload", strings.NewReader(secret)) + request.Header.Set("Content-Type", "application/octet-stream") + response, err := app.Test(request, 10000) + if err != nil { + t.Fatalf("执行超长敏感请求失败：%v", err) + } + _ = response.Body.Close() + logged := output.String() + if strings.Contains(logged, "long-file-secret") || !strings.Contains(logged, `"request_body_truncated":true`) { + t.Fatalf("超长敏感载荷未安全摘要或未标记截断：%s", logged) + } +} + +func TestSanitizeInvalidJSONNeverFallsBackToRawBody(t *testing.T) { + t.Parallel() + + raw := []byte(`{"password":"should-never-appear"`) + result := sanitizeBody(raw, BodyPolicyJSON) + if strings.Contains(result.Content, "should-never-appear") { + t.Fatalf("解析失败时不得回退原文：%s", result.Content) + } + if result.SHA256 == "" || result.Size != len(raw) { + t.Fatalf("解析失败摘要缺少大小或哈希：%+v", result) + } +} + +func TestSanitizeQueryUsesSameSensitiveFieldRules(t *testing.T) { + t.Parallel() + + query := sanitizeQuery("page=1&Authorization=Bearer-secret&nonce=abc") + if strings.Contains(query.Content, "Bearer-secret") || strings.Contains(query.Content, "abc") { + t.Fatalf("query 敏感值未脱敏：%s", query.Content) + } +} diff --git a/pkg/openapi/handlers.go b/pkg/openapi/handlers.go index 5219757..937384e 100644 --- a/pkg/openapi/handlers.go +++ b/pkg/openapi/handlers.go @@ -67,6 +67,7 @@ func BuildDocHandlers() *bootstrap.Handlers { OrderPackageInvalidate: admin.NewOrderPackageInvalidateHandler(nil), ClientWechat: app.NewClientWechatHandler(nil, nil, nil), SuperAdmin: admin.NewSuperAdminHandler(nil), + SystemConfig: admin.NewSystemConfigHandler(nil, nil), AgentOpenAPI: openapiHandler.NewHandler(nil, nil), } } diff --git a/pkg/queue/client.go b/pkg/queue/client.go index 5341ccb..e960d04 100644 --- a/pkg/queue/client.go +++ b/pkg/queue/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/break/junhong_cmp_fiber/pkg/asynctask" "github.com/break/junhong_cmp_fiber/pkg/config" "github.com/break/junhong_cmp_fiber/pkg/constants" "github.com/bytedance/sonic" @@ -39,10 +40,10 @@ func NewClient(redisClient *redis.Client, logger *zap.Logger) *Client { // payload 必须传入 struct 或 map，禁止传入 []byte。 // 传入 []byte 会导致 sonic.Marshal 将其 base64 编码，handler 解析时类型不匹配。 func (c *Client) EnqueueTask(ctx context.Context, taskType string, payload interface{}, opts ...asynq.Option) error { - if _, ok := payload.([]byte); ok { + if err := asynctask.ValidatePayload(payload); err != nil { c.logger.Error("任务载荷类型错误，禁止传入 []byte", - zap.String("task_type", taskType)) - return fmt.Errorf("task payload must be struct or map, got []byte") + zap.String("task_type", taskType), zap.Error(err)) + return err } // 内部统一序列化，调用方无需预先 Marshal diff --git a/pkg/queue/client_test.go b/pkg/queue/client_test.go new file mode 100644 index 0000000..6bb2027 --- /dev/null +++ b/pkg/queue/client_test.go @@ -0,0 +1,17 @@ +package queue + +import ( + "context" + "testing" + + "go.uber.org/zap" +) + +func TestEnqueueTaskRejectsPreSerializedBytesBeforeQueueAccess(t *testing.T) { + t.Parallel() + + client := &Client{logger: zap.NewNop()} + if err := client.EnqueueTask(context.Background(), "test:bytes", []byte(`{"task_id":1}`)); err == nil { + t.Fatal("预序列化 []byte 载荷必须在访问 Asynq 前被拒绝") + } +} diff --git a/scripts/test-tech-public-foundation.sh b/scripts/test-tech-public-foundation.sh new file mode 100755 index 0000000..f20aae4 --- /dev/null +++ b/scripts/test-tech-public-foundation.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# 本验收只运行公共基础拥有的公开接缝；真实依赖连接信息沿用项目本地配置。 +if [[ -f .env.local ]]; then + set -a + # shellcheck disable=SC1091 + source .env.local + set +a +fi + +GOCACHE="${GOCACHE:-/tmp/junhong-go-cache}" go test \ + ./pkg/idempotency \ + ./pkg/asynctask \ + ./pkg/logger \ + ./pkg/queue \ + ./internal/infrastructure/asynctask \ + ./internal/infrastructure/releasegate \ + ./internal/infrastructure/messaging/outbox \ + ./internal/query/outbox \ + ./internal/application/outbox \ + ./internal/infrastructure/systemconfig \ + ./internal/query/systemconfig \ + ./internal/application/systemconfig \ + ./internal/routes \ + -count=1