feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口

- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id
- tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情
- 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月
- 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选
- 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选
- 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选
- 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录
- 迁移 000272、000273 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
邹进
2026-11-05 18:47:00 +09:00
parent 8eb7d403c4
commit 94cbf2adab
39 changed files with 1794 additions and 140 deletions

View File

@@ -281,6 +281,8 @@ type PackageTrafficAdjustmentListRequest struct {
PackageUsageID uint `path:"package_usage_id" description:"套餐使用记录ID" required:"true"`
Page int `query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码,默认 1"`
PageSize int `query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页条数,默认 20"`
StartTime string `query:"start_time" description:"开始时间(RFC3339 或 2006-01-02 15:04:05,上海时间),选填;与结束时间同时传,跨度不超过 31 天"`
EndTime string `query:"end_time" description:"结束时间(不含),选填"`
}
// PackageTrafficAdjustmentItem 冲正流水。
@@ -312,6 +314,11 @@ type CardTrafficCollectionListRequest struct {
EndTime string `query:"end_time" validate:"required" required:"true" description:"结束时间(不含),最长跨度 31 天"`
Page int `query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码,默认 1"`
PageSize int `query:"page_size" validate:"omitempty,min=1,max=200" minimum:"1" maximum:"200" description:"每页条数,默认 50"`
Decision string `query:"decision" validate:"omitempty,max=32" description:"写入判断筛选(只作用于成功记录)"`
Source string `query:"source" validate:"omitempty,max=32" description:"来源筛选"`
Scene string `query:"scene" validate:"omitempty,max=64" description:"场景筛选"`
// IncludeFailures 为 true 时合并采集失败记录,按采集时间倒序返回。
IncludeFailures bool `query:"include_failures" description:"是否包含采集失败记录,默认 false"`
}
// CardTrafficCollectionItem 卡流量采集记录。
@@ -329,6 +336,13 @@ type CardTrafficCollectionItem struct {
StaleGuarded *bool `json:"stale_guarded" description:"写入时是否触发重置日防旧值"`
DeltaMB *float64 `json:"delta_mb" description:"写入时本次增量(MB)"`
EventID string `json:"event_id" description:"流量事件 ID"`
// RecordKind 区分成功表与失败表记录:success/failure。
RecordKind string `json:"record_kind" description:"记录类型:success 采集成功,failure 采集失败"`
ErrorCategory string `json:"error_category,omitempty" description:"错误类别(失败记录)"`
ProviderCode string `json:"provider_code,omitempty" description:"上游返回码(失败记录)"`
ProviderMessage string `json:"provider_message,omitempty" description:"上游返回信息或错误信息(失败记录)"`
DurationMS int `json:"duration_ms,omitempty" description:"调用耗时毫秒(失败记录)"`
GatewayRawItem
}
// CardTrafficCollectionListResponse 卡流量采集记录分页结果。

View File

@@ -140,6 +140,10 @@ type StopResumeRecordListRequest struct {
Page int `json:"page" query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码"`
PageSize int `json:"page_size" query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页数量"`
ICCID string `json:"iccid" query:"iccid" validate:"omitempty,max=30" maxLength:"30" description:"ICCID"`
Identifier string `json:"identifier" query:"identifier" validate:"omitempty,max=50" maxLength:"50" description:"资产标识(虚拟号/ICCID/MSISDN,设备支持虚拟号/IMEI/SN,取其绑定卡)"`
Scene string `json:"scene" query:"scene" validate:"omitempty,max=50" maxLength:"50" description:"场景"`
Reason string `json:"reason" query:"reason" validate:"omitempty,max=50" maxLength:"50" description:"停机原因"`
CommandID string `json:"command_id" query:"command_id" validate:"omitempty,max=64" maxLength:"64" description:"指令号(同一次停复机命令的多次尝试共用)"`
CarrierID *uint `json:"carrier_id" query:"carrier_id" validate:"omitempty" description:"运营商账户ID"`
Action string `json:"action" query:"action" validate:"omitempty,oneof=stop start" description:"动作 (stop:停机, start:复机)"`
Source string `json:"source" query:"source" validate:"omitempty,max=50" maxLength:"50" description:"来源"`
@@ -156,6 +160,9 @@ type CardStopResumeRecordListRequest struct {
PageSize int `json:"page_size" query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页数量"`
StartTime time.Time `json:"start_time" query:"start_time" description:"开始时间(RFC3339),为空取最近 31 天"`
EndTime time.Time `json:"end_time" query:"end_time" description:"结束时间(RFC3339),为空取当前"`
Action string `json:"action" query:"action" validate:"omitempty,oneof=stop start" description:"动作 (stop:停机, start:复机)"`
Result string `json:"result" query:"result" validate:"omitempty,oneof=success failed unknown already_target skipped" description:"结果 (success:成功, failed:失败, unknown:结果未知, already_target:已是目标状态, skipped:跳过)"`
Source string `json:"source" query:"source" validate:"omitempty,max=50" maxLength:"50" description:"来源"`
}
// StopResumeRecordResponse 停复机记录响应

View File

@@ -194,5 +194,8 @@ type PackageUsageDetailResponse struct {
PackageUsageID uint `json:"package_usage_id" description:"套餐使用记录ID"`
PackageName string `json:"package_name" description:"套餐名称"`
Records []PackageUsageDailyRecordResponse `json:"records" description:"流量日记录列表"`
TotalUsageMB int64 `json:"total_usage_mb" description:"总使用流量(MB)"`
TotalUsageMB int64 `json:"total_usage_mb" description:"总使用流量(MB),为筛选范围内全部记录之和"`
Total int64 `json:"total" description:"筛选范围内记录总数"`
Page int `json:"page" description:"页码;未传 page_size 时为 0,返回全部记录"`
PageSize int `json:"page_size" description:"每页条数;未传时为 0,返回全部记录"`
}

View File

@@ -0,0 +1,215 @@
package dto
import (
"encoding/json"
"time"
)
// RecordRangeQuery 是记录查询的公共条件:资产标识选填(不传查全部),时间范围必填(跨度不超过 31 天),分页。
type RecordRangeQuery struct {
Identifier string `query:"identifier" validate:"omitempty,max=50" maxLength:"50" description:"资产标识(虚拟号/ICCID/MSISDN,设备支持 IMEI/SN);不传查全部"`
StartTime string `query:"start_time" validate:"required" required:"true" description:"开始时间(RFC3339 或 2006-01-02 15:04:05,上海时间)"`
EndTime string `query:"end_time" validate:"required" required:"true" description:"结束时间(不含),跨度不超过 31 天"`
Page int `query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码,默认 1"`
PageSize int `query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页条数,默认 20,最大 100"`
}
// RecordDateRangeQuery 是按日期查询的公共条件:资产标识选填,日期范围必填(含首尾,跨度不超过 31 天),分页。
type RecordDateRangeQuery struct {
Identifier string `query:"identifier" validate:"omitempty,max=50" maxLength:"50" description:"资产标识(虚拟号/ICCID/MSISDN,设备支持 IMEI/SN);不传查全部"`
StartDate string `query:"start_date" validate:"required" required:"true" description:"开始日期(2006-01-02,含)"`
EndDate string `query:"end_date" validate:"required" required:"true" description:"结束日期(2006-01-02,含),跨度不超过 31 天"`
Page int `query:"page" validate:"omitempty,min=1" minimum:"1" description:"页码,默认 1"`
PageSize int `query:"page_size" validate:"omitempty,min=1,max=100" minimum:"1" maximum:"100" description:"每页条数,默认 20,最大 100"`
}
// GatewayRawItem 是采集记录保存的网关原始请求与响应。
type GatewayRawItem struct {
RawRequest json.RawMessage `json:"raw_request" description:"网关明文业务请求(加密前),未收集时为 null"`
HTTPStatus *int `json:"http_status" description:"HTTP 状态码,未拿到响应时为 null"`
GatewayCode *int `json:"gateway_code" description:"网关响应 code"`
GatewayMsg string `json:"gateway_msg" description:"网关响应 msg"`
GatewayTraceID string `json:"gateway_trace_id" description:"网关响应 trace_id"`
RawResponse json.RawMessage `json:"raw_response" description:"网关原始响应体(JSON 且不超过 4KB 时)"`
RawResponseText string `json:"raw_response_text" description:"网关原始响应体文本(非 JSON 或超过 4KB 截断时)"`
RawTruncated bool `json:"raw_truncated" description:"原始响应是否被截断"`
GatewayAttempts int `json:"gateway_attempts" description:"网关客户端内部 HTTP 尝试次数(原始数据为最后一次)"`
}
// TrafficCollectionListRequest 采集成功记录列表请求。
type TrafficCollectionListRequest struct {
RecordRangeQuery
CarrierID *uint `query:"carrier_id" validate:"omitempty,min=1" description:"运营商账户ID"`
Decision string `query:"decision" validate:"omitempty,max=32" description:"写入判断:written/baseline/queue_not_full/slope_negative/out_of_order/settled_existing"`
Source string `query:"source" validate:"omitempty,max=32" description:"来源:polling/manual_sync/business_event"`
Scene string `query:"scene" validate:"omitempty,max=64" description:"场景"`
HasDelta *bool `query:"has_delta" description:"是否产生增量:true 只看增量大于 0,false 只看没有增量"`
StaleGuarded *bool `query:"stale_guarded" description:"是否触发重置日防旧值"`
}
// TrafficCollectionRecordItem 采集成功记录。
type TrafficCollectionRecordItem struct {
CardTrafficCollectionItem
IotCardID uint `json:"iot_card_id" description:"卡ID"`
CarrierID uint `json:"carrier_id" description:"运营商账户ID(历史记录为 0)"`
}
// TrafficCollectionRecordListResponse 采集成功记录分页结果。
type TrafficCollectionRecordListResponse struct {
Items []TrafficCollectionRecordItem `json:"items" description:"采集成功记录"`
Total int64 `json:"total" description:"总数"`
Page int `json:"page" description:"页码"`
PageSize int `json:"page_size" description:"每页条数"`
}
// TrafficCollectionFailureListRequest 采集失败记录列表请求。
type TrafficCollectionFailureListRequest struct {
RecordRangeQuery
CarrierID *uint `query:"carrier_id" validate:"omitempty,min=1" description:"运营商账户ID"`
Result string `query:"result" validate:"omitempty,oneof=failed rate_limited invalid_payload no_reading invalid_reading" description:"失败类型:failed/rate_limited/invalid_payload/no_reading/invalid_reading"`
ErrorCategory string `query:"error_category" validate:"omitempty,max=30" description:"错误类别:timeout/network/http/invalid_response/upstream 等"`
ProviderCode string `query:"provider_code" validate:"omitempty,max=30" description:"上游返回码"`
Source string `query:"source" validate:"omitempty,max=32" description:"来源"`
Scene string `query:"scene" validate:"omitempty,max=64" description:"场景"`
}
// TrafficCollectionFailureItem 采集失败记录。
type TrafficCollectionFailureItem struct {
ID uint `json:"id" description:"记录ID"`
IotCardID uint `json:"iot_card_id" description:"卡ID"`
ICCID string `json:"iccid" description:"ICCID"`
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
ObservedAt time.Time `json:"observed_at" description:"采集时间"`
ObservationID string `json:"observation_id" description:"采集 ID(等于对接日志 integration_id)"`
Source string `json:"source" description:"来源"`
Scene string `json:"scene" description:"场景"`
Result string `json:"result" description:"失败类型"`
ErrorCategory string `json:"error_category" description:"错误类别"`
ProviderCode string `json:"provider_code" description:"上游返回码"`
ProviderMessage string `json:"provider_message" description:"上游返回信息或错误信息"`
DurationMS int `json:"duration_ms" description:"调用耗时(毫秒)"`
GatewayRawItem
}
// TrafficCollectionFailureListResponse 采集失败记录分页结果。
type TrafficCollectionFailureListResponse struct {
Items []TrafficCollectionFailureItem `json:"items" description:"采集失败记录"`
Total int64 `json:"total" description:"总数"`
Page int `json:"page" description:"页码"`
PageSize int `json:"page_size" description:"每页条数"`
}
// CardDailyUsageListRequest 卡日流量列表请求。
type CardDailyUsageListRequest struct {
RecordDateRangeQuery
CarrierID *uint `query:"carrier_id" validate:"omitempty,min=1" description:"运营商账户ID"`
MinUsageMB *float64 `query:"min_usage_mb" validate:"omitempty,min=0" description:"单日用量下限(MB),用于排查异常大流量"`
}
// CardDailyUsageItem 卡日流量。
type CardDailyUsageItem struct {
ID uint `json:"id" description:"记录ID"`
IotCardID uint `json:"iot_card_id" description:"卡ID"`
ICCID string `json:"iccid" description:"ICCID"`
CarrierID uint `json:"carrier_id" description:"运营商账户ID"`
Date string `json:"date" description:"日期"`
UsageMB float64 `json:"usage_mb" description:"当日流量(MB)"`
UpdatedAt time.Time `json:"updated_at" description:"更新时间"`
}
// CardDailyUsageListResponse 卡日流量分页结果。
type CardDailyUsageListResponse struct {
Items []CardDailyUsageItem `json:"items" description:"卡日流量"`
Total int64 `json:"total" description:"总数"`
Page int `json:"page" description:"页码"`
PageSize int `json:"page_size" description:"每页条数"`
}
// PackageUsageDailyRecordListRequest 套餐日流量详单列表请求。
type PackageUsageDailyRecordListRequest struct {
RecordDateRangeQuery
PackageUsageID *uint `query:"package_usage_id" validate:"omitempty,min=1" description:"套餐使用记录ID"`
PackageID *uint `query:"package_id" validate:"omitempty,min=1" description:"套餐ID"`
CarrierID *uint `query:"carrier_id" validate:"omitempty,min=1" description:"运营商账户ID(按卡所属账户)"`
}
// PackageUsageDailyRecordItem 套餐日流量详单。
type PackageUsageDailyRecordItem struct {
ID uint `json:"id" description:"记录ID"`
PackageUsageID uint `json:"package_usage_id" description:"套餐使用记录ID"`
PackageID uint `json:"package_id" description:"套餐ID"`
IotCardID uint `json:"iot_card_id" description:"卡ID,设备套餐为 0"`
DeviceID uint `json:"device_id" description:"设备ID,卡套餐为 0"`
Date string `json:"date" description:"日期"`
DailyUsageMB int `json:"daily_usage_mb" description:"当日流量(MB)"`
CumulativeUsageMB int64 `json:"cumulative_usage_mb" description:"截止当日累计(MB)"`
UpdatedAt time.Time `json:"updated_at" description:"更新时间"`
}
// PackageUsageDailyRecordListResponse 套餐日流量详单分页结果。
type PackageUsageDailyRecordListResponse struct {
Items []PackageUsageDailyRecordItem `json:"items" description:"套餐日流量详单"`
Total int64 `json:"total" description:"总数"`
Page int `json:"page" description:"页码"`
PageSize int `json:"page_size" description:"每页条数"`
}
// PackageTrafficAdjustmentQueryRequest 套餐流量冲正流水列表请求。
type PackageTrafficAdjustmentQueryRequest struct {
RecordRangeQuery
PackageUsageID *uint `query:"package_usage_id" validate:"omitempty,min=1" description:"套餐使用记录ID"`
OperatorID *uint `query:"operator_id" validate:"omitempty,min=1" description:"操作人账号ID"`
Direction string `query:"direction" validate:"omitempty,oneof=deduct refund" description:"调整方向:deduct 补扣(增量为正),refund 退回(增量为负)"`
}
// PackageTrafficAdjustmentRecordItem 冲正流水(跨卡列表)。
type PackageTrafficAdjustmentRecordItem struct {
PackageTrafficAdjustmentItem
IotCardID uint `json:"iot_card_id" description:"卡ID"`
DeviceID uint `json:"device_id" description:"设备ID"`
}
// PackageTrafficAdjustmentRecordListResponse 冲正流水分页结果(跨卡列表)。
type PackageTrafficAdjustmentRecordListResponse struct {
Items []PackageTrafficAdjustmentRecordItem `json:"items" description:"冲正流水"`
Total int64 `json:"total" description:"总数"`
Page int `json:"page" description:"页码"`
PageSize int `json:"page_size" description:"每页条数"`
}
// RefundPostProcessAttemptListRequest 退款资产后处理尝试记录列表请求。
type RefundPostProcessAttemptListRequest struct {
RecordRangeQuery
RefundID *uint `query:"refund_id" validate:"omitempty,min=1" description:"退款单ID"`
RefundNo string `query:"refund_no" validate:"omitempty,max=64" description:"退款单号"`
Step string `query:"step" validate:"omitempty,oneof=load invalidate activate check_main stop mark done" description:"处理步骤:load/invalidate/activate/check_main/stop/mark/done"`
Result string `query:"result" validate:"omitempty,oneof=success failed" description:"结果:success/failed"`
ErrorCode string `query:"error_code" validate:"omitempty,max=32" description:"错误码"`
Trigger string `query:"trigger" validate:"omitempty,oneof=outbox manual" description:"触发方式:outbox(含补偿扫描)/manual(人工重试)"`
}
// RefundPostProcessAttemptItem 退款资产后处理尝试记录。
type RefundPostProcessAttemptItem struct {
ID uint `json:"id" description:"记录ID"`
RefundID uint `json:"refund_id" description:"退款单ID"`
RefundNo string `json:"refund_no" description:"退款单号"`
Step string `json:"step" description:"处理步骤(失败时为失败步骤,成功为 done)"`
AssetType string `json:"asset_type" description:"资产类型"`
AssetID uint `json:"asset_id" description:"资产ID"`
ICCID string `json:"iccid" description:"ICCID(单卡资产)"`
Attempt int `json:"attempt" description:"尝试序号"`
Trigger string `json:"trigger" description:"触发方式"`
Result string `json:"result" description:"结果"`
ErrorCode string `json:"error_code" description:"错误码"`
ErrorMessage string `json:"error_message" description:"错误信息"`
DurationMS int `json:"duration_ms" description:"耗时(毫秒)"`
CreatedAt time.Time `json:"created_at" description:"记录时间"`
}
// RefundPostProcessAttemptListResponse 退款资产后处理尝试记录分页结果。
type RefundPostProcessAttemptListResponse struct {
Items []RefundPostProcessAttemptItem `json:"items" description:"尝试记录"`
Total int64 `json:"total" description:"总数"`
Page int `json:"page" description:"页码"`
PageSize int `json:"page_size" description:"每页条数"`
}

View File

@@ -1,5 +1,7 @@
package dto
import "time"
// CreateRefundRequest 创建退款申请请求
type CreateRefundRequest struct {
OrderID uint `json:"order_id" validate:"required" required:"true" description:"关联订单ID"`
@@ -60,6 +62,7 @@ type RefundListRequest struct {
OrderID *uint `json:"order_id" query:"order_id" validate:"omitempty" description:"关联订单ID"`
ShopID *uint `json:"shop_id" query:"shop_id" validate:"omitempty" description:"店铺ID"`
AssetIdentifier string `json:"asset_identifier" query:"asset_identifier" validate:"omitempty,max=100" maxLength:"100" description:"资产标识精确检索(ICCID 或 设备虚拟号,非空时精确匹配)"`
AssetPostStatus *int `json:"asset_post_status" query:"asset_post_status" validate:"omitempty,min=0,max=3" minimum:"0" maximum:"3" description:"资产后处理状态 (0:未开始, 1:待重试, 2:已完成, 3:需人工处理)"`
}
// RefundResponse 退款申请详情响应
@@ -116,6 +119,10 @@ type RefundResponse struct {
Remark string `json:"remark,omitempty" description:"审批备注"`
CommissionDeducted bool `json:"commission_deducted" description:"佣金是否已回扣"`
AssetReset bool `json:"asset_reset" description:"退款后资产处理是否完成"`
AssetPostStatus int `json:"asset_post_status" description:"资产后处理状态 (0:未开始, 1:待重试, 2:已完成, 3:需人工处理)"`
AssetPostAttempts int `json:"asset_post_attempts" description:"资产后处理已尝试次数(自动重试上限 10 次)"`
AssetPostLastError string `json:"asset_post_last_error" description:"资产后处理最后一次错误"`
AssetPostNextAt *time.Time `json:"asset_post_next_at,omitempty" description:"资产后处理下次自动重试时间"`
SubmitterID uint `json:"submitter_id" description:"提交人账号ID"`
SubmitterName string `json:"submitter_name" description:"提交人账号名称"`
ApprovalInstanceID *uint `json:"approval_instance_id,omitempty" description:"通用审批实例ID"`