Files
junhong_cmp_fiber/internal/application/cardobservation/traffic_collection.go
邹进 94cbf2adab feat: 采集记录成功失败分表并保存网关原始数据,到期归档 csv.gz,新增记录分页查询接口
- 网关客户端新增原始数据收集器,记录最后一次尝试的明文请求、HTTP 状态、响应体与 code/msg/trace_id
- tb_card_traffic_collection 只写成功读数,新增 carrier_id 与原始数据列;新增 tb_card_traffic_collection_failure,失败不判断有无套餐并带错误详情
- 采集记录在线保留当月加最近 2 个整月,到期分区摘出、导出 csv.gz、校验行数与 SHA-256 后删除,本地保留 3 个月
- 新增采集成功/失败、卡日流量、套餐日流量详单、冲正流水、退款后处理尝试列表接口:可查单个也可查全部,分页与条件筛选
- 停复机记录支持资产标识、场景、原因、指令号筛选;单卡历史支持动作、结果、来源筛选
- 单卡采集记录支持筛选与合并失败记录;套餐日流量详单支持分页;退款列表支持后处理状态筛选
- 测试环境 api、worker 日志分目录,保留 40 个压缩备份;worker 挂载 archive 目录
- 迁移 000272、000273 需维护者在测试库执行后再部署

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 18:47:00 +09:00

147 lines
5.5 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package cardobservation
import (
"context"
"encoding/json"
"time"
"unicode/utf8"
"go.uber.org/zap"
"gorm.io/datatypes"
cardflow "github.com/break/junhong_cmp_fiber/internal/domain/cardflow"
domain "github.com/break/junhong_cmp_fiber/internal/domain/cardobservation"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store/postgres"
"github.com/break/junhong_cmp_fiber/pkg/constants"
)
// collectionRecordTimeout 是事务提交后写采集记录的超时;采集记录只用于排查,写失败只告警。
const collectionRecordTimeout = 3 * time.Second
// rawResponseMaxBytes 是原始响应体保存上限,超过时截断并标记。
const rawResponseMaxBytes = 4096
// collectionRecord 是一次采集的留痕内容;HasPackage 在观测事务内判断(只对成功读数有意义)。
type collectionRecord struct {
model.CardTrafficCollection
HasPackage bool
failure *domain.CollectionFailure
}
func newCollectionRecord(observation domain.TrafficObservation) *collectionRecord {
record := &collectionRecord{CardTrafficCollection: model.CardTrafficCollection{
IotCardID: observation.CardID, ObservedAt: observation.Metadata.ObservedAt,
ObservationID: observation.Metadata.ObservationID, Source: observation.Metadata.Source,
Scene: observation.Metadata.Scene, Result: domain.ResolveReadingResult(observation),
CollectionRaw: buildCollectionRaw(observation.Raw),
}, failure: observation.Failure}
if record.Result == constants.TrafficCollectionResultOK {
reading := observation.GatewayReadingMB
record.VendorUsedMB = &reading
}
return record
}
func (r *collectionRecord) setQueue(points []cardflow.QueuePoint) {
if encoded, err := json.Marshal(points); err == nil {
r.QueuePoints = encoded
}
}
func (r *collectionRecord) applyOutcome(outcome *flowOutcome) {
r.Decision = outcome.Decision
if !outcome.Written {
return
}
monthUsed, delta, stale := outcome.MonthUsedMB, outcome.DeltaMB, outcome.StaleGuarded
r.MonthUsedMB, r.DeltaMB, r.StaleGuarded, r.EventID = &monthUsed, &delta, &stale, outcome.EventID
}
// writeCollectionRecord 在观测事务之外写采集记录:成功读数只为采集时刻有套餐的卡写成功表;
// 失败读数不判断有无套餐,写失败表。
func (s *Service) writeCollectionRecord(ctx context.Context, record *collectionRecord, _ time.Time) {
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), collectionRecordTimeout)
defer cancel()
if record.Result != constants.TrafficCollectionResultOK {
s.writeCollectionFailure(writeCtx, record)
return
}
if !record.HasPackage {
return
}
if err := postgres.NewCardTrafficCollectionStore(s.db).Create(writeCtx, &record.CardTrafficCollection); err != nil {
s.logTrafficWarn("写卡流量采集记录失败", zap.Uint("card_id", record.IotCardID),
zap.String("observation_id", record.ObservationID), zap.Error(err))
}
}
func (s *Service) writeCollectionFailure(ctx context.Context, record *collectionRecord) {
row := &model.CardTrafficCollectionFailure{
IotCardID: record.IotCardID, ObservedAt: record.ObservedAt, ObservationID: record.ObservationID,
Source: record.Source, Scene: record.Scene, Result: record.Result, CollectionRaw: record.CollectionRaw,
}
if failure := record.failure; failure != nil {
row.ErrorCategory, row.ProviderCode = failure.ErrorCategory, failure.ProviderCode
row.ProviderMessage, row.DurationMS = truncateText(failure.ProviderMessage, 500), failure.DurationMS
}
var card model.IotCard
if err := s.db.WithContext(ctx).Unscoped().Select("iccid", "carrier_id").Where("id = ?", record.IotCardID).Take(&card).Error; err == nil {
row.ICCID, row.CarrierID = card.ICCID, card.CarrierID
}
if err := postgres.NewCardTrafficCollectionFailureStore(s.db).Create(ctx, row); err != nil {
s.logTrafficWarn("写卡流量采集失败记录失败", zap.Uint("card_id", record.IotCardID),
zap.String("observation_id", record.ObservationID), zap.Error(err))
}
}
// buildCollectionRaw 把网关原始数据转为留存字段:响应可解析为 JSON 且不超过 4KB 时存 JSON,否则存文本并按 4KB 截断。
func buildCollectionRaw(raw *domain.GatewayRaw) model.CollectionRaw {
if raw == nil {
return model.CollectionRaw{}
}
out := model.CollectionRaw{
GatewayCode: raw.GatewayCode, GatewayMsg: truncateText(raw.GatewayMsg, 500),
GatewayTraceID: truncateText(raw.TraceID, 128), GatewayAttempts: raw.Attempts,
}
if raw.HTTPStatus > 0 {
status := raw.HTTPStatus
out.HTTPStatus = &status
}
if len(raw.Request) > 0 && json.Valid(raw.Request) {
out.RawRequest = datatypes.JSON(raw.Request)
}
switch {
case len(raw.Response) == 0:
case len(raw.Response) > rawResponseMaxBytes:
out.RawResponseText, out.RawTruncated = validUTF8Prefix(raw.Response, rawResponseMaxBytes), true
case json.Valid(raw.Response):
out.RawResponse = datatypes.JSON(raw.Response)
default:
out.RawResponseText = validUTF8Prefix(raw.Response, rawResponseMaxBytes)
}
return out
}
// validUTF8Prefix 返回不超过 limit 字节且不截断多字节字符的前缀,并替换非法 UTF-8(PostgreSQL text 不接受)。
func validUTF8Prefix(data []byte, limit int) string {
if len(data) > limit {
data = data[:limit]
for len(data) > 0 && !utf8.Valid(data) {
data = data[:len(data)-1]
}
}
if !utf8.Valid(data) {
return string([]rune(string(data)))
}
return string(data)
}
func truncateText(value string, limit int) string {
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}