perf: IoT 卡 30M 行分页查询优化(P95 17.9s → <500ms)
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 7m6s

- 新增 is_standalone 物化列 + 触发器自动维护(迁移 056)
- 并行查询拆分:多店铺 IN 查询拆为 per-shop goroutine 并行 Index Scan
- 两阶段延迟 Join:深度分页(page≥50)走覆盖索引 Index Only Scan 取 ID 再回表
- COUNT 缓存:per-shop 并行 COUNT + Redis 30 分钟 TTL
- 索引优化:删除有害全局索引、新增 partial composite indexes(迁移 057/058)
- ICCID 模糊搜索路径隔离:trigram GIN 索引走独立查询路径
- 慢查询阈值从 100ms 调整为 500ms
- 新增 30M 测试数据种子脚本和 benchmark 工具
This commit is contained in:
2026-02-24 16:23:02 +08:00
parent c665f32976
commit f32d32cd36
20 changed files with 2705 additions and 50 deletions

View File

@@ -2,14 +2,25 @@ package postgres
import (
"context"
"fmt"
"hash/fnv"
"sort"
"sync"
"time"
"github.com/break/junhong_cmp_fiber/internal/model"
"github.com/break/junhong_cmp_fiber/internal/store"
"github.com/break/junhong_cmp_fiber/pkg/constants"
pkggorm "github.com/break/junhong_cmp_fiber/pkg/gorm"
"github.com/break/junhong_cmp_fiber/pkg/logger"
"github.com/break/junhong_cmp_fiber/pkg/middleware"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
"gorm.io/gorm"
)
const listCountCacheTTL = 30 * time.Minute
type IotCardStore struct {
db *gorm.DB
redis *redis.Client
@@ -102,9 +113,11 @@ func (s *IotCardStore) List(ctx context.Context, opts *store.QueryOptions, filte
query := s.db.WithContext(ctx).Model(&model.IotCard{})
// 企业用户特殊处理:只能看到授权给自己的卡
// 子查询跳过数据权限过滤,权限已由外层查询的 GORM callback 保证
skipCtx := pkggorm.SkipDataPermission(ctx)
if enterpriseID, ok := filters["authorized_enterprise_id"].(uint); ok && enterpriseID > 0 {
query = query.Where("id IN (?)",
s.db.Table("tb_enterprise_card_authorization").
s.db.WithContext(skipCtx).Table("tb_enterprise_card_authorization").
Select("card_id").
Where("enterprise_id = ? AND revoked_at IS NULL AND deleted_at IS NULL", enterpriseID))
}
@@ -130,7 +143,7 @@ func (s *IotCardStore) List(ctx context.Context, opts *store.QueryOptions, filte
}
if packageID, ok := filters["package_id"].(uint); ok && packageID > 0 {
query = query.Where("id IN (?)",
s.db.Table("tb_package_usage").
s.db.WithContext(skipCtx).Table("tb_package_usage").
Select("iot_card_id").
Where("package_id = ? AND deleted_at IS NULL", packageID))
}
@@ -151,9 +164,13 @@ func (s *IotCardStore) List(ctx context.Context, opts *store.QueryOptions, filte
query = query.Where("series_id = ?", seriesID)
}
// 统计总数
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
if cachedTotal, ok := s.getCachedCount(ctx, "iot_card", filters); ok {
total = cachedTotal
} else {
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
s.cacheCount(ctx, "iot_card", filters, total)
}
// 分页处理
@@ -181,16 +198,393 @@ func (s *IotCardStore) List(ctx context.Context, opts *store.QueryOptions, filte
return cards, total, nil
}
// standaloneListColumns 列表查询只选取响应需要的列,避免 SELECT * 的宽行 I/O
var standaloneListColumns = []string{
"id", "iccid", "card_category", "carrier_id", "carrier_type", "carrier_name",
"imsi", "msisdn", "batch_no", "supplier", "cost_price", "distribute_price",
"status", "shop_id", "activated_at", "activation_status", "real_name_status",
"network_status", "data_usage_mb", "current_month_usage_mb", "current_month_start_date",
"last_month_total_mb", "last_data_check_at", "last_real_name_check_at",
"enable_polling", "series_id", "first_commission_paid", "accumulated_recharge",
"created_at", "updated_at",
}
// ListStandalone 独立卡列表查询入口
// 路由逻辑:
// - ICCID/MSISDN 模糊搜索 → listStandaloneDefaulttrigram GIN 索引)
// - 多店铺(代理用户) → listStandaloneParallelper-shop 并行,内含深度分页两阶段优化)
// - 深度分页page >= 50→ listStandaloneTwoPhase覆盖索引延迟 Join
// - 其他 → listStandaloneDefault
func (s *IotCardStore) ListStandalone(ctx context.Context, opts *store.QueryOptions, filters map[string]any) ([]*model.IotCard, int64, error) {
if opts == nil {
opts = &store.QueryOptions{
Page: 1,
PageSize: constants.DefaultPageSize,
}
}
_, hasICCID := filters["iccid"].(string)
_, hasMSISDN := filters["msisdn"].(string)
useFuzzySearch := (hasICCID && filters["iccid"] != "") || (hasMSISDN && filters["msisdn"] != "")
if !useFuzzySearch {
if shopIDs, ok := filters["subordinate_shop_ids"].([]uint); ok && len(shopIDs) > 1 {
return s.listStandaloneParallel(ctx, opts, filters, shopIDs)
}
}
// 非多店铺场景的深度分页走两阶段路径(单店铺或平台用户)
if !useFuzzySearch && opts.Page >= 50 {
return s.listStandaloneTwoPhase(ctx, opts, filters)
}
return s.listStandaloneDefault(ctx, opts, filters)
}
// listStandaloneTwoPhase 单路径两阶段延迟 Join
// 适用于非多店铺场景平台用户、单店铺的深度分页page >= 50
// Phase 1: SELECT id 走覆盖索引Phase 2: WHERE id IN 回表
func (s *IotCardStore) listStandaloneTwoPhase(ctx context.Context, opts *store.QueryOptions, filters map[string]any) ([]*model.IotCard, int64, error) {
var total int64
query := s.db.WithContext(ctx).Model(&model.IotCard{}).
Where("is_standalone = true")
query = s.applyStandaloneFilters(ctx, query, filters)
if cachedTotal, ok := s.getCachedCount(ctx, "standalone", filters); ok {
total = cachedTotal
} else {
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
s.cacheCount(ctx, "standalone", filters, total)
}
offset := (opts.Page - 1) * opts.PageSize
// Phase 1: 仅取 ID覆盖索引 Index Only Scan
var ids []uint
if err := query.Select("id").
Order("created_at DESC, id").
Offset(offset).Limit(opts.PageSize).
Pluck("id", &ids).Error; err != nil {
return nil, 0, err
}
if len(ids) == 0 {
return []*model.IotCard{}, total, nil
}
// Phase 2: 用 ID 精确回表
var cards []*model.IotCard
if err := s.db.WithContext(ctx).Model(&model.IotCard{}).
Select(standaloneListColumns).
Where("id IN ?", ids).
Find(&cards).Error; err != nil {
return nil, 0, err
}
idOrder := make(map[uint]int, len(ids))
for i, id := range ids {
idOrder[id] = i
}
sort.Slice(cards, func(i, j int) bool {
return idOrder[cards[i].ID] < idOrder[cards[j].ID]
})
logger.GetAppLogger().Debug("两阶段查询完成",
zap.Int("page", opts.Page),
zap.Int("phase2_fetched", len(cards)),
zap.Int64("total_count", total),
)
return cards, total, nil
}
// listStandaloneDefault 原始查询路径
// 适用于平台/超管用户(无 shop_id 过滤)或单店铺场景
func (s *IotCardStore) listStandaloneDefault(ctx context.Context, opts *store.QueryOptions, filters map[string]any) ([]*model.IotCard, int64, error) {
var cards []*model.IotCard
var total int64
query := s.db.WithContext(ctx).Model(&model.IotCard{})
query := s.db.WithContext(ctx).Model(&model.IotCard{}).
Where("is_standalone = true")
query = s.applyStandaloneFilters(ctx, query, filters)
query = query.Where("id NOT IN (?)",
s.db.Model(&model.DeviceSimBinding{}).
Select("iot_card_id").
Where("bind_status = ?", 1))
if cachedTotal, ok := s.getCachedCount(ctx, "standalone", filters); ok {
total = cachedTotal
} else {
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
s.cacheCount(ctx, "standalone", filters, total)
}
offset := (opts.Page - 1) * opts.PageSize
if opts.OrderBy != "" {
query = query.Order(opts.OrderBy)
} else {
query = query.Order("created_at DESC")
}
if err := query.Select(standaloneListColumns).Offset(offset).Limit(opts.PageSize).Find(&cards).Error; err != nil {
return nil, 0, err
}
return cards, total, nil
}
// listStandaloneParallel 并行查询路径
// 将 shop_id IN (...) 拆分为 per-shop 独立查询,每个查询走 Index Scan
// 然后在应用层归并排序,避免 PG 对多值 IN + ORDER BY 选择全表扫描
func (s *IotCardStore) listStandaloneParallel(ctx context.Context, opts *store.QueryOptions, filters map[string]any, shopIDs []uint) ([]*model.IotCard, int64, error) {
skipCtx := pkggorm.SkipDataPermission(ctx)
fetchLimit := (opts.Page-1)*opts.PageSize + opts.PageSize
// 深度分页page >= 50走两阶段延迟 Join
// Phase 1 只取 ID覆盖索引 Index Only ScanPhase 2 用 ID 回表
// 避免 OFFSET 跳过大量宽行数据
if opts.Page >= 50 {
return s.listStandaloneParallelTwoPhase(ctx, opts, filters, shopIDs)
}
type shopResult struct {
cards []*model.IotCard
count int64
err error
}
results := make([]shopResult, len(shopIDs))
var wg sync.WaitGroup
cachedTotal, hasCachedTotal := s.getCachedCount(ctx, "standalone", filters)
for i, shopID := range shopIDs {
wg.Add(1)
go func(idx int, sid uint) {
defer wg.Done()
q := s.db.WithContext(skipCtx).Model(&model.IotCard{}).
Where("is_standalone = true AND deleted_at IS NULL AND shop_id = ?", sid)
q = s.applyStandaloneFilters(skipCtx, q, filters)
var cards []*model.IotCard
if err := q.Select(standaloneListColumns).
Order("created_at DESC, id").
Limit(fetchLimit).
Find(&cards).Error; err != nil {
results[idx] = shopResult{err: err}
return
}
var count int64
if !hasCachedTotal {
countQ := s.db.WithContext(skipCtx).Model(&model.IotCard{}).
Where("is_standalone = true AND deleted_at IS NULL AND shop_id = ?", sid)
countQ = s.applyStandaloneFilters(skipCtx, countQ, filters)
if err := countQ.Count(&count).Error; err != nil {
results[idx] = shopResult{err: err}
return
}
}
results[idx] = shopResult{cards: cards, count: count}
}(i, shopID)
}
wg.Wait()
allCards := make([]*model.IotCard, 0, fetchLimit)
var totalCount int64
if hasCachedTotal {
totalCount = cachedTotal
}
for _, r := range results {
if r.err != nil {
return nil, 0, r.err
}
allCards = append(allCards, r.cards...)
if !hasCachedTotal {
totalCount += r.count
}
}
if !hasCachedTotal && totalCount > 0 {
s.cacheCount(ctx, "standalone", filters, totalCount)
}
sort.Slice(allCards, func(i, j int) bool {
if allCards[i].CreatedAt.Equal(allCards[j].CreatedAt) {
return allCards[i].ID < allCards[j].ID
}
return allCards[i].CreatedAt.After(allCards[j].CreatedAt)
})
offset := (opts.Page - 1) * opts.PageSize
end := offset + opts.PageSize
if offset >= len(allCards) {
return []*model.IotCard{}, totalCount, nil
}
if end > len(allCards) {
end = len(allCards)
}
logger.GetAppLogger().Debug("并行查询完成",
zap.Int("shop_count", len(shopIDs)),
zap.Int("total_fetched", len(allCards)),
zap.Int("offset", offset),
zap.Int("page_size", opts.PageSize),
zap.Int64("total_count", totalCount),
)
return allCards[offset:end], totalCount, nil
}
// cardIDWithTime 仅存储 ID 和排序键,用于两阶段查询的轻量归并排序
type cardIDWithTime struct {
ID uint
CreatedAt time.Time
}
// listStandaloneParallelTwoPhase 并行+两阶段延迟 Join 查询路径
// 解决深度分页page >= 50时 OFFSET 跳过大量宽行数据的性能问题:
// Phase 1: 每个 shop 并行查询 SELECT id, created_at走覆盖索引 Index Only Scan~20字节/元组)
// 归并排序后取目标页的 20 个 ID
// Phase 2: SELECT 完整列 WHERE id IN (20 IDs)PK 精确回表)
func (s *IotCardStore) listStandaloneParallelTwoPhase(ctx context.Context, opts *store.QueryOptions, filters map[string]any, shopIDs []uint) ([]*model.IotCard, int64, error) {
skipCtx := pkggorm.SkipDataPermission(ctx)
fetchLimit := (opts.Page-1)*opts.PageSize + opts.PageSize
type shopResult struct {
ids []cardIDWithTime
count int64
err error
}
results := make([]shopResult, len(shopIDs))
var wg sync.WaitGroup
cachedTotal, hasCachedTotal := s.getCachedCount(ctx, "standalone", filters)
// Phase 1: 并行获取每个 shop 的 (id, created_at),走覆盖索引
for i, shopID := range shopIDs {
wg.Add(1)
go func(idx int, sid uint) {
defer wg.Done()
q := s.db.WithContext(skipCtx).Model(&model.IotCard{}).
Where("is_standalone = true AND deleted_at IS NULL AND shop_id = ?", sid)
q = s.applyStandaloneFilters(skipCtx, q, filters)
var ids []cardIDWithTime
if err := q.Select("id, created_at").
Order("created_at DESC, id").
Limit(fetchLimit).
Find(&ids).Error; err != nil {
results[idx] = shopResult{err: err}
return
}
var count int64
if !hasCachedTotal {
countQ := s.db.WithContext(skipCtx).Model(&model.IotCard{}).
Where("is_standalone = true AND deleted_at IS NULL AND shop_id = ?", sid)
countQ = s.applyStandaloneFilters(skipCtx, countQ, filters)
if err := countQ.Count(&count).Error; err != nil {
results[idx] = shopResult{err: err}
return
}
}
results[idx] = shopResult{ids: ids, count: count}
}(i, shopID)
}
wg.Wait()
allIDs := make([]cardIDWithTime, 0, fetchLimit)
var totalCount int64
if hasCachedTotal {
totalCount = cachedTotal
}
for _, r := range results {
if r.err != nil {
return nil, 0, r.err
}
allIDs = append(allIDs, r.ids...)
if !hasCachedTotal {
totalCount += r.count
}
}
if !hasCachedTotal && totalCount > 0 {
s.cacheCount(ctx, "standalone", filters, totalCount)
}
sort.Slice(allIDs, func(i, j int) bool {
if allIDs[i].CreatedAt.Equal(allIDs[j].CreatedAt) {
return allIDs[i].ID < allIDs[j].ID
}
return allIDs[i].CreatedAt.After(allIDs[j].CreatedAt)
})
offset := (opts.Page - 1) * opts.PageSize
end := offset + opts.PageSize
if offset >= len(allIDs) {
return []*model.IotCard{}, totalCount, nil
}
if end > len(allIDs) {
end = len(allIDs)
}
pageIDs := make([]uint, 0, opts.PageSize)
for _, item := range allIDs[offset:end] {
pageIDs = append(pageIDs, item.ID)
}
if len(pageIDs) == 0 {
return []*model.IotCard{}, totalCount, nil
}
// Phase 2: 用 ID 精确回表获取完整数据PK Index Scan仅 20 行)
var cards []*model.IotCard
if err := s.db.WithContext(skipCtx).Model(&model.IotCard{}).
Select(standaloneListColumns).
Where("id IN ?", pageIDs).
Find(&cards).Error; err != nil {
return nil, 0, err
}
// Phase 2 结果按原始排序顺序重排
idOrder := make(map[uint]int, len(pageIDs))
for i, id := range pageIDs {
idOrder[id] = i
}
sort.Slice(cards, func(i, j int) bool {
return idOrder[cards[i].ID] < idOrder[cards[j].ID]
})
logger.GetAppLogger().Debug("两阶段并行查询完成",
zap.Int("shop_count", len(shopIDs)),
zap.Int("phase1_total_ids", len(allIDs)),
zap.Int("phase2_fetched", len(cards)),
zap.Int("page", opts.Page),
zap.Int64("total_count", totalCount),
)
return cards, totalCount, nil
}
// applyStandaloneFilters 应用独立卡列表的通用过滤条件
// 注意:不包含 is_standalone、shop_id、deleted_at 条件(由调用方控制)
// 也不包含 subordinate_shop_ids仅用于路由选择不作为查询条件
func (s *IotCardStore) applyStandaloneFilters(ctx context.Context, query *gorm.DB, filters map[string]any) *gorm.DB {
skipCtx := pkggorm.SkipDataPermission(ctx)
if status, ok := filters["status"].(int); ok && status > 0 {
query = query.Where("status = ?", status)
@@ -198,6 +592,7 @@ func (s *IotCardStore) ListStandalone(ctx context.Context, opts *store.QueryOpti
if carrierID, ok := filters["carrier_id"].(uint); ok && carrierID > 0 {
query = query.Where("carrier_id = ?", carrierID)
}
// 并行路径下 shop_id 已由调用方设置,此处仅处理显式的 shop_id 过滤
if shopID, ok := filters["shop_id"].(uint); ok && shopID > 0 {
query = query.Where("shop_id = ?", shopID)
}
@@ -212,7 +607,7 @@ func (s *IotCardStore) ListStandalone(ctx context.Context, opts *store.QueryOpti
}
if packageID, ok := filters["package_id"].(uint); ok && packageID > 0 {
query = query.Where("id IN (?)",
s.db.Table("tb_package_usage").
s.db.WithContext(skipCtx).Table("tb_package_usage").
Select("iot_card_id").
Where("package_id = ? AND deleted_at IS NULL", packageID))
}
@@ -232,12 +627,12 @@ func (s *IotCardStore) ListStandalone(ctx context.Context, opts *store.QueryOpti
if isReplaced, ok := filters["is_replaced"].(bool); ok {
if isReplaced {
query = query.Where("id IN (?)",
s.db.Table("tb_card_replacement_record").
s.db.WithContext(skipCtx).Table("tb_card_replacement_record").
Select("old_iot_card_id").
Where("deleted_at IS NULL"))
} else {
query = query.Where("id NOT IN (?)",
s.db.Table("tb_card_replacement_record").
s.db.WithContext(skipCtx).Table("tb_card_replacement_record").
Select("old_iot_card_id").
Where("deleted_at IS NULL"))
}
@@ -246,30 +641,7 @@ func (s *IotCardStore) ListStandalone(ctx context.Context, opts *store.QueryOpti
query = query.Where("series_id = ?", seriesID)
}
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
if opts == nil {
opts = &store.QueryOptions{
Page: 1,
PageSize: constants.DefaultPageSize,
}
}
offset := (opts.Page - 1) * opts.PageSize
query = query.Offset(offset).Limit(opts.PageSize)
if opts.OrderBy != "" {
query = query.Order(opts.OrderBy)
} else {
query = query.Order("created_at DESC")
}
if err := query.Find(&cards).Error; err != nil {
return nil, 0, err
}
return cards, total, nil
return query
}
func (s *IotCardStore) GetByICCIDs(ctx context.Context, iccids []string) ([]*model.IotCard, error) {
@@ -285,10 +657,7 @@ func (s *IotCardStore) GetByICCIDs(ctx context.Context, iccids []string) ([]*mod
func (s *IotCardStore) GetStandaloneByICCIDRange(ctx context.Context, iccidStart, iccidEnd string, shopID *uint) ([]*model.IotCard, error) {
query := s.db.WithContext(ctx).Model(&model.IotCard{}).
Where("id NOT IN (?)",
s.db.Model(&model.DeviceSimBinding{}).
Select("iot_card_id").
Where("bind_status = ?", 1)).
Where("is_standalone = true").
Where("iccid >= ? AND iccid <= ?", iccidStart, iccidEnd)
if shopID == nil {
@@ -306,10 +675,7 @@ func (s *IotCardStore) GetStandaloneByICCIDRange(ctx context.Context, iccidStart
func (s *IotCardStore) GetStandaloneByFilters(ctx context.Context, filters map[string]any, shopID *uint) ([]*model.IotCard, error) {
query := s.db.WithContext(ctx).Model(&model.IotCard{}).
Where("id NOT IN (?)",
s.db.Model(&model.DeviceSimBinding{}).
Select("iot_card_id").
Where("bind_status = ?", 1))
Where("is_standalone = true")
if shopID == nil {
query = query.Where("shop_id IS NULL")
@@ -365,9 +731,10 @@ func (s *IotCardStore) GetByIDsWithEnterpriseFilter(ctx context.Context, cardIDs
query := s.db.WithContext(ctx).Model(&model.IotCard{})
if enterpriseID != nil && *enterpriseID > 0 {
skipCtx := pkggorm.SkipDataPermission(ctx)
query = query.Where("id IN (?) AND id IN (?)",
cardIDs,
s.db.Table("tb_enterprise_card_authorization").
s.db.WithContext(skipCtx).Table("tb_enterprise_card_authorization").
Select("card_id").
Where("enterprise_id = ? AND revoked_at IS NULL AND deleted_at IS NULL", *enterpriseID))
} else {
@@ -475,3 +842,43 @@ func (s *IotCardStore) BatchDelete(ctx context.Context, cardIDs []uint) error {
Where("id IN ?", cardIDs).
Delete(&model.IotCard{}).Error
}
// ==================== 列表计数缓存 ====================
func (s *IotCardStore) getCachedCount(ctx context.Context, table string, filters map[string]any) (int64, bool) {
if s.redis == nil {
return 0, false
}
key := constants.RedisListCountKey(table, middleware.GetUserIDFromContext(ctx), hashFilters(filters))
val, err := s.redis.Get(ctx, key).Int64()
if err != nil {
return 0, false
}
return val, true
}
func (s *IotCardStore) cacheCount(ctx context.Context, table string, filters map[string]any, total int64) {
if s.redis == nil {
return
}
key := constants.RedisListCountKey(table, middleware.GetUserIDFromContext(ctx), hashFilters(filters))
s.redis.Set(ctx, key, total, listCountCacheTTL)
}
func hashFilters(filters map[string]any) string {
if len(filters) == 0 {
return "0"
}
keys := make([]string, 0, len(filters))
for k := range filters {
keys = append(keys, k)
}
sort.Strings(keys)
h := fnv.New32a()
for _, k := range keys {
h.Write([]byte(k))
h.Write([]byte(fmt.Sprint(filters[k])))
}
return fmt.Sprintf("%08x", h.Sum32())
}