feat: 实现 IoT 卡轮询系统（支持千万级卡规模）
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 6m35s
All checks were successful
构建并部署到测试环境（无 SSH） / build-and-deploy (push) Successful in 6m35s
实现功能： - 实名状态检查轮询（可配置间隔） - 卡流量检查轮询（支持跨月流量追踪） - 套餐检查与超额自动停机 - 分布式并发控制（Redis 信号量） - 手动触发轮询（单卡/批量/条件筛选） - 数据清理配置与执行 - 告警规则与历史记录 - 实时监控统计（队列/性能/并发） 性能优化： - Redis 缓存卡信息，减少 DB 查询 - Pipeline 批量写入 Redis - 异步流量记录写入 - 渐进式初始化（10万卡/批） 压测工具（scripts/benchmark/）： - Mock Gateway 模拟上游服务 - 测试卡生成器 - 配置初始化脚本 - 实时监控脚本 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
224
scripts/benchmark/monitor.sh
Executable file
224
scripts/benchmark/monitor.sh
Executable file
@@ -0,0 +1,224 @@
|
||||
#!/bin/bash
|
||||
# 压测监控脚本 - 增强版
|
||||
|
||||
set -e
|
||||
|
||||
# 检查 Redis 连接
|
||||
REDIS_HOST="${JUNHONG_REDIS_ADDRESS:-127.0.0.1}"
|
||||
REDIS_PORT="${JUNHONG_REDIS_PORT:-6379}"
|
||||
REDIS_CLI="redis-cli -h $REDIS_HOST -p $REDIS_PORT"
|
||||
|
||||
# 上一次的统计值（用于计算增量）
|
||||
LAST_REALNAME_SUCCESS=0
|
||||
LAST_REALNAME_FAILURE=0
|
||||
LAST_CARDDATA_SUCCESS=0
|
||||
LAST_CARDDATA_FAILURE=0
|
||||
LAST_PACKAGE_SUCCESS=0
|
||||
LAST_PACKAGE_FAILURE=0
|
||||
LAST_TIME=$(date +%s)
|
||||
|
||||
echo "=== 轮询系统压测监控（增强版）==="
|
||||
echo "Redis 地址: $REDIS_HOST:$REDIS_PORT"
|
||||
echo ""
|
||||
|
||||
# 循环监控
|
||||
while true; do
|
||||
clear
|
||||
NOW=$(date +%s)
|
||||
INTERVAL=$((NOW - LAST_TIME))
|
||||
if [ $INTERVAL -eq 0 ]; then
|
||||
INTERVAL=1
|
||||
fi
|
||||
|
||||
echo "╔══════════════════════════════════════════════════════════════════════╗"
|
||||
echo "║ 轮询系统压测监控 $(date '+%Y-%m-%d %H:%M:%S') ║"
|
||||
echo "╚══════════════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
# ========== Redis 队列状态 ==========
|
||||
echo "【📊 Redis 队列状态】"
|
||||
REALNAME_QUEUE=$($REDIS_CLI ZCARD "polling:queue:realname" 2>/dev/null || echo "0")
|
||||
CARDDATA_QUEUE=$($REDIS_CLI ZCARD "polling:queue:carddata" 2>/dev/null || echo "0")
|
||||
PACKAGE_QUEUE=$($REDIS_CLI ZCARD "polling:queue:package" 2>/dev/null || echo "0")
|
||||
MANUAL_REALNAME=$($REDIS_CLI LLEN "polling:manual:realname" 2>/dev/null || echo "0")
|
||||
MANUAL_CARDDATA=$($REDIS_CLI LLEN "polling:manual:carddata" 2>/dev/null || echo "0")
|
||||
MANUAL_PACKAGE=$($REDIS_CLI LLEN "polling:manual:package" 2>/dev/null || echo "0")
|
||||
|
||||
printf " %-20s %'12d\n" "实名检查队列:" "$REALNAME_QUEUE"
|
||||
printf " %-20s %'12d\n" "流量检查队列:" "$CARDDATA_QUEUE"
|
||||
printf " %-20s %'12d\n" "套餐检查队列:" "$PACKAGE_QUEUE"
|
||||
printf " %-20s %'12d\n" "手动触发(实名):" "$MANUAL_REALNAME"
|
||||
printf " %-20s %'12d\n" "手动触发(流量):" "$MANUAL_CARDDATA"
|
||||
printf " %-20s %'12d\n" "手动触发(套餐):" "$MANUAL_PACKAGE"
|
||||
echo ""
|
||||
|
||||
# ========== 处理性能统计 ==========
|
||||
echo "【⚡ 处理性能统计】"
|
||||
|
||||
# 获取当前统计值（注意：key 格式是 polling:stats:polling:xxx）
|
||||
REALNAME_SUCCESS=$($REDIS_CLI HGET "polling:stats:polling:realname" "success_count_1h" 2>/dev/null || echo "0")
|
||||
REALNAME_FAILURE=$($REDIS_CLI HGET "polling:stats:polling:realname" "failure_count_1h" 2>/dev/null || echo "0")
|
||||
REALNAME_DURATION=$($REDIS_CLI HGET "polling:stats:polling:realname" "total_duration_1h" 2>/dev/null || echo "0")
|
||||
|
||||
CARDDATA_SUCCESS=$($REDIS_CLI HGET "polling:stats:polling:carddata" "success_count_1h" 2>/dev/null || echo "0")
|
||||
CARDDATA_FAILURE=$($REDIS_CLI HGET "polling:stats:polling:carddata" "failure_count_1h" 2>/dev/null || echo "0")
|
||||
CARDDATA_DURATION=$($REDIS_CLI HGET "polling:stats:polling:carddata" "total_duration_1h" 2>/dev/null || echo "0")
|
||||
|
||||
PACKAGE_SUCCESS=$($REDIS_CLI HGET "polling:stats:polling:package" "success_count_1h" 2>/dev/null || echo "0")
|
||||
PACKAGE_FAILURE=$($REDIS_CLI HGET "polling:stats:polling:package" "failure_count_1h" 2>/dev/null || echo "0")
|
||||
PACKAGE_DURATION=$($REDIS_CLI HGET "polling:stats:polling:package" "total_duration_1h" 2>/dev/null || echo "0")
|
||||
|
||||
# 设置默认值
|
||||
REALNAME_SUCCESS=${REALNAME_SUCCESS:-0}
|
||||
REALNAME_FAILURE=${REALNAME_FAILURE:-0}
|
||||
REALNAME_DURATION=${REALNAME_DURATION:-0}
|
||||
CARDDATA_SUCCESS=${CARDDATA_SUCCESS:-0}
|
||||
CARDDATA_FAILURE=${CARDDATA_FAILURE:-0}
|
||||
CARDDATA_DURATION=${CARDDATA_DURATION:-0}
|
||||
PACKAGE_SUCCESS=${PACKAGE_SUCCESS:-0}
|
||||
PACKAGE_FAILURE=${PACKAGE_FAILURE:-0}
|
||||
PACKAGE_DURATION=${PACKAGE_DURATION:-0}
|
||||
|
||||
# 计算增量和 QPS
|
||||
REALNAME_SUCCESS_DELTA=$((REALNAME_SUCCESS - LAST_REALNAME_SUCCESS))
|
||||
REALNAME_FAILURE_DELTA=$((REALNAME_FAILURE - LAST_REALNAME_FAILURE))
|
||||
CARDDATA_SUCCESS_DELTA=$((CARDDATA_SUCCESS - LAST_CARDDATA_SUCCESS))
|
||||
CARDDATA_FAILURE_DELTA=$((CARDDATA_FAILURE - LAST_CARDDATA_FAILURE))
|
||||
PACKAGE_SUCCESS_DELTA=$((PACKAGE_SUCCESS - LAST_PACKAGE_SUCCESS))
|
||||
PACKAGE_FAILURE_DELTA=$((PACKAGE_FAILURE - LAST_PACKAGE_FAILURE))
|
||||
|
||||
REALNAME_QPS=$((REALNAME_SUCCESS_DELTA / INTERVAL))
|
||||
CARDDATA_QPS=$((CARDDATA_SUCCESS_DELTA / INTERVAL))
|
||||
PACKAGE_QPS=$((PACKAGE_SUCCESS_DELTA / INTERVAL))
|
||||
TOTAL_QPS=$((REALNAME_QPS + CARDDATA_QPS + PACKAGE_QPS))
|
||||
|
||||
# 计算成功率
|
||||
REALNAME_TOTAL=$((REALNAME_SUCCESS + REALNAME_FAILURE))
|
||||
CARDDATA_TOTAL=$((CARDDATA_SUCCESS + CARDDATA_FAILURE))
|
||||
PACKAGE_TOTAL=$((PACKAGE_SUCCESS + PACKAGE_FAILURE))
|
||||
|
||||
if [ $REALNAME_TOTAL -gt 0 ]; then
|
||||
REALNAME_RATE=$(echo "scale=1; $REALNAME_SUCCESS * 100 / $REALNAME_TOTAL" | bc)
|
||||
else
|
||||
REALNAME_RATE="0.0"
|
||||
fi
|
||||
if [ $CARDDATA_TOTAL -gt 0 ]; then
|
||||
CARDDATA_RATE=$(echo "scale=1; $CARDDATA_SUCCESS * 100 / $CARDDATA_TOTAL" | bc)
|
||||
else
|
||||
CARDDATA_RATE="0.0"
|
||||
fi
|
||||
if [ $PACKAGE_TOTAL -gt 0 ]; then
|
||||
PACKAGE_RATE=$(echo "scale=1; $PACKAGE_SUCCESS * 100 / $PACKAGE_TOTAL" | bc)
|
||||
else
|
||||
PACKAGE_RATE="0.0"
|
||||
fi
|
||||
|
||||
# 计算平均延迟
|
||||
if [ $REALNAME_SUCCESS -gt 0 ]; then
|
||||
REALNAME_AVG_MS=$((REALNAME_DURATION / REALNAME_SUCCESS))
|
||||
else
|
||||
REALNAME_AVG_MS=0
|
||||
fi
|
||||
if [ $CARDDATA_SUCCESS -gt 0 ]; then
|
||||
CARDDATA_AVG_MS=$((CARDDATA_DURATION / CARDDATA_SUCCESS))
|
||||
else
|
||||
CARDDATA_AVG_MS=0
|
||||
fi
|
||||
if [ $PACKAGE_SUCCESS -gt 0 ]; then
|
||||
PACKAGE_AVG_MS=$((PACKAGE_DURATION / PACKAGE_SUCCESS))
|
||||
else
|
||||
PACKAGE_AVG_MS=0
|
||||
fi
|
||||
|
||||
printf " %-10s | %8s | %8s | %6s | %6s | %8s\n" "任务类型" "成功" "失败" "成功率" "QPS" "平均延迟"
|
||||
printf " %-10s | %8s | %8s | %6s | %6s | %8s\n" "----------" "--------" "--------" "------" "------" "--------"
|
||||
printf " %-10s | %'8d | %'8d | %5.1f%% | %6d | %6dms\n" "实名检查" "$REALNAME_SUCCESS" "$REALNAME_FAILURE" "$REALNAME_RATE" "$REALNAME_QPS" "$REALNAME_AVG_MS"
|
||||
printf " %-10s | %'8d | %'8d | %5.1f%% | %6d | %6dms\n" "流量检查" "$CARDDATA_SUCCESS" "$CARDDATA_FAILURE" "$CARDDATA_RATE" "$CARDDATA_QPS" "$CARDDATA_AVG_MS"
|
||||
printf " %-10s | %'8d | %'8d | %5.1f%% | %6d | %6dms\n" "套餐检查" "$PACKAGE_SUCCESS" "$PACKAGE_FAILURE" "$PACKAGE_RATE" "$PACKAGE_QPS" "$PACKAGE_AVG_MS"
|
||||
printf " %-10s | %8s | %8s | %6s | %6d | %8s\n" "总计" "-" "-" "-" "$TOTAL_QPS" "-"
|
||||
echo ""
|
||||
|
||||
# 更新上次值
|
||||
LAST_REALNAME_SUCCESS=$REALNAME_SUCCESS
|
||||
LAST_REALNAME_FAILURE=$REALNAME_FAILURE
|
||||
LAST_CARDDATA_SUCCESS=$CARDDATA_SUCCESS
|
||||
LAST_CARDDATA_FAILURE=$CARDDATA_FAILURE
|
||||
LAST_PACKAGE_SUCCESS=$PACKAGE_SUCCESS
|
||||
LAST_PACKAGE_FAILURE=$PACKAGE_FAILURE
|
||||
LAST_TIME=$NOW
|
||||
|
||||
# ========== 并发控制状态 ==========
|
||||
echo "【🔒 并发控制状态】"
|
||||
# 注意：current key 包含 polling: 前缀，config key 不包含
|
||||
REALNAME_CURRENT=$($REDIS_CLI GET "polling:concurrency:current:polling:realname" 2>/dev/null || echo "0")
|
||||
REALNAME_MAX=$($REDIS_CLI GET "polling:concurrency:config:realname" 2>/dev/null || echo "50")
|
||||
CARDDATA_CURRENT=$($REDIS_CLI GET "polling:concurrency:current:polling:carddata" 2>/dev/null || echo "0")
|
||||
CARDDATA_MAX=$($REDIS_CLI GET "polling:concurrency:config:carddata" 2>/dev/null || echo "50")
|
||||
PACKAGE_CURRENT=$($REDIS_CLI GET "polling:concurrency:current:polling:package" 2>/dev/null || echo "0")
|
||||
PACKAGE_MAX=$($REDIS_CLI GET "polling:concurrency:config:package" 2>/dev/null || echo "50")
|
||||
|
||||
REALNAME_CURRENT=${REALNAME_CURRENT:-0}
|
||||
REALNAME_MAX=${REALNAME_MAX:-50}
|
||||
CARDDATA_CURRENT=${CARDDATA_CURRENT:-0}
|
||||
CARDDATA_MAX=${CARDDATA_MAX:-50}
|
||||
PACKAGE_CURRENT=${PACKAGE_CURRENT:-0}
|
||||
PACKAGE_MAX=${PACKAGE_MAX:-50}
|
||||
|
||||
if [ "$REALNAME_MAX" = "50" ] && [ -z "$($REDIS_CLI GET "polling:concurrency:config:realname" 2>/dev/null)" ]; then
|
||||
echo " (未启动 Worker，并发配置未加载)"
|
||||
else
|
||||
printf " 实名检查: %d / %s\n" "$REALNAME_CURRENT" "$REALNAME_MAX"
|
||||
printf " 流量检查: %d / %s\n" "$CARDDATA_CURRENT" "$CARDDATA_MAX"
|
||||
printf " 套餐检查: %d / %s\n" "$PACKAGE_CURRENT" "$PACKAGE_MAX"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ========== Mock Gateway 统计 ==========
|
||||
if curl -s http://127.0.0.1:8888/stats > /dev/null 2>&1; then
|
||||
echo "【🌐 Mock Gateway 统计】"
|
||||
GATEWAY_STATS=$(curl -s http://127.0.0.1:8888/stats 2>/dev/null)
|
||||
if [ -n "$GATEWAY_STATS" ]; then
|
||||
echo "$GATEWAY_STATS" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
uptime = data.get('uptime_seconds', 0)
|
||||
total = data.get('total_requests', 0)
|
||||
success = data.get('success_count', 0)
|
||||
failed = data.get('failed_count', 0)
|
||||
qps = data.get('qps', 0)
|
||||
rate = data.get('success_rate', '0%')
|
||||
print(f' 运行时长: {uptime:.0f}s | 总请求: {total:,} | QPS: {qps:.1f} | 成功率: {rate}')
|
||||
except Exception as e:
|
||||
print(f' 解析失败: {e}')
|
||||
" 2>/dev/null || echo " 解析失败"
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ========== Redis 内存 ==========
|
||||
echo "【💾 Redis 内存使用】"
|
||||
REDIS_INFO=$($REDIS_CLI INFO memory 2>/dev/null)
|
||||
if [ -n "$REDIS_INFO" ]; then
|
||||
USED_MEMORY=$(echo "$REDIS_INFO" | grep "used_memory_human:" | cut -d: -f2 | tr -d '\r')
|
||||
MAX_MEMORY=$(echo "$REDIS_INFO" | grep "maxmemory_human:" | cut -d: -f2 | tr -d '\r')
|
||||
printf " 已用: %s / 最大: %s\n" "$USED_MEMORY" "$MAX_MEMORY"
|
||||
else
|
||||
echo " 无法获取 Redis 信息"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ========== 数据库统计（从 Redis 计算）==========
|
||||
echo "【📦 卡统计（队列推算）】"
|
||||
TOTAL_QUEUE=$((REALNAME_QUEUE + CARDDATA_QUEUE + PACKAGE_QUEUE))
|
||||
# 根据配置推算：未实名进入实名队列，已激活进入流量和套餐队列
|
||||
# 这只是近似值，实际统计需要查数据库
|
||||
printf " 队列总卡数: %'d\n" "$TOTAL_QUEUE"
|
||||
printf " 未实名(估): %'d | 已激活(估): %'d\n" "$REALNAME_QUEUE" "$CARDDATA_QUEUE"
|
||||
echo " (注: 精确统计需要数据库连接)"
|
||||
echo ""
|
||||
|
||||
echo "────────────────────────────────────────────────────────────────────────"
|
||||
echo "按 Ctrl+C 退出监控... (每 5 秒刷新)"
|
||||
sleep 5
|
||||
done
|
||||
Reference in New Issue
Block a user