feat: 实现 IoT 卡轮询系统(支持千万级卡规模)
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 6m35s

实现功能:
- 实名状态检查轮询(可配置间隔)
- 卡流量检查轮询(支持跨月流量追踪)
- 套餐检查与超额自动停机
- 分布式并发控制(Redis 信号量)
- 手动触发轮询(单卡/批量/条件筛选)
- 数据清理配置与执行
- 告警规则与历史记录
- 实时监控统计(队列/性能/并发)

性能优化:
- Redis 缓存卡信息,减少 DB 查询
- Pipeline 批量写入 Redis
- 异步流量记录写入
- 渐进式初始化(10万卡/批)

压测工具(scripts/benchmark/):
- Mock Gateway 模拟上游服务
- 测试卡生成器
- 配置初始化脚本
- 实时监控脚本

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-02-05 17:32:44 +08:00
parent b11edde720
commit 931e140e8e
104 changed files with 16883 additions and 87 deletions

View File

@@ -0,0 +1,2 @@
-- 删除轮询配置表
DROP TABLE IF EXISTS tb_polling_config;

View File

@@ -0,0 +1,44 @@
-- 轮询配置表
-- 先删除旧表(如果存在)
DROP TABLE IF EXISTS tb_polling_config CASCADE;
CREATE TABLE tb_polling_config (
id BIGSERIAL PRIMARY KEY,
config_name VARCHAR(100) NOT NULL,
card_condition VARCHAR(50),
card_category VARCHAR(50),
carrier_id BIGINT,
priority INT NOT NULL DEFAULT 100,
realname_check_interval INT,
carddata_check_interval INT,
package_check_interval INT,
status SMALLINT NOT NULL DEFAULT 1,
description TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
created_by BIGINT,
updated_by BIGINT
);
-- 索引
CREATE INDEX idx_polling_config_status_priority ON tb_polling_config(status, priority);
CREATE INDEX idx_polling_config_carrier_id ON tb_polling_config(carrier_id);
-- 表注释
COMMENT ON TABLE tb_polling_config IS '轮询配置表 - 定义不同条件下的卡轮询策略';
-- 列注释
COMMENT ON COLUMN tb_polling_config.config_name IS '配置名称';
COMMENT ON COLUMN tb_polling_config.card_condition IS '卡状态条件not_real_name/real_name/activated/suspended';
COMMENT ON COLUMN tb_polling_config.card_category IS '卡业务类型normal/industry';
COMMENT ON COLUMN tb_polling_config.carrier_id IS '运营商ID可选精确匹配';
COMMENT ON COLUMN tb_polling_config.priority IS '优先级(数字越小优先级越高)';
COMMENT ON COLUMN tb_polling_config.realname_check_interval IS '实名检查间隔NULL表示不检查';
COMMENT ON COLUMN tb_polling_config.carddata_check_interval IS '流量检查间隔NULL表示不检查';
COMMENT ON COLUMN tb_polling_config.package_check_interval IS '套餐检查间隔NULL表示不检查';
COMMENT ON COLUMN tb_polling_config.status IS '状态0-禁用1-启用';
COMMENT ON COLUMN tb_polling_config.description IS '配置说明';
COMMENT ON COLUMN tb_polling_config.created_at IS '创建时间';
COMMENT ON COLUMN tb_polling_config.updated_at IS '更新时间';
COMMENT ON COLUMN tb_polling_config.created_by IS '创建人ID';
COMMENT ON COLUMN tb_polling_config.updated_by IS '更新人ID';

View File

@@ -0,0 +1,2 @@
-- 删除并发控制配置表
DROP TABLE IF EXISTS tb_polling_concurrency_config;

View File

@@ -0,0 +1,24 @@
-- 并发控制配置表
CREATE TABLE IF NOT EXISTS tb_polling_concurrency_config (
id BIGSERIAL PRIMARY KEY,
task_type VARCHAR(50) NOT NULL UNIQUE,
max_concurrency INT NOT NULL DEFAULT 50,
description TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_by BIGINT
);
-- 索引
CREATE INDEX idx_polling_concurrency_task_type ON tb_polling_concurrency_config(task_type);
-- 表注释
COMMENT ON TABLE tb_polling_concurrency_config IS '并发控制配置表 - 控制不同类型任务的最大并发数';
-- 列注释
COMMENT ON COLUMN tb_polling_concurrency_config.task_type IS '任务类型realname/carddata/package/stop_start';
COMMENT ON COLUMN tb_polling_concurrency_config.max_concurrency IS '最大并发数';
COMMENT ON COLUMN tb_polling_concurrency_config.description IS '配置说明';
COMMENT ON COLUMN tb_polling_concurrency_config.created_at IS '创建时间';
COMMENT ON COLUMN tb_polling_concurrency_config.updated_at IS '更新时间';
COMMENT ON COLUMN tb_polling_concurrency_config.updated_by IS '更新人ID';

View File

@@ -0,0 +1,2 @@
-- 删除告警规则表
DROP TABLE IF EXISTS tb_polling_alert_rule;

View File

@@ -0,0 +1,45 @@
-- 告警规则表
CREATE TABLE IF NOT EXISTS tb_polling_alert_rule (
id BIGSERIAL PRIMARY KEY,
rule_name VARCHAR(100) NOT NULL,
task_type VARCHAR(50) NOT NULL,
metric_type VARCHAR(50) NOT NULL,
operator VARCHAR(20) NOT NULL,
threshold DECIMAL(10, 2) NOT NULL,
duration_minutes INT NOT NULL DEFAULT 5,
alert_level VARCHAR(20) NOT NULL DEFAULT 'warning',
notification_channels TEXT,
notification_config TEXT,
status SMALLINT NOT NULL DEFAULT 1,
cooldown_minutes INT NOT NULL DEFAULT 5,
description TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
created_by BIGINT,
updated_by BIGINT
);
-- 索引
CREATE INDEX idx_polling_alert_rule_status ON tb_polling_alert_rule(status);
CREATE INDEX idx_polling_alert_rule_task_type ON tb_polling_alert_rule(task_type);
-- 表注释
COMMENT ON TABLE tb_polling_alert_rule IS '告警规则表 - 定义轮询系统的告警条件和通知方式';
-- 列注释
COMMENT ON COLUMN tb_polling_alert_rule.rule_name IS '规则名称';
COMMENT ON COLUMN tb_polling_alert_rule.task_type IS '任务类型realname/carddata/package';
COMMENT ON COLUMN tb_polling_alert_rule.metric_type IS '指标类型queue_size/success_rate/avg_duration/concurrency';
COMMENT ON COLUMN tb_polling_alert_rule.operator IS '比较运算符gt/lt/gte/lte/eq';
COMMENT ON COLUMN tb_polling_alert_rule.threshold IS '阈值';
COMMENT ON COLUMN tb_polling_alert_rule.duration_minutes IS '持续时长(分钟),避免短暂波动';
COMMENT ON COLUMN tb_polling_alert_rule.alert_level IS '告警级别info/warning/error/critical';
COMMENT ON COLUMN tb_polling_alert_rule.notification_channels IS '通知渠道JSON数组["email","sms","webhook"]';
COMMENT ON COLUMN tb_polling_alert_rule.notification_config IS '通知配置JSON';
COMMENT ON COLUMN tb_polling_alert_rule.status IS '状态0-禁用1-启用';
COMMENT ON COLUMN tb_polling_alert_rule.cooldown_minutes IS '冷却期(分钟),避免重复告警';
COMMENT ON COLUMN tb_polling_alert_rule.description IS '规则说明';
COMMENT ON COLUMN tb_polling_alert_rule.created_at IS '创建时间';
COMMENT ON COLUMN tb_polling_alert_rule.updated_at IS '更新时间';
COMMENT ON COLUMN tb_polling_alert_rule.created_by IS '创建人ID';
COMMENT ON COLUMN tb_polling_alert_rule.updated_by IS '更新人ID';

View File

@@ -0,0 +1,2 @@
-- 删除告警历史表
DROP TABLE IF EXISTS tb_polling_alert_history;

View File

@@ -0,0 +1,36 @@
-- 告警历史表
CREATE TABLE IF NOT EXISTS tb_polling_alert_history (
id BIGSERIAL PRIMARY KEY,
rule_id BIGINT NOT NULL,
task_type VARCHAR(50) NOT NULL,
metric_type VARCHAR(50) NOT NULL,
alert_level VARCHAR(20) NOT NULL,
current_value DECIMAL(10, 2) NOT NULL,
threshold DECIMAL(10, 2) NOT NULL,
alert_message TEXT NOT NULL,
notification_channels TEXT,
notification_status VARCHAR(20),
notification_result TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- 索引
CREATE INDEX idx_polling_alert_history_rule_id ON tb_polling_alert_history(rule_id);
CREATE INDEX idx_polling_alert_history_created_at ON tb_polling_alert_history(created_at);
CREATE INDEX idx_polling_alert_history_task_type ON tb_polling_alert_history(task_type);
-- 表注释
COMMENT ON TABLE tb_polling_alert_history IS '告警历史表 - 记录所有触发的告警';
-- 列注释
COMMENT ON COLUMN tb_polling_alert_history.rule_id IS '告警规则ID';
COMMENT ON COLUMN tb_polling_alert_history.task_type IS '任务类型';
COMMENT ON COLUMN tb_polling_alert_history.metric_type IS '指标类型';
COMMENT ON COLUMN tb_polling_alert_history.alert_level IS '告警级别';
COMMENT ON COLUMN tb_polling_alert_history.current_value IS '当前值';
COMMENT ON COLUMN tb_polling_alert_history.threshold IS '阈值';
COMMENT ON COLUMN tb_polling_alert_history.alert_message IS '告警消息';
COMMENT ON COLUMN tb_polling_alert_history.notification_channels IS '通知渠道JSON数组';
COMMENT ON COLUMN tb_polling_alert_history.notification_status IS '通知状态pending/sent/failed';
COMMENT ON COLUMN tb_polling_alert_history.notification_result IS '通知结果JSON';
COMMENT ON COLUMN tb_polling_alert_history.created_at IS '告警时间';

View File

@@ -0,0 +1,2 @@
-- 删除数据清理配置表
DROP TABLE IF EXISTS tb_data_cleanup_config;

View File

@@ -0,0 +1,28 @@
-- 数据清理配置表
CREATE TABLE IF NOT EXISTS tb_data_cleanup_config (
id BIGSERIAL PRIMARY KEY,
table_name VARCHAR(100) NOT NULL UNIQUE,
retention_days INT NOT NULL,
enabled SMALLINT NOT NULL DEFAULT 1,
batch_size INT NOT NULL DEFAULT 10000,
description TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_by BIGINT
);
-- 索引
CREATE INDEX idx_data_cleanup_config_enabled ON tb_data_cleanup_config(enabled);
-- 表注释
COMMENT ON TABLE tb_data_cleanup_config IS '数据清理配置表 - 定义各表的数据保留策略';
-- 列注释
COMMENT ON COLUMN tb_data_cleanup_config.table_name IS '表名';
COMMENT ON COLUMN tb_data_cleanup_config.retention_days IS '保留天数';
COMMENT ON COLUMN tb_data_cleanup_config.enabled IS '是否启用0-禁用1-启用';
COMMENT ON COLUMN tb_data_cleanup_config.batch_size IS '每批删除条数';
COMMENT ON COLUMN tb_data_cleanup_config.description IS '配置说明';
COMMENT ON COLUMN tb_data_cleanup_config.created_at IS '创建时间';
COMMENT ON COLUMN tb_data_cleanup_config.updated_at IS '更新时间';
COMMENT ON COLUMN tb_data_cleanup_config.updated_by IS '更新人ID';

View File

@@ -0,0 +1,2 @@
-- 删除数据清理日志表
DROP TABLE IF EXISTS tb_data_cleanup_log;

View File

@@ -0,0 +1,34 @@
-- 数据清理日志表
CREATE TABLE IF NOT EXISTS tb_data_cleanup_log (
id BIGSERIAL PRIMARY KEY,
table_name VARCHAR(100) NOT NULL,
cleanup_type VARCHAR(50) NOT NULL,
retention_days INT NOT NULL,
deleted_count BIGINT NOT NULL DEFAULT 0,
duration_ms BIGINT NOT NULL DEFAULT 0,
status VARCHAR(20) NOT NULL,
error_message TEXT,
started_at TIMESTAMP NOT NULL,
completed_at TIMESTAMP,
triggered_by BIGINT
);
-- 索引
CREATE INDEX idx_data_cleanup_log_table_name ON tb_data_cleanup_log(table_name);
CREATE INDEX idx_data_cleanup_log_started_at ON tb_data_cleanup_log(started_at);
CREATE INDEX idx_data_cleanup_log_status ON tb_data_cleanup_log(status);
-- 表注释
COMMENT ON TABLE tb_data_cleanup_log IS '数据清理日志表 - 记录所有数据清理操作';
-- 列注释
COMMENT ON COLUMN tb_data_cleanup_log.table_name IS '表名';
COMMENT ON COLUMN tb_data_cleanup_log.cleanup_type IS '清理类型scheduled/manual';
COMMENT ON COLUMN tb_data_cleanup_log.retention_days IS '保留天数';
COMMENT ON COLUMN tb_data_cleanup_log.deleted_count IS '删除记录数';
COMMENT ON COLUMN tb_data_cleanup_log.duration_ms IS '执行耗时(毫秒)';
COMMENT ON COLUMN tb_data_cleanup_log.status IS '状态success/failed/running';
COMMENT ON COLUMN tb_data_cleanup_log.error_message IS '错误信息';
COMMENT ON COLUMN tb_data_cleanup_log.started_at IS '开始时间';
COMMENT ON COLUMN tb_data_cleanup_log.completed_at IS '完成时间';
COMMENT ON COLUMN tb_data_cleanup_log.triggered_by IS '触发人ID手动触发时';

View File

@@ -0,0 +1,2 @@
-- 删除手动触发日志表
DROP TABLE IF EXISTS tb_polling_manual_trigger_log;

View File

@@ -0,0 +1,39 @@
-- 手动触发日志表
CREATE TABLE IF NOT EXISTS tb_polling_manual_trigger_log (
id BIGSERIAL PRIMARY KEY,
task_type VARCHAR(50) NOT NULL,
trigger_type VARCHAR(50) NOT NULL,
card_ids TEXT,
condition_filter TEXT,
total_count INT NOT NULL DEFAULT 0,
processed_count INT NOT NULL DEFAULT 0,
success_count INT NOT NULL DEFAULT 0,
failed_count INT NOT NULL DEFAULT 0,
status VARCHAR(20) NOT NULL,
triggered_by BIGINT NOT NULL,
triggered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
completed_at TIMESTAMP
);
-- 索引
CREATE INDEX idx_polling_manual_trigger_log_task_type ON tb_polling_manual_trigger_log(task_type);
CREATE INDEX idx_polling_manual_trigger_log_triggered_by ON tb_polling_manual_trigger_log(triggered_by);
CREATE INDEX idx_polling_manual_trigger_log_triggered_at ON tb_polling_manual_trigger_log(triggered_at);
CREATE INDEX idx_polling_manual_trigger_log_status ON tb_polling_manual_trigger_log(status);
-- 表注释
COMMENT ON TABLE tb_polling_manual_trigger_log IS '手动触发日志表 - 记录手动触发轮询检查的操作';
-- 列注释
COMMENT ON COLUMN tb_polling_manual_trigger_log.task_type IS '任务类型realname/carddata/package';
COMMENT ON COLUMN tb_polling_manual_trigger_log.trigger_type IS '触发类型single/batch/by_condition';
COMMENT ON COLUMN tb_polling_manual_trigger_log.card_ids IS '卡ID列表JSON数组';
COMMENT ON COLUMN tb_polling_manual_trigger_log.condition_filter IS '筛选条件JSON';
COMMENT ON COLUMN tb_polling_manual_trigger_log.total_count IS '总卡数';
COMMENT ON COLUMN tb_polling_manual_trigger_log.processed_count IS '已处理数';
COMMENT ON COLUMN tb_polling_manual_trigger_log.success_count IS '成功数';
COMMENT ON COLUMN tb_polling_manual_trigger_log.failed_count IS '失败数';
COMMENT ON COLUMN tb_polling_manual_trigger_log.status IS '状态pending/processing/completed/cancelled';
COMMENT ON COLUMN tb_polling_manual_trigger_log.triggered_by IS '触发人ID';
COMMENT ON COLUMN tb_polling_manual_trigger_log.triggered_at IS '触发时间';
COMMENT ON COLUMN tb_polling_manual_trigger_log.completed_at IS '完成时间';

View File

@@ -0,0 +1,8 @@
-- 删除 tb_iot_card 的月流量追踪字段
ALTER TABLE tb_iot_card
DROP COLUMN IF EXISTS current_month_usage_mb,
DROP COLUMN IF EXISTS current_month_start_date,
DROP COLUMN IF EXISTS last_month_total_mb;
DROP INDEX IF EXISTS idx_iot_card_current_month_start_date;

View File

@@ -0,0 +1,15 @@
-- 为 tb_iot_card 添加月流量追踪字段
-- 用于处理 Gateway 返回的自然月流量总量并计算增量
ALTER TABLE tb_iot_card
ADD COLUMN current_month_usage_mb DECIMAL(10, 2) DEFAULT 0,
ADD COLUMN current_month_start_date DATE,
ADD COLUMN last_month_total_mb DECIMAL(10, 2) DEFAULT 0;
-- 索引
CREATE INDEX idx_iot_card_current_month_start_date ON tb_iot_card(current_month_start_date);
-- 字段注释
COMMENT ON COLUMN tb_iot_card.current_month_usage_mb IS '本月已用流量MB - Gateway返回的自然月流量总量';
COMMENT ON COLUMN tb_iot_card.current_month_start_date IS '本月开始日期 - 用于检测跨月流量重置';
COMMENT ON COLUMN tb_iot_card.last_month_total_mb IS '上月结束时的总流量MB - 用于跨月流量计算';

View File

@@ -0,0 +1,2 @@
-- 删除流量使用记录表
DROP TABLE IF EXISTS tb_data_usage_record;

View File

@@ -0,0 +1,26 @@
-- 创建流量使用记录表
CREATE TABLE IF NOT EXISTS tb_data_usage_record (
id SERIAL PRIMARY KEY,
iot_card_id INTEGER NOT NULL,
data_usage_mb BIGINT NOT NULL DEFAULT 0,
data_increase_mb BIGINT DEFAULT 0,
check_time TIMESTAMP NOT NULL,
source VARCHAR(50) DEFAULT 'polling',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT fk_data_usage_record_iot_card FOREIGN KEY (iot_card_id) REFERENCES tb_iot_card(id) ON DELETE CASCADE
);
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_data_usage_record_iot_card_id ON tb_data_usage_record(iot_card_id);
CREATE INDEX IF NOT EXISTS idx_data_usage_record_check_time ON tb_data_usage_record(check_time);
CREATE INDEX IF NOT EXISTS idx_data_usage_record_iot_card_check_time ON tb_data_usage_record(iot_card_id, check_time DESC);
COMMENT ON TABLE tb_data_usage_record IS '流量使用记录表';
COMMENT ON COLUMN tb_data_usage_record.id IS '流量使用记录ID';
COMMENT ON COLUMN tb_data_usage_record.iot_card_id IS 'IoT卡ID';
COMMENT ON COLUMN tb_data_usage_record.data_usage_mb IS '流量使用量(MB)';
COMMENT ON COLUMN tb_data_usage_record.data_increase_mb IS '相比上次的增量(MB)';
COMMENT ON COLUMN tb_data_usage_record.check_time IS '检查时间';
COMMENT ON COLUMN tb_data_usage_record.source IS '数据来源 polling-轮询 manual-手动 gateway-回调';
COMMENT ON COLUMN tb_data_usage_record.created_at IS '创建时间';