csxj2026/junhong_cmp_fiber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

每日清理

Browse Source
This commit is contained in:
break
2026-08-20 12:03:17 +08:00
parent 143df60485
commit 22b95db2f9
23 changed files with 614 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/bootstrap/directories.go
View File

@@ -10,10 +10,11 @@ import (
)
type DirectoryResult struct {
TempDir string
AppLogDir string
AccessLogDir string
Fallbacks []string
TempDir string
AppLogDir string
AccessLogDir string
RetentionLogDir string
Fallbacks []string
}
func EnsureDirectories(cfg *config.Config, logger *zap.Logger) (*DirectoryResult, error) {
@@ -27,6 +28,7 @@ func EnsureDirectories(cfg *config.Config, logger *zap.Logger) (*DirectoryResult
{cfg.Storage.TempDir, "storage.temp_dir", &result.TempDir},
{filepath.Dir(cfg.Logging.AppLog.Filename), "logging.app_log.filename", &result.AppLogDir},
{filepath.Dir(cfg.Logging.AccessLog.Filename), "logging.access_log.filename", &result.AccessLogDir},
{filepath.Dir(cfg.Logging.RetentionLog.Filename), "logging.retention_log.filename", &result.RetentionLogDir},
}
for _, dir := range directories {

17
pkg/config/config.go
View File

@@ -79,10 +79,11 @@ type QueueConfig struct {
// LoggingConfig 日志配置
type LoggingConfig struct {
Level string `mapstructure:"level"` // debug, info, warn, error
Development bool `mapstructure:"development"` // 启用开发模式（美化输出）
AppLog LogRotationConfig `mapstructure:"app_log"` // 应用日志配置
AccessLog LogRotationConfig `mapstructure:"access_log"` // HTTP 访问日志配置
Level string `mapstructure:"level"` // debug, info, warn, error
Development bool `mapstructure:"development"` // 启用开发模式（美化输出）
AppLog LogRotationConfig `mapstructure:"app_log"` // 应用日志配置
AccessLog LogRotationConfig `mapstructure:"access_log"` // HTTP 访问日志配置
RetentionLog LogRotationConfig `mapstructure:"retention_log"` // Worker 日留存日志配置
}
// LogRotationConfig Lumberjack 日志轮转配置
@@ -168,7 +169,7 @@ type GatewayConfig struct {
type WorkerConfig struct {
Role string `mapstructure:"role"` // Worker 运行角色：all、leader、consumer
InstanceName string `mapstructure:"instance_name"` // Worker 实例名称，用于多实例日志区分
AuditRetentionCleanupEnabled bool `mapstructure:"audit_retention_cleanup_enabled"` // 是否启用审计日志月度物理清理
AuditRetentionCleanupEnabled bool `mapstructure:"audit_retention_cleanup_enabled"` // 是否启用审计日志物理清理
}
// ApprovalConfig 审批新旧入口切换配置。
@@ -291,12 +292,18 @@ func (c *Config) Validate() error {
if c.Logging.AccessLog.Filename == "" {
return fmt.Errorf("invalid configuration: logging.access_log.filename: must be non-empty valid file path")
}
if c.Logging.RetentionLog.Filename == "" {
return fmt.Errorf("invalid configuration: logging.retention_log.filename: must be non-empty valid file path")
}
if c.Logging.AppLog.MaxSize < 1 || c.Logging.AppLog.MaxSize > 1000 {
return fmt.Errorf("invalid configuration: logging.app_log.max_size: size out of range (current value: %d, expected: 1-1000 MB)", c.Logging.AppLog.MaxSize)
}
if c.Logging.AccessLog.MaxSize < 1 || c.Logging.AccessLog.MaxSize > 1000 {
return fmt.Errorf("invalid configuration: logging.access_log.max_size: size out of range (current value: %d, expected: 1-1000 MB)", c.Logging.AccessLog.MaxSize)
}
if c.Logging.RetentionLog.MaxSize < 1 || c.Logging.RetentionLog.MaxSize > 1000 {
return fmt.Errorf("invalid configuration: logging.retention_log.max_size: size out of range (current value: %d, expected: 1-1000 MB)", c.Logging.RetentionLog.MaxSize)
}
// 中间件验证
if c.Middleware.RateLimiter.Max <= 0 {

8
pkg/config/defaults/config.yaml
View File

@@ -65,6 +65,12 @@ logging:
max_backups: 3
max_age: 7
compress: true
retention_log:
filename: "logs/audit-retention.log"
max_size: 100
max_backups: 3
max_age: 7
compress: true
# 任务队列配置
queue:
@@ -137,7 +143,7 @@ polling_auto_trigger:
worker:
role: "all"
instance_name: ""
# 完整自然月灰度验收通过前必须保持关闭
# 日留存只读演练验收通过前必须保持关闭
audit_retention_cleanup_enabled: false
# 审批新旧入口切换配置

5
pkg/config/loader.go
View File

@@ -94,6 +94,11 @@ func bindEnvVariables(v *viper.Viper) {
"logging.access_log.max_backups",
"logging.access_log.max_age",
"logging.access_log.compress",
"logging.retention_log.filename",
"logging.retention_log.max_size",
"logging.retention_log.max_backups",
"logging.retention_log.max_age",
"logging.retention_log.compress",
"queue.concurrency",
"queue.retry_max",
"queue.timeout",

8
pkg/constants/audit_archive.go
View File

@@ -5,10 +5,8 @@ const (
TaskTypeAuditDailyArchive = "audit:daily:archive"
// TaskTypeIntegrationDailyArchive 表示 Integration Log 每日冷归档任务。
TaskTypeIntegrationDailyArchive = "integration:daily:archive"
// TaskTypeIntegrationMonthlyFinalize 表示 Integration Log 月度最终版本复核任务。
TaskTypeIntegrationMonthlyFinalize = "integration:monthly:finalize"
// TaskTypeAuditMonthlyRetention 表示审计日志月度物理清理任务。
TaskTypeAuditMonthlyRetention = "audit:monthly:retention"
// TaskTypeAuditDailyRetention 表示审计日志日留存任务。
TaskTypeAuditDailyRetention = "audit:daily:retention"
// AuditArchiveSource 表示 Audit Event 与 Event Resource 归档数据源。
AuditArchiveSource = "audit"
@@ -32,6 +30,6 @@ const (
// ArchiveStatusFailed 表示归档任务执行失败并等待重试。
ArchiveStatusFailed = "failed"
// AuditRetentionDeleteBatchSize 表示月度物理清理单批删除上限。
// AuditRetentionDeleteBatchSize 表示日物理清理单批删除上限。
AuditRetentionDeleteBatchSize = 1000
)

2
pkg/constants/constants.go
View File

@@ -298,7 +298,7 @@ func QueueForTaskType(taskType string) string {
return QueueDataCleanup
case TaskTypeDailyTrafficFlush:
return QueueDailyTrafficFlush
case TaskTypeAuditDailyArchive, TaskTypeIntegrationDailyArchive, TaskTypeIntegrationMonthlyFinalize, TaskTypeAuditMonthlyRetention:
case TaskTypeAuditDailyArchive, TaskTypeIntegrationDailyArchive, TaskTypeAuditDailyRetention:
return QueueDataCleanup
case TaskTypeOutboxDeliver:
return QueueOutboxDeliver

17
pkg/logger/retention.go Normal file
View File

@@ -0,0 +1,17 @@
package logger
import (
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
// NewRetentionLogger 创建仅供 Worker 日留存使用的独立轮转日志。
func NewRetentionLogger(level string, config LogRotationConfig) (*zap.Logger, func() error) {
encoder := zapcore.NewJSONEncoder(zapcore.EncoderConfig{
TimeKey: "time", LevelKey: "level", MessageKey: "msg", CallerKey: "caller",
EncodeTime: zapcore.ISO8601TimeEncoder, EncodeLevel: zapcore.CapitalLevelEncoder,
EncodeCaller: zapcore.ShortCallerEncoder,
})
logger := zap.New(zapcore.NewCore(encoder, zapcore.AddSync(newLumberjackLogger(config)), parseLevel(level)), zap.AddCaller())
return logger, logger.Sync
}