feat: 添加环境变量管理工具和部署配置改版
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 5m33s
All checks were successful
构建并部署到测试环境(无 SSH) / build-and-deploy (push) Successful in 5m33s
主要改动: - 新增交互式环境配置脚本 (scripts/setup-env.sh) - 新增本地启动快捷脚本 (scripts/run-local.sh) - 新增环境变量模板文件 (.env.example) - 部署模式改版:使用嵌入式配置 + 环境变量覆盖 - 添加对象存储功能支持 - 改进 IoT 卡片导入任务 - 优化 OpenAPI 文档生成 - 删除旧的配置文件,改用嵌入式默认配置
This commit is contained in:
184
pkg/storage/s3.go
Normal file
184
pkg/storage/s3.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
)
|
||||
|
||||
type S3Provider struct {
|
||||
client *s3.S3
|
||||
uploader *s3manager.Uploader
|
||||
bucket string
|
||||
tempDir string
|
||||
}
|
||||
|
||||
func NewS3Provider(cfg *config.StorageConfig) (*S3Provider, error) {
|
||||
if cfg.S3.Endpoint == "" || cfg.S3.Bucket == "" {
|
||||
return nil, fmt.Errorf("S3 配置不完整:endpoint 和 bucket 必填")
|
||||
}
|
||||
|
||||
if cfg.S3.AccessKeyID == "" || cfg.S3.SecretAccessKey == "" {
|
||||
return nil, fmt.Errorf("S3 凭证未配置:access_key_id 和 secret_access_key 必填")
|
||||
}
|
||||
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Endpoint: aws.String(cfg.S3.Endpoint),
|
||||
Region: aws.String(cfg.S3.Region),
|
||||
Credentials: credentials.NewStaticCredentials(cfg.S3.AccessKeyID, cfg.S3.SecretAccessKey, ""),
|
||||
DisableSSL: aws.Bool(!cfg.S3.UseSSL),
|
||||
S3ForcePathStyle: aws.Bool(cfg.S3.PathStyle),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建 S3 session 失败: %w", err)
|
||||
}
|
||||
|
||||
tempDir := cfg.TempDir
|
||||
if tempDir == "" {
|
||||
tempDir = "/tmp/junhong-storage"
|
||||
}
|
||||
|
||||
return &S3Provider{
|
||||
client: s3.New(sess),
|
||||
uploader: s3manager.NewUploader(sess),
|
||||
bucket: cfg.S3.Bucket,
|
||||
tempDir: tempDir,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) Upload(ctx context.Context, key string, reader io.Reader, contentType string) error {
|
||||
input := &s3manager.UploadInput{
|
||||
Bucket: aws.String(p.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: reader,
|
||||
ContentType: aws.String(contentType),
|
||||
}
|
||||
|
||||
_, err := p.uploader.UploadWithContext(ctx, input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("上传文件失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) Download(ctx context.Context, key string, writer io.Writer) error {
|
||||
input := &s3.GetObjectInput{
|
||||
Bucket: aws.String(p.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
result, err := p.client.GetObjectWithContext(ctx, input)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "NoSuchKey") {
|
||||
return fmt.Errorf("文件不存在: %s", key)
|
||||
}
|
||||
return fmt.Errorf("下载文件失败: %w", err)
|
||||
}
|
||||
defer result.Body.Close()
|
||||
|
||||
_, err = io.Copy(writer, result.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("写入文件内容失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) DownloadToTemp(ctx context.Context, key string) (string, func(), error) {
|
||||
ext := filepath.Ext(key)
|
||||
if ext == "" {
|
||||
ext = ".tmp"
|
||||
}
|
||||
|
||||
tempFile, err := os.CreateTemp(p.tempDir, "download-*"+ext)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("创建临时文件失败: %w", err)
|
||||
}
|
||||
tempPath := tempFile.Name()
|
||||
|
||||
cleanup := func() {
|
||||
_ = os.Remove(tempPath)
|
||||
}
|
||||
|
||||
if err := p.Download(ctx, key, tempFile); err != nil {
|
||||
tempFile.Close()
|
||||
cleanup()
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
if err := tempFile.Close(); err != nil {
|
||||
cleanup()
|
||||
return "", nil, fmt.Errorf("关闭临时文件失败: %w", err)
|
||||
}
|
||||
|
||||
return tempPath, cleanup, nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) Delete(ctx context.Context, key string) error {
|
||||
input := &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(p.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
_, err := p.client.DeleteObjectWithContext(ctx, input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除文件失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) Exists(ctx context.Context, key string) (bool, error) {
|
||||
input := &s3.HeadObjectInput{
|
||||
Bucket: aws.String(p.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
_, err := p.client.HeadObjectWithContext(ctx, input)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "NotFound") || strings.Contains(err.Error(), "404") {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("检查文件存在性失败: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) GetUploadURL(ctx context.Context, key string, contentType string, expires time.Duration) (string, error) {
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(p.bucket),
|
||||
Key: aws.String(key),
|
||||
ContentType: aws.String(contentType),
|
||||
}
|
||||
|
||||
req, _ := p.client.PutObjectRequest(input)
|
||||
url, err := req.Presign(expires)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("生成上传预签名 URL 失败: %w", err)
|
||||
}
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func (p *S3Provider) GetDownloadURL(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||
input := &s3.GetObjectInput{
|
||||
Bucket: aws.String(p.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
req, _ := p.client.GetObjectRequest(input)
|
||||
url, err := req.Presign(expires)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("生成下载预签名 URL 失败: %w", err)
|
||||
}
|
||||
return url, nil
|
||||
}
|
||||
112
pkg/storage/service.go
Normal file
112
pkg/storage/service.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/break/junhong_cmp_fiber/pkg/config"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
provider Provider
|
||||
config *config.StorageConfig
|
||||
}
|
||||
|
||||
func NewService(provider Provider, cfg *config.StorageConfig) *Service {
|
||||
return &Service{
|
||||
provider: provider,
|
||||
config: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) GenerateFileKey(purpose, fileName string) (string, error) {
|
||||
mapping, ok := PurposeMappings[purpose]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("不支持的文件用途: %s", purpose)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(fileName)
|
||||
if ext == "" {
|
||||
ext = ".bin"
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
id := uuid.New().String()
|
||||
|
||||
key := fmt.Sprintf("%s/%04d/%02d/%02d/%s%s",
|
||||
mapping.Prefix,
|
||||
now.Year(),
|
||||
now.Month(),
|
||||
now.Day(),
|
||||
id,
|
||||
strings.ToLower(ext),
|
||||
)
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetUploadURL(ctx context.Context, purpose, fileName, contentType string) (*PresignResult, error) {
|
||||
fileKey, err := s.GenerateFileKey(purpose, fileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if contentType == "" {
|
||||
if mapping, ok := PurposeMappings[purpose]; ok && mapping.ContentType != "" {
|
||||
contentType = mapping.ContentType
|
||||
} else {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
expires := s.config.Presign.UploadExpires
|
||||
if expires == 0 {
|
||||
expires = 15 * time.Minute
|
||||
}
|
||||
|
||||
url, err := s.provider.GetUploadURL(ctx, fileKey, contentType, expires)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &PresignResult{
|
||||
URL: url,
|
||||
FileKey: fileKey,
|
||||
ExpiresIn: int(expires.Seconds()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetDownloadURL(ctx context.Context, fileKey string) (*PresignResult, error) {
|
||||
expires := s.config.Presign.DownloadExpires
|
||||
if expires == 0 {
|
||||
expires = 24 * time.Hour
|
||||
}
|
||||
|
||||
url, err := s.provider.GetDownloadURL(ctx, fileKey, expires)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &PresignResult{
|
||||
URL: url,
|
||||
FileKey: fileKey,
|
||||
ExpiresIn: int(expires.Seconds()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) DownloadToTemp(ctx context.Context, fileKey string) (string, func(), error) {
|
||||
return s.provider.DownloadToTemp(ctx, fileKey)
|
||||
}
|
||||
|
||||
func (s *Service) Provider() Provider {
|
||||
return s.provider
|
||||
}
|
||||
|
||||
func (s *Service) Bucket() string {
|
||||
return s.config.S3.Bucket
|
||||
}
|
||||
17
pkg/storage/storage.go
Normal file
17
pkg/storage/storage.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Provider interface {
|
||||
Upload(ctx context.Context, key string, reader io.Reader, contentType string) error
|
||||
Download(ctx context.Context, key string, writer io.Writer) error
|
||||
DownloadToTemp(ctx context.Context, key string) (localPath string, cleanup func(), err error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
Exists(ctx context.Context, key string) (bool, error)
|
||||
GetUploadURL(ctx context.Context, key string, contentType string, expires time.Duration) (string, error)
|
||||
GetDownloadURL(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||
}
|
||||
18
pkg/storage/types.go
Normal file
18
pkg/storage/types.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package storage
|
||||
|
||||
type PresignResult struct {
|
||||
URL string `json:"url"`
|
||||
FileKey string `json:"file_key"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
}
|
||||
|
||||
type PurposeMapping struct {
|
||||
Prefix string
|
||||
ContentType string
|
||||
}
|
||||
|
||||
var PurposeMappings = map[string]PurposeMapping{
|
||||
"iot_import": {Prefix: "imports", ContentType: "text/csv"},
|
||||
"export": {Prefix: "exports", ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
"attachment": {Prefix: "attachments", ContentType: ""},
|
||||
}
|
||||
Reference in New Issue
Block a user