更新环境配置，优化数据库和Redis连接设置

- 修改`.env.example`文件，更新数据库和Redis的默认主机地址为`192.168.9.127` - 在`docker-compose.yml`中移除不必要的卷挂载配置 - 在`go.mod`中添加`github.com/joho/godotenv`依赖以支持环境变量加载 - 更新`README.md`以反映新的数据库和Redis连接信息 - 在`main.go`中加载环境变量以支持动态配置 - 新增缓存存在性检查功能，避免重复预热缓存 - 优化CSV读取逻辑，支持从S3获取数据
2026-06-08 14:52:23 +08:00
parent 99785672a5
commit 87bdad34fc
15 changed files with 136 additions and 110 deletions
@@ -6,23 +6,23 @@ APP_PORT=8080
 # Gin mode: debug | release | test
 GIN_MODE=debug

-# Database (development defaults apply when APP_ENV=development and vars are unset)
-# DATABASE_HOST=localhost
-# DATABASE_PORT=3306
-# DATABASE_NAME=audio
-# DATABASE_USER=root
-# DATABASE_PASSWORD=root123
+# Database (development reads from env, default host 192.168.9.127)
+DATABASE_HOST=192.168.9.127
+DATABASE_PORT=3306
+DATABASE_NAME=audio
+DATABASE_USER=root
+DATABASE_PASSWORD=root123

 # Meilisearch (development defaults apply when APP_ENV=development and vars are unset)
 # MEILISEARCH_HOST=http://ec2-18-184-205-87.eu-central-1.compute.amazonaws.com:7700
 # MEILISEARCH_API_KEY=your-api-key
 # MEILISEARCH_INDEX=models

-# Redis (development defaults apply when APP_ENV=development and vars are unset)
-# REDIS_HOST=localhost
-# REDIS_PORT=6379
-# REDIS_PASSWORD=
-# REDIS_DATABASE=1
+# Redis (development reads from env, default host 192.168.9.127)
+REDIS_HOST=192.168.9.127
+REDIS_PORT=6379
+REDIS_PASSWORD=
+REDIS_DATABASE=1

 # S3 (local development only; production uses EC2 IAM role)
 # AWS_REGION=eu-central-1
@@ -49,19 +49,23 @@ cp .env.example .env

 | 环境 | `APP_ENV` | 默认连接 |
 |------|-----------|----------|
-| 开发 | `development` | `localhost:3306/audio`，用户 `root`，密码 `root123` |
+| 开发 | `development` | `192.168.9.127:3306/audio`（可通过 `DATABASE_HOST` 覆盖），用户 `root`，密码 `root123` |
 | 正式 | `production` | AWS RDS `database-1.chmuueamo72p.eu-central-1.rds.amazonaws.com:3306/audio` |

 正式环境密码**必须**通过环境变量 `DATABASE_PASSWORD` 提供（不要写入代码仓库）。可参考 `.env.production.example`。

-开发环境可在 `.env` 中覆盖：
+开发环境通过 `.env` 管理 MySQL / Redis 地址（host 变更时只需改 `.env`）：

 ```bash
-DATABASE_HOST=localhost
+DATABASE_HOST=192.168.9.127
 DATABASE_PORT=3306
 DATABASE_NAME=audio
 DATABASE_USER=root
 DATABASE_PASSWORD=root123
+
+REDIS_HOST=192.168.9.127
+REDIS_PORT=6379
+REDIS_DATABASE=1
 ```

 正式环境启动示例（PowerShell）：
@@ -11,6 +11,7 @@ import (
 	"time"

 	"github.com/gin-gonic/gin"
+	"github.com/joho/godotenv"
 	"github.com/luxsin/app-api/internal/cache"
 	"github.com/luxsin/app-api/internal/config"
 	"github.com/luxsin/app-api/internal/database"
@@ -26,6 +27,8 @@ import (
 )

 func main() {
+	_ = godotenv.Load()
+
 	cfg, err := config.Load()
 	if err != nil {
 		panic(err)
@@ -121,13 +124,26 @@ func main() {
 	log.Info("server stopped")
 }

-// warmUpCache 启动时从数据库加载数据到 Redis
+// warmUpCache 启动时从数据库加载数据到 Redis（缓存已存在则跳过，避免多实例重复预热）
 func warmUpCache(log *zap.Logger, db *sql.DB, rdb *redis.Client) {
 	ctx := context.Background()

 	brandCache := cache.NewBrandCache(rdb)
 	modelCache := cache.NewModelCache(rdb)

+	brandExists, err := brandCache.Exists(ctx)
+	if err != nil {
+		log.Warn("check brand cache failed, will warm up", zap.Error(err))
+	}
+	modelExists, err := modelCache.Exists(ctx)
+	if err != nil {
+		log.Warn("check model cache failed, will warm up", zap.Error(err))
+	}
+	if brandExists && modelExists {
+		log.Info("cache already warmed, skip warm-up")
+		return
+	}
+
 	brandRepo := repository.NewBrandRepository(db, brandCache)
 	modelRepo := repository.NewModelRepository(db, modelCache)

@@ -17,11 +17,6 @@ services:
      - ENABLE_PERSIST_TASK=false
      - AWS_REGION=eu-central-1
      - S3_BUCKET=luxsin-app-bucket
-      - MEASUREMENT_BASE_PATH=/app/measurements
-      - TARGET_BASE_PATH=/app/targets
-    volumes:
-      - /data/project/autoeq/measurements:/app/measurements:ro
-      - /data/project/autoeq/targets:/app/targets:ro
    healthcheck:
      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:8080/api/v1/health"]
      interval: 30s
@@ -9,6 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2
 	github.com/gin-gonic/gin v1.10.0
 	github.com/go-sql-driver/mysql v1.10.0
+	github.com/joho/godotenv v1.5.1
 	github.com/meilisearch/meilisearch-go v0.36.2
 	github.com/redis/go-redis/v9 v9.19.0
 	go.uber.org/zap v1.27.0
@@ -78,6 +78,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -23,6 +23,14 @@ func NewBrandCache(rdb *redis.Client) *BrandCache {
 	return &BrandCache{rdb: rdb}
 }

+func (c *BrandCache) Exists(ctx context.Context) (bool, error) {
+	n, err := c.rdb.Exists(ctx, brandAllKey).Result()
+	if err != nil {
+		return false, err
+	}
+	return n > 0, nil
+}
+
 func (c *BrandCache) GetAll(ctx context.Context) ([]model.Brand, error) {
 	data, err := c.rdb.Get(ctx, brandAllKey).Bytes()
 	if err != nil {
@@ -24,6 +24,14 @@ func NewModelCache(rdb *redis.Client) *ModelCache {
 	return &ModelCache{rdb: rdb}
 }

+func (c *ModelCache) Exists(ctx context.Context) (bool, error) {
+	n, err := c.rdb.Exists(ctx, modelAllKey).Result()
+	if err != nil {
+		return false, err
+	}
+	return n > 0, nil
+}
+
 func modelBrandKey(brandName string) string {
 	return modelBrandPrefix + brandName
 }
@@ -15,12 +15,11 @@ type DatabaseConfig struct {
 }

 func loadDatabase(env string) (DatabaseConfig, error) {
-	if os.Getenv("DATABASE_HOST") != "" {
-		return databaseFromEnv()
-	}
-
 	switch env {
 	case "production":
+		if os.Getenv("DATABASE_HOST") != "" {
+			return databaseFromEnv()
+		}
 		return DatabaseConfig{
 			Host:     "database-1.chmuueamo72p.eu-central-1.rds.amazonaws.com",
 			Port:     3306,
@@ -29,12 +28,16 @@ func loadDatabase(env string) (DatabaseConfig, error) {
 			Password: os.Getenv("DATABASE_PASSWORD"),
 		}, nil
 	default:
+		port, err := strconv.Atoi(getEnv("DATABASE_PORT", "3306"))
+		if err != nil {
+			return DatabaseConfig{}, fmt.Errorf("invalid DATABASE_PORT: %w", err)
+		}
 		return DatabaseConfig{
-			Host:     "192.168.9.137",
-			Port:     3306,
-			Name:     "audio",
-			User:     "root",
-			Password: "root123",
+			Host:     getEnv("DATABASE_HOST", "192.168.9.127"),
+			Port:     port,
+			Name:     getEnv("DATABASE_NAME", "audio"),
+			User:     getEnv("DATABASE_USER", "root"),
+			Password: getEnv("DATABASE_PASSWORD", "root123"),
 		}, nil
 	}
 }
@@ -3,32 +3,18 @@ package config
 import "os"

 type EqualizeConfig struct {
-	APIURL              string
-	MeasurementBasePath string
-	TargetBasePath      string
+	APIURL string
 }

 func loadEqualize(env string) EqualizeConfig {
 	if os.Getenv("EQ_API_URL") != "" {
-		return EqualizeConfig{
-			APIURL:              os.Getenv("EQ_API_URL"),
-			MeasurementBasePath: getEnv("MEASUREMENT_BASE_PATH", "/app/measurements"),
-			TargetBasePath:      getEnv("TARGET_BASE_PATH", "/app/targets"),
-		}
+		return EqualizeConfig{APIURL: os.Getenv("EQ_API_URL")}
 	}

 	switch env {
 	case "production":
-		return EqualizeConfig{
-			APIURL:              "http://172.31.18.70:8000/equalize",
-			MeasurementBasePath: getEnv("MEASUREMENT_BASE_PATH", "/app/measurements"),
-			TargetBasePath:      getEnv("TARGET_BASE_PATH", "/app/targets"),
-		}
+		return EqualizeConfig{APIURL: "http://172.31.18.70:8000/equalize"}
 	default:
-		return EqualizeConfig{
-			APIURL:              "https://autoeq.app/equalize",
-			MeasurementBasePath: getEnv("MEASUREMENT_BASE_PATH", "/app/measurements"),
-			TargetBasePath:      getEnv("TARGET_BASE_PATH", "/app/targets"),
-		}
+		return EqualizeConfig{APIURL: "https://autoeq.app/equalize"}
 	}
 }
@@ -14,12 +14,11 @@ type RedisConfig struct {
 }

 func loadRedis(env string) RedisConfig {
-	if os.Getenv("REDIS_HOST") != "" {
-		return redisFromEnv()
-	}
-
 	switch env {
 	case "production":
+		if os.Getenv("REDIS_HOST") != "" {
+			return redisFromEnv("16279")
+		}
 		return RedisConfig{
 			Host:     "172.31.38.162",
 			Port:     16279,
@@ -27,17 +26,19 @@ func loadRedis(env string) RedisConfig {
 			Database: 1,
 		}
 	default:
+		port, _ := strconv.Atoi(getEnv("REDIS_PORT", "6379"))
+		db, _ := strconv.Atoi(getEnv("REDIS_DATABASE", "1"))
 		return RedisConfig{
-			Host:     "192.168.9.137",
-			Port:     6379,
-			Password: "",
-			Database: 1,
+			Host:     getEnv("REDIS_HOST", "192.168.9.127"),
+			Port:     port,
+			Password: getEnv("REDIS_PASSWORD", ""),
+			Database: db,
 		}
 	}
 }

-func redisFromEnv() RedisConfig {
-	port, _ := strconv.Atoi(getEnv("REDIS_PORT", "16279"))
+func redisFromEnv(defaultPort string) RedisConfig {
+	port, _ := strconv.Atoi(getEnv("REDIS_PORT", defaultPort))
 	db, _ := strconv.Atoi(getEnv("REDIS_DATABASE", "1"))

 	return RedisConfig{
@@ -3,22 +3,23 @@ package handler
 import (
 	"bytes"
 	"context"
-	"encoding/csv"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"strconv"
 	"strings"
 	"time"

+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/gin-gonic/gin"
 	"github.com/luxsin/app-api/internal/cache"
 	"github.com/luxsin/app-api/internal/config"
 	"github.com/luxsin/app-api/internal/repository"
 	"github.com/luxsin/app-api/internal/response"
+	"github.com/luxsin/app-api/internal/storage"
 	"github.com/luxsin/app-api/pkg/encode"
 	"go.uber.org/zap"
 )
@@ -27,13 +28,14 @@ type CurveHandler struct {
 	repo  *repository.CurveRepository
 	cache *cache.CurveCache
 	cfg   config.EqualizeConfig
+	s3    *storage.S3Storage
 	log   *zap.Logger
 }

 const defaultModelCurveTarget = "Harman over-ear 2018"

-func NewCurveHandler(repo *repository.CurveRepository, curveCache *cache.CurveCache, cfg config.EqualizeConfig, log *zap.Logger) *CurveHandler {
-	return &CurveHandler{repo: repo, cache: curveCache, cfg: cfg, log: log}
+func NewCurveHandler(repo *repository.CurveRepository, curveCache *cache.CurveCache, cfg config.EqualizeConfig, s3 *storage.S3Storage, log *zap.Logger) *CurveHandler {
+	return &CurveHandler{repo: repo, cache: curveCache, cfg: cfg, s3: s3, log: log}
 }

 // ModelCurve 获取机型默认曲线（固定 target: Harman over-ear 2018）
@@ -82,7 +84,8 @@ func (h *CurveHandler) ModelCurve(c *gin.Context) {
 func (h *CurveHandler) GetCurve(c *gin.Context) {
 	brand := strings.TrimSpace(queryParam(c, "brand"))
 	name := strings.TrimSpace(queryParam(c, "name"))
-	target := strings.TrimSpace(queryParam(c, "target"))
+	// target 使用标准 query 解码：+ 表示空格（如 Harman+over-ear+2018 → Harman over-ear 2018）
+	target := strings.TrimSpace(c.Query("target"))
 	base64Resp := encode.ParseBase64Param(c)

 	if brand == "" || name == "" || target == "" {
@@ -215,10 +218,11 @@ func (h *CurveHandler) getCurvePointFromPEQ(ctx context.Context, brand, name, ta
 		headPhone = name
 	}

-	// 获取 measurement 数据（仅 Eafonyoung 源需要读 CSV）
+	// 获取 measurement 数据（仅 Eafonyoung 源需要从 S3 读 CSV）
 	var measurement map[string]any
 	if m.Source != nil && *m.Source == "Eafonyoung" {
-		measurement, err = h.readCSV(h.cfg.MeasurementBasePath + "/Eafonyoung/data/" + deref(m.Form) + "/" + headPhone + ".csv")
+		key := modelCSVKey(brand, name, deref(m.Form))
+		measurement, err = h.readCSVFromS3(ctx, key)
 		if err != nil || measurement == nil {
 			return "", nil
 		}
@@ -229,7 +233,8 @@ func (h *CurveHandler) getCurvePointFromPEQ(ctx context.Context, brand, name, ta
 	var targetRaw map[string]any

 	if bool(t.ReadCSV) {
-		targetRaw, err = h.readCSV(h.cfg.TargetBasePath + "/" + deref(t.File))
+		key := targetCSVKey(deref(t.File))
+		targetRaw, err = h.readCSVFromS3(ctx, key)
 		if err != nil || targetRaw == nil {
 			return "", nil
 		}
@@ -369,55 +374,24 @@ func (h *CurveHandler) reqEqualize(headPhone string, measurement map[string]any,
 	return string(resultJSON), nil
 }

-// readCSV 读取 CSV 文件并返回 {frequency: [...], raw: [...]}
-func (h *CurveHandler) readCSV(path string) (map[string]any, error) {
-	f, err := os.Open(path)
+// readCSVFromS3 从 S3 读取 CSV 并返回 {frequency: [...], raw: [...]}
+func (h *CurveHandler) readCSVFromS3(ctx context.Context, key string) (map[string]any, error) {
+	h.log.Info("reading csv from s3", zap.String("key", key))
+	data, err := h.s3.GetObject(ctx, key)
 	if err != nil {
-		if os.IsNotExist(err) {
-			h.log.Info("csv file not found", zap.String("path", path))
+		var noSuchKey *types.NoSuchKey
+		if errors.As(err, &noSuchKey) {
+			h.log.Info("csv not found in s3", zap.String("key", key))
 			return nil, nil
 		}
-		return nil, fmt.Errorf("open csv: %w", err)
-	}
-	defer f.Close()
-
-	reader := csv.NewReader(f)
-	// 跳过表头
-	if _, err := reader.Read(); err != nil {
-		return nil, fmt.Errorf("read csv header: %w", err)
+		return nil, fmt.Errorf("get csv from s3: %w", err)
 	}

-	frequency := make([]float64, 0)
-	raw := make([]float64, 0)
-
-	for {
-		record, err := reader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, fmt.Errorf("read csv row: %w", err)
-		}
-		if len(record) < 2 {
-			continue
-		}
-		freq, err1 := strconv.ParseFloat(record[0], 64)
-		val, err2 := strconv.ParseFloat(record[1], 64)
-		if err1 != nil || err2 != nil {
-			continue
-		}
-		frequency = append(frequency, freq)
-		raw = append(raw, val)
+	parsed, err := parseCSVData(data)
+	if err != nil {
+		return nil, fmt.Errorf("parse csv: %w", err)
 	}
-
-	if len(frequency) == 0 {
-		return nil, nil
-	}
-
-	return map[string]any{
-		"frequency": frequency,
-		"raw":       raw,
-	}, nil
+	return parsed, nil
 }

 // 辅助函数
@@ -46,6 +46,7 @@ func (h *ModelCSVHandler) GetModelCSV(c *gin.Context) {
 	key := modelCSVKey(brand, model, form)
 	ctx := c.Request.Context()

+	h.log.Info("reading csv from s3", zap.String("key", key))
 	data, err := h.s3.GetObject(ctx, key)
 	if err != nil {
 		var noSuchKey *types.NoSuchKey
@@ -86,6 +87,11 @@ func (h *ModelCSVHandler) GetModelCSV(c *gin.Context) {
 	})
 }

+func targetCSVKey(file string) string {
+	file = strings.TrimPrefix(strings.TrimSpace(file), "/")
+	return "autoeq/targets/" + file
+}
+
 func modelCSVKey(brand, model, form string) string {
 	filename := brand + " " + model + ".csv"
 	return fmt.Sprintf("autoeq/measurements/Eafonyoung/data/%s/%s/%s",
@@ -40,7 +40,7 @@ func New(log *zap.Logger, db *sql.DB, searchClient *search.Client, rdb *redis.Cl
 	modelList := handler.NewModelListHandler(searchClient, log)
 	device := handler.NewDeviceHandler(rdb, log)
 	ota := handler.NewOTAHandler(otaRepo, log)
-	curve := handler.NewCurveHandler(curveRepo, curveCache, eqCfg, log)
+	curve := handler.NewCurveHandler(curveRepo, curveCache, eqCfg, s3, log)
 	modelCSV := handler.NewModelCSVHandler(s3, log)

 	v1 := r.Group("/api/v1")
@@ -1,8 +1,29 @@
 #!/usr/bin/env bash
 set -euo pipefail

-REMOTE="ubuntu@ec2-18-184-205-87.eu-central-1.compute.amazonaws.com:/data/project/app-api/"
 KEY="$HOME/.ssh/aws_eafon.pem"
+REMOTE_DIR="/data/project/app-api/"
+
+SERVER="api1"
+if [ $# -gt 0 ] && { [ "$1" = "api1" ] || [ "$1" = "api2" ]; }; then
+  SERVER="$1"
+  shift
+fi
+
+case "$SERVER" in
+  api1)
+    REMOTE_HOST="ubuntu@ec2-18-184-205-87.eu-central-1.compute.amazonaws.com"
+    ;;
+  api2)
+    REMOTE_HOST="ubuntu@ec2-3-71-153-45.eu-central-1.compute.amazonaws.com"
+    ;;
+  *)
+    echo "未知服务器: $SERVER（可选: api1, api2）" >&2
+    exit 1
+    ;;
+esac
+
+REMOTE="${REMOTE_HOST}:${REMOTE_DIR}"

 if [ $# -eq 0 ]; then
  PATHS=(
@@ -18,4 +39,5 @@ else
  PATHS=("$@")
 fi

+echo "上传到 ${SERVER} (${REMOTE_HOST}) ..."
 scp -i "$KEY" -r "${PATHS[@]}" "$REMOTE"