From 87bdad34fc18443628f9d4b4abc750ddc9bda898 Mon Sep 17 00:00:00 2001 From: eafonyang Date: Mon, 8 Jun 2026 14:52:23 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=8E=AF=E5=A2=83=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=92=8CRedis=E8=BF=9E=E6=8E=A5=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修改`.env.example`文件，更新数据库和Redis的默认主机地址为`192.168.9.127` - 在`docker-compose.yml`中移除不必要的卷挂载配置 - 在`go.mod`中添加`github.com/joho/godotenv`依赖以支持环境变量加载 - 更新`README.md`以反映新的数据库和Redis连接信息 - 在`main.go`中加载环境变量以支持动态配置 - 新增缓存存在性检查功能，避免重复预热缓存 - 优化CSV读取逻辑，支持从S3获取数据 --- .env.example | 22 +++++----- README.md | 10 +++-- cmd/server/main.go | 18 ++++++++- docker-compose.yml | 5 --- go.mod | 1 + go.sum | 2 + internal/cache/brand_cache.go | 8 ++++ internal/cache/model_cache.go | 8 ++++ internal/config/database.go | 21 +++++----- internal/config/equalize.go | 22 ++-------- internal/config/redis.go | 21 +++++----- internal/handler/curve.go | 76 ++++++++++++----------------------- internal/handler/model_csv.go | 6 +++ internal/router/router.go | 2 +- scripts/upload.sh | 24 ++++++++++- 15 files changed, 136 insertions(+), 110 deletions(-) diff --git a/.env.example b/.env.example index 1a9cd0a..8bac6fc 100644 --- a/.env.example +++ b/.env.example @@ -6,23 +6,23 @@ APP_PORT=8080 # Gin mode: debug | release | test GIN_MODE=debug -# Database (development defaults apply when APP_ENV=development and vars are unset) -# DATABASE_HOST=localhost -# DATABASE_PORT=3306 -# DATABASE_NAME=audio -# DATABASE_USER=root -# DATABASE_PASSWORD=root123 +# Database (development reads from env, default host 192.168.9.127) +DATABASE_HOST=192.168.9.127 +DATABASE_PORT=3306 +DATABASE_NAME=audio +DATABASE_USER=root +DATABASE_PASSWORD=root123 # Meilisearch (development defaults apply when APP_ENV=development and vars are unset) # MEILISEARCH_HOST=http://ec2-18-184-205-87.eu-central-1.compute.amazonaws.com:7700 # MEILISEARCH_API_KEY=your-api-key # MEILISEARCH_INDEX=models -# Redis (development defaults apply when APP_ENV=development and vars are unset) -# REDIS_HOST=localhost -# REDIS_PORT=6379 -# REDIS_PASSWORD= -# REDIS_DATABASE=1 +# Redis (development reads from env, default host 192.168.9.127) +REDIS_HOST=192.168.9.127 +REDIS_PORT=6379 +REDIS_PASSWORD= +REDIS_DATABASE=1 # S3 (local development only; production uses EC2 IAM role) # AWS_REGION=eu-central-1 diff --git a/README.md b/README.md index 1d09671..faaf65b 100644 --- a/README.md +++ b/README.md @@ -49,19 +49,23 @@ cp .env.example .env | 环境 | `APP_ENV` | 默认连接 | |------|-----------|----------| -| 开发 | `development` | `localhost:3306/audio`，用户 `root`，密码 `root123` | +| 开发 | `development` | `192.168.9.127:3306/audio`（可通过 `DATABASE_HOST` 覆盖），用户 `root`，密码 `root123` | | 正式 | `production` | AWS RDS `database-1.chmuueamo72p.eu-central-1.rds.amazonaws.com:3306/audio` | 正式环境密码**必须**通过环境变量 `DATABASE_PASSWORD` 提供（不要写入代码仓库）。可参考 `.env.production.example`。 -开发环境可在 `.env` 中覆盖： +开发环境通过 `.env` 管理 MySQL / Redis 地址（host 变更时只需改 `.env`）： ```bash -DATABASE_HOST=localhost +DATABASE_HOST=192.168.9.127 DATABASE_PORT=3306 DATABASE_NAME=audio DATABASE_USER=root DATABASE_PASSWORD=root123 + +REDIS_HOST=192.168.9.127 +REDIS_PORT=6379 +REDIS_DATABASE=1 ``` 正式环境启动示例（PowerShell）： diff --git a/cmd/server/main.go b/cmd/server/main.go index c2e1f5b..f6098a4 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -11,6 +11,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/joho/godotenv" "github.com/luxsin/app-api/internal/cache" "github.com/luxsin/app-api/internal/config" "github.com/luxsin/app-api/internal/database" @@ -26,6 +27,8 @@ import ( ) func main() { + _ = godotenv.Load() + cfg, err := config.Load() if err != nil { panic(err) @@ -121,13 +124,26 @@ func main() { log.Info("server stopped") } -// warmUpCache 启动时从数据库加载数据到 Redis +// warmUpCache 启动时从数据库加载数据到 Redis（缓存已存在则跳过，避免多实例重复预热） func warmUpCache(log *zap.Logger, db *sql.DB, rdb *redis.Client) { ctx := context.Background() brandCache := cache.NewBrandCache(rdb) modelCache := cache.NewModelCache(rdb) + brandExists, err := brandCache.Exists(ctx) + if err != nil { + log.Warn("check brand cache failed, will warm up", zap.Error(err)) + } + modelExists, err := modelCache.Exists(ctx) + if err != nil { + log.Warn("check model cache failed, will warm up", zap.Error(err)) + } + if brandExists && modelExists { + log.Info("cache already warmed, skip warm-up") + return + } + brandRepo := repository.NewBrandRepository(db, brandCache) modelRepo := repository.NewModelRepository(db, modelCache) diff --git a/docker-compose.yml b/docker-compose.yml index 7b87d3b..d863155 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,11 +17,6 @@ services: - ENABLE_PERSIST_TASK=false - AWS_REGION=eu-central-1 - S3_BUCKET=luxsin-app-bucket - - MEASUREMENT_BASE_PATH=/app/measurements - - TARGET_BASE_PATH=/app/targets - volumes: - - /data/project/autoeq/measurements:/app/measurements:ro - - /data/project/autoeq/targets:/app/targets:ro healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:8080/api/v1/health"] interval: 30s diff --git a/go.mod b/go.mod index 27c988e..6a51883 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2 github.com/gin-gonic/gin v1.10.0 github.com/go-sql-driver/mysql v1.10.0 + github.com/joho/godotenv v1.5.1 github.com/meilisearch/meilisearch-go v0.36.2 github.com/redis/go-redis/v9 v9.19.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 8f44e70..e865b81 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= diff --git a/internal/cache/brand_cache.go b/internal/cache/brand_cache.go index 9168cdd..168c8da 100644 --- a/internal/cache/brand_cache.go +++ b/internal/cache/brand_cache.go @@ -23,6 +23,14 @@ func NewBrandCache(rdb *redis.Client) *BrandCache { return &BrandCache{rdb: rdb} } +func (c *BrandCache) Exists(ctx context.Context) (bool, error) { + n, err := c.rdb.Exists(ctx, brandAllKey).Result() + if err != nil { + return false, err + } + return n > 0, nil +} + func (c *BrandCache) GetAll(ctx context.Context) ([]model.Brand, error) { data, err := c.rdb.Get(ctx, brandAllKey).Bytes() if err != nil { diff --git a/internal/cache/model_cache.go b/internal/cache/model_cache.go index a4dcdfe..182aeb3 100644 --- a/internal/cache/model_cache.go +++ b/internal/cache/model_cache.go @@ -24,6 +24,14 @@ func NewModelCache(rdb *redis.Client) *ModelCache { return &ModelCache{rdb: rdb} } +func (c *ModelCache) Exists(ctx context.Context) (bool, error) { + n, err := c.rdb.Exists(ctx, modelAllKey).Result() + if err != nil { + return false, err + } + return n > 0, nil +} + func modelBrandKey(brandName string) string { return modelBrandPrefix + brandName } diff --git a/internal/config/database.go b/internal/config/database.go index 8f200d1..eebaa95 100644 --- a/internal/config/database.go +++ b/internal/config/database.go @@ -15,12 +15,11 @@ type DatabaseConfig struct { } func loadDatabase(env string) (DatabaseConfig, error) { - if os.Getenv("DATABASE_HOST") != "" { - return databaseFromEnv() - } - switch env { case "production": + if os.Getenv("DATABASE_HOST") != "" { + return databaseFromEnv() + } return DatabaseConfig{ Host: "database-1.chmuueamo72p.eu-central-1.rds.amazonaws.com", Port: 3306, @@ -29,12 +28,16 @@ func loadDatabase(env string) (DatabaseConfig, error) { Password: os.Getenv("DATABASE_PASSWORD"), }, nil default: + port, err := strconv.Atoi(getEnv("DATABASE_PORT", "3306")) + if err != nil { + return DatabaseConfig{}, fmt.Errorf("invalid DATABASE_PORT: %w", err) + } return DatabaseConfig{ - Host: "192.168.9.137", - Port: 3306, - Name: "audio", - User: "root", - Password: "root123", + Host: getEnv("DATABASE_HOST", "192.168.9.127"), + Port: port, + Name: getEnv("DATABASE_NAME", "audio"), + User: getEnv("DATABASE_USER", "root"), + Password: getEnv("DATABASE_PASSWORD", "root123"), }, nil } } diff --git a/internal/config/equalize.go b/internal/config/equalize.go index 4873d2f..261e199 100644 --- a/internal/config/equalize.go +++ b/internal/config/equalize.go @@ -3,32 +3,18 @@ package config import "os" type EqualizeConfig struct { - APIURL string - MeasurementBasePath string - TargetBasePath string + APIURL string } func loadEqualize(env string) EqualizeConfig { if os.Getenv("EQ_API_URL") != "" { - return EqualizeConfig{ - APIURL: os.Getenv("EQ_API_URL"), - MeasurementBasePath: getEnv("MEASUREMENT_BASE_PATH", "/app/measurements"), - TargetBasePath: getEnv("TARGET_BASE_PATH", "/app/targets"), - } + return EqualizeConfig{APIURL: os.Getenv("EQ_API_URL")} } switch env { case "production": - return EqualizeConfig{ - APIURL: "http://172.31.18.70:8000/equalize", - MeasurementBasePath: getEnv("MEASUREMENT_BASE_PATH", "/app/measurements"), - TargetBasePath: getEnv("TARGET_BASE_PATH", "/app/targets"), - } + return EqualizeConfig{APIURL: "http://172.31.18.70:8000/equalize"} default: - return EqualizeConfig{ - APIURL: "https://autoeq.app/equalize", - MeasurementBasePath: getEnv("MEASUREMENT_BASE_PATH", "/app/measurements"), - TargetBasePath: getEnv("TARGET_BASE_PATH", "/app/targets"), - } + return EqualizeConfig{APIURL: "https://autoeq.app/equalize"} } } diff --git a/internal/config/redis.go b/internal/config/redis.go index 9a00ca6..8f969b0 100644 --- a/internal/config/redis.go +++ b/internal/config/redis.go @@ -14,12 +14,11 @@ type RedisConfig struct { } func loadRedis(env string) RedisConfig { - if os.Getenv("REDIS_HOST") != "" { - return redisFromEnv() - } - switch env { case "production": + if os.Getenv("REDIS_HOST") != "" { + return redisFromEnv("16279") + } return RedisConfig{ Host: "172.31.38.162", Port: 16279, @@ -27,17 +26,19 @@ func loadRedis(env string) RedisConfig { Database: 1, } default: + port, _ := strconv.Atoi(getEnv("REDIS_PORT", "6379")) + db, _ := strconv.Atoi(getEnv("REDIS_DATABASE", "1")) return RedisConfig{ - Host: "192.168.9.137", - Port: 6379, - Password: "", - Database: 1, + Host: getEnv("REDIS_HOST", "192.168.9.127"), + Port: port, + Password: getEnv("REDIS_PASSWORD", ""), + Database: db, } } } -func redisFromEnv() RedisConfig { - port, _ := strconv.Atoi(getEnv("REDIS_PORT", "16279")) +func redisFromEnv(defaultPort string) RedisConfig { + port, _ := strconv.Atoi(getEnv("REDIS_PORT", defaultPort)) db, _ := strconv.Atoi(getEnv("REDIS_DATABASE", "1")) return RedisConfig{ diff --git a/internal/handler/curve.go b/internal/handler/curve.go index 507dbf7..5a36233 100644 --- a/internal/handler/curve.go +++ b/internal/handler/curve.go @@ -3,22 +3,23 @@ package handler import ( "bytes" "context" - "encoding/csv" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" - "os" "strconv" "strings" "time" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gin-gonic/gin" "github.com/luxsin/app-api/internal/cache" "github.com/luxsin/app-api/internal/config" "github.com/luxsin/app-api/internal/repository" "github.com/luxsin/app-api/internal/response" + "github.com/luxsin/app-api/internal/storage" "github.com/luxsin/app-api/pkg/encode" "go.uber.org/zap" ) @@ -27,13 +28,14 @@ type CurveHandler struct { repo *repository.CurveRepository cache *cache.CurveCache cfg config.EqualizeConfig + s3 *storage.S3Storage log *zap.Logger } const defaultModelCurveTarget = "Harman over-ear 2018" -func NewCurveHandler(repo *repository.CurveRepository, curveCache *cache.CurveCache, cfg config.EqualizeConfig, log *zap.Logger) *CurveHandler { - return &CurveHandler{repo: repo, cache: curveCache, cfg: cfg, log: log} +func NewCurveHandler(repo *repository.CurveRepository, curveCache *cache.CurveCache, cfg config.EqualizeConfig, s3 *storage.S3Storage, log *zap.Logger) *CurveHandler { + return &CurveHandler{repo: repo, cache: curveCache, cfg: cfg, s3: s3, log: log} } // ModelCurve 获取机型默认曲线（固定 target: Harman over-ear 2018） @@ -82,7 +84,8 @@ func (h *CurveHandler) ModelCurve(c *gin.Context) { func (h *CurveHandler) GetCurve(c *gin.Context) { brand := strings.TrimSpace(queryParam(c, "brand")) name := strings.TrimSpace(queryParam(c, "name")) - target := strings.TrimSpace(queryParam(c, "target")) + // target 使用标准 query 解码：+ 表示空格（如 Harman+over-ear+2018 → Harman over-ear 2018） + target := strings.TrimSpace(c.Query("target")) base64Resp := encode.ParseBase64Param(c) if brand == "" || name == "" || target == "" { @@ -215,10 +218,11 @@ func (h *CurveHandler) getCurvePointFromPEQ(ctx context.Context, brand, name, ta headPhone = name } - // 获取 measurement 数据（仅 Eafonyoung 源需要读 CSV） + // 获取 measurement 数据（仅 Eafonyoung 源需要从 S3 读 CSV） var measurement map[string]any if m.Source != nil && *m.Source == "Eafonyoung" { - measurement, err = h.readCSV(h.cfg.MeasurementBasePath + "/Eafonyoung/data/" + deref(m.Form) + "/" + headPhone + ".csv") + key := modelCSVKey(brand, name, deref(m.Form)) + measurement, err = h.readCSVFromS3(ctx, key) if err != nil || measurement == nil { return "", nil } @@ -229,7 +233,8 @@ func (h *CurveHandler) getCurvePointFromPEQ(ctx context.Context, brand, name, ta var targetRaw map[string]any if bool(t.ReadCSV) { - targetRaw, err = h.readCSV(h.cfg.TargetBasePath + "/" + deref(t.File)) + key := targetCSVKey(deref(t.File)) + targetRaw, err = h.readCSVFromS3(ctx, key) if err != nil || targetRaw == nil { return "", nil } @@ -369,55 +374,24 @@ func (h *CurveHandler) reqEqualize(headPhone string, measurement map[string]any, return string(resultJSON), nil } -// readCSV 读取 CSV 文件并返回 {frequency: [...], raw: [...]} -func (h *CurveHandler) readCSV(path string) (map[string]any, error) { - f, err := os.Open(path) +// readCSVFromS3 从 S3 读取 CSV 并返回 {frequency: [...], raw: [...]} +func (h *CurveHandler) readCSVFromS3(ctx context.Context, key string) (map[string]any, error) { + h.log.Info("reading csv from s3", zap.String("key", key)) + data, err := h.s3.GetObject(ctx, key) if err != nil { - if os.IsNotExist(err) { - h.log.Info("csv file not found", zap.String("path", path)) + var noSuchKey *types.NoSuchKey + if errors.As(err, &noSuchKey) { + h.log.Info("csv not found in s3", zap.String("key", key)) return nil, nil } - return nil, fmt.Errorf("open csv: %w", err) - } - defer f.Close() - - reader := csv.NewReader(f) - // 跳过表头 - if _, err := reader.Read(); err != nil { - return nil, fmt.Errorf("read csv header: %w", err) + return nil, fmt.Errorf("get csv from s3: %w", err) } - frequency := make([]float64, 0) - raw := make([]float64, 0) - - for { - record, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, fmt.Errorf("read csv row: %w", err) - } - if len(record) < 2 { - continue - } - freq, err1 := strconv.ParseFloat(record[0], 64) - val, err2 := strconv.ParseFloat(record[1], 64) - if err1 != nil || err2 != nil { - continue - } - frequency = append(frequency, freq) - raw = append(raw, val) + parsed, err := parseCSVData(data) + if err != nil { + return nil, fmt.Errorf("parse csv: %w", err) } - - if len(frequency) == 0 { - return nil, nil - } - - return map[string]any{ - "frequency": frequency, - "raw": raw, - }, nil + return parsed, nil } // 辅助函数 diff --git a/internal/handler/model_csv.go b/internal/handler/model_csv.go index ffd57dc..7e5ce80 100644 --- a/internal/handler/model_csv.go +++ b/internal/handler/model_csv.go @@ -46,6 +46,7 @@ func (h *ModelCSVHandler) GetModelCSV(c *gin.Context) { key := modelCSVKey(brand, model, form) ctx := c.Request.Context() + h.log.Info("reading csv from s3", zap.String("key", key)) data, err := h.s3.GetObject(ctx, key) if err != nil { var noSuchKey *types.NoSuchKey @@ -86,6 +87,11 @@ func (h *ModelCSVHandler) GetModelCSV(c *gin.Context) { }) } +func targetCSVKey(file string) string { + file = strings.TrimPrefix(strings.TrimSpace(file), "/") + return "autoeq/targets/" + file +} + func modelCSVKey(brand, model, form string) string { filename := brand + " " + model + ".csv" return fmt.Sprintf("autoeq/measurements/Eafonyoung/data/%s/%s/%s", diff --git a/internal/router/router.go b/internal/router/router.go index 6547d14..a434721 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -40,7 +40,7 @@ func New(log *zap.Logger, db *sql.DB, searchClient *search.Client, rdb *redis.Cl modelList := handler.NewModelListHandler(searchClient, log) device := handler.NewDeviceHandler(rdb, log) ota := handler.NewOTAHandler(otaRepo, log) - curve := handler.NewCurveHandler(curveRepo, curveCache, eqCfg, log) + curve := handler.NewCurveHandler(curveRepo, curveCache, eqCfg, s3, log) modelCSV := handler.NewModelCSVHandler(s3, log) v1 := r.Group("/api/v1") diff --git a/scripts/upload.sh b/scripts/upload.sh index 7885f24..1e55140 100755 --- a/scripts/upload.sh +++ b/scripts/upload.sh @@ -1,8 +1,29 @@ #!/usr/bin/env bash set -euo pipefail -REMOTE="ubuntu@ec2-18-184-205-87.eu-central-1.compute.amazonaws.com:/data/project/app-api/" KEY="$HOME/.ssh/aws_eafon.pem" +REMOTE_DIR="/data/project/app-api/" + +SERVER="api1" +if [ $# -gt 0 ] && { [ "$1" = "api1" ] || [ "$1" = "api2" ]; }; then + SERVER="$1" + shift +fi + +case "$SERVER" in + api1) + REMOTE_HOST="ubuntu@ec2-18-184-205-87.eu-central-1.compute.amazonaws.com" + ;; + api2) + REMOTE_HOST="ubuntu@ec2-3-71-153-45.eu-central-1.compute.amazonaws.com" + ;; + *) + echo "未知服务器: $SERVER（可选: api1, api2）" >&2 + exit 1 + ;; +esac + +REMOTE="${REMOTE_HOST}:${REMOTE_DIR}" if [ $# -eq 0 ]; then PATHS=( @@ -18,4 +39,5 @@ else PATHS=("$@") fi +echo "上传到 ${SERVER} (${REMOTE_HOST}) ..." scp -i "$KEY" -r "${PATHS[@]}" "$REMOTE"