From e98f8561f7a7896d79ea25933be9a8a1a154e8c4 Mon Sep 17 00:00:00 2001 From: MiFaZhan Date: Tue, 3 Nov 2026 10:40:48 +0800 Subject: [PATCH] =?UTF-8?q?fix(deploy):=20nginx=20=E4=B8=8A=E6=B8=B8?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=9C=9F=E9=87=8D=E6=96=B0=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=20+=20=E5=B0=B1=E7=BB=AA=E6=8E=A2=E9=92=88=EF=BC=9B=E5=86=85?= =?UTF-8?q?=E5=AD=98=E4=B8=8A=E9=99=90=E6=8C=89=208C/32G=20=E9=87=8D?= =?UTF-8?q?=E6=96=B0=E5=88=86=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 问题：容器重建换 IP 后，网关与 admin-ui 仍往旧 IP 打，/openapi 与 /api 整片 502，必须手动重启 nginx 容器才恢复（2026-11-03 实机踩到）。 编排 - admin-svr / openapi / admin-ui / docs 增加 healthcheck - admin-ui、gateway、nginx 改 depends_on: condition: service_healthy， 保证 nginx 启动时后端已就绪 - 新增 deploy/compose.memlimit.yml（原来只存在于某台演示机上）： 全套上限 8.5G、实测常驻 2G，留给同机其它服务 20G+； MySQL buffer pool 128M→768M、max_connections 151→300 nginx - gateway / admin-ui / iotlcm.conf 示例 统一改 resolver 127.0.0.11 valid=10s + upstream zone/resolve：运行期跟随 IP 变化，不必重启（实测 6s 内自愈） --- deploy/compose.memlimit.yml | 175 ++++++++++++++++++++++++++++ deploy/config/admin-ui/default.conf | 8 +- deploy/config/gateway/nginx.conf | 11 +- deploy/config/nginx/iotlcm.conf | 26 ++++- deploy/docker-compose.yml | 53 +++++++-- 5 files changed, 255 insertions(+), 18 deletions(-) create mode 100644 deploy/compose.memlimit.yml diff --git a/deploy/compose.memlimit.yml b/deploy/compose.memlimit.yml new file mode 100644 index 0000000..6d70299 --- /dev/null +++ b/deploy/compose.memlimit.yml @@ -0,0 +1,175 @@ +# ============================================================ +# iotlcm 内存上限 override —— 8 vCPU / 32 GB 宿主机，与其他服务共用 +# +# 定位：这台机器是「一机多环境」，iotlcm 只是其中一个占用方。 +# 所以只给全套 20 个服务留 **约 8.5 GB 上限**（实测常驻约 2 GB）， +# 剩余 20+ GB 留给宿主机上要新增的其它服务。 +# +# 没有本文件会怎样：编排里没有内存限制，任何一个服务泄漏/跑批都会 +# 把整机内存吃到 OOM Killer 随机杀进程（表现为容器反复 Restarting +# 且日志里看不到业务异常，只能 dmesg 里看到 oom-kill）。 +# +# 用法（顺序不能反）： +# docker compose -f docker-compose.yml -f compose.memlimit.yml up -d +# +# 只调 resources（+ MySQL 自身缓存参数），不动镜像、端口、卷、依赖。 +# 换成更小的机器（4 GB）时按比例下调：mysql 1024M / tdengine 512M / +# worker-svr 768M / 其余 320M；8 GB 机器可用 1024M / 768M / 512M。 +# ============================================================ + +services: + # ---------- 基础组件 ---------- + mysql: + # buffer pool 由 128M（默认）提到 768M，连接数 151 → 300 + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_general_ci + - --default-time-zone=+08:00 + - --innodb-buffer-pool-size=768M + - --max-connections=300 + deploy: + resources: + limits: + memory: 2048M + reservations: + memory: 768M + + redis: + deploy: + resources: + limits: + memory: 384M + reservations: + memory: 96M + + rabbitmq: + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 192M + + tdengine: + deploy: + resources: + limits: + memory: 1024M + reservations: + memory: 256M + + # 一次性建表任务，跑完即退出 + tdengine-init: + deploy: + resources: + limits: + memory: 128M + + # 单节点 consul 常驻约 110~120 MB，给足余量避免贴边 + consul: + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 96M + + # ---------- 业务服务 ---------- + admin-svr: + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 128M + + admin-ui: + deploy: + resources: + limits: + memory: 64M + + openapi: + deploy: + resources: + limits: + memory: 384M + reservations: + memory: 128M + + upstream-chan: + deploy: + resources: + limits: + memory: 384M + reservations: + memory: 128M + + upstream-proxy: + deploy: + resources: + limits: + memory: 384M + reservations: + memory: 128M + + # 全量同步（流量/状态/IMEI）时内存抬升最明显，按文档「500 MB ~ 1 GB」给 1 GB + worker-svr: + deploy: + resources: + limits: + memory: 1024M + reservations: + memory: 256M + + app-server: + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 128M + + app-web: + deploy: + resources: + limits: + memory: 64M + + sms: + deploy: + resources: + limits: + memory: 192M + + docs: + deploy: + resources: + limits: + memory: 64M + + # ---------- profile 可选服务 ---------- + bot: + deploy: + resources: + limits: + memory: 192M + + onelink-mock: + deploy: + resources: + limits: + memory: 192M + + nginx: + deploy: + resources: + limits: + memory: 128M + + # 单端口入口（with-gateway）；之前没有任何上限 + gateway: + deploy: + resources: + limits: + memory: 256M diff --git a/deploy/config/admin-ui/default.conf b/deploy/config/admin-ui/default.conf index 8ccfa24..5bd178c 100644 --- a/deploy/config/admin-ui/default.conf +++ b/deploy/config/admin-ui/default.conf @@ -1,3 +1,9 @@ +# DNS 解析：nginx 只在启动时解析一次 upstream 主机名，admin-svr 重建换 IP 后 +# /api/ 会一直 502。127.0.0.11 是 Docker 内置 DNS，resolve + zone 让运行期自动跟随。 +resolver 127.0.0.11 valid=10s ipv6=off; + +upstream gw_admin_svr { zone gw_admin_svr 64k; server admin-svr:8008 resolve; } + server { listen 80; root /usr/share/nginx/html; @@ -40,7 +46,7 @@ server { proxy_set_header Upgrade $http_upgrade; proxy_http_version 1.1; chunked_transfer_encoding off; - proxy_pass http://admin-svr:8008/api/v1/; + proxy_pass http://gw_admin_svr/api/v1/; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/deploy/config/gateway/nginx.conf b/deploy/config/gateway/nginx.conf index e004e73..895a61e 100644 --- a/deploy/config/gateway/nginx.conf +++ b/deploy/config/gateway/nginx.conf @@ -16,9 +16,14 @@ # 小程序 / H5 端的静态资源是根路径部署的，不能挂在同一域名的子路径下， # 需要单独域名或单独端口（见 config/nginx/iotlcm.conf 的 app 段）。 -upstream gw_admin_ui { server admin-ui:80; } -upstream gw_openapi { server openapi:8000; } -upstream gw_docs { server docs:80; } +# DNS 解析：nginx 默认只在启动时把 upstream 主机名解析一次，容器重建/重启换了 IP +# 就一直往旧地址打（2026-11-03 踩到：整站 /openapi 全 502，重启网关才恢复）。 +# 127.0.0.11 是 Docker 内置 DNS；下面的 resolve + zone 让它在运行期自动跟随。 +resolver 127.0.0.11 valid=10s ipv6=off; + +upstream gw_admin_ui { zone gw_admin_ui 64k; server admin-ui:80 resolve; } +upstream gw_openapi { zone gw_openapi 64k; server openapi:8000 resolve; } +upstream gw_docs { zone gw_docs 64k; server docs:80 resolve; } server { listen 80; diff --git a/deploy/config/nginx/iotlcm.conf b/deploy/config/nginx/iotlcm.conf index 5c3e1d3..8d36e4d 100644 --- a/deploy/config/nginx/iotlcm.conf +++ b/deploy/config/nginx/iotlcm.conf @@ -8,28 +8,42 @@ # 把 your-domain.example.com 替换为你的实际域名。 # ============================================================ +# ---------- 后端地址解析 ---------- +# 这些 upstream 写的是 compose 服务名。nginx 默认只在**启动时**解析一次， +# 容器重建后 IP 变了就会持续 502（需重启 nginx）。resolve + zone 让它在运行期自动跟随。 +# 注意：127.0.0.11 是 Docker 内置 DNS，仅在容器内有效： +# ・在容器内用本文件 → 保持下面的 resolver 不动 +# ・把本文件并入宿主机 nginx → 把 resolver 换成宿主机 DNS（或删掉 resolver 与各 upstream 的 resolve 参数） +resolver 127.0.0.11 valid=10s ipv6=off; + upstream iotlcm_admin_api { - server admin-svr:8008; + zone iotlcm_admin_api 64k; + server admin-svr:8008 resolve; } upstream iotlcm_admin_ui { - server admin-ui:80; + zone iotlcm_admin_ui 64k; + server admin-ui:80 resolve; } upstream iotlcm_openapi_api { - server openapi:8000; + zone iotlcm_openapi_api 64k; + server openapi:8000 resolve; } upstream iotlcm_docs { - server docs:80; + zone iotlcm_docs 64k; + server docs:80 resolve; } upstream iotlcm_app_web { - server app-web:80; + zone iotlcm_app_web 64k; + server app-web:80 resolve; } upstream iotlcm_app_server { - server app-server:8010; + zone iotlcm_app_server 64k; + server app-server:8010 resolve; } # ---------- OpenAPI 对外域名 ---------- diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 309cb98..44e7199 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -175,6 +175,14 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + # 就绪探针：nginx 类容器只在启动时解析一次后端 IP， + # 没有它就无法用 depends_on 的 service_healthy 保证启动顺序（见 gateway） + healthcheck: + test: ["CMD", "wget", "--quiet", "--output-document=/dev/null", "http://127.0.0.1:8008/api.json"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 20s admin-ui: image: 'crpi-qflrqrxoohlkkvgc.cn-chengdu.personal.cr.aliyuncs.com/iotlcm/iotlcm:iotlcm-admin-ui-v1.0.4' @@ -191,7 +199,14 @@ services: networks: - iotlcm-network depends_on: - - admin-svr + admin-svr: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--quiet", "--output-document=/dev/null", "http://127.0.0.1/"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 5s openapi: image: 'crpi-qflrqrxoohlkkvgc.cn-chengdu.personal.cr.aliyuncs.com/iotlcm/iotlcm:iotlcm-openapi-v1.0.4' @@ -213,6 +228,12 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--quiet", "--output-document=/dev/null", "http://127.0.0.1:8000/api.json"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 20s upstream-chan: image: 'crpi-qflrqrxoohlkkvgc.cn-chengdu.personal.cr.aliyuncs.com/iotlcm/iotlcm:iotlcm-upstream-chan-v1.0.4' @@ -370,6 +391,12 @@ services: - "127.0.0.1:8019:80" networks: - iotlcm-network + healthcheck: + test: ["CMD", "wget", "--quiet", "--output-document=/dev/null", "http://127.0.0.1/docs/"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s # ---------------- 可选：运营商接口挡板 ---------------- # 模拟中国移动 OneLink 接口，用于在没有真实运营商环境时联调整条链路： @@ -418,10 +445,14 @@ services: networks: - iotlcm-network depends_on: - - admin-ui - - openapi - - app-web - - docs + admin-ui: + condition: service_healthy + openapi: + condition: service_healthy + docs: + condition: service_healthy + app-web: + condition: service_started profiles: - with-nginx @@ -437,10 +468,16 @@ services: - ./data/logs/gateway:/var/log/nginx networks: - iotlcm-network + # 后端必须"健康"后才起网关：否则 nginx 会在后端拿到 IP 之前把 upstream + # 解析结果缓存下来，容器重启换 IP 后所有请求变成 502（2026-11-03 踩过）。 + # 配置里另有 resolver + resolve，可在运行期自动跟随 IP 变化。 depends_on: - - admin-ui - - openapi - - docs + admin-ui: + condition: service_healthy + openapi: + condition: service_healthy + docs: + condition: service_healthy profiles: - with-gateway