yeuimu
5aaddbbf7c
- docker-compose: prefect-worker 加 init: true。pull 步骤的 git 子进程被杀后 无人 wait 变僵尸,9/17 宿主机重启后爆发 3.7 万僵尸占满容器 cgroup PID 配额, 导致所有 flow run "Failed to start flow run process"(全量 CRASH 约 40h)。 - label_ocr: OCR 编排服务(lm-server-canary)偶发秒级重启,原逻辑对批量请求 失败零重试并整批标永久 ERROR,9/1、9/5、9/9 三波共 ~2500 单误标。 现改为请求级重试 3 次,仍失败则保持未处理,留给下个小时 run 自动重试。
64 lines
1.8 KiB
YAML
64 lines
1.8 KiB
YAML
services:
|
|
prefect-server:
|
|
image: prefecthq/prefect:3-python3.12
|
|
container_name: prefect-server
|
|
command: prefect server start --host 0.0.0.0
|
|
environment:
|
|
PREFECT_UI_API_URL: http://192.168.124.92:4200/api
|
|
TZ: Asia/Shanghai
|
|
ports:
|
|
- "4200:4200"
|
|
volumes:
|
|
- prefect-data:/root/.prefect
|
|
restart: unless-stopped
|
|
|
|
prefect-worker:
|
|
build: .
|
|
container_name: prefect-worker
|
|
# init 进程(tini)回收孤儿/僵尸:pull 步骤的 git 子进程被杀后没人 wait 会变僵尸,
|
|
# 堆满 cgroup PID 配额后容器内无法再 fork(2026-09-17 事故根因)
|
|
init: true
|
|
environment:
|
|
PREFECT_API_URL: http://prefect-server:4200/api
|
|
TZ: Asia/Shanghai
|
|
depends_on:
|
|
- prefect-server
|
|
volumes:
|
|
# 设备监控(device-check flow):宿主机磁盘使用率 + Docker 容器状态 + 指标采样
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
- /:/host:ro
|
|
- ./data/metrics:/data/metrics:ro
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
# GPU 监控(nvidia-smi);依赖宿主机 nvidia-container-toolkit
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
restart: unless-stopped
|
|
|
|
# 宿主机指标采样侧车:每 5 分钟写内存/swap/GPU,每日 du 快照(供 device-check 日报)
|
|
metrics-agent:
|
|
build: .
|
|
container_name: metrics-agent
|
|
command: python /opt/metrics_agent.py
|
|
environment:
|
|
TZ: Asia/Shanghai
|
|
METRICS_DIR: /data/metrics
|
|
volumes:
|
|
- /:/host:ro
|
|
- ./data/metrics:/data/metrics
|
|
- ./scripts/metrics_agent.py:/opt/metrics_agent.py:ro
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
prefect-data:
|