fix: worker 加 init 回收僵尸进程;label-ocr 批量请求重试且失败不落 ERROR

- docker-compose: prefect-worker 加 init: true。pull 步骤的 git 子进程被杀后
  无人 wait 变僵尸,9/17 宿主机重启后爆发 3.7 万僵尸占满容器 cgroup PID 配额,
  导致所有 flow run "Failed to start flow run process"(全量 CRASH 约 40h)。
- label_ocr: OCR 编排服务(lm-server-canary)偶发秒级重启,原逻辑对批量请求
  失败零重试并整批标永久 ERROR,9/1、9/5、9/9 三波共 ~2500 单误标。
  现改为请求级重试 3 次,仍失败则保持未处理,留给下个小时 run 自动重试。
This commit is contained in:
yeuimu
2026-09-18 10:09:24 +00:00
parent fbba147413
commit 5aaddbbf7c
2 changed files with 28 additions and 8 deletions
+25 -8
View File
@@ -7,6 +7,7 @@
from __future__ import annotations
import os
import time
import httpx
from prefect import flow, task
@@ -16,6 +17,8 @@ from api.ocr import OcrClient, country_to_ocr_code
from flows.shared import SyncDB, build_stats_rows, emit_sync_summary
BATCH_SIZE = 200
BATCH_RETRIES = 3 # 批量请求级重试(OCR 服务偶发秒级重启)
BATCH_RETRY_BACKOFF = [5, 15] # 重试间隔(秒)
_OCR_FIELD_MAP = [
("layout", "ocrLayout"), ("courier", "ocrCourier"), ("tracking", "ocrTracking"),
@@ -63,7 +66,7 @@ def _has_running_peer() -> bool:
def process_all(pending: list) -> dict[str, int]:
db = SyncDB()
ocr = OcrClient()
stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0}
stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0, "deferred": 0}
try:
# 1. 按国家分组;不支持的国家直接标 UNSUPPORTED
by_country: dict[str, list[tuple[str, object, str]]] = {}
@@ -89,12 +92,24 @@ def process_all(pending: list) -> dict[str, int]:
urls = [{"url": url, "filename": no} for no, _, url in batch]
batch_no = i // BATCH_SIZE + 1
total_batches = -(-total // BATCH_SIZE)
try:
results = ocr.recognize_batch(urls, country_code)
except Exception as err: # noqa: BLE001
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败: {str(err)[:100]}")
db.update_ocr_batch([(no, created_map[no], {"ocrLayout": "ERROR"}) for no in created_map])
stats["error"] += len(batch)
results = None
for attempt in range(BATCH_RETRIES):
try:
results = ocr.recognize_batch(urls, country_code)
break
except Exception as err: # noqa: BLE001
if attempt < BATCH_RETRIES - 1:
wait = BATCH_RETRY_BACKOFF[min(attempt, len(BATCH_RETRY_BACKOFF) - 1)]
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败"
f"({str(err)[:80]}),{wait}s 后重试({attempt + 2}/{BATCH_RETRIES})")
time.sleep(wait)
else:
# OCR 服务端短暂不可用(容器秒级重启类故障):订单保持未处理,
# 留给下个小时的 run 自动重试,不落 ERROR(那会是永久终态)。
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} "
f"重试 {BATCH_RETRIES} 次仍失败,留待下次 run: {str(err)[:100]}")
if results is None:
stats["deferred"] += len(batch)
continue
updates = []
@@ -152,11 +167,13 @@ def label_ocr(
after = db.count_ocr_status()
total_processed = stats["success"] + stats["error"] + stats["unsupported"]
print(f"完成{scope}: 处理 {total_processed} 单 / {stats['batches']} 批 | "
f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']}")
f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']} "
f"遗留待重试 {stats['deferred']}")
print(f"累计: {after}")
emit_sync_summary(f"面单 OCR{scope}", build_stats_rows("ALL", {}, [
("本批处理", total_processed), ("成功", stats["success"]),
("失败", stats["error"]), ("不支持", stats["unsupported"]),
("遗留待重试", stats["deferred"]),
("累计已完成", after.get("done", 0)), ("累计待处理", after.get("pending", 0)),
]))
return {"processed": total_processed, **stats, **after}