diff --git a/docker-compose.yml b/docker-compose.yml index 81db5f6..303a4b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,9 @@ services: prefect-worker: build: . container_name: prefect-worker + # init 进程(tini)回收孤儿/僵尸:pull 步骤的 git 子进程被杀后没人 wait 会变僵尸, + # 堆满 cgroup PID 配额后容器内无法再 fork(2026-09-17 事故根因) + init: true environment: PREFECT_API_URL: http://prefect-server:4200/api TZ: Asia/Shanghai diff --git a/flows/label_ocr.py b/flows/label_ocr.py index ff8d92a..9362d77 100644 --- a/flows/label_ocr.py +++ b/flows/label_ocr.py @@ -7,6 +7,7 @@ from __future__ import annotations import os +import time import httpx from prefect import flow, task @@ -16,6 +17,8 @@ from api.ocr import OcrClient, country_to_ocr_code from flows.shared import SyncDB, build_stats_rows, emit_sync_summary BATCH_SIZE = 200 +BATCH_RETRIES = 3 # 批量请求级重试(OCR 服务偶发秒级重启) +BATCH_RETRY_BACKOFF = [5, 15] # 重试间隔(秒) _OCR_FIELD_MAP = [ ("layout", "ocrLayout"), ("courier", "ocrCourier"), ("tracking", "ocrTracking"), @@ -63,7 +66,7 @@ def _has_running_peer() -> bool: def process_all(pending: list) -> dict[str, int]: db = SyncDB() ocr = OcrClient() - stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0} + stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0, "deferred": 0} try: # 1. 按国家分组;不支持的国家直接标 UNSUPPORTED by_country: dict[str, list[tuple[str, object, str]]] = {} @@ -89,12 +92,24 @@ def process_all(pending: list) -> dict[str, int]: urls = [{"url": url, "filename": no} for no, _, url in batch] batch_no = i // BATCH_SIZE + 1 total_batches = -(-total // BATCH_SIZE) - try: - results = ocr.recognize_batch(urls, country_code) - except Exception as err: # noqa: BLE001 - print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败: {str(err)[:100]}") - db.update_ocr_batch([(no, created_map[no], {"ocrLayout": "ERROR"}) for no in created_map]) - stats["error"] += len(batch) + results = None + for attempt in range(BATCH_RETRIES): + try: + results = ocr.recognize_batch(urls, country_code) + break + except Exception as err: # noqa: BLE001 + if attempt < BATCH_RETRIES - 1: + wait = BATCH_RETRY_BACKOFF[min(attempt, len(BATCH_RETRY_BACKOFF) - 1)] + print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败" + f"({str(err)[:80]}),{wait}s 后重试({attempt + 2}/{BATCH_RETRIES})") + time.sleep(wait) + else: + # OCR 服务端短暂不可用(容器秒级重启类故障):订单保持未处理, + # 留给下个小时的 run 自动重试,不落 ERROR(那会是永久终态)。 + print(f"[OCR] {country_code} batch {batch_no}/{total_batches} " + f"重试 {BATCH_RETRIES} 次仍失败,留待下次 run: {str(err)[:100]}") + if results is None: + stats["deferred"] += len(batch) continue updates = [] @@ -152,11 +167,13 @@ def label_ocr( after = db.count_ocr_status() total_processed = stats["success"] + stats["error"] + stats["unsupported"] print(f"完成{scope}: 处理 {total_processed} 单 / {stats['batches']} 批 | " - f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']}") + f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']} " + f"遗留待重试 {stats['deferred']}") print(f"累计: {after}") emit_sync_summary(f"面单 OCR{scope}", build_stats_rows("ALL", {}, [ ("本批处理", total_processed), ("成功", stats["success"]), ("失败", stats["error"]), ("不支持", stats["unsupported"]), + ("遗留待重试", stats["deferred"]), ("累计已完成", after.get("done", 0)), ("累计待处理", after.get("pending", 0)), ])) return {"processed": total_processed, **stats, **after}