Merge pull request 'Develop' (#4) from develop into main
Reviewed-on: #4
This commit was merged in pull request #4.
This commit is contained in:
@@ -15,6 +15,9 @@ services:
|
|||||||
prefect-worker:
|
prefect-worker:
|
||||||
build: .
|
build: .
|
||||||
container_name: prefect-worker
|
container_name: prefect-worker
|
||||||
|
# init 进程(tini)回收孤儿/僵尸:pull 步骤的 git 子进程被杀后没人 wait 会变僵尸,
|
||||||
|
# 堆满 cgroup PID 配额后容器内无法再 fork(2026-09-17 事故根因)
|
||||||
|
init: true
|
||||||
environment:
|
environment:
|
||||||
PREFECT_API_URL: http://prefect-server:4200/api
|
PREFECT_API_URL: http://prefect-server:4200/api
|
||||||
TZ: Asia/Shanghai
|
TZ: Asia/Shanghai
|
||||||
|
|||||||
+22
-5
@@ -7,6 +7,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from prefect import flow, task
|
from prefect import flow, task
|
||||||
@@ -16,6 +17,8 @@ from api.ocr import OcrClient, country_to_ocr_code
|
|||||||
from flows.shared import SyncDB, build_stats_rows, emit_sync_summary
|
from flows.shared import SyncDB, build_stats_rows, emit_sync_summary
|
||||||
|
|
||||||
BATCH_SIZE = 200
|
BATCH_SIZE = 200
|
||||||
|
BATCH_RETRIES = 3 # 批量请求级重试(OCR 服务偶发秒级重启)
|
||||||
|
BATCH_RETRY_BACKOFF = [5, 15] # 重试间隔(秒)
|
||||||
|
|
||||||
_OCR_FIELD_MAP = [
|
_OCR_FIELD_MAP = [
|
||||||
("layout", "ocrLayout"), ("courier", "ocrCourier"), ("tracking", "ocrTracking"),
|
("layout", "ocrLayout"), ("courier", "ocrCourier"), ("tracking", "ocrTracking"),
|
||||||
@@ -63,7 +66,7 @@ def _has_running_peer() -> bool:
|
|||||||
def process_all(pending: list) -> dict[str, int]:
|
def process_all(pending: list) -> dict[str, int]:
|
||||||
db = SyncDB()
|
db = SyncDB()
|
||||||
ocr = OcrClient()
|
ocr = OcrClient()
|
||||||
stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0}
|
stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0, "deferred": 0}
|
||||||
try:
|
try:
|
||||||
# 1. 按国家分组;不支持的国家直接标 UNSUPPORTED
|
# 1. 按国家分组;不支持的国家直接标 UNSUPPORTED
|
||||||
by_country: dict[str, list[tuple[str, object, str]]] = {}
|
by_country: dict[str, list[tuple[str, object, str]]] = {}
|
||||||
@@ -89,12 +92,24 @@ def process_all(pending: list) -> dict[str, int]:
|
|||||||
urls = [{"url": url, "filename": no} for no, _, url in batch]
|
urls = [{"url": url, "filename": no} for no, _, url in batch]
|
||||||
batch_no = i // BATCH_SIZE + 1
|
batch_no = i // BATCH_SIZE + 1
|
||||||
total_batches = -(-total // BATCH_SIZE)
|
total_batches = -(-total // BATCH_SIZE)
|
||||||
|
results = None
|
||||||
|
for attempt in range(BATCH_RETRIES):
|
||||||
try:
|
try:
|
||||||
results = ocr.recognize_batch(urls, country_code)
|
results = ocr.recognize_batch(urls, country_code)
|
||||||
|
break
|
||||||
except Exception as err: # noqa: BLE001
|
except Exception as err: # noqa: BLE001
|
||||||
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败: {str(err)[:100]}")
|
if attempt < BATCH_RETRIES - 1:
|
||||||
db.update_ocr_batch([(no, created_map[no], {"ocrLayout": "ERROR"}) for no in created_map])
|
wait = BATCH_RETRY_BACKOFF[min(attempt, len(BATCH_RETRY_BACKOFF) - 1)]
|
||||||
stats["error"] += len(batch)
|
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败"
|
||||||
|
f"({str(err)[:80]}),{wait}s 后重试({attempt + 2}/{BATCH_RETRIES})")
|
||||||
|
time.sleep(wait)
|
||||||
|
else:
|
||||||
|
# OCR 服务端短暂不可用(容器秒级重启类故障):订单保持未处理,
|
||||||
|
# 留给下个小时的 run 自动重试,不落 ERROR(那会是永久终态)。
|
||||||
|
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} "
|
||||||
|
f"重试 {BATCH_RETRIES} 次仍失败,留待下次 run: {str(err)[:100]}")
|
||||||
|
if results is None:
|
||||||
|
stats["deferred"] += len(batch)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
updates = []
|
updates = []
|
||||||
@@ -152,11 +167,13 @@ def label_ocr(
|
|||||||
after = db.count_ocr_status()
|
after = db.count_ocr_status()
|
||||||
total_processed = stats["success"] + stats["error"] + stats["unsupported"]
|
total_processed = stats["success"] + stats["error"] + stats["unsupported"]
|
||||||
print(f"完成{scope}: 处理 {total_processed} 单 / {stats['batches']} 批 | "
|
print(f"完成{scope}: 处理 {total_processed} 单 / {stats['batches']} 批 | "
|
||||||
f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']}")
|
f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']} "
|
||||||
|
f"遗留待重试 {stats['deferred']}")
|
||||||
print(f"累计: {after}")
|
print(f"累计: {after}")
|
||||||
emit_sync_summary(f"面单 OCR{scope}", build_stats_rows("ALL", {}, [
|
emit_sync_summary(f"面单 OCR{scope}", build_stats_rows("ALL", {}, [
|
||||||
("本批处理", total_processed), ("成功", stats["success"]),
|
("本批处理", total_processed), ("成功", stats["success"]),
|
||||||
("失败", stats["error"]), ("不支持", stats["unsupported"]),
|
("失败", stats["error"]), ("不支持", stats["unsupported"]),
|
||||||
|
("遗留待重试", stats["deferred"]),
|
||||||
("累计已完成", after.get("done", 0)), ("累计待处理", after.get("pending", 0)),
|
("累计已完成", after.get("done", 0)), ("累计待处理", after.get("pending", 0)),
|
||||||
]))
|
]))
|
||||||
return {"processed": total_processed, **stats, **after}
|
return {"processed": total_processed, **stats, **after}
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ RIIN_LOGIN_URL = "http://tshirt.riin.com/auth/api/auth/login"
|
|||||||
RIIN_ORDER_URL = "https://tshirt.riin.com/distribution/order/page"
|
RIIN_ORDER_URL = "https://tshirt.riin.com/distribution/order/page"
|
||||||
USERNAME_RIIN = "预上网"
|
USERNAME_RIIN = "预上网"
|
||||||
PASSWORD_RIIN = "Aa123456."
|
PASSWORD_RIIN = "Aa123456."
|
||||||
SUB_ID_LIST = ["10368", "10283", "16905", "11502", "13615", "11531"]
|
SUB_ID_LIST = ["10368", "10283", "16905", "11502", "13615", "11531","12279"]
|
||||||
MAX_RETRY = 3; REQUEST_TIMEOUT = 30; PAGE_SIZE_RIIN = 200
|
MAX_RETRY = 3; REQUEST_TIMEOUT = 30; PAGE_SIZE_RIIN = 200
|
||||||
|
|
||||||
LOOKBACK_DAYS = 6 # 回查14天避免漏单
|
LOOKBACK_DAYS = 6 # 回查14天避免漏单
|
||||||
@@ -208,4 +208,5 @@ async def riin_platform_sync():
|
|||||||
log_msg("===== 完成 =====")
|
log_msg("===== 完成 =====")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
riin_platform_sync.serve(name="platform-order-riin", cron="15 * * * *")
|
# 直跑一次用于本地调试;勿改回 serve():同名 upsert 会覆盖 server 上挂 work pool 的正式 deployment
|
||||||
|
riin_platform_sync()
|
||||||
|
|||||||
@@ -273,4 +273,5 @@ def rouhui_platform_sync():
|
|||||||
log_msg("===== 提取完成 =====")
|
log_msg("===== 提取完成 =====")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
rouhui_platform_sync.serve(name="platform-order-rouhui", cron="*/15 * * * *")
|
# 直跑一次用于本地调试;勿改回 serve():同名 upsert 会覆盖 server 上挂 work pool 的正式 deployment
|
||||||
|
rouhui_platform_sync()
|
||||||
@@ -471,4 +471,5 @@ async def sdspod_platform_sync():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
sdspod_platform_sync.serve(name="platform-order-sdspod", cron="*/15 * * * *")
|
# 直跑一次用于本地调试;勿改回 serve():同名 upsert 会覆盖 server 上挂 work pool 的正式 deployment
|
||||||
|
sdspod_platform_sync()
|
||||||
Reference in New Issue
Block a user