Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a7c44a354 | ||
|
|
740bd803a9 | ||
|
|
f87662ba4b | ||
|
|
eb97ee1647 |
@@ -246,6 +246,14 @@ class RiinDisClient(TokenHttpClient):
|
||||
data = self.post("distribution/order/goods", body={"orderId": order_id})
|
||||
return OrderGoodsResponse.model_validate(data)
|
||||
|
||||
def get_order_goods_batch(self, ids: list[str]) -> OrderGoodsResponse:
|
||||
"""批量获取订单商品(distribution/order/goods/list)。
|
||||
|
||||
上游要求:单批 ≤200 个 id 且串行调用;高频/并发会被软限流(成功但空 data)。
|
||||
"""
|
||||
data = self.post("distribution/order/goods/list", body={"ids": ids})
|
||||
return OrderGoodsResponse.model_validate(data)
|
||||
|
||||
def get_order_count(
|
||||
self, params: OrderPageRequest | None = None
|
||||
) -> OrderListCountResponse:
|
||||
|
||||
+8
-1
@@ -29,13 +29,20 @@ _OCR_FIELD_MAP = [
|
||||
]
|
||||
|
||||
|
||||
# 单号不需要 "-" 的承运商(Evri 单号两段式带 "-",下游查询用无 "-" 形式)
|
||||
_DASH_STRIP_COURIERS = {"EVRI"}
|
||||
|
||||
|
||||
def _result_to_fields(r) -> dict[str, str | None]:
|
||||
if not r.ok:
|
||||
err = (r.error or "").lower()
|
||||
if "reject" in err or "not a recognized layout" in err:
|
||||
return {"ocrLayout": "UNSUPPORTED"}
|
||||
return {"ocrLayout": "ERROR"}
|
||||
return {col: (getattr(r, attr) or "") for attr, col in _OCR_FIELD_MAP}
|
||||
fields = {col: (getattr(r, attr) or "") for attr, col in _OCR_FIELD_MAP}
|
||||
if (r.courier or "").strip().upper() in _DASH_STRIP_COURIERS:
|
||||
fields["ocrTracking"] = fields["ocrTracking"].replace("-", "")
|
||||
return fields
|
||||
|
||||
|
||||
def _has_running_peer() -> bool:
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""RIIN 同步公共管道:映射订单 → 并发拉商品 → upsert 到 PG。"""
|
||||
"""RIIN 同步公共管道:映射订单 → 批量拉商品 → upsert 到 PG。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from flows.shared import fetch_goods_concurrent
|
||||
from flows.shared import fetch_goods_for_orders
|
||||
from flows.shared.db import SyncDB
|
||||
from flows.shared.mappers import map_riin_goods, map_riin_order
|
||||
from api import RiinDisClient
|
||||
@@ -31,7 +31,7 @@ def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB)
|
||||
|
||||
goods_inputs: list[Any] = []
|
||||
if order_meta:
|
||||
goods_map = fetch_goods_concurrent(client, list(order_meta.keys()))
|
||||
goods_map = fetch_goods_for_orders(client, list(order_meta.keys()))
|
||||
for oid, (order_no, created_at) in order_meta.items():
|
||||
items = goods_map.get(oid, [])
|
||||
if not items:
|
||||
|
||||
@@ -5,7 +5,7 @@ from flows.shared.artifacts import build_stats_rows, emit_sync_summary, emit_ver
|
||||
from flows.shared.clients import (
|
||||
create_riin_client,
|
||||
create_sds_client,
|
||||
fetch_goods_concurrent,
|
||||
fetch_goods_for_orders,
|
||||
fetch_riin_all_pages,
|
||||
fetch_sds_all_pages,
|
||||
floor_to_hour,
|
||||
@@ -23,7 +23,7 @@ __all__ = [
|
||||
"clients", "db", "mappers", "SyncDB",
|
||||
"OrderUpsertInput", "GoodsUpsertInput",
|
||||
"create_riin_client", "create_sds_client",
|
||||
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent",
|
||||
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_for_orders",
|
||||
"get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source",
|
||||
"format_date", "start_of_day", "floor_to_hour",
|
||||
"build_stats_rows", "emit_sync_summary", "emit_verify_report",
|
||||
|
||||
+42
-24
@@ -21,7 +21,6 @@ from flows.shared.mappers import map_sds_channel_type
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
GOODS_CONCURRENCY = 20
|
||||
PAGE_DELAY_MS = 500
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY_MS = 3000
|
||||
@@ -144,39 +143,58 @@ def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[An
|
||||
|
||||
# ---------- RIIN 商品 ----------
|
||||
|
||||
def fetch_goods_for_order(client: RiinDisClient, order_id: str) -> list[Any]:
|
||||
try:
|
||||
resp = client.get_order_goods(order_id)
|
||||
return list(resp.data)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[RIIN sync] 订单 {order_id} 商品拉取失败: {err}")
|
||||
return []
|
||||
# 批量接口 distribution/order/goods/list:单批 ≤200 个 id,且要求串行调用(分销侧约定)。
|
||||
# 相比旧逐单接口(软限流:高频返回成功但空 data),调用量从日均 ~1 万次降到 ~50 次。
|
||||
GOODS_BATCH_SIZE = 200
|
||||
GOODS_BATCH_INTERVAL_MS = 2000 # 相邻两批的串行间隔
|
||||
|
||||
|
||||
def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[str, list[Any]]:
|
||||
"""以 GOODS_CONCURRENCY 并发拉取多订单商品,返回 {order_id: [goods...]}。"""
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
def fetch_goods_for_orders(client: RiinDisClient, order_ids: list[str]) -> dict[str, list[Any]]:
|
||||
"""按 goods/list 批量接口串行拉取多订单商品,返回 {order_id: [goods...]}。
|
||||
|
||||
results: dict[str, list[Any]] = {}
|
||||
单批失败(网络/业务错误)整批重试最多 MAX_RETRIES 次,仍失败则该批订单记空列表,
|
||||
由调用方按"无商品订单"计数(与旧逐单路径的容错语义一致)。
|
||||
"""
|
||||
results: dict[str, list[Any]] = {oid: [] for oid in order_ids}
|
||||
if not order_ids:
|
||||
return results
|
||||
concurrency = min(GOODS_CONCURRENCY, len(order_ids))
|
||||
print(f"[RIIN sync] 商品拉取开始: {len(order_ids)} 个订单 (并发={concurrency})")
|
||||
|
||||
def make(oid: str) -> Callable[[], None]:
|
||||
def run() -> None:
|
||||
results[oid] = fetch_goods_for_order(client, oid)
|
||||
return run
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
|
||||
total_batches = -(-len(order_ids) // GOODS_BATCH_SIZE)
|
||||
print(f"[RIIN sync] 商品批量拉取开始: {len(order_ids)} 个订单 / {total_batches} 批"
|
||||
f" (goods/list 串行, 批间 {GOODS_BATCH_INTERVAL_MS}ms)")
|
||||
for bi in range(total_batches):
|
||||
chunk = order_ids[bi * GOODS_BATCH_SIZE : (bi + 1) * GOODS_BATCH_SIZE]
|
||||
items = _fetch_goods_batch_with_retry(client, chunk, bi + 1)
|
||||
for g in items:
|
||||
if g.order_id in results:
|
||||
results[g.order_id].append(g)
|
||||
if bi < total_batches - 1:
|
||||
time.sleep(GOODS_BATCH_INTERVAL_MS / 1000)
|
||||
total_goods = sum(len(v) for v in results.values())
|
||||
empty_orders = sum(1 for v in results.values() if not v)
|
||||
print(f"[RIIN sync] 商品拉取完成: 共 {total_goods} 条商品, {empty_orders} 个订单无商品")
|
||||
print(f"[RIIN sync] 商品批量拉取完成: 共 {total_goods} 条商品, {empty_orders} 个订单无商品")
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_goods_batch_with_retry(client: RiinDisClient, chunk: list[str], batch_no: int) -> list[Any]:
|
||||
last_err: Exception | None = None
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
resp = fetch_with_retry(
|
||||
lambda: client.get_order_goods_batch(chunk), f"RIIN goods 批次 {batch_no}"
|
||||
)
|
||||
if resp.successful is False:
|
||||
raise RuntimeError(f"goods/list 业务失败: {resp.message}")
|
||||
return list(resp.data)
|
||||
except Exception as err: # noqa: BLE001
|
||||
last_err = err
|
||||
print(f"[RIIN sync] 商品批次 {batch_no} 第 {attempt}/{MAX_RETRIES} 次失败: {str(err)[:120]}")
|
||||
if attempt < MAX_RETRIES:
|
||||
time.sleep(RETRY_DELAY_MS * attempt / 1000)
|
||||
assert last_err is not None
|
||||
print(f"[RIIN sync] 商品批次 {batch_no} 放弃, {len(chunk)} 个订单本轮无商品")
|
||||
return []
|
||||
|
||||
|
||||
# ---------- SDS 物流渠道字典 ----------
|
||||
|
||||
_channel_cache: dict[str, Any] = {}
|
||||
|
||||
+4
-2
@@ -460,9 +460,11 @@ class SyncDB:
|
||||
stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0}
|
||||
with conn.cursor() as cur:
|
||||
# 统计已存在 → created/updated
|
||||
# 注:必须 cast 成 timestamptz——入参是带时区偏移的 tz-aware datetime,
|
||||
# cast 成 timestamp 会丢弃偏移,在会话时区非 +08 时匹配不到既有行,导致 created/updated 统计全错。
|
||||
pairs = [(o.orderNo, o.createdAt) for o in orders]
|
||||
cur.execute(
|
||||
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamp[]) AS t(order_no,created_at) "
|
||||
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamptz[]) AS t(order_no,created_at) "
|
||||
"JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at",
|
||||
_unnest_pairs(pairs),
|
||||
)
|
||||
@@ -623,6 +625,6 @@ class SyncDB:
|
||||
def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]:
|
||||
"""构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。
|
||||
|
||||
配合 SQL:unnest(%s::text[], %s::timestamp[]) 或 unnest(%s::text[], %s::text[])。
|
||||
配合 SQL:unnest(%s::text[], %s::timestamptz[]) 或 unnest(%s::text[], %s::text[])。
|
||||
"""
|
||||
return [p[0] for p in pairs], [p[1] for p in pairs]
|
||||
|
||||
@@ -21,6 +21,7 @@ deployments:
|
||||
schedules:
|
||||
- cron: 30 * * * *
|
||||
timezone: Asia/Shanghai
|
||||
active: false # 暂停:每小时重拉近7天全部订单商品,对 RIIN distribution/order/goods 调用量过大(增量/刷新未完结/日报核验已覆盖)
|
||||
work_pool:
|
||||
name: inkreach-pool
|
||||
- name: riin-refresh-pending
|
||||
@@ -39,6 +40,7 @@ deployments:
|
||||
schedules:
|
||||
- cron: 0 6 * * *
|
||||
timezone: Asia/Shanghai
|
||||
active: false # 暂停:与 daily-report 18:30 的核验子流程重复,保留日报一处即可
|
||||
work_pool:
|
||||
name: inkreach-pool
|
||||
- name: sds-order-sync
|
||||
|
||||
Reference in New Issue
Block a user