feat: 详细日志(分页/映射/批次进度)+ Artifacts 摘要(sync-summary/verify-report)
This commit is contained in:
+14
-1
@@ -108,12 +108,16 @@ def fetch_riin_all_pages(client: RiinDisClient, params: dict[str, Any], max_page
|
||||
total = first.data.total
|
||||
records = list(first.data.records)
|
||||
total_pages = min(-(-total // page_size), max_pages)
|
||||
print(f"[RIIN sync] 分页开始: 共 {total} 条 / {total_pages} 页 (page_size={page_size})")
|
||||
for page in range(2, total_pages + 1):
|
||||
resp = fetch(page)
|
||||
records.extend(resp.data.records)
|
||||
if page % 10 == 0 or page == total_pages:
|
||||
print(f"[RIIN sync] 分页进度: {page}/{total_pages} 页, 累计 {len(records)} 条")
|
||||
time.sleep(PAGE_DELAY_MS / 1000)
|
||||
if total_pages == max_pages:
|
||||
print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条")
|
||||
print(f"[RIIN sync] 分页完成: 共拉取 {len(records)} 条")
|
||||
return records
|
||||
|
||||
|
||||
@@ -127,10 +131,14 @@ def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[An
|
||||
first = fetch(1)
|
||||
records = list(first.records)
|
||||
total_pages = -(-first.total_count // page_size)
|
||||
print(f"[SDS sync] 分页开始: 共 {first.total_count} 条 / {total_pages} 页 (page_size={page_size})")
|
||||
for page in range(2, total_pages + 1):
|
||||
resp = fetch(page)
|
||||
records.extend(resp.records)
|
||||
if page % 10 == 0 or page == total_pages:
|
||||
print(f"[SDS sync] 分页进度: {page}/{total_pages} 页, 累计 {len(records)} 条")
|
||||
time.sleep(PAGE_DELAY_MS / 1000)
|
||||
print(f"[SDS sync] 分页完成: 共拉取 {len(records)} 条")
|
||||
return records
|
||||
|
||||
|
||||
@@ -153,14 +161,19 @@ def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[
|
||||
results: dict[str, list[Any]] = {}
|
||||
if not order_ids:
|
||||
return results
|
||||
concurrency = min(GOODS_CONCURRENCY, len(order_ids))
|
||||
print(f"[RIIN sync] 商品拉取开始: {len(order_ids)} 个订单 (并发={concurrency})")
|
||||
|
||||
def make(oid: str) -> Callable[[], None]:
|
||||
def run() -> None:
|
||||
results[oid] = fetch_goods_for_order(client, oid)
|
||||
return run
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(GOODS_CONCURRENCY, len(order_ids))) as pool:
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
|
||||
total_goods = sum(len(v) for v in results.values())
|
||||
empty_orders = sum(1 for v in results.values() if not v)
|
||||
print(f"[RIIN sync] 商品拉取完成: 共 {total_goods} 条商品, {empty_orders} 个订单无商品")
|
||||
return results
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user