feat: 详细日志(分页/映射/批次进度)+ Artifacts 摘要(sync-summary/verify-report)

This commit is contained in:
yeuimu
2026-07-23 09:01:39 +00:00
parent 367168bcdc
commit 4ec7cf2dd6
18 changed files with 154 additions and 22 deletions
+6
View File
@@ -14,17 +14,21 @@ def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB)
order_inputs: list[Any] = [] order_inputs: list[Any] = []
order_meta: dict[str, tuple[str, Any]] = {} # raw.id -> (orderNo, createdAt) order_meta: dict[str, tuple[str, Any]] = {} # raw.id -> (orderNo, createdAt)
goods_failed = 0 goods_failed = 0
order_map_failed = 0
for raw in raw_orders: for raw in raw_orders:
try: try:
o = map_riin_order(raw) o = map_riin_order(raw)
except Exception as err: # noqa: BLE001 except Exception as err: # noqa: BLE001
order_map_failed += 1
print(f"[RIIN] 映射订单失败 {getattr(raw, 'platform_oid', '?')}: {err}") print(f"[RIIN] 映射订单失败 {getattr(raw, 'platform_oid', '?')}: {err}")
continue continue
order_inputs.append(o) order_inputs.append(o)
if raw.id: if raw.id:
order_meta[raw.id] = (o.orderNo, o.createdAt) order_meta[raw.id] = (o.orderNo, o.createdAt)
print(f"[RIIN] 订单映射完成: 成功 {len(order_inputs)} / 失败 {order_map_failed} (共 {len(raw_orders)})")
goods_inputs: list[Any] = [] goods_inputs: list[Any] = []
if order_meta: if order_meta:
goods_map = fetch_goods_concurrent(client, list(order_meta.keys())) goods_map = fetch_goods_concurrent(client, list(order_meta.keys()))
@@ -38,6 +42,8 @@ def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB)
except Exception as err: # noqa: BLE001 except Exception as err: # noqa: BLE001
print(f"[RIIN] 映射商品失败 order={order_no}: {err}") print(f"[RIIN] 映射商品失败 order={order_no}: {err}")
print(f"[RIIN] 商品映射完成: {len(goods_inputs)} 条, 商品拉取失败订单 {goods_failed} 个")
stats = db.sync_orders_and_goods(order_inputs, goods_inputs) stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
stats["orderCount"] = len(order_inputs) stats["orderCount"] = len(order_inputs)
stats["goodsCount"] = len(goods_inputs) stats["goodsCount"] = len(goods_inputs)
+2 -1
View File
@@ -11,7 +11,7 @@ from prefect import flow, task
from api.riin.models import OrderPageRequest from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client from flows.shared import SyncDB, build_stats_rows, create_riin_client, emit_sync_summary
def _split_order_nos(text: str) -> list[str]: def _split_order_nos(text: str) -> list[str]:
@@ -52,6 +52,7 @@ def riin_by_orders_sync(order_nos: str) -> dict:
raw = fetch_by_orders(nos) raw = fetch_by_orders(nos)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}") print(f"完成: 匹配 {len(raw)} | {stats}")
emit_sync_summary("RIIN 指定单号同步", build_stats_rows("RIIN", stats, [("输入单号", len(nos)), ("匹配", len(raw))]))
return {"inputCount": len(nos), "matchedCount": len(raw), **stats} return {"inputCount": len(nos), "matchedCount": len(raw), **stats}
+3 -1
View File
@@ -11,7 +11,8 @@ from prefect import flow, task
from flows.riin._pipeline import sync_riin_records from flows.riin._pipeline import sync_riin_records
from flows.shared import ( from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, floor_to_hour, format_date, SyncDB, build_stats_rows, create_riin_client, emit_sync_summary, fetch_riin_all_pages,
floor_to_hour, format_date,
) )
@@ -55,6 +56,7 @@ def riin_incremental_sync() -> dict:
raw = extract_orders(params) raw = extract_orders(params)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}") print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
emit_sync_summary("RIIN 增量同步", build_stats_rows("RIIN", stats, [("时间范围", f"{params['start_time']} ~ {params['end_time']}")]))
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, **stats} return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, **stats}
+4 -1
View File
@@ -11,7 +11,9 @@ from prefect import flow, task
from api.riin.models import OrderPageRequest from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client, fetch_riin_all_pages, format_date from flows.shared import (
SyncDB, build_stats_rows, create_riin_client, emit_sync_summary, fetch_riin_all_pages, format_date,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9]) @task(retries=3, retry_delay_seconds=[3, 6, 9])
@@ -70,6 +72,7 @@ def riin_order_sync(
raw = extract_orders(params) raw = extract_orders(params)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} | {stats}") print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} | {stats}")
emit_sync_summary("RIIN 订单同步", build_stats_rows("RIIN", stats, [("时间范围", f"{order_date_begin} ~ {order_date_end}")]))
return {"syncRange": {"from": order_date_begin, "to": order_date_end}, **stats} return {"syncRange": {"from": order_date_begin, "to": order_date_end}, **stats}
+3 -1
View File
@@ -11,7 +11,8 @@ from prefect import flow, task
from flows.riin._pipeline import sync_riin_records from flows.riin._pipeline import sync_riin_records
from flows.shared import ( from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day, SyncDB, build_stats_rows, create_riin_client, emit_sync_summary,
fetch_riin_all_pages, format_date, start_of_day,
) )
@@ -49,6 +50,7 @@ def riin_recent_sync(recent_days: int = 7) -> dict:
raw = extract_orders(params) raw = extract_orders(params)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}") print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
emit_sync_summary("RIIN 近期同步", build_stats_rows("RIIN", stats, [("近N天", days), ("时间范围", f"{params['start_time']} ~ {params['end_time']}")]))
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, "recentDays": days, **stats} return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, "recentDays": days, **stats}
+2 -1
View File
@@ -9,7 +9,7 @@ from prefect import flow, task
from api.riin.models import OrderPageRequest from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client from flows.shared import SyncDB, build_stats_rows, create_riin_client, emit_sync_summary
RIIN_FINISHED_STATUSES = ["已退款", "已关闭", "已发货"] RIIN_FINISHED_STATUSES = ["已退款", "已关闭", "已发货"]
ORDER_BATCH = 500 ORDER_BATCH = 500
@@ -51,6 +51,7 @@ def riin_refresh_pending() -> dict:
raw = fetch_pending(order_nos) raw = fetch_pending(order_nos)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}") print(f"完成: 匹配 {len(raw)} | {stats}")
emit_sync_summary("RIIN 刷新未完结", build_stats_rows("RIIN", stats, [("未完结订单", len(order_nos)), ("匹配", len(raw))]))
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats} return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
+5 -3
View File
@@ -12,7 +12,7 @@ from prefect import flow, task
from api.riin.models import OrderPageRequest from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records from flows.riin._pipeline import sync_riin_records
from flows.shared import ( from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day, SyncDB, create_riin_client, emit_verify_report, fetch_riin_all_pages, format_date, start_of_day,
) )
RIIN_COUNT_STATUS_MAP = { RIIN_COUNT_STATUS_MAP = {
@@ -96,7 +96,7 @@ def riin_verify(mode: str = "recent", recent_days: int = 1, date_range: str | No
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}" print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+ (f" 差异 {diff}" if has_diff else " 一致,跳过")) + (f" 差异 {diff}" if has_diff else " 一致,跳过"))
if not has_diff: if not has_diff:
day_reports.append({"date": day_str, "diff": diff, "ok": True}) day_reports.append({"date": day_str, "remote": remote_total, "local": sum(local_counts.values()), "diff": diff, "ok": True})
continue continue
days_with_diff += 1 days_with_diff += 1
@@ -111,11 +111,13 @@ def riin_verify(mode: str = "recent", recent_days: int = 1, date_range: str | No
totals["ordersUpdated"] += stats["ordersUpdated"] totals["ordersUpdated"] += stats["ordersUpdated"]
totals["goodsAdded"] += stats["goodsCreated"] totals["goodsAdded"] += stats["goodsCreated"]
totals["goodsUpdated"] += stats["goodsUpdated"] totals["goodsUpdated"] += stats["goodsUpdated"]
day_reports.append({"date": day_str, "diff": diff, "ok": False, day_reports.append({"date": day_str, "remote": remote_total, "local": sum(local_counts.values()), "diff": diff, "ok": False,
"orders": stats["orderCount"], "goods": stats["goodsCount"]}) "orders": stats["orderCount"], "goods": stats["goodsCount"]})
finally: finally:
client.close() client.close()
print(f"[核验汇总] 共 {len(days)} 天 / 有差异 {days_with_diff} 天 / {totals}")
emit_verify_report("RIIN 数据核验", day_reports, totals, len(days), days_with_diff)
return { return {
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}}, "verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""}, "syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
+1
View File
@@ -51,6 +51,7 @@ def sync_sds_records(
order_inputs, goods_inputs, skipped = map_sds_records( order_inputs, goods_inputs, skipped = map_sds_records(
raw_orders, channel_map, channel_refresh, skip_unpaid raw_orders, channel_map, channel_refresh, skip_unpaid
) )
print(f"[SDS] 映射完成: 订单 {len(order_inputs)} / 商品 {len(goods_inputs)} / 跳过待付款 {skipped} (共 {len(raw_orders)})")
stats = db.sync_orders_and_goods(order_inputs, goods_inputs) stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
stats["orderCount"] = len(order_inputs) stats["orderCount"] = len(order_inputs)
stats["goodsCount"] = len(goods_inputs) stats["goodsCount"] = len(goods_inputs)
+3 -2
View File
@@ -11,8 +11,8 @@ from prefect import flow, task
from flows.sds._pipeline import sync_sds_records from flows.sds._pipeline import sync_sds_records
from flows.shared import ( from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, floor_to_hour, format_date, SyncDB, build_stats_rows, create_sds_client, emit_sync_summary, fetch_sds_all_pages,
get_channel_type_map, refresh_channel_type_map, floor_to_hour, format_date, get_channel_type_map, refresh_channel_type_map,
) )
@@ -52,6 +52,7 @@ def sds_incremental_sync() -> dict:
raw = extract_orders(params) raw = extract_orders(params)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}") print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
emit_sync_summary("SDS 增量同步", build_stats_rows("SDS", stats, [("时间范围", f"{params['pay_date_begin']} ~ {params['pay_date_end']}")]))
return {"syncRange": {"from": params["pay_date_begin"], "to": params["pay_date_end"]}, **stats} return {"syncRange": {"from": params["pay_date_begin"], "to": params["pay_date_end"]}, **stats}
+3 -1
View File
@@ -11,7 +11,8 @@ from prefect import flow, task
from flows.sds._pipeline import sync_sds_records from flows.sds._pipeline import sync_sds_records
from flows.shared import ( from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, get_channel_type_map, refresh_channel_type_map, SyncDB, build_stats_rows, create_sds_client, emit_sync_summary,
fetch_sds_all_pages, get_channel_type_map, refresh_channel_type_map,
) )
@@ -69,6 +70,7 @@ def sds_order_sync(
raw = extract_orders(params) raw = extract_orders(params)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} / 跳过待付款 {stats.get('skippedUnpaid',0)}") print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} / 跳过待付款 {stats.get('skippedUnpaid',0)}")
emit_sync_summary("SDS 订单同步", build_stats_rows("SDS", stats, [("时间范围", f"{pay_date_begin} ~ {pay_date_end}")]))
return {"syncRange": {"from": pay_date_begin, "to": pay_date_end}, **stats} return {"syncRange": {"from": pay_date_begin, "to": pay_date_end}, **stats}
+3 -2
View File
@@ -11,8 +11,8 @@ from prefect import flow, task
from flows.sds._pipeline import sync_sds_records from flows.sds._pipeline import sync_sds_records
from flows.shared import ( from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map, SyncDB, build_stats_rows, create_sds_client, emit_sync_summary, fetch_sds_all_pages,
refresh_channel_type_map, start_of_day, format_date, get_channel_type_map, refresh_channel_type_map, start_of_day,
) )
@@ -48,6 +48,7 @@ def sds_recent_sync(recent_days: int = 7) -> dict:
raw = extract_orders(params) raw = extract_orders(params)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}") print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
emit_sync_summary("SDS 近期同步", build_stats_rows("SDS", stats, [("近N天", days), ("时间范围", f"{params['created_date_begin']} ~ {params['created_date_end']}")]))
return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)}, return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)},
"recentDays": days, **stats} "recentDays": days, **stats}
+3 -2
View File
@@ -9,8 +9,8 @@ from prefect import flow, task
from api.sds.models import OrderPageRequest from api.sds.models import OrderPageRequest
from flows.shared import ( from flows.shared import (
SyncDB, create_sds_client, get_channel_type_map, SyncDB, build_stats_rows, create_sds_client, emit_sync_summary,
refresh_channel_type_map, resolve_courier_source, get_channel_type_map, refresh_channel_type_map, resolve_courier_source,
) )
from flows.shared.mappers import map_sds_goods, map_sds_order from flows.shared.mappers import map_sds_goods, map_sds_order
@@ -99,6 +99,7 @@ def sds_refresh_100_prefix() -> dict:
remote = fetch_remote(order_nos) remote = fetch_remote(order_nos)
stats = upsert_records(remote, order_nos) stats = upsert_records(remote, order_nos)
print(f"完成: 刷新 {stats['orderCount']} / 仍待分配 {stats.get('skippedStillPending',0)} / 待付款 {stats.get('skippedUnpaid',0)}") print(f"完成: 刷新 {stats['orderCount']} / 仍待分配 {stats.get('skippedStillPending',0)} / 待付款 {stats.get('skippedUnpaid',0)}")
emit_sync_summary("SDS 100前缀刷新", build_stats_rows("SDS", stats, [("目标订单", len(order_nos))]))
return {"targetCount": len(order_nos), **stats} return {"targetCount": len(order_nos), **stats}
+3 -1
View File
@@ -10,7 +10,8 @@ from prefect import flow, task
from api.sds.models import OrderPageRequest from api.sds.models import OrderPageRequest
from flows.sds._pipeline import sync_sds_records from flows.sds._pipeline import sync_sds_records
from flows.shared import ( from flows.shared import (
SyncDB, create_sds_client, get_channel_type_map, refresh_channel_type_map, SyncDB, build_stats_rows, create_sds_client, emit_sync_summary,
get_channel_type_map, refresh_channel_type_map,
) )
SDS_FINISHED_STATUSES = ["已完成", "已取消"] SDS_FINISHED_STATUSES = ["已完成", "已取消"]
@@ -54,6 +55,7 @@ def sds_refresh_pending() -> dict:
raw = fetch_pending(order_nos) raw = fetch_pending(order_nos)
stats = upsert_records(raw) stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}") print(f"完成: 匹配 {len(raw)} | {stats}")
emit_sync_summary("SDS 刷新未完结", build_stats_rows("SDS", stats, [("未完结订单", len(order_nos)), ("匹配", len(raw))]))
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats} return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
+6 -4
View File
@@ -12,8 +12,8 @@ from prefect import flow, task
from api.sds.models import OrderQueryParams from api.sds.models import OrderQueryParams
from flows.sds._pipeline import sync_sds_records from flows.sds._pipeline import sync_sds_records
from flows.shared import ( from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map, SyncDB, create_sds_client, emit_verify_report, fetch_sds_all_pages, format_date,
refresh_channel_type_map, start_of_day, get_channel_type_map, refresh_channel_type_map, start_of_day,
) )
@@ -75,7 +75,7 @@ def sds_verify(mode: str = "recent", recent_days: int = 1, date_range: str | Non
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}" print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+ (f" 差异 {diff}" if has_diff else " 一致,跳过")) + (f" 差异 {diff}" if has_diff else " 一致,跳过"))
if not has_diff: if not has_diff:
day_reports.append({"date": day_str, "diff": diff, "ok": True}) day_reports.append({"date": day_str, "remote": remote_total, "local": sum(local_counts.values()), "diff": diff, "ok": True})
continue continue
days_with_diff += 1 days_with_diff += 1
@@ -89,11 +89,13 @@ def sds_verify(mode: str = "recent", recent_days: int = 1, date_range: str | Non
totals["ordersUpdated"] += stats["ordersUpdated"] totals["ordersUpdated"] += stats["ordersUpdated"]
totals["goodsAdded"] += stats["goodsCreated"] totals["goodsAdded"] += stats["goodsCreated"]
totals["goodsUpdated"] += stats["goodsUpdated"] totals["goodsUpdated"] += stats["goodsUpdated"]
day_reports.append({"date": day_str, "diff": diff, "ok": False, day_reports.append({"date": day_str, "remote": remote_total, "local": sum(local_counts.values()), "diff": diff, "ok": False,
"orders": stats["orderCount"], "goods": stats["goodsCount"]}) "orders": stats["orderCount"], "goods": stats["goodsCount"]})
finally: finally:
client.close() client.close()
print(f"[核验汇总] 共 {len(days)} 天 / 有差异 {days_with_diff} 天 / {totals}")
emit_verify_report("SDS 数据核验", day_reports, totals, len(days), days_with_diff)
return { return {
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}}, "verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""}, "syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
+3 -1
View File
@@ -1,6 +1,7 @@
"""同步编排共享层:客户端/分页、映射、PG 写入。""" """同步编排共享层:客户端/分页、映射、PG 写入、Artifacts。"""
from flows.shared import clients, db, mappers from flows.shared import clients, db, mappers
from flows.shared.artifacts import build_stats_rows, emit_sync_summary, emit_verify_report
from flows.shared.clients import ( from flows.shared.clients import (
create_riin_client, create_riin_client,
create_sds_client, create_sds_client,
@@ -24,4 +25,5 @@ __all__ = [
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent", "fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent",
"get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source", "get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source",
"format_date", "start_of_day", "floor_to_hour", "format_date", "start_of_day", "floor_to_hour",
"build_stats_rows", "emit_sync_summary", "emit_verify_report",
] ]
+82
View File
@@ -0,0 +1,82 @@
"""Prefect Artifacts:同步结果摘要(markdown),展示在 flow run 的 Artifacts 标签页。
所有函数均容错——artifact 生成失败只打印警告,绝不中断 flow run。
"""
from __future__ import annotations
from typing import Any
from prefect.artifacts import create_markdown_artifact
_STATS_LABELS: list[tuple[str, str]] = [
("orderCount", "拉取订单"),
("goodsCount", "拉取商品"),
("ordersCreated", "订单新增"),
("ordersUpdated", "订单更新"),
("ordersDeleted", "订单删除"),
("goodsCreated", "商品新增"),
("goodsUpdated", "商品更新"),
("goodsRemoved", "商品移除"),
("goodsFetchFailed", "商品拉取失败"),
("skippedUnpaid", "跳过待付款"),
("skippedStillPending", "仍待分配"),
]
def build_stats_rows(
system: str, stats: dict[str, Any], extra: list[tuple[str, Any]] | None = None
) -> list[tuple[str, Any]]:
"""从 stats dict 构造摘要行(仅展示实际存在的键)。"""
rows: list[tuple[str, Any]] = [("系统", system)]
if extra:
rows.extend(extra)
for key, label in _STATS_LABELS:
if key in stats:
rows.append((label, stats[key]))
return rows
def emit_sync_summary(title: str, rows: list[tuple[str, Any]]) -> None:
"""生成同步结果摘要 markdown artifact(表格)。
Args:
title: 摘要标题(如 "RIIN 订单同步")。
rows: [(标签, 值), ...],每项渲染为表格一行。
"""
lines = [f"# {title}", "", "| 指标 | 值 |", "| --- | --- |"]
for label, value in rows:
lines.append(f"| {label} | {value} |")
try:
create_markdown_artifact(key="sync-summary", markdown="\n".join(lines), description=title)
except Exception as err: # noqa: BLE001
print(f"[artifact] 摘要生成失败: {err}")
def emit_verify_report(
title: str,
day_reports: list[dict],
totals: dict[str, int],
total_days: int,
days_with_diff: int,
) -> None:
"""生成核验报告 markdown artifact(逐天差异表 + 汇总)。"""
lines = [f"# {title}", "", "## 逐天核验", "", "| 日期 | 远程 | 本地 | 差异 | 结果 |", "| --- | --- | --- | --- | --- |"]
for r in day_reports:
diff = r.get("diff", {})
diff_str = ", ".join(f"{k}:{v:+d}" for k, v in diff.items()) if diff else "—"
ok = r.get("ok")
if ok:
result = "✅ 一致"
else:
result = f"🔧 已修复(订单 {r.get('orders', 0)} / 商品 {r.get('goods', 0)})"
lines.append(f"| {r['date']} | {r.get('remote', '—')} | {r.get('local', '—')} | {diff_str} | {result} |")
lines += ["", "## 汇总", "", "| 指标 | 值 |", "| --- | --- |"]
lines.append(f"| 核验天数 | {total_days} |")
lines.append(f"| 有差异天数 | {days_with_diff} |")
for k, v in totals.items():
lines.append(f"| {k} | {v} |")
try:
create_markdown_artifact(key="verify-report", markdown="\n".join(lines), description=title)
except Exception as err: # noqa: BLE001
print(f"[artifact] 核验报告生成失败: {err}")
+14 -1
View File
@@ -108,12 +108,16 @@ def fetch_riin_all_pages(client: RiinDisClient, params: dict[str, Any], max_page
total = first.data.total total = first.data.total
records = list(first.data.records) records = list(first.data.records)
total_pages = min(-(-total // page_size), max_pages) total_pages = min(-(-total // page_size), max_pages)
print(f"[RIIN sync] 分页开始: 共 {total} 条 / {total_pages} 页 (page_size={page_size})")
for page in range(2, total_pages + 1): for page in range(2, total_pages + 1):
resp = fetch(page) resp = fetch(page)
records.extend(resp.data.records) records.extend(resp.data.records)
if page % 10 == 0 or page == total_pages:
print(f"[RIIN sync] 分页进度: {page}/{total_pages} 页, 累计 {len(records)} 条")
time.sleep(PAGE_DELAY_MS / 1000) time.sleep(PAGE_DELAY_MS / 1000)
if total_pages == max_pages: if total_pages == max_pages:
print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条") print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条")
print(f"[RIIN sync] 分页完成: 共拉取 {len(records)} 条")
return records return records
@@ -127,10 +131,14 @@ def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[An
first = fetch(1) first = fetch(1)
records = list(first.records) records = list(first.records)
total_pages = -(-first.total_count // page_size) total_pages = -(-first.total_count // page_size)
print(f"[SDS sync] 分页开始: 共 {first.total_count} 条 / {total_pages} 页 (page_size={page_size})")
for page in range(2, total_pages + 1): for page in range(2, total_pages + 1):
resp = fetch(page) resp = fetch(page)
records.extend(resp.records) records.extend(resp.records)
if page % 10 == 0 or page == total_pages:
print(f"[SDS sync] 分页进度: {page}/{total_pages} 页, 累计 {len(records)} 条")
time.sleep(PAGE_DELAY_MS / 1000) time.sleep(PAGE_DELAY_MS / 1000)
print(f"[SDS sync] 分页完成: 共拉取 {len(records)} 条")
return records return records
@@ -153,14 +161,19 @@ def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[
results: dict[str, list[Any]] = {} results: dict[str, list[Any]] = {}
if not order_ids: if not order_ids:
return results return results
concurrency = min(GOODS_CONCURRENCY, len(order_ids))
print(f"[RIIN sync] 商品拉取开始: {len(order_ids)} 个订单 (并发={concurrency})")
def make(oid: str) -> Callable[[], None]: def make(oid: str) -> Callable[[], None]:
def run() -> None: def run() -> None:
results[oid] = fetch_goods_for_order(client, oid) results[oid] = fetch_goods_for_order(client, oid)
return run return run
with ThreadPoolExecutor(max_workers=min(GOODS_CONCURRENCY, len(order_ids))) as pool: with ThreadPoolExecutor(max_workers=concurrency) as pool:
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids])) list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
total_goods = sum(len(v) for v in results.values())
empty_orders = sum(1 for v in results.values() if not v)
print(f"[RIIN sync] 商品拉取完成: 共 {total_goods} 条商品, {empty_orders} 个订单无商品")
return results return results
+8
View File
@@ -111,6 +111,7 @@ class SyncDB:
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0, "goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
} }
if not orders and not goods and not delete_order_nos: if not orders and not goods and not delete_order_nos:
print("[DB sync] 无数据可写入,跳过")
return result return result
conn = self._conn() conn = self._conn()
@@ -163,12 +164,18 @@ class SyncDB:
for g in goods_dedup: for g in goods_dedup:
goods_by_order.setdefault(g.orderNo, []).append(g) goods_by_order.setdefault(g.orderNo, []).append(g)
print(f"[DB sync] 准备写入: {len(orders_dedup)} 订单 / {len(goods_dedup)} 商品"
+ (f" (系统={system_name})" if system_name else ""))
# 3. 分批 upsert orders + 关联 goods # 3. 分批 upsert orders + 关联 goods
total_order_batches = -(-len(orders_dedup) // ORDER_CHUNK)
for i in range(0, len(orders_dedup), ORDER_CHUNK): for i in range(0, len(orders_dedup), ORDER_CHUNK):
chunk = orders_dedup[i : i + ORDER_CHUNK] chunk = orders_dedup[i : i + ORDER_CHUNK]
related: list[GoodsUpsertInput] = [] related: list[GoodsUpsertInput] = []
for o in chunk: for o in chunk:
related.extend(goods_by_order.get(o.orderNo, [])) related.extend(goods_by_order.get(o.orderNo, []))
batch_no = i // ORDER_CHUNK + 1
print(f"[DB sync] 订单批次 {batch_no}/{total_order_batches}: {len(chunk)} 订单 / {len(related)} 商品")
stats = self._upsert_order_batch(conn, chunk, related, system_name) stats = self._upsert_order_batch(conn, chunk, related, system_name)
for k, v in stats.items(): for k, v in stats.items():
result[k] += v result[k] += v
@@ -199,6 +206,7 @@ class SyncDB:
raise raise
finally: finally:
conn.close() conn.close()
print(f"[DB sync] 写入完成: {result}")
return result return result
def _upsert_order_batch( def _upsert_order_batch(