init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows

This commit is contained in:
yeuimu
2026-07-23 08:13:09 +00:00
commit 4cba1da4e1
41 changed files with 6219 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""RIIN 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / by_orders / verify。"""
+45
View File
@@ -0,0 +1,45 @@
"""RIIN 同步公共管道:映射订单 → 并发拉商品 → upsert 到 PG。"""
from __future__ import annotations
from typing import Any
from flows.shared import fetch_goods_concurrent
from flows.shared.db import SyncDB
from flows.shared.mappers import map_riin_goods, map_riin_order
from api import RiinDisClient
def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB) -> dict[str, int]:
order_inputs: list[Any] = []
order_meta: dict[str, tuple[str, Any]] = {} # raw.id -> (orderNo, createdAt)
goods_failed = 0
for raw in raw_orders:
try:
o = map_riin_order(raw)
except Exception as err: # noqa: BLE001
print(f"[RIIN] 映射订单失败 {getattr(raw, 'platform_oid', '?')}: {err}")
continue
order_inputs.append(o)
if raw.id:
order_meta[raw.id] = (o.orderNo, o.createdAt)
goods_inputs: list[Any] = []
if order_meta:
goods_map = fetch_goods_concurrent(client, list(order_meta.keys()))
for oid, (order_no, created_at) in order_meta.items():
items = goods_map.get(oid, [])
if not items:
goods_failed += 1
for g in items:
try:
goods_inputs.append(map_riin_goods(g, order_no, created_at))
except Exception as err: # noqa: BLE001
print(f"[RIIN] 映射商品失败 order={order_no}: {err}")
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
stats["orderCount"] = len(order_inputs)
stats["goodsCount"] = len(goods_inputs)
stats["goodsFetchFailed"] = goods_failed
return stats
+59
View File
@@ -0,0 +1,59 @@
"""RIIN 指定单号同步:通过订单号列表(逗号/换行分隔)拉取指定订单。
对应 riin-by-orders。
"""
from __future__ import annotations
import re
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client
def _split_order_nos(text: str) -> list[str]:
seen: dict[str, None] = {}
for part in re.split(r"[\n\r,,\s]+", text or ""):
part = part.strip()
if part and part not in seen:
seen[part] = None
return list(seen.keys())
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_by_orders(order_nos: list[str]) -> list:
client = create_riin_client()
try:
resp = client.get_orders(OrderPageRequest(platform_oid_list=",".join(order_nos), all=True))
return list(resp.data.records)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-by-orders-sync", log_prints=True)
def riin_by_orders_sync(order_nos: str) -> dict:
nos = _split_order_nos(order_nos)
print(f"RIIN 指定单号同步 输入 {len(nos)} 个")
if not nos:
return {"inputCount": 0, "matchedCount": 0}
raw = fetch_by_orders(nos)
stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}")
return {"inputCount": len(nos), "matchedCount": len(raw), **stats}
if __name__ == "__main__":
riin_by_orders_sync.serve(name="riin-by-orders-sync")
+60
View File
@@ -0,0 +1,60 @@
"""RIIN 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
对应 riin-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now。
"""
from __future__ import annotations
from datetime import datetime
from prefect import flow, task
from flows.riin._pipeline import sync_riin_records
from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, floor_to_hour, format_date,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_riin_client()
try:
return fetch_riin_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-incremental-sync", log_prints=True)
def riin_incremental_sync() -> dict:
db = SyncDB()
latest = db.get_latest_ordered_at("RIIN")
now = datetime.now()
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
to_dt = now
if from_dt >= to_dt:
from_dt = floor_to_hour(now)
params = {
"time_type": 3,
"start_time": format_date(from_dt),
"end_time": format_date(to_dt),
"page_size": 200,
}
print(f"RIIN 增量同步 {params['start_time']} ~ {params['end_time']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, **stats}
if __name__ == "__main__":
riin_incremental_sync.serve(name="riin-incremental-sync", cron="0 * * * *")
+77
View File
@@ -0,0 +1,77 @@
"""RIIN 订单同步:按时间窗口(+可选筛选)拉取订单与商品。
对应 riin-order-sync。timeType 固定 3(商家推送时间)。
"""
from __future__ import annotations
from typing import Any
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client, fetch_riin_all_pages, format_date
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict[str, Any]) -> list[Any]:
client = create_riin_client()
try:
return fetch_riin_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list[Any]) -> dict[str, int]:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-order-sync", log_prints=True)
def riin_order_sync(
order_date_begin: str,
order_date_end: str,
platform_type: int | None = None,
order_status: int | None = None,
factory_ids: list[str] | None = None,
express_source: int | None = None,
urgent_order: int | None = None,
has_refund: bool | None = None,
page_size: int = 200,
) -> dict[str, Any]:
if not order_date_begin or not order_date_end:
raise ValueError("order_date_begin / order_date_end 为必填")
params: dict[str, Any] = {
"time_type": 3,
"start_time": order_date_begin,
"end_time": order_date_end,
"page_size": page_size,
}
if platform_type is not None:
params["platform_type"] = platform_type
if order_status is not None:
params["order_status"] = order_status
if factory_ids:
params["factory_ids"] = factory_ids
if express_source is not None:
params["express_source"] = express_source
if urgent_order is not None:
params["urgent_order"] = urgent_order
if has_refund is not None:
params["has_refund"] = has_refund
print(f"RIIN 订单同步 {order_date_begin} ~ {order_date_end}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} | {stats}")
return {"syncRange": {"from": order_date_begin, "to": order_date_end}, **stats}
if __name__ == "__main__":
riin_order_sync.serve(name="riin-order-sync")
+56
View File
@@ -0,0 +1,56 @@
"""RIIN 近期同步:拉取最近 N 天订单(默认 7 天,1~90)。
对应 riin-recent-sync。from = startOfDay(now - N days),to = now。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from flows.riin._pipeline import sync_riin_records
from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_riin_client()
try:
return fetch_riin_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-recent-sync", log_prints=True)
def riin_recent_sync(recent_days: int = 7) -> dict:
days = min(max(recent_days, 1), 90)
now = datetime.now()
from_dt = start_of_day(now - timedelta(days=days))
params = {
"time_type": 3,
"start_time": format_date(from_dt),
"end_time": format_date(now),
"page_size": 200,
}
print(f"RIIN 近期同步 近 {days} 天 {params['start_time']} ~ {params['end_time']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, "recentDays": days, **stats}
if __name__ == "__main__":
riin_recent_sync.serve(name="riin-recent-sync", cron="30 * * * *")
+58
View File
@@ -0,0 +1,58 @@
"""RIIN 刷新未完结:查 DB 中未完结订单号,按 platformOidList 批量(500)刷新。
对应 riin-refresh-pending。终态:已退款 / 已关闭 / 已发货。
"""
from __future__ import annotations
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client
RIIN_FINISHED_STATUSES = ["已退款", "已关闭", "已发货"]
ORDER_BATCH = 500
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_pending(order_nos: list[str]) -> list:
client = create_riin_client()
try:
all_raw: list = []
for i in range(0, len(order_nos), ORDER_BATCH):
batch = order_nos[i : i + ORDER_BATCH]
resp = client.get_orders(
OrderPageRequest(platform_oid_list=",".join(batch), all=True)
)
all_raw.extend(resp.data.records)
return all_raw
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-refresh-pending", log_prints=True)
def riin_refresh_pending() -> dict:
db = SyncDB()
order_nos = db.get_pending_orders("RIIN", RIIN_FINISHED_STATUSES)
print(f"RIIN 未完结订单 {len(order_nos)} 个")
if not order_nos:
return {"pendingCount": 0, "matchedCount": 0}
raw = fetch_pending(order_nos)
stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}")
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
if __name__ == "__main__":
riin_refresh_pending.serve(name="riin-refresh-pending", cron="0 */3 * * *")
+126
View File
@@ -0,0 +1,126 @@
"""RIIN 数据核验:逐天比对远程/本地按状态计数,有差异则拉取修复。
对应 riin-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
)
RIIN_COUNT_STATUS_MAP = {
"assignedSuceessed": "已分配",
"factoryAuditing": "工厂审核中",
"inProducing": "生产中",
"shipped": "已发货",
"closed": "已关闭",
"refunding": "退款中",
"refunded": "已退款",
"rejecting": "拒审中",
"rejectedToShop": "已退回店铺",
}
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
today = start_of_day(datetime.now())
if mode == "range" and date_range:
parts = [p.strip() for p in date_range.split("~")]
from_dt = datetime.fromisoformat(parts[0])
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
days = []
cur = from_dt
while cur <= to_dt:
days.append(cur)
cur += timedelta(days=1)
return days
days_n = min(max(recent_days, 1), 90)
return [today - timedelta(days=i) for i in range(days_n, 0, -1)]
def _count_to_status_map(data: dict) -> dict[str, int]:
out: dict[str, int] = {}
for key, count in data.items():
if key.startswith("dayRange"):
continue
label = RIIN_COUNT_STATUS_MAP.get(key)
if label and count and count > 0:
out[label] = count
return out
def _diff(remote: dict[str, int], local: dict[str, int]) -> tuple[dict[str, int], bool]:
diff: dict[str, int] = {}
has_diff = False
for status in set(remote) | set(local):
r = remote.get(status, 0)
l = local.get(status, 0)
if r != l:
diff[status] = l - r
has_diff = True
return diff, has_diff
@flow(name="riin-verify", log_prints=True)
def riin_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
days = _parse_days(mode, recent_days, date_range)
db = SyncDB()
client = create_riin_client()
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "ordersRemoved",
"goodsAdded", "goodsUpdated", "goodsRemoved")}
days_with_diff = 0
day_reports = []
try:
for day in days:
day_str = format_date(day).split(" ")[0]
day_start = start_of_day(day)
day_end = day_start + timedelta(days=1)
count_resp = client.get_order_count(
OrderPageRequest(time_type=3, start_time=format_date(day_start), end_time=format_date(day_end))
)
remote_map = _count_to_status_map(count_resp.data.model_dump(by_alias=False))
remote_total = sum(remote_map.values())
local_counts = db.get_order_count_by_status("RIIN", day_start, day_end)
local_goods = db.get_local_goods_count("RIIN", day_start, day_end)
diff, has_diff = _diff(remote_map, local_counts)
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
if not has_diff:
day_reports.append({"date": day_str, "diff": diff, "ok": True})
continue
days_with_diff += 1
raw_orders = fetch_riin_all_pages(
client,
{"time_type": 3, "start_time": format_date(day_start), "end_time": format_date(day_end), "page_size": 200},
)
stats = sync_riin_records(raw_orders, client, db)
for k in ("ordersCreated", "ordersUpdated", "goodsCreated", "goodsUpdated"):
pass
totals["ordersAdded"] += stats["ordersCreated"]
totals["ordersUpdated"] += stats["ordersUpdated"]
totals["goodsAdded"] += stats["goodsCreated"]
totals["goodsUpdated"] += stats["goodsUpdated"]
day_reports.append({"date": day_str, "diff": diff, "ok": False,
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
finally:
client.close()
return {
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
}
if __name__ == "__main__":
riin_verify.serve(name="riin-verify", cron="0 6 * * *")
+1
View File
@@ -0,0 +1 @@
"""SDS 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / refresh_100_prefix / verify。"""
+58
View File
@@ -0,0 +1,58 @@
"""SDS 同步公共管道:映射订单(含内联 items)→ upsert 到 PG。"""
from __future__ import annotations
from typing import Any, Callable
from flows.shared import resolve_courier_source
from flows.shared.db import SyncDB
from flows.shared.mappers import map_sds_goods, map_sds_order
def map_sds_records(
raw_orders: list[Any],
channel_map: dict[int, str | None],
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
skip_unpaid: bool = True,
) -> tuple[list[Any], list[Any], int]:
"""映射 SDS 订单+商品。返回 (order_inputs, goods_inputs, skipped_unpaid)。"""
order_inputs: list[Any] = []
goods_inputs: list[Any] = []
refreshed: set[int] = set()
skipped_unpaid = 0
for raw in raw_orders:
courier = None
if channel_refresh is not None:
courier = resolve_courier_source(raw.logistics_id, channel_map, channel_refresh, refreshed)
try:
o = map_sds_order(raw, channel_map, courier)
except Exception as err: # noqa: BLE001
print(f"[SDS] 映射订单失败 {getattr(raw, 'no', '?')}: {err}")
continue
if skip_unpaid and o.systemStatus == "待付款":
skipped_unpaid += 1
continue
order_inputs.append(o)
for item in raw.items or []:
try:
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
except Exception as err: # noqa: BLE001
print(f"[SDS] 映射商品失败 order={o.orderNo}: {err}")
return order_inputs, goods_inputs, skipped_unpaid
def sync_sds_records(
raw_orders: list[Any], channel_map: dict[int, str | None], db: SyncDB,
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
skip_unpaid: bool = True,
) -> dict[str, int]:
order_inputs, goods_inputs, skipped = map_sds_records(
raw_orders, channel_map, channel_refresh, skip_unpaid
)
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
stats["orderCount"] = len(order_inputs)
stats["goodsCount"] = len(goods_inputs)
stats["skippedUnpaid"] = skipped
return stats
+57
View File
@@ -0,0 +1,57 @@
"""SDS 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
对应 sds-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now,按 payDate。
"""
from __future__ import annotations
from datetime import datetime
from prefect import flow, task
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, floor_to_hour, format_date,
get_channel_type_map, refresh_channel_type_map,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_sds_client()
try:
return fetch_sds_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client))
finally:
client.close()
@flow(name="sds-incremental-sync", log_prints=True)
def sds_incremental_sync() -> dict:
db = SyncDB()
latest = db.get_latest_ordered_at("SDS")
now = datetime.now()
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
if from_dt >= now:
from_dt = floor_to_hour(now)
params = {"pay_date_begin": format_date(from_dt), "pay_date_end": format_date(now), "size": 500}
print(f"SDS 增量同步 {params['pay_date_begin']} ~ {params['pay_date_end']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["pay_date_begin"], "to": params["pay_date_end"]}, **stats}
if __name__ == "__main__":
sds_incremental_sync.serve(name="sds-incremental-sync", cron="15 * * * *")
+76
View File
@@ -0,0 +1,76 @@
"""SDS 订单同步:按下单时间窗口(+筛选)拉取订单与商品。
对应 sds-order-sync。主时间维度 payDate(下单时间)。
"""
from __future__ import annotations
from typing import Any
from prefect import flow, task
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, get_channel_type_map, refresh_channel_type_map,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict[str, Any]) -> list:
client = create_sds_client()
try:
return fetch_sds_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client))
finally:
client.close()
@flow(name="sds-order-sync", log_prints=True)
def sds_order_sync(
pay_date_begin: str,
pay_date_end: str,
order_status_code: str | None = None,
factory_ids: list[int] | None = None,
merchant_ids: list[int] | None = None,
logistics_ids: list[int] | None = None,
issuing_bay_area_id: int | None = None,
keywords: str | None = None,
page_size: int = 500,
) -> dict[str, Any]:
params: dict[str, Any] = {
"pay_date_begin": pay_date_begin,
"pay_date_end": pay_date_end,
"size": page_size,
}
if order_status_code is not None:
params["order_status_code"] = order_status_code
if factory_ids:
params["factory_ids"] = factory_ids
if merchant_ids:
params["merchant_ids"] = merchant_ids
if logistics_ids:
params["logistics_ids"] = logistics_ids
if issuing_bay_area_id is not None:
params["issuing_bay_area_id"] = issuing_bay_area_id
if keywords:
params["keywords"] = keywords
print(f"SDS 订单同步 {pay_date_begin} ~ {pay_date_end}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} / 跳过待付款 {stats.get('skippedUnpaid',0)}")
return {"syncRange": {"from": pay_date_begin, "to": pay_date_end}, **stats}
if __name__ == "__main__":
sds_order_sync.serve(name="sds-order-sync")
+56
View File
@@ -0,0 +1,56 @@
"""SDS 近期同步:拉取最近 N 天订单(默认 7,1~90),按 createdDate。
对应 sds-recent-sync。from = startOfDay(now - N days),to = startOfDay(today)。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
refresh_channel_type_map, start_of_day,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_sds_client()
try:
return fetch_sds_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client))
finally:
client.close()
@flow(name="sds-recent-sync", log_prints=True)
def sds_recent_sync(recent_days: int = 7) -> dict:
days = min(max(recent_days, 1), 90)
now = datetime.now()
from_dt = start_of_day(now - timedelta(days=days))
to_dt = start_of_day(now)
params = {"created_date_begin": format_date(from_dt), "created_date_end": format_date(to_dt), "size": 500}
print(f"SDS 近期同步 近 {days} 天 {params['created_date_begin']} ~ {params['created_date_end']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)},
"recentDays": days, **stats}
if __name__ == "__main__":
sds_recent_sync.serve(name="sds-recent-sync", cron="45 * * * *")
+104
View File
@@ -0,0 +1,104 @@
"""SDS 100前缀刷新:备货中/待确认且 productionNo 以 100 开头的订单,工厂分配后用新数据替换。
对应 sds-refresh-100-prefix。
"""
from __future__ import annotations
from prefect import flow, task
from api.sds.models import OrderPageRequest
from flows.shared import (
SyncDB, create_sds_client, get_channel_type_map,
refresh_channel_type_map, resolve_courier_source,
)
from flows.shared.mappers import map_sds_goods, map_sds_order
TARGET_STATUSES = ["备货中", "待确认"]
KEYWORDS_BATCH = 200
def _has_100_prefix_item(raw) -> bool:
for item in raw.items or []:
prod_no = (item.no or item.id) if hasattr(item, "no") else None
if prod_no and str(prod_no).startswith("100"):
return True
return False
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_remote(order_nos: list[str]) -> dict:
client = create_sds_client()
try:
remote: dict[str, object] = {}
for i in range(0, len(order_nos), KEYWORDS_BATCH):
batch = order_nos[i : i + KEYWORDS_BATCH]
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
for o in resp.records:
if o.no:
remote[o.no] = o
return remote
finally:
client.close()
@task
def upsert_records(remote_map: dict, target_order_nos: list[str]) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
refreshed: set[int] = set()
order_inputs: list = []
goods_inputs: list = []
skipped_pending = skipped_unpaid = 0
for no in target_order_nos:
raw = remote_map.get(no)
if not raw:
continue
if _has_100_prefix_item(raw):
skipped_pending += 1
continue
courier = resolve_courier_source(raw.logistics_id, channel_map,
lambda: refresh_channel_type_map(client), refreshed)
try:
o = map_sds_order(raw, channel_map, courier)
except Exception as err: # noqa: BLE001
print(f"[SDS 100] 映射失败 {no}: {err}")
continue
if o.systemStatus == "待付款":
skipped_unpaid += 1
continue
order_inputs.append(o)
for item in raw.items or []:
try:
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
except Exception: # noqa: BLE001
pass
stats = SyncDB().sync_orders_and_goods(order_inputs, goods_inputs)
stats["skippedStillPending"] = skipped_pending
stats["skippedUnpaid"] = skipped_unpaid
return stats
finally:
client.close()
@flow(name="sds-refresh-100-prefix", log_prints=True)
def sds_refresh_100_prefix() -> dict:
db = SyncDB()
targets = db.get_orders_with_100_prefix("SDS", TARGET_STATUSES)
order_nos = list({o for o, _ in targets})
print(f"SDS 100前缀目标订单 {len(order_nos)} 个")
if not order_nos:
return {"targetCount": 0, "refreshed": 0}
remote = fetch_remote(order_nos)
stats = upsert_records(remote, order_nos)
print(f"完成: 刷新 {stats['orderCount']} / 仍待分配 {stats.get('skippedStillPending',0)} / 待付款 {stats.get('skippedUnpaid',0)}")
return {"targetCount": len(order_nos), **stats}
if __name__ == "__main__":
sds_refresh_100_prefix.serve(name="sds-refresh-100-prefix", cron="30 */2 * * *")
+61
View File
@@ -0,0 +1,61 @@
"""SDS 刷新未完结:查 DB 未完结订单号,按 keywords 批量刷新。
对应 sds-refresh-pending。终态:已完成 / 已取消。注意不跳过待付款。
"""
from __future__ import annotations
from prefect import flow, task
from api.sds.models import OrderPageRequest
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, get_channel_type_map, refresh_channel_type_map,
)
SDS_FINISHED_STATUSES = ["已完成", "已取消"]
KEYWORDS_BATCH = 500
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_pending(order_nos: list[str]) -> list:
client = create_sds_client()
try:
all_raw: list = []
for i in range(0, len(order_nos), KEYWORDS_BATCH):
batch = order_nos[i : i + KEYWORDS_BATCH]
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
all_raw.extend(resp.records)
return all_raw
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client),
skip_unpaid=False)
finally:
client.close()
@flow(name="sds-refresh-pending", log_prints=True)
def sds_refresh_pending() -> dict:
db = SyncDB()
order_nos = db.get_pending_orders("SDS", SDS_FINISHED_STATUSES)
print(f"SDS 未完结订单 {len(order_nos)} 个")
if not order_nos:
return {"pendingCount": 0, "matchedCount": 0}
raw = fetch_pending(order_nos)
stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}")
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
if __name__ == "__main__":
sds_refresh_pending.serve(name="sds-refresh-pending", cron="0 */2 * * *")
+104
View File
@@ -0,0 +1,104 @@
"""SDS 数据核验:逐天比对远程/本地按状态计数(排除"售后"),有差异则拉取修复。
对应 sds-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from api.sds.models import OrderQueryParams
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
refresh_channel_type_map, start_of_day,
)
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
today = start_of_day(datetime.now())
if mode == "range" and date_range:
parts = [p.strip() for p in date_range.split("~")]
from_dt = datetime.fromisoformat(parts[0])
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
days = []
cur = from_dt
while cur <= to_dt:
days.append(cur)
cur += timedelta(days=1)
return days
n = min(max(recent_days, 1), 90)
return [today - timedelta(days=i) for i in range(n, 0, -1)]
def _diff(remote_stats: list, local: dict[str, int]) -> tuple[dict[str, int], bool]:
diff: dict[str, int] = {}
has_diff = False
statuses = {getattr(s, "name", None) for s in remote_stats} | set(local)
statuses.discard("售后")
statuses.discard(None)
for status in statuses:
remote = next((getattr(s, "orderNum", 0) for s in remote_stats if getattr(s, "name", None) == status), 0)
local_n = local.get(status, 0)
if remote != local_n:
diff[status] = local_n - remote
has_diff = True
return diff, has_diff
@flow(name="sds-verify", log_prints=True)
def sds_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
days = _parse_days(mode, recent_days, date_range)
db = SyncDB()
client = create_sds_client()
channel_map = get_channel_type_map(client)
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "goodsAdded", "goodsUpdated")}
days_with_diff = 0
day_reports = []
try:
for day in days:
day_str = format_date(day).split(" ")[0]
day_start = start_of_day(day)
day_end = day_start + timedelta(days=1)
remote_stats = client.get_status(
OrderQueryParams(pay_date_begin=format_date(day_start), pay_date_end=format_date(day_end))
)
remote_total = sum(getattr(s, "orderNum", 0) or 0 for s in remote_stats)
local_counts = db.get_order_count_by_status("SDS", day_start, day_end)
diff, has_diff = _diff(remote_stats, local_counts)
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
if not has_diff:
day_reports.append({"date": day_str, "diff": diff, "ok": True})
continue
days_with_diff += 1
raw = fetch_sds_all_pages(
client,
{"pay_date_begin": format_date(day_start), "pay_date_end": format_date(day_end), "size": 500},
)
stats = sync_sds_records(raw, channel_map, db,
channel_refresh=lambda: refresh_channel_type_map(client))
totals["ordersAdded"] += stats["ordersCreated"]
totals["ordersUpdated"] += stats["ordersUpdated"]
totals["goodsAdded"] += stats["goodsCreated"]
totals["goodsUpdated"] += stats["goodsUpdated"]
day_reports.append({"date": day_str, "diff": diff, "ok": False,
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
finally:
client.close()
return {
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
}
if __name__ == "__main__":
sds_verify.serve(name="sds-verify", cron="0 7 * * *")
+27
View File
@@ -0,0 +1,27 @@
"""同步编排共享层:客户端/分页、映射、PG 写入。"""
from flows.shared import clients, db, mappers
from flows.shared.clients import (
create_riin_client,
create_sds_client,
fetch_goods_concurrent,
fetch_riin_all_pages,
fetch_sds_all_pages,
floor_to_hour,
format_date,
get_channel_type_map,
refresh_channel_type_map,
resolve_courier_source,
start_of_day,
)
from flows.shared.db import SyncDB
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
__all__ = [
"clients", "db", "mappers", "SyncDB",
"OrderUpsertInput", "GoodsUpsertInput",
"create_riin_client", "create_sds_client",
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent",
"get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source",
"format_date", "start_of_day", "floor_to_hour",
]
+229
View File
@@ -0,0 +1,229 @@
"""同步共享:客户端工厂、顺序分页(含重试/延迟)、RIIN 商品拉取、SDS 物流渠道字典、日期辅助。
行为对齐 inkreach-bi-auth-mng 的 sync/tasks/clients.ts:
- 顺序分页(页间 500ms),非并发,对外部 API 友好。
- fetch_with_retry:仅对 5xx / 429 / 超时重试,最多 3 次,退避 3s*次数。
"""
from __future__ import annotations
import json
import time
from datetime import datetime, timedelta, timezone
from typing import Any, Callable, TypeVar
from prefect.blocks.system import Secret
from api import RiinDisClient, SdsMngClient
from api.riin.models import OrderPageRequest as RiinReq
from api.sds.models import LogisticsPageRequest, OrderPageRequest as SdsReq
from flows.shared.mappers import map_sds_channel_type
T = TypeVar("T")
GOODS_CONCURRENCY = 20
PAGE_DELAY_MS = 500
MAX_RETRIES = 3
RETRY_DELAY_MS = 3000
TZ_CN = timezone(timedelta(hours=8))
_RIIN_FINISHED = ["已退款", "已关闭", "已发货"]
_SDS_FINISHED = ["已完成", "已取消"]
# ---------- 客户端工厂 ----------
def create_riin_client() -> RiinDisClient:
cred = _load_secret("riin-credentials")
return RiinDisClient(cred["username"], cred["password"])
def create_sds_client() -> SdsMngClient:
cred = _load_secret("sds-credentials")
return SdsMngClient(cred["username"], cred["password"])
def _load_secret(name: str) -> dict[str, str]:
value = Secret.load(name).get()
if isinstance(value, str):
value = json.loads(value)
return value
# ---------- 日期辅助 ----------
def format_date(d: datetime) -> str:
return d.strftime("%Y-%m-%d %H:%M:%S")
def start_of_day(d: datetime) -> datetime:
return d.replace(hour=0, minute=0, second=0, microsecond=0)
def floor_to_hour(d: datetime) -> datetime:
return d.replace(minute=0, second=0, microsecond=0)
# ---------- 重试 ----------
def fetch_with_retry(fn: Callable[[], T], label: str) -> T:
last_err: Exception | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
return fn()
except Exception as err: # noqa: BLE001
last_err = err
status = _status_of(err)
msg = str(err)
is_timeout = "timeout" in msg.lower() or "timed out" in msg.lower()
retryable = status == 429 or (status is not None and status >= 500) or is_timeout
if attempt < MAX_RETRIES and retryable:
print(f"[sync] {label} 第 {attempt}/{MAX_RETRIES} 次重试 ({status}: {msg[:120]})")
time.sleep(RETRY_DELAY_MS * attempt / 1000)
continue
raise
assert last_err is not None
raise last_err
def _status_of(err: Exception) -> int | None:
for attr in ("status_code", "statusCode", "status"):
v = getattr(err, attr, None)
if isinstance(v, int):
return v
resp = getattr(err, "response", None)
return getattr(resp, "status_code", None) if resp is not None else None
# ---------- 分页 ----------
def fetch_riin_all_pages(client: RiinDisClient, params: dict[str, Any], max_pages: int = 1000) -> list[Any]:
page_size = params.get("page_size", 200)
base = {**params, "page_size": page_size}
def fetch(idx: int):
return fetch_with_retry(lambda: client.get_orders(RiinReq(**{**base, "page_index": idx})), f"RIIN page {idx}")
first = fetch(1)
total = first.data.total
records = list(first.data.records)
total_pages = min(-(-total // page_size), max_pages)
for page in range(2, total_pages + 1):
resp = fetch(page)
records.extend(resp.data.records)
time.sleep(PAGE_DELAY_MS / 1000)
if total_pages == max_pages:
print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条")
return records
def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[Any]:
page_size = params.get("size", 500)
base = {**params, "size": page_size}
def fetch(idx: int):
return fetch_with_retry(lambda: client.get_orders(SdsReq(**{**base, "page": idx})), f"SDS page {idx}")
first = fetch(1)
records = list(first.records)
total_pages = -(-first.total_count // page_size)
for page in range(2, total_pages + 1):
resp = fetch(page)
records.extend(resp.records)
time.sleep(PAGE_DELAY_MS / 1000)
return records
# ---------- RIIN 商品 ----------
def fetch_goods_for_order(client: RiinDisClient, order_id: str) -> list[Any]:
try:
resp = client.get_order_goods(order_id)
return list(resp.data)
except Exception as err: # noqa: BLE001
print(f"[RIIN sync] 订单 {order_id} 商品拉取失败: {err}")
return []
def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[str, list[Any]]:
"""以 GOODS_CONCURRENCY 并发拉取多订单商品,返回 {order_id: [goods...]}。"""
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
results: dict[str, list[Any]] = {}
if not order_ids:
return results
def make(oid: str) -> Callable[[], None]:
def run() -> None:
results[oid] = fetch_goods_for_order(client, oid)
return run
with ThreadPoolExecutor(max_workers=min(GOODS_CONCURRENCY, len(order_ids))) as pool:
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
return results
# ---------- SDS 物流渠道字典 ----------
_channel_cache: dict[str, Any] = {}
_CHANNEL_TTL = 24 * 3600
_CHANNEL_KEY = "all"
def get_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
cached = _channel_cache.get(_CHANNEL_KEY)
if cached and cached["expire_at"] > time.time():
return cached["data"]
mapping: dict[int, str | None] = {}
try:
areas = client.get_logistics_areas()
area_ids = [a.id for a in areas.items if a.id is not None]
except Exception as err: # noqa: BLE001
print(f"[SDS sync] 物流区域拉取失败: {err}")
area_ids = []
for area_id in area_ids:
try:
resp = client.get_logistics_page(LogisticsPageRequest(issuing_bay_area_id=area_id, all=True, size=200))
for item in resp.items:
if item.id is None:
continue
raw = getattr(item, "channelType", None) or getattr(item, "channel_type", None)
label = map_sds_channel_type(raw)
if label is None:
continue
mapping[int(item.id)] = label
except Exception as err: # noqa: BLE001
print(f"[SDS sync] 区域 {area_id} 物流字典拉取失败: {err}")
_channel_cache[_CHANNEL_KEY] = {"data": mapping, "expire_at": time.time() + _CHANNEL_TTL}
return mapping
def refresh_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
_channel_cache.pop(_CHANNEL_KEY, None)
return get_channel_type_map(client)
def resolve_courier_source(
raw_logistics_id: Any,
mapping: dict[int, str | None],
refresh: Callable[[], dict[int, str | None]],
refreshed_ids: set[int],
) -> str | None:
if raw_logistics_id is None:
return None
try:
lid = int(raw_logistics_id)
except (TypeError, ValueError):
return None
if lid in mapping:
return mapping[lid]
if lid in refreshed_ids:
return None
refreshed_ids.add(lid)
try:
new_map = refresh()
mapping.update(new_map)
except Exception: # noqa: BLE001
pass
return mapping.get(lid)
+365
View File
@@ -0,0 +1,365 @@
"""PG 同步写入层:Orders / Goods / Merchant 的 upsert + 同步所需的查询。
忠实移植自 inkreach-bi-auth-mng 的 sync/lib/db.service.ts。
- Orders 主键:("orderNo", "createdAt")
- Goods 主键:("orderNo", "productionNo")
关键:写入前用库中已有 createdAt 覆盖,避免同一订单因 createdAt 不同产生重复主键行。
"""
from __future__ import annotations
from datetime import datetime
from typing import Any
from psycopg2.extras import execute_values
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
ORDER_CHUNK = 1000
GOODS_CHUNK = 2000
LOOKUP_BATCH = 10000
class SyncDB:
def __init__(self, block_name: str = "pg-inkreach-dmp") -> None:
from prefect_sqlalchemy import SqlAlchemyConnector
self._engine = SqlAlchemyConnector.load(block_name).get_engine()
def _conn(self):
return self._engine.raw_connection()
# ---------- 查询辅助 ----------
def get_latest_ordered_at(self, system_name: str) -> datetime | None:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT MAX(COALESCE("orderedAt","createdAt")) FROM "Orders" WHERE "systemName"=%s',
(system_name,),
)
row = cur.fetchone()
return row[0] if row and row[0] else None
def get_pending_orders(self, system_name: str, exclude_statuses: list[str]) -> list[str]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
'AND "systemStatus" IS DISTINCT FROM ALL %s',
(system_name, exclude_statuses),
)
rows = cur.fetchall()
return [r[0] for r in rows]
def get_order_count_by_status(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT "systemStatus", COUNT(*) FROM "Orders" WHERE "systemName"=%s '
'AND "orderedAt">=%s AND "orderedAt"<%s GROUP BY "systemStatus"',
(system_name, start, end),
)
rows = cur.fetchall()
return {r[0]: int(r[1]) for r in rows}
def get_local_order_nos(self, system_name: str, start: datetime, end: datetime) -> list[str]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
'AND "orderedAt">=%s AND "orderedAt"<%s',
(system_name, start, end),
)
rows = cur.fetchall()
return [r[0] for r in rows]
def get_local_goods_count(self, system_name: str, start: datetime, end: datetime) -> int:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT COUNT(*) FROM "Goods" g JOIN "Orders" o '
'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
(system_name, start, end),
)
row = cur.fetchone()
return int(row[0]) if row else 0
def get_orders_with_100_prefix(self, system_name: str, statuses: list[str]) -> list[tuple[str, datetime]]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT DISTINCT o."orderNo", o."createdAt" FROM "Orders" o '
'JOIN "Goods" g ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
'WHERE o."systemName"=%s AND o."systemStatus"=ANY(%s) AND g."productionNo" LIKE \'100%\'',
(system_name, statuses),
)
rows = cur.fetchall()
return [(r[0], r[1]) for r in rows]
# ---------- 主 upsert ----------
def sync_orders_and_goods(
self,
orders: list[OrderUpsertInput],
goods: list[GoodsUpsertInput],
delete_order_nos: list[str] | None = None,
) -> dict[str, int]:
result = {
"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0,
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
}
if not orders and not goods and not delete_order_nos:
return result
conn = self._conn()
try:
system_name = orders[0].systemName if orders else None
all_order_nos = list({o.orderNo for o in orders} | {g.orderNo for g in goods})
# 1. 解析已有 createdAt,覆盖到 orders(保证幂等 upsert)
existing_created: dict[str, datetime] = {}
for i in range(0, len(all_order_nos), LOOKUP_BATCH):
chunk = all_order_nos[i : i + LOOKUP_BATCH]
with conn.cursor() as cur:
cur.execute(
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
+ (f' AND "systemName"=%s' if system_name else ""),
(chunk, system_name) if system_name else (chunk,),
)
for r in cur.fetchall():
existing_created[r[0]] = r[1]
for o in orders:
if o.orderNo in existing_created:
o.createdAt = existing_created[o.orderNo]
# 2. 去重 orders / goods
seen_o: set[str] = set()
orders_dedup: list[OrderUpsertInput] = []
for o in orders:
key = f'{o.orderNo}|{o.createdAt.isoformat()}'
if key in seen_o:
continue
seen_o.add(key)
orders_dedup.append(o)
resolved_created: dict[str, datetime] = {}
for o in orders_dedup:
resolved_created.setdefault(o.orderNo, o.createdAt)
seen_g: set[str] = set()
goods_dedup: list[GoodsUpsertInput] = []
for g in goods:
if g.orderNo in resolved_created:
g.createdAt = resolved_created[g.orderNo]
key = f'{g.orderNo}|{g.productionNo}'
if key in seen_g:
continue
seen_g.add(key)
goods_dedup.append(g)
goods_by_order: dict[str, list[GoodsUpsertInput]] = {}
for g in goods_dedup:
goods_by_order.setdefault(g.orderNo, []).append(g)
# 3. 分批 upsert orders + 关联 goods
for i in range(0, len(orders_dedup), ORDER_CHUNK):
chunk = orders_dedup[i : i + ORDER_CHUNK]
related: list[GoodsUpsertInput] = []
for o in chunk:
related.extend(goods_by_order.get(o.orderNo, []))
stats = self._upsert_order_batch(conn, chunk, related, system_name)
for k, v in stats.items():
result[k] += v
# 4. 剩余 goods(其 order 不在本次 orders 列表)
orders_set = {o.orderNo for o in orders_dedup}
remaining = [g for g in goods_dedup if g.orderNo not in orders_set]
if remaining:
c, u = self._upsert_goods_only(conn, remaining, system_name)
result["goodsCreated"] += c
result["goodsUpdated"] += u
# 5. 删除订单
if delete_order_nos:
with conn.cursor() as cur:
cur.execute(
'DELETE FROM "Orders" WHERE "orderNo"=ANY(%s)'
+ (f' AND "systemName"=%s' if system_name else ""),
(delete_order_nos, system_name) if system_name else (delete_order_nos,),
)
result["ordersDeleted"] = cur.rowcount
# 6. 商家占位
self._ensure_merchants(conn, orders_dedup)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
return result
def _upsert_order_batch(
self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], system_name: str | None
) -> dict[str, int]:
stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0}
with conn.cursor() as cur:
# 统计已存在 → created/updated
pairs = [(o.orderNo, o.createdAt) for o in orders]
cur.execute(
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamp[]) AS t(order_no,created_at) "
"JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at",
_unnest_pairs(pairs),
)
existing_set = {(r[0], r[1]) for r in cur.fetchall()}
updated = sum(1 for o in orders if (o.orderNo, o.createdAt) in existing_set)
stats["ordersUpdated"] = updated
stats["ordersCreated"] = len(orders) - updated
execute_values(
cur,
'''INSERT INTO "Orders" (
"orderNo","outOrderNo","courierSource","courierName","courierNo","merchantId",
"countryRegion","factoryName","systemName","systemStatus","platformName","platformStatus",
"createdAt","orderedAt","confirmedAt","completedAt","shipDeadline"
) VALUES %s
ON CONFLICT ("orderNo","createdAt") DO UPDATE SET
"outOrderNo"=EXCLUDED."outOrderNo","courierSource"=EXCLUDED."courierSource",
"courierName"=EXCLUDED."courierName","courierNo"=EXCLUDED."courierNo",
"merchantId"=EXCLUDED."merchantId","countryRegion"=EXCLUDED."countryRegion",
"factoryName"=EXCLUDED."factoryName","systemName"=EXCLUDED."systemName",
"systemStatus"=EXCLUDED."systemStatus","platformName"=EXCLUDED."platformName",
"platformStatus"=EXCLUDED."platformStatus","orderedAt"=EXCLUDED."orderedAt",
"confirmedAt"=EXCLUDED."confirmedAt","completedAt"=EXCLUDED."completedAt",
"shipDeadline"=EXCLUDED."shipDeadline"''',
[(
o.orderNo, o.outOrderNo, o.courierSource, o.courierName, o.courierNo, o.merchantId,
o.countryRegion, o.factoryName, o.systemName, o.systemStatus, o.platformName, o.platformStatus,
o.createdAt, o.orderedAt, o.confirmedAt, o.completedAt, o.shipDeadline,
) for o in orders],
page_size=ORDER_CHUNK,
)
# 关联 goods:先删本次不再出现的,再 upsert
if goods:
order_nos = list({g.orderNo for g in goods})
prod_nos = [g.productionNo for g in goods]
cur.execute(
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)
AND ("orderNo","createdAt") IN (
SELECT "orderNo","createdAt" FROM "Orders"
WHERE "orderNo"=ANY(%s)''' + (f' AND "systemName"=%s' if system_name else "") + ')',
(order_nos, prod_nos, order_nos, system_name) if system_name else (order_nos, prod_nos, order_nos),
)
stats["goodsRemoved"] += cur.rowcount
g_pairs = [(g.orderNo, g.productionNo) for g in goods]
cur.execute(
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
_unnest_pairs(g_pairs),
)
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
g_updated = sum(1 for g in goods if (g.orderNo, g.productionNo) in g_existing)
stats["goodsUpdated"] = g_updated
stats["goodsCreated"] = len(goods) - g_updated
execute_values(
cur,
'''INSERT INTO "Goods" (
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
"itemSize","itemStyleNo","quantity","status"
) VALUES %s
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
[(
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
g.itemSize, g.itemStyleNo, g.quantity, g.status,
) for g in goods],
page_size=GOODS_CHUNK,
)
return stats
def _upsert_goods_only(
self, conn, goods: list[GoodsUpsertInput], system_name: str | None
) -> tuple[int, int]:
# 用库中实际 createdAt 覆盖
order_nos = list({g.orderNo for g in goods})
actual: dict[str, datetime] = {}
with conn.cursor() as cur:
cur.execute(
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
+ (f' AND "systemName"=%s' if system_name else ""),
(order_nos, system_name) if system_name else (order_nos,),
)
for r in cur.fetchall():
actual[r[0]] = r[1]
for g in goods:
if g.orderNo in actual:
g.createdAt = actual[g.orderNo]
created = updated = 0
for i in range(0, len(goods), GOODS_CHUNK):
chunk = goods[i : i + GOODS_CHUNK]
with conn.cursor() as cur:
c_order = list({g.orderNo for g in chunk})
c_prod = [g.productionNo for g in chunk]
cur.execute(
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)'''
+ (f''' AND ("orderNo","createdAt") IN (SELECT "orderNo","createdAt" FROM "Orders"
WHERE "orderNo"=ANY(%s) AND "systemName"=%s)''' if system_name else ""),
(c_order, c_prod, c_order, system_name) if system_name else (c_order, c_prod),
)
g_pairs = [(g.orderNo, g.productionNo) for g in chunk]
cur.execute(
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
_unnest_pairs(g_pairs),
)
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
u = sum(1 for g in chunk if (g.orderNo, g.productionNo) in g_existing)
updated += u
created += len(chunk) - u
execute_values(
cur,
'''INSERT INTO "Goods" (
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
"itemSize","itemStyleNo","quantity","status"
) VALUES %s
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
[(
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
g.itemSize, g.itemStyleNo, g.quantity, g.status,
) for g in chunk],
page_size=GOODS_CHUNK,
)
return created, updated
def _ensure_merchants(self, conn, orders: list[OrderUpsertInput]) -> None:
ids = list({o.merchantId for o in orders if o.merchantId and o.merchantId.strip()})
if not ids:
return
with conn.cursor() as cur:
execute_values(
cur,
'''INSERT INTO "Merchant" ("merchantId","merchantGroup","responsibleName","responsibleId","tags")
VALUES %s ON CONFLICT ("merchantId") DO NOTHING''',
[(mid, "", None, None, "") for mid in ids],
)
def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]:
"""构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。
配合 SQL:unnest(%s::text[], %s::timestamp[]) 或 unnest(%s::text[], %s::text[])。
"""
return [p[0] for p in pairs], [p[1] for p in pairs]
+317
View File
@@ -0,0 +1,317 @@
"""数据映射:RIIN / SDS 原始订单 → 统一 Order/Goods 写入格式。
忠实移植自 inkreach-bi-auth-mng 的 sync/mappers/{riin,sds,normalize-country}.ts。
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
TZ_CN = timezone(timedelta(hours=8))
# ---------- RIIN 状态 / 平台 / 快递来源映射 ----------
RIIN_SYSTEM_STATUS_MAP: dict[int, str] = {
-4: "待推送", -3: "工厂反审", -5: "待审核", -6: "分配失败",
1: "待推送", 3: "已反审", 4: "生产中", 5: "生产中",
12: "已发货", 13: "已关闭", 14: "已取消", 15: "已关闭",
}
RIIN_PLATFORM_TYPE_MAP: dict[int, str] = {
2: "淘宝", 3: "拼多多", 6: "Shopee", 7: "LAZADA", 8: "1688",
11: "TIKTOK", 13: "TEMU", 15: "自建站(海外)", 16: "SHOPIFY",
17: "SHEIN", 18: "自建站(国内)", 19: "WALMART", 21: "AMAZON",
26: "OZON", 27: "Wildberries", 30: "ETSY", 31: "MERCADO",
}
RIIN_EXPRESS_SOURCE_MAP: dict[int, str] = {
1: "工厂发货", 2: "平台面单", 3: "自有面单", 4: "分销面单", 5: "电商面单",
}
RIIN_GOODS_STATUS_MAP: dict[str, str] = {
"NOT_SHIPPED": "待发货", "SHIPPED": "已发货", "CANCEL": "已取消",
"CANCELLED": "已取消", "REFUNDING": "退款中",
}
RIIN_PLATFORM_STATUS_MAP: dict[str, str] = {
"SHIPPED": "已发货", "NOT_SHIPPED": "待发货", "CANCEL": "已取消",
"PAUSE": "暂停", "COMPLETE": "已完成",
}
# ---------- SDS 状态 / 物流渠道映射 ----------
SDS_SYSTEM_STATUS_MAP: dict[int, str] = {
1: "待付款", 2: "待确认", 3: "备货中", 4: "已完成", 5: "搁置中",
7: "待设计", 9: "部分发货", 98: "已取消",
}
SDS_GOODS_STATUS_MAP: dict[int, str] = {**SDS_SYSTEM_STATUS_MAP, 99: "已删除"}
SDS_CHANNEL_TYPE_LABEL_MAP: dict[str, str] = {
"online": "线上物流", "offline": "普通物流", "manual": "手动物流", "self_pickup": "自提",
}
# ---------- 日期解析 ----------
def safe_date(value: Any, context: str = "") -> datetime:
"""解析日期字符串(无时区视为 +08:00),返回 tz-aware datetime。"""
if not value or not isinstance(value, str):
raise ValueError(f"Invalid date for {context}: {value!r}")
s = value.strip()
if "+" not in s and "T" not in s and not s.endswith("Z"):
s = f"{s}T00:00:00+08:00" if len(s) == 10 else f"{s}+08:00"
try:
dt = datetime.fromisoformat(s)
except ValueError:
raise ValueError(f"Invalid date for {context}: {value!r}")
return dt if dt.tzinfo else dt.replace(tzinfo=TZ_CN)
def _from_epoch_ms(value: Any) -> datetime | None:
try:
n = int(value)
except (TypeError, ValueError):
return None
return datetime.fromtimestamp(n / 1000, tz=TZ_CN) if n > 0 else None
# ---------- 国家/地区归一 ----------
_US_ALIASES = {"US", "UNITED STATES", "USA", "ESTADOS UNIDOS", "美国加州发货仓"}
_MX_ALIASES = {"MX", "MEXICO", "MÉXICO", "MEX", "ME"}
_CA_ALIASES = {"加拿大工厂", "CA"}
_PL_ALIASES = {"欧州波兰发货仓", "欧洲波兰法兰克福发货仓"}
_WAREHOUSE_MAP = {
"印美达澳洲发货仓": "澳洲", "欧洲德国发货仓": "德国", "欧洲匈牙利发货仓": "匈牙利",
"欧洲西班牙发货仓": "西班牙", "欧洲意大利发货仓": "意大利",
}
_DIRECT_MAP = {"Saudi Arabia": "沙特", "Japan": "日本"}
_EXCLUDED_COUNTRIES = {
"Croatia", "Czech Republic", "丹麦", "保加利亚", "克罗地亚", "卢森堡", "奥地利", "希腊",
"拉脱维亚", "捷克", "斯洛伐克", "斯洛文尼亚", "比利时", "法国", "爱尔兰", "爱沙尼亚",
"瑞典", "瑞士", "立陶宛", "罗马尼亚", "芬兰", "荷兰", "Denmark", "Austria", "Greece",
"Latvia", "Slovakia", "Slovenia", "Belgium", "France", "Ireland", "Estonia", "Sweden",
"Switzerland", "Lithuania", "Romania", "Finland", "Netherlands", "Luxembourg", "Bulgaria",
}
def normalize_country_region(raw: str | None, factory_name: str | None = None) -> str | None:
if factory_name == "广州印美达":
return "中国"
if not raw:
return None
trimmed = raw.strip()
if trimmed in _EXCLUDED_COUNTRIES:
return None
upper = trimmed.upper()
if upper in _US_ALIASES:
return "美国"
if upper in _MX_ALIASES:
return "墨西哥"
if upper in _CA_ALIASES:
return "加拿大"
if trimmed in _PL_ALIASES:
return "波兰"
if trimmed in _WAREHOUSE_MAP:
return _WAREHOUSE_MAP[trimmed]
if raw in _DIRECT_MAP:
return _DIRECT_MAP[raw]
return raw
# ---------- 统一写入模型 ----------
class OrderUpsertInput:
__slots__ = (
"orderNo", "outOrderNo", "courierSource", "courierName", "courierNo",
"merchantId", "countryRegion", "factoryName", "systemName", "systemStatus",
"platformName", "platformStatus", "createdAt", "orderedAt", "confirmedAt",
"completedAt", "shipDeadline",
)
def __init__(self, **kwargs: Any) -> None:
for k in self.__slots__:
setattr(self, k, kwargs.get(k))
class GoodsUpsertInput:
__slots__ = (
"orderNo", "productionNo", "createdAt", "outOrderNo", "itemName",
"itemColor", "itemSize", "itemStyleNo", "quantity", "status",
)
def __init__(self, **kwargs: Any) -> None:
for k in self.__slots__:
setattr(self, k, kwargs.get(k))
# ---------- RIIN 映射 ----------
def map_riin_system_status(order_status: int) -> str | None:
return RIIN_SYSTEM_STATUS_MAP.get(order_status)
def map_riin_order(raw: Any) -> OrderUpsertInput:
platform_oid = raw.platform_oid
ordered_at = _parse(raw.push_time, f"RIIN order {platform_oid} pushTime")
confirmed_at = _parse(raw.distributor_push_time, f"RIIN order {platform_oid} distributorPushTime")
system_status = map_riin_system_status(raw.order_status) if raw.order_status is not None else None
is_terminal = system_status in ("已发货", "已关闭", "已取消")
completed_at: datetime | None = None
if is_terminal:
completed_at = _parse(raw.shipping_time, "") or _parse(raw.finish_time, "")
ship_deadline = _parse(raw.except_last_shipping_time, "")
courier_source = RIIN_EXPRESS_SOURCE_MAP.get(raw.express_source) if raw.express_source is not None else None
factory_name = raw.distributor_push_target_name
return OrderUpsertInput(
orderNo=platform_oid,
outOrderNo=raw.source_platform_oid,
courierSource=courier_source,
courierName=raw.delivery_courier,
courierNo=raw.courier_number,
merchantId=raw.sub_name,
countryRegion=normalize_country_region(raw.receiver_country, factory_name),
factoryName=factory_name,
systemName="RIIN",
systemStatus=system_status,
platformName=RIIN_PLATFORM_TYPE_MAP.get(raw.platform_type, str(raw.platform_type)) if raw.platform_type is not None else None,
platformStatus=RIIN_PLATFORM_STATUS_MAP.get(raw.platform_order_status, raw.platform_order_status) if raw.platform_order_status else None,
createdAt=safe_date(raw.order_create_time, f"RIIN order {platform_oid} orderCreateTime"),
orderedAt=ordered_at,
confirmedAt=confirmed_at,
completedAt=completed_at,
shipDeadline=ship_deadline,
)
def map_riin_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
product_code = raw.product_code
item_name = product_code.split("-")[0] if product_code else None
return GoodsUpsertInput(
orderNo=parent_order_no,
productionNo=raw.id,
createdAt=parent_created_at,
outOrderNo=raw.platform_oid,
itemName=item_name,
itemColor=raw.color_code,
itemSize=raw.size_code,
itemStyleNo=item_name,
quantity=raw.num if raw.num is not None else 1,
status=RIIN_GOODS_STATUS_MAP.get(raw.goods_status) if raw.goods_status else None,
)
# ---------- SDS 映射 ----------
def map_sds_channel_type(channel_type: Any) -> str | None:
if not isinstance(channel_type, str):
return None
return SDS_CHANNEL_TYPE_LABEL_MAP.get(channel_type)
def map_sds_system_status(status: int) -> str | None:
return SDS_SYSTEM_STATUS_MAP.get(status)
def map_sds_order(raw: Any, channel_type_map: dict[int, str | None], pre_courier: str | None = None) -> OrderUpsertInput:
ordered_at = _parse(raw.gmt_pay, f"SDS order {raw.no} gmtPay")
system_status = map_sds_system_status(raw.status) if raw.status is not None else None
is_terminal = system_status in ("已完成", "已取消")
completed_at: datetime | None = None
if is_terminal and raw.finish_time and raw.finish_time > 0:
completed_at = _from_epoch_ms(raw.finish_time)
ship_deadline = _from_epoch_ms(raw.latest_ship_time) if raw.latest_ship_time else None
courier_source = pre_courier
if courier_source is None and raw.logistics_id is not None:
try:
lid = int(raw.logistics_id)
except (TypeError, ValueError):
lid = None
if lid is not None and lid in channel_type_map:
courier_source = channel_type_map[lid]
courier_no: str | None = None
if raw.parcel_list:
first = raw.parcel_list[0]
try:
courier_no = first["logistics"]["carriage"]["no"]
except (TypeError, KeyError, IndexError):
courier_no = None
factory_name: str | None = None
for item in raw.items or []:
fname = getattr(getattr(item, "factory", None), "name", None)
if not fname and isinstance(item, dict):
fname = (item.get("factory") or {}).get("name")
if fname:
factory_name = fname
break
area_name = raw.issuing_bay_area.name if raw.issuing_bay_area else None
return OrderUpsertInput(
orderNo=raw.no,
outOrderNo=raw.out_order_no,
courierSource=courier_source,
courierName=raw.carriage_name,
courierNo=courier_no,
merchantId=raw.merchant.name if raw.merchant else None,
countryRegion=normalize_country_region(area_name, factory_name),
factoryName=factory_name,
systemName="SDS",
systemStatus=system_status,
platformName=raw.merchant_store.platform_name if raw.merchant_store else None,
platformStatus=raw.online_order_status_name,
createdAt=safe_date(raw.gmt_created, f"SDS order {raw.no}"),
orderedAt=ordered_at,
confirmedAt=None,
completedAt=completed_at,
shipDeadline=ship_deadline,
)
def _parse_sds_style_no(name: str | None) -> str | None:
if not name:
return None
parts = name.split("()")
if len(parts) < 3:
return None
subs = parts[2].split("-")
if len(subs) == 2:
return subs[0] or None
if len(subs) >= 3:
return subs[1] or None
return None
def map_sds_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
product_name = raw.product_name
color_name = None
block = raw.product_color_block
if block is not None:
color_name = getattr(block, "color_name", None) or (block.get("colorName") if isinstance(block, dict) else None)
return GoodsUpsertInput(
orderNo=parent_order_no,
productionNo=raw.no or raw.id,
createdAt=parent_created_at,
outOrderNo=raw.out_order_item_id,
itemName=product_name,
itemColor=color_name,
itemSize=raw.product_size,
itemStyleNo=_parse_sds_style_no(product_name),
quantity=raw.num if raw.num is not None else 1,
status=SDS_GOODS_STATUS_MAP.get(raw.status) if raw.status is not None else None,
)
def _parse(value: Any, context: str) -> datetime | None:
if not value:
return None
if isinstance(value, (int, float)):
return _from_epoch_ms(value)
try:
return safe_date(value, context)
except ValueError:
return None