init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""RIIN 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / by_orders / verify。"""
|
||||
@@ -0,0 +1,45 @@
|
||||
"""RIIN 同步公共管道:映射订单 → 并发拉商品 → upsert 到 PG。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from flows.shared import fetch_goods_concurrent
|
||||
from flows.shared.db import SyncDB
|
||||
from flows.shared.mappers import map_riin_goods, map_riin_order
|
||||
from api import RiinDisClient
|
||||
|
||||
|
||||
def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB) -> dict[str, int]:
|
||||
order_inputs: list[Any] = []
|
||||
order_meta: dict[str, tuple[str, Any]] = {} # raw.id -> (orderNo, createdAt)
|
||||
goods_failed = 0
|
||||
|
||||
for raw in raw_orders:
|
||||
try:
|
||||
o = map_riin_order(raw)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[RIIN] 映射订单失败 {getattr(raw, 'platform_oid', '?')}: {err}")
|
||||
continue
|
||||
order_inputs.append(o)
|
||||
if raw.id:
|
||||
order_meta[raw.id] = (o.orderNo, o.createdAt)
|
||||
|
||||
goods_inputs: list[Any] = []
|
||||
if order_meta:
|
||||
goods_map = fetch_goods_concurrent(client, list(order_meta.keys()))
|
||||
for oid, (order_no, created_at) in order_meta.items():
|
||||
items = goods_map.get(oid, [])
|
||||
if not items:
|
||||
goods_failed += 1
|
||||
for g in items:
|
||||
try:
|
||||
goods_inputs.append(map_riin_goods(g, order_no, created_at))
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[RIIN] 映射商品失败 order={order_no}: {err}")
|
||||
|
||||
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
|
||||
stats["orderCount"] = len(order_inputs)
|
||||
stats["goodsCount"] = len(goods_inputs)
|
||||
stats["goodsFetchFailed"] = goods_failed
|
||||
return stats
|
||||
@@ -0,0 +1,59 @@
|
||||
"""RIIN 指定单号同步:通过订单号列表(逗号/换行分隔)拉取指定订单。
|
||||
|
||||
对应 riin-by-orders。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.riin.models import OrderPageRequest
|
||||
from flows.riin._pipeline import sync_riin_records
|
||||
from flows.shared import SyncDB, create_riin_client
|
||||
|
||||
|
||||
def _split_order_nos(text: str) -> list[str]:
|
||||
seen: dict[str, None] = {}
|
||||
for part in re.split(r"[\n\r,,\s]+", text or ""):
|
||||
part = part.strip()
|
||||
if part and part not in seen:
|
||||
seen[part] = None
|
||||
return list(seen.keys())
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def fetch_by_orders(order_nos: list[str]) -> list:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
resp = client.get_orders(OrderPageRequest(platform_oid_list=",".join(order_nos), all=True))
|
||||
return list(resp.data.records)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return sync_riin_records(raw_orders, client, SyncDB())
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="riin-by-orders-sync", log_prints=True)
|
||||
def riin_by_orders_sync(order_nos: str) -> dict:
|
||||
nos = _split_order_nos(order_nos)
|
||||
print(f"RIIN 指定单号同步 输入 {len(nos)} 个")
|
||||
if not nos:
|
||||
return {"inputCount": 0, "matchedCount": 0}
|
||||
|
||||
raw = fetch_by_orders(nos)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 匹配 {len(raw)} | {stats}")
|
||||
return {"inputCount": len(nos), "matchedCount": len(raw), **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_by_orders_sync.serve(name="riin-by-orders-sync")
|
||||
@@ -0,0 +1,60 @@
|
||||
"""RIIN 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
|
||||
|
||||
对应 riin-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.riin._pipeline import sync_riin_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_riin_client, fetch_riin_all_pages, floor_to_hour, format_date,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict) -> list:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return fetch_riin_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return sync_riin_records(raw_orders, client, SyncDB())
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="riin-incremental-sync", log_prints=True)
|
||||
def riin_incremental_sync() -> dict:
|
||||
db = SyncDB()
|
||||
latest = db.get_latest_ordered_at("RIIN")
|
||||
now = datetime.now()
|
||||
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
|
||||
to_dt = now
|
||||
if from_dt >= to_dt:
|
||||
from_dt = floor_to_hour(now)
|
||||
|
||||
params = {
|
||||
"time_type": 3,
|
||||
"start_time": format_date(from_dt),
|
||||
"end_time": format_date(to_dt),
|
||||
"page_size": 200,
|
||||
}
|
||||
print(f"RIIN 增量同步 {params['start_time']} ~ {params['end_time']}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
|
||||
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_incremental_sync.serve(name="riin-incremental-sync", cron="0 * * * *")
|
||||
@@ -0,0 +1,77 @@
|
||||
"""RIIN 订单同步:按时间窗口(+可选筛选)拉取订单与商品。
|
||||
|
||||
对应 riin-order-sync。timeType 固定 3(商家推送时间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.riin.models import OrderPageRequest
|
||||
from flows.riin._pipeline import sync_riin_records
|
||||
from flows.shared import SyncDB, create_riin_client, fetch_riin_all_pages, format_date
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict[str, Any]) -> list[Any]:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return fetch_riin_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list[Any]) -> dict[str, int]:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return sync_riin_records(raw_orders, client, SyncDB())
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="riin-order-sync", log_prints=True)
|
||||
def riin_order_sync(
|
||||
order_date_begin: str,
|
||||
order_date_end: str,
|
||||
platform_type: int | None = None,
|
||||
order_status: int | None = None,
|
||||
factory_ids: list[str] | None = None,
|
||||
express_source: int | None = None,
|
||||
urgent_order: int | None = None,
|
||||
has_refund: bool | None = None,
|
||||
page_size: int = 200,
|
||||
) -> dict[str, Any]:
|
||||
if not order_date_begin or not order_date_end:
|
||||
raise ValueError("order_date_begin / order_date_end 为必填")
|
||||
|
||||
params: dict[str, Any] = {
|
||||
"time_type": 3,
|
||||
"start_time": order_date_begin,
|
||||
"end_time": order_date_end,
|
||||
"page_size": page_size,
|
||||
}
|
||||
if platform_type is not None:
|
||||
params["platform_type"] = platform_type
|
||||
if order_status is not None:
|
||||
params["order_status"] = order_status
|
||||
if factory_ids:
|
||||
params["factory_ids"] = factory_ids
|
||||
if express_source is not None:
|
||||
params["express_source"] = express_source
|
||||
if urgent_order is not None:
|
||||
params["urgent_order"] = urgent_order
|
||||
if has_refund is not None:
|
||||
params["has_refund"] = has_refund
|
||||
|
||||
print(f"RIIN 订单同步 {order_date_begin} ~ {order_date_end}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} | {stats}")
|
||||
return {"syncRange": {"from": order_date_begin, "to": order_date_end}, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_order_sync.serve(name="riin-order-sync")
|
||||
@@ -0,0 +1,56 @@
|
||||
"""RIIN 近期同步:拉取最近 N 天订单(默认 7 天,1~90)。
|
||||
|
||||
对应 riin-recent-sync。from = startOfDay(now - N days),to = now。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.riin._pipeline import sync_riin_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict) -> list:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return fetch_riin_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return sync_riin_records(raw_orders, client, SyncDB())
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="riin-recent-sync", log_prints=True)
|
||||
def riin_recent_sync(recent_days: int = 7) -> dict:
|
||||
days = min(max(recent_days, 1), 90)
|
||||
now = datetime.now()
|
||||
from_dt = start_of_day(now - timedelta(days=days))
|
||||
|
||||
params = {
|
||||
"time_type": 3,
|
||||
"start_time": format_date(from_dt),
|
||||
"end_time": format_date(now),
|
||||
"page_size": 200,
|
||||
}
|
||||
print(f"RIIN 近期同步 近 {days} 天 {params['start_time']} ~ {params['end_time']}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
|
||||
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, "recentDays": days, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_recent_sync.serve(name="riin-recent-sync", cron="30 * * * *")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""RIIN 刷新未完结:查 DB 中未完结订单号,按 platformOidList 批量(500)刷新。
|
||||
|
||||
对应 riin-refresh-pending。终态:已退款 / 已关闭 / 已发货。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.riin.models import OrderPageRequest
|
||||
from flows.riin._pipeline import sync_riin_records
|
||||
from flows.shared import SyncDB, create_riin_client
|
||||
|
||||
RIIN_FINISHED_STATUSES = ["已退款", "已关闭", "已发货"]
|
||||
ORDER_BATCH = 500
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def fetch_pending(order_nos: list[str]) -> list:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
all_raw: list = []
|
||||
for i in range(0, len(order_nos), ORDER_BATCH):
|
||||
batch = order_nos[i : i + ORDER_BATCH]
|
||||
resp = client.get_orders(
|
||||
OrderPageRequest(platform_oid_list=",".join(batch), all=True)
|
||||
)
|
||||
all_raw.extend(resp.data.records)
|
||||
return all_raw
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_riin_client()
|
||||
try:
|
||||
return sync_riin_records(raw_orders, client, SyncDB())
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="riin-refresh-pending", log_prints=True)
|
||||
def riin_refresh_pending() -> dict:
|
||||
db = SyncDB()
|
||||
order_nos = db.get_pending_orders("RIIN", RIIN_FINISHED_STATUSES)
|
||||
print(f"RIIN 未完结订单 {len(order_nos)} 个")
|
||||
if not order_nos:
|
||||
return {"pendingCount": 0, "matchedCount": 0}
|
||||
|
||||
raw = fetch_pending(order_nos)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 匹配 {len(raw)} | {stats}")
|
||||
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_refresh_pending.serve(name="riin-refresh-pending", cron="0 */3 * * *")
|
||||
@@ -0,0 +1,126 @@
|
||||
"""RIIN 数据核验:逐天比对远程/本地按状态计数,有差异则拉取修复。
|
||||
|
||||
对应 riin-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.riin.models import OrderPageRequest
|
||||
from flows.riin._pipeline import sync_riin_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
|
||||
)
|
||||
|
||||
RIIN_COUNT_STATUS_MAP = {
|
||||
"assignedSuceessed": "已分配",
|
||||
"factoryAuditing": "工厂审核中",
|
||||
"inProducing": "生产中",
|
||||
"shipped": "已发货",
|
||||
"closed": "已关闭",
|
||||
"refunding": "退款中",
|
||||
"refunded": "已退款",
|
||||
"rejecting": "拒审中",
|
||||
"rejectedToShop": "已退回店铺",
|
||||
}
|
||||
|
||||
|
||||
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
|
||||
today = start_of_day(datetime.now())
|
||||
if mode == "range" and date_range:
|
||||
parts = [p.strip() for p in date_range.split("~")]
|
||||
from_dt = datetime.fromisoformat(parts[0])
|
||||
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
|
||||
days = []
|
||||
cur = from_dt
|
||||
while cur <= to_dt:
|
||||
days.append(cur)
|
||||
cur += timedelta(days=1)
|
||||
return days
|
||||
days_n = min(max(recent_days, 1), 90)
|
||||
return [today - timedelta(days=i) for i in range(days_n, 0, -1)]
|
||||
|
||||
|
||||
def _count_to_status_map(data: dict) -> dict[str, int]:
|
||||
out: dict[str, int] = {}
|
||||
for key, count in data.items():
|
||||
if key.startswith("dayRange"):
|
||||
continue
|
||||
label = RIIN_COUNT_STATUS_MAP.get(key)
|
||||
if label and count and count > 0:
|
||||
out[label] = count
|
||||
return out
|
||||
|
||||
|
||||
def _diff(remote: dict[str, int], local: dict[str, int]) -> tuple[dict[str, int], bool]:
|
||||
diff: dict[str, int] = {}
|
||||
has_diff = False
|
||||
for status in set(remote) | set(local):
|
||||
r = remote.get(status, 0)
|
||||
l = local.get(status, 0)
|
||||
if r != l:
|
||||
diff[status] = l - r
|
||||
has_diff = True
|
||||
return diff, has_diff
|
||||
|
||||
|
||||
@flow(name="riin-verify", log_prints=True)
|
||||
def riin_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
|
||||
days = _parse_days(mode, recent_days, date_range)
|
||||
db = SyncDB()
|
||||
client = create_riin_client()
|
||||
|
||||
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "ordersRemoved",
|
||||
"goodsAdded", "goodsUpdated", "goodsRemoved")}
|
||||
days_with_diff = 0
|
||||
day_reports = []
|
||||
|
||||
try:
|
||||
for day in days:
|
||||
day_str = format_date(day).split(" ")[0]
|
||||
day_start = start_of_day(day)
|
||||
day_end = day_start + timedelta(days=1)
|
||||
|
||||
count_resp = client.get_order_count(
|
||||
OrderPageRequest(time_type=3, start_time=format_date(day_start), end_time=format_date(day_end))
|
||||
)
|
||||
remote_map = _count_to_status_map(count_resp.data.model_dump(by_alias=False))
|
||||
remote_total = sum(remote_map.values())
|
||||
local_counts = db.get_order_count_by_status("RIIN", day_start, day_end)
|
||||
local_goods = db.get_local_goods_count("RIIN", day_start, day_end)
|
||||
diff, has_diff = _diff(remote_map, local_counts)
|
||||
|
||||
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
|
||||
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
|
||||
if not has_diff:
|
||||
day_reports.append({"date": day_str, "diff": diff, "ok": True})
|
||||
continue
|
||||
|
||||
days_with_diff += 1
|
||||
raw_orders = fetch_riin_all_pages(
|
||||
client,
|
||||
{"time_type": 3, "start_time": format_date(day_start), "end_time": format_date(day_end), "page_size": 200},
|
||||
)
|
||||
stats = sync_riin_records(raw_orders, client, db)
|
||||
for k in ("ordersCreated", "ordersUpdated", "goodsCreated", "goodsUpdated"):
|
||||
pass
|
||||
totals["ordersAdded"] += stats["ordersCreated"]
|
||||
totals["ordersUpdated"] += stats["ordersUpdated"]
|
||||
totals["goodsAdded"] += stats["goodsCreated"]
|
||||
totals["goodsUpdated"] += stats["goodsUpdated"]
|
||||
day_reports.append({"date": day_str, "diff": diff, "ok": False,
|
||||
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
return {
|
||||
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
|
||||
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_verify.serve(name="riin-verify", cron="0 6 * * *")
|
||||
@@ -0,0 +1 @@
|
||||
"""SDS 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / refresh_100_prefix / verify。"""
|
||||
@@ -0,0 +1,58 @@
|
||||
"""SDS 同步公共管道:映射订单(含内联 items)→ upsert 到 PG。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
from flows.shared import resolve_courier_source
|
||||
from flows.shared.db import SyncDB
|
||||
from flows.shared.mappers import map_sds_goods, map_sds_order
|
||||
|
||||
|
||||
def map_sds_records(
|
||||
raw_orders: list[Any],
|
||||
channel_map: dict[int, str | None],
|
||||
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
|
||||
skip_unpaid: bool = True,
|
||||
) -> tuple[list[Any], list[Any], int]:
|
||||
"""映射 SDS 订单+商品。返回 (order_inputs, goods_inputs, skipped_unpaid)。"""
|
||||
order_inputs: list[Any] = []
|
||||
goods_inputs: list[Any] = []
|
||||
refreshed: set[int] = set()
|
||||
skipped_unpaid = 0
|
||||
|
||||
for raw in raw_orders:
|
||||
courier = None
|
||||
if channel_refresh is not None:
|
||||
courier = resolve_courier_source(raw.logistics_id, channel_map, channel_refresh, refreshed)
|
||||
try:
|
||||
o = map_sds_order(raw, channel_map, courier)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS] 映射订单失败 {getattr(raw, 'no', '?')}: {err}")
|
||||
continue
|
||||
if skip_unpaid and o.systemStatus == "待付款":
|
||||
skipped_unpaid += 1
|
||||
continue
|
||||
order_inputs.append(o)
|
||||
for item in raw.items or []:
|
||||
try:
|
||||
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS] 映射商品失败 order={o.orderNo}: {err}")
|
||||
|
||||
return order_inputs, goods_inputs, skipped_unpaid
|
||||
|
||||
|
||||
def sync_sds_records(
|
||||
raw_orders: list[Any], channel_map: dict[int, str | None], db: SyncDB,
|
||||
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
|
||||
skip_unpaid: bool = True,
|
||||
) -> dict[str, int]:
|
||||
order_inputs, goods_inputs, skipped = map_sds_records(
|
||||
raw_orders, channel_map, channel_refresh, skip_unpaid
|
||||
)
|
||||
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
|
||||
stats["orderCount"] = len(order_inputs)
|
||||
stats["goodsCount"] = len(goods_inputs)
|
||||
stats["skippedUnpaid"] = skipped
|
||||
return stats
|
||||
@@ -0,0 +1,57 @@
|
||||
"""SDS 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
|
||||
|
||||
对应 sds-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now,按 payDate。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, floor_to_hour, format_date,
|
||||
get_channel_type_map, refresh_channel_type_map,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
return fetch_sds_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-incremental-sync", log_prints=True)
|
||||
def sds_incremental_sync() -> dict:
|
||||
db = SyncDB()
|
||||
latest = db.get_latest_ordered_at("SDS")
|
||||
now = datetime.now()
|
||||
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
|
||||
if from_dt >= now:
|
||||
from_dt = floor_to_hour(now)
|
||||
|
||||
params = {"pay_date_begin": format_date(from_dt), "pay_date_end": format_date(now), "size": 500}
|
||||
print(f"SDS 增量同步 {params['pay_date_begin']} ~ {params['pay_date_end']}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
|
||||
return {"syncRange": {"from": params["pay_date_begin"], "to": params["pay_date_end"]}, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_incremental_sync.serve(name="sds-incremental-sync", cron="15 * * * *")
|
||||
@@ -0,0 +1,76 @@
|
||||
"""SDS 订单同步:按下单时间窗口(+筛选)拉取订单与商品。
|
||||
|
||||
对应 sds-order-sync。主时间维度 payDate(下单时间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, get_channel_type_map, refresh_channel_type_map,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict[str, Any]) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
return fetch_sds_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-order-sync", log_prints=True)
|
||||
def sds_order_sync(
|
||||
pay_date_begin: str,
|
||||
pay_date_end: str,
|
||||
order_status_code: str | None = None,
|
||||
factory_ids: list[int] | None = None,
|
||||
merchant_ids: list[int] | None = None,
|
||||
logistics_ids: list[int] | None = None,
|
||||
issuing_bay_area_id: int | None = None,
|
||||
keywords: str | None = None,
|
||||
page_size: int = 500,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {
|
||||
"pay_date_begin": pay_date_begin,
|
||||
"pay_date_end": pay_date_end,
|
||||
"size": page_size,
|
||||
}
|
||||
if order_status_code is not None:
|
||||
params["order_status_code"] = order_status_code
|
||||
if factory_ids:
|
||||
params["factory_ids"] = factory_ids
|
||||
if merchant_ids:
|
||||
params["merchant_ids"] = merchant_ids
|
||||
if logistics_ids:
|
||||
params["logistics_ids"] = logistics_ids
|
||||
if issuing_bay_area_id is not None:
|
||||
params["issuing_bay_area_id"] = issuing_bay_area_id
|
||||
if keywords:
|
||||
params["keywords"] = keywords
|
||||
|
||||
print(f"SDS 订单同步 {pay_date_begin} ~ {pay_date_end}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} / 跳过待付款 {stats.get('skippedUnpaid',0)}")
|
||||
return {"syncRange": {"from": pay_date_begin, "to": pay_date_end}, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_order_sync.serve(name="sds-order-sync")
|
||||
@@ -0,0 +1,56 @@
|
||||
"""SDS 近期同步:拉取最近 N 天订单(默认 7,1~90),按 createdDate。
|
||||
|
||||
对应 sds-recent-sync。from = startOfDay(now - N days),to = startOfDay(today)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
|
||||
refresh_channel_type_map, start_of_day,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
return fetch_sds_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-recent-sync", log_prints=True)
|
||||
def sds_recent_sync(recent_days: int = 7) -> dict:
|
||||
days = min(max(recent_days, 1), 90)
|
||||
now = datetime.now()
|
||||
from_dt = start_of_day(now - timedelta(days=days))
|
||||
to_dt = start_of_day(now)
|
||||
|
||||
params = {"created_date_begin": format_date(from_dt), "created_date_end": format_date(to_dt), "size": 500}
|
||||
print(f"SDS 近期同步 近 {days} 天 {params['created_date_begin']} ~ {params['created_date_end']}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
|
||||
return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)},
|
||||
"recentDays": days, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_recent_sync.serve(name="sds-recent-sync", cron="45 * * * *")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""SDS 100前缀刷新:备货中/待确认且 productionNo 以 100 开头的订单,工厂分配后用新数据替换。
|
||||
|
||||
对应 sds-refresh-100-prefix。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.sds.models import OrderPageRequest
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, get_channel_type_map,
|
||||
refresh_channel_type_map, resolve_courier_source,
|
||||
)
|
||||
from flows.shared.mappers import map_sds_goods, map_sds_order
|
||||
|
||||
TARGET_STATUSES = ["备货中", "待确认"]
|
||||
KEYWORDS_BATCH = 200
|
||||
|
||||
|
||||
def _has_100_prefix_item(raw) -> bool:
|
||||
for item in raw.items or []:
|
||||
prod_no = (item.no or item.id) if hasattr(item, "no") else None
|
||||
if prod_no and str(prod_no).startswith("100"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def fetch_remote(order_nos: list[str]) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
remote: dict[str, object] = {}
|
||||
for i in range(0, len(order_nos), KEYWORDS_BATCH):
|
||||
batch = order_nos[i : i + KEYWORDS_BATCH]
|
||||
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
|
||||
for o in resp.records:
|
||||
if o.no:
|
||||
remote[o.no] = o
|
||||
return remote
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(remote_map: dict, target_order_nos: list[str]) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
refreshed: set[int] = set()
|
||||
|
||||
order_inputs: list = []
|
||||
goods_inputs: list = []
|
||||
skipped_pending = skipped_unpaid = 0
|
||||
|
||||
for no in target_order_nos:
|
||||
raw = remote_map.get(no)
|
||||
if not raw:
|
||||
continue
|
||||
if _has_100_prefix_item(raw):
|
||||
skipped_pending += 1
|
||||
continue
|
||||
courier = resolve_courier_source(raw.logistics_id, channel_map,
|
||||
lambda: refresh_channel_type_map(client), refreshed)
|
||||
try:
|
||||
o = map_sds_order(raw, channel_map, courier)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS 100] 映射失败 {no}: {err}")
|
||||
continue
|
||||
if o.systemStatus == "待付款":
|
||||
skipped_unpaid += 1
|
||||
continue
|
||||
order_inputs.append(o)
|
||||
for item in raw.items or []:
|
||||
try:
|
||||
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
stats = SyncDB().sync_orders_and_goods(order_inputs, goods_inputs)
|
||||
stats["skippedStillPending"] = skipped_pending
|
||||
stats["skippedUnpaid"] = skipped_unpaid
|
||||
return stats
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-refresh-100-prefix", log_prints=True)
|
||||
def sds_refresh_100_prefix() -> dict:
|
||||
db = SyncDB()
|
||||
targets = db.get_orders_with_100_prefix("SDS", TARGET_STATUSES)
|
||||
order_nos = list({o for o, _ in targets})
|
||||
print(f"SDS 100前缀目标订单 {len(order_nos)} 个")
|
||||
if not order_nos:
|
||||
return {"targetCount": 0, "refreshed": 0}
|
||||
|
||||
remote = fetch_remote(order_nos)
|
||||
stats = upsert_records(remote, order_nos)
|
||||
print(f"完成: 刷新 {stats['orderCount']} / 仍待分配 {stats.get('skippedStillPending',0)} / 待付款 {stats.get('skippedUnpaid',0)}")
|
||||
return {"targetCount": len(order_nos), **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_refresh_100_prefix.serve(name="sds-refresh-100-prefix", cron="30 */2 * * *")
|
||||
@@ -0,0 +1,61 @@
|
||||
"""SDS 刷新未完结:查 DB 未完结订单号,按 keywords 批量刷新。
|
||||
|
||||
对应 sds-refresh-pending。终态:已完成 / 已取消。注意不跳过待付款。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.sds.models import OrderPageRequest
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, get_channel_type_map, refresh_channel_type_map,
|
||||
)
|
||||
|
||||
SDS_FINISHED_STATUSES = ["已完成", "已取消"]
|
||||
KEYWORDS_BATCH = 500
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def fetch_pending(order_nos: list[str]) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
all_raw: list = []
|
||||
for i in range(0, len(order_nos), KEYWORDS_BATCH):
|
||||
batch = order_nos[i : i + KEYWORDS_BATCH]
|
||||
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
|
||||
all_raw.extend(resp.records)
|
||||
return all_raw
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client),
|
||||
skip_unpaid=False)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-refresh-pending", log_prints=True)
|
||||
def sds_refresh_pending() -> dict:
|
||||
db = SyncDB()
|
||||
order_nos = db.get_pending_orders("SDS", SDS_FINISHED_STATUSES)
|
||||
print(f"SDS 未完结订单 {len(order_nos)} 个")
|
||||
if not order_nos:
|
||||
return {"pendingCount": 0, "matchedCount": 0}
|
||||
|
||||
raw = fetch_pending(order_nos)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 匹配 {len(raw)} | {stats}")
|
||||
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_refresh_pending.serve(name="sds-refresh-pending", cron="0 */2 * * *")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""SDS 数据核验:逐天比对远程/本地按状态计数(排除"售后"),有差异则拉取修复。
|
||||
|
||||
对应 sds-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.sds.models import OrderQueryParams
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
|
||||
refresh_channel_type_map, start_of_day,
|
||||
)
|
||||
|
||||
|
||||
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
|
||||
today = start_of_day(datetime.now())
|
||||
if mode == "range" and date_range:
|
||||
parts = [p.strip() for p in date_range.split("~")]
|
||||
from_dt = datetime.fromisoformat(parts[0])
|
||||
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
|
||||
days = []
|
||||
cur = from_dt
|
||||
while cur <= to_dt:
|
||||
days.append(cur)
|
||||
cur += timedelta(days=1)
|
||||
return days
|
||||
n = min(max(recent_days, 1), 90)
|
||||
return [today - timedelta(days=i) for i in range(n, 0, -1)]
|
||||
|
||||
|
||||
def _diff(remote_stats: list, local: dict[str, int]) -> tuple[dict[str, int], bool]:
|
||||
diff: dict[str, int] = {}
|
||||
has_diff = False
|
||||
statuses = {getattr(s, "name", None) for s in remote_stats} | set(local)
|
||||
statuses.discard("售后")
|
||||
statuses.discard(None)
|
||||
for status in statuses:
|
||||
remote = next((getattr(s, "orderNum", 0) for s in remote_stats if getattr(s, "name", None) == status), 0)
|
||||
local_n = local.get(status, 0)
|
||||
if remote != local_n:
|
||||
diff[status] = local_n - remote
|
||||
has_diff = True
|
||||
return diff, has_diff
|
||||
|
||||
|
||||
@flow(name="sds-verify", log_prints=True)
|
||||
def sds_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
|
||||
days = _parse_days(mode, recent_days, date_range)
|
||||
db = SyncDB()
|
||||
client = create_sds_client()
|
||||
channel_map = get_channel_type_map(client)
|
||||
|
||||
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "goodsAdded", "goodsUpdated")}
|
||||
days_with_diff = 0
|
||||
day_reports = []
|
||||
|
||||
try:
|
||||
for day in days:
|
||||
day_str = format_date(day).split(" ")[0]
|
||||
day_start = start_of_day(day)
|
||||
day_end = day_start + timedelta(days=1)
|
||||
|
||||
remote_stats = client.get_status(
|
||||
OrderQueryParams(pay_date_begin=format_date(day_start), pay_date_end=format_date(day_end))
|
||||
)
|
||||
remote_total = sum(getattr(s, "orderNum", 0) or 0 for s in remote_stats)
|
||||
local_counts = db.get_order_count_by_status("SDS", day_start, day_end)
|
||||
diff, has_diff = _diff(remote_stats, local_counts)
|
||||
|
||||
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
|
||||
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
|
||||
if not has_diff:
|
||||
day_reports.append({"date": day_str, "diff": diff, "ok": True})
|
||||
continue
|
||||
|
||||
days_with_diff += 1
|
||||
raw = fetch_sds_all_pages(
|
||||
client,
|
||||
{"pay_date_begin": format_date(day_start), "pay_date_end": format_date(day_end), "size": 500},
|
||||
)
|
||||
stats = sync_sds_records(raw, channel_map, db,
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
totals["ordersAdded"] += stats["ordersCreated"]
|
||||
totals["ordersUpdated"] += stats["ordersUpdated"]
|
||||
totals["goodsAdded"] += stats["goodsCreated"]
|
||||
totals["goodsUpdated"] += stats["goodsUpdated"]
|
||||
day_reports.append({"date": day_str, "diff": diff, "ok": False,
|
||||
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
return {
|
||||
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
|
||||
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_verify.serve(name="sds-verify", cron="0 7 * * *")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""同步编排共享层:客户端/分页、映射、PG 写入。"""
|
||||
|
||||
from flows.shared import clients, db, mappers
|
||||
from flows.shared.clients import (
|
||||
create_riin_client,
|
||||
create_sds_client,
|
||||
fetch_goods_concurrent,
|
||||
fetch_riin_all_pages,
|
||||
fetch_sds_all_pages,
|
||||
floor_to_hour,
|
||||
format_date,
|
||||
get_channel_type_map,
|
||||
refresh_channel_type_map,
|
||||
resolve_courier_source,
|
||||
start_of_day,
|
||||
)
|
||||
from flows.shared.db import SyncDB
|
||||
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
|
||||
|
||||
__all__ = [
|
||||
"clients", "db", "mappers", "SyncDB",
|
||||
"OrderUpsertInput", "GoodsUpsertInput",
|
||||
"create_riin_client", "create_sds_client",
|
||||
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent",
|
||||
"get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source",
|
||||
"format_date", "start_of_day", "floor_to_hour",
|
||||
]
|
||||
@@ -0,0 +1,229 @@
|
||||
"""同步共享:客户端工厂、顺序分页(含重试/延迟)、RIIN 商品拉取、SDS 物流渠道字典、日期辅助。
|
||||
|
||||
行为对齐 inkreach-bi-auth-mng 的 sync/tasks/clients.ts:
|
||||
- 顺序分页(页间 500ms),非并发,对外部 API 友好。
|
||||
- fetch_with_retry:仅对 5xx / 429 / 超时重试,最多 3 次,退避 3s*次数。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
from prefect.blocks.system import Secret
|
||||
|
||||
from api import RiinDisClient, SdsMngClient
|
||||
from api.riin.models import OrderPageRequest as RiinReq
|
||||
from api.sds.models import LogisticsPageRequest, OrderPageRequest as SdsReq
|
||||
from flows.shared.mappers import map_sds_channel_type
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
GOODS_CONCURRENCY = 20
|
||||
PAGE_DELAY_MS = 500
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY_MS = 3000
|
||||
TZ_CN = timezone(timedelta(hours=8))
|
||||
|
||||
_RIIN_FINISHED = ["已退款", "已关闭", "已发货"]
|
||||
_SDS_FINISHED = ["已完成", "已取消"]
|
||||
|
||||
|
||||
# ---------- 客户端工厂 ----------
|
||||
|
||||
def create_riin_client() -> RiinDisClient:
|
||||
cred = _load_secret("riin-credentials")
|
||||
return RiinDisClient(cred["username"], cred["password"])
|
||||
|
||||
|
||||
def create_sds_client() -> SdsMngClient:
|
||||
cred = _load_secret("sds-credentials")
|
||||
return SdsMngClient(cred["username"], cred["password"])
|
||||
|
||||
|
||||
def _load_secret(name: str) -> dict[str, str]:
|
||||
value = Secret.load(name).get()
|
||||
if isinstance(value, str):
|
||||
value = json.loads(value)
|
||||
return value
|
||||
|
||||
|
||||
# ---------- 日期辅助 ----------
|
||||
|
||||
def format_date(d: datetime) -> str:
|
||||
return d.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def start_of_day(d: datetime) -> datetime:
|
||||
return d.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def floor_to_hour(d: datetime) -> datetime:
|
||||
return d.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
# ---------- 重试 ----------
|
||||
|
||||
def fetch_with_retry(fn: Callable[[], T], label: str) -> T:
|
||||
last_err: Exception | None = None
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
return fn()
|
||||
except Exception as err: # noqa: BLE001
|
||||
last_err = err
|
||||
status = _status_of(err)
|
||||
msg = str(err)
|
||||
is_timeout = "timeout" in msg.lower() or "timed out" in msg.lower()
|
||||
retryable = status == 429 or (status is not None and status >= 500) or is_timeout
|
||||
if attempt < MAX_RETRIES and retryable:
|
||||
print(f"[sync] {label} 第 {attempt}/{MAX_RETRIES} 次重试 ({status}: {msg[:120]})")
|
||||
time.sleep(RETRY_DELAY_MS * attempt / 1000)
|
||||
continue
|
||||
raise
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
|
||||
def _status_of(err: Exception) -> int | None:
|
||||
for attr in ("status_code", "statusCode", "status"):
|
||||
v = getattr(err, attr, None)
|
||||
if isinstance(v, int):
|
||||
return v
|
||||
resp = getattr(err, "response", None)
|
||||
return getattr(resp, "status_code", None) if resp is not None else None
|
||||
|
||||
|
||||
# ---------- 分页 ----------
|
||||
|
||||
def fetch_riin_all_pages(client: RiinDisClient, params: dict[str, Any], max_pages: int = 1000) -> list[Any]:
|
||||
page_size = params.get("page_size", 200)
|
||||
base = {**params, "page_size": page_size}
|
||||
|
||||
def fetch(idx: int):
|
||||
return fetch_with_retry(lambda: client.get_orders(RiinReq(**{**base, "page_index": idx})), f"RIIN page {idx}")
|
||||
|
||||
first = fetch(1)
|
||||
total = first.data.total
|
||||
records = list(first.data.records)
|
||||
total_pages = min(-(-total // page_size), max_pages)
|
||||
for page in range(2, total_pages + 1):
|
||||
resp = fetch(page)
|
||||
records.extend(resp.data.records)
|
||||
time.sleep(PAGE_DELAY_MS / 1000)
|
||||
if total_pages == max_pages:
|
||||
print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条")
|
||||
return records
|
||||
|
||||
|
||||
def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[Any]:
|
||||
page_size = params.get("size", 500)
|
||||
base = {**params, "size": page_size}
|
||||
|
||||
def fetch(idx: int):
|
||||
return fetch_with_retry(lambda: client.get_orders(SdsReq(**{**base, "page": idx})), f"SDS page {idx}")
|
||||
|
||||
first = fetch(1)
|
||||
records = list(first.records)
|
||||
total_pages = -(-first.total_count // page_size)
|
||||
for page in range(2, total_pages + 1):
|
||||
resp = fetch(page)
|
||||
records.extend(resp.records)
|
||||
time.sleep(PAGE_DELAY_MS / 1000)
|
||||
return records
|
||||
|
||||
|
||||
# ---------- RIIN 商品 ----------
|
||||
|
||||
def fetch_goods_for_order(client: RiinDisClient, order_id: str) -> list[Any]:
|
||||
try:
|
||||
resp = client.get_order_goods(order_id)
|
||||
return list(resp.data)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[RIIN sync] 订单 {order_id} 商品拉取失败: {err}")
|
||||
return []
|
||||
|
||||
|
||||
def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[str, list[Any]]:
|
||||
"""以 GOODS_CONCURRENCY 并发拉取多订单商品,返回 {order_id: [goods...]}。"""
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
results: dict[str, list[Any]] = {}
|
||||
if not order_ids:
|
||||
return results
|
||||
|
||||
def make(oid: str) -> Callable[[], None]:
|
||||
def run() -> None:
|
||||
results[oid] = fetch_goods_for_order(client, oid)
|
||||
return run
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(GOODS_CONCURRENCY, len(order_ids))) as pool:
|
||||
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
|
||||
return results
|
||||
|
||||
|
||||
# ---------- SDS 物流渠道字典 ----------
|
||||
|
||||
_channel_cache: dict[str, Any] = {}
|
||||
_CHANNEL_TTL = 24 * 3600
|
||||
_CHANNEL_KEY = "all"
|
||||
|
||||
|
||||
def get_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
|
||||
cached = _channel_cache.get(_CHANNEL_KEY)
|
||||
if cached and cached["expire_at"] > time.time():
|
||||
return cached["data"]
|
||||
mapping: dict[int, str | None] = {}
|
||||
try:
|
||||
areas = client.get_logistics_areas()
|
||||
area_ids = [a.id for a in areas.items if a.id is not None]
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS sync] 物流区域拉取失败: {err}")
|
||||
area_ids = []
|
||||
for area_id in area_ids:
|
||||
try:
|
||||
resp = client.get_logistics_page(LogisticsPageRequest(issuing_bay_area_id=area_id, all=True, size=200))
|
||||
for item in resp.items:
|
||||
if item.id is None:
|
||||
continue
|
||||
raw = getattr(item, "channelType", None) or getattr(item, "channel_type", None)
|
||||
label = map_sds_channel_type(raw)
|
||||
if label is None:
|
||||
continue
|
||||
mapping[int(item.id)] = label
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS sync] 区域 {area_id} 物流字典拉取失败: {err}")
|
||||
_channel_cache[_CHANNEL_KEY] = {"data": mapping, "expire_at": time.time() + _CHANNEL_TTL}
|
||||
return mapping
|
||||
|
||||
|
||||
def refresh_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
|
||||
_channel_cache.pop(_CHANNEL_KEY, None)
|
||||
return get_channel_type_map(client)
|
||||
|
||||
|
||||
def resolve_courier_source(
|
||||
raw_logistics_id: Any,
|
||||
mapping: dict[int, str | None],
|
||||
refresh: Callable[[], dict[int, str | None]],
|
||||
refreshed_ids: set[int],
|
||||
) -> str | None:
|
||||
if raw_logistics_id is None:
|
||||
return None
|
||||
try:
|
||||
lid = int(raw_logistics_id)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if lid in mapping:
|
||||
return mapping[lid]
|
||||
if lid in refreshed_ids:
|
||||
return None
|
||||
refreshed_ids.add(lid)
|
||||
try:
|
||||
new_map = refresh()
|
||||
mapping.update(new_map)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return mapping.get(lid)
|
||||
@@ -0,0 +1,365 @@
|
||||
"""PG 同步写入层:Orders / Goods / Merchant 的 upsert + 同步所需的查询。
|
||||
|
||||
忠实移植自 inkreach-bi-auth-mng 的 sync/lib/db.service.ts。
|
||||
- Orders 主键:("orderNo", "createdAt")
|
||||
- Goods 主键:("orderNo", "productionNo")
|
||||
关键:写入前用库中已有 createdAt 覆盖,避免同一订单因 createdAt 不同产生重复主键行。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
|
||||
|
||||
ORDER_CHUNK = 1000
|
||||
GOODS_CHUNK = 2000
|
||||
LOOKUP_BATCH = 10000
|
||||
|
||||
|
||||
class SyncDB:
|
||||
def __init__(self, block_name: str = "pg-inkreach-dmp") -> None:
|
||||
from prefect_sqlalchemy import SqlAlchemyConnector
|
||||
|
||||
self._engine = SqlAlchemyConnector.load(block_name).get_engine()
|
||||
|
||||
def _conn(self):
|
||||
return self._engine.raw_connection()
|
||||
|
||||
# ---------- 查询辅助 ----------
|
||||
|
||||
def get_latest_ordered_at(self, system_name: str) -> datetime | None:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT MAX(COALESCE("orderedAt","createdAt")) FROM "Orders" WHERE "systemName"=%s',
|
||||
(system_name,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return row[0] if row and row[0] else None
|
||||
|
||||
def get_pending_orders(self, system_name: str, exclude_statuses: list[str]) -> list[str]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
|
||||
'AND "systemStatus" IS DISTINCT FROM ALL %s',
|
||||
(system_name, exclude_statuses),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
def get_order_count_by_status(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT "systemStatus", COUNT(*) FROM "Orders" WHERE "systemName"=%s '
|
||||
'AND "orderedAt">=%s AND "orderedAt"<%s GROUP BY "systemStatus"',
|
||||
(system_name, start, end),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return {r[0]: int(r[1]) for r in rows}
|
||||
|
||||
def get_local_order_nos(self, system_name: str, start: datetime, end: datetime) -> list[str]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
|
||||
'AND "orderedAt">=%s AND "orderedAt"<%s',
|
||||
(system_name, start, end),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
def get_local_goods_count(self, system_name: str, start: datetime, end: datetime) -> int:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT COUNT(*) FROM "Goods" g JOIN "Orders" o '
|
||||
'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
||||
'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
|
||||
(system_name, start, end),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def get_orders_with_100_prefix(self, system_name: str, statuses: list[str]) -> list[tuple[str, datetime]]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT o."orderNo", o."createdAt" FROM "Orders" o '
|
||||
'JOIN "Goods" g ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
||||
'WHERE o."systemName"=%s AND o."systemStatus"=ANY(%s) AND g."productionNo" LIKE \'100%\'',
|
||||
(system_name, statuses),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [(r[0], r[1]) for r in rows]
|
||||
|
||||
# ---------- 主 upsert ----------
|
||||
|
||||
def sync_orders_and_goods(
|
||||
self,
|
||||
orders: list[OrderUpsertInput],
|
||||
goods: list[GoodsUpsertInput],
|
||||
delete_order_nos: list[str] | None = None,
|
||||
) -> dict[str, int]:
|
||||
result = {
|
||||
"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0,
|
||||
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
|
||||
}
|
||||
if not orders and not goods and not delete_order_nos:
|
||||
return result
|
||||
|
||||
conn = self._conn()
|
||||
try:
|
||||
system_name = orders[0].systemName if orders else None
|
||||
all_order_nos = list({o.orderNo for o in orders} | {g.orderNo for g in goods})
|
||||
|
||||
# 1. 解析已有 createdAt,覆盖到 orders(保证幂等 upsert)
|
||||
existing_created: dict[str, datetime] = {}
|
||||
for i in range(0, len(all_order_nos), LOOKUP_BATCH):
|
||||
chunk = all_order_nos[i : i + LOOKUP_BATCH]
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
||||
+ (f' AND "systemName"=%s' if system_name else ""),
|
||||
(chunk, system_name) if system_name else (chunk,),
|
||||
)
|
||||
for r in cur.fetchall():
|
||||
existing_created[r[0]] = r[1]
|
||||
for o in orders:
|
||||
if o.orderNo in existing_created:
|
||||
o.createdAt = existing_created[o.orderNo]
|
||||
|
||||
# 2. 去重 orders / goods
|
||||
seen_o: set[str] = set()
|
||||
orders_dedup: list[OrderUpsertInput] = []
|
||||
for o in orders:
|
||||
key = f'{o.orderNo}|{o.createdAt.isoformat()}'
|
||||
if key in seen_o:
|
||||
continue
|
||||
seen_o.add(key)
|
||||
orders_dedup.append(o)
|
||||
|
||||
resolved_created: dict[str, datetime] = {}
|
||||
for o in orders_dedup:
|
||||
resolved_created.setdefault(o.orderNo, o.createdAt)
|
||||
|
||||
seen_g: set[str] = set()
|
||||
goods_dedup: list[GoodsUpsertInput] = []
|
||||
for g in goods:
|
||||
if g.orderNo in resolved_created:
|
||||
g.createdAt = resolved_created[g.orderNo]
|
||||
key = f'{g.orderNo}|{g.productionNo}'
|
||||
if key in seen_g:
|
||||
continue
|
||||
seen_g.add(key)
|
||||
goods_dedup.append(g)
|
||||
|
||||
goods_by_order: dict[str, list[GoodsUpsertInput]] = {}
|
||||
for g in goods_dedup:
|
||||
goods_by_order.setdefault(g.orderNo, []).append(g)
|
||||
|
||||
# 3. 分批 upsert orders + 关联 goods
|
||||
for i in range(0, len(orders_dedup), ORDER_CHUNK):
|
||||
chunk = orders_dedup[i : i + ORDER_CHUNK]
|
||||
related: list[GoodsUpsertInput] = []
|
||||
for o in chunk:
|
||||
related.extend(goods_by_order.get(o.orderNo, []))
|
||||
stats = self._upsert_order_batch(conn, chunk, related, system_name)
|
||||
for k, v in stats.items():
|
||||
result[k] += v
|
||||
|
||||
# 4. 剩余 goods(其 order 不在本次 orders 列表)
|
||||
orders_set = {o.orderNo for o in orders_dedup}
|
||||
remaining = [g for g in goods_dedup if g.orderNo not in orders_set]
|
||||
if remaining:
|
||||
c, u = self._upsert_goods_only(conn, remaining, system_name)
|
||||
result["goodsCreated"] += c
|
||||
result["goodsUpdated"] += u
|
||||
|
||||
# 5. 删除订单
|
||||
if delete_order_nos:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'DELETE FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
||||
+ (f' AND "systemName"=%s' if system_name else ""),
|
||||
(delete_order_nos, system_name) if system_name else (delete_order_nos,),
|
||||
)
|
||||
result["ordersDeleted"] = cur.rowcount
|
||||
|
||||
# 6. 商家占位
|
||||
self._ensure_merchants(conn, orders_dedup)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
def _upsert_order_batch(
|
||||
self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], system_name: str | None
|
||||
) -> dict[str, int]:
|
||||
stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0}
|
||||
with conn.cursor() as cur:
|
||||
# 统计已存在 → created/updated
|
||||
pairs = [(o.orderNo, o.createdAt) for o in orders]
|
||||
cur.execute(
|
||||
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamp[]) AS t(order_no,created_at) "
|
||||
"JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at",
|
||||
_unnest_pairs(pairs),
|
||||
)
|
||||
existing_set = {(r[0], r[1]) for r in cur.fetchall()}
|
||||
updated = sum(1 for o in orders if (o.orderNo, o.createdAt) in existing_set)
|
||||
stats["ordersUpdated"] = updated
|
||||
stats["ordersCreated"] = len(orders) - updated
|
||||
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Orders" (
|
||||
"orderNo","outOrderNo","courierSource","courierName","courierNo","merchantId",
|
||||
"countryRegion","factoryName","systemName","systemStatus","platformName","platformStatus",
|
||||
"createdAt","orderedAt","confirmedAt","completedAt","shipDeadline"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("orderNo","createdAt") DO UPDATE SET
|
||||
"outOrderNo"=EXCLUDED."outOrderNo","courierSource"=EXCLUDED."courierSource",
|
||||
"courierName"=EXCLUDED."courierName","courierNo"=EXCLUDED."courierNo",
|
||||
"merchantId"=EXCLUDED."merchantId","countryRegion"=EXCLUDED."countryRegion",
|
||||
"factoryName"=EXCLUDED."factoryName","systemName"=EXCLUDED."systemName",
|
||||
"systemStatus"=EXCLUDED."systemStatus","platformName"=EXCLUDED."platformName",
|
||||
"platformStatus"=EXCLUDED."platformStatus","orderedAt"=EXCLUDED."orderedAt",
|
||||
"confirmedAt"=EXCLUDED."confirmedAt","completedAt"=EXCLUDED."completedAt",
|
||||
"shipDeadline"=EXCLUDED."shipDeadline"''',
|
||||
[(
|
||||
o.orderNo, o.outOrderNo, o.courierSource, o.courierName, o.courierNo, o.merchantId,
|
||||
o.countryRegion, o.factoryName, o.systemName, o.systemStatus, o.platformName, o.platformStatus,
|
||||
o.createdAt, o.orderedAt, o.confirmedAt, o.completedAt, o.shipDeadline,
|
||||
) for o in orders],
|
||||
page_size=ORDER_CHUNK,
|
||||
)
|
||||
|
||||
# 关联 goods:先删本次不再出现的,再 upsert
|
||||
if goods:
|
||||
order_nos = list({g.orderNo for g in goods})
|
||||
prod_nos = [g.productionNo for g in goods]
|
||||
cur.execute(
|
||||
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)
|
||||
AND ("orderNo","createdAt") IN (
|
||||
SELECT "orderNo","createdAt" FROM "Orders"
|
||||
WHERE "orderNo"=ANY(%s)''' + (f' AND "systemName"=%s' if system_name else "") + ')',
|
||||
(order_nos, prod_nos, order_nos, system_name) if system_name else (order_nos, prod_nos, order_nos),
|
||||
)
|
||||
stats["goodsRemoved"] += cur.rowcount
|
||||
|
||||
g_pairs = [(g.orderNo, g.productionNo) for g in goods]
|
||||
cur.execute(
|
||||
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
|
||||
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
|
||||
_unnest_pairs(g_pairs),
|
||||
)
|
||||
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
|
||||
g_updated = sum(1 for g in goods if (g.orderNo, g.productionNo) in g_existing)
|
||||
stats["goodsUpdated"] = g_updated
|
||||
stats["goodsCreated"] = len(goods) - g_updated
|
||||
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Goods" (
|
||||
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
|
||||
"itemSize","itemStyleNo","quantity","status"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
|
||||
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
|
||||
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
|
||||
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
|
||||
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
|
||||
[(
|
||||
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
|
||||
g.itemSize, g.itemStyleNo, g.quantity, g.status,
|
||||
) for g in goods],
|
||||
page_size=GOODS_CHUNK,
|
||||
)
|
||||
return stats
|
||||
|
||||
def _upsert_goods_only(
|
||||
self, conn, goods: list[GoodsUpsertInput], system_name: str | None
|
||||
) -> tuple[int, int]:
|
||||
# 用库中实际 createdAt 覆盖
|
||||
order_nos = list({g.orderNo for g in goods})
|
||||
actual: dict[str, datetime] = {}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
||||
+ (f' AND "systemName"=%s' if system_name else ""),
|
||||
(order_nos, system_name) if system_name else (order_nos,),
|
||||
)
|
||||
for r in cur.fetchall():
|
||||
actual[r[0]] = r[1]
|
||||
for g in goods:
|
||||
if g.orderNo in actual:
|
||||
g.createdAt = actual[g.orderNo]
|
||||
|
||||
created = updated = 0
|
||||
for i in range(0, len(goods), GOODS_CHUNK):
|
||||
chunk = goods[i : i + GOODS_CHUNK]
|
||||
with conn.cursor() as cur:
|
||||
c_order = list({g.orderNo for g in chunk})
|
||||
c_prod = [g.productionNo for g in chunk]
|
||||
cur.execute(
|
||||
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)'''
|
||||
+ (f''' AND ("orderNo","createdAt") IN (SELECT "orderNo","createdAt" FROM "Orders"
|
||||
WHERE "orderNo"=ANY(%s) AND "systemName"=%s)''' if system_name else ""),
|
||||
(c_order, c_prod, c_order, system_name) if system_name else (c_order, c_prod),
|
||||
)
|
||||
g_pairs = [(g.orderNo, g.productionNo) for g in chunk]
|
||||
cur.execute(
|
||||
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
|
||||
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
|
||||
_unnest_pairs(g_pairs),
|
||||
)
|
||||
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
|
||||
u = sum(1 for g in chunk if (g.orderNo, g.productionNo) in g_existing)
|
||||
updated += u
|
||||
created += len(chunk) - u
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Goods" (
|
||||
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
|
||||
"itemSize","itemStyleNo","quantity","status"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
|
||||
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
|
||||
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
|
||||
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
|
||||
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
|
||||
[(
|
||||
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
|
||||
g.itemSize, g.itemStyleNo, g.quantity, g.status,
|
||||
) for g in chunk],
|
||||
page_size=GOODS_CHUNK,
|
||||
)
|
||||
return created, updated
|
||||
|
||||
def _ensure_merchants(self, conn, orders: list[OrderUpsertInput]) -> None:
|
||||
ids = list({o.merchantId for o in orders if o.merchantId and o.merchantId.strip()})
|
||||
if not ids:
|
||||
return
|
||||
with conn.cursor() as cur:
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Merchant" ("merchantId","merchantGroup","responsibleName","responsibleId","tags")
|
||||
VALUES %s ON CONFLICT ("merchantId") DO NOTHING''',
|
||||
[(mid, "", None, None, "") for mid in ids],
|
||||
)
|
||||
|
||||
|
||||
def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]:
|
||||
"""构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。
|
||||
|
||||
配合 SQL:unnest(%s::text[], %s::timestamp[]) 或 unnest(%s::text[], %s::text[])。
|
||||
"""
|
||||
return [p[0] for p in pairs], [p[1] for p in pairs]
|
||||
@@ -0,0 +1,317 @@
|
||||
"""数据映射:RIIN / SDS 原始订单 → 统一 Order/Goods 写入格式。
|
||||
|
||||
忠实移植自 inkreach-bi-auth-mng 的 sync/mappers/{riin,sds,normalize-country}.ts。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
TZ_CN = timezone(timedelta(hours=8))
|
||||
|
||||
# ---------- RIIN 状态 / 平台 / 快递来源映射 ----------
|
||||
|
||||
RIIN_SYSTEM_STATUS_MAP: dict[int, str] = {
|
||||
-4: "待推送", -3: "工厂反审", -5: "待审核", -6: "分配失败",
|
||||
1: "待推送", 3: "已反审", 4: "生产中", 5: "生产中",
|
||||
12: "已发货", 13: "已关闭", 14: "已取消", 15: "已关闭",
|
||||
}
|
||||
|
||||
RIIN_PLATFORM_TYPE_MAP: dict[int, str] = {
|
||||
2: "淘宝", 3: "拼多多", 6: "Shopee", 7: "LAZADA", 8: "1688",
|
||||
11: "TIKTOK", 13: "TEMU", 15: "自建站(海外)", 16: "SHOPIFY",
|
||||
17: "SHEIN", 18: "自建站(国内)", 19: "WALMART", 21: "AMAZON",
|
||||
26: "OZON", 27: "Wildberries", 30: "ETSY", 31: "MERCADO",
|
||||
}
|
||||
|
||||
RIIN_EXPRESS_SOURCE_MAP: dict[int, str] = {
|
||||
1: "工厂发货", 2: "平台面单", 3: "自有面单", 4: "分销面单", 5: "电商面单",
|
||||
}
|
||||
|
||||
RIIN_GOODS_STATUS_MAP: dict[str, str] = {
|
||||
"NOT_SHIPPED": "待发货", "SHIPPED": "已发货", "CANCEL": "已取消",
|
||||
"CANCELLED": "已取消", "REFUNDING": "退款中",
|
||||
}
|
||||
|
||||
RIIN_PLATFORM_STATUS_MAP: dict[str, str] = {
|
||||
"SHIPPED": "已发货", "NOT_SHIPPED": "待发货", "CANCEL": "已取消",
|
||||
"PAUSE": "暂停", "COMPLETE": "已完成",
|
||||
}
|
||||
|
||||
# ---------- SDS 状态 / 物流渠道映射 ----------
|
||||
|
||||
SDS_SYSTEM_STATUS_MAP: dict[int, str] = {
|
||||
1: "待付款", 2: "待确认", 3: "备货中", 4: "已完成", 5: "搁置中",
|
||||
7: "待设计", 9: "部分发货", 98: "已取消",
|
||||
}
|
||||
SDS_GOODS_STATUS_MAP: dict[int, str] = {**SDS_SYSTEM_STATUS_MAP, 99: "已删除"}
|
||||
|
||||
SDS_CHANNEL_TYPE_LABEL_MAP: dict[str, str] = {
|
||||
"online": "线上物流", "offline": "普通物流", "manual": "手动物流", "self_pickup": "自提",
|
||||
}
|
||||
|
||||
|
||||
# ---------- 日期解析 ----------
|
||||
|
||||
def safe_date(value: Any, context: str = "") -> datetime:
|
||||
"""解析日期字符串(无时区视为 +08:00),返回 tz-aware datetime。"""
|
||||
if not value or not isinstance(value, str):
|
||||
raise ValueError(f"Invalid date for {context}: {value!r}")
|
||||
s = value.strip()
|
||||
if "+" not in s and "T" not in s and not s.endswith("Z"):
|
||||
s = f"{s}T00:00:00+08:00" if len(s) == 10 else f"{s}+08:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
raise ValueError(f"Invalid date for {context}: {value!r}")
|
||||
return dt if dt.tzinfo else dt.replace(tzinfo=TZ_CN)
|
||||
|
||||
|
||||
def _from_epoch_ms(value: Any) -> datetime | None:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return datetime.fromtimestamp(n / 1000, tz=TZ_CN) if n > 0 else None
|
||||
|
||||
|
||||
# ---------- 国家/地区归一 ----------
|
||||
|
||||
_US_ALIASES = {"US", "UNITED STATES", "USA", "ESTADOS UNIDOS", "美国加州发货仓"}
|
||||
_MX_ALIASES = {"MX", "MEXICO", "MÉXICO", "MEX", "ME"}
|
||||
_CA_ALIASES = {"加拿大工厂", "CA"}
|
||||
_PL_ALIASES = {"欧州波兰发货仓", "欧洲波兰法兰克福发货仓"}
|
||||
_WAREHOUSE_MAP = {
|
||||
"印美达澳洲发货仓": "澳洲", "欧洲德国发货仓": "德国", "欧洲匈牙利发货仓": "匈牙利",
|
||||
"欧洲西班牙发货仓": "西班牙", "欧洲意大利发货仓": "意大利",
|
||||
}
|
||||
_DIRECT_MAP = {"Saudi Arabia": "沙特", "Japan": "日本"}
|
||||
_EXCLUDED_COUNTRIES = {
|
||||
"Croatia", "Czech Republic", "丹麦", "保加利亚", "克罗地亚", "卢森堡", "奥地利", "希腊",
|
||||
"拉脱维亚", "捷克", "斯洛伐克", "斯洛文尼亚", "比利时", "法国", "爱尔兰", "爱沙尼亚",
|
||||
"瑞典", "瑞士", "立陶宛", "罗马尼亚", "芬兰", "荷兰", "Denmark", "Austria", "Greece",
|
||||
"Latvia", "Slovakia", "Slovenia", "Belgium", "France", "Ireland", "Estonia", "Sweden",
|
||||
"Switzerland", "Lithuania", "Romania", "Finland", "Netherlands", "Luxembourg", "Bulgaria",
|
||||
}
|
||||
|
||||
|
||||
def normalize_country_region(raw: str | None, factory_name: str | None = None) -> str | None:
|
||||
if factory_name == "广州印美达":
|
||||
return "中国"
|
||||
if not raw:
|
||||
return None
|
||||
trimmed = raw.strip()
|
||||
if trimmed in _EXCLUDED_COUNTRIES:
|
||||
return None
|
||||
upper = trimmed.upper()
|
||||
if upper in _US_ALIASES:
|
||||
return "美国"
|
||||
if upper in _MX_ALIASES:
|
||||
return "墨西哥"
|
||||
if upper in _CA_ALIASES:
|
||||
return "加拿大"
|
||||
if trimmed in _PL_ALIASES:
|
||||
return "波兰"
|
||||
if trimmed in _WAREHOUSE_MAP:
|
||||
return _WAREHOUSE_MAP[trimmed]
|
||||
if raw in _DIRECT_MAP:
|
||||
return _DIRECT_MAP[raw]
|
||||
return raw
|
||||
|
||||
|
||||
# ---------- 统一写入模型 ----------
|
||||
|
||||
class OrderUpsertInput:
|
||||
__slots__ = (
|
||||
"orderNo", "outOrderNo", "courierSource", "courierName", "courierNo",
|
||||
"merchantId", "countryRegion", "factoryName", "systemName", "systemStatus",
|
||||
"platformName", "platformStatus", "createdAt", "orderedAt", "confirmedAt",
|
||||
"completedAt", "shipDeadline",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
for k in self.__slots__:
|
||||
setattr(self, k, kwargs.get(k))
|
||||
|
||||
|
||||
class GoodsUpsertInput:
|
||||
__slots__ = (
|
||||
"orderNo", "productionNo", "createdAt", "outOrderNo", "itemName",
|
||||
"itemColor", "itemSize", "itemStyleNo", "quantity", "status",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
for k in self.__slots__:
|
||||
setattr(self, k, kwargs.get(k))
|
||||
|
||||
|
||||
# ---------- RIIN 映射 ----------
|
||||
|
||||
def map_riin_system_status(order_status: int) -> str | None:
|
||||
return RIIN_SYSTEM_STATUS_MAP.get(order_status)
|
||||
|
||||
|
||||
def map_riin_order(raw: Any) -> OrderUpsertInput:
|
||||
platform_oid = raw.platform_oid
|
||||
ordered_at = _parse(raw.push_time, f"RIIN order {platform_oid} pushTime")
|
||||
confirmed_at = _parse(raw.distributor_push_time, f"RIIN order {platform_oid} distributorPushTime")
|
||||
system_status = map_riin_system_status(raw.order_status) if raw.order_status is not None else None
|
||||
is_terminal = system_status in ("已发货", "已关闭", "已取消")
|
||||
completed_at: datetime | None = None
|
||||
if is_terminal:
|
||||
completed_at = _parse(raw.shipping_time, "") or _parse(raw.finish_time, "")
|
||||
ship_deadline = _parse(raw.except_last_shipping_time, "")
|
||||
courier_source = RIIN_EXPRESS_SOURCE_MAP.get(raw.express_source) if raw.express_source is not None else None
|
||||
factory_name = raw.distributor_push_target_name
|
||||
|
||||
return OrderUpsertInput(
|
||||
orderNo=platform_oid,
|
||||
outOrderNo=raw.source_platform_oid,
|
||||
courierSource=courier_source,
|
||||
courierName=raw.delivery_courier,
|
||||
courierNo=raw.courier_number,
|
||||
merchantId=raw.sub_name,
|
||||
countryRegion=normalize_country_region(raw.receiver_country, factory_name),
|
||||
factoryName=factory_name,
|
||||
systemName="RIIN",
|
||||
systemStatus=system_status,
|
||||
platformName=RIIN_PLATFORM_TYPE_MAP.get(raw.platform_type, str(raw.platform_type)) if raw.platform_type is not None else None,
|
||||
platformStatus=RIIN_PLATFORM_STATUS_MAP.get(raw.platform_order_status, raw.platform_order_status) if raw.platform_order_status else None,
|
||||
createdAt=safe_date(raw.order_create_time, f"RIIN order {platform_oid} orderCreateTime"),
|
||||
orderedAt=ordered_at,
|
||||
confirmedAt=confirmed_at,
|
||||
completedAt=completed_at,
|
||||
shipDeadline=ship_deadline,
|
||||
)
|
||||
|
||||
|
||||
def map_riin_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
|
||||
product_code = raw.product_code
|
||||
item_name = product_code.split("-")[0] if product_code else None
|
||||
return GoodsUpsertInput(
|
||||
orderNo=parent_order_no,
|
||||
productionNo=raw.id,
|
||||
createdAt=parent_created_at,
|
||||
outOrderNo=raw.platform_oid,
|
||||
itemName=item_name,
|
||||
itemColor=raw.color_code,
|
||||
itemSize=raw.size_code,
|
||||
itemStyleNo=item_name,
|
||||
quantity=raw.num if raw.num is not None else 1,
|
||||
status=RIIN_GOODS_STATUS_MAP.get(raw.goods_status) if raw.goods_status else None,
|
||||
)
|
||||
|
||||
|
||||
# ---------- SDS 映射 ----------
|
||||
|
||||
def map_sds_channel_type(channel_type: Any) -> str | None:
|
||||
if not isinstance(channel_type, str):
|
||||
return None
|
||||
return SDS_CHANNEL_TYPE_LABEL_MAP.get(channel_type)
|
||||
|
||||
|
||||
def map_sds_system_status(status: int) -> str | None:
|
||||
return SDS_SYSTEM_STATUS_MAP.get(status)
|
||||
|
||||
|
||||
def map_sds_order(raw: Any, channel_type_map: dict[int, str | None], pre_courier: str | None = None) -> OrderUpsertInput:
|
||||
ordered_at = _parse(raw.gmt_pay, f"SDS order {raw.no} gmtPay")
|
||||
system_status = map_sds_system_status(raw.status) if raw.status is not None else None
|
||||
is_terminal = system_status in ("已完成", "已取消")
|
||||
completed_at: datetime | None = None
|
||||
if is_terminal and raw.finish_time and raw.finish_time > 0:
|
||||
completed_at = _from_epoch_ms(raw.finish_time)
|
||||
ship_deadline = _from_epoch_ms(raw.latest_ship_time) if raw.latest_ship_time else None
|
||||
|
||||
courier_source = pre_courier
|
||||
if courier_source is None and raw.logistics_id is not None:
|
||||
try:
|
||||
lid = int(raw.logistics_id)
|
||||
except (TypeError, ValueError):
|
||||
lid = None
|
||||
if lid is not None and lid in channel_type_map:
|
||||
courier_source = channel_type_map[lid]
|
||||
|
||||
courier_no: str | None = None
|
||||
if raw.parcel_list:
|
||||
first = raw.parcel_list[0]
|
||||
try:
|
||||
courier_no = first["logistics"]["carriage"]["no"]
|
||||
except (TypeError, KeyError, IndexError):
|
||||
courier_no = None
|
||||
|
||||
factory_name: str | None = None
|
||||
for item in raw.items or []:
|
||||
fname = getattr(getattr(item, "factory", None), "name", None)
|
||||
if not fname and isinstance(item, dict):
|
||||
fname = (item.get("factory") or {}).get("name")
|
||||
if fname:
|
||||
factory_name = fname
|
||||
break
|
||||
|
||||
area_name = raw.issuing_bay_area.name if raw.issuing_bay_area else None
|
||||
return OrderUpsertInput(
|
||||
orderNo=raw.no,
|
||||
outOrderNo=raw.out_order_no,
|
||||
courierSource=courier_source,
|
||||
courierName=raw.carriage_name,
|
||||
courierNo=courier_no,
|
||||
merchantId=raw.merchant.name if raw.merchant else None,
|
||||
countryRegion=normalize_country_region(area_name, factory_name),
|
||||
factoryName=factory_name,
|
||||
systemName="SDS",
|
||||
systemStatus=system_status,
|
||||
platformName=raw.merchant_store.platform_name if raw.merchant_store else None,
|
||||
platformStatus=raw.online_order_status_name,
|
||||
createdAt=safe_date(raw.gmt_created, f"SDS order {raw.no}"),
|
||||
orderedAt=ordered_at,
|
||||
confirmedAt=None,
|
||||
completedAt=completed_at,
|
||||
shipDeadline=ship_deadline,
|
||||
)
|
||||
|
||||
|
||||
def _parse_sds_style_no(name: str | None) -> str | None:
|
||||
if not name:
|
||||
return None
|
||||
parts = name.split("()")
|
||||
if len(parts) < 3:
|
||||
return None
|
||||
subs = parts[2].split("-")
|
||||
if len(subs) == 2:
|
||||
return subs[0] or None
|
||||
if len(subs) >= 3:
|
||||
return subs[1] or None
|
||||
return None
|
||||
|
||||
|
||||
def map_sds_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
|
||||
product_name = raw.product_name
|
||||
color_name = None
|
||||
block = raw.product_color_block
|
||||
if block is not None:
|
||||
color_name = getattr(block, "color_name", None) or (block.get("colorName") if isinstance(block, dict) else None)
|
||||
return GoodsUpsertInput(
|
||||
orderNo=parent_order_no,
|
||||
productionNo=raw.no or raw.id,
|
||||
createdAt=parent_created_at,
|
||||
outOrderNo=raw.out_order_item_id,
|
||||
itemName=product_name,
|
||||
itemColor=color_name,
|
||||
itemSize=raw.product_size,
|
||||
itemStyleNo=_parse_sds_style_no(product_name),
|
||||
quantity=raw.num if raw.num is not None else 1,
|
||||
status=SDS_GOODS_STATUS_MAP.get(raw.status) if raw.status is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _parse(value: Any, context: str) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return _from_epoch_ms(value)
|
||||
try:
|
||||
return safe_date(value, context)
|
||||
except ValueError:
|
||||
return None
|
||||
Reference in New Issue
Block a user